feat: add gateway-api support

Makefile

@@ -10,7 +10,7 @@ KINDCONFIG := $(or $(KINDCONFIG),control-plane/kind.yaml)
 KUBECONFIG := $(shell pwd)/.kubeconfig
 
 METALCTL_HMAC := $(or $(METALCTL_HMAC),metal-admin)
-METALCTL_API_URL := $(or $(METALCTL_API_URL),http://api.172.17.0.1.nip.io:8080/metal)
+METALCTL_API_URL := $(or $(METALCTL_API_URL),http://api.172.42.0.42.nip.io:8080/metal)
 
 MKE2FS_CONFIG := $(shell pwd)/mke2fs.conf
 # Default values
@@ -26,6 +26,8 @@ MINI_LAB_VM_IMAGE := $(or $(MINI_LAB_VM_IMAGE),ghcr.io/metal-stack/mini-lab-vms:
 MINI_LAB_SONIC_IMAGE := $(or $(MINI_LAB_SONIC_IMAGE),ghcr.io/metal-stack/mini-lab-sonic:latest)
 MINI_LAB_DELL_SONIC_VERSION := $(or $(MINI_LAB_DELL_SONIC_VERSION),4.5.1)
 
+MINI_LAB_INTERNAL_NETWORK=mini_lab_internal
+
 MACHINE_OS=debian-12.0
 MAX_RETRIES := 30
 
@@ -117,13 +119,16 @@ create-proxy-registries:
 
 .PHONY: control-plane-bake
 control-plane-bake:
+
+	@if ! docker network ls | grep -q mini_lab_internal; then docker network create mini_lab_internal --gateway 172.42.0.1 --ip-range=172.42.0.0/24 --subnet=172.42.0.0/24 --ipv6=false ; fi
 	@if ! which kind > /dev/null; then echo "kind needs to be installed"; exit 1; fi
 	@if ! kind get clusters | grep metal-control-plane > /dev/null; then \
 		kind create cluster $(KIND_ARGS) \
 			--name metal-control-plane \
 			--config $(KINDCONFIG) \
 			--kubeconfig $(KUBECONFIG); fi
 	$(MAKE) create-proxy-registries
+	docker compose up -d --force-recreate cloud-provider-kind
 
 .PHONY: partition
 partition: partition-bake
@@ -166,6 +171,7 @@ env:
 
 .PHONY: cleanup
 cleanup: cleanup-control-plane cleanup-partition
+	docker network rm --force mini_lab_internal
 
 .PHONY: cleanup-control-plane
 cleanup-control-plane:
@@ -438,7 +444,7 @@ build-dell-sonic:
 fetch-virtual-kubeconfig:
 	# TODO: it's hard to get the latest issued generic kubeconfig secret... just take the first result for now
 	kubectl --kubeconfig=$(KUBECONFIG) get secret -n garden $(shell kubectl --kubeconfig=$(KUBECONFIG) get secret -n garden -l managed-by=secrets-manager,manager-identity=gardener-operator,name=generic-token-kubeconfig --no-headers | awk '{ print $$1 }') -o jsonpath='{.data.kubeconfig}' | base64 -d > .virtual-kubeconfig
-	@kubectl --kubeconfig=.virtual-kubeconfig config set-cluster garden --server=https://api.gardener-kube-apiserver.172.17.0.1.nip.io:4443
+	@kubectl --kubeconfig=.virtual-kubeconfig config set-cluster garden --server=https://api.gardener-kube-apiserver.172.42.0.1.nip.io:4443
 	@kubectl --kubeconfig=.virtual-kubeconfig config set-credentials garden --token=$(shell kubectl --kubeconfig=$(KUBECONFIG) get secret -n garden shoot-access-virtual-garden -o jsonpath='{.data.token}' | base64 -d)
 	@kubectl --kubeconfig=$(KUBECONFIG) config unset users.garden
 	@kubectl --kubeconfig=$(KUBECONFIG) config unset contexts.garden

compose.yaml

@@ -128,6 +128,20 @@ services:
       - REGISTRY_PROXY_TTL=168h
       - REGISTRY_STORAGE_DELETE_ENABLED=true
       - OTEL_TRACES_EXPORTER=none
+  cloud-provider-kind:
+    image: registry.k8s.io/cloud-provider-kind/cloud-controller-manager:v0.10.0
+    restart: always
+    networks:
+      - kind
+    environment:
+      - KIND_EXPERIMENTAL_DOCKER_NETWORK=${KIND_EXPERIMENTAL_DOCKER_NETWORK:-kind}
+    command:
+    # v0.10.0 of cloud controller does not support tcproutes, since it does not support the experimental gateway api channel
+    # using envoy-gateway deployed via roles/gateway instead
+      - --gateway-channel
+      - disabled
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
 volumes:
   proxy-docker:
   proxy-gcr:

control-plane/kind.yaml

@@ -2,7 +2,7 @@ kind: Cluster
 apiVersion: kind.x-k8s.io/v1alpha4
 networking:
   apiServerPort: 6443
-  apiServerAddress: 0.0.0.0
+  apiServerAddress: 172.42.0.1
 nodes:
   - role: control-plane
     extraMounts:
@@ -13,10 +13,6 @@ nodes:
         hostPort: 4443
       - containerPort: 8080
         hostPort: 8080
-      - containerPort: 4150
-        hostPort: 4150
-      - containerPort: 50051
-        hostPort: 50051
     # if you want to run gardener operator + metal-stack, you need more pods
     kubeadmConfigPatches:
       - |

deploy_control_plane.yaml

@@ -6,6 +6,8 @@
   roles:
     - name: ansible-common
       tags: always
+    - name: gateway
+      tags: gateway
     - name: ingress-controller
       tags: ingress-controller
     - name: metal-roles/control-plane/roles/prepare

deploy_gardener.yaml

@@ -13,7 +13,7 @@
             name: shoot-info
             namespace: kube-system
           data:
-            nodeNetwork: 172.18.0.0/16
+            nodeNetwork: 172.42.0.0/16
             podNetwork: 10.244.0.0/24
             serviceNetwork: 10.96.0.0/16
       tags: gardener
@@ -81,7 +81,7 @@
             status:
               loadBalancer:
                 ingress:
-                  - ip: "172.17.0.1"
+                  - ip: "172.42.0.1"
         tags: gardener
 
       - name: Expose istio gateway through ingress-nginx (for local environments)

env.sh

@@ -24,4 +24,5 @@ DEPLOYMENT_BASE_IMAGE_TAG=${DEPLOYMENT_BASE_IMAGE_TAG}
 CI=${CI:=false}
 DOCKER_HUB_USER=${DOCKER_HUB_USER:=}
 DOCKER_HUB_TOKEN=${DOCKER_HUB_TOKEN:=}
+KIND_EXPERIMENTAL_DOCKER_NETWORK=${MINI_LAB_INTERNAL_NETWORK:=}
 EOF

files/certs/default-gateway/server.json

@@ -0,0 +1,20 @@
+{
+  "CN": "default-gateway",
+  "hosts": [
+    "api.172.42.0.42.nip.io",
+    "v2.api.172.42.0.42.nip.io"
+  ],
+  "key": {
+    "algo": "rsa",
+    "size": 4096
+  },
+  "names": [
+    {
+      "C":  "DE",
+      "L":  "Munich",
+      "O":  "metal-stack",
+      "OU": "DevOps",
+      "ST": "Bavaria"
+    }
+  ]
+}

files/certs/grpc/server.json

@@ -1,7 +1,7 @@
 {
   "CN": "metal-api",
   "hosts": [
-    "172.17.0.1",
+    "172.42.0.42",
     "203.0.113.1"
   ],
   "key": {

files/dev_images.yaml

@@ -1,6 +1,6 @@
 ---
 # Do not change these values
 metal_api_image_tag: dev
-metal_core_image_name: 172.17.0.1:5000/metalstack/metal-core
+metal_core_image_name: 172.42.0.42:5000/metalstack/metal-core
 metal_core_image_tag: dev
-metal_hammer_image_url: http://172.17.0.1:20015/metal-hammer-initrd.img.lz4
+metal_hammer_image_url: http://172.42.0.42:20015/metal-hammer-initrd.img.lz4

files/startup-config/leaf01_4.4.3.json

@@ -11,7 +11,7 @@
     }
   },
   "DNS_SERVER": {
-    "172.17.0.1": {},
+    "172.42.0.1": {},
     "1.1.1.1": {},
     "1.0.0.1": {}
   },

files/startup-config/leaf01_4.5.1.json

@@ -11,7 +11,7 @@
     }
   },
   "DNS_SERVER": {
-    "172.17.0.1": {},
+    "172.42.0.1": {},
     "1.1.1.1": {},
     "1.0.0.1": {}
   },

files/startup-config/leaf02_4.4.3.json

@@ -11,7 +11,7 @@
     }
   },
   "DNS_SERVER": {
-    "172.17.0.1": {},
+    "172.42.0.1": {},
     "1.1.1.1": {},
     "1.0.0.1": {}
   },

files/startup-config/leaf02_4.5.1.json

@@ -11,7 +11,7 @@
     }
   },
   "DNS_SERVER": {
-    "172.17.0.1": {},
+    "172.42.0.1": {},
     "1.1.1.1": {},
     "1.0.0.1": {}
   },

inventories/group_vars/all/control_plane.yaml

@@ -1,6 +1,7 @@
 ---
 metal_control_plane_provider_tenant: metal-stack
-metal_control_plane_ingress_dns: 172.17.0.1.nip.io
+metal_control_plane_ingress_dns: 172.42.0.1.nip.io
+metal_control_plane_gateway_dns: 172.42.0.42.nip.io
 metal_control_plane_stage_name: test
 metal_control_plane_namespace: metal-control-plane
 metal_control_plane_image_pull_policy: Always

inventories/group_vars/control_plane/dex.yaml

@@ -8,7 +8,7 @@ auth_dex_static_clients:
   name: "metal-stack"
   secret: secret
   redirectURIs:
-  - 'http://v2.api.172.17.0.1.nip.io:8080/auth/oidc/callback'
+  - 'http://v2.api.{{ metal_control_plane_gateway_dns }}:8080/auth/oidc/callback'
 
 auth_dex_static_passwords:
 - email: admin@metal-stack.io

inventories/group_vars/control_plane/gardener/gardenlet.yaml

@@ -1,5 +1,5 @@
 ---
-gardener_gardenlet_default_dns_domain: "gardener.172.17.0.1.nip.io"
+gardener_gardenlet_default_dns_domain: "gardener.172.42.0.1.nip.io"
 gardener_gardenlet_default_dns_provider: powerdns
 gardener_gardenlet_default_dns_credentials:
   apiKey: "{{ powerdns_api_key | b64encode }}"

inventories/group_vars/control_plane/gardener/operator.yaml

@@ -1,5 +1,5 @@
 ---
-gardener_operator_ingress_dns_domain: "gardener.172.17.0.1.nip.io"
+gardener_operator_ingress_dns_domain: "gardener.{{ metal_control_plane_ingress_dns }}"
 
 gardener_operator_backup_infrastructure:
   provider: S3
@@ -17,7 +17,7 @@ gardener_operator_backup_infrastructure_secret:
   s3ForcePathStyle: "{{ 'true' | b64encode }}"
 
 # enable mini-lab patches
-gardener_operator_patch_istio_ingress_gateway_service_ip: 172.17.0.1
+gardener_operator_patch_istio_ingress_gateway_service_ip: 172.42.0.1
 gardener_operator_expose_virtual_garden_through_ingress_nginx: true
 
 # for local setups this should be sufficient

inventories/group_vars/control_plane/ingress.yaml

@@ -1,4 +1,4 @@
 ---
-ingress_tcp_service_exposals:
-  "4150": "{{ metal_control_plane_namespace }}/nsqd:4150"
-  "50051": "{{ metal_control_plane_namespace }}/metal-api:50051"
+gateway_tcp_listeners:
+  nsq: 4150
+  metal-api-grpc: 50051

inventories/group_vars/control_plane/metal.yml

@@ -1,9 +1,23 @@
 ---
 metal_set_resource_limits: no
-metal_check_api_health_endpoint: http://api.{{ metal_control_plane_ingress_dns }}:8080/metal/v1/health
-metal_api_headscale_control_plane_address: "http://headscale.{{ metal_control_plane_ingress_dns }}:8080"
+metal_check_api_health_endpoint: http://api.{{ metal_control_plane_gateway_dns }}:8080/metal/v1/health
+metal_api_headscale_control_plane_address: "http://headscale.{{ metal_control_plane_gateway_dns }}:8080"
 
-# metal_helm_chart_local_path: /helm-charts/charts/metal-control-plane
+metal_helm_chart_local_path: /helm-charts/charts/metal-control-plane
+
+metal_deploy_ingress: false
+
+metal_api_httproute_enabled: true
+metal_api_httproute_parent_refs:
+- name: metal-control-plane
+  namespace: "{{ metal_control_plane_namespace }}"
+  sectionName: http
+
+metal_api_tcproute_enabled: true
+metal_api_tcproute_parent_refs:
+- name: metal-control-plane
+  namespace: "{{ metal_control_plane_namespace }}"
+  sectionName: metal-api-grpc
 
 metal_api_pdb_min_available: 1
 metal_api_replicas: 1
@@ -16,14 +30,20 @@ metal_api_nsq_tcp_address: nsqd:4150
 metal_apiserver_pdb_min_available: 1
 
 metal_apiserver_enabled: true
-metal_apiserver_url: http://v2.api.{{ metal_control_plane_ingress_dns }}:8080
+metal_apiserver_url: http://v2.api.{{ metal_control_plane_gateway_dns }}:8080
 
 metal_apiserver_oidc_secret_name: zitadel-client-credentials
-metal_apiserver_oidc_discovery_url: https://zitadel.{{ metal_control_plane_ingress_dns }}:4443/.well-known/openid-configuration
-metal_apiserver_oidc_end_session_url: "https://zitadel.{{ metal_control_plane_ingress_dns }}:4443/oidc/v1/end_session"
+metal_apiserver_oidc_discovery_url: https://zitadel.{{ metal_control_plane_gateway_dns }}:4443/.well-known/openid-configuration
+metal_apiserver_oidc_end_session_url: "https://zitadel.{{ metal_control_plane_gateway_dns }}:4443/oidc/v1/end_session"
 
 metal_apiserver_redis_password: change-me-soon
-metal_apiserver_admin_subjects: "admin@metal-stack.zitadel.172.17.0.1.nip.io@openid-connect"
+metal_apiserver_admin_subjects: "admin@metal-stack.zitadel.{{ metal_control_plane_gateway_dns }}@openid-connect"
+
+metal_apiserver_httproute_enabled: true
+metal_apiserver_httproute_parent_refs:
+- name: metal-control-plane
+  namespace: "{{ metal_control_plane_namespace }}"
+  sectionName: http
 
 metal_api_images:
   - id: firewall-ubuntu-3.0

inventories/group_vars/control_plane/minio.yaml

@@ -2,4 +2,4 @@
 minio_root_user: mini-lab
 minio_root_password: change-me
 
-minio_dns_name: minio.172.17.0.1.nip.io
+minio_dns_name: minio.172.42.0.1.nip.io

inventories/group_vars/control_plane/nsq.yaml

@@ -9,3 +9,8 @@ nsq_certs_client_cert: "{{  lookup('file', 'certs/nsq/client.crt') }}"
 nsq_certs_ca_cert: "{{ lookup('file', 'certs/ca.pem') }}"
 
 nsq_broadcast_address: nsqd
+
+nsq_tcproute_enabled: true
+nsq_tcproute_parent_refs:
+- name: metal-control-plane
+  sectionName: nsq

inventories/group_vars/control_plane/powerdns.yaml

@@ -10,11 +10,11 @@ powerdns_load_balancer_dns_name: "ns.{{ metal_control_plane_ingress_dns }}"
 powerdns_api_dns_name: "powerdns-api.{{ metal_control_plane_ingress_dns }}"
 
 powerdns_zones:
-  - name: "gardener.172.17.0.1.nip.io."
+  - name: "gardener.172.42.0.1.nip.io."
     kind: Master
     nameservers:
       - "{{ powerdns_load_balancer_dns_name }}."
-  - name: "gardener-kube-apiserver.172.17.0.1.nip.io."
+  - name: "gardener-kube-apiserver.172.42.0.1.nip.io."
     kind: Master
     nameservers:
       - "{{ powerdns_load_balancer_dns_name }}."

inventories/group_vars/control_plane/zitadel.yaml

@@ -1,11 +1,16 @@
 ---
 zitadel_endpoint: zitadel.{{ metal_control_plane_namespace }}.svc.cluster.local
-zitadel_external_domain: zitadel.{{ metal_control_plane_ingress_dns }}
+zitadel_external_domain: zitadel.{{ metal_control_plane_gateway_dns }}
 zitadel_ingress_dns: https://{{ zitadel_external_domain }}:4443
 zitadel_port: 8080
 zitadel_skip_verify_tls: true
 zitadel_insecure: true
 
+zitadel_httproute_enabled: true
+zitadel_httproute_parent_refs:
+- name: metal-control-plane
+  sectionName: https
+
 zitadel_init_config:
   static_users:
     - first_name: Olli
@@ -23,4 +28,4 @@ zitadel_init_config:
     # later id will be added but currently not possible with zitadel
     id: metal-stack
     name: metal-stack
-    redirect_uri: http://v2.api.172.17.0.1.nip.io:8080/auth/openid-connect/callback
+    redirect_uri: http://v2.api.172.42.0.42.nip.io:8080/auth/openid-connect/callback

inventories/group_vars/leaves/main.yaml

@@ -12,7 +12,7 @@ sonic_config_frr_render: false
 sonic_config_loopback_address: "{{ lo }}"
 sonic_config_mgmt_interface:
   ip: "{{ ansible_host }}/16`"
-  gateway_address: "172.17.0.1"
+  gateway_address: "172.42.0.1"
 
 sonic_config_mgmt_vrf: false
 sonic_config_nameservers: "{{ router_nameservers }}"