[DNM] Add Support for Kubernetes Version 1.33.5

cmd/generate_test.go

@@ -553,8 +553,8 @@ func TestExampleAPIModels(t *testing.T) {
 			setArgs:      defaultSet,
 		},
 		{
-			name:         "1.30 example",
-			apiModelPath: "../examples/kubernetes-releases/kubernetes1.30.json",
+			name:         "1.33 example",
+			apiModelPath: "../examples/kubernetes-releases/kubernetes1.33.json",
 			setArgs:      defaultSet,
 		},
 		{

examples/azure-stack/kubernetes-azurestack.json

@@ -3,8 +3,8 @@
     "location": "",
     "properties": {
         "orchestratorProfile": {
-            "orchestratorRelease": "1.30",
-            "orchestratorVersion": "1.30.14",
+            "orchestratorRelease": "1.31",
+            "orchestratorVersion": "1.31.11",
             "kubernetesConfig": {
                 "cloudProviderBackoff": true,
                 "cloudProviderBackoffRetries": 1,

examples/azure-stack/kubernetes-windows.json

@@ -3,8 +3,8 @@
     "location": "",
     "properties": {
         "orchestratorProfile": {
-            "orchestratorRelease": "1.30",
-            "orchestratorVersion": "1.30.14",
+            "orchestratorRelease": "1.31",
+            "orchestratorVersion": "1.31.11",
             "kubernetesConfig": {
                 "cloudProviderBackoff": true,
                 "cloudProviderBackoffRetries": 1,

examples/kubernetes-releases/kubernetes1.33.json

@@ -0,0 +1,30 @@
+{
+  "apiVersion": "vlabs",
+  "properties": {
+    "orchestratorProfile": {
+      "orchestratorRelease": "1.33"
+    },
+    "masterProfile": {
+      "count": 1,
+      "dnsPrefix": "",
+      "vmSize": "Standard_D2_v3"
+    },
+    "agentPoolProfiles": [
+      {
+        "name": "agentpool1",
+        "count": 3,
+        "vmSize": "Standard_D2_v3"
+      }
+    ],
+    "linuxProfile": {
+      "adminUsername": "azureuser",
+      "ssh": {
+        "publicKeys": [
+          {
+            "keyData": ""
+          }
+        ]
+      }
+    }
+  }
+}

hack/tools/Makefile

@@ -3,7 +3,7 @@ LOCALBIN		= $(shell pwd)/bin
 SHELLCHECK_VERSION ?= v0.8.0
 AZCLI_VERSION   ?= 2.56.0
 PYWINRM_VERSION ?= 0.4.3
-KUBECTL_VERSION ?= v1.30.14
+KUBECTL_VERSION ?= v1.31.11
 
 all: install
 

pkg/api/common/const.go

@@ -57,13 +57,13 @@ const (
 
 const (
 	// KubernetesDefaultRelease is the default Kubernetes release
-	KubernetesDefaultRelease string = "1.30"
+	KubernetesDefaultRelease string = "1.32"
 	// KubernetesDefaultReleaseWindows is the default Kubernetes release for Windows
-	KubernetesDefaultReleaseWindows string = "1.30"
+	KubernetesDefaultReleaseWindows string = "1.32"
 	// KubernetesDefaultReleaseAzureStack is the default Kubernetes release on Azure Stack
-	KubernetesDefaultReleaseAzureStack string = "1.30"
+	KubernetesDefaultReleaseAzureStack string = "1.32"
 	// KubernetesDefaultReleaseWindowsAzureStack is the default Kubernetes release for Windows on Azure Stack
-	KubernetesDefaultReleaseWindowsAzureStack string = "1.30"
+	KubernetesDefaultReleaseWindowsAzureStack string = "1.32"
 	// PodSecurityPolicyRemovedVersion is the first Kubernetes version that does not includes the PSP admission plugin
 	PodSecurityPolicyRemovedVersion = "1.25.0"
 )

pkg/api/common/versions.go

@@ -368,8 +368,9 @@ var AllKubernetesSupportedVersions = map[string]bool{
 	"1.29.10":        false,
 	"1.29.15":        false,
 	"1.30.10":        false,
-	"1.30.14":        true,
+	"1.30.14":        false,
 	"1.31.11":        true,
+	"1.33.5":         true,
 }
 
 // AllKubernetesSupportedVersionsAzureStack is a hash table of all supported Kubernetes version strings on Azure Stack
@@ -428,8 +429,9 @@ var AllKubernetesSupportedVersionsAzureStack = map[string]bool{
 	"1.29.10": false,
 	"1.29.15": false,
 	"1.30.10": false,
-	"1.30.14": true,
+	"1.30.14": false,
 	"1.31.11": true,
+	"1.33.5":  true,
 }
 
 // AllKubernetesWindowsSupportedVersionsAzureStack maintain a set of available k8s Windows versions in aks-engine on Azure Stack
@@ -483,8 +485,9 @@ var AllKubernetesWindowsSupportedVersionsAzureStack = map[string]bool{
 	"1.29.10": false,
 	"1.29.15": false,
 	"1.30.10": false,
-	"1.30.14": true,
+	"1.30.14": false,
 	"1.31.11": true,
+	"1.33.5":  true,
 }
 
 // GetDefaultKubernetesVersion returns the default Kubernetes version, that is the latest patch of the default release

pkg/api/defaults-apiserver.go

@@ -396,6 +396,55 @@ func (cs *ContainerService) overrideAPIServerConfig() {
 		// Remove --feature-gate SkipReadOnlyValidationGCE starting with 1.31
 		invalidFeatureGates = append(invalidFeatureGates, "SkipReadOnlyValidationGCE")
 	}
+	if common.IsKubernetesVersionGe(o.OrchestratorVersion, "1.33.0") {
+		// Remove --feature-gate AdmissionWebhookMatchConditions starting with 1.33
+		invalidFeatureGates = append(invalidFeatureGates, "AdmissionWebhookMatchConditions")
+
+		// Remove --feature-gate AggregatedDiscoveryEndpoint starting with 1.33
+		invalidFeatureGates = append(invalidFeatureGates, "AggregatedDiscoveryEndpoint")
+
+		// Remove --feature-gate APIListChunking starting with 1.33
+		invalidFeatureGates = append(invalidFeatureGates, "APIListChunking")
+
+		// Remove --feature-gate AppArmor starting with 1.33
+		invalidFeatureGates = append(invalidFeatureGates, "AppArmor")
+
+		// Remove --feature-gate AppArmorFields starting with 1.33
+		invalidFeatureGates = append(invalidFeatureGates, "AppArmorFields")
+
+		// Remove --feature-gate CPUManager starting with 1.33
+		invalidFeatureGates = append(invalidFeatureGates, "CPUManager")
+
+		// Remove --feature-gate DisableCloudProviders starting with 1.33
+		invalidFeatureGates = append(invalidFeatureGates, "DisableCloudProviders")
+
+		// Remove --feature-gate DisableKubeletCloudCredentialProviders starting with 1.33
+		invalidFeatureGates = append(invalidFeatureGates, "DisableKubeletCloudCredentialProviders")
+
+		// Remove --feature-gate EfficientWatchResumption starting with 1.33
+		invalidFeatureGates = append(invalidFeatureGates, "EfficientWatchResumption")
+
+		// Remove --feature-gate JobPodFailurePolicy starting with 1.33
+		invalidFeatureGates = append(invalidFeatureGates, "JobPodFailurePolicy")
+
+		// Remove --feature-gate KubeProxyDrainingTerminatingNodes starting with 1.33
+		invalidFeatureGates = append(invalidFeatureGates, "KubeProxyDrainingTerminatingNodes")
+
+		// Remove --feature-gate PDBUnhealthyPodEvictionPolicy starting with 1.33
+		invalidFeatureGates = append(invalidFeatureGates, "PDBUnhealthyPodEvictionPolicy")
+
+		// Remove --feature-gate PersistentVolumeLastPhaseTransitionTime starting with 1.33
+		invalidFeatureGates = append(invalidFeatureGates, "PersistentVolumeLastPhaseTransitionTime")
+
+		// Remove --feature-gate RemainingItemCount starting with 1.33
+		invalidFeatureGates = append(invalidFeatureGates, "RemainingItemCount")
+
+		// Remove --feature-gate VolumeCapacityPriority starting with 1.33
+		invalidFeatureGates = append(invalidFeatureGates, "VolumeCapacityPriority")
+
+		// Remove --feature-gate WatchBookmark starting with 1.33
+		invalidFeatureGates = append(invalidFeatureGates, "WatchBookmark")
+	}
 	removeInvalidFeatureGates(o.KubernetesConfig.APIServerConfig, invalidFeatureGates)
 
 	if common.ShouldDisablePodSecurityPolicyAddon(o.OrchestratorVersion) {

pkg/api/defaults-apiserver_test.go

@@ -782,6 +782,48 @@ func TestAPIServerFeatureGates(t *testing.T) {
 		t.Fatalf("got unexpected '--feature-gates' for %s \n API server config original value  %s \n, actual sanitized value: %s \n, expected sanitized value: %s \n ",
 			"1.30.0", featuregate131, a["--feature-gates"], featuregate130Sanitized)
 	}
+
+	// test user-overrides, removal of feature gates for k8s versions >= 1.33
+	cs = CreateMockContainerService("testcluster", defaultTestClusterVer, 3, 2, false)
+	cs.Properties.OrchestratorProfile.OrchestratorVersion = "1.33.0"
+	cs.Properties.OrchestratorProfile.KubernetesConfig.APIServerConfig = make(map[string]string)
+	a = cs.Properties.OrchestratorProfile.KubernetesConfig.APIServerConfig
+	featuregate133 := "AdmissionWebhookMatchConditions=true,AggregatedDiscoveryEndpoint=true,APIListChunking=true,AppArmor=true,AppArmorFields=true,CPUManager=true,DisableCloudProviders=true,DisableKubeletCloudCredentialProviders=true,EfficientWatchResumption=true,JobPodFailurePolicy=true,KubeProxyDrainingTerminatingNodes=true,PDBUnhealthyPodEvictionPolicy=true,PersistentVolumeLastPhaseTransitionTime=true,RemainingItemCount=true,VolumeCapacityPriority=true,WatchBookmark=true"
+	a["--feature-gates"] = featuregate133
+	cs.setAPIServerConfig()
+	// split both strings by ", " and ensure no original item exists in the sanitized list
+	originalList := strings.Split(featuregate133, ",")
+	sanitizedList := strings.Split(a["--feature-gates"], ",")
+	for _, of := range originalList {
+		for _, sf := range sanitizedList {
+			if of == sf {
+				t.Fatalf("feature-gate %q should not exist in sanitized list for %s\nfeaturegate133 (original): %q\nfeaturegate133Sanitized (actual): %q", sf, "1.33", featuregate133, a["--feature-gates"])
+			}
+		}
+	}
+
+	// test user-overrides, no removal of feature gates for k8s versions < 1.33
+	cs = CreateMockContainerService("testcluster", defaultTestClusterVer, 3, 2, false)
+	cs.Properties.OrchestratorProfile.OrchestratorVersion = "1.32.0"
+	cs.Properties.OrchestratorProfile.KubernetesConfig.APIServerConfig = make(map[string]string)
+	a = cs.Properties.OrchestratorProfile.KubernetesConfig.APIServerConfig
+	a["--feature-gates"] = featuregate133
+	cs.setAPIServerConfig()
+	actualList := strings.Split(a["--feature-gates"], ",")
+	expectedList := strings.Split(featuregate133, ",")
+	for _, exp := range expectedList {
+		found := false
+		for _, act := range actualList {
+			if act == exp {
+				found = true
+				break
+			}
+		}
+		if !found {
+			t.Fatalf("missing feature-gate %q in actual '--feature-gates' for %s\nfeaturegate133 (expected subset): %q\nactual: %q",
+				exp, "1.32.0", featuregate133, a["--feature-gates"])
+		}
+	}
 }
 
 func TestAPIServerInsecureFlag(t *testing.T) {

pkg/api/defaults-cloud-controller-manager.go

@@ -239,6 +239,55 @@ func (cs *ContainerService) setCloudControllerManagerConfig() {
 		// Remove --feature-gate SkipReadOnlyValidationGCE starting with 1.31
 		invalidFeatureGates = append(invalidFeatureGates, "SkipReadOnlyValidationGCE")
 	}
+	if common.IsKubernetesVersionGe(o.OrchestratorVersion, "1.33.0") {
+		// Remove --feature-gate AdmissionWebhookMatchConditions starting with 1.33
+		invalidFeatureGates = append(invalidFeatureGates, "AdmissionWebhookMatchConditions")
+
+		// Remove --feature-gate AggregatedDiscoveryEndpoint starting with 1.33
+		invalidFeatureGates = append(invalidFeatureGates, "AggregatedDiscoveryEndpoint")
+
+		// Remove --feature-gate APIListChunking starting with 1.33
+		invalidFeatureGates = append(invalidFeatureGates, "APIListChunking")
+
+		// Remove --feature-gate AppArmor starting with 1.33
+		invalidFeatureGates = append(invalidFeatureGates, "AppArmor")
+
+		// Remove --feature-gate AppArmorFields starting with 1.33
+		invalidFeatureGates = append(invalidFeatureGates, "AppArmorFields")
+
+		// Remove --feature-gate CPUManager starting with 1.33
+		invalidFeatureGates = append(invalidFeatureGates, "CPUManager")
+
+		// Remove --feature-gate DisableCloudProviders starting with 1.33
+		invalidFeatureGates = append(invalidFeatureGates, "DisableCloudProviders")
+
+		// Remove --feature-gate DisableKubeletCloudCredentialProviders starting with 1.33
+		invalidFeatureGates = append(invalidFeatureGates, "DisableKubeletCloudCredentialProviders")
+
+		// Remove --feature-gate EfficientWatchResumption starting with 1.33
+		invalidFeatureGates = append(invalidFeatureGates, "EfficientWatchResumption")
+
+		// Remove --feature-gate JobPodFailurePolicy starting with 1.33
+		invalidFeatureGates = append(invalidFeatureGates, "JobPodFailurePolicy")
+
+		// Remove --feature-gate KubeProxyDrainingTerminatingNodes starting with 1.33
+		invalidFeatureGates = append(invalidFeatureGates, "KubeProxyDrainingTerminatingNodes")
+
+		// Remove --feature-gate PDBUnhealthyPodEvictionPolicy starting with 1.33
+		invalidFeatureGates = append(invalidFeatureGates, "PDBUnhealthyPodEvictionPolicy")
+
+		// Remove --feature-gate PersistentVolumeLastPhaseTransitionTime starting with 1.33
+		invalidFeatureGates = append(invalidFeatureGates, "PersistentVolumeLastPhaseTransitionTime")
+
+		// Remove --feature-gate RemainingItemCount starting with 1.33
+		invalidFeatureGates = append(invalidFeatureGates, "RemainingItemCount")
+
+		// Remove --feature-gate VolumeCapacityPriority starting with 1.33
+		invalidFeatureGates = append(invalidFeatureGates, "VolumeCapacityPriority")
+
+		// Remove --feature-gate WatchBookmark starting with 1.33
+		invalidFeatureGates = append(invalidFeatureGates, "WatchBookmark")
+	}
 
 	removeInvalidFeatureGates(o.KubernetesConfig.CloudControllerManagerConfig, invalidFeatureGates)
 

pkg/api/defaults-cloud-controller-manager_test.go

@@ -4,6 +4,7 @@
 package api
 
 import (
+	"strings"
 	"testing"
 
 	"github.com/Azure/aks-engine-azurestack/pkg/api/common"
@@ -205,4 +206,46 @@ func TestCloudControllerManagerFeatureGates(t *testing.T) {
 		t.Fatalf("got unexpected '--feature-gates' for %s \n Cloud Controller Manager config original value  %s \n, actual sanitized value: %s \n, expected sanitized value: %s \n ",
 			"1.30.0", featuregate131, ccm["--feature-gates"], featuregate130Sanitized)
 	}
+
+	// test user-overrides, removal of feature gates for k8s versions >= 1.33
+	cs = CreateMockContainerService("testcluster", defaultTestClusterVer, 3, 2, false)
+	cs.Properties.OrchestratorProfile.OrchestratorVersion = "1.33.0"
+	cs.Properties.OrchestratorProfile.KubernetesConfig.CloudControllerManagerConfig = make(map[string]string)
+	ccm = cs.Properties.OrchestratorProfile.KubernetesConfig.CloudControllerManagerConfig
+	featuregate133 := "AdmissionWebhookMatchConditions=true,AggregatedDiscoveryEndpoint=true,APIListChunking=true,AppArmor=true,AppArmorFields=true,CPUManager=true,DisableCloudProviders=true,DisableKubeletCloudCredentialProviders=true,EfficientWatchResumption=true,JobPodFailurePolicy=true,KubeProxyDrainingTerminatingNodes=true,PDBUnhealthyPodEvictionPolicy=true,PersistentVolumeLastPhaseTransitionTime=true,RemainingItemCount=true,VolumeCapacityPriority=true,WatchBookmark=true"
+	ccm["--feature-gates"] = featuregate133
+	cs.setCloudControllerManagerConfig()
+	// split both strings by ", " and ensure no original item exists in the sanitized list
+	originalList := strings.Split(featuregate133, ",")
+	sanitizedList := strings.Split(ccm["--feature-gates"], ",")
+	for _, of := range originalList {
+		for _, sf := range sanitizedList {
+			if of == sf {
+				t.Fatalf("feature-gate %q should not exist in sanitized list for %s\nfeaturegate133 (original): %q\nfeaturegate133Sanitized (actual): %q", sf, "1.33", featuregate133, ccm["--feature-gates"])
+			}
+		}
+	}
+
+	// test user-overrides, no removal of feature gates for k8s versions < 1.33
+	cs = CreateMockContainerService("testcluster", defaultTestClusterVer, 3, 2, false)
+	cs.Properties.OrchestratorProfile.OrchestratorVersion = "1.32.0"
+	cs.Properties.OrchestratorProfile.KubernetesConfig.CloudControllerManagerConfig = make(map[string]string)
+	ccm = cs.Properties.OrchestratorProfile.KubernetesConfig.CloudControllerManagerConfig
+	ccm["--feature-gates"] = featuregate133
+	cs.setCloudControllerManagerConfig()
+	actualList := strings.Split(ccm["--feature-gates"], ",")
+	expectedList := strings.Split(featuregate133, ",")
+	for _, exp := range expectedList {
+		found := false
+		for _, act := range actualList {
+			if act == exp {
+				found = true
+				break
+			}
+		}
+		if !found {
+			t.Fatalf("missing feature-gate %q in actual '--feature-gates' for %s\nfeaturegate133 (expected subset): %q\nactual: %q",
+				exp, "1.32.0", featuregate133, ccm["--feature-gates"])
+		}
+	}
 }

pkg/api/defaults-controller-manager.go

@@ -313,5 +313,54 @@ func (cs *ContainerService) setControllerManagerConfig() {
 		// Remove --feature-gate SkipReadOnlyValidationGCE starting with 1.31
 		invalidFeatureGates = append(invalidFeatureGates, "SkipReadOnlyValidationGCE")
 	}
+	if common.IsKubernetesVersionGe(o.OrchestratorVersion, "1.33.0") {
+		// Remove --feature-gate AdmissionWebhookMatchConditions starting with 1.33
+		invalidFeatureGates = append(invalidFeatureGates, "AdmissionWebhookMatchConditions")
+
+		// Remove --feature-gate AggregatedDiscoveryEndpoint starting with 1.33
+		invalidFeatureGates = append(invalidFeatureGates, "AggregatedDiscoveryEndpoint")
+
+		// Remove --feature-gate APIListChunking starting with 1.33
+		invalidFeatureGates = append(invalidFeatureGates, "APIListChunking")
+
+		// Remove --feature-gate AppArmor starting with 1.33
+		invalidFeatureGates = append(invalidFeatureGates, "AppArmor")
+
+		// Remove --feature-gate AppArmorFields starting with 1.33
+		invalidFeatureGates = append(invalidFeatureGates, "AppArmorFields")
+
+		// Remove --feature-gate CPUManager starting with 1.33
+		invalidFeatureGates = append(invalidFeatureGates, "CPUManager")
+
+		// Remove --feature-gate DisableCloudProviders starting with 1.33
+		invalidFeatureGates = append(invalidFeatureGates, "DisableCloudProviders")
+
+		// Remove --feature-gate DisableKubeletCloudCredentialProviders starting with 1.33
+		invalidFeatureGates = append(invalidFeatureGates, "DisableKubeletCloudCredentialProviders")
+
+		// Remove --feature-gate EfficientWatchResumption starting with 1.33
+		invalidFeatureGates = append(invalidFeatureGates, "EfficientWatchResumption")
+
+		// Remove --feature-gate JobPodFailurePolicy starting with 1.33
+		invalidFeatureGates = append(invalidFeatureGates, "JobPodFailurePolicy")
+
+		// Remove --feature-gate KubeProxyDrainingTerminatingNodes starting with 1.33
+		invalidFeatureGates = append(invalidFeatureGates, "KubeProxyDrainingTerminatingNodes")
+
+		// Remove --feature-gate PDBUnhealthyPodEvictionPolicy starting with 1.33
+		invalidFeatureGates = append(invalidFeatureGates, "PDBUnhealthyPodEvictionPolicy")
+
+		// Remove --feature-gate PersistentVolumeLastPhaseTransitionTime starting with 1.33
+		invalidFeatureGates = append(invalidFeatureGates, "PersistentVolumeLastPhaseTransitionTime")
+
+		// Remove --feature-gate RemainingItemCount starting with 1.33
+		invalidFeatureGates = append(invalidFeatureGates, "RemainingItemCount")
+
+		// Remove --feature-gate VolumeCapacityPriority starting with 1.33
+		invalidFeatureGates = append(invalidFeatureGates, "VolumeCapacityPriority")
+
+		// Remove --feature-gate WatchBookmark starting with 1.33
+		invalidFeatureGates = append(invalidFeatureGates, "WatchBookmark")
+	}
 	removeInvalidFeatureGates(o.KubernetesConfig.ControllerManagerConfig, invalidFeatureGates)
 }