diff --git a/cmd/generate_test.go b/cmd/generate_test.go index 8a899eadd..00fe24837 100644 --- a/cmd/generate_test.go +++ b/cmd/generate_test.go @@ -553,8 +553,8 @@ func TestExampleAPIModels(t *testing.T) { setArgs: defaultSet, }, { - name: "1.30 example", - apiModelPath: "../examples/kubernetes-releases/kubernetes1.30.json", + name: "1.33 example", + apiModelPath: "../examples/kubernetes-releases/kubernetes1.33.json", setArgs: defaultSet, }, { diff --git a/examples/azure-stack/kubernetes-azurestack.json b/examples/azure-stack/kubernetes-azurestack.json index d8f759c0d..30a31fc5e 100644 --- a/examples/azure-stack/kubernetes-azurestack.json +++ b/examples/azure-stack/kubernetes-azurestack.json @@ -3,8 +3,8 @@ "location": "", "properties": { "orchestratorProfile": { - "orchestratorRelease": "1.30", - "orchestratorVersion": "1.30.14", + "orchestratorRelease": "1.31", + "orchestratorVersion": "1.31.11", "kubernetesConfig": { "cloudProviderBackoff": true, "cloudProviderBackoffRetries": 1, diff --git a/examples/azure-stack/kubernetes-windows.json b/examples/azure-stack/kubernetes-windows.json index 1d3c26436..d6fa23f62 100644 --- a/examples/azure-stack/kubernetes-windows.json +++ b/examples/azure-stack/kubernetes-windows.json @@ -3,8 +3,8 @@ "location": "", "properties": { "orchestratorProfile": { - "orchestratorRelease": "1.30", - "orchestratorVersion": "1.30.14", + "orchestratorRelease": "1.31", + "orchestratorVersion": "1.31.11", "kubernetesConfig": { "cloudProviderBackoff": true, "cloudProviderBackoffRetries": 1, diff --git a/examples/kubernetes-releases/kubernetes1.33.json b/examples/kubernetes-releases/kubernetes1.33.json new file mode 100644 index 000000000..481ecf207 --- /dev/null +++ b/examples/kubernetes-releases/kubernetes1.33.json @@ -0,0 +1,30 @@ +{ + "apiVersion": "vlabs", + "properties": { + "orchestratorProfile": { + "orchestratorRelease": "1.33" + }, + "masterProfile": { + "count": 1, + "dnsPrefix": "", + "vmSize": "Standard_D2_v3" + }, + "agentPoolProfiles": [ + { + "name": "agentpool1", + "count": 3, + "vmSize": "Standard_D2_v3" + } + ], + "linuxProfile": { + "adminUsername": "azureuser", + "ssh": { + "publicKeys": [ + { + "keyData": "" + } + ] + } + } + } +} \ No newline at end of file diff --git a/hack/tools/Makefile b/hack/tools/Makefile index 8f69617f3..dbaf8a78a 100644 --- a/hack/tools/Makefile +++ b/hack/tools/Makefile @@ -3,7 +3,7 @@ LOCALBIN = $(shell pwd)/bin SHELLCHECK_VERSION ?= v0.8.0 AZCLI_VERSION ?= 2.56.0 PYWINRM_VERSION ?= 0.4.3 -KUBECTL_VERSION ?= v1.30.14 +KUBECTL_VERSION ?= v1.31.11 all: install diff --git a/pkg/api/common/const.go b/pkg/api/common/const.go index 375a40f8e..28ec96fb1 100644 --- a/pkg/api/common/const.go +++ b/pkg/api/common/const.go @@ -57,13 +57,13 @@ const ( const ( // KubernetesDefaultRelease is the default Kubernetes release - KubernetesDefaultRelease string = "1.30" + KubernetesDefaultRelease string = "1.32" // KubernetesDefaultReleaseWindows is the default Kubernetes release for Windows - KubernetesDefaultReleaseWindows string = "1.30" + KubernetesDefaultReleaseWindows string = "1.32" // KubernetesDefaultReleaseAzureStack is the default Kubernetes release on Azure Stack - KubernetesDefaultReleaseAzureStack string = "1.30" + KubernetesDefaultReleaseAzureStack string = "1.32" // KubernetesDefaultReleaseWindowsAzureStack is the default Kubernetes release for Windows on Azure Stack - KubernetesDefaultReleaseWindowsAzureStack string = "1.30" + KubernetesDefaultReleaseWindowsAzureStack string = "1.32" // PodSecurityPolicyRemovedVersion is the first Kubernetes version that does not includes the PSP admission plugin PodSecurityPolicyRemovedVersion = "1.25.0" ) diff --git a/pkg/api/common/versions.go b/pkg/api/common/versions.go index 2f1b78d35..ce400aa59 100644 --- a/pkg/api/common/versions.go +++ b/pkg/api/common/versions.go @@ -368,8 +368,9 @@ var AllKubernetesSupportedVersions = map[string]bool{ "1.29.10": false, "1.29.15": false, "1.30.10": false, - "1.30.14": true, + "1.30.14": false, "1.31.11": true, + "1.33.5": true, } // AllKubernetesSupportedVersionsAzureStack is a hash table of all supported Kubernetes version strings on Azure Stack @@ -428,8 +429,9 @@ var AllKubernetesSupportedVersionsAzureStack = map[string]bool{ "1.29.10": false, "1.29.15": false, "1.30.10": false, - "1.30.14": true, + "1.30.14": false, "1.31.11": true, + "1.33.5": true, } // AllKubernetesWindowsSupportedVersionsAzureStack maintain a set of available k8s Windows versions in aks-engine on Azure Stack @@ -483,8 +485,9 @@ var AllKubernetesWindowsSupportedVersionsAzureStack = map[string]bool{ "1.29.10": false, "1.29.15": false, "1.30.10": false, - "1.30.14": true, + "1.30.14": false, "1.31.11": true, + "1.33.5": true, } // GetDefaultKubernetesVersion returns the default Kubernetes version, that is the latest patch of the default release diff --git a/pkg/api/defaults-apiserver.go b/pkg/api/defaults-apiserver.go index 927f12ac0..e96bba8f4 100644 --- a/pkg/api/defaults-apiserver.go +++ b/pkg/api/defaults-apiserver.go @@ -396,6 +396,55 @@ func (cs *ContainerService) overrideAPIServerConfig() { // Remove --feature-gate SkipReadOnlyValidationGCE starting with 1.31 invalidFeatureGates = append(invalidFeatureGates, "SkipReadOnlyValidationGCE") } + if common.IsKubernetesVersionGe(o.OrchestratorVersion, "1.33.0") { + // Remove --feature-gate AdmissionWebhookMatchConditions starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "AdmissionWebhookMatchConditions") + + // Remove --feature-gate AggregatedDiscoveryEndpoint starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "AggregatedDiscoveryEndpoint") + + // Remove --feature-gate APIListChunking starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "APIListChunking") + + // Remove --feature-gate AppArmor starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "AppArmor") + + // Remove --feature-gate AppArmorFields starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "AppArmorFields") + + // Remove --feature-gate CPUManager starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "CPUManager") + + // Remove --feature-gate DisableCloudProviders starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "DisableCloudProviders") + + // Remove --feature-gate DisableKubeletCloudCredentialProviders starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "DisableKubeletCloudCredentialProviders") + + // Remove --feature-gate EfficientWatchResumption starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "EfficientWatchResumption") + + // Remove --feature-gate JobPodFailurePolicy starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "JobPodFailurePolicy") + + // Remove --feature-gate KubeProxyDrainingTerminatingNodes starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "KubeProxyDrainingTerminatingNodes") + + // Remove --feature-gate PDBUnhealthyPodEvictionPolicy starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "PDBUnhealthyPodEvictionPolicy") + + // Remove --feature-gate PersistentVolumeLastPhaseTransitionTime starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "PersistentVolumeLastPhaseTransitionTime") + + // Remove --feature-gate RemainingItemCount starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "RemainingItemCount") + + // Remove --feature-gate VolumeCapacityPriority starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "VolumeCapacityPriority") + + // Remove --feature-gate WatchBookmark starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "WatchBookmark") + } removeInvalidFeatureGates(o.KubernetesConfig.APIServerConfig, invalidFeatureGates) if common.ShouldDisablePodSecurityPolicyAddon(o.OrchestratorVersion) { diff --git a/pkg/api/defaults-apiserver_test.go b/pkg/api/defaults-apiserver_test.go index 305515e36..13e40aa75 100644 --- a/pkg/api/defaults-apiserver_test.go +++ b/pkg/api/defaults-apiserver_test.go @@ -782,6 +782,48 @@ func TestAPIServerFeatureGates(t *testing.T) { t.Fatalf("got unexpected '--feature-gates' for %s \n API server config original value %s \n, actual sanitized value: %s \n, expected sanitized value: %s \n ", "1.30.0", featuregate131, a["--feature-gates"], featuregate130Sanitized) } + + // test user-overrides, removal of feature gates for k8s versions >= 1.33 + cs = CreateMockContainerService("testcluster", defaultTestClusterVer, 3, 2, false) + cs.Properties.OrchestratorProfile.OrchestratorVersion = "1.33.0" + cs.Properties.OrchestratorProfile.KubernetesConfig.APIServerConfig = make(map[string]string) + a = cs.Properties.OrchestratorProfile.KubernetesConfig.APIServerConfig + featuregate133 := "AdmissionWebhookMatchConditions=true,AggregatedDiscoveryEndpoint=true,APIListChunking=true,AppArmor=true,AppArmorFields=true,CPUManager=true,DisableCloudProviders=true,DisableKubeletCloudCredentialProviders=true,EfficientWatchResumption=true,JobPodFailurePolicy=true,KubeProxyDrainingTerminatingNodes=true,PDBUnhealthyPodEvictionPolicy=true,PersistentVolumeLastPhaseTransitionTime=true,RemainingItemCount=true,VolumeCapacityPriority=true,WatchBookmark=true" + a["--feature-gates"] = featuregate133 + cs.setAPIServerConfig() + // split both strings by ", " and ensure no original item exists in the sanitized list + originalList := strings.Split(featuregate133, ",") + sanitizedList := strings.Split(a["--feature-gates"], ",") + for _, of := range originalList { + for _, sf := range sanitizedList { + if of == sf { + t.Fatalf("feature-gate %q should not exist in sanitized list for %s\nfeaturegate133 (original): %q\nfeaturegate133Sanitized (actual): %q", sf, "1.33", featuregate133, a["--feature-gates"]) + } + } + } + + // test user-overrides, no removal of feature gates for k8s versions < 1.33 + cs = CreateMockContainerService("testcluster", defaultTestClusterVer, 3, 2, false) + cs.Properties.OrchestratorProfile.OrchestratorVersion = "1.32.0" + cs.Properties.OrchestratorProfile.KubernetesConfig.APIServerConfig = make(map[string]string) + a = cs.Properties.OrchestratorProfile.KubernetesConfig.APIServerConfig + a["--feature-gates"] = featuregate133 + cs.setAPIServerConfig() + actualList := strings.Split(a["--feature-gates"], ",") + expectedList := strings.Split(featuregate133, ",") + for _, exp := range expectedList { + found := false + for _, act := range actualList { + if act == exp { + found = true + break + } + } + if !found { + t.Fatalf("missing feature-gate %q in actual '--feature-gates' for %s\nfeaturegate133 (expected subset): %q\nactual: %q", + exp, "1.32.0", featuregate133, a["--feature-gates"]) + } + } } func TestAPIServerInsecureFlag(t *testing.T) { diff --git a/pkg/api/defaults-cloud-controller-manager.go b/pkg/api/defaults-cloud-controller-manager.go index 073915bb6..a8331cd03 100644 --- a/pkg/api/defaults-cloud-controller-manager.go +++ b/pkg/api/defaults-cloud-controller-manager.go @@ -239,6 +239,55 @@ func (cs *ContainerService) setCloudControllerManagerConfig() { // Remove --feature-gate SkipReadOnlyValidationGCE starting with 1.31 invalidFeatureGates = append(invalidFeatureGates, "SkipReadOnlyValidationGCE") } + if common.IsKubernetesVersionGe(o.OrchestratorVersion, "1.33.0") { + // Remove --feature-gate AdmissionWebhookMatchConditions starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "AdmissionWebhookMatchConditions") + + // Remove --feature-gate AggregatedDiscoveryEndpoint starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "AggregatedDiscoveryEndpoint") + + // Remove --feature-gate APIListChunking starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "APIListChunking") + + // Remove --feature-gate AppArmor starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "AppArmor") + + // Remove --feature-gate AppArmorFields starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "AppArmorFields") + + // Remove --feature-gate CPUManager starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "CPUManager") + + // Remove --feature-gate DisableCloudProviders starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "DisableCloudProviders") + + // Remove --feature-gate DisableKubeletCloudCredentialProviders starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "DisableKubeletCloudCredentialProviders") + + // Remove --feature-gate EfficientWatchResumption starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "EfficientWatchResumption") + + // Remove --feature-gate JobPodFailurePolicy starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "JobPodFailurePolicy") + + // Remove --feature-gate KubeProxyDrainingTerminatingNodes starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "KubeProxyDrainingTerminatingNodes") + + // Remove --feature-gate PDBUnhealthyPodEvictionPolicy starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "PDBUnhealthyPodEvictionPolicy") + + // Remove --feature-gate PersistentVolumeLastPhaseTransitionTime starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "PersistentVolumeLastPhaseTransitionTime") + + // Remove --feature-gate RemainingItemCount starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "RemainingItemCount") + + // Remove --feature-gate VolumeCapacityPriority starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "VolumeCapacityPriority") + + // Remove --feature-gate WatchBookmark starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "WatchBookmark") + } removeInvalidFeatureGates(o.KubernetesConfig.CloudControllerManagerConfig, invalidFeatureGates) diff --git a/pkg/api/defaults-cloud-controller-manager_test.go b/pkg/api/defaults-cloud-controller-manager_test.go index f7af378db..177d0d553 100644 --- a/pkg/api/defaults-cloud-controller-manager_test.go +++ b/pkg/api/defaults-cloud-controller-manager_test.go @@ -4,6 +4,7 @@ package api import ( + "strings" "testing" "github.com/Azure/aks-engine-azurestack/pkg/api/common" @@ -205,4 +206,46 @@ func TestCloudControllerManagerFeatureGates(t *testing.T) { t.Fatalf("got unexpected '--feature-gates' for %s \n Cloud Controller Manager config original value %s \n, actual sanitized value: %s \n, expected sanitized value: %s \n ", "1.30.0", featuregate131, ccm["--feature-gates"], featuregate130Sanitized) } + + // test user-overrides, removal of feature gates for k8s versions >= 1.33 + cs = CreateMockContainerService("testcluster", defaultTestClusterVer, 3, 2, false) + cs.Properties.OrchestratorProfile.OrchestratorVersion = "1.33.0" + cs.Properties.OrchestratorProfile.KubernetesConfig.CloudControllerManagerConfig = make(map[string]string) + ccm = cs.Properties.OrchestratorProfile.KubernetesConfig.CloudControllerManagerConfig + featuregate133 := "AdmissionWebhookMatchConditions=true,AggregatedDiscoveryEndpoint=true,APIListChunking=true,AppArmor=true,AppArmorFields=true,CPUManager=true,DisableCloudProviders=true,DisableKubeletCloudCredentialProviders=true,EfficientWatchResumption=true,JobPodFailurePolicy=true,KubeProxyDrainingTerminatingNodes=true,PDBUnhealthyPodEvictionPolicy=true,PersistentVolumeLastPhaseTransitionTime=true,RemainingItemCount=true,VolumeCapacityPriority=true,WatchBookmark=true" + ccm["--feature-gates"] = featuregate133 + cs.setCloudControllerManagerConfig() + // split both strings by ", " and ensure no original item exists in the sanitized list + originalList := strings.Split(featuregate133, ",") + sanitizedList := strings.Split(ccm["--feature-gates"], ",") + for _, of := range originalList { + for _, sf := range sanitizedList { + if of == sf { + t.Fatalf("feature-gate %q should not exist in sanitized list for %s\nfeaturegate133 (original): %q\nfeaturegate133Sanitized (actual): %q", sf, "1.33", featuregate133, ccm["--feature-gates"]) + } + } + } + + // test user-overrides, no removal of feature gates for k8s versions < 1.33 + cs = CreateMockContainerService("testcluster", defaultTestClusterVer, 3, 2, false) + cs.Properties.OrchestratorProfile.OrchestratorVersion = "1.32.0" + cs.Properties.OrchestratorProfile.KubernetesConfig.CloudControllerManagerConfig = make(map[string]string) + ccm = cs.Properties.OrchestratorProfile.KubernetesConfig.CloudControllerManagerConfig + ccm["--feature-gates"] = featuregate133 + cs.setCloudControllerManagerConfig() + actualList := strings.Split(ccm["--feature-gates"], ",") + expectedList := strings.Split(featuregate133, ",") + for _, exp := range expectedList { + found := false + for _, act := range actualList { + if act == exp { + found = true + break + } + } + if !found { + t.Fatalf("missing feature-gate %q in actual '--feature-gates' for %s\nfeaturegate133 (expected subset): %q\nactual: %q", + exp, "1.32.0", featuregate133, ccm["--feature-gates"]) + } + } } diff --git a/pkg/api/defaults-controller-manager.go b/pkg/api/defaults-controller-manager.go index 68853fa83..f0a0e43c2 100644 --- a/pkg/api/defaults-controller-manager.go +++ b/pkg/api/defaults-controller-manager.go @@ -313,5 +313,54 @@ func (cs *ContainerService) setControllerManagerConfig() { // Remove --feature-gate SkipReadOnlyValidationGCE starting with 1.31 invalidFeatureGates = append(invalidFeatureGates, "SkipReadOnlyValidationGCE") } + if common.IsKubernetesVersionGe(o.OrchestratorVersion, "1.33.0") { + // Remove --feature-gate AdmissionWebhookMatchConditions starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "AdmissionWebhookMatchConditions") + + // Remove --feature-gate AggregatedDiscoveryEndpoint starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "AggregatedDiscoveryEndpoint") + + // Remove --feature-gate APIListChunking starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "APIListChunking") + + // Remove --feature-gate AppArmor starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "AppArmor") + + // Remove --feature-gate AppArmorFields starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "AppArmorFields") + + // Remove --feature-gate CPUManager starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "CPUManager") + + // Remove --feature-gate DisableCloudProviders starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "DisableCloudProviders") + + // Remove --feature-gate DisableKubeletCloudCredentialProviders starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "DisableKubeletCloudCredentialProviders") + + // Remove --feature-gate EfficientWatchResumption starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "EfficientWatchResumption") + + // Remove --feature-gate JobPodFailurePolicy starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "JobPodFailurePolicy") + + // Remove --feature-gate KubeProxyDrainingTerminatingNodes starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "KubeProxyDrainingTerminatingNodes") + + // Remove --feature-gate PDBUnhealthyPodEvictionPolicy starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "PDBUnhealthyPodEvictionPolicy") + + // Remove --feature-gate PersistentVolumeLastPhaseTransitionTime starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "PersistentVolumeLastPhaseTransitionTime") + + // Remove --feature-gate RemainingItemCount starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "RemainingItemCount") + + // Remove --feature-gate VolumeCapacityPriority starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "VolumeCapacityPriority") + + // Remove --feature-gate WatchBookmark starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "WatchBookmark") + } removeInvalidFeatureGates(o.KubernetesConfig.ControllerManagerConfig, invalidFeatureGates) } diff --git a/pkg/api/defaults-controller-manager_test.go b/pkg/api/defaults-controller-manager_test.go index 4cb13dfd9..62d4c46d9 100644 --- a/pkg/api/defaults-controller-manager_test.go +++ b/pkg/api/defaults-controller-manager_test.go @@ -4,6 +4,7 @@ package api import ( + "strings" "testing" "github.com/Azure/aks-engine-azurestack/pkg/helpers/to" @@ -300,6 +301,48 @@ func TestControllerManagerConfigFeatureGates(t *testing.T) { t.Fatalf("got unexpected '--feature-gates' for %s \n controller manager config original value %s \n, actual sanitized value: %s \n, expected sanitized value: %s \n ", "1.30.0", featuregate131, cm["--feature-gates"], featuregate130Sanitized) } + + // test user-overrides, removal of feature gates for k8s versions >= 1.33 + cs = CreateMockContainerService("testcluster", defaultTestClusterVer, 3, 2, false) + cs.Properties.OrchestratorProfile.OrchestratorVersion = "1.33.0" + cs.Properties.OrchestratorProfile.KubernetesConfig.ControllerManagerConfig = make(map[string]string) + cm = cs.Properties.OrchestratorProfile.KubernetesConfig.ControllerManagerConfig + featuregate133 := "AdmissionWebhookMatchConditions=true,AggregatedDiscoveryEndpoint=true,APIListChunking=true,AppArmor=true,AppArmorFields=true,CPUManager=true,DisableCloudProviders=true,DisableKubeletCloudCredentialProviders=true,EfficientWatchResumption=true,JobPodFailurePolicy=true,KubeProxyDrainingTerminatingNodes=true,PDBUnhealthyPodEvictionPolicy=true,PersistentVolumeLastPhaseTransitionTime=true,RemainingItemCount=true,VolumeCapacityPriority=true,WatchBookmark=true" + cm["--feature-gates"] = featuregate133 + cs.setControllerManagerConfig() + // split both strings by ", " and ensure no original item exists in the sanitized list + originalList := strings.Split(featuregate133, ",") + sanitizedList := strings.Split(cm["--feature-gates"], ",") + for _, of := range originalList { + for _, sf := range sanitizedList { + if of == sf { + t.Fatalf("feature-gate %q should not exist in sanitized list for %s\nfeaturegate133 (original): %q\nfeaturegate133Sanitized (actual): %q", sf, "1.33", featuregate133, cm["--feature-gates"]) + } + } + } + + // test user-overrides, no removal of feature gates for k8s versions < 1.33 + cs = CreateMockContainerService("testcluster", defaultTestClusterVer, 3, 2, false) + cs.Properties.OrchestratorProfile.OrchestratorVersion = "1.32.0" + cs.Properties.OrchestratorProfile.KubernetesConfig.ControllerManagerConfig = make(map[string]string) + cm = cs.Properties.OrchestratorProfile.KubernetesConfig.ControllerManagerConfig + cm["--feature-gates"] = featuregate133 + cs.setControllerManagerConfig() + actualList := strings.Split(cm["--feature-gates"], ",") + expectedList := strings.Split(featuregate133, ",") + for _, exp := range expectedList { + found := false + for _, act := range actualList { + if act == exp { + found = true + break + } + } + if !found { + t.Fatalf("missing feature-gate %q in actual '--feature-gates' for %s\nfeaturegate133 (expected subset): %q\nactual: %q", + exp, "1.32.0", featuregate133, cm["--feature-gates"]) + } + } } func TestControllerManagerDefaultConfig(t *testing.T) { diff --git a/pkg/api/defaults-kubelet.go b/pkg/api/defaults-kubelet.go index 863d1e746..61396d18c 100644 --- a/pkg/api/defaults-kubelet.go +++ b/pkg/api/defaults-kubelet.go @@ -361,6 +361,55 @@ func (cs *ContainerService) setKubeletConfig(isUpgrade bool) { // Remove --feature-gate SkipReadOnlyValidationGCE starting with 1.31 invalidFeatureGates = append(invalidFeatureGates, "SkipReadOnlyValidationGCE") } + if common.IsKubernetesVersionGe(o.OrchestratorVersion, "1.33.0") { + // Remove --feature-gate AdmissionWebhookMatchConditions starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "AdmissionWebhookMatchConditions") + + // Remove --feature-gate AggregatedDiscoveryEndpoint starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "AggregatedDiscoveryEndpoint") + + // Remove --feature-gate APIListChunking starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "APIListChunking") + + // Remove --feature-gate AppArmor starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "AppArmor") + + // Remove --feature-gate AppArmorFields starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "AppArmorFields") + + // Remove --feature-gate CPUManager starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "CPUManager") + + // Remove --feature-gate DisableCloudProviders starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "DisableCloudProviders") + + // Remove --feature-gate DisableKubeletCloudCredentialProviders starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "DisableKubeletCloudCredentialProviders") + + // Remove --feature-gate EfficientWatchResumption starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "EfficientWatchResumption") + + // Remove --feature-gate JobPodFailurePolicy starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "JobPodFailurePolicy") + + // Remove --feature-gate KubeProxyDrainingTerminatingNodes starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "KubeProxyDrainingTerminatingNodes") + + // Remove --feature-gate PDBUnhealthyPodEvictionPolicy starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "PDBUnhealthyPodEvictionPolicy") + + // Remove --feature-gate PersistentVolumeLastPhaseTransitionTime starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "PersistentVolumeLastPhaseTransitionTime") + + // Remove --feature-gate RemainingItemCount starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "RemainingItemCount") + + // Remove --feature-gate VolumeCapacityPriority starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "VolumeCapacityPriority") + + // Remove --feature-gate WatchBookmark starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "WatchBookmark") + } removeInvalidFeatureGates(o.KubernetesConfig.KubeletConfig, invalidFeatureGates) // Master-specific kubelet config changes go here diff --git a/pkg/api/defaults-kubelet_test.go b/pkg/api/defaults-kubelet_test.go index 0585d8eac..c2544f2c5 100644 --- a/pkg/api/defaults-kubelet_test.go +++ b/pkg/api/defaults-kubelet_test.go @@ -1083,6 +1083,48 @@ func TestKubeletConfigFeatureGates(t *testing.T) { t.Fatalf("got unexpected '--feature-gates' for %s \n kubelet config original value %s \n, actual sanitized value: %s \n, expected sanitized value: %s \n ", "1.30.0", featuregate131, k["--feature-gates"], featuregate130Sanitized) } + + // test user-overrides, removal of feature gates for k8s versions >= 1.33 + cs = CreateMockContainerService("testcluster", defaultTestClusterVer, 3, 2, false) + cs.Properties.OrchestratorProfile.OrchestratorVersion = "1.33.0" + cs.Properties.OrchestratorProfile.KubernetesConfig.KubeletConfig = make(map[string]string) + k = cs.Properties.OrchestratorProfile.KubernetesConfig.KubeletConfig + featuregate133 := "AdmissionWebhookMatchConditions=true,AggregatedDiscoveryEndpoint=true,APIListChunking=true,AppArmor=true,AppArmorFields=true,CPUManager=true,DisableCloudProviders=true,DisableKubeletCloudCredentialProviders=true,EfficientWatchResumption=true,JobPodFailurePolicy=true,KubeProxyDrainingTerminatingNodes=true,PDBUnhealthyPodEvictionPolicy=true,PersistentVolumeLastPhaseTransitionTime=true,RemainingItemCount=true,VolumeCapacityPriority=true,WatchBookmark=true" + k["--feature-gates"] = featuregate133 + cs.setKubeletConfig(false) + // split both strings by ", " and ensure no original item exists in the sanitized list + originalList := strings.Split(featuregate133, ",") + sanitizedList := strings.Split(k["--feature-gates"], ",") + for _, of := range originalList { + for _, sf := range sanitizedList { + if of == sf { + t.Fatalf("feature-gate %q should not exist in sanitized list for %s\nfeaturegate133 (original): %q\nfeaturegate133Sanitized (actual): %q", sf, "1.33", featuregate133, k["--feature-gates"]) + } + } + } + + // test user-overrides, no removal of feature gates for k8s versions < 1.33 + cs = CreateMockContainerService("testcluster", defaultTestClusterVer, 3, 2, false) + cs.Properties.OrchestratorProfile.OrchestratorVersion = "1.32.0" + cs.Properties.OrchestratorProfile.KubernetesConfig.KubeletConfig = make(map[string]string) + k = cs.Properties.OrchestratorProfile.KubernetesConfig.KubeletConfig + k["--feature-gates"] = featuregate133 + cs.setKubeletConfig(false) + actualList := strings.Split(k["--feature-gates"], ",") + expectedList := strings.Split(featuregate133, ",") + for _, exp := range expectedList { + found := false + for _, act := range actualList { + if act == exp { + found = true + break + } + } + if !found { + t.Fatalf("missing feature-gate %q in actual '--feature-gates' for %s\nfeaturegate133 (expected subset): %q\nactual: %q", + exp, "1.32.0", featuregate133, k["--feature-gates"]) + } + } } func TestKubeletStrongCipherSuites(t *testing.T) { diff --git a/pkg/api/defaults-scheduler.go b/pkg/api/defaults-scheduler.go index cd235ae16..bd98a9925 100644 --- a/pkg/api/defaults-scheduler.go +++ b/pkg/api/defaults-scheduler.go @@ -219,6 +219,55 @@ func (cs *ContainerService) setSchedulerConfig() { // Remove --feature-gate SkipReadOnlyValidationGCE starting with 1.31 invalidFeatureGates = append(invalidFeatureGates, "SkipReadOnlyValidationGCE") } + if common.IsKubernetesVersionGe(o.OrchestratorVersion, "1.33.0") { + // Remove --feature-gate AdmissionWebhookMatchConditions starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "AdmissionWebhookMatchConditions") + + // Remove --feature-gate AggregatedDiscoveryEndpoint starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "AggregatedDiscoveryEndpoint") + + // Remove --feature-gate APIListChunking starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "APIListChunking") + + // Remove --feature-gate AppArmor starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "AppArmor") + + // Remove --feature-gate AppArmorFields starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "AppArmorFields") + + // Remove --feature-gate CPUManager starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "CPUManager") + + // Remove --feature-gate DisableCloudProviders starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "DisableCloudProviders") + + // Remove --feature-gate DisableKubeletCloudCredentialProviders starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "DisableKubeletCloudCredentialProviders") + + // Remove --feature-gate EfficientWatchResumption starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "EfficientWatchResumption") + + // Remove --feature-gate JobPodFailurePolicy starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "JobPodFailurePolicy") + + // Remove --feature-gate KubeProxyDrainingTerminatingNodes starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "KubeProxyDrainingTerminatingNodes") + + // Remove --feature-gate PDBUnhealthyPodEvictionPolicy starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "PDBUnhealthyPodEvictionPolicy") + + // Remove --feature-gate PersistentVolumeLastPhaseTransitionTime starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "PersistentVolumeLastPhaseTransitionTime") + + // Remove --feature-gate RemainingItemCount starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "RemainingItemCount") + + // Remove --feature-gate VolumeCapacityPriority starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "VolumeCapacityPriority") + + // Remove --feature-gate WatchBookmark starting with 1.33 + invalidFeatureGates = append(invalidFeatureGates, "WatchBookmark") + } removeInvalidFeatureGates(o.KubernetesConfig.SchedulerConfig, invalidFeatureGates) // Replace the flag names diff --git a/pkg/api/defaults-scheduler_test.go b/pkg/api/defaults-scheduler_test.go index 6ab242ba1..0fc958d7f 100644 --- a/pkg/api/defaults-scheduler_test.go +++ b/pkg/api/defaults-scheduler_test.go @@ -4,6 +4,7 @@ package api import ( + "strings" "testing" ) @@ -331,4 +332,46 @@ func TestSchedulerFeatureGates(t *testing.T) { t.Fatalf("got unexpected '--feature-gates' for %s \n Scheduler config original value %s \n, actual sanitized value: %s \n, expected sanitized value: %s \n ", "1.30.0", featuregate131, s["--feature-gates"], featuregate130Sanitized) } + + // test user-overrides, removal of feature gates for k8s versions >= 1.33 + cs = CreateMockContainerService("testcluster", defaultTestClusterVer, 3, 2, false) + cs.Properties.OrchestratorProfile.OrchestratorVersion = "1.33.0" + cs.Properties.OrchestratorProfile.KubernetesConfig.SchedulerConfig = make(map[string]string) + s = cs.Properties.OrchestratorProfile.KubernetesConfig.SchedulerConfig + featuregate133 := "AdmissionWebhookMatchConditions=true,AggregatedDiscoveryEndpoint=true,APIListChunking=true,AppArmor=true,AppArmorFields=true,CPUManager=true,DisableCloudProviders=true,DisableKubeletCloudCredentialProviders=true,EfficientWatchResumption=true,JobPodFailurePolicy=true,KubeProxyDrainingTerminatingNodes=true,PDBUnhealthyPodEvictionPolicy=true,PersistentVolumeLastPhaseTransitionTime=true,RemainingItemCount=true,VolumeCapacityPriority=true,WatchBookmark=true" + s["--feature-gates"] = featuregate133 + cs.setSchedulerConfig() + // split both strings by ", " and ensure no original item exists in the sanitized list + originalList := strings.Split(featuregate133, ",") + sanitizedList := strings.Split(s["--feature-gates"], ",") + for _, of := range originalList { + for _, sf := range sanitizedList { + if of == sf { + t.Fatalf("feature-gate %q should not exist in sanitized list for %s\nfeaturegate133 (original): %q\nfeaturegate133Sanitized (actual): %q", sf, "1.33", featuregate133, s["--feature-gates"]) + } + } + } + + // test user-overrides, no removal of feature gates for k8s versions < 1.33 + cs = CreateMockContainerService("testcluster", defaultTestClusterVer, 3, 2, false) + cs.Properties.OrchestratorProfile.OrchestratorVersion = "1.32.0" + cs.Properties.OrchestratorProfile.KubernetesConfig.SchedulerConfig = make(map[string]string) + s = cs.Properties.OrchestratorProfile.KubernetesConfig.SchedulerConfig + s["--feature-gates"] = featuregate133 + cs.setSchedulerConfig() + actualList := strings.Split(s["--feature-gates"], ",") + expectedList := strings.Split(featuregate133, ",") + for _, exp := range expectedList { + found := false + for _, act := range actualList { + if act == exp { + found = true + break + } + } + if !found { + t.Fatalf("missing feature-gate %q in actual '--feature-gates' for %s\nfeaturegate133 (expected subset): %q\nactual: %q", + exp, "1.32.0", featuregate133, s["--feature-gates"]) + } + } } diff --git a/pkg/api/k8s_versions.go b/pkg/api/k8s_versions.go index 7d1b88d6e..243a7b0d7 100644 --- a/pkg/api/k8s_versions.go +++ b/pkg/api/k8s_versions.go @@ -102,6 +102,22 @@ func getDefaultImage(image, kubernetesImageBaseType string) string { // The map supports GCR or MCR image string flavors var kubernetesImageBaseVersionedImages = map[string]map[string]map[string]string{ common.KubernetesImageBaseTypeGCR: { + "1.33": { + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v5.3.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v4.9.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.16.0", + common.CSILivenessProbeWindowsContainerName: "oss/kubernetes-csi/livenessprobe:v2.16.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.14.0", + common.CSINodeDriverRegistrarWindowsContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.14.0", + common.CSISnapshotterContainerName: "oss/kubernetes-csi/csi-snapshotter:v8.3.0", + common.CSISnapshotControllerContainerName: "oss/kubernetes-csi/snapshot-controller:v8.3.0", + common.CSIResizerContainerName: "oss/kubernetes-csi/csi-resizer:v1.14.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.33.4", + common.AddonResizerComponentName: "addon-resizer:1.8.23", + common.MetricsServerAddonName: "metrics-server/metrics-server:v0.8.0", + common.AddonManagerComponentName: "kube-addon-manager-amd64:v9.1.8", + common.ClusterAutoscalerAddonName: "cluster-autoscaler:v1.33.1", + }, "1.31": { common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v5.1.0", common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v4.7.0", @@ -449,6 +465,22 @@ var kubernetesImageBaseVersionedImages = map[string]map[string]map[string]string }, }, common.KubernetesImageBaseTypeMCR: { + "1.33": { + common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v5.3.0", + common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v4.9.0", + common.CSILivenessProbeContainerName: "oss/kubernetes-csi/livenessprobe:v2.16.0", + common.CSILivenessProbeWindowsContainerName: "oss/kubernetes-csi/livenessprobe:v2.16.0", + common.CSINodeDriverRegistrarContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.14.0", + common.CSINodeDriverRegistrarWindowsContainerName: "oss/kubernetes-csi/csi-node-driver-registrar:v2.14.0", + common.CSISnapshotterContainerName: "oss/kubernetes-csi/csi-snapshotter:v8.3.0", + common.CSISnapshotControllerContainerName: "oss/kubernetes-csi/snapshot-controller:v8.3.0", + common.CSIResizerContainerName: "oss/kubernetes-csi/csi-resizer:v1.14.0", + common.CSIAzureDiskContainerName: "oss/kubernetes-csi/azuredisk-csi:v1.33.4", + common.AddonResizerComponentName: "oss/kubernetes/autoscaler/addon-resizer:1.8.23", + common.MetricsServerAddonName: "oss/kubernetes/metrics-server:v0.8.0", + common.AddonManagerComponentName: "oss/kubernetes/kube-addon-manager:v9.1.8", + common.ClusterAutoscalerAddonName: "oss/kubernetes/autoscaler/cluster-autoscaler:v1.33.1", + }, "1.31": { common.CSIProvisionerContainerName: "oss/kubernetes-csi/csi-provisioner:v5.1.0", common.CSIAttacherContainerName: "oss/kubernetes-csi/csi-attacher:v4.7.0", @@ -912,14 +944,14 @@ func getK8sVersionComponents(version, kubernetesImageBaseType string, overrides var ret map[string]string k8sComponent := kubernetesImageBaseVersionedImages[kubernetesImageBaseType][majorMinor] switch majorMinor { - case "1.31": + case "1.33": ret = map[string]string{ common.APIServerComponentName: getDefaultImage(common.APIServerComponentName, kubernetesImageBaseType) + ":v" + version, common.ControllerManagerComponentName: getDefaultImage(common.ControllerManagerComponentName, kubernetesImageBaseType) + ":v" + version, common.KubeProxyAddonName: getDefaultImage(common.KubeProxyAddonName, kubernetesImageBaseType) + ":v" + version, common.SchedulerComponentName: getDefaultImage(common.SchedulerComponentName, kubernetesImageBaseType) + ":v" + version, - common.CloudControllerManagerComponentName: "oss/kubernetes/azure-cloud-controller-manager:v1.31.1", - common.CloudNodeManagerAddonName: "oss/kubernetes/azure-cloud-node-manager:v1.31.1", + common.CloudControllerManagerComponentName: "oss/kubernetes/azure-cloud-controller-manager:v1.33.1", + common.CloudNodeManagerAddonName: "oss/kubernetes/azure-cloud-node-manager:v1.33.0", common.WindowsArtifactComponentName: "v" + version + "/windowszip/v" + version + "-1int.zip", common.WindowsArtifactAzureStackComponentName: "v" + version + "/windowszip/v" + version + "-1int.zip", common.DashboardAddonName: dashboardImageReference, diff --git a/vhd/packer/configure-windows-vhd-phase2.ps1 b/vhd/packer/configure-windows-vhd-phase2.ps1 index f6bf8413b..a63c4bef9 100644 --- a/vhd/packer/configure-windows-vhd-phase2.ps1 +++ b/vhd/packer/configure-windows-vhd-phase2.ps1 @@ -24,13 +24,17 @@ function Get-ContainerImages { "mcr.microsoft.com/windows/servercore:ltsc2019", "mcr.microsoft.com/windows/nanoserver:1809", "mcr.microsoft.com/oss/kubernetes/pause:3.8", - "mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.30.8", + "mcr.microsoft.com/oss/kubernetes/windows-host-process-containers-base-image:v1.0.0", "mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.31.1", + "mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.33.0", "mcr.microsoft.com/oss/kubernetes-csi/azuredisk-csi:v1.29.12", "mcr.microsoft.com/oss/kubernetes-csi/azuredisk-csi:v1.29.12-windows-hp", + "mcr.microsoft.com/oss/kubernetes-csi/azuredisk-csi:v1.33.4", + "mcr.microsoft.com/oss/kubernetes-csi/azuredisk-csi:v1.33.4-windows-hp", "mcr.microsoft.com/oss/kubernetes-csi/csi-node-driver-registrar:v2.12.0", + "mcr.microsoft.com/oss/kubernetes-csi/csi-node-driver-registrar:v2.14.0", "mcr.microsoft.com/oss/kubernetes-csi/livenessprobe:v2.14.0", - "mcr.microsoft.com/oss/kubernetes/windows-host-process-containers-base-image:v1.0.0") + "mcr.microsoft.com/oss/kubernetes-csi/livenessprobe:v2.16.0") # start containerd to pre-pull the images to disk on VHD # CSE will configure and register containerd as a service at deployment time @@ -72,8 +76,8 @@ function Get-FilesToCacheOnVHD { "https://packages.aks.azure.com/csi-proxy/v1.1.3/binaries/csi-proxy-v1.1.3.tar.gz" ); "c:\akse-cache\win-k8s\" = @( - "https://packages.aks.azure.com/kubernetes/v1.31.11/windowszip/v1.31.11-1int.zip", - "https://packages.aks.azure.com/kubernetes/v1.30.14/windowszip/v1.30.14-1int.zip" + "https://packages.aks.azure.com/kubernetes/v1.33.5/windowszip/v1.33.5-1int.zip", + "https://packages.aks.azure.com/kubernetes/v1.31.11/windowszip/v1.31.11-1int.zip" ); "c:\akse-cache\win-vnet-cni\" = @( "https://packages.aks.azure.com/azure-cni/v1.4.59/binaries/azure-vnet-cni-windows-amd64-v1.4.59.zip" @@ -96,8 +100,8 @@ function Get-FilesToCacheOnVHD { } $acrCredentialProviderUrls = @( - @{ Url = "https://github.com/kubernetes-sigs/cloud-provider-azure/releases/download/v1.31.7/azure-acr-credential-provider-windows-amd64.exe"; K8sVersion = "v1.31" }, - @{ Url = "https://github.com/kubernetes-sigs/cloud-provider-azure/releases/download/v1.30.13/azure-acr-credential-provider-windows-amd64.exe"; K8sVersion = "v1.30" } + @{ Url = "https://github.com/kubernetes-sigs/cloud-provider-azure/releases/download/v1.33.1/azure-acr-credential-provider-windows-amd64.exe"; K8sVersion = "v1.33" }, + @{ Url = "https://github.com/kubernetes-sigs/cloud-provider-azure/releases/download/v1.31.1/azure-acr-credential-provider-windows-amd64.exe"; K8sVersion = "v1.31" } ) $credentialProviderDir = "c:\k\credential-provider\" diff --git a/vhd/packer/configure-windows-vhd.ps1 b/vhd/packer/configure-windows-vhd.ps1 index 599dd1d48..40ef6a692 100644 --- a/vhd/packer/configure-windows-vhd.ps1 +++ b/vhd/packer/configure-windows-vhd.ps1 @@ -48,16 +48,17 @@ function Get-ContainerImages { "mcr.microsoft.com/windows/servercore:ltsc2019", "mcr.microsoft.com/windows/nanoserver:1809", "mcr.microsoft.com/oss/kubernetes/pause:3.8", - "mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.30.10", - "mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.31.9", - "mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.32.0", + "mcr.microsoft.com/oss/kubernetes/windows-host-process-containers-base-image:v1.0.0", + "mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.31.1", + "mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.33.0", "mcr.microsoft.com/oss/kubernetes-csi/azuredisk-csi:v1.29.12", "mcr.microsoft.com/oss/kubernetes-csi/azuredisk-csi:v1.29.12-windows-hp", - "mcr.microsoft.com/oss/kubernetes-csi/csi-node-driver-registrar:v2.8.0", + "mcr.microsoft.com/oss/kubernetes-csi/azuredisk-csi:v1.33.4", + "mcr.microsoft.com/oss/kubernetes-csi/azuredisk-csi:v1.33.4-windows-hp", "mcr.microsoft.com/oss/kubernetes-csi/csi-node-driver-registrar:v2.12.0", - "mcr.microsoft.com/oss/kubernetes-csi/livenessprobe:v2.15.0", + "mcr.microsoft.com/oss/kubernetes-csi/csi-node-driver-registrar:v2.14.0", "mcr.microsoft.com/oss/kubernetes-csi/livenessprobe:v2.14.0", - "mcr.microsoft.com/oss/kubernetes/windows-host-process-containers-base-image:v1.0.0") + "mcr.microsoft.com/oss/kubernetes-csi/livenessprobe:v2.16.0") } '2004' { $imagesToPull = @( @@ -114,9 +115,8 @@ function Get-FilesToCacheOnVHD { "https://packages.aks.azure.com/csi-proxy/v1.1.3/binaries/csi-proxy-v1.1.3.tar.gz" ); "c:\akse-cache\win-k8s\" = @( - "https://packages.aks.azure.com/kubernetes/v1.32.5/windowszip/v1.32.5-1int.zip", - "https://packages.aks.azure.com/kubernetes/v1.31.9/windowszip/v1.31.9-1int.zip", - "https://packages.aks.azure.com/kubernetes/v1.30.10/windowszip/v1.30.10-1int.zip" + "https://packages.aks.azure.com/kubernetes/v1.33.5/windowszip/v1.33.5-1int.zip", + "https://packages.aks.azure.com/kubernetes/v1.31.11/windowszip/v1.31.11-1int.zip" ); "c:\akse-cache\win-vnet-cni\" = @( "https://packages.aks.azure.com/azure-cni/v1.4.59/binaries/azure-vnet-cni-windows-amd64-v1.4.59.zip" diff --git a/vhd/packer/install-dependencies.sh b/vhd/packer/install-dependencies.sh index fcab6117a..c0f2a579f 100644 --- a/vhd/packer/install-dependencies.sh +++ b/vhd/packer/install-dependencies.sh @@ -125,6 +125,7 @@ systemctl status docker --no-pager || exit 1 echo "Docker images pre-pulled:" >> ${VHD_LOGS_FILEPATH} METRICS_SERVER_VERSIONS=" +0.8.0 0.5.2 " for METRICS_SERVER_VERSION in ${METRICS_SERVER_VERSIONS}; do @@ -134,7 +135,7 @@ for METRICS_SERVER_VERSION in ${METRICS_SERVER_VERSIONS}; do done KUBE_ADDON_MANAGER_VERSIONS=" -9.1.5 +9.1.8 9.1.6 " for KUBE_ADDON_MANAGER_VERSION in ${KUBE_ADDON_MANAGER_VERSIONS}; do @@ -154,6 +155,7 @@ for PAUSE_VERSION in ${MCR_PAUSE_VERSIONS}; do done CLUSTER_AUTOSCALER_VERSIONS=" +1.33.1 1.22.1 " for CLUSTER_AUTOSCALER_VERSION in ${CLUSTER_AUTOSCALER_VERSIONS}; do @@ -211,8 +213,8 @@ loadContainerImage ${BUSYBOX_IMAGE} echo " - ${BUSYBOX_IMAGE}" >> ${VHD_LOGS_FILEPATH} K8S_VERSIONS=" +1.33.5 1.31.11 -1.30.14 " for KUBERNETES_VERSION in ${K8S_VERSIONS}; do for component in kube-apiserver kube-controller-manager kube-proxy kube-scheduler; do @@ -226,8 +228,8 @@ done # Starting with 1.16 we pull cloud-controller-manager and cloud-node-manager CLOUD_CONTROLLER_MANAGER_VERSIONS=" +1.33.1 1.31.1 -1.30.7 " for CLOUD_CONTROLLER_MANAGER_VERSION in ${CLOUD_CONTROLLER_MANAGER_VERSIONS}; do CONTAINER_IMAGE="mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v${CLOUD_CONTROLLER_MANAGER_VERSION}" @@ -237,8 +239,8 @@ done # Starting with 1.16 we pull cloud-controller-manager and cloud-node-manager CLOUD_NODE_MANAGER_VERSIONS=" +1.33.0 1.31.1 -1.30.8 " for CLOUD_NODE_MANAGER_VERSION in ${CLOUD_NODE_MANAGER_VERSIONS}; do CONTAINER_IMAGE="mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v${CLOUD_NODE_MANAGER_VERSION}" @@ -247,6 +249,7 @@ for CLOUD_NODE_MANAGER_VERSION in ${CLOUD_NODE_MANAGER_VERSIONS}; do done AZUREDISK_CSI_VERSIONS=" +1.33.4 1.29.12 " for AZUREDISK_CSI_VERSION in ${AZUREDISK_CSI_VERSIONS}; do @@ -265,7 +268,7 @@ for AZUREFILE_CSI_VERSION in ${AZUREFILE_CSI_VERSIONS}; do done CSI_ATTACHER_VERSIONS=" -4.8.0 +4.9.0 4.7.0 " for CSI_ATTACHER_VERSION in ${CSI_ATTACHER_VERSIONS}; do @@ -275,6 +278,7 @@ for CSI_ATTACHER_VERSION in ${CSI_ATTACHER_VERSIONS}; do done CSI_NODE_DRIVER_REGISTRAR_VERSIONS=" +2.14.0 2.12.0 " for CSI_NODE_DRIVER_REGISTRAR_VERSION in ${CSI_NODE_DRIVER_REGISTRAR_VERSIONS}; do @@ -284,7 +288,7 @@ for CSI_NODE_DRIVER_REGISTRAR_VERSION in ${CSI_NODE_DRIVER_REGISTRAR_VERSIONS}; done CSI_PROVISIONER_VERSIONS=" -5.2.0 +5.3.0 5.1.0 " for CSI_PROVISIONER_VERSION in ${CSI_PROVISIONER_VERSIONS}; do @@ -293,7 +297,7 @@ for CSI_PROVISIONER_VERSION in ${CSI_PROVISIONER_VERSIONS}; do echo " - ${CONTAINER_IMAGE}" >> ${VHD_LOGS_FILEPATH} done -LIVENESSPROBE_VERSIONS=" +2.16.0 2.14.0 " for LIVENESSPROBE_VERSION in ${LIVENESSPROBE_VERSIONS}; do @@ -303,7 +307,7 @@ for LIVENESSPROBE_VERSION in ${LIVENESSPROBE_VERSIONS}; do done CSI_RESIZER_VERSIONS=" -1.13.1 +1.14.0 1.12.0 " for CSI_RESIZER_VERSION in ${CSI_RESIZER_VERSIONS}; do @@ -312,9 +316,8 @@ for CSI_RESIZER_VERSION in ${CSI_RESIZER_VERSIONS}; do echo " - ${CONTAINER_IMAGE}" >> ${VHD_LOGS_FILEPATH} done -CSI_SNAPSHOTTER_VERSIONS=" +8.3.0 8.1.0 -6.2.2 " for CSI_SNAPSHOTTER_VERSION in ${CSI_SNAPSHOTTER_VERSIONS}; do CONTAINER_IMAGE="mcr.microsoft.com/oss/kubernetes-csi/csi-snapshotter:v${CSI_SNAPSHOTTER_VERSION}" @@ -322,9 +325,8 @@ for CSI_SNAPSHOTTER_VERSION in ${CSI_SNAPSHOTTER_VERSIONS}; do echo " - ${CONTAINER_IMAGE}" >> ${VHD_LOGS_FILEPATH} done -SNAPSHOT_CONTROLLER_VERSIONS=" +8.3.0 8.1.0 -6.2.2 " for SNAPSHOT_CONTROLLER_VERSION in ${SNAPSHOT_CONTROLLER_VERSIONS}; do CONTAINER_IMAGE="mcr.microsoft.com/oss/kubernetes-csi/snapshot-controller:v${SNAPSHOT_CONTROLLER_VERSION}" @@ -369,7 +371,8 @@ for AAD_POD_IDENTITY_NMI_VERSION in ${AAD_POD_IDENTITY_NMI_VERSIONS}; do done CLUSTER_PROPORTIONAL_AUTOSCALER_VERSIONS=" -1.8.5 +1.8.23 +1.8.7 " for CLUSTER_PROPORTIONAL_AUTOSCALER_VERSION in ${CLUSTER_PROPORTIONAL_AUTOSCALER_VERSIONS}; do CONTAINER_IMAGE="mcr.microsoft.com/oss/kubernetes/autoscaler/cluster-proportional-autoscaler:${CLUSTER_PROPORTIONAL_AUTOSCALER_VERSION}"