From 118ad16c22336a0d80e6641fe9e8d9a07198d956 Mon Sep 17 00:00:00 2001 From: Danil-Grigorev Date: Tue, 29 Jun 2021 15:42:39 +0200 Subject: [PATCH 1/8] Add feature gate exclusion list to kubelet config --- pkg/controller/kubelet-config/kubelet_config_controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/controller/kubelet-config/kubelet_config_controller.go b/pkg/controller/kubelet-config/kubelet_config_controller.go index 814f09d1b0..8cd30d185b 100644 --- a/pkg/controller/kubelet-config/kubelet_config_controller.go +++ b/pkg/controller/kubelet-config/kubelet_config_controller.go @@ -478,7 +478,7 @@ func (ctrl *Controller) syncKubeletConfig(key string) error { err := fmt.Errorf("could not fetch FeatureGates: %v", err) return ctrl.syncStatusOnly(cfg, err) } - featureGates, err := generateFeatureMap(features) + featureGates, err := generateFeatureMap(features, openshiftOnlyFeatureGates...) if err != nil { err := fmt.Errorf("could not generate FeatureMap: %v", err) glog.V(2).Infof("%v", err) From 64cdcdfad4145cadf8c76b7442dd896e5ea1ce53 Mon Sep 17 00:00:00 2001 From: Danil-Grigorev Date: Thu, 1 Jul 2021 13:42:13 +0200 Subject: [PATCH 2/8] Add featureGate evaluation in manifests folder to bootstrap implementaion --- pkg/controller/bootstrap/bootstrap.go | 8 +++++++- pkg/controller/bootstrap/bootstrap_test.go | 13 +++++++++++++ pkg/controller/template/template_controller.go | 4 ++-- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/pkg/controller/bootstrap/bootstrap.go b/pkg/controller/bootstrap/bootstrap.go index c18f391a8e..12dd895c50 100644 --- a/pkg/controller/bootstrap/bootstrap.go +++ b/pkg/controller/bootstrap/bootstrap.go @@ -20,6 +20,7 @@ import ( apicfgv1 "github.com/openshift/api/config/v1" apioperatorsv1alpha1 "github.com/openshift/api/operator/v1alpha1" mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" + ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common" containerruntimeconfig "github.com/openshift/machine-config-operator/pkg/controller/container-runtime-config" "github.com/openshift/machine-config-operator/pkg/controller/render" "github.com/openshift/machine-config-operator/pkg/controller/template" @@ -70,6 +71,7 @@ func (b *Bootstrap) Run(destDir string) error { decoder := codecFactory.UniversalDecoder(mcfgv1.GroupVersion, apioperatorsv1alpha1.GroupVersion, apicfgv1.GroupVersion) var cconfig *mcfgv1.ControllerConfig + var featureGate *apicfgv1.FeatureGate var pools []*mcfgv1.MachineConfigPool var configs []*mcfgv1.MachineConfig var icspRules []*apioperatorsv1alpha1.ImageContentSourcePolicy @@ -112,6 +114,10 @@ func (b *Bootstrap) Run(destDir string) error { icspRules = append(icspRules, obj) case *apicfgv1.Image: imgCfg = obj + case *apicfgv1.FeatureGate: + if obj.GetName() == ctrlcommon.ClusterFeatureInstanceName { + featureGate = obj + } default: glog.Infof("skipping %q [%d] manifest because of unhandled %T", file.Name(), idx+1, obji) } @@ -121,7 +127,7 @@ func (b *Bootstrap) Run(destDir string) error { if cconfig == nil { return fmt.Errorf("error: no controllerconfig found in dir: %q", destDir) } - iconfigs, err := template.RunBootstrap(b.templatesDir, cconfig, psraw) + iconfigs, err := template.RunBootstrap(b.templatesDir, cconfig, psraw, featureGate) if err != nil { return err } diff --git a/pkg/controller/bootstrap/bootstrap_test.go b/pkg/controller/bootstrap/bootstrap_test.go index 4d5fe08743..469a0d070b 100644 --- a/pkg/controller/bootstrap/bootstrap_test.go +++ b/pkg/controller/bootstrap/bootstrap_test.go @@ -44,6 +44,19 @@ spec: want: []manifest{{ Raw: []byte(`{"apiVersion":"extensions/v1beta1","kind":"Ingress","metadata":{"name":"test-ingress","namespace":"test-namespace"},"spec":{"rules":[{"http":{"paths":[{"backend":{"serviceName":"test","servicePort":80},"path":"/testpath"}]}}]}}`), }}, + }, { + name: "feature gate", + raw: ` +apiVersion: config.openshift.io/v1 +kind: FeatureGate +metadata: + name: cluster +spec: + featureSet: TechPreviewNoUpgrade +`, + want: []manifest{{ + Raw: []byte(`{"apiVersion":"config.openshift.io/v1","kind":"FeatureGate","metadata":{"name":"cluster"},"spec":{"featureSet":"TechPreviewNoUpgrade"}}`), + }}, }, { name: "two-resources", raw: ` diff --git a/pkg/controller/template/template_controller.go b/pkg/controller/template/template_controller.go index 909224bd03..96fa61e296 100644 --- a/pkg/controller/template/template_controller.go +++ b/pkg/controller/template/template_controller.go @@ -523,6 +523,6 @@ func getMachineConfigsForControllerConfig(templatesDir string, config *mcfgv1.Co } // RunBootstrap runs the tempate controller in boostrap mode. -func RunBootstrap(templatesDir string, config *mcfgv1.ControllerConfig, pullSecretRaw []byte) ([]*mcfgv1.MachineConfig, error) { - return getMachineConfigsForControllerConfig(templatesDir, config, pullSecretRaw, nil) +func RunBootstrap(templatesDir string, config *mcfgv1.ControllerConfig, pullSecretRaw []byte, featureGate *configv1.FeatureGate) ([]*mcfgv1.MachineConfig, error) { + return getMachineConfigsForControllerConfig(templatesDir, config, pullSecretRaw, featureGate) } From 0e5157327f32eee1450aeb7859ce224c3c718e6e Mon Sep 17 00:00:00 2001 From: Joel Speed Date: Thu, 8 Jul 2021 11:27:50 +0100 Subject: [PATCH 3/8] Extract KubeConfig FeatureGate ignition to a standalone function --- .../kubelet_config_controller.go | 14 +-- .../kubelet-config/kubelet_config_features.go | 85 ++++++++++++------- .../kubelet_config_features_test.go | 2 +- 3 files changed, 61 insertions(+), 40 deletions(-) diff --git a/pkg/controller/kubelet-config/kubelet_config_controller.go b/pkg/controller/kubelet-config/kubelet_config_controller.go index 8cd30d185b..0282bdd6e3 100644 --- a/pkg/controller/kubelet-config/kubelet_config_controller.go +++ b/pkg/controller/kubelet-config/kubelet_config_controller.go @@ -330,14 +330,10 @@ func (ctrl *Controller) handleFeatureErr(err error, key interface{}) { ctrl.featureQueue.AddAfter(key, 1*time.Minute) } -func (ctrl *Controller) generateOriginalKubeletConfig(role string, featureGate *configv1.FeatureGate) (*ign3types.File, error) { - cc, err := ctrl.ccLister.Get(ctrlcommon.ControllerConfigName) - if err != nil { - return nil, fmt.Errorf("could not get ControllerConfig %v", err) - } +func generateOriginalKubeletConfig(cc *mcfgv1.ControllerConfig, templatesDir, role string, featureGate *configv1.FeatureGate) (*ign3types.File, error) { // Render the default templates rc := &mtmpl.RenderConfig{ControllerConfigSpec: &cc.Spec, FeatureGate: featureGate} - generatedConfigs, err := mtmpl.GenerateMachineConfigsForRole(rc, role, ctrl.templatesDir) + generatedConfigs, err := mtmpl.GenerateMachineConfigsForRole(rc, role, templatesDir) if err != nil { return nil, fmt.Errorf("GenerateMachineConfigsforRole failed with error %s", err) } @@ -512,7 +508,11 @@ func (ctrl *Controller) syncKubeletConfig(key string) error { userDefinedSystemReserved := make(map[string]string, 2) // Generate the original KubeletConfig - originalKubeletIgn, err := ctrl.generateOriginalKubeletConfig(role, features) + cc, err := ctrl.ccLister.Get(ctrlcommon.ControllerConfigName) + if err != nil { + return fmt.Errorf("could not get ControllerConfig %v", err) + } + originalKubeletIgn, err := generateOriginalKubeletConfig(cc, ctrl.templatesDir, role, features) if err != nil { return ctrl.syncStatusOnly(cfg, err, "could not generate the original Kubelet config: %v", err) } diff --git a/pkg/controller/kubelet-config/kubelet_config_features.go b/pkg/controller/kubelet-config/kubelet_config_features.go index 965a79f8e5..ab4fdfe09e 100644 --- a/pkg/controller/kubelet-config/kubelet_config_features.go +++ b/pkg/controller/kubelet-config/kubelet_config_features.go @@ -20,6 +20,7 @@ import ( "k8s.io/client-go/util/retry" kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" + mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common" "github.com/openshift/machine-config-operator/pkg/version" ) @@ -74,6 +75,11 @@ func (ctrl *Controller) syncFeatureHandler(key string) error { return err } + cc, err := ctrl.ccLister.Get(ctrlcommon.ControllerConfigName) + if err != nil { + return fmt.Errorf("could not get ControllerConfig %v", err) + } + // Find all MachineConfigPools mcpPools, err := ctrl.mcpLister.List(labels.Everything()) if err != nil { @@ -100,43 +106,15 @@ func (ctrl *Controller) syncFeatureHandler(key string) error { return err } } - // Generate the original KubeletConfig - originalKubeletIgn, err := ctrl.generateOriginalKubeletConfig(role, nil) - if err != nil { - return err - } - if originalKubeletIgn.Contents.Source == nil { - return fmt.Errorf("could not find original Kubelet config to decode") - } - dataURL, err := dataurl.DecodeString(*originalKubeletIgn.Contents.Source) - if err != nil { - return err - } - originalKubeConfig, err := decodeKubeletConfig(dataURL.Data) + + rawCfgIgn, err := generateKubeConfigIgnFromFeatures(cc, ctrl.templatesDir, role, *featureGates) if err != nil { return err } - // Check to see if FeatureGates are equal - if reflect.DeepEqual(originalKubeConfig.FeatureGates, *featureGates) { + if rawCfgIgn == nil { continue } - // Merge in Feature Gates - err = mergo.Merge(&originalKubeConfig.FeatureGates, featureGates, mergo.WithOverride) - if err != nil { - return err - } - // Encode the new config into raw JSON - cfgJSON, err := EncodeKubeletConfig(originalKubeConfig, kubeletconfigv1beta1.SchemeGroupVersion) - if err != nil { - return err - } - tempIgnConfig := ctrlcommon.NewIgnConfig() - cfgIgn := createNewKubeletIgnition(cfgJSON) - tempIgnConfig.Storage.Files = append(tempIgnConfig.Storage.Files, *cfgIgn) - rawCfgIgn, err := json.Marshal(tempIgnConfig) - if err != nil { - return err - } + mc.Spec.Config.Raw = rawCfgIgn mc.ObjectMeta.Annotations = map[string]string{ ctrlcommon.GeneratedByControllerVersionAnnotationKey: version.Hash, @@ -231,3 +209,46 @@ func generateFeatureMap(features *osev1.FeatureGate, exclusions ...string) (*map } return &rv, nil } + +func generateKubeConfigIgnFromFeatures(cc *mcfgv1.ControllerConfig, templatesDir, role string, featureGates map[string]bool) ([]byte, error) { + // Generate the original KubeletConfig + originalKubeletIgn, err := generateOriginalKubeletConfig(cc, templatesDir, role, nil) + if err != nil { + return nil, err + } + if originalKubeletIgn.Contents.Source == nil { + return nil, fmt.Errorf("could not find original Kubelet config to decode") + } + dataURL, err := dataurl.DecodeString(*originalKubeletIgn.Contents.Source) + if err != nil { + return nil, err + } + originalKubeConfig, err := decodeKubeletConfig(dataURL.Data) + if err != nil { + return nil, err + } + // Check to see if FeatureGates are equal + if reflect.DeepEqual(originalKubeConfig.FeatureGates, featureGates) { + // When there is no difference, this isn't an error, but no machine config should be created + return nil, nil + } + + // Merge in Feature Gates + err = mergo.Merge(&originalKubeConfig.FeatureGates, featureGates, mergo.WithOverride) + if err != nil { + return nil, err + } + // Encode the new config into raw JSON + cfgJSON, err := EncodeKubeletConfig(originalKubeConfig, kubeletconfigv1beta1.SchemeGroupVersion) + if err != nil { + return nil, err + } + tempIgnConfig := ctrlcommon.NewIgnConfig() + cfgIgn := createNewKubeletIgnition(cfgJSON) + tempIgnConfig.Storage.Files = append(tempIgnConfig.Storage.Files, *cfgIgn) + rawCfgIgn, err := json.Marshal(tempIgnConfig) + if err != nil { + return nil, err + } + return rawCfgIgn, nil +} diff --git a/pkg/controller/kubelet-config/kubelet_config_features_test.go b/pkg/controller/kubelet-config/kubelet_config_features_test.go index 6625468d13..847983961c 100644 --- a/pkg/controller/kubelet-config/kubelet_config_features_test.go +++ b/pkg/controller/kubelet-config/kubelet_config_features_test.go @@ -23,7 +23,7 @@ func TestFeatureGateDrift(t *testing.T) { f.ccLister = append(f.ccLister, cc) ctrl := f.newController() - kubeletConfig, err := ctrl.generateOriginalKubeletConfig("master", nil) + kubeletConfig, err := generateOriginalKubeletConfig(cc, ctrl.templatesDir, "master", nil) if err != nil { t.Errorf("could not generate kubelet config from templates %v", err) } From 5f9562976b32507658724becf5e61f97a834ddfa Mon Sep 17 00:00:00 2001 From: Joel Speed Date: Thu, 8 Jul 2021 11:35:57 +0100 Subject: [PATCH 4/8] Add Kubelet Feature Gate bootstrap --- pkg/controller/bootstrap/bootstrap.go | 9 ++++ .../kubelet-config/kubelet_config_features.go | 41 +++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/pkg/controller/bootstrap/bootstrap.go b/pkg/controller/bootstrap/bootstrap.go index 12dd895c50..6933200f9f 100644 --- a/pkg/controller/bootstrap/bootstrap.go +++ b/pkg/controller/bootstrap/bootstrap.go @@ -22,6 +22,7 @@ import ( mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common" containerruntimeconfig "github.com/openshift/machine-config-operator/pkg/controller/container-runtime-config" + kubeletconfig "github.com/openshift/machine-config-operator/pkg/controller/kubelet-config" "github.com/openshift/machine-config-operator/pkg/controller/render" "github.com/openshift/machine-config-operator/pkg/controller/template" ) @@ -139,6 +140,14 @@ func (b *Bootstrap) Run(destDir string) error { } configs = append(configs, rconfigs...) + if featureGate != nil { + kConfigs, err := kubeletconfig.RunFeatureGateBootstrap(b.templatesDir, featureGate, cconfig, pools) + if err != nil { + return err + } + configs = append(configs, kConfigs...) + } + fpools, gconfigs, err := render.RunBootstrap(pools, configs, cconfig) if err != nil { return err diff --git a/pkg/controller/kubelet-config/kubelet_config_features.go b/pkg/controller/kubelet-config/kubelet_config_features.go index ab4fdfe09e..26c9464e93 100644 --- a/pkg/controller/kubelet-config/kubelet_config_features.go +++ b/pkg/controller/kubelet-config/kubelet_config_features.go @@ -252,3 +252,44 @@ func generateKubeConfigIgnFromFeatures(cc *mcfgv1.ControllerConfig, templatesDir } return rawCfgIgn, nil } + +func RunFeatureGateBootstrap(templateDir string, features *osev1.FeatureGate, controllerConfig *mcfgv1.ControllerConfig, mcpPools []*mcfgv1.MachineConfigPool) ([]*mcfgv1.MachineConfig, error) { + machineConfigs := []*mcfgv1.MachineConfig{} + + featureGates, err := generateFeatureMap(features, openshiftOnlyFeatureGates...) + if err != nil { + return nil, err + } + + for _, pool := range mcpPools { + role := pool.Name + rawCfgIgn, err := generateKubeConfigIgnFromFeatures(controllerConfig, templateDir, role, *featureGates) + if err != nil { + return nil, err + } + if rawCfgIgn == nil { + continue + } + + // Get MachineConfig + managedKey, err := getManagedFeaturesKey(pool, nil) + if err != nil { + return nil, err + } + + ignConfig := ctrlcommon.NewIgnConfig() + mc, err := ctrlcommon.MachineConfigFromIgnConfig(role, managedKey, ignConfig) + if err != nil { + return nil, err + } + + mc.Spec.Config.Raw = rawCfgIgn + mc.ObjectMeta.Annotations = map[string]string{ + ctrlcommon.GeneratedByControllerVersionAnnotationKey: version.Hash, + } + + machineConfigs = append(machineConfigs, mc) + } + + return machineConfigs, nil +} From 36b4168b0028c2617c0b7853e51e8fb5f202914a Mon Sep 17 00:00:00 2001 From: Joel Speed Date: Thu, 8 Jul 2021 12:47:45 +0100 Subject: [PATCH 5/8] Add tests for KubeletConfig Feature bootstrap --- .../kubelet_config_features_test.go | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/pkg/controller/kubelet-config/kubelet_config_features_test.go b/pkg/controller/kubelet-config/kubelet_config_features_test.go index 847983961c..b2bd07f0aa 100644 --- a/pkg/controller/kubelet-config/kubelet_config_features_test.go +++ b/pkg/controller/kubelet-config/kubelet_config_features_test.go @@ -11,8 +11,10 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" + mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common" "github.com/openshift/machine-config-operator/test/helpers" + "github.com/stretchr/testify/require" ) func TestFeatureGateDrift(t *testing.T) { @@ -131,3 +133,78 @@ func TestFeaturesCustomNoUpgrade(t *testing.T) { }) } } + +func TestBootstrapFeaturesDefault(t *testing.T) { + for _, platform := range []configv1.PlatformType{configv1.AWSPlatformType, configv1.NonePlatformType, "unrecognized"} { + t.Run(string(platform), func(t *testing.T) { + + cc := newControllerConfig(ctrlcommon.ControllerConfigName, platform) + mcp := helpers.NewMachineConfigPool("master", nil, helpers.MasterSelector, "v0") + mcp2 := helpers.NewMachineConfigPool("worker", nil, helpers.WorkerSelector, "v0") + mcps := []*mcfgv1.MachineConfigPool{mcp, mcp2} + + features := createNewDefaultFeatureGate() + + mcs, err := RunFeatureGateBootstrap("../../../templates", features, cc, mcps) + if err != nil { + t.Errorf("could not run feature gate bootstrap: %v", err) + } + if len(mcs) > 0 { + t.Errorf("expected no machine config generated with the default feature gate, got %d configs", len(mcs)) + } + }) + } +} + +func TestBootstrapFeaturesCustomNoUpgrade(t *testing.T) { + for _, platform := range []configv1.PlatformType{configv1.AWSPlatformType, configv1.NonePlatformType, "unrecognized"} { + t.Run(string(platform), func(t *testing.T) { + + cc := newControllerConfig(ctrlcommon.ControllerConfigName, platform) + mcp := helpers.NewMachineConfigPool("master", nil, helpers.MasterSelector, "v0") + mcp2 := helpers.NewMachineConfigPool("worker", nil, helpers.WorkerSelector, "v0") + mcps := []*mcfgv1.MachineConfigPool{mcp, mcp2} + + features := &osev1.FeatureGate{ + ObjectMeta: metav1.ObjectMeta{ + Name: ctrlcommon.ClusterFeatureInstanceName, + }, + Spec: osev1.FeatureGateSpec{ + FeatureGateSelection: osev1.FeatureGateSelection{ + FeatureSet: osev1.CustomNoUpgrade, + CustomNoUpgrade: &osev1.CustomFeatureGates{ + Enabled: []string{"CSIMigration"}, + }, + }, + }, + } + + mcs, err := RunFeatureGateBootstrap("../../../templates", features, cc, mcps) + if err != nil { + t.Errorf("could not run feature gate bootstrap: %v", err) + } + if len(mcs) != 2 { + t.Errorf("expected 2 machine configs generated with the custom feature gate, got %d configs", len(mcs)) + } + + for _, mc := range mcs { + ignCfg, err := ctrlcommon.ParseAndConvertConfig(mc.Spec.Config.Raw) + regfile := ignCfg.Storage.Files[0] + conf, err := dataurl.DecodeString(*regfile.Contents.Source) + require.NoError(t, err) + + originalKubeConfig, _ := decodeKubeletConfig(conf.Data) + defaultFeatureGates, err := generateFeatureMap(createNewDefaultFeatureGate()) + if err != nil { + t.Errorf("could not generate defaultFeatureGates: %v", err) + } + if reflect.DeepEqual(originalKubeConfig.FeatureGates, *defaultFeatureGates) { + t.Errorf("template FeatureGates should not match default openshift/api FeatureGates: (default=%v)", defaultFeatureGates) + } + if !originalKubeConfig.FeatureGates["CSIMigration"] { + t.Errorf("template FeatureGates should contain CSIMigration: %v", originalKubeConfig.FeatureGates) + } + } + }) + } +} From 0c555186774b89964c3a6156095107a35c9d5359 Mon Sep 17 00:00:00 2001 From: Joel Speed Date: Thu, 8 Jul 2021 18:02:51 +0100 Subject: [PATCH 6/8] Prevent go cyclo lint error on bootstrap Run --- pkg/controller/bootstrap/bootstrap.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/controller/bootstrap/bootstrap.go b/pkg/controller/bootstrap/bootstrap.go index 6933200f9f..5f0a0a5b1f 100644 --- a/pkg/controller/bootstrap/bootstrap.go +++ b/pkg/controller/bootstrap/bootstrap.go @@ -48,6 +48,7 @@ func New(templatesDir, manifestDir, pullSecretFile string) *Bootstrap { // Run runs boostrap for Machine Config Controller // It writes all the assets to destDir +// nolint:gocyclo func (b *Bootstrap) Run(destDir string) error { infos, err := ioutil.ReadDir(b.manifestDir) if err != nil { From 494e62d6291fd224cb9a244d35c8dad3b392aed2 Mon Sep 17 00:00:00 2001 From: Denis Moiseev Date: Fri, 9 Jul 2021 13:09:59 +0200 Subject: [PATCH 7/8] Update library go dependency for out-of-tree providers support on Azure platform --- go.mod | 8 ++++---- go.sum | 16 +++++----------- pkg/controller/template/render.go | 4 ++-- .../library-go/pkg/cloudprovider/external.go | 16 ++++++++++++++-- vendor/modules.txt | 12 ++++++------ 5 files changed, 31 insertions(+), 25 deletions(-) diff --git a/go.mod b/go.mod index fba6adf5fa..041597f51a 100644 --- a/go.mod +++ b/go.mod @@ -36,8 +36,8 @@ require ( github.com/imdario/mergo v0.3.9 github.com/opencontainers/go-digest v1.0.0 github.com/openshift/api v0.0.0-20210629145910-15a1cae1fca8 - github.com/openshift/client-go v0.0.0-20210112165513-ebc401615f47 - github.com/openshift/library-go v0.0.0-20210301154249-aa29957b8a9c + github.com/openshift/client-go v0.0.0-20210521082421-73d9475a9142 + github.com/openshift/library-go v0.0.0-20210702104503-39570b4a2ae8 github.com/openshift/runtime-utils v0.0.0-20200415173359-c45d4ff3f912 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.7.1 @@ -52,9 +52,9 @@ require ( golang.org/x/net v0.0.0-20210224082022-3d97a244fca7 golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba k8s.io/api v0.21.1 - k8s.io/apiextensions-apiserver v0.21.0-rc.0 + k8s.io/apiextensions-apiserver v0.21.1 k8s.io/apimachinery v0.21.1 - k8s.io/client-go v0.21.0-rc.0 + k8s.io/client-go v0.21.1 k8s.io/code-generator v0.21.1 k8s.io/kubectl v0.21.0-rc.0 k8s.io/kubelet v0.21.0-rc.0 diff --git a/go.sum b/go.sum index a7e2ef0f20..e66ee17bfb 100644 --- a/go.sum +++ b/go.sum @@ -632,14 +632,12 @@ github.com/opencontainers/selinux v1.5.2/go.mod h1:yTcKuYAh6R95iDpefGLQaPaRwJFwy github.com/openshift/api v0.0.0-20210629145910-15a1cae1fca8 h1:/wwYnC6F/XQJW2DyvoEZRgO2UFExBl9nSQaQbEEgbXA= github.com/openshift/api v0.0.0-20210629145910-15a1cae1fca8/go.mod h1:izBmoXbUu3z5kUa4FjZhvekTsyzIWiOoaIgJiZBBMQs= github.com/openshift/build-machinery-go v0.0.0-20200211121458-5e3d6e570160/go.mod h1:1CkcsT3aVebzRBzVTSbiKSkJMsC/CASqxesfqEMfJEc= -github.com/openshift/build-machinery-go v0.0.0-20200917070002-f171684f77ab/go.mod h1:b1BuldmJlbA/xYtdZvKi+7j5YGB44qJUJDZ9zwiNCfE= github.com/openshift/build-machinery-go v0.0.0-20210423112049-9415d7ebd33e/go.mod h1:b1BuldmJlbA/xYtdZvKi+7j5YGB44qJUJDZ9zwiNCfE= -github.com/openshift/client-go v0.0.0-20201214125552-e615e336eb49/go.mod h1:9/jG4I6sh+5QublJpZZ4Zs/P4/QCXMsQQ/K/058bSB8= -github.com/openshift/client-go v0.0.0-20210112165513-ebc401615f47 h1:+TEY29DK0XhqB7HFC9OfV8qf3wffSyi7MWv3AP28DGQ= -github.com/openshift/client-go v0.0.0-20210112165513-ebc401615f47/go.mod h1:u7NRAjtYVAKokiI9LouzTv4mhds8P4S1TwdVAfbjKSk= +github.com/openshift/client-go v0.0.0-20210521082421-73d9475a9142 h1:ZHRIMCFIJN1p9LsJt4HQ+akDrys4PrYnXzOWI5LK03I= +github.com/openshift/client-go v0.0.0-20210521082421-73d9475a9142/go.mod h1:fjS8r9mqDVsPb5td3NehsNOAWa4uiFkYEfVZioQ2gH0= github.com/openshift/library-go v0.0.0-20191003152030-97c62d8a2901/go.mod h1:NBttNjZpWwup/nthuLbPAPSYC8Qyo+BBK5bCtFoyYjo= -github.com/openshift/library-go v0.0.0-20210301154249-aa29957b8a9c h1:6yulc45KX+/9qqlW+D8oMp8C0lgqkUMYKM+oGbxX98M= -github.com/openshift/library-go v0.0.0-20210301154249-aa29957b8a9c/go.mod h1:hip2mwymSDfiip0aiTaePpUC1dxOHkjzMqaS0e3lWhw= +github.com/openshift/library-go v0.0.0-20210702104503-39570b4a2ae8 h1:vgvVr4EnUiUpl2u5QltMw3/IoBLx8SW2GRcGEXnzmGQ= +github.com/openshift/library-go v0.0.0-20210702104503-39570b4a2ae8/go.mod h1:PkxPzGpVgObkue/pqw6JSKE+z58AUcHVYgME6W4pM0g= github.com/openshift/runtime-utils v0.0.0-20200415173359-c45d4ff3f912 h1:zPJYcr+2g68K+pEqw7crr0vLSVsYb3B82LS3y7/kxpc= github.com/openshift/runtime-utils v0.0.0-20200415173359-c45d4ff3f912/go.mod h1:0OXNy7VoqFexkxKqyQbHJLPwn1MFp1/CxRJAgKHM+/o= github.com/ostreedev/ostree-go v0.0.0-20190702140239-759a8c1ac913/go.mod h1:J6OG6YJVEWopen4avK3VNQSnALmmjvniMmni/YFYAwc= @@ -996,7 +994,6 @@ golang.org/x/sys v0.0.0-20200610111108-226ff32320da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073 h1:8qxJSnu+7dRq6upnbntrmriWByIakBuct5OM/MdQC1M= @@ -1017,7 +1014,6 @@ golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20170915040203-e531a2a1c15f/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1193,7 +1189,6 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20190502103701-55513cacd4ae/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= @@ -1237,7 +1232,6 @@ k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.8.0 h1:Q3gmuM9hKEjefWFFYF0Mat+YyFJvsUyYuwyNNJ5C9Ts= k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/kube-aggregator v0.21.0-rc.0/go.mod h1:M+whOmsAeQf8ObJ0/eO9Af1Dz2UQEB9OW9BWmt9b2sU= @@ -1261,7 +1255,7 @@ rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/controller-tools v0.2.8/go.mod h1:9VKHPszmf2DHz/QmHkcfZoewO6BL7pPs9uAiBVsaJSE= -sigs.k8s.io/kube-storage-version-migrator v0.0.3/go.mod h1:mXfSLkx9xbJHQsgNDDUZK/iQTs2tMbx/hsJlWe6Fthw= +sigs.k8s.io/kube-storage-version-migrator v0.0.4/go.mod h1:mXfSLkx9xbJHQsgNDDUZK/iQTs2tMbx/hsJlWe6Fthw= sigs.k8s.io/kustomize/api v0.8.5 h1:bfCXGXDAbFbb/Jv5AhMj2BB8a5VAJuuQ5/KU69WtDjQ= sigs.k8s.io/kustomize/api v0.8.5/go.mod h1:M377apnKT5ZHJS++6H4rQoCHmWtt6qTpp3mbe7p6OLY= sigs.k8s.io/kustomize/cmd/config v0.9.7/go.mod h1:MvXCpHs77cfyxRmCNUQjIqCmZyYsbn5PyQpWiq44nW0= diff --git a/pkg/controller/template/render.go b/pkg/controller/template/render.go index a42d3d197b..d54d4d6cf8 100644 --- a/pkg/controller/template/render.go +++ b/pkg/controller/template/render.go @@ -334,7 +334,7 @@ func skipMissing(key string) (interface{}, error) { func cloudProvider(cfg RenderConfig) (interface{}, error) { if cfg.Infra.Status.PlatformStatus != nil { - external, err := cloudprovider.IsCloudProviderExternal(cfg.Infra.Status.PlatformStatus.Type, cfg.FeatureGate) + external, err := cloudprovider.IsCloudProviderExternal(cfg.Infra.Status.PlatformStatus, cfg.FeatureGate) if err != nil { glog.Error(err) } else if external { @@ -379,7 +379,7 @@ func cloudConfigFlag(cfg RenderConfig) interface{} { } } - external, err := cloudprovider.IsCloudProviderExternal(cfg.Infra.Status.PlatformStatus.Type, cfg.FeatureGate) + external, err := cloudprovider.IsCloudProviderExternal(cfg.Infra.Status.PlatformStatus, cfg.FeatureGate) if err != nil { glog.Error(err) } else if external { diff --git a/vendor/github.com/openshift/library-go/pkg/cloudprovider/external.go b/vendor/github.com/openshift/library-go/pkg/cloudprovider/external.go index 2704225fa1..4e8c5a0fa1 100644 --- a/vendor/github.com/openshift/library-go/pkg/cloudprovider/external.go +++ b/vendor/github.com/openshift/library-go/pkg/cloudprovider/external.go @@ -17,18 +17,30 @@ const ( // IsCloudProviderExternal is used to check whether external cloud provider settings should be used in a component. // It checks whether the ExternalCloudProvider feature gate is enabled and whether the ExternalCloudProvider feature // has been implemented for the platform. -func IsCloudProviderExternal(platformType configv1.PlatformType, featureGate *configv1.FeatureGate) (bool, error) { - switch platformType { +func IsCloudProviderExternal(platformStatus *configv1.PlatformStatus, featureGate *configv1.FeatureGate) (bool, error) { + if platformStatus == nil { + return false, fmt.Errorf("platformStatus is required") + } + switch platformStatus.Type { case configv1.AWSPlatformType, configv1.OpenStackPlatformType: // Platforms that are external based on feature gate presence return isExternalFeatureGateEnabled(featureGate) + case configv1.AzurePlatformType: + if isAzureStackHub(platformStatus) { + return true, nil + } + return isExternalFeatureGateEnabled(featureGate) default: // Platforms that do not have external cloud providers implemented return false, nil } } +func isAzureStackHub(platformStatus *configv1.PlatformStatus) bool { + return platformStatus.Azure != nil && platformStatus.Azure.CloudName == configv1.AzureStackCloud +} + // isExternalFeatureGateEnabled determines whether the ExternalCloudProvider feature gate is present in the current // feature set. func isExternalFeatureGateEnabled(featureGate *configv1.FeatureGate) (bool, error) { diff --git a/vendor/modules.txt b/vendor/modules.txt index 2baecbd808..69f443e120 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -456,7 +456,7 @@ github.com/opencontainers/image-spec/specs-go/v1 github.com/openshift/api/config/v1 github.com/openshift/api/operator/v1 github.com/openshift/api/operator/v1alpha1 -# github.com/openshift/client-go v0.0.0-20210112165513-ebc401615f47 +# github.com/openshift/client-go v0.0.0-20210521082421-73d9475a9142 ## explicit github.com/openshift/client-go/config/clientset/versioned github.com/openshift/client-go/config/clientset/versioned/fake @@ -482,7 +482,7 @@ github.com/openshift/client-go/operator/informers/externalversions/operator/v1 github.com/openshift/client-go/operator/informers/externalversions/operator/v1alpha1 github.com/openshift/client-go/operator/listers/operator/v1 github.com/openshift/client-go/operator/listers/operator/v1alpha1 -# github.com/openshift/library-go v0.0.0-20210301154249-aa29957b8a9c +# github.com/openshift/library-go v0.0.0-20210702104503-39570b4a2ae8 ## explicit github.com/openshift/library-go/pkg/cloudprovider github.com/openshift/library-go/pkg/config/clusteroperator/v1helpers @@ -776,7 +776,7 @@ k8s.io/api/scheduling/v1beta1 k8s.io/api/storage/v1 k8s.io/api/storage/v1alpha1 k8s.io/api/storage/v1beta1 -# k8s.io/apiextensions-apiserver v0.21.0-rc.0 => k8s.io/apiextensions-apiserver v0.21.0-rc.0 +# k8s.io/apiextensions-apiserver v0.21.1 => k8s.io/apiextensions-apiserver v0.21.0-rc.0 ## explicit k8s.io/apiextensions-apiserver/pkg/apis/apiextensions k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1 @@ -847,13 +847,13 @@ k8s.io/apimachinery/pkg/watch k8s.io/apimachinery/third_party/forked/golang/json k8s.io/apimachinery/third_party/forked/golang/netutil k8s.io/apimachinery/third_party/forked/golang/reflect -# k8s.io/apiserver v0.21.0-rc.0 => k8s.io/apiserver v0.21.0-rc.0 +# k8s.io/apiserver v0.21.1 => k8s.io/apiserver v0.21.0-rc.0 k8s.io/apiserver/pkg/authentication/user # k8s.io/cli-runtime v0.21.0-rc.0 => k8s.io/cli-runtime v0.21.0-rc.0 k8s.io/cli-runtime/pkg/genericclioptions k8s.io/cli-runtime/pkg/printers k8s.io/cli-runtime/pkg/resource -# k8s.io/client-go v0.21.0-rc.0 => k8s.io/client-go v0.21.0-rc.0 +# k8s.io/client-go v0.21.1 => k8s.io/client-go v0.21.0-rc.0 ## explicit k8s.io/client-go/applyconfigurations/admissionregistration/v1 k8s.io/client-go/applyconfigurations/admissionregistration/v1beta1 @@ -1166,7 +1166,7 @@ k8s.io/code-generator/cmd/set-gen k8s.io/code-generator/pkg/namer k8s.io/code-generator/pkg/util k8s.io/code-generator/third_party/forked/golang/reflect -# k8s.io/component-base v0.21.0-rc.0 => k8s.io/component-base v0.21.0-rc.0 +# k8s.io/component-base v0.21.1 => k8s.io/component-base v0.21.0-rc.0 k8s.io/component-base/config k8s.io/component-base/config/v1alpha1 k8s.io/component-base/version From fb6ee602beee6e23393eb3fb4db40ee67a6b1741 Mon Sep 17 00:00:00 2001 From: Denis Moiseev Date: Fri, 9 Jul 2021 13:28:10 +0200 Subject: [PATCH 8/8] fix testcase, extend TestCloudProvider for accounting Azure --- pkg/controller/template/render_test.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/controller/template/render_test.go b/pkg/controller/template/render_test.go index fad4bd8d00..c40e58c5e8 100644 --- a/pkg/controller/template/render_test.go +++ b/pkg/controller/template/render_test.go @@ -41,6 +41,10 @@ func TestCloudProvider(t *testing.T) { platform: configv1.OpenStackPlatformType, featureGate: newFeatures("cluster", "CustomNoUpgrade", []string{cloudprovider.ExternalCloudProviderFeature}, nil), res: "external", + }, { + platform: configv1.AzurePlatformType, + featureGate: newFeatures("cluster", "CustomNoUpgrade", []string{cloudprovider.ExternalCloudProviderFeature}, nil), + res: "external", }, { platform: configv1.GCPPlatformType, featureGate: newFeatures("cluster", "CustomNoUpgrade", []string{cloudprovider.ExternalCloudProviderFeature}, nil), @@ -161,7 +165,7 @@ func TestCloudConfigFlag(t *testing.T) { option = a `, featureGate: newFeatures("cluster", "CustomNoUpgrade", []string{cloudprovider.ExternalCloudProviderFeature}, nil), - res: "--cloud-config=/etc/kubernetes/cloud.conf", + res: "", }, { platform: configv1.OpenStackPlatformType, content: `