USHIFT-3778: Drop-in configuration directory

docs/user/howto_config.md

@@ -294,4 +294,63 @@ driver, the cluster has no knowledge of the underlying storage interface and thu
 provisioning/deprovisioning or mount/unmount operations. Workloads with attached volumes must be manually stopped, and
 those volumes must then be manually deleted by the user. Once the MicroShift config `storage` section is specified with
 supported values, the user may restart MicroShift. They should see that MicroShift does not redeploy the disabled
-components after restart.
+components after restart.
+
+## Drop-in configuration directory
+
+In addition to the existing `/etc/microshift/config.yaml` configuration file there is a `/etc/microshift/config.d` configuration directory where you can place fragments of configuration.
+
+At runtime, `/etc/microshift/config.yaml` and `.yaml` files inside `/etc/microshift/config.d` are merged together to create one configuration file which overrides the defaults.
+
+Files in `/etc/microshift/config.d` are sorted lexicographilly. It is recommended to use numerical prefix for easy reasoning about the priority of the fragments.
+
+For example, given following files:
+- `/etc/microshift/config.yaml`
+- `/etc/microshift/config.d/10-subjectAltNames.yaml`
+- `/etc/microshift/config.d/20-kubelet.yaml`
+
+The final user config will be created by using `config.yaml` as a base, and then overwriting it with `10-subjectAltNames.yaml`, and then overwriting it with `20-kubelet.yaml`:
+```
+20-kubelet.yaml
+     ||
+     \/
+10-subjectAltNames.yaml
+     ||
+     \/
+config.yaml
+```
+
+Some additional rules:
+- Lists are not merged together, they are overwritten. For example:
+  ```yaml
+  # 10-san.yaml
+  apiServer:
+    subjectAltNames:
+      - host1
+      - host2
+
+  # 20-san.yaml
+  apiServer:
+    subjectAltNames:
+      - hostZ
+
+  # end result
+  apiServer:
+    subjectAltNames:
+      - hostZ
+  ```
+- Contents of `kubelet:` configuration are merged together (unless specific field is a list). For example:
+  ```yaml
+  # 10-kubelet.yaml
+  kubelet:
+    some_setting: True
+
+  # 20-kubelet.yaml
+  kubelet:
+    another_setting: True
+
+  # end result
+  kubelet:
+    some_setting: True
+    another_setting: True
+  ```

packaging/rpm/microshift.spec

@@ -263,6 +263,7 @@ install -p -m644 packaging/systemd/microshift.service %{buildroot}%{_unitdir}/mi
 install -d -m755 %{buildroot}/%{_sysconfdir}/microshift
 install -d -m755 %{buildroot}/%{_sysconfdir}/microshift/manifests
 install -d -m755 %{buildroot}/%{_sysconfdir}/microshift/manifests.d
+install -d -m755 %{buildroot}/%{_sysconfdir}/microshift/config.d
 install -p -m644 packaging/microshift/config.yaml %{buildroot}%{_sysconfdir}/microshift/config.yaml.default
 install -p -m644 packaging/microshift/lvmd.yaml %{buildroot}%{_sysconfdir}/microshift/lvmd.yaml.default
 install -p -m644 packaging/microshift/ovn.yaml %{buildroot}%{_sysconfdir}/microshift/ovn.yaml.default
@@ -446,6 +447,7 @@ fi
 %{_sysconfdir}/crio/crio.conf.d/10-microshift.conf
 %{_datadir}/microshift/spec/config-openapi-spec.json
 %dir %{_sysconfdir}/microshift
+%dir %{_sysconfdir}/microshift/config.d
 %dir %{_sysconfdir}/microshift/manifests
 %dir %{_sysconfdir}/microshift/manifests.d
 %config(noreplace) %{_sysconfdir}/microshift/config.yaml.default
@@ -519,6 +521,9 @@ fi
 # Use Git command to generate the log and replace the VERSION string
 # LANG=C git log --date="format:%a %b %d %Y" --pretty="tformat:* %cd %an <%ae> VERSION%n- %s%n" packaging/rpm/microshift.spec
 %changelog
+* Fri Aug 30 2024 Patryk Matuszak <pmatusza@redhat.com> 4.18.0
+- Support for config drop-in directory
+
 * Mon Aug 26 2024 Nadia Pinaeva <n.m.pinaeva@gmail.com> 4.17.0
 - Update openvswitch to 3.4
 

pkg/config/config_test.go

@@ -392,7 +392,7 @@ func TestGetActiveConfigFromYAML(t *testing.T) {
 
 	for _, tt := range ttests {
 		t.Run(tt.name, func(t *testing.T) {
-			config, err := getActiveConfigFromYAML([]byte(tt.config))
+			config, err := getActiveConfigFromYAMLDropins([][]byte{[]byte(tt.config)})
 			// If we have any warnings, drop them. Use an empty array
 			// instead of nil so that we can differentiate between
 			// unexpected warnings (where we get an array instead of
@@ -418,6 +418,78 @@ func TestGetActiveConfigFromYAML(t *testing.T) {
 			}
 		})
 	}
+
+	t.Run("multiple-drop-ins", func(t *testing.T) {
+		dropins := [][]byte{
+			// Individual fields should be overwritten
+			[]byte(dedent(`
+            ingress:
+              ports:
+                http: 1234
+                https: 9876
+            `)),
+			[]byte(dedent(`
+            ingress:
+              ports:
+                http: 2345
+            `)),
+			[]byte(dedent(`
+            ingress:
+              ports:
+                https: 8765
+            `)),
+
+			// Arrays are overwritten completely (no addition)
+			[]byte(dedent(`
+            ingress:
+              listenAddress:
+                - eth1
+                - eth2
+            `)),
+			[]byte(dedent(`
+            ingress:
+              listenAddress:
+                - lo
+            `)),
+
+			// Even though kubelet is map[string]any, we want to merge individual settings
+			[]byte(dedent(`
+            kubelet:
+              cpuManagerPolicy: static
+              evictionHard:
+                imagefs.available: 15%
+                memory.available: 100Mi
+            `)),
+			[]byte(dedent(`
+            kubelet:
+              memoryManagerPolicy: Static
+              evictionHard:
+                nodefs.available: 10%
+                nodefs.inodesFree: 5%
+            `)),
+		}
+
+		expected := mkDefaultConfig()
+		expected.Ingress.Ports.Http = ptr.To[int](2345)
+		expected.Ingress.Ports.Https = ptr.To[int](8765)
+		expected.Ingress.ListenAddress = []string{"lo"}
+		expected.Kubelet = map[string]any{
+			"cpuManagerPolicy":    "static",
+			"memoryManagerPolicy": "Static",
+			"evictionHard": map[string]any{
+				"imagefs.available": "15%",
+				"memory.available":  "100Mi",
+				"nodefs.available":  "10%",
+				"nodefs.inodesFree": "5%",
+			},
+		}
+
+		config, err := getActiveConfigFromYAMLDropins(dropins)
+		assert.NoError(t, err)
+
+		config.userSettings = nil
+		assert.Equal(t, expected, config)
+	})
 }
 
 // Test the validation logic

pkg/config/files.go

@@ -1,64 +1,127 @@
 package config
 
 import (
+	"encoding/json"
 	"fmt"
+	"io/fs"
 	"os"
+	"path/filepath"
+	"strings"
 
+	jsonpatch "github.com/evanphx/json-patch"
+	"github.com/openshift/microshift/pkg/util"
 	"sigs.k8s.io/yaml"
 )
 
 const (
-	ConfigFile = "/etc/microshift/config.yaml"
-	DataDir    = "/var/lib/microshift"
-	BackupsDir = "/var/lib/microshift-backups"
+	ConfigFile      = "/etc/microshift/config.yaml"
+	DataDir         = "/var/lib/microshift"
+	BackupsDir      = "/var/lib/microshift-backups"
+	ConfigDropInDir = "/etc/microshift/config.d"
 )
 
-func parse(contents []byte) (*Config, error) {
-	c := &Config{}
-	if err := yaml.Unmarshal(contents, c); err != nil {
-		return nil, fmt.Errorf("failed to decode configuration: %v", err)
+func getActiveConfigFromYAMLDropins(yamlDropins [][]byte) (*Config, error) {
+	var mergedUserConfigPatch []byte
+
+	// Convert YAMLs to JSONs and merge them together to get a single configuration from the user.
+	for _, dropin := range yamlDropins {
+		if strings.TrimSpace(string(dropin)) == "" {
+			continue
+		}
+
+		jsonDropin, err := yaml.YAMLToJSON(dropin)
+		if err != nil {
+			return nil, fmt.Errorf("failed to convert config yaml (%q) to json: %w", string(dropin), err)
+		}
+
+		if mergedUserConfigPatch == nil {
+			mergedUserConfigPatch = jsonDropin
+			continue
+		}
+
+		patched, err := jsonpatch.MergePatch(mergedUserConfigPatch, jsonDropin)
+		if err != nil {
+			return nil, fmt.Errorf("failed to merge dropin (%q) into the config patch (%q): %w", string(jsonDropin), string(mergedUserConfigPatch), err)
+		}
+		mergedUserConfigPatch = patched
 	}
-	return c, nil
-}
 
-func getActiveConfigFromYAML(contents []byte) (*Config, error) {
-	userSettings, err := parse(contents)
-	if err != nil {
-		return nil, fmt.Errorf("failed to parse config file %q: %v", ConfigFile, err)
+	cfg := &Config{}
+	if err := cfg.fillDefaults(); err != nil {
+		return nil, fmt.Errorf("failed to fill config's defaults: %w", err)
 	}
 
-	// Start with the defaults, then apply the user settings and
-	// recompute dynamic values.
-	results := &Config{}
-	if err := results.fillDefaults(); err != nil {
-		return nil, fmt.Errorf("invalid configuration: %v", err)
+	if len(mergedUserConfigPatch) != 0 {
+		userSettings := &Config{}
+		if err := json.Unmarshal(mergedUserConfigPatch, userSettings); err != nil {
+			return nil, fmt.Errorf("failed to unmarshal user cfg json to config: %w", err)
+		}
+		cfg.incorporateUserSettings(userSettings)
 	}
-	results.incorporateUserSettings(userSettings)
-	if err := results.updateComputedValues(); err != nil {
-		return nil, fmt.Errorf("invalid configuration: %v", err)
+
+	if err := cfg.updateComputedValues(); err != nil {
+		return nil, fmt.Errorf("invalid configuration: %w", err)
 	}
-	if err := results.validate(); err != nil {
-		return nil, fmt.Errorf("invalid configuration: %v", err)
+
+	if err := cfg.validate(); err != nil {
+		return nil, fmt.Errorf("invalid configuration: %w", err)
 	}
-	return results, nil
+
+	return cfg, nil
 }
 
-// ActiveConfig returns the active configuration. If the configuration
-// file exists, read it and require it to be valid. Otherwise return
-// the default settings.
-func ActiveConfig() (*Config, error) {
-	_, err := os.Stat(ConfigFile)
-	if os.IsNotExist(err) {
-		// No configuration file, use the default settings
-		return NewDefault(), nil
-	} else if err != nil {
+// collectUserProvidedConfigs loads all the user provided yaml config files:
+// - main MicroShift config (/etc/microshift/config.yaml), and
+// - YAML files from config drop-in directory (/etc/microshift/config.d)
+func collectUserProvidedConfigs() ([][]byte, error) {
+	dropins := [][]byte{}
+
+	if exists, err := util.PathExists(ConfigFile); err != nil {
 		return nil, err
+	} else if exists {
+		contents, err := os.ReadFile(ConfigFile)
+		if err != nil {
+			return nil, fmt.Errorf("error reading config file %q: %v", ConfigFile, err)
+		}
+		dropins = append(dropins, contents)
 	}
 
-	// Read the file and merge user-provided settings with the defaults
-	contents, err := os.ReadFile(ConfigFile)
+	dropInDirExists, err := util.PathExistsAndIsNotEmpty(ConfigDropInDir)
 	if err != nil {
-		return nil, fmt.Errorf("error reading config file %q: %v", ConfigFile, err)
+		return nil, err
 	}
-	return getActiveConfigFromYAML(contents)
+
+	if !dropInDirExists {
+		return dropins, nil
+	}
+
+	err = filepath.WalkDir(ConfigDropInDir, func(path string, info fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if !info.IsDir() && filepath.Ext(info.Name()) == ".yaml" {
+			contents, err := os.ReadFile(path)
+			if err != nil {
+				return fmt.Errorf("error reading config file %q: %v", path, err)
+			}
+			dropins = append(dropins, contents)
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to walk the config drop-in dir %q: %w", ConfigDropInDir, err)
+	}
+
+	return dropins, nil
+}
+
+// ActiveConfig returns the active configuration which is default config with overrides
+// from user provided config files.
+func ActiveConfig() (*Config, error) {
+	dropins, err := collectUserProvidedConfigs()
+	if err != nil {
+		return nil, err
+	}
+
+	return getActiveConfigFromYAMLDropins(dropins)
 }

test/resources/microshift-config.resource

@@ -67,6 +67,18 @@ Upload MicroShift Config
     [Arguments]    ${config_content}
     Upload String To File    ${config_content}    /etc/microshift/config.yaml
 
+Drop In MicroShift Config
+    [Documentation]    Upload a drop-in configuration file to the MicroShift host
+    [Arguments]    ${config_content}    ${name}
+    Upload String To File    ${config_content}    /etc/microshift/config.d/${name}.yaml
+
+Remove Drop In MicroShift Config
+    [Documentation]    Remove a drop-in configuration file from MicroShift host
+    [Arguments]    ${name}
+    ${stdout}    ${rc}=    Execute Command
+    ...    rm -f /etc/microshift/config.d/${name}.yaml
+    ...    sudo=True    return_rc=True
+
 Save Lvmd Config
     [Documentation]    If an lvmd.yaml file already exists, preserver it
     ${stdout}    ${rc}=    Execute Command

test/suites/greenboot/greenboot.robot

@@ -57,7 +57,8 @@ Simulate Pod Failure
     ...    Restart Greenboot And Wait For Success
 
     [Teardown]    Run Keywords
-    ...    Restore Default MicroShift Config
+    ...    Remove Drop In MicroShift Config    10-svcNetwork
+    ...    AND
     ...    Cleanup And Start
 
 
@@ -122,16 +123,14 @@ Restore Service
 
 Disrupt Pod Network
     [Documentation]    Prevent Microshift pods From starting correctly
-    Save Default MicroShift Config
     ${configuration}=    Catenate    SEPARATOR=\n
     ...    network:
     ...    \ clusterNetwork:
     ...    \ - 10.42.0.0/16
     ...    \ serviceNetwork:
     ...    \ - 10.66.0.0/16
     ...
-    ${newconfig}=    Extend MicroShift Config    ${configuration}
-    Upload MicroShift Config    ${newconfig}
+    Drop In MicroShift Config    ${configuration}    10-svcNetwork
 
 Cleanup And Start
     [Documentation]    Wipe Microshift data and start it.