From 8a0c98684bf056bef89e852bbc01e050c443ae23 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Sat, 6 May 2023 13:05:21 -0400 Subject: [PATCH] USHIFT-1199: add automated tests for etcd memory limit management Signed-off-by: Doug Hellmann --- test/resources/YAML.py | 61 ++++++++++++++++- test/resources/microshift-config.resource | 80 ++++++++++++++++++++++ test/resources/microshift-process.resource | 43 +++++++++++- test/resources/systemd.resource | 77 +++++++++++++++++++++ test/suites/etcd.robot | 78 +++++++++++++++++++++ 5 files changed, 337 insertions(+), 2 deletions(-) create mode 100644 test/resources/microshift-config.resource create mode 100644 test/resources/systemd.resource create mode 100644 test/suites/etcd.robot diff --git a/test/resources/YAML.py b/test/resources/YAML.py index de066175e6..26f0fe152c 100644 --- a/test/resources/YAML.py +++ b/test/resources/YAML.py @@ -5,6 +5,65 @@ def yaml_parse(data): - """Parse input string as YAML and return DotDict instance.""" + """Parse input string as YAML and return DotDict instance. + + If the input data is empty, return an empty DotDict. + """ + if not data: + return DotDict() parsed = yaml.safe_load(data) return DotDict(parsed) + + +def _merge(dest, addition): + """Implements data structure merge. + + The dest value is modified in place by adding missing values found + in the addition parameter. Existing containers (dict, list) are + updated. Scalar values (int, string) are replaced. + + >>> _merge({}, {}) + {} + + >>> _merge({}, {'new-key': 'value'}) + {'new-key': 'value'} + + >>> _merge({'same-key': 'old-value'}, {'same-key': 'new-value'}) + {'same-key': 'new-value'} + + >>> _merge({'nested': {}}, {'nested': {'new-key': 'new-value'}}) + {'nested': {'new-key': 'new-value'}} + + >>> _merge({'list': ['a']}, {'list': ['b']}) + {'list': ['a', 'b']} + + >>> _merge({'nested-list': {'list': ['a']}}, {'nested-list': {'list': ['b']}}) + {'nested-list': {'list': ['a', 'b']}} + + """ + for key, value in addition.items(): + if key in dest: + if isinstance(value, dict): + _merge(dest[key], value) + continue + if isinstance(value, list): + dest[key].extend(value) + continue + # Either a new item or replacing a non-container type + dest[key] = value + # The return value is for doctest + return dest + + +def yaml_merge(base, addition): + """Return combination of both YAML data structures, additively.""" + if not base: + combined = {} + else: + combined = yaml.safe_load(base) + if not addition: + parsed_addition = {} + else: + parsed_addition = yaml.safe_load(addition) + _merge(combined, parsed_addition) + return yaml.dump(combined) diff --git a/test/resources/microshift-config.resource b/test/resources/microshift-config.resource new file mode 100644 index 0000000000..ca2703940a --- /dev/null +++ b/test/resources/microshift-config.resource @@ -0,0 +1,80 @@ +*** Settings *** +Documentation Keywords for running the microshift command line. + +Library Process +Library String +Library OperatingSystem + +Library SSHLibrary +Library ../resources/YAML.py + + +*** Keywords *** +Save Default MicroShift Config + [Documentation] Fetch the current config settings and preserve them as the default + ... + ... Sets the suite variable DEFAULT_MICROSHIFT_CONFIG to the text value + ... based on the contents of /etc/microshift/config.yaml, if it exists, or + ... an empty string if the file does not exist. + ... + ... This keyword is meant to be used from a Setup step. + ${stdout} ${rc}= Execute Command + ... cat /etc/microshift/config.yaml + ... sudo=True return_rc=True + IF ${rc} == 0 + Set Suite Variable \${DEFAULT_MICROSHIFT_CONFIG} ${stdout} + ELSE + Set Suite Variable \${DEFAULT_MICROSHIFT_CONFIG} ${EMPTY} + END + +Restore Default MicroShift Config + [Documentation] Replace the microshift config file with the original defaults. + ... + ... If there was no configuration file originally, delete any file that is there now. + ${len}= Get Length ${DEFAULT_MICROSHIFT_CONFIG} + IF ${len} == 0 + # If there was no configuration file to start with, we do not want to create + # a new one, even if it is empty. + Clear MicroShift Config + ELSE + Upload MicroShift Config ${DEFAULT_MICROSHIFT_CONFIG} + END + +Extend MicroShift Config + [Documentation] Return combination of default config and input argument as a string. + ... + ... The values are parsed as YAML and merged additively (no keys are deleted + ... and list values are extended but not replaced) by 'Yaml Merge'. + [Arguments] ${config} + ${merged}= Yaml Merge ${DEFAULT_MICROSHIFT_CONFIG} ${config} + RETURN ${merged} + +Clear MicroShift Config + [Documentation] Remove any configuration file + ${stdout} ${rc}= Execute Command + ... rm -f /etc/microshift/config.yaml + ... sudo=True return_rc=True + +Upload MicroShift Config # robocop: disable=too-many-calls-in-keyword + [Documentation] Upload a new configuration file to the MicroShift host + [Arguments] ${config_content} + + ${rand}= Generate Random String + ${local_tmp}= Join Path /tmp ${rand} + Create File ${local_tmp} ${config_content} + + ${rand}= Generate Random String + ${remote_tmp}= Join Path /tmp ${rand} + Put File ${local_tmp} ${remote_tmp} mode=0644 + + Remove File ${local_tmp} + + ${stdout} ${rc}= Execute Command + ... mv ${remote_tmp} /etc/microshift/config.yaml + ... sudo=True return_rc=True + Should Be Equal As Integers 0 ${rc} + + ${stdout} ${rc}= Execute Command + ... chown root:root /etc/microshift/config.yaml + ... sudo=True return_rc=True + Should Be Equal As Integers 0 ${rc} diff --git a/test/resources/microshift-process.resource b/test/resources/microshift-process.resource index ddf57de475..e388837d7d 100644 --- a/test/resources/microshift-process.resource +++ b/test/resources/microshift-process.resource @@ -6,8 +6,10 @@ Library String Library OperatingSystem Library SSHLibrary +Resource ../resources/oc.resource +Resource ../resources/systemd.resource Resource ../resources/microshift-host.resource -Library ../resources/YAML.py +Library ../resources/YAML.py *** Keywords *** @@ -20,3 +22,42 @@ MicroShift Version Should Not Be Empty ${version_text} ${version}= Yaml Parse ${version_text} RETURN ${version} + +MicroShift Is Ready + [Documentation] Check the /readyz endpoint + ${stdout}= Run With Kubeconfig oc get --raw='/readyz' + Should Be Equal As Strings ${stdout} ok strip_spaces=True + +MicroShift Is Live + [Documentation] Check the /livez endpoint + ${stdout}= Run With Kubeconfig oc get --raw='/livez' + Should Be Equal As Strings ${stdout} ok strip_spaces=True + +Wait For MicroShift + [Documentation] Wait for various checks to ensure MicroShift is online. + Wait Until Keyword Succeeds + ... 30x + ... 10s + ... MicroShift Is Ready + Wait Until Keyword Succeeds + ... 30x + ... 10s + ... MicroShift Is Live + # We could also wait for relevant pods. Can we restructure the + # greenboot check script to let us use it even when not on a + # greenboot host? + +Restart MicroShift + [Documentation] Restart the MicroShift service + # Use separate stop and start steps to avoid a race condition with + # restart where we can't tell when it is down and so we can't tell + # whether we should count it as back up yet. Forcing the service + # down, then back up, ensures that when we do things like check + # systemd settings for the running process we get the values we + # expect without worrying about the race. Any test that modifies + # the MicroShift configuration file will be more reliable as a + # result. + Systemctl With Retry stop microshift.service + Sleep 10 seconds + Systemctl With Retry start microshift.service + Wait For MicroShift diff --git a/test/resources/systemd.resource b/test/resources/systemd.resource new file mode 100644 index 0000000000..4a71859db0 --- /dev/null +++ b/test/resources/systemd.resource @@ -0,0 +1,77 @@ +*** Settings *** +Documentation Keywords for interacting with systemd + +Library Process +Library String +Resource ../resources/microshift-host.resource + + +*** Keywords *** +Get Systemd Setting + [Documentation] Fetch one setting from systemd for the named unit. + ... Take care to get the unit_name value _exactly_ right, or + ... systemd will report a default value without reporting any error + ... or warning. + [Arguments] ${unit_name} ${property} + + ${stdout} ${rc}= Execute Command + ... systemctl show --property=${property} --value ${unit_name} + ... sudo=True return_rc=True + Should Be Equal As Integers ${rc} 0 + Should Not Be Empty ${stdout} + + ${result}= Strip String ${stdout} + RETURN ${result} + +Systemctl # robocop: disable=too-long-keyword + [Documentation] Run a systemctl command on the microshift host. + ... The intent is to start, stop, or restart a service. Other + ... commands should be implemented separately. When the verb is + ... "start" or "restart", this keyword will wait for the unit + ... to report that it is "running". When the verb is "stop", this + ... keyword will wait for the unit to report that it is "dead". + [Arguments] ${verb} ${unit_name} + + IF "${verb}" in {"restart", "start"} + ${state}= Set Variable running + ELSE + ${state}= Set Variable dead + END + + ${stdout} ${stderr} ${rc}= Execute Command + ... systemctl ${verb} ${unit_name} + ... sudo=True + ... return_stdout=True + ... return_stderr=True + ... return_rc=True + IF ${rc} != 0 + ${status_text}= Execute Command + ... systemctl status ${unit_name} + ... sudo=True + ... return_stdout=True + ${log_text}= Execute Command + ... journalctl -u ${unit_name} -o short | tail -n 100 + ... sudo=True + ... return_stdout=True + END + Should Be Equal As Integers 0 ${rc} + + # It takes a bit for systemd to respond, and if we check too soon + # then it looks like microshift is up, even though it is about to + # be restarted. + Sleep 5s + + Wait Until Keyword Succeeds + ... 10x + ... 10s + ... Execute Command + ... [ $(systemctl show -p SubState --value ${unit_name}) = ${state} ] + ... timeout=10s return_stdout=True return_stderr=True + +Systemctl With Retry + [Documentation] Run Systemctl keyword but retry 10 times + [Arguments] ${verb} ${unit_name} + Wait Until Keyword Succeeds + ... 10x + ... 10s + ... Systemctl ${verb} ${unit_name} diff --git a/test/suites/etcd.robot b/test/suites/etcd.robot new file mode 100644 index 0000000000..fc39175ab7 --- /dev/null +++ b/test/suites/etcd.robot @@ -0,0 +1,78 @@ +*** Settings *** +Documentation Tests related to how etcd is managed + +Resource ../resources/common.resource +Resource ../resources/systemd.resource +Resource ../resources/microshift-config.resource +Resource ../resources/microshift-process.resource +Library Collections + +Suite Setup Setup +Suite Teardown Teardown + +Test Tags etcd configuration restart slow + + +*** Variables *** +${ETCD_SYSTEMD_UNIT} microshift-etcd.scope +${MEMLIMIT128} SEPARATOR=\n +... --- +... etcd: +... \ \ memoryLimitMB: 128 +${MEMLIMIT0} SEPARATOR=\n +... --- +... etcd: +... \ \ memoryLimitMB: 0 + + +*** Test Cases *** +Set MemoryHigh Limit Unlimited + [Documentation] The default configuration should not limit RAM + ... + ... Since we cannot assume that the default configuration file is + ... being used, the test explicitly configures a '0' limit, which + ... is equivalent to not having any configuration at all. + [Setup] Setup With Custom Config ${MEMLIMIT0} + Expect MemoryHigh infinity + +Set MemoryHigh Limit 128MB + [Documentation] Set the memory limit for etcd to 128MB and ensure it takes effect + [Setup] Setup With Custom Config ${MEMLIMIT128} + # Expecting the setting to be 128 * 1024 * 1024 + Expect MemoryHigh 134217728 + [Teardown] Restore Default Config + + +*** Keywords *** +Setup + [Documentation] Test suite setup + Check Required Env Variables + Login MicroShift Host + Setup Kubeconfig # for readiness checks + Save Default MicroShift Config + +Teardown + [Documentation] Test suite teardown + Restore Default Config + Logout MicroShift Host + Remove Kubeconfig + +Restore Default Config + [Documentation] Remove any custom config and restart MicroShift + Restore Default MicroShift Config + Restart MicroShift + +Setup With Custom Config + [Documentation] Install a custom config and restart MicroShift + [Arguments] ${config_content} + ${merged}= Extend MicroShift Config ${config_content} + Upload MicroShift Config ${merged} + Restart MicroShift + +Expect MemoryHigh + [Documentation] Verify that the MemoryHigh setting for etcd matches the expected value + [Arguments] ${expected} + ${actual}= Get Systemd Setting microshift-etcd.scope MemoryHigh + # Using integer comparison is complicated here because sometimes + # the returned or expected value is 'infinity'. + Should Be Equal ${expected} ${actual}