USHIFT-1199: add automated tests for etcd memory limit management

test/resources/YAML.py

@@ -5,6 +5,65 @@
 
 
 def yaml_parse(data):
-    """Parse input string as YAML and return DotDict instance."""
+    """Parse input string as YAML and return DotDict instance.
+
+    If the input data is empty, return an empty DotDict.
+    """
+    if not data:
+        return DotDict()
     parsed = yaml.safe_load(data)
     return DotDict(parsed)
+
+
+def _merge(dest, addition):
+    """Implements data structure merge.
+
+    The dest value is modified in place by adding missing values found
+    in the addition parameter.  Existing containers (dict, list) are
+    updated. Scalar values (int, string) are replaced.
+
+    >>> _merge({}, {})
+    {}
+
+    >>> _merge({}, {'new-key': 'value'})
+    {'new-key': 'value'}
+
+    >>> _merge({'same-key': 'old-value'}, {'same-key': 'new-value'})
+    {'same-key': 'new-value'}
+
+    >>> _merge({'nested': {}}, {'nested': {'new-key': 'new-value'}})
+    {'nested': {'new-key': 'new-value'}}
+
+    >>> _merge({'list': ['a']}, {'list': ['b']})
+    {'list': ['a', 'b']}
+
+    >>> _merge({'nested-list': {'list': ['a']}}, {'nested-list': {'list': ['b']}})
+    {'nested-list': {'list': ['a', 'b']}}
+
+    """
+    for key, value in addition.items():
+        if key in dest:
+            if isinstance(value, dict):
+                _merge(dest[key], value)
+                continue
+            if isinstance(value, list):
+                dest[key].extend(value)
+                continue
+        # Either a new item or replacing a non-container type
+        dest[key] = value
+    # The return value is for doctest
+    return dest
+
+
+def yaml_merge(base, addition):
+    """Return combination of both YAML data structures, additively."""
+    if not base:
+        combined = {}
+    else:
+        combined = yaml.safe_load(base)
+    if not addition:
+        parsed_addition = {}
+    else:
+        parsed_addition = yaml.safe_load(addition)
+    _merge(combined, parsed_addition)
+    return yaml.dump(combined)

test/resources/microshift-config.resource

@@ -0,0 +1,80 @@
+*** Settings ***
+Documentation  Keywords for running the microshift command line.
+
+Library    Process
+Library    String
+Library    OperatingSystem
+
+Library    SSHLibrary
+Library     ../resources/YAML.py
+
+
+*** Keywords ***
+Save Default MicroShift Config
+    [Documentation]  Fetch the current config settings and preserve them as the default
+    ...
+    ...  Sets the suite variable DEFAULT_MICROSHIFT_CONFIG to the text value
+    ...  based on the contents of /etc/microshift/config.yaml, if it exists, or
+    ...  an empty string if the file does not exist.
+    ...
+    ...  This keyword is meant to be used from a Setup step.
+    ${stdout}  ${rc}=  Execute Command
+    ...    cat /etc/microshift/config.yaml
+    ...    sudo=True  return_rc=True
+    IF  ${rc} == 0
+        Set Suite Variable  \${DEFAULT_MICROSHIFT_CONFIG}  ${stdout}
+    ELSE
+        Set Suite Variable  \${DEFAULT_MICROSHIFT_CONFIG}  ${EMPTY}
+    END
+
+Restore Default MicroShift Config
+    [Documentation]  Replace the microshift config file with the original defaults.
+    ...
+    ...  If there was no configuration file originally, delete any file that is there now.
+    ${len}=  Get Length  ${DEFAULT_MICROSHIFT_CONFIG}
+    IF  ${len} == 0
+        # If there was no configuration file to start with, we do not want to create
+        # a new one, even if it is empty.
+        Clear MicroShift Config
+    ELSE
+        Upload MicroShift Config  ${DEFAULT_MICROSHIFT_CONFIG}
+    END
+
+Extend MicroShift Config
+    [Documentation]  Return combination of default config and input argument as a string.
+    ...
+    ...  The values are parsed as YAML and merged additively (no keys are deleted
+    ...  and list values are extended but not replaced) by 'Yaml Merge'.
+    [Arguments]  ${config}
+    ${merged}=  Yaml Merge  ${DEFAULT_MICROSHIFT_CONFIG}  ${config}
+    RETURN  ${merged}
+
+Clear MicroShift Config
+    [Documentation]  Remove any configuration file
+    ${stdout}  ${rc}=  Execute Command
+    ...    rm -f /etc/microshift/config.yaml
+    ...    sudo=True  return_rc=True
+
+Upload MicroShift Config  # robocop: disable=too-many-calls-in-keyword
+    [Documentation]  Upload a new configuration file to the MicroShift host
+    [Arguments]  ${config_content}
+
+    ${rand}=    Generate Random String
+    ${local_tmp}=    Join Path    /tmp    ${rand}
+    Create File    ${local_tmp}    ${config_content}
+
+    ${rand}=    Generate Random String
+    ${remote_tmp}=    Join Path    /tmp    ${rand}
+    Put File  ${local_tmp}  ${remote_tmp}  mode=0644
+
+    Remove File  ${local_tmp}
+
+    ${stdout}  ${rc}=  Execute Command
+    ...    mv ${remote_tmp} /etc/microshift/config.yaml
+    ...    sudo=True  return_rc=True
+    Should Be Equal As Integers  0  ${rc}
+
+    ${stdout}  ${rc}=  Execute Command
+    ...    chown root:root /etc/microshift/config.yaml
+    ...    sudo=True  return_rc=True
+    Should Be Equal As Integers  0  ${rc}

test/resources/microshift-process.resource

@@ -6,8 +6,10 @@ Library    String
 Library    OperatingSystem
 
 Library    SSHLibrary
+Resource    ../resources/oc.resource
+Resource    ../resources/systemd.resource
 Resource    ../resources/microshift-host.resource
-Library    ../resources/YAML.py
+Library     ../resources/YAML.py
 
 
 *** Keywords ***
@@ -20,3 +22,42 @@ MicroShift Version
     Should Not Be Empty    ${version_text}
     ${version}=  Yaml Parse  ${version_text}
     RETURN  ${version}
+
+MicroShift Is Ready
+    [Documentation]  Check the /readyz endpoint
+    ${stdout}=  Run With Kubeconfig  oc get --raw='/readyz'
+    Should Be Equal As Strings  ${stdout}  ok  strip_spaces=True
+
+MicroShift Is Live
+    [Documentation]  Check the /livez endpoint
+    ${stdout}=  Run With Kubeconfig  oc get --raw='/livez'
+    Should Be Equal As Strings  ${stdout}  ok  strip_spaces=True
+
+Wait For MicroShift
+    [Documentation]  Wait for various checks to ensure MicroShift is online.
+    Wait Until Keyword Succeeds
+    ...  30x
+    ...  10s
+    ...  MicroShift Is Ready
+    Wait Until Keyword Succeeds
+    ...  30x
+    ...  10s
+    ...  MicroShift Is Live
+    # We could also wait for relevant pods. Can we restructure the
+    # greenboot check script to let us use it even when not on a
+    # greenboot host?
+
+Restart MicroShift
+    [Documentation]  Restart the MicroShift service
+    # Use separate stop and start steps to avoid a race condition with
+    # restart where we can't tell when it is down and so we can't tell
+    # whether we should count it as back up yet. Forcing the service
+    # down, then back up, ensures that when we do things like check
+    # systemd settings for the running process we get the values we
+    # expect without worrying about the race. Any test that modifies
+    # the MicroShift configuration file will be more reliable as a
+    # result.
+    Systemctl With Retry  stop  microshift.service
+    Sleep  10 seconds
+    Systemctl With Retry  start  microshift.service
+    Wait For MicroShift

test/resources/systemd.resource

@@ -0,0 +1,77 @@
+*** Settings ***
+Documentation    Keywords for interacting with systemd
+
+Library    Process
+Library    String
+Resource   ../resources/microshift-host.resource
+
+
+*** Keywords ***
+Get Systemd Setting
+    [Documentation]  Fetch one setting from systemd for the named unit.
+    ...  Take care to get the unit_name value _exactly_ right, or
+    ...  systemd will report a default value without reporting any error
+    ...  or warning.
+    [Arguments]  ${unit_name}  ${property}
+
+    ${stdout}    ${rc}=    Execute Command
+    ...    systemctl show --property=${property} --value ${unit_name}
+    ...    sudo=True    return_rc=True
+    Should Be Equal As Integers    ${rc}    0
+    Should Not Be Empty    ${stdout}
+
+    ${result}=  Strip String  ${stdout}
+    RETURN  ${result}
+
+Systemctl    # robocop: disable=too-long-keyword
+    [Documentation]  Run a systemctl command on the microshift host.
+    ...  The intent is to start, stop, or restart a service. Other
+    ...  commands should be implemented separately.  When the verb is
+    ...  "start" or "restart", this keyword will wait for the unit
+    ...  to report that it is "running". When the verb is "stop", this
+    ...  keyword will wait for the unit to report that it is "dead".
+    [Arguments]  ${verb}  ${unit_name}
+
+    IF  "${verb}" in {"restart", "start"}
+        ${state}=  Set Variable  running
+    ELSE
+        ${state}=  Set Variable  dead
+    END
+
+    ${stdout}  ${stderr}  ${rc}=  Execute Command
+    ...    systemctl ${verb} ${unit_name}
+    ...    sudo=True
+    ...    return_stdout=True
+    ...    return_stderr=True
+    ...    return_rc=True
+    IF  ${rc} != 0
+        ${status_text}=  Execute Command
+        ...    systemctl status ${unit_name}
+        ...    sudo=True
+        ...    return_stdout=True
+        ${log_text}=  Execute Command
+        ...    journalctl -u ${unit_name} -o short | tail -n 100
+        ...    sudo=True
+        ...    return_stdout=True
+    END
+    Should Be Equal As Integers  0  ${rc}
+
+    # It takes a bit for systemd to respond, and if we check too soon
+    # then it looks like microshift is up, even though it is about to
+    # be restarted.
+    Sleep    5s
+
+    Wait Until Keyword Succeeds
+    ...    10x
+    ...    10s
+    ...    Execute Command
+    ...    [ $(systemctl show -p SubState --value ${unit_name}) = ${state} ]
+    ...    timeout=10s    return_stdout=True    return_stderr=True
+
+Systemctl With Retry
+    [Documentation]  Run Systemctl keyword but retry 10 times
+    [Arguments]  ${verb}  ${unit_name}
+    Wait Until Keyword Succeeds
+    ...    10x
+    ...    10s
+    ...    Systemctl  ${verb}  ${unit_name}

test/suites/etcd.robot

@@ -0,0 +1,78 @@
+*** Settings ***
+Documentation       Tests related to how etcd is managed
+
+Resource            ../resources/common.resource
+Resource            ../resources/systemd.resource
+Resource            ../resources/microshift-config.resource
+Resource            ../resources/microshift-process.resource
+Library             Collections
+
+Suite Setup         Setup
+Suite Teardown      Teardown
+
+Test Tags           etcd  configuration  restart  slow
+
+
+*** Variables ***
+${ETCD_SYSTEMD_UNIT}    microshift-etcd.scope
+${MEMLIMIT128}      SEPARATOR=\n
+...  ---
+...  etcd:
+...  \ \ memoryLimitMB: 128
+${MEMLIMIT0}      SEPARATOR=\n
+...  ---
+...  etcd:
+...  \ \ memoryLimitMB: 0
+
+
+*** Test Cases ***
+Set MemoryHigh Limit Unlimited
+    [Documentation]  The default configuration should not limit RAM
+    ...
+    ...  Since we cannot assume that the default configuration file is
+    ...  being used, the test explicitly configures a '0' limit, which
+    ...  is equivalent to not having any configuration at all.
+    [Setup]  Setup With Custom Config  ${MEMLIMIT0}
+    Expect MemoryHigh  infinity
+
+Set MemoryHigh Limit 128MB
+    [Documentation]  Set the memory limit for etcd to 128MB and ensure it takes effect
+    [Setup]  Setup With Custom Config  ${MEMLIMIT128}
+    # Expecting the setting to be 128 * 1024 * 1024
+    Expect MemoryHigh  134217728
+    [Teardown]  Restore Default Config
+
+
+*** Keywords ***
+Setup
+    [Documentation]  Test suite setup
+    Check Required Env Variables
+    Login MicroShift Host
+    Setup Kubeconfig  # for readiness checks
+    Save Default MicroShift Config
+
+Teardown
+    [Documentation]  Test suite teardown
+    Restore Default Config
+    Logout MicroShift Host
+    Remove Kubeconfig
+
+Restore Default Config
+    [Documentation]  Remove any custom config and restart MicroShift
+    Restore Default MicroShift Config
+    Restart MicroShift
+
+Setup With Custom Config
+    [Documentation]  Install a custom config and restart MicroShift
+    [Arguments]  ${config_content}
+    ${merged}=  Extend MicroShift Config  ${config_content}
+    Upload MicroShift Config  ${merged}
+    Restart MicroShift
+
+Expect MemoryHigh
+    [Documentation]  Verify that the MemoryHigh setting for etcd matches the expected value
+    [Arguments]  ${expected}
+    ${actual}=  Get Systemd Setting  microshift-etcd.scope  MemoryHigh
+    # Using integer comparison is complicated here because sometimes
+    # the returned or expected value is 'infinity'.
+    Should Be Equal  ${expected}  ${actual}