diff --git a/test/scenarios/releases/el98-lrel@osconfig.sh b/test/scenarios/releases/el98-lrel@osconfig.sh index d1c00e651b..2e6b00d979 100644 --- a/test/scenarios/releases/el98-lrel@osconfig.sh +++ b/test/scenarios/releases/el98-lrel@osconfig.sh @@ -22,5 +22,6 @@ scenario_run_tests() { run_tests host1 \ suites/osconfig/clusterid.robot \ - suites/osconfig/systemd-resolved.robot + suites/osconfig/systemd-resolved.robot \ + suites/osconfig/sos-report-plugins.robot } diff --git a/test/suites/configuration/configuration.robot b/test/suites/configuration/configuration.robot index 5a79c3f6f2..059b80c0b2 100644 --- a/test/suites/configuration/configuration.robot +++ b/test/suites/configuration/configuration.robot @@ -50,6 +50,10 @@ ${LVMS_CSI_SNAPSHOT_DISABLED} SEPARATOR=\n ... \ \ driver: "none" ... \ \ optionalCsiComponents: [ none ] ${APISERVER_ETCD_CLIENT_CERT} /var/lib/microshift/certs/etcd-signer/apiserver-etcd-client +${KCM_LOG_VLEVEL} SEPARATOR=\n +... --- +... debugging: +... \ \ logVLevel: 2 *** Test Cases *** @@ -153,6 +157,42 @@ Http Proxy Not Defined In Bootc Image ${is_bootc}= Is System Bootc IF ${is_bootc} Check HTTP Proxy Env In Bootc Image +Verify KCM Runs With OpenShift Flavor Flags + [Documentation] Verify that kube-controller-manager runs with the + ... OpenShift flavor flags logged in MicroShift journal. + ... OCP-56673 + [Setup] Setup KCM Log VLevel + + ${kcm_flags}= Command Should Work + ... journalctl -u microshift --no-pager | grep kube-controller-manager | grep FLAG + + FOR ${flag} IN + ... --enable-dynamic-provisioning\="true" + ... --allocate-node-cidrs\="true" + ... --use-service-account-credentials\="true" + ... --leader-elect\="false" + ... --leader-elect-retry-period\="3s" + ... --leader-elect-resource-lock\="leases" + ... --cluster-signing-duration\="720h0m0s" + ... --secure-port\="10257" + ... --cert-dir\="/var/run/kubernetes" + ... --root-ca-file\="/var/lib/microshift/certs/ca-bundle/service-account-token-ca.crt" + ... --service-account-private-key-file\="/var/lib/microshift/resources/kube-apiserver/secrets/service-account-key/service-account.key" + ... --cluster-signing-cert-file\="/var/lib/microshift/certs/kubelet-csr-signer-signer/csr-signer/ca.crt" + ... --cluster-signing-key-file\="/var/lib/microshift/certs/kubelet-csr-signer-signer/csr-signer/ca.key" + ... --kube-api-qps\="150" + ... --kube-api-burst\="300" + Should Contain ${kcm_flags} ${flag} + END + + Should Match Regexp ${kcm_flags} --controllers\="\\[.*-bootstrapsigner.*-tokencleaner.*-ttl.*\\]" + + ${openshift_config}= Command Should Work + ... journalctl -u microshift --no-pager | grep kube-controller-manager | grep openshift-config + Should Contain ${openshift_config} --openshift-config="" + + [Teardown] Teardown KCM Log VLevel + *** Keywords *** Setup @@ -166,6 +206,7 @@ Teardown [Documentation] Test suite teardown Remove Drop In MicroShift Config 10-loglevel Remove Drop In MicroShift Config 10-audit + Remove Drop In MicroShift Config 10-kcm-vlevel Restart MicroShift Logout MicroShift Host Remove Kubeconfig @@ -197,6 +238,16 @@ Setup Audit Flags Drop In MicroShift Config ${AUDIT_FLAGS} 10-audit Restart MicroShift +Setup KCM Log VLevel + [Documentation] Set verbose log level and restart to capture KCM flags + Drop In MicroShift Config ${KCM_LOG_VLEVEL} 10-kcm-vlevel + Restart MicroShift + +Teardown KCM Log VLevel + [Documentation] Remove KCM verbose log level drop-in and restart + Remove Drop In MicroShift Config 10-kcm-vlevel + Restart MicroShift + Deploy Storage Config [Documentation] Applies a storage ${config} to the exist MicroShift config, pushes it to the MicroShift host, ... and restarts microshift.service diff --git a/test/suites/osconfig/sos-report-plugins.robot b/test/suites/osconfig/sos-report-plugins.robot new file mode 100644 index 0000000000..56786bcf97 --- /dev/null +++ b/test/suites/osconfig/sos-report-plugins.robot @@ -0,0 +1,178 @@ +*** Settings *** +Documentation Tests verifying MicroShift SOS report plugins are listed and enabled, +... SOS report collects MicroShift config and OVN related information, +... and greenboot logs reside in a separate directory. +... +... Ported from openshift-tests-private: +... Critical-60924, Critical-60929, High-68257, High-60930, High-68256 + +Resource ../../resources/microshift-host.resource +Resource ../../resources/microshift-process.resource +Resource ../../resources/common.resource + +Suite Setup Setup +Suite Teardown Teardown + +Test Tags slow + + +*** Variables *** +${SOS_REPORT_BASE_DIR} /tmp/rf-test/sos-report-plugins + + +*** Test Cases *** +Verify SOS Report Lists Enabled MicroShift Plugins + [Documentation] Verify sos report -l lists both microshift and + ... microshift_ovn plugins and they are enabled. + ... OCP-60924 + ${output}= Command Should Work sos report -l + Should Contain ${output} microshift + Should Contain ${output} microshift_ovn + Plugin Should Be Enabled ${output} microshift + Plugin Should Be Enabled ${output} microshift_ovn + +Verify SOS Report Collects MicroShift And OVN Information + [Documentation] Verify running sos report with microshift profile + ... collects microshift config, oc adm inspect output, + ... and OVN related information. + ... OCP-60929, OCP-68257 + ${sos_report_tarfile}= Create Sos Report + ${sos_report_extracted}= Extract Sos Report ${sos_report_tarfile} + + ${sos_text}= Command Should Work cat ${sos_report_extracted}/sos_reports/sos.txt + Should Contain ${sos_text} oc adm inspect + Should Contain ${sos_text} ovs-appctl -t /var/run/ovn/ovn-controller.*.ctl ct-zone-list + + Verify Remote Directory Exists With Sudo ${sos_report_extracted}/etc/microshift + Verify Remote Directory Exists With Sudo ${sos_report_extracted}/etc/microshift/manifests + Verify Remote Directory Exists With Sudo ${sos_report_extracted}/etc/microshift/manifests.d + + ${system_config}= Command Should Work + ... ls -l /etc/microshift | awk '{print $NF}' | tail -n+2 + ${report_config}= Command Should Work + ... ls -l ${sos_report_extracted}/etc/microshift | awk '{print $NF}' | tail -n+2 + Should Be Equal As Strings ${report_config} ${system_config} + ... msg=Not all microshift config files are collected in sosreport + + ${system_manifests}= Command Should Work ls -l /etc/microshift/manifests + ${report_manifests}= Command Should Work + ... ls -l ${sos_report_extracted}/etc/microshift/manifests + Should Be Equal As Strings ${report_manifests} ${system_manifests} + ... msg=Files inside /etc/microshift/manifests directory does not match between system and sosreport + + ${system_manifests_d}= Command Should Work ls -l /etc/microshift/manifests.d/ + ${report_manifests_d}= Command Should Work + ... ls -l ${sos_report_extracted}/etc/microshift/manifests.d/ + Should Be Equal As Strings ${report_manifests_d} ${system_manifests_d} + ... msg=Files inside /etc/microshift/manifests.d directory does not match between system and sosreport + +Verify SOS Report Collects Data When MicroShift Is Stopped + [Documentation] Verify that sos report -p microshift still collects + ... config files, journal logs, and OVN data when MicroShift + ... service is stopped, but does not collect oc adm inspect data. + [Tags] restart + + Stop MicroShift + + ${rand_str}= Generate Random String 4 [NUMBERS] + ${sos_report_dir}= Catenate SEPARATOR=${EMPTY} ${SOS_REPORT_BASE_DIR}_ ${rand_str} + Command Should Work mkdir -p ${sos_report_dir} + Command Should Work + ... sos report --batch --tmp-dir ${sos_report_dir} -p microshift + ${sos_report_tarfile}= Command Should Work + ... find ${sos_report_dir} -type f -name "sosreport-*.tar.xz" + Should Not Be Empty ${sos_report_tarfile} + ${sos_report_extracted}= Extract Sos Report ${sos_report_tarfile} + + ${sos_text}= Command Should Work cat ${sos_report_extracted}/sos_reports/sos.txt + Should Not Contain ${sos_text} oc adm inspect + + Verify Remote Directory Exists With Sudo ${sos_report_extracted}/etc/microshift + Verify Remote Directory Exists With Sudo ${sos_report_extracted}/etc/microshift/manifests + Verify Remote Directory Exists With Sudo ${sos_report_extracted}/etc/microshift/manifests.d + Verify Remote Directory Exists With Sudo ${sos_report_extracted}/sos_commands/microshift + Verify Remote Directory Exists With Sudo ${sos_report_extracted}/sos_commands/microshift_ovn + + ${microshift_cmds}= Command Should Work + ... ls ${sos_report_extracted}/sos_commands/microshift + Should Contain ${microshift_cmds} journalctl_--no-pager_--unit_microshift + Should Contain ${microshift_cmds} journalctl_--no-pager_--unit_microshift-etcd.scope + Should Contain ${microshift_cmds} microshift_version + Should Contain ${microshift_cmds} systemctl_status_microshift + + [Teardown] Start MicroShift + +Verify Greenboot Logs In SOS Report + [Documentation] Verify greenboot logs reside in a separate directory + ... and are easy to access in the SOS report. + ... OCP-68256 + ${sos_report_tarfile}= Create Sos Report With Profile system + ${sos_report_extracted}= Extract Sos Report ${sos_report_tarfile} + + ${greenboot_files}= Command Should Work + ... ls -l ${sos_report_extracted}/sos_commands/greenboot + Should Contain ${greenboot_files} journalctl_--no-pager_--unit_greenboot-healthcheck + Should Contain ${greenboot_files} systemctl_status_greenboot-healthcheck + + ${greenboot_conf}= Command Should Work + ... ls -l ${sos_report_extracted}/etc/greenboot + Should Contain ${greenboot_conf} greenboot.conf + + +*** Keywords *** +Setup + [Documentation] Set up all of the tests in this suite + Check Required Env Variables + Login MicroShift Host + +Teardown + [Documentation] Test suite teardown + Cleanup Sos Report Directory + Logout MicroShift Host + +Plugin Should Be Enabled + [Documentation] Verify that the named plugin is not disabled in the sos report -l output. + [Arguments] ${output} ${plugin_name} + Should Not Match Regexp ${output} ${plugin_name}\\s+.*disabled + +Create Sos Report + [Documentation] Create a MicroShift SOS Report using microshift-sos-report + ... and return the tar file path. + ${rand_str}= Generate Random String 4 [NUMBERS] + ${sos_report_dir}= Catenate SEPARATOR=${EMPTY} ${SOS_REPORT_BASE_DIR}_ ${rand_str} + Command Should Work mkdir -p ${sos_report_dir} + ${output}= Command Should Work + ... /usr/bin/microshift-sos-report --tmp-dir ${sos_report_dir} + Should Contain ${output} Your sos report has been generated and saved in + ${sos_report_tarfile}= Command Should Work + ... find ${sos_report_dir} -type f -name "sosreport-*.tar.xz" + Should Not Be Empty ${sos_report_tarfile} + RETURN ${sos_report_tarfile} + +Create Sos Report With Profile + [Documentation] Create a SOS report with the specified profile and return the tar file path. + [Arguments] ${profile} + ${rand_str}= Generate Random String 4 [NUMBERS] + ${sos_report_dir}= Catenate SEPARATOR=${EMPTY} ${SOS_REPORT_BASE_DIR}_ ${rand_str} + Command Should Work mkdir -p ${sos_report_dir} + Command Should Work + ... sos report --batch --clean --all-logs --profile ${profile} --tmp-dir ${sos_report_dir} + ${sos_report_tarfile}= Command Should Work + ... find ${sos_report_dir} -type f -name "sosreport-*.tar.xz" + Should Not Be Empty ${sos_report_tarfile} + RETURN ${sos_report_tarfile} + +Extract Sos Report + [Documentation] Extract SOS report from the tar file and return the extracted directory. + [Arguments] ${sos_report_tarfile} + ${sos_report_dir}= Command Should Work dirname ${sos_report_tarfile} + Command Should Work tar xf ${sos_report_tarfile} -C ${sos_report_dir} + ${sos_report_untared}= Command Should Work + ... find ${sos_report_dir} -maxdepth 1 -type d -name "sosreport-*" + Should Not Be Empty ${sos_report_untared} + RETURN ${sos_report_untared} + +Cleanup Sos Report Directory + [Documentation] Remove temporary SOS report files created during tests. + Command Should Work rm -rf ${SOS_REPORT_BASE_DIR}_* + Command Should Work rm -rf /var/tmp/sosreport-*