From f0f28176816a31b670ce55f9d5a2b026806c341a Mon Sep 17 00:00:00 2001 From: James Falcon Date: Wed, 18 Nov 2020 09:34:26 -0600 Subject: [PATCH 1/2] Collect logs from integration test runs During teardown of every cloud instance, run 'cloud-init collect-logs', then transfer and unpack locally. Two new integration settings have been added to specify whether to perform this action, and where to store these logs. --- tests/integration_tests/conftest.py | 30 +++++++++++++++++++ tests/integration_tests/instances.py | 8 ++--- .../integration_tests/integration_settings.py | 6 ++++ 3 files changed, 40 insertions(+), 4 deletions(-) diff --git a/tests/integration_tests/conftest.py b/tests/integration_tests/conftest.py index 73b44bfc894..45d51c00b1b 100644 --- a/tests/integration_tests/conftest.py +++ b/tests/integration_tests/conftest.py @@ -1,9 +1,12 @@ # This file is part of cloud-init. See LICENSE file for license information. +import datetime import logging import os import pytest import sys +from tarfile import TarFile from contextlib import contextmanager +from pathlib import Path from tests.integration_tests import integration_settings from tests.integration_tests.clouds import ( @@ -29,6 +32,8 @@ 'lxd_vm': LxdVmCloud, } +session_start_time = datetime.datetime.now().strftime('%y%m%d%H%M%S') + def pytest_runtest_setup(item): """Skip tests on unsupported clouds. @@ -114,6 +119,29 @@ def setup_image(session_cloud): log.info('Done with environment setup') +def _collect_logs(instance, node_id): + instance.execute( + 'cloud-init collect-logs -u -t /var/tmp/cloud-init.tar.gz') + node_id_path = Path( + node_id + .replace('.py', '') # Having a directory with '.py' would be weird + .replace('::', os.path.sep) # Turn classes/tests into paths + .replace('[', '-') # For parametrized names + .replace(']', '') # For parameterized names + ) + log_dir = Path( + integration_settings.LOCAL_LOG_PATH + ) / session_start_time / node_id_path + if not log_dir.exists(): + log_dir.mkdir(parents=True) + tarball_path = log_dir / 'cloud-init.tar.gz' + instance.pull_file('/var/tmp/cloud-init.tar.gz', tarball_path) + + tarball = TarFile.open(str(tarball_path)) + tarball.extractall(path=str(log_dir)) + tarball_path.unlink() + + @contextmanager def _client(request, fixture_utils, session_cloud): """Fixture implementation for the client fixtures. @@ -133,6 +161,8 @@ def _client(request, fixture_utils, session_cloud): user_data=user_data, launch_kwargs=launch_kwargs ) as instance: yield instance + if integration_settings.COLLECT_LOGS: + _collect_logs(instance, request.node.nodeid) @pytest.yield_fixture diff --git a/tests/integration_tests/instances.py b/tests/integration_tests/instances.py index 9b13288c696..a0a5fb6b118 100644 --- a/tests/integration_tests/instances.py +++ b/tests/integration_tests/instances.py @@ -47,14 +47,14 @@ def execute(self, command, *, use_sudo=None) -> Result: def pull_file(self, remote_path, local_path): # First copy to a temporary directory because of permissions issues tmp_path = _get_tmp_path() - self.instance.execute('cp {} {}'.format(remote_path, tmp_path)) - self.instance.pull_file(tmp_path, local_path) + self.instance.execute('cp {} {}'.format(str(remote_path), tmp_path)) + self.instance.pull_file(tmp_path, str(local_path)) def push_file(self, local_path, remote_path): # First push to a temporary directory because of permissions issues tmp_path = _get_tmp_path() - self.instance.push_file(local_path, tmp_path) - self.execute('mv {} {}'.format(tmp_path, remote_path)) + self.instance.push_file(str(local_path), tmp_path) + self.execute('mv {} {}'.format(tmp_path, str(remote_path))) def read_from_file(self, remote_path) -> str: result = self.execute('cat {}'.format(remote_path)) diff --git a/tests/integration_tests/integration_settings.py b/tests/integration_tests/integration_settings.py index a0609f7ec19..4ba7a648da9 100644 --- a/tests/integration_tests/integration_settings.py +++ b/tests/integration_tests/integration_settings.py @@ -55,6 +55,12 @@ # A path to a valid package to be uploaded and installed CLOUD_INIT_SOURCE = 'NONE' +# Before an instance is torn down, we run `cloud-init collect-logs` +# and transfer them locally. These settings specify whether to collect these +# logs at all, and if so, where do we put them on the local filesystem +COLLECT_LOGS = True +LOCAL_LOG_PATH = '/tmp/cloud_init_test_logs' + ################################################################## # GCE SPECIFIC SETTINGS ################################################################## From b577434d3cdd5d3d3ce22012407a8853c75b6123 Mon Sep 17 00:00:00 2001 From: James Falcon Date: Mon, 23 Nov 2020 13:22:45 -0600 Subject: [PATCH 2/2] [squash] Adding ALWAYS,ON_ERROR,NEVER options --- tests/integration_tests/conftest.py | 22 ++++++++++++++++--- .../integration_tests/integration_settings.py | 10 ++++++--- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/tests/integration_tests/conftest.py b/tests/integration_tests/conftest.py index 45d51c00b1b..6e1465beb36 100644 --- a/tests/integration_tests/conftest.py +++ b/tests/integration_tests/conftest.py @@ -17,6 +17,7 @@ LxdContainerCloud, LxdVmCloud, ) +from tests.integration_tests.instances import IntegrationInstance log = logging.getLogger('integration_testing') @@ -119,7 +120,21 @@ def setup_image(session_cloud): log.info('Done with environment setup') -def _collect_logs(instance, node_id): +def _collect_logs(instance: IntegrationInstance, node_id: str, + test_failed: bool): + """Collect logs from remote instance. + + Args: + instance: The current IntegrationInstance to collect logs from + node_id: The pytest representation of this test, E.g.: + tests/integration_tests/test_example.py::TestExample.test_example + test_failed: If test failed or not + """ + if any([ + integration_settings.COLLECT_LOGS == 'NEVER', + integration_settings.COLLECT_LOGS == 'ON_ERROR' and not test_failed + ]): + return instance.execute( 'cloud-init collect-logs -u -t /var/tmp/cloud-init.tar.gz') node_id_path = Path( @@ -160,9 +175,10 @@ def _client(request, fixture_utils, session_cloud): with session_cloud.launch( user_data=user_data, launch_kwargs=launch_kwargs ) as instance: + previous_failures = request.session.testsfailed yield instance - if integration_settings.COLLECT_LOGS: - _collect_logs(instance, request.node.nodeid) + test_failed = request.session.testsfailed - previous_failures > 0 + _collect_logs(instance, request.node.nodeid, test_failed) @pytest.yield_fixture diff --git a/tests/integration_tests/integration_settings.py b/tests/integration_tests/integration_settings.py index 4ba7a648da9..9be9a94f9e8 100644 --- a/tests/integration_tests/integration_settings.py +++ b/tests/integration_tests/integration_settings.py @@ -56,9 +56,13 @@ CLOUD_INIT_SOURCE = 'NONE' # Before an instance is torn down, we run `cloud-init collect-logs` -# and transfer them locally. These settings specify whether to collect these -# logs at all, and if so, where do we put them on the local filesystem -COLLECT_LOGS = True +# and transfer them locally. These settings specify when to collect these +# logs and where to put them on the local filesystem +# One of: +# 'ALWAYS' +# 'ON_ERROR' +# 'NEVER' +COLLECT_LOGS = 'ON_ERROR' LOCAL_LOG_PATH = '/tmp/cloud_init_test_logs' ##################################################################