From ec2fbba9bda9fb13e810b66ebd8c4b499173426e Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Fri, 20 Nov 2020 15:05:13 -0800 Subject: [PATCH 001/194] =?UTF-8?q?Move=20first=20prototype=20into=20?= =?UTF-8?q?=E2=80=98.archive=E2=80=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It’s hidden so that tools like `pytest` and `rg` ignore it by default. --- .flake8 => .archive/.flake8 | 0 .gitattributes => .archive/.gitattributes | 0 {.github => .archive/.github}/CODEOWNERS | 0 .../.github}/workflows/continuous-integration-workflow.yml | 0 .gitignore => .archive/.gitignore | 0 CODE_OF_CONDUCT.md => .archive/CODE_OF_CONDUCT.md | 0 CONTRIBUTING.md => .archive/CONTRIBUTING.md | 0 LICENSE-2.0.txt => .archive/LICENSE-2.0.txt | 0 Makefile => .archive/Makefile | 0 README.md => .archive/README.md | 0 {examples => .archive/examples}/runbook/hello_world.yml | 0 {examples => .archive/examples}/runbook/hello_world_azure.yml | 0 {examples => .archive/examples}/runbook/hello_world_remote.yml | 0 {examples => .archive/examples}/testsuites/__init__.py | 0 {examples => .archive/examples}/testsuites/helloworld.py | 0 {examples => .archive/examples}/testsuites/multinodes.py | 0 {examples => .archive/examples}/testsuites/scripts/echo.sh | 0 {examples => .archive/examples}/testsuites/withscript.py | 0 {lisa => .archive/lisa}/__init__.py | 0 {lisa => .archive/lisa}/action.py | 0 {lisa => .archive/lisa}/commands.py | 0 {lisa => .archive/lisa}/environment.py | 0 {lisa => .archive/lisa}/executable.py | 0 {lisa => .archive/lisa}/feature.py | 0 {lisa => .archive/lisa}/features/__init__.py | 0 {lisa => .archive/lisa}/features/serial_console.py | 0 {lisa => .archive/lisa}/features/startstop.py | 0 {lisa => .archive/lisa}/main.py | 0 {lisa => .archive/lisa}/node.py | 0 {lisa => .archive/lisa}/notifier.py | 0 {lisa => .archive/lisa}/notifiers/__init__.py | 0 {lisa => .archive/lisa}/notifiers/console.py | 0 {lisa => .archive/lisa}/operating_system.py | 0 {lisa => .archive/lisa}/parameter_parser/__init__.py | 0 {lisa => .archive/lisa}/parameter_parser/argparser.py | 0 {lisa => .archive/lisa}/parameter_parser/runbook.py | 0 {lisa => .archive/lisa}/platform_.py | 0 {lisa => .archive/lisa}/runner.py | 0 {lisa => .archive/lisa}/schema.py | 0 {lisa => .archive/lisa}/search_space.py | 0 {lisa => .archive/lisa}/secret.py | 0 {lisa => .archive/lisa}/sut_orchestrator/__init__.py | 0 {lisa => .archive/lisa}/sut_orchestrator/azure/__init__.py | 0 {lisa => .archive/lisa}/sut_orchestrator/azure/arm_template.json | 0 {lisa => .archive/lisa}/sut_orchestrator/azure/common.py | 0 {lisa => .archive/lisa}/sut_orchestrator/azure/cred_wrapper.py | 0 {lisa => .archive/lisa}/sut_orchestrator/azure/features.py | 0 {lisa => .archive/lisa}/sut_orchestrator/azure/platform_.py | 0 {lisa => .archive/lisa}/sut_orchestrator/azure/tests/__init__.py | 0 .../azure/tests/azure_locations_australiaeast.json | 0 .../sut_orchestrator/azure/tests/azure_locations_brazilsouth.json | 0 .../sut_orchestrator/azure/tests/azure_locations_eastus.json | 0 .../sut_orchestrator/azure/tests/azure_locations_eastus2.json | 0 .../sut_orchestrator/azure/tests/azure_locations_northeurope.json | 0 .../sut_orchestrator/azure/tests/azure_locations_notreal.json | 0 .../azure/tests/azure_locations_southcentralus.json | 0 .../azure/tests/azure_locations_southeastasia.json | 0 .../sut_orchestrator/azure/tests/azure_locations_uksouth.json | 0 .../sut_orchestrator/azure/tests/azure_locations_westeurope.json | 0 .../sut_orchestrator/azure/tests/azure_locations_westus2.json | 0 .../lisa}/sut_orchestrator/azure/tests/test_prepare.py | 0 {lisa => .archive/lisa}/sut_orchestrator/azure/tools.py | 0 {lisa => .archive/lisa}/sut_orchestrator/ready.py | 0 {lisa => .archive/lisa}/tests/__init__.py | 0 {lisa => .archive/lisa}/tests/test_env_requirement.py | 0 {lisa => .archive/lisa}/tests/test_environment.py | 0 {lisa => .archive/lisa}/tests/test_platform.py | 0 {lisa => .archive/lisa}/tests/test_runner.py | 0 {lisa => .archive/lisa}/tests/test_search_space.py | 0 {lisa => .archive/lisa}/tests/test_secret.py | 0 {lisa => .archive/lisa}/tests/test_testselector.py | 0 {lisa => .archive/lisa}/tests/test_testsuite.py | 0 {lisa => .archive/lisa}/tests/test_variable.py | 0 {lisa => .archive/lisa}/tests/variable_normal.yml | 0 {lisa => .archive/lisa}/tests/variable_secret.yml | 0 {lisa => .archive/lisa}/testselector.py | 0 {lisa => .archive/lisa}/testsuite.py | 0 {lisa => .archive/lisa}/tools/__init__.py | 0 {lisa => .archive/lisa}/tools/cat.py | 0 {lisa => .archive/lisa}/tools/date.py | 0 {lisa => .archive/lisa}/tools/dmesg.py | 0 {lisa => .archive/lisa}/tools/echo.py | 0 {lisa => .archive/lisa}/tools/gcc.py | 0 {lisa => .archive/lisa}/tools/git.py | 0 {lisa => .archive/lisa}/tools/lscpu.py | 0 {lisa => .archive/lisa}/tools/make.py | 0 {lisa => .archive/lisa}/tools/modinfo.py | 0 {lisa => .archive/lisa}/tools/ntttcp.py | 0 {lisa => .archive/lisa}/tools/reboot.py | 0 {lisa => .archive/lisa}/tools/uname.py | 0 {lisa => .archive/lisa}/tools/who.py | 0 {lisa => .archive/lisa}/util/__init__.py | 0 {lisa => .archive/lisa}/util/constants.py | 0 {lisa => .archive/lisa}/util/logger.py | 0 {lisa => .archive/lisa}/util/module.py | 0 {lisa => .archive/lisa}/util/perf_timer.py | 0 {lisa => .archive/lisa}/util/process.py | 0 {lisa => .archive/lisa}/util/shell.py | 0 {lisa => .archive/lisa}/util/subclasses.py | 0 {lisa => .archive/lisa}/variable.py | 0 poetry.lock => .archive/poetry.lock | 0 pyproject.toml => .archive/pyproject.toml | 0 {testsuites => .archive/testsuites}/basic/provisioning.py | 0 {testsuites => .archive/testsuites}/runbooks/azure/p0.yml | 0 {testsuites => .archive/testsuites}/runbooks/azure/secret.yml | 0 105 files changed, 0 insertions(+), 0 deletions(-) rename .flake8 => .archive/.flake8 (100%) rename .gitattributes => .archive/.gitattributes (100%) rename {.github => .archive/.github}/CODEOWNERS (100%) rename {.github => .archive/.github}/workflows/continuous-integration-workflow.yml (100%) rename .gitignore => .archive/.gitignore (100%) rename CODE_OF_CONDUCT.md => .archive/CODE_OF_CONDUCT.md (100%) rename CONTRIBUTING.md => .archive/CONTRIBUTING.md (100%) rename LICENSE-2.0.txt => .archive/LICENSE-2.0.txt (100%) rename Makefile => .archive/Makefile (100%) rename README.md => .archive/README.md (100%) rename {examples => .archive/examples}/runbook/hello_world.yml (100%) rename {examples => .archive/examples}/runbook/hello_world_azure.yml (100%) rename {examples => .archive/examples}/runbook/hello_world_remote.yml (100%) rename {examples => .archive/examples}/testsuites/__init__.py (100%) rename {examples => .archive/examples}/testsuites/helloworld.py (100%) rename {examples => .archive/examples}/testsuites/multinodes.py (100%) rename {examples => .archive/examples}/testsuites/scripts/echo.sh (100%) rename {examples => .archive/examples}/testsuites/withscript.py (100%) rename {lisa => .archive/lisa}/__init__.py (100%) rename {lisa => .archive/lisa}/action.py (100%) rename {lisa => .archive/lisa}/commands.py (100%) rename {lisa => .archive/lisa}/environment.py (100%) rename {lisa => .archive/lisa}/executable.py (100%) rename {lisa => .archive/lisa}/feature.py (100%) rename {lisa => .archive/lisa}/features/__init__.py (100%) rename {lisa => .archive/lisa}/features/serial_console.py (100%) rename {lisa => .archive/lisa}/features/startstop.py (100%) rename {lisa => .archive/lisa}/main.py (100%) rename {lisa => .archive/lisa}/node.py (100%) rename {lisa => .archive/lisa}/notifier.py (100%) rename {lisa => .archive/lisa}/notifiers/__init__.py (100%) rename {lisa => .archive/lisa}/notifiers/console.py (100%) rename {lisa => .archive/lisa}/operating_system.py (100%) rename {lisa => .archive/lisa}/parameter_parser/__init__.py (100%) rename {lisa => .archive/lisa}/parameter_parser/argparser.py (100%) rename {lisa => .archive/lisa}/parameter_parser/runbook.py (100%) rename {lisa => .archive/lisa}/platform_.py (100%) rename {lisa => .archive/lisa}/runner.py (100%) rename {lisa => .archive/lisa}/schema.py (100%) rename {lisa => .archive/lisa}/search_space.py (100%) rename {lisa => .archive/lisa}/secret.py (100%) rename {lisa => .archive/lisa}/sut_orchestrator/__init__.py (100%) rename {lisa => .archive/lisa}/sut_orchestrator/azure/__init__.py (100%) rename {lisa => .archive/lisa}/sut_orchestrator/azure/arm_template.json (100%) rename {lisa => .archive/lisa}/sut_orchestrator/azure/common.py (100%) rename {lisa => .archive/lisa}/sut_orchestrator/azure/cred_wrapper.py (100%) rename {lisa => .archive/lisa}/sut_orchestrator/azure/features.py (100%) rename {lisa => .archive/lisa}/sut_orchestrator/azure/platform_.py (100%) rename {lisa => .archive/lisa}/sut_orchestrator/azure/tests/__init__.py (100%) rename {lisa => .archive/lisa}/sut_orchestrator/azure/tests/azure_locations_australiaeast.json (100%) rename {lisa => .archive/lisa}/sut_orchestrator/azure/tests/azure_locations_brazilsouth.json (100%) rename {lisa => .archive/lisa}/sut_orchestrator/azure/tests/azure_locations_eastus.json (100%) rename {lisa => .archive/lisa}/sut_orchestrator/azure/tests/azure_locations_eastus2.json (100%) rename {lisa => .archive/lisa}/sut_orchestrator/azure/tests/azure_locations_northeurope.json (100%) rename {lisa => .archive/lisa}/sut_orchestrator/azure/tests/azure_locations_notreal.json (100%) rename {lisa => .archive/lisa}/sut_orchestrator/azure/tests/azure_locations_southcentralus.json (100%) rename {lisa => .archive/lisa}/sut_orchestrator/azure/tests/azure_locations_southeastasia.json (100%) rename {lisa => .archive/lisa}/sut_orchestrator/azure/tests/azure_locations_uksouth.json (100%) rename {lisa => .archive/lisa}/sut_orchestrator/azure/tests/azure_locations_westeurope.json (100%) rename {lisa => .archive/lisa}/sut_orchestrator/azure/tests/azure_locations_westus2.json (100%) rename {lisa => .archive/lisa}/sut_orchestrator/azure/tests/test_prepare.py (100%) rename {lisa => .archive/lisa}/sut_orchestrator/azure/tools.py (100%) rename {lisa => .archive/lisa}/sut_orchestrator/ready.py (100%) rename {lisa => .archive/lisa}/tests/__init__.py (100%) rename {lisa => .archive/lisa}/tests/test_env_requirement.py (100%) rename {lisa => .archive/lisa}/tests/test_environment.py (100%) rename {lisa => .archive/lisa}/tests/test_platform.py (100%) rename {lisa => .archive/lisa}/tests/test_runner.py (100%) rename {lisa => .archive/lisa}/tests/test_search_space.py (100%) rename {lisa => .archive/lisa}/tests/test_secret.py (100%) rename {lisa => .archive/lisa}/tests/test_testselector.py (100%) rename {lisa => .archive/lisa}/tests/test_testsuite.py (100%) rename {lisa => .archive/lisa}/tests/test_variable.py (100%) rename {lisa => .archive/lisa}/tests/variable_normal.yml (100%) rename {lisa => .archive/lisa}/tests/variable_secret.yml (100%) rename {lisa => .archive/lisa}/testselector.py (100%) rename {lisa => .archive/lisa}/testsuite.py (100%) rename {lisa => .archive/lisa}/tools/__init__.py (100%) rename {lisa => .archive/lisa}/tools/cat.py (100%) rename {lisa => .archive/lisa}/tools/date.py (100%) rename {lisa => .archive/lisa}/tools/dmesg.py (100%) rename {lisa => .archive/lisa}/tools/echo.py (100%) rename {lisa => .archive/lisa}/tools/gcc.py (100%) rename {lisa => .archive/lisa}/tools/git.py (100%) rename {lisa => .archive/lisa}/tools/lscpu.py (100%) rename {lisa => .archive/lisa}/tools/make.py (100%) rename {lisa => .archive/lisa}/tools/modinfo.py (100%) rename {lisa => .archive/lisa}/tools/ntttcp.py (100%) rename {lisa => .archive/lisa}/tools/reboot.py (100%) rename {lisa => .archive/lisa}/tools/uname.py (100%) rename {lisa => .archive/lisa}/tools/who.py (100%) rename {lisa => .archive/lisa}/util/__init__.py (100%) rename {lisa => .archive/lisa}/util/constants.py (100%) rename {lisa => .archive/lisa}/util/logger.py (100%) rename {lisa => .archive/lisa}/util/module.py (100%) rename {lisa => .archive/lisa}/util/perf_timer.py (100%) rename {lisa => .archive/lisa}/util/process.py (100%) rename {lisa => .archive/lisa}/util/shell.py (100%) rename {lisa => .archive/lisa}/util/subclasses.py (100%) rename {lisa => .archive/lisa}/variable.py (100%) rename poetry.lock => .archive/poetry.lock (100%) rename pyproject.toml => .archive/pyproject.toml (100%) rename {testsuites => .archive/testsuites}/basic/provisioning.py (100%) rename {testsuites => .archive/testsuites}/runbooks/azure/p0.yml (100%) rename {testsuites => .archive/testsuites}/runbooks/azure/secret.yml (100%) diff --git a/.flake8 b/.archive/.flake8 similarity index 100% rename from .flake8 rename to .archive/.flake8 diff --git a/.gitattributes b/.archive/.gitattributes similarity index 100% rename from .gitattributes rename to .archive/.gitattributes diff --git a/.github/CODEOWNERS b/.archive/.github/CODEOWNERS similarity index 100% rename from .github/CODEOWNERS rename to .archive/.github/CODEOWNERS diff --git a/.github/workflows/continuous-integration-workflow.yml b/.archive/.github/workflows/continuous-integration-workflow.yml similarity index 100% rename from .github/workflows/continuous-integration-workflow.yml rename to .archive/.github/workflows/continuous-integration-workflow.yml diff --git a/.gitignore b/.archive/.gitignore similarity index 100% rename from .gitignore rename to .archive/.gitignore diff --git a/CODE_OF_CONDUCT.md b/.archive/CODE_OF_CONDUCT.md similarity index 100% rename from CODE_OF_CONDUCT.md rename to .archive/CODE_OF_CONDUCT.md diff --git a/CONTRIBUTING.md b/.archive/CONTRIBUTING.md similarity index 100% rename from CONTRIBUTING.md rename to .archive/CONTRIBUTING.md diff --git a/LICENSE-2.0.txt b/.archive/LICENSE-2.0.txt similarity index 100% rename from LICENSE-2.0.txt rename to .archive/LICENSE-2.0.txt diff --git a/Makefile b/.archive/Makefile similarity index 100% rename from Makefile rename to .archive/Makefile diff --git a/README.md b/.archive/README.md similarity index 100% rename from README.md rename to .archive/README.md diff --git a/examples/runbook/hello_world.yml b/.archive/examples/runbook/hello_world.yml similarity index 100% rename from examples/runbook/hello_world.yml rename to .archive/examples/runbook/hello_world.yml diff --git a/examples/runbook/hello_world_azure.yml b/.archive/examples/runbook/hello_world_azure.yml similarity index 100% rename from examples/runbook/hello_world_azure.yml rename to .archive/examples/runbook/hello_world_azure.yml diff --git a/examples/runbook/hello_world_remote.yml b/.archive/examples/runbook/hello_world_remote.yml similarity index 100% rename from examples/runbook/hello_world_remote.yml rename to .archive/examples/runbook/hello_world_remote.yml diff --git a/examples/testsuites/__init__.py b/.archive/examples/testsuites/__init__.py similarity index 100% rename from examples/testsuites/__init__.py rename to .archive/examples/testsuites/__init__.py diff --git a/examples/testsuites/helloworld.py b/.archive/examples/testsuites/helloworld.py similarity index 100% rename from examples/testsuites/helloworld.py rename to .archive/examples/testsuites/helloworld.py diff --git a/examples/testsuites/multinodes.py b/.archive/examples/testsuites/multinodes.py similarity index 100% rename from examples/testsuites/multinodes.py rename to .archive/examples/testsuites/multinodes.py diff --git a/examples/testsuites/scripts/echo.sh b/.archive/examples/testsuites/scripts/echo.sh similarity index 100% rename from examples/testsuites/scripts/echo.sh rename to .archive/examples/testsuites/scripts/echo.sh diff --git a/examples/testsuites/withscript.py b/.archive/examples/testsuites/withscript.py similarity index 100% rename from examples/testsuites/withscript.py rename to .archive/examples/testsuites/withscript.py diff --git a/lisa/__init__.py b/.archive/lisa/__init__.py similarity index 100% rename from lisa/__init__.py rename to .archive/lisa/__init__.py diff --git a/lisa/action.py b/.archive/lisa/action.py similarity index 100% rename from lisa/action.py rename to .archive/lisa/action.py diff --git a/lisa/commands.py b/.archive/lisa/commands.py similarity index 100% rename from lisa/commands.py rename to .archive/lisa/commands.py diff --git a/lisa/environment.py b/.archive/lisa/environment.py similarity index 100% rename from lisa/environment.py rename to .archive/lisa/environment.py diff --git a/lisa/executable.py b/.archive/lisa/executable.py similarity index 100% rename from lisa/executable.py rename to .archive/lisa/executable.py diff --git a/lisa/feature.py b/.archive/lisa/feature.py similarity index 100% rename from lisa/feature.py rename to .archive/lisa/feature.py diff --git a/lisa/features/__init__.py b/.archive/lisa/features/__init__.py similarity index 100% rename from lisa/features/__init__.py rename to .archive/lisa/features/__init__.py diff --git a/lisa/features/serial_console.py b/.archive/lisa/features/serial_console.py similarity index 100% rename from lisa/features/serial_console.py rename to .archive/lisa/features/serial_console.py diff --git a/lisa/features/startstop.py b/.archive/lisa/features/startstop.py similarity index 100% rename from lisa/features/startstop.py rename to .archive/lisa/features/startstop.py diff --git a/lisa/main.py b/.archive/lisa/main.py similarity index 100% rename from lisa/main.py rename to .archive/lisa/main.py diff --git a/lisa/node.py b/.archive/lisa/node.py similarity index 100% rename from lisa/node.py rename to .archive/lisa/node.py diff --git a/lisa/notifier.py b/.archive/lisa/notifier.py similarity index 100% rename from lisa/notifier.py rename to .archive/lisa/notifier.py diff --git a/lisa/notifiers/__init__.py b/.archive/lisa/notifiers/__init__.py similarity index 100% rename from lisa/notifiers/__init__.py rename to .archive/lisa/notifiers/__init__.py diff --git a/lisa/notifiers/console.py b/.archive/lisa/notifiers/console.py similarity index 100% rename from lisa/notifiers/console.py rename to .archive/lisa/notifiers/console.py diff --git a/lisa/operating_system.py b/.archive/lisa/operating_system.py similarity index 100% rename from lisa/operating_system.py rename to .archive/lisa/operating_system.py diff --git a/lisa/parameter_parser/__init__.py b/.archive/lisa/parameter_parser/__init__.py similarity index 100% rename from lisa/parameter_parser/__init__.py rename to .archive/lisa/parameter_parser/__init__.py diff --git a/lisa/parameter_parser/argparser.py b/.archive/lisa/parameter_parser/argparser.py similarity index 100% rename from lisa/parameter_parser/argparser.py rename to .archive/lisa/parameter_parser/argparser.py diff --git a/lisa/parameter_parser/runbook.py b/.archive/lisa/parameter_parser/runbook.py similarity index 100% rename from lisa/parameter_parser/runbook.py rename to .archive/lisa/parameter_parser/runbook.py diff --git a/lisa/platform_.py b/.archive/lisa/platform_.py similarity index 100% rename from lisa/platform_.py rename to .archive/lisa/platform_.py diff --git a/lisa/runner.py b/.archive/lisa/runner.py similarity index 100% rename from lisa/runner.py rename to .archive/lisa/runner.py diff --git a/lisa/schema.py b/.archive/lisa/schema.py similarity index 100% rename from lisa/schema.py rename to .archive/lisa/schema.py diff --git a/lisa/search_space.py b/.archive/lisa/search_space.py similarity index 100% rename from lisa/search_space.py rename to .archive/lisa/search_space.py diff --git a/lisa/secret.py b/.archive/lisa/secret.py similarity index 100% rename from lisa/secret.py rename to .archive/lisa/secret.py diff --git a/lisa/sut_orchestrator/__init__.py b/.archive/lisa/sut_orchestrator/__init__.py similarity index 100% rename from lisa/sut_orchestrator/__init__.py rename to .archive/lisa/sut_orchestrator/__init__.py diff --git a/lisa/sut_orchestrator/azure/__init__.py b/.archive/lisa/sut_orchestrator/azure/__init__.py similarity index 100% rename from lisa/sut_orchestrator/azure/__init__.py rename to .archive/lisa/sut_orchestrator/azure/__init__.py diff --git a/lisa/sut_orchestrator/azure/arm_template.json b/.archive/lisa/sut_orchestrator/azure/arm_template.json similarity index 100% rename from lisa/sut_orchestrator/azure/arm_template.json rename to .archive/lisa/sut_orchestrator/azure/arm_template.json diff --git a/lisa/sut_orchestrator/azure/common.py b/.archive/lisa/sut_orchestrator/azure/common.py similarity index 100% rename from lisa/sut_orchestrator/azure/common.py rename to .archive/lisa/sut_orchestrator/azure/common.py diff --git a/lisa/sut_orchestrator/azure/cred_wrapper.py b/.archive/lisa/sut_orchestrator/azure/cred_wrapper.py similarity index 100% rename from lisa/sut_orchestrator/azure/cred_wrapper.py rename to .archive/lisa/sut_orchestrator/azure/cred_wrapper.py diff --git a/lisa/sut_orchestrator/azure/features.py b/.archive/lisa/sut_orchestrator/azure/features.py similarity index 100% rename from lisa/sut_orchestrator/azure/features.py rename to .archive/lisa/sut_orchestrator/azure/features.py diff --git a/lisa/sut_orchestrator/azure/platform_.py b/.archive/lisa/sut_orchestrator/azure/platform_.py similarity index 100% rename from lisa/sut_orchestrator/azure/platform_.py rename to .archive/lisa/sut_orchestrator/azure/platform_.py diff --git a/lisa/sut_orchestrator/azure/tests/__init__.py b/.archive/lisa/sut_orchestrator/azure/tests/__init__.py similarity index 100% rename from lisa/sut_orchestrator/azure/tests/__init__.py rename to .archive/lisa/sut_orchestrator/azure/tests/__init__.py diff --git a/lisa/sut_orchestrator/azure/tests/azure_locations_australiaeast.json b/.archive/lisa/sut_orchestrator/azure/tests/azure_locations_australiaeast.json similarity index 100% rename from lisa/sut_orchestrator/azure/tests/azure_locations_australiaeast.json rename to .archive/lisa/sut_orchestrator/azure/tests/azure_locations_australiaeast.json diff --git a/lisa/sut_orchestrator/azure/tests/azure_locations_brazilsouth.json b/.archive/lisa/sut_orchestrator/azure/tests/azure_locations_brazilsouth.json similarity index 100% rename from lisa/sut_orchestrator/azure/tests/azure_locations_brazilsouth.json rename to .archive/lisa/sut_orchestrator/azure/tests/azure_locations_brazilsouth.json diff --git a/lisa/sut_orchestrator/azure/tests/azure_locations_eastus.json b/.archive/lisa/sut_orchestrator/azure/tests/azure_locations_eastus.json similarity index 100% rename from lisa/sut_orchestrator/azure/tests/azure_locations_eastus.json rename to .archive/lisa/sut_orchestrator/azure/tests/azure_locations_eastus.json diff --git a/lisa/sut_orchestrator/azure/tests/azure_locations_eastus2.json b/.archive/lisa/sut_orchestrator/azure/tests/azure_locations_eastus2.json similarity index 100% rename from lisa/sut_orchestrator/azure/tests/azure_locations_eastus2.json rename to .archive/lisa/sut_orchestrator/azure/tests/azure_locations_eastus2.json diff --git a/lisa/sut_orchestrator/azure/tests/azure_locations_northeurope.json b/.archive/lisa/sut_orchestrator/azure/tests/azure_locations_northeurope.json similarity index 100% rename from lisa/sut_orchestrator/azure/tests/azure_locations_northeurope.json rename to .archive/lisa/sut_orchestrator/azure/tests/azure_locations_northeurope.json diff --git a/lisa/sut_orchestrator/azure/tests/azure_locations_notreal.json b/.archive/lisa/sut_orchestrator/azure/tests/azure_locations_notreal.json similarity index 100% rename from lisa/sut_orchestrator/azure/tests/azure_locations_notreal.json rename to .archive/lisa/sut_orchestrator/azure/tests/azure_locations_notreal.json diff --git a/lisa/sut_orchestrator/azure/tests/azure_locations_southcentralus.json b/.archive/lisa/sut_orchestrator/azure/tests/azure_locations_southcentralus.json similarity index 100% rename from lisa/sut_orchestrator/azure/tests/azure_locations_southcentralus.json rename to .archive/lisa/sut_orchestrator/azure/tests/azure_locations_southcentralus.json diff --git a/lisa/sut_orchestrator/azure/tests/azure_locations_southeastasia.json b/.archive/lisa/sut_orchestrator/azure/tests/azure_locations_southeastasia.json similarity index 100% rename from lisa/sut_orchestrator/azure/tests/azure_locations_southeastasia.json rename to .archive/lisa/sut_orchestrator/azure/tests/azure_locations_southeastasia.json diff --git a/lisa/sut_orchestrator/azure/tests/azure_locations_uksouth.json b/.archive/lisa/sut_orchestrator/azure/tests/azure_locations_uksouth.json similarity index 100% rename from lisa/sut_orchestrator/azure/tests/azure_locations_uksouth.json rename to .archive/lisa/sut_orchestrator/azure/tests/azure_locations_uksouth.json diff --git a/lisa/sut_orchestrator/azure/tests/azure_locations_westeurope.json b/.archive/lisa/sut_orchestrator/azure/tests/azure_locations_westeurope.json similarity index 100% rename from lisa/sut_orchestrator/azure/tests/azure_locations_westeurope.json rename to .archive/lisa/sut_orchestrator/azure/tests/azure_locations_westeurope.json diff --git a/lisa/sut_orchestrator/azure/tests/azure_locations_westus2.json b/.archive/lisa/sut_orchestrator/azure/tests/azure_locations_westus2.json similarity index 100% rename from lisa/sut_orchestrator/azure/tests/azure_locations_westus2.json rename to .archive/lisa/sut_orchestrator/azure/tests/azure_locations_westus2.json diff --git a/lisa/sut_orchestrator/azure/tests/test_prepare.py b/.archive/lisa/sut_orchestrator/azure/tests/test_prepare.py similarity index 100% rename from lisa/sut_orchestrator/azure/tests/test_prepare.py rename to .archive/lisa/sut_orchestrator/azure/tests/test_prepare.py diff --git a/lisa/sut_orchestrator/azure/tools.py b/.archive/lisa/sut_orchestrator/azure/tools.py similarity index 100% rename from lisa/sut_orchestrator/azure/tools.py rename to .archive/lisa/sut_orchestrator/azure/tools.py diff --git a/lisa/sut_orchestrator/ready.py b/.archive/lisa/sut_orchestrator/ready.py similarity index 100% rename from lisa/sut_orchestrator/ready.py rename to .archive/lisa/sut_orchestrator/ready.py diff --git a/lisa/tests/__init__.py b/.archive/lisa/tests/__init__.py similarity index 100% rename from lisa/tests/__init__.py rename to .archive/lisa/tests/__init__.py diff --git a/lisa/tests/test_env_requirement.py b/.archive/lisa/tests/test_env_requirement.py similarity index 100% rename from lisa/tests/test_env_requirement.py rename to .archive/lisa/tests/test_env_requirement.py diff --git a/lisa/tests/test_environment.py b/.archive/lisa/tests/test_environment.py similarity index 100% rename from lisa/tests/test_environment.py rename to .archive/lisa/tests/test_environment.py diff --git a/lisa/tests/test_platform.py b/.archive/lisa/tests/test_platform.py similarity index 100% rename from lisa/tests/test_platform.py rename to .archive/lisa/tests/test_platform.py diff --git a/lisa/tests/test_runner.py b/.archive/lisa/tests/test_runner.py similarity index 100% rename from lisa/tests/test_runner.py rename to .archive/lisa/tests/test_runner.py diff --git a/lisa/tests/test_search_space.py b/.archive/lisa/tests/test_search_space.py similarity index 100% rename from lisa/tests/test_search_space.py rename to .archive/lisa/tests/test_search_space.py diff --git a/lisa/tests/test_secret.py b/.archive/lisa/tests/test_secret.py similarity index 100% rename from lisa/tests/test_secret.py rename to .archive/lisa/tests/test_secret.py diff --git a/lisa/tests/test_testselector.py b/.archive/lisa/tests/test_testselector.py similarity index 100% rename from lisa/tests/test_testselector.py rename to .archive/lisa/tests/test_testselector.py diff --git a/lisa/tests/test_testsuite.py b/.archive/lisa/tests/test_testsuite.py similarity index 100% rename from lisa/tests/test_testsuite.py rename to .archive/lisa/tests/test_testsuite.py diff --git a/lisa/tests/test_variable.py b/.archive/lisa/tests/test_variable.py similarity index 100% rename from lisa/tests/test_variable.py rename to .archive/lisa/tests/test_variable.py diff --git a/lisa/tests/variable_normal.yml b/.archive/lisa/tests/variable_normal.yml similarity index 100% rename from lisa/tests/variable_normal.yml rename to .archive/lisa/tests/variable_normal.yml diff --git a/lisa/tests/variable_secret.yml b/.archive/lisa/tests/variable_secret.yml similarity index 100% rename from lisa/tests/variable_secret.yml rename to .archive/lisa/tests/variable_secret.yml diff --git a/lisa/testselector.py b/.archive/lisa/testselector.py similarity index 100% rename from lisa/testselector.py rename to .archive/lisa/testselector.py diff --git a/lisa/testsuite.py b/.archive/lisa/testsuite.py similarity index 100% rename from lisa/testsuite.py rename to .archive/lisa/testsuite.py diff --git a/lisa/tools/__init__.py b/.archive/lisa/tools/__init__.py similarity index 100% rename from lisa/tools/__init__.py rename to .archive/lisa/tools/__init__.py diff --git a/lisa/tools/cat.py b/.archive/lisa/tools/cat.py similarity index 100% rename from lisa/tools/cat.py rename to .archive/lisa/tools/cat.py diff --git a/lisa/tools/date.py b/.archive/lisa/tools/date.py similarity index 100% rename from lisa/tools/date.py rename to .archive/lisa/tools/date.py diff --git a/lisa/tools/dmesg.py b/.archive/lisa/tools/dmesg.py similarity index 100% rename from lisa/tools/dmesg.py rename to .archive/lisa/tools/dmesg.py diff --git a/lisa/tools/echo.py b/.archive/lisa/tools/echo.py similarity index 100% rename from lisa/tools/echo.py rename to .archive/lisa/tools/echo.py diff --git a/lisa/tools/gcc.py b/.archive/lisa/tools/gcc.py similarity index 100% rename from lisa/tools/gcc.py rename to .archive/lisa/tools/gcc.py diff --git a/lisa/tools/git.py b/.archive/lisa/tools/git.py similarity index 100% rename from lisa/tools/git.py rename to .archive/lisa/tools/git.py diff --git a/lisa/tools/lscpu.py b/.archive/lisa/tools/lscpu.py similarity index 100% rename from lisa/tools/lscpu.py rename to .archive/lisa/tools/lscpu.py diff --git a/lisa/tools/make.py b/.archive/lisa/tools/make.py similarity index 100% rename from lisa/tools/make.py rename to .archive/lisa/tools/make.py diff --git a/lisa/tools/modinfo.py b/.archive/lisa/tools/modinfo.py similarity index 100% rename from lisa/tools/modinfo.py rename to .archive/lisa/tools/modinfo.py diff --git a/lisa/tools/ntttcp.py b/.archive/lisa/tools/ntttcp.py similarity index 100% rename from lisa/tools/ntttcp.py rename to .archive/lisa/tools/ntttcp.py diff --git a/lisa/tools/reboot.py b/.archive/lisa/tools/reboot.py similarity index 100% rename from lisa/tools/reboot.py rename to .archive/lisa/tools/reboot.py diff --git a/lisa/tools/uname.py b/.archive/lisa/tools/uname.py similarity index 100% rename from lisa/tools/uname.py rename to .archive/lisa/tools/uname.py diff --git a/lisa/tools/who.py b/.archive/lisa/tools/who.py similarity index 100% rename from lisa/tools/who.py rename to .archive/lisa/tools/who.py diff --git a/lisa/util/__init__.py b/.archive/lisa/util/__init__.py similarity index 100% rename from lisa/util/__init__.py rename to .archive/lisa/util/__init__.py diff --git a/lisa/util/constants.py b/.archive/lisa/util/constants.py similarity index 100% rename from lisa/util/constants.py rename to .archive/lisa/util/constants.py diff --git a/lisa/util/logger.py b/.archive/lisa/util/logger.py similarity index 100% rename from lisa/util/logger.py rename to .archive/lisa/util/logger.py diff --git a/lisa/util/module.py b/.archive/lisa/util/module.py similarity index 100% rename from lisa/util/module.py rename to .archive/lisa/util/module.py diff --git a/lisa/util/perf_timer.py b/.archive/lisa/util/perf_timer.py similarity index 100% rename from lisa/util/perf_timer.py rename to .archive/lisa/util/perf_timer.py diff --git a/lisa/util/process.py b/.archive/lisa/util/process.py similarity index 100% rename from lisa/util/process.py rename to .archive/lisa/util/process.py diff --git a/lisa/util/shell.py b/.archive/lisa/util/shell.py similarity index 100% rename from lisa/util/shell.py rename to .archive/lisa/util/shell.py diff --git a/lisa/util/subclasses.py b/.archive/lisa/util/subclasses.py similarity index 100% rename from lisa/util/subclasses.py rename to .archive/lisa/util/subclasses.py diff --git a/lisa/variable.py b/.archive/lisa/variable.py similarity index 100% rename from lisa/variable.py rename to .archive/lisa/variable.py diff --git a/poetry.lock b/.archive/poetry.lock similarity index 100% rename from poetry.lock rename to .archive/poetry.lock diff --git a/pyproject.toml b/.archive/pyproject.toml similarity index 100% rename from pyproject.toml rename to .archive/pyproject.toml diff --git a/testsuites/basic/provisioning.py b/.archive/testsuites/basic/provisioning.py similarity index 100% rename from testsuites/basic/provisioning.py rename to .archive/testsuites/basic/provisioning.py diff --git a/testsuites/runbooks/azure/p0.yml b/.archive/testsuites/runbooks/azure/p0.yml similarity index 100% rename from testsuites/runbooks/azure/p0.yml rename to .archive/testsuites/runbooks/azure/p0.yml diff --git a/testsuites/runbooks/azure/secret.yml b/.archive/testsuites/runbooks/azure/secret.yml similarity index 100% rename from testsuites/runbooks/azure/secret.yml rename to .archive/testsuites/runbooks/azure/secret.yml From 37756a1401606b6539122846f28ee2653bf633b9 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Fri, 9 Oct 2020 17:49:45 -0700 Subject: [PATCH 002/194] Demo LISAv3 as simply pytest This is a _working_ test. --- .editorconfig | 2 + .flake8 | 5 + CODE_OF_CONDUCT.md | 9 + CONTRIBUTING.md | 269 ++++++++++++ Makefile | 13 + README.md | 105 +++++ mypy.ini | 20 + poetry.lock | 935 +++++++++++++++++++++++++++++++++++++++++ pyproject.toml | 43 ++ pytest.ini | 4 + testsuites/test_lis.py | 30 ++ 11 files changed, 1435 insertions(+) create mode 100644 .editorconfig create mode 100644 .flake8 create mode 100644 CODE_OF_CONDUCT.md create mode 100644 CONTRIBUTING.md create mode 100644 Makefile create mode 100644 README.md create mode 100644 mypy.ini create mode 100644 poetry.lock create mode 100644 pyproject.toml create mode 100644 pytest.ini create mode 100644 testsuites/test_lis.py diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000000..15e6a1f149 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,2 @@ +# Ignore parent project’s config +root = true diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000000..f855799a35 --- /dev/null +++ b/.flake8 @@ -0,0 +1,5 @@ +[flake8] +max-line-length = 88 +select = B,BLK,C90,E,F,I,W +max-complexity = 15 +extend-ignore = E203 diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000000..f9ba8cf65f --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,9 @@ +# Microsoft Open Source Code of Conduct + +This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). + +Resources: + +- [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/) +- [Microsoft Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) +- Contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with questions or concerns diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000000..f74461187d --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,269 @@ +# Contributing Guidelines + +This document describes the existing developer tooling we have in place (and what to +expect of it), as well as our design and development philosophy. + +## Naming Conventions + +Naming conventions are not automatically enforced, so please read the [naming +conventions](https://www.python.org/dev/peps/pep-0008/#naming-conventions) +section of PEP 8, which describes what each of the different styles means. A +short summary of the most important parts: + +* Modules (and hence files) should have short, all-lowercase names. +* Class (and exception) names should normally use the `CapWords` convention + (also known as `CamelCase`). +* Function and variable names should be lowercase, with words separated by + underscores as necessary to improve readability (also known as `snake_case`). +* To avoid collisions with the standard library, an underscore can be appended, + such as `id_`. +* Always use `self` for the first argument to instance methods. +* Always use `cls` for the first argument to class methods. +* Use one leading underscore only for non-public methods and instance variables, + such as `_data`. Do not activate name mangling with `__` unless necessary. +* If there is a pair of `get_x` and `set_x` methods, they should instead be a + proper property, which is easy to do with the built-in `@property` decorator. +* Constants should be `CAPITALIZED_SNAKE_CASE`. +* When importing a function, try to avoid renaming it with `import as` because + it introduces cognitive overhead to track yet another name. +* When deriving another module’s class (such as `unittest.TestCase`), reuse the + class name to avoid confusion, such as `LisaTestCase`, instead of introducing + a different connotation like `TestSuite`. + +When in doubt, adhere to existing conventions, or check the style guide. + +## Automated Tooling + +If you have ran pytest-lisa already, then you have installed and used the `poetry` +tool. [Poetry][] is a [PEP 518][] compliant and cross-platform build system +which handles our Python dependencies and environment. + +This project’s dependencies are found in the [`pyproject.toml`](pyproject.toml) +file. This is similar to but more powerful than the familiar `requirements.txt`. +With [PEP 518][] and [PEP 621][]. + +[Poetry]: https://python-poetry.org/docs/ +[PEP 518]: https://www.python.org/dev/peps/pep-0518/ +[PEP 621]: https://www.python.org/dev/peps/pep-0621/ + +### Metadata + +The first section, `tool.poetry`, defines the project’s metadata (name, version, +description, authors, and license) which will be embedded in the final built +package. + +The chosen version follows [Semantic Versioning][], with the [Python specific +pre-release versioning suffix][pre-release] ‘.dev1’. Since this is “pytest-lisa” it +seemed appropriate to set our version to ‘3.0.0.dev1’, that is, “the first +development release of pytest-lisa.” + +[Semantic Versioning]: https://semver.org/ +[pre-release]: https://packaging.python.org/guides/distributing-packages-using-setuptools/#choosing-a-versioning-scheme + +### Package Dependencies + +The next section, `tool.poetry.dependencies`, is where `poetry add +` records our required packages. + +Poetry automatically creates and manages [isolated +environments](https://python-poetry.org/docs/managing-environments/). + +From the documentation: + +> Poetry will first check if it’s currently running inside a virtual +> environment. If it is, it will use it directly without creating a new one. But +> if it’s not, it will use one that it has already created or create a brand new +> one for you. + +On Linux, your initial run of `poetry install` will cause Poetry to +automatically setup a new [virtualenv][] using [pyenv][]. If you are developing +on Windows, you will want to setup your own, perhaps using [Conda][]. + +[virtualenv]: https://docs.python-guide.org/dev/virtualenvs/ +[pyenv]: https://github.com/pyenv/pyenv +[Conda]: https://docs.conda.io/en/latest/ + +* python: We pinned Python to version 3.8 so everyone uses the same version. + +### Developer Dependencies + +Similar to the previous section, `tool.poetry.dev-dependencies` is where `poetry +add --dev ` records our _developer_ packages. These are not +necessary for LISAv3 to execute, but are used by developers to automatically +adhere to our coding standards. + +* [Black](https://github.com/psf/black), the opinionated code formatter which + settles all debates as to how our Python files should be formatted. It follows + [PEP 8](https://www.python.org/dev/peps/pep-0008/), the official Python style + guide, and where ambiguous makes the decision for us. + +* [Flake8](https://flake8.pycqa.org/en/latest/) (and integrations), the semantic + analyzer, used to coordinate most of the other tools. + +* [isort](https://timothycrosley.github.io/isort/), the `import` sorter, which + automatically splits imports into the expected, alphabetized sections. + +* [mypy](http://mypy-lang.org/), the static type checker, which coupled with + type annotations allows us to avoid the pitfalls of Python being a dynamically + typed language. + +* [python-language-server](https://github.com/palantir/python-language-server) + (and integrations), the de facto LSP server. While Microsoft is developing + their own LSP servers, they do not integrate with the existing ecosystem of + tools, and their latest tool, Pyright, simply does not support + `pyproject.toml`. Since pyls is used far more widely, and supports every + editor, we use it. + +* [rope](https://github.com/python-rope/rope), to provide completions and + renaming support to pyls. + +With these packages installed and a correctly setup editor (see the readme and +feel free to reach out to us), your code should automatically follow all the +standards which we could automate. + +The final sections, `tool.black`, `tool.isort`, `build-system`, and the +`.flake8` file (Flake8 does not yet support `pyproject.toml`) configure the +tools per their recommendations. + +## Type Annotations + +We are using [mypy][] to enforce static type checking of our Python code. This +may surprise you as Python is not a statically typed language. While dynamic +typing can be useful, for a complex tool such as LISA it is more likely to +introduce bugs that are found only at runtime (which the user experiences as a +crash). For more information on why we (and others) do this, see [Dropbox’s +journey to type checking 4 million lines of Python][dropbox]. [PEP 484][] and +[PEP 526][] (among others) introduced and defined [type hints][] for the Python +language. You can probably figuring out the syntax based on the surrounding +code, but you can also see this [Intro to Using Python Type Hints][intro] and +mypy’s [cheat sheet][]. + +[mypy]: http://mypy-lang.org/ +[dropbox]: https://dropbox.tech/application/our-journey-to-type-checking-4-million-lines-of-python +[PEP 484]: https://www.python.org/dev/peps/pep-0484/ +[PEP 526]: https://www.python.org/dev/peps/pep-0526/ +[type hints]: https://docs.python.org/3/library/typing.html +[intro]: https://kishstats.com/python/2019/01/07/python-type-hinting.html +[cheat sheet]: https://mypy.readthedocs.io/en/latest/cheat_sheet_py3.html + +## Runbook schema + +Some plugins like Platform need follow this section to extend runbook schema. Runbook is the configurations of LISA runs. Every LISA run need a runbook. + +The runbook uses [dataclass](https://docs.python.org/3/library/dataclasses.html) to define, [dataclass-json](https://github.com/lidatong/dataclasses-json/) to deserialize, and [marshmallow](https://marshmallow.readthedocs.io/en/3.0/api_reference.html) to validate the schema. + +See more examples in [schema.py](lisa/schema.py), if you need to extend runbook schema. + +## Committing Guidelines + +A best practice when using [Git](https://git-scm.com/book/en/v2) is to create a +series of independent and well-documented commits. Each commit should “do one +thing” and do it correctly. If a mistake is made (you need to fix a bug or +adjust formatting), you should amend it (or use an [interactive +rebase](https://thoughtbot.com/blog/git-interactive-rebase-squash-amend-rewriting-history) +to edit it). If you’re using Emacs, the [Magit](https://magit.vc/) package makes +all of this easy. Some of the reasons for making each commit polished is that it +aids immensely in future debugging. It lets us use tools like [`git +bisect`](https://git-scm.com/docs/git-bisect) to automatically find bugs, and +understand why prior code was written. Although some of it has gone out of date, +see this otherwise great essay on [Git best +practices](http://sethrobertson.github.io/GitBestPractices/). For how Git works, +read [Git from the Bottom +Up](https://jwiegley.github.io/git-from-the-bottom-up/). + +For writing your commit messages, see this modification of [Tim Pope’s +example](https://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html): + +> Capitalized, short (72 chars or less) summary +> +> More detailed explanatory text, if necessary. Wrap it to about 72 +> characters or so. In some contexts, the first line is treated as the +> subject of an email and the rest of the text as the body. The blank line +> separating the summary from the body is critical (unless you omit the +> body entirely); tools like rebase can get confused if you run the two +> together. +> +> Write your commit message in the imperative: “Fix bug” and not “Fixed +> bug” or “Fixes bug.” This convention matches up with commit messages +> generated by commands like git merge and git revert. +> +> Further paragraphs come after blank lines. +> +> * Bullet points are okay, too +> +> * Typically a hyphen or asterisk is used for the bullet, followed by a +> single space, with blank lines in between, but conventions vary here +> +> * Use a hanging indent + +You should also feel free to use Markdown in the commit messages, as our project +is hosted on GitHub which renders it (and Markdown is human readable). + +## Design Patterns + +The most important goal we are attempting to accomplish with LISAv3 is for it to +be “simple, clean, and with a low maintenance cost.” + +We should use caution when using Object Oriented Design, because when it is used +without critical analysis, it creates unmaintainable code. A great talk on this +subject is [Stop Writing Classes](https://www.youtube.com/watch?v=o9pEzgHorH0), +by Jack Diederich. As he says, “classes are great but they are also overused.” + +This [Python Design Patterns](https://python-patterns.guide/) is a fantastic +collection of material for writing maintainable Python code. It specifically +details many of the common “Object Oriented” patterns from the Gang of Four book +(which, in fact, were patterns geared toward languages like C++, and no longer +apply to modern languages like Python), what lessons can be learned from them, +and how to apply them (or their modern alternatives) today. It also serves as an +easy-to-read guide to the Gang of Four book itself, as its principles still +serve us well today. + +Every time a developer chooses to use a design pattern, that person needs to +reason through and document why it was chosen, and what alternatives were +considered. We will recreate the problems with LISAv2 unless we take our time to +carefully create a well-designed and maintainable framework. + +Several popular patterns that actually _do not_ work well in Python are: + +* [The Abstract Factory Pattern](https://python-patterns.guide/gang-of-four/abstract-factory/) +* [The Factory Method Pattern](https://python-patterns.guide/gang-of-four/factory-method/) +* [The Prototype Pattern](https://python-patterns.guide/gang-of-four/prototype/) +* [The Singleton Pattern](https://python-patterns.guide/gang-of-four/singleton/) + +Conversely, patterns that are a natural fit to Python include: + +* [The Composite Pattern](https://python-patterns.guide/gang-of-four/composite/) +* [The Iterator Pattern](https://python-patterns.guide/gang-of-four/iterator/) + (caution: it is actually better to implement these with `yield`!) + +Finally, a high-level guide to all things Python is [The Hitchhiker’s Guide to +Python](https://docs.python-guide.org/). It covers just about everything in the +Python world. If you make it through even some of these guides, you will be well +on your way to being a “Pythonista” (a Python developer) writing “Pythonic” +(canonically correct Python) code left and right. + +### Async IO + +With Python 3.4, the Async IO pattern found in languages such as C# and Go is +available through the keywords `async` and `await`, along with the Python module +`asyncio`. Please read [Async IO in Python: A Complete +Walkthrough](https://realpython.com/async-io-python/) to understand at a high +level how asynchronous programming works. As of Python 3.7, One major “gotcha” +is that `asyncio.run(...)` should be used [exactly once in +`main`](https://docs.python.org/3/library/asyncio-task.html), it starts the +event loop. Everything else should be a coroutine or task which the event loop +schedules. + +## Future Sections + +Just a collection of reminders for the author to expand on later. + +* [unittest](https://docs.python.org/3/library/unittest.html) +* [doctest](https://docs.python.org/3/library/doctest.html) +* [subprocess](https://pymotw.com/3/subprocess/index.html) +* [GitHub Actions](https://github.com/LIS/LISAv2/actions) +* [ShellCheck](https://www.shellcheck.net/) +* [Governance](https://opensource.guide/leadership-and-governance/) +* [Maintenance Cost](https://web.archive.org/web/20120313070806/http://users.jyu.fi/~koskinen/smcosts.htm) +* Parallelism and multi-plexing +* Versioned inputs and outputs diff --git a/Makefile b/Makefile new file mode 100644 index 0000000000..5c052c4707 --- /dev/null +++ b/Makefile @@ -0,0 +1,13 @@ +all: setup run + +# Install Python packages +setup: + @poetry install --no-ansi --remove-untracked + +# Run Pytest +run: + @poetry run python -X dev -m pytest --flake8 --mypy -rA + +# Print current Python virtualenv +venv: + @poetry env list --no-ansi --full-path diff --git a/README.md b/README.md new file mode 100644 index 0000000000..38c1ae798b --- /dev/null +++ b/README.md @@ -0,0 +1,105 @@ +# LISAv3 via pytest-lisa + +[Pytest](https://docs.pytest.org/en/stable/) is an [incredibly +popular](https://docs.pytest.org/en/stable/talks.html) MIT licensed open source +Python testing framework. It has a thriving community and plugin framework, with +[over 750 plugins](https://plugincompat.herokuapp.com/). There is even a YAML +example of writing a Domain Specific Language +[DSL](https://docs.pytest.org/en/stable/example/nonpython.html#yaml-plugin) for +specifying tests. Instead of writing yet another test framework, LISAv3 could be +written as pytest-lisa, a [plugin for +Pytest](https://docs.pytest.org/en/stable/writing_plugins.html) which implements +our requirements. In fact, most of Pytest itself is implemented via [built-in +plugins](https://docs.pytest.org/en/stable/plugins.html), providing us with a +lot to leverage. + +The [fundamental features](https://www.youtube.com/watch?v=CMuSn9cofbI) of +Pytest match our needs very well: + +* Automatic test discovery, no boiler-plate test code +* Useful information when a test fails (assertions are introspected) +* Test parameterization +* Modular setup/teardown via fixtures +* Customizable (as detailed above) + +So all the logic for discovering, running, skipping based on requirements, and +reporting the tests is already written and maintained by the greater open source +community, leaving us to focus on the hard and unique problem: creating an API +to launch the necessary nodes. It would also allow us the space to abstract the +installation of tools required by tests. In this way, LISAv3 could solve the +difficulties we have at hand without creating yet another unit test framework. + +## Design + +### pytest-mark + +The [pytest-mark](https://docs.pytest.org/en/stable/mark.html) already provides +functionality for adding metadata to tests, where we specifically want: + +* Owner +* Category +* Area +* Tags +* Priority + +We could simply reuse this built-in plugin with minimal logic to enforce our +required metadata, with sane defaults (such as setting the area to the name of +the module), and to list statistics about our test coverage. + +It also through pytest-mark that [skipping +functionality](https://docs.pytest.org/en/stable/skipping.html) exists, which we +would leverage for ensuring our environmental requirements are met. + +Note that Pytest leverages Python’s docstrings for built-in documentation (and +can even run tests discovered in such strings, like doctest). + +### Fixtures + +Pytest supports [fixtures](https://docs.pytest.org/en/stable/fixture.html), +which are the primary way of setting up test requirements. They replace less +flexible alternatives like setup/teardown functions. It is through fixtures that +pytest-lisa would implement remote node setup/teardown. Our node fixture would +implement (with more as found to be required): + +* Provision a node based on parameterized requirements +* Reboot the node if requested +* Run a command (perhaps asynchronously) on the node using SSH +* Download and upload files to the node (with retries and timeouts) + +Our abstraction would leverage +[Fabric](https://docs.fabfile.org/en/stable/index.html), which uses +[paramiko](https://docs.paramiko.org/en/stable/) underneath, directly to +implement the SSH commands, and it would use existing modules to deploy +[Azure](https://aka.ms/azsdk/python/all) and AWS nodes. We would need implement +specific logic for Hyper-V and similar platforms where APIs do not currently +exist, and this would be the bulk of our work instead of rewriting a unit test +framework. + +Other test specific requirements, such as installing software and daemons or +downloading files from remote storage, would similarly be implemented via +fixtures and shared among tests. + +Note that Paramiko is less complex (smaller library footprint) than Fabric, but +is a bit more difficult to use, and doesn’t support reading existing SSH config +files, nor does it support “ProxyJump” which we use heavily. + +## pytest-xdist + +With the [pytest-xdist plugin](https://github.com/pytest-dev/pytest-xdist) there +already exists support for running a folder of tests on an arbitrary remote host +via SSH. + +The LISA tests could be written as Python code suitable for running on the +target test system, which means direct access to the system in the test code +itself (subprocesses are still available, without having to use SSH within the +test, but would become far less necessary), something that is not possible with +the current prototype. Where the pytest-xdist plugin copies the package of code +to the target node and runs it, the pytest-lisa plugin could instantiate that +node (boot the necessary image on a remote machine or launch a new Hyper-V or +Azure VM, etc.) for the tests. YAML playbooks (AKA “runbooks” in the current +prototype) could be interpreted by the pytest-lisa plugin to determine how to +create those nodes. + +However, this is only one approach, and we may prefer to run the Python code on +the user’s machine, with pytest-lisa instead providing the previously mentioned +node fixtures, default marks, and requirements logic. diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 0000000000..b5a41dd5b5 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,20 @@ +[mypy] +namespace_packages = True +pretty = True + +warn_unused_configs = True +disallow_any_generics = True +disallow_subclassing_any = True +disallow_untyped_calls = True +disallow_untyped_defs = True +disallow_incomplete_defs = True +check_untyped_defs = True +disallow_untyped_decorators = True +no_implicit_optional = True +warn_redundant_casts = True +warn_unused_ignores = True +warn_return_any = True +no_implicit_reexport = True +strict_equality = True + +warn_unreachable = True diff --git a/poetry.lock b/poetry.lock new file mode 100644 index 0000000000..a0a87e8d83 --- /dev/null +++ b/poetry.lock @@ -0,0 +1,935 @@ +[[package]] +name = "appdirs" +version = "1.4.4" +description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "atomicwrites" +version = "1.4.0" +description = "Atomic file writes." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[[package]] +name = "attrs" +version = "20.2.0" +description = "Classes Without Boilerplate" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[package.extras] +dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "sphinx", "sphinx-rtd-theme", "pre-commit"] +docs = ["sphinx", "sphinx-rtd-theme", "zope.interface"] +tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface"] +tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six"] + +[[package]] +name = "bcrypt" +version = "3.2.0" +description = "Modern password hashing for your software and your servers" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +cffi = ">=1.1" +six = ">=1.4.1" + +[package.extras] +tests = ["pytest (>=3.2.1,<3.3.0 || >3.3.0)"] +typecheck = ["mypy"] + +[[package]] +name = "black" +version = "20.8b1" +description = "The uncompromising code formatter." +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +appdirs = "*" +click = ">=7.1.2" +mypy-extensions = ">=0.4.3" +pathspec = ">=0.6,<1" +regex = ">=2020.1.8" +toml = ">=0.10.1" +typed-ast = ">=1.4.0" +typing-extensions = ">=3.7.4" + +[package.extras] +colorama = ["colorama (>=0.4.3)"] +d = ["aiohttp (>=3.3.2)", "aiohttp-cors"] + +[[package]] +name = "cffi" +version = "1.14.3" +description = "Foreign Function Interface for Python calling C code." +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +pycparser = "*" + +[[package]] +name = "click" +version = "7.1.2" +description = "Composable command line interface toolkit" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[[package]] +name = "colorama" +version = "0.4.3" +description = "Cross-platform colored terminal text." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[[package]] +name = "cryptography" +version = "3.1.1" +description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +category = "main" +optional = false +python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*" + +[package.dependencies] +cffi = ">=1.8,<1.11.3 || >1.11.3" +six = ">=1.4.1" + +[package.extras] +docs = ["sphinx (>=1.6.5,<1.8.0 || >1.8.0,<3.1.0 || >3.1.0,<3.1.1 || >3.1.1)", "sphinx-rtd-theme"] +docstest = ["doc8", "pyenchant (>=1.6.11)", "twine (>=1.12.0)", "sphinxcontrib-spelling (>=4.0.1)"] +pep8test = ["black", "flake8", "flake8-import-order", "pep8-naming"] +ssh = ["bcrypt (>=3.1.5)"] +test = ["pytest (>=3.6.0,<3.9.0 || >3.9.0,<3.9.1 || >3.9.1,<3.9.2 || >3.9.2)", "pretend", "iso8601", "pytz", "hypothesis (>=1.11.4,<3.79.2 || >3.79.2)"] + +[[package]] +name = "fabric" +version = "2.5.0" +description = "High level SSH command execution" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +invoke = ">=1.3,<2.0" +paramiko = ">=2.4" + +[package.extras] +pytest = ["mock (>=2.0.0,<3.0)", "pytest (>=3.2.5,<4.0)"] +testing = ["mock (>=2.0.0,<3.0)"] + +[[package]] +name = "filelock" +version = "3.0.12" +description = "A platform independent file lock." +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "flake8" +version = "3.8.4" +description = "the modular source code checker: pep8 pyflakes and co" +category = "dev" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7" + +[package.dependencies] +mccabe = ">=0.6.0,<0.7.0" +pycodestyle = ">=2.6.0a1,<2.7.0" +pyflakes = ">=2.2.0,<2.3.0" + +[[package]] +name = "flake8-black" +version = "0.2.1" +description = "flake8 plugin to call black as a code style validator" +category = "dev" +optional = false +python-versions = "*" + +[package.dependencies] +black = "*" +flake8 = ">=3.0.0" + +[[package]] +name = "flake8-bugbear" +version = "20.1.4" +description = "A plugin for flake8 finding likely bugs and design problems in your program. Contains warnings that don't belong in pyflakes and pycodestyle." +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +attrs = ">=19.2.0" +flake8 = ">=3.0.0" + +[[package]] +name = "flake8-isort" +version = "4.0.0" +description = "flake8 plugin that integrates isort ." +category = "dev" +optional = false +python-versions = "*" + +[package.dependencies] +flake8 = ">=3.2.1,<4" +isort = ">=4.3.5,<6" +testfixtures = ">=6.8.0,<7" + +[package.extras] +test = ["pytest (>=4.0.2,<6)", "toml"] + +[[package]] +name = "iniconfig" +version = "1.0.1" +description = "iniconfig: brain-dead simple config-ini parsing" +category = "main" +optional = false +python-versions = "*" + +[[package]] +name = "invoke" +version = "1.4.1" +description = "Pythonic task execution" +category = "main" +optional = false +python-versions = "*" + +[[package]] +name = "isort" +version = "5.6.1" +description = "A Python utility / library to sort Python imports." +category = "dev" +optional = false +python-versions = ">=3.6,<4.0" + +[package.extras] +pipfile_deprecated_finder = ["pipreqs", "requirementslib"] +requirements_deprecated_finder = ["pipreqs", "pip-api"] +colors = ["colorama (>=0.4.3,<0.5.0)"] + +[[package]] +name = "jedi" +version = "0.17.2" +description = "An autocompletion tool for Python that can be used for text editors." +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[package.dependencies] +parso = ">=0.7.0,<0.8.0" + +[package.extras] +qa = ["flake8 (3.7.9)"] +testing = ["Django (<3.1)", "colorama", "docopt", "pytest (>=3.9.0,<5.0.0)"] + +[[package]] +name = "mccabe" +version = "0.6.1" +description = "McCabe checker, plugin for flake8" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "mypy" +version = "0.782" +description = "Optional static typing for Python" +category = "dev" +optional = false +python-versions = ">=3.5" + +[package.dependencies] +mypy-extensions = ">=0.4.3,<0.5.0" +typed-ast = ">=1.4.0,<1.5.0" +typing-extensions = ">=3.7.4" + +[package.extras] +dmypy = ["psutil (>=4.0)"] + +[[package]] +name = "mypy-extensions" +version = "0.4.3" +description = "Experimental type system extensions for programs checked with the mypy typechecker." +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "packaging" +version = "20.4" +description = "Core utilities for Python packages" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[package.dependencies] +pyparsing = ">=2.0.2" +six = "*" + +[[package]] +name = "paramiko" +version = "2.7.2" +description = "SSH2 protocol library" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +bcrypt = ">=3.1.3" +cryptography = ">=2.5" +pynacl = ">=1.0.1" + +[package.extras] +all = ["pyasn1 (>=0.1.7)", "pynacl (>=1.0.1)", "bcrypt (>=3.1.3)", "invoke (>=1.3)", "gssapi (>=1.4.1)", "pywin32 (>=2.1.8)"] +ed25519 = ["pynacl (>=1.0.1)", "bcrypt (>=3.1.3)"] +gssapi = ["pyasn1 (>=0.1.7)", "gssapi (>=1.4.1)", "pywin32 (>=2.1.8)"] +invoke = ["invoke (>=1.3)"] + +[[package]] +name = "parso" +version = "0.7.1" +description = "A Python Parser" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[package.extras] +testing = ["docopt", "pytest (>=3.0.7)"] + +[[package]] +name = "pathspec" +version = "0.8.0" +description = "Utility library for gitignore style pattern matching of file paths." +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[[package]] +name = "pluggy" +version = "0.13.1" +description = "plugin and hook calling mechanisms for python" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[package.extras] +dev = ["pre-commit", "tox"] + +[[package]] +name = "py" +version = "1.9.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[[package]] +name = "pycodestyle" +version = "2.6.0" +description = "Python style guide checker" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[[package]] +name = "pycparser" +version = "2.20" +description = "C parser in Python" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[[package]] +name = "pyflakes" +version = "2.2.0" +description = "passive checker of Python programs" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[[package]] +name = "pyls-black" +version = "0.4.6" +description = "Black plugin for the Python Language Server" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +black = ">=19.3b0" +python-language-server = "*" +toml = "*" + +[package.extras] +dev = ["isort (>=5.0)", "flake8", "pytest", "mypy"] + +[[package]] +name = "pyls-isort" +version = "0.2.0" +description = "Isort plugin for python-language-server" +category = "dev" +optional = false +python-versions = "*" + +[package.dependencies] +isort = "*" +python-language-server = "*" + +[[package]] +name = "pyls-mypy" +version = "0.1.8" +description = "Mypy linter for the Python Language Server" +category = "dev" +optional = false +python-versions = "*" + +[package.dependencies] +mypy = "*" +python-language-server = "*" + +[package.extras] +test = ["tox", "versioneer", "pytest", "pytest-cov", "coverage"] + +[[package]] +name = "pynacl" +version = "1.4.0" +description = "Python binding to the Networking and Cryptography (NaCl) library" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[package.dependencies] +cffi = ">=1.4.1" +six = "*" + +[package.extras] +docs = ["sphinx (>=1.6.5)", "sphinx-rtd-theme"] +tests = ["pytest (>=3.2.1,<3.3.0 || >3.3.0)", "hypothesis (>=3.27.0)"] + +[[package]] +name = "pyparsing" +version = "2.4.7" +description = "Python parsing module" +category = "main" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" + +[[package]] +name = "pytest" +version = "6.1.1" +description = "pytest: simple powerful testing with Python" +category = "main" +optional = false +python-versions = ">=3.5" + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=17.4.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<1.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +checkqa_mypy = ["mypy (0.780)"] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "pytest-flake8" +version = "1.0.6" +description = "pytest plugin to check FLAKE8 requirements" +category = "dev" +optional = false +python-versions = "*" + +[package.dependencies] +flake8 = ">=3.5" +pytest = ">=3.5" + +[[package]] +name = "pytest-mypy" +version = "0.7.0" +description = "Mypy static type checker plugin for Pytest" +category = "dev" +optional = false +python-versions = ">=3.5" + +[package.dependencies] +filelock = ">=3.0" +mypy = {version = ">=0.700", markers = "python_version >= \"3.8\""} +pytest = ">=3.5" + +[[package]] +name = "python-jsonrpc-server" +version = "0.4.0" +description = "JSON RPC 2.0 server library" +category = "dev" +optional = false +python-versions = "*" + +[package.dependencies] +ujson = ">=3.0.0" + +[package.extras] +test = ["versioneer", "pylint", "pycodestyle", "pyflakes", "pytest", "mock", "pytest-cov", "coverage"] + +[[package]] +name = "python-language-server" +version = "0.35.1" +description = "Python Language Server for the Language Server Protocol" +category = "dev" +optional = false +python-versions = "*" + +[package.dependencies] +jedi = ">=0.17.0,<0.18.0" +pluggy = "*" +python-jsonrpc-server = ">=0.4.0" + +[package.extras] +all = ["autopep8", "flake8 (>=3.8.0)", "mccabe (>=0.6.0,<0.7.0)", "pycodestyle (>=2.6.0,<2.7.0)", "pydocstyle (>=2.0.0)", "pyflakes (>=2.2.0,<2.3.0)", "pylint (>=2.5.0)", "rope (>=0.10.5)", "yapf"] +autopep8 = ["autopep8"] +flake8 = ["flake8 (>=3.8.0)"] +mccabe = ["mccabe (>=0.6.0,<0.7.0)"] +pycodestyle = ["pycodestyle (>=2.6.0,<2.7.0)"] +pydocstyle = ["pydocstyle (>=2.0.0)"] +pyflakes = ["pyflakes (>=2.2.0,<2.3.0)"] +pylint = ["pylint (>=2.5.0)"] +rope = ["rope (>0.10.5)"] +test = ["versioneer", "pylint (>=2.5.0)", "pytest", "mock", "pytest-cov", "coverage", "numpy", "pandas", "matplotlib", "flaky", "pyqt5"] +yapf = ["yapf"] + +[[package]] +name = "regex" +version = "2020.9.27" +description = "Alternative regular expression module, to replace re." +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "rope" +version = "0.18.0" +description = "a python refactoring library..." +category = "dev" +optional = false +python-versions = "*" + +[package.extras] +dev = ["pytest"] + +[[package]] +name = "six" +version = "1.15.0" +description = "Python 2 and 3 compatibility utilities" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" + +[[package]] +name = "testfixtures" +version = "6.15.0" +description = "A collection of helpers and mock objects for unit tests and doc tests." +category = "dev" +optional = false +python-versions = "*" + +[package.extras] +build = ["setuptools-git", "wheel", "twine"] +docs = ["sphinx", "zope.component", "sybil", "twisted", "mock", "django (<2)", "django"] +test = ["pytest (>=3.6)", "pytest-cov", "pytest-django", "zope.component", "sybil", "twisted", "mock", "django (<2)", "django"] + +[[package]] +name = "toml" +version = "0.10.1" +description = "Python Library for Tom's Obvious, Minimal Language" +category = "main" +optional = false +python-versions = "*" + +[[package]] +name = "typed-ast" +version = "1.4.1" +description = "a fork of Python 2 and 3 ast modules with type comment support" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "typing-extensions" +version = "3.7.4.3" +description = "Backported and Experimental Type Hints for Python 3.5+" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "ujson" +version = "4.0.1" +description = "Ultra fast JSON encoder and decoder for Python" +category = "dev" +optional = false +python-versions = ">=3.6" + +[metadata] +lock-version = "1.1" +python-versions = "^3.8" +content-hash = "40a1da4f76e519932e1c86deaa81cb3331cc282c37ff78f39607936730cbb322" + +[metadata.files] +appdirs = [ + {file = "appdirs-1.4.4-py2.py3-none-any.whl", hash = "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128"}, + {file = "appdirs-1.4.4.tar.gz", hash = "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41"}, +] +atomicwrites = [ + {file = "atomicwrites-1.4.0-py2.py3-none-any.whl", hash = "sha256:6d1784dea7c0c8d4a5172b6c620f40b6e4cbfdf96d783691f2e1302a7b88e197"}, + {file = "atomicwrites-1.4.0.tar.gz", hash = "sha256:ae70396ad1a434f9c7046fd2dd196fc04b12f9e91ffb859164193be8b6168a7a"}, +] +attrs = [ + {file = "attrs-20.2.0-py2.py3-none-any.whl", hash = "sha256:fce7fc47dfc976152e82d53ff92fa0407700c21acd20886a13777a0d20e655dc"}, + {file = "attrs-20.2.0.tar.gz", hash = "sha256:26b54ddbbb9ee1d34d5d3668dd37d6cf74990ab23c828c2888dccdceee395594"}, +] +bcrypt = [ + {file = "bcrypt-3.2.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:c95d4cbebffafcdd28bd28bb4e25b31c50f6da605c81ffd9ad8a3d1b2ab7b1b6"}, + {file = "bcrypt-3.2.0-cp36-abi3-manylinux1_x86_64.whl", hash = "sha256:63d4e3ff96188e5898779b6057878fecf3f11cfe6ec3b313ea09955d587ec7a7"}, + {file = "bcrypt-3.2.0-cp36-abi3-manylinux2010_x86_64.whl", hash = "sha256:cd1ea2ff3038509ea95f687256c46b79f5fc382ad0aa3664d200047546d511d1"}, + {file = "bcrypt-3.2.0-cp36-abi3-manylinux2014_aarch64.whl", hash = "sha256:cdcdcb3972027f83fe24a48b1e90ea4b584d35f1cc279d76de6fc4b13376239d"}, + {file = "bcrypt-3.2.0-cp36-abi3-win32.whl", hash = "sha256:a67fb841b35c28a59cebed05fbd3e80eea26e6d75851f0574a9273c80f3e9b55"}, + {file = "bcrypt-3.2.0-cp36-abi3-win_amd64.whl", hash = "sha256:81fec756feff5b6818ea7ab031205e1d323d8943d237303baca2c5f9c7846f34"}, + {file = "bcrypt-3.2.0.tar.gz", hash = "sha256:5b93c1726e50a93a033c36e5ca7fdcd29a5c7395af50a6892f5d9e7c6cfbfb29"}, +] +black = [ + {file = "black-20.8b1.tar.gz", hash = "sha256:1c02557aa099101b9d21496f8a914e9ed2222ef70336404eeeac8edba836fbea"}, +] +cffi = [ + {file = "cffi-1.14.3-2-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:3eeeb0405fd145e714f7633a5173318bd88d8bbfc3dd0a5751f8c4f70ae629bc"}, + {file = "cffi-1.14.3-2-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:cb763ceceae04803adcc4e2d80d611ef201c73da32d8f2722e9d0ab0c7f10768"}, + {file = "cffi-1.14.3-2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:44f60519595eaca110f248e5017363d751b12782a6f2bd6a7041cba275215f5d"}, + {file = "cffi-1.14.3-2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c53af463f4a40de78c58b8b2710ade243c81cbca641e34debf3396a9640d6ec1"}, + {file = "cffi-1.14.3-2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:33c6cdc071ba5cd6d96769c8969a0531be2d08c2628a0143a10a7dcffa9719ca"}, + {file = "cffi-1.14.3-2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c11579638288e53fc94ad60022ff1b67865363e730ee41ad5e6f0a17188b327a"}, + {file = "cffi-1.14.3-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:3cb3e1b9ec43256c4e0f8d2837267a70b0e1ca8c4f456685508ae6106b1f504c"}, + {file = "cffi-1.14.3-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:f0620511387790860b249b9241c2f13c3a80e21a73e0b861a2df24e9d6f56730"}, + {file = "cffi-1.14.3-cp27-cp27m-win32.whl", hash = "sha256:005f2bfe11b6745d726dbb07ace4d53f057de66e336ff92d61b8c7e9c8f4777d"}, + {file = "cffi-1.14.3-cp27-cp27m-win_amd64.whl", hash = "sha256:2f9674623ca39c9ebe38afa3da402e9326c245f0f5ceff0623dccdac15023e05"}, + {file = "cffi-1.14.3-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:09e96138280241bd355cd585148dec04dbbedb4f46128f340d696eaafc82dd7b"}, + {file = "cffi-1.14.3-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:3363e77a6176afb8823b6e06db78c46dbc4c7813b00a41300a4873b6ba63b171"}, + {file = "cffi-1.14.3-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:0ef488305fdce2580c8b2708f22d7785ae222d9825d3094ab073e22e93dfe51f"}, + {file = "cffi-1.14.3-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:0b1ad452cc824665ddc682400b62c9e4f5b64736a2ba99110712fdee5f2505c4"}, + {file = "cffi-1.14.3-cp35-cp35m-win32.whl", hash = "sha256:85ba797e1de5b48aa5a8427b6ba62cf69607c18c5d4eb747604b7302f1ec382d"}, + {file = "cffi-1.14.3-cp35-cp35m-win_amd64.whl", hash = "sha256:e66399cf0fc07de4dce4f588fc25bfe84a6d1285cc544e67987d22663393926d"}, + {file = "cffi-1.14.3-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:15f351bed09897fbda218e4db5a3d5c06328862f6198d4fb385f3e14e19decb3"}, + {file = "cffi-1.14.3-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:4d7c26bfc1ea9f92084a1d75e11999e97b62d63128bcc90c3624d07813c52808"}, + {file = "cffi-1.14.3-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:23e5d2040367322824605bc29ae8ee9175200b92cb5483ac7d466927a9b3d537"}, + {file = "cffi-1.14.3-cp36-cp36m-win32.whl", hash = "sha256:a624fae282e81ad2e4871bdb767e2c914d0539708c0f078b5b355258293c98b0"}, + {file = "cffi-1.14.3-cp36-cp36m-win_amd64.whl", hash = "sha256:de31b5164d44ef4943db155b3e8e17929707cac1e5bd2f363e67a56e3af4af6e"}, + {file = "cffi-1.14.3-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:f92cdecb618e5fa4658aeb97d5eb3d2f47aa94ac6477c6daf0f306c5a3b9e6b1"}, + {file = "cffi-1.14.3-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:22399ff4870fb4c7ef19fff6eeb20a8bbf15571913c181c78cb361024d574579"}, + {file = "cffi-1.14.3-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:f4eae045e6ab2bb54ca279733fe4eb85f1effda392666308250714e01907f394"}, + {file = "cffi-1.14.3-cp37-cp37m-win32.whl", hash = "sha256:b0358e6fefc74a16f745afa366acc89f979040e0cbc4eec55ab26ad1f6a9bfbc"}, + {file = "cffi-1.14.3-cp37-cp37m-win_amd64.whl", hash = "sha256:6642f15ad963b5092d65aed022d033c77763515fdc07095208f15d3563003869"}, + {file = "cffi-1.14.3-cp38-cp38-manylinux1_i686.whl", hash = "sha256:2791f68edc5749024b4722500e86303a10d342527e1e3bcac47f35fbd25b764e"}, + {file = "cffi-1.14.3-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:529c4ed2e10437c205f38f3691a68be66c39197d01062618c55f74294a4a4828"}, + {file = "cffi-1.14.3-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:8f0f1e499e4000c4c347a124fa6a27d37608ced4fe9f7d45070563b7c4c370c9"}, + {file = "cffi-1.14.3-cp38-cp38-win32.whl", hash = "sha256:3b8eaf915ddc0709779889c472e553f0d3e8b7bdf62dab764c8921b09bf94522"}, + {file = "cffi-1.14.3-cp38-cp38-win_amd64.whl", hash = "sha256:bbd2f4dfee1079f76943767fce837ade3087b578aeb9f69aec7857d5bf25db15"}, + {file = "cffi-1.14.3-cp39-cp39-manylinux1_i686.whl", hash = "sha256:cc75f58cdaf043fe6a7a6c04b3b5a0e694c6a9e24050967747251fb80d7bce0d"}, + {file = "cffi-1.14.3-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:bf39a9e19ce7298f1bd6a9758fa99707e9e5b1ebe5e90f2c3913a47bc548747c"}, + {file = "cffi-1.14.3-cp39-cp39-win32.whl", hash = "sha256:d80998ed59176e8cba74028762fbd9b9153b9afc71ea118e63bbf5d4d0f9552b"}, + {file = "cffi-1.14.3-cp39-cp39-win_amd64.whl", hash = "sha256:c150eaa3dadbb2b5339675b88d4573c1be3cb6f2c33a6c83387e10cc0bf05bd3"}, + {file = "cffi-1.14.3.tar.gz", hash = "sha256:f92f789e4f9241cd262ad7a555ca2c648a98178a953af117ef7fad46aa1d5591"}, +] +click = [ + {file = "click-7.1.2-py2.py3-none-any.whl", hash = "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc"}, + {file = "click-7.1.2.tar.gz", hash = "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a"}, +] +colorama = [ + {file = "colorama-0.4.3-py2.py3-none-any.whl", hash = "sha256:7d73d2a99753107a36ac6b455ee49046802e59d9d076ef8e47b61499fa29afff"}, + {file = "colorama-0.4.3.tar.gz", hash = "sha256:e96da0d330793e2cb9485e9ddfd918d456036c7149416295932478192f4436a1"}, +] +cryptography = [ + {file = "cryptography-3.1.1-cp27-cp27m-macosx_10_10_x86_64.whl", hash = "sha256:65beb15e7f9c16e15934569d29fb4def74ea1469d8781f6b3507ab896d6d8719"}, + {file = "cryptography-3.1.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:983c0c3de4cb9fcba68fd3f45ed846eb86a2a8b8d8bc5bb18364c4d00b3c61fe"}, + {file = "cryptography-3.1.1-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:e97a3b627e3cb63c415a16245d6cef2139cca18bb1183d1b9375a1c14e83f3b3"}, + {file = "cryptography-3.1.1-cp27-cp27m-win32.whl", hash = "sha256:cb179acdd4ae1e4a5a160d80b87841b3d0e0be84af46c7bb2cd7ece57a39c4ba"}, + {file = "cryptography-3.1.1-cp27-cp27m-win_amd64.whl", hash = "sha256:b372026ebf32fe2523159f27d9f0e9f485092e43b00a5adacf732192a70ba118"}, + {file = "cryptography-3.1.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:680da076cad81cdf5ffcac50c477b6790be81768d30f9da9e01960c4b18a66db"}, + {file = "cryptography-3.1.1-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:5d52c72449bb02dd45a773a203196e6d4fae34e158769c896012401f33064396"}, + {file = "cryptography-3.1.1-cp35-abi3-macosx_10_10_x86_64.whl", hash = "sha256:f0e099fc4cc697450c3dd4031791559692dd941a95254cb9aeded66a7aa8b9bc"}, + {file = "cryptography-3.1.1-cp35-abi3-manylinux1_x86_64.whl", hash = "sha256:a7597ffc67987b37b12e09c029bd1dc43965f75d328076ae85721b84046e9ca7"}, + {file = "cryptography-3.1.1-cp35-abi3-manylinux2010_x86_64.whl", hash = "sha256:4549b137d8cbe3c2eadfa56c0c858b78acbeff956bd461e40000b2164d9167c6"}, + {file = "cryptography-3.1.1-cp35-abi3-manylinux2014_aarch64.whl", hash = "sha256:89aceb31cd5f9fc2449fe8cf3810797ca52b65f1489002d58fe190bfb265c536"}, + {file = "cryptography-3.1.1-cp35-cp35m-win32.whl", hash = "sha256:559d622aef2a2dff98a892eef321433ba5bc55b2485220a8ca289c1ecc2bd54f"}, + {file = "cryptography-3.1.1-cp35-cp35m-win_amd64.whl", hash = "sha256:451cdf60be4dafb6a3b78802006a020e6cd709c22d240f94f7a0696240a17154"}, + {file = "cryptography-3.1.1-cp36-abi3-win32.whl", hash = "sha256:762bc5a0df03c51ee3f09c621e1cee64e3a079a2b5020de82f1613873d79ee70"}, + {file = "cryptography-3.1.1-cp36-abi3-win_amd64.whl", hash = "sha256:b12e715c10a13ca1bd27fbceed9adc8c5ff640f8e1f7ea76416352de703523c8"}, + {file = "cryptography-3.1.1-cp36-cp36m-win32.whl", hash = "sha256:21b47c59fcb1c36f1113f3709d37935368e34815ea1d7073862e92f810dc7499"}, + {file = "cryptography-3.1.1-cp36-cp36m-win_amd64.whl", hash = "sha256:48ee615a779ffa749d7d50c291761dc921d93d7cf203dca2db663b4f193f0e49"}, + {file = "cryptography-3.1.1-cp37-cp37m-win32.whl", hash = "sha256:b2bded09c578d19e08bd2c5bb8fed7f103e089752c9cf7ca7ca7de522326e921"}, + {file = "cryptography-3.1.1-cp37-cp37m-win_amd64.whl", hash = "sha256:f99317a0fa2e49917689b8cf977510addcfaaab769b3f899b9c481bbd76730c2"}, + {file = "cryptography-3.1.1-cp38-cp38-win32.whl", hash = "sha256:ab010e461bb6b444eaf7f8c813bb716be2d78ab786103f9608ffd37a4bd7d490"}, + {file = "cryptography-3.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:99d4984aabd4c7182050bca76176ce2dbc9fa9748afe583a7865c12954d714ba"}, + {file = "cryptography-3.1.1.tar.gz", hash = "sha256:9d9fc6a16357965d282dd4ab6531013935425d0dc4950df2e0cf2a1b1ac1017d"}, +] +fabric = [ + {file = "fabric-2.5.0-py2.py3-none-any.whl", hash = "sha256:160331934ea60036604928e792fa8e9f813266b098ef5562aa82b88527740389"}, + {file = "fabric-2.5.0.tar.gz", hash = "sha256:24842d7d51556adcabd885ac3cf5e1df73fc622a1708bf3667bf5927576cdfa6"}, +] +filelock = [ + {file = "filelock-3.0.12-py3-none-any.whl", hash = "sha256:929b7d63ec5b7d6b71b0fa5ac14e030b3f70b75747cef1b10da9b879fef15836"}, + {file = "filelock-3.0.12.tar.gz", hash = "sha256:18d82244ee114f543149c66a6e0c14e9c4f8a1044b5cdaadd0f82159d6a6ff59"}, +] +flake8 = [ + {file = "flake8-3.8.4-py2.py3-none-any.whl", hash = "sha256:749dbbd6bfd0cf1318af27bf97a14e28e5ff548ef8e5b1566ccfb25a11e7c839"}, + {file = "flake8-3.8.4.tar.gz", hash = "sha256:aadae8761ec651813c24be05c6f7b4680857ef6afaae4651a4eccaef97ce6c3b"}, +] +flake8-black = [ + {file = "flake8-black-0.2.1.tar.gz", hash = "sha256:f26651bc10db786c03f4093414f7c9ea982ed8a244cec323c984feeffdf4c118"}, +] +flake8-bugbear = [ + {file = "flake8-bugbear-20.1.4.tar.gz", hash = "sha256:bd02e4b009fb153fe6072c31c52aeab5b133d508095befb2ffcf3b41c4823162"}, + {file = "flake8_bugbear-20.1.4-py36.py37.py38-none-any.whl", hash = "sha256:a3ddc03ec28ba2296fc6f89444d1c946a6b76460f859795b35b77d4920a51b63"}, +] +flake8-isort = [ + {file = "flake8-isort-4.0.0.tar.gz", hash = "sha256:2b91300f4f1926b396c2c90185844eb1a3d5ec39ea6138832d119da0a208f4d9"}, + {file = "flake8_isort-4.0.0-py2.py3-none-any.whl", hash = "sha256:729cd6ef9ba3659512dee337687c05d79c78e1215fdf921ed67e5fe46cce2f3c"}, +] +iniconfig = [ + {file = "iniconfig-1.0.1-py3-none-any.whl", hash = "sha256:80cf40c597eb564e86346103f609d74efce0f6b4d4f30ec8ce9e2c26411ba437"}, + {file = "iniconfig-1.0.1.tar.gz", hash = "sha256:e5f92f89355a67de0595932a6c6c02ab4afddc6fcdc0bfc5becd0d60884d3f69"}, +] +invoke = [ + {file = "invoke-1.4.1-py2-none-any.whl", hash = "sha256:93e12876d88130c8e0d7fd6618dd5387d6b36da55ad541481dfa5e001656f134"}, + {file = "invoke-1.4.1-py3-none-any.whl", hash = "sha256:87b3ef9d72a1667e104f89b159eaf8a514dbf2f3576885b2bbdefe74c3fb2132"}, + {file = "invoke-1.4.1.tar.gz", hash = "sha256:de3f23bfe669e3db1085789fd859eb8ca8e0c5d9c20811e2407fa042e8a5e15d"}, +] +isort = [ + {file = "isort-5.6.1-py3-none-any.whl", hash = "sha256:dd3211f513f4a92ec1ec1876fc1dc3c686649c349d49523f5b5adbb0814e5960"}, + {file = "isort-5.6.1.tar.gz", hash = "sha256:2f510f34ae18a8d0958c53eec51ef84fd099f07c4c639676525acbcd7b5bd3ff"}, +] +jedi = [ + {file = "jedi-0.17.2-py2.py3-none-any.whl", hash = "sha256:98cc583fa0f2f8304968199b01b6b4b94f469a1f4a74c1560506ca2a211378b5"}, + {file = "jedi-0.17.2.tar.gz", hash = "sha256:86ed7d9b750603e4ba582ea8edc678657fb4007894a12bcf6f4bb97892f31d20"}, +] +mccabe = [ + {file = "mccabe-0.6.1-py2.py3-none-any.whl", hash = "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42"}, + {file = "mccabe-0.6.1.tar.gz", hash = "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"}, +] +mypy = [ + {file = "mypy-0.782-cp35-cp35m-macosx_10_6_x86_64.whl", hash = "sha256:2c6cde8aa3426c1682d35190b59b71f661237d74b053822ea3d748e2c9578a7c"}, + {file = "mypy-0.782-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9c7a9a7ceb2871ba4bac1cf7217a7dd9ccd44c27c2950edbc6dc08530f32ad4e"}, + {file = "mypy-0.782-cp35-cp35m-win_amd64.whl", hash = "sha256:c05b9e4fb1d8a41d41dec8786c94f3b95d3c5f528298d769eb8e73d293abc48d"}, + {file = "mypy-0.782-cp36-cp36m-macosx_10_6_x86_64.whl", hash = "sha256:6731603dfe0ce4352c555c6284c6db0dc935b685e9ce2e4cf220abe1e14386fd"}, + {file = "mypy-0.782-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:f05644db6779387ccdb468cc47a44b4356fc2ffa9287135d05b70a98dc83b89a"}, + {file = "mypy-0.782-cp36-cp36m-win_amd64.whl", hash = "sha256:b7fbfabdbcc78c4f6fc4712544b9b0d6bf171069c6e0e3cb82440dd10ced3406"}, + {file = "mypy-0.782-cp37-cp37m-macosx_10_6_x86_64.whl", hash = "sha256:3fdda71c067d3ddfb21da4b80e2686b71e9e5c72cca65fa216d207a358827f86"}, + {file = "mypy-0.782-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:d7df6eddb6054d21ca4d3c6249cae5578cb4602951fd2b6ee2f5510ffb098707"}, + {file = "mypy-0.782-cp37-cp37m-win_amd64.whl", hash = "sha256:a4a2cbcfc4cbf45cd126f531dedda8485671545b43107ded25ce952aac6fb308"}, + {file = "mypy-0.782-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6bb93479caa6619d21d6e7160c552c1193f6952f0668cdda2f851156e85186fc"}, + {file = "mypy-0.782-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:81c7908b94239c4010e16642c9102bfc958ab14e36048fa77d0be3289dda76ea"}, + {file = "mypy-0.782-cp38-cp38-win_amd64.whl", hash = "sha256:5dd13ff1f2a97f94540fd37a49e5d255950ebcdf446fb597463a40d0df3fac8b"}, + {file = "mypy-0.782-py3-none-any.whl", hash = "sha256:e0b61738ab504e656d1fe4ff0c0601387a5489ca122d55390ade31f9ca0e252d"}, + {file = "mypy-0.782.tar.gz", hash = "sha256:eff7d4a85e9eea55afa34888dfeaccde99e7520b51f867ac28a48492c0b1130c"}, +] +mypy-extensions = [ + {file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"}, + {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"}, +] +packaging = [ + {file = "packaging-20.4-py2.py3-none-any.whl", hash = "sha256:998416ba6962ae7fbd6596850b80e17859a5753ba17c32284f67bfff33784181"}, + {file = "packaging-20.4.tar.gz", hash = "sha256:4357f74f47b9c12db93624a82154e9b120fa8293699949152b22065d556079f8"}, +] +paramiko = [ + {file = "paramiko-2.7.2-py2.py3-none-any.whl", hash = "sha256:4f3e316fef2ac628b05097a637af35685183111d4bc1b5979bd397c2ab7b5898"}, + {file = "paramiko-2.7.2.tar.gz", hash = "sha256:7f36f4ba2c0d81d219f4595e35f70d56cc94f9ac40a6acdf51d6ca210ce65035"}, +] +parso = [ + {file = "parso-0.7.1-py2.py3-none-any.whl", hash = "sha256:97218d9159b2520ff45eb78028ba8b50d2bc61dcc062a9682666f2dc4bd331ea"}, + {file = "parso-0.7.1.tar.gz", hash = "sha256:caba44724b994a8a5e086460bb212abc5a8bc46951bf4a9a1210745953622eb9"}, +] +pathspec = [ + {file = "pathspec-0.8.0-py2.py3-none-any.whl", hash = "sha256:7d91249d21749788d07a2d0f94147accd8f845507400749ea19c1ec9054a12b0"}, + {file = "pathspec-0.8.0.tar.gz", hash = "sha256:da45173eb3a6f2a5a487efba21f050af2b41948be6ab52b6a1e3ff22bb8b7061"}, +] +pluggy = [ + {file = "pluggy-0.13.1-py2.py3-none-any.whl", hash = "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"}, + {file = "pluggy-0.13.1.tar.gz", hash = "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0"}, +] +py = [ + {file = "py-1.9.0-py2.py3-none-any.whl", hash = "sha256:366389d1db726cd2fcfc79732e75410e5fe4d31db13692115529d34069a043c2"}, + {file = "py-1.9.0.tar.gz", hash = "sha256:9ca6883ce56b4e8da7e79ac18787889fa5206c79dcc67fb065376cd2fe03f342"}, +] +pycodestyle = [ + {file = "pycodestyle-2.6.0-py2.py3-none-any.whl", hash = "sha256:2295e7b2f6b5bd100585ebcb1f616591b652db8a741695b3d8f5d28bdc934367"}, + {file = "pycodestyle-2.6.0.tar.gz", hash = "sha256:c58a7d2815e0e8d7972bf1803331fb0152f867bd89adf8a01dfd55085434192e"}, +] +pycparser = [ + {file = "pycparser-2.20-py2.py3-none-any.whl", hash = "sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705"}, + {file = "pycparser-2.20.tar.gz", hash = "sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0"}, +] +pyflakes = [ + {file = "pyflakes-2.2.0-py2.py3-none-any.whl", hash = "sha256:0d94e0e05a19e57a99444b6ddcf9a6eb2e5c68d3ca1e98e90707af8152c90a92"}, + {file = "pyflakes-2.2.0.tar.gz", hash = "sha256:35b2d75ee967ea93b55750aa9edbbf72813e06a66ba54438df2cfac9e3c27fc8"}, +] +pyls-black = [ + {file = "pyls-black-0.4.6.tar.gz", hash = "sha256:33700e5ed605636ea7ba39188a1362d2f8602f7301f8f2b8544773886f965663"}, + {file = "pyls_black-0.4.6-py3-none-any.whl", hash = "sha256:8f5fb8fed503588c10435d2d48e2c3751437f1bdb8116134b05a4591c4899940"}, +] +pyls-isort = [ + {file = "pyls-isort-0.2.0.tar.gz", hash = "sha256:a6c292332746d3dc690f2a3dcdb9a01d913b9ee8444defe3cbffcddb7e3874eb"}, +] +pyls-mypy = [ + {file = "pyls-mypy-0.1.8.tar.gz", hash = "sha256:3fd83028961f0ca9eb3048b7a01cf42a9e3d46d8ea4935c1424c33da22c3eb03"}, +] +pynacl = [ + {file = "PyNaCl-1.4.0-cp27-cp27m-macosx_10_10_x86_64.whl", hash = "sha256:ea6841bc3a76fa4942ce00f3bda7d436fda21e2d91602b9e21b7ca9ecab8f3ff"}, + {file = "PyNaCl-1.4.0-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:d452a6746f0a7e11121e64625109bc4468fc3100452817001dbe018bb8b08514"}, + {file = "PyNaCl-1.4.0-cp27-cp27m-win32.whl", hash = "sha256:2fe0fc5a2480361dcaf4e6e7cea00e078fcda07ba45f811b167e3f99e8cff574"}, + {file = "PyNaCl-1.4.0-cp27-cp27m-win_amd64.whl", hash = "sha256:f8851ab9041756003119368c1e6cd0b9c631f46d686b3904b18c0139f4419f80"}, + {file = "PyNaCl-1.4.0-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:7757ae33dae81c300487591c68790dfb5145c7d03324000433d9a2c141f82af7"}, + {file = "PyNaCl-1.4.0-cp35-abi3-macosx_10_10_x86_64.whl", hash = "sha256:757250ddb3bff1eecd7e41e65f7f833a8405fede0194319f87899690624f2122"}, + {file = "PyNaCl-1.4.0-cp35-abi3-manylinux1_x86_64.whl", hash = "sha256:30f9b96db44e09b3304f9ea95079b1b7316b2b4f3744fe3aaecccd95d547063d"}, + {file = "PyNaCl-1.4.0-cp35-abi3-win32.whl", hash = "sha256:4e10569f8cbed81cb7526ae137049759d2a8d57726d52c1a000a3ce366779634"}, + {file = "PyNaCl-1.4.0-cp35-abi3-win_amd64.whl", hash = "sha256:c914f78da4953b33d4685e3cdc7ce63401247a21425c16a39760e282075ac4a6"}, + {file = "PyNaCl-1.4.0-cp35-cp35m-win32.whl", hash = "sha256:06cbb4d9b2c4bd3c8dc0d267416aaed79906e7b33f114ddbf0911969794b1cc4"}, + {file = "PyNaCl-1.4.0-cp35-cp35m-win_amd64.whl", hash = "sha256:511d269ee845037b95c9781aa702f90ccc36036f95d0f31373a6a79bd8242e25"}, + {file = "PyNaCl-1.4.0-cp36-cp36m-win32.whl", hash = "sha256:11335f09060af52c97137d4ac54285bcb7df0cef29014a1a4efe64ac065434c4"}, + {file = "PyNaCl-1.4.0-cp36-cp36m-win_amd64.whl", hash = "sha256:cd401ccbc2a249a47a3a1724c2918fcd04be1f7b54eb2a5a71ff915db0ac51c6"}, + {file = "PyNaCl-1.4.0-cp37-cp37m-win32.whl", hash = "sha256:8122ba5f2a2169ca5da936b2e5a511740ffb73979381b4229d9188f6dcb22f1f"}, + {file = "PyNaCl-1.4.0-cp37-cp37m-win_amd64.whl", hash = "sha256:537a7ccbea22905a0ab36ea58577b39d1fa9b1884869d173b5cf111f006f689f"}, + {file = "PyNaCl-1.4.0-cp38-cp38-win32.whl", hash = "sha256:9c4a7ea4fb81536c1b1f5cc44d54a296f96ae78c1ebd2311bd0b60be45a48d96"}, + {file = "PyNaCl-1.4.0-cp38-cp38-win_amd64.whl", hash = "sha256:7c6092102219f59ff29788860ccb021e80fffd953920c4a8653889c029b2d420"}, + {file = "PyNaCl-1.4.0.tar.gz", hash = "sha256:54e9a2c849c742006516ad56a88f5c74bf2ce92c9f67435187c3c5953b346505"}, +] +pyparsing = [ + {file = "pyparsing-2.4.7-py2.py3-none-any.whl", hash = "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"}, + {file = "pyparsing-2.4.7.tar.gz", hash = "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1"}, +] +pytest = [ + {file = "pytest-6.1.1-py3-none-any.whl", hash = "sha256:7a8190790c17d79a11f847fba0b004ee9a8122582ebff4729a082c109e81a4c9"}, + {file = "pytest-6.1.1.tar.gz", hash = "sha256:8f593023c1a0f916110285b6efd7f99db07d59546e3d8c36fc60e2ab05d3be92"}, +] +pytest-flake8 = [ + {file = "pytest-flake8-1.0.6.tar.gz", hash = "sha256:1b82bb58c88eb1db40524018d3fcfd0424575029703b4e2d8e3ee873f2b17027"}, + {file = "pytest_flake8-1.0.6-py2.py3-none-any.whl", hash = "sha256:2e91578ecd9b200066f99c1e1de0f510fbb85bcf43712d46ea29fe47607cc234"}, +] +pytest-mypy = [ + {file = "pytest-mypy-0.7.0.tar.gz", hash = "sha256:5a667d9a2b66bf98b3a494411f221923a6e2c3eafbe771104951aaec8985673d"}, + {file = "pytest_mypy-0.7.0-py3-none-any.whl", hash = "sha256:e0505ace48d2b19fe686366fce6b4a2ac0d090423736bb6aa2e39554d18974b7"}, +] +python-jsonrpc-server = [ + {file = "python-jsonrpc-server-0.4.0.tar.gz", hash = "sha256:62c543e541f101ec5b57dc654efc212d2c2e3ea47ff6f54b2e7dcb36ecf20595"}, + {file = "python_jsonrpc_server-0.4.0-py3-none-any.whl", hash = "sha256:e5a908ff182e620aac07db5f57887eeb0afe33993008f57dc1b85b594cea250c"}, +] +python-language-server = [ + {file = "python-language-server-0.35.1.tar.gz", hash = "sha256:6e0c9a3b2ae98e0eb22e98ed6b3c4e190a6bf9e27af53efd2396da60cd92b221"}, + {file = "python_language_server-0.35.1-py2.py3-none-any.whl", hash = "sha256:7051090259e3e81c0cdb140de8e32b8f11219808cda4427e6faf61f9ff9a3bf4"}, +] +regex = [ + {file = "regex-2020.9.27-cp27-cp27m-win32.whl", hash = "sha256:d23a18037313714fb3bb5a94434d3151ee4300bae631894b1ac08111abeaa4a3"}, + {file = "regex-2020.9.27-cp27-cp27m-win_amd64.whl", hash = "sha256:84e9407db1b2eb368b7ecc283121b5e592c9aaedbe8c78b1a2f1102eb2e21d19"}, + {file = "regex-2020.9.27-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:5f18875ac23d9aa2f060838e8b79093e8bb2313dbaaa9f54c6d8e52a5df097be"}, + {file = "regex-2020.9.27-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:ae91972f8ac958039920ef6e8769277c084971a142ce2b660691793ae44aae6b"}, + {file = "regex-2020.9.27-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:9a02d0ae31d35e1ec12a4ea4d4cca990800f66a917d0fb997b20fbc13f5321fc"}, + {file = "regex-2020.9.27-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:ebbe29186a3d9b0c591e71b7393f1ae08c83cb2d8e517d2a822b8f7ec99dfd8b"}, + {file = "regex-2020.9.27-cp36-cp36m-win32.whl", hash = "sha256:4707f3695b34335afdfb09be3802c87fa0bc27030471dbc082f815f23688bc63"}, + {file = "regex-2020.9.27-cp36-cp36m-win_amd64.whl", hash = "sha256:9bc13e0d20b97ffb07821aa3e113f9998e84994fe4d159ffa3d3a9d1b805043b"}, + {file = "regex-2020.9.27-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:f1b3afc574a3db3b25c89161059d857bd4909a1269b0b3cb3c904677c8c4a3f7"}, + {file = "regex-2020.9.27-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:5533a959a1748a5c042a6da71fe9267a908e21eded7a4f373efd23a2cbdb0ecc"}, + {file = "regex-2020.9.27-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:1fe0a41437bbd06063aa184c34804efa886bcc128222e9916310c92cd54c3b4c"}, + {file = "regex-2020.9.27-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:c570f6fa14b9c4c8a4924aaad354652366577b4f98213cf76305067144f7b100"}, + {file = "regex-2020.9.27-cp37-cp37m-win32.whl", hash = "sha256:eda4771e0ace7f67f58bc5b560e27fb20f32a148cbc993b0c3835970935c2707"}, + {file = "regex-2020.9.27-cp37-cp37m-win_amd64.whl", hash = "sha256:60b0e9e6dc45683e569ec37c55ac20c582973841927a85f2d8a7d20ee80216ab"}, + {file = "regex-2020.9.27-cp38-cp38-manylinux1_i686.whl", hash = "sha256:088afc8c63e7bd187a3c70a94b9e50ab3f17e1d3f52a32750b5b77dbe99ef5ef"}, + {file = "regex-2020.9.27-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:eaf548d117b6737df379fdd53bdde4f08870e66d7ea653e230477f071f861121"}, + {file = "regex-2020.9.27-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:41bb65f54bba392643557e617316d0d899ed5b4946dccee1cb6696152b29844b"}, + {file = "regex-2020.9.27-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:8d69cef61fa50c8133382e61fd97439de1ae623fe943578e477e76a9d9471637"}, + {file = "regex-2020.9.27-cp38-cp38-win32.whl", hash = "sha256:f2388013e68e750eaa16ccbea62d4130180c26abb1d8e5d584b9baf69672b30f"}, + {file = "regex-2020.9.27-cp38-cp38-win_amd64.whl", hash = "sha256:4318d56bccfe7d43e5addb272406ade7a2274da4b70eb15922a071c58ab0108c"}, + {file = "regex-2020.9.27-cp39-cp39-manylinux1_i686.whl", hash = "sha256:84cada8effefe9a9f53f9b0d2ba9b7b6f5edf8d2155f9fdbe34616e06ececf81"}, + {file = "regex-2020.9.27-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:816064fc915796ea1f26966163f6845de5af78923dfcecf6551e095f00983650"}, + {file = "regex-2020.9.27-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:5d892a4f1c999834eaa3c32bc9e8b976c5825116cde553928c4c8e7e48ebda67"}, + {file = "regex-2020.9.27-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:c9443124c67b1515e4fe0bb0aa18df640965e1030f468a2a5dc2589b26d130ad"}, + {file = "regex-2020.9.27-cp39-cp39-win32.whl", hash = "sha256:49f23ebd5ac073765ecbcf046edc10d63dcab2f4ae2bce160982cb30df0c0302"}, + {file = "regex-2020.9.27-cp39-cp39-win_amd64.whl", hash = "sha256:3d20024a70b97b4f9546696cbf2fd30bae5f42229fbddf8661261b1eaff0deb7"}, + {file = "regex-2020.9.27.tar.gz", hash = "sha256:a6f32aea4260dfe0e55dc9733ea162ea38f0ea86aa7d0f77b15beac5bf7b369d"}, +] +rope = [ + {file = "rope-0.18.0.tar.gz", hash = "sha256:786b5c38c530d4846aa68a42604f61b4e69a493390e3ca11b88df0fbfdc3ed04"}, +] +six = [ + {file = "six-1.15.0-py2.py3-none-any.whl", hash = "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"}, + {file = "six-1.15.0.tar.gz", hash = "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259"}, +] +testfixtures = [ + {file = "testfixtures-6.15.0-py2.py3-none-any.whl", hash = "sha256:e17f4f526fc90b0ac9bc7f8ca62b7dec17d9faf3d721f56bda4f0fd94d02f85a"}, + {file = "testfixtures-6.15.0.tar.gz", hash = "sha256:409f77cfbdad822d12a8ce5c4aa8fb4d0bb38073f4a5444fede3702716a2cec2"}, +] +toml = [ + {file = "toml-0.10.1-py2.py3-none-any.whl", hash = "sha256:bda89d5935c2eac546d648028b9901107a595863cb36bae0c73ac804a9b4ce88"}, + {file = "toml-0.10.1.tar.gz", hash = "sha256:926b612be1e5ce0634a2ca03470f95169cf16f939018233a670519cb4ac58b0f"}, +] +typed-ast = [ + {file = "typed_ast-1.4.1-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:73d785a950fc82dd2a25897d525d003f6378d1cb23ab305578394694202a58c3"}, + {file = "typed_ast-1.4.1-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:aaee9905aee35ba5905cfb3c62f3e83b3bec7b39413f0a7f19be4e547ea01ebb"}, + {file = "typed_ast-1.4.1-cp35-cp35m-win32.whl", hash = "sha256:0c2c07682d61a629b68433afb159376e24e5b2fd4641d35424e462169c0a7919"}, + {file = "typed_ast-1.4.1-cp35-cp35m-win_amd64.whl", hash = "sha256:4083861b0aa07990b619bd7ddc365eb7fa4b817e99cf5f8d9cf21a42780f6e01"}, + {file = "typed_ast-1.4.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:269151951236b0f9a6f04015a9004084a5ab0d5f19b57de779f908621e7d8b75"}, + {file = "typed_ast-1.4.1-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:24995c843eb0ad11a4527b026b4dde3da70e1f2d8806c99b7b4a7cf491612652"}, + {file = "typed_ast-1.4.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:fe460b922ec15dd205595c9b5b99e2f056fd98ae8f9f56b888e7a17dc2b757e7"}, + {file = "typed_ast-1.4.1-cp36-cp36m-win32.whl", hash = "sha256:4e3e5da80ccbebfff202a67bf900d081906c358ccc3d5e3c8aea42fdfdfd51c1"}, + {file = "typed_ast-1.4.1-cp36-cp36m-win_amd64.whl", hash = "sha256:249862707802d40f7f29f6e1aad8d84b5aa9e44552d2cc17384b209f091276aa"}, + {file = "typed_ast-1.4.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8ce678dbaf790dbdb3eba24056d5364fb45944f33553dd5869b7580cdbb83614"}, + {file = "typed_ast-1.4.1-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:c9e348e02e4d2b4a8b2eedb48210430658df6951fa484e59de33ff773fbd4b41"}, + {file = "typed_ast-1.4.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:bcd3b13b56ea479b3650b82cabd6b5343a625b0ced5429e4ccad28a8973f301b"}, + {file = "typed_ast-1.4.1-cp37-cp37m-win32.whl", hash = "sha256:d5d33e9e7af3b34a40dc05f498939f0ebf187f07c385fd58d591c533ad8562fe"}, + {file = "typed_ast-1.4.1-cp37-cp37m-win_amd64.whl", hash = "sha256:0666aa36131496aed8f7be0410ff974562ab7eeac11ef351def9ea6fa28f6355"}, + {file = "typed_ast-1.4.1-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:d205b1b46085271b4e15f670058ce182bd1199e56b317bf2ec004b6a44f911f6"}, + {file = "typed_ast-1.4.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:6daac9731f172c2a22ade6ed0c00197ee7cc1221aa84cfdf9c31defeb059a907"}, + {file = "typed_ast-1.4.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:498b0f36cc7054c1fead3d7fc59d2150f4d5c6c56ba7fb150c013fbc683a8d2d"}, + {file = "typed_ast-1.4.1-cp38-cp38-win32.whl", hash = "sha256:715ff2f2df46121071622063fc7543d9b1fd19ebfc4f5c8895af64a77a8c852c"}, + {file = "typed_ast-1.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:fc0fea399acb12edbf8a628ba8d2312f583bdbdb3335635db062fa98cf71fca4"}, + {file = "typed_ast-1.4.1-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:d43943ef777f9a1c42bf4e552ba23ac77a6351de620aa9acf64ad54933ad4d34"}, + {file = "typed_ast-1.4.1.tar.gz", hash = "sha256:8c8aaad94455178e3187ab22c8b01a3837f8ee50e09cf31f1ba129eb293ec30b"}, +] +typing-extensions = [ + {file = "typing_extensions-3.7.4.3-py2-none-any.whl", hash = "sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f"}, + {file = "typing_extensions-3.7.4.3-py3-none-any.whl", hash = "sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918"}, + {file = "typing_extensions-3.7.4.3.tar.gz", hash = "sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c"}, +] +ujson = [ + {file = "ujson-4.0.1-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:5fe1536465b1c86e32a47113abd3178001b7c2dcd61f95f336fe2febf4661e74"}, + {file = "ujson-4.0.1-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:0f412c3f59b1ab0f40018235224ca0cf29232d0201ff5085618565a8a9c810ed"}, + {file = "ujson-4.0.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:4f12b0b4e235b35d49f15227b0a827e614c52dda903c58a8f5523936c233dfc7"}, + {file = "ujson-4.0.1-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:7a1545ac2476db4cc1f0f236603ccbb50991fc1bba480cda1bc06348cc2a2bf0"}, + {file = "ujson-4.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:078808c385036cba73cad96f498310c61e9b5ae5ac9ea01e7c3996ece544b556"}, + {file = "ujson-4.0.1-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:4fe8c6112b732cba5a722f7cbe22f18d405f6f44415794a5b46473a477635233"}, + {file = "ujson-4.0.1-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:71703a269f074ff65b9d7746662e4b3e76a4af443e532218af1e8ce15d9b1e7b"}, + {file = "ujson-4.0.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:b87379a3f8046d6d111762d81f3384bf38ab24b1535c841fe867a4a097d84523"}, + {file = "ujson-4.0.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:a79bca47eafb31c74b38e68623bc9b2bb930cb48fab1af31c8f2cb68cf473421"}, + {file = "ujson-4.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:e7ab24942b2d57920d75b817b8eead293026db003247e26f99506bdad86c61b4"}, + {file = "ujson-4.0.1-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:51480048373cf97a6b97fcd70c3586ca0a31f27e22ab680fb14c1f22bedbf743"}, + {file = "ujson-4.0.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:c604024bd853b5df6be7d933e934da8dd139e6159564db7c55b92a9937678093"}, + {file = "ujson-4.0.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:568bb3e7f035006147af4ce3a9ced7d126c92e1a8607c7b2266007b1c1162c53"}, + {file = "ujson-4.0.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:bd4c77aee3ffb920e2dbc21a9e0c7945a400557ce671cfd57dbd569f5ebc619d"}, + {file = "ujson-4.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:c354c1617b0a4378b6279d0cd511b769500cf3fa7c42e8e004cbbbb6b4c2a875"}, + {file = "ujson-4.0.1-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:a5200a68f1dcf3ce275e1cefbcfa3914b70c2b5e2f71c2e31556aa1f7244c845"}, + {file = "ujson-4.0.1-cp39-cp39-manylinux1_i686.whl", hash = "sha256:a618af22407baeadb3f046f81e7a5ee5e9f8b0b716d2b564f92276a54d26a823"}, + {file = "ujson-4.0.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:0a2e1b211714eb1ec0772a013ec9967f8f95f21c84e8f46382e9f8a32ae781fe"}, + {file = "ujson-4.0.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:2b2d9264ac76aeb11f590f7a1ccff0689ba1313adacbb6d38d3b15f21a392897"}, + {file = "ujson-4.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:f8a60928737a9a47e692fcd661ef2b5d75ba22c7c930025bd95e338f2a6e15bc"}, + {file = "ujson-4.0.1.tar.gz", hash = "sha256:26cf6241b36ff5ce4539ae687b6b02673109c5e3efc96148806a7873eaa229d3"}, +] diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000..10f302440b --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,43 @@ +[tool.poetry] +name = "pytest-lisa" +version = "0.1.0" +description = "LISA plugin for pytest" +authors = ["Andrew Schwartzmeyer "] +license = "MIT License" + +[tool.poetry.dependencies] +python = "^3.8" +pytest = "^6.1.1" +fabric = "^2.5.0" + +[tool.poetry.dev-dependencies] +black = "^20.8b1" +flake8 = "^3.8.4" +flake8-black = "^0.2.1" +flake8-bugbear = "^20.1.4" +flake8-isort = "^4.0.0" +isort = "^5.6.1" +mypy = "^0.782" +python-language-server = "^0.35.1" +pyls-black = "^0.4.6" +pyls-isort = "^0.2.0" +pyls-mypy = "^0.1.8" +rope = "^0.18.0" +pytest-flake8 = "^1.0.6" +pytest-mypy = "^0.7.0" + +[tool.black] +line-length = 88 +target-version = ['py38'] + +[tool.isort] +multi_line_output = 3 +include_trailing_comma = true +force_grid_wrap = 0 +use_parentheses = true +ensure_newline_before_comments = true +line_length = 88 + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000000..91a21cac60 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,4 @@ +[pytest] +filterwarnings = + error + ignore:the imp module is deprecated in favour of importlib:DeprecationWarning diff --git a/testsuites/test_lis.py b/testsuites/test_lis.py new file mode 100644 index 0000000000..7fe5c022d0 --- /dev/null +++ b/testsuites/test_lis.py @@ -0,0 +1,30 @@ +from pathlib import Path + +from fabric import Config, Connection # type: ignore + +import pytest + +LINUX_SCRIPTS = Path("../Testscripts/Linux") + + +# TODO: Make the hostname a parameter. +@pytest.fixture +def node() -> Connection: + config = Config(overrides={"run": {"in_stream": False}}) + with Connection("centos", config=config) as connection: + yield connection + + +def test_lis_version(node: Connection) -> None: + # TODO: Include “utils.sh” automatically? Or something... + for f in ["utils.sh", "LIS-VERSION-CHECK.sh"]: + node.put(LINUX_SCRIPTS / f) + node.run(f"chmod +x {f}") + node.sudo("yum install -y bc") + # TODO: Fix this PATH issue. + node.run( + "PATH=$PATH:/usr/local/sbin:/usr/sbin ./LIS-VERSION-CHECK.sh", + ) + node.get("state.txt") + with open("state.txt") as f: + assert f.readline().strip() == "TestCompleted" From b94555b59688ba387046c9a0b051eb3b799caa23 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Fri, 9 Oct 2020 23:28:13 -0700 Subject: [PATCH 003/194] Make Node fixture more reusable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extend Fabric’s Connection class with a new (and simple) command “cat” which return the value of a remote file as a string. Setup a Config for the Connection when creating it that echoes every command, disables the stdin forwarding (since we’re running under Pytest), and fixes the PATH since the remote commands don’t run under a login shell. --- mypy.ini | 2 +- testsuites/test_lis.py | 43 ++++++++++++++++++++++++++++++------------ 2 files changed, 32 insertions(+), 13 deletions(-) diff --git a/mypy.ini b/mypy.ini index b5a41dd5b5..85e42268e0 100644 --- a/mypy.ini +++ b/mypy.ini @@ -4,7 +4,7 @@ pretty = True warn_unused_configs = True disallow_any_generics = True -disallow_subclassing_any = True +disallow_subclassing_any = False disallow_untyped_calls = True disallow_untyped_defs = True disallow_incomplete_defs = True diff --git a/testsuites/test_lis.py b/testsuites/test_lis.py index 7fe5c022d0..f36423caa1 100644 --- a/testsuites/test_lis.py +++ b/testsuites/test_lis.py @@ -1,4 +1,6 @@ +from io import BytesIO from pathlib import Path +from typing import Iterator from fabric import Config, Connection # type: ignore @@ -7,24 +9,41 @@ LINUX_SCRIPTS = Path("../Testscripts/Linux") +class Node(Connection): + """Extends 'fabric.Connection' with our own utilities.""" + + def cat(self, path: str) -> str: + """Gets the value of a remote file without a temporary file.""" + with BytesIO() as buf: + self.get(path, buf) + return buf.getvalue().decode("utf-8").strip() + + # TODO: Make the hostname a parameter. @pytest.fixture -def node() -> Connection: - config = Config(overrides={"run": {"in_stream": False}}) - with Connection("centos", config=config) as connection: - yield connection +def node() -> Iterator[Node]: + """Yields a safe remote Node on which to run commands.""" + config = Config( + overrides={ + "run": { + # Show each command as its run. + "echo": True, + # Disable stdin forwarding. + "in_stream": False, + # Set PATH since it’s not a login shell. + "env": {"PATH": "$PATH:/usr/local/sbin:/usr/sbin"}, + } + } + ) + with Node("centos", config=config, inline_ssh_env=True) as n: + yield n -def test_lis_version(node: Connection) -> None: +def test_lis_driver_version(node: Node) -> None: # TODO: Include “utils.sh” automatically? Or something... for f in ["utils.sh", "LIS-VERSION-CHECK.sh"]: node.put(LINUX_SCRIPTS / f) node.run(f"chmod +x {f}") node.sudo("yum install -y bc") - # TODO: Fix this PATH issue. - node.run( - "PATH=$PATH:/usr/local/sbin:/usr/sbin ./LIS-VERSION-CHECK.sh", - ) - node.get("state.txt") - with open("state.txt") as f: - assert f.readline().strip() == "TestCompleted" + node.run("./LIS-VERSION-CHECK.sh") + assert node.cat("state.txt") == "TestCompleted" From 87273178e0c54a14a1802525a2945bf5fb3473d9 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Fri, 9 Oct 2020 23:48:37 -0700 Subject: [PATCH 004/194] Add 300 second timeout to all tests --- poetry.lock | 17 ++++++++++++++++- pyproject.toml | 1 + pytest.ini | 1 + 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index a0a87e8d83..82eabae938 100644 --- a/poetry.lock +++ b/poetry.lock @@ -472,6 +472,17 @@ filelock = ">=3.0" mypy = {version = ">=0.700", markers = "python_version >= \"3.8\""} pytest = ">=3.5" +[[package]] +name = "pytest-timeout" +version = "1.4.2" +description = "py.test plugin to abort hanging tests" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +pytest = ">=3.6.0" + [[package]] name = "python-jsonrpc-server" version = "0.4.0" @@ -587,7 +598,7 @@ python-versions = ">=3.6" [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "40a1da4f76e519932e1c86deaa81cb3331cc282c37ff78f39607936730cbb322" +content-hash = "307896057c574edcbf704e9060caabb3eb14b01c06ba841b8de5c7715ce86ecb" [metadata.files] appdirs = [ @@ -830,6 +841,10 @@ pytest-mypy = [ {file = "pytest-mypy-0.7.0.tar.gz", hash = "sha256:5a667d9a2b66bf98b3a494411f221923a6e2c3eafbe771104951aaec8985673d"}, {file = "pytest_mypy-0.7.0-py3-none-any.whl", hash = "sha256:e0505ace48d2b19fe686366fce6b4a2ac0d090423736bb6aa2e39554d18974b7"}, ] +pytest-timeout = [ + {file = "pytest-timeout-1.4.2.tar.gz", hash = "sha256:20b3113cf6e4e80ce2d403b6fb56e9e1b871b510259206d40ff8d609f48bda76"}, + {file = "pytest_timeout-1.4.2-py2.py3-none-any.whl", hash = "sha256:541d7aa19b9a6b4e475c759fd6073ef43d7cdc9a92d95644c260076eb257a063"}, +] python-jsonrpc-server = [ {file = "python-jsonrpc-server-0.4.0.tar.gz", hash = "sha256:62c543e541f101ec5b57dc654efc212d2c2e3ea47ff6f54b2e7dcb36ecf20595"}, {file = "python_jsonrpc_server-0.4.0-py3-none-any.whl", hash = "sha256:e5a908ff182e620aac07db5f57887eeb0afe33993008f57dc1b85b594cea250c"}, diff --git a/pyproject.toml b/pyproject.toml index 10f302440b..c083e43bef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,6 +9,7 @@ license = "MIT License" python = "^3.8" pytest = "^6.1.1" fabric = "^2.5.0" +pytest-timeout = "^1.4.2" [tool.poetry.dev-dependencies] black = "^20.8b1" diff --git a/pytest.ini b/pytest.ini index 91a21cac60..f3efd3b4f3 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,4 +1,5 @@ [pytest] +timeout = 300 filterwarnings = error ignore:the imp module is deprecated in favour of importlib:DeprecationWarning From c7564d189cb87112dc8a9baaec3aaa95678cc5a3 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Sun, 11 Oct 2020 20:21:10 -0700 Subject: [PATCH 005/194] Move node implementation to custom plugin --- conftest.py | 6 ++++++ node_plugin.py | 37 +++++++++++++++++++++++++++++++++++++ testsuites/test_lis.py | 37 ++----------------------------------- 3 files changed, 45 insertions(+), 35 deletions(-) create mode 100644 conftest.py create mode 100644 node_plugin.py diff --git a/conftest.py b/conftest.py new file mode 100644 index 0000000000..5f84e8864c --- /dev/null +++ b/conftest.py @@ -0,0 +1,6 @@ +"""This file sets up custom plugins. + +https://docs.pytest.org/en/stable/writing_plugins.html + +""" +pytest_plugins = "node_plugin" diff --git a/node_plugin.py b/node_plugin.py new file mode 100644 index 0000000000..509b7450d4 --- /dev/null +++ b/node_plugin.py @@ -0,0 +1,37 @@ +"""Pytest plugin implementing a Node fixture for running remote commands.""" +from io import BytesIO +from typing import Iterator + +from fabric import Config, Connection # type: ignore + +import pytest + + +class Node(Connection): + """Extends 'fabric.Connection' with our own utilities.""" + + def cat(self, path: str) -> str: + """Gets the value of a remote file without a temporary file.""" + with BytesIO() as buf: + self.get(path, buf) + return buf.getvalue().decode("utf-8").strip() + + +# TODO: Make the hostname a parameter. +@pytest.fixture +def node() -> Iterator[Node]: + """Yields a safe remote Node on which to run commands.""" + config = Config( + overrides={ + "run": { + # Show each command as its run. + "echo": True, + # Disable stdin forwarding. + "in_stream": False, + # Set PATH since it’s not a login shell. + "env": {"PATH": "$PATH:/usr/local/sbin:/usr/sbin"}, + } + } + ) + with Node("centos", config=config, inline_ssh_env=True) as n: + yield n diff --git a/testsuites/test_lis.py b/testsuites/test_lis.py index f36423caa1..a2eeba74a2 100644 --- a/testsuites/test_lis.py +++ b/testsuites/test_lis.py @@ -1,44 +1,11 @@ -from io import BytesIO +"""Runs 'LIS-Tests.xml' using Pytest.""" from pathlib import Path -from typing import Iterator -from fabric import Config, Connection # type: ignore - -import pytest +from node_plugin import Node LINUX_SCRIPTS = Path("../Testscripts/Linux") -class Node(Connection): - """Extends 'fabric.Connection' with our own utilities.""" - - def cat(self, path: str) -> str: - """Gets the value of a remote file without a temporary file.""" - with BytesIO() as buf: - self.get(path, buf) - return buf.getvalue().decode("utf-8").strip() - - -# TODO: Make the hostname a parameter. -@pytest.fixture -def node() -> Iterator[Node]: - """Yields a safe remote Node on which to run commands.""" - config = Config( - overrides={ - "run": { - # Show each command as its run. - "echo": True, - # Disable stdin forwarding. - "in_stream": False, - # Set PATH since it’s not a login shell. - "env": {"PATH": "$PATH:/usr/local/sbin:/usr/sbin"}, - } - } - ) - with Node("centos", config=config, inline_ssh_env=True) as n: - yield n - - def test_lis_driver_version(node: Node) -> None: # TODO: Include “utils.sh” automatically? Or something... for f in ["utils.sh", "LIS-VERSION-CHECK.sh"]: From 9246b0ea61f7b3dbdd71f242878f95bd431e1f7f Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Sun, 11 Oct 2020 20:21:20 -0700 Subject: [PATCH 006/194] Shorten tracebacks --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 5c052c4707..a30b69a904 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ setup: # Run Pytest run: - @poetry run python -X dev -m pytest --flake8 --mypy -rA + @poetry run python -X dev -X tracemalloc -m pytest --flake8 --mypy -rA --tb=short # Print current Python virtualenv venv: From 5204a1c2f3bfc4d1ef07f781984d8da52c5a930a Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Sun, 11 Oct 2020 21:37:33 -0700 Subject: [PATCH 007/194] Demo connecting to host based on mark MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This could be extended to instead deploy a host of the specified distro. Most likely we’ll want a command-line parameter that the fixture uses to create a Node with the given requirements, and then tests will be skipped if their requirements aren’t met. Further more, the mark here is very simple. It can instead take keyword arguments, which would map to our metadata. --- node_plugin.py | 11 ++++++++--- pytest.ini | 2 ++ testsuites/test_lis.py | 2 ++ 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/node_plugin.py b/node_plugin.py index 509b7450d4..4fb2d17eca 100644 --- a/node_plugin.py +++ b/node_plugin.py @@ -2,6 +2,7 @@ from io import BytesIO from typing import Iterator +import _pytest from fabric import Config, Connection # type: ignore import pytest @@ -17,9 +18,8 @@ def cat(self, path: str) -> str: return buf.getvalue().decode("utf-8").strip() -# TODO: Make the hostname a parameter. @pytest.fixture -def node() -> Iterator[Node]: +def node(request: _pytest.fixtures.FixtureRequest) -> Iterator[Node]: """Yields a safe remote Node on which to run commands.""" config = Config( overrides={ @@ -33,5 +33,10 @@ def node() -> Iterator[Node]: } } ) - with Node("centos", config=config, inline_ssh_env=True) as n: + # Get the host from the test’s marker. + host = "localhost" + marker = request.node.get_closest_marker("host") + if marker is not None: + host = marker.args[0] + with Node(host, config=config, inline_ssh_env=True) as n: yield n diff --git a/pytest.ini b/pytest.ini index f3efd3b4f3..68d0e9d973 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,4 +1,6 @@ [pytest] +markers = + host timeout = 300 filterwarnings = error diff --git a/testsuites/test_lis.py b/testsuites/test_lis.py index a2eeba74a2..b5c6f29204 100644 --- a/testsuites/test_lis.py +++ b/testsuites/test_lis.py @@ -1,11 +1,13 @@ """Runs 'LIS-Tests.xml' using Pytest.""" from pathlib import Path +import pytest from node_plugin import Node LINUX_SCRIPTS = Path("../Testscripts/Linux") +@pytest.mark.host("centos") # type: ignore def test_lis_driver_version(node: Node) -> None: # TODO: Include “utils.sh” automatically? Or something... for f in ["utils.sh", "LIS-VERSION-CHECK.sh"]: From 790f224d06f4689384bcbd90ff09040ccb87ab02 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Sun, 11 Oct 2020 21:46:00 -0700 Subject: [PATCH 008/194] Add considered early alternatives --- README.md | 55 ++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 52 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 38c1ae798b..c538711d39 100644 --- a/README.md +++ b/README.md @@ -79,9 +79,7 @@ Other test specific requirements, such as installing software and daemons or downloading files from remote storage, would similarly be implemented via fixtures and shared among tests. -Note that Paramiko is less complex (smaller library footprint) than Fabric, but -is a bit more difficult to use, and doesn’t support reading existing SSH config -files, nor does it support “ProxyJump” which we use heavily. +### Alternatives considered ## pytest-xdist @@ -103,3 +101,54 @@ create those nodes. However, this is only one approach, and we may prefer to run the Python code on the user’s machine, with pytest-lisa instead providing the previously mentioned node fixtures, default marks, and requirements logic. + +## Paramiko instead of Fabric + +The Paramiko library is less complex (smaller library footprint) than Fabric, as +the latter wraps the former, but it is a bit more difficult to use, and doesn’t +support reading existing SSH config files, nor does it support “ProxyJump” which +we use heavily. Fabric instead provides a clean high-level interface for +existing shell commands, handling all the connection abstractions for us. + +It looked a like this: + +```python +from pathlib import Path +from typing import List + +from paramiko import SSHClient + +import pytest + +@pytest.fixture +def node() -> SSHClient: + with SSHClient() as client: + client.load_system_host_keys() + client.connect(hostname="...") + yield client + + +def test_lis_version(node: SSHClient) -> None: + with node.open_sftp() as sftp: + for f in ["utils.sh", "LIS-VERSION-CHECK.sh"]: + sftp.put(LINUX_SCRIPTS / f, f) + _, stdout, stderr = node.exec_command("./LIS-VERSION-CHECK.sh") + sftp.get("state.txt", "state.txt") + with Path("state.txt").open as f: + assert f.readline() == "TestCompleted" +``` +## StringIO + +For `Node.cat()` it would seem we could use `StringIO` like so: + +```python +from io import StringIO + +with StringIO() as result: + node.get("state.txt", result) + assert result.getvalue().strip() == "TestCompleted" +``` + +However, the data returned by Paramiko is in bytes, which in Python 3 are not +equivalent to strings, hence the existing implementation which uses `BytesIO` +and decodes the bytes to a string. From 9f31ce1f69f16956535be95243b561208a8fe901 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Mon, 12 Oct 2020 11:56:42 -0700 Subject: [PATCH 009/194] Allow untyped decorators As supplying types for these would be supremely annoying. --- mypy.ini | 2 +- testsuites/test_lis.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mypy.ini b/mypy.ini index 85e42268e0..6b513808fb 100644 --- a/mypy.ini +++ b/mypy.ini @@ -9,7 +9,7 @@ disallow_untyped_calls = True disallow_untyped_defs = True disallow_incomplete_defs = True check_untyped_defs = True -disallow_untyped_decorators = True +disallow_untyped_decorators = False no_implicit_optional = True warn_redundant_casts = True warn_unused_ignores = True diff --git a/testsuites/test_lis.py b/testsuites/test_lis.py index b5c6f29204..bd1bf0d598 100644 --- a/testsuites/test_lis.py +++ b/testsuites/test_lis.py @@ -7,7 +7,7 @@ LINUX_SCRIPTS = Path("../Testscripts/Linux") -@pytest.mark.host("centos") # type: ignore +@pytest.mark.host("centos") def test_lis_driver_version(node: Node) -> None: # TODO: Include “utils.sh” automatically? Or something... for f in ["utils.sh", "LIS-VERSION-CHECK.sh"]: From fe596b28a2cfd981d176835cf4288248c1389b95 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Mon, 12 Oct 2020 13:22:45 -0700 Subject: [PATCH 010/194] Setup markers for generation from XML --- conftest.py | 4 ++++ node_plugin.py | 3 ++- pytest.ini | 4 +++- testsuites/test_lis.py | 13 +++++++------ testsuites/test_xdp.py | 27 +++++++++++++++++++++++++++ 5 files changed, 43 insertions(+), 8 deletions(-) create mode 100644 testsuites/test_xdp.py diff --git a/conftest.py b/conftest.py index 5f84e8864c..fb5ebaeec6 100644 --- a/conftest.py +++ b/conftest.py @@ -3,4 +3,8 @@ https://docs.pytest.org/en/stable/writing_plugins.html """ +from pathlib import Path + pytest_plugins = "node_plugin" + +LINUX_SCRIPTS = Path("../Testscripts/Linux") diff --git a/node_plugin.py b/node_plugin.py index 4fb2d17eca..888af4358c 100644 --- a/node_plugin.py +++ b/node_plugin.py @@ -21,6 +21,7 @@ def cat(self, path: str) -> str: @pytest.fixture def node(request: _pytest.fixtures.FixtureRequest) -> Iterator[Node]: """Yields a safe remote Node on which to run commands.""" + # TODO: If test has ‘deploy’ marker, do so. config = Config( overrides={ "run": { @@ -35,7 +36,7 @@ def node(request: _pytest.fixtures.FixtureRequest) -> Iterator[Node]: ) # Get the host from the test’s marker. host = "localhost" - marker = request.node.get_closest_marker("host") + marker = request.node.get_closest_marker("connect") if marker is not None: host = marker.args[0] with Node(host, config=config, inline_ssh_env=True) as n: diff --git a/pytest.ini b/pytest.ini index 68d0e9d973..ca01a9b06f 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,6 +1,8 @@ [pytest] markers = - host + lisa + deploy + connect timeout = 300 filterwarnings = error diff --git a/testsuites/test_lis.py b/testsuites/test_lis.py index bd1bf0d598..bb050d7d0f 100644 --- a/testsuites/test_lis.py +++ b/testsuites/test_lis.py @@ -1,17 +1,18 @@ """Runs 'LIS-Tests.xml' using Pytest.""" -from pathlib import Path - +import conftest import pytest from node_plugin import Node -LINUX_SCRIPTS = Path("../Testscripts/Linux") - -@pytest.mark.host("centos") +@pytest.mark.lisa( + platform="Azure", category="Functional", area="LIS_DEPLOY", tags=["lis"], priority=0 +) +# @pytest.mark.deploy(setup="OneVM") +@pytest.mark.connect("centos") def test_lis_driver_version(node: Node) -> None: # TODO: Include “utils.sh” automatically? Or something... for f in ["utils.sh", "LIS-VERSION-CHECK.sh"]: - node.put(LINUX_SCRIPTS / f) + node.put(conftest.LINUX_SCRIPTS / f) node.run(f"chmod +x {f}") node.sudo("yum install -y bc") node.run("./LIS-VERSION-CHECK.sh") diff --git a/testsuites/test_xdp.py b/testsuites/test_xdp.py new file mode 100644 index 0000000000..cad6d97b76 --- /dev/null +++ b/testsuites/test_xdp.py @@ -0,0 +1,27 @@ +"""Runs 'FunctionalTests-XDP.xml' using Pytest.""" + + +import conftest +import pytest +from node_plugin import Node + + +@pytest.mark.lisa( + platform="Azure", + category="Functional", + area="XDP", + tags=["xdp", "network", "hv_netvsc", "sriov"], + priority=0, +) +@pytest.mark.deploy(setup="OneVM2NIC", networking="SRIOV", vm_size="Standard_DS4_v2") +@pytest.mark.skip(reason="Not Implemented") +def test_verify_xdp_compliance(node: Node) -> None: + for f in [ + "xdpdumpsetup.sh", + "xdputils.sh", + "utils.sh", + "enable_passwordless_root.sh", + "enable_root.sh", + ]: + node.put(conftest.LINUX_SCRIPTS / f) + node.run(f"chmod +x {f}") From 3a1ea7fd3e526fb84657724fc1c27ddb9a4c1cef Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Mon, 12 Oct 2020 13:26:35 -0700 Subject: [PATCH 011/194] Add libvirt note --- README.md | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index c538711d39..90b7052530 100644 --- a/README.md +++ b/README.md @@ -69,15 +69,14 @@ implement (with more as found to be required): Our abstraction would leverage [Fabric](https://docs.fabfile.org/en/stable/index.html), which uses [paramiko](https://docs.paramiko.org/en/stable/) underneath, directly to -implement the SSH commands, and it would use existing modules to deploy -[Azure](https://aka.ms/azsdk/python/all) and AWS nodes. We would need implement -specific logic for Hyper-V and similar platforms where APIs do not currently -exist, and this would be the bulk of our work instead of rewriting a unit test -framework. - -Other test specific requirements, such as installing software and daemons or -downloading files from remote storage, would similarly be implemented via -fixtures and shared among tests. +implement the SSH commands. For deployment logic, it would use existing Python APIs to deploy +[Azure](https://aka.ms/azsdk/python/all) nodes, and for Hyper-V (and other +virtualization platforms), it would use +[libvirt](https://libvirt.org/python.html). + +Other test specific requirements, such as installing software and daemons, +downloading files from remote storage, or checking the state of our Bash test +scripts, would similarly be implemented via fixtures and shared among tests. ### Alternatives considered From d1688f519eab3b65f897e81b1452b6aa36518f22 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Mon, 12 Oct 2020 13:36:18 -0700 Subject: [PATCH 012/194] Add pytest-azurepipelines note --- README.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 90b7052530..5dc023742b 100644 --- a/README.md +++ b/README.md @@ -78,9 +78,17 @@ Other test specific requirements, such as installing software and daemons, downloading files from remote storage, or checking the state of our Bash test scripts, would similarly be implemented via fixtures and shared among tests. -### Alternatives considered +### Test result output -## pytest-xdist +Instead of writing our own test result output, we can leverage existing plugins. +For instance, there already exists +[pytest-azurepipelines](https://pypi.org/project/pytest-azurepipelines/) which +transforms results into the format consumed by ADO. It has over 90,000 downloads +a month. We don’t need to rewrite this. + +## Alternatives considered + +### pytest-xdist With the [pytest-xdist plugin](https://github.com/pytest-dev/pytest-xdist) there already exists support for running a folder of tests on an arbitrary remote host From b6f4e91bb04ddd1c76b71a126da87470b73888a9 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Mon, 12 Oct 2020 13:36:46 -0700 Subject: [PATCH 013/194] Add skeleton to parse deploy marker --- node_plugin.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/node_plugin.py b/node_plugin.py index 888af4358c..e493ff793e 100644 --- a/node_plugin.py +++ b/node_plugin.py @@ -21,7 +21,21 @@ def cat(self, path: str) -> str: @pytest.fixture def node(request: _pytest.fixtures.FixtureRequest) -> Iterator[Node]: """Yields a safe remote Node on which to run commands.""" - # TODO: If test has ‘deploy’ marker, do so. + # TODO: The deploy and connect markers should be mutually + # exclusive. + host = "localhost" + + # Deploy a node. + deploy_marker = request.node.get_closest_marker("deploy") + if deploy_marker: + pass + + # Get the host from the test’s marker. + connect_marker = request.node.get_closest_marker("connect") + if connect_marker: + host = connect_marker.args[0] + + # Yield the configured Node connection. config = Config( overrides={ "run": { @@ -34,10 +48,5 @@ def node(request: _pytest.fixtures.FixtureRequest) -> Iterator[Node]: } } ) - # Get the host from the test’s marker. - host = "localhost" - marker = request.node.get_closest_marker("connect") - if marker is not None: - host = marker.args[0] with Node(host, config=config, inline_ssh_env=True) as n: yield n From 2ffda5f4d792ac09faa6c40a3d8dd5d022484001 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Mon, 12 Oct 2020 16:10:02 -0700 Subject: [PATCH 014/194] Create and delete VM resource --- node_plugin.py | 61 +++++++++++++++++++++++++++++++++++++++++- testsuites/test_xdp.py | 18 ++++++++++--- 2 files changed, 74 insertions(+), 5 deletions(-) diff --git a/node_plugin.py b/node_plugin.py index e493ff793e..7e3ae9499a 100644 --- a/node_plugin.py +++ b/node_plugin.py @@ -1,13 +1,66 @@ """Pytest plugin implementing a Node fixture for running remote commands.""" +import json from io import BytesIO from typing import Iterator +from uuid import uuid4 import _pytest +import invoke # type: ignore from fabric import Config, Connection # type: ignore +from invoke.runners import Result # type: ignore import pytest +def install_az_cli() -> None: + if not invoke.run("which az", warn=True, echo=False, in_stream=False): + # TODO: Use Invoke for pipes. + invoke.run( + "curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash", + echo=True, + in_stream=False, + ) + # TODO: Login with service principal (az login) and set + # default subscription (az account set -s) using secrets. + + +def deploy_vm( + name: str, + location="westus2", + vm_image="UbuntuLTS", + vm_size="Standard_DS1_v2", + setup="", + networking="", +) -> str: + install_az_cli() + invoke.run( + f"az group create --name {name}-rg --location {location}", + echo=True, + in_stream=False, + ) + vm_command = [ + "az vm create", + f"--resource-group {name}-rg", + f"--name {name}", + f"--image {vm_image}", + f"--size {vm_size}", + "--generate-ssh-keys", + ] + if networking == "SRIOV": + vm_command.append("--accelerated-networking true") + vm_result: Result = invoke.run( + " ".join(vm_command), + echo=True, + in_stream=False, + ) + vm_data = json.loads(vm_result.stdout) + return vm_data["publicIpAddress"] + + +def delete_vm(name: str) -> None: + invoke.run(f"az group delete --name {name}-rg --yes", echo=True) + + class Node(Connection): """Extends 'fabric.Connection' with our own utilities.""" @@ -18,6 +71,7 @@ def cat(self, path: str) -> str: return buf.getvalue().decode("utf-8").strip() +# TODO: Scope this to a module. @pytest.fixture def node(request: _pytest.fixtures.FixtureRequest) -> Iterator[Node]: """Yields a safe remote Node on which to run commands.""" @@ -26,9 +80,10 @@ def node(request: _pytest.fixtures.FixtureRequest) -> Iterator[Node]: host = "localhost" # Deploy a node. + name = f"pytest-{uuid4()}" deploy_marker = request.node.get_closest_marker("deploy") if deploy_marker: - pass + host = deploy_vm(name, **deploy_marker.kwargs) # Get the host from the test’s marker. connect_marker = request.node.get_closest_marker("connect") @@ -48,5 +103,9 @@ def node(request: _pytest.fixtures.FixtureRequest) -> Iterator[Node]: } } ) + print(f"Host is {host}") with Node(host, config=config, inline_ssh_env=True) as n: yield n + # Clean up! + if deploy_marker: + delete_vm(name) diff --git a/testsuites/test_xdp.py b/testsuites/test_xdp.py index cad6d97b76..d386ff2bfc 100644 --- a/testsuites/test_xdp.py +++ b/testsuites/test_xdp.py @@ -13,15 +13,25 @@ tags=["xdp", "network", "hv_netvsc", "sriov"], priority=0, ) -@pytest.mark.deploy(setup="OneVM2NIC", networking="SRIOV", vm_size="Standard_DS4_v2") -@pytest.mark.skip(reason="Not Implemented") +@pytest.mark.deploy( + setup="OneVM2NIC", + networking="SRIOV", + vm_image="Canonical:0001-com-ubuntu-server-focal:20_04-lts:latest", + vm_size="Standard_DS4_v2", +) +@pytest.mark.skip(reason="Not Finished") def test_verify_xdp_compliance(node: Node) -> None: for f in [ - "xdpdumpsetup.sh", - "xdputils.sh", "utils.sh", + "XDPDumpSetup.sh", + "XDPUtils.sh", "enable_passwordless_root.sh", "enable_root.sh", ]: node.put(conftest.LINUX_SCRIPTS / f) node.run(f"chmod +x {f}") + node.run("./enable_root.sh") + node.run("./enable_passwordless_root.sh") + synth_interface = node.run("source XDPUtils.sh ; get_extra_synth_nic").stdout + node.run(f"./XDPDumpSetup.sh {node.internal_address} {synth_interface}") + assert node.cat("state.txt") == "TestCompleted" From f65940f6846bcd90bfc091abc145b217b8bd7c5b Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Wed, 14 Oct 2020 12:54:31 -0700 Subject: [PATCH 015/194] Move semantic analysis testing to separate target --- Makefile | 8 ++++++-- node_plugin.py | 14 +++++++------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index a30b69a904..3ca09385c5 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -all: setup run +all: setup test run # Install Python packages setup: @@ -6,7 +6,11 @@ setup: # Run Pytest run: - @poetry run python -X dev -X tracemalloc -m pytest --flake8 --mypy -rA --tb=short + @poetry run python -m pytest --setup-show -rA --tb=short + +# Run semantic analysis +test: + @poetry run python -X dev -X tracemalloc -m pytest --flake8 --mypy -m 'flake8 or mypy' # Print current Python virtualenv venv: diff --git a/node_plugin.py b/node_plugin.py index 7e3ae9499a..2900100567 100644 --- a/node_plugin.py +++ b/node_plugin.py @@ -1,7 +1,7 @@ """Pytest plugin implementing a Node fixture for running remote commands.""" import json from io import BytesIO -from typing import Iterator +from typing import Dict, Iterator from uuid import uuid4 import _pytest @@ -26,11 +26,11 @@ def install_az_cli() -> None: def deploy_vm( name: str, - location="westus2", - vm_image="UbuntuLTS", - vm_size="Standard_DS1_v2", - setup="", - networking="", + location: str = "westus2", + vm_image: str = "UbuntuLTS", + vm_size: str = "Standard_DS1_v2", + setup: str = "", + networking: str = "", ) -> str: install_az_cli() invoke.run( @@ -53,7 +53,7 @@ def deploy_vm( echo=True, in_stream=False, ) - vm_data = json.loads(vm_result.stdout) + vm_data: Dict[str, str] = json.loads(vm_result.stdout) return vm_data["publicIpAddress"] From 6f4ed2655a1be6b87eccfe5a371719d8059f837f Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Wed, 14 Oct 2020 15:20:12 -0700 Subject: [PATCH 016/194] Cache deployed VM --- node_plugin.py | 50 ++++++++++++++++++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 16 deletions(-) diff --git a/node_plugin.py b/node_plugin.py index 2900100567..1897299274 100644 --- a/node_plugin.py +++ b/node_plugin.py @@ -1,13 +1,12 @@ """Pytest plugin implementing a Node fixture for running remote commands.""" import json from io import BytesIO -from typing import Dict, Iterator +from typing import Dict, Iterator, Optional, Tuple from uuid import uuid4 import _pytest import invoke # type: ignore from fabric import Config, Connection # type: ignore -from invoke.runners import Result # type: ignore import pytest @@ -25,19 +24,32 @@ def install_az_cli() -> None: def deploy_vm( - name: str, + request: _pytest.fixtures.FixtureRequest, location: str = "westus2", vm_image: str = "UbuntuLTS", vm_size: str = "Standard_DS1_v2", setup: str = "", networking: str = "", -) -> str: +) -> Tuple[str, Dict[str, str]]: + + key = f"{location}/{vm_image}/{vm_size}" + name: Optional[str] = request.config.cache.get(key, None) + if name: + result: Dict[str, str] = request.config.cache.get(name, {}) + assert result, "There was a cache problem, use --cache-clear and try again." + return name, result + + name = f"pytest-{uuid4()}" + request.config.cache.set(key, name) + install_az_cli() + invoke.run( f"az group create --name {name}-rg --location {location}", echo=True, in_stream=False, ) + vm_command = [ "az vm create", f"--resource-group {name}-rg", @@ -48,17 +60,20 @@ def deploy_vm( ] if networking == "SRIOV": vm_command.append("--accelerated-networking true") - vm_result: Result = invoke.run( - " ".join(vm_command), - echo=True, - in_stream=False, + + result: Dict[str, str] = json.loads( + invoke.run( + " ".join(vm_command), + echo=True, + in_stream=False, + ).stdout ) - vm_data: Dict[str, str] = json.loads(vm_result.stdout) - return vm_data["publicIpAddress"] + request.config.cache.set(name, result) + return name, result def delete_vm(name: str) -> None: - invoke.run(f"az group delete --name {name}-rg --yes", echo=True) + invoke.run(f"az group delete --name {name}-rg --yes", echo=True, in_stream=False) class Node(Connection): @@ -71,23 +86,24 @@ def cat(self, path: str) -> str: return buf.getvalue().decode("utf-8").strip() -# TODO: Scope this to a module. @pytest.fixture def node(request: _pytest.fixtures.FixtureRequest) -> Iterator[Node]: """Yields a safe remote Node on which to run commands.""" + # TODO: The deploy and connect markers should be mutually # exclusive. host = "localhost" # Deploy a node. - name = f"pytest-{uuid4()}" deploy_marker = request.node.get_closest_marker("deploy") if deploy_marker: - host = deploy_vm(name, **deploy_marker.kwargs) + name, result = deploy_vm(request, **deploy_marker.kwargs) + host = result["publicIpAddress"] # Get the host from the test’s marker. connect_marker = request.node.get_closest_marker("connect") if connect_marker: + name = "local" host = connect_marker.args[0] # Yield the configured Node connection. @@ -103,9 +119,11 @@ def node(request: _pytest.fixtures.FixtureRequest) -> Iterator[Node]: } } ) - print(f"Host is {host}") + with Node(host, config=config, inline_ssh_env=True) as n: yield n + # Clean up! - if deploy_marker: + # TODO: This logic is wrong. + if request.config.getoption("cacheclear") and name: delete_vm(name) From 3f52e5c3570605d701061562fac095858176ec68 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Wed, 14 Oct 2020 17:04:52 -0700 Subject: [PATCH 017/194] Enable boot diagnostics when creating a VM --- node_plugin.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/node_plugin.py b/node_plugin.py index 1897299274..50c1d96445 100644 --- a/node_plugin.py +++ b/node_plugin.py @@ -45,18 +45,21 @@ def deploy_vm( install_az_cli() invoke.run( - f"az group create --name {name}-rg --location {location}", + f"az group create -n {name}-rg --location {location}", echo=True, in_stream=False, ) vm_command = [ "az vm create", - f"--resource-group {name}-rg", - f"--name {name}", + f"-g {name}-rg", + f"-n {name}", f"--image {vm_image}", f"--size {vm_size}", "--generate-ssh-keys", + # TODO: Create unique boot diagnostics storage account. + # `az storage account create -g {name}-rg -n pytestbootdiag` + f"--boot-diagnostics-storage pytestbootdiag", ] if networking == "SRIOV": vm_command.append("--accelerated-networking true") @@ -73,7 +76,7 @@ def deploy_vm( def delete_vm(name: str) -> None: - invoke.run(f"az group delete --name {name}-rg --yes", echo=True, in_stream=False) + invoke.run(f"az group delete -n {name}-rg --yes", echo=True, in_stream=False) class Node(Connection): From 6b4d8f00e7602e53cf04aaa8ba6070dbf37e2f18 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Wed, 14 Oct 2020 17:05:25 -0700 Subject: [PATCH 018/194] Include /usr/bin etc. in remote path --- node_plugin.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/node_plugin.py b/node_plugin.py index 50c1d96445..66871d37a5 100644 --- a/node_plugin.py +++ b/node_plugin.py @@ -118,7 +118,9 @@ def node(request: _pytest.fixtures.FixtureRequest) -> Iterator[Node]: # Disable stdin forwarding. "in_stream": False, # Set PATH since it’s not a login shell. - "env": {"PATH": "$PATH:/usr/local/sbin:/usr/sbin"}, + "env": { + "PATH": "/sbin:/usr/sbin:/usr/local/sbin:/bin:/usr/bin:/usr/local/bin" + }, } } ) From 779c617bc3dfa01290a6f43e1cd1094ac3c0f0be Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Wed, 14 Oct 2020 17:05:54 -0700 Subject: [PATCH 019/194] Add node functions to restart and get boot diagnostics --- node_plugin.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/node_plugin.py b/node_plugin.py index 66871d37a5..49901aa0d8 100644 --- a/node_plugin.py +++ b/node_plugin.py @@ -82,6 +82,18 @@ def delete_vm(name: str) -> None: class Node(Connection): """Extends 'fabric.Connection' with our own utilities.""" + name: str + + def get_boot_diagnostics(self): + """Gets the serial console logs.""" + return self.local( + f"az vm boot-diagnostics get-boot-log -n {self.name} -g {self.name}-rg" + ) + + def platform_restart(self): + """TODO: Should this '--force' and redeploy?""" + return self.local(f"az vm restart -n {self.name} -g {self.name}-rg") + def cat(self, path: str) -> str: """Gets the value of a remote file without a temporary file.""" with BytesIO() as buf: @@ -126,6 +138,7 @@ def node(request: _pytest.fixtures.FixtureRequest) -> Iterator[Node]: ) with Node(host, config=config, inline_ssh_env=True) as n: + n.name = name yield n # Clean up! From 7f8aa960d82974ce917ab29eda7a8b69a9fa3f4a Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Wed, 14 Oct 2020 17:06:11 -0700 Subject: [PATCH 020/194] Add a basic smoke test --- Makefile | 3 +++ testsuites/test_smoke.py | 51 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) create mode 100644 testsuites/test_smoke.py diff --git a/Makefile b/Makefile index 3ca09385c5..fd931c8170 100644 --- a/Makefile +++ b/Makefile @@ -12,6 +12,9 @@ run: test: @poetry run python -X dev -X tracemalloc -m pytest --flake8 --mypy -m 'flake8 or mypy' +smoke: + @poetry run python -m pytest -rA -k smoke + # Print current Python virtualenv venv: @poetry env list --no-ansi --full-path diff --git a/testsuites/test_smoke.py b/testsuites/test_smoke.py new file mode 100644 index 0000000000..cbc8dfac82 --- /dev/null +++ b/testsuites/test_smoke.py @@ -0,0 +1,51 @@ +"""Runs a 'smoke' test for an Azure Linux VM deployment.""" +import socket + +from invoke.runners import Result # type: ignore +from paramiko import SSHException + +import pytest +from node_plugin import Node + + +@pytest.mark.deploy(setup="OneVM", vm_size="Standard_DS2_v2") +def test_smoke(node: Node) -> None: + """Check that a VM can be deployed and is responsive. + + 1. Deploy the VM (via 'node' fixture) and log it. + 2. Ping the VM. + 3. Connect to the VM via SSH. + 4. Attempt to reboot via SSH, otherwise use the platform. + 5. Fetch the serial console logs. + + For commands where we expect a possible non-zero exit code, we + pass 'warn=True' to prevent it from throwing 'UnexpectedExit' and + we instead check its result at the end. + + SSH failures DO NOT fail this test. + TODO: Log warnings instead of printing. + """ + # TODO: Can’t ping by default, need to enable. + ping1_result: Result = node.local(f"ping {node.host} -c 1", warn=True) + + try: + node.run("uptime") # If SSH fails, we catch it. + reboot_result: Result = node.sudo("reboot", warn=True) # Expect -1 + except (TimeoutError, SSHException, socket.error) as e: + print(f"SSH failed '{e}', using platform to reboot...") + node.platform_restart() + + # Try pinging and SSH again. + ping2_result: Result = node.local(f"ping {node.host} -c 1", warn=True) + + try: + node.run("uptime") + except (TimeoutError, SSHException, socket.error) as e: + print(f"SSH failed '{e}' after the reboot.") + + # Always download the serial console logs. + node.get_boot_diagnostics() + + assert ping1_result.ok + assert reboot_result.exited == -1, "Reboot failed, used platform instead" + assert ping2_result.ok From 0fba2a8b53a647d5a5359160708afa091dc2c747 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Wed, 14 Oct 2020 18:59:04 -0700 Subject: [PATCH 021/194] Set default node command timeout to 1 minute --- node_plugin.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/node_plugin.py b/node_plugin.py index 49901aa0d8..64a179f067 100644 --- a/node_plugin.py +++ b/node_plugin.py @@ -133,6 +133,9 @@ def node(request: _pytest.fixtures.FixtureRequest) -> Iterator[Node]: "env": { "PATH": "/sbin:/usr/sbin:/usr/local/sbin:/bin:/usr/bin:/usr/local/bin" }, + # Don’t let remote commands take longer than a minute + # (unless later overridden). + "timeout": 60, } } ) From aadd9f6b651e1a4e4bcfa7bc2c90d7e0094b2551 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Wed, 14 Oct 2020 19:15:59 -0700 Subject: [PATCH 022/194] Clean up Invoke configuration --- node_plugin.py | 61 +++++++++++++++++++++++++------------------------- 1 file changed, 30 insertions(+), 31 deletions(-) diff --git a/node_plugin.py b/node_plugin.py index 64a179f067..28379ba89f 100644 --- a/node_plugin.py +++ b/node_plugin.py @@ -5,19 +5,40 @@ from uuid import uuid4 import _pytest -import invoke # type: ignore -from fabric import Config, Connection # type: ignore +from fabric import Connection # type: ignore +from invoke import Config, Context # type: ignore import pytest +# Setup a sane configuration for local and remote commands. +config = Config( + overrides={ + "run": { + # Show each command as its run. + "echo": True, + # Disable stdin forwarding. + "in_stream": False, + # Set PATH since it’s not a login shell. + "env": { + "PATH": "/sbin:/usr/sbin:/usr/local/sbin:/bin:/usr/bin:/usr/local/bin" + }, + # Don’t let remote commands take longer than a minute + # (unless later overridden). + "timeout": 60, + } + } +) + +# Provide a configured local Invoke context for running commands +# before establishing a connection. (Use like `local.run(...)`). +local = Context(config=config) + def install_az_cli() -> None: - if not invoke.run("which az", warn=True, echo=False, in_stream=False): + if not local.run("which az", warn=True, echo=False): # TODO: Use Invoke for pipes. - invoke.run( + local.run( "curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash", - echo=True, - in_stream=False, ) # TODO: Login with service principal (az login) and set # default subscription (az account set -s) using secrets. @@ -44,10 +65,8 @@ def deploy_vm( install_az_cli() - invoke.run( + local.run( f"az group create -n {name}-rg --location {location}", - echo=True, - in_stream=False, ) vm_command = [ @@ -65,10 +84,8 @@ def deploy_vm( vm_command.append("--accelerated-networking true") result: Dict[str, str] = json.loads( - invoke.run( + local.run( " ".join(vm_command), - echo=True, - in_stream=False, ).stdout ) request.config.cache.set(name, result) @@ -76,7 +93,7 @@ def deploy_vm( def delete_vm(name: str) -> None: - invoke.run(f"az group delete -n {name}-rg --yes", echo=True, in_stream=False) + local.run(f"az group delete -n {name}-rg --yes") class Node(Connection): @@ -122,24 +139,6 @@ def node(request: _pytest.fixtures.FixtureRequest) -> Iterator[Node]: host = connect_marker.args[0] # Yield the configured Node connection. - config = Config( - overrides={ - "run": { - # Show each command as its run. - "echo": True, - # Disable stdin forwarding. - "in_stream": False, - # Set PATH since it’s not a login shell. - "env": { - "PATH": "/sbin:/usr/sbin:/usr/local/sbin:/bin:/usr/bin:/usr/local/bin" - }, - # Don’t let remote commands take longer than a minute - # (unless later overridden). - "timeout": 60, - } - } - ) - with Node(host, config=config, inline_ssh_env=True) as n: n.name = name yield n From 9bd0d709b0fc421af8ef86e7c4a86ed60a7f956e Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Wed, 14 Oct 2020 19:24:59 -0700 Subject: [PATCH 023/194] Create boot storage account and resource group automatically --- node_plugin.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/node_plugin.py b/node_plugin.py index 28379ba89f..112b9ec2cd 100644 --- a/node_plugin.py +++ b/node_plugin.py @@ -44,6 +44,17 @@ def install_az_cli() -> None: # default subscription (az account set -s) using secrets. +def create_boot_storage(location: str) -> str: + """Create a separate resource group and storage account for boot diagnostics.""" + account = "pytestbootdiag" + # This command always exits with 0 but returns a string. + if local.run("az group exists -n pytest-lisa").stdout.strip() == "false": + local.run(f"az group create -n pytest-lisa --location {location}") + if not local.run(f"az storage account show -g pytest-lisa -n {account}", warn=True): + local.run(f"az storage account create -g pytest-lisa -n {account}") + return account + + def deploy_vm( request: _pytest.fixtures.FixtureRequest, location: str = "westus2", @@ -64,6 +75,7 @@ def deploy_vm( request.config.cache.set(key, name) install_az_cli() + boot_storage = create_boot_storage(location) local.run( f"az group create -n {name}-rg --location {location}", @@ -75,10 +87,8 @@ def deploy_vm( f"-n {name}", f"--image {vm_image}", f"--size {vm_size}", + f"--boot-diagnostics-storage {boot_storage}", "--generate-ssh-keys", - # TODO: Create unique boot diagnostics storage account. - # `az storage account create -g {name}-rg -n pytestbootdiag` - f"--boot-diagnostics-storage pytestbootdiag", ] if networking == "SRIOV": vm_command.append("--accelerated-networking true") From c39badb36e97864d1fcda25951d3d9fa4710a644 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Wed, 14 Oct 2020 19:59:49 -0700 Subject: [PATCH 024/194] Check that az cli is logged in with a default subscription --- node_plugin.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/node_plugin.py b/node_plugin.py index 112b9ec2cd..8573f47488 100644 --- a/node_plugin.py +++ b/node_plugin.py @@ -7,6 +7,7 @@ import _pytest from fabric import Connection # type: ignore from invoke import Config, Context # type: ignore +from invoke.runners import Result # type: ignore import pytest @@ -34,14 +35,19 @@ local = Context(config=config) -def install_az_cli() -> None: - if not local.run("which az", warn=True, echo=False): +def check_az_cli() -> None: + if not local.run("which az", warn=True): # TODO: Use Invoke for pipes. local.run( "curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash", ) - # TODO: Login with service principal (az login) and set - # default subscription (az account set -s) using secrets. + + # TODO: Login with service principal (az login) and set + # default subscription (az account set -s) using secrets. + account: Result = local.run("az account show") + assert account.ok, "Please `az login`!" + subs = json.loads(account.stdout) + assert subs["isDefault"], "Please `az account set -s `!" def create_boot_storage(location: str) -> str: @@ -74,7 +80,7 @@ def deploy_vm( name = f"pytest-{uuid4()}" request.config.cache.set(key, name) - install_az_cli() + check_az_cli() boot_storage = create_boot_storage(location) local.run( From bd363c3581ad498d09f3e7eff17046e1d838765e Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Wed, 14 Oct 2020 20:42:46 -0700 Subject: [PATCH 025/194] Set default node command timeout to 5 minutes instead --- node_plugin.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/node_plugin.py b/node_plugin.py index 8573f47488..ecc55df97a 100644 --- a/node_plugin.py +++ b/node_plugin.py @@ -23,9 +23,9 @@ "env": { "PATH": "/sbin:/usr/sbin:/usr/local/sbin:/bin:/usr/bin:/usr/local/bin" }, - # Don’t let remote commands take longer than a minute - # (unless later overridden). - "timeout": 60, + # Don’t let remote commands take longer than five minutes + # (unless later overridden). This is to prevent hangs. + "timeout": 300, } } ) From 3c41920eabced43b8d1586ba13c1387e4ee863b3 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Wed, 14 Oct 2020 21:03:09 -0700 Subject: [PATCH 026/194] Display logged stderr/stdout as it happens --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index fd931c8170..1c071a30b6 100644 --- a/Makefile +++ b/Makefile @@ -6,14 +6,14 @@ setup: # Run Pytest run: - @poetry run python -m pytest --setup-show -rA --tb=short + @poetry run python -m pytest -rA --capture=tee-sys --tb=short # Run semantic analysis test: @poetry run python -X dev -X tracemalloc -m pytest --flake8 --mypy -m 'flake8 or mypy' smoke: - @poetry run python -m pytest -rA -k smoke + @poetry run python -m pytest -rA --capture=tee-sys -k smoke # Print current Python virtualenv venv: From e95420fce8aeb42a341483658a8f6a8de709c13d Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Wed, 14 Oct 2020 21:17:59 -0700 Subject: [PATCH 027/194] Fix subtle break in Fabric MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Accidentally eliminated Fabric’s default overrides of Invoke by supplying my own config based on `invoke.Config` to Fabric. Oops. --- node_plugin.py | 42 ++++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/node_plugin.py b/node_plugin.py index ecc55df97a..99c591abb2 100644 --- a/node_plugin.py +++ b/node_plugin.py @@ -5,34 +5,35 @@ from uuid import uuid4 import _pytest +import fabric +import invoke from fabric import Connection # type: ignore -from invoke import Config, Context # type: ignore +from invoke import Context # type: ignore from invoke.runners import Result # type: ignore import pytest -# Setup a sane configuration for local and remote commands. -config = Config( - overrides={ - "run": { - # Show each command as its run. - "echo": True, - # Disable stdin forwarding. - "in_stream": False, - # Set PATH since it’s not a login shell. - "env": { - "PATH": "/sbin:/usr/sbin:/usr/local/sbin:/bin:/usr/bin:/usr/local/bin" - }, - # Don’t let remote commands take longer than five minutes - # (unless later overridden). This is to prevent hangs. - "timeout": 300, - } +# Setup a sane configuration for local and remote commands. Note that +# the defaults between Fabric and Invoke are different, so we use +# their Config classes explicitly. +config = { + "run": { + # Show each command as its run. + "echo": True, + # Disable stdin forwarding. + "in_stream": False, + # Set PATH since it’s not a login shell. + "env": {"PATH": "/sbin:/usr/sbin:/usr/local/sbin:/bin:/usr/bin:/usr/local/bin"}, + # Don’t let remote commands take longer than five minutes + # (unless later overridden). This is to prevent hangs. + "timeout": 300, } -) +} # Provide a configured local Invoke context for running commands # before establishing a connection. (Use like `local.run(...)`). -local = Context(config=config) +invoke_config = invoke.Config(overrides=config) +local = Context(config=invoke_config) def check_az_cli() -> None: @@ -155,7 +156,8 @@ def node(request: _pytest.fixtures.FixtureRequest) -> Iterator[Node]: host = connect_marker.args[0] # Yield the configured Node connection. - with Node(host, config=config, inline_ssh_env=True) as n: + fabric_config = fabric.Config(overrides=config) + with Node(host, config=fabric_config, inline_ssh_env=True) as n: n.name = name yield n From ef234b9fd73ed90977eb9a1a38299590433ce689 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Thu, 15 Oct 2020 12:07:36 -0700 Subject: [PATCH 028/194] Note use of az CLI instead of Azure Python APIs --- README.md | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 5dc023742b..dd7d3cd183 100644 --- a/README.md +++ b/README.md @@ -66,12 +66,12 @@ implement (with more as found to be required): * Run a command (perhaps asynchronously) on the node using SSH * Download and upload files to the node (with retries and timeouts) -Our abstraction would leverage +Our abstraction leverages [Fabric](https://docs.fabfile.org/en/stable/index.html), which uses [paramiko](https://docs.paramiko.org/en/stable/) underneath, directly to -implement the SSH commands. For deployment logic, it would use existing Python APIs to deploy -[Azure](https://aka.ms/azsdk/python/all) nodes, and for Hyper-V (and other -virtualization platforms), it would use +implement the SSH commands. For deployment logic, it uses the [`az` +CLI](https://aka.ms/azureclidocs), wrapped by Fabric. For Hyper-V (and other +virtualization platforms), it could use [libvirt](https://libvirt.org/python.html). Other test specific requirements, such as installing software and daemons, @@ -88,6 +88,16 @@ a month. We don’t need to rewrite this. ## Alternatives considered +### Azure Python APIs instead of `az` CLI + +We do not use the [Azure Python APIs](https://aka.ms/azsdk/python/all) directly +because they are more complicated (and less documented) than the `az` CLI. Given +Fabric (and its underlying Invoke library), the CLI becomes incredibly easy to +work with. The `az` CLI lead developer states that they have [feature +parity](https://stackoverflow.com/a/50005660/1028665) and that the CLI is more +straightforward to use. Considering our ease-of-maintenance requirement, this +seems the apt choice. + ### pytest-xdist With the [pytest-xdist plugin](https://github.com/pytest-dev/pytest-xdist) there From c8658460d337cf96dced829568ef5c8c7bf759e7 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Thu, 15 Oct 2020 15:50:29 -0700 Subject: [PATCH 029/194] Add a basic self test --- Makefile | 6 +++++- node_plugin.py | 3 ++- selftests/test_basic.py | 7 +++++++ 3 files changed, 14 insertions(+), 2 deletions(-) create mode 100644 selftests/test_basic.py diff --git a/Makefile b/Makefile index 1c071a30b6..b823ce5adb 100644 --- a/Makefile +++ b/Makefile @@ -8,8 +8,12 @@ setup: run: @poetry run python -m pytest -rA --capture=tee-sys --tb=short -# Run semantic analysis +# Run local tests test: + @poetry run python -m pytest -rA --capture=tee-sys --tb=short selftests/ + +# Run semantic analysis +check: @poetry run python -X dev -X tracemalloc -m pytest --flake8 --mypy -m 'flake8 or mypy' smoke: diff --git a/node_plugin.py b/node_plugin.py index 99c591abb2..77a6abab40 100644 --- a/node_plugin.py +++ b/node_plugin.py @@ -141,6 +141,7 @@ def node(request: _pytest.fixtures.FixtureRequest) -> Iterator[Node]: # TODO: The deploy and connect markers should be mutually # exclusive. + name = "local" host = "localhost" # Deploy a node. @@ -152,8 +153,8 @@ def node(request: _pytest.fixtures.FixtureRequest) -> Iterator[Node]: # Get the host from the test’s marker. connect_marker = request.node.get_closest_marker("connect") if connect_marker: - name = "local" host = connect_marker.args[0] + name = host # Yield the configured Node connection. fabric_config = fabric.Config(overrides=config) diff --git a/selftests/test_basic.py b/selftests/test_basic.py new file mode 100644 index 0000000000..a644b137ae --- /dev/null +++ b/selftests/test_basic.py @@ -0,0 +1,7 @@ +"""These tests are meant to run in a CI environment.""" +from node_plugin import Node + + +def test_basic(node: Node) -> None: + """Basic test which creates a Node connection to 'localhost'.""" + node.local("echo Hello World") From f2514990c0f11ef45fc397af035dedb9cf50b3fa Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Thu, 15 Oct 2020 16:05:56 -0700 Subject: [PATCH 030/194] Make az CLI check compatible with Windows --- node_plugin.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/node_plugin.py b/node_plugin.py index 77a6abab40..bdb008f75b 100644 --- a/node_plugin.py +++ b/node_plugin.py @@ -37,12 +37,8 @@ def check_az_cli() -> None: - if not local.run("which az", warn=True): - # TODO: Use Invoke for pipes. - local.run( - "curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash", - ) - + # E.g. on Ubuntu: `curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash` + assert local.run("az --version", warn=True), "Please install the `az` CLI!" # TODO: Login with service principal (az login) and set # default subscription (az account set -s) using secrets. account: Result = local.run("az account show") From 8220a6040c5f3afa6a418e4883660d3fe49ff0e6 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Thu, 15 Oct 2020 16:10:07 -0700 Subject: [PATCH 031/194] Only set PATH for SSH commands MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This also required patching the `local()` method of Fabric’s `Connection` class to _not_ replace the environment for local commands, since a Linux PATH on Windows does not work out. --- node_plugin.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/node_plugin.py b/node_plugin.py index bdb008f75b..d933a347fc 100644 --- a/node_plugin.py +++ b/node_plugin.py @@ -22,14 +22,13 @@ "echo": True, # Disable stdin forwarding. "in_stream": False, - # Set PATH since it’s not a login shell. - "env": {"PATH": "/sbin:/usr/sbin:/usr/local/sbin:/bin:/usr/bin:/usr/local/bin"}, # Don’t let remote commands take longer than five minutes # (unless later overridden). This is to prevent hangs. "timeout": 300, } } + # Provide a configured local Invoke context for running commands # before establishing a connection. (Use like `local.run(...)`). invoke_config = invoke.Config(overrides=config) @@ -114,6 +113,10 @@ class Node(Connection): name: str + def local(self, *args, **kwargs): + """This patches Fabric's 'local()' function to ignore SSH environment.""" + return super(Connection, self).run(replace_env=False, env={}, *args, **kwargs) + def get_boot_diagnostics(self): """Gets the serial console logs.""" return self.local( @@ -153,7 +156,12 @@ def node(request: _pytest.fixtures.FixtureRequest) -> Iterator[Node]: name = host # Yield the configured Node connection. - fabric_config = fabric.Config(overrides=config) + ssh_config = config.copy() + ssh_config["run"]["env"] = { + # Set PATH since it’s not a login shell. + "PATH": "/sbin:/usr/sbin:/usr/local/sbin:/bin:/usr/bin:/usr/local/bin" + } + fabric_config = fabric.Config(overrides=ssh_config) with Node(host, config=fabric_config, inline_ssh_env=True) as n: n.name = name yield n From d10825e8206307441d319cb8817178330400ec26 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Thu, 15 Oct 2020 16:59:42 -0700 Subject: [PATCH 032/194] =?UTF-8?q?Make=20=E2=80=98ping=E2=80=99=20cross-p?= =?UTF-8?q?latform?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- testsuites/test_smoke.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/testsuites/test_smoke.py b/testsuites/test_smoke.py index cbc8dfac82..d52ed6082b 100644 --- a/testsuites/test_smoke.py +++ b/testsuites/test_smoke.py @@ -1,4 +1,5 @@ """Runs a 'smoke' test for an Azure Linux VM deployment.""" +import platform import socket from invoke.runners import Result # type: ignore @@ -25,8 +26,10 @@ def test_smoke(node: Node) -> None: SSH failures DO NOT fail this test. TODO: Log warnings instead of printing. """ + # TODO: Move to ‘Node.ping()’ + ping_flag = "-c 1" if platform.system() == "Linux" else "-n 1" # TODO: Can’t ping by default, need to enable. - ping1_result: Result = node.local(f"ping {node.host} -c 1", warn=True) + ping1_result: Result = node.local(f"ping {ping_flag} {node.host}", warn=True) try: node.run("uptime") # If SSH fails, we catch it. @@ -36,7 +39,7 @@ def test_smoke(node: Node) -> None: node.platform_restart() # Try pinging and SSH again. - ping2_result: Result = node.local(f"ping {node.host} -c 1", warn=True) + ping2_result: Result = node.local(f"ping {ping_flag} {node.host}", warn=True) try: node.run("uptime") From a4d5f49e632f4a955c9d7e1d8e44f56d371e2094 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Thu, 15 Oct 2020 17:48:38 -0700 Subject: [PATCH 033/194] Fix types --- node_plugin.py | 29 +++++++++++++++-------------- testsuites/test_smoke.py | 2 +- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/node_plugin.py b/node_plugin.py index d933a347fc..38cfe6f81f 100644 --- a/node_plugin.py +++ b/node_plugin.py @@ -1,14 +1,14 @@ """Pytest plugin implementing a Node fixture for running remote commands.""" import json from io import BytesIO -from typing import Dict, Iterator, Optional, Tuple +from typing import Any, Dict, Iterator, Optional, Tuple from uuid import uuid4 import _pytest -import fabric -import invoke -from fabric import Connection # type: ignore -from invoke import Context # type: ignore +import fabric # type: ignore +import invoke # type: ignore +from fabric import Connection +from invoke import Context from invoke.runners import Result # type: ignore import pytest @@ -67,14 +67,15 @@ def deploy_vm( ) -> Tuple[str, Dict[str, str]]: key = f"{location}/{vm_image}/{vm_size}" - name: Optional[str] = request.config.cache.get(key, None) + name: Optional[str] = request.config.cache.get(key, None) # type: ignore + result: Dict[str, str] = dict() if name: - result: Dict[str, str] = request.config.cache.get(name, {}) + result = request.config.cache.get(name, {}) # type: ignore assert result, "There was a cache problem, use --cache-clear and try again." return name, result name = f"pytest-{uuid4()}" - request.config.cache.set(key, name) + request.config.cache.set(key, name) # type: ignore check_az_cli() boot_storage = create_boot_storage(location) @@ -95,12 +96,12 @@ def deploy_vm( if networking == "SRIOV": vm_command.append("--accelerated-networking true") - result: Dict[str, str] = json.loads( + result = json.loads( local.run( " ".join(vm_command), ).stdout ) - request.config.cache.set(name, result) + request.config.cache.set(name, result) # type: ignore return name, result @@ -113,17 +114,17 @@ class Node(Connection): name: str - def local(self, *args, **kwargs): + def local(self, *args: Any, **kwargs: Any) -> Result: """This patches Fabric's 'local()' function to ignore SSH environment.""" return super(Connection, self).run(replace_env=False, env={}, *args, **kwargs) - def get_boot_diagnostics(self): + def get_boot_diagnostics(self) -> Result: """Gets the serial console logs.""" return self.local( f"az vm boot-diagnostics get-boot-log -n {self.name} -g {self.name}-rg" ) - def platform_restart(self): + def platform_restart(self) -> Result: """TODO: Should this '--force' and redeploy?""" return self.local(f"az vm restart -n {self.name} -g {self.name}-rg") @@ -157,7 +158,7 @@ def node(request: _pytest.fixtures.FixtureRequest) -> Iterator[Node]: # Yield the configured Node connection. ssh_config = config.copy() - ssh_config["run"]["env"] = { + ssh_config["run"]["env"] = { # type: ignore # Set PATH since it’s not a login shell. "PATH": "/sbin:/usr/sbin:/usr/local/sbin:/bin:/usr/bin:/usr/local/bin" } diff --git a/testsuites/test_smoke.py b/testsuites/test_smoke.py index d52ed6082b..1fe16cad04 100644 --- a/testsuites/test_smoke.py +++ b/testsuites/test_smoke.py @@ -3,7 +3,7 @@ import socket from invoke.runners import Result # type: ignore -from paramiko import SSHException +from paramiko import SSHException # type: ignore import pytest from node_plugin import Node From cbbbcbdd53da3f833220cb277b1808dfb7460133 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Thu, 15 Oct 2020 18:11:24 -0700 Subject: [PATCH 034/194] Add tenacity package --- poetry.lock | 20 +++++++++++++++++++- pyproject.toml | 1 + 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index 82eabae938..ac379eee62 100644 --- a/poetry.lock +++ b/poetry.lock @@ -550,6 +550,20 @@ category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +[[package]] +name = "tenacity" +version = "6.2.0" +description = "Retry code until it succeeds" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +six = ">=1.9.0" + +[package.extras] +doc = ["reno", "sphinx", "tornado (>=4.5)"] + [[package]] name = "testfixtures" version = "6.15.0" @@ -598,7 +612,7 @@ python-versions = ">=3.6" [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "307896057c574edcbf704e9060caabb3eb14b01c06ba841b8de5c7715ce86ecb" +content-hash = "42ece43921b68dfda0693eb450c7da2797f9971ee0824cc4ab7752691fd71552" [metadata.files] appdirs = [ @@ -889,6 +903,10 @@ six = [ {file = "six-1.15.0-py2.py3-none-any.whl", hash = "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"}, {file = "six-1.15.0.tar.gz", hash = "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259"}, ] +tenacity = [ + {file = "tenacity-6.2.0-py2.py3-none-any.whl", hash = "sha256:5a5d3dcd46381abe8b4f82b5736b8726fd3160c6c7161f53f8af7f1eb9b82173"}, + {file = "tenacity-6.2.0.tar.gz", hash = "sha256:29ae90e7faf488a8628432154bb34ace1cca58244c6ea399fd33f066ac71339a"}, +] testfixtures = [ {file = "testfixtures-6.15.0-py2.py3-none-any.whl", hash = "sha256:e17f4f526fc90b0ac9bc7f8ca62b7dec17d9faf3d721f56bda4f0fd94d02f85a"}, {file = "testfixtures-6.15.0.tar.gz", hash = "sha256:409f77cfbdad822d12a8ce5c4aa8fb4d0bb38073f4a5444fede3702716a2cec2"}, diff --git a/pyproject.toml b/pyproject.toml index c083e43bef..68b90e4371 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,6 +10,7 @@ python = "^3.8" pytest = "^6.1.1" fabric = "^2.5.0" pytest-timeout = "^1.4.2" +tenacity = "^6.2.0" [tool.poetry.dev-dependencies] black = "^20.8b1" From bbc4c178a99ec4282f944057bd58bd4d061975ce Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Thu, 15 Oct 2020 18:11:31 -0700 Subject: [PATCH 035/194] Add retry to get boot diagnostics with exponential wait --- node_plugin.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/node_plugin.py b/node_plugin.py index 38cfe6f81f..4d4bbc91c0 100644 --- a/node_plugin.py +++ b/node_plugin.py @@ -10,6 +10,7 @@ from fabric import Connection from invoke import Context from invoke.runners import Result # type: ignore +from tenacity import retry, stop_after_delay, wait_exponential import pytest @@ -118,6 +119,7 @@ def local(self, *args: Any, **kwargs: Any) -> Result: """This patches Fabric's 'local()' function to ignore SSH environment.""" return super(Connection, self).run(replace_env=False, env={}, *args, **kwargs) + @retry(wait=wait_exponential(), stop=stop_after_delay(60)) def get_boot_diagnostics(self) -> Result: """Gets the serial console logs.""" return self.local( From bf14056ed6e3c170d9907feffccd91848a265de9 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Thu, 15 Oct 2020 18:25:11 -0700 Subject: [PATCH 036/194] Ignore unclosed file/socket resource warnings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Need to figure out the unclosed files at some point, but punting. The unclosed socket happens because Paramiko doesn’t close sockets when connections abruptly end, which we’ll need to fix upstream. --- pytest.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/pytest.ini b/pytest.ini index ca01a9b06f..eb42e41f02 100644 --- a/pytest.ini +++ b/pytest.ini @@ -6,4 +6,5 @@ markers = timeout = 300 filterwarnings = error + ignore:unclosed:ResourceWarning ignore:the imp module is deprecated in favour of importlib:DeprecationWarning From b31ec91bb962b5635226665517422c7fdd9c8e11 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Fri, 16 Oct 2020 10:57:35 -0700 Subject: [PATCH 037/194] Fix VM caching --- conftest.py | 17 +++++++++++ node_plugin.py | 83 +++++++++++++++++++++++++++++++------------------- 2 files changed, 68 insertions(+), 32 deletions(-) diff --git a/conftest.py b/conftest.py index fb5ebaeec6..80269b41ab 100644 --- a/conftest.py +++ b/conftest.py @@ -5,6 +5,23 @@ """ from pathlib import Path +from _pytest.config.argparsing import Parser + pytest_plugins = "node_plugin" + +def pytest_addoption(parser: Parser) -> None: + """Pytest hook for adding arbitrary CLI options. + + https://docs.pytest.org/en/latest/example/simple.html + + """ + parser.addoption( + "--keep-vms", + action="store_true", + default=False, + help="Keeps deployed VMs cached between test runs, useful for developers.", + ) + + LINUX_SCRIPTS = Path("../Testscripts/Linux") diff --git a/node_plugin.py b/node_plugin.py index 4d4bbc91c0..bd9618e49c 100644 --- a/node_plugin.py +++ b/node_plugin.py @@ -37,6 +37,7 @@ def check_az_cli() -> None: + """Assert that the `az` CLI is installed and logged in.""" # E.g. on Ubuntu: `curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash` assert local.run("az --version", warn=True), "Please install the `az` CLI!" # TODO: Login with service principal (az login) and set @@ -59,25 +60,22 @@ def create_boot_storage(location: str) -> str: def deploy_vm( - request: _pytest.fixtures.FixtureRequest, + name: str, location: str = "westus2", vm_image: str = "UbuntuLTS", vm_size: str = "Standard_DS1_v2", setup: str = "", networking: str = "", ) -> Tuple[str, Dict[str, str]]: + """Given deployment info, deploy a new VM. - key = f"{location}/{vm_image}/{vm_size}" - name: Optional[str] = request.config.cache.get(key, None) # type: ignore - result: Dict[str, str] = dict() - if name: - result = request.config.cache.get(name, {}) # type: ignore - assert result, "There was a cache problem, use --cache-clear and try again." - return name, result - - name = f"pytest-{uuid4()}" - request.config.cache.set(key, name) # type: ignore + TODO: This along with the functions it calls are Azure specific + and so would be refactored to support other platforms. Hence it + returns both the host and the deployment data so that calling + functions don't have to know which field in the data corresponds + to the host. + """ check_az_cli() boot_storage = create_boot_storage(location) @@ -97,16 +95,14 @@ def deploy_vm( if networking == "SRIOV": vm_command.append("--accelerated-networking true") - result = json.loads( - local.run( - " ".join(vm_command), - ).stdout - ) - request.config.cache.set(name, result) # type: ignore - return name, result + data: Dict[str, str] = json.loads(local.run(" ".join(vm_command)).stdout) + host = data["publicIpAddress"] + return host, data def delete_vm(name: str) -> None: + """Delete the entire allocated resource group.""" + # TODO: Maybe don’t wait for this command to complete. local.run(f"az group delete -n {name}-rg --yes") @@ -114,6 +110,7 @@ class Node(Connection): """Extends 'fabric.Connection' with our own utilities.""" name: str + data: Dict[str, str] def local(self, *args: Any, **kwargs: Any) -> Result: """This patches Fabric's 'local()' function to ignore SSH environment.""" @@ -139,23 +136,44 @@ def cat(self, path: str) -> str: @pytest.fixture def node(request: _pytest.fixtures.FixtureRequest) -> Iterator[Node]: - """Yields a safe remote Node on which to run commands.""" + """Yields a safe remote Node on which to run commands. - # TODO: The deploy and connect markers should be mutually - # exclusive. - name = "local" - host = "localhost" + TODO: Currently this also manages the caching of the deployed VMs. + However, we should make a node pool (perhaps a session-scoped + fixture) which caches and deploys VMs, leaving this to perform its + original work as a connection creator. - # Deploy a node. + """ deploy_marker = request.node.get_closest_marker("deploy") - if deploy_marker: - name, result = deploy_vm(request, **deploy_marker.kwargs) - host = result["publicIpAddress"] - - # Get the host from the test’s marker. connect_marker = request.node.get_closest_marker("connect") - if connect_marker: + + data: Dict[str, str] = dict() + name: Optional[str] = None + host: Optional[str] = None + + # TODO: The deploy and connect markers should be mutually + # exclusive. + if deploy_marker: + # NOTE: https://docs.pytest.org/en/stable/cache.html + key = "/".join(["node"] + list(filter(None, deploy_marker.kwargs.values()))) + data = request.config.cache.get(key, None) # type: ignore + if not data: + # Cache miss, deploy new node... + name = f"pytest-{uuid4()}" + host, data = deploy_vm(name, **deploy_marker.kwargs) + data["name"] = name + data["host"] = host + request.config.cache.set(key, data) # type: ignore + name = data["name"] + host = data["host"] + elif connect_marker: + # Get the host from the test’s marker. host = connect_marker.args[0] + name = f"pre-deployed:{host}" + else: + # NOTE: This still uses SSH so the localhost must be + # connectable. + host = "localhost" name = host # Yield the configured Node connection. @@ -167,9 +185,10 @@ def node(request: _pytest.fixtures.FixtureRequest) -> Iterator[Node]: fabric_config = fabric.Config(overrides=ssh_config) with Node(host, config=fabric_config, inline_ssh_env=True) as n: n.name = name + n.data = data yield n # Clean up! - # TODO: This logic is wrong. - if request.config.getoption("cacheclear") and name: + if not request.config.getoption("keep_vms") and key: delete_vm(name) + request.config.cache.set(key, None) # type: ignore From d6d01987ebf8a0840bef77d195472da803e526b7 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Fri, 16 Oct 2020 11:20:14 -0700 Subject: [PATCH 038/194] Clean up types --- node_plugin.py | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/node_plugin.py b/node_plugin.py index bd9618e49c..e94b04ffc0 100644 --- a/node_plugin.py +++ b/node_plugin.py @@ -1,19 +1,25 @@ """Pytest plugin implementing a Node fixture for running remote commands.""" +from __future__ import annotations + import json +import typing from io import BytesIO -from typing import Any, Dict, Iterator, Optional, Tuple from uuid import uuid4 -import _pytest import fabric # type: ignore import invoke # type: ignore from fabric import Connection from invoke import Context from invoke.runners import Result # type: ignore -from tenacity import retry, stop_after_delay, wait_exponential +from tenacity import retry, stop_after_delay, wait_exponential # type: ignore import pytest +if typing.TYPE_CHECKING: + from typing import Any, Dict, Iterator, Optional, Tuple + + from _pytest.fixtures import FixtureRequest + # Setup a sane configuration for local and remote commands. Note that # the defaults between Fabric and Invoke are different, so we use # their Config classes explicitly. @@ -135,7 +141,7 @@ def cat(self, path: str) -> str: @pytest.fixture -def node(request: _pytest.fixtures.FixtureRequest) -> Iterator[Node]: +def node(request: FixtureRequest) -> Iterator[Node]: """Yields a safe remote Node on which to run commands. TODO: Currently this also manages the caching of the deployed VMs. @@ -156,14 +162,15 @@ def node(request: _pytest.fixtures.FixtureRequest) -> Iterator[Node]: if deploy_marker: # NOTE: https://docs.pytest.org/en/stable/cache.html key = "/".join(["node"] + list(filter(None, deploy_marker.kwargs.values()))) - data = request.config.cache.get(key, None) # type: ignore + assert request.config.cache is not None + data = request.config.cache.get(key, None) if not data: # Cache miss, deploy new node... name = f"pytest-{uuid4()}" host, data = deploy_vm(name, **deploy_marker.kwargs) data["name"] = name data["host"] = host - request.config.cache.set(key, data) # type: ignore + request.config.cache.set(key, data) name = data["name"] host = data["host"] elif connect_marker: @@ -177,8 +184,8 @@ def node(request: _pytest.fixtures.FixtureRequest) -> Iterator[Node]: name = host # Yield the configured Node connection. - ssh_config = config.copy() - ssh_config["run"]["env"] = { # type: ignore + ssh_config: Dict[str, Any] = config.copy() + ssh_config["run"]["env"] = { # Set PATH since it’s not a login shell. "PATH": "/sbin:/usr/sbin:/usr/local/sbin:/bin:/usr/bin:/usr/local/bin" } @@ -191,4 +198,5 @@ def node(request: _pytest.fixtures.FixtureRequest) -> Iterator[Node]: # Clean up! if not request.config.getoption("keep_vms") and key: delete_vm(name) - request.config.cache.set(key, None) # type: ignore + assert request.config.cache is not None + request.config.cache.set(key, None) From b29ac6a2e26286056d43718bc7049ffe512e004e Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Fri, 16 Oct 2020 15:02:54 -0700 Subject: [PATCH 039/194] Enable all junit logging --- pytest.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/pytest.ini b/pytest.ini index eb42e41f02..73894c5bf1 100644 --- a/pytest.ini +++ b/pytest.ini @@ -3,6 +3,7 @@ markers = lisa deploy connect +junit_logging = all timeout = 300 filterwarnings = error From 4cd522da2fe19622819ac835e51d8402ce9eccb4 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Fri, 16 Oct 2020 15:03:25 -0700 Subject: [PATCH 040/194] Add logging to node plugin --- node_plugin.py | 20 +++++++++++++++----- pytest.ini | 2 ++ 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/node_plugin.py b/node_plugin.py index e94b04ffc0..63082a00f3 100644 --- a/node_plugin.py +++ b/node_plugin.py @@ -2,6 +2,7 @@ from __future__ import annotations import json +import logging import typing from io import BytesIO from uuid import uuid4 @@ -50,8 +51,11 @@ def check_az_cli() -> None: # default subscription (az account set -s) using secrets. account: Result = local.run("az account show") assert account.ok, "Please `az login`!" - subs = json.loads(account.stdout) - assert subs["isDefault"], "Please `az account set -s `!" + sub = json.loads(account.stdout) + assert sub["isDefault"], "Please `az account set -s `!" + logging.info( + f"Using account '{sub['user']['name']}' with subscription '{sub['name']}'" + ) def create_boot_storage(location: str) -> str: @@ -85,10 +89,13 @@ def deploy_vm( check_az_cli() boot_storage = create_boot_storage(location) - local.run( - f"az group create -n {name}-rg --location {location}", + logging.info( + f"Deploying VM to resource group '{name}-rg' in '{location}' " + "with image '{vm_image}' and size '{vm_size}'" ) + local.run(f"az group create -n {name}-rg --location {location}") + vm_command = [ "az vm create", f"-g {name}-rg", @@ -109,6 +116,7 @@ def deploy_vm( def delete_vm(name: str) -> None: """Delete the entire allocated resource group.""" # TODO: Maybe don’t wait for this command to complete. + logging.info(f"Deleting resource group '{name}-rg'") local.run(f"az group delete -n {name}-rg --yes") @@ -164,7 +172,9 @@ def node(request: FixtureRequest) -> Iterator[Node]: key = "/".join(["node"] + list(filter(None, deploy_marker.kwargs.values()))) assert request.config.cache is not None data = request.config.cache.get(key, None) - if not data: + if data: + logging.info(f"Reusing node for cached key '{key}'") + else: # Cache miss, deploy new node... name = f"pytest-{uuid4()}" host, data = deploy_vm(name, **deploy_marker.kwargs) diff --git a/pytest.ini b/pytest.ini index 73894c5bf1..890ddea1a0 100644 --- a/pytest.ini +++ b/pytest.ini @@ -3,6 +3,8 @@ markers = lisa deploy connect +log_cli = true +log_cli_level = INFO junit_logging = all timeout = 300 filterwarnings = From 52bd62111de840aec7255ed268b60370d20ec309 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Fri, 16 Oct 2020 15:05:40 -0700 Subject: [PATCH 041/194] Split Node scopes to function and class fixtures --- node_plugin.py | 49 +++++++++++++++++++++++++++++++++++++------------ 1 file changed, 37 insertions(+), 12 deletions(-) diff --git a/node_plugin.py b/node_plugin.py index 63082a00f3..ff31286cc5 100644 --- a/node_plugin.py +++ b/node_plugin.py @@ -91,7 +91,7 @@ def deploy_vm( logging.info( f"Deploying VM to resource group '{name}-rg' in '{location}' " - "with image '{vm_image}' and size '{vm_size}'" + f"with image '{vm_image}' and size '{vm_size}'" ) local.run(f"az group create -n {name}-rg --location {location}") @@ -148,8 +148,40 @@ def cat(self, path: str) -> str: return buf.getvalue().decode("utf-8").strip() -@pytest.fixture +@pytest.fixture(scope="function") def node(request: FixtureRequest) -> Iterator[Node]: + key, name, host, data, fabric_config = get_node(request) + with Node(host, config=fabric_config, inline_ssh_env=True) as n: + n.name = name + n.data = data + yield n + + # Clean up! + if not request.config.getoption("keep_vms") and key: + delete_vm(name) + assert request.config.cache is not None + request.config.cache.set(key, None) + + +@pytest.fixture(scope="class") +def class_node(request: FixtureRequest) -> Iterator[None]: + key, name, host, data, fabric_config = get_node(request) + with Node(host, config=fabric_config, inline_ssh_env=True) as n: + n.name = name + n.data = data + request.cls.n = n + yield + + # Clean up! + if not request.config.getoption("keep_vms") and key: + delete_vm(name) + assert request.config.cache is not None + request.config.cache.set(key, None) + + +def get_node( + request: FixtureRequest, +) -> Tuple[Optional[str], str, Optional[str], Dict[str, str], fabric.Config]: """Yields a safe remote Node on which to run commands. TODO: Currently this also manages the caching of the deployed VMs. @@ -157,10 +189,12 @@ def node(request: FixtureRequest) -> Iterator[Node]: fixture) which caches and deploys VMs, leaving this to perform its original work as a connection creator. + TODO: It's return type is garbage. """ deploy_marker = request.node.get_closest_marker("deploy") connect_marker = request.node.get_closest_marker("connect") + key: Optional[str] = None data: Dict[str, str] = dict() name: Optional[str] = None host: Optional[str] = None @@ -200,13 +234,4 @@ def node(request: FixtureRequest) -> Iterator[Node]: "PATH": "/sbin:/usr/sbin:/usr/local/sbin:/bin:/usr/bin:/usr/local/bin" } fabric_config = fabric.Config(overrides=ssh_config) - with Node(host, config=fabric_config, inline_ssh_env=True) as n: - n.name = name - n.data = data - yield n - - # Clean up! - if not request.config.getoption("keep_vms") and key: - delete_vm(name) - assert request.config.cache is not None - request.config.cache.set(key, None) + return key, name, host, data, fabric_config From 778c97c2e8ae62e48470db2b29183572271c926b Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Fri, 16 Oct 2020 15:05:55 -0700 Subject: [PATCH 042/194] Split smoke test into component tests with less verbose output --- Makefile | 2 +- testsuites/test_smoke.py | 70 ++++++++++++++++++++++------------------ 2 files changed, 40 insertions(+), 32 deletions(-) diff --git a/Makefile b/Makefile index b823ce5adb..90ffb1ca4e 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,7 @@ check: @poetry run python -X dev -X tracemalloc -m pytest --flake8 --mypy -m 'flake8 or mypy' smoke: - @poetry run python -m pytest -rA --capture=tee-sys -k smoke + @poetry run python -m pytest --quiet --junit-xml=tests.xml --tb=no -rA --show-capture=log -k smoke # Print current Python virtualenv venv: diff --git a/testsuites/test_smoke.py b/testsuites/test_smoke.py index 1fe16cad04..5f33d6c24d 100644 --- a/testsuites/test_smoke.py +++ b/testsuites/test_smoke.py @@ -1,6 +1,8 @@ """Runs a 'smoke' test for an Azure Linux VM deployment.""" +import logging import platform import socket +import time from invoke.runners import Result # type: ignore from paramiko import SSHException # type: ignore @@ -10,7 +12,8 @@ @pytest.mark.deploy(setup="OneVM", vm_size="Standard_DS2_v2") -def test_smoke(node: Node) -> None: +@pytest.mark.usefixtures("class_node") +class TestSmoke: """Check that a VM can be deployed and is responsive. 1. Deploy the VM (via 'node' fixture) and log it. @@ -19,36 +22,41 @@ def test_smoke(node: Node) -> None: 4. Attempt to reboot via SSH, otherwise use the platform. 5. Fetch the serial console logs. - For commands where we expect a possible non-zero exit code, we - pass 'warn=True' to prevent it from throwing 'UnexpectedExit' and - we instead check its result at the end. - - SSH failures DO NOT fail this test. - TODO: Log warnings instead of printing. """ + + n: Node + # TODO: Move to ‘Node.ping()’ ping_flag = "-c 1" if platform.system() == "Linux" else "-n 1" - # TODO: Can’t ping by default, need to enable. - ping1_result: Result = node.local(f"ping {ping_flag} {node.host}", warn=True) - - try: - node.run("uptime") # If SSH fails, we catch it. - reboot_result: Result = node.sudo("reboot", warn=True) # Expect -1 - except (TimeoutError, SSHException, socket.error) as e: - print(f"SSH failed '{e}', using platform to reboot...") - node.platform_restart() - - # Try pinging and SSH again. - ping2_result: Result = node.local(f"ping {ping_flag} {node.host}", warn=True) - - try: - node.run("uptime") - except (TimeoutError, SSHException, socket.error) as e: - print(f"SSH failed '{e}' after the reboot.") - - # Always download the serial console logs. - node.get_boot_diagnostics() - - assert ping1_result.ok - assert reboot_result.exited == -1, "Reboot failed, used platform instead" - assert ping2_result.ok + + def test_ping_1(self) -> None: + # TODO: Can’t ping by default, need to enable. + logging.warning("Expecting ping to fail because it's not enabled yet") + r: Result = self.n.local(f"ping {self.ping_flag} {self.n.host}", warn=True) + assert r.ok, f"Pinging {self.n.host} failed" + + def test_ssh_1(self) -> None: + self.n.run("uptime") + + def test_reboot(self) -> None: + try: + # If this succeeds, we should expect the exit code to be -1 + r: Result = self.n.sudo("reboot", warn=True) + except (TimeoutError, SSHException, socket.error) as e: + logging.warning(f"SSH failed '{e}', using platform to reboot") + self.n.platform_restart() + logging.info("Waiting 10 seconds for reboot to finish") + time.sleep(10) + assert r.exited == -1, "While SSH worked, reboot failed" + + def test_ping_2(self) -> None: + # TODO: Can’t ping by default, need to enable. + logging.warning("Expecting ping to fail for the same reason as above") + r: Result = self.n.local(f"ping {self.ping_flag} {self.n.host}", warn=True) + assert r.ok, f"Pinging {self.n.host} failed" + + def test_ssh_2(self) -> None: + self.n.run("uptime") + + def test_serial_log(self) -> None: + self.n.get_boot_diagnostics() From 0a0e8b3522a4c3b800f5cd1c0a2cbaa00d6033bb Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Fri, 16 Oct 2020 16:06:13 -0700 Subject: [PATCH 043/194] =?UTF-8?q?Add=20a=20=E2=80=98clean=E2=80=99=20mak?= =?UTF-8?q?e=20target=20to=20clear=20the=20cache=20and=20show=20the=20setu?= =?UTF-8?q?p=20plan?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index 90ffb1ca4e..a1c74c8246 100644 --- a/Makefile +++ b/Makefile @@ -16,6 +16,9 @@ test: check: @poetry run python -X dev -X tracemalloc -m pytest --flake8 --mypy -m 'flake8 or mypy' +clean: + @poetry run python -m pytest --cache-clear --setup-plan + smoke: @poetry run python -m pytest --quiet --junit-xml=tests.xml --tb=no -rA --show-capture=log -k smoke From 49b0edf81163729798b01fdde581e7bf45f808ec Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Fri, 16 Oct 2020 16:19:50 -0700 Subject: [PATCH 044/194] =?UTF-8?q?Don=E2=80=99t=20wait=20for=20deletion?= =?UTF-8?q?=20of=20resource=20group?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- node_plugin.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/node_plugin.py b/node_plugin.py index ff31286cc5..3336e88acd 100644 --- a/node_plugin.py +++ b/node_plugin.py @@ -115,9 +115,8 @@ def deploy_vm( def delete_vm(name: str) -> None: """Delete the entire allocated resource group.""" - # TODO: Maybe don’t wait for this command to complete. logging.info(f"Deleting resource group '{name}-rg'") - local.run(f"az group delete -n {name}-rg --yes") + local.run(f"az group delete -n {name}-rg --yes --no-wait") class Node(Connection): @@ -148,6 +147,8 @@ def cat(self, path: str) -> str: return buf.getvalue().decode("utf-8").strip() +# TODO: The fixtures need to be fixed up since we now have a pair, one +# for each scope. They need documentation and de-duplication too. @pytest.fixture(scope="function") def node(request: FixtureRequest) -> Iterator[Node]: key, name, host, data, fabric_config = get_node(request) @@ -158,9 +159,9 @@ def node(request: FixtureRequest) -> Iterator[Node]: # Clean up! if not request.config.getoption("keep_vms") and key: - delete_vm(name) assert request.config.cache is not None request.config.cache.set(key, None) + delete_vm(name) @pytest.fixture(scope="class") @@ -174,9 +175,9 @@ def class_node(request: FixtureRequest) -> Iterator[None]: # Clean up! if not request.config.getoption("keep_vms") and key: - delete_vm(name) assert request.config.cache is not None request.config.cache.set(key, None) + delete_vm(name) def get_node( From 33fcb31fd5600ac5454ab841e875a4a9d834318b Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Fri, 16 Oct 2020 17:14:57 -0700 Subject: [PATCH 045/194] Better output --- Makefile | 2 +- node_plugin.py | 15 +++++++++++++-- pytest.ini | 2 ++ testsuites/test_smoke.py | 2 -- 4 files changed, 16 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index a1c74c8246..2548396019 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ clean: @poetry run python -m pytest --cache-clear --setup-plan smoke: - @poetry run python -m pytest --quiet --junit-xml=tests.xml --tb=no -rA --show-capture=log -k smoke + @poetry run python -m pytest --quiet --junit-xml=tests.xml --tb=line --show-capture=log -k smoke # Print current Python virtualenv venv: diff --git a/node_plugin.py b/node_plugin.py index 3336e88acd..32dfe89f67 100644 --- a/node_plugin.py +++ b/node_plugin.py @@ -90,8 +90,11 @@ def deploy_vm( boot_storage = create_boot_storage(location) logging.info( - f"Deploying VM to resource group '{name}-rg' in '{location}' " - f"with image '{vm_image}' and size '{vm_size}'" + f"""Deploying VM... + Resource Group: '{name}-rg' + Region: '{location}' + Image: '{vm_image}' + Size: '{vm_size}'""" ) local.run(f"az group create -n {name}-rg --location {location}") @@ -171,6 +174,14 @@ def class_node(request: FixtureRequest) -> Iterator[None]: n.name = name n.data = data request.cls.n = n + logging.info(f"Using VM at: '{host}'") + try: + r: Result = n.run("uname -r") + except Exception as e: + logging.warning(f"Kernel Version: Unknown due to '{e}'") + else: + assert r.ok + logging.info(f"Kernel Version: '{r.stdout.strip()}'") yield # Clean up! diff --git a/pytest.ini b/pytest.ini index 890ddea1a0..5c6d5212b2 100644 --- a/pytest.ini +++ b/pytest.ini @@ -5,6 +5,8 @@ markers = connect log_cli = true log_cli_level = INFO +log_cli_format = %(asctime)s %(levelname)s %(message)s +log_cli_date_format = %Y-%m-%d %H:%M:%S junit_logging = all timeout = 300 filterwarnings = diff --git a/testsuites/test_smoke.py b/testsuites/test_smoke.py index 5f33d6c24d..0e1f157468 100644 --- a/testsuites/test_smoke.py +++ b/testsuites/test_smoke.py @@ -31,7 +31,6 @@ class TestSmoke: def test_ping_1(self) -> None: # TODO: Can’t ping by default, need to enable. - logging.warning("Expecting ping to fail because it's not enabled yet") r: Result = self.n.local(f"ping {self.ping_flag} {self.n.host}", warn=True) assert r.ok, f"Pinging {self.n.host} failed" @@ -51,7 +50,6 @@ def test_reboot(self) -> None: def test_ping_2(self) -> None: # TODO: Can’t ping by default, need to enable. - logging.warning("Expecting ping to fail for the same reason as above") r: Result = self.n.local(f"ping {self.ping_flag} {self.n.host}", warn=True) assert r.ok, f"Pinging {self.n.host} failed" From 27db8541d0b57fea557af2755e95383700c8fcff Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Fri, 16 Oct 2020 22:27:09 -0700 Subject: [PATCH 046/194] Enable ICMP on deployed Azure VMs --- conftest.py | 2 ++ node_plugin.py | 26 ++++++++++++++++++++++++++ testsuites/test_smoke.py | 2 -- 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/conftest.py b/conftest.py index 80269b41ab..be29b17567 100644 --- a/conftest.py +++ b/conftest.py @@ -22,6 +22,8 @@ def pytest_addoption(parser: Parser) -> None: default=False, help="Keeps deployed VMs cached between test runs, useful for developers.", ) + # TODO: Add “--lisa” (and “--debug” etc.) options which set up our + # defaults, instead of encoding them in the Makefile LINUX_SCRIPTS = Path("../Testscripts/Linux") diff --git a/node_plugin.py b/node_plugin.py index 32dfe89f67..5c9f12979d 100644 --- a/node_plugin.py +++ b/node_plugin.py @@ -69,6 +69,27 @@ def create_boot_storage(location: str) -> str: return account +def allow_ping(name: str) -> None: + """Create NSG rules to enable ICMP ping. + + ICMP ping is disallowed by the Azure load balancer by default, but + there’s strong debate about if this is necessary, and our tests + like to check if the host is up using ping, so we create inbound + and outbound rules in the VM's network security group to allow it. + + """ + try: + for d in ["Inbound", "Outbound"]: + local.run( + f"az network nsg rule create --name allow{d}ICMP " + f"--nsg-name {name}NSG --priority 100 --resource-group {name}-rg " + f"--access Allow --direction '{d}' --protocol Icmp " + "--source-port-ranges '*' --destination-port-ranges '*'" + ) + except Exception as e: + logging.warning(f"Failed to create ICMP allow rules in NSG due to '{e}'") + + def deploy_vm( name: str, location: str = "westus2", @@ -108,11 +129,16 @@ def deploy_vm( f"--boot-diagnostics-storage {boot_storage}", "--generate-ssh-keys", ] + # TODO: Support setting up to NICs. if networking == "SRIOV": vm_command.append("--accelerated-networking true") data: Dict[str, str] = json.loads(local.run(" ".join(vm_command)).stdout) host = data["publicIpAddress"] + + allow_ping(name) + # TODO: Enable auto-shutdown 4 hours from deployment. + return host, data diff --git a/testsuites/test_smoke.py b/testsuites/test_smoke.py index 0e1f157468..70bbe75af6 100644 --- a/testsuites/test_smoke.py +++ b/testsuites/test_smoke.py @@ -30,7 +30,6 @@ class TestSmoke: ping_flag = "-c 1" if platform.system() == "Linux" else "-n 1" def test_ping_1(self) -> None: - # TODO: Can’t ping by default, need to enable. r: Result = self.n.local(f"ping {self.ping_flag} {self.n.host}", warn=True) assert r.ok, f"Pinging {self.n.host} failed" @@ -49,7 +48,6 @@ def test_reboot(self) -> None: assert r.exited == -1, "While SSH worked, reboot failed" def test_ping_2(self) -> None: - # TODO: Can’t ping by default, need to enable. r: Result = self.n.local(f"ping {self.ping_flag} {self.n.host}", warn=True) assert r.ok, f"Pinging {self.n.host} failed" From b62eff0c388cec77f20f8ab1b439ab34c9add91d Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Fri, 16 Oct 2020 23:40:43 -0700 Subject: [PATCH 047/194] Add retry with exponential backoff after reboot --- testsuites/test_smoke.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/testsuites/test_smoke.py b/testsuites/test_smoke.py index 70bbe75af6..948818a5f6 100644 --- a/testsuites/test_smoke.py +++ b/testsuites/test_smoke.py @@ -2,10 +2,10 @@ import logging import platform import socket -import time from invoke.runners import Result # type: ignore from paramiko import SSHException # type: ignore +from tenacity import Retrying, stop_after_delay, wait_exponential # type: ignore import pytest from node_plugin import Node @@ -43,16 +43,24 @@ def test_reboot(self) -> None: except (TimeoutError, SSHException, socket.error) as e: logging.warning(f"SSH failed '{e}', using platform to reboot") self.n.platform_restart() - logging.info("Waiting 10 seconds for reboot to finish") - time.sleep(10) assert r.exited == -1, "While SSH worked, reboot failed" def test_ping_2(self) -> None: - r: Result = self.n.local(f"ping {self.ping_flag} {self.n.host}", warn=True) - assert r.ok, f"Pinging {self.n.host} failed" + for attempt in Retrying( + wait=wait_exponential(), stop=stop_after_delay(30) + ): # type: ignore + with attempt: + r: Result = self.n.local( + f"ping {self.ping_flag} {self.n.host}", warn=True + ) + assert r.ok, f"Pinging {self.n.host} failed" def test_ssh_2(self) -> None: - self.n.run("uptime") + for attempt in Retrying( + wait=wait_exponential(), stop=stop_after_delay(30) + ): # type: ignore + with attempt: + self.n.run("uptime") def test_serial_log(self) -> None: self.n.get_boot_diagnostics() From 7be742500ad836d1958a47aaab93c05327ee9884 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Sat, 17 Oct 2020 18:16:21 -0700 Subject: [PATCH 048/194] Replace tenacity with pytest-rerunfailures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tenacity’s `stop_after_delay` seems to get confused underneath pytest, applying the delay as a total to all code using it, meaning the last test `get_boot_diagnostics` started always failing due to being canceled. There appears to be a similar issue (though fixed) in the library for async functions, where the state on their `RetryState` object doesn’t get reset properly. --- node_plugin.py | 2 -- poetry.lock | 35 ++++++++++++++++------------------- pyproject.toml | 2 +- testsuites/test_smoke.py | 22 ++++++++-------------- 4 files changed, 25 insertions(+), 36 deletions(-) diff --git a/node_plugin.py b/node_plugin.py index 5c9f12979d..1c4514876e 100644 --- a/node_plugin.py +++ b/node_plugin.py @@ -12,7 +12,6 @@ from fabric import Connection from invoke import Context from invoke.runners import Result # type: ignore -from tenacity import retry, stop_after_delay, wait_exponential # type: ignore import pytest @@ -158,7 +157,6 @@ def local(self, *args: Any, **kwargs: Any) -> Result: """This patches Fabric's 'local()' function to ignore SSH environment.""" return super(Connection, self).run(replace_env=False, env={}, *args, **kwargs) - @retry(wait=wait_exponential(), stop=stop_after_delay(60)) def get_boot_diagnostics(self) -> Result: """Gets the serial console logs.""" return self.local( diff --git a/poetry.lock b/poetry.lock index ac379eee62..35ffe4d457 100644 --- a/poetry.lock +++ b/poetry.lock @@ -472,6 +472,17 @@ filelock = ">=3.0" mypy = {version = ">=0.700", markers = "python_version >= \"3.8\""} pytest = ">=3.5" +[[package]] +name = "pytest-rerunfailures" +version = "9.1.1" +description = "pytest plugin to re-run tests to eliminate flaky failures" +category = "main" +optional = false +python-versions = ">=3.5" + +[package.dependencies] +pytest = ">=5.0" + [[package]] name = "pytest-timeout" version = "1.4.2" @@ -550,20 +561,6 @@ category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" -[[package]] -name = "tenacity" -version = "6.2.0" -description = "Retry code until it succeeds" -category = "main" -optional = false -python-versions = "*" - -[package.dependencies] -six = ">=1.9.0" - -[package.extras] -doc = ["reno", "sphinx", "tornado (>=4.5)"] - [[package]] name = "testfixtures" version = "6.15.0" @@ -612,7 +609,7 @@ python-versions = ">=3.6" [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "42ece43921b68dfda0693eb450c7da2797f9971ee0824cc4ab7752691fd71552" +content-hash = "700617dcc49319fa2ef80157ce05864824e81482bd31f36306be622aff7f385c" [metadata.files] appdirs = [ @@ -855,6 +852,10 @@ pytest-mypy = [ {file = "pytest-mypy-0.7.0.tar.gz", hash = "sha256:5a667d9a2b66bf98b3a494411f221923a6e2c3eafbe771104951aaec8985673d"}, {file = "pytest_mypy-0.7.0-py3-none-any.whl", hash = "sha256:e0505ace48d2b19fe686366fce6b4a2ac0d090423736bb6aa2e39554d18974b7"}, ] +pytest-rerunfailures = [ + {file = "pytest-rerunfailures-9.1.1.tar.gz", hash = "sha256:1cb11a17fc121b3918414eb5eaf314ee325f2e693ac7cb3f6abf7560790827f2"}, + {file = "pytest_rerunfailures-9.1.1-py3-none-any.whl", hash = "sha256:2eb7d0ad651761fbe80e064b0fd415cf6730cdbc53c16a145fd84b66143e609f"}, +] pytest-timeout = [ {file = "pytest-timeout-1.4.2.tar.gz", hash = "sha256:20b3113cf6e4e80ce2d403b6fb56e9e1b871b510259206d40ff8d609f48bda76"}, {file = "pytest_timeout-1.4.2-py2.py3-none-any.whl", hash = "sha256:541d7aa19b9a6b4e475c759fd6073ef43d7cdc9a92d95644c260076eb257a063"}, @@ -903,10 +904,6 @@ six = [ {file = "six-1.15.0-py2.py3-none-any.whl", hash = "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"}, {file = "six-1.15.0.tar.gz", hash = "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259"}, ] -tenacity = [ - {file = "tenacity-6.2.0-py2.py3-none-any.whl", hash = "sha256:5a5d3dcd46381abe8b4f82b5736b8726fd3160c6c7161f53f8af7f1eb9b82173"}, - {file = "tenacity-6.2.0.tar.gz", hash = "sha256:29ae90e7faf488a8628432154bb34ace1cca58244c6ea399fd33f066ac71339a"}, -] testfixtures = [ {file = "testfixtures-6.15.0-py2.py3-none-any.whl", hash = "sha256:e17f4f526fc90b0ac9bc7f8ca62b7dec17d9faf3d721f56bda4f0fd94d02f85a"}, {file = "testfixtures-6.15.0.tar.gz", hash = "sha256:409f77cfbdad822d12a8ce5c4aa8fb4d0bb38073f4a5444fede3702716a2cec2"}, diff --git a/pyproject.toml b/pyproject.toml index 68b90e4371..c14a068ac7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ python = "^3.8" pytest = "^6.1.1" fabric = "^2.5.0" pytest-timeout = "^1.4.2" -tenacity = "^6.2.0" +pytest-rerunfailures = "^9.1.1" [tool.poetry.dev-dependencies] black = "^20.8b1" diff --git a/testsuites/test_smoke.py b/testsuites/test_smoke.py index 948818a5f6..588f8b0a94 100644 --- a/testsuites/test_smoke.py +++ b/testsuites/test_smoke.py @@ -5,7 +5,6 @@ from invoke.runners import Result # type: ignore from paramiko import SSHException # type: ignore -from tenacity import Retrying, stop_after_delay, wait_exponential # type: ignore import pytest from node_plugin import Node @@ -29,10 +28,12 @@ class TestSmoke: # TODO: Move to ‘Node.ping()’ ping_flag = "-c 1" if platform.system() == "Linux" else "-n 1" + @pytest.mark.flaky(reruns=5, reruns_delay=5) def test_ping_1(self) -> None: r: Result = self.n.local(f"ping {self.ping_flag} {self.n.host}", warn=True) assert r.ok, f"Pinging {self.n.host} failed" + @pytest.mark.flaky(reruns=5, reruns_delay=5) def test_ssh_1(self) -> None: self.n.run("uptime") @@ -45,22 +46,15 @@ def test_reboot(self) -> None: self.n.platform_restart() assert r.exited == -1, "While SSH worked, reboot failed" + @pytest.mark.flaky(reruns=5, reruns_delay=5) def test_ping_2(self) -> None: - for attempt in Retrying( - wait=wait_exponential(), stop=stop_after_delay(30) - ): # type: ignore - with attempt: - r: Result = self.n.local( - f"ping {self.ping_flag} {self.n.host}", warn=True - ) - assert r.ok, f"Pinging {self.n.host} failed" + r: Result = self.n.local(f"ping {self.ping_flag} {self.n.host}", warn=True) + assert r.ok, f"Pinging {self.n.host} failed" + @pytest.mark.flaky(reruns=5, reruns_delay=5) def test_ssh_2(self) -> None: - for attempt in Retrying( - wait=wait_exponential(), stop=stop_after_delay(30) - ): # type: ignore - with attempt: - self.n.run("uptime") + self.n.run("uptime") + @pytest.mark.flaky(reruns=5, reruns_delay=5) def test_serial_log(self) -> None: self.n.get_boot_diagnostics() From e1c3c03c3c5494f6fb585199b5b8454c7d190d73 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Sat, 17 Oct 2020 20:50:23 -0700 Subject: [PATCH 049/194] Generate HTML report instead of JUnit (XML) --- Makefile | 6 +++--- conftest.py | 4 ++++ poetry.lock | 33 ++++++++++++++++++++++++++++++++- pyproject.toml | 1 + pytest.ini | 1 + 5 files changed, 41 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 2548396019..72cb97df8b 100644 --- a/Makefile +++ b/Makefile @@ -10,17 +10,17 @@ run: # Run local tests test: - @poetry run python -m pytest -rA --capture=tee-sys --tb=short selftests/ + @poetry run python -m pytest --html=test.html -rA --capture=tee-sys --tb=short selftests/ # Run semantic analysis check: - @poetry run python -X dev -X tracemalloc -m pytest --flake8 --mypy -m 'flake8 or mypy' + @poetry run python -X dev -X tracemalloc -m pytest --html=check.html --flake8 --mypy -m 'flake8 or mypy' clean: @poetry run python -m pytest --cache-clear --setup-plan smoke: - @poetry run python -m pytest --quiet --junit-xml=tests.xml --tb=line --show-capture=log -k smoke + @poetry run python -m pytest --quiet --html=smoke.html --self-contained-html --tb=line --show-capture=log -k smoke # Print current Python virtualenv venv: diff --git a/conftest.py b/conftest.py index be29b17567..68b1af6ba7 100644 --- a/conftest.py +++ b/conftest.py @@ -26,4 +26,8 @@ def pytest_addoption(parser: Parser) -> None: # defaults, instead of encoding them in the Makefile +def pytest_html_report_title(report): # type: ignore + report.title = "LISAv3 (Using Pytest) Results" + + LINUX_SCRIPTS = Path("../Testscripts/Linux") diff --git a/poetry.lock b/poetry.lock index 35ffe4d457..9e8582d0cd 100644 --- a/poetry.lock +++ b/poetry.lock @@ -459,6 +459,29 @@ python-versions = "*" flake8 = ">=3.5" pytest = ">=3.5" +[[package]] +name = "pytest-html" +version = "2.1.1" +description = "pytest plugin for generating HTML reports" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +pytest = ">=5.0" +pytest-metadata = "*" + +[[package]] +name = "pytest-metadata" +version = "1.10.0" +description = "pytest plugin for test session metadata" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" + +[package.dependencies] +pytest = ">=2.9.0" + [[package]] name = "pytest-mypy" version = "0.7.0" @@ -609,7 +632,7 @@ python-versions = ">=3.6" [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "700617dcc49319fa2ef80157ce05864824e81482bd31f36306be622aff7f385c" +content-hash = "42edf55535a6b5670f6c14a231f00ae5740743eac5d8599b7e32304b9983049e" [metadata.files] appdirs = [ @@ -848,6 +871,14 @@ pytest-flake8 = [ {file = "pytest-flake8-1.0.6.tar.gz", hash = "sha256:1b82bb58c88eb1db40524018d3fcfd0424575029703b4e2d8e3ee873f2b17027"}, {file = "pytest_flake8-1.0.6-py2.py3-none-any.whl", hash = "sha256:2e91578ecd9b200066f99c1e1de0f510fbb85bcf43712d46ea29fe47607cc234"}, ] +pytest-html = [ + {file = "pytest-html-2.1.1.tar.gz", hash = "sha256:6a4ac391e105e391208e3eb9bd294a60dd336447fd8e1acddff3a6de7f4e57c5"}, + {file = "pytest_html-2.1.1-py2.py3-none-any.whl", hash = "sha256:9e4817e8be8ddde62e8653c8934d0f296b605da3d2277a052f762c56a8b32df2"}, +] +pytest-metadata = [ + {file = "pytest-metadata-1.10.0.tar.gz", hash = "sha256:b7e6e0a45adacb17a03a97bf7a2ef60cc1f4e172bcce9732ce5e814191932315"}, + {file = "pytest_metadata-1.10.0-py2.py3-none-any.whl", hash = "sha256:fcbcc5781aee450107c620c79c57e50796b6777b82b3c504be9cbc3017201169"}, +] pytest-mypy = [ {file = "pytest-mypy-0.7.0.tar.gz", hash = "sha256:5a667d9a2b66bf98b3a494411f221923a6e2c3eafbe771104951aaec8985673d"}, {file = "pytest_mypy-0.7.0-py3-none-any.whl", hash = "sha256:e0505ace48d2b19fe686366fce6b4a2ac0d090423736bb6aa2e39554d18974b7"}, diff --git a/pyproject.toml b/pyproject.toml index c14a068ac7..4e8b6ef2e0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,7 @@ pytest = "^6.1.1" fabric = "^2.5.0" pytest-timeout = "^1.4.2" pytest-rerunfailures = "^9.1.1" +pytest-html = "^2.1.1" [tool.poetry.dev-dependencies] black = "^20.8b1" diff --git a/pytest.ini b/pytest.ini index 5c6d5212b2..ee05d83097 100644 --- a/pytest.ini +++ b/pytest.ini @@ -7,6 +7,7 @@ log_cli = true log_cli_level = INFO log_cli_format = %(asctime)s %(levelname)s %(message)s log_cli_date_format = %Y-%m-%d %H:%M:%S +render_collapsed = true junit_logging = all timeout = 300 filterwarnings = From 788fa16ee5899c8f0fa47cf40706f05b14bd8019 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Sat, 17 Oct 2020 20:58:15 -0700 Subject: [PATCH 050/194] Add reports to gitignore --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000..e1711c78f3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +# Pytest report files +/*.xml +/*.html +/assets From 1e7b0e97368c31a6a5a0e7b54a1141ef3be62875 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Sat, 17 Oct 2020 22:19:26 -0700 Subject: [PATCH 051/194] Revert "Replace tenacity with pytest-rerunfailures" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit a9e4afffa26cc0d7275e6ef593225419afbaf716. The plugin is incompatible with class fixtures, so when the last test in a test class fails, the fixture is torn down (in this case, deleting the node) before the test is retried, which doesn’t work for us. --- node_plugin.py | 2 ++ poetry.lock | 35 +++++++++++++++++++---------------- pyproject.toml | 2 +- testsuites/test_smoke.py | 22 ++++++++++++++-------- 4 files changed, 36 insertions(+), 25 deletions(-) diff --git a/node_plugin.py b/node_plugin.py index 1c4514876e..5c9f12979d 100644 --- a/node_plugin.py +++ b/node_plugin.py @@ -12,6 +12,7 @@ from fabric import Connection from invoke import Context from invoke.runners import Result # type: ignore +from tenacity import retry, stop_after_delay, wait_exponential # type: ignore import pytest @@ -157,6 +158,7 @@ def local(self, *args: Any, **kwargs: Any) -> Result: """This patches Fabric's 'local()' function to ignore SSH environment.""" return super(Connection, self).run(replace_env=False, env={}, *args, **kwargs) + @retry(wait=wait_exponential(), stop=stop_after_delay(60)) def get_boot_diagnostics(self) -> Result: """Gets the serial console logs.""" return self.local( diff --git a/poetry.lock b/poetry.lock index 9e8582d0cd..a665674e39 100644 --- a/poetry.lock +++ b/poetry.lock @@ -495,17 +495,6 @@ filelock = ">=3.0" mypy = {version = ">=0.700", markers = "python_version >= \"3.8\""} pytest = ">=3.5" -[[package]] -name = "pytest-rerunfailures" -version = "9.1.1" -description = "pytest plugin to re-run tests to eliminate flaky failures" -category = "main" -optional = false -python-versions = ">=3.5" - -[package.dependencies] -pytest = ">=5.0" - [[package]] name = "pytest-timeout" version = "1.4.2" @@ -584,6 +573,20 @@ category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +[[package]] +name = "tenacity" +version = "6.2.0" +description = "Retry code until it succeeds" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +six = ">=1.9.0" + +[package.extras] +doc = ["reno", "sphinx", "tornado (>=4.5)"] + [[package]] name = "testfixtures" version = "6.15.0" @@ -632,7 +635,7 @@ python-versions = ">=3.6" [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "42edf55535a6b5670f6c14a231f00ae5740743eac5d8599b7e32304b9983049e" +content-hash = "b46f526aeb728c87b2bb8e3cfb91d66634e9c81e82c0f70c2e48045eba56c915" [metadata.files] appdirs = [ @@ -883,10 +886,6 @@ pytest-mypy = [ {file = "pytest-mypy-0.7.0.tar.gz", hash = "sha256:5a667d9a2b66bf98b3a494411f221923a6e2c3eafbe771104951aaec8985673d"}, {file = "pytest_mypy-0.7.0-py3-none-any.whl", hash = "sha256:e0505ace48d2b19fe686366fce6b4a2ac0d090423736bb6aa2e39554d18974b7"}, ] -pytest-rerunfailures = [ - {file = "pytest-rerunfailures-9.1.1.tar.gz", hash = "sha256:1cb11a17fc121b3918414eb5eaf314ee325f2e693ac7cb3f6abf7560790827f2"}, - {file = "pytest_rerunfailures-9.1.1-py3-none-any.whl", hash = "sha256:2eb7d0ad651761fbe80e064b0fd415cf6730cdbc53c16a145fd84b66143e609f"}, -] pytest-timeout = [ {file = "pytest-timeout-1.4.2.tar.gz", hash = "sha256:20b3113cf6e4e80ce2d403b6fb56e9e1b871b510259206d40ff8d609f48bda76"}, {file = "pytest_timeout-1.4.2-py2.py3-none-any.whl", hash = "sha256:541d7aa19b9a6b4e475c759fd6073ef43d7cdc9a92d95644c260076eb257a063"}, @@ -935,6 +934,10 @@ six = [ {file = "six-1.15.0-py2.py3-none-any.whl", hash = "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"}, {file = "six-1.15.0.tar.gz", hash = "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259"}, ] +tenacity = [ + {file = "tenacity-6.2.0-py2.py3-none-any.whl", hash = "sha256:5a5d3dcd46381abe8b4f82b5736b8726fd3160c6c7161f53f8af7f1eb9b82173"}, + {file = "tenacity-6.2.0.tar.gz", hash = "sha256:29ae90e7faf488a8628432154bb34ace1cca58244c6ea399fd33f066ac71339a"}, +] testfixtures = [ {file = "testfixtures-6.15.0-py2.py3-none-any.whl", hash = "sha256:e17f4f526fc90b0ac9bc7f8ca62b7dec17d9faf3d721f56bda4f0fd94d02f85a"}, {file = "testfixtures-6.15.0.tar.gz", hash = "sha256:409f77cfbdad822d12a8ce5c4aa8fb4d0bb38073f4a5444fede3702716a2cec2"}, diff --git a/pyproject.toml b/pyproject.toml index 4e8b6ef2e0..dfa4d7df93 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,8 +10,8 @@ python = "^3.8" pytest = "^6.1.1" fabric = "^2.5.0" pytest-timeout = "^1.4.2" -pytest-rerunfailures = "^9.1.1" pytest-html = "^2.1.1" +tenacity = "^6.2.0" [tool.poetry.dev-dependencies] black = "^20.8b1" diff --git a/testsuites/test_smoke.py b/testsuites/test_smoke.py index 588f8b0a94..948818a5f6 100644 --- a/testsuites/test_smoke.py +++ b/testsuites/test_smoke.py @@ -5,6 +5,7 @@ from invoke.runners import Result # type: ignore from paramiko import SSHException # type: ignore +from tenacity import Retrying, stop_after_delay, wait_exponential # type: ignore import pytest from node_plugin import Node @@ -28,12 +29,10 @@ class TestSmoke: # TODO: Move to ‘Node.ping()’ ping_flag = "-c 1" if platform.system() == "Linux" else "-n 1" - @pytest.mark.flaky(reruns=5, reruns_delay=5) def test_ping_1(self) -> None: r: Result = self.n.local(f"ping {self.ping_flag} {self.n.host}", warn=True) assert r.ok, f"Pinging {self.n.host} failed" - @pytest.mark.flaky(reruns=5, reruns_delay=5) def test_ssh_1(self) -> None: self.n.run("uptime") @@ -46,15 +45,22 @@ def test_reboot(self) -> None: self.n.platform_restart() assert r.exited == -1, "While SSH worked, reboot failed" - @pytest.mark.flaky(reruns=5, reruns_delay=5) def test_ping_2(self) -> None: - r: Result = self.n.local(f"ping {self.ping_flag} {self.n.host}", warn=True) - assert r.ok, f"Pinging {self.n.host} failed" + for attempt in Retrying( + wait=wait_exponential(), stop=stop_after_delay(30) + ): # type: ignore + with attempt: + r: Result = self.n.local( + f"ping {self.ping_flag} {self.n.host}", warn=True + ) + assert r.ok, f"Pinging {self.n.host} failed" - @pytest.mark.flaky(reruns=5, reruns_delay=5) def test_ssh_2(self) -> None: - self.n.run("uptime") + for attempt in Retrying( + wait=wait_exponential(), stop=stop_after_delay(30) + ): # type: ignore + with attempt: + self.n.run("uptime") - @pytest.mark.flaky(reruns=5, reruns_delay=5) def test_serial_log(self) -> None: self.n.get_boot_diagnostics() From 1a10501188698d8b8bb3ee1c2b9a53f126862dfe Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Sun, 18 Oct 2020 15:10:43 -0700 Subject: [PATCH 052/194] Document use of Tenacity over pytest-rerunfailures --- README.md | 16 ++++++++++++++-- node_plugin.py | 4 ++-- testsuites/test_smoke.py | 6 +++--- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index dd7d3cd183..b82fbfb7f5 100644 --- a/README.md +++ b/README.md @@ -119,7 +119,7 @@ However, this is only one approach, and we may prefer to run the Python code on the user’s machine, with pytest-lisa instead providing the previously mentioned node fixtures, default marks, and requirements logic. -## Paramiko instead of Fabric +### Paramiko instead of Fabric The Paramiko library is less complex (smaller library footprint) than Fabric, as the latter wraps the former, but it is a bit more difficult to use, and doesn’t @@ -154,7 +154,8 @@ def test_lis_version(node: SSHClient) -> None: with Path("state.txt").open as f: assert f.readline() == "TestCompleted" ``` -## StringIO + +### StringIO For `Node.cat()` it would seem we could use `StringIO` like so: @@ -169,3 +170,14 @@ with StringIO() as result: However, the data returned by Paramiko is in bytes, which in Python 3 are not equivalent to strings, hence the existing implementation which uses `BytesIO` and decodes the bytes to a string. + +### pytest-rerunfailures instead of Tenacity + +Due to an open +[bug](https://github.com/pytest-dev/pytest-rerunfailures/issues/51) this popular +Pytest plugin is incompatible with module/class/session fixtures. What this +means is given a class of tests with a class fixture (say a shared `Node`), if +the last test is marked as flaky and is rerun, the class fixture is unexpectedly +torn down and then the test is rerun. That is, the rerun happens too late, and +the test is then performed against a new `Node`. So while slightly more verbose, +we’re back to using [Tenacity](https://github.com/jd/tenacity). diff --git a/node_plugin.py b/node_plugin.py index 5c9f12979d..2eb642ba4d 100644 --- a/node_plugin.py +++ b/node_plugin.py @@ -12,7 +12,7 @@ from fabric import Connection from invoke import Context from invoke.runners import Result # type: ignore -from tenacity import retry, stop_after_delay, wait_exponential # type: ignore +from tenacity import retry, stop_after_attempt, wait_exponential # type: ignore import pytest @@ -158,7 +158,7 @@ def local(self, *args: Any, **kwargs: Any) -> Result: """This patches Fabric's 'local()' function to ignore SSH environment.""" return super(Connection, self).run(replace_env=False, env={}, *args, **kwargs) - @retry(wait=wait_exponential(), stop=stop_after_delay(60)) + @retry(wait=wait_exponential(), stop=stop_after_attempt(5)) def get_boot_diagnostics(self) -> Result: """Gets the serial console logs.""" return self.local( diff --git a/testsuites/test_smoke.py b/testsuites/test_smoke.py index 948818a5f6..279858c72b 100644 --- a/testsuites/test_smoke.py +++ b/testsuites/test_smoke.py @@ -5,7 +5,7 @@ from invoke.runners import Result # type: ignore from paramiko import SSHException # type: ignore -from tenacity import Retrying, stop_after_delay, wait_exponential # type: ignore +from tenacity import Retrying, stop_after_attempt, wait_exponential # type: ignore import pytest from node_plugin import Node @@ -47,7 +47,7 @@ def test_reboot(self) -> None: def test_ping_2(self) -> None: for attempt in Retrying( - wait=wait_exponential(), stop=stop_after_delay(30) + wait=wait_exponential(), stop=stop_after_attempt(5) ): # type: ignore with attempt: r: Result = self.n.local( @@ -57,7 +57,7 @@ def test_ping_2(self) -> None: def test_ssh_2(self) -> None: for attempt in Retrying( - wait=wait_exponential(), stop=stop_after_delay(30) + wait=wait_exponential(), stop=stop_after_attempt(5) ): # type: ignore with attempt: self.n.run("uptime") From b4e8ad9ae2aacfb7d19d997ac0d1db504b426ef9 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Tue, 20 Oct 2020 11:29:29 -0700 Subject: [PATCH 053/194] Add `ping()` to Node --- node_plugin.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/node_plugin.py b/node_plugin.py index 2eb642ba4d..46be85a6c7 100644 --- a/node_plugin.py +++ b/node_plugin.py @@ -3,6 +3,7 @@ import json import logging +import platform import typing from io import BytesIO from uuid import uuid4 @@ -12,7 +13,7 @@ from fabric import Connection from invoke import Context from invoke.runners import Result # type: ignore -from tenacity import retry, stop_after_attempt, wait_exponential # type: ignore +from tenacity import retry, stop_after_delay, wait_exponential # type: ignore import pytest @@ -158,13 +159,18 @@ def local(self, *args: Any, **kwargs: Any) -> Result: """This patches Fabric's 'local()' function to ignore SSH environment.""" return super(Connection, self).run(replace_env=False, env={}, *args, **kwargs) - @retry(wait=wait_exponential(), stop=stop_after_attempt(5)) + @retry(wait=wait_exponential(), stop=stop_after_delay(60)) def get_boot_diagnostics(self) -> Result: """Gets the serial console logs.""" return self.local( f"az vm boot-diagnostics get-boot-log -n {self.name} -g {self.name}-rg" ) + @retry(wait=wait_exponential(), stop=stop_after_delay(30)) + def ping(self, **kwargs: Any) -> Result: + flag = "-c 1" if platform.system() == "Linux" else "-n 1" + return self.local(f"ping {flag} {self.host}", **kwargs) + def platform_restart(self) -> Result: """TODO: Should this '--force' and redeploy?""" return self.local(f"az vm restart -n {self.name} -g {self.name}-rg") From f9bc739706c0a017ed640f5c7bb8a7f34b83a3f1 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Tue, 20 Oct 2020 13:52:42 -0700 Subject: [PATCH 054/194] Redo smoke test using single function --- README.md | 16 ++++++++- testsuites/test_smoke.py | 78 ++++++++++++++++++++-------------------- 2 files changed, 55 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index b82fbfb7f5..33770ab600 100644 --- a/README.md +++ b/README.md @@ -119,6 +119,8 @@ However, this is only one approach, and we may prefer to run the Python code on the user’s machine, with pytest-lisa instead providing the previously mentioned node fixtures, default marks, and requirements logic. +Note that pytest-dist can still be useful for locally running tests in parallel. + ### Paramiko instead of Fabric The Paramiko library is less complex (smaller library footprint) than Fabric, as @@ -171,7 +173,7 @@ However, the data returned by Paramiko is in bytes, which in Python 3 are not equivalent to strings, hence the existing implementation which uses `BytesIO` and decodes the bytes to a string. -### pytest-rerunfailures instead of Tenacity +### Tenacity instead of pytest-rerunfailures Due to an open [bug](https://github.com/pytest-dev/pytest-rerunfailures/issues/51) this popular @@ -181,3 +183,15 @@ the last test is marked as flaky and is rerun, the class fixture is unexpectedly torn down and then the test is rerun. That is, the rerun happens too late, and the test is then performed against a new `Node`. So while slightly more verbose, we’re back to using [Tenacity](https://github.com/jd/tenacity). + +### Function per test instead of class + +An option I explored to make an “executive summary” of the smoke test was to use +a class where each functionality was tested as individual function (meaning they +could fail independently without failing the whole smoke test), accompanied by a +class-scoped node fixture. This had its advantages, however, it was difficult to +parameterize and also overly verbose. We should instead keep each test as Pytest +intends: as a function. This allows the fixtures to be written in a simpler +manner (not rely on caching between functions) and allows parameterization using +the built-in decorator +[`@pytest.mark.parametrize`](https://docs.pytest.org/en/stable/parametrize.html). diff --git a/testsuites/test_smoke.py b/testsuites/test_smoke.py index 279858c72b..315f500c15 100644 --- a/testsuites/test_smoke.py +++ b/testsuites/test_smoke.py @@ -1,19 +1,16 @@ """Runs a 'smoke' test for an Azure Linux VM deployment.""" import logging -import platform import socket from invoke.runners import Result # type: ignore from paramiko import SSHException # type: ignore -from tenacity import Retrying, stop_after_attempt, wait_exponential # type: ignore import pytest from node_plugin import Node @pytest.mark.deploy(setup="OneVM", vm_size="Standard_DS2_v2") -@pytest.mark.usefixtures("class_node") -class TestSmoke: +def test_smoke(urn: str, node: Node) -> None: """Check that a VM can be deployed and is responsive. 1. Deploy the VM (via 'node' fixture) and log it. @@ -22,45 +19,50 @@ class TestSmoke: 4. Attempt to reboot via SSH, otherwise use the platform. 5. Fetch the serial console logs. - """ + For commands where we expect a possible non-zero exit code, we + pass 'warn=True' to prevent it from throwing 'UnexpectedExit' and + we instead check its result at the end. + + SSH failures DO NOT fail this test. - n: Node + """ + logging.info("Pinging before reboot...") + ping1: Result = node.ping(warn=True) - # TODO: Move to ‘Node.ping()’ - ping_flag = "-c 1" if platform.system() == "Linux" else "-n 1" + ssh_errors = (TimeoutError, SSHException, socket.error) - def test_ping_1(self) -> None: - r: Result = self.n.local(f"ping {self.ping_flag} {self.n.host}", warn=True) - assert r.ok, f"Pinging {self.n.host} failed" + try: + logging.info("SSHing before reboot...") + ssh1: Result = node.run("uptime", warn=True) + except ssh_errors as e: + logging.warning(f"SSH before reboot failed: '{e}'") - def test_ssh_1(self) -> None: - self.n.run("uptime") + try: + logging.info("Rebooting...") + # If this succeeds, we should expect the exit code to be -1 + reboot: Result = node.sudo("reboot", warn=True) + except ssh_errors as e: + logging.warning(f"SSH failed, using platform to reboot: '{e}'") + node.platform_restart() + else: + if reboot.exited != -1: + logging.warning("While SSH worked, 'reboot' command failed") - def test_reboot(self) -> None: - try: - # If this succeeds, we should expect the exit code to be -1 - r: Result = self.n.sudo("reboot", warn=True) - except (TimeoutError, SSHException, socket.error) as e: - logging.warning(f"SSH failed '{e}', using platform to reboot") - self.n.platform_restart() - assert r.exited == -1, "While SSH worked, reboot failed" + logging.info("Pinging after reboot...") + ping2: Result = node.ping(warn=True) - def test_ping_2(self) -> None: - for attempt in Retrying( - wait=wait_exponential(), stop=stop_after_attempt(5) - ): # type: ignore - with attempt: - r: Result = self.n.local( - f"ping {self.ping_flag} {self.n.host}", warn=True - ) - assert r.ok, f"Pinging {self.n.host} failed" + try: + logging.info("SSHing after reboot...") + ssh2: Result = node.run("uptime", warn=True) + except ssh_errors as e: + logging.warning(f"SSH after reboot failed: '{e}'") - def test_ssh_2(self) -> None: - for attempt in Retrying( - wait=wait_exponential(), stop=stop_after_attempt(5) - ): # type: ignore - with attempt: - self.n.run("uptime") + logging.info("Retrieving boot diagnostics...") + node.get_boot_diagnostics() - def test_serial_log(self) -> None: - self.n.get_boot_diagnostics() + assert ping1.ok, f"Pinging {node.host} before reboot failed" + if not ssh1.ok: + logging.warning(f"SSH command '{ssh1.command}' before reboot failed") + assert ping2.ok, f"Pinging {node.host} after reboot failed" + if not ssh2.ok: + logging.warning(f"SSH command '{ssh2.command}' after reboot failed") From e94f06ac0b5ac6b10fd0ce30e939f2704b840ec1 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Tue, 20 Oct 2020 14:53:18 -0700 Subject: [PATCH 055/194] Demo test parameterization using smoke test --- node_plugin.py | 10 +++++++++- pytest.ini | 2 +- testsuites/test_smoke.py | 15 ++++++++++++++- 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/node_plugin.py b/node_plugin.py index 46be85a6c7..57897ead64 100644 --- a/node_plugin.py +++ b/node_plugin.py @@ -120,6 +120,12 @@ def deploy_vm( ) local.run(f"az group create -n {name}-rg --location {location}") + # TODO: Accept EULA terms when necessary. Like: + # + # local.run(f"az vm image terms accept --urn {vm_image}") + # + # However, this command fails unless the terms exist and have yet + # to be accepted. vm_command = [ "az vm create", @@ -166,7 +172,7 @@ def get_boot_diagnostics(self) -> Result: f"az vm boot-diagnostics get-boot-log -n {self.name} -g {self.name}-rg" ) - @retry(wait=wait_exponential(), stop=stop_after_delay(30)) + @retry(wait=wait_exponential(), stop=stop_after_delay(60)) def ping(self, **kwargs: Any) -> Result: flag = "-c 1" if platform.system() == "Linux" else "-n 1" return self.local(f"ping {flag} {self.host}", **kwargs) @@ -199,6 +205,7 @@ def node(request: FixtureRequest) -> Iterator[Node]: delete_vm(name) +# TODO: Delete this and resurrect at a later date if we need it again. @pytest.fixture(scope="class") def class_node(request: FixtureRequest) -> Iterator[None]: key, name, host, data, fabric_config = get_node(request) @@ -278,4 +285,5 @@ def get_node( "PATH": "/sbin:/usr/sbin:/usr/local/sbin:/bin:/usr/bin:/usr/local/bin" } fabric_config = fabric.Config(overrides=ssh_config) + logging.info(f"Using VM at: '{host}'") return key, name, host, data, fabric_config diff --git a/pytest.ini b/pytest.ini index ee05d83097..6ade0166fe 100644 --- a/pytest.ini +++ b/pytest.ini @@ -9,7 +9,7 @@ log_cli_format = %(asctime)s %(levelname)s %(message)s log_cli_date_format = %Y-%m-%d %H:%M:%S render_collapsed = true junit_logging = all -timeout = 300 +timeout = 600 filterwarnings = error ignore:unclosed:ResourceWarning diff --git a/testsuites/test_smoke.py b/testsuites/test_smoke.py index 315f500c15..98736d5f8a 100644 --- a/testsuites/test_smoke.py +++ b/testsuites/test_smoke.py @@ -8,8 +8,21 @@ import pytest from node_plugin import Node +# TODO: This is an example of leveraging Pytest’s parameterization +# support. We can implement a small YAML parser to read a playbook at +# runtime to generate this instead of using the below list. +params = [ + pytest.param(i, marks=pytest.mark.deploy(vm_image=i, vm_size="Standard_DS2_v2")) + for i in [ + "citrix:netscalervpx-130:netscalerbyol:latest", + "audiocodes:mediantsessionbordercontroller:mediantvirtualsbcazure:latest", + "credativ:Debian:9:9.0.201706190", + "github:github-enterprise:github-enterprise:latest", + ] +] -@pytest.mark.deploy(setup="OneVM", vm_size="Standard_DS2_v2") + +@pytest.mark.parametrize("urn", params) def test_smoke(urn: str, node: Node) -> None: """Check that a VM can be deployed and is responsive. From ae339c69994ae8584644c12dd8de3e3e301d3561 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Tue, 20 Oct 2020 16:34:16 -0700 Subject: [PATCH 056/194] Change SSH test to just connecting --- testsuites/test_smoke.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/testsuites/test_smoke.py b/testsuites/test_smoke.py index 98736d5f8a..6e7d523c78 100644 --- a/testsuites/test_smoke.py +++ b/testsuites/test_smoke.py @@ -46,7 +46,7 @@ def test_smoke(urn: str, node: Node) -> None: try: logging.info("SSHing before reboot...") - ssh1: Result = node.run("uptime", warn=True) + node.open() except ssh_errors as e: logging.warning(f"SSH before reboot failed: '{e}'") @@ -66,7 +66,7 @@ def test_smoke(urn: str, node: Node) -> None: try: logging.info("SSHing after reboot...") - ssh2: Result = node.run("uptime", warn=True) + node.open() except ssh_errors as e: logging.warning(f"SSH after reboot failed: '{e}'") @@ -74,8 +74,4 @@ def test_smoke(urn: str, node: Node) -> None: node.get_boot_diagnostics() assert ping1.ok, f"Pinging {node.host} before reboot failed" - if not ssh1.ok: - logging.warning(f"SSH command '{ssh1.command}' before reboot failed") assert ping2.ok, f"Pinging {node.host} after reboot failed" - if not ssh2.ok: - logging.warning(f"SSH command '{ssh2.command}' after reboot failed") From 6e0b5cd00c50b617fcf60cd037ac7d9323fbac67 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Tue, 20 Oct 2020 17:02:45 -0700 Subject: [PATCH 057/194] Fix timeout of reboot command MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Note that `warn=True` causes timeouts to be ignored. On some images, the invoked program (it’s not even a shell) just hangs waiting for manual input. So we set a proper timeout. --- node_plugin.py | 5 ++++- testsuites/test_smoke.py | 18 ++++++++++++------ 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/node_plugin.py b/node_plugin.py index 57897ead64..b711f7b4a7 100644 --- a/node_plugin.py +++ b/node_plugin.py @@ -33,7 +33,7 @@ "in_stream": False, # Don’t let remote commands take longer than five minutes # (unless later overridden). This is to prevent hangs. - "timeout": 300, + "command_timeout": 300, } } @@ -168,6 +168,9 @@ def local(self, *args: Any, **kwargs: Any) -> Result: @retry(wait=wait_exponential(), stop=stop_after_delay(60)) def get_boot_diagnostics(self) -> Result: """Gets the serial console logs.""" + # NOTE: Some images can cause the `az` CLI to crash because + # their logs aren’t UTF-8 encoded. I’ve filed a bug: + # https://github.com/Azure/azure-cli/issues/15590 return self.local( f"az vm boot-diagnostics get-boot-log -n {self.name} -g {self.name}-rg" ) diff --git a/testsuites/test_smoke.py b/testsuites/test_smoke.py index 6e7d523c78..e443355e40 100644 --- a/testsuites/test_smoke.py +++ b/testsuites/test_smoke.py @@ -2,7 +2,7 @@ import logging import socket -from invoke.runners import Result # type: ignore +from invoke.runners import CommandTimedOut, Result, UnexpectedExit # type: ignore from paramiko import SSHException # type: ignore import pytest @@ -42,7 +42,7 @@ def test_smoke(urn: str, node: Node) -> None: logging.info("Pinging before reboot...") ping1: Result = node.ping(warn=True) - ssh_errors = (TimeoutError, SSHException, socket.error) + ssh_errors = (TimeoutError, CommandTimedOut, SSHException, socket.error) try: logging.info("SSHing before reboot...") @@ -50,15 +50,18 @@ def test_smoke(urn: str, node: Node) -> None: except ssh_errors as e: logging.warning(f"SSH before reboot failed: '{e}'") + reboot_exit = 0 try: logging.info("Rebooting...") # If this succeeds, we should expect the exit code to be -1 - reboot: Result = node.sudo("reboot", warn=True) + reboot_exit = node.sudo("reboot", timeout=5).exited except ssh_errors as e: logging.warning(f"SSH failed, using platform to reboot: '{e}'") node.platform_restart() - else: - if reboot.exited != -1: + except UnexpectedExit: + # TODO: How do we differentiate reboot working and the SSH + # connection disconnecting for other reasons? + if reboot_exit != -1: logging.warning("While SSH worked, 'reboot' command failed") logging.info("Pinging after reboot...") @@ -71,7 +74,10 @@ def test_smoke(urn: str, node: Node) -> None: logging.warning(f"SSH after reboot failed: '{e}'") logging.info("Retrieving boot diagnostics...") - node.get_boot_diagnostics() + if node.get_boot_diagnostics(warn=True).ok: + logging.info("See full report for boot diagnostics.") + else: + logging.warning("Retrieving boot diagnostics failed.") assert ping1.ok, f"Pinging {node.host} before reboot failed" assert ping2.ok, f"Pinging {node.host} after reboot failed" From 753e83fe633e0aa5ef35e510f5f31e8a336dc441 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Wed, 21 Oct 2020 10:29:10 -0700 Subject: [PATCH 058/194] Improve retry logic and increase command timeouts --- node_plugin.py | 13 +++++++------ pytest.ini | 2 +- testsuites/test_smoke.py | 29 ++++++++++++++++++++++------- 3 files changed, 30 insertions(+), 14 deletions(-) diff --git a/node_plugin.py b/node_plugin.py index b711f7b4a7..b5453eca1f 100644 --- a/node_plugin.py +++ b/node_plugin.py @@ -13,7 +13,7 @@ from fabric import Connection from invoke import Context from invoke.runners import Result # type: ignore -from tenacity import retry, stop_after_delay, wait_exponential # type: ignore +from tenacity import retry, stop_after_attempt, wait_exponential # type: ignore import pytest @@ -33,7 +33,7 @@ "in_stream": False, # Don’t let remote commands take longer than five minutes # (unless later overridden). This is to prevent hangs. - "command_timeout": 300, + "command_timeout": 1200, } } @@ -165,17 +165,18 @@ def local(self, *args: Any, **kwargs: Any) -> Result: """This patches Fabric's 'local()' function to ignore SSH environment.""" return super(Connection, self).run(replace_env=False, env={}, *args, **kwargs) - @retry(wait=wait_exponential(), stop=stop_after_delay(60)) - def get_boot_diagnostics(self) -> Result: + @retry(reraise=True, wait=wait_exponential(), stop=stop_after_attempt(3)) + def get_boot_diagnostics(self, **kwargs: Any) -> Result: """Gets the serial console logs.""" # NOTE: Some images can cause the `az` CLI to crash because # their logs aren’t UTF-8 encoded. I’ve filed a bug: # https://github.com/Azure/azure-cli/issues/15590 return self.local( - f"az vm boot-diagnostics get-boot-log -n {self.name} -g {self.name}-rg" + f"az vm boot-diagnostics get-boot-log -n {self.name} -g {self.name}-rg", + **kwargs, ) - @retry(wait=wait_exponential(), stop=stop_after_delay(60)) + @retry(reraise=True, wait=wait_exponential(), stop=stop_after_attempt(3)) def ping(self, **kwargs: Any) -> Result: flag = "-c 1" if platform.system() == "Linux" else "-n 1" return self.local(f"ping {flag} {self.host}", **kwargs) diff --git a/pytest.ini b/pytest.ini index 6ade0166fe..37ec5db1b4 100644 --- a/pytest.ini +++ b/pytest.ini @@ -9,7 +9,7 @@ log_cli_format = %(asctime)s %(levelname)s %(message)s log_cli_date_format = %Y-%m-%d %H:%M:%S render_collapsed = true junit_logging = all -timeout = 600 +timeout = 1200 filterwarnings = error ignore:unclosed:ResourceWarning diff --git a/testsuites/test_smoke.py b/testsuites/test_smoke.py index e443355e40..4ee97e0bd8 100644 --- a/testsuites/test_smoke.py +++ b/testsuites/test_smoke.py @@ -1,6 +1,7 @@ """Runs a 'smoke' test for an Azure Linux VM deployment.""" import logging import socket +import time from invoke.runners import CommandTimedOut, Result, UnexpectedExit # type: ignore from paramiko import SSHException # type: ignore @@ -40,7 +41,11 @@ def test_smoke(urn: str, node: Node) -> None: """ logging.info("Pinging before reboot...") - ping1: Result = node.ping(warn=True) + ping1 = Result() + try: + ping1 = node.ping() + except UnexpectedExit: + logging.warning(f"Pinging {node.host} before reboot failed") ssh_errors = (TimeoutError, CommandTimedOut, SSHException, socket.error) @@ -64,8 +69,15 @@ def test_smoke(urn: str, node: Node) -> None: if reboot_exit != -1: logging.warning("While SSH worked, 'reboot' command failed") + logging.info("Sleeping for 10 seconds after reboot...") + time.sleep(10) + logging.info("Pinging after reboot...") - ping2: Result = node.ping(warn=True) + ping2 = Result() + try: + ping2 = node.ping() + except UnexpectedExit: + logging.warning(f"Pinging {node.host} after reboot failed") try: logging.info("SSHing after reboot...") @@ -74,10 +86,13 @@ def test_smoke(urn: str, node: Node) -> None: logging.warning(f"SSH after reboot failed: '{e}'") logging.info("Retrieving boot diagnostics...") - if node.get_boot_diagnostics(warn=True).ok: - logging.info("See full report for boot diagnostics.") - else: + try: + node.get_boot_diagnostics() + except UnexpectedExit: logging.warning("Retrieving boot diagnostics failed.") + else: + logging.info("See full report for boot diagnostics.") - assert ping1.ok, f"Pinging {node.host} before reboot failed" - assert ping2.ok, f"Pinging {node.host} after reboot failed" + # NOTE: The test criteria is to fail only if ping fails. + assert ping1.ok + assert ping2.ok From 9d3713de166f9d51eb1281b755786715461392df Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Wed, 21 Oct 2020 10:40:05 -0700 Subject: [PATCH 059/194] Use pytest-rerunfailures in addition to Tenacity They can serve completely different purposes. In this case, re-running the whole test and reporting it as a second test run. --- README.md | 34 +++++++++++++++++++++++----------- poetry.lock | 17 ++++++++++++++++- pyproject.toml | 1 + testsuites/test_smoke.py | 1 + 4 files changed, 41 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 33770ab600..778372dbba 100644 --- a/README.md +++ b/README.md @@ -173,17 +173,6 @@ However, the data returned by Paramiko is in bytes, which in Python 3 are not equivalent to strings, hence the existing implementation which uses `BytesIO` and decodes the bytes to a string. -### Tenacity instead of pytest-rerunfailures - -Due to an open -[bug](https://github.com/pytest-dev/pytest-rerunfailures/issues/51) this popular -Pytest plugin is incompatible with module/class/session fixtures. What this -means is given a class of tests with a class fixture (say a shared `Node`), if -the last test is marked as flaky and is rerun, the class fixture is unexpectedly -torn down and then the test is rerun. That is, the rerun happens too late, and -the test is then performed against a new `Node`. So while slightly more verbose, -we’re back to using [Tenacity](https://github.com/jd/tenacity). - ### Function per test instead of class An option I explored to make an “executive summary” of the smoke test was to use @@ -195,3 +184,26 @@ intends: as a function. This allows the fixtures to be written in a simpler manner (not rely on caching between functions) and allows parameterization using the built-in decorator [`@pytest.mark.parametrize`](https://docs.pytest.org/en/stable/parametrize.html). + +### Tenacity _and_ pytest-rerunfailures + +Due to an open +[bug](https://github.com/pytest-dev/pytest-rerunfailures/issues/51) this popular +Pytest plugin is incompatible with module/class/session fixtures. What this +means is given a class of tests with a class fixture (say a shared `Node`), if +the last test is marked as flaky and is rerun, the class fixture is unexpectedly +torn down and then the test is rerun. That is, the rerun happens too late, and +the test is then performed against a new `Node`. For this reason, to use this +plugin effectively tests would need to be contained to one function per test, +but as written above, that seems to be the best route. + +However, this plugin is otherwise very useful for marking tests as flaky, and is +already integrated with pytest-html such that reruns are reported correctly in +the report. + +For instances where particular parts of code are flaky and need to be rerun, +such as `ping`, we use the modern Python retry library, +[Tenacity](https://github.com/jd/tenacity), which has easy-to-use decorators to +retry functions (and context managers to use within functions), as well as good +wait and timeout support. The `ping()` function currently uses it with +exponential back-off to great effect. diff --git a/poetry.lock b/poetry.lock index a665674e39..6c4d1291ed 100644 --- a/poetry.lock +++ b/poetry.lock @@ -495,6 +495,17 @@ filelock = ">=3.0" mypy = {version = ">=0.700", markers = "python_version >= \"3.8\""} pytest = ">=3.5" +[[package]] +name = "pytest-rerunfailures" +version = "9.1.1" +description = "pytest plugin to re-run tests to eliminate flaky failures" +category = "main" +optional = false +python-versions = ">=3.5" + +[package.dependencies] +pytest = ">=5.0" + [[package]] name = "pytest-timeout" version = "1.4.2" @@ -635,7 +646,7 @@ python-versions = ">=3.6" [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "b46f526aeb728c87b2bb8e3cfb91d66634e9c81e82c0f70c2e48045eba56c915" +content-hash = "d41a721b35ca455e53b898026dcd034d40c64d8837141393e335d4829e293c71" [metadata.files] appdirs = [ @@ -886,6 +897,10 @@ pytest-mypy = [ {file = "pytest-mypy-0.7.0.tar.gz", hash = "sha256:5a667d9a2b66bf98b3a494411f221923a6e2c3eafbe771104951aaec8985673d"}, {file = "pytest_mypy-0.7.0-py3-none-any.whl", hash = "sha256:e0505ace48d2b19fe686366fce6b4a2ac0d090423736bb6aa2e39554d18974b7"}, ] +pytest-rerunfailures = [ + {file = "pytest-rerunfailures-9.1.1.tar.gz", hash = "sha256:1cb11a17fc121b3918414eb5eaf314ee325f2e693ac7cb3f6abf7560790827f2"}, + {file = "pytest_rerunfailures-9.1.1-py3-none-any.whl", hash = "sha256:2eb7d0ad651761fbe80e064b0fd415cf6730cdbc53c16a145fd84b66143e609f"}, +] pytest-timeout = [ {file = "pytest-timeout-1.4.2.tar.gz", hash = "sha256:20b3113cf6e4e80ce2d403b6fb56e9e1b871b510259206d40ff8d609f48bda76"}, {file = "pytest_timeout-1.4.2-py2.py3-none-any.whl", hash = "sha256:541d7aa19b9a6b4e475c759fd6073ef43d7cdc9a92d95644c260076eb257a063"}, diff --git a/pyproject.toml b/pyproject.toml index dfa4d7df93..a6499d9d10 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,7 @@ fabric = "^2.5.0" pytest-timeout = "^1.4.2" pytest-html = "^2.1.1" tenacity = "^6.2.0" +pytest-rerunfailures = "^9.1.1" [tool.poetry.dev-dependencies] black = "^20.8b1" diff --git a/testsuites/test_smoke.py b/testsuites/test_smoke.py index 4ee97e0bd8..bb807b07ab 100644 --- a/testsuites/test_smoke.py +++ b/testsuites/test_smoke.py @@ -24,6 +24,7 @@ @pytest.mark.parametrize("urn", params) +@pytest.mark.flaky(reruns=1) def test_smoke(urn: str, node: Node) -> None: """Check that a VM can be deployed and is responsive. From 0a87e6065b9db6770b1f4f477096a25615867a38 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Wed, 21 Oct 2020 10:40:36 -0700 Subject: [PATCH 060/194] Use East US 2 Azure region by default --- node_plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node_plugin.py b/node_plugin.py index b5453eca1f..f118f64843 100644 --- a/node_plugin.py +++ b/node_plugin.py @@ -93,7 +93,7 @@ def allow_ping(name: str) -> None: def deploy_vm( name: str, - location: str = "westus2", + location: str = "eastus2", vm_image: str = "UbuntuLTS", vm_size: str = "Standard_DS1_v2", setup: str = "", From 3441b763e5b3fd8a8b8af2d404c7227d37494ec8 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Wed, 21 Oct 2020 13:31:48 -0700 Subject: [PATCH 061/194] Move pytest/Makefile to root directory For simpler use. --- Makefile | 27 --------------------------- 1 file changed, 27 deletions(-) delete mode 100644 Makefile diff --git a/Makefile b/Makefile deleted file mode 100644 index 72cb97df8b..0000000000 --- a/Makefile +++ /dev/null @@ -1,27 +0,0 @@ -all: setup test run - -# Install Python packages -setup: - @poetry install --no-ansi --remove-untracked - -# Run Pytest -run: - @poetry run python -m pytest -rA --capture=tee-sys --tb=short - -# Run local tests -test: - @poetry run python -m pytest --html=test.html -rA --capture=tee-sys --tb=short selftests/ - -# Run semantic analysis -check: - @poetry run python -X dev -X tracemalloc -m pytest --html=check.html --flake8 --mypy -m 'flake8 or mypy' - -clean: - @poetry run python -m pytest --cache-clear --setup-plan - -smoke: - @poetry run python -m pytest --quiet --html=smoke.html --self-contained-html --tb=line --show-capture=log -k smoke - -# Print current Python virtualenv -venv: - @poetry env list --no-ansi --full-path From 7618b50d5bd327795a1b73ae9ef6fab19eccc450 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Wed, 21 Oct 2020 17:19:24 -0700 Subject: [PATCH 062/194] Add PyYAML package --- poetry.lock | 23 ++++++++++++++++++++++- pyproject.toml | 1 + 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index 6c4d1291ed..5d7c68ca9b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -557,6 +557,14 @@ rope = ["rope (>0.10.5)"] test = ["versioneer", "pylint (>=2.5.0)", "pytest", "mock", "pytest-cov", "coverage", "numpy", "pandas", "matplotlib", "flaky", "pyqt5"] yapf = ["yapf"] +[[package]] +name = "pyyaml" +version = "5.3.1" +description = "YAML parser and emitter for Python" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + [[package]] name = "regex" version = "2020.9.27" @@ -646,7 +654,7 @@ python-versions = ">=3.6" [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "d41a721b35ca455e53b898026dcd034d40c64d8837141393e335d4829e293c71" +content-hash = "42b398ae9b15852176c7d822f2e27cfb2a50892e031b1e187475ffa0deabcef9" [metadata.files] appdirs = [ @@ -913,6 +921,19 @@ python-language-server = [ {file = "python-language-server-0.35.1.tar.gz", hash = "sha256:6e0c9a3b2ae98e0eb22e98ed6b3c4e190a6bf9e27af53efd2396da60cd92b221"}, {file = "python_language_server-0.35.1-py2.py3-none-any.whl", hash = "sha256:7051090259e3e81c0cdb140de8e32b8f11219808cda4427e6faf61f9ff9a3bf4"}, ] +pyyaml = [ + {file = "PyYAML-5.3.1-cp27-cp27m-win32.whl", hash = "sha256:74809a57b329d6cc0fdccee6318f44b9b8649961fa73144a98735b0aaf029f1f"}, + {file = "PyYAML-5.3.1-cp27-cp27m-win_amd64.whl", hash = "sha256:240097ff019d7c70a4922b6869d8a86407758333f02203e0fc6ff79c5dcede76"}, + {file = "PyYAML-5.3.1-cp35-cp35m-win32.whl", hash = "sha256:4f4b913ca1a7319b33cfb1369e91e50354d6f07a135f3b901aca02aa95940bd2"}, + {file = "PyYAML-5.3.1-cp35-cp35m-win_amd64.whl", hash = "sha256:cc8955cfbfc7a115fa81d85284ee61147059a753344bc51098f3ccd69b0d7e0c"}, + {file = "PyYAML-5.3.1-cp36-cp36m-win32.whl", hash = "sha256:7739fc0fa8205b3ee8808aea45e968bc90082c10aef6ea95e855e10abf4a37b2"}, + {file = "PyYAML-5.3.1-cp36-cp36m-win_amd64.whl", hash = "sha256:69f00dca373f240f842b2931fb2c7e14ddbacd1397d57157a9b005a6a9942648"}, + {file = "PyYAML-5.3.1-cp37-cp37m-win32.whl", hash = "sha256:d13155f591e6fcc1ec3b30685d50bf0711574e2c0dfffd7644babf8b5102ca1a"}, + {file = "PyYAML-5.3.1-cp37-cp37m-win_amd64.whl", hash = "sha256:73f099454b799e05e5ab51423c7bcf361c58d3206fa7b0d555426b1f4d9a3eaf"}, + {file = "PyYAML-5.3.1-cp38-cp38-win32.whl", hash = "sha256:06a0d7ba600ce0b2d2fe2e78453a470b5a6e000a985dd4a4e54e436cc36b0e97"}, + {file = "PyYAML-5.3.1-cp38-cp38-win_amd64.whl", hash = "sha256:95f71d2af0ff4227885f7a6605c37fd53d3a106fcab511b8860ecca9fcf400ee"}, + {file = "PyYAML-5.3.1.tar.gz", hash = "sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d"}, +] regex = [ {file = "regex-2020.9.27-cp27-cp27m-win32.whl", hash = "sha256:d23a18037313714fb3bb5a94434d3151ee4300bae631894b1ac08111abeaa4a3"}, {file = "regex-2020.9.27-cp27-cp27m-win_amd64.whl", hash = "sha256:84e9407db1b2eb368b7ecc283121b5e592c9aaedbe8c78b1a2f1102eb2e21d19"}, diff --git a/pyproject.toml b/pyproject.toml index a6499d9d10..c75adfb2d8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,7 @@ pytest-timeout = "^1.4.2" pytest-html = "^2.1.1" tenacity = "^6.2.0" pytest-rerunfailures = "^9.1.1" +PyYAML = "^5.3.1" [tool.poetry.dev-dependencies] black = "^20.8b1" From df78c88d7b250e22e790a3dcfdcf66de05ad639a Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Wed, 21 Oct 2020 17:20:00 -0700 Subject: [PATCH 063/194] Add proof-of-concept YAML playbook parsing --- conftest.py | 91 ++++++++++++++++++++++++++++++++++++++-- criteria.yaml | 18 ++++++++ testsuites/test_smoke.py | 1 + 3 files changed, 107 insertions(+), 3 deletions(-) create mode 100644 criteria.yaml diff --git a/conftest.py b/conftest.py index 68b1af6ba7..274af6e3f3 100644 --- a/conftest.py +++ b/conftest.py @@ -3,9 +3,26 @@ https://docs.pytest.org/en/stable/writing_plugins.html """ +from __future__ import annotations + +import typing +from functools import partial from pathlib import Path -from _pytest.config.argparsing import Parser +import yaml + +try: + from yaml import CLoader as Loader +except ImportError: + from yaml import Loader # type: ignore + +if typing.TYPE_CHECKING: + from typing import List, Optional + + from _pytest.config import Config + from _pytest.config.argparsing import Parser + + from pytest import Item, Session pytest_plugins = "node_plugin" @@ -16,14 +33,82 @@ def pytest_addoption(parser: Parser) -> None: https://docs.pytest.org/en/latest/example/simple.html """ + # TODO: Add “--lisa” (and “--debug” etc.) options which set up our + # defaults, instead of encoding them in the Makefile parser.addoption( "--keep-vms", action="store_true", default=False, help="Keeps deployed VMs cached between test runs, useful for developers.", ) - # TODO: Add “--lisa” (and “--debug” etc.) options which set up our - # defaults, instead of encoding them in the Makefile + parser.addoption( + "--playbook", type=Path, help="Path to playbook of test selection criteria." + ) + + +def pytest_collection_modifyitems( + session: Session, config: Config, items: List[Item] +) -> None: + """Pytest hook for modifying the selected items (tests). + + https://docs.pytest.org/en/latest/reference.html#pytest.hookspec.pytest_collection_modifyitems + + """ + playbook_path: Optional[Path] = config.getoption("--playbook") + new_items: List[Item] = [] + force_exclude: List[Item] = [] + + def select_item(action: Optional[str], times: int, item: Item) -> None: + """Includes or excludes the item as appropriate.""" + if action == "forceExclude": + print(f" Forcing exclusion of item {item}") + force_exclude.append(item) + else: + print(f" Keeping {item} selected {times} times") + for _ in range(times - new_items.count(item)): + new_items.append(item) + + # TODO: Review, refactor, and fix logging. If we do schema + # validation and have reasonable defaults we can delete most of + # the `is not None` checks. Suggest using: + # https://pypi.org/project/schema/ + if playbook_path: + playbook = dict() + with open(playbook_path) as f: + playbook = yaml.load(f, Loader=Loader) + for play in playbook: + criteria = play.get("criteria") + if criteria is None: + print(f"Criteria missing, cannot parse play {play}") + continue + else: + print(f"Parsing criteria {criteria}") + select_action = play.get("select_action", "forceInclude") + times = play.get("times", 1) + select = partial(select_item, select_action, times) + + name = criteria.get("name") + priority = criteria.get("priority") + area = criteria.get("area") + for i in items: + marker = i.get_closest_marker("lisa") + if marker is None: + # TODO: This should be a warning. + continue + lisa = marker.kwargs + if name is not None: + if i.name.startswith(name): + print(f" Selecting test {i} because name is {name}!") + select(i) + if priority is not None: + if lisa.get("priority") == priority: + print(f" Selecting test {i} because priority is {priority}!") + select(i) + if area and lisa.get("area"): + if lisa["area"].lower() == area: + print(f" Selecting test {i} because area is {area}!") + select(i) + items[:] = [i for i in new_items if i not in force_exclude] def pytest_html_report_title(report): # type: ignore diff --git a/criteria.yaml b/criteria.yaml new file mode 100644 index 0000000000..0758fa3f0a --- /dev/null +++ b/criteria.yaml @@ -0,0 +1,18 @@ +# NOTE: This is a proof-of-concept ask from Chi. + +# select all p0 cases +# for example, selected three cases: a,b,c +- criteria: + priority: 0 +# drop all cases of xdp, +# because it's not ready on a tested distro. +# for example, droped c, so now is: a,b +- criteria: + area: xdp + # forceExclude means not to pick up it again in next rules. + select_action: forceExclude +# run smoke_test cases twice, to prove a distro stable enough +# after this rule, the picked test cases is like a,b,b +- criteria: + name: test_smoke + times: 2 diff --git a/testsuites/test_smoke.py b/testsuites/test_smoke.py index bb807b07ab..fa0108493b 100644 --- a/testsuites/test_smoke.py +++ b/testsuites/test_smoke.py @@ -23,6 +23,7 @@ ] +@pytest.mark.lisa(priority=0) @pytest.mark.parametrize("urn", params) @pytest.mark.flaky(reruns=1) def test_smoke(urn: str, node: Node) -> None: From 8dbb8f62a800632b4821b42928b0ceca25ec2ffc Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Tue, 27 Oct 2020 12:12:55 -0700 Subject: [PATCH 064/194] Add basic user modes --- conftest.py | 43 +++++++++++++++++++++++++++++++------------ pytest.ini | 8 +++++++- 2 files changed, 38 insertions(+), 13 deletions(-) diff --git a/conftest.py b/conftest.py index 274af6e3f3..01f043e022 100644 --- a/conftest.py +++ b/conftest.py @@ -17,7 +17,7 @@ from yaml import Loader # type: ignore if typing.TYPE_CHECKING: - from typing import List, Optional + from typing import Any, Dict, List, Optional from _pytest.config import Config from _pytest.config.argparsing import Parser @@ -33,17 +33,36 @@ def pytest_addoption(parser: Parser) -> None: https://docs.pytest.org/en/latest/example/simple.html """ - # TODO: Add “--lisa” (and “--debug” etc.) options which set up our - # defaults, instead of encoding them in the Makefile - parser.addoption( - "--keep-vms", - action="store_true", - default=False, - help="Keeps deployed VMs cached between test runs, useful for developers.", - ) - parser.addoption( - "--playbook", type=Path, help="Path to playbook of test selection criteria." - ) + parser.addoption("--keep-vms", action="store_true", help="Keeps deployed VMs.") + parser.addoption("--check", action="store_true", help="Run semantic analysis.") + parser.addoption("--demo", action="store_true", help="Run in demo mode.") + parser.addoption("--playbook", type=Path, help="Path to test playbook.") + + +def pytest_configure(config: Config) -> None: + """Set default configurations passed on custom flags.""" + # Search ‘_pytest’ for ‘addoption’ to find these. + options: Dict[str, Any] = {} # See ‘pytest.ini’ for defaults. + if config.getoption("--check"): + options.update( + { + "flake8": True, + "mypy": True, + "markexpr": "flake8 or mypy", + "reportchars": "fE", + } + ) + if config.getoption("--demo"): + options.update( + { + "html": "demo.html", + "no_header": True, + "showcapture": "log", + "tb": "line", + } + ) + for attr, value in options.items(): + setattr(config.option, attr, value) def pytest_collection_modifyitems( diff --git a/pytest.ini b/pytest.ini index 37ec5db1b4..543a90e16a 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,10 +1,16 @@ [pytest] +addopts = + --strict-markers + --self-contained-html + --capture=tee-sys + --tb=short + -rA markers = lisa deploy connect log_cli = true -log_cli_level = INFO +log_cli_level = WARNING log_cli_format = %(asctime)s %(levelname)s %(message)s log_cli_date_format = %Y-%m-%d %H:%M:%S render_collapsed = true From cb0a21a2f5077275280a63d4dca4a4dcd969c372 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Tue, 27 Oct 2020 16:13:06 -0700 Subject: [PATCH 065/194] Draft the Technical Specification Document --- DESIGN.md | 704 +++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 218 ++------------- node_plugin.py | 1 + 3 files changed, 725 insertions(+), 198 deletions(-) create mode 100644 DESIGN.md diff --git a/DESIGN.md b/DESIGN.md new file mode 100644 index 0000000000..57acb4848b --- /dev/null +++ b/DESIGN.md @@ -0,0 +1,704 @@ +# LISAv3 Technical Specification Document + +This document outlines the technical specifications for LISAv3. We are +evaluating the feasibility of leveraging +[Pytest](https://docs.pytest.org/en/stable/) as our test runner. + +Please see [PR #1065](https://github.com/LIS/LISAv2/pull/1065) for a working, +proof-of-concept prototype. + +Authored by Andrew Schwartzmeyer (he/him), version 0.1.0. + +## Why Pytest? + +Pytest is an [incredibly popular](https://docs.pytest.org/en/stable/talks.html) +MIT licensed open source Python testing framework. It has a thriving community +and plugin framework, with over 750 +[plugins](https://plugincompat.herokuapp.com/). Instead of writing (and +therefore maintaining) yet another test framework, we would do less with more by +reusing Pytest and existing plugins. This will allow us to focus on our unique +problems: organizing and understanding our tests, deploying necessary resources +(such as Azure or Hyper-V virtual machines), and analyzing our results. + +In fact, most of Pytest itself is implemented via [built-in +plugins](https://docs.pytest.org/en/stable/plugins.html), providing us with many +useful and well-documented examples. Furthermore, when others were confronted +with a problem similar to our own they also chose to use Pytest. +[Labgrid](https://github.com/labgrid-project/labgrid) is an open source embedded +board control library that delegated the testing framework logic to Pytest in +their [design](https://labgrid.readthedocs.io/en/latest/design_decisions.html), +and [U-Boot](https://github.com/u-boot/u-boot), an embedded board boot loader, +similarly leveraged Pytest in their +[tests](https://github.com/u-boot/u-boot/tree/master/test/py). KernelCI and +Avocado were also evaluated by the Labgrid developers at an [Embedded Linux +Conference](https://youtu.be/S0EJJM5bVUY) and both ruled out for reasons similar +to our own before they settled on Pytest. + +The [fundamental features](https://youtu.be/CMuSn9cofbI) of Pytest match our +needs very well: + +* Automatic test discovery, no boiler-plate test code +* Useful information when a test fails (assertions are introspected) +* Test and fixture parameterization +* Modular setup/teardown via fixtures +* Incredibly customizable (as detailed above) + +So all the logic for describing, discovering, running, skipping based on +requirements, and reporting results of the tests is already written and +maintained by the greater open source community, leaving us to focus on our hard +and specific problem: creating an abstraction to launch the necessary nodes in +our environments. Using Pytest would also allow us the space to abstract other +commonalities in our specific tests. In this way, LISAv3 could solve the +difficulties we have at hand without creating yet another test framework. + +## High-Level Design Decisions + +### What are the User Modes? + +Because Pytest is infinitely customizable, we want to provide a few sets of +reasonable default configurations for some common scenarios. We will add a flag +like `--mode=[dev,debug,ci,demo]` to change the default options and output of +Pytest. Doing so is readily supported by Pytest via the `pytest_addoption` and +`pytest_configure` hooks. We call these the provided “user modes.” + +* The dev(eloper) mode is intended for use by test developers while writing a + new test. It is verbose, caches the deployed VMs between runs, and generates a + digestible [HTML](https://pypi.org/project/pytest-html/) report. + +* The debug mode is like dev mode but with all possible information shown, and + will open the Python debugger automatically on failures (which is provided by + Pytest with the `--pdb` flag). + +* The CI mode will be fairly quiet on the console, showing all test results, but + putting the full info output into the generated report file (HTML for sharing + with humans and + [JUnit](https://docs.pytest.org/en/stable/_modules/_pytest/junitxml.html) for + the associated CI environment, which presents as native test results). + +* The demo mode will show the “executive summary” (a lot like CI, but finely + tuned for demos). For example, what `make smoke` currently shows. + +### How Are Tests Described? + +The built-in [pytest-mark](https://docs.pytest.org/en/stable/mark.html) plugin +already provides functionality for adding metadata to tests, where we +specifically want: + +* Platform: used to skip tests inapplicable to the current system-under-test +* Category: our high-level test organization +* Area: feature being tested (could default to module name) +* Priority: self-explanatory +* Tags: optional additional metadata for test organization + +We simply reuse this with minimal logic to enforce our required metadata, with +sane defaults (perhaps setting the area to the name of the module), and to list +statistics about our test coverage. This is already included in the prototype. +It looks like this: + +```python +import pytest + +@pytest.mark.lisa( + platform="Azure", category="Functional", area="LIS_DEPLOY", priority=0, tags=["lis"] +) +def test_lis_driver_version(node: Node) -> None: + """Checks that the installed drivers have the correct version.""" + ... +``` + +This is a functional example, which takes zero implementation. With this simple +decorator, all test collection hooks can introspect the metadata, enforce +required parameters and set defaults, select tests based on arbitrary criteria, +and list test coverage statistics. + +Note that Pytest leverages Python’s docstrings for built-in documentation (and +can even run tests discovered in such strings, like doctest). Being just Python +code, this decorator need not be `@pytest.mark.lisa(...)` but can trivially be +provided as simply `@lisa(...)`. + +This mark also does need to be repeated for each test, as marks can be scoped to +a module, and so one line could describe defaults for every test in a file, with +individual tests overriding parameters as needed. We may also introduce marks +such as `@pytest.mark.slow` to allow for easier test selection. + +We even have a prototype +[generator](https://github.com/LIS/LISAv2/tree/pytest/generator) which parses +LISAv2 XML test descriptions and generates stubs with this mark filled in +correctly. + +### How Are Tests Selected? + +Pytest already allows a user to specify which exact tests to run: + +* Listing folders on the CLI (see below on where tests should live) +* Specifying a name expression on the CLI (e.g. `-k smoke and xdp`) +* Specifying a mark expression on the CLI (e.g. `-m functional and not slow`) + +We can also implement any other mechanism via the +`pytest_collection_modifyitems` hook. There’s already a +[proof-of-concept](https://github.com/LIS/LISAv2/blob/ab01c33f1f1e1ffac7100f6a69beda07192f05bb/pytest/conftest.py#L49) +which uses selection criteria read from a YAML file: + +```yaml +# Select all Priority 0 tests +- criteria: + priority: 0 +# Exclude all tests in Area "xdp" +- criteria: + area: xdp + select_action: forceExclude +# Run test with name `test_smoke` twice +- criteria: + name: test_smoke + times: 2 +``` + +However, before we settle on the basic schema understood by the +proof-of-concept, we should write and _review_ a full schema. + +### How Are Results Reported? + +Parsing the results of a large test suite can be difficult. Fortunately, because +Pytest is a testing framework, there already exists support for generating +excellent reports. For developers, the +[HTML](https://pypi.org/project/pytest-html/) report is easy to read: it is +self-contained, holds all the results and logs, and each test can be expanded +and collapsed. Tests which were rerun are recorded separately. For CI pipelines, +Pytest has integrated +[JUnit](https://docs.pytest.org/en/stable/_modules/_pytest/junitxml.html) XML +test report support. This is the standard method of reporting results to CI +servers like Jenkins and are natively parsed into the CI system’s built-in test +display page. Finally, Azure DevOps pipelines are even supported with a +community plugin +[pytest-azurepipelines](https://pypi.org/project/pytest-azurepipelines/) which +enhances the standard JUnit report for ADO. + +### How Are Nodes Provided and Accessed? + +First we need to define “node” as an instance of a system-under-test. That is, +given some environment requirements, such an Azure image (URN) and image (SKU), +a node would be a virtual machine deployed by Pytest with SSH access provided to +the tests. A node could optionally be deployed outside Pytest. + +Pytest uses [fixtures](https://docs.pytest.org/en/stable/fixture.html), which +are the primary way of setting up test requirements. They replace less flexible +alternatives like setup/teardown functions. It is through fixtures that we +implement remote node setup/teardown. Our node fixture currently provides: + +* Automatic provisioning of an Azure VM given URN and SKU +* Remote shell access via SSH +* Data including hostname / IP address for local tools +* Cross-platform ping functionality with exponential back-off +* Allowing ICMP ping via Azure firewall rules +* Platform API reboot +* Uploading of local files to arbitrary remote destinations +* Downloading of remote file contents into local string variable +* Downloading boot diagnostics (serial console log) from platform +* Asynchronous remote command execution with promises + +The prototype demonstrates how easy it is to quickly implement these features. +As we need more features, they can be readily added and shared among tests. + +Our abstraction leverages [Fabric](https://www.fabfile.org/) which is a popular +high-level Python library for executing shell commands on remote systems over +SSH. Underneath the covers it uses +[paramiko](https://docs.paramiko.org/en/stable/), the most popular low-level +Python SSH library. Fabric does the heavy lifting of safely connecting and +disconnecting from the node, executing the shell command (synchronously or +asynchronously), reporting the exit status, gathering the stdout and stderr, +providing stdin (or interactive auto-responses, similar to `expect`), uploading +and downloading files, and much more. In fact, these APIs are all available and +implemented for the local machine by the underlying +[Inovke](https://www.pyinvoke.org/) library, which is essentially a Python +`subprocess` wrapper with “a powerful and clean feature set.” + +Other test specific requirements, such as installing software and daemons, +downloading files from remote storage, or checking the state of our Bash test +scripts, would similarly be implemented by methods on the `Node` class or via +additional fixtures and thus shared among tests. + +For Azure, we use the [Azure CLI](https://aka.ms/azureclidocs) to deploy a +virtual machine. For Hyper-V (and other virtualization platforms), we would like +to use [libvirt](https://libvirt.org/python.html), and for embedded +environments we are evaluating +[labgrid](https://github.com/labgrid-project/labgrid). + +Tests do not need to explicitly call for a node to be provided, and we do not +need to write much code to setup this resource-provider logic. We simply define +a `Node` class and a Pytest fixture which returns one: + +```python +@pytest.fixture(scope="session") +def node(request: FixtureRequest) -> Iterator[Node]: + """Return the current node for any test which requests it.""" + with Node() as n: + yield n + +@pytest.mark.lisa(...) +def test_uptime(node: Node) -> None + """Automatically has access to the current node because of the argument.""" + # Runs `uname` via SSH and asserts it's Linux. + assert node.run("uname").stdout.strip() == "Linux" +``` + +When created, the `Node` instance either uses a cached node or deploys a new one +based on the given parameters (which can be provided at runtime). When the scope +of the fixture is exited (in this example, the test session), the `Node` +instance deletes its deployed resource unless requested not to by the user, +which is currently controlled by the `--keep-vms` flag. + +To provide the parameters to the node fixture, the prototype currently +implements a simple `@pytest.mark.deploy(...)` mark which takes `vm_image`, +`vm_size`, etc., and it’s applied to each function. This worked for the demo, +and proved the concept; however, we will want to provide a mechanism for +specifying lists of environments and their required resources to the tests at +runtime. This will likely be a YAML file that is parsed at initialization and +used to parameterize the node fixture itself, causing all the tests to be +executed for each environment. For more details, see the section “Where Does +Parameterization Happen?” + +See the Detailed Design Decisions below for what the `Node` class looks like. + +#### Interaction with Azure + +We do not use the [Azure Python APIs](https://aka.ms/azsdk/python/all) directly +because they are more complicated (and less documented) than the [Azure +CLI](https://aka.ms/azureclidocs). With Invoke (as discussed above), `az` +becomes incredibly easy to work with. The Azure CLI lead developer states that +they have [feature parity](https://stackoverflow.com/a/50005660/1028665) and +that the CLI is more straightforward to use. Considering our ease-of-maintenance +requirement, this seems the apt choice. If it later becomes necessary to use the +Python APIs directly, that is, of course, still allowed by our design. + +### How Are Tests Timed Out? + +The [pytest-timeout](https://pypi.org/project/pytest-timeout/) plugin provides +integrated timeouts via `@pytest.mark.timeout()`, a configuration +file option, environment variable, and CLI flag. The Fabric library provides +timeouts in both the configuration and per-command usage. These are already used +to satisfaction in the prototype. + +### How Are Tests Organized? + +That is, what does a folder of tests map to: a platform, feature, or owner? + +In my opinion it is likely to be both. Tests which are common to a platform and +written by our team are probably best placed in a folder like `tests/azure` +whereas tests for a particular scenario which limits their image and SKU +applicability should be in a folder like `tests/acc`. It’s going to depend on +how often the tests are run together. + +Because Pytest can run tests and `conftest.py` files from arbitrary folders, +maintaining sets of tests and plugins separately from the base LISA repository +is easy. Custom repositories with new tests, plugins, fixtures, +platform-specific support, etc. can simply be cloned anywhere, and provided on +the command-line to Pytest. + +Test authors should keep tests which share requirements and are otherwise +similar to a single module (Python file). Not only is this well-organized, but +because marks can be applied at the module level, setting all the tests to be +skipped or expected to fail (with the built-in `skip` and `xfail` Pytest marks) +becomes even easier. + +An open question is if we really want to bring every test from LISAv2 directly +over, or if we should carefully analyze our tests to craft a new set of +high-level scenarios. An interesting result of reorganizing and rewriting the +tests would be the ability to have test layers, where the result of a high-level +test dictates if the tests below it should be skipped. If it passes, it implies +the tests underneath it would pass, and so skips them; but if it fails, the next +test below it runs and so on until a passing layer is found. + +### How Will We Port LISAv2 Tests? + +Given the above, we still must decide if we want to put the engineering effort +into porting _every_ LISAv2 test. However, the prototype started by porting the +`LIS-DRIVER-VERSION-CHECK` test, proving that tests which exclusively use Bash +scripts are trivially portable. Unfortunately, most tests use an associated +PowerShell script which is tightly coupled to the LISAv2 framework. + +We believe that it is _possible_ to port these tests without untoward +modifications. We would need to write a mock library that implements (or stubs +where appropriate) LISAv2 framework functionality such as +`Provision-VMsForLisa`, `Copy-RemoteFiles`, `Run-LinuxCmd`, etc., and provides +both the expected “global” objects and the test function parameters `AllVmData` +and `CurrentTestData`. + +This work needs to be done regardless of the approach we take with our framework +(leveraging Pytest or writing our own), and it is not inconsequential work. It +needs to be thoroughly planned and executed, and is certainly a ways off. + +### What Do Parallel Tests Mean? + +While our original list of goals stated that we want to run tests “in parallel” +we were not specific about what was meant, and the topic of parallelism and +concurrency is understandably complex. We certainly don’t mean running two tests +at once on the same node, as this would undoubtedly lead to flaky tests. + +Assuming that we care about a set of tests passing on a particular image and +size combination, but not necessarily on a particular deployed instance, then we +can run tests concurrently by deploying multiple “identical” nodes and splitting +the tests across them. The tests would still run in isolation on each node. This +sounds hard, but actually it’s practically free with Pytest if the node fixture +is session scoped and we use +[pytest-xdist](https://pypi.org/project/pytest-xdist/) as described below. + +It’s also unlikely that we want to write our tests using the Async I/O pattern, +because we do not want tests to accidentally conflict with each other. While +[pytest-asyncio](https://pypi.org/project/pytest-asyncio/) exists, our +concurrency model is probably as described above: split the tests among multiple +identical nodes. + +### How Are Tests and Functions Retried? + +Testing remote instances is inherently flaky, so we take a two-pronged approach +to dealing with the flakiness. + +The [pytest-rerunfailures](https://pypi.org/project/pytest-rerunfailures/) +plugin will be used to easily mark a test itself as flaky. It has the nice +feature of recording each rerun in the produced report. It looks like this: + +```python +@pytest.mark.flaky(reruns=5) +def test_something_flaky(...): + """This fails most of the time.""" + ... +``` + +> Note that there is an open +> [bug](https://github.com/pytest-dev/pytest-rerunfailures/issues/51) in this +> plugin which can cause issues with fixtures using scopes other than “function” +> but it can be worked around. + +The [Tenacity](https://tenacity.readthedocs.io/en/latest/) library should be +used to retry flaky functions that are not tests, such as downloading boot +diagnostics or pinging a node. As the modern Python retry library it has +easy-to-use decorators to retry functions (and context managers to use within +functions), as well as excellent wait and timeout support. It looks like this: + +```python +from tenacity import retry, stop_after_attempt, wait_exponential + +class Node: + ... + @retry(reraise=True, wait=wait_exponential(), stop=stop_after_attempt(3)) + def ping(self, **kwargs): + """Ping the node from the local system in a cross-platform manner.""" + flag = "-c 1" if platform.system() == "Linux" else "-n 1" + return self.local(f"ping {flag} {self.host}", **kwargs) + ... +``` + +We can additionally list a test twice when modifying the items collection, as +implemented in the criteria proof-of-concept. However, given the above +abilities, this may not be desired. + +### Where Does Parameterization Happen? + +Do we parameterize +[tests](https://docs.pytest.org/en/stable/parametrize.html#parametrizemark) or +[fixtures](https://docs.pytest.org/en/stable/fixture.html#fixture-parametrize)? + +This all comes down to how we want to use LISA. If we want to put a single +system under test at a time, and run all possible tests against it, then it +would make sense to parameterize the node fixture across the set of images to +test. I believe this to likely be the case. + +A parameterized node fixture would be session-scoped. This would enable us to +take advantage of [pytest-xdist](https://pypi.org/project/pytest-xdist/) for +running the tests concurrently against multiple nodes, where each forked runner +has its own node. Note that the cache key for deployed nodes will need to +include an identifier to separate the parallel runs, but this is available. + +This approach would let us list a number of images and sizes (or a matrix +combination of them) and then run all requested tests against each of those. +However, it means that tests will need to be intelligent enough to [skip or +xfail](https://docs.pytest.org/en/stable/skipping.html) on systems where they do +not apply. This can be done in test code to start with. As commonalities are +realized they can be refactored into simple, reusable feature checks. + +Finally, while the base (and most common) case of tests which require one node +becomes trivially solved, we still have to deal with the edge cases of tests +which use two or three nodes. Determining the best course of action here +requires investigating how and when those tests are run, and if the node pair or +triple all use the same image and size. An easy solution would be to have a test +which requires a second or third node to simply deploy them through a +function-scoped fixture, and tear them down at the end. This may be costly in +terms of time if there are many of these tests and they run frequently, but for +long “performance” tests it would be an adequate option. Alternatively, we could +have a node pool that the session-scoped node fixture uses, where each node is +locked while in use. While this would take more engineering effort, it means we +could use the nodes for running tests concurrently, and “borrow” a runner when a +test needs another. + +Other ideas are welcome, but what we don’t want to do is change the environment +a user is expecting their tests to run in. I do not think that we should use a +“least common denominator” approach that collects feature requests and deploys +nodes which match those features, as the user will lose control over their +environment. We still want to enumerate features so tests can check if they’re +applicable, but the user’s environment request should be respected. + +Alternatively, parameterizing tests means that each test (or module, or class, +as the fixture could no longer be session-scoped) specifies in some way (whether +in code or read at runtime from a file) what image/size combinations it should +run against. This generally eliminates having to check if it should skip, but +means that running the test suite will put multiple systems under test at once, +the results of which may be difficult to interpret. While this is a viable +route, it means maintaining a comprehensive list of which environments each +tests use, and I think feature-checking is more scalable. + +This is an open question which we need to settle as the two methods can +technically be combined, but we will want to be careful if we do this. + +Regardless of approach, we will want to write and _review_ a simple YAML schema +for specifying the system-under-test targets. As described above, the prototype +currently reads this information from a mark, but if we move forward with the +suggestion above, the scope of the node fixture will change to session and it +will become parameterized. Those parameters would be set at runtime by reading a +given YAML file. + +### When Do We Export a Plugin? + +The current prototype is simply using Pytest. All the implementation is in the files +`conftest.py` and `node_plugin.py`, the former of which is Pytest’s default +“user plugin” file. We likely want to create a proper `pytest-lisa` package +which provides our marks, fixtures, command-line parameters, user modes, and +hook modifications for reading YAML files. + +This requires more research as doing so is obviously not necessary but is nice. + +## Detailed Design Decisions + +This section contains truly technical specifications of our current plans to +bring the prototype to production. + +### Planned `Node` Class Refactor + +#### Basic Shape + +`Node` should still subclass `fabric.Connection`. It should be a partially +abstract class with platform-specific subclasses (Azure, libvirt, an embedded +device, etc.). However, the initializer and context manager methods _should not_ +need to be reimplemented by a platform subclass. Most added methods like +`ping()` and `reboot()` should also be shared. This is where static type +checking will help. + +An `Environment` class will be a collection of nodes in a group, for tests which +require multiple nodes. It is important that `Node` is self-contained and does +not require an `Environment` instance because the base case of most tests is to +use a `Node`. + +#### Caching + +A `Node` should be able to be cached. If `--keep-vms` is given to Pytest, it +should not delete the deployed VM resource and should instead cache its data so +that a subsequent invocation can connect directly to it. A `Node` should also be +able to connect directly to a system deployed outside Pytest, reusing the cache +hydration logic. The `init()` and `__exit__()` methods will handle checking and +updating the cache so that this logic is shared. + +Note that cross-session [caching](https://docs.pytest.org/en/stable/cache.html) +is provided by Pytest, and very easy to work with. The existing prototype +already implements `--keep-vms`. + +#### Initializing + +The `init()` method does the following: + +* Takes an optional group ID (provided by Environment for instance so that it’s + easy to create/deploy multiple nodes into one group) to generate its name and + deduce its group. + +* Checks the cache for the node’s key. + +* On a cache miss, calls `deploy()` and saves the returned host to the field + inherited from `Connection` and the rest of the platform-specific information + to a `data` dictionary field. Caches the data dictionary for the node’s key. + +* On a cache hit, saves the cached host and data to the instance. + +* Calls `super()` to setup `Connection` with our default Fabric configuration. + +#### Deploy and Delete + +* The `deploy()` and `delete()` methods are abstract and implemented by + platform-specific node classes to actually deploy the VM. For Azure, note that + `deploy()` will check if the resource group exists, and if not, creates it. + For `delete()` it will check if it is the last VM in the group, and if so + deletes the group too. Again this is to keep `Environment` from being a + requirement. + +* The group ID is `pytest-{uuid4()}` (maybe with `pytest` being replaced by a + user- or run-specific short identifier). The ID should be returned by a static + method so that when an `Environment` creates a collection of nodes, it can + simply use the static method to generate a shared group ID. + +* The context manager’s `__exit__()` method calls `super()` to disconnect and + potentially `delete()` the VM. If it’s to be deleted, the key/value pair is + also removed from the cache. + +* Because of how Python’s context managers work, we may not need to reimplement + `__enter__()` but will want to check its inherited implementation. + +#### Common Tasks + +Common tasks for systems under tests like rebooting and pinging should be +implemented on the `Node` class. + +* Methods inherited from `Connection` include `run()`, `sudo()` and `local()` + which are used to easily run arbitrary commands, and `get()` and `put()` to + download and upload arbitrary files. + +* The `cat()` method (already implemented in the prototype) wraps `get()` and + returns the file as data in a string. This makes test code like this possible: + + ```python + assert node.cat("state.txt") == "TestCompleted" + ``` + +* Reboot should first try to use `self.sudo("reboot", timeout=5)` (with a short + timeout to avoid a hung SSH session). It should retry with an exponential + back-off to see if the machine has rebooted by checking either `uptime` or the + existence of a file created before the reboot. This is to avoid having to + `sleep()` and just guess the amount of time it takes to reboot. + +* Restart should “power cycle” the machine using the platform’s API, and thus is + in abstract method. It should optionally be able to redeploy the node too, + which can be used by tests which require a completely fresh node. + +* Note that the `local()` method is already overridden to patch Fabric so as to + ignore the provided SSH environment. This demonstrates that we can easily + provide necessary changes to users while still leveraging the library. For + instance, we may want an alternative to `run()` which, instead of taking a + string, takes a list of arguments and quotes them correctly so as to deal with + difficult shell quoting edge cases. + +* One new method we’ve already identified is `copy_scripts()` which will copy a + list of scripts to the node and mark them executable. It could even be a + context manager which deletes the scripts when exited. + +## Alternatives Considered + +### Writing Another Framework + +I believe the above set of technical specifications clearly describes how we can +leverage Pytest for our needs. Furthermore, the existing prototype proves this +is a viable option. Therefore I do not think we should consider writing and +maintaining a _new_ Python testing framework. We should avoid falling for “not +invented here” syndrome. The alternative prototype which does implement a new +framework required over five thousand lines of code, the Pytest-based prototype +used less than two hundred, or less than three percent. We do not want to take +on the maintenance cost of yet another framework, the maintenance cost of LISAv2 +already caused this mess in the first place. I think the work of prototyping +said new framework was valuable, as it provided insight into the eventual +technical design of LISAv3. + +### Using Remote Capabilities of pytest-xdist + +With the [pytest-xdist plugin](https://github.com/pytest-dev/pytest-xdist) there +already exists support for running a folder of tests on an arbitrary remote host +via SSH. + +The LISA tests could be written as Python code suitable for running on the +target test system, which means direct access to the system in the test code +itself (subprocesses are still available, without having to use SSH within the +test, but would become far less necessary), something that is not possible with +any current prototype. Where the pytest-xdist plugin copies the package of code +to the target node and runs it, the pytest-lisa plugin could instantiate that +node (boot the necessary image on a remote machine or launch a new Hyper-V or +Azure VM, etc.) for the tests. + +However, this use of pytest-dist requires full Python support on the target +machines, and drastically changes how developers write tests. Furthermore, it +would not support running local commands against the remote node (like ping) or +running the test across a reboot of the node. Thus we do not want to use this +functionality of pytest-xdist. That said, pytest-xdist will still be useful for +running tests concurrently, as described above. + +### Using Paramiko Instead of Fabric + +The Paramiko library is less complex (smaller library footprint) than Fabric, as +the latter wraps the former, but it is a bit more difficult to use, and doesn’t +support reading existing SSH config files, nor does it support “ProxyJump” which +we use heavily. Fabric instead provides a clean high-level interface for +existing shell commands, handling all the connection abstractions for us. + +Using Paramiko looked like this: + +```python +from pathlib import Path +from typing import List + +from paramiko import SSHClient + +import pytest + +@pytest.fixture +def node() -> SSHClient: + with SSHClient() as client: + client.load_system_host_keys() + client.connect(hostname="...") + yield client + + +def test_lis_version(node: SSHClient) -> None: + with node.open_sftp() as sftp: + for f in ["utils.sh", "LIS-VERSION-CHECK.sh"]: + sftp.put(LINUX_SCRIPTS / f, f) + _, stdout, stderr = node.exec_command("./LIS-VERSION-CHECK.sh") + sftp.get("state.txt", "state.txt") + with Path("state.txt").open as f: + assert f.readline() == "TestCompleted" +``` + +It is more verbose than necessary when compared to Fabric. + +### StringIO + +For `Node.cat()` it would seem we could use `StringIO` like so: + +```python +from io import StringIO + +with StringIO() as result: + node.get("state.txt", result) + assert result.getvalue().strip() == "TestCompleted" +``` + +However, the data returned by Paramiko is in bytes, which in Python 3 are not +equivalent to strings, hence the existing implementation which uses `BytesIO` +and decodes the bytes to a string. + +### Writing a Class of Individual Test Methods + +An option I explored to make an “executive summary” of the smoke test was to use +a class where each functionality was tested as individual function (meaning they +could fail independently without failing the whole smoke test), accompanied by a +class-scoped node fixture. This had its advantages, however, it was difficult to +parameterize and also overly verbose. We should instead keep each test as Pytest +intends: as a function. This allows the fixtures to be written in a simpler +manner (not rely on caching between functions) and allows +[parameterization](https://docs.pytest.org/en/stable/parametrize.html) using the +built-in decorator `@pytest.mark.parametrize`. + +However, this decision may be reconsidered if we session-scope and parameterize +the `Node` fixture, in which case these issues are resolved. + +## What Else? + +There’s still a lot more to think about and design. A non-exhaustive list of +future topics (some touched on above): + +* Tests inventory (generating statistics from metadata) +* ARM template support (with Azure CLI) +* Servicing Azure CLI (how stable is their API?) +* libvirt driver support (gives us Hyper-V and more) +* Duration reporting (built-in) +* Self-documentation (via Pydoc) +* Environment class design +* Feature requests (NICs in particular) +* Selection and targets YAML schema +* Secret management +* External results reporting (database and emails) +* Embedded systems / bare metal support +* Managing Python `logging` records +* Managing shell command stdout/stderr diff --git a/README.md b/README.md index 778372dbba..c564d256cf 100644 --- a/README.md +++ b/README.md @@ -1,209 +1,31 @@ # LISAv3 via pytest-lisa -[Pytest](https://docs.pytest.org/en/stable/) is an [incredibly -popular](https://docs.pytest.org/en/stable/talks.html) MIT licensed open source -Python testing framework. It has a thriving community and plugin framework, with -[over 750 plugins](https://plugincompat.herokuapp.com/). There is even a YAML -example of writing a Domain Specific Language -[DSL](https://docs.pytest.org/en/stable/example/nonpython.html#yaml-plugin) for -specifying tests. Instead of writing yet another test framework, LISAv3 could be -written as pytest-lisa, a [plugin for -Pytest](https://docs.pytest.org/en/stable/writing_plugins.html) which implements -our requirements. In fact, most of Pytest itself is implemented via [built-in -plugins](https://docs.pytest.org/en/stable/plugins.html), providing us with a -lot to leverage. +Basic instructions for testing the prototype: -The [fundamental features](https://www.youtube.com/watch?v=CMuSn9cofbI) of -Pytest match our needs very well: +```bash +# Install Poetry, make sure `poetry` is in your `PATH` +curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python -* Automatic test discovery, no boiler-plate test code -* Useful information when a test fails (assertions are introspected) -* Test parameterization -* Modular setup/teardown via fixtures -* Customizable (as detailed above) +# Install Azure CLI, make sure `az` is in your `PATH` +curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash -So all the logic for discovering, running, skipping based on requirements, and -reporting the tests is already written and maintained by the greater open source -community, leaving us to focus on the hard and unique problem: creating an API -to launch the necessary nodes. It would also allow us the space to abstract the -installation of tools required by tests. In this way, LISAv3 could solve the -difficulties we have at hand without creating yet another unit test framework. +# Login and set subscription +az login +az account set -s -## Design +# Clone LISAv2 with the Pytest prototype +git clone -b pytest/main https://github.com/LIS/LISAv2.git +cd LISAv2 -### pytest-mark +# Install Python packages +make setup -The [pytest-mark](https://docs.pytest.org/en/stable/mark.html) already provides -functionality for adding metadata to tests, where we specifically want: +# Run some local demos +make test +make yaml -* Owner -* Category -* Area -* Tags -* Priority - -We could simply reuse this built-in plugin with minimal logic to enforce our -required metadata, with sane defaults (such as setting the area to the name of -the module), and to list statistics about our test coverage. - -It also through pytest-mark that [skipping -functionality](https://docs.pytest.org/en/stable/skipping.html) exists, which we -would leverage for ensuring our environmental requirements are met. - -Note that Pytest leverages Python’s docstrings for built-in documentation (and -can even run tests discovered in such strings, like doctest). - -### Fixtures - -Pytest supports [fixtures](https://docs.pytest.org/en/stable/fixture.html), -which are the primary way of setting up test requirements. They replace less -flexible alternatives like setup/teardown functions. It is through fixtures that -pytest-lisa would implement remote node setup/teardown. Our node fixture would -implement (with more as found to be required): - -* Provision a node based on parameterized requirements -* Reboot the node if requested -* Run a command (perhaps asynchronously) on the node using SSH -* Download and upload files to the node (with retries and timeouts) - -Our abstraction leverages -[Fabric](https://docs.fabfile.org/en/stable/index.html), which uses -[paramiko](https://docs.paramiko.org/en/stable/) underneath, directly to -implement the SSH commands. For deployment logic, it uses the [`az` -CLI](https://aka.ms/azureclidocs), wrapped by Fabric. For Hyper-V (and other -virtualization platforms), it could use -[libvirt](https://libvirt.org/python.html). - -Other test specific requirements, such as installing software and daemons, -downloading files from remote storage, or checking the state of our Bash test -scripts, would similarly be implemented via fixtures and shared among tests. - -### Test result output - -Instead of writing our own test result output, we can leverage existing plugins. -For instance, there already exists -[pytest-azurepipelines](https://pypi.org/project/pytest-azurepipelines/) which -transforms results into the format consumed by ADO. It has over 90,000 downloads -a month. We don’t need to rewrite this. - -## Alternatives considered - -### Azure Python APIs instead of `az` CLI - -We do not use the [Azure Python APIs](https://aka.ms/azsdk/python/all) directly -because they are more complicated (and less documented) than the `az` CLI. Given -Fabric (and its underlying Invoke library), the CLI becomes incredibly easy to -work with. The `az` CLI lead developer states that they have [feature -parity](https://stackoverflow.com/a/50005660/1028665) and that the CLI is more -straightforward to use. Considering our ease-of-maintenance requirement, this -seems the apt choice. - -### pytest-xdist - -With the [pytest-xdist plugin](https://github.com/pytest-dev/pytest-xdist) there -already exists support for running a folder of tests on an arbitrary remote host -via SSH. - -The LISA tests could be written as Python code suitable for running on the -target test system, which means direct access to the system in the test code -itself (subprocesses are still available, without having to use SSH within the -test, but would become far less necessary), something that is not possible with -the current prototype. Where the pytest-xdist plugin copies the package of code -to the target node and runs it, the pytest-lisa plugin could instantiate that -node (boot the necessary image on a remote machine or launch a new Hyper-V or -Azure VM, etc.) for the tests. YAML playbooks (AKA “runbooks” in the current -prototype) could be interpreted by the pytest-lisa plugin to determine how to -create those nodes. - -However, this is only one approach, and we may prefer to run the Python code on -the user’s machine, with pytest-lisa instead providing the previously mentioned -node fixtures, default marks, and requirements logic. - -Note that pytest-dist can still be useful for locally running tests in parallel. - -### Paramiko instead of Fabric - -The Paramiko library is less complex (smaller library footprint) than Fabric, as -the latter wraps the former, but it is a bit more difficult to use, and doesn’t -support reading existing SSH config files, nor does it support “ProxyJump” which -we use heavily. Fabric instead provides a clean high-level interface for -existing shell commands, handling all the connection abstractions for us. - -It looked a like this: - -```python -from pathlib import Path -from typing import List - -from paramiko import SSHClient - -import pytest - -@pytest.fixture -def node() -> SSHClient: - with SSHClient() as client: - client.load_system_host_keys() - client.connect(hostname="...") - yield client - - -def test_lis_version(node: SSHClient) -> None: - with node.open_sftp() as sftp: - for f in ["utils.sh", "LIS-VERSION-CHECK.sh"]: - sftp.put(LINUX_SCRIPTS / f, f) - _, stdout, stderr = node.exec_command("./LIS-VERSION-CHECK.sh") - sftp.get("state.txt", "state.txt") - with Path("state.txt").open as f: - assert f.readline() == "TestCompleted" +# Run a demo which deployes Azure resources +make smoke ``` -### StringIO - -For `Node.cat()` it would seem we could use `StringIO` like so: - -```python -from io import StringIO - -with StringIO() as result: - node.get("state.txt", result) - assert result.getvalue().strip() == "TestCompleted" -``` - -However, the data returned by Paramiko is in bytes, which in Python 3 are not -equivalent to strings, hence the existing implementation which uses `BytesIO` -and decodes the bytes to a string. - -### Function per test instead of class - -An option I explored to make an “executive summary” of the smoke test was to use -a class where each functionality was tested as individual function (meaning they -could fail independently without failing the whole smoke test), accompanied by a -class-scoped node fixture. This had its advantages, however, it was difficult to -parameterize and also overly verbose. We should instead keep each test as Pytest -intends: as a function. This allows the fixtures to be written in a simpler -manner (not rely on caching between functions) and allows parameterization using -the built-in decorator -[`@pytest.mark.parametrize`](https://docs.pytest.org/en/stable/parametrize.html). - -### Tenacity _and_ pytest-rerunfailures - -Due to an open -[bug](https://github.com/pytest-dev/pytest-rerunfailures/issues/51) this popular -Pytest plugin is incompatible with module/class/session fixtures. What this -means is given a class of tests with a class fixture (say a shared `Node`), if -the last test is marked as flaky and is rerun, the class fixture is unexpectedly -torn down and then the test is rerun. That is, the rerun happens too late, and -the test is then performed against a new `Node`. For this reason, to use this -plugin effectively tests would need to be contained to one function per test, -but as written above, that seems to be the best route. - -However, this plugin is otherwise very useful for marking tests as flaky, and is -already integrated with pytest-html such that reruns are reported correctly in -the report. - -For instances where particular parts of code are flaky and need to be rerun, -such as `ping`, we use the modern Python retry library, -[Tenacity](https://github.com/jd/tenacity), which has easy-to-use decorators to -retry functions (and context managers to use within functions), as well as good -wait and timeout support. The `ping()` function currently uses it with -exponential back-off to great effect. +See the [design document](DESIGN.md) for details. diff --git a/node_plugin.py b/node_plugin.py index f118f64843..b2b3cc0a3d 100644 --- a/node_plugin.py +++ b/node_plugin.py @@ -178,6 +178,7 @@ def get_boot_diagnostics(self, **kwargs: Any) -> Result: @retry(reraise=True, wait=wait_exponential(), stop=stop_after_attempt(3)) def ping(self, **kwargs: Any) -> Result: + """Ping the node from the local system in a cross-platform manner.""" flag = "-c 1" if platform.system() == "Linux" else "-n 1" return self.local(f"ping {flag} {self.host}", **kwargs) From 0dcb30c13985377e61bf73da0a4f3bf1bf17f495 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Tue, 3 Nov 2020 12:54:11 -0800 Subject: [PATCH 066/194] Basic automatic grouping of tests based on feature requirement --- selftests/setup_plan/__init__.py | 0 selftests/setup_plan/conftest.py | 6 ++++++ selftests/setup_plan/test_plan_A.py | 16 ++++++++++++++++ selftests/setup_plan/test_plan_B.py | 16 ++++++++++++++++ 4 files changed, 38 insertions(+) create mode 100644 selftests/setup_plan/__init__.py create mode 100644 selftests/setup_plan/conftest.py create mode 100644 selftests/setup_plan/test_plan_A.py create mode 100644 selftests/setup_plan/test_plan_B.py diff --git a/selftests/setup_plan/__init__.py b/selftests/setup_plan/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/selftests/setup_plan/conftest.py b/selftests/setup_plan/conftest.py new file mode 100644 index 0000000000..86f2de80fc --- /dev/null +++ b/selftests/setup_plan/conftest.py @@ -0,0 +1,6 @@ +import pytest + + +@pytest.fixture(scope="session", params=["xdp", "gpu", "rdma"]) +def feature(request) -> str: + yield request.param diff --git a/selftests/setup_plan/test_plan_A.py b/selftests/setup_plan/test_plan_A.py new file mode 100644 index 0000000000..dcc713205d --- /dev/null +++ b/selftests/setup_plan/test_plan_A.py @@ -0,0 +1,16 @@ +import pytest + + +def test_xdp_a(feature) -> None: + if feature != "xdp": + pytest.skip("Required feature missing") + + +def test_gpu_a(feature) -> None: + if feature != "gpu": + pytest.skip("Required feature missing") + + +def test_rdma_a(feature) -> None: + if feature != "rdma": + pytest.skip("Required feature missing") diff --git a/selftests/setup_plan/test_plan_B.py b/selftests/setup_plan/test_plan_B.py new file mode 100644 index 0000000000..2104200ad5 --- /dev/null +++ b/selftests/setup_plan/test_plan_B.py @@ -0,0 +1,16 @@ +import pytest + + +def test_xdp_b(feature) -> None: + if feature != "xdp": + pytest.skip("Required feature missing") + + +def test_gpu_b(feature) -> None: + if feature != "gpu": + pytest.skip("Required feature missing") + + +def test_rdma_b(feature) -> None: + if feature != "rdma": + pytest.skip("Required feature missing") From d3e088e698babf50c983d31352bd3dbb83e50820 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Tue, 3 Nov 2020 16:42:52 -0800 Subject: [PATCH 067/194] Slightly extended example which removes instead of skips tests --- pytest.ini | 1 + selftests/setup_plan/conftest.py | 29 ++++++++++++++++++++++++++++- selftests/setup_plan/test_plan_A.py | 18 +++++++++--------- selftests/setup_plan/test_plan_B.py | 18 +++++++++--------- selftests/setup_plan/test_plan_C.py | 16 ++++++++++++++++ 5 files changed, 63 insertions(+), 19 deletions(-) create mode 100644 selftests/setup_plan/test_plan_C.py diff --git a/pytest.ini b/pytest.ini index 543a90e16a..b242158aa3 100644 --- a/pytest.ini +++ b/pytest.ini @@ -9,6 +9,7 @@ markers = lisa deploy connect + feature log_cli = true log_cli_level = WARNING log_cli_format = %(asctime)s %(levelname)s %(message)s diff --git a/selftests/setup_plan/conftest.py b/selftests/setup_plan/conftest.py index 86f2de80fc..854916983e 100644 --- a/selftests/setup_plan/conftest.py +++ b/selftests/setup_plan/conftest.py @@ -1,6 +1,33 @@ +"""Proof-of-concept to schedule a comprehensive test plan.""" +from __future__ import annotations + +import typing + import pytest +if typing.TYPE_CHECKING: + from typing import List + + from _pytest.config import Config + + from pytest import Item, Session + + +def pytest_collection_modifyitems( + session: Session, config: Config, items: List[Item] +) -> None: + """For each item keep only instances using the feature.""" + keep: List[Item] = [] + for item in items: + marker = item.get_closest_marker("feature") + if marker is None: + continue + feature = marker.args[0] + if item.name.endswith(f"[{feature}]"): + keep.append(item) + items[:] = keep + -@pytest.fixture(scope="session", params=["xdp", "gpu", "rdma"]) +@pytest.fixture(scope="session", autouse=True, params=["xdp", "gpu", "rdma"]) def feature(request) -> str: yield request.param diff --git a/selftests/setup_plan/test_plan_A.py b/selftests/setup_plan/test_plan_A.py index dcc713205d..d0edba2ba1 100644 --- a/selftests/setup_plan/test_plan_A.py +++ b/selftests/setup_plan/test_plan_A.py @@ -1,16 +1,16 @@ import pytest -def test_xdp_a(feature) -> None: - if feature != "xdp": - pytest.skip("Required feature missing") +@pytest.mark.feature("xdp") +def test_xdp_a() -> None: + pass -def test_gpu_a(feature) -> None: - if feature != "gpu": - pytest.skip("Required feature missing") +@pytest.mark.feature("gpu") +def test_gpu_a() -> None: + pass -def test_rdma_a(feature) -> None: - if feature != "rdma": - pytest.skip("Required feature missing") +@pytest.mark.feature("rdma") +def test_rdma_a() -> None: + pass diff --git a/selftests/setup_plan/test_plan_B.py b/selftests/setup_plan/test_plan_B.py index 2104200ad5..c415d8bda7 100644 --- a/selftests/setup_plan/test_plan_B.py +++ b/selftests/setup_plan/test_plan_B.py @@ -1,16 +1,16 @@ import pytest -def test_xdp_b(feature) -> None: - if feature != "xdp": - pytest.skip("Required feature missing") +@pytest.mark.feature("xdp") +def test_xdp_b() -> None: + pass -def test_gpu_b(feature) -> None: - if feature != "gpu": - pytest.skip("Required feature missing") +@pytest.mark.feature("gpu") +def test_gpu_b() -> None: + pass -def test_rdma_b(feature) -> None: - if feature != "rdma": - pytest.skip("Required feature missing") +@pytest.mark.feature("rdma") +def test_rdma_b() -> None: + pass diff --git a/selftests/setup_plan/test_plan_C.py b/selftests/setup_plan/test_plan_C.py new file mode 100644 index 0000000000..ce9d2155fe --- /dev/null +++ b/selftests/setup_plan/test_plan_C.py @@ -0,0 +1,16 @@ +import pytest + + +@pytest.mark.feature("xdp") +def test_xdp_c() -> None: + pass + + +@pytest.mark.feature("gpu") +def test_gpu_c() -> None: + pass + + +@pytest.mark.feature("rdma") +def test_rdma_c() -> None: + pass From ed255098893481a73521a234e38732d3c8b8aabf Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Tue, 3 Nov 2020 16:43:30 -0800 Subject: [PATCH 068/194] Add pytest-xdist --- poetry.lock | 69 +++++++++++++++++++++++++++++++++++++++++++++++++- pyproject.toml | 1 + 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index 5d7c68ca9b..a4a4ef9725 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,3 +1,11 @@ +[[package]] +name = "apipkg" +version = "1.5" +description = "apipkg: namespace control and lazy-import mechanism" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + [[package]] name = "appdirs" version = "1.4.4" @@ -112,6 +120,20 @@ pep8test = ["black", "flake8", "flake8-import-order", "pep8-naming"] ssh = ["bcrypt (>=3.1.5)"] test = ["pytest (>=3.6.0,<3.9.0 || >3.9.0,<3.9.1 || >3.9.1,<3.9.2 || >3.9.2)", "pretend", "iso8601", "pytz", "hypothesis (>=1.11.4,<3.79.2 || >3.79.2)"] +[[package]] +name = "execnet" +version = "1.7.1" +description = "execnet: rapid multi-Python deployment" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[package.dependencies] +apipkg = ">=1.4" + +[package.extras] +testing = ["pre-commit"] + [[package]] name = "fabric" version = "2.5.0" @@ -459,6 +481,18 @@ python-versions = "*" flake8 = ">=3.5" pytest = ">=3.5" +[[package]] +name = "pytest-forked" +version = "1.3.0" +description = "run tests in isolated forked subprocesses" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[package.dependencies] +py = "*" +pytest = ">=3.10" + [[package]] name = "pytest-html" version = "2.1.1" @@ -517,6 +551,23 @@ python-versions = "*" [package.dependencies] pytest = ">=3.6.0" +[[package]] +name = "pytest-xdist" +version = "2.1.0" +description = "pytest xdist plugin for distributed testing and loop-on-failing modes" +category = "main" +optional = false +python-versions = ">=3.5" + +[package.dependencies] +execnet = ">=1.1" +pytest = ">=6.0.0" +pytest-forked = "*" + +[package.extras] +psutil = ["psutil (>=3.0)"] +testing = ["filelock"] + [[package]] name = "python-jsonrpc-server" version = "0.4.0" @@ -654,9 +705,13 @@ python-versions = ">=3.6" [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "42b398ae9b15852176c7d822f2e27cfb2a50892e031b1e187475ffa0deabcef9" +content-hash = "6b221105c12de9baa5fda48ab1839efcb4337e94f02560942d8fa165128fd74f" [metadata.files] +apipkg = [ + {file = "apipkg-1.5-py2.py3-none-any.whl", hash = "sha256:58587dd4dc3daefad0487f6d9ae32b4542b185e1c36db6993290e7c41ca2b47c"}, + {file = "apipkg-1.5.tar.gz", hash = "sha256:37228cda29411948b422fae072f57e31d3396d2ee1c9783775980ee9c9990af6"}, +] appdirs = [ {file = "appdirs-1.4.4-py2.py3-none-any.whl", hash = "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128"}, {file = "appdirs-1.4.4.tar.gz", hash = "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41"}, @@ -751,6 +806,10 @@ cryptography = [ {file = "cryptography-3.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:99d4984aabd4c7182050bca76176ce2dbc9fa9748afe583a7865c12954d714ba"}, {file = "cryptography-3.1.1.tar.gz", hash = "sha256:9d9fc6a16357965d282dd4ab6531013935425d0dc4950df2e0cf2a1b1ac1017d"}, ] +execnet = [ + {file = "execnet-1.7.1-py2.py3-none-any.whl", hash = "sha256:d4efd397930c46415f62f8a31388d6be4f27a91d7550eb79bc64a756e0056547"}, + {file = "execnet-1.7.1.tar.gz", hash = "sha256:cacb9df31c9680ec5f95553976c4da484d407e85e41c83cb812aa014f0eddc50"}, +] fabric = [ {file = "fabric-2.5.0-py2.py3-none-any.whl", hash = "sha256:160331934ea60036604928e792fa8e9f813266b098ef5562aa82b88527740389"}, {file = "fabric-2.5.0.tar.gz", hash = "sha256:24842d7d51556adcabd885ac3cf5e1df73fc622a1708bf3667bf5927576cdfa6"}, @@ -893,6 +952,10 @@ pytest-flake8 = [ {file = "pytest-flake8-1.0.6.tar.gz", hash = "sha256:1b82bb58c88eb1db40524018d3fcfd0424575029703b4e2d8e3ee873f2b17027"}, {file = "pytest_flake8-1.0.6-py2.py3-none-any.whl", hash = "sha256:2e91578ecd9b200066f99c1e1de0f510fbb85bcf43712d46ea29fe47607cc234"}, ] +pytest-forked = [ + {file = "pytest-forked-1.3.0.tar.gz", hash = "sha256:6aa9ac7e00ad1a539c41bec6d21011332de671e938c7637378ec9710204e37ca"}, + {file = "pytest_forked-1.3.0-py2.py3-none-any.whl", hash = "sha256:dc4147784048e70ef5d437951728825a131b81714b398d5d52f17c7c144d8815"}, +] pytest-html = [ {file = "pytest-html-2.1.1.tar.gz", hash = "sha256:6a4ac391e105e391208e3eb9bd294a60dd336447fd8e1acddff3a6de7f4e57c5"}, {file = "pytest_html-2.1.1-py2.py3-none-any.whl", hash = "sha256:9e4817e8be8ddde62e8653c8934d0f296b605da3d2277a052f762c56a8b32df2"}, @@ -913,6 +976,10 @@ pytest-timeout = [ {file = "pytest-timeout-1.4.2.tar.gz", hash = "sha256:20b3113cf6e4e80ce2d403b6fb56e9e1b871b510259206d40ff8d609f48bda76"}, {file = "pytest_timeout-1.4.2-py2.py3-none-any.whl", hash = "sha256:541d7aa19b9a6b4e475c759fd6073ef43d7cdc9a92d95644c260076eb257a063"}, ] +pytest-xdist = [ + {file = "pytest-xdist-2.1.0.tar.gz", hash = "sha256:82d938f1a24186520e2d9d3a64ef7d9ac7ecdf1a0659e095d18e596b8cbd0672"}, + {file = "pytest_xdist-2.1.0-py3-none-any.whl", hash = "sha256:7c629016b3bb006b88ac68e2b31551e7becf173c76b977768848e2bbed594d90"}, +] python-jsonrpc-server = [ {file = "python-jsonrpc-server-0.4.0.tar.gz", hash = "sha256:62c543e541f101ec5b57dc654efc212d2c2e3ea47ff6f54b2e7dcb36ecf20595"}, {file = "python_jsonrpc_server-0.4.0-py3-none-any.whl", hash = "sha256:e5a908ff182e620aac07db5f57887eeb0afe33993008f57dc1b85b594cea250c"}, diff --git a/pyproject.toml b/pyproject.toml index c75adfb2d8..7b24e9418d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,7 @@ pytest-timeout = "^1.4.2" pytest-html = "^2.1.1" tenacity = "^6.2.0" pytest-rerunfailures = "^9.1.1" +pytest-xdist = "^2.1.0" PyYAML = "^5.3.1" [tool.poetry.dev-dependencies] From c200847cac1ef92a884cca2b35d1f44613bb3299 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Tue, 3 Nov 2020 17:15:25 -0800 Subject: [PATCH 069/194] Add filelock package --- poetry.lock | 4 ++-- pyproject.toml | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/poetry.lock b/poetry.lock index a4a4ef9725..fc2c2f3273 100644 --- a/poetry.lock +++ b/poetry.lock @@ -154,7 +154,7 @@ testing = ["mock (>=2.0.0,<3.0)"] name = "filelock" version = "3.0.12" description = "A platform independent file lock." -category = "dev" +category = "main" optional = false python-versions = "*" @@ -705,7 +705,7 @@ python-versions = ">=3.6" [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "6b221105c12de9baa5fda48ab1839efcb4337e94f02560942d8fa165128fd74f" +content-hash = "ee86abdeec8b63e0ff22b16e7d9d6e8aedce399ada42a0cbe395ecd917a42703" [metadata.files] apipkg = [ diff --git a/pyproject.toml b/pyproject.toml index 7b24e9418d..01ba6f0db0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,6 +8,7 @@ license = "MIT License" [tool.poetry.dependencies] python = "^3.8" pytest = "^6.1.1" +filelock = "^3.0.12" fabric = "^2.5.0" pytest-timeout = "^1.4.2" pytest-html = "^2.1.1" From 8be09cbb3fecd5e1e37bcff6c5c98c6c04caaa83 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Tue, 3 Nov 2020 17:45:30 -0800 Subject: [PATCH 070/194] Demonstrate parallelism with shared feature fixture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Running with ‘-n 1’ takes 10 seconds, or 3 seconds for each feature in sequence plus overhead. Running with ‘-n 8’ takes 4 seconds, 3 seconds for each in parallel plus overhead. --- selftests/setup_plan/conftest.py | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/selftests/setup_plan/conftest.py b/selftests/setup_plan/conftest.py index 854916983e..fb9db52c51 100644 --- a/selftests/setup_plan/conftest.py +++ b/selftests/setup_plan/conftest.py @@ -1,14 +1,19 @@ """Proof-of-concept to schedule a comprehensive test plan.""" from __future__ import annotations +import time import typing +from filelock import FileLock # type: ignore + import pytest if typing.TYPE_CHECKING: from typing import List from _pytest.config import Config + from _pytest.fixtures import SubRequest + from _pytest.tmpdir import TempPathFactory from pytest import Item, Session @@ -21,6 +26,7 @@ def pytest_collection_modifyitems( for item in items: marker = item.get_closest_marker("feature") if marker is None: + keep.append(item) continue feature = marker.args[0] if item.name.endswith(f"[{feature}]"): @@ -29,5 +35,24 @@ def pytest_collection_modifyitems( @pytest.fixture(scope="session", autouse=True, params=["xdp", "gpu", "rdma"]) -def feature(request) -> str: - yield request.param +def feature( + request: SubRequest, tmp_path_factory: TempPathFactory, worker_id: str +) -> str: + """Pretend that this sets up the environment.""" + assert request.param + if worker_id == "master": + return str(request.param) + # Get the shared temp directory. + tmp_dir = tmp_path_factory.getbasetemp().parent + fn = tmp_dir / request.param + data: str = "" + with FileLock(str(fn) + ".lock"): + print(f"Worker {worker_id} using feature {request.param}") + if fn.is_file(): + data = fn.read_text() + else: + # Pretend to do some expensive setup and cache it. + time.sleep(3) + data = request.param + fn.write_text(data) + return data From 1c972a0317d9b429bffe80f3ad53bdf06e280716 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Thu, 5 Nov 2020 17:43:35 -0800 Subject: [PATCH 071/194] Proof-of-concept redux with dynamic feature requests Example from `make test`: ``` Created target: set() / {'platform': 'Azure', 'image': 'citrix:netscalervpx-130:netscalerbyol:latest', 'sku': 'Standard_DS1_v2'} Created target: set() / {'platform': 'Azure', 'image': 'audiocodes:mediantsessionbordercontroller:mediantvirtualsbcazure:latest', 'sku': 'Standard_DS1_v2'} Created target: set() / {'platform': 'Azure', 'image': 'credativ:Debian:9:9.0.201706190', 'sku': 'Standard_DS1_v2'} Created target: set() / {'platform': 'Azure', 'image': 'github:github-enterprise:github-enterprise:latest', 'sku': 'Standard_DS1_v2'} Created target: {'xdp'} / {'platform': 'Azure', 'image': 'citrix:netscalervpx-130:netscalerbyol:latest', 'sku': 'Standard_DS1_v2'} Created target: {'xdp'} / {'platform': 'Azure', 'image': 'audiocodes:mediantsessionbordercontroller:mediantvirtualsbcazure:latest', 'sku': 'Standard_DS1_v2'} Created target: {'xdp'} / {'platform': 'Azure', 'image': 'credativ:Debian:9:9.0.201706190', 'sku': 'Standard_DS1_v2'} Created target: {'xdp'} / {'platform': 'Azure', 'image': 'github:github-enterprise:github-enterprise:latest', 'sku': 'Standard_DS1_v2'} Created target: {'gpu'} / {'platform': 'Azure', 'image': 'citrix:netscalervpx-130:netscalerbyol:latest', 'sku': 'Standard_DS1_v2'} Created target: {'gpu'} / {'platform': 'Azure', 'image': 'audiocodes:mediantsessionbordercontroller:mediantvirtualsbcazure:latest', 'sku': 'Standard_DS1_v2'} Created target: {'gpu'} / {'platform': 'Azure', 'image': 'credativ:Debian:9:9.0.201706190', 'sku': 'Standard_DS1_v2'} Created target: {'gpu'} / {'platform': 'Azure', 'image': 'github:github-enterprise:github-enterprise:latest', 'sku': 'Standard_DS1_v2'} Created target: {'rdma'} / {'platform': 'Azure', 'image': 'citrix:netscalervpx-130:netscalerbyol:latest', 'sku': 'Standard_DS1_v2'} Created target: {'rdma'} / {'platform': 'Azure', 'image': 'audiocodes:mediantsessionbordercontroller:mediantvirtualsbcazure:latest', 'sku': 'Standard_DS1_v2'} Created target: {'rdma'} / {'platform': 'Azure', 'image': 'credativ:Debian:9:9.0.201706190', 'sku': 'Standard_DS1_v2'} Created target: {'rdma'} / {'platform': 'Azure', 'image': 'github:github-enterprise:github-enterprise:latest', 'sku': 'Standard_DS1_v2'} ``` --- azure.py | 138 +++++++++++++ conftest.py | 141 +++++++++++-- criteria.yaml | 1 + lisa.py | 48 +++++ node_plugin.py | 294 ---------------------------- selftests/setup_plan/conftest.py | 58 ------ selftests/setup_plan/test_plan_A.py | 15 +- selftests/setup_plan/test_plan_B.py | 15 +- selftests/setup_plan/test_plan_C.py | 15 +- selftests/test_basic.py | 8 +- target.py | 95 +++++++++ targets.yaml | 12 ++ testsuites/test_lis.py | 29 +-- testsuites/test_smoke.py | 45 ++--- 14 files changed, 477 insertions(+), 437 deletions(-) create mode 100644 azure.py create mode 100644 lisa.py delete mode 100644 node_plugin.py delete mode 100644 selftests/setup_plan/conftest.py create mode 100644 target.py create mode 100644 targets.yaml diff --git a/azure.py b/azure.py new file mode 100644 index 0000000000..a20b4bbb32 --- /dev/null +++ b/azure.py @@ -0,0 +1,138 @@ +from __future__ import annotations + +import json +import logging +import typing + +from invoke.runners import Result # type: ignore +from tenacity import retry, stop_after_attempt, wait_exponential # type: ignore + +from target import Target + +if typing.TYPE_CHECKING: + from typing import Any + + +class Azure(Target): + """Implements Azure-specific target methods.""" + + az_ok = False + + def check_az_cli(self) -> None: + """Assert that the `az` CLI is installed and logged in.""" + if Azure.az_ok: + return + # E.g. on Ubuntu: `curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash` + assert self.local("az --version", warn=True), "Please install the `az` CLI!" + # TODO: Login with service principal (az login) and set + # default subscription (az account set -s) using secrets. + account: Result = self.local("az account show") + assert account.ok, "Please `az login`!" + sub = json.loads(account.stdout) + assert sub["isDefault"], "Please `az account set -s `!" + logging.info( + f"Using account '{sub['user']['name']}' with subscription '{sub['name']}'" + ) + Azure.az_ok = True + + def create_boot_storage(self, location: str) -> str: + """Create a separate resource group and storage account for boot diagnostics.""" + account = "pytestbootdiag" + # This command always exits with 0 but returns a string. + if self.local("az group exists -n pytest-lisa").stdout.strip() == "false": + self.local(f"az group create -n pytest-lisa --location {location}") + if not self.local( + f"az storage account show -g pytest-lisa -n {account}", warn=True + ): + self.local(f"az storage account create -g pytest-lisa -n {account}") + return account + + def allow_ping(self) -> None: + """Create NSG rules to enable ICMP ping. + + ICMP ping is disallowed by the Azure load balancer by default, but + there’s strong debate about if this is necessary, and our tests + like to check if the host is up using ping, so we create inbound + and outbound rules in the VM's network security group to allow it. + + """ + try: + for d in ["Inbound", "Outbound"]: + self.local( + f"az network nsg rule create --name allow{d}ICMP " + f"--nsg-name {self.name}NSG --priority 100 --resource-group {self.name}-rg " + f"--access Allow --direction '{d}' --protocol Icmp " + "--source-port-ranges '*' --destination-port-ranges '*'" + ) + except Exception as e: + logging.warning(f"Failed to create ICMP allow rules in NSG due to '{e}'") + + def deploy(self): + """Given deployment info, deploy a new VM.""" + image = self.params["image"] + sku = self.params["sku"] + location = self.params.get("location", "eastus2") + networking = self.params.get("networking", "") + + self.check_az_cli() + + logging.info( + f"""Deploying VM... + Resource Group: '{self.name}-rg' + Region: '{location}' + Image: '{image}' + SKU: '{sku}'""" + ) + + boot_storage = self.create_boot_storage(location) + + self.local(f"az group create -n {self.name}-rg --location {location}") + # TODO: Accept EULA terms when necessary. Like: + # + # local.run(f"az vm image terms accept --urn {vm_image}") + # + # However, this command fails unless the terms exist and have yet + # to be accepted. + + vm_command = [ + "az vm create", + f"-g {self.name}-rg", + f"-n {self.name}", + f"--image {image}", + f"--size {sku}", + f"--boot-diagnostics-storage {boot_storage}", + "--generate-ssh-keys", + ] + # TODO: Support setting up to NICs. + if networking == "SRIOV": + vm_command.append("--accelerated-networking true") + + self.data = json.loads(self.local(" ".join(vm_command)).stdout) + self.allow_ping(self.name) + # TODO: Enable auto-shutdown 4 hours from deployment. + return self.data["publicIpAddress"] + + def delete(self) -> None: + """Delete the entire allocated resource group. + + TODO: Delete VM itself. Only if it was the last VM then delete + the entire resource group. + + """ + logging.info(f"Deleting resource group '{self.name}-rg'") + self.local(f"az group delete -n {self.name}-rg --yes --no-wait") + + @retry(reraise=True, wait=wait_exponential(), stop=stop_after_attempt(3)) + def get_boot_diagnostics(self, **kwargs: Any) -> Result: + """Gets the serial console logs.""" + # NOTE: Some images can cause the `az` CLI to crash because + # their logs aren’t UTF-8 encoded. I’ve filed a bug: + # https://github.com/Azure/azure-cli/issues/15590 + return self.local( + f"az vm boot-diagnostics get-boot-log -n {self.name} -g {self.name}-rg", + **kwargs, + ) + + def platform_restart(self) -> Result: + """TODO: Should this '--force' and redeploy?""" + return self.local(f"az vm restart -n {self.name} -g {self.name}-rg") diff --git a/conftest.py b/conftest.py index 01f043e022..0c80e01648 100644 --- a/conftest.py +++ b/conftest.py @@ -5,26 +5,92 @@ """ from __future__ import annotations +import sys import typing from functools import partial from pathlib import Path import yaml +import lisa + +# TODO: Use importlib instead +from azure import Azure +from target import Target + try: from yaml import CLoader as Loader except ImportError: from yaml import Loader # type: ignore +import pytest + if typing.TYPE_CHECKING: - from typing import Any, Dict, List, Optional + from typing import Any, Dict, Iterator, List, Optional from _pytest.config import Config from _pytest.config.argparsing import Parser + from _pytest.fixtures import FixtureRequest + from _pytest.python import Metafunc from pytest import Item, Session -pytest_plugins = "node_plugin" + +LISA = pytest.mark.lisa +LINUX_SCRIPTS = Path("../Testscripts/Linux") + + +@pytest.fixture(scope="session") +def pool(request: FixtureRequest) -> Iterator[List[Target]]: + """This fixture tracks all deployed target resources.""" + targets: List[Target] = [] + yield targets + for t in targets: + print(f"Created target: {t.features} / {t.params}") + if not request.config.getoption("keep_vms"): + t.delete() + + +@pytest.fixture +def target(pool, worker_id, request: FixtureRequest) -> Iterator[Target]: + """This fixture provides a connected target for each test. + + It is parametrized indirectly in 'pytest_generate_tests'. + + In this fixture we can check if any existing target matches all + the requirements. If so, we can re-use that target, and if not, we + can deallocate the currently running target and allocate a new + one. When all tests are finished, the pool fixture above will + delete all created VMs. Coupled with performing discrete + optimization in the test collection phase and ordering the tests + such that the test(s) with the lowest common denominator + requirements are executed first, we have the two-layer scheduling + as asked. + + However, this feels like putting the cart before the horse to me. + It would be much simpler in terms of design, implementation, and + usage that features are specified upfront when the targets are + specified. Then all this goes away, and tests are skipped when the + feature is missing, which also leaves users in full control of + their environments. + + """ + params = request.param + marker = request.node.get_closest_marker("lisa") + features = marker.kwargs["features"] + for t in pool: + # TODO: Implement full feature comparison, etc. and not just + # proof-of-concept string set comparison. + if params == t.params and features <= t.features: + yield t + break + else: + # TODO: Reimplement caching. + # TODO: Dynamically load platform module and use it. + t = Azure(params, features) + pool.append(t) + yield t + t.connection.close() def pytest_addoption(parser: Parser) -> None: @@ -36,11 +102,39 @@ def pytest_addoption(parser: Parser) -> None: parser.addoption("--keep-vms", action="store_true", help="Keeps deployed VMs.") parser.addoption("--check", action="store_true", help="Run semantic analysis.") parser.addoption("--demo", action="store_true", help="Run in demo mode.") - parser.addoption("--playbook", type=Path, help="Path to test playbook.") + parser.addoption("--targets", type=Path, help="Path to targets playbook.") + parser.addoption("--criteria", type=Path, help="Path to criteria playbook.") + + +TARGETS = [] +TARGET_IDS = [] def pytest_configure(config: Config) -> None: - """Set default configurations passed on custom flags.""" + """Parse provided user inputs to setup configuration. + + Determines the targets based on the playbook and sets default + configurations based user mode. + """ + playbook_path: Optional[Path] = config.getoption("--targets") + if playbook_path: + playbook = dict() + with open(playbook_path) as f: + playbook = yaml.load(f, Loader=Loader) + for play in playbook: + t = play.get("target") + if t is None: + continue + else: + print(f"Parsing target {t}") + setup = { + "platform": t.get("platform", "Azure"), + "image": t.get("image", "UbuntuLTS"), + "sku": t.get("sku", "Standard_DS1_v2"), + } + TARGETS.append(setup) + TARGET_IDS.append("-".join(setup.values())) + # Search ‘_pytest’ for ‘addoption’ to find these. options: Dict[str, Any] = {} # See ‘pytest.ini’ for defaults. if config.getoption("--check"): @@ -65,6 +159,19 @@ def pytest_configure(config: Config) -> None: setattr(config.option, attr, value) +def pytest_generate_tests(metafunc: Metafunc): + """Parametrize the tests based on our inputs. + + Note that this hook is run for each test, so we do the file I/O in + 'pytest_configure' and save the results. + + """ + # TODO: Provide a default target? + assert TARGETS, "No targets specified!" + if "target" in metafunc.fixturenames: + metafunc.parametrize("target", TARGETS, True, TARGET_IDS) + + def pytest_collection_modifyitems( session: Session, config: Config, items: List[Item] ) -> None: @@ -73,7 +180,14 @@ def pytest_collection_modifyitems( https://docs.pytest.org/en/latest/reference.html#pytest.hookspec.pytest_collection_modifyitems """ - playbook_path: Optional[Path] = config.getoption("--playbook") + # Validate LISA mark on every item. + for item in items: + mark = item.get_closest_marker("lisa") + assert mark, f"{item} is missing required LISA marker!" + lisa.validate(mark) + + # Optionally select tests based on a playbook. + playbook_path: Optional[Path] = config.getoption("--criteria") new_items: List[Item] = [] force_exclude: List[Item] = [] @@ -98,7 +212,6 @@ def select_item(action: Optional[str], times: int, item: Item) -> None: for play in playbook: criteria = play.get("criteria") if criteria is None: - print(f"Criteria missing, cannot parse play {play}") continue else: print(f"Parsing criteria {criteria}") @@ -110,21 +223,18 @@ def select_item(action: Optional[str], times: int, item: Item) -> None: priority = criteria.get("priority") area = criteria.get("area") for i in items: - marker = i.get_closest_marker("lisa") - if marker is None: - # TODO: This should be a warning. - continue - lisa = marker.kwargs + marker = i.get_closest_marker("LISA") + args = marker.kwargs if name is not None: if i.name.startswith(name): print(f" Selecting test {i} because name is {name}!") select(i) if priority is not None: - if lisa.get("priority") == priority: + if args.get("priority") == priority: print(f" Selecting test {i} because priority is {priority}!") select(i) - if area and lisa.get("area"): - if lisa["area"].lower() == area: + if area and args.get("area"): + if args["area"].lower() == area: print(f" Selecting test {i} because area is {area}!") select(i) items[:] = [i for i in new_items if i not in force_exclude] @@ -132,6 +242,3 @@ def select_item(action: Optional[str], times: int, item: Item) -> None: def pytest_html_report_title(report): # type: ignore report.title = "LISAv3 (Using Pytest) Results" - - -LINUX_SCRIPTS = Path("../Testscripts/Linux") diff --git a/criteria.yaml b/criteria.yaml index 0758fa3f0a..3e08a20942 100644 --- a/criteria.yaml +++ b/criteria.yaml @@ -1,5 +1,6 @@ # NOTE: This is a proof-of-concept ask from Chi. + # select all p0 cases # for example, selected three cases: a,b,c - criteria: diff --git a/lisa.py b/lisa.py new file mode 100644 index 0000000000..d0d906a9c6 --- /dev/null +++ b/lisa.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +import typing + +if typing.TYPE_CHECKING: + from _pytest.mark.structures import Mark + +# Setup a sane configuration for local and remote commands. Note that +# the defaults between Fabric and Invoke are different, so we use +# their Config classes explicitly. +config = { + "run": { + # Show each command as its run. + "echo": True, + # Disable stdin forwarding. + "in_stream": False, + # Don’t let remote commands take longer than five minutes + # (unless later overridden). This is to prevent hangs. + "command_timeout": 1200, + } +} + + +def validate(mark: Mark): + """Validate each test's LISA parameters.""" + assert not mark.args, "LISA marker cannot have positional arguments!" + args = mark.kwargs + + if args.get("platform"): + assert type(args["platform"]) is str, "Platform must be a string!" + + if args.get("priority") is not None: + assert type(args["priority"]) is int, "Priority must be an integer!" + + if args.get("features") is not None: + if type(args["features"]) is str: + # Convert single ‘str’ argument to ‘Set[str]’ + features = set() + features.add(args["features"]) + args["features"] = features + elif type(args["features"]) is list: + # Convert ‘list’ to ‘set’ + args["features"] = set(args["features"]) + assert type(args["features"]) is set, "Features must be a set!" + for feature in args["features"]: + assert type(feature) is str, "Features must be strings!" + else: + args["features"] = set() diff --git a/node_plugin.py b/node_plugin.py deleted file mode 100644 index b2b3cc0a3d..0000000000 --- a/node_plugin.py +++ /dev/null @@ -1,294 +0,0 @@ -"""Pytest plugin implementing a Node fixture for running remote commands.""" -from __future__ import annotations - -import json -import logging -import platform -import typing -from io import BytesIO -from uuid import uuid4 - -import fabric # type: ignore -import invoke # type: ignore -from fabric import Connection -from invoke import Context -from invoke.runners import Result # type: ignore -from tenacity import retry, stop_after_attempt, wait_exponential # type: ignore - -import pytest - -if typing.TYPE_CHECKING: - from typing import Any, Dict, Iterator, Optional, Tuple - - from _pytest.fixtures import FixtureRequest - -# Setup a sane configuration for local and remote commands. Note that -# the defaults between Fabric and Invoke are different, so we use -# their Config classes explicitly. -config = { - "run": { - # Show each command as its run. - "echo": True, - # Disable stdin forwarding. - "in_stream": False, - # Don’t let remote commands take longer than five minutes - # (unless later overridden). This is to prevent hangs. - "command_timeout": 1200, - } -} - - -# Provide a configured local Invoke context for running commands -# before establishing a connection. (Use like `local.run(...)`). -invoke_config = invoke.Config(overrides=config) -local = Context(config=invoke_config) - - -def check_az_cli() -> None: - """Assert that the `az` CLI is installed and logged in.""" - # E.g. on Ubuntu: `curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash` - assert local.run("az --version", warn=True), "Please install the `az` CLI!" - # TODO: Login with service principal (az login) and set - # default subscription (az account set -s) using secrets. - account: Result = local.run("az account show") - assert account.ok, "Please `az login`!" - sub = json.loads(account.stdout) - assert sub["isDefault"], "Please `az account set -s `!" - logging.info( - f"Using account '{sub['user']['name']}' with subscription '{sub['name']}'" - ) - - -def create_boot_storage(location: str) -> str: - """Create a separate resource group and storage account for boot diagnostics.""" - account = "pytestbootdiag" - # This command always exits with 0 but returns a string. - if local.run("az group exists -n pytest-lisa").stdout.strip() == "false": - local.run(f"az group create -n pytest-lisa --location {location}") - if not local.run(f"az storage account show -g pytest-lisa -n {account}", warn=True): - local.run(f"az storage account create -g pytest-lisa -n {account}") - return account - - -def allow_ping(name: str) -> None: - """Create NSG rules to enable ICMP ping. - - ICMP ping is disallowed by the Azure load balancer by default, but - there’s strong debate about if this is necessary, and our tests - like to check if the host is up using ping, so we create inbound - and outbound rules in the VM's network security group to allow it. - - """ - try: - for d in ["Inbound", "Outbound"]: - local.run( - f"az network nsg rule create --name allow{d}ICMP " - f"--nsg-name {name}NSG --priority 100 --resource-group {name}-rg " - f"--access Allow --direction '{d}' --protocol Icmp " - "--source-port-ranges '*' --destination-port-ranges '*'" - ) - except Exception as e: - logging.warning(f"Failed to create ICMP allow rules in NSG due to '{e}'") - - -def deploy_vm( - name: str, - location: str = "eastus2", - vm_image: str = "UbuntuLTS", - vm_size: str = "Standard_DS1_v2", - setup: str = "", - networking: str = "", -) -> Tuple[str, Dict[str, str]]: - """Given deployment info, deploy a new VM. - - TODO: This along with the functions it calls are Azure specific - and so would be refactored to support other platforms. Hence it - returns both the host and the deployment data so that calling - functions don't have to know which field in the data corresponds - to the host. - - """ - check_az_cli() - boot_storage = create_boot_storage(location) - - logging.info( - f"""Deploying VM... - Resource Group: '{name}-rg' - Region: '{location}' - Image: '{vm_image}' - Size: '{vm_size}'""" - ) - - local.run(f"az group create -n {name}-rg --location {location}") - # TODO: Accept EULA terms when necessary. Like: - # - # local.run(f"az vm image terms accept --urn {vm_image}") - # - # However, this command fails unless the terms exist and have yet - # to be accepted. - - vm_command = [ - "az vm create", - f"-g {name}-rg", - f"-n {name}", - f"--image {vm_image}", - f"--size {vm_size}", - f"--boot-diagnostics-storage {boot_storage}", - "--generate-ssh-keys", - ] - # TODO: Support setting up to NICs. - if networking == "SRIOV": - vm_command.append("--accelerated-networking true") - - data: Dict[str, str] = json.loads(local.run(" ".join(vm_command)).stdout) - host = data["publicIpAddress"] - - allow_ping(name) - # TODO: Enable auto-shutdown 4 hours from deployment. - - return host, data - - -def delete_vm(name: str) -> None: - """Delete the entire allocated resource group.""" - logging.info(f"Deleting resource group '{name}-rg'") - local.run(f"az group delete -n {name}-rg --yes --no-wait") - - -class Node(Connection): - """Extends 'fabric.Connection' with our own utilities.""" - - name: str - data: Dict[str, str] - - def local(self, *args: Any, **kwargs: Any) -> Result: - """This patches Fabric's 'local()' function to ignore SSH environment.""" - return super(Connection, self).run(replace_env=False, env={}, *args, **kwargs) - - @retry(reraise=True, wait=wait_exponential(), stop=stop_after_attempt(3)) - def get_boot_diagnostics(self, **kwargs: Any) -> Result: - """Gets the serial console logs.""" - # NOTE: Some images can cause the `az` CLI to crash because - # their logs aren’t UTF-8 encoded. I’ve filed a bug: - # https://github.com/Azure/azure-cli/issues/15590 - return self.local( - f"az vm boot-diagnostics get-boot-log -n {self.name} -g {self.name}-rg", - **kwargs, - ) - - @retry(reraise=True, wait=wait_exponential(), stop=stop_after_attempt(3)) - def ping(self, **kwargs: Any) -> Result: - """Ping the node from the local system in a cross-platform manner.""" - flag = "-c 1" if platform.system() == "Linux" else "-n 1" - return self.local(f"ping {flag} {self.host}", **kwargs) - - def platform_restart(self) -> Result: - """TODO: Should this '--force' and redeploy?""" - return self.local(f"az vm restart -n {self.name} -g {self.name}-rg") - - def cat(self, path: str) -> str: - """Gets the value of a remote file without a temporary file.""" - with BytesIO() as buf: - self.get(path, buf) - return buf.getvalue().decode("utf-8").strip() - - -# TODO: The fixtures need to be fixed up since we now have a pair, one -# for each scope. They need documentation and de-duplication too. -@pytest.fixture(scope="function") -def node(request: FixtureRequest) -> Iterator[Node]: - key, name, host, data, fabric_config = get_node(request) - with Node(host, config=fabric_config, inline_ssh_env=True) as n: - n.name = name - n.data = data - yield n - - # Clean up! - if not request.config.getoption("keep_vms") and key: - assert request.config.cache is not None - request.config.cache.set(key, None) - delete_vm(name) - - -# TODO: Delete this and resurrect at a later date if we need it again. -@pytest.fixture(scope="class") -def class_node(request: FixtureRequest) -> Iterator[None]: - key, name, host, data, fabric_config = get_node(request) - with Node(host, config=fabric_config, inline_ssh_env=True) as n: - n.name = name - n.data = data - request.cls.n = n - logging.info(f"Using VM at: '{host}'") - try: - r: Result = n.run("uname -r") - except Exception as e: - logging.warning(f"Kernel Version: Unknown due to '{e}'") - else: - assert r.ok - logging.info(f"Kernel Version: '{r.stdout.strip()}'") - yield - - # Clean up! - if not request.config.getoption("keep_vms") and key: - assert request.config.cache is not None - request.config.cache.set(key, None) - delete_vm(name) - - -def get_node( - request: FixtureRequest, -) -> Tuple[Optional[str], str, Optional[str], Dict[str, str], fabric.Config]: - """Yields a safe remote Node on which to run commands. - - TODO: Currently this also manages the caching of the deployed VMs. - However, we should make a node pool (perhaps a session-scoped - fixture) which caches and deploys VMs, leaving this to perform its - original work as a connection creator. - - TODO: It's return type is garbage. - """ - deploy_marker = request.node.get_closest_marker("deploy") - connect_marker = request.node.get_closest_marker("connect") - - key: Optional[str] = None - data: Dict[str, str] = dict() - name: Optional[str] = None - host: Optional[str] = None - - # TODO: The deploy and connect markers should be mutually - # exclusive. - if deploy_marker: - # NOTE: https://docs.pytest.org/en/stable/cache.html - key = "/".join(["node"] + list(filter(None, deploy_marker.kwargs.values()))) - assert request.config.cache is not None - data = request.config.cache.get(key, None) - if data: - logging.info(f"Reusing node for cached key '{key}'") - else: - # Cache miss, deploy new node... - name = f"pytest-{uuid4()}" - host, data = deploy_vm(name, **deploy_marker.kwargs) - data["name"] = name - data["host"] = host - request.config.cache.set(key, data) - name = data["name"] - host = data["host"] - elif connect_marker: - # Get the host from the test’s marker. - host = connect_marker.args[0] - name = f"pre-deployed:{host}" - else: - # NOTE: This still uses SSH so the localhost must be - # connectable. - host = "localhost" - name = host - - # Yield the configured Node connection. - ssh_config: Dict[str, Any] = config.copy() - ssh_config["run"]["env"] = { - # Set PATH since it’s not a login shell. - "PATH": "/sbin:/usr/sbin:/usr/local/sbin:/bin:/usr/bin:/usr/local/bin" - } - fabric_config = fabric.Config(overrides=ssh_config) - logging.info(f"Using VM at: '{host}'") - return key, name, host, data, fabric_config diff --git a/selftests/setup_plan/conftest.py b/selftests/setup_plan/conftest.py deleted file mode 100644 index fb9db52c51..0000000000 --- a/selftests/setup_plan/conftest.py +++ /dev/null @@ -1,58 +0,0 @@ -"""Proof-of-concept to schedule a comprehensive test plan.""" -from __future__ import annotations - -import time -import typing - -from filelock import FileLock # type: ignore - -import pytest - -if typing.TYPE_CHECKING: - from typing import List - - from _pytest.config import Config - from _pytest.fixtures import SubRequest - from _pytest.tmpdir import TempPathFactory - - from pytest import Item, Session - - -def pytest_collection_modifyitems( - session: Session, config: Config, items: List[Item] -) -> None: - """For each item keep only instances using the feature.""" - keep: List[Item] = [] - for item in items: - marker = item.get_closest_marker("feature") - if marker is None: - keep.append(item) - continue - feature = marker.args[0] - if item.name.endswith(f"[{feature}]"): - keep.append(item) - items[:] = keep - - -@pytest.fixture(scope="session", autouse=True, params=["xdp", "gpu", "rdma"]) -def feature( - request: SubRequest, tmp_path_factory: TempPathFactory, worker_id: str -) -> str: - """Pretend that this sets up the environment.""" - assert request.param - if worker_id == "master": - return str(request.param) - # Get the shared temp directory. - tmp_dir = tmp_path_factory.getbasetemp().parent - fn = tmp_dir / request.param - data: str = "" - with FileLock(str(fn) + ".lock"): - print(f"Worker {worker_id} using feature {request.param}") - if fn.is_file(): - data = fn.read_text() - else: - # Pretend to do some expensive setup and cache it. - time.sleep(3) - data = request.param - fn.write_text(data) - return data diff --git a/selftests/setup_plan/test_plan_A.py b/selftests/setup_plan/test_plan_A.py index d0edba2ba1..211361fad3 100644 --- a/selftests/setup_plan/test_plan_A.py +++ b/selftests/setup_plan/test_plan_A.py @@ -1,16 +1,17 @@ -import pytest +from conftest import LISA +from target import Target -@pytest.mark.feature("xdp") -def test_xdp_a() -> None: +@LISA(platform="Azure", features="xdp") +def test_xdp_a(target: Target) -> None: pass -@pytest.mark.feature("gpu") -def test_gpu_a() -> None: +@LISA(platform="Azure", features="gpu") +def test_gpu_a(target: Target) -> None: pass -@pytest.mark.feature("rdma") -def test_rdma_a() -> None: +@LISA(platform="Azure", features="rdma") +def test_rdma_a(target: Target) -> None: pass diff --git a/selftests/setup_plan/test_plan_B.py b/selftests/setup_plan/test_plan_B.py index c415d8bda7..4576d89bcd 100644 --- a/selftests/setup_plan/test_plan_B.py +++ b/selftests/setup_plan/test_plan_B.py @@ -1,16 +1,17 @@ -import pytest +from conftest import LISA +from target import Target -@pytest.mark.feature("xdp") -def test_xdp_b() -> None: +@LISA(platform="Azure", features="xdp") +def test_xdp_b(target: Target) -> None: pass -@pytest.mark.feature("gpu") -def test_gpu_b() -> None: +@LISA(platform="Azure", features="gpu") +def test_gpu_b(target: Target) -> None: pass -@pytest.mark.feature("rdma") -def test_rdma_b() -> None: +@LISA(platform="Azure", features="rdma") +def test_rdma_b(target: Target) -> None: pass diff --git a/selftests/setup_plan/test_plan_C.py b/selftests/setup_plan/test_plan_C.py index ce9d2155fe..5380ee00f2 100644 --- a/selftests/setup_plan/test_plan_C.py +++ b/selftests/setup_plan/test_plan_C.py @@ -1,16 +1,17 @@ -import pytest +from conftest import LISA +from target import Target -@pytest.mark.feature("xdp") -def test_xdp_c() -> None: +@LISA(platform="Azure", features="xdp") +def test_xdp_c(target: Target) -> None: pass -@pytest.mark.feature("gpu") -def test_gpu_c() -> None: +@LISA(platform="Azure", features="gpu") +def test_gpu_c(target: Target) -> None: pass -@pytest.mark.feature("rdma") -def test_rdma_c() -> None: +@LISA(platform="Azure", features="rdma") +def test_rdma_c(target: Target) -> None: pass diff --git a/selftests/test_basic.py b/selftests/test_basic.py index a644b137ae..b8ffdd4125 100644 --- a/selftests/test_basic.py +++ b/selftests/test_basic.py @@ -1,7 +1,9 @@ """These tests are meant to run in a CI environment.""" -from node_plugin import Node +from conftest import LISA +from target import Target -def test_basic(node: Node) -> None: +@LISA +def test_basic(target: Target) -> None: """Basic test which creates a Node connection to 'localhost'.""" - node.local("echo Hello World") + target.local("echo Hello World") diff --git a/target.py b/target.py new file mode 100644 index 0000000000..a20333df7e --- /dev/null +++ b/target.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +import platform +import typing +from abc import ABC, abstractmethod +from io import BytesIO +from uuid import uuid4 + +from fabric import Config as FabricConfig +from fabric import Connection +from invoke import Config as InvokeConfig +from invoke import Context +from invoke.runners import Result # type: ignore +from tenacity import retry, stop_after_attempt, wait_exponential # type: ignore + +import lisa + +if typing.TYPE_CHECKING: + from typing import Any, Dict, Set + + +class Target(ABC): + """Extends 'fabric.Connection' with our own utilities.""" + + local_context = Context(config=InvokeConfig(overrides=lisa.config)) + + def __init__( + self, + params: Dict[str, str], + features: Set[str], + name: str = f"pytest-{uuid4()}", + ): + """If not given a name, generates one uniquely. + + Name is a unique identifier for the group of associated + resources. Features is a list of requirements such as sriov, + rdma, gpu, xdp. + + """ + self.params: Dict[str, str] = params + self.features: Set[str] = features + self.name: str = name + + # TODO: Fix this. + self.host = self.deploy() + + config = lisa.config.copy() + config["run"]["env"] = { + # Set PATH since it’s not a login shell. + "PATH": "/sbin:/usr/sbin:/usr/local/sbin:/bin:/usr/bin:/usr/local/bin" + } + self.connection = Connection( + self.host, config=FabricConfig(overrides=config), inline_ssh_env=True + ) + + @abstractmethod + def deploy(self) -> str: + """Must deploy the target resources and return hostname.""" + ... + + @abstractmethod + def delete(self) -> None: + """Must delete the target resources.""" + ... + + @classmethod + def local(self, *args: Any, **kwargs: Any) -> Result: + """This patches Fabric's 'local()' function to ignore SSH environment.""" + return Target.local_context.run(*args, **kwargs) + + # TODO: Generate these automatically. There’s some weird bug with + # inheriting from ‘Connection’ that causes infinite recursion. + def run(self, *args: Any, **kwargs: Any) -> Result: + return self.connection.run(*args, **kwargs) + + def sudo(self, *args: Any, **kwargs: Any) -> Result: + return self.connection.sudo(*args, **kwargs) + + def get(self, *args: Any, **kwargs: Any) -> Result: + return self.connection.get(*args, **kwargs) + + def put(self, *args: Any, **kwargs: Any) -> Result: + return self.connection.put(*args, **kwargs) + + @retry(reraise=True, wait=wait_exponential(), stop=stop_after_attempt(3)) + def ping(self, **kwargs: Any) -> Result: + """Ping the node from the local system in a cross-platform manner.""" + flag = "-c 1" if platform.system() == "Linux" else "-n 1" + return self.local(f"ping {flag} {self.host}", **kwargs) + + def cat(self, path: str) -> str: + """Gets the value of a remote file without a temporary file.""" + with BytesIO() as buf: + self.get(path, buf) + return buf.getvalue().decode("utf-8").strip() diff --git a/targets.yaml b/targets.yaml new file mode 100644 index 0000000000..58c3d04797 --- /dev/null +++ b/targets.yaml @@ -0,0 +1,12 @@ +# TODO: We need to actually think about the schema here. +- target: + image: "citrix:netscalervpx-130:netscalerbyol:latest" + +- target: + image: "audiocodes:mediantsessionbordercontroller:mediantvirtualsbcazure:latest" + +- target: + image: "credativ:Debian:9:9.0.201706190" + +- target: + image: "github:github-enterprise:github-enterprise:latest" diff --git a/testsuites/test_lis.py b/testsuites/test_lis.py index bb050d7d0f..ea91f7840d 100644 --- a/testsuites/test_lis.py +++ b/testsuites/test_lis.py @@ -1,19 +1,20 @@ """Runs 'LIS-Tests.xml' using Pytest.""" -import conftest -import pytest -from node_plugin import Node +from __future__ import annotations +import typing -@pytest.mark.lisa( - platform="Azure", category="Functional", area="LIS_DEPLOY", tags=["lis"], priority=0 -) -# @pytest.mark.deploy(setup="OneVM") -@pytest.mark.connect("centos") -def test_lis_driver_version(node: Node) -> None: +if typing.TYPE_CHECKING: + from azure import Azure + +from conftest import LINUX_SCRIPTS, LISA + + +@LISA(platform="Azure", category="Functional", priority=0, area="LIS_DEPLOY") +def test_lis_driver_version(target: Azure) -> None: # TODO: Include “utils.sh” automatically? Or something... for f in ["utils.sh", "LIS-VERSION-CHECK.sh"]: - node.put(conftest.LINUX_SCRIPTS / f) - node.run(f"chmod +x {f}") - node.sudo("yum install -y bc") - node.run("./LIS-VERSION-CHECK.sh") - assert node.cat("state.txt") == "TestCompleted" + target.put(LINUX_SCRIPTS / f) + target.run(f"chmod +x {f}") + target.sudo("yum install -y bc") + target.run("./LIS-VERSION-CHECK.sh") + assert target.cat("state.txt") == "TestCompleted" diff --git a/testsuites/test_smoke.py b/testsuites/test_smoke.py index fa0108493b..28d1a14c35 100644 --- a/testsuites/test_smoke.py +++ b/testsuites/test_smoke.py @@ -6,27 +6,12 @@ from invoke.runners import CommandTimedOut, Result, UnexpectedExit # type: ignore from paramiko import SSHException # type: ignore -import pytest -from node_plugin import Node - -# TODO: This is an example of leveraging Pytest’s parameterization -# support. We can implement a small YAML parser to read a playbook at -# runtime to generate this instead of using the below list. -params = [ - pytest.param(i, marks=pytest.mark.deploy(vm_image=i, vm_size="Standard_DS2_v2")) - for i in [ - "citrix:netscalervpx-130:netscalerbyol:latest", - "audiocodes:mediantsessionbordercontroller:mediantvirtualsbcazure:latest", - "credativ:Debian:9:9.0.201706190", - "github:github-enterprise:github-enterprise:latest", - ] -] - - -@pytest.mark.lisa(priority=0) -@pytest.mark.parametrize("urn", params) -@pytest.mark.flaky(reruns=1) -def test_smoke(urn: str, node: Node) -> None: +from azure import Azure +from conftest import LISA + + +@LISA(platform="Azure", priority=0, sku="Standard_DS2_v2") +def test_smoke(target: Azure) -> None: """Check that a VM can be deployed and is responsive. 1. Deploy the VM (via 'node' fixture) and log it. @@ -45,15 +30,15 @@ def test_smoke(urn: str, node: Node) -> None: logging.info("Pinging before reboot...") ping1 = Result() try: - ping1 = node.ping() + ping1 = target.ping() except UnexpectedExit: - logging.warning(f"Pinging {node.host} before reboot failed") + logging.warning(f"Pinging {target.host} before reboot failed") ssh_errors = (TimeoutError, CommandTimedOut, SSHException, socket.error) try: logging.info("SSHing before reboot...") - node.open() + target.connection.open() except ssh_errors as e: logging.warning(f"SSH before reboot failed: '{e}'") @@ -61,10 +46,10 @@ def test_smoke(urn: str, node: Node) -> None: try: logging.info("Rebooting...") # If this succeeds, we should expect the exit code to be -1 - reboot_exit = node.sudo("reboot", timeout=5).exited + reboot_exit = target.sudo("reboot", timeout=5).exited except ssh_errors as e: logging.warning(f"SSH failed, using platform to reboot: '{e}'") - node.platform_restart() + target.platform_restart() except UnexpectedExit: # TODO: How do we differentiate reboot working and the SSH # connection disconnecting for other reasons? @@ -77,19 +62,19 @@ def test_smoke(urn: str, node: Node) -> None: logging.info("Pinging after reboot...") ping2 = Result() try: - ping2 = node.ping() + ping2 = target.ping() except UnexpectedExit: - logging.warning(f"Pinging {node.host} after reboot failed") + logging.warning(f"Pinging {target.host} after reboot failed") try: logging.info("SSHing after reboot...") - node.open() + target.connection.open() except ssh_errors as e: logging.warning(f"SSH after reboot failed: '{e}'") logging.info("Retrieving boot diagnostics...") try: - node.get_boot_diagnostics() + target.get_boot_diagnostics() except UnexpectedExit: logging.warning("Retrieving boot diagnostics failed.") else: From 72dfe08ce6ff8131fa9b35ce73d60dbad1d597ee Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Fri, 6 Nov 2020 20:41:04 -0800 Subject: [PATCH 072/194] Use schema to massively simplify playbook --- conftest.py | 141 +++++++++++++++++++++----------------------------- criteria.yaml | 33 +++++------- lisa.py | 37 ++++++------- playbook.py | 54 +++++++++++++++++++ playbook.yaml | 15 ++++++ target.py | 10 ++-- 6 files changed, 161 insertions(+), 129 deletions(-) create mode 100644 playbook.py create mode 100644 playbook.yaml diff --git a/conftest.py b/conftest.py index 0c80e01648..749b213f95 100644 --- a/conftest.py +++ b/conftest.py @@ -5,26 +5,19 @@ """ from __future__ import annotations -import sys import typing -from functools import partial from pathlib import Path -import yaml +import schema import lisa +import playbook +import pytest # TODO: Use importlib instead from azure import Azure from target import Target -try: - from yaml import CLoader as Loader -except ImportError: - from yaml import Loader # type: ignore - -import pytest - if typing.TYPE_CHECKING: from typing import Any, Dict, Iterator, List, Optional @@ -77,7 +70,7 @@ def target(pool, worker_id, request: FixtureRequest) -> Iterator[Target]: """ params = request.param marker = request.node.get_closest_marker("lisa") - features = marker.kwargs["features"] + features = set(marker.kwargs["features"]) for t in pool: # TODO: Implement full feature comparison, etc. and not just # proof-of-concept string set comparison. @@ -102,38 +95,33 @@ def pytest_addoption(parser: Parser) -> None: parser.addoption("--keep-vms", action="store_true", help="Keeps deployed VMs.") parser.addoption("--check", action="store_true", help="Run semantic analysis.") parser.addoption("--demo", action="store_true", help="Run in demo mode.") - parser.addoption("--targets", type=Path, help="Path to targets playbook.") - parser.addoption("--criteria", type=Path, help="Path to criteria playbook.") + parser.addoption("--playbook", type=Path, help="Path to playbook.") TARGETS = [] TARGET_IDS = [] +def get_playbook(path: Optional[Path]) -> dict(): + book = dict() + if not path: + return book + with open(path) as f: + book = playbook.schema.validate(f) + return book + + def pytest_configure(config: Config) -> None: """Parse provided user inputs to setup configuration. Determines the targets based on the playbook and sets default configurations based user mode. - """ - playbook_path: Optional[Path] = config.getoption("--targets") - if playbook_path: - playbook = dict() - with open(playbook_path) as f: - playbook = yaml.load(f, Loader=Loader) - for play in playbook: - t = play.get("target") - if t is None: - continue - else: - print(f"Parsing target {t}") - setup = { - "platform": t.get("platform", "Azure"), - "image": t.get("image", "UbuntuLTS"), - "sku": t.get("sku", "Standard_DS1_v2"), - } - TARGETS.append(setup) - TARGET_IDS.append("-".join(setup.values())) + + configurations based user mode.""" + book = get_playbook(config.getoption("--playbook")) + for t in book.get("targets"): + TARGETS.append(t) + TARGET_IDS.append(t["name"]) # Search ‘_pytest’ for ‘addoption’ to find these. options: Dict[str, Any] = {} # See ‘pytest.ini’ for defaults. @@ -182,62 +170,49 @@ def pytest_collection_modifyitems( """ # Validate LISA mark on every item. for item in items: - mark = item.get_closest_marker("lisa") - assert mark, f"{item} is missing required LISA marker!" - lisa.validate(mark) + m = item.get_closest_marker("lisa") + assert m, f"{item} is missing required LISA marker!" + try: + lisa.validate(m) + except schema.SchemaMissingKeyError as e: + print(f"Test {item.name} failed LISA validation {e}!") + items[:] = [] + return # Optionally select tests based on a playbook. - playbook_path: Optional[Path] = config.getoption("--criteria") - new_items: List[Item] = [] - force_exclude: List[Item] = [] + included: List[Item] = [] + excluded: List[Item] = [] - def select_item(action: Optional[str], times: int, item: Item) -> None: + # TODO: Remove logging. + def select(item: Item, times: int, exclude: bool) -> None: """Includes or excludes the item as appropriate.""" - if action == "forceExclude": - print(f" Forcing exclusion of item {item}") - force_exclude.append(item) + if exclude: + print(f" Excluding {item}") + excluded.append(item) else: - print(f" Keeping {item} selected {times} times") - for _ in range(times - new_items.count(item)): - new_items.append(item) - - # TODO: Review, refactor, and fix logging. If we do schema - # validation and have reasonable defaults we can delete most of - # the `is not None` checks. Suggest using: - # https://pypi.org/project/schema/ - if playbook_path: - playbook = dict() - with open(playbook_path) as f: - playbook = yaml.load(f, Loader=Loader) - for play in playbook: - criteria = play.get("criteria") - if criteria is None: - continue - else: - print(f"Parsing criteria {criteria}") - select_action = play.get("select_action", "forceInclude") - times = play.get("times", 1) - select = partial(select_item, select_action, times) - - name = criteria.get("name") - priority = criteria.get("priority") - area = criteria.get("area") - for i in items: - marker = i.get_closest_marker("LISA") - args = marker.kwargs - if name is not None: - if i.name.startswith(name): - print(f" Selecting test {i} because name is {name}!") - select(i) - if priority is not None: - if args.get("priority") == priority: - print(f" Selecting test {i} because priority is {priority}!") - select(i) - if area and args.get("area"): - if args["area"].lower() == area: - print(f" Selecting test {i} because area is {area}!") - select(i) - items[:] = [i for i in new_items if i not in force_exclude] + print(f" Including {item} {times} times") + for _ in range(times - included.count(item)): + included.append(item) + + book = get_playbook(config.getoption("--playbook")) + for c in book.get("criteria"): + print(f"Parsing criteria {c}") + for item in items: + m = item.get_closest_marker("lisa").kwargs + if any( + [ + c["name"] and c["name"] in item.name, + c["area"] and c["area"].casefold() == m["area"].casefold(), + c["category"] + and c["category"].casefold() == m["category"].casefold(), + c["priority"] and c["priority"] == m["priority"], + c["tags"] and set(c["tags"]) <= set(m["tags"]), + ] + ): + select(item, c["times"], c["exclude"]) + if not included: + included = items + items[:] = [i for i in included if i not in excluded] def pytest_html_report_title(report): # type: ignore diff --git a/criteria.yaml b/criteria.yaml index 3e08a20942..cec1b91c3f 100644 --- a/criteria.yaml +++ b/criteria.yaml @@ -1,19 +1,14 @@ -# NOTE: This is a proof-of-concept ask from Chi. - - -# select all p0 cases -# for example, selected three cases: a,b,c -- criteria: - priority: 0 -# drop all cases of xdp, -# because it's not ready on a tested distro. -# for example, droped c, so now is: a,b -- criteria: - area: xdp - # forceExclude means not to pick up it again in next rules. - select_action: forceExclude -# run smoke_test cases twice, to prove a distro stable enough -# after this rule, the picked test cases is like a,b,b -- criteria: - name: test_smoke - times: 2 +# NOTE: This is an adjusted proof-of-concept ask from Chi. +criteria: + # select all p0 cases + # for example, selected three cases: a,b,c + - priority: 0 + # run smoke_test cases twice, to prove a distro stable enough + # after this rule, the picked test cases is like a,b,b + - name: smoke + times: 2 + # drop all cases of xdp, + # because it's not ready on a tested distro. + # for example, droped c, so now is: a,b + - area: xdp + exclude: true diff --git a/lisa.py b/lisa.py index d0d906a9c6..da98d92140 100644 --- a/lisa.py +++ b/lisa.py @@ -2,6 +2,8 @@ import typing +from schema import Optional, Or, Schema + if typing.TYPE_CHECKING: from _pytest.mark.structures import Mark @@ -20,29 +22,20 @@ } } +lisa_schema = Schema( + { + "platform": str, + "category": Or("Functional", "Performance", "Stress", "Community", "Longhaul"), + "area": str, + "priority": Or(0, 1, 2, 3), + Optional("features", default=list): [str], + Optional(object): object, + }, + ignore_extra_keys=True, +) + def validate(mark: Mark): """Validate each test's LISA parameters.""" assert not mark.args, "LISA marker cannot have positional arguments!" - args = mark.kwargs - - if args.get("platform"): - assert type(args["platform"]) is str, "Platform must be a string!" - - if args.get("priority") is not None: - assert type(args["priority"]) is int, "Priority must be an integer!" - - if args.get("features") is not None: - if type(args["features"]) is str: - # Convert single ‘str’ argument to ‘Set[str]’ - features = set() - features.add(args["features"]) - args["features"] = features - elif type(args["features"]) is list: - # Convert ‘list’ to ‘set’ - args["features"] = set(args["features"]) - assert type(args["features"]) is set, "Features must be a set!" - for feature in args["features"]: - assert type(feature) is str, "Features must be strings!" - else: - args["features"] = set() + mark.kwargs.update(lisa_schema.validate(mark.kwargs)) diff --git a/playbook.py b/playbook.py new file mode 100644 index 0000000000..ccbf34e44a --- /dev/null +++ b/playbook.py @@ -0,0 +1,54 @@ +import yaml + +try: + from yaml import CLoader as Loader +except ImportError: + from yaml import Loader # type: ignore + +from schema import And, Optional, Schema, Use + +criteria_schema = Schema( + { + # TODO: Validate that these strings are valid regular + # expressions if we change our matching logic. + Optional("name", default=None): str, + Optional("area", default=None): str, + Optional("category", default=None): str, + Optional("priority", default=None): int, + Optional("tags", default=list): [str], + Optional("times", default=1): int, + Optional("exclude", default=False): bool, + } +) + +# NOTE: We could have each platform register its own schema and +# “Or(...)” them together, so this is actually quite flexible. Again, +# so far just writing a proof-of-concept because we need to peer +# review our design. +target_schema = Schema( + { + # TODO: Maybe set name to image if unset. + "name": str, + # TODO: Use ‘Or([list of registered platforms])’ + "platform": str, + # TODO: Maybe validate as URN or path etc. + Optional("image", default=None): str, + Optional("sku", default=None): str, + } +) + +default_target = {"name": "Default", "platform": "Local"} + +schema = Schema( + And( + # NOTE: This is “magic” that automatically loads and validates + # YAML input. See https://pypi.org/project/schema/ and + # https://pyyaml.org/wiki/PyYAMLDocumentation for + # documentation. + Use(lambda x: yaml.load(x, Loader=Loader)), + { + Optional("targets", default=[default_target]): [target_schema], + Optional("criteria", default=list): [criteria_schema], + }, + ) +) diff --git a/playbook.yaml b/playbook.yaml new file mode 100644 index 0000000000..1ab2a7a16a --- /dev/null +++ b/playbook.yaml @@ -0,0 +1,15 @@ +# NOTE: This is a suggested playbook example. See the schema. +targets: + - name: Ubuntu + platform: Azure + image: UbuntuLTS + sku: Standard_DS1_v2 + - name: Debian + platform: Azure + image: credativ:Debian:9:9.0.201706190 + - name: GitHub + platform: Azure + image: github:github-enterprise:github-enterprise:latest + +criteria: + - name: smoke diff --git a/target.py b/target.py index a20333df7e..d78f615aa5 100644 --- a/target.py +++ b/target.py @@ -16,7 +16,7 @@ import lisa if typing.TYPE_CHECKING: - from typing import Any, Dict, Set + from typing import Any, Mapping, Sequence, Set class Target(ABC): @@ -26,8 +26,8 @@ class Target(ABC): def __init__( self, - params: Dict[str, str], - features: Set[str], + params: Mapping[str, str], + features: Sequence[str], name: str = f"pytest-{uuid4()}", ): """If not given a name, generates one uniquely. @@ -37,8 +37,8 @@ def __init__( rdma, gpu, xdp. """ - self.params: Dict[str, str] = params - self.features: Set[str] = features + self.params: Mapping[str, str] = params + self.features: Set[str] = set(features) self.name: str = name # TODO: Fix this. From 75a52c171d73d7e6de57e3005f194128dc0b7df2 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Fri, 6 Nov 2020 20:41:38 -0800 Subject: [PATCH 073/194] Add schema package --- poetry.lock | 29 ++++++++++++++++++++++++++++- pyproject.toml | 1 + 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index fc2c2f3273..ea01a1d5f2 100644 --- a/poetry.lock +++ b/poetry.lock @@ -101,6 +101,14 @@ category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +[[package]] +name = "contextlib2" +version = "0.6.0.post1" +description = "Backports and enhancements for the contextlib module" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + [[package]] name = "cryptography" version = "3.1.1" @@ -635,6 +643,17 @@ python-versions = "*" [package.extras] dev = ["pytest"] +[[package]] +name = "schema" +version = "0.7.3" +description = "Simple data validation library" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +contextlib2 = ">=0.5.5" + [[package]] name = "six" version = "1.15.0" @@ -705,7 +724,7 @@ python-versions = ">=3.6" [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "ee86abdeec8b63e0ff22b16e7d9d6e8aedce399ada42a0cbe395ecd917a42703" +content-hash = "ff9d853cf9f58598aa01e465e2c673172b9e573fd7a8569bf29236348884c748" [metadata.files] apipkg = [ @@ -782,6 +801,10 @@ colorama = [ {file = "colorama-0.4.3-py2.py3-none-any.whl", hash = "sha256:7d73d2a99753107a36ac6b455ee49046802e59d9d076ef8e47b61499fa29afff"}, {file = "colorama-0.4.3.tar.gz", hash = "sha256:e96da0d330793e2cb9485e9ddfd918d456036c7149416295932478192f4436a1"}, ] +contextlib2 = [ + {file = "contextlib2-0.6.0.post1-py2.py3-none-any.whl", hash = "sha256:3355078a159fbb44ee60ea80abd0d87b80b78c248643b49aa6d94673b413609b"}, + {file = "contextlib2-0.6.0.post1.tar.gz", hash = "sha256:01f490098c18b19d2bd5bb5dc445b2054d2fa97f09a4280ba2c5f3c394c8162e"}, +] cryptography = [ {file = "cryptography-3.1.1-cp27-cp27m-macosx_10_10_x86_64.whl", hash = "sha256:65beb15e7f9c16e15934569d29fb4def74ea1469d8781f6b3507ab896d6d8719"}, {file = "cryptography-3.1.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:983c0c3de4cb9fcba68fd3f45ed846eb86a2a8b8d8bc5bb18364c4d00b3c61fe"}, @@ -1033,6 +1056,10 @@ regex = [ rope = [ {file = "rope-0.18.0.tar.gz", hash = "sha256:786b5c38c530d4846aa68a42604f61b4e69a493390e3ca11b88df0fbfdc3ed04"}, ] +schema = [ + {file = "schema-0.7.3-py2.py3-none-any.whl", hash = "sha256:c331438b60f634cab5664ab720d3083cc444f924d55269530c36b33e3354276f"}, + {file = "schema-0.7.3.tar.gz", hash = "sha256:4cf529318cfd1e844ecbe02f41f7e5aa027463e7403666a52746f31f04f47a5e"}, +] six = [ {file = "six-1.15.0-py2.py3-none-any.whl", hash = "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"}, {file = "six-1.15.0.tar.gz", hash = "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259"}, diff --git a/pyproject.toml b/pyproject.toml index 01ba6f0db0..47db86c319 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,7 @@ tenacity = "^6.2.0" pytest-rerunfailures = "^9.1.1" pytest-xdist = "^2.1.0" PyYAML = "^5.3.1" +schema = "^0.7.3" [tool.poetry.dev-dependencies] black = "^20.8b1" From 199458a3bf2821a200c9bf8ac0ccea560d16d93f Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Fri, 6 Nov 2020 20:41:57 -0800 Subject: [PATCH 074/194] Fix up tests for schema validation --- selftests/setup_plan/test_plan_A.py | 14 ++++++++++---- selftests/setup_plan/test_plan_B.py | 14 ++++++++++---- selftests/setup_plan/test_plan_C.py | 14 ++++++++++---- selftests/test_basic.py | 4 +++- testsuites/test_smoke.py | 8 +++++++- testsuites/test_xdp.py | 22 +++++++++++----------- 6 files changed, 51 insertions(+), 25 deletions(-) diff --git a/selftests/setup_plan/test_plan_A.py b/selftests/setup_plan/test_plan_A.py index 211361fad3..6f5e004937 100644 --- a/selftests/setup_plan/test_plan_A.py +++ b/selftests/setup_plan/test_plan_A.py @@ -1,17 +1,23 @@ -from conftest import LISA +import functools + +import conftest from target import Target +LISA = functools.partial( + conftest.LISA, platform="Azure", category="Functional", area="self-test", priority=1 +) + -@LISA(platform="Azure", features="xdp") +@LISA(features=["xdp"]) def test_xdp_a(target: Target) -> None: pass -@LISA(platform="Azure", features="gpu") +@LISA(features=["gpu"]) def test_gpu_a(target: Target) -> None: pass -@LISA(platform="Azure", features="rdma") +@LISA(features=["rdma"]) def test_rdma_a(target: Target) -> None: pass diff --git a/selftests/setup_plan/test_plan_B.py b/selftests/setup_plan/test_plan_B.py index 4576d89bcd..b1e8fc7a26 100644 --- a/selftests/setup_plan/test_plan_B.py +++ b/selftests/setup_plan/test_plan_B.py @@ -1,17 +1,23 @@ -from conftest import LISA +import functools + +import conftest from target import Target +LISA = functools.partial( + conftest.LISA, platform="Azure", category="Functional", area="self-test", priority=1 +) + -@LISA(platform="Azure", features="xdp") +@LISA(features=["xdp"]) def test_xdp_b(target: Target) -> None: pass -@LISA(platform="Azure", features="gpu") +@LISA(features=["gpu"]) def test_gpu_b(target: Target) -> None: pass -@LISA(platform="Azure", features="rdma") +@LISA(features=["rdma"]) def test_rdma_b(target: Target) -> None: pass diff --git a/selftests/setup_plan/test_plan_C.py b/selftests/setup_plan/test_plan_C.py index 5380ee00f2..60698e3194 100644 --- a/selftests/setup_plan/test_plan_C.py +++ b/selftests/setup_plan/test_plan_C.py @@ -1,17 +1,23 @@ -from conftest import LISA +import functools + +import conftest from target import Target +LISA = functools.partial( + conftest.LISA, platform="Azure", category="Functional", area="self-test", priority=1 +) + -@LISA(platform="Azure", features="xdp") +@LISA(features=["xdp"]) def test_xdp_c(target: Target) -> None: pass -@LISA(platform="Azure", features="gpu") +@LISA(features=["gpu"]) def test_gpu_c(target: Target) -> None: pass -@LISA(platform="Azure", features="rdma") +@LISA(features=["rdma"]) def test_rdma_c(target: Target) -> None: pass diff --git a/selftests/test_basic.py b/selftests/test_basic.py index b8ffdd4125..92c835fcf1 100644 --- a/selftests/test_basic.py +++ b/selftests/test_basic.py @@ -2,8 +2,10 @@ from conftest import LISA from target import Target +pytestmark = [] -@LISA + +@LISA(platform="Local", category="Functional", area="self-test", priority=1) def test_basic(target: Target) -> None: """Basic test which creates a Node connection to 'localhost'.""" target.local("echo Hello World") diff --git a/testsuites/test_smoke.py b/testsuites/test_smoke.py index 28d1a14c35..2867ee6317 100644 --- a/testsuites/test_smoke.py +++ b/testsuites/test_smoke.py @@ -10,7 +10,13 @@ from conftest import LISA -@LISA(platform="Azure", priority=0, sku="Standard_DS2_v2") +@LISA( + platform="Azure", + category="Functional", + area="deploy", + priority=0, + sku="Standard_DS2_v2", +) def test_smoke(target: Azure) -> None: """Check that a VM can be deployed and is responsive. diff --git a/testsuites/test_xdp.py b/testsuites/test_xdp.py index d386ff2bfc..c5f98a70d6 100644 --- a/testsuites/test_xdp.py +++ b/testsuites/test_xdp.py @@ -1,12 +1,12 @@ """Runs 'FunctionalTests-XDP.xml' using Pytest.""" -import conftest import pytest -from node_plugin import Node +from conftest import LINUX_SCRIPTS, LISA +from target import Target -@pytest.mark.lisa( +@LISA( platform="Azure", category="Functional", area="XDP", @@ -20,7 +20,7 @@ vm_size="Standard_DS4_v2", ) @pytest.mark.skip(reason="Not Finished") -def test_verify_xdp_compliance(node: Node) -> None: +def test_verify_xdp_compliance(target: Target) -> None: for f in [ "utils.sh", "XDPDumpSetup.sh", @@ -28,10 +28,10 @@ def test_verify_xdp_compliance(node: Node) -> None: "enable_passwordless_root.sh", "enable_root.sh", ]: - node.put(conftest.LINUX_SCRIPTS / f) - node.run(f"chmod +x {f}") - node.run("./enable_root.sh") - node.run("./enable_passwordless_root.sh") - synth_interface = node.run("source XDPUtils.sh ; get_extra_synth_nic").stdout - node.run(f"./XDPDumpSetup.sh {node.internal_address} {synth_interface}") - assert node.cat("state.txt") == "TestCompleted" + target.put(LINUX_SCRIPTS / f) + target.run(f"chmod +x {f}") + target.run("./enable_root.sh") + target.run("./enable_passwordless_root.sh") + synth_interface = target.run("source XDPUtils.sh ; get_extra_synth_nic").stdout + target.run(f"./XDPDumpSetup.sh {target.internal_address} {synth_interface}") + assert target.cat("state.txt") == "TestCompleted" From 9aff173717916af099e0c8549f0df8ddd85017e3 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Tue, 10 Nov 2020 15:04:44 -0800 Subject: [PATCH 075/194] Move LISA marker and update tests --- conftest.py | 4 ---- lisa.py | 8 +++++++- pytest.ini | 3 --- selftests/setup_plan/test_plan_A.py | 4 ++-- selftests/setup_plan/test_plan_B.py | 4 ++-- selftests/setup_plan/test_plan_C.py | 4 ++-- selftests/test_basic.py | 4 +--- testsuites/test_lis.py | 2 +- testsuites/test_smoke.py | 2 +- testsuites/test_xdp.py | 17 ++++++++--------- 10 files changed, 24 insertions(+), 28 deletions(-) diff --git a/conftest.py b/conftest.py index 749b213f95..cb61607632 100644 --- a/conftest.py +++ b/conftest.py @@ -29,10 +29,6 @@ from pytest import Item, Session -LISA = pytest.mark.lisa -LINUX_SCRIPTS = Path("../Testscripts/Linux") - - @pytest.fixture(scope="session") def pool(request: FixtureRequest) -> Iterator[List[Target]]: """This fixture tracks all deployed target resources.""" diff --git a/lisa.py b/lisa.py index da98d92140..20996c5c51 100644 --- a/lisa.py +++ b/lisa.py @@ -1,12 +1,18 @@ from __future__ import annotations import typing +from pathlib import Path -from schema import Optional, Or, Schema +from schema import Optional, Or, Schema # type: ignore + +import pytest if typing.TYPE_CHECKING: from _pytest.mark.structures import Mark +LISA = pytest.mark.lisa +LINUX_SCRIPTS = Path("../Testscripts/Linux") + # Setup a sane configuration for local and remote commands. Note that # the defaults between Fabric and Invoke are different, so we use # their Config classes explicitly. diff --git a/pytest.ini b/pytest.ini index b242158aa3..c80e7c884d 100644 --- a/pytest.ini +++ b/pytest.ini @@ -7,9 +7,6 @@ addopts = -rA markers = lisa - deploy - connect - feature log_cli = true log_cli_level = WARNING log_cli_format = %(asctime)s %(levelname)s %(message)s diff --git a/selftests/setup_plan/test_plan_A.py b/selftests/setup_plan/test_plan_A.py index 6f5e004937..54f2e2543f 100644 --- a/selftests/setup_plan/test_plan_A.py +++ b/selftests/setup_plan/test_plan_A.py @@ -1,10 +1,10 @@ import functools -import conftest +import lisa from target import Target LISA = functools.partial( - conftest.LISA, platform="Azure", category="Functional", area="self-test", priority=1 + lisa.LISA, platform="Azure", category="Functional", area="self-test", priority=1 ) diff --git a/selftests/setup_plan/test_plan_B.py b/selftests/setup_plan/test_plan_B.py index b1e8fc7a26..90b214e58c 100644 --- a/selftests/setup_plan/test_plan_B.py +++ b/selftests/setup_plan/test_plan_B.py @@ -1,10 +1,10 @@ import functools -import conftest +import lisa from target import Target LISA = functools.partial( - conftest.LISA, platform="Azure", category="Functional", area="self-test", priority=1 + lisa.LISA, platform="Azure", category="Functional", area="self-test", priority=1 ) diff --git a/selftests/setup_plan/test_plan_C.py b/selftests/setup_plan/test_plan_C.py index 60698e3194..8aff95e2fb 100644 --- a/selftests/setup_plan/test_plan_C.py +++ b/selftests/setup_plan/test_plan_C.py @@ -1,10 +1,10 @@ import functools -import conftest +import lisa from target import Target LISA = functools.partial( - conftest.LISA, platform="Azure", category="Functional", area="self-test", priority=1 + lisa.LISA, platform="Azure", category="Functional", area="self-test", priority=1 ) diff --git a/selftests/test_basic.py b/selftests/test_basic.py index 92c835fcf1..5b3012d8a4 100644 --- a/selftests/test_basic.py +++ b/selftests/test_basic.py @@ -1,9 +1,7 @@ """These tests are meant to run in a CI environment.""" -from conftest import LISA +from lisa import LISA from target import Target -pytestmark = [] - @LISA(platform="Local", category="Functional", area="self-test", priority=1) def test_basic(target: Target) -> None: diff --git a/testsuites/test_lis.py b/testsuites/test_lis.py index ea91f7840d..86921f4fa1 100644 --- a/testsuites/test_lis.py +++ b/testsuites/test_lis.py @@ -6,7 +6,7 @@ if typing.TYPE_CHECKING: from azure import Azure -from conftest import LINUX_SCRIPTS, LISA +from lisa import LINUX_SCRIPTS, LISA @LISA(platform="Azure", category="Functional", priority=0, area="LIS_DEPLOY") diff --git a/testsuites/test_smoke.py b/testsuites/test_smoke.py index 2867ee6317..cfc911707f 100644 --- a/testsuites/test_smoke.py +++ b/testsuites/test_smoke.py @@ -7,7 +7,7 @@ from paramiko import SSHException # type: ignore from azure import Azure -from conftest import LISA +from lisa import LISA @LISA( diff --git a/testsuites/test_xdp.py b/testsuites/test_xdp.py index c5f98a70d6..b526db1f0f 100644 --- a/testsuites/test_xdp.py +++ b/testsuites/test_xdp.py @@ -2,8 +2,8 @@ import pytest -from conftest import LINUX_SCRIPTS, LISA -from target import Target +from azure import Azure +from lisa import LINUX_SCRIPTS, LISA @LISA( @@ -13,14 +13,13 @@ tags=["xdp", "network", "hv_netvsc", "sriov"], priority=0, ) -@pytest.mark.deploy( - setup="OneVM2NIC", - networking="SRIOV", - vm_image="Canonical:0001-com-ubuntu-server-focal:20_04-lts:latest", - vm_size="Standard_DS4_v2", -) +# TODO: This example is pending an update. +# setup="OneVM2NIC", +# networking="SRIOV", +# vm_image="Canonical:0001-com-ubuntu-server-focal:20_04-lts:latest", +# vm_size="Standard_DS4_v2", @pytest.mark.skip(reason="Not Finished") -def test_verify_xdp_compliance(target: Target) -> None: +def test_verify_xdp_compliance(target: Azure) -> None: for f in [ "utils.sh", "XDPDumpSetup.sh", From 9fc8d178ccf0b778c78a064b4d79d4e5630fac51 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Tue, 10 Nov 2020 15:15:20 -0800 Subject: [PATCH 076/194] Cleanup types in Target and Azure modules --- azure.py | 38 +++++++++++++++++++++++--------------- target.py | 35 +++++++++++++++++++++++------------ 2 files changed, 46 insertions(+), 27 deletions(-) diff --git a/azure.py b/azure.py index a20b4bbb32..3eade8e4a9 100644 --- a/azure.py +++ b/azure.py @@ -16,24 +16,29 @@ class Azure(Target): """Implements Azure-specific target methods.""" + # Custom instance attribute(s). + internal_address: str + + # A class attribute because it’s defined. az_ok = False - def check_az_cli(self) -> None: + @classmethod + def check_az_cli(cls) -> None: """Assert that the `az` CLI is installed and logged in.""" - if Azure.az_ok: + if cls.az_ok: # Shortcut if we already checked. return # E.g. on Ubuntu: `curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash` - assert self.local("az --version", warn=True), "Please install the `az` CLI!" + assert cls.local("az --version", warn=True), "Please install the `az` CLI!" # TODO: Login with service principal (az login) and set # default subscription (az account set -s) using secrets. - account: Result = self.local("az account show") + account: Result = cls.local("az account show") assert account.ok, "Please `az login`!" sub = json.loads(account.stdout) assert sub["isDefault"], "Please `az account set -s `!" logging.info( f"Using account '{sub['user']['name']}' with subscription '{sub['name']}'" ) - Azure.az_ok = True + cls.az_ok = True def create_boot_storage(self, location: str) -> str: """Create a separate resource group and storage account for boot diagnostics.""" @@ -59,22 +64,23 @@ def allow_ping(self) -> None: try: for d in ["Inbound", "Outbound"]: self.local( - f"az network nsg rule create --name allow{d}ICMP " - f"--nsg-name {self.name}NSG --priority 100 --resource-group {self.name}-rg " + f"az network nsg rule create " + f"--name allow{d}ICMP --resource-group {self.name}-rg " + f"--nsg-name {self.name}NSG --priority 100 " f"--access Allow --direction '{d}' --protocol Icmp " "--source-port-ranges '*' --destination-port-ranges '*'" ) except Exception as e: logging.warning(f"Failed to create ICMP allow rules in NSG due to '{e}'") - def deploy(self): + def deploy(self) -> str: """Given deployment info, deploy a new VM.""" - image = self.params["image"] - sku = self.params["sku"] - location = self.params.get("location", "eastus2") - networking = self.params.get("networking", "") + image = self.parameters["image"] + sku = self.parameters["sku"] + location = self.parameters["location"] + networking = self.parameters["networking"] - self.check_az_cli() + Azure.check_az_cli() logging.info( f"""Deploying VM... @@ -108,9 +114,11 @@ def deploy(self): vm_command.append("--accelerated-networking true") self.data = json.loads(self.local(" ".join(vm_command)).stdout) - self.allow_ping(self.name) + self.allow_ping() # TODO: Enable auto-shutdown 4 hours from deployment. - return self.data["publicIpAddress"] + self.internal_address = self.data["internal_address"] + hostname: str = self.data["publicIpAddress"] + return hostname def delete(self) -> None: """Delete the entire allocated resource group. diff --git a/target.py b/target.py index d78f615aa5..e41da654b9 100644 --- a/target.py +++ b/target.py @@ -6,28 +6,34 @@ from io import BytesIO from uuid import uuid4 -from fabric import Config as FabricConfig +from fabric import Config as FabricConfig # type: ignore from fabric import Connection -from invoke import Config as InvokeConfig +from invoke import Config as InvokeConfig # type: ignore from invoke import Context from invoke.runners import Result # type: ignore +from schema import Schema # type: ignore from tenacity import retry, stop_after_attempt, wait_exponential # type: ignore import lisa if typing.TYPE_CHECKING: - from typing import Any, Mapping, Sequence, Set + from typing import Any, Mapping, Set class Target(ABC): """Extends 'fabric.Connection' with our own utilities.""" - local_context = Context(config=InvokeConfig(overrides=lisa.config)) + # Typed instance attributes, not class attributes. + parameters: Mapping[str, str] + features: Set[str] + name: str + host: str + connection: Connection def __init__( self, - params: Mapping[str, str], - features: Sequence[str], + parameters: Mapping[str, str], + features: Set[str], name: str = f"pytest-{uuid4()}", ): """If not given a name, generates one uniquely. @@ -37,15 +43,17 @@ def __init__( rdma, gpu, xdp. """ - self.params: Mapping[str, str] = params - self.features: Set[str] = set(features) - self.name: str = name + # TODO: Do we need to re-validate the parameters here? + self.parameters = parameters + self.features = features + self.name = name - # TODO: Fix this. + # TODO: Review this thoroughly as currently it depends on + # parameters which is side-effecty. self.host = self.deploy() config = lisa.config.copy() - config["run"]["env"] = { + config["run"]["env"] = { # type: ignore # Set PATH since it’s not a login shell. "PATH": "/sbin:/usr/sbin:/usr/local/sbin:/bin:/usr/bin:/usr/local/bin" } @@ -63,8 +71,11 @@ def delete(self) -> None: """Must delete the target resources.""" ... + # A class attribute because it’s defined. + local_context = Context(config=InvokeConfig(overrides=lisa.config)) + @classmethod - def local(self, *args: Any, **kwargs: Any) -> Result: + def local(cls, *args: Any, **kwargs: Any) -> Result: """This patches Fabric's 'local()' function to ignore SSH environment.""" return Target.local_context.run(*args, **kwargs) From 16e30d7f378d23c706b3854f865be466f76de019 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Tue, 10 Nov 2020 15:21:41 -0800 Subject: [PATCH 077/194] Handle LISA marker being optional --- conftest.py | 25 ++++++++++++++----------- lisa.py | 4 +++- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/conftest.py b/conftest.py index cb61607632..10b56849a5 100644 --- a/conftest.py +++ b/conftest.py @@ -8,7 +8,7 @@ import typing from pathlib import Path -import schema +from schema import SchemaMissingKeyError # type: ignore import lisa import playbook @@ -164,13 +164,11 @@ def pytest_collection_modifyitems( https://docs.pytest.org/en/latest/reference.html#pytest.hookspec.pytest_collection_modifyitems """ - # Validate LISA mark on every item. + # Validate all LISA marks. for item in items: - m = item.get_closest_marker("lisa") - assert m, f"{item} is missing required LISA marker!" try: - lisa.validate(m) - except schema.SchemaMissingKeyError as e: + lisa.validate(item.get_closest_marker("lisa")) + except SchemaMissingKeyError as e: print(f"Test {item.name} failed LISA validation {e}!") items[:] = [] return @@ -194,15 +192,20 @@ def select(item: Item, times: int, exclude: bool) -> None: for c in book.get("criteria"): print(f"Parsing criteria {c}") for item in items: - m = item.get_closest_marker("lisa").kwargs + marker = item.get_closest_marker("lisa") + if not marker: + # Not all tests will have the LISA marker, such as + # static analysis tests. + continue + i = marker.kwargs if any( [ c["name"] and c["name"] in item.name, - c["area"] and c["area"].casefold() == m["area"].casefold(), + c["area"] and c["area"].casefold() == i["area"].casefold(), c["category"] - and c["category"].casefold() == m["category"].casefold(), - c["priority"] and c["priority"] == m["priority"], - c["tags"] and set(c["tags"]) <= set(m["tags"]), + and c["category"].casefold() == i["category"].casefold(), + c["priority"] and c["priority"] == i["priority"], + c["tags"] and set(c["tags"]) <= set(i["tags"]), ] ): select(item, c["times"], c["exclude"]) diff --git a/lisa.py b/lisa.py index 20996c5c51..b80346f9a8 100644 --- a/lisa.py +++ b/lisa.py @@ -41,7 +41,9 @@ ) -def validate(mark: Mark): +def validate(mark: Optional[Mark]) -> None: """Validate each test's LISA parameters.""" + if not mark: + return assert not mark.args, "LISA marker cannot have positional arguments!" mark.kwargs.update(lisa_schema.validate(mark.kwargs)) From 9a199b97a35705a5365b7ea2a96fad87c407c4ff Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Tue, 10 Nov 2020 15:25:58 -0800 Subject: [PATCH 078/194] Load platforms dynamically with their own parameters schema --- azure.py | 16 +++++++++ conftest.py | 73 +++++++++++++++++++++++++------------- playbook.py | 82 +++++++++++++++++++++++++------------------ playbook.yaml | 26 +++++++++++--- selftests/__init__.py | 0 selftests/conftest.py | 17 +++++++++ target.py | 20 +++++++++++ targets.yaml | 12 ------- 8 files changed, 171 insertions(+), 75 deletions(-) create mode 100644 selftests/__init__.py create mode 100644 selftests/conftest.py delete mode 100644 targets.yaml diff --git a/azure.py b/azure.py index 3eade8e4a9..aadb2b5eda 100644 --- a/azure.py +++ b/azure.py @@ -5,6 +5,7 @@ import typing from invoke.runners import Result # type: ignore +from schema import Optional, Schema # type: ignore from tenacity import retry, stop_after_attempt, wait_exponential # type: ignore from target import Target @@ -19,6 +20,21 @@ class Azure(Target): # Custom instance attribute(s). internal_address: str + # @property + # @classmethod + # def schema(cls) -> Schema: + # return + + schema: Schema = Schema( + { + # TODO: Maybe validate as URN or path etc. + "image": str, + Optional("sku", default="Standard_DS1_v2"): str, + Optional("location", default="eastus2"): str, + Optional("networking", default=""): str, + } + ) + # A class attribute because it’s defined. az_ok = False diff --git a/conftest.py b/conftest.py index 10b56849a5..f5cd7f3bb8 100644 --- a/conftest.py +++ b/conftest.py @@ -10,38 +10,35 @@ from schema import SchemaMissingKeyError # type: ignore +import azure # noqa import lisa -import playbook import pytest - -# TODO: Use importlib instead -from azure import Azure from target import Target if typing.TYPE_CHECKING: - from typing import Any, Dict, Iterator, List, Optional + from typing import Any, Dict, Iterator, List, Optional, Type from _pytest.config import Config from _pytest.config.argparsing import Parser - from _pytest.fixtures import FixtureRequest + from _pytest.fixtures import SubRequest from _pytest.python import Metafunc from pytest import Item, Session @pytest.fixture(scope="session") -def pool(request: FixtureRequest) -> Iterator[List[Target]]: +def pool(request: SubRequest) -> Iterator[List[Target]]: """This fixture tracks all deployed target resources.""" targets: List[Target] = [] yield targets for t in targets: - print(f"Created target: {t.features} / {t.params}") + print(f"Created target: {t.features} / {t.parameters}") if not request.config.getoption("keep_vms"): t.delete() @pytest.fixture -def target(pool, worker_id, request: FixtureRequest) -> Iterator[Target]: +def target(pool: List[Target], request: SubRequest) -> Iterator[Target]: """This fixture provides a connected target for each test. It is parametrized indirectly in 'pytest_generate_tests'. @@ -64,19 +61,28 @@ def target(pool, worker_id, request: FixtureRequest) -> Iterator[Target]: their environments. """ - params = request.param + import playbook + + platform: Type[Target] = playbook.PLATFORMS[request.param["platform"]] + parameters: Dict[str, Any] = request.param["parameters"] marker = request.node.get_closest_marker("lisa") features = set(marker.kwargs["features"]) + for t in pool: # TODO: Implement full feature comparison, etc. and not just # proof-of-concept string set comparison. - if params == t.params and features <= t.features: + if all( + [ + isinstance(t, platform), + t.parameters == parameters, + t.features >= features, + ] + ): yield t break else: # TODO: Reimplement caching. - # TODO: Dynamically load platform module and use it. - t = Azure(params, features) + t = platform(parameters, features) pool.append(t) yield t t.connection.close() @@ -94,16 +100,34 @@ def pytest_addoption(parser: Parser) -> None: parser.addoption("--playbook", type=Path, help="Path to playbook.") -TARGETS = [] -TARGET_IDS = [] +TARGETS: List[Dict[str, Any]] = [] +TARGET_IDS: List[str] = [] -def get_playbook(path: Optional[Path]) -> dict(): +def get_playbook(path: Optional[Path]) -> Dict[str, Any]: + """Loads and validates the playbook file. + + This imports the playbook module at runtime to ensure all + subclasses of 'Target' (e.g. all supported platforms, including + those defined in arbitrary 'conftest.py' files) are defined. + + """ + import playbook + book = dict() - if not path: - return book - with open(path) as f: - book = playbook.schema.validate(f) + if path: + # See https://pyyaml.org/wiki/PyYAMLDocumentation + import yaml + + try: + from yaml import CLoader as Loader + except ImportError: + from yaml import Loader # type: ignore + + with open(path) as f: + book = playbook.schema.validate(yaml.load(f, Loader=Loader)) + else: + book = playbook.schema.validate({}) return book @@ -115,7 +139,7 @@ def pytest_configure(config: Config) -> None: configurations based user mode.""" book = get_playbook(config.getoption("--playbook")) - for t in book.get("targets"): + for t in book.get("targets", []): TARGETS.append(t) TARGET_IDS.append(t["name"]) @@ -143,16 +167,15 @@ def pytest_configure(config: Config) -> None: setattr(config.option, attr, value) -def pytest_generate_tests(metafunc: Metafunc): +def pytest_generate_tests(metafunc: Metafunc) -> None: """Parametrize the tests based on our inputs. Note that this hook is run for each test, so we do the file I/O in 'pytest_configure' and save the results. """ - # TODO: Provide a default target? - assert TARGETS, "No targets specified!" if "target" in metafunc.fixturenames: + assert TARGETS, "No targets specified!" metafunc.parametrize("target", TARGETS, True, TARGET_IDS) @@ -189,7 +212,7 @@ def select(item: Item, times: int, exclude: bool) -> None: included.append(item) book = get_playbook(config.getoption("--playbook")) - for c in book.get("criteria"): + for c in book.get("criteria", []): print(f"Parsing criteria {c}") for item in items: marker = item.get_closest_marker("lisa") diff --git a/playbook.py b/playbook.py index ccbf34e44a..0d3a79267c 100644 --- a/playbook.py +++ b/playbook.py @@ -1,11 +1,50 @@ -import yaml +"""Describes the YAML schema for the playbook file. -try: - from yaml import CLoader as Loader -except ImportError: - from yaml import Loader # type: ignore +This module should be imported at runtime such that 'PLATFORMS' is +defined after all 'Target' subclasses have been defined. -from schema import And, Optional, Schema, Use +PLATFORMS is a mapping of platform names (strings) to the implementing +subclass of 'Target' where each subclass defines its own 'parameters' +schema, 'deploy' and 'delete' methods, and other platform-specific +functionality. A 'Target' subclass need only be defined in a file +loaded by Pytest, so a 'contest.py' file works just fine. No manual +registration is required, it will be discovered automatically. + +TODO: Add field annotations, friendly error reporting, automatic case +transformations, etc. + +""" +from __future__ import annotations + +import typing + +# See https://pypi.org/project/schema/ +from schema import Optional, Or, Schema # type: ignore + +from target import Target + +if typing.TYPE_CHECKING: + from typing import Mapping, Type + +# See https://github.com/python/mypy/issues/4717 for why we ignore the type. +PLATFORMS: Mapping[str, Type[Target]] = { + cls.__name__: cls for cls in Target.__subclasses__() # type: ignore +} + +target_schema = Schema( + { + "name": str, + "platform": Or(*[platform for platform in PLATFORMS.keys()]), + # TODO: What should we do when lacking parameters? Ideally we + # use the platform’s defaults from its own schema, but that + # means this value must be set, even if to an empty dict. + Optional("parameters", default=dict): Or( + *[cls.schema for cls in PLATFORMS.values()] + ), + } +) + +default_target = {"name": "Default", "platform": "Local"} criteria_schema = Schema( { @@ -21,34 +60,9 @@ } ) -# NOTE: We could have each platform register its own schema and -# “Or(...)” them together, so this is actually quite flexible. Again, -# so far just writing a proof-of-concept because we need to peer -# review our design. -target_schema = Schema( +schema = Schema( { - # TODO: Maybe set name to image if unset. - "name": str, - # TODO: Use ‘Or([list of registered platforms])’ - "platform": str, - # TODO: Maybe validate as URN or path etc. - Optional("image", default=None): str, - Optional("sku", default=None): str, + Optional("targets", default=[default_target]): [target_schema], + Optional("criteria", default=list): [criteria_schema], } ) - -default_target = {"name": "Default", "platform": "Local"} - -schema = Schema( - And( - # NOTE: This is “magic” that automatically loads and validates - # YAML input. See https://pypi.org/project/schema/ and - # https://pyyaml.org/wiki/PyYAMLDocumentation for - # documentation. - Use(lambda x: yaml.load(x, Loader=Loader)), - { - Optional("targets", default=[default_target]): [target_schema], - Optional("criteria", default=list): [criteria_schema], - }, - ) -) diff --git a/playbook.yaml b/playbook.yaml index 1ab2a7a16a..a291dda079 100644 --- a/playbook.yaml +++ b/playbook.yaml @@ -1,15 +1,33 @@ # NOTE: This is a suggested playbook example. See the schema. targets: + - name: Test + platform: Local + - name: Ubuntu platform: Azure - image: UbuntuLTS - sku: Standard_DS1_v2 + parameters: + image: UbuntuLTS + sku: Standard_DS1_v2 + - name: Debian platform: Azure - image: credativ:Debian:9:9.0.201706190 + parameters: + image: credativ:Debian:9:9.0.201706190 + - name: GitHub platform: Azure - image: github:github-enterprise:github-enterprise:latest + parameters: + image: github:github-enterprise:github-enterprise:latest + + - name: Citrix + platform: Azure + parameters: + image: citrix:netscalervpx-130:netscalerbyol:latest + + - name: AudioCodes + platform: Azure + parameters: + image: audiocodes:mediantsessionbordercontroller:mediantvirtualsbcazure:latest criteria: - name: smoke diff --git a/selftests/__init__.py b/selftests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/selftests/conftest.py b/selftests/conftest.py new file mode 100644 index 0000000000..cc7fd59999 --- /dev/null +++ b/selftests/conftest.py @@ -0,0 +1,17 @@ +from schema import Schema # type: ignore + +from target import Target + + +class Custom(Target): + schema: Schema = Schema(None) + # @property + # @classmethod + # def schema(cls) -> Schema: + # return Schema() + + def deploy(self) -> str: + return "localhost" + + def delete(self) -> None: + pass diff --git a/target.py b/target.py index e41da654b9..a9f1e382a2 100644 --- a/target.py +++ b/target.py @@ -61,6 +61,16 @@ def __init__( self.host, config=FabricConfig(overrides=config), inline_ssh_env=True ) + # TODO: Use an abstract class property to ensure this is defined. + schema: Schema = Schema(None) + + # @property + # @classmethod + # @abstractmethod + # def schema(cls) -> Schema: + # """Must return the parameters schema for setup.""" + # ... + @abstractmethod def deploy(self) -> str: """Must deploy the target resources and return hostname.""" @@ -104,3 +114,13 @@ def cat(self, path: str) -> str: with BytesIO() as buf: self.get(path, buf) return buf.getvalue().decode("utf-8").strip() + + +class Local(Target): + schema: Schema = Schema(None) + + def deploy(self) -> str: + return "localhost" + + def delete(self) -> None: + pass diff --git a/targets.yaml b/targets.yaml deleted file mode 100644 index 58c3d04797..0000000000 --- a/targets.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# TODO: We need to actually think about the schema here. -- target: - image: "citrix:netscalervpx-130:netscalerbyol:latest" - -- target: - image: "audiocodes:mediantsessionbordercontroller:mediantvirtualsbcazure:latest" - -- target: - image: "credativ:Debian:9:9.0.201706190" - -- target: - image: "github:github-enterprise:github-enterprise:latest" From af478155601a0564dd995443678c427ce1a41277 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Tue, 10 Nov 2020 15:43:13 -0800 Subject: [PATCH 079/194] Move smoke test targets to smoke.yaml --- playbook.yaml | 23 ----------------------- smoke.yaml | 23 +++++++++++++++++++++++ 2 files changed, 23 insertions(+), 23 deletions(-) create mode 100644 smoke.yaml diff --git a/playbook.yaml b/playbook.yaml index a291dda079..df9c3178b2 100644 --- a/playbook.yaml +++ b/playbook.yaml @@ -8,26 +8,3 @@ targets: parameters: image: UbuntuLTS sku: Standard_DS1_v2 - - - name: Debian - platform: Azure - parameters: - image: credativ:Debian:9:9.0.201706190 - - - name: GitHub - platform: Azure - parameters: - image: github:github-enterprise:github-enterprise:latest - - - name: Citrix - platform: Azure - parameters: - image: citrix:netscalervpx-130:netscalerbyol:latest - - - name: AudioCodes - platform: Azure - parameters: - image: audiocodes:mediantsessionbordercontroller:mediantvirtualsbcazure:latest - -criteria: - - name: smoke diff --git a/smoke.yaml b/smoke.yaml new file mode 100644 index 0000000000..ea1a15ab60 --- /dev/null +++ b/smoke.yaml @@ -0,0 +1,23 @@ +targets: + - name: Debian + platform: Azure + parameters: + image: credativ:Debian:9:9.0.201706190 + + - name: GitHub + platform: Azure + parameters: + image: github:github-enterprise:github-enterprise:latest + + - name: Citrix + platform: Azure + parameters: + image: citrix:netscalervpx-130:netscalerbyol:latest + + - name: AudioCodes + platform: Azure + parameters: + image: audiocodes:mediantsessionbordercontroller:mediantvirtualsbcazure:latest + +criteria: + - name: smoke From 2edf257499d4563da4821fdd6bc2c27b6a0f68ec Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Wed, 11 Nov 2020 12:46:28 -0800 Subject: [PATCH 080/194] =?UTF-8?q?Move=20playbooks=20and=20fix=20self=20t?= =?UTF-8?q?ests=20to=20use=20=E2=80=9CCustom=E2=80=9D=20platform?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instead of “Azure” which of course doesn’t work in CI. --- conftest.py | 9 ++++++++- playbook.yaml | 10 ---------- criteria.yaml => playbooks/criteria.yaml | 0 smoke.yaml => playbooks/smoke.yaml | 0 playbooks/test.yaml | 5 +++++ selftests/setup_plan/test_plan_A.py | 2 +- selftests/setup_plan/test_plan_B.py | 2 +- selftests/setup_plan/test_plan_C.py | 2 +- 8 files changed, 16 insertions(+), 14 deletions(-) delete mode 100644 playbook.yaml rename criteria.yaml => playbooks/criteria.yaml (100%) rename smoke.yaml => playbooks/smoke.yaml (100%) create mode 100644 playbooks/test.yaml diff --git a/conftest.py b/conftest.py index f5cd7f3bb8..6c8ca9c30b 100644 --- a/conftest.py +++ b/conftest.py @@ -92,6 +92,7 @@ def pytest_addoption(parser: Parser) -> None: """Pytest hook for adding arbitrary CLI options. https://docs.pytest.org/en/latest/example/simple.html + https://docs.pytest.org/en/latest/reference.html#pytest.hookspec.pytest_addoption """ parser.addoption("--keep-vms", action="store_true", help="Keeps deployed VMs.") @@ -112,6 +113,8 @@ def get_playbook(path: Optional[Path]) -> Dict[str, Any]: those defined in arbitrary 'conftest.py' files) are defined. """ + # TODO: Move to 'playbook.py' and setup 'PLATFORMS' when called so + # that the import can take place at any time. import playbook book = dict() @@ -137,7 +140,9 @@ def pytest_configure(config: Config) -> None: Determines the targets based on the playbook and sets default configurations based user mode. - configurations based user mode.""" + https://docs.pytest.org/en/latest/reference.html#pytest.hookspec.pytest_configure + + """ book = get_playbook(config.getoption("--playbook")) for t in book.get("targets", []): TARGETS.append(t) @@ -173,6 +178,8 @@ def pytest_generate_tests(metafunc: Metafunc) -> None: Note that this hook is run for each test, so we do the file I/O in 'pytest_configure' and save the results. + https://docs.pytest.org/en/latest/reference.html#pytest.hookspec.pytest_generate_tests + """ if "target" in metafunc.fixturenames: assert TARGETS, "No targets specified!" diff --git a/playbook.yaml b/playbook.yaml deleted file mode 100644 index df9c3178b2..0000000000 --- a/playbook.yaml +++ /dev/null @@ -1,10 +0,0 @@ -# NOTE: This is a suggested playbook example. See the schema. -targets: - - name: Test - platform: Local - - - name: Ubuntu - platform: Azure - parameters: - image: UbuntuLTS - sku: Standard_DS1_v2 diff --git a/criteria.yaml b/playbooks/criteria.yaml similarity index 100% rename from criteria.yaml rename to playbooks/criteria.yaml diff --git a/smoke.yaml b/playbooks/smoke.yaml similarity index 100% rename from smoke.yaml rename to playbooks/smoke.yaml diff --git a/playbooks/test.yaml b/playbooks/test.yaml new file mode 100644 index 0000000000..0495002ab0 --- /dev/null +++ b/playbooks/test.yaml @@ -0,0 +1,5 @@ +targets: + - name: Local Tests + platform: Local + - name: Setup Plan + platform: Custom diff --git a/selftests/setup_plan/test_plan_A.py b/selftests/setup_plan/test_plan_A.py index 54f2e2543f..5a4e049dd6 100644 --- a/selftests/setup_plan/test_plan_A.py +++ b/selftests/setup_plan/test_plan_A.py @@ -4,7 +4,7 @@ from target import Target LISA = functools.partial( - lisa.LISA, platform="Azure", category="Functional", area="self-test", priority=1 + lisa.LISA, platform="Custom", category="Functional", area="self-test", priority=1 ) diff --git a/selftests/setup_plan/test_plan_B.py b/selftests/setup_plan/test_plan_B.py index 90b214e58c..0d89896300 100644 --- a/selftests/setup_plan/test_plan_B.py +++ b/selftests/setup_plan/test_plan_B.py @@ -4,7 +4,7 @@ from target import Target LISA = functools.partial( - lisa.LISA, platform="Azure", category="Functional", area="self-test", priority=1 + lisa.LISA, platform="Custom", category="Functional", area="self-test", priority=1 ) diff --git a/selftests/setup_plan/test_plan_C.py b/selftests/setup_plan/test_plan_C.py index 8aff95e2fb..efc579fe10 100644 --- a/selftests/setup_plan/test_plan_C.py +++ b/selftests/setup_plan/test_plan_C.py @@ -4,7 +4,7 @@ from target import Target LISA = functools.partial( - lisa.LISA, platform="Azure", category="Functional", area="self-test", priority=1 + lisa.LISA, platform="Custom", category="Functional", area="self-test", priority=1 ) From 6c1c7469624c8c5dcd7221bfabafdb400effc602 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Wed, 11 Nov 2020 19:58:59 -0800 Subject: [PATCH 081/194] Small cleanups found during update of design document --- conftest.py | 17 +++++++++++------ lisa.py | 5 +++-- playbooks/criteria.yaml | 10 +++------- target.py | 19 +++++++++---------- testsuites/test_lis.py | 1 + 5 files changed, 27 insertions(+), 25 deletions(-) diff --git a/conftest.py b/conftest.py index 6c8ca9c30b..2dfe991bd2 100644 --- a/conftest.py +++ b/conftest.py @@ -68,15 +68,16 @@ def target(pool: List[Target], request: SubRequest) -> Iterator[Target]: marker = request.node.get_closest_marker("lisa") features = set(marker.kwargs["features"]) + # TODO: If `t` is not already in use, deallocate the previous + # target, and ensure the tests have been sorted (and so grouped) + # by their requirements. for t in pool: # TODO: Implement full feature comparison, etc. and not just # proof-of-concept string set comparison. - if all( - [ - isinstance(t, platform), - t.parameters == parameters, - t.features >= features, - ] + if ( + isinstance(t, platform) + and t.parameters == parameters + and t.features >= features ): yield t break @@ -194,6 +195,10 @@ def pytest_collection_modifyitems( https://docs.pytest.org/en/latest/reference.html#pytest.hookspec.pytest_collection_modifyitems """ + # TODO: The ‘Item’ object has a ‘user_properties’ attribute which + # is a list of tuples and could be used to hold the validated + # marker data, simplifying later usage. + # Validate all LISA marks. for item in items: try: diff --git a/lisa.py b/lisa.py index b80346f9a8..bc3b16506e 100644 --- a/lisa.py +++ b/lisa.py @@ -35,15 +35,16 @@ "area": str, "priority": Or(0, 1, 2, 3), Optional("features", default=list): [str], + Optional("tags", default=list): [str], Optional(object): object, }, ignore_extra_keys=True, ) -def validate(mark: Optional[Mark]) -> None: +def validate(mark: typing.Optional[Mark]) -> None: """Validate each test's LISA parameters.""" if not mark: return assert not mark.args, "LISA marker cannot have positional arguments!" - mark.kwargs.update(lisa_schema.validate(mark.kwargs)) + mark.kwargs.update(lisa_schema.validate(mark.kwargs)) # type: ignore diff --git a/playbooks/criteria.yaml b/playbooks/criteria.yaml index cec1b91c3f..3e86bc538d 100644 --- a/playbooks/criteria.yaml +++ b/playbooks/criteria.yaml @@ -1,14 +1,10 @@ # NOTE: This is an adjusted proof-of-concept ask from Chi. criteria: - # select all p0 cases - # for example, selected three cases: a,b,c + # Select all Priority 0 tests. - priority: 0 - # run smoke_test cases twice, to prove a distro stable enough - # after this rule, the picked test cases is like a,b,b + # Run tests with 'smoke' in the name twice. - name: smoke times: 2 - # drop all cases of xdp, - # because it's not ready on a tested distro. - # for example, droped c, so now is: a,b + # Exclude all tests in Area "xdp" - area: xdp exclude: true diff --git a/target.py b/target.py index a9f1e382a2..2608039cd5 100644 --- a/target.py +++ b/target.py @@ -6,10 +6,8 @@ from io import BytesIO from uuid import uuid4 -from fabric import Config as FabricConfig # type: ignore -from fabric import Connection -from invoke import Config as InvokeConfig # type: ignore -from invoke import Context +import fabric # type: ignore +import invoke # type: ignore from invoke.runners import Result # type: ignore from schema import Schema # type: ignore from tenacity import retry, stop_after_attempt, wait_exponential # type: ignore @@ -28,7 +26,7 @@ class Target(ABC): features: Set[str] name: str host: str - connection: Connection + conn: fabric.Connection def __init__( self, @@ -57,8 +55,8 @@ def __init__( # Set PATH since it’s not a login shell. "PATH": "/sbin:/usr/sbin:/usr/local/sbin:/bin:/usr/bin:/usr/local/bin" } - self.connection = Connection( - self.host, config=FabricConfig(overrides=config), inline_ssh_env=True + self.connection = fabric.Connection( + self.host, config=fabric.Config(overrides=config), inline_ssh_env=True ) # TODO: Use an abstract class property to ensure this is defined. @@ -82,15 +80,16 @@ def delete(self) -> None: ... # A class attribute because it’s defined. - local_context = Context(config=InvokeConfig(overrides=lisa.config)) + local_context = invoke.Context(config=invoke.Config(overrides=lisa.config)) @classmethod def local(cls, *args: Any, **kwargs: Any) -> Result: """This patches Fabric's 'local()' function to ignore SSH environment.""" return Target.local_context.run(*args, **kwargs) - # TODO: Generate these automatically. There’s some weird bug with - # inheriting from ‘Connection’ that causes infinite recursion. + # TODO: Refactor this. We don’t want to inherit from `Connection` + # because that’s overly complicated. Honestly we probably just + # want users to call `target.conn.run()` etc. def run(self, *args: Any, **kwargs: Any) -> Result: return self.connection.run(*args, **kwargs) diff --git a/testsuites/test_lis.py b/testsuites/test_lis.py index 86921f4fa1..a35e6fe94e 100644 --- a/testsuites/test_lis.py +++ b/testsuites/test_lis.py @@ -11,6 +11,7 @@ @LISA(platform="Azure", category="Functional", priority=0, area="LIS_DEPLOY") def test_lis_driver_version(target: Azure) -> None: + """Checks that the installed drivers have the correct version.""" # TODO: Include “utils.sh” automatically? Or something... for f in ["utils.sh", "LIS-VERSION-CHECK.sh"]: target.put(LINUX_SCRIPTS / f) From 03125ce7edcc26e3d89dad773d65309a0b917975 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Wed, 11 Nov 2020 20:00:17 -0800 Subject: [PATCH 082/194] Write version 0.2 of design document --- DESIGN.md | 874 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 484 insertions(+), 390 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 57acb4848b..50ebb9a4c2 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -7,7 +7,7 @@ evaluating the feasibility of leveraging Please see [PR #1065](https://github.com/LIS/LISAv2/pull/1065) for a working, proof-of-concept prototype. -Authored by Andrew Schwartzmeyer (he/him), version 0.1.0. +Authored by Andrew Schwartzmeyer (he/him), version 0.2.0. ## Why Pytest? @@ -15,10 +15,11 @@ Pytest is an [incredibly popular](https://docs.pytest.org/en/stable/talks.html) MIT licensed open source Python testing framework. It has a thriving community and plugin framework, with over 750 [plugins](https://plugincompat.herokuapp.com/). Instead of writing (and -therefore maintaining) yet another test framework, we would do less with more by +therefore maintaining) yet another test framework, we would do more with less by reusing Pytest and existing plugins. This will allow us to focus on our unique problems: organizing and understanding our tests, deploying necessary resources -(such as Azure or Hyper-V virtual machines), and analyzing our results. +(such as Azure, Hyper-V, or bare metal machines, collectively known as +“targets”), and analyzing our results. In fact, most of Pytest itself is implemented via [built-in plugins](https://docs.pytest.org/en/stable/plugins.html), providing us with many @@ -43,23 +44,315 @@ needs very well: * Modular setup/teardown via fixtures * Incredibly customizable (as detailed above) -So all the logic for describing, discovering, running, skipping based on -requirements, and reporting results of the tests is already written and -maintained by the greater open source community, leaving us to focus on our hard -and specific problem: creating an abstraction to launch the necessary nodes in -our environments. Using Pytest would also allow us the space to abstract other +So all the logic for describing, discovering, running, skipping and reporting +results of the tests, as well as enabling and importing users’ plugins is +already written and maintained by the open source community. This leaves us to +focus on our hard and specific problems: creating an abstraction to launch the +necessary targets, organizing and publishing our tests, and reporting test +results upstream. Using Pytest would also allow us the space to abstract other commonalities in our specific tests. In this way, LISAv3 could solve the difficulties we have at hand without creating yet another test framework. -## High-Level Design Decisions +Finally, by leveraging such a popular framework and reducing the amount of code +we need to maintain, we drastically increase our chances of receiving pull +requests instead of bug reports from users. This is important because despite +our best efforts it is practically guaranteed that as adoption of LISAv3 +increases, users will want changes to be made, and we need to empower them to do +so themselves. + +## What are we maintaining? + +The current proof-of-concept implementation uses the top-level `conftest.py` +file to define our “plugin” functionality. This works, but it is not ideal. I +believe that we will want to publish two open source Pytest plugins as packages +on [PyPI](https://pypi.org/), the Python Package Index: `pytest-target` and +`pytest-lisa`. We will also maintain our set of public “LISA” tests, but these +should simply install and use our plugins. + +The `pytest-target` plugin should encapsulate all our logic for _how_ and _when_ +to deploy targets (local or cloud virtual machines, or bare metal machines, and +all the associated resources), run tests on the specified targets, and delete +the targets. This includes specifying which features and resources each test +needs and each given target provides (such as number of cores, amount of RAM, +and other hardware like a GPU etc.), how to deploy and delete each target based +on its platform, and parameterization of the `target` fixture based on CLI or +YAML file input. In fact, some tests (like networking) will require multiple +targets at once. This plugin will need to manage resources intelligently, being +able to optimize for both time and cost, and make it easy for tests to request +and use various resources. + +The `pytest-lisa` plugin should encapsulate all our logic for how to organize +and select tests, as well as our opinions on displaying test results. This +includes the user modes, test metadata and inventory, test selection based on +criteria against that metadata, required and pre-configured upstream plugins, +and result notifiers. It will similarly support both CLI and YAML file input. + +We should strive to keep these plugins from depending on each other in order to +keep their scope well-defined. In the “LISA” repository of tests we will depend +on the two plugins and maintain additional fixtures for our tests’ unique +requirements. Similarly, we and others may have private test repositories which +build upon the above by defining new platform support and internal service +integrations. + +## pytest-target + +### How are targets provided and accessed? + +First we need to define “target” as an instance of a system-under-test. That is, +given some environment requirements, such an Azure image (URN) and size (SKU), a +target would be a virtual machine deployed by `pytest-target` with SSH access +provided to the requesting test. A target could optionally be pre-deployed and +simply connected. Some tests may request multiple targets as well. -### What are the User Modes? +Pytest uses [fixtures](https://docs.pytest.org/en/stable/fixture.html), which +are the primary way of setting up test requirements. They replace less flexible +alternatives like setup/teardown functions. It is through fixtures that we +implement remote target setup/teardown. Our `target` fixture returns a `Target` +instance, which currently provides: + +* Remote shell access via SSH +* Data including hostname / IP address +* Cross-platform ping functionality with exponential back-off +* Uploading of local files to arbitrary remote destinations +* Downloading of remote file contents into local string variable +* Asynchronous remote command execution with promises + +The `Azure(Target)` subclass additionally provides: + +* Automatic provisioning of an Azure VM given URN and SKU +* Allowing ICMP ping via Azure firewall rules +* Azure platform forced reboot by API +* Downloading boot diagnostics (serial console log) from platform + +The prototype demonstrates how easy it is to quickly implement these features. +As we need more features, they can be readily added and shared among tests. + +The `Target` class leverages [Fabric](https://www.fabfile.org/) which is a +popular high-level Python library for executing shell commands on remote systems +over SSH. Underneath the covers Fabric uses +[paramiko](https://docs.paramiko.org/en/stable/), the most popular low-level +Python SSH library. Fabric does the heavy lifting of safely connecting and +disconnecting from the node, executing the shell command (synchronously or +asynchronously), reporting the exit status, gathering the stdout and stderr, +providing stdin (or interactive auto-responses, similar to `expect`), uploading +and downloading files, and much more. In fact, these APIs are all available and +implemented for the local machine by the underlying +[Inovke](https://www.pyinvoke.org/) library, which is essentially a Python +`subprocess` wrapper with “a powerful and clean feature set.” + +Other test specific requirements, such as installing software and daemons, +downloading files from remote storage, or checking the state of our Bash test +scripts, would similarly be implemented by methods on the `Target` class or via +additional fixtures and thus shared among tests. + +### How do we interact with Azure? + +For Azure, we currently use the [Azure CLI](https://aka.ms/azureclidocs) to +deploy a virtual machine. For Hyper-V (and other virtualization platforms), we +would like to use [libvirt](https://libvirt.org/python.html), and for embedded +environments we are evaluating +[labgrid](https://github.com/labgrid-project/labgrid). + +If possible, we do not want to use the [Azure Python +APIs](https://aka.ms/azsdk/python/all) directly because they are more +complicated (and less documented) than the [Azure +CLI](https://aka.ms/azureclidocs). With Invoke (as discussed above), `az` +becomes incredibly easy to work with. The Azure CLI lead developer states that +they have [feature parity](https://stackoverflow.com/a/50005660/1028665) and +that the CLI is more straightforward to use. Considering our ease-of-maintenance +requirement, this seems the apt choice. If it later becomes necessary to use the +Python APIs directly, that is, of course, still doable. + +### What’s the `Target` class? + +In version 0.1 of this design document we detailed a planned refactor of what +was then called the `Node` class. This has since been executed with just a few +modifications (one being the rename to `Target`, as `Node` was found to be an +overloaded term in the context of data centers). This class and its subclasses +are decoupled from Pytest, and are used via fixtures. It looks like this: + +```python +from abc import ABC, abstractmethod +from schema import Schema +import fabric + +class Target(ABC): + parameters: Mapping[str, str] + features: Set[str] + name: str + host: str + conn: fabric.Connection # Provides run, sudo, get, put etc. + + def __init__(...): + ... + self.host = self.deploy() + self.conn = fabric.Connection(self.host) + + @classmethod + @property + @abstractmethod + def schema(cls) -> Schema: + """Must return the parameters schema for setup.""" + ... + + @abstractmethod + def deploy(self) -> str: + """Must deploy the target resources and return hostname.""" + ... + + @abstractmethod + def delete(self) -> None: + """Must delete the target resources.""" + ... + + @classmethod + def local(...) -> Result: + """Runs a local shell command.""" + ... +``` + +#### How are platforms implemented? + +Platform support is implemented by subclassing `Target` and defining the +`schema` property, `deploy` method, `delete` method, and any platform-specific +methods. Using the `__subclasses__` attribute of `Target` the available +platforms and their parameter schemata are automatically gathered from users’ +own `conftest.py` files and other plugins. This enables the `target` fixture to +dynamically instantiate a target from the gathered requirements and parameters. + +#### How are requirements examined? + +The `features` attribute is currently a set of strings and (combined with the +parameters dictionary) was used to demonstrate how we can test if an existing +target instance (representing a deployed machine) met a test’s requirements. It +should be updated with a `Requirements` class that represents all physical +attributes of the target, and a `requires` Pytest mark should be added which +takes instances of this class. Two `Requirements` should be comparable to +determine if one set meets (or exceeds) the other set. + +#### How do we share common tasks? + +Common tasks for targets like rebooting and pinging should be implemented on the +`Target` class, and platform-specific tasks on the respective subclass. + +Methods available from `Connection` include `run()` and `sudo()` which are used +to easily run arbitrary commands, and `get()` and `put()` to download and upload +arbitrary files. + +The `cat()` method wraps `get()` and returns the file as data in a string. This +makes test code like this possible: + +```python +assert target.conn.cat("state.txt") == "TestCompleted" +``` -Because Pytest is infinitely customizable, we want to provide a few sets of +A `reboot()` method should be added that first tries to use `sudo("reboot", +timeout=5)` (with a short timeout to avoid a hung SSH session). It should retry +with an exponential back-off to see if the machine has rebooted by checking +either `uptime` or the existence of a file created before the reboot. This is to +avoid having to `sleep()` and just guess the amount of time it takes to reboot. + +A `restart()` method should “power cycle” the machine using the platform’s API, +and thus is in abstract method. + +Other tools and shared logic should be implemented as necessary. A major area of +concern is the automatic and package-manager agnostic installation of necessary +tools, much of which has been implemented previously and can be integrated. + +### How are targets requested and managed? + +We implement a pair of Pytest fixtures to provide targets. The first is the +`pool` fixture, which looks like: + +```python +@pytest.fixture(scope="session") +def pool(request: SubRequest) -> Iterator[List[Target]]: + """This fixture tracks all deployed target resources.""" + targets: List[Target] = [] + yield targets + for t in targets: + t.delete() +``` + +The `pool` fixture is setup once at the beginning of the test session, at which +point the `targets` list is then provided as input to every instance of the +`target` fixture. While currently a list, to support optimal scheduling we will +likely want to use a priority queue, where the priority of a target represents +its cost (whether in terms of time or money), allowing us to provide either the +fastest or the cheapest target to each request. Targets not in use will be +deallocated, and all targets will be automatically deleted after the tests are +finished (unless the user requested otherwise, in which case they’ll be cached). + +Note that cross-session [caching](https://docs.pytest.org/en/stable/cache.html) +is provided by Pytest, and very easy to work with. An early prototype +implemented a `--keep-vms` flag successfully, and this will be implemented again +with the updated design. + +The second is the `target` fixture, which looks like: + +```python +@pytest.fixture +def target(pool: List[Target], request: SubRequest) -> Iterator[Target]: + """This fixture provides a connected target for each test.""" + platform: Type[Target] = playbook.PLATFORMS[request.param["platform"]] + parameters: Dict[str, Any] = request.param["parameters"] + marker = request.node.get_closest_marker("lisa") + features = set(marker.kwargs["features"]) + + # TODO: If `t` is not already in use, deallocate the previous target. + for t in pool: + if isinstance(t, platform) and t.parameters == parameters and t.features >= features: + yield t + break + else: + t = platform(parameters, features) + pool.append(t) + yield t + t.connection.close() +``` + +This is obviously still an early implementation, but it is viable. By using the +[pytest_collection_modifyitems][] hook to sort (and so group) the tests by their +requirements, the tests would efficiently reuse targets. This fixture is +indirectly parameterized during setup with the [pytest_generate_tests][] hook. +Test and fixture [parameterization][] is a huge feature of Pytest. When we +parameterize the `target` fixture for multiple targets (e.g. “Ubuntu” and +“Debian”), Pytest automatically creates a set of tests for each target. So +`test_smoke` turns into `test_smoke[Ubuntu]` and `test_smoke[Debian]`. This +allows us to run a collection of tests against multiple targets with ease. These +targets are defined in a YAML file and validated against the parameters +collected from the previously described platform subclasses. + +### How are tests executed in parallel? + +While our original list of goals stated that we want to run tests “in parallel” +we were not specific about what was meant, and the topic of parallelism and +concurrency is understandably complex. We certainly don’t mean running two tests +at once on the same target, as this would undoubtedly lead to flaky tests. + +Assuming that we care about a set of tests passing on a particular image and +size combination, but not necessarily on a particular deployed instance, then we +can run tests concurrently by deploying multiple “identical” targets and +splitting the tests across them. The tests would still run in isolation on each +target. This sounds hard, but actually it’s practically free with Pytest via +[pytest-xdist][]. + +The default `pytest-xdist` implementation simply takes the list of tests and +runs them in a round-robin fashion with the desired number of executors. We’ve +talked at length about being able to schedule groups of tests to run in +particular executors and using particular targets. While there are many paths +open to us, this plugin actually provides a hook, `pytest_xdist_make_scheduler` +that exists specifically to “implement custom tests distribution logic.” + +## pytest-lisa + +### What are the user modes? + +Because Pytest is incredibly customizable, we want to provide a few sets of reasonable default configurations for some common scenarios. We will add a flag -like `--mode=[dev,debug,ci,demo]` to change the default options and output of -Pytest. Doing so is readily supported by Pytest via the `pytest_addoption` and -`pytest_configure` hooks. We call these the provided “user modes.” +like `--lisa-mode=[dev,debug,ci,demo]` to change the default options and output +of Pytest. Doing so is readily supported by Pytest via the [pytest_addoption][] +and [pytest_configure][] hooks. We call these the provided “user modes.” * The dev(eloper) mode is intended for use by test developers while writing a new test. It is verbose, caches the deployed VMs between runs, and generates a @@ -78,7 +371,7 @@ Pytest. Doing so is readily supported by Pytest via the `pytest_addoption` and * The demo mode will show the “executive summary” (a lot like CI, but finely tuned for demos). For example, what `make smoke` currently shows. -### How Are Tests Described? +### How are tests described? The built-in [pytest-mark](https://docs.pytest.org/en/stable/mark.html) plugin already provides functionality for adding metadata to tests, where we @@ -86,7 +379,7 @@ specifically want: * Platform: used to skip tests inapplicable to the current system-under-test * Category: our high-level test organization -* Area: feature being tested (could default to module name) +* Area: feature being tested * Priority: self-explanatory * Tags: optional additional metadata for test organization @@ -98,35 +391,75 @@ It looks like this: ```python import pytest -@pytest.mark.lisa( - platform="Azure", category="Functional", area="LIS_DEPLOY", priority=0, tags=["lis"] -) -def test_lis_driver_version(node: Node) -> None: +@pytest.mark.lisa(platform="Azure", category="Functional", priority=0, area="LIS_DEPLOY") +def test_lis_driver_version(target: Azure) -> None: """Checks that the installed drivers have the correct version.""" ... ``` This is a functional example, which takes zero implementation. With this simple -decorator, all test collection hooks can introspect the metadata, enforce +decorator, all test [collection hooks][] can introspect the metadata, enforce required parameters and set defaults, select tests based on arbitrary criteria, and list test coverage statistics. Note that Pytest leverages Python’s docstrings for built-in documentation (and -can even run tests discovered in such strings, like doctest). Being just Python -code, this decorator need not be `@pytest.mark.lisa(...)` but can trivially be -provided as simply `@lisa(...)`. +can even run tests discovered in such strings, like doctest). Hence we do not +have a separate field for the test’s documentation. + +Being just Python code, this decorator need not be `@pytest.mark.lisa(...)` but +can trivially be provided as simply `@LISA(...)`. In fact, we provide this in +`lisa.py` with: + +```python +LISA = pytest.mark.lisa + +@LISA(...) +def test_something(...) +``` + +Currently we validate the parameters given to this mark during test collection, +by using the following code, which leverages the [schema][] library: + +```python +from schema import Optional, Or, Schema + +lisa_schema = Schema( + { + "platform": str, + "category": Or("Functional", "Performance", "Stress", "Community", "Longhaul"), + "area": str, + "priority": Or(0, 1, 2, 3), + Optional("tags", default=list): [str], + }, +) + +def validate(mark: Mark) -> None: + """Validate each test's LISA parameters.""" + assert not mark.args, "LISA marker cannot have positional arguments!" + mark.kwargs.update(lisa_schema.validate(mark.kwargs)) +``` + +In the future we could change `LISA` to be a function with these keyword +arguments so that IDE auto-completion is enabled. However, this is not mandatory +to move forward, and parameter validation is enabled succinctly with the above. This mark also does need to be repeated for each test, as marks can be scoped to a module, and so one line could describe defaults for every test in a file, with -individual tests overriding parameters as needed. We may also introduce marks -such as `@pytest.mark.slow` to allow for easier test selection. +individual tests overriding parameters as needed. -We even have a prototype +In the current implementation, we also take a `features: List[str]` argument +that is used to prove the concept deploying (or reusing) a target based on the +test’s required and the target’s available sets of features. However, as we move +forward we should define a separate `requires` mark that takes well-defined +classes describing the minimal required resources for a test. This will be part +of the refactor into the two Pytest plugins mentioned above. + +Furthermore, we have a prototype [generator](https://github.com/LIS/LISAv2/tree/pytest/generator) which parses LISAv2 XML test descriptions and generates stubs with this mark filled in correctly. -### How Are Tests Selected? +### How are tests selected? Pytest already allows a user to specify which exact tests to run: @@ -135,28 +468,86 @@ Pytest already allows a user to specify which exact tests to run: * Specifying a mark expression on the CLI (e.g. `-m functional and not slow`) We can also implement any other mechanism via the -`pytest_collection_modifyitems` hook. There’s already a -[proof-of-concept](https://github.com/LIS/LISAv2/blob/ab01c33f1f1e1ffac7100f6a69beda07192f05bb/pytest/conftest.py#L49) -which uses selection criteria read from a YAML file: +[pytest_collection_modifyitems][] hook. The proof-of-concept supports gathering +selection criteria from a YAML file: ```yaml -# Select all Priority 0 tests -- criteria: - priority: 0 -# Exclude all tests in Area "xdp" -- criteria: - area: xdp - select_action: forceExclude -# Run test with name `test_smoke` twice -- criteria: - name: test_smoke - times: 2 +criteria: + # Select all Priority 0 tests. + - priority: 0 + # Run tests with 'smoke' in the name twice. + - name: smoke + times: 2 + # Exclude all tests in Area "xdp" + - area: xdp + exclude: true +``` + +This criteria is validated against the following [schema][]: + +```python +from schema import Schema, Optional + +criteria_schema = Schema( + { + # TODO: Validate that these strings are valid regular + # expressions if we change our matching logic. + Optional("name", default=None): str, + Optional("area", default=None): str, + Optional("category", default=None): str, + Optional("priority", default=None): int, + Optional("tags", default=list): [str], + Optional("times", default=1): int, + Optional("exclude", default=False): bool, + } +) +``` + +The test collection is then modified using the Pytest hook, +[pytest_collection_modifyitems][]: + +```python +def pytest_collection_modifyitems( + session: Session, config: Config, items: List[Item] +) -> None: + included: List[Item] = [] + excluded: List[Item] = [] + + def select(item: Item, times: int, exclude: bool) -> None: + if exclude: + excluded.append(item) + else: + for _ in range(times - included.count(item)): + included.append(item) + + for c in criteria: # Where `criteria` is from the schema. + for item in items: + marker = item.get_closest_marker("lisa") + if not marker: + # Not all tests will have the LISA marker, such as + # static analysis tests. + continue + i = marker.kwargs + if any( + [ + c["name"] and c["name"] in item.name, + c["area"] and c["area"].casefold() == i["area"].casefold(), + c["category"] + and c["category"].casefold() == i["category"].casefold(), + c["priority"] and c["priority"] == i["priority"], + c["tags"] and set(c["tags"]) <= set(i["tags"]), + ] + ): + select(item, c["times"], c["exclude"]) + items[:] = [i for i in included if i not in excluded] ``` -However, before we settle on the basic schema understood by the -proof-of-concept, we should write and _review_ a full schema. +Because this is simply a Python list, we can also sort the tests according to +our needs, such as by priority. If the `python-targets` plugin has already +sorted by requirements, that’s just fine, Python’s `sorted()` built-in is +guaranteed to be stable (meaning we can sort in multiple passes). -### How Are Results Reported? +### How are results reported? Parsing the results of a large test suite can be difficult. Fortunately, because Pytest is a testing framework, there already exists support for generating @@ -173,104 +564,15 @@ community plugin [pytest-azurepipelines](https://pypi.org/project/pytest-azurepipelines/) which enhances the standard JUnit report for ADO. -### How Are Nodes Provided and Accessed? - -First we need to define “node” as an instance of a system-under-test. That is, -given some environment requirements, such an Azure image (URN) and image (SKU), -a node would be a virtual machine deployed by Pytest with SSH access provided to -the tests. A node could optionally be deployed outside Pytest. - -Pytest uses [fixtures](https://docs.pytest.org/en/stable/fixture.html), which -are the primary way of setting up test requirements. They replace less flexible -alternatives like setup/teardown functions. It is through fixtures that we -implement remote node setup/teardown. Our node fixture currently provides: - -* Automatic provisioning of an Azure VM given URN and SKU -* Remote shell access via SSH -* Data including hostname / IP address for local tools -* Cross-platform ping functionality with exponential back-off -* Allowing ICMP ping via Azure firewall rules -* Platform API reboot -* Uploading of local files to arbitrary remote destinations -* Downloading of remote file contents into local string variable -* Downloading boot diagnostics (serial console log) from platform -* Asynchronous remote command execution with promises +However, we also have internal requirements to report test results throughout +the test life cycle to a database to be consumed by other tools. In this sense, +LISAv3 (the composition of our published plugins, tests, and fixtures) is simply +a producer. Our repository’s `conftest.py` can implement the necessary logic +using Pytest’s ample [test running hooks][]. In particular, the hook +[pytest_runtest_makereport][] is called for each of the setup, call and teardown +phases of a test. As such it can used for precisely this purpose. -The prototype demonstrates how easy it is to quickly implement these features. -As we need more features, they can be readily added and shared among tests. - -Our abstraction leverages [Fabric](https://www.fabfile.org/) which is a popular -high-level Python library for executing shell commands on remote systems over -SSH. Underneath the covers it uses -[paramiko](https://docs.paramiko.org/en/stable/), the most popular low-level -Python SSH library. Fabric does the heavy lifting of safely connecting and -disconnecting from the node, executing the shell command (synchronously or -asynchronously), reporting the exit status, gathering the stdout and stderr, -providing stdin (or interactive auto-responses, similar to `expect`), uploading -and downloading files, and much more. In fact, these APIs are all available and -implemented for the local machine by the underlying -[Inovke](https://www.pyinvoke.org/) library, which is essentially a Python -`subprocess` wrapper with “a powerful and clean feature set.” - -Other test specific requirements, such as installing software and daemons, -downloading files from remote storage, or checking the state of our Bash test -scripts, would similarly be implemented by methods on the `Node` class or via -additional fixtures and thus shared among tests. - -For Azure, we use the [Azure CLI](https://aka.ms/azureclidocs) to deploy a -virtual machine. For Hyper-V (and other virtualization platforms), we would like -to use [libvirt](https://libvirt.org/python.html), and for embedded -environments we are evaluating -[labgrid](https://github.com/labgrid-project/labgrid). - -Tests do not need to explicitly call for a node to be provided, and we do not -need to write much code to setup this resource-provider logic. We simply define -a `Node` class and a Pytest fixture which returns one: - -```python -@pytest.fixture(scope="session") -def node(request: FixtureRequest) -> Iterator[Node]: - """Return the current node for any test which requests it.""" - with Node() as n: - yield n - -@pytest.mark.lisa(...) -def test_uptime(node: Node) -> None - """Automatically has access to the current node because of the argument.""" - # Runs `uname` via SSH and asserts it's Linux. - assert node.run("uname").stdout.strip() == "Linux" -``` - -When created, the `Node` instance either uses a cached node or deploys a new one -based on the given parameters (which can be provided at runtime). When the scope -of the fixture is exited (in this example, the test session), the `Node` -instance deletes its deployed resource unless requested not to by the user, -which is currently controlled by the `--keep-vms` flag. - -To provide the parameters to the node fixture, the prototype currently -implements a simple `@pytest.mark.deploy(...)` mark which takes `vm_image`, -`vm_size`, etc., and it’s applied to each function. This worked for the demo, -and proved the concept; however, we will want to provide a mechanism for -specifying lists of environments and their required resources to the tests at -runtime. This will likely be a YAML file that is parsed at initialization and -used to parameterize the node fixture itself, causing all the tests to be -executed for each environment. For more details, see the section “Where Does -Parameterization Happen?” - -See the Detailed Design Decisions below for what the `Node` class looks like. - -#### Interaction with Azure - -We do not use the [Azure Python APIs](https://aka.ms/azsdk/python/all) directly -because they are more complicated (and less documented) than the [Azure -CLI](https://aka.ms/azureclidocs). With Invoke (as discussed above), `az` -becomes incredibly easy to work with. The Azure CLI lead developer states that -they have [feature parity](https://stackoverflow.com/a/50005660/1028665) and -that the CLI is more straightforward to use. Considering our ease-of-maintenance -requirement, this seems the apt choice. If it later becomes necessary to use the -Python APIs directly, that is, of course, still allowed by our design. - -### How Are Tests Timed Out? +### How are tests timed out? The [pytest-timeout](https://pypi.org/project/pytest-timeout/) plugin provides integrated timeouts via `@pytest.mark.timeout()`, a configuration @@ -278,7 +580,7 @@ file option, environment variable, and CLI flag. The Fabric library provides timeouts in both the configuration and per-command usage. These are already used to satisfaction in the prototype. -### How Are Tests Organized? +### How are tests organized? That is, what does a folder of tests map to: a platform, feature, or owner? @@ -308,7 +610,7 @@ test dictates if the tests below it should be skipped. If it passes, it implies the tests underneath it would pass, and so skips them; but if it fails, the next test below it runs and so on until a passing layer is found. -### How Will We Port LISAv2 Tests? +### How will we port LISAv2 tests? Given the above, we still must decide if we want to put the engineering effort into porting _every_ LISAv2 test. However, the prototype started by porting the @@ -327,31 +629,10 @@ This work needs to be done regardless of the approach we take with our framework (leveraging Pytest or writing our own), and it is not inconsequential work. It needs to be thoroughly planned and executed, and is certainly a ways off. -### What Do Parallel Tests Mean? - -While our original list of goals stated that we want to run tests “in parallel” -we were not specific about what was meant, and the topic of parallelism and -concurrency is understandably complex. We certainly don’t mean running two tests -at once on the same node, as this would undoubtedly lead to flaky tests. - -Assuming that we care about a set of tests passing on a particular image and -size combination, but not necessarily on a particular deployed instance, then we -can run tests concurrently by deploying multiple “identical” nodes and splitting -the tests across them. The tests would still run in isolation on each node. This -sounds hard, but actually it’s practically free with Pytest if the node fixture -is session scoped and we use -[pytest-xdist](https://pypi.org/project/pytest-xdist/) as described below. - -It’s also unlikely that we want to write our tests using the Async I/O pattern, -because we do not want tests to accidentally conflict with each other. While -[pytest-asyncio](https://pypi.org/project/pytest-asyncio/) exists, our -concurrency model is probably as described above: split the tests among multiple -identical nodes. - -### How Are Tests and Functions Retried? +### How are tests and functions retried? -Testing remote instances is inherently flaky, so we take a two-pronged approach -to dealing with the flakiness. +Testing remote targets is inherently flaky, so we take a two-pronged approach to +dealing with the flakiness. The [pytest-rerunfailures](https://pypi.org/project/pytest-rerunfailures/) plugin will be used to easily mark a test itself as flaky. It has the nice @@ -392,191 +673,29 @@ We can additionally list a test twice when modifying the items collection, as implemented in the criteria proof-of-concept. However, given the above abilities, this may not be desired. -### Where Does Parameterization Happen? - -Do we parameterize -[tests](https://docs.pytest.org/en/stable/parametrize.html#parametrizemark) or -[fixtures](https://docs.pytest.org/en/stable/fixture.html#fixture-parametrize)? - -This all comes down to how we want to use LISA. If we want to put a single -system under test at a time, and run all possible tests against it, then it -would make sense to parameterize the node fixture across the set of images to -test. I believe this to likely be the case. - -A parameterized node fixture would be session-scoped. This would enable us to -take advantage of [pytest-xdist](https://pypi.org/project/pytest-xdist/) for -running the tests concurrently against multiple nodes, where each forked runner -has its own node. Note that the cache key for deployed nodes will need to -include an identifier to separate the parallel runs, but this is available. - -This approach would let us list a number of images and sizes (or a matrix -combination of them) and then run all requested tests against each of those. -However, it means that tests will need to be intelligent enough to [skip or -xfail](https://docs.pytest.org/en/stable/skipping.html) on systems where they do -not apply. This can be done in test code to start with. As commonalities are -realized they can be refactored into simple, reusable feature checks. - -Finally, while the base (and most common) case of tests which require one node -becomes trivially solved, we still have to deal with the edge cases of tests -which use two or three nodes. Determining the best course of action here -requires investigating how and when those tests are run, and if the node pair or -triple all use the same image and size. An easy solution would be to have a test -which requires a second or third node to simply deploy them through a -function-scoped fixture, and tear them down at the end. This may be costly in -terms of time if there are many of these tests and they run frequently, but for -long “performance” tests it would be an adequate option. Alternatively, we could -have a node pool that the session-scoped node fixture uses, where each node is -locked while in use. While this would take more engineering effort, it means we -could use the nodes for running tests concurrently, and “borrow” a runner when a -test needs another. - -Other ideas are welcome, but what we don’t want to do is change the environment -a user is expecting their tests to run in. I do not think that we should use a -“least common denominator” approach that collects feature requests and deploys -nodes which match those features, as the user will lose control over their -environment. We still want to enumerate features so tests can check if they’re -applicable, but the user’s environment request should be respected. - -Alternatively, parameterizing tests means that each test (or module, or class, -as the fixture could no longer be session-scoped) specifies in some way (whether -in code or read at runtime from a file) what image/size combinations it should -run against. This generally eliminates having to check if it should skip, but -means that running the test suite will put multiple systems under test at once, -the results of which may be difficult to interpret. While this is a viable -route, it means maintaining a comprehensive list of which environments each -tests use, and I think feature-checking is more scalable. - -This is an open question which we need to settle as the two methods can -technically be combined, but we will want to be careful if we do this. - -Regardless of approach, we will want to write and _review_ a simple YAML schema -for specifying the system-under-test targets. As described above, the prototype -currently reads this information from a mark, but if we move forward with the -suggestion above, the scope of the node fixture will change to session and it -will become parameterized. Those parameters would be set at runtime by reading a -given YAML file. - -### When Do We Export a Plugin? - -The current prototype is simply using Pytest. All the implementation is in the files -`conftest.py` and `node_plugin.py`, the former of which is Pytest’s default -“user plugin” file. We likely want to create a proper `pytest-lisa` package -which provides our marks, fixtures, command-line parameters, user modes, and -hook modifications for reading YAML files. - -This requires more research as doing so is obviously not necessary but is nice. - -## Detailed Design Decisions - -This section contains truly technical specifications of our current plans to -bring the prototype to production. - -### Planned `Node` Class Refactor - -#### Basic Shape - -`Node` should still subclass `fabric.Connection`. It should be a partially -abstract class with platform-specific subclasses (Azure, libvirt, an embedded -device, etc.). However, the initializer and context manager methods _should not_ -need to be reimplemented by a platform subclass. Most added methods like -`ping()` and `reboot()` should also be shared. This is where static type -checking will help. - -An `Environment` class will be a collection of nodes in a group, for tests which -require multiple nodes. It is important that `Node` is self-contained and does -not require an `Environment` instance because the base case of most tests is to -use a `Node`. - -#### Caching - -A `Node` should be able to be cached. If `--keep-vms` is given to Pytest, it -should not delete the deployed VM resource and should instead cache its data so -that a subsequent invocation can connect directly to it. A `Node` should also be -able to connect directly to a system deployed outside Pytest, reusing the cache -hydration logic. The `init()` and `__exit__()` methods will handle checking and -updating the cache so that this logic is shared. - -Note that cross-session [caching](https://docs.pytest.org/en/stable/cache.html) -is provided by Pytest, and very easy to work with. The existing prototype -already implements `--keep-vms`. - -#### Initializing - -The `init()` method does the following: - -* Takes an optional group ID (provided by Environment for instance so that it’s - easy to create/deploy multiple nodes into one group) to generate its name and - deduce its group. - -* Checks the cache for the node’s key. - -* On a cache miss, calls `deploy()` and saves the returned host to the field - inherited from `Connection` and the rest of the platform-specific information - to a `data` dictionary field. Caches the data dictionary for the node’s key. - -* On a cache hit, saves the cached host and data to the instance. - -* Calls `super()` to setup `Connection` with our default Fabric configuration. - -#### Deploy and Delete - -* The `deploy()` and `delete()` methods are abstract and implemented by - platform-specific node classes to actually deploy the VM. For Azure, note that - `deploy()` will check if the resource group exists, and if not, creates it. - For `delete()` it will check if it is the last VM in the group, and if so - deletes the group too. Again this is to keep `Environment` from being a - requirement. - -* The group ID is `pytest-{uuid4()}` (maybe with `pytest` being replaced by a - user- or run-specific short identifier). The ID should be returned by a static - method so that when an `Environment` creates a collection of nodes, it can - simply use the static method to generate a shared group ID. - -* The context manager’s `__exit__()` method calls `super()` to disconnect and - potentially `delete()` the VM. If it’s to be deleted, the key/value pair is - also removed from the cache. - -* Because of how Python’s context managers work, we may not need to reimplement - `__enter__()` but will want to check its inherited implementation. - -#### Common Tasks - -Common tasks for systems under tests like rebooting and pinging should be -implemented on the `Node` class. - -* Methods inherited from `Connection` include `run()`, `sudo()` and `local()` - which are used to easily run arbitrary commands, and `get()` and `put()` to - download and upload arbitrary files. - -* The `cat()` method (already implemented in the prototype) wraps `get()` and - returns the file as data in a string. This makes test code like this possible: - - ```python - assert node.cat("state.txt") == "TestCompleted" - ``` - -* Reboot should first try to use `self.sudo("reboot", timeout=5)` (with a short - timeout to avoid a hung SSH session). It should retry with an exponential - back-off to see if the machine has rebooted by checking either `uptime` or the - existence of a file created before the reboot. This is to avoid having to - `sleep()` and just guess the amount of time it takes to reboot. +## What Else? -* Restart should “power cycle” the machine using the platform’s API, and thus is - in abstract method. It should optionally be able to redeploy the node too, - which can be used by tests which require a completely fresh node. +There’s still a lot more to think about and design. A non-exhaustive list of +future topics (some touched on above): -* Note that the `local()` method is already overridden to patch Fabric so as to - ignore the provided SSH environment. This demonstrates that we can easily - provide necessary changes to users while still leveraging the library. For - instance, we may want an alternative to `run()` which, instead of taking a - string, takes a list of arguments and quotes them correctly so as to deal with - difficult shell quoting edge cases. +* Tests inventory (generating statistics from metadata) +* ARM template support (with Azure CLI) +* Servicing Azure CLI (how stable is their API?) +* libvirt driver support (gives us Hyper-V and more) +* Duration reporting (built-in) +* Self-documentation (via Pydoc) +* Environment class design +* Feature requests (NICs in particular) +* Selection and targets YAML schema +* Secret management +* External results reporting (database and emails) +* Embedded systems / bare metal support +* Managing Python `logging` records +* Managing shell command stdout/stderr -* One new method we’ve already identified is `copy_scripts()` which will copy a - list of scripts to the node and mark them executable. It could even be a - context manager which deletes the scripts when exited. +## What alternatives were tried? -## Alternatives Considered +These are notes from things tried that did not work out, and why. ### Writing Another Framework @@ -592,17 +711,16 @@ already caused this mess in the first place. I think the work of prototyping said new framework was valuable, as it provided insight into the eventual technical design of LISAv3. -### Using Remote Capabilities of pytest-xdist +### Using Remote Capabilities of `pytest-xdist` -With the [pytest-xdist plugin](https://github.com/pytest-dev/pytest-xdist) there -already exists support for running a folder of tests on an arbitrary remote host -via SSH. +With the [pytest-xdist][] plugin there already exists support for running a +folder of tests on an arbitrary remote host via SSH. The LISA tests could be written as Python code suitable for running on the target test system, which means direct access to the system in the test code itself (subprocesses are still available, without having to use SSH within the test, but would become far less necessary), something that is not possible with -any current prototype. Where the pytest-xdist plugin copies the package of code +any current prototype. Where the `pytest-xdist` plugin copies the package of code to the target node and runs it, the pytest-lisa plugin could instantiate that node (boot the necessary image on a remote machine or launch a new Hyper-V or Azure VM, etc.) for the tests. @@ -611,8 +729,8 @@ However, this use of pytest-dist requires full Python support on the target machines, and drastically changes how developers write tests. Furthermore, it would not support running local commands against the remote node (like ping) or running the test across a reboot of the node. Thus we do not want to use this -functionality of pytest-xdist. That said, pytest-xdist will still be useful for -running tests concurrently, as described above. +functionality of `pytest-xdist`. That said, `pytest-xdist` will still be useful +for running tests concurrently, as described above. ### Using Paramiko Instead of Fabric @@ -668,37 +786,13 @@ However, the data returned by Paramiko is in bytes, which in Python 3 are not equivalent to strings, hence the existing implementation which uses `BytesIO` and decodes the bytes to a string. -### Writing a Class of Individual Test Methods - -An option I explored to make an “executive summary” of the smoke test was to use -a class where each functionality was tested as individual function (meaning they -could fail independently without failing the whole smoke test), accompanied by a -class-scoped node fixture. This had its advantages, however, it was difficult to -parameterize and also overly verbose. We should instead keep each test as Pytest -intends: as a function. This allows the fixtures to be written in a simpler -manner (not rely on caching between functions) and allows -[parameterization](https://docs.pytest.org/en/stable/parametrize.html) using the -built-in decorator `@pytest.mark.parametrize`. - -However, this decision may be reconsidered if we session-scope and parameterize -the `Node` fixture, in which case these issues are resolved. - -## What Else? - -There’s still a lot more to think about and design. A non-exhaustive list of -future topics (some touched on above): - -* Tests inventory (generating statistics from metadata) -* ARM template support (with Azure CLI) -* Servicing Azure CLI (how stable is their API?) -* libvirt driver support (gives us Hyper-V and more) -* Duration reporting (built-in) -* Self-documentation (via Pydoc) -* Environment class design -* Feature requests (NICs in particular) -* Selection and targets YAML schema -* Secret management -* External results reporting (database and emails) -* Embedded systems / bare metal support -* Managing Python `logging` records -* Managing shell command stdout/stderr +[pytest-xdist]: https://github.com/pytest-dev/pytest-xdist +[collection hooks]: https://docs.pytest.org/en/latest/reference.html#collection-hooks +[parameterization]: https://docs.pytest.org/en/stable/parametrize.html +[pytest_addoption]: https://docs.pytest.org/en/latest/reference.html#pytest.hookspec.pytest_addoption +[pytest_collection_modifyitems]: https://docs.pytest.org/en/latest/reference.html#pytest.hookspec.pytest_collection_modifyitems +[pytest_configure]: https://docs.pytest.org/en/latest/reference.html#pytest.hookspec.pytest_configure +[pytest_generate_tests]: https://docs.pytest.org/en/latest/reference.html#pytest.hookspec.pytest_generate_tests +[pytest_runtest_makereport]: https://docs.pytest.org/en/latest/reference.html#pytest.hookspec.pytest_runtest_makereport +[schema]: https://pypi.org/project/schema/ +[test running hooks]: https://docs.pytest.org/en/latest/reference.html#test-running-runtest-hooks From 0120baef6391aa445e04ffcceecd278f72645e15 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Tue, 17 Nov 2020 16:10:59 -0800 Subject: [PATCH 083/194] Write version 0.3 of design document --- DESIGN.md | 273 +++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 228 insertions(+), 45 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 50ebb9a4c2..31647e5dc2 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -7,7 +7,7 @@ evaluating the feasibility of leveraging Please see [PR #1065](https://github.com/LIS/LISAv2/pull/1065) for a working, proof-of-concept prototype. -Authored by Andrew Schwartzmeyer (he/him), version 0.2.0. +Authored by Andrew Schwartzmeyer (he/him), version 0.3.0. ## Why Pytest? @@ -40,8 +40,8 @@ needs very well: * Automatic test discovery, no boiler-plate test code * Useful information when a test fails (assertions are introspected) -* Test and fixture parameterization -* Modular setup/teardown via fixtures +* Test and fixture [parameterization][] +* Modular setup/teardown via [fixtures][] * Incredibly customizable (as detailed above) So all the logic for describing, discovering, running, skipping and reporting @@ -89,10 +89,18 @@ and result notifiers. It will similarly support both CLI and YAML file input. We should strive to keep these plugins from depending on each other in order to keep their scope well-defined. In the “LISA” repository of tests we will depend -on the two plugins and maintain additional fixtures for our tests’ unique +on the two plugins and maintain additional [fixtures][] for our tests’ unique requirements. Similarly, we and others may have private test repositories which build upon the above by defining new platform support and internal service -integrations. +integrations. The built-in plugin discovery of Pytest (via `conftest.py` files) +enables us to satisfy one of our requirements to “support plugins to orchestrate +the test environment.” + +Finally, a third smaller utility plugin, `pytest-schema` may be written in order +to share the common functionality of registering component schemata (e.g. +platform and target parameters from `pytest-target` and selection criteria from +`pytest-lisa`). This is somewhat of an implementation detail, but would be a +third and lower-level library we can publish. ## pytest-target @@ -104,11 +112,11 @@ target would be a virtual machine deployed by `pytest-target` with SSH access provided to the requesting test. A target could optionally be pre-deployed and simply connected. Some tests may request multiple targets as well. -Pytest uses [fixtures](https://docs.pytest.org/en/stable/fixture.html), which -are the primary way of setting up test requirements. They replace less flexible -alternatives like setup/teardown functions. It is through fixtures that we -implement remote target setup/teardown. Our `target` fixture returns a `Target` -instance, which currently provides: +Pytest uses [fixtures][], which are the primary way of setting up test +requirements. They replace less flexible alternatives like setup/teardown +functions. It is through fixtures that we implement remote target +setup/teardown. Our `target` fixture returns a `Target` instance, which +currently provides: * Remote shell access via SSH * Data including hostname / IP address @@ -149,8 +157,8 @@ additional fixtures and thus shared among tests. For Azure, we currently use the [Azure CLI](https://aka.ms/azureclidocs) to deploy a virtual machine. For Hyper-V (and other virtualization platforms), we -would like to use [libvirt](https://libvirt.org/python.html), and for embedded -environments we are evaluating +would like to use [libvirt](https://libvirt.org/python.html), and for embedded / +bare metal environments we are evaluating [labgrid](https://github.com/labgrid-project/labgrid). If possible, we do not want to use the [Azure Python @@ -160,8 +168,17 @@ CLI](https://aka.ms/azureclidocs). With Invoke (as discussed above), `az` becomes incredibly easy to work with. The Azure CLI lead developer states that they have [feature parity](https://stackoverflow.com/a/50005660/1028665) and that the CLI is more straightforward to use. Considering our ease-of-maintenance -requirement, this seems the apt choice. If it later becomes necessary to use the -Python APIs directly, that is, of course, still doable. +requirement, this seems the apt choice, especially since the Azure CLI supports +deploying resources with [ARM +templates](https://docs.microsoft.com/en-us/azure/azure-resource-manager/templates/deploy-cli). +If it later becomes necessary to use the Python APIs directly, that is, of +course, still doable (and we can reuse existing code doing it). + +On the topic of “servicing” the Azure CLI, its developers state that “at command +level, packages only upgrading the PATCH version guarantee backward +compatibility.” The tool is also intended to be used in scripts, so servicing +would amount to documenting the tested version and having the Azure class check +that it’s compatible before using it (or warning and then trying its best). ### What’s the `Target` class? @@ -220,6 +237,51 @@ platforms and their parameter schemata are automatically gathered from users’ own `conftest.py` files and other plugins. This enables the `target` fixture to dynamically instantiate a target from the gathered requirements and parameters. +For example, the `Azure(Target)` class defines its required parameters using the +[schema][] library like this: + +```python +from schema import Optional, Schema +from target import Target + +class Azure(Target): + ... + schema: Schema = Schema( + { + # TODO: Maybe validate as URN or path etc. + "image": str, + Optional("sku", default="Standard_DS1_v2"): str, + Optional("location", default="eastus2"): str, + Optional("networking", default=""): str, + } + ) +``` + +In the YAML playbook, a set of Azure targets can then be defined like this: + +```yaml +targets: + - name: Debian + platform: Azure + parameters: + image: credativ:Debian:9:9.0.201706190 + location: westus2 + + - name: Ubuntu + platform: Azure + parameters: + image: UbuntuLTS + sku: Standard_DS3_v2 +``` + +These targets are then used to parameterize the `target` fixture in the +[pytest_generate_tests][] hook (see below for more details). + +This demonstrated how we can have platforms define their own schema and register +that schema automatically. A pending update to this is to have two schemata per +`Target` subclass: target-level and platform-level (the former is what’s +demonstrated above, the latter would be common settings, such as subscription). + #### How are requirements examined? The `features` attribute is currently a set of strings and (combined with the @@ -228,7 +290,8 @@ target instance (representing a deployed machine) met a test’s requirements. I should be updated with a `Requirements` class that represents all physical attributes of the target, and a `requires` Pytest mark should be added which takes instances of this class. Two `Requirements` should be comparable to -determine if one set meets (or exceeds) the other set. +determine if one set meets (or exceeds) the other set. Existing code that does +this can be reused here. #### How do we share common tasks? @@ -257,7 +320,7 @@ and thus is in abstract method. Other tools and shared logic should be implemented as necessary. A major area of concern is the automatic and package-manager agnostic installation of necessary -tools, much of which has been implemented previously and can be integrated. +tools, much of which has been implemented previously and can be reused. ### How are targets requested and managed? @@ -323,6 +386,44 @@ allows us to run a collection of tests against multiple targets with ease. These targets are defined in a YAML file and validated against the parameters collected from the previously described platform subclasses. +The entire implementation looks like so: + +```python +TARGETS: List[Dict[str, Any]] = [] +TARGET_IDS: List[str] = [] + +def pytest_configure(config: Config) -> None: + book = get_playbook(config.getoption("--playbook")) + for t in book.get("targets", []): + TARGETS.append(t) + TARGET_IDS.append(t["name"]) + +def pytest_generate_tests(metafunc: Metafunc) -> None: + if "target" in metafunc.fixturenames: + assert TARGETS, "No targets specified!" + metafunc.parametrize("target", TARGETS, True, TARGET_IDS) +``` + +The function `get_playbook()` only imports the [PyYAML][] library, opens the +playbook file `f` within a context manager, and returns +`playbook.schema.validate(yaml.load(f))`. This is leveraging Pytest’s existing +parameterization technology to achieve one of our “test entrance” goals of +requesting environments with a YAML playbook, and one of our “test parameter +validation” goals of validating platforms before executing tests so that we can +fail fast if a target has insufficient information to be setup. Parsing the same +parameters from a CLI can also be implemented. + +Finally, once the `target` fixture has returned a working and sanity-checked +environment to the requesting test, the test is capable of examining any and all +attributes of the `Target` and quickly marking itself as skipped, expected to +fail, or failed before executing the body of the test. Our static type checking +enables developers to ensure that the platform they requested supports all +methods and fields they use by annotating the test’s `target` parameter with the +expected platform type (or types). Ensuring the effectiveness of this type +checking will require us to carefully update our platform implementations, and +not rely on arbitrary objects of data. (For example, add an `internal_address` +field to `Azure`, don’t just look up `data["internal_address"]`.) + ### How are tests executed in parallel? While our original list of goals stated that we want to run tests “in parallel” @@ -344,6 +445,11 @@ particular executors and using particular targets. While there are many paths open to us, this plugin actually provides a hook, `pytest_xdist_make_scheduler` that exists specifically to “implement custom tests distribution logic.” +Figuring out the requirements of our test scheduler and designing the best +algorithm will require further discussion and design review. For the purposes of +moving forward, we are not blocked, as the eventual implementation can be +dropped in-place with minimal effort. + ## pytest-lisa ### What are the user modes? @@ -352,7 +458,10 @@ Because Pytest is incredibly customizable, we want to provide a few sets of reasonable default configurations for some common scenarios. We will add a flag like `--lisa-mode=[dev,debug,ci,demo]` to change the default options and output of Pytest. Doing so is readily supported by Pytest via the [pytest_addoption][] -and [pytest_configure][] hooks. We call these the provided “user modes.” +and [pytest_configure][] hooks. We call these the provided “user modes.” Note +that by “output” we mean not just logging (because that implies the Python +`logger` module, which Pytest allows full control over) but also commands’ +stdout and stderr as well as Pytest-provided information. * The dev(eloper) mode is intended for use by test developers while writing a new test. It is verbose, caches the deployed VMs between runs, and generates a @@ -400,11 +509,15 @@ def test_lis_driver_version(target: Azure) -> None: This is a functional example, which takes zero implementation. With this simple decorator, all test [collection hooks][] can introspect the metadata, enforce required parameters and set defaults, select tests based on arbitrary criteria, -and list test coverage statistics. +and list test coverage statistics (test inventory). Designing and implementing +the test inventory algorithm is still under development, but it’s tractable. Note that Pytest leverages Python’s docstrings for built-in documentation (and can even run tests discovered in such strings, like doctest). Hence we do not -have a separate field for the test’s documentation. +have a separate field for the test’s documentation. As long as we continue to +follow the practice of using docstrings for our modules, classes, and functions, +we can automatically use [pydoc](https://docs.python.org/3/library/pydoc.html) +to generate full documentation for each plugin and test. Being just Python code, this decorator need not be `@pytest.mark.lisa(...)` but can trivially be provided as simply `@LISA(...)`. In fact, we provide this in @@ -441,7 +554,8 @@ def validate(mark: Mark) -> None: In the future we could change `LISA` to be a function with these keyword arguments so that IDE auto-completion is enabled. However, this is not mandatory -to move forward, and parameter validation is enabled succinctly with the above. +to move forward, and parameter validation is enabled succinctly with the above, +which satisfies one of our “test parameter validation” requirements. This mark also does need to be repeated for each test, as marks can be scoped to a module, and so one line could describe defaults for every test in a file, with @@ -452,7 +566,10 @@ that is used to prove the concept deploying (or reusing) a target based on the test’s required and the target’s available sets of features. However, as we move forward we should define a separate `requires` mark that takes well-defined classes describing the minimal required resources for a test. This will be part -of the refactor into the two Pytest plugins mentioned above. +of the refactor into the two Pytest plugins mentioned above. Coupled with the +test’s requested `target` fixture being parameterized (see discussion in +`pytest-target`) this demonstrates at least one way we can satisfy our “test run +planner/scheduler” requirement. Furthermore, we have a prototype [generator](https://github.com/LIS/LISAv2/tree/pytest/generator) which parses @@ -542,6 +659,10 @@ def pytest_collection_modifyitems( items[:] = [i for i in included if i not in excluded] ``` +Together, the CLI support and YAML playbook satisfy one of our “test entrance” +requirements. We can also generate our own binary called `lisa` which simply +delegates to Pytest, if we really want to do so. + Because this is simply a Python list, we can also sort the tests according to our needs, such as by priority. If the `python-targets` plugin has already sorted by requirements, that’s just fine, Python’s `sorted()` built-in is @@ -564,13 +685,34 @@ community plugin [pytest-azurepipelines](https://pypi.org/project/pytest-azurepipelines/) which enhances the standard JUnit report for ADO. +One of our requirements is to support the lookup of previous tests’ execution +metrics, such as recorded performance metrics and duration, so that performance +tests can check regressions. This is the perfect example of carrying a small +fixture which provides access to our internal database and is dynamically added +to our tests when run internally, and the tests can lookup and record whatever +they need through the fixture. + However, we also have internal requirements to report test results throughout -the test life cycle to a database to be consumed by other tools. In this sense, -LISAv3 (the composition of our published plugins, tests, and fixtures) is simply -a producer. Our repository’s `conftest.py` can implement the necessary logic -using Pytest’s ample [test running hooks][]. In particular, the hook -[pytest_runtest_makereport][] is called for each of the setup, call and teardown -phases of a test. As such it can used for precisely this purpose. +the test life cycle to a database (the “result manager” and “progress tracker”) +to be consumed by other tools. In this sense, LISAv3 (the composition of our +published plugins, tests, and fixtures) is simply a producer, and the consumers +can parse the test results, send emails, archive the collected logs, update a +GUI display of test progress, etc. Our repository’s `conftest.py` can implement +the necessary logic using Pytest’s ample [test running hooks][]. In particular, +the hook [pytest_runtest_makereport][] is called for each of the setup, call and +teardown phases of a test. As such it can used for precisely this purpose. + +### How is setup, run, and cleanup handled? + +Pytest strives to require minimal boiler-plate code. Thus the classic +“xunit-style” of defining a class with setup and teardown functions in addition +to test functions is not recommended (nor necessary). Generally Pytest expects +[fixtures][] to be used for dependency injection (which is what setup/teardown +functions usually do). For users that really want the classic style, it is +nonetheless fully +[supported](https://docs.pytest.org/en/stable/xunit_setup.html) and documented +(and can be applied at the module, class, and method scopes). Thus our “test +runner” requirement is satisfied. ### How are tests timed out? @@ -578,7 +720,9 @@ The [pytest-timeout](https://pypi.org/project/pytest-timeout/) plugin provides integrated timeouts via `@pytest.mark.timeout()`, a configuration file option, environment variable, and CLI flag. The Fabric library provides timeouts in both the configuration and per-command usage. These are already used -to satisfaction in the prototype. +to satisfaction in the prototype. Additionally, Pytest has built-in support for +measuring the duration of each fixture’s setup and teardown and each test (it’s +simply the `--durations` and `--durations-min` flags). ### How are tests organized? @@ -645,10 +789,10 @@ def test_something_flaky(...): ... ``` -> Note that there is an open -> [bug](https://github.com/pytest-dev/pytest-rerunfailures/issues/51) in this -> plugin which can cause issues with fixtures using scopes other than “function” -> but it can be worked around. +Note that there is an open +[bug](https://github.com/pytest-dev/pytest-rerunfailures/issues/51) in this +plugin which can cause issues with fixtures using scopes other than “function” +but it can be worked around. The [Tenacity](https://tenacity.readthedocs.io/en/latest/) library should be used to retry flaky functions that are not tests, such as downloading boot @@ -673,25 +817,62 @@ We can additionally list a test twice when modifying the items collection, as implemented in the criteria proof-of-concept. However, given the above abilities, this may not be desired. +## What does the “flow” of Pytest look-like? + +This is best described in Pythonic pseudo-code, where the context manager +encapsulates each scope and the for loop encapsulates processing: + +```python +pool_fixture: a session-scoped context manager +target_fixture: a function-scoped context manager +items: a collection of tests +targets: a collection of targets +criteria: a collection of test selection criteria + +def pytest_addoption(parser): + """Add CLI options etc.""" + parser.addoption("--playbook", type=Path) + +pytest_addoption(parser) # Pytest fills in parser. + +def pytest_configure(config): + """Setup the run's configuration.""" + targets = playbook.get_targets() + criteria = playbook.get_criteria() + +pytest_configure(config) # Pytest fills in config. + +# pytest_generate_tests(metafunc) does this: +for test_metafunc in metafuncs: + for target in targets: + # items is tests * targets in size + items.append(test_metafunc[target]) + +# pytest_collection_modifyitems(session, config, items) does this: +for test in items: + validate(test) + include_or_exclude(test, criteria) + +# finally, each executor/session does this: +session_items = items.split() # based on scheduler algorithm +with pool_fixture as pool: + # the fixture has setup a pool to track the deployed targets + for test_function in session_items: + with target_fixture as target: + # the fixture has found or deployed an appropriate target + test_function(target) +``` + ## What Else? There’s still a lot more to think about and design. A non-exhaustive list of future topics (some touched on above): * Tests inventory (generating statistics from metadata) -* ARM template support (with Azure CLI) -* Servicing Azure CLI (how stable is their API?) -* libvirt driver support (gives us Hyper-V and more) -* Duration reporting (built-in) -* Self-documentation (via Pydoc) -* Environment class design -* Feature requests (NICs in particular) -* Selection and targets YAML schema +* Environment / multiple targets class design +* Feature/requirement requests (NICs in particular) +* Custom test scheduler algorithm * Secret management -* External results reporting (database and emails) -* Embedded systems / bare metal support -* Managing Python `logging` records -* Managing shell command stdout/stderr ## What alternatives were tried? @@ -786,9 +967,11 @@ However, the data returned by Paramiko is in bytes, which in Python 3 are not equivalent to strings, hence the existing implementation which uses `BytesIO` and decodes the bytes to a string. -[pytest-xdist]: https://github.com/pytest-dev/pytest-xdist +[PyYAML]: https://pyyaml.org/wiki/PyYAMLDocumentation [collection hooks]: https://docs.pytest.org/en/latest/reference.html#collection-hooks +[fixtures]: https://docs.pytest.org/en/stable/fixture.html [parameterization]: https://docs.pytest.org/en/stable/parametrize.html +[pytest-xdist]: https://github.com/pytest-dev/pytest-xdist [pytest_addoption]: https://docs.pytest.org/en/latest/reference.html#pytest.hookspec.pytest_addoption [pytest_collection_modifyitems]: https://docs.pytest.org/en/latest/reference.html#pytest.hookspec.pytest_collection_modifyitems [pytest_configure]: https://docs.pytest.org/en/latest/reference.html#pytest.hookspec.pytest_configure From ed42f7af04abce5d91ddc3cb5f10b223ad3d9b52 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Wed, 18 Nov 2020 13:44:48 -0800 Subject: [PATCH 084/194] Add initial Pytest plugin trees --- pytest-lisa/pyproject.toml | 17 +++++++++++++++++ pytest-playbook/pyproject.toml | 19 +++++++++++++++++++ pytest-target/pyproject.toml | 20 ++++++++++++++++++++ 3 files changed, 56 insertions(+) create mode 100644 pytest-lisa/pyproject.toml create mode 100644 pytest-playbook/pyproject.toml create mode 100644 pytest-target/pyproject.toml diff --git a/pytest-lisa/pyproject.toml b/pytest-lisa/pyproject.toml new file mode 100644 index 0000000000..cca0f9beaa --- /dev/null +++ b/pytest-lisa/pyproject.toml @@ -0,0 +1,17 @@ +[tool.poetry] +name = "pytest-lisa" +version = "0.1.0" +description = "Pytest plugin for Linux Integration Services Automation (LISA)." +authors = ["Andrew Schwartzmeyer "] +license = "MIT" +classifiers = ["Framework :: Pytest"] + +[tool.poetry.dependencies] +python = "^3.7" +pytest = "^6.1.2" + +[tool.poetry.dev-dependencies] + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" diff --git a/pytest-playbook/pyproject.toml b/pytest-playbook/pyproject.toml new file mode 100644 index 0000000000..5ecf851772 --- /dev/null +++ b/pytest-playbook/pyproject.toml @@ -0,0 +1,19 @@ +[tool.poetry] +name = "pytest-playbook" +version = "0.1.0" +description = "Pytest plugin for reading playbooks." +authors = ["Andrew Schwartzmeyer "] +license = "MIT" +classifiers = ["Framework :: Pytest"] + +[tool.poetry.dependencies] +python = "^3.7" +pytest = "^6.1.2" +schema = "^0.7.3" +PyYAML = {version = "^5.3.1", optional = true} + +[tool.poetry.dev-dependencies] + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" diff --git a/pytest-target/pyproject.toml b/pytest-target/pyproject.toml new file mode 100644 index 0000000000..a5a4beb56b --- /dev/null +++ b/pytest-target/pyproject.toml @@ -0,0 +1,20 @@ +[tool.poetry] +name = "pytest-target" +version = "0.1.0" +description = "Pytest plugin for remote target orchestration." +authors = ["Andrew Schwartzmeyer "] +license = "MIT" +classifiers = ["Framework :: Pytest"] + +[tool.poetry.dependencies] +python = "^3.7" +pytest = "^6.1.2" +fabric = "^2.5.0" +invoke = "^1.4.1" +tenacity = "^6.2.0" + +[tool.poetry.dev-dependencies] + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" From 55aa90cb646ad68e2999cd8454514eb912358d95 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Wed, 18 Nov 2020 16:37:44 -0800 Subject: [PATCH 085/194] Update Python packages --- poetry.lock | 176 ++++++++++++++++++++++++++++------------------------ 1 file changed, 95 insertions(+), 81 deletions(-) diff --git a/poetry.lock b/poetry.lock index ea01a1d5f2..792bb074f4 100644 --- a/poetry.lock +++ b/poetry.lock @@ -24,15 +24,15 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "attrs" -version = "20.2.0" +version = "20.3.0" description = "Classes Without Boilerplate" category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [package.extras] -dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "sphinx", "sphinx-rtd-theme", "pre-commit"] -docs = ["sphinx", "sphinx-rtd-theme", "zope.interface"] +dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "furo", "sphinx", "pre-commit"] +docs = ["furo", "sphinx", "zope.interface"] tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface"] tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six"] @@ -49,7 +49,7 @@ cffi = ">=1.1" six = ">=1.4.1" [package.extras] -tests = ["pytest (>=3.2.1,<3.3.0 || >3.3.0)"] +tests = ["pytest (>=3.2.1,!=3.3.0)"] typecheck = ["mypy"] [[package]] @@ -95,7 +95,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [[package]] name = "colorama" -version = "0.4.3" +version = "0.4.4" description = "Cross-platform colored terminal text." category = "main" optional = false @@ -111,7 +111,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "cryptography" -version = "3.1.1" +version = "3.2.1" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." category = "main" optional = false @@ -122,11 +122,11 @@ cffi = ">=1.8,<1.11.3 || >1.11.3" six = ">=1.4.1" [package.extras] -docs = ["sphinx (>=1.6.5,<1.8.0 || >1.8.0,<3.1.0 || >3.1.0,<3.1.1 || >3.1.1)", "sphinx-rtd-theme"] +docs = ["sphinx (>=1.6.5,!=1.8.0,!=3.1.0,!=3.1.1)", "sphinx-rtd-theme"] docstest = ["doc8", "pyenchant (>=1.6.11)", "twine (>=1.12.0)", "sphinxcontrib-spelling (>=4.0.1)"] pep8test = ["black", "flake8", "flake8-import-order", "pep8-naming"] ssh = ["bcrypt (>=3.1.5)"] -test = ["pytest (>=3.6.0,<3.9.0 || >3.9.0,<3.9.1 || >3.9.1,<3.9.2 || >3.9.2)", "pretend", "iso8601", "pytz", "hypothesis (>=1.11.4,<3.79.2 || >3.79.2)"] +test = ["pytest (>=3.6.0,!=3.9.0,!=3.9.1,!=3.9.2)", "pretend", "iso8601", "pytz", "hypothesis (>=1.11.4,!=3.79.2)"] [[package]] name = "execnet" @@ -221,7 +221,7 @@ test = ["pytest (>=4.0.2,<6)", "toml"] [[package]] name = "iniconfig" -version = "1.0.1" +version = "1.1.1" description = "iniconfig: brain-dead simple config-ini parsing" category = "main" optional = false @@ -237,7 +237,7 @@ python-versions = "*" [[package]] name = "isort" -version = "5.6.1" +version = "5.6.4" description = "A Python utility / library to sort Python imports." category = "dev" optional = false @@ -260,7 +260,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" parso = ">=0.7.0,<0.8.0" [package.extras] -qa = ["flake8 (3.7.9)"] +qa = ["flake8 (==3.7.9)"] testing = ["Django (<3.1)", "colorama", "docopt", "pytest (>=3.9.0,<5.0.0)"] [[package]] @@ -339,7 +339,7 @@ testing = ["docopt", "pytest (>=3.0.7)"] [[package]] name = "pathspec" -version = "0.8.0" +version = "0.8.1" description = "Utility library for gitignore style pattern matching of file paths." category = "dev" optional = false @@ -445,7 +445,7 @@ six = "*" [package.extras] docs = ["sphinx (>=1.6.5)", "sphinx-rtd-theme"] -tests = ["pytest (>=3.2.1,<3.3.0 || >3.3.0)", "hypothesis (>=3.27.0)"] +tests = ["pytest (>=3.2.1,!=3.3.0)", "hypothesis (>=3.27.0)"] [[package]] name = "pyparsing" @@ -457,7 +457,7 @@ python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" [[package]] name = "pytest" -version = "6.1.1" +version = "6.1.2" description = "pytest: simple powerful testing with Python" category = "main" optional = false @@ -474,7 +474,7 @@ py = ">=1.8.2" toml = "*" [package.extras] -checkqa_mypy = ["mypy (0.780)"] +checkqa_mypy = ["mypy (==0.780)"] testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] [[package]] @@ -626,7 +626,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [[package]] name = "regex" -version = "2020.9.27" +version = "2020.11.13" description = "Alternative regular expression module, to replace re." category = "dev" optional = false @@ -691,11 +691,11 @@ test = ["pytest (>=3.6)", "pytest-cov", "pytest-django", "zope.component", "sybi [[package]] name = "toml" -version = "0.10.1" +version = "0.10.2" description = "Python Library for Tom's Obvious, Minimal Language" category = "main" optional = false -python-versions = "*" +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" [[package]] name = "typed-ast" @@ -740,8 +740,8 @@ atomicwrites = [ {file = "atomicwrites-1.4.0.tar.gz", hash = "sha256:ae70396ad1a434f9c7046fd2dd196fc04b12f9e91ffb859164193be8b6168a7a"}, ] attrs = [ - {file = "attrs-20.2.0-py2.py3-none-any.whl", hash = "sha256:fce7fc47dfc976152e82d53ff92fa0407700c21acd20886a13777a0d20e655dc"}, - {file = "attrs-20.2.0.tar.gz", hash = "sha256:26b54ddbbb9ee1d34d5d3668dd37d6cf74990ab23c828c2888dccdceee395594"}, + {file = "attrs-20.3.0-py2.py3-none-any.whl", hash = "sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6"}, + {file = "attrs-20.3.0.tar.gz", hash = "sha256:832aa3cde19744e49938b91fea06d69ecb9e649c93ba974535d08ad92164f700"}, ] bcrypt = [ {file = "bcrypt-3.2.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:c95d4cbebffafcdd28bd28bb4e25b31c50f6da605c81ffd9ad8a3d1b2ab7b1b6"}, @@ -798,36 +798,36 @@ click = [ {file = "click-7.1.2.tar.gz", hash = "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a"}, ] colorama = [ - {file = "colorama-0.4.3-py2.py3-none-any.whl", hash = "sha256:7d73d2a99753107a36ac6b455ee49046802e59d9d076ef8e47b61499fa29afff"}, - {file = "colorama-0.4.3.tar.gz", hash = "sha256:e96da0d330793e2cb9485e9ddfd918d456036c7149416295932478192f4436a1"}, + {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"}, + {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"}, ] contextlib2 = [ {file = "contextlib2-0.6.0.post1-py2.py3-none-any.whl", hash = "sha256:3355078a159fbb44ee60ea80abd0d87b80b78c248643b49aa6d94673b413609b"}, {file = "contextlib2-0.6.0.post1.tar.gz", hash = "sha256:01f490098c18b19d2bd5bb5dc445b2054d2fa97f09a4280ba2c5f3c394c8162e"}, ] cryptography = [ - {file = "cryptography-3.1.1-cp27-cp27m-macosx_10_10_x86_64.whl", hash = "sha256:65beb15e7f9c16e15934569d29fb4def74ea1469d8781f6b3507ab896d6d8719"}, - {file = "cryptography-3.1.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:983c0c3de4cb9fcba68fd3f45ed846eb86a2a8b8d8bc5bb18364c4d00b3c61fe"}, - {file = "cryptography-3.1.1-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:e97a3b627e3cb63c415a16245d6cef2139cca18bb1183d1b9375a1c14e83f3b3"}, - {file = "cryptography-3.1.1-cp27-cp27m-win32.whl", hash = "sha256:cb179acdd4ae1e4a5a160d80b87841b3d0e0be84af46c7bb2cd7ece57a39c4ba"}, - {file = "cryptography-3.1.1-cp27-cp27m-win_amd64.whl", hash = "sha256:b372026ebf32fe2523159f27d9f0e9f485092e43b00a5adacf732192a70ba118"}, - {file = "cryptography-3.1.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:680da076cad81cdf5ffcac50c477b6790be81768d30f9da9e01960c4b18a66db"}, - {file = "cryptography-3.1.1-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:5d52c72449bb02dd45a773a203196e6d4fae34e158769c896012401f33064396"}, - {file = "cryptography-3.1.1-cp35-abi3-macosx_10_10_x86_64.whl", hash = "sha256:f0e099fc4cc697450c3dd4031791559692dd941a95254cb9aeded66a7aa8b9bc"}, - {file = "cryptography-3.1.1-cp35-abi3-manylinux1_x86_64.whl", hash = "sha256:a7597ffc67987b37b12e09c029bd1dc43965f75d328076ae85721b84046e9ca7"}, - {file = "cryptography-3.1.1-cp35-abi3-manylinux2010_x86_64.whl", hash = "sha256:4549b137d8cbe3c2eadfa56c0c858b78acbeff956bd461e40000b2164d9167c6"}, - {file = "cryptography-3.1.1-cp35-abi3-manylinux2014_aarch64.whl", hash = "sha256:89aceb31cd5f9fc2449fe8cf3810797ca52b65f1489002d58fe190bfb265c536"}, - {file = "cryptography-3.1.1-cp35-cp35m-win32.whl", hash = "sha256:559d622aef2a2dff98a892eef321433ba5bc55b2485220a8ca289c1ecc2bd54f"}, - {file = "cryptography-3.1.1-cp35-cp35m-win_amd64.whl", hash = "sha256:451cdf60be4dafb6a3b78802006a020e6cd709c22d240f94f7a0696240a17154"}, - {file = "cryptography-3.1.1-cp36-abi3-win32.whl", hash = "sha256:762bc5a0df03c51ee3f09c621e1cee64e3a079a2b5020de82f1613873d79ee70"}, - {file = "cryptography-3.1.1-cp36-abi3-win_amd64.whl", hash = "sha256:b12e715c10a13ca1bd27fbceed9adc8c5ff640f8e1f7ea76416352de703523c8"}, - {file = "cryptography-3.1.1-cp36-cp36m-win32.whl", hash = "sha256:21b47c59fcb1c36f1113f3709d37935368e34815ea1d7073862e92f810dc7499"}, - {file = "cryptography-3.1.1-cp36-cp36m-win_amd64.whl", hash = "sha256:48ee615a779ffa749d7d50c291761dc921d93d7cf203dca2db663b4f193f0e49"}, - {file = "cryptography-3.1.1-cp37-cp37m-win32.whl", hash = "sha256:b2bded09c578d19e08bd2c5bb8fed7f103e089752c9cf7ca7ca7de522326e921"}, - {file = "cryptography-3.1.1-cp37-cp37m-win_amd64.whl", hash = "sha256:f99317a0fa2e49917689b8cf977510addcfaaab769b3f899b9c481bbd76730c2"}, - {file = "cryptography-3.1.1-cp38-cp38-win32.whl", hash = "sha256:ab010e461bb6b444eaf7f8c813bb716be2d78ab786103f9608ffd37a4bd7d490"}, - {file = "cryptography-3.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:99d4984aabd4c7182050bca76176ce2dbc9fa9748afe583a7865c12954d714ba"}, - {file = "cryptography-3.1.1.tar.gz", hash = "sha256:9d9fc6a16357965d282dd4ab6531013935425d0dc4950df2e0cf2a1b1ac1017d"}, + {file = "cryptography-3.2.1-cp27-cp27m-macosx_10_10_x86_64.whl", hash = "sha256:6dc59630ecce8c1f558277ceb212c751d6730bd12c80ea96b4ac65637c4f55e7"}, + {file = "cryptography-3.2.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:75e8e6684cf0034f6bf2a97095cb95f81537b12b36a8fedf06e73050bb171c2d"}, + {file = "cryptography-3.2.1-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:4e7268a0ca14536fecfdf2b00297d4e407da904718658c1ff1961c713f90fd33"}, + {file = "cryptography-3.2.1-cp27-cp27m-win32.whl", hash = "sha256:7117319b44ed1842c617d0a452383a5a052ec6aa726dfbaffa8b94c910444297"}, + {file = "cryptography-3.2.1-cp27-cp27m-win_amd64.whl", hash = "sha256:a733671100cd26d816eed39507e585c156e4498293a907029969234e5e634bc4"}, + {file = "cryptography-3.2.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:a75f306a16d9f9afebfbedc41c8c2351d8e61e818ba6b4c40815e2b5740bb6b8"}, + {file = "cryptography-3.2.1-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:5849d59358547bf789ee7e0d7a9036b2d29e9a4ddf1ce5e06bb45634f995c53e"}, + {file = "cryptography-3.2.1-cp35-abi3-macosx_10_10_x86_64.whl", hash = "sha256:bd717aa029217b8ef94a7d21632a3bb5a4e7218a4513d2521c2a2fd63011e98b"}, + {file = "cryptography-3.2.1-cp35-abi3-manylinux1_x86_64.whl", hash = "sha256:efe15aca4f64f3a7ea0c09c87826490e50ed166ce67368a68f315ea0807a20df"}, + {file = "cryptography-3.2.1-cp35-abi3-manylinux2010_x86_64.whl", hash = "sha256:32434673d8505b42c0de4de86da8c1620651abd24afe91ae0335597683ed1b77"}, + {file = "cryptography-3.2.1-cp35-abi3-manylinux2014_aarch64.whl", hash = "sha256:7b8d9d8d3a9bd240f453342981f765346c87ade811519f98664519696f8e6ab7"}, + {file = "cryptography-3.2.1-cp35-cp35m-win32.whl", hash = "sha256:d3545829ab42a66b84a9aaabf216a4dce7f16dbc76eb69be5c302ed6b8f4a29b"}, + {file = "cryptography-3.2.1-cp35-cp35m-win_amd64.whl", hash = "sha256:a4e27ed0b2504195f855b52052eadcc9795c59909c9d84314c5408687f933fc7"}, + {file = "cryptography-3.2.1-cp36-abi3-win32.whl", hash = "sha256:13b88a0bd044b4eae1ef40e265d006e34dbcde0c2f1e15eb9896501b2d8f6c6f"}, + {file = "cryptography-3.2.1-cp36-abi3-win_amd64.whl", hash = "sha256:07ca431b788249af92764e3be9a488aa1d39a0bc3be313d826bbec690417e538"}, + {file = "cryptography-3.2.1-cp36-cp36m-win32.whl", hash = "sha256:a035a10686532b0587d58a606004aa20ad895c60c4d029afa245802347fab57b"}, + {file = "cryptography-3.2.1-cp36-cp36m-win_amd64.whl", hash = "sha256:d26a2557d8f9122f9bf445fc7034242f4375bd4e95ecda007667540270965b13"}, + {file = "cryptography-3.2.1-cp37-cp37m-win32.whl", hash = "sha256:545a8550782dda68f8cdc75a6e3bf252017aa8f75f19f5a9ca940772fc0cb56e"}, + {file = "cryptography-3.2.1-cp37-cp37m-win_amd64.whl", hash = "sha256:55d0b896631412b6f0c7de56e12eb3e261ac347fbaa5d5e705291a9016e5f8cb"}, + {file = "cryptography-3.2.1-cp38-cp38-win32.whl", hash = "sha256:3cd75a683b15576cfc822c7c5742b3276e50b21a06672dc3a800a2d5da4ecd1b"}, + {file = "cryptography-3.2.1-cp38-cp38-win_amd64.whl", hash = "sha256:d25cecbac20713a7c3bc544372d42d8eafa89799f492a43b79e1dfd650484851"}, + {file = "cryptography-3.2.1.tar.gz", hash = "sha256:d3d5e10be0cf2a12214ddee45c6bd203dab435e3d83b4560c03066eda600bfe3"}, ] execnet = [ {file = "execnet-1.7.1-py2.py3-none-any.whl", hash = "sha256:d4efd397930c46415f62f8a31388d6be4f27a91d7550eb79bc64a756e0056547"}, @@ -857,8 +857,8 @@ flake8-isort = [ {file = "flake8_isort-4.0.0-py2.py3-none-any.whl", hash = "sha256:729cd6ef9ba3659512dee337687c05d79c78e1215fdf921ed67e5fe46cce2f3c"}, ] iniconfig = [ - {file = "iniconfig-1.0.1-py3-none-any.whl", hash = "sha256:80cf40c597eb564e86346103f609d74efce0f6b4d4f30ec8ce9e2c26411ba437"}, - {file = "iniconfig-1.0.1.tar.gz", hash = "sha256:e5f92f89355a67de0595932a6c6c02ab4afddc6fcdc0bfc5becd0d60884d3f69"}, + {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, + {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, ] invoke = [ {file = "invoke-1.4.1-py2-none-any.whl", hash = "sha256:93e12876d88130c8e0d7fd6618dd5387d6b36da55ad541481dfa5e001656f134"}, @@ -866,8 +866,8 @@ invoke = [ {file = "invoke-1.4.1.tar.gz", hash = "sha256:de3f23bfe669e3db1085789fd859eb8ca8e0c5d9c20811e2407fa042e8a5e15d"}, ] isort = [ - {file = "isort-5.6.1-py3-none-any.whl", hash = "sha256:dd3211f513f4a92ec1ec1876fc1dc3c686649c349d49523f5b5adbb0814e5960"}, - {file = "isort-5.6.1.tar.gz", hash = "sha256:2f510f34ae18a8d0958c53eec51ef84fd099f07c4c639676525acbcd7b5bd3ff"}, + {file = "isort-5.6.4-py3-none-any.whl", hash = "sha256:dcab1d98b469a12a1a624ead220584391648790275560e1a43e54c5dceae65e7"}, + {file = "isort-5.6.4.tar.gz", hash = "sha256:dcaeec1b5f0eca77faea2a35ab790b4f3680ff75590bfcb7145986905aab2f58"}, ] jedi = [ {file = "jedi-0.17.2-py2.py3-none-any.whl", hash = "sha256:98cc583fa0f2f8304968199b01b6b4b94f469a1f4a74c1560506ca2a211378b5"}, @@ -910,8 +910,8 @@ parso = [ {file = "parso-0.7.1.tar.gz", hash = "sha256:caba44724b994a8a5e086460bb212abc5a8bc46951bf4a9a1210745953622eb9"}, ] pathspec = [ - {file = "pathspec-0.8.0-py2.py3-none-any.whl", hash = "sha256:7d91249d21749788d07a2d0f94147accd8f845507400749ea19c1ec9054a12b0"}, - {file = "pathspec-0.8.0.tar.gz", hash = "sha256:da45173eb3a6f2a5a487efba21f050af2b41948be6ab52b6a1e3ff22bb8b7061"}, + {file = "pathspec-0.8.1-py2.py3-none-any.whl", hash = "sha256:aa0cb481c4041bf52ffa7b0d8fa6cd3e88a2ca4879c533c9153882ee2556790d"}, + {file = "pathspec-0.8.1.tar.gz", hash = "sha256:86379d6b86d75816baba717e64b1a3a3469deb93bb76d613c9ce79edc5cb68fd"}, ] pluggy = [ {file = "pluggy-0.13.1-py2.py3-none-any.whl", hash = "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"}, @@ -968,8 +968,8 @@ pyparsing = [ {file = "pyparsing-2.4.7.tar.gz", hash = "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1"}, ] pytest = [ - {file = "pytest-6.1.1-py3-none-any.whl", hash = "sha256:7a8190790c17d79a11f847fba0b004ee9a8122582ebff4729a082c109e81a4c9"}, - {file = "pytest-6.1.1.tar.gz", hash = "sha256:8f593023c1a0f916110285b6efd7f99db07d59546e3d8c36fc60e2ab05d3be92"}, + {file = "pytest-6.1.2-py3-none-any.whl", hash = "sha256:4288fed0d9153d9646bfcdf0c0428197dba1ecb27a33bb6e031d002fa88653fe"}, + {file = "pytest-6.1.2.tar.gz", hash = "sha256:c0a7e94a8cdbc5422a51ccdad8e6f1024795939cc89159a0ae7f0b316ad3823e"}, ] pytest-flake8 = [ {file = "pytest-flake8-1.0.6.tar.gz", hash = "sha256:1b82bb58c88eb1db40524018d3fcfd0424575029703b4e2d8e3ee873f2b17027"}, @@ -1025,33 +1025,47 @@ pyyaml = [ {file = "PyYAML-5.3.1.tar.gz", hash = "sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d"}, ] regex = [ - {file = "regex-2020.9.27-cp27-cp27m-win32.whl", hash = "sha256:d23a18037313714fb3bb5a94434d3151ee4300bae631894b1ac08111abeaa4a3"}, - {file = "regex-2020.9.27-cp27-cp27m-win_amd64.whl", hash = "sha256:84e9407db1b2eb368b7ecc283121b5e592c9aaedbe8c78b1a2f1102eb2e21d19"}, - {file = "regex-2020.9.27-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:5f18875ac23d9aa2f060838e8b79093e8bb2313dbaaa9f54c6d8e52a5df097be"}, - {file = "regex-2020.9.27-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:ae91972f8ac958039920ef6e8769277c084971a142ce2b660691793ae44aae6b"}, - {file = "regex-2020.9.27-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:9a02d0ae31d35e1ec12a4ea4d4cca990800f66a917d0fb997b20fbc13f5321fc"}, - {file = "regex-2020.9.27-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:ebbe29186a3d9b0c591e71b7393f1ae08c83cb2d8e517d2a822b8f7ec99dfd8b"}, - {file = "regex-2020.9.27-cp36-cp36m-win32.whl", hash = "sha256:4707f3695b34335afdfb09be3802c87fa0bc27030471dbc082f815f23688bc63"}, - {file = "regex-2020.9.27-cp36-cp36m-win_amd64.whl", hash = "sha256:9bc13e0d20b97ffb07821aa3e113f9998e84994fe4d159ffa3d3a9d1b805043b"}, - {file = "regex-2020.9.27-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:f1b3afc574a3db3b25c89161059d857bd4909a1269b0b3cb3c904677c8c4a3f7"}, - {file = "regex-2020.9.27-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:5533a959a1748a5c042a6da71fe9267a908e21eded7a4f373efd23a2cbdb0ecc"}, - {file = "regex-2020.9.27-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:1fe0a41437bbd06063aa184c34804efa886bcc128222e9916310c92cd54c3b4c"}, - {file = "regex-2020.9.27-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:c570f6fa14b9c4c8a4924aaad354652366577b4f98213cf76305067144f7b100"}, - {file = "regex-2020.9.27-cp37-cp37m-win32.whl", hash = "sha256:eda4771e0ace7f67f58bc5b560e27fb20f32a148cbc993b0c3835970935c2707"}, - {file = "regex-2020.9.27-cp37-cp37m-win_amd64.whl", hash = "sha256:60b0e9e6dc45683e569ec37c55ac20c582973841927a85f2d8a7d20ee80216ab"}, - {file = "regex-2020.9.27-cp38-cp38-manylinux1_i686.whl", hash = "sha256:088afc8c63e7bd187a3c70a94b9e50ab3f17e1d3f52a32750b5b77dbe99ef5ef"}, - {file = "regex-2020.9.27-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:eaf548d117b6737df379fdd53bdde4f08870e66d7ea653e230477f071f861121"}, - {file = "regex-2020.9.27-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:41bb65f54bba392643557e617316d0d899ed5b4946dccee1cb6696152b29844b"}, - {file = "regex-2020.9.27-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:8d69cef61fa50c8133382e61fd97439de1ae623fe943578e477e76a9d9471637"}, - {file = "regex-2020.9.27-cp38-cp38-win32.whl", hash = "sha256:f2388013e68e750eaa16ccbea62d4130180c26abb1d8e5d584b9baf69672b30f"}, - {file = "regex-2020.9.27-cp38-cp38-win_amd64.whl", hash = "sha256:4318d56bccfe7d43e5addb272406ade7a2274da4b70eb15922a071c58ab0108c"}, - {file = "regex-2020.9.27-cp39-cp39-manylinux1_i686.whl", hash = "sha256:84cada8effefe9a9f53f9b0d2ba9b7b6f5edf8d2155f9fdbe34616e06ececf81"}, - {file = "regex-2020.9.27-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:816064fc915796ea1f26966163f6845de5af78923dfcecf6551e095f00983650"}, - {file = "regex-2020.9.27-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:5d892a4f1c999834eaa3c32bc9e8b976c5825116cde553928c4c8e7e48ebda67"}, - {file = "regex-2020.9.27-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:c9443124c67b1515e4fe0bb0aa18df640965e1030f468a2a5dc2589b26d130ad"}, - {file = "regex-2020.9.27-cp39-cp39-win32.whl", hash = "sha256:49f23ebd5ac073765ecbcf046edc10d63dcab2f4ae2bce160982cb30df0c0302"}, - {file = "regex-2020.9.27-cp39-cp39-win_amd64.whl", hash = "sha256:3d20024a70b97b4f9546696cbf2fd30bae5f42229fbddf8661261b1eaff0deb7"}, - {file = "regex-2020.9.27.tar.gz", hash = "sha256:a6f32aea4260dfe0e55dc9733ea162ea38f0ea86aa7d0f77b15beac5bf7b369d"}, + {file = "regex-2020.11.13-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:8b882a78c320478b12ff024e81dc7d43c1462aa4a3341c754ee65d857a521f85"}, + {file = "regex-2020.11.13-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:a63f1a07932c9686d2d416fb295ec2c01ab246e89b4d58e5fa468089cab44b70"}, + {file = "regex-2020.11.13-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:6e4b08c6f8daca7d8f07c8d24e4331ae7953333dbd09c648ed6ebd24db5a10ee"}, + {file = "regex-2020.11.13-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:bba349276b126947b014e50ab3316c027cac1495992f10e5682dc677b3dfa0c5"}, + {file = "regex-2020.11.13-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:56e01daca75eae420bce184edd8bb341c8eebb19dd3bce7266332258f9fb9dd7"}, + {file = "regex-2020.11.13-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:6a8ce43923c518c24a2579fda49f093f1397dad5d18346211e46f134fc624e31"}, + {file = "regex-2020.11.13-cp36-cp36m-manylinux2014_i686.whl", hash = "sha256:1ab79fcb02b930de09c76d024d279686ec5d532eb814fd0ed1e0051eb8bd2daa"}, + {file = "regex-2020.11.13-cp36-cp36m-manylinux2014_x86_64.whl", hash = "sha256:9801c4c1d9ae6a70aeb2128e5b4b68c45d4f0af0d1535500884d644fa9b768c6"}, + {file = "regex-2020.11.13-cp36-cp36m-win32.whl", hash = "sha256:49cae022fa13f09be91b2c880e58e14b6da5d10639ed45ca69b85faf039f7a4e"}, + {file = "regex-2020.11.13-cp36-cp36m-win_amd64.whl", hash = "sha256:749078d1eb89484db5f34b4012092ad14b327944ee7f1c4f74d6279a6e4d1884"}, + {file = "regex-2020.11.13-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b2f4007bff007c96a173e24dcda236e5e83bde4358a557f9ccf5e014439eae4b"}, + {file = "regex-2020.11.13-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:38c8fd190db64f513fe4e1baa59fed086ae71fa45083b6936b52d34df8f86a88"}, + {file = "regex-2020.11.13-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:5862975b45d451b6db51c2e654990c1820523a5b07100fc6903e9c86575202a0"}, + {file = "regex-2020.11.13-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:262c6825b309e6485ec2493ffc7e62a13cf13fb2a8b6d212f72bd53ad34118f1"}, + {file = "regex-2020.11.13-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:bafb01b4688833e099d79e7efd23f99172f501a15c44f21ea2118681473fdba0"}, + {file = "regex-2020.11.13-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:e32f5f3d1b1c663af7f9c4c1e72e6ffe9a78c03a31e149259f531e0fed826512"}, + {file = "regex-2020.11.13-cp37-cp37m-manylinux2014_i686.whl", hash = "sha256:3bddc701bdd1efa0d5264d2649588cbfda549b2899dc8d50417e47a82e1387ba"}, + {file = "regex-2020.11.13-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:02951b7dacb123d8ea6da44fe45ddd084aa6777d4b2454fa0da61d569c6fa538"}, + {file = "regex-2020.11.13-cp37-cp37m-win32.whl", hash = "sha256:0d08e71e70c0237883d0bef12cad5145b84c3705e9c6a588b2a9c7080e5af2a4"}, + {file = "regex-2020.11.13-cp37-cp37m-win_amd64.whl", hash = "sha256:1fa7ee9c2a0e30405e21031d07d7ba8617bc590d391adfc2b7f1e8b99f46f444"}, + {file = "regex-2020.11.13-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:baf378ba6151f6e272824b86a774326f692bc2ef4cc5ce8d5bc76e38c813a55f"}, + {file = "regex-2020.11.13-cp38-cp38-manylinux1_i686.whl", hash = "sha256:e3faaf10a0d1e8e23a9b51d1900b72e1635c2d5b0e1bea1c18022486a8e2e52d"}, + {file = "regex-2020.11.13-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:2a11a3e90bd9901d70a5b31d7dd85114755a581a5da3fc996abfefa48aee78af"}, + {file = "regex-2020.11.13-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:d1ebb090a426db66dd80df8ca85adc4abfcbad8a7c2e9a5ec7513ede522e0a8f"}, + {file = "regex-2020.11.13-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:b2b1a5ddae3677d89b686e5c625fc5547c6e492bd755b520de5332773a8af06b"}, + {file = "regex-2020.11.13-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:2c99e97d388cd0a8d30f7c514d67887d8021541b875baf09791a3baad48bb4f8"}, + {file = "regex-2020.11.13-cp38-cp38-manylinux2014_i686.whl", hash = "sha256:c084582d4215593f2f1d28b65d2a2f3aceff8342aa85afd7be23a9cad74a0de5"}, + {file = "regex-2020.11.13-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:a3d748383762e56337c39ab35c6ed4deb88df5326f97a38946ddd19028ecce6b"}, + {file = "regex-2020.11.13-cp38-cp38-win32.whl", hash = "sha256:7913bd25f4ab274ba37bc97ad0e21c31004224ccb02765ad984eef43e04acc6c"}, + {file = "regex-2020.11.13-cp38-cp38-win_amd64.whl", hash = "sha256:6c54ce4b5d61a7129bad5c5dc279e222afd00e721bf92f9ef09e4fae28755683"}, + {file = "regex-2020.11.13-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1862a9d9194fae76a7aaf0150d5f2a8ec1da89e8b55890b1786b8f88a0f619dc"}, + {file = "regex-2020.11.13-cp39-cp39-manylinux1_i686.whl", hash = "sha256:4902e6aa086cbb224241adbc2f06235927d5cdacffb2425c73e6570e8d862364"}, + {file = "regex-2020.11.13-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:7a25fcbeae08f96a754b45bdc050e1fb94b95cab046bf56b016c25e9ab127b3e"}, + {file = "regex-2020.11.13-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:d2d8ce12b7c12c87e41123997ebaf1a5767a5be3ec545f64675388970f415e2e"}, + {file = "regex-2020.11.13-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:f7d29a6fc4760300f86ae329e3b6ca28ea9c20823df123a2ea8693e967b29917"}, + {file = "regex-2020.11.13-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:717881211f46de3ab130b58ec0908267961fadc06e44f974466d1887f865bd5b"}, + {file = "regex-2020.11.13-cp39-cp39-manylinux2014_i686.whl", hash = "sha256:3128e30d83f2e70b0bed9b2a34e92707d0877e460b402faca908c6667092ada9"}, + {file = "regex-2020.11.13-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:8f6a2229e8ad946e36815f2a03386bb8353d4bde368fdf8ca5f0cb97264d3b5c"}, + {file = "regex-2020.11.13-cp39-cp39-win32.whl", hash = "sha256:f8f295db00ef5f8bae530fc39af0b40486ca6068733fb860b42115052206466f"}, + {file = "regex-2020.11.13-cp39-cp39-win_amd64.whl", hash = "sha256:a15f64ae3a027b64496a71ab1f722355e570c3fac5ba2801cafce846bf5af01d"}, + {file = "regex-2020.11.13.tar.gz", hash = "sha256:83d6b356e116ca119db8e7c6fc2983289d87b27b3fac238cfe5dca529d884562"}, ] rope = [ {file = "rope-0.18.0.tar.gz", hash = "sha256:786b5c38c530d4846aa68a42604f61b4e69a493390e3ca11b88df0fbfdc3ed04"}, @@ -1073,8 +1087,8 @@ testfixtures = [ {file = "testfixtures-6.15.0.tar.gz", hash = "sha256:409f77cfbdad822d12a8ce5c4aa8fb4d0bb38073f4a5444fede3702716a2cec2"}, ] toml = [ - {file = "toml-0.10.1-py2.py3-none-any.whl", hash = "sha256:bda89d5935c2eac546d648028b9901107a595863cb36bae0c73ac804a9b4ce88"}, - {file = "toml-0.10.1.tar.gz", hash = "sha256:926b612be1e5ce0634a2ca03470f95169cf16f939018233a670519cb4ac58b0f"}, + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, ] typed-ast = [ {file = "typed_ast-1.4.1-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:73d785a950fc82dd2a25897d525d003f6378d1cb23ab305578394694202a58c3"}, From 16ef4e76df81e48a8995de7d242951fd3a4bc0fe Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Wed, 18 Nov 2020 17:55:41 -0800 Subject: [PATCH 086/194] Create pytest-playbook plugin --- conftest.py | 114 +++++++------ playbook.py | 68 -------- poetry.lock | 21 ++- pyproject.toml | 2 +- pytest-playbook/poetry.lock | 252 +++++++++++++++++++++++++++++ pytest-playbook/pyproject.toml | 5 +- pytest-playbook/pytest_playbook.py | 66 ++++++++ 7 files changed, 412 insertions(+), 116 deletions(-) delete mode 100644 playbook.py create mode 100644 pytest-playbook/poetry.lock create mode 100644 pytest-playbook/pytest_playbook.py diff --git a/conftest.py b/conftest.py index 2dfe991bd2..e674adaa08 100644 --- a/conftest.py +++ b/conftest.py @@ -6,9 +6,11 @@ from __future__ import annotations import typing -from pathlib import Path -from schema import SchemaMissingKeyError # type: ignore +import pytest_playbook + +# See https://pypi.org/project/schema/ +from schema import Optional, Or, Schema, SchemaMissingKeyError # type: ignore import azure # noqa import lisa @@ -16,7 +18,7 @@ from target import Target if typing.TYPE_CHECKING: - from typing import Any, Dict, Iterator, List, Optional, Type + from typing import Any, Dict, Iterator, List, Type from _pytest.config import Config from _pytest.config.argparsing import Parser @@ -25,6 +27,8 @@ from pytest import Item, Session +pytest_plugins = ["playbook"] + @pytest.fixture(scope="session") def pool(request: SubRequest) -> Iterator[List[Target]]: @@ -61,9 +65,7 @@ def target(pool: List[Target], request: SubRequest) -> Iterator[Target]: their environments. """ - import playbook - - platform: Type[Target] = playbook.PLATFORMS[request.param["platform"]] + platform: Type[Target] = platforms[request.param["platform"]] parameters: Dict[str, Any] = request.param["parameters"] marker = request.node.get_closest_marker("lisa") features = set(marker.kwargs["features"]) @@ -99,56 +101,80 @@ def pytest_addoption(parser: Parser) -> None: parser.addoption("--keep-vms", action="store_true", help="Keeps deployed VMs.") parser.addoption("--check", action="store_true", help="Run semantic analysis.") parser.addoption("--demo", action="store_true", help="Run in demo mode.") - parser.addoption("--playbook", type=Path, help="Path to playbook.") -TARGETS: List[Dict[str, Any]] = [] -TARGET_IDS: List[str] = [] +platforms: Dict[str, Type[Target]] = dict() -def get_playbook(path: Optional[Path]) -> Dict[str, Any]: - """Loads and validates the playbook file. +def pytest_playbook_schema(schema: Dict[Any, Any], config: Config) -> None: + """Describes the YAML schema for the playbook file. - This imports the playbook module at runtime to ensure all - subclasses of 'Target' (e.g. all supported platforms, including - those defined in arbitrary 'conftest.py' files) are defined. + 'platforms' is a mapping of platform names (strings) to the + implementing subclass of 'Target' where each subclass defines its + own 'parameters' schema, 'deploy' and 'delete' methods, and other + platform-specific functionality. A 'Target' subclass need only be + defined in a file loaded by Pytest, so a 'contest.py' file works + just fine. No manual subclass of 'Target' where each subc ass + defines its own 'parameters' schema, 'deploy' and 'delete' + methods, and other platform-specific functionality. A 'Target' + subclass need only be defined in a file loaded by Pytest, so a 'c - """ - # TODO: Move to 'playbook.py' and setup 'PLATFORMS' when called so - # that the import can take place at any time. - import playbook - - book = dict() - if path: - # See https://pyyaml.org/wiki/PyYAMLDocumentation - import yaml + TODO: Add field annotations, friendly error reporting, automatic + case transformations, etc. - try: - from yaml import CLoader as Loader - except ImportError: - from yaml import Loader # type: ignore - - with open(path) as f: - book = playbook.schema.validate(yaml.load(f, Loader=Loader)) - else: - book = playbook.schema.validate({}) - return book + """ + global platforms + platforms = {cls.__name__: cls for cls in Target.__subclasses__()} # type: ignore + target_schema = Schema( + { + "name": str, + "platform": Or(*[platform for platform in platforms.keys()]), + # TODO: What should we do when lacking parameters? Ideally we + # use the platform’s defaults from its own schema, but that + # means this value must be set, even if to an empty dict. + Optional("parameters", default=dict): Or( + *[cls.schema for cls in platforms.values()] + ), + } + ) + + default_target = {"name": "Default", "platform": "Local"} + + criteria_schema = Schema( + { + # TODO: Validate that these strings are valid regular + # expressions if we change our matching logic. + Optional("name", default=None): str, + Optional("area", default=None): str, + Optional("category", default=None): str, + Optional("priority", default=None): int, + Optional("tags", default=list): [str], + Optional("times", default=1): int, + Optional("exclude", default=False): bool, + } + ) + + schema[Optional("targets", default=[default_target])] = [target_schema] + schema[Optional("criteria", default=list)] = [criteria_schema] + + +targets: List[Dict[str, Any]] = [] +target_ids: List[str] = [] + + +def pytest_sessionstart(session: Session) -> None: + """Determines the targets based on the playbook.""" + for t in pytest_playbook.playbook.get("targets", []): + targets.append(t) + target_ids.append(t["name"]) def pytest_configure(config: Config) -> None: """Parse provided user inputs to setup configuration. - Determines the targets based on the playbook and sets default - configurations based user mode. - https://docs.pytest.org/en/latest/reference.html#pytest.hookspec.pytest_configure """ - book = get_playbook(config.getoption("--playbook")) - for t in book.get("targets", []): - TARGETS.append(t) - TARGET_IDS.append(t["name"]) - # Search ‘_pytest’ for ‘addoption’ to find these. options: Dict[str, Any] = {} # See ‘pytest.ini’ for defaults. if config.getoption("--check"): @@ -183,8 +209,7 @@ def pytest_generate_tests(metafunc: Metafunc) -> None: """ if "target" in metafunc.fixturenames: - assert TARGETS, "No targets specified!" - metafunc.parametrize("target", TARGETS, True, TARGET_IDS) + metafunc.parametrize("target", targets, True, target_ids) def pytest_collection_modifyitems( @@ -223,8 +248,7 @@ def select(item: Item, times: int, exclude: bool) -> None: for _ in range(times - included.count(item)): included.append(item) - book = get_playbook(config.getoption("--playbook")) - for c in book.get("criteria", []): + for c in pytest_playbook.playbook.get("criteria", []): print(f"Parsing criteria {c}") for item in items: marker = item.get_closest_marker("lisa") diff --git a/playbook.py b/playbook.py deleted file mode 100644 index 0d3a79267c..0000000000 --- a/playbook.py +++ /dev/null @@ -1,68 +0,0 @@ -"""Describes the YAML schema for the playbook file. - -This module should be imported at runtime such that 'PLATFORMS' is -defined after all 'Target' subclasses have been defined. - -PLATFORMS is a mapping of platform names (strings) to the implementing -subclass of 'Target' where each subclass defines its own 'parameters' -schema, 'deploy' and 'delete' methods, and other platform-specific -functionality. A 'Target' subclass need only be defined in a file -loaded by Pytest, so a 'contest.py' file works just fine. No manual -registration is required, it will be discovered automatically. - -TODO: Add field annotations, friendly error reporting, automatic case -transformations, etc. - -""" -from __future__ import annotations - -import typing - -# See https://pypi.org/project/schema/ -from schema import Optional, Or, Schema # type: ignore - -from target import Target - -if typing.TYPE_CHECKING: - from typing import Mapping, Type - -# See https://github.com/python/mypy/issues/4717 for why we ignore the type. -PLATFORMS: Mapping[str, Type[Target]] = { - cls.__name__: cls for cls in Target.__subclasses__() # type: ignore -} - -target_schema = Schema( - { - "name": str, - "platform": Or(*[platform for platform in PLATFORMS.keys()]), - # TODO: What should we do when lacking parameters? Ideally we - # use the platform’s defaults from its own schema, but that - # means this value must be set, even if to an empty dict. - Optional("parameters", default=dict): Or( - *[cls.schema for cls in PLATFORMS.values()] - ), - } -) - -default_target = {"name": "Default", "platform": "Local"} - -criteria_schema = Schema( - { - # TODO: Validate that these strings are valid regular - # expressions if we change our matching logic. - Optional("name", default=None): str, - Optional("area", default=None): str, - Optional("category", default=None): str, - Optional("priority", default=None): int, - Optional("tags", default=list): [str], - Optional("times", default=1): int, - Optional("exclude", default=False): bool, - } -) - -schema = Schema( - { - Optional("targets", default=[default_target]): [target_schema], - Optional("criteria", default=list): [criteria_schema], - } -) diff --git a/poetry.lock b/poetry.lock index 792bb074f4..9e9c0d70b6 100644 --- a/poetry.lock +++ b/poetry.lock @@ -537,6 +537,24 @@ filelock = ">=3.0" mypy = {version = ">=0.700", markers = "python_version >= \"3.8\""} pytest = ">=3.5" +[[package]] +name = "pytest-playbook" +version = "0.1.0" +description = "Pytest plugin for reading playbooks." +category = "main" +optional = false +python-versions = "^3.7" +develop = true + +[package.dependencies] +pytest = "^6.1.2" +PyYAML = "^5.3.1" +schema = "^0.7.3" + +[package.source] +type = "directory" +url = "pytest-playbook" + [[package]] name = "pytest-rerunfailures" version = "9.1.1" @@ -724,7 +742,7 @@ python-versions = ">=3.6" [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "ff9d853cf9f58598aa01e465e2c673172b9e573fd7a8569bf29236348884c748" +content-hash = "6b1d924f0bc47c51bc55daa27f1c15c0dfb72b4902625f2577ef75c6d3ab992f" [metadata.files] apipkg = [ @@ -991,6 +1009,7 @@ pytest-mypy = [ {file = "pytest-mypy-0.7.0.tar.gz", hash = "sha256:5a667d9a2b66bf98b3a494411f221923a6e2c3eafbe771104951aaec8985673d"}, {file = "pytest_mypy-0.7.0-py3-none-any.whl", hash = "sha256:e0505ace48d2b19fe686366fce6b4a2ac0d090423736bb6aa2e39554d18974b7"}, ] +pytest-playbook = [] pytest-rerunfailures = [ {file = "pytest-rerunfailures-9.1.1.tar.gz", hash = "sha256:1cb11a17fc121b3918414eb5eaf314ee325f2e693ac7cb3f6abf7560790827f2"}, {file = "pytest_rerunfailures-9.1.1-py3-none-any.whl", hash = "sha256:2eb7d0ad651761fbe80e064b0fd415cf6730cdbc53c16a145fd84b66143e609f"}, diff --git a/pyproject.toml b/pyproject.toml index 47db86c319..3f72c622ff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,8 +15,8 @@ pytest-html = "^2.1.1" tenacity = "^6.2.0" pytest-rerunfailures = "^9.1.1" pytest-xdist = "^2.1.0" -PyYAML = "^5.3.1" schema = "^0.7.3" +pytest-playbook = {path = "pytest-playbook", develop = true} [tool.poetry.dev-dependencies] black = "^20.8b1" diff --git a/pytest-playbook/poetry.lock b/pytest-playbook/poetry.lock new file mode 100644 index 0000000000..6f2d824232 --- /dev/null +++ b/pytest-playbook/poetry.lock @@ -0,0 +1,252 @@ +[[package]] +name = "atomicwrites" +version = "1.4.0" +description = "Atomic file writes." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[[package]] +name = "attrs" +version = "20.3.0" +description = "Classes Without Boilerplate" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[package.extras] +dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "furo", "sphinx", "pre-commit"] +docs = ["furo", "sphinx", "zope.interface"] +tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface"] +tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six"] + +[[package]] +name = "colorama" +version = "0.4.4" +description = "Cross-platform colored terminal text." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[[package]] +name = "contextlib2" +version = "0.6.0.post1" +description = "Backports and enhancements for the contextlib module" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[[package]] +name = "importlib-metadata" +version = "2.0.0" +description = "Read metadata from Python packages" +category = "main" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" + +[package.dependencies] +zipp = ">=0.5" + +[package.extras] +docs = ["sphinx", "rst.linker"] +testing = ["packaging", "pep517", "importlib-resources (>=1.3)"] + +[[package]] +name = "iniconfig" +version = "1.1.1" +description = "iniconfig: brain-dead simple config-ini parsing" +category = "main" +optional = false +python-versions = "*" + +[[package]] +name = "packaging" +version = "20.4" +description = "Core utilities for Python packages" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[package.dependencies] +pyparsing = ">=2.0.2" +six = "*" + +[[package]] +name = "pluggy" +version = "0.13.1" +description = "plugin and hook calling mechanisms for python" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[package.dependencies] +importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} + +[package.extras] +dev = ["pre-commit", "tox"] + +[[package]] +name = "py" +version = "1.9.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[[package]] +name = "pyparsing" +version = "2.4.7" +description = "Python parsing module" +category = "main" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" + +[[package]] +name = "pytest" +version = "6.1.2" +description = "pytest: simple powerful testing with Python" +category = "main" +optional = false +python-versions = ">=3.5" + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=17.4.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<1.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +checkqa_mypy = ["mypy (==0.780)"] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "pyyaml" +version = "5.3.1" +description = "YAML parser and emitter for Python" +category = "main" +optional = true +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[[package]] +name = "schema" +version = "0.7.3" +description = "Simple data validation library" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +contextlib2 = ">=0.5.5" + +[[package]] +name = "six" +version = "1.15.0" +description = "Python 2 and 3 compatibility utilities" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +category = "main" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" + +[[package]] +name = "zipp" +version = "3.4.0" +description = "Backport of pathlib-compatible object wrapper for zip files" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.extras] +docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"] +testing = ["pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake8", "pytest-cov", "jaraco.test (>=3.2.0)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy"] + +[metadata] +lock-version = "1.1" +python-versions = "^3.7" +content-hash = "0365579f4be5c800fa2a7e73898fadf530f5c3c2808113744a73b10b5b7ccc7d" + +[metadata.files] +atomicwrites = [ + {file = "atomicwrites-1.4.0-py2.py3-none-any.whl", hash = "sha256:6d1784dea7c0c8d4a5172b6c620f40b6e4cbfdf96d783691f2e1302a7b88e197"}, + {file = "atomicwrites-1.4.0.tar.gz", hash = "sha256:ae70396ad1a434f9c7046fd2dd196fc04b12f9e91ffb859164193be8b6168a7a"}, +] +attrs = [ + {file = "attrs-20.3.0-py2.py3-none-any.whl", hash = "sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6"}, + {file = "attrs-20.3.0.tar.gz", hash = "sha256:832aa3cde19744e49938b91fea06d69ecb9e649c93ba974535d08ad92164f700"}, +] +colorama = [ + {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"}, + {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"}, +] +contextlib2 = [ + {file = "contextlib2-0.6.0.post1-py2.py3-none-any.whl", hash = "sha256:3355078a159fbb44ee60ea80abd0d87b80b78c248643b49aa6d94673b413609b"}, + {file = "contextlib2-0.6.0.post1.tar.gz", hash = "sha256:01f490098c18b19d2bd5bb5dc445b2054d2fa97f09a4280ba2c5f3c394c8162e"}, +] +importlib-metadata = [ + {file = "importlib_metadata-2.0.0-py2.py3-none-any.whl", hash = "sha256:cefa1a2f919b866c5beb7c9f7b0ebb4061f30a8a9bf16d609b000e2dfaceb9c3"}, + {file = "importlib_metadata-2.0.0.tar.gz", hash = "sha256:77a540690e24b0305878c37ffd421785a6f7e53c8b5720d211b211de8d0e95da"}, +] +iniconfig = [ + {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, + {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, +] +packaging = [ + {file = "packaging-20.4-py2.py3-none-any.whl", hash = "sha256:998416ba6962ae7fbd6596850b80e17859a5753ba17c32284f67bfff33784181"}, + {file = "packaging-20.4.tar.gz", hash = "sha256:4357f74f47b9c12db93624a82154e9b120fa8293699949152b22065d556079f8"}, +] +pluggy = [ + {file = "pluggy-0.13.1-py2.py3-none-any.whl", hash = "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"}, + {file = "pluggy-0.13.1.tar.gz", hash = "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0"}, +] +py = [ + {file = "py-1.9.0-py2.py3-none-any.whl", hash = "sha256:366389d1db726cd2fcfc79732e75410e5fe4d31db13692115529d34069a043c2"}, + {file = "py-1.9.0.tar.gz", hash = "sha256:9ca6883ce56b4e8da7e79ac18787889fa5206c79dcc67fb065376cd2fe03f342"}, +] +pyparsing = [ + {file = "pyparsing-2.4.7-py2.py3-none-any.whl", hash = "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"}, + {file = "pyparsing-2.4.7.tar.gz", hash = "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1"}, +] +pytest = [ + {file = "pytest-6.1.2-py3-none-any.whl", hash = "sha256:4288fed0d9153d9646bfcdf0c0428197dba1ecb27a33bb6e031d002fa88653fe"}, + {file = "pytest-6.1.2.tar.gz", hash = "sha256:c0a7e94a8cdbc5422a51ccdad8e6f1024795939cc89159a0ae7f0b316ad3823e"}, +] +pyyaml = [ + {file = "PyYAML-5.3.1-cp27-cp27m-win32.whl", hash = "sha256:74809a57b329d6cc0fdccee6318f44b9b8649961fa73144a98735b0aaf029f1f"}, + {file = "PyYAML-5.3.1-cp27-cp27m-win_amd64.whl", hash = "sha256:240097ff019d7c70a4922b6869d8a86407758333f02203e0fc6ff79c5dcede76"}, + {file = "PyYAML-5.3.1-cp35-cp35m-win32.whl", hash = "sha256:4f4b913ca1a7319b33cfb1369e91e50354d6f07a135f3b901aca02aa95940bd2"}, + {file = "PyYAML-5.3.1-cp35-cp35m-win_amd64.whl", hash = "sha256:cc8955cfbfc7a115fa81d85284ee61147059a753344bc51098f3ccd69b0d7e0c"}, + {file = "PyYAML-5.3.1-cp36-cp36m-win32.whl", hash = "sha256:7739fc0fa8205b3ee8808aea45e968bc90082c10aef6ea95e855e10abf4a37b2"}, + {file = "PyYAML-5.3.1-cp36-cp36m-win_amd64.whl", hash = "sha256:69f00dca373f240f842b2931fb2c7e14ddbacd1397d57157a9b005a6a9942648"}, + {file = "PyYAML-5.3.1-cp37-cp37m-win32.whl", hash = "sha256:d13155f591e6fcc1ec3b30685d50bf0711574e2c0dfffd7644babf8b5102ca1a"}, + {file = "PyYAML-5.3.1-cp37-cp37m-win_amd64.whl", hash = "sha256:73f099454b799e05e5ab51423c7bcf361c58d3206fa7b0d555426b1f4d9a3eaf"}, + {file = "PyYAML-5.3.1-cp38-cp38-win32.whl", hash = "sha256:06a0d7ba600ce0b2d2fe2e78453a470b5a6e000a985dd4a4e54e436cc36b0e97"}, + {file = "PyYAML-5.3.1-cp38-cp38-win_amd64.whl", hash = "sha256:95f71d2af0ff4227885f7a6605c37fd53d3a106fcab511b8860ecca9fcf400ee"}, + {file = "PyYAML-5.3.1.tar.gz", hash = "sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d"}, +] +schema = [ + {file = "schema-0.7.3-py2.py3-none-any.whl", hash = "sha256:c331438b60f634cab5664ab720d3083cc444f924d55269530c36b33e3354276f"}, + {file = "schema-0.7.3.tar.gz", hash = "sha256:4cf529318cfd1e844ecbe02f41f7e5aa027463e7403666a52746f31f04f47a5e"}, +] +six = [ + {file = "six-1.15.0-py2.py3-none-any.whl", hash = "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"}, + {file = "six-1.15.0.tar.gz", hash = "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259"}, +] +toml = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] +zipp = [ + {file = "zipp-3.4.0-py3-none-any.whl", hash = "sha256:102c24ef8f171fd729d46599845e95c7ab894a4cf45f5de11a44cc7444fb1108"}, + {file = "zipp-3.4.0.tar.gz", hash = "sha256:ed5eee1974372595f9e416cc7bbeeb12335201d8081ca8a0743c954d4446e5cb"}, +] diff --git a/pytest-playbook/pyproject.toml b/pytest-playbook/pyproject.toml index 5ecf851772..787a88034d 100644 --- a/pytest-playbook/pyproject.toml +++ b/pytest-playbook/pyproject.toml @@ -10,10 +10,13 @@ classifiers = ["Framework :: Pytest"] python = "^3.7" pytest = "^6.1.2" schema = "^0.7.3" -PyYAML = {version = "^5.3.1", optional = true} +PyYAML = "^5.3.1" [tool.poetry.dev-dependencies] +[tool.poetry.plugins] +pytest11 = {playbook = "pytest_playbook"} + [build-system] requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" diff --git a/pytest-playbook/pytest_playbook.py b/pytest-playbook/pytest_playbook.py new file mode 100644 index 0000000000..bff379a2a1 --- /dev/null +++ b/pytest-playbook/pytest_playbook.py @@ -0,0 +1,66 @@ +"""A plugin for creating, validating, and loading a playbook. + +After the last call to 'pytest_configure', 'playbook' will be +populated with the validated data. + +""" + +from __future__ import annotations + +import typing +from pathlib import Path + +import yaml # TODO: Optionally load yaml. +from schema import Schema # type: ignore + +# See https://pyyaml.org/wiki/PyYAMLDocumentation +try: + from yaml import CLoader as Loader +except ImportError: + from yaml import Loader # type: ignore + +import pytest + +if typing.TYPE_CHECKING: + from typing import Any, Dict, Optional + + from _pytest.config import Config, PytestPluginManager + from _pytest.config.argparsing import Parser + + +class Hooks: + """Class which contains our hooks.""" + + def pytest_playbook_schema(self, schema: Dict[Any, Any], config: Config) -> None: + """Update the Playbook's schema dict.""" + + +def pytest_addhooks(pluginmanager: PytestPluginManager) -> None: + """Pytest hook to register hooks.""" + pluginmanager.add_hookspecs(Hooks) + + +def pytest_addoption(parser: Parser, pluginmanager: PytestPluginManager) -> None: + """Pytest hook to add CLI options.""" + group = parser.getgroup("playbook") + group.addoption("--playbook", type=Path, help="Path to playbook.") + + +playbook: Dict[Any, Any] = dict() + + +@pytest.hookimpl(trylast=True) +def pytest_configure(config: Config) -> None: + """Pytest hook to configure each plugin.""" + path: Optional[Path] = config.getoption("playbook") + if not path or not path.is_file(): + # TODO: Log an appropriate warning. + return + + schema: Dict[Any, Any] = dict() + config.hook.pytest_playbook_schema(schema=schema, config=config) + + global playbook + with open(path) as f: + # TODO: Handle ‘SchemaMissingKeyError’. + playbook = Schema(schema).validate(yaml.load(f, Loader=Loader)) From 02beaf0e6831348244e028b7d22e6642f8619dd2 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Thu, 19 Nov 2020 13:18:10 -0800 Subject: [PATCH 087/194] Settle on plugin source code layout --- conftest.py | 6 +- pytest-playbook/playbook.py | 98 ++++++++++++++++++++++++++++++ pytest-playbook/pyproject.toml | 3 +- pytest-playbook/pytest_playbook.py | 66 -------------------- 4 files changed, 103 insertions(+), 70 deletions(-) create mode 100644 pytest-playbook/playbook.py delete mode 100644 pytest-playbook/pytest_playbook.py diff --git a/conftest.py b/conftest.py index e674adaa08..ad93ac227c 100644 --- a/conftest.py +++ b/conftest.py @@ -7,7 +7,7 @@ import typing -import pytest_playbook +import playbook # See https://pypi.org/project/schema/ from schema import Optional, Or, Schema, SchemaMissingKeyError # type: ignore @@ -164,7 +164,7 @@ def pytest_playbook_schema(schema: Dict[Any, Any], config: Config) -> None: def pytest_sessionstart(session: Session) -> None: """Determines the targets based on the playbook.""" - for t in pytest_playbook.playbook.get("targets", []): + for t in playbook.playbook.get("targets", []): targets.append(t) target_ids.append(t["name"]) @@ -248,7 +248,7 @@ def select(item: Item, times: int, exclude: bool) -> None: for _ in range(times - included.count(item)): included.append(item) - for c in pytest_playbook.playbook.get("criteria", []): + for c in playbook.playbook.get("criteria", []): print(f"Parsing criteria {c}") for item in items: marker = item.get_closest_marker("lisa") diff --git a/pytest-playbook/playbook.py b/pytest-playbook/playbook.py new file mode 100644 index 0000000000..a0784dfc09 --- /dev/null +++ b/pytest-playbook/playbook.py @@ -0,0 +1,98 @@ +"""A plugin for creating, validating, and reading a playbook. + +Use the `pytest_playbook_schema` hook to modify the schema dictionary +representing the data expected to be read and validated from a +`playbook.yaml` file, the path to which is provided by the user with +the command-line flag `--playbook`. + +This module's `playbook` attribute will hold the read and validated +data after all `pytest_configure` hooks have run. See +`pytest_playbook_schema` for example usage. + +Remember not to use `from playbook import playbook` because then the +attribute will not contain the shared data. Instead use `import +playbook` and reference `playbook.playbook`. + +""" + +from __future__ import annotations + +import typing +from pathlib import Path + +import yaml # TODO: Optionally load yaml. +from schema import Schema # type: ignore + +# See https://pyyaml.org/wiki/PyYAMLDocumentation +try: + from yaml import CLoader as Loader +except ImportError: + from yaml import Loader # type: ignore + +import pytest + +if typing.TYPE_CHECKING: + from typing import Any, Dict, Optional + + from _pytest.config import Config, PytestPluginManager + from _pytest.config.argparsing import Parser + +# TODO: I’m not a fan of this name. +playbook: Dict[Any, Any] = dict() + + +class Hooks: + """Provides the hook specifications.""" + + @pytest.hookspec + def pytest_playbook_schema(self, schema: Dict[Any, Any], config: Config) -> None: + """Update the Playbook's schema dict. + + The 'schema' is a mutable dict, and 'config' is optional. + Example usage:: + + import playbook + from schema import Schema + + def pytest_playbook_schema(schema): + schema["targets"] = Schema({"name": str, "platform": str, "cpus": int}) + + def pytest_sessionstart(session): + for target in playbook.playbook["targets"]: + print(target["name"]) + + """ + + +# Now provide the hook implementations. +def pytest_addhooks(pluginmanager: PytestPluginManager) -> None: + """Pytest hook to register our hooks.""" + pluginmanager.add_hookspecs(Hooks) + + +def pytest_addoption(parser: Parser, pluginmanager: PytestPluginManager) -> None: + """Pytest hook to add our CLI options.""" + group = parser.getgroup("playbook") + group.addoption("--playbook", type=Path, help="Path to playbook.") + + +@pytest.hookimpl(trylast=True) +def pytest_configure(config: Config) -> None: + """Pytest hook to configure our plugin. + + This is set to be tried last so that all other plugins have been + loaded and defined their `pytest_playbook_schema` hooks. + + """ + path: Optional[Path] = config.getoption("playbook") + if not path or not path.is_file(): + # TODO: Log an appropriate warning. + return + + schema: Dict[Any, Any] = dict() + config.hook.pytest_playbook_schema(schema=schema, config=config) + + global playbook + with open(path) as f: + # TODO: Handle ‘SchemaMissingKeyError’. + playbook = Schema(schema).validate(yaml.load(f, Loader=Loader)) diff --git a/pytest-playbook/pyproject.toml b/pytest-playbook/pyproject.toml index 787a88034d..4ad1234d06 100644 --- a/pytest-playbook/pyproject.toml +++ b/pytest-playbook/pyproject.toml @@ -5,6 +5,7 @@ description = "Pytest plugin for reading playbooks." authors = ["Andrew Schwartzmeyer "] license = "MIT" classifiers = ["Framework :: Pytest"] +packages = [{include = "playbook.py"}] [tool.poetry.dependencies] python = "^3.7" @@ -15,7 +16,7 @@ PyYAML = "^5.3.1" [tool.poetry.dev-dependencies] [tool.poetry.plugins] -pytest11 = {playbook = "pytest_playbook"} +pytest11 = {playbook = "playbook"} [build-system] requires = ["poetry-core>=1.0.0"] diff --git a/pytest-playbook/pytest_playbook.py b/pytest-playbook/pytest_playbook.py deleted file mode 100644 index bff379a2a1..0000000000 --- a/pytest-playbook/pytest_playbook.py +++ /dev/null @@ -1,66 +0,0 @@ -"""A plugin for creating, validating, and loading a playbook. - -After the last call to 'pytest_configure', 'playbook' will be -populated with the validated data. - -""" - -from __future__ import annotations - -import typing -from pathlib import Path - -import yaml # TODO: Optionally load yaml. -from schema import Schema # type: ignore - -# See https://pyyaml.org/wiki/PyYAMLDocumentation -try: - from yaml import CLoader as Loader -except ImportError: - from yaml import Loader # type: ignore - -import pytest - -if typing.TYPE_CHECKING: - from typing import Any, Dict, Optional - - from _pytest.config import Config, PytestPluginManager - from _pytest.config.argparsing import Parser - - -class Hooks: - """Class which contains our hooks.""" - - def pytest_playbook_schema(self, schema: Dict[Any, Any], config: Config) -> None: - """Update the Playbook's schema dict.""" - - -def pytest_addhooks(pluginmanager: PytestPluginManager) -> None: - """Pytest hook to register hooks.""" - pluginmanager.add_hookspecs(Hooks) - - -def pytest_addoption(parser: Parser, pluginmanager: PytestPluginManager) -> None: - """Pytest hook to add CLI options.""" - group = parser.getgroup("playbook") - group.addoption("--playbook", type=Path, help="Path to playbook.") - - -playbook: Dict[Any, Any] = dict() - - -@pytest.hookimpl(trylast=True) -def pytest_configure(config: Config) -> None: - """Pytest hook to configure each plugin.""" - path: Optional[Path] = config.getoption("playbook") - if not path or not path.is_file(): - # TODO: Log an appropriate warning. - return - - schema: Dict[Any, Any] = dict() - config.hook.pytest_playbook_schema(schema=schema, config=config) - - global playbook - with open(path) as f: - # TODO: Handle ‘SchemaMissingKeyError’. - playbook = Schema(schema).validate(yaml.load(f, Loader=Loader)) From 886aa12e77385a360aeab21147a7993b86e24191 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Thu, 19 Nov 2020 16:36:40 -0800 Subject: [PATCH 088/194] Create pytest-target plugin --- conftest.py | 135 +----- poetry.lock | 23 +- pyproject.toml | 3 +- pytest-target/poetry.lock | 511 ++++++++++++++++++++ pytest-target/pyproject.toml | 5 + pytest-target/target/__init__.py | 36 ++ azure.py => pytest-target/target/azure.py | 4 +- pytest-target/target/plugin.py | 143 ++++++ target.py => pytest-target/target/target.py | 10 +- 9 files changed, 728 insertions(+), 142 deletions(-) create mode 100644 pytest-target/poetry.lock create mode 100644 pytest-target/target/__init__.py rename azure.py => pytest-target/target/azure.py (98%) create mode 100644 pytest-target/target/plugin.py rename target.py => pytest-target/target/target.py (95%) diff --git a/conftest.py b/conftest.py index ad93ac227c..cba8c7ddf0 100644 --- a/conftest.py +++ b/conftest.py @@ -10,85 +10,19 @@ import playbook # See https://pypi.org/project/schema/ -from schema import Optional, Or, Schema, SchemaMissingKeyError # type: ignore +from schema import Optional, Schema, SchemaMissingKeyError # type: ignore -import azure # noqa import lisa -import pytest -from target import Target if typing.TYPE_CHECKING: - from typing import Any, Dict, Iterator, List, Type + from typing import Any, Dict, List from _pytest.config import Config from _pytest.config.argparsing import Parser - from _pytest.fixtures import SubRequest - from _pytest.python import Metafunc from pytest import Item, Session -pytest_plugins = ["playbook"] - - -@pytest.fixture(scope="session") -def pool(request: SubRequest) -> Iterator[List[Target]]: - """This fixture tracks all deployed target resources.""" - targets: List[Target] = [] - yield targets - for t in targets: - print(f"Created target: {t.features} / {t.parameters}") - if not request.config.getoption("keep_vms"): - t.delete() - - -@pytest.fixture -def target(pool: List[Target], request: SubRequest) -> Iterator[Target]: - """This fixture provides a connected target for each test. - - It is parametrized indirectly in 'pytest_generate_tests'. - - In this fixture we can check if any existing target matches all - the requirements. If so, we can re-use that target, and if not, we - can deallocate the currently running target and allocate a new - one. When all tests are finished, the pool fixture above will - delete all created VMs. Coupled with performing discrete - optimization in the test collection phase and ordering the tests - such that the test(s) with the lowest common denominator - requirements are executed first, we have the two-layer scheduling - as asked. - - However, this feels like putting the cart before the horse to me. - It would be much simpler in terms of design, implementation, and - usage that features are specified upfront when the targets are - specified. Then all this goes away, and tests are skipped when the - feature is missing, which also leaves users in full control of - their environments. - - """ - platform: Type[Target] = platforms[request.param["platform"]] - parameters: Dict[str, Any] = request.param["parameters"] - marker = request.node.get_closest_marker("lisa") - features = set(marker.kwargs["features"]) - - # TODO: If `t` is not already in use, deallocate the previous - # target, and ensure the tests have been sorted (and so grouped) - # by their requirements. - for t in pool: - # TODO: Implement full feature comparison, etc. and not just - # proof-of-concept string set comparison. - if ( - isinstance(t, platform) - and t.parameters == parameters - and t.features >= features - ): - yield t - break - else: - # TODO: Reimplement caching. - t = platform(parameters, features) - pool.append(t) - yield t - t.connection.close() +pytest_plugins = ["playbook", "target"] def pytest_addoption(parser: Parser) -> None: @@ -98,48 +32,11 @@ def pytest_addoption(parser: Parser) -> None: https://docs.pytest.org/en/latest/reference.html#pytest.hookspec.pytest_addoption """ - parser.addoption("--keep-vms", action="store_true", help="Keeps deployed VMs.") parser.addoption("--check", action="store_true", help="Run semantic analysis.") parser.addoption("--demo", action="store_true", help="Run in demo mode.") -platforms: Dict[str, Type[Target]] = dict() - - def pytest_playbook_schema(schema: Dict[Any, Any], config: Config) -> None: - """Describes the YAML schema for the playbook file. - - 'platforms' is a mapping of platform names (strings) to the - implementing subclass of 'Target' where each subclass defines its - own 'parameters' schema, 'deploy' and 'delete' methods, and other - platform-specific functionality. A 'Target' subclass need only be - defined in a file loaded by Pytest, so a 'contest.py' file works - just fine. No manual subclass of 'Target' where each subc ass - defines its own 'parameters' schema, 'deploy' and 'delete' - methods, and other platform-specific functionality. A 'Target' - subclass need only be defined in a file loaded by Pytest, so a 'c - - TODO: Add field annotations, friendly error reporting, automatic - case transformations, etc. - - """ - global platforms - platforms = {cls.__name__: cls for cls in Target.__subclasses__()} # type: ignore - target_schema = Schema( - { - "name": str, - "platform": Or(*[platform for platform in platforms.keys()]), - # TODO: What should we do when lacking parameters? Ideally we - # use the platform’s defaults from its own schema, but that - # means this value must be set, even if to an empty dict. - Optional("parameters", default=dict): Or( - *[cls.schema for cls in platforms.values()] - ), - } - ) - - default_target = {"name": "Default", "platform": "Local"} - criteria_schema = Schema( { # TODO: Validate that these strings are valid regular @@ -153,22 +50,9 @@ def pytest_playbook_schema(schema: Dict[Any, Any], config: Config) -> None: Optional("exclude", default=False): bool, } ) - - schema[Optional("targets", default=[default_target])] = [target_schema] schema[Optional("criteria", default=list)] = [criteria_schema] -targets: List[Dict[str, Any]] = [] -target_ids: List[str] = [] - - -def pytest_sessionstart(session: Session) -> None: - """Determines the targets based on the playbook.""" - for t in playbook.playbook.get("targets", []): - targets.append(t) - target_ids.append(t["name"]) - - def pytest_configure(config: Config) -> None: """Parse provided user inputs to setup configuration. @@ -199,19 +83,6 @@ def pytest_configure(config: Config) -> None: setattr(config.option, attr, value) -def pytest_generate_tests(metafunc: Metafunc) -> None: - """Parametrize the tests based on our inputs. - - Note that this hook is run for each test, so we do the file I/O in - 'pytest_configure' and save the results. - - https://docs.pytest.org/en/latest/reference.html#pytest.hookspec.pytest_generate_tests - - """ - if "target" in metafunc.fixturenames: - metafunc.parametrize("target", targets, True, target_ids) - - def pytest_collection_modifyitems( session: Session, config: Config, items: List[Item] ) -> None: diff --git a/poetry.lock b/poetry.lock index 9e9c0d70b6..7326958076 100644 --- a/poetry.lock +++ b/poetry.lock @@ -566,6 +566,26 @@ python-versions = ">=3.5" [package.dependencies] pytest = ">=5.0" +[[package]] +name = "pytest-target" +version = "0.1.0" +description = "Pytest plugin for remote target orchestration." +category = "main" +optional = false +python-versions = "^3.7" +develop = true + +[package.dependencies] +fabric = "^2.5.0" +invoke = "^1.4.1" +pytest = "^6.1.2" +pytest-playbook = "0.1.0" +tenacity = "^6.2.0" + +[package.source] +type = "directory" +url = "pytest-target" + [[package]] name = "pytest-timeout" version = "1.4.2" @@ -742,7 +762,7 @@ python-versions = ">=3.6" [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "6b1d924f0bc47c51bc55daa27f1c15c0dfb72b4902625f2577ef75c6d3ab992f" +content-hash = "86fba290150f934a1cd430ea0e34d7a847bf084fac1b3d5ca356e4f919707cef" [metadata.files] apipkg = [ @@ -1014,6 +1034,7 @@ pytest-rerunfailures = [ {file = "pytest-rerunfailures-9.1.1.tar.gz", hash = "sha256:1cb11a17fc121b3918414eb5eaf314ee325f2e693ac7cb3f6abf7560790827f2"}, {file = "pytest_rerunfailures-9.1.1-py3-none-any.whl", hash = "sha256:2eb7d0ad651761fbe80e064b0fd415cf6730cdbc53c16a145fd84b66143e609f"}, ] +pytest-target = [] pytest-timeout = [ {file = "pytest-timeout-1.4.2.tar.gz", hash = "sha256:20b3113cf6e4e80ce2d403b6fb56e9e1b871b510259206d40ff8d609f48bda76"}, {file = "pytest_timeout-1.4.2-py2.py3-none-any.whl", hash = "sha256:541d7aa19b9a6b4e475c759fd6073ef43d7cdc9a92d95644c260076eb257a063"}, diff --git a/pyproject.toml b/pyproject.toml index 3f72c622ff..9ed769c70a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,14 +9,13 @@ license = "MIT License" python = "^3.8" pytest = "^6.1.1" filelock = "^3.0.12" -fabric = "^2.5.0" pytest-timeout = "^1.4.2" pytest-html = "^2.1.1" -tenacity = "^6.2.0" pytest-rerunfailures = "^9.1.1" pytest-xdist = "^2.1.0" schema = "^0.7.3" pytest-playbook = {path = "pytest-playbook", develop = true} +pytest-target = {path = "pytest-target", develop = true} [tool.poetry.dev-dependencies] black = "^20.8b1" diff --git a/pytest-target/poetry.lock b/pytest-target/poetry.lock new file mode 100644 index 0000000000..c6dd5bfa58 --- /dev/null +++ b/pytest-target/poetry.lock @@ -0,0 +1,511 @@ +[[package]] +name = "atomicwrites" +version = "1.4.0" +description = "Atomic file writes." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[[package]] +name = "attrs" +version = "20.3.0" +description = "Classes Without Boilerplate" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[package.extras] +dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "furo", "sphinx", "pre-commit"] +docs = ["furo", "sphinx", "zope.interface"] +tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface"] +tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six"] + +[[package]] +name = "bcrypt" +version = "3.2.0" +description = "Modern password hashing for your software and your servers" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +cffi = ">=1.1" +six = ">=1.4.1" + +[package.extras] +tests = ["pytest (>=3.2.1,!=3.3.0)"] +typecheck = ["mypy"] + +[[package]] +name = "cffi" +version = "1.14.4" +description = "Foreign Function Interface for Python calling C code." +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +pycparser = "*" + +[[package]] +name = "colorama" +version = "0.4.4" +description = "Cross-platform colored terminal text." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[[package]] +name = "contextlib2" +version = "0.6.0.post1" +description = "Backports and enhancements for the contextlib module" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[[package]] +name = "cryptography" +version = "3.2.1" +description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +category = "main" +optional = false +python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*" + +[package.dependencies] +cffi = ">=1.8,<1.11.3 || >1.11.3" +six = ">=1.4.1" + +[package.extras] +docs = ["sphinx (>=1.6.5,!=1.8.0,!=3.1.0,!=3.1.1)", "sphinx-rtd-theme"] +docstest = ["doc8", "pyenchant (>=1.6.11)", "twine (>=1.12.0)", "sphinxcontrib-spelling (>=4.0.1)"] +pep8test = ["black", "flake8", "flake8-import-order", "pep8-naming"] +ssh = ["bcrypt (>=3.1.5)"] +test = ["pytest (>=3.6.0,!=3.9.0,!=3.9.1,!=3.9.2)", "pretend", "iso8601", "pytz", "hypothesis (>=1.11.4,!=3.79.2)"] + +[[package]] +name = "fabric" +version = "2.5.0" +description = "High level SSH command execution" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +invoke = ">=1.3,<2.0" +paramiko = ">=2.4" + +[package.extras] +pytest = ["mock (>=2.0.0,<3.0)", "pytest (>=3.2.5,<4.0)"] +testing = ["mock (>=2.0.0,<3.0)"] + +[[package]] +name = "importlib-metadata" +version = "3.1.1" +description = "Read metadata from Python packages" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +zipp = ">=0.5" + +[package.extras] +docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"] +testing = ["pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake8", "pytest-cov", "jaraco.test (>=3.2.0)", "packaging", "pep517", "pyfakefs", "flufl.flake8", "pytest-black (>=0.3.7)", "pytest-mypy", "importlib-resources (>=1.3)"] + +[[package]] +name = "iniconfig" +version = "1.1.1" +description = "iniconfig: brain-dead simple config-ini parsing" +category = "main" +optional = false +python-versions = "*" + +[[package]] +name = "invoke" +version = "1.4.1" +description = "Pythonic task execution" +category = "main" +optional = false +python-versions = "*" + +[[package]] +name = "packaging" +version = "20.7" +description = "Core utilities for Python packages" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[package.dependencies] +pyparsing = ">=2.0.2" + +[[package]] +name = "paramiko" +version = "2.7.2" +description = "SSH2 protocol library" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +bcrypt = ">=3.1.3" +cryptography = ">=2.5" +pynacl = ">=1.0.1" + +[package.extras] +all = ["pyasn1 (>=0.1.7)", "pynacl (>=1.0.1)", "bcrypt (>=3.1.3)", "invoke (>=1.3)", "gssapi (>=1.4.1)", "pywin32 (>=2.1.8)"] +ed25519 = ["pynacl (>=1.0.1)", "bcrypt (>=3.1.3)"] +gssapi = ["pyasn1 (>=0.1.7)", "gssapi (>=1.4.1)", "pywin32 (>=2.1.8)"] +invoke = ["invoke (>=1.3)"] + +[[package]] +name = "pluggy" +version = "0.13.1" +description = "plugin and hook calling mechanisms for python" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[package.dependencies] +importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} + +[package.extras] +dev = ["pre-commit", "tox"] + +[[package]] +name = "py" +version = "1.9.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[[package]] +name = "pycparser" +version = "2.20" +description = "C parser in Python" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[[package]] +name = "pynacl" +version = "1.4.0" +description = "Python binding to the Networking and Cryptography (NaCl) library" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[package.dependencies] +cffi = ">=1.4.1" +six = "*" + +[package.extras] +docs = ["sphinx (>=1.6.5)", "sphinx-rtd-theme"] +tests = ["pytest (>=3.2.1,!=3.3.0)", "hypothesis (>=3.27.0)"] + +[[package]] +name = "pyparsing" +version = "2.4.7" +description = "Python parsing module" +category = "main" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" + +[[package]] +name = "pytest" +version = "6.1.2" +description = "pytest: simple powerful testing with Python" +category = "main" +optional = false +python-versions = ">=3.5" + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=17.4.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<1.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +checkqa_mypy = ["mypy (==0.780)"] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "pytest-playbook" +version = "0.1.0" +description = "Pytest plugin for reading playbooks." +category = "main" +optional = false +python-versions = "^3.7" +develop = true + +[package.dependencies] +pytest = "^6.1.2" +PyYAML = "^5.3.1" +schema = "^0.7.3" + +[package.source] +type = "directory" +url = "../pytest-playbook" + +[[package]] +name = "pyyaml" +version = "5.3.1" +description = "YAML parser and emitter for Python" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[[package]] +name = "schema" +version = "0.7.3" +description = "Simple data validation library" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +contextlib2 = ">=0.5.5" + +[[package]] +name = "six" +version = "1.15.0" +description = "Python 2 and 3 compatibility utilities" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" + +[[package]] +name = "tenacity" +version = "6.2.0" +description = "Retry code until it succeeds" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +six = ">=1.9.0" + +[package.extras] +doc = ["reno", "sphinx", "tornado (>=4.5)"] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +category = "main" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" + +[[package]] +name = "zipp" +version = "3.4.0" +description = "Backport of pathlib-compatible object wrapper for zip files" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.extras] +docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"] +testing = ["pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake8", "pytest-cov", "jaraco.test (>=3.2.0)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy"] + +[metadata] +lock-version = "1.1" +python-versions = "^3.7" +content-hash = "d0e0d3cfad95b0ead424722cf46a2ad67bc1140e5d7c08ad2821855e58086c88" + +[metadata.files] +atomicwrites = [ + {file = "atomicwrites-1.4.0-py2.py3-none-any.whl", hash = "sha256:6d1784dea7c0c8d4a5172b6c620f40b6e4cbfdf96d783691f2e1302a7b88e197"}, + {file = "atomicwrites-1.4.0.tar.gz", hash = "sha256:ae70396ad1a434f9c7046fd2dd196fc04b12f9e91ffb859164193be8b6168a7a"}, +] +attrs = [ + {file = "attrs-20.3.0-py2.py3-none-any.whl", hash = "sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6"}, + {file = "attrs-20.3.0.tar.gz", hash = "sha256:832aa3cde19744e49938b91fea06d69ecb9e649c93ba974535d08ad92164f700"}, +] +bcrypt = [ + {file = "bcrypt-3.2.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:c95d4cbebffafcdd28bd28bb4e25b31c50f6da605c81ffd9ad8a3d1b2ab7b1b6"}, + {file = "bcrypt-3.2.0-cp36-abi3-manylinux1_x86_64.whl", hash = "sha256:63d4e3ff96188e5898779b6057878fecf3f11cfe6ec3b313ea09955d587ec7a7"}, + {file = "bcrypt-3.2.0-cp36-abi3-manylinux2010_x86_64.whl", hash = "sha256:cd1ea2ff3038509ea95f687256c46b79f5fc382ad0aa3664d200047546d511d1"}, + {file = "bcrypt-3.2.0-cp36-abi3-manylinux2014_aarch64.whl", hash = "sha256:cdcdcb3972027f83fe24a48b1e90ea4b584d35f1cc279d76de6fc4b13376239d"}, + {file = "bcrypt-3.2.0-cp36-abi3-win32.whl", hash = "sha256:a67fb841b35c28a59cebed05fbd3e80eea26e6d75851f0574a9273c80f3e9b55"}, + {file = "bcrypt-3.2.0-cp36-abi3-win_amd64.whl", hash = "sha256:81fec756feff5b6818ea7ab031205e1d323d8943d237303baca2c5f9c7846f34"}, + {file = "bcrypt-3.2.0.tar.gz", hash = "sha256:5b93c1726e50a93a033c36e5ca7fdcd29a5c7395af50a6892f5d9e7c6cfbfb29"}, +] +cffi = [ + {file = "cffi-1.14.4-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:ebb253464a5d0482b191274f1c8bf00e33f7e0b9c66405fbffc61ed2c839c775"}, + {file = "cffi-1.14.4-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:2c24d61263f511551f740d1a065eb0212db1dbbbbd241db758f5244281590c06"}, + {file = "cffi-1.14.4-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:9f7a31251289b2ab6d4012f6e83e58bc3b96bd151f5b5262467f4bb6b34a7c26"}, + {file = "cffi-1.14.4-cp27-cp27m-win32.whl", hash = "sha256:5cf4be6c304ad0b6602f5c4e90e2f59b47653ac1ed9c662ed379fe48a8f26b0c"}, + {file = "cffi-1.14.4-cp27-cp27m-win_amd64.whl", hash = "sha256:f60567825f791c6f8a592f3c6e3bd93dd2934e3f9dac189308426bd76b00ef3b"}, + {file = "cffi-1.14.4-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:c6332685306b6417a91b1ff9fae889b3ba65c2292d64bd9245c093b1b284809d"}, + {file = "cffi-1.14.4-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:d9efd8b7a3ef378dd61a1e77367f1924375befc2eba06168b6ebfa903a5e59ca"}, + {file = "cffi-1.14.4-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:51a8b381b16ddd370178a65360ebe15fbc1c71cf6f584613a7ea08bfad946698"}, + {file = "cffi-1.14.4-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:1d2c4994f515e5b485fd6d3a73d05526aa0fcf248eb135996b088d25dfa1865b"}, + {file = "cffi-1.14.4-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:af5c59122a011049aad5dd87424b8e65a80e4a6477419c0c1015f73fb5ea0293"}, + {file = "cffi-1.14.4-cp35-cp35m-win32.whl", hash = "sha256:594234691ac0e9b770aee9fcdb8fa02c22e43e5c619456efd0d6c2bf276f3eb2"}, + {file = "cffi-1.14.4-cp35-cp35m-win_amd64.whl", hash = "sha256:64081b3f8f6f3c3de6191ec89d7dc6c86a8a43911f7ecb422c60e90c70be41c7"}, + {file = "cffi-1.14.4-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:f803eaa94c2fcda012c047e62bc7a51b0bdabda1cad7a92a522694ea2d76e49f"}, + {file = "cffi-1.14.4-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:105abaf8a6075dc96c1fe5ae7aae073f4696f2905fde6aeada4c9d2926752362"}, + {file = "cffi-1.14.4-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:0638c3ae1a0edfb77c6765d487fee624d2b1ee1bdfeffc1f0b58c64d149e7eec"}, + {file = "cffi-1.14.4-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:7c6b1dece89874d9541fc974917b631406233ea0440d0bdfbb8e03bf39a49b3b"}, + {file = "cffi-1.14.4-cp36-cp36m-win32.whl", hash = "sha256:155136b51fd733fa94e1c2ea5211dcd4c8879869008fc811648f16541bf99668"}, + {file = "cffi-1.14.4-cp36-cp36m-win_amd64.whl", hash = "sha256:6bc25fc545a6b3d57b5f8618e59fc13d3a3a68431e8ca5fd4c13241cd70d0009"}, + {file = "cffi-1.14.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a7711edca4dcef1a75257b50a2fbfe92a65187c47dab5a0f1b9b332c5919a3fb"}, + {file = "cffi-1.14.4-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:00e28066507bfc3fe865a31f325c8391a1ac2916219340f87dfad602c3e48e5d"}, + {file = "cffi-1.14.4-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:798caa2a2384b1cbe8a2a139d80734c9db54f9cc155c99d7cc92441a23871c03"}, + {file = "cffi-1.14.4-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:a5ed8c05548b54b998b9498753fb9cadbfd92ee88e884641377d8a8b291bcc01"}, + {file = "cffi-1.14.4-cp37-cp37m-win32.whl", hash = "sha256:00a1ba5e2e95684448de9b89888ccd02c98d512064b4cb987d48f4b40aa0421e"}, + {file = "cffi-1.14.4-cp37-cp37m-win_amd64.whl", hash = "sha256:9cc46bc107224ff5b6d04369e7c595acb700c3613ad7bcf2e2012f62ece80c35"}, + {file = "cffi-1.14.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:df5169c4396adc04f9b0a05f13c074df878b6052430e03f50e68adf3a57aa28d"}, + {file = "cffi-1.14.4-cp38-cp38-manylinux1_i686.whl", hash = "sha256:9ffb888f19d54a4d4dfd4b3f29bc2c16aa4972f1c2ab9c4ab09b8ab8685b9c2b"}, + {file = "cffi-1.14.4-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:8d6603078baf4e11edc4168a514c5ce5b3ba6e3e9c374298cb88437957960a53"}, + {file = "cffi-1.14.4-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:d5ff0621c88ce83a28a10d2ce719b2ee85635e85c515f12bac99a95306da4b2e"}, + {file = "cffi-1.14.4-cp38-cp38-win32.whl", hash = "sha256:b4e248d1087abf9f4c10f3c398896c87ce82a9856494a7155823eb45a892395d"}, + {file = "cffi-1.14.4-cp38-cp38-win_amd64.whl", hash = "sha256:ec80dc47f54e6e9a78181ce05feb71a0353854cc26999db963695f950b5fb375"}, + {file = "cffi-1.14.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:840793c68105fe031f34d6a086eaea153a0cd5c491cde82a74b420edd0a2b909"}, + {file = "cffi-1.14.4-cp39-cp39-manylinux1_i686.whl", hash = "sha256:b18e0a9ef57d2b41f5c68beefa32317d286c3d6ac0484efd10d6e07491bb95dd"}, + {file = "cffi-1.14.4-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:045d792900a75e8b1e1b0ab6787dd733a8190ffcf80e8c8ceb2fb10a29ff238a"}, + {file = "cffi-1.14.4-cp39-cp39-win32.whl", hash = "sha256:ba4e9e0ae13fc41c6b23299545e5ef73055213e466bd107953e4a013a5ddd7e3"}, + {file = "cffi-1.14.4-cp39-cp39-win_amd64.whl", hash = "sha256:f032b34669220030f905152045dfa27741ce1a6db3324a5bc0b96b6c7420c87b"}, + {file = "cffi-1.14.4.tar.gz", hash = "sha256:1a465cbe98a7fd391d47dce4b8f7e5b921e6cd805ef421d04f5f66ba8f06086c"}, +] +colorama = [ + {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"}, + {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"}, +] +contextlib2 = [ + {file = "contextlib2-0.6.0.post1-py2.py3-none-any.whl", hash = "sha256:3355078a159fbb44ee60ea80abd0d87b80b78c248643b49aa6d94673b413609b"}, + {file = "contextlib2-0.6.0.post1.tar.gz", hash = "sha256:01f490098c18b19d2bd5bb5dc445b2054d2fa97f09a4280ba2c5f3c394c8162e"}, +] +cryptography = [ + {file = "cryptography-3.2.1-cp27-cp27m-macosx_10_10_x86_64.whl", hash = "sha256:6dc59630ecce8c1f558277ceb212c751d6730bd12c80ea96b4ac65637c4f55e7"}, + {file = "cryptography-3.2.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:75e8e6684cf0034f6bf2a97095cb95f81537b12b36a8fedf06e73050bb171c2d"}, + {file = "cryptography-3.2.1-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:4e7268a0ca14536fecfdf2b00297d4e407da904718658c1ff1961c713f90fd33"}, + {file = "cryptography-3.2.1-cp27-cp27m-win32.whl", hash = "sha256:7117319b44ed1842c617d0a452383a5a052ec6aa726dfbaffa8b94c910444297"}, + {file = "cryptography-3.2.1-cp27-cp27m-win_amd64.whl", hash = "sha256:a733671100cd26d816eed39507e585c156e4498293a907029969234e5e634bc4"}, + {file = "cryptography-3.2.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:a75f306a16d9f9afebfbedc41c8c2351d8e61e818ba6b4c40815e2b5740bb6b8"}, + {file = "cryptography-3.2.1-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:5849d59358547bf789ee7e0d7a9036b2d29e9a4ddf1ce5e06bb45634f995c53e"}, + {file = "cryptography-3.2.1-cp35-abi3-macosx_10_10_x86_64.whl", hash = "sha256:bd717aa029217b8ef94a7d21632a3bb5a4e7218a4513d2521c2a2fd63011e98b"}, + {file = "cryptography-3.2.1-cp35-abi3-manylinux1_x86_64.whl", hash = "sha256:efe15aca4f64f3a7ea0c09c87826490e50ed166ce67368a68f315ea0807a20df"}, + {file = "cryptography-3.2.1-cp35-abi3-manylinux2010_x86_64.whl", hash = "sha256:32434673d8505b42c0de4de86da8c1620651abd24afe91ae0335597683ed1b77"}, + {file = "cryptography-3.2.1-cp35-abi3-manylinux2014_aarch64.whl", hash = "sha256:7b8d9d8d3a9bd240f453342981f765346c87ade811519f98664519696f8e6ab7"}, + {file = "cryptography-3.2.1-cp35-cp35m-win32.whl", hash = "sha256:d3545829ab42a66b84a9aaabf216a4dce7f16dbc76eb69be5c302ed6b8f4a29b"}, + {file = "cryptography-3.2.1-cp35-cp35m-win_amd64.whl", hash = "sha256:a4e27ed0b2504195f855b52052eadcc9795c59909c9d84314c5408687f933fc7"}, + {file = "cryptography-3.2.1-cp36-abi3-win32.whl", hash = "sha256:13b88a0bd044b4eae1ef40e265d006e34dbcde0c2f1e15eb9896501b2d8f6c6f"}, + {file = "cryptography-3.2.1-cp36-abi3-win_amd64.whl", hash = "sha256:07ca431b788249af92764e3be9a488aa1d39a0bc3be313d826bbec690417e538"}, + {file = "cryptography-3.2.1-cp36-cp36m-win32.whl", hash = "sha256:a035a10686532b0587d58a606004aa20ad895c60c4d029afa245802347fab57b"}, + {file = "cryptography-3.2.1-cp36-cp36m-win_amd64.whl", hash = "sha256:d26a2557d8f9122f9bf445fc7034242f4375bd4e95ecda007667540270965b13"}, + {file = "cryptography-3.2.1-cp37-cp37m-win32.whl", hash = "sha256:545a8550782dda68f8cdc75a6e3bf252017aa8f75f19f5a9ca940772fc0cb56e"}, + {file = "cryptography-3.2.1-cp37-cp37m-win_amd64.whl", hash = "sha256:55d0b896631412b6f0c7de56e12eb3e261ac347fbaa5d5e705291a9016e5f8cb"}, + {file = "cryptography-3.2.1-cp38-cp38-win32.whl", hash = "sha256:3cd75a683b15576cfc822c7c5742b3276e50b21a06672dc3a800a2d5da4ecd1b"}, + {file = "cryptography-3.2.1-cp38-cp38-win_amd64.whl", hash = "sha256:d25cecbac20713a7c3bc544372d42d8eafa89799f492a43b79e1dfd650484851"}, + {file = "cryptography-3.2.1.tar.gz", hash = "sha256:d3d5e10be0cf2a12214ddee45c6bd203dab435e3d83b4560c03066eda600bfe3"}, +] +fabric = [ + {file = "fabric-2.5.0-py2.py3-none-any.whl", hash = "sha256:160331934ea60036604928e792fa8e9f813266b098ef5562aa82b88527740389"}, + {file = "fabric-2.5.0.tar.gz", hash = "sha256:24842d7d51556adcabd885ac3cf5e1df73fc622a1708bf3667bf5927576cdfa6"}, +] +importlib-metadata = [ + {file = "importlib_metadata-3.1.1-py3-none-any.whl", hash = "sha256:6112e21359ef8f344e7178aa5b72dc6e62b38b0d008e6d3cb212c5b84df72013"}, + {file = "importlib_metadata-3.1.1.tar.gz", hash = "sha256:b0c2d3b226157ae4517d9625decf63591461c66b3a808c2666d538946519d170"}, +] +iniconfig = [ + {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, + {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, +] +invoke = [ + {file = "invoke-1.4.1-py2-none-any.whl", hash = "sha256:93e12876d88130c8e0d7fd6618dd5387d6b36da55ad541481dfa5e001656f134"}, + {file = "invoke-1.4.1-py3-none-any.whl", hash = "sha256:87b3ef9d72a1667e104f89b159eaf8a514dbf2f3576885b2bbdefe74c3fb2132"}, + {file = "invoke-1.4.1.tar.gz", hash = "sha256:de3f23bfe669e3db1085789fd859eb8ca8e0c5d9c20811e2407fa042e8a5e15d"}, +] +packaging = [ + {file = "packaging-20.7-py2.py3-none-any.whl", hash = "sha256:eb41423378682dadb7166144a4926e443093863024de508ca5c9737d6bc08376"}, + {file = "packaging-20.7.tar.gz", hash = "sha256:05af3bb85d320377db281cf254ab050e1a7ebcbf5410685a9a407e18a1f81236"}, +] +paramiko = [ + {file = "paramiko-2.7.2-py2.py3-none-any.whl", hash = "sha256:4f3e316fef2ac628b05097a637af35685183111d4bc1b5979bd397c2ab7b5898"}, + {file = "paramiko-2.7.2.tar.gz", hash = "sha256:7f36f4ba2c0d81d219f4595e35f70d56cc94f9ac40a6acdf51d6ca210ce65035"}, +] +pluggy = [ + {file = "pluggy-0.13.1-py2.py3-none-any.whl", hash = "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"}, + {file = "pluggy-0.13.1.tar.gz", hash = "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0"}, +] +py = [ + {file = "py-1.9.0-py2.py3-none-any.whl", hash = "sha256:366389d1db726cd2fcfc79732e75410e5fe4d31db13692115529d34069a043c2"}, + {file = "py-1.9.0.tar.gz", hash = "sha256:9ca6883ce56b4e8da7e79ac18787889fa5206c79dcc67fb065376cd2fe03f342"}, +] +pycparser = [ + {file = "pycparser-2.20-py2.py3-none-any.whl", hash = "sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705"}, + {file = "pycparser-2.20.tar.gz", hash = "sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0"}, +] +pynacl = [ + {file = "PyNaCl-1.4.0-cp27-cp27m-macosx_10_10_x86_64.whl", hash = "sha256:ea6841bc3a76fa4942ce00f3bda7d436fda21e2d91602b9e21b7ca9ecab8f3ff"}, + {file = "PyNaCl-1.4.0-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:d452a6746f0a7e11121e64625109bc4468fc3100452817001dbe018bb8b08514"}, + {file = "PyNaCl-1.4.0-cp27-cp27m-win32.whl", hash = "sha256:2fe0fc5a2480361dcaf4e6e7cea00e078fcda07ba45f811b167e3f99e8cff574"}, + {file = "PyNaCl-1.4.0-cp27-cp27m-win_amd64.whl", hash = "sha256:f8851ab9041756003119368c1e6cd0b9c631f46d686b3904b18c0139f4419f80"}, + {file = "PyNaCl-1.4.0-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:7757ae33dae81c300487591c68790dfb5145c7d03324000433d9a2c141f82af7"}, + {file = "PyNaCl-1.4.0-cp35-abi3-macosx_10_10_x86_64.whl", hash = "sha256:757250ddb3bff1eecd7e41e65f7f833a8405fede0194319f87899690624f2122"}, + {file = "PyNaCl-1.4.0-cp35-abi3-manylinux1_x86_64.whl", hash = "sha256:30f9b96db44e09b3304f9ea95079b1b7316b2b4f3744fe3aaecccd95d547063d"}, + {file = "PyNaCl-1.4.0-cp35-abi3-win32.whl", hash = "sha256:4e10569f8cbed81cb7526ae137049759d2a8d57726d52c1a000a3ce366779634"}, + {file = "PyNaCl-1.4.0-cp35-abi3-win_amd64.whl", hash = "sha256:c914f78da4953b33d4685e3cdc7ce63401247a21425c16a39760e282075ac4a6"}, + {file = "PyNaCl-1.4.0-cp35-cp35m-win32.whl", hash = "sha256:06cbb4d9b2c4bd3c8dc0d267416aaed79906e7b33f114ddbf0911969794b1cc4"}, + {file = "PyNaCl-1.4.0-cp35-cp35m-win_amd64.whl", hash = "sha256:511d269ee845037b95c9781aa702f90ccc36036f95d0f31373a6a79bd8242e25"}, + {file = "PyNaCl-1.4.0-cp36-cp36m-win32.whl", hash = "sha256:11335f09060af52c97137d4ac54285bcb7df0cef29014a1a4efe64ac065434c4"}, + {file = "PyNaCl-1.4.0-cp36-cp36m-win_amd64.whl", hash = "sha256:cd401ccbc2a249a47a3a1724c2918fcd04be1f7b54eb2a5a71ff915db0ac51c6"}, + {file = "PyNaCl-1.4.0-cp37-cp37m-win32.whl", hash = "sha256:8122ba5f2a2169ca5da936b2e5a511740ffb73979381b4229d9188f6dcb22f1f"}, + {file = "PyNaCl-1.4.0-cp37-cp37m-win_amd64.whl", hash = "sha256:537a7ccbea22905a0ab36ea58577b39d1fa9b1884869d173b5cf111f006f689f"}, + {file = "PyNaCl-1.4.0-cp38-cp38-win32.whl", hash = "sha256:9c4a7ea4fb81536c1b1f5cc44d54a296f96ae78c1ebd2311bd0b60be45a48d96"}, + {file = "PyNaCl-1.4.0-cp38-cp38-win_amd64.whl", hash = "sha256:7c6092102219f59ff29788860ccb021e80fffd953920c4a8653889c029b2d420"}, + {file = "PyNaCl-1.4.0.tar.gz", hash = "sha256:54e9a2c849c742006516ad56a88f5c74bf2ce92c9f67435187c3c5953b346505"}, +] +pyparsing = [ + {file = "pyparsing-2.4.7-py2.py3-none-any.whl", hash = "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"}, + {file = "pyparsing-2.4.7.tar.gz", hash = "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1"}, +] +pytest = [ + {file = "pytest-6.1.2-py3-none-any.whl", hash = "sha256:4288fed0d9153d9646bfcdf0c0428197dba1ecb27a33bb6e031d002fa88653fe"}, + {file = "pytest-6.1.2.tar.gz", hash = "sha256:c0a7e94a8cdbc5422a51ccdad8e6f1024795939cc89159a0ae7f0b316ad3823e"}, +] +pytest-playbook = [] +pyyaml = [ + {file = "PyYAML-5.3.1-cp27-cp27m-win32.whl", hash = "sha256:74809a57b329d6cc0fdccee6318f44b9b8649961fa73144a98735b0aaf029f1f"}, + {file = "PyYAML-5.3.1-cp27-cp27m-win_amd64.whl", hash = "sha256:240097ff019d7c70a4922b6869d8a86407758333f02203e0fc6ff79c5dcede76"}, + {file = "PyYAML-5.3.1-cp35-cp35m-win32.whl", hash = "sha256:4f4b913ca1a7319b33cfb1369e91e50354d6f07a135f3b901aca02aa95940bd2"}, + {file = "PyYAML-5.3.1-cp35-cp35m-win_amd64.whl", hash = "sha256:cc8955cfbfc7a115fa81d85284ee61147059a753344bc51098f3ccd69b0d7e0c"}, + {file = "PyYAML-5.3.1-cp36-cp36m-win32.whl", hash = "sha256:7739fc0fa8205b3ee8808aea45e968bc90082c10aef6ea95e855e10abf4a37b2"}, + {file = "PyYAML-5.3.1-cp36-cp36m-win_amd64.whl", hash = "sha256:69f00dca373f240f842b2931fb2c7e14ddbacd1397d57157a9b005a6a9942648"}, + {file = "PyYAML-5.3.1-cp37-cp37m-win32.whl", hash = "sha256:d13155f591e6fcc1ec3b30685d50bf0711574e2c0dfffd7644babf8b5102ca1a"}, + {file = "PyYAML-5.3.1-cp37-cp37m-win_amd64.whl", hash = "sha256:73f099454b799e05e5ab51423c7bcf361c58d3206fa7b0d555426b1f4d9a3eaf"}, + {file = "PyYAML-5.3.1-cp38-cp38-win32.whl", hash = "sha256:06a0d7ba600ce0b2d2fe2e78453a470b5a6e000a985dd4a4e54e436cc36b0e97"}, + {file = "PyYAML-5.3.1-cp38-cp38-win_amd64.whl", hash = "sha256:95f71d2af0ff4227885f7a6605c37fd53d3a106fcab511b8860ecca9fcf400ee"}, + {file = "PyYAML-5.3.1-cp39-cp39-win32.whl", hash = "sha256:ad9c67312c84def58f3c04504727ca879cb0013b2517c85a9a253f0cb6380c0a"}, + {file = "PyYAML-5.3.1-cp39-cp39-win_amd64.whl", hash = "sha256:6034f55dab5fea9e53f436aa68fa3ace2634918e8b5994d82f3621c04ff5ed2e"}, + {file = "PyYAML-5.3.1.tar.gz", hash = "sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d"}, +] +schema = [ + {file = "schema-0.7.3-py2.py3-none-any.whl", hash = "sha256:c331438b60f634cab5664ab720d3083cc444f924d55269530c36b33e3354276f"}, + {file = "schema-0.7.3.tar.gz", hash = "sha256:4cf529318cfd1e844ecbe02f41f7e5aa027463e7403666a52746f31f04f47a5e"}, +] +six = [ + {file = "six-1.15.0-py2.py3-none-any.whl", hash = "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"}, + {file = "six-1.15.0.tar.gz", hash = "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259"}, +] +tenacity = [ + {file = "tenacity-6.2.0-py2.py3-none-any.whl", hash = "sha256:5a5d3dcd46381abe8b4f82b5736b8726fd3160c6c7161f53f8af7f1eb9b82173"}, + {file = "tenacity-6.2.0.tar.gz", hash = "sha256:29ae90e7faf488a8628432154bb34ace1cca58244c6ea399fd33f066ac71339a"}, +] +toml = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] +zipp = [ + {file = "zipp-3.4.0-py3-none-any.whl", hash = "sha256:102c24ef8f171fd729d46599845e95c7ab894a4cf45f5de11a44cc7444fb1108"}, + {file = "zipp-3.4.0.tar.gz", hash = "sha256:ed5eee1974372595f9e416cc7bbeeb12335201d8081ca8a0743c954d4446e5cb"}, +] diff --git a/pytest-target/pyproject.toml b/pytest-target/pyproject.toml index a5a4beb56b..bb6a272586 100644 --- a/pytest-target/pyproject.toml +++ b/pytest-target/pyproject.toml @@ -5,6 +5,7 @@ description = "Pytest plugin for remote target orchestration." authors = ["Andrew Schwartzmeyer "] license = "MIT" classifiers = ["Framework :: Pytest"] +packages = [{include = "target"}] [tool.poetry.dependencies] python = "^3.7" @@ -12,9 +13,13 @@ pytest = "^6.1.2" fabric = "^2.5.0" invoke = "^1.4.1" tenacity = "^6.2.0" +pytest-playbook = {path = "../pytest-playbook", develop = true} [tool.poetry.dev-dependencies] +[tool.poetry.plugins] +pytest11 = {target = "target.plugin"} + [build-system] requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" diff --git a/pytest-target/target/__init__.py b/pytest-target/target/__init__.py new file mode 100644 index 0000000000..40f5a87afb --- /dev/null +++ b/pytest-target/target/__init__.py @@ -0,0 +1,36 @@ +"""A plugin for creating, using, and managing remote targets. + +The abstract base `Target` class provides an interface for adding +platform-specific support through sub-classes. A usable reference +implementation is the `Azure` class. A class for testing on the local +system is the `Local` class. Sub-classes can be implemented in a +`conftest.py` file and will be found automatically. + +Tests can request access to a target through the function-scoped +`target` Pytest fixture, which returns an instance based on the +targets listed in a `playbook.yaml` file. The fixture is parameterized +across the list of provided targets. For example: + + targets: + - name: Debian + platform: Azure + image: Debian:debian-10:10:latest + - name: Ubuntu + platform: Azure + image: Canonical:UbuntuServer:18.04-LTS:latest + - name: OpenSUSE + platform: Azure + image: SUSE:openSUSE-Leap:42.3:latest + +Will run all selected tests against each target. The `pool` fixture is +session-scoped and used by the `target` fixture to efficiently re-use +deployed targets. + +""" +# Provide common types in the package's namespace. +from target.azure import Azure +from target.plugin import pool, target +from target.target import Local, Target + +# NOTE: This is mostly to avoid “imported but not used.” +__all__ = ["Azure", "Target", "Local", "pool", "target"] diff --git a/azure.py b/pytest-target/target/azure.py similarity index 98% rename from azure.py rename to pytest-target/target/azure.py index aadb2b5eda..51de417b58 100644 --- a/azure.py +++ b/pytest-target/target/azure.py @@ -1,3 +1,4 @@ +"""Provides an `Azure(Target)` implementation using the Azure CLI.""" from __future__ import annotations import json @@ -6,10 +7,9 @@ from invoke.runners import Result # type: ignore from schema import Optional, Schema # type: ignore +from target.target import Target from tenacity import retry, stop_after_attempt, wait_exponential # type: ignore -from target import Target - if typing.TYPE_CHECKING: from typing import Any diff --git a/pytest-target/target/plugin.py b/pytest-target/target/plugin.py new file mode 100644 index 0000000000..b0852efff8 --- /dev/null +++ b/pytest-target/target/plugin.py @@ -0,0 +1,143 @@ +"""Provides and parameterizes the `pool` and `target` fixtures.""" +from __future__ import annotations + +import typing +from uuid import uuid4 + +import playbook +import pytest + +# See https://pypi.org/project/schema/ +from schema import Optional, Or, Schema # type: ignore +from target.target import Target + +if typing.TYPE_CHECKING: + from typing import Any, Dict, Iterator, List, Type + + from _pytest.config.argparsing import Parser + from _pytest.fixtures import SubRequest + from _pytest.python import Metafunc + + +def pytest_addoption(parser: Parser) -> None: + """Pytest hook to add our CLI options.""" + group = parser.getgroup("target") + group.addoption("--keep-vms", action="store_true", help="Keeps deployed VMs.") + + +platforms: Dict[str, Type[Target]] = dict() + + +def pytest_playbook_schema(schema: Dict[Any, Any]) -> None: + """pytest-playbook hook to update the playbook schema. + + The `platforms` global is a mapping of platform names (strings) to + the implementing subclasses of `Target` where each subclass + defines its own `parameters` schema, `deploy` and `delete` + methods, and other platform-specific functionality. A `Target` + subclass need only be defined in a file loaded by Pytest, so a + `contest.py` file works just fine. + + TODO: Add field annotations, friendly error reporting, automatic + case transformations, etc. + + """ + global platforms + platforms = {cls.__name__: cls for cls in Target.__subclasses__()} # type: ignore + target_schema = Schema( + { + "name": str, + "platform": Or(*[platform for platform in platforms.keys()]), + # TODO: What should we do when lacking parameters? Ideally we + # use the platform’s defaults from its own schema, but that + # means this value must be set, even if to an empty dict. + Optional("parameters", default=dict): Or( + *[cls.schema for cls in platforms.values()] + ), + } + ) + default_targets = [{"name": "Default", "platform": "Local"}] + schema[Optional("targets", default=default_targets)] = [target_schema] + + +@pytest.fixture(scope="session") +def pool(request: SubRequest) -> Iterator[List[Target]]: + """This fixture tracks all deployed target resources.""" + targets: List[Target] = [] + yield targets + for t in targets: + print(f"Created target: {t.features} / {t.parameters}") + if not request.config.getoption("keep_vms"): + t.delete() + + +@pytest.fixture +def target(pool: List[Target], request: SubRequest) -> Iterator[Target]: + """This fixture provides a connected target for each test. + + It is parametrized indirectly in `pytest_generate_tests`. + + In this fixture we can check if any existing target matches all + the requirements. If so, we can re-use that target, and if not, we + can deallocate the currently running target and allocate a new + one. When all tests are finished, the pool fixture above will + delete all created VMs. Coupled with performing discrete + optimization in the test collection phase and ordering the tests + such that the test(s) with the lowest common denominator + requirements are executed first, we have the two-layer scheduling + as asked. + + However, this feels like putting the cart before the horse to me. + It would be much simpler in terms of design, implementation, and + usage that features are specified upfront when the targets are + specified. Then all this goes away, and tests are skipped when the + feature is missing, which also leaves users in full control of + their environments. + + """ + platform: Type[Target] = platforms[request.param["platform"]] + parameters: Dict[str, Any] = request.param["parameters"] + marker = request.node.get_closest_marker("lisa") + features = set(marker.kwargs["features"]) + + # TODO: If `t` is not already in use, deallocate the previous + # target, and ensure the tests have been sorted (and so grouped) + # by their requirements. + for t in pool: + # TODO: Implement full feature comparison, etc. and not just + # proof-of-concept string set comparison. + if ( + isinstance(t, platform) + and t.parameters == parameters + and t.features >= features + ): + yield t + break + else: + # TODO: Reimplement caching. + t = platform(f"pytest-{uuid4()}", parameters, features) + pool.append(t) + yield t + t.connection.close() + + +targets: List[Dict[str, Any]] = [] +target_ids: List[str] = [] + + +def pytest_sessionstart() -> None: + """Gather the `targets` from the playbook.""" + for target in playbook.playbook.get("targets", []): + targets.append(target) + target_ids.append(target["name"]) + + +def pytest_generate_tests(metafunc: Metafunc) -> None: + """Indirectly parametrize the `target` fixture based on the playbook. + + This hook is run for each test, so we gather the `targets` in + `pytest_sessionstart`. + + """ + if "target" in metafunc.fixturenames: + metafunc.parametrize("target", targets, True, target_ids) diff --git a/target.py b/pytest-target/target/target.py similarity index 95% rename from target.py rename to pytest-target/target/target.py index 2608039cd5..b4f727d7e7 100644 --- a/target.py +++ b/pytest-target/target/target.py @@ -1,10 +1,10 @@ +"""Provides the abstract base `Target` class.""" from __future__ import annotations import platform import typing from abc import ABC, abstractmethod from io import BytesIO -from uuid import uuid4 import fabric # type: ignore import invoke # type: ignore @@ -30,21 +30,21 @@ class Target(ABC): def __init__( self, + name: str, parameters: Mapping[str, str], features: Set[str], - name: str = f"pytest-{uuid4()}", ): - """If not given a name, generates one uniquely. + """Requires a unique name. Name is a unique identifier for the group of associated resources. Features is a list of requirements such as sriov, - rdma, gpu, xdp. + rdma, gpu, xdp. Parameters are used by `deploy()`. """ + self.name = name # TODO: Do we need to re-validate the parameters here? self.parameters = parameters self.features = features - self.name = name # TODO: Review this thoroughly as currently it depends on # parameters which is side-effecty. From bb476b34ae4044748f44b3504f17a428a9a0bf5e Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Thu, 19 Nov 2020 16:51:02 -0800 Subject: [PATCH 089/194] Fix tests for target plugin refactor --- selftests/conftest.py | 1 - selftests/setup_plan/test_plan_A.py | 7 ++++++- selftests/setup_plan/test_plan_B.py | 7 ++++++- selftests/setup_plan/test_plan_C.py | 7 ++++++- selftests/test_basic.py | 10 ++++++++-- testsuites/test_lis.py | 2 +- testsuites/test_smoke.py | 8 +++++++- testsuites/test_xdp.py | 6 +++++- 8 files changed, 39 insertions(+), 9 deletions(-) diff --git a/selftests/conftest.py b/selftests/conftest.py index cc7fd59999..d24231012d 100644 --- a/selftests/conftest.py +++ b/selftests/conftest.py @@ -1,5 +1,4 @@ from schema import Schema # type: ignore - from target import Target diff --git a/selftests/setup_plan/test_plan_A.py b/selftests/setup_plan/test_plan_A.py index 5a4e049dd6..6d27842635 100644 --- a/selftests/setup_plan/test_plan_A.py +++ b/selftests/setup_plan/test_plan_A.py @@ -1,7 +1,12 @@ +from __future__ import annotations + import functools +import typing + +if typing.TYPE_CHECKING: + from target import Target import lisa -from target import Target LISA = functools.partial( lisa.LISA, platform="Custom", category="Functional", area="self-test", priority=1 diff --git a/selftests/setup_plan/test_plan_B.py b/selftests/setup_plan/test_plan_B.py index 0d89896300..bce121d133 100644 --- a/selftests/setup_plan/test_plan_B.py +++ b/selftests/setup_plan/test_plan_B.py @@ -1,7 +1,12 @@ +from __future__ import annotations + import functools +import typing + +if typing.TYPE_CHECKING: + from target import Target import lisa -from target import Target LISA = functools.partial( lisa.LISA, platform="Custom", category="Functional", area="self-test", priority=1 diff --git a/selftests/setup_plan/test_plan_C.py b/selftests/setup_plan/test_plan_C.py index efc579fe10..35c1684f1e 100644 --- a/selftests/setup_plan/test_plan_C.py +++ b/selftests/setup_plan/test_plan_C.py @@ -1,7 +1,12 @@ +from __future__ import annotations + import functools +import typing + +if typing.TYPE_CHECKING: + from target import Target import lisa -from target import Target LISA = functools.partial( lisa.LISA, platform="Custom", category="Functional", area="self-test", priority=1 diff --git a/selftests/test_basic.py b/selftests/test_basic.py index 5b3012d8a4..f592b28cba 100644 --- a/selftests/test_basic.py +++ b/selftests/test_basic.py @@ -1,9 +1,15 @@ """These tests are meant to run in a CI environment.""" +from __future__ import annotations + +import typing + +if typing.TYPE_CHECKING: + from target import Local + from lisa import LISA -from target import Target @LISA(platform="Local", category="Functional", area="self-test", priority=1) -def test_basic(target: Target) -> None: +def test_basic(target: Local) -> None: """Basic test which creates a Node connection to 'localhost'.""" target.local("echo Hello World") diff --git a/testsuites/test_lis.py b/testsuites/test_lis.py index a35e6fe94e..e68db2708c 100644 --- a/testsuites/test_lis.py +++ b/testsuites/test_lis.py @@ -4,7 +4,7 @@ import typing if typing.TYPE_CHECKING: - from azure import Azure + from target import Azure from lisa import LINUX_SCRIPTS, LISA diff --git a/testsuites/test_smoke.py b/testsuites/test_smoke.py index cfc911707f..c8e0b2e424 100644 --- a/testsuites/test_smoke.py +++ b/testsuites/test_smoke.py @@ -1,4 +1,11 @@ """Runs a 'smoke' test for an Azure Linux VM deployment.""" +from __future__ import annotations + +import typing + +if typing.TYPE_CHECKING: + from target import Azure + import logging import socket import time @@ -6,7 +13,6 @@ from invoke.runners import CommandTimedOut, Result, UnexpectedExit # type: ignore from paramiko import SSHException # type: ignore -from azure import Azure from lisa import LISA diff --git a/testsuites/test_xdp.py b/testsuites/test_xdp.py index b526db1f0f..ffbd5a9417 100644 --- a/testsuites/test_xdp.py +++ b/testsuites/test_xdp.py @@ -1,8 +1,12 @@ """Runs 'FunctionalTests-XDP.xml' using Pytest.""" +from __future__ import annotations +import typing + +if typing.TYPE_CHECKING: + from target import Azure import pytest -from azure import Azure from lisa import LINUX_SCRIPTS, LISA From a72559c001ffb0f4bee97bd349e3eda809623163 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Thu, 19 Nov 2020 17:15:47 -0800 Subject: [PATCH 090/194] Make `Target.schema()` an abstract classmethod --- pytest-target/target/azure.py | 25 +++++++++++-------------- pytest-target/target/plugin.py | 2 +- pytest-target/target/target.py | 24 +++++++++++++++--------- selftests/conftest.py | 8 +++----- 4 files changed, 30 insertions(+), 29 deletions(-) diff --git a/pytest-target/target/azure.py b/pytest-target/target/azure.py index 51de417b58..3d02cc256b 100644 --- a/pytest-target/target/azure.py +++ b/pytest-target/target/azure.py @@ -20,20 +20,17 @@ class Azure(Target): # Custom instance attribute(s). internal_address: str - # @property - # @classmethod - # def schema(cls) -> Schema: - # return - - schema: Schema = Schema( - { - # TODO: Maybe validate as URN or path etc. - "image": str, - Optional("sku", default="Standard_DS1_v2"): str, - Optional("location", default="eastus2"): str, - Optional("networking", default=""): str, - } - ) + @classmethod + def schema(cls) -> Schema: + return Schema( + { + # TODO: Maybe validate as URN or path etc. + "image": str, + Optional("sku", default="Standard_DS1_v2"): str, + Optional("location", default="eastus2"): str, + Optional("networking", default=""): str, + } + ) # A class attribute because it’s defined. az_ok = False diff --git a/pytest-target/target/plugin.py b/pytest-target/target/plugin.py index b0852efff8..1d1e510d42 100644 --- a/pytest-target/target/plugin.py +++ b/pytest-target/target/plugin.py @@ -52,7 +52,7 @@ def pytest_playbook_schema(schema: Dict[Any, Any]) -> None: # use the platform’s defaults from its own schema, but that # means this value must be set, even if to an empty dict. Optional("parameters", default=dict): Or( - *[cls.schema for cls in platforms.values()] + *[cls.schema() for cls in platforms.values()] ), } ) diff --git a/pytest-target/target/target.py b/pytest-target/target/target.py index b4f727d7e7..e584b445ad 100644 --- a/pytest-target/target/target.py +++ b/pytest-target/target/target.py @@ -59,15 +59,19 @@ def __init__( self.host, config=fabric.Config(overrides=config), inline_ssh_env=True ) - # TODO: Use an abstract class property to ensure this is defined. - schema: Schema = Schema(None) + # NOTE: This ought to be a property, but the combination of + # @classmethod, @property, and @abstractmethod is only supported + # in Python 3.9 and up. + @classmethod + @abstractmethod + def schema(cls) -> Schema: + """Must return a schema for expected instance parameters. + + TODO: This schema is used for each instance. We may want to + define platform-level shared schemata too. - # @property - # @classmethod - # @abstractmethod - # def schema(cls) -> Schema: - # """Must return the parameters schema for setup.""" - # ... + """ + ... @abstractmethod def deploy(self) -> str: @@ -116,7 +120,9 @@ def cat(self, path: str) -> str: class Local(Target): - schema: Schema = Schema(None) + @classmethod + def schema(cls) -> Schema: + return Schema(None) def deploy(self) -> str: return "localhost" diff --git a/selftests/conftest.py b/selftests/conftest.py index d24231012d..cbcfcd1b5e 100644 --- a/selftests/conftest.py +++ b/selftests/conftest.py @@ -3,11 +3,9 @@ class Custom(Target): - schema: Schema = Schema(None) - # @property - # @classmethod - # def schema(cls) -> Schema: - # return Schema() + @classmethod + def schema(cls) -> Schema: + return Schema(None) def deploy(self) -> str: return "localhost" From 81718d36de54cf0a0bc72d9f98a49c164a91f29f Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Thu, 19 Nov 2020 19:09:47 -0800 Subject: [PATCH 091/194] Handle schema, yaml, and OS errors in playbook --- pytest-playbook/playbook.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/pytest-playbook/playbook.py b/pytest-playbook/playbook.py index a0784dfc09..f2a840d315 100644 --- a/pytest-playbook/playbook.py +++ b/pytest-playbook/playbook.py @@ -21,7 +21,7 @@ from pathlib import Path import yaml # TODO: Optionally load yaml. -from schema import Schema # type: ignore +from schema import Schema, SchemaMissingKeyError # type: ignore # See https://pyyaml.org/wiki/PyYAMLDocumentation try: @@ -93,6 +93,9 @@ def pytest_configure(config: Config) -> None: config.hook.pytest_playbook_schema(schema=schema, config=config) global playbook - with open(path) as f: - # TODO: Handle ‘SchemaMissingKeyError’. - playbook = Schema(schema).validate(yaml.load(f, Loader=Loader)) + try: + with open(path) as f: + data = yaml.load(f, Loader=Loader) + playbook = Schema(schema).validate(data) + except (yaml.YAMLError, SchemaMissingKeyError, OSError) as e: + pytest.exit(f"Error loading playbook '{path}': {e}") From 3b44e89732a516febb430a59f06843fea694c583 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Thu, 19 Nov 2020 19:19:55 -0800 Subject: [PATCH 092/194] =?UTF-8?q?Rename=20top-level=20package=20to=20?= =?UTF-8?q?=E2=80=98LISA=E2=80=99=20so=20we=20can=20make=20pytest-lisa?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- poetry.lock | 22 +- pyproject.toml | 8 +- pytest-lisa/poetry.lock | 531 +++++++++++++++++++++++++++++++++++++ pytest-lisa/pyproject.toml | 7 + 4 files changed, 562 insertions(+), 6 deletions(-) create mode 100644 pytest-lisa/poetry.lock diff --git a/poetry.lock b/poetry.lock index 7326958076..ce576ff018 100644 --- a/poetry.lock +++ b/poetry.lock @@ -513,6 +513,25 @@ python-versions = ">=3.6" pytest = ">=5.0" pytest-metadata = "*" +[[package]] +name = "pytest-lisa" +version = "0.1.0" +description = "Pytest plugin for Linux Integration Services Automation (LISA)." +category = "main" +optional = false +python-versions = "^3.7" +develop = true + +[package.dependencies] +pytest = "^6.1.2" +pytest-playbook = "0.1.0" +pytest-target = "0.1.0" +schema = "^0.7.3" + +[package.source] +type = "directory" +url = "pytest-lisa" + [[package]] name = "pytest-metadata" version = "1.10.0" @@ -762,7 +781,7 @@ python-versions = ">=3.6" [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "86fba290150f934a1cd430ea0e34d7a847bf084fac1b3d5ca356e4f919707cef" +content-hash = "709c5ad614869a72de95d29708421298cd94048f6c5a4f2a2d00e83c8abccf37" [metadata.files] apipkg = [ @@ -1021,6 +1040,7 @@ pytest-html = [ {file = "pytest-html-2.1.1.tar.gz", hash = "sha256:6a4ac391e105e391208e3eb9bd294a60dd336447fd8e1acddff3a6de7f4e57c5"}, {file = "pytest_html-2.1.1-py2.py3-none-any.whl", hash = "sha256:9e4817e8be8ddde62e8653c8934d0f296b605da3d2277a052f762c56a8b32df2"}, ] +pytest-lisa = [] pytest-metadata = [ {file = "pytest-metadata-1.10.0.tar.gz", hash = "sha256:b7e6e0a45adacb17a03a97bf7a2ef60cc1f4e172bcce9732ce5e814191932315"}, {file = "pytest_metadata-1.10.0-py2.py3-none-any.whl", hash = "sha256:fcbcc5781aee450107c620c79c57e50796b6777b82b3c504be9cbc3017201169"}, diff --git a/pyproject.toml b/pyproject.toml index 9ed769c70a..1fb99c7d30 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [tool.poetry] -name = "pytest-lisa" +name = "LISA" version = "0.1.0" -description = "LISA plugin for pytest" +description = "Linux Integration Services Automation (LISA)" authors = ["Andrew Schwartzmeyer "] license = "MIT License" @@ -13,9 +13,7 @@ pytest-timeout = "^1.4.2" pytest-html = "^2.1.1" pytest-rerunfailures = "^9.1.1" pytest-xdist = "^2.1.0" -schema = "^0.7.3" -pytest-playbook = {path = "pytest-playbook", develop = true} -pytest-target = {path = "pytest-target", develop = true} +pytest-lisa = {path = "pytest-lisa", develop = true} [tool.poetry.dev-dependencies] black = "^20.8b1" diff --git a/pytest-lisa/poetry.lock b/pytest-lisa/poetry.lock new file mode 100644 index 0000000000..9c1579df82 --- /dev/null +++ b/pytest-lisa/poetry.lock @@ -0,0 +1,531 @@ +[[package]] +name = "atomicwrites" +version = "1.4.0" +description = "Atomic file writes." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[[package]] +name = "attrs" +version = "20.3.0" +description = "Classes Without Boilerplate" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[package.extras] +dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "furo", "sphinx", "pre-commit"] +docs = ["furo", "sphinx", "zope.interface"] +tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface"] +tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six"] + +[[package]] +name = "bcrypt" +version = "3.2.0" +description = "Modern password hashing for your software and your servers" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +cffi = ">=1.1" +six = ">=1.4.1" + +[package.extras] +tests = ["pytest (>=3.2.1,!=3.3.0)"] +typecheck = ["mypy"] + +[[package]] +name = "cffi" +version = "1.14.3" +description = "Foreign Function Interface for Python calling C code." +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +pycparser = "*" + +[[package]] +name = "colorama" +version = "0.4.4" +description = "Cross-platform colored terminal text." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[[package]] +name = "contextlib2" +version = "0.6.0.post1" +description = "Backports and enhancements for the contextlib module" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[[package]] +name = "cryptography" +version = "3.2.1" +description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +category = "main" +optional = false +python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*" + +[package.dependencies] +cffi = ">=1.8,<1.11.3 || >1.11.3" +six = ">=1.4.1" + +[package.extras] +docs = ["sphinx (>=1.6.5,!=1.8.0,!=3.1.0,!=3.1.1)", "sphinx-rtd-theme"] +docstest = ["doc8", "pyenchant (>=1.6.11)", "twine (>=1.12.0)", "sphinxcontrib-spelling (>=4.0.1)"] +pep8test = ["black", "flake8", "flake8-import-order", "pep8-naming"] +ssh = ["bcrypt (>=3.1.5)"] +test = ["pytest (>=3.6.0,!=3.9.0,!=3.9.1,!=3.9.2)", "pretend", "iso8601", "pytz", "hypothesis (>=1.11.4,!=3.79.2)"] + +[[package]] +name = "fabric" +version = "2.5.0" +description = "High level SSH command execution" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +invoke = ">=1.3,<2.0" +paramiko = ">=2.4" + +[package.extras] +pytest = ["mock (>=2.0.0,<3.0)", "pytest (>=3.2.5,<4.0)"] +testing = ["mock (>=2.0.0,<3.0)"] + +[[package]] +name = "importlib-metadata" +version = "2.0.0" +description = "Read metadata from Python packages" +category = "main" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" + +[package.dependencies] +zipp = ">=0.5" + +[package.extras] +docs = ["sphinx", "rst.linker"] +testing = ["packaging", "pep517", "importlib-resources (>=1.3)"] + +[[package]] +name = "iniconfig" +version = "1.1.1" +description = "iniconfig: brain-dead simple config-ini parsing" +category = "main" +optional = false +python-versions = "*" + +[[package]] +name = "invoke" +version = "1.4.1" +description = "Pythonic task execution" +category = "main" +optional = false +python-versions = "*" + +[[package]] +name = "packaging" +version = "20.4" +description = "Core utilities for Python packages" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[package.dependencies] +pyparsing = ">=2.0.2" +six = "*" + +[[package]] +name = "paramiko" +version = "2.7.2" +description = "SSH2 protocol library" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +bcrypt = ">=3.1.3" +cryptography = ">=2.5" +pynacl = ">=1.0.1" + +[package.extras] +all = ["pyasn1 (>=0.1.7)", "pynacl (>=1.0.1)", "bcrypt (>=3.1.3)", "invoke (>=1.3)", "gssapi (>=1.4.1)", "pywin32 (>=2.1.8)"] +ed25519 = ["pynacl (>=1.0.1)", "bcrypt (>=3.1.3)"] +gssapi = ["pyasn1 (>=0.1.7)", "gssapi (>=1.4.1)", "pywin32 (>=2.1.8)"] +invoke = ["invoke (>=1.3)"] + +[[package]] +name = "pluggy" +version = "0.13.1" +description = "plugin and hook calling mechanisms for python" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[package.dependencies] +importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} + +[package.extras] +dev = ["pre-commit", "tox"] + +[[package]] +name = "py" +version = "1.9.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[[package]] +name = "pycparser" +version = "2.20" +description = "C parser in Python" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[[package]] +name = "pynacl" +version = "1.4.0" +description = "Python binding to the Networking and Cryptography (NaCl) library" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[package.dependencies] +cffi = ">=1.4.1" +six = "*" + +[package.extras] +docs = ["sphinx (>=1.6.5)", "sphinx-rtd-theme"] +tests = ["pytest (>=3.2.1,!=3.3.0)", "hypothesis (>=3.27.0)"] + +[[package]] +name = "pyparsing" +version = "2.4.7" +description = "Python parsing module" +category = "main" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" + +[[package]] +name = "pytest" +version = "6.1.2" +description = "pytest: simple powerful testing with Python" +category = "main" +optional = false +python-versions = ">=3.5" + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=17.4.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<1.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +checkqa_mypy = ["mypy (==0.780)"] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "pytest-playbook" +version = "0.1.0" +description = "Pytest plugin for reading playbooks." +category = "main" +optional = false +python-versions = "^3.7" +develop = true + +[package.dependencies] +pytest = "^6.1.2" +PyYAML = "^5.3.1" +schema = "^0.7.3" + +[package.source] +type = "directory" +url = "../pytest-playbook" + +[[package]] +name = "pytest-target" +version = "0.1.0" +description = "Pytest plugin for remote target orchestration." +category = "main" +optional = false +python-versions = "^3.7" +develop = true + +[package.dependencies] +fabric = "^2.5.0" +invoke = "^1.4.1" +pytest = "^6.1.2" +pytest-playbook = "0.1.0" +tenacity = "^6.2.0" + +[package.source] +type = "directory" +url = "../pytest-target" + +[[package]] +name = "pyyaml" +version = "5.3.1" +description = "YAML parser and emitter for Python" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[[package]] +name = "schema" +version = "0.7.3" +description = "Simple data validation library" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +contextlib2 = ">=0.5.5" + +[[package]] +name = "six" +version = "1.15.0" +description = "Python 2 and 3 compatibility utilities" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" + +[[package]] +name = "tenacity" +version = "6.2.0" +description = "Retry code until it succeeds" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +six = ">=1.9.0" + +[package.extras] +doc = ["reno", "sphinx", "tornado (>=4.5)"] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +category = "main" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" + +[[package]] +name = "zipp" +version = "3.4.0" +description = "Backport of pathlib-compatible object wrapper for zip files" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.extras] +docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"] +testing = ["pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake8", "pytest-cov", "jaraco.test (>=3.2.0)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy"] + +[metadata] +lock-version = "1.1" +python-versions = "^3.7" +content-hash = "bc55352e00a00a309afc084b0584068709a437c282532207c1f0d1ea623e7588" + +[metadata.files] +atomicwrites = [ + {file = "atomicwrites-1.4.0-py2.py3-none-any.whl", hash = "sha256:6d1784dea7c0c8d4a5172b6c620f40b6e4cbfdf96d783691f2e1302a7b88e197"}, + {file = "atomicwrites-1.4.0.tar.gz", hash = "sha256:ae70396ad1a434f9c7046fd2dd196fc04b12f9e91ffb859164193be8b6168a7a"}, +] +attrs = [ + {file = "attrs-20.3.0-py2.py3-none-any.whl", hash = "sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6"}, + {file = "attrs-20.3.0.tar.gz", hash = "sha256:832aa3cde19744e49938b91fea06d69ecb9e649c93ba974535d08ad92164f700"}, +] +bcrypt = [ + {file = "bcrypt-3.2.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:c95d4cbebffafcdd28bd28bb4e25b31c50f6da605c81ffd9ad8a3d1b2ab7b1b6"}, + {file = "bcrypt-3.2.0-cp36-abi3-manylinux1_x86_64.whl", hash = "sha256:63d4e3ff96188e5898779b6057878fecf3f11cfe6ec3b313ea09955d587ec7a7"}, + {file = "bcrypt-3.2.0-cp36-abi3-manylinux2010_x86_64.whl", hash = "sha256:cd1ea2ff3038509ea95f687256c46b79f5fc382ad0aa3664d200047546d511d1"}, + {file = "bcrypt-3.2.0-cp36-abi3-manylinux2014_aarch64.whl", hash = "sha256:cdcdcb3972027f83fe24a48b1e90ea4b584d35f1cc279d76de6fc4b13376239d"}, + {file = "bcrypt-3.2.0-cp36-abi3-win32.whl", hash = "sha256:a67fb841b35c28a59cebed05fbd3e80eea26e6d75851f0574a9273c80f3e9b55"}, + {file = "bcrypt-3.2.0-cp36-abi3-win_amd64.whl", hash = "sha256:81fec756feff5b6818ea7ab031205e1d323d8943d237303baca2c5f9c7846f34"}, + {file = "bcrypt-3.2.0.tar.gz", hash = "sha256:5b93c1726e50a93a033c36e5ca7fdcd29a5c7395af50a6892f5d9e7c6cfbfb29"}, +] +cffi = [ + {file = "cffi-1.14.3-2-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:3eeeb0405fd145e714f7633a5173318bd88d8bbfc3dd0a5751f8c4f70ae629bc"}, + {file = "cffi-1.14.3-2-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:cb763ceceae04803adcc4e2d80d611ef201c73da32d8f2722e9d0ab0c7f10768"}, + {file = "cffi-1.14.3-2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:44f60519595eaca110f248e5017363d751b12782a6f2bd6a7041cba275215f5d"}, + {file = "cffi-1.14.3-2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c53af463f4a40de78c58b8b2710ade243c81cbca641e34debf3396a9640d6ec1"}, + {file = "cffi-1.14.3-2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:33c6cdc071ba5cd6d96769c8969a0531be2d08c2628a0143a10a7dcffa9719ca"}, + {file = "cffi-1.14.3-2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c11579638288e53fc94ad60022ff1b67865363e730ee41ad5e6f0a17188b327a"}, + {file = "cffi-1.14.3-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:3cb3e1b9ec43256c4e0f8d2837267a70b0e1ca8c4f456685508ae6106b1f504c"}, + {file = "cffi-1.14.3-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:f0620511387790860b249b9241c2f13c3a80e21a73e0b861a2df24e9d6f56730"}, + {file = "cffi-1.14.3-cp27-cp27m-win32.whl", hash = "sha256:005f2bfe11b6745d726dbb07ace4d53f057de66e336ff92d61b8c7e9c8f4777d"}, + {file = "cffi-1.14.3-cp27-cp27m-win_amd64.whl", hash = "sha256:2f9674623ca39c9ebe38afa3da402e9326c245f0f5ceff0623dccdac15023e05"}, + {file = "cffi-1.14.3-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:09e96138280241bd355cd585148dec04dbbedb4f46128f340d696eaafc82dd7b"}, + {file = "cffi-1.14.3-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:3363e77a6176afb8823b6e06db78c46dbc4c7813b00a41300a4873b6ba63b171"}, + {file = "cffi-1.14.3-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:0ef488305fdce2580c8b2708f22d7785ae222d9825d3094ab073e22e93dfe51f"}, + {file = "cffi-1.14.3-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:0b1ad452cc824665ddc682400b62c9e4f5b64736a2ba99110712fdee5f2505c4"}, + {file = "cffi-1.14.3-cp35-cp35m-win32.whl", hash = "sha256:85ba797e1de5b48aa5a8427b6ba62cf69607c18c5d4eb747604b7302f1ec382d"}, + {file = "cffi-1.14.3-cp35-cp35m-win_amd64.whl", hash = "sha256:e66399cf0fc07de4dce4f588fc25bfe84a6d1285cc544e67987d22663393926d"}, + {file = "cffi-1.14.3-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:15f351bed09897fbda218e4db5a3d5c06328862f6198d4fb385f3e14e19decb3"}, + {file = "cffi-1.14.3-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:4d7c26bfc1ea9f92084a1d75e11999e97b62d63128bcc90c3624d07813c52808"}, + {file = "cffi-1.14.3-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:23e5d2040367322824605bc29ae8ee9175200b92cb5483ac7d466927a9b3d537"}, + {file = "cffi-1.14.3-cp36-cp36m-win32.whl", hash = "sha256:a624fae282e81ad2e4871bdb767e2c914d0539708c0f078b5b355258293c98b0"}, + {file = "cffi-1.14.3-cp36-cp36m-win_amd64.whl", hash = "sha256:de31b5164d44ef4943db155b3e8e17929707cac1e5bd2f363e67a56e3af4af6e"}, + {file = "cffi-1.14.3-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:f92cdecb618e5fa4658aeb97d5eb3d2f47aa94ac6477c6daf0f306c5a3b9e6b1"}, + {file = "cffi-1.14.3-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:22399ff4870fb4c7ef19fff6eeb20a8bbf15571913c181c78cb361024d574579"}, + {file = "cffi-1.14.3-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:f4eae045e6ab2bb54ca279733fe4eb85f1effda392666308250714e01907f394"}, + {file = "cffi-1.14.3-cp37-cp37m-win32.whl", hash = "sha256:b0358e6fefc74a16f745afa366acc89f979040e0cbc4eec55ab26ad1f6a9bfbc"}, + {file = "cffi-1.14.3-cp37-cp37m-win_amd64.whl", hash = "sha256:6642f15ad963b5092d65aed022d033c77763515fdc07095208f15d3563003869"}, + {file = "cffi-1.14.3-cp38-cp38-manylinux1_i686.whl", hash = "sha256:2791f68edc5749024b4722500e86303a10d342527e1e3bcac47f35fbd25b764e"}, + {file = "cffi-1.14.3-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:529c4ed2e10437c205f38f3691a68be66c39197d01062618c55f74294a4a4828"}, + {file = "cffi-1.14.3-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:8f0f1e499e4000c4c347a124fa6a27d37608ced4fe9f7d45070563b7c4c370c9"}, + {file = "cffi-1.14.3-cp38-cp38-win32.whl", hash = "sha256:3b8eaf915ddc0709779889c472e553f0d3e8b7bdf62dab764c8921b09bf94522"}, + {file = "cffi-1.14.3-cp38-cp38-win_amd64.whl", hash = "sha256:bbd2f4dfee1079f76943767fce837ade3087b578aeb9f69aec7857d5bf25db15"}, + {file = "cffi-1.14.3-cp39-cp39-manylinux1_i686.whl", hash = "sha256:cc75f58cdaf043fe6a7a6c04b3b5a0e694c6a9e24050967747251fb80d7bce0d"}, + {file = "cffi-1.14.3-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:bf39a9e19ce7298f1bd6a9758fa99707e9e5b1ebe5e90f2c3913a47bc548747c"}, + {file = "cffi-1.14.3-cp39-cp39-win32.whl", hash = "sha256:d80998ed59176e8cba74028762fbd9b9153b9afc71ea118e63bbf5d4d0f9552b"}, + {file = "cffi-1.14.3-cp39-cp39-win_amd64.whl", hash = "sha256:c150eaa3dadbb2b5339675b88d4573c1be3cb6f2c33a6c83387e10cc0bf05bd3"}, + {file = "cffi-1.14.3.tar.gz", hash = "sha256:f92f789e4f9241cd262ad7a555ca2c648a98178a953af117ef7fad46aa1d5591"}, +] +colorama = [ + {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"}, + {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"}, +] +contextlib2 = [ + {file = "contextlib2-0.6.0.post1-py2.py3-none-any.whl", hash = "sha256:3355078a159fbb44ee60ea80abd0d87b80b78c248643b49aa6d94673b413609b"}, + {file = "contextlib2-0.6.0.post1.tar.gz", hash = "sha256:01f490098c18b19d2bd5bb5dc445b2054d2fa97f09a4280ba2c5f3c394c8162e"}, +] +cryptography = [ + {file = "cryptography-3.2.1-cp27-cp27m-macosx_10_10_x86_64.whl", hash = "sha256:6dc59630ecce8c1f558277ceb212c751d6730bd12c80ea96b4ac65637c4f55e7"}, + {file = "cryptography-3.2.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:75e8e6684cf0034f6bf2a97095cb95f81537b12b36a8fedf06e73050bb171c2d"}, + {file = "cryptography-3.2.1-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:4e7268a0ca14536fecfdf2b00297d4e407da904718658c1ff1961c713f90fd33"}, + {file = "cryptography-3.2.1-cp27-cp27m-win32.whl", hash = "sha256:7117319b44ed1842c617d0a452383a5a052ec6aa726dfbaffa8b94c910444297"}, + {file = "cryptography-3.2.1-cp27-cp27m-win_amd64.whl", hash = "sha256:a733671100cd26d816eed39507e585c156e4498293a907029969234e5e634bc4"}, + {file = "cryptography-3.2.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:a75f306a16d9f9afebfbedc41c8c2351d8e61e818ba6b4c40815e2b5740bb6b8"}, + {file = "cryptography-3.2.1-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:5849d59358547bf789ee7e0d7a9036b2d29e9a4ddf1ce5e06bb45634f995c53e"}, + {file = "cryptography-3.2.1-cp35-abi3-macosx_10_10_x86_64.whl", hash = "sha256:bd717aa029217b8ef94a7d21632a3bb5a4e7218a4513d2521c2a2fd63011e98b"}, + {file = "cryptography-3.2.1-cp35-abi3-manylinux1_x86_64.whl", hash = "sha256:efe15aca4f64f3a7ea0c09c87826490e50ed166ce67368a68f315ea0807a20df"}, + {file = "cryptography-3.2.1-cp35-abi3-manylinux2010_x86_64.whl", hash = "sha256:32434673d8505b42c0de4de86da8c1620651abd24afe91ae0335597683ed1b77"}, + {file = "cryptography-3.2.1-cp35-abi3-manylinux2014_aarch64.whl", hash = "sha256:7b8d9d8d3a9bd240f453342981f765346c87ade811519f98664519696f8e6ab7"}, + {file = "cryptography-3.2.1-cp35-cp35m-win32.whl", hash = "sha256:d3545829ab42a66b84a9aaabf216a4dce7f16dbc76eb69be5c302ed6b8f4a29b"}, + {file = "cryptography-3.2.1-cp35-cp35m-win_amd64.whl", hash = "sha256:a4e27ed0b2504195f855b52052eadcc9795c59909c9d84314c5408687f933fc7"}, + {file = "cryptography-3.2.1-cp36-abi3-win32.whl", hash = "sha256:13b88a0bd044b4eae1ef40e265d006e34dbcde0c2f1e15eb9896501b2d8f6c6f"}, + {file = "cryptography-3.2.1-cp36-abi3-win_amd64.whl", hash = "sha256:07ca431b788249af92764e3be9a488aa1d39a0bc3be313d826bbec690417e538"}, + {file = "cryptography-3.2.1-cp36-cp36m-win32.whl", hash = "sha256:a035a10686532b0587d58a606004aa20ad895c60c4d029afa245802347fab57b"}, + {file = "cryptography-3.2.1-cp36-cp36m-win_amd64.whl", hash = "sha256:d26a2557d8f9122f9bf445fc7034242f4375bd4e95ecda007667540270965b13"}, + {file = "cryptography-3.2.1-cp37-cp37m-win32.whl", hash = "sha256:545a8550782dda68f8cdc75a6e3bf252017aa8f75f19f5a9ca940772fc0cb56e"}, + {file = "cryptography-3.2.1-cp37-cp37m-win_amd64.whl", hash = "sha256:55d0b896631412b6f0c7de56e12eb3e261ac347fbaa5d5e705291a9016e5f8cb"}, + {file = "cryptography-3.2.1-cp38-cp38-win32.whl", hash = "sha256:3cd75a683b15576cfc822c7c5742b3276e50b21a06672dc3a800a2d5da4ecd1b"}, + {file = "cryptography-3.2.1-cp38-cp38-win_amd64.whl", hash = "sha256:d25cecbac20713a7c3bc544372d42d8eafa89799f492a43b79e1dfd650484851"}, + {file = "cryptography-3.2.1.tar.gz", hash = "sha256:d3d5e10be0cf2a12214ddee45c6bd203dab435e3d83b4560c03066eda600bfe3"}, +] +fabric = [ + {file = "fabric-2.5.0-py2.py3-none-any.whl", hash = "sha256:160331934ea60036604928e792fa8e9f813266b098ef5562aa82b88527740389"}, + {file = "fabric-2.5.0.tar.gz", hash = "sha256:24842d7d51556adcabd885ac3cf5e1df73fc622a1708bf3667bf5927576cdfa6"}, +] +importlib-metadata = [ + {file = "importlib_metadata-2.0.0-py2.py3-none-any.whl", hash = "sha256:cefa1a2f919b866c5beb7c9f7b0ebb4061f30a8a9bf16d609b000e2dfaceb9c3"}, + {file = "importlib_metadata-2.0.0.tar.gz", hash = "sha256:77a540690e24b0305878c37ffd421785a6f7e53c8b5720d211b211de8d0e95da"}, +] +iniconfig = [ + {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, + {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, +] +invoke = [ + {file = "invoke-1.4.1-py2-none-any.whl", hash = "sha256:93e12876d88130c8e0d7fd6618dd5387d6b36da55ad541481dfa5e001656f134"}, + {file = "invoke-1.4.1-py3-none-any.whl", hash = "sha256:87b3ef9d72a1667e104f89b159eaf8a514dbf2f3576885b2bbdefe74c3fb2132"}, + {file = "invoke-1.4.1.tar.gz", hash = "sha256:de3f23bfe669e3db1085789fd859eb8ca8e0c5d9c20811e2407fa042e8a5e15d"}, +] +packaging = [ + {file = "packaging-20.4-py2.py3-none-any.whl", hash = "sha256:998416ba6962ae7fbd6596850b80e17859a5753ba17c32284f67bfff33784181"}, + {file = "packaging-20.4.tar.gz", hash = "sha256:4357f74f47b9c12db93624a82154e9b120fa8293699949152b22065d556079f8"}, +] +paramiko = [ + {file = "paramiko-2.7.2-py2.py3-none-any.whl", hash = "sha256:4f3e316fef2ac628b05097a637af35685183111d4bc1b5979bd397c2ab7b5898"}, + {file = "paramiko-2.7.2.tar.gz", hash = "sha256:7f36f4ba2c0d81d219f4595e35f70d56cc94f9ac40a6acdf51d6ca210ce65035"}, +] +pluggy = [ + {file = "pluggy-0.13.1-py2.py3-none-any.whl", hash = "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"}, + {file = "pluggy-0.13.1.tar.gz", hash = "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0"}, +] +py = [ + {file = "py-1.9.0-py2.py3-none-any.whl", hash = "sha256:366389d1db726cd2fcfc79732e75410e5fe4d31db13692115529d34069a043c2"}, + {file = "py-1.9.0.tar.gz", hash = "sha256:9ca6883ce56b4e8da7e79ac18787889fa5206c79dcc67fb065376cd2fe03f342"}, +] +pycparser = [ + {file = "pycparser-2.20-py2.py3-none-any.whl", hash = "sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705"}, + {file = "pycparser-2.20.tar.gz", hash = "sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0"}, +] +pynacl = [ + {file = "PyNaCl-1.4.0-cp27-cp27m-macosx_10_10_x86_64.whl", hash = "sha256:ea6841bc3a76fa4942ce00f3bda7d436fda21e2d91602b9e21b7ca9ecab8f3ff"}, + {file = "PyNaCl-1.4.0-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:d452a6746f0a7e11121e64625109bc4468fc3100452817001dbe018bb8b08514"}, + {file = "PyNaCl-1.4.0-cp27-cp27m-win32.whl", hash = "sha256:2fe0fc5a2480361dcaf4e6e7cea00e078fcda07ba45f811b167e3f99e8cff574"}, + {file = "PyNaCl-1.4.0-cp27-cp27m-win_amd64.whl", hash = "sha256:f8851ab9041756003119368c1e6cd0b9c631f46d686b3904b18c0139f4419f80"}, + {file = "PyNaCl-1.4.0-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:7757ae33dae81c300487591c68790dfb5145c7d03324000433d9a2c141f82af7"}, + {file = "PyNaCl-1.4.0-cp35-abi3-macosx_10_10_x86_64.whl", hash = "sha256:757250ddb3bff1eecd7e41e65f7f833a8405fede0194319f87899690624f2122"}, + {file = "PyNaCl-1.4.0-cp35-abi3-manylinux1_x86_64.whl", hash = "sha256:30f9b96db44e09b3304f9ea95079b1b7316b2b4f3744fe3aaecccd95d547063d"}, + {file = "PyNaCl-1.4.0-cp35-abi3-win32.whl", hash = "sha256:4e10569f8cbed81cb7526ae137049759d2a8d57726d52c1a000a3ce366779634"}, + {file = "PyNaCl-1.4.0-cp35-abi3-win_amd64.whl", hash = "sha256:c914f78da4953b33d4685e3cdc7ce63401247a21425c16a39760e282075ac4a6"}, + {file = "PyNaCl-1.4.0-cp35-cp35m-win32.whl", hash = "sha256:06cbb4d9b2c4bd3c8dc0d267416aaed79906e7b33f114ddbf0911969794b1cc4"}, + {file = "PyNaCl-1.4.0-cp35-cp35m-win_amd64.whl", hash = "sha256:511d269ee845037b95c9781aa702f90ccc36036f95d0f31373a6a79bd8242e25"}, + {file = "PyNaCl-1.4.0-cp36-cp36m-win32.whl", hash = "sha256:11335f09060af52c97137d4ac54285bcb7df0cef29014a1a4efe64ac065434c4"}, + {file = "PyNaCl-1.4.0-cp36-cp36m-win_amd64.whl", hash = "sha256:cd401ccbc2a249a47a3a1724c2918fcd04be1f7b54eb2a5a71ff915db0ac51c6"}, + {file = "PyNaCl-1.4.0-cp37-cp37m-win32.whl", hash = "sha256:8122ba5f2a2169ca5da936b2e5a511740ffb73979381b4229d9188f6dcb22f1f"}, + {file = "PyNaCl-1.4.0-cp37-cp37m-win_amd64.whl", hash = "sha256:537a7ccbea22905a0ab36ea58577b39d1fa9b1884869d173b5cf111f006f689f"}, + {file = "PyNaCl-1.4.0-cp38-cp38-win32.whl", hash = "sha256:9c4a7ea4fb81536c1b1f5cc44d54a296f96ae78c1ebd2311bd0b60be45a48d96"}, + {file = "PyNaCl-1.4.0-cp38-cp38-win_amd64.whl", hash = "sha256:7c6092102219f59ff29788860ccb021e80fffd953920c4a8653889c029b2d420"}, + {file = "PyNaCl-1.4.0.tar.gz", hash = "sha256:54e9a2c849c742006516ad56a88f5c74bf2ce92c9f67435187c3c5953b346505"}, +] +pyparsing = [ + {file = "pyparsing-2.4.7-py2.py3-none-any.whl", hash = "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"}, + {file = "pyparsing-2.4.7.tar.gz", hash = "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1"}, +] +pytest = [ + {file = "pytest-6.1.2-py3-none-any.whl", hash = "sha256:4288fed0d9153d9646bfcdf0c0428197dba1ecb27a33bb6e031d002fa88653fe"}, + {file = "pytest-6.1.2.tar.gz", hash = "sha256:c0a7e94a8cdbc5422a51ccdad8e6f1024795939cc89159a0ae7f0b316ad3823e"}, +] +pytest-playbook = [] +pytest-target = [] +pyyaml = [ + {file = "PyYAML-5.3.1-cp27-cp27m-win32.whl", hash = "sha256:74809a57b329d6cc0fdccee6318f44b9b8649961fa73144a98735b0aaf029f1f"}, + {file = "PyYAML-5.3.1-cp27-cp27m-win_amd64.whl", hash = "sha256:240097ff019d7c70a4922b6869d8a86407758333f02203e0fc6ff79c5dcede76"}, + {file = "PyYAML-5.3.1-cp35-cp35m-win32.whl", hash = "sha256:4f4b913ca1a7319b33cfb1369e91e50354d6f07a135f3b901aca02aa95940bd2"}, + {file = "PyYAML-5.3.1-cp35-cp35m-win_amd64.whl", hash = "sha256:cc8955cfbfc7a115fa81d85284ee61147059a753344bc51098f3ccd69b0d7e0c"}, + {file = "PyYAML-5.3.1-cp36-cp36m-win32.whl", hash = "sha256:7739fc0fa8205b3ee8808aea45e968bc90082c10aef6ea95e855e10abf4a37b2"}, + {file = "PyYAML-5.3.1-cp36-cp36m-win_amd64.whl", hash = "sha256:69f00dca373f240f842b2931fb2c7e14ddbacd1397d57157a9b005a6a9942648"}, + {file = "PyYAML-5.3.1-cp37-cp37m-win32.whl", hash = "sha256:d13155f591e6fcc1ec3b30685d50bf0711574e2c0dfffd7644babf8b5102ca1a"}, + {file = "PyYAML-5.3.1-cp37-cp37m-win_amd64.whl", hash = "sha256:73f099454b799e05e5ab51423c7bcf361c58d3206fa7b0d555426b1f4d9a3eaf"}, + {file = "PyYAML-5.3.1-cp38-cp38-win32.whl", hash = "sha256:06a0d7ba600ce0b2d2fe2e78453a470b5a6e000a985dd4a4e54e436cc36b0e97"}, + {file = "PyYAML-5.3.1-cp38-cp38-win_amd64.whl", hash = "sha256:95f71d2af0ff4227885f7a6605c37fd53d3a106fcab511b8860ecca9fcf400ee"}, + {file = "PyYAML-5.3.1.tar.gz", hash = "sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d"}, +] +schema = [ + {file = "schema-0.7.3-py2.py3-none-any.whl", hash = "sha256:c331438b60f634cab5664ab720d3083cc444f924d55269530c36b33e3354276f"}, + {file = "schema-0.7.3.tar.gz", hash = "sha256:4cf529318cfd1e844ecbe02f41f7e5aa027463e7403666a52746f31f04f47a5e"}, +] +six = [ + {file = "six-1.15.0-py2.py3-none-any.whl", hash = "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"}, + {file = "six-1.15.0.tar.gz", hash = "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259"}, +] +tenacity = [ + {file = "tenacity-6.2.0-py2.py3-none-any.whl", hash = "sha256:5a5d3dcd46381abe8b4f82b5736b8726fd3160c6c7161f53f8af7f1eb9b82173"}, + {file = "tenacity-6.2.0.tar.gz", hash = "sha256:29ae90e7faf488a8628432154bb34ace1cca58244c6ea399fd33f066ac71339a"}, +] +toml = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] +zipp = [ + {file = "zipp-3.4.0-py3-none-any.whl", hash = "sha256:102c24ef8f171fd729d46599845e95c7ab894a4cf45f5de11a44cc7444fb1108"}, + {file = "zipp-3.4.0.tar.gz", hash = "sha256:ed5eee1974372595f9e416cc7bbeeb12335201d8081ca8a0743c954d4446e5cb"}, +] diff --git a/pytest-lisa/pyproject.toml b/pytest-lisa/pyproject.toml index cca0f9beaa..66bd65a2ff 100644 --- a/pytest-lisa/pyproject.toml +++ b/pytest-lisa/pyproject.toml @@ -5,13 +5,20 @@ description = "Pytest plugin for Linux Integration Services Automation (LISA)." authors = ["Andrew Schwartzmeyer "] license = "MIT" classifiers = ["Framework :: Pytest"] +packages = [{include = "lisa.py"}] [tool.poetry.dependencies] python = "^3.7" pytest = "^6.1.2" +pytest-playbook = {path = "../pytest-playbook", develop = true} +pytest-target = {path = "../pytest-target", develop = true} +schema = "^0.7.3" [tool.poetry.dev-dependencies] +[tool.poetry.plugins] +pytest11 = {lisa = "lisa"} + [build-system] requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" From a573d0afb6f632df5ef1a8f412b58cc2fb193627 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Thu, 19 Nov 2020 20:05:33 -0800 Subject: [PATCH 093/194] Move `lisa.config` into `Target` to break coupling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It didn’t belong there in the first place! --- pytest-target/target/target.py | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/pytest-target/target/target.py b/pytest-target/target/target.py index e584b445ad..5353fc5044 100644 --- a/pytest-target/target/target.py +++ b/pytest-target/target/target.py @@ -12,8 +12,6 @@ from schema import Schema # type: ignore from tenacity import retry, stop_after_attempt, wait_exponential # type: ignore -import lisa - if typing.TYPE_CHECKING: from typing import Any, Mapping, Set @@ -28,6 +26,21 @@ class Target(ABC): host: str conn: fabric.Connection + # Setup a sane configuration for local and remote commands. Note + # that the defaults between Fabric and Invoke are different, so we + # use their Config classes explicitly later. + config = { + "run": { + # Show each command as its run. + "echo": True, + # Disable stdin forwarding. + "in_stream": False, + # Don’t let remote commands take longer than five minutes + # (unless later overridden). This is to prevent hangs. + "command_timeout": 1200, + } + } + def __init__( self, name: str, @@ -50,13 +63,15 @@ def __init__( # parameters which is side-effecty. self.host = self.deploy() - config = lisa.config.copy() - config["run"]["env"] = { # type: ignore + fabric_config = self.config.copy() + fabric_config["run"]["env"] = { # type: ignore # Set PATH since it’s not a login shell. "PATH": "/sbin:/usr/sbin:/usr/local/sbin:/bin:/usr/bin:/usr/local/bin" } self.connection = fabric.Connection( - self.host, config=fabric.Config(overrides=config), inline_ssh_env=True + self.host, + config=fabric.Config(overrides=fabric_config), + inline_ssh_env=True, ) # NOTE: This ought to be a property, but the combination of @@ -84,7 +99,7 @@ def delete(self) -> None: ... # A class attribute because it’s defined. - local_context = invoke.Context(config=invoke.Config(overrides=lisa.config)) + local_context = invoke.Context(config=invoke.Config(overrides=config)) @classmethod def local(cls, *args: Any, **kwargs: Any) -> Result: From af043cb83fa9701747079b1f1379abcb1939393c Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Thu, 19 Nov 2020 20:20:47 -0800 Subject: [PATCH 094/194] Create pytest-lisa plugin --- conftest.py | 105 +++-------------------------- lisa.py | 50 -------------- pytest-lisa/lisa.py | 141 +++++++++++++++++++++++++++++++++++++++ testsuites/test_lis.py | 5 +- testsuites/test_smoke.py | 3 +- testsuites/test_xdp.py | 4 +- 6 files changed, 158 insertions(+), 150 deletions(-) delete mode 100644 lisa.py create mode 100644 pytest-lisa/lisa.py diff --git a/conftest.py b/conftest.py index cba8c7ddf0..e272ff4ffd 100644 --- a/conftest.py +++ b/conftest.py @@ -1,58 +1,32 @@ -"""This file sets up custom plugins. +"""LISA tests' specific configurations go here. -https://docs.pytest.org/en/stable/writing_plugins.html +This file is essentially the staging ground for contributions to +`pytest-lisa`, the plugin (and package). Anything that is reusable and +stable should be sent upstream. """ from __future__ import annotations import typing - -import playbook - -# See https://pypi.org/project/schema/ -from schema import Optional, Schema, SchemaMissingKeyError # type: ignore - -import lisa +from pathlib import Path if typing.TYPE_CHECKING: - from typing import Any, Dict, List + from typing import Any, Dict from _pytest.config import Config from _pytest.config.argparsing import Parser - from pytest import Item, Session +pytest_plugins = ["playbook", "target", "lisa"] -pytest_plugins = ["playbook", "target"] +LINUX_SCRIPTS = Path("../Testscripts/Linux") def pytest_addoption(parser: Parser) -> None: - """Pytest hook for adding arbitrary CLI options. - - https://docs.pytest.org/en/latest/example/simple.html - https://docs.pytest.org/en/latest/reference.html#pytest.hookspec.pytest_addoption - - """ + """Pytest hook to add our CLI options.""" parser.addoption("--check", action="store_true", help="Run semantic analysis.") parser.addoption("--demo", action="store_true", help="Run in demo mode.") -def pytest_playbook_schema(schema: Dict[Any, Any], config: Config) -> None: - criteria_schema = Schema( - { - # TODO: Validate that these strings are valid regular - # expressions if we change our matching logic. - Optional("name", default=None): str, - Optional("area", default=None): str, - Optional("category", default=None): str, - Optional("priority", default=None): int, - Optional("tags", default=list): [str], - Optional("times", default=1): int, - Optional("exclude", default=False): bool, - } - ) - schema[Optional("criteria", default=list)] = [criteria_schema] - - def pytest_configure(config: Config) -> None: """Parse provided user inputs to setup configuration. @@ -83,66 +57,5 @@ def pytest_configure(config: Config) -> None: setattr(config.option, attr, value) -def pytest_collection_modifyitems( - session: Session, config: Config, items: List[Item] -) -> None: - """Pytest hook for modifying the selected items (tests). - - https://docs.pytest.org/en/latest/reference.html#pytest.hookspec.pytest_collection_modifyitems - - """ - # TODO: The ‘Item’ object has a ‘user_properties’ attribute which - # is a list of tuples and could be used to hold the validated - # marker data, simplifying later usage. - - # Validate all LISA marks. - for item in items: - try: - lisa.validate(item.get_closest_marker("lisa")) - except SchemaMissingKeyError as e: - print(f"Test {item.name} failed LISA validation {e}!") - items[:] = [] - return - - # Optionally select tests based on a playbook. - included: List[Item] = [] - excluded: List[Item] = [] - - # TODO: Remove logging. - def select(item: Item, times: int, exclude: bool) -> None: - """Includes or excludes the item as appropriate.""" - if exclude: - print(f" Excluding {item}") - excluded.append(item) - else: - print(f" Including {item} {times} times") - for _ in range(times - included.count(item)): - included.append(item) - - for c in playbook.playbook.get("criteria", []): - print(f"Parsing criteria {c}") - for item in items: - marker = item.get_closest_marker("lisa") - if not marker: - # Not all tests will have the LISA marker, such as - # static analysis tests. - continue - i = marker.kwargs - if any( - [ - c["name"] and c["name"] in item.name, - c["area"] and c["area"].casefold() == i["area"].casefold(), - c["category"] - and c["category"].casefold() == i["category"].casefold(), - c["priority"] and c["priority"] == i["priority"], - c["tags"] and set(c["tags"]) <= set(i["tags"]), - ] - ): - select(item, c["times"], c["exclude"]) - if not included: - included = items - items[:] = [i for i in included if i not in excluded] - - def pytest_html_report_title(report): # type: ignore report.title = "LISAv3 (Using Pytest) Results" diff --git a/lisa.py b/lisa.py deleted file mode 100644 index bc3b16506e..0000000000 --- a/lisa.py +++ /dev/null @@ -1,50 +0,0 @@ -from __future__ import annotations - -import typing -from pathlib import Path - -from schema import Optional, Or, Schema # type: ignore - -import pytest - -if typing.TYPE_CHECKING: - from _pytest.mark.structures import Mark - -LISA = pytest.mark.lisa -LINUX_SCRIPTS = Path("../Testscripts/Linux") - -# Setup a sane configuration for local and remote commands. Note that -# the defaults between Fabric and Invoke are different, so we use -# their Config classes explicitly. -config = { - "run": { - # Show each command as its run. - "echo": True, - # Disable stdin forwarding. - "in_stream": False, - # Don’t let remote commands take longer than five minutes - # (unless later overridden). This is to prevent hangs. - "command_timeout": 1200, - } -} - -lisa_schema = Schema( - { - "platform": str, - "category": Or("Functional", "Performance", "Stress", "Community", "Longhaul"), - "area": str, - "priority": Or(0, 1, 2, 3), - Optional("features", default=list): [str], - Optional("tags", default=list): [str], - Optional(object): object, - }, - ignore_extra_keys=True, -) - - -def validate(mark: typing.Optional[Mark]) -> None: - """Validate each test's LISA parameters.""" - if not mark: - return - assert not mark.args, "LISA marker cannot have positional arguments!" - mark.kwargs.update(lisa_schema.validate(mark.kwargs)) # type: ignore diff --git a/pytest-lisa/lisa.py b/pytest-lisa/lisa.py new file mode 100644 index 0000000000..98e858a57b --- /dev/null +++ b/pytest-lisa/lisa.py @@ -0,0 +1,141 @@ +"""A plugin for organizing, analyzing, and selecting tests. + +This plugin provides the mark `pytest.mark.lisa`, aliased as `LISA`, +for marking up tests metadata beyond that which Pytest provides by +default. See the `lisa_schema` for the expected metadata input. + +Tests can be selected through a `playbook.yaml` file using the +criteria schema. For example:: + + criteria: + # Select all Priority 0 tests. + - priority: 0 + # Run tests with 'smoke' in the name twice. + - name: smoke + times: 2 + # Exclude all tests in Area "xdp" + - area: xdp + exclude: true + +TODO +==== +* Provide test metadata statistics via a command-line flag. +* Improve schemata with annotations, error messages, etc. +* Assert every test has a LISA marker. +* Register custom marker. + +""" +from __future__ import annotations + +import typing + +import playbook +import pytest +from schema import Optional, Or, Schema, SchemaMissingKeyError # type: ignore + +if typing.TYPE_CHECKING: + from typing import Any, Dict, List + + from _pytest.config import Config + from _pytest.mark.structures import Mark + from pytest import Item, Session + +LISA = pytest.mark.lisa + + +def pytest_playbook_schema(schema: Dict[Any, Any]) -> None: + """pytest-playbook hook to update the playbook schema.""" + criteria_schema = Schema( + { + # TODO: Validate that these strings are valid regular + # expressions if we change our matching logic. + Optional("name", default=None): str, + Optional("area", default=None): str, + Optional("category", default=None): str, + Optional("priority", default=None): int, + Optional("tags", default=list): [str], + Optional("times", default=1): int, + Optional("exclude", default=False): bool, + } + ) + schema[Optional("criteria", default=list)] = [criteria_schema] + + +lisa_schema = Schema( + { + "platform": str, + "category": Or("Functional", "Performance", "Stress", "Community", "Longhaul"), + "area": str, + "priority": Or(0, 1, 2, 3), + Optional("features", default=list): [str], + Optional("tags", default=list): [str], + Optional(object): object, + }, + ignore_extra_keys=True, +) + + +def validate_mark(mark: typing.Optional[Mark]) -> None: + """Validate each test's LISA parameters.""" + if not mark: + # TODO: `assert mark, "LISA marker is missing!"` but not all + # tests will have it, such as static analysis tests. + return + assert not mark.args, "LISA marker cannot have positional arguments!" + mark.kwargs.update(lisa_schema.validate(mark.kwargs)) # type: ignore + + +def pytest_collection_modifyitems( + session: Session, config: Config, items: List[Item] +) -> None: + """Pytest hook for modifying the selected items (tests). + + https://docs.pytest.org/en/latest/reference.html#pytest.hookspec.pytest_collection_modifyitems + + """ + # TODO: The ‘Item’ object has a ‘user_properties’ attribute which + # is a list of tuples and could be used to hold the validated + # marker data, simplifying later usage. + + # Validate all LISA marks. + for item in items: + try: + validate_mark(item.get_closest_marker("lisa")) + except (SchemaMissingKeyError, AssertionError) as e: + pytest.exit(f"Error validating test '{item.name}' metadata: {e}") + + # Optionally select tests based on a playbook. + included: List[Item] = [] + excluded: List[Item] = [] + + def select(item: Item, times: int, exclude: bool) -> None: + """Includes or excludes the item as appropriate.""" + if exclude: + excluded.append(item) + else: + for _ in range(times - included.count(item)): + included.append(item) + + for c in playbook.playbook.get("criteria", []): + for item in items: + mark = item.get_closest_marker("lisa") + if not mark: + # Not all tests will have the LISA marker, such as + # static analysis tests. + continue + i = mark.kwargs + if any( + [ + c["name"] and c["name"] in item.name, + c["area"] and c["area"].casefold() == i["area"].casefold(), + c["category"] + and c["category"].casefold() == i["category"].casefold(), + c["priority"] and c["priority"] == i["priority"], + c["tags"] and set(c["tags"]) <= set(i["tags"]), + ] + ): + select(item, c["times"], c["exclude"]) + # Handle edge case of no items selected for inclusion. + if not included: + included = items + items[:] = [i for i in included if i not in excluded] diff --git a/testsuites/test_lis.py b/testsuites/test_lis.py index e68db2708c..e07512b760 100644 --- a/testsuites/test_lis.py +++ b/testsuites/test_lis.py @@ -6,7 +6,10 @@ if typing.TYPE_CHECKING: from target import Azure -from lisa import LINUX_SCRIPTS, LISA +import pytest +from lisa import LISA + +from conftest import LINUX_SCRIPTS @LISA(platform="Azure", category="Functional", priority=0, area="LIS_DEPLOY") diff --git a/testsuites/test_smoke.py b/testsuites/test_smoke.py index c8e0b2e424..c9f8c44d78 100644 --- a/testsuites/test_smoke.py +++ b/testsuites/test_smoke.py @@ -11,9 +11,8 @@ import time from invoke.runners import CommandTimedOut, Result, UnexpectedExit # type: ignore -from paramiko import SSHException # type: ignore - from lisa import LISA +from paramiko import SSHException # type: ignore @LISA( diff --git a/testsuites/test_xdp.py b/testsuites/test_xdp.py index ffbd5a9417..7e81410673 100644 --- a/testsuites/test_xdp.py +++ b/testsuites/test_xdp.py @@ -7,7 +7,9 @@ from target import Azure import pytest -from lisa import LINUX_SCRIPTS, LISA +from lisa import LISA + +from conftest import LINUX_SCRIPTS @LISA( From 635b0aa9a01df13de8dae5ab8435dee7a69d2484 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Thu, 19 Nov 2020 20:22:38 -0800 Subject: [PATCH 095/194] Improve `target.plugin` documentation --- pytest-target/target/plugin.py | 15 ++++++++++++++- pytest-target/target/target.py | 1 + 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/pytest-target/target/plugin.py b/pytest-target/target/plugin.py index 1d1e510d42..2a34046030 100644 --- a/pytest-target/target/plugin.py +++ b/pytest-target/target/plugin.py @@ -1,4 +1,17 @@ -"""Provides and parameterizes the `pool` and `target` fixtures.""" +"""Provides and parameterizes the `pool` and `target` fixtures. + +TODO +==== +* Provide a `targets` fixture for tests which use more than one target + at a time. +* Deallocate targets when switching to a new target. +* Use richer feature/requirements comparison for targets. +* Add `pytest.mark.target` instead of LISA mark for target + requirements. +* Reimplement caching of targets between runs. +* Improve schema with annotations, error messages, etc. + +""" from __future__ import annotations import typing diff --git a/pytest-target/target/target.py b/pytest-target/target/target.py index 5353fc5044..f8546d2013 100644 --- a/pytest-target/target/target.py +++ b/pytest-target/target/target.py @@ -24,6 +24,7 @@ class Target(ABC): features: Set[str] name: str host: str + # TODO: Use `self.conn` and remove forwarding methods. conn: fabric.Connection # Setup a sane configuration for local and remote commands. Note From 66bc2c89fa55c58bc11ca7d60924c78ef7e61955 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Thu, 19 Nov 2020 20:25:58 -0800 Subject: [PATCH 096/194] Use self.conn directly instead of forwarding --- pytest-target/target/plugin.py | 2 +- pytest-target/target/target.py | 20 ++------------------ testsuites/test_lis.py | 8 ++++---- testsuites/test_smoke.py | 6 +++--- testsuites/test_xdp.py | 12 ++++++------ 5 files changed, 16 insertions(+), 32 deletions(-) diff --git a/pytest-target/target/plugin.py b/pytest-target/target/plugin.py index 2a34046030..5c7629c110 100644 --- a/pytest-target/target/plugin.py +++ b/pytest-target/target/plugin.py @@ -131,7 +131,7 @@ def target(pool: List[Target], request: SubRequest) -> Iterator[Target]: t = platform(f"pytest-{uuid4()}", parameters, features) pool.append(t) yield t - t.connection.close() + t.conn.close() targets: List[Dict[str, Any]] = [] diff --git a/pytest-target/target/target.py b/pytest-target/target/target.py index f8546d2013..11d5f5e4b1 100644 --- a/pytest-target/target/target.py +++ b/pytest-target/target/target.py @@ -24,7 +24,6 @@ class Target(ABC): features: Set[str] name: str host: str - # TODO: Use `self.conn` and remove forwarding methods. conn: fabric.Connection # Setup a sane configuration for local and remote commands. Note @@ -69,7 +68,7 @@ def __init__( # Set PATH since it’s not a login shell. "PATH": "/sbin:/usr/sbin:/usr/local/sbin:/bin:/usr/bin:/usr/local/bin" } - self.connection = fabric.Connection( + self.conn = fabric.Connection( self.host, config=fabric.Config(overrides=fabric_config), inline_ssh_env=True, @@ -107,21 +106,6 @@ def local(cls, *args: Any, **kwargs: Any) -> Result: """This patches Fabric's 'local()' function to ignore SSH environment.""" return Target.local_context.run(*args, **kwargs) - # TODO: Refactor this. We don’t want to inherit from `Connection` - # because that’s overly complicated. Honestly we probably just - # want users to call `target.conn.run()` etc. - def run(self, *args: Any, **kwargs: Any) -> Result: - return self.connection.run(*args, **kwargs) - - def sudo(self, *args: Any, **kwargs: Any) -> Result: - return self.connection.sudo(*args, **kwargs) - - def get(self, *args: Any, **kwargs: Any) -> Result: - return self.connection.get(*args, **kwargs) - - def put(self, *args: Any, **kwargs: Any) -> Result: - return self.connection.put(*args, **kwargs) - @retry(reraise=True, wait=wait_exponential(), stop=stop_after_attempt(3)) def ping(self, **kwargs: Any) -> Result: """Ping the node from the local system in a cross-platform manner.""" @@ -131,7 +115,7 @@ def ping(self, **kwargs: Any) -> Result: def cat(self, path: str) -> str: """Gets the value of a remote file without a temporary file.""" with BytesIO() as buf: - self.get(path, buf) + self.conn.get(path, buf) return buf.getvalue().decode("utf-8").strip() diff --git a/testsuites/test_lis.py b/testsuites/test_lis.py index e07512b760..12116a20cc 100644 --- a/testsuites/test_lis.py +++ b/testsuites/test_lis.py @@ -17,8 +17,8 @@ def test_lis_driver_version(target: Azure) -> None: """Checks that the installed drivers have the correct version.""" # TODO: Include “utils.sh” automatically? Or something... for f in ["utils.sh", "LIS-VERSION-CHECK.sh"]: - target.put(LINUX_SCRIPTS / f) - target.run(f"chmod +x {f}") - target.sudo("yum install -y bc") - target.run("./LIS-VERSION-CHECK.sh") + target.conn.put(LINUX_SCRIPTS / f) + target.conn.run(f"chmod +x {f}") + target.conn.sudo("yum install -y bc") + target.conn.run("./LIS-VERSION-CHECK.sh") assert target.cat("state.txt") == "TestCompleted" diff --git a/testsuites/test_smoke.py b/testsuites/test_smoke.py index c9f8c44d78..365e3ae248 100644 --- a/testsuites/test_smoke.py +++ b/testsuites/test_smoke.py @@ -49,7 +49,7 @@ def test_smoke(target: Azure) -> None: try: logging.info("SSHing before reboot...") - target.connection.open() + target.conn.open() except ssh_errors as e: logging.warning(f"SSH before reboot failed: '{e}'") @@ -57,7 +57,7 @@ def test_smoke(target: Azure) -> None: try: logging.info("Rebooting...") # If this succeeds, we should expect the exit code to be -1 - reboot_exit = target.sudo("reboot", timeout=5).exited + reboot_exit = target.conn.sudo("reboot", timeout=5).exited except ssh_errors as e: logging.warning(f"SSH failed, using platform to reboot: '{e}'") target.platform_restart() @@ -79,7 +79,7 @@ def test_smoke(target: Azure) -> None: try: logging.info("SSHing after reboot...") - target.connection.open() + target.conn.open() except ssh_errors as e: logging.warning(f"SSH after reboot failed: '{e}'") diff --git a/testsuites/test_xdp.py b/testsuites/test_xdp.py index 7e81410673..4c6379b120 100644 --- a/testsuites/test_xdp.py +++ b/testsuites/test_xdp.py @@ -33,10 +33,10 @@ def test_verify_xdp_compliance(target: Azure) -> None: "enable_passwordless_root.sh", "enable_root.sh", ]: - target.put(LINUX_SCRIPTS / f) - target.run(f"chmod +x {f}") - target.run("./enable_root.sh") - target.run("./enable_passwordless_root.sh") - synth_interface = target.run("source XDPUtils.sh ; get_extra_synth_nic").stdout - target.run(f"./XDPDumpSetup.sh {target.internal_address} {synth_interface}") + target.conn.put(LINUX_SCRIPTS / f) + target.conn.run(f"chmod +x {f}") + target.conn.run("./enable_root.sh") + target.conn.run("./enable_passwordless_root.sh") + synth_interface = target.conn.run("source XDPUtils.sh ; get_extra_synth_nic").stdout + target.conn.run(f"./XDPDumpSetup.sh {target.internal_address} {synth_interface}") assert target.cat("state.txt") == "TestCompleted" From 32b2c761947d5cce0a933a89ea5ee658ad720caf Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Thu, 19 Nov 2020 20:29:59 -0800 Subject: [PATCH 097/194] Update .gitignore --- .gitignore | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index e1711c78f3..b2c361f63a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,10 @@ -# Pytest report files -/*.xml +*$py.class +*.py[cod] +*.so +.mypy_cache/ +.pytest_cache/ /*.html -/assets +/*.xml +/assets/ +__pycache__/ +dist/ From b9b62ed127dfd635fc0371b4d4de4f0245cd5a32 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Thu, 19 Nov 2020 20:33:46 -0800 Subject: [PATCH 098/194] Remove filelock, drop Python to 3.7, and test it --- poetry.lock | 51 ++++++++++++++++++++++++++++++++++++++++++++++---- pyproject.toml | 3 +-- 2 files changed, 48 insertions(+), 6 deletions(-) diff --git a/poetry.lock b/poetry.lock index ce576ff018..4711bfc384 100644 --- a/poetry.lock +++ b/poetry.lock @@ -162,7 +162,7 @@ testing = ["mock (>=2.0.0,<3.0)"] name = "filelock" version = "3.0.12" description = "A platform independent file lock." -category = "main" +category = "dev" optional = false python-versions = "*" @@ -175,6 +175,7 @@ optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7" [package.dependencies] +importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} mccabe = ">=0.6.0,<0.7.0" pycodestyle = ">=2.6.0a1,<2.7.0" pyflakes = ">=2.2.0,<2.3.0" @@ -219,6 +220,21 @@ testfixtures = ">=6.8.0,<7" [package.extras] test = ["pytest (>=4.0.2,<6)", "toml"] +[[package]] +name = "importlib-metadata" +version = "2.0.0" +description = "Read metadata from Python packages" +category = "main" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" + +[package.dependencies] +zipp = ">=0.5" + +[package.extras] +docs = ["sphinx", "rst.linker"] +testing = ["packaging", "pep517", "importlib-resources (>=1.3)"] + [[package]] name = "iniconfig" version = "1.1.1" @@ -353,6 +369,9 @@ category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +[package.dependencies] +importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} + [package.extras] dev = ["pre-commit", "tox"] @@ -467,6 +486,7 @@ python-versions = ">=3.5" atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} attrs = ">=17.4.0" colorama = {version = "*", markers = "sys_platform == \"win32\""} +importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} iniconfig = "*" packaging = "*" pluggy = ">=0.12,<1.0" @@ -553,7 +573,10 @@ python-versions = ">=3.5" [package.dependencies] filelock = ">=3.0" -mypy = {version = ">=0.700", markers = "python_version >= \"3.8\""} +mypy = [ + {version = ">=0.500", markers = "python_version < \"3.8\""}, + {version = ">=0.700", markers = "python_version >= \"3.8\""}, +] pytest = ">=3.5" [[package]] @@ -778,10 +801,22 @@ category = "dev" optional = false python-versions = ">=3.6" +[[package]] +name = "zipp" +version = "3.4.0" +description = "Backport of pathlib-compatible object wrapper for zip files" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.extras] +docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"] +testing = ["pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake8", "pytest-cov", "jaraco.test (>=3.2.0)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy"] + [metadata] lock-version = "1.1" -python-versions = "^3.8" -content-hash = "709c5ad614869a72de95d29708421298cd94048f6c5a4f2a2d00e83c8abccf37" +python-versions = "^3.7" +content-hash = "2446fb23de7593ab9aed977563ed514f029bfed67f5f69e102c72ff994e2bed6" [metadata.files] apipkg = [ @@ -913,6 +948,10 @@ flake8-isort = [ {file = "flake8-isort-4.0.0.tar.gz", hash = "sha256:2b91300f4f1926b396c2c90185844eb1a3d5ec39ea6138832d119da0a208f4d9"}, {file = "flake8_isort-4.0.0-py2.py3-none-any.whl", hash = "sha256:729cd6ef9ba3659512dee337687c05d79c78e1215fdf921ed67e5fe46cce2f3c"}, ] +importlib-metadata = [ + {file = "importlib_metadata-2.0.0-py2.py3-none-any.whl", hash = "sha256:cefa1a2f919b866c5beb7c9f7b0ebb4061f30a8a9bf16d609b000e2dfaceb9c3"}, + {file = "importlib_metadata-2.0.0.tar.gz", hash = "sha256:77a540690e24b0305878c37ffd421785a6f7e53c8b5720d211b211de8d0e95da"}, +] iniconfig = [ {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, @@ -1201,3 +1240,7 @@ ujson = [ {file = "ujson-4.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:f8a60928737a9a47e692fcd661ef2b5d75ba22c7c930025bd95e338f2a6e15bc"}, {file = "ujson-4.0.1.tar.gz", hash = "sha256:26cf6241b36ff5ce4539ae687b6b02673109c5e3efc96148806a7873eaa229d3"}, ] +zipp = [ + {file = "zipp-3.4.0-py3-none-any.whl", hash = "sha256:102c24ef8f171fd729d46599845e95c7ab894a4cf45f5de11a44cc7444fb1108"}, + {file = "zipp-3.4.0.tar.gz", hash = "sha256:ed5eee1974372595f9e416cc7bbeeb12335201d8081ca8a0743c954d4446e5cb"}, +] diff --git a/pyproject.toml b/pyproject.toml index 1fb99c7d30..b01394ed87 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,9 +6,8 @@ authors = ["Andrew Schwartzmeyer "] license = "MIT License" [tool.poetry.dependencies] -python = "^3.8" +python = "^3.7" pytest = "^6.1.1" -filelock = "^3.0.12" pytest-timeout = "^1.4.2" pytest-html = "^2.1.1" pytest-rerunfailures = "^9.1.1" From d06a425cb52739089dbd5fd290b5bf3458588aed Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Thu, 19 Nov 2020 20:46:22 -0800 Subject: [PATCH 099/194] Programmatically register LISA marker --- pytest-lisa/lisa.py | 18 +++++++++++++++++- pytest.ini | 2 -- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/pytest-lisa/lisa.py b/pytest-lisa/lisa.py index 98e858a57b..90ea870d00 100644 --- a/pytest-lisa/lisa.py +++ b/pytest-lisa/lisa.py @@ -22,7 +22,7 @@ * Provide test metadata statistics via a command-line flag. * Improve schemata with annotations, error messages, etc. * Assert every test has a LISA marker. -* Register custom marker. +* Remove 'features' from marker. """ from __future__ import annotations @@ -43,6 +43,22 @@ LISA = pytest.mark.lisa +def pytest_configure(config: Config) -> None: + """Pytest hook to perform initial configuration. + + We're registering our custom marker so that it passes + `--strict-markers`. + + """ + config.addinivalue_line( + "markers", + ( + "lisa(platform, category, area, priority, features, tags): " + "Annotate a test with metadata." + ), + ) + + def pytest_playbook_schema(schema: Dict[Any, Any]) -> None: """pytest-playbook hook to update the playbook schema.""" criteria_schema = Schema( diff --git a/pytest.ini b/pytest.ini index c80e7c884d..c538e219dd 100644 --- a/pytest.ini +++ b/pytest.ini @@ -5,8 +5,6 @@ addopts = --capture=tee-sys --tb=short -rA -markers = - lisa log_cli = true log_cli_level = WARNING log_cli_format = %(asctime)s %(levelname)s %(message)s From 47c88458099d5ea164a218534e967e46076f4a99 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Fri, 20 Nov 2020 14:55:35 -0800 Subject: [PATCH 100/194] Add `lisa` binary Just forwards to Pytest, but hey! --- pytest-lisa/lisa.py | 6 ++++++ pytest-lisa/pyproject.toml | 3 +++ 2 files changed, 9 insertions(+) diff --git a/pytest-lisa/lisa.py b/pytest-lisa/lisa.py index 90ea870d00..bd97f61601 100644 --- a/pytest-lisa/lisa.py +++ b/pytest-lisa/lisa.py @@ -27,6 +27,7 @@ """ from __future__ import annotations +import sys import typing import playbook @@ -43,6 +44,11 @@ LISA = pytest.mark.lisa +def main() -> None: + """Wrapper function so we can have a `lisa` binary.""" + sys.exit(pytest.main()) + + def pytest_configure(config: Config) -> None: """Pytest hook to perform initial configuration. diff --git a/pytest-lisa/pyproject.toml b/pytest-lisa/pyproject.toml index 66bd65a2ff..97b61056a4 100644 --- a/pytest-lisa/pyproject.toml +++ b/pytest-lisa/pyproject.toml @@ -19,6 +19,9 @@ schema = "^0.7.3" [tool.poetry.plugins] pytest11 = {lisa = "lisa"} +[tool.poetry.scripts] +lisa = "lisa:main" + [build-system] requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" From db2d17b17144c5f494ff521d53c65a790341e79d Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Fri, 20 Nov 2020 16:23:29 -0800 Subject: [PATCH 101/194] Update readme, CI, license, etc. --- .github/workflows/ci-workflow.yaml | 44 ++++ LICENSE-DOCS.md | 395 +++++++++++++++++++++++++++++ LICENSE.md | 21 ++ README.md | 191 ++++++++++++-- pytest.ini | 5 +- testsuites/test_lis.py | 1 + 6 files changed, 638 insertions(+), 19 deletions(-) create mode 100644 .github/workflows/ci-workflow.yaml create mode 100644 LICENSE-DOCS.md create mode 100644 LICENSE.md diff --git a/.github/workflows/ci-workflow.yaml b/.github/workflows/ci-workflow.yaml new file mode 100644 index 0000000000..778aa4243b --- /dev/null +++ b/.github/workflows/ci-workflow.yaml @@ -0,0 +1,44 @@ +name: LISA/Pytest CI Workflow + +on: [push, pull_request] + +jobs: + build: + strategy: + matrix: + os: [ubuntu-20.04, windows-2019] + python-version: [3.7, 3.8, 3.9] + fail-fast: false + runs-on: ${{ matrix.os }} + steps: + - name: Checkout repository to $GITHUB_WORKSPACE + uses: actions/checkout@v2 + + - name: Setup bootstrap Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Install Poetry for Linux + if: runner.os == 'Linux' + run: | + curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python + echo "$HOME/.poetry/bin" >> $GITHUB_PATH + + - name: Install Poetry for Windows + if: runner.os == 'Windows' + run: | + (Invoke-WebRequest -Uri https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py -UseBasicParsing).Content | python + echo "$env:USERPROFILE\.poetry\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + + - name: Install Python dependencies + run: poetry install + + # TODO: There's a bug in Poetry where `poetry run lisa` tries + # to call the Bash script it generated instead of `lisa.cmd`, + # which of course fails on Windows. + - name: Run self tests + run: poetry run pytest --verbose --playbook=playbooks/test.yaml --setup-show selftests/ + + - name: Run semantic analysis + run: poetry run pytest --quiet --flake8 --mypy -m "flake8 or mypy" diff --git a/LICENSE-DOCS.md b/LICENSE-DOCS.md new file mode 100644 index 0000000000..9912bb9c01 --- /dev/null +++ b/LICENSE-DOCS.md @@ -0,0 +1,395 @@ +Attribution 4.0 International + +======================================================================= + +Creative Commons Corporation ("Creative Commons") is not a law firm and +does not provide legal services or legal advice. Distribution of +Creative Commons public licenses does not create a lawyer-client or +other relationship. Creative Commons makes its licenses and related +information available on an "as-is" basis. Creative Commons gives no +warranties regarding its licenses, any material licensed under their +terms and conditions, or any related information. Creative Commons +disclaims all liability for damages resulting from their use to the +fullest extent possible. + +Using Creative Commons Public Licenses + +Creative Commons public licenses provide a standard set of terms and +conditions that creators and other rights holders may use to share +original works of authorship and other material subject to copyright +and certain other rights specified in the public license below. The +following considerations are for informational purposes only, are not +exhaustive, and do not form part of our licenses. + + Considerations for licensors: Our public licenses are + intended for use by those authorized to give the public + permission to use material in ways otherwise restricted by + copyright and certain other rights. Our licenses are + irrevocable. Licensors should read and understand the terms + and conditions of the license they choose before applying it. + Licensors should also secure all rights necessary before + applying our licenses so that the public can reuse the + material as expected. Licensors should clearly mark any + material not subject to the license. This includes other CC- + licensed material, or material used under an exception or + limitation to copyright. More considerations for licensors: + wiki.creativecommons.org/Considerations_for_licensors + + Considerations for the public: By using one of our public + licenses, a licensor grants the public permission to use the + licensed material under specified terms and conditions. If + the licensor's permission is not necessary for any reason--for + example, because of any applicable exception or limitation to + copyright--then that use is not regulated by the license. Our + licenses grant only permissions under copyright and certain + other rights that a licensor has authority to grant. Use of + the licensed material may still be restricted for other + reasons, including because others have copyright or other + rights in the material. A licensor may make special requests, + such as asking that all changes be marked or described. + Although not required by our licenses, you are encouraged to + respect those requests where reasonable. More_considerations + for the public: + wiki.creativecommons.org/Considerations_for_licensees + +======================================================================= + +Creative Commons Attribution 4.0 International Public License + +By exercising the Licensed Rights (defined below), You accept and agree +to be bound by the terms and conditions of this Creative Commons +Attribution 4.0 International Public License ("Public License"). To the +extent this Public License may be interpreted as a contract, You are +granted the Licensed Rights in consideration of Your acceptance of +these terms and conditions, and the Licensor grants You such rights in +consideration of benefits the Licensor receives from making the +Licensed Material available under these terms and conditions. + + +Section 1 -- Definitions. + + a. Adapted Material means material subject to Copyright and Similar + Rights that is derived from or based upon the Licensed Material + and in which the Licensed Material is translated, altered, + arranged, transformed, or otherwise modified in a manner requiring + permission under the Copyright and Similar Rights held by the + Licensor. For purposes of this Public License, where the Licensed + Material is a musical work, performance, or sound recording, + Adapted Material is always produced where the Licensed Material is + synched in timed relation with a moving image. + + b. Adapter's License means the license You apply to Your Copyright + and Similar Rights in Your contributions to Adapted Material in + accordance with the terms and conditions of this Public License. + + c. Copyright and Similar Rights means copyright and/or similar rights + closely related to copyright including, without limitation, + performance, broadcast, sound recording, and Sui Generis Database + Rights, without regard to how the rights are labeled or + categorized. For purposes of this Public License, the rights + specified in Section 2(b)(1)-(2) are not Copyright and Similar + Rights. + + d. Effective Technological Measures means those measures that, in the + absence of proper authority, may not be circumvented under laws + fulfilling obligations under Article 11 of the WIPO Copyright + Treaty adopted on December 20, 1996, and/or similar international + agreements. + + e. Exceptions and Limitations means fair use, fair dealing, and/or + any other exception or limitation to Copyright and Similar Rights + that applies to Your use of the Licensed Material. + + f. Licensed Material means the artistic or literary work, database, + or other material to which the Licensor applied this Public + License. + + g. Licensed Rights means the rights granted to You subject to the + terms and conditions of this Public License, which are limited to + all Copyright and Similar Rights that apply to Your use of the + Licensed Material and that the Licensor has authority to license. + + h. Licensor means the individual(s) or entity(ies) granting rights + under this Public License. + + i. Share means to provide material to the public by any means or + process that requires permission under the Licensed Rights, such + as reproduction, public display, public performance, distribution, + dissemination, communication, or importation, and to make material + available to the public including in ways that members of the + public may access the material from a place and at a time + individually chosen by them. + + j. Sui Generis Database Rights means rights other than copyright + resulting from Directive 96/9/EC of the European Parliament and of + the Council of 11 March 1996 on the legal protection of databases, + as amended and/or succeeded, as well as other essentially + equivalent rights anywhere in the world. + + k. You means the individual or entity exercising the Licensed Rights + under this Public License. Your has a corresponding meaning. + + +Section 2 -- Scope. + + a. License grant. + + 1. Subject to the terms and conditions of this Public License, + the Licensor hereby grants You a worldwide, royalty-free, + non-sublicensable, non-exclusive, irrevocable license to + exercise the Licensed Rights in the Licensed Material to: + + a. reproduce and Share the Licensed Material, in whole or + in part; and + + b. produce, reproduce, and Share Adapted Material. + + 2. Exceptions and Limitations. For the avoidance of doubt, where + Exceptions and Limitations apply to Your use, this Public + License does not apply, and You do not need to comply with + its terms and conditions. + + 3. Term. The term of this Public License is specified in Section + 6(a). + + 4. Media and formats; technical modifications allowed. The + Licensor authorizes You to exercise the Licensed Rights in + all media and formats whether now known or hereafter created, + and to make technical modifications necessary to do so. The + Licensor waives and/or agrees not to assert any right or + authority to forbid You from making technical modifications + necessary to exercise the Licensed Rights, including + technical modifications necessary to circumvent Effective + Technological Measures. For purposes of this Public License, + simply making modifications authorized by this Section 2(a) + (4) never produces Adapted Material. + + 5. Downstream recipients. + + a. Offer from the Licensor -- Licensed Material. Every + recipient of the Licensed Material automatically + receives an offer from the Licensor to exercise the + Licensed Rights under the terms and conditions of this + Public License. + + b. No downstream restrictions. You may not offer or impose + any additional or different terms or conditions on, or + apply any Effective Technological Measures to, the + Licensed Material if doing so restricts exercise of the + Licensed Rights by any recipient of the Licensed + Material. + + 6. No endorsement. Nothing in this Public License constitutes or + may be construed as permission to assert or imply that You + are, or that Your use of the Licensed Material is, connected + with, or sponsored, endorsed, or granted official status by, + the Licensor or others designated to receive attribution as + provided in Section 3(a)(1)(A)(i). + + b. Other rights. + + 1. Moral rights, such as the right of integrity, are not + licensed under this Public License, nor are publicity, + privacy, and/or other similar personality rights; however, to + the extent possible, the Licensor waives and/or agrees not to + assert any such rights held by the Licensor to the limited + extent necessary to allow You to exercise the Licensed + Rights, but not otherwise. + + 2. Patent and trademark rights are not licensed under this + Public License. + + 3. To the extent possible, the Licensor waives any right to + collect royalties from You for the exercise of the Licensed + Rights, whether directly or through a collecting society + under any voluntary or waivable statutory or compulsory + licensing scheme. In all other cases the Licensor expressly + reserves any right to collect such royalties. + + +Section 3 -- License Conditions. + +Your exercise of the Licensed Rights is expressly made subject to the +following conditions. + + a. Attribution. + + 1. If You Share the Licensed Material (including in modified + form), You must: + + a. retain the following if it is supplied by the Licensor + with the Licensed Material: + + i. identification of the creator(s) of the Licensed + Material and any others designated to receive + attribution, in any reasonable manner requested by + the Licensor (including by pseudonym if + designated); + + ii. a copyright notice; + + iii. a notice that refers to this Public License; + + iv. a notice that refers to the disclaimer of + warranties; + + v. a URI or hyperlink to the Licensed Material to the + extent reasonably practicable; + + b. indicate if You modified the Licensed Material and + retain an indication of any previous modifications; and + + c. indicate the Licensed Material is licensed under this + Public License, and include the text of, or the URI or + hyperlink to, this Public License. + + 2. You may satisfy the conditions in Section 3(a)(1) in any + reasonable manner based on the medium, means, and context in + which You Share the Licensed Material. For example, it may be + reasonable to satisfy the conditions by providing a URI or + hyperlink to a resource that includes the required + information. + + 3. If requested by the Licensor, You must remove any of the + information required by Section 3(a)(1)(A) to the extent + reasonably practicable. + + 4. If You Share Adapted Material You produce, the Adapter's + License You apply must not prevent recipients of the Adapted + Material from complying with this Public License. + + +Section 4 -- Sui Generis Database Rights. + +Where the Licensed Rights include Sui Generis Database Rights that +apply to Your use of the Licensed Material: + + a. for the avoidance of doubt, Section 2(a)(1) grants You the right + to extract, reuse, reproduce, and Share all or a substantial + portion of the contents of the database; + + b. if You include all or a substantial portion of the database + contents in a database in which You have Sui Generis Database + Rights, then the database in which You have Sui Generis Database + Rights (but not its individual contents) is Adapted Material; and + + c. You must comply with the conditions in Section 3(a) if You Share + all or a substantial portion of the contents of the database. + +For the avoidance of doubt, this Section 4 supplements and does not +replace Your obligations under this Public License where the Licensed +Rights include other Copyright and Similar Rights. + + +Section 5 -- Disclaimer of Warranties and Limitation of Liability. + + a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE + EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS + AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF + ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS, + IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION, + WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR + PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS, + ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT + KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT + ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU. + + b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE + TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION, + NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT, + INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES, + COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR + USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN + ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR + DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR + IN PART, THIS LIMITATION MAY NOT APPLY TO YOU. + + c. The disclaimer of warranties and limitation of liability provided + above shall be interpreted in a manner that, to the extent + possible, most closely approximates an absolute disclaimer and + waiver of all liability. + + +Section 6 -- Term and Termination. + + a. This Public License applies for the term of the Copyright and + Similar Rights licensed here. However, if You fail to comply with + this Public License, then Your rights under this Public License + terminate automatically. + + b. Where Your right to use the Licensed Material has terminated under + Section 6(a), it reinstates: + + 1. automatically as of the date the violation is cured, provided + it is cured within 30 days of Your discovery of the + violation; or + + 2. upon express reinstatement by the Licensor. + + For the avoidance of doubt, this Section 6(b) does not affect any + right the Licensor may have to seek remedies for Your violations + of this Public License. + + c. For the avoidance of doubt, the Licensor may also offer the + Licensed Material under separate terms or conditions or stop + distributing the Licensed Material at any time; however, doing so + will not terminate this Public License. + + d. Sections 1, 5, 6, 7, and 8 survive termination of this Public + License. + + +Section 7 -- Other Terms and Conditions. + + a. The Licensor shall not be bound by any additional or different + terms or conditions communicated by You unless expressly agreed. + + b. Any arrangements, understandings, or agreements regarding the + Licensed Material not stated herein are separate from and + independent of the terms and conditions of this Public License. + + +Section 8 -- Interpretation. + + a. For the avoidance of doubt, this Public License does not, and + shall not be interpreted to, reduce, limit, restrict, or impose + conditions on any use of the Licensed Material that could lawfully + be made without permission under this Public License. + + b. To the extent possible, if any provision of this Public License is + deemed unenforceable, it shall be automatically reformed to the + minimum extent necessary to make it enforceable. If the provision + cannot be reformed, it shall be severed from this Public License + without affecting the enforceability of the remaining terms and + conditions. + + c. No term or condition of this Public License will be waived and no + failure to comply consented to unless expressly agreed to by the + Licensor. + + d. Nothing in this Public License constitutes or may be interpreted + as a limitation upon, or waiver of, any privileges and immunities + that apply to the Licensor or You, including from the legal + processes of any jurisdiction or authority. + + +======================================================================= + +Creative Commons is not a party to its public +licenses. Notwithstanding, Creative Commons may elect to apply one of +its public licenses to material it publishes and in those instances +will be considered the “Licensor.” The text of the Creative Commons +public licenses is dedicated to the public domain under the CC0 Public +Domain Dedication. Except for the limited purpose of indicating that +material is shared under a Creative Commons public license or as +otherwise permitted by the Creative Commons policies published at +creativecommons.org/policies, Creative Commons does not authorize the +use of the trademark "Creative Commons" or any other trademark or logo +of Creative Commons without its prior written consent including, +without limitation, in connection with any unauthorized modifications +to any of its public licenses or any other arrangements, +understandings, or agreements concerning use of licensed material. For +the avoidance of doubt, this paragraph does not form part of the +public licenses. + +Creative Commons may be contacted at creativecommons.org. diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000000..b2f52a2bad --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,21 @@ +Copyright (c) Microsoft Corporation. + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index c564d256cf..583bc81b38 100644 --- a/README.md +++ b/README.md @@ -1,31 +1,192 @@ -# LISAv3 via pytest-lisa +# Linux Integration Services Automation 3.0 (LISAv3) -Basic instructions for testing the prototype: +[![CI Workflow for LISAv3](https://github.com/LIS/LISAv2/workflows/CI%20Workflow%20for%20LISAv3/badge.svg?branch=main)](https://github.com/LIS/LISAv2/actions?query=workflow%3A%22CI+Workflow+for+LISAv3%22+event%3Apush+branch%3Amain) +[![Code Style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) +[![GitHub License](https://img.shields.io/github/license/LIS/LISAv2)](https://github.com/LIS/LISAv2/blob/main/LICENSE.md) + +LISA is a Linux test automation framework with built-in test cases to verify the +quality of Linux distributions on multiple platforms (such as Azure, Hyper-V, +and bare metal). + +## Getting Started: + +### Install Python 3: + +Install Python 3.7 or newer from your Linux distribution’s package repositories, +or [python.org](https://www.python.org/). + +### Install Poetry: + +[Poetry](https://python-poetry.org/docs/) is our preferred tool for Python +dependency management and packaging. We’ll use it to automatically setup a +‘virtualenv’ and install everything we need. + +#### On Linux (or WSL): ```bash -# Install Poetry, make sure `poetry` is in your `PATH` -curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python +curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python - +source $HOME/.poetry/env +``` + +If you are using WSL, installing Poetry on both Windows and Linux may cause both +platforms’ versions of Poetry to be on your path, as Windows binaries are mapped +into WSL’s `PATH`. This means that the Linux `poetry` binary _must_ appear in +your `PATH` before the Windows version, or this error will appear: + +``` +`/usr/bin/env: ‘python\r’: No such file or directory` +``` + +Adjust your `PATH` appropriately to fix it. + +#### On Windows (in PowerShell): + +```powershell +(Invoke-WebRequest -Uri https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py -UseBasicParsing).Content | python - +$env:PATH += ";$env:USERPROFILE\.poetry\bin" +``` + +### Clone LISA and `cd` into the Git repo: + +```bash +git clone -b main https://github.com/LIS/LISAv2.git lisa +cd lisa +``` + +### Install Python dependencies: + +```bash +# Install the Python packages +poetry install + +# Enter the virtual environment +poetry shell +``` + +### Use LISA: + +```bash +# Run some self-tests +lisa --playbook=playbooks/test.yml selftests/ + +# Run a demo which deployes Azure resources +lisa --playbooks/smoke.yaml +``` +#### Enable Azure: + +To run the demo you’ll need the [Azure CLI][] tool installed and configured: + +```bash # Install Azure CLI, make sure `az` is in your `PATH` curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash # Login and set subscription az login az account set -s +``` -# Clone LISAv2 with the Pytest prototype -git clone -b pytest/main https://github.com/LIS/LISAv2.git -cd LISAv2 +See the [design document](DESIGN.md) for details. -# Install Python packages -make setup +## Contributing -# Run some local demos -make test -make yaml +The path to the virtualenv used by Poetry can found with this command: -# Run a demo which deployes Azure resources -make smoke +```bash +poetry env list --full-path ``` -See the [design document](DESIGN.md) for details. +Use it to configure your editor. + +### Editor Setup + +#### Visual Studio Code + +First, click the Python version in the bottom left, then enter the path emitted +by the command above. This will point Code to the Poetry virtual environment. + +Make sure below settings are in root level of `.vscode/settings.json` + +```json +{ + "python.analysis.typeCheckingMode": "strict", + "python.formatting.provider": "black", + "python.linting.enabled": true, + "python.linting.flake8Enabled": true, + "python.linting.mypyEnabled": true, + "python.linting.pylintEnabled": false, + "editor.formatOnSave": true, + "python.linting.mypyArgs": [ + "--strict", + "--namespace-packages", + "--show-column-numbers", + ], + "python.sortImports.path": "isort", + "python.analysis.useLibraryCodeForTypes": false, + "python.analysis.autoImportCompletions": false, + "files.eol": "\n", +} +``` + +#### Emacs + +Use the [pyvenv](https://github.com/jorgenschaefer/pyvenv) package: + +```emacs-lisp +(use-package pyvenv + :ensure t + :hook (python-mode . pyvenv-tracking-mode)) +``` + +Then run `M-x add-dir-local-variable RET python-mode RET pyvenv-activate RET +` where the value is the path given by the command above. +This will create a `.dir-locals.el` file which looks like this: + +```emacs-lisp +;;; Directory Local Variables +;;; For more information see (info "(emacs) Directory Variables") + +((nil . ((pyvenv-activate . "~/.cache/pypoetry/virtualenvs/")))) +``` + +### Contributor License Agreement + +This project welcomes contributions and suggestions. Most contributions require +you to agree to a Contributor License Agreement (CLA) declaring that you have +the right to, and actually do, grant us the rights to use your contribution. For +details, visit https://cla.opensource.microsoft.com. + +When you submit a pull request, a CLA bot will automatically determine whether +you need to provide a CLA and decorate the PR appropriately (e.g., status check, +comment). Simply follow the instructions provided by the bot. You will only need +to do this once across all repos using our CLA. + +This project has adopted the [Microsoft Open Source Code of +Conduct](https://opensource.microsoft.com/codeofconduct/). For more information +see the [Code of Conduct +FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact +[opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional +questions or comments. + +## Legal Notices + +Microsoft and any contributors grant you a license to the Microsoft +documentation and other content in this repository under the [Creative Commons +Attribution 4.0 International Public +License](https://creativecommons.org/licenses/by/4.0/legalcode), see the +[LICENSE-DOCS](LICENSE-DOCS.md) file, and grant you a license to any code in the +repository under the [MIT License](https://opensource.org/licenses/MIT), see the +[LICENSE](LICENSE.md) file. + +Microsoft, Windows, Microsoft Azure and/or other Microsoft products and services +referenced in the documentation may be either trademarks or registered +trademarks of Microsoft in the United States and/or other countries. The +licenses for this project do not grant you rights to use any Microsoft names, +logos, or trademarks. Microsoft's general trademark guidelines can be found at +http://go.microsoft.com/fwlink/?LinkID=254653. + +Privacy information can be found at https://privacy.microsoft.com/en-us/ + +Microsoft and any contributors reserve all other rights, whether under their +respective copyrights, patents, or trademarks, whether by implication, estoppel +or otherwise. diff --git a/pytest.ini b/pytest.ini index c538e219dd..6539eb3164 100644 --- a/pytest.ini +++ b/pytest.ini @@ -4,15 +4,12 @@ addopts = --self-contained-html --capture=tee-sys --tb=short - -rA log_cli = true -log_cli_level = WARNING +log_cli_level = ERROR log_cli_format = %(asctime)s %(levelname)s %(message)s log_cli_date_format = %Y-%m-%d %H:%M:%S render_collapsed = true junit_logging = all timeout = 1200 filterwarnings = - error - ignore:unclosed:ResourceWarning ignore:the imp module is deprecated in favour of importlib:DeprecationWarning diff --git a/testsuites/test_lis.py b/testsuites/test_lis.py index 12116a20cc..cec2226c7f 100644 --- a/testsuites/test_lis.py +++ b/testsuites/test_lis.py @@ -13,6 +13,7 @@ @LISA(platform="Azure", category="Functional", priority=0, area="LIS_DEPLOY") +@pytest.mark.skip(reason="Scripts missing") def test_lis_driver_version(target: Azure) -> None: """Checks that the installed drivers have the correct version.""" # TODO: Include “utils.sh” automatically? Or something... From b5f3274879a807d7466059ee20a4113bf120ceb1 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Fri, 20 Nov 2020 18:35:31 -0800 Subject: [PATCH 102/194] Remove references to LINUX_SCRIPTS and unnecessary CLI options --- conftest.py | 49 ++++-------------------------------------- testsuites/test_lis.py | 4 +--- testsuites/test_xdp.py | 4 +--- 3 files changed, 6 insertions(+), 51 deletions(-) diff --git a/conftest.py b/conftest.py index e272ff4ffd..eada04e42e 100644 --- a/conftest.py +++ b/conftest.py @@ -8,54 +8,13 @@ from __future__ import annotations import typing -from pathlib import Path if typing.TYPE_CHECKING: - from typing import Any, Dict - - from _pytest.config import Config - from _pytest.config.argparsing import Parser + from pytest_html.plugin import HTMLReport # type: ignore pytest_plugins = ["playbook", "target", "lisa"] -LINUX_SCRIPTS = Path("../Testscripts/Linux") - - -def pytest_addoption(parser: Parser) -> None: - """Pytest hook to add our CLI options.""" - parser.addoption("--check", action="store_true", help="Run semantic analysis.") - parser.addoption("--demo", action="store_true", help="Run in demo mode.") - - -def pytest_configure(config: Config) -> None: - """Parse provided user inputs to setup configuration. - - https://docs.pytest.org/en/latest/reference.html#pytest.hookspec.pytest_configure - - """ - # Search ‘_pytest’ for ‘addoption’ to find these. - options: Dict[str, Any] = {} # See ‘pytest.ini’ for defaults. - if config.getoption("--check"): - options.update( - { - "flake8": True, - "mypy": True, - "markexpr": "flake8 or mypy", - "reportchars": "fE", - } - ) - if config.getoption("--demo"): - options.update( - { - "html": "demo.html", - "no_header": True, - "showcapture": "log", - "tb": "line", - } - ) - for attr, value in options.items(): - setattr(config.option, attr, value) - -def pytest_html_report_title(report): # type: ignore - report.title = "LISAv3 (Using Pytest) Results" +def pytest_html_report_title(report: HTMLReport) -> None: + """Set HTML report title.""" + report.title = "LISAv3 Results" diff --git a/testsuites/test_lis.py b/testsuites/test_lis.py index cec2226c7f..d902282cfa 100644 --- a/testsuites/test_lis.py +++ b/testsuites/test_lis.py @@ -9,8 +9,6 @@ import pytest from lisa import LISA -from conftest import LINUX_SCRIPTS - @LISA(platform="Azure", category="Functional", priority=0, area="LIS_DEPLOY") @pytest.mark.skip(reason="Scripts missing") @@ -18,7 +16,7 @@ def test_lis_driver_version(target: Azure) -> None: """Checks that the installed drivers have the correct version.""" # TODO: Include “utils.sh” automatically? Or something... for f in ["utils.sh", "LIS-VERSION-CHECK.sh"]: - target.conn.put(LINUX_SCRIPTS / f) + target.conn.put(f) target.conn.run(f"chmod +x {f}") target.conn.sudo("yum install -y bc") target.conn.run("./LIS-VERSION-CHECK.sh") diff --git a/testsuites/test_xdp.py b/testsuites/test_xdp.py index 4c6379b120..8aafe577b4 100644 --- a/testsuites/test_xdp.py +++ b/testsuites/test_xdp.py @@ -9,8 +9,6 @@ import pytest from lisa import LISA -from conftest import LINUX_SCRIPTS - @LISA( platform="Azure", @@ -33,7 +31,7 @@ def test_verify_xdp_compliance(target: Azure) -> None: "enable_passwordless_root.sh", "enable_root.sh", ]: - target.conn.put(LINUX_SCRIPTS / f) + target.conn.put(f) target.conn.run(f"chmod +x {f}") target.conn.run("./enable_root.sh") target.conn.run("./enable_passwordless_root.sh") From 03d5a2f0f2660891777d9a13ed062cba99aac316 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Mon, 23 Nov 2020 19:26:08 -0800 Subject: [PATCH 103/194] =?UTF-8?q?Fix=20Azure.deploy()=20(forgot=20to=20f?= =?UTF-8?q?inish=20=E2=80=98internal=5Faddress=E2=80=99)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pytest-target/target/azure.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pytest-target/target/azure.py b/pytest-target/target/azure.py index 3d02cc256b..0ee4649da9 100644 --- a/pytest-target/target/azure.py +++ b/pytest-target/target/azure.py @@ -127,10 +127,10 @@ def deploy(self) -> str: vm_command.append("--accelerated-networking true") self.data = json.loads(self.local(" ".join(vm_command)).stdout) + hostname: str = self.data["publicIpAddress"] + self.internal_address = self.data["privateIpAddress"] self.allow_ping() # TODO: Enable auto-shutdown 4 hours from deployment. - self.internal_address = self.data["internal_address"] - hostname: str = self.data["publicIpAddress"] return hostname def delete(self) -> None: From 8b064b0186857732cc094e9cc311566bf6bc9250 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Mon, 23 Nov 2020 19:41:09 -0800 Subject: [PATCH 104/194] Build as a Python package (so we can take a dependency on it) --- conftest.py | 2 -- pyproject.toml | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conftest.py b/conftest.py index eada04e42e..c63fcdc909 100644 --- a/conftest.py +++ b/conftest.py @@ -12,8 +12,6 @@ if typing.TYPE_CHECKING: from pytest_html.plugin import HTMLReport # type: ignore -pytest_plugins = ["playbook", "target", "lisa"] - def pytest_html_report_title(report: HTMLReport) -> None: """Set HTML report title.""" diff --git a/pyproject.toml b/pyproject.toml index b01394ed87..65965600c4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,6 +4,8 @@ version = "0.1.0" description = "Linux Integration Services Automation (LISA)" authors = ["Andrew Schwartzmeyer "] license = "MIT License" +include = [".md", "playbooks/*.yaml"] +packages = [{include = "*.py"}, {include = "testsuites"}] [tool.poetry.dependencies] python = "^3.7" From 776fcfee1346bf3c89f622a435c5b86ef2d43c3f Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Mon, 7 Dec 2020 15:21:07 -0800 Subject: [PATCH 105/194] Add demo-like Pytest options Which means a single-line traceback, no header, and INFO and above logging messages. --- pytest.ini | 5 +++-- testsuites/test_smoke.py | 2 ++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/pytest.ini b/pytest.ini index 6539eb3164..7da0eb70c2 100644 --- a/pytest.ini +++ b/pytest.ini @@ -3,9 +3,10 @@ addopts = --strict-markers --self-contained-html --capture=tee-sys - --tb=short + --tb=line + --no-header log_cli = true -log_cli_level = ERROR +log_cli_level = WARNING log_cli_format = %(asctime)s %(levelname)s %(message)s log_cli_date_format = %Y-%m-%d %H:%M:%S render_collapsed = true diff --git a/testsuites/test_smoke.py b/testsuites/test_smoke.py index 365e3ae248..dc7b017fde 100644 --- a/testsuites/test_smoke.py +++ b/testsuites/test_smoke.py @@ -37,6 +37,8 @@ def test_smoke(target: Azure) -> None: SSH failures DO NOT fail this test. + TODO: Capture these logs without capturing all INFO logs. + """ logging.info("Pinging before reboot...") ping1 = Result() From f4fb61f362b4866d8d1eb2a7289d2176529d8f8c Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Mon, 7 Dec 2020 15:21:41 -0800 Subject: [PATCH 106/194] Add a TODO to the design doc --- DESIGN.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/DESIGN.md b/DESIGN.md index 31647e5dc2..151472711b 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -817,7 +817,7 @@ We can additionally list a test twice when modifying the items collection, as implemented in the criteria proof-of-concept. However, given the above abilities, this may not be desired. -## What does the “flow” of Pytest look-like? +## What does the “flow” of Pytest look like? This is best described in Pythonic pseudo-code, where the context manager encapsulates each scope and the for loop encapsulates processing: @@ -868,6 +868,7 @@ with pool_fixture as pool: There’s still a lot more to think about and design. A non-exhaustive list of future topics (some touched on above): +* Terminology table * Tests inventory (generating statistics from metadata) * Environment / multiple targets class design * Feature/requirement requests (NICs in particular) From 00d8556f28c28a8d0865cc9d2fd1691a60ed5836 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Tue, 8 Dec 2020 10:46:35 -0800 Subject: [PATCH 107/194] Downgrade schema package to 0.7.2 manually Since 0.7.3 was yanked, this was preventing new installations. This also unfortunately included package updates, specifically isort, which changed our isort order in a couple files. --- poetry.lock | 129 ++++++++++++++++++--------------- pytest-lisa/poetry.lock | 105 ++++++++++++++------------- pytest-lisa/pyproject.toml | 2 +- pytest-playbook/poetry.lock | 43 ++++------- pytest-playbook/pyproject.toml | 2 +- pytest-target/poetry.lock | 8 +- testsuites/test_lis.py | 1 + testsuites/test_xdp.py | 1 + 8 files changed, 148 insertions(+), 143 deletions(-) diff --git a/poetry.lock b/poetry.lock index 4711bfc384..4513f0aea6 100644 --- a/poetry.lock +++ b/poetry.lock @@ -76,7 +76,7 @@ d = ["aiohttp (>=3.3.2)", "aiohttp-cors"] [[package]] name = "cffi" -version = "1.14.3" +version = "1.14.4" description = "Foreign Function Interface for Python calling C code." category = "main" optional = false @@ -194,7 +194,7 @@ flake8 = ">=3.0.0" [[package]] name = "flake8-bugbear" -version = "20.1.4" +version = "20.11.1" description = "A plugin for flake8 finding likely bugs and design problems in your program. Contains warnings that don't belong in pyflakes and pycodestyle." category = "dev" optional = false @@ -204,6 +204,9 @@ python-versions = ">=3.6" attrs = ">=19.2.0" flake8 = ">=3.0.0" +[package.extras] +dev = ["coverage", "black", "hypothesis", "hypothesmith"] + [[package]] name = "flake8-isort" version = "4.0.0" @@ -222,18 +225,18 @@ test = ["pytest (>=4.0.2,<6)", "toml"] [[package]] name = "importlib-metadata" -version = "2.0.0" +version = "3.1.1" description = "Read metadata from Python packages" category = "main" optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" +python-versions = ">=3.6" [package.dependencies] zipp = ">=0.5" [package.extras] -docs = ["sphinx", "rst.linker"] -testing = ["packaging", "pep517", "importlib-resources (>=1.3)"] +docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"] +testing = ["pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake8", "pytest-cov", "jaraco.test (>=3.2.0)", "packaging", "pep517", "pyfakefs", "flufl.flake8", "pytest-black (>=0.3.7)", "pytest-mypy", "importlib-resources (>=1.3)"] [[package]] name = "iniconfig" @@ -313,7 +316,7 @@ python-versions = "*" [[package]] name = "packaging" -version = "20.4" +version = "20.7" description = "Core utilities for Python packages" category = "main" optional = false @@ -321,7 +324,6 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [package.dependencies] pyparsing = ">=2.0.2" -six = "*" [[package]] name = "paramiko" @@ -546,7 +548,7 @@ develop = true pytest = "^6.1.2" pytest-playbook = "0.1.0" pytest-target = "0.1.0" -schema = "^0.7.3" +schema = "0.7.2" [package.source] type = "directory" @@ -554,7 +556,7 @@ url = "pytest-lisa" [[package]] name = "pytest-metadata" -version = "1.10.0" +version = "1.11.0" description = "pytest plugin for test session metadata" category = "main" optional = false @@ -591,7 +593,7 @@ develop = true [package.dependencies] pytest = "^6.1.2" PyYAML = "^5.3.1" -schema = "^0.7.3" +schema = "0.7.2" [package.source] type = "directory" @@ -725,7 +727,7 @@ dev = ["pytest"] [[package]] name = "schema" -version = "0.7.3" +version = "0.7.2" description = "Simple data validation library" category = "main" optional = false @@ -848,42 +850,42 @@ black = [ {file = "black-20.8b1.tar.gz", hash = "sha256:1c02557aa099101b9d21496f8a914e9ed2222ef70336404eeeac8edba836fbea"}, ] cffi = [ - {file = "cffi-1.14.3-2-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:3eeeb0405fd145e714f7633a5173318bd88d8bbfc3dd0a5751f8c4f70ae629bc"}, - {file = "cffi-1.14.3-2-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:cb763ceceae04803adcc4e2d80d611ef201c73da32d8f2722e9d0ab0c7f10768"}, - {file = "cffi-1.14.3-2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:44f60519595eaca110f248e5017363d751b12782a6f2bd6a7041cba275215f5d"}, - {file = "cffi-1.14.3-2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c53af463f4a40de78c58b8b2710ade243c81cbca641e34debf3396a9640d6ec1"}, - {file = "cffi-1.14.3-2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:33c6cdc071ba5cd6d96769c8969a0531be2d08c2628a0143a10a7dcffa9719ca"}, - {file = "cffi-1.14.3-2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c11579638288e53fc94ad60022ff1b67865363e730ee41ad5e6f0a17188b327a"}, - {file = "cffi-1.14.3-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:3cb3e1b9ec43256c4e0f8d2837267a70b0e1ca8c4f456685508ae6106b1f504c"}, - {file = "cffi-1.14.3-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:f0620511387790860b249b9241c2f13c3a80e21a73e0b861a2df24e9d6f56730"}, - {file = "cffi-1.14.3-cp27-cp27m-win32.whl", hash = "sha256:005f2bfe11b6745d726dbb07ace4d53f057de66e336ff92d61b8c7e9c8f4777d"}, - {file = "cffi-1.14.3-cp27-cp27m-win_amd64.whl", hash = "sha256:2f9674623ca39c9ebe38afa3da402e9326c245f0f5ceff0623dccdac15023e05"}, - {file = "cffi-1.14.3-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:09e96138280241bd355cd585148dec04dbbedb4f46128f340d696eaafc82dd7b"}, - {file = "cffi-1.14.3-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:3363e77a6176afb8823b6e06db78c46dbc4c7813b00a41300a4873b6ba63b171"}, - {file = "cffi-1.14.3-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:0ef488305fdce2580c8b2708f22d7785ae222d9825d3094ab073e22e93dfe51f"}, - {file = "cffi-1.14.3-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:0b1ad452cc824665ddc682400b62c9e4f5b64736a2ba99110712fdee5f2505c4"}, - {file = "cffi-1.14.3-cp35-cp35m-win32.whl", hash = "sha256:85ba797e1de5b48aa5a8427b6ba62cf69607c18c5d4eb747604b7302f1ec382d"}, - {file = "cffi-1.14.3-cp35-cp35m-win_amd64.whl", hash = "sha256:e66399cf0fc07de4dce4f588fc25bfe84a6d1285cc544e67987d22663393926d"}, - {file = "cffi-1.14.3-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:15f351bed09897fbda218e4db5a3d5c06328862f6198d4fb385f3e14e19decb3"}, - {file = "cffi-1.14.3-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:4d7c26bfc1ea9f92084a1d75e11999e97b62d63128bcc90c3624d07813c52808"}, - {file = "cffi-1.14.3-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:23e5d2040367322824605bc29ae8ee9175200b92cb5483ac7d466927a9b3d537"}, - {file = "cffi-1.14.3-cp36-cp36m-win32.whl", hash = "sha256:a624fae282e81ad2e4871bdb767e2c914d0539708c0f078b5b355258293c98b0"}, - {file = "cffi-1.14.3-cp36-cp36m-win_amd64.whl", hash = "sha256:de31b5164d44ef4943db155b3e8e17929707cac1e5bd2f363e67a56e3af4af6e"}, - {file = "cffi-1.14.3-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:f92cdecb618e5fa4658aeb97d5eb3d2f47aa94ac6477c6daf0f306c5a3b9e6b1"}, - {file = "cffi-1.14.3-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:22399ff4870fb4c7ef19fff6eeb20a8bbf15571913c181c78cb361024d574579"}, - {file = "cffi-1.14.3-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:f4eae045e6ab2bb54ca279733fe4eb85f1effda392666308250714e01907f394"}, - {file = "cffi-1.14.3-cp37-cp37m-win32.whl", hash = "sha256:b0358e6fefc74a16f745afa366acc89f979040e0cbc4eec55ab26ad1f6a9bfbc"}, - {file = "cffi-1.14.3-cp37-cp37m-win_amd64.whl", hash = "sha256:6642f15ad963b5092d65aed022d033c77763515fdc07095208f15d3563003869"}, - {file = "cffi-1.14.3-cp38-cp38-manylinux1_i686.whl", hash = "sha256:2791f68edc5749024b4722500e86303a10d342527e1e3bcac47f35fbd25b764e"}, - {file = "cffi-1.14.3-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:529c4ed2e10437c205f38f3691a68be66c39197d01062618c55f74294a4a4828"}, - {file = "cffi-1.14.3-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:8f0f1e499e4000c4c347a124fa6a27d37608ced4fe9f7d45070563b7c4c370c9"}, - {file = "cffi-1.14.3-cp38-cp38-win32.whl", hash = "sha256:3b8eaf915ddc0709779889c472e553f0d3e8b7bdf62dab764c8921b09bf94522"}, - {file = "cffi-1.14.3-cp38-cp38-win_amd64.whl", hash = "sha256:bbd2f4dfee1079f76943767fce837ade3087b578aeb9f69aec7857d5bf25db15"}, - {file = "cffi-1.14.3-cp39-cp39-manylinux1_i686.whl", hash = "sha256:cc75f58cdaf043fe6a7a6c04b3b5a0e694c6a9e24050967747251fb80d7bce0d"}, - {file = "cffi-1.14.3-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:bf39a9e19ce7298f1bd6a9758fa99707e9e5b1ebe5e90f2c3913a47bc548747c"}, - {file = "cffi-1.14.3-cp39-cp39-win32.whl", hash = "sha256:d80998ed59176e8cba74028762fbd9b9153b9afc71ea118e63bbf5d4d0f9552b"}, - {file = "cffi-1.14.3-cp39-cp39-win_amd64.whl", hash = "sha256:c150eaa3dadbb2b5339675b88d4573c1be3cb6f2c33a6c83387e10cc0bf05bd3"}, - {file = "cffi-1.14.3.tar.gz", hash = "sha256:f92f789e4f9241cd262ad7a555ca2c648a98178a953af117ef7fad46aa1d5591"}, + {file = "cffi-1.14.4-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:ebb253464a5d0482b191274f1c8bf00e33f7e0b9c66405fbffc61ed2c839c775"}, + {file = "cffi-1.14.4-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:2c24d61263f511551f740d1a065eb0212db1dbbbbd241db758f5244281590c06"}, + {file = "cffi-1.14.4-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:9f7a31251289b2ab6d4012f6e83e58bc3b96bd151f5b5262467f4bb6b34a7c26"}, + {file = "cffi-1.14.4-cp27-cp27m-win32.whl", hash = "sha256:5cf4be6c304ad0b6602f5c4e90e2f59b47653ac1ed9c662ed379fe48a8f26b0c"}, + {file = "cffi-1.14.4-cp27-cp27m-win_amd64.whl", hash = "sha256:f60567825f791c6f8a592f3c6e3bd93dd2934e3f9dac189308426bd76b00ef3b"}, + {file = "cffi-1.14.4-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:c6332685306b6417a91b1ff9fae889b3ba65c2292d64bd9245c093b1b284809d"}, + {file = "cffi-1.14.4-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:d9efd8b7a3ef378dd61a1e77367f1924375befc2eba06168b6ebfa903a5e59ca"}, + {file = "cffi-1.14.4-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:51a8b381b16ddd370178a65360ebe15fbc1c71cf6f584613a7ea08bfad946698"}, + {file = "cffi-1.14.4-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:1d2c4994f515e5b485fd6d3a73d05526aa0fcf248eb135996b088d25dfa1865b"}, + {file = "cffi-1.14.4-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:af5c59122a011049aad5dd87424b8e65a80e4a6477419c0c1015f73fb5ea0293"}, + {file = "cffi-1.14.4-cp35-cp35m-win32.whl", hash = "sha256:594234691ac0e9b770aee9fcdb8fa02c22e43e5c619456efd0d6c2bf276f3eb2"}, + {file = "cffi-1.14.4-cp35-cp35m-win_amd64.whl", hash = "sha256:64081b3f8f6f3c3de6191ec89d7dc6c86a8a43911f7ecb422c60e90c70be41c7"}, + {file = "cffi-1.14.4-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:f803eaa94c2fcda012c047e62bc7a51b0bdabda1cad7a92a522694ea2d76e49f"}, + {file = "cffi-1.14.4-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:105abaf8a6075dc96c1fe5ae7aae073f4696f2905fde6aeada4c9d2926752362"}, + {file = "cffi-1.14.4-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:0638c3ae1a0edfb77c6765d487fee624d2b1ee1bdfeffc1f0b58c64d149e7eec"}, + {file = "cffi-1.14.4-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:7c6b1dece89874d9541fc974917b631406233ea0440d0bdfbb8e03bf39a49b3b"}, + {file = "cffi-1.14.4-cp36-cp36m-win32.whl", hash = "sha256:155136b51fd733fa94e1c2ea5211dcd4c8879869008fc811648f16541bf99668"}, + {file = "cffi-1.14.4-cp36-cp36m-win_amd64.whl", hash = "sha256:6bc25fc545a6b3d57b5f8618e59fc13d3a3a68431e8ca5fd4c13241cd70d0009"}, + {file = "cffi-1.14.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a7711edca4dcef1a75257b50a2fbfe92a65187c47dab5a0f1b9b332c5919a3fb"}, + {file = "cffi-1.14.4-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:00e28066507bfc3fe865a31f325c8391a1ac2916219340f87dfad602c3e48e5d"}, + {file = "cffi-1.14.4-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:798caa2a2384b1cbe8a2a139d80734c9db54f9cc155c99d7cc92441a23871c03"}, + {file = "cffi-1.14.4-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:a5ed8c05548b54b998b9498753fb9cadbfd92ee88e884641377d8a8b291bcc01"}, + {file = "cffi-1.14.4-cp37-cp37m-win32.whl", hash = "sha256:00a1ba5e2e95684448de9b89888ccd02c98d512064b4cb987d48f4b40aa0421e"}, + {file = "cffi-1.14.4-cp37-cp37m-win_amd64.whl", hash = "sha256:9cc46bc107224ff5b6d04369e7c595acb700c3613ad7bcf2e2012f62ece80c35"}, + {file = "cffi-1.14.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:df5169c4396adc04f9b0a05f13c074df878b6052430e03f50e68adf3a57aa28d"}, + {file = "cffi-1.14.4-cp38-cp38-manylinux1_i686.whl", hash = "sha256:9ffb888f19d54a4d4dfd4b3f29bc2c16aa4972f1c2ab9c4ab09b8ab8685b9c2b"}, + {file = "cffi-1.14.4-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:8d6603078baf4e11edc4168a514c5ce5b3ba6e3e9c374298cb88437957960a53"}, + {file = "cffi-1.14.4-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:d5ff0621c88ce83a28a10d2ce719b2ee85635e85c515f12bac99a95306da4b2e"}, + {file = "cffi-1.14.4-cp38-cp38-win32.whl", hash = "sha256:b4e248d1087abf9f4c10f3c398896c87ce82a9856494a7155823eb45a892395d"}, + {file = "cffi-1.14.4-cp38-cp38-win_amd64.whl", hash = "sha256:ec80dc47f54e6e9a78181ce05feb71a0353854cc26999db963695f950b5fb375"}, + {file = "cffi-1.14.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:840793c68105fe031f34d6a086eaea153a0cd5c491cde82a74b420edd0a2b909"}, + {file = "cffi-1.14.4-cp39-cp39-manylinux1_i686.whl", hash = "sha256:b18e0a9ef57d2b41f5c68beefa32317d286c3d6ac0484efd10d6e07491bb95dd"}, + {file = "cffi-1.14.4-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:045d792900a75e8b1e1b0ab6787dd733a8190ffcf80e8c8ceb2fb10a29ff238a"}, + {file = "cffi-1.14.4-cp39-cp39-win32.whl", hash = "sha256:ba4e9e0ae13fc41c6b23299545e5ef73055213e466bd107953e4a013a5ddd7e3"}, + {file = "cffi-1.14.4-cp39-cp39-win_amd64.whl", hash = "sha256:f032b34669220030f905152045dfa27741ce1a6db3324a5bc0b96b6c7420c87b"}, + {file = "cffi-1.14.4.tar.gz", hash = "sha256:1a465cbe98a7fd391d47dce4b8f7e5b921e6cd805ef421d04f5f66ba8f06086c"}, ] click = [ {file = "click-7.1.2-py2.py3-none-any.whl", hash = "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc"}, @@ -941,16 +943,16 @@ flake8-black = [ {file = "flake8-black-0.2.1.tar.gz", hash = "sha256:f26651bc10db786c03f4093414f7c9ea982ed8a244cec323c984feeffdf4c118"}, ] flake8-bugbear = [ - {file = "flake8-bugbear-20.1.4.tar.gz", hash = "sha256:bd02e4b009fb153fe6072c31c52aeab5b133d508095befb2ffcf3b41c4823162"}, - {file = "flake8_bugbear-20.1.4-py36.py37.py38-none-any.whl", hash = "sha256:a3ddc03ec28ba2296fc6f89444d1c946a6b76460f859795b35b77d4920a51b63"}, + {file = "flake8-bugbear-20.11.1.tar.gz", hash = "sha256:528020129fea2dea33a466b9d64ab650aa3e5f9ffc788b70ea4bc6cf18283538"}, + {file = "flake8_bugbear-20.11.1-py36.py37.py38-none-any.whl", hash = "sha256:f35b8135ece7a014bc0aee5b5d485334ac30a6da48494998cc1fabf7ec70d703"}, ] flake8-isort = [ {file = "flake8-isort-4.0.0.tar.gz", hash = "sha256:2b91300f4f1926b396c2c90185844eb1a3d5ec39ea6138832d119da0a208f4d9"}, {file = "flake8_isort-4.0.0-py2.py3-none-any.whl", hash = "sha256:729cd6ef9ba3659512dee337687c05d79c78e1215fdf921ed67e5fe46cce2f3c"}, ] importlib-metadata = [ - {file = "importlib_metadata-2.0.0-py2.py3-none-any.whl", hash = "sha256:cefa1a2f919b866c5beb7c9f7b0ebb4061f30a8a9bf16d609b000e2dfaceb9c3"}, - {file = "importlib_metadata-2.0.0.tar.gz", hash = "sha256:77a540690e24b0305878c37ffd421785a6f7e53c8b5720d211b211de8d0e95da"}, + {file = "importlib_metadata-3.1.1-py3-none-any.whl", hash = "sha256:6112e21359ef8f344e7178aa5b72dc6e62b38b0d008e6d3cb212c5b84df72013"}, + {file = "importlib_metadata-3.1.1.tar.gz", hash = "sha256:b0c2d3b226157ae4517d9625decf63591461c66b3a808c2666d538946519d170"}, ] iniconfig = [ {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, @@ -994,8 +996,8 @@ mypy-extensions = [ {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"}, ] packaging = [ - {file = "packaging-20.4-py2.py3-none-any.whl", hash = "sha256:998416ba6962ae7fbd6596850b80e17859a5753ba17c32284f67bfff33784181"}, - {file = "packaging-20.4.tar.gz", hash = "sha256:4357f74f47b9c12db93624a82154e9b120fa8293699949152b22065d556079f8"}, + {file = "packaging-20.7-py2.py3-none-any.whl", hash = "sha256:eb41423378682dadb7166144a4926e443093863024de508ca5c9737d6bc08376"}, + {file = "packaging-20.7.tar.gz", hash = "sha256:05af3bb85d320377db281cf254ab050e1a7ebcbf5410685a9a407e18a1f81236"}, ] paramiko = [ {file = "paramiko-2.7.2-py2.py3-none-any.whl", hash = "sha256:4f3e316fef2ac628b05097a637af35685183111d4bc1b5979bd397c2ab7b5898"}, @@ -1081,8 +1083,8 @@ pytest-html = [ ] pytest-lisa = [] pytest-metadata = [ - {file = "pytest-metadata-1.10.0.tar.gz", hash = "sha256:b7e6e0a45adacb17a03a97bf7a2ef60cc1f4e172bcce9732ce5e814191932315"}, - {file = "pytest_metadata-1.10.0-py2.py3-none-any.whl", hash = "sha256:fcbcc5781aee450107c620c79c57e50796b6777b82b3c504be9cbc3017201169"}, + {file = "pytest-metadata-1.11.0.tar.gz", hash = "sha256:71b506d49d34e539cc3cfdb7ce2c5f072bea5c953320002c95968e0238f8ecf1"}, + {file = "pytest_metadata-1.11.0-py2.py3-none-any.whl", hash = "sha256:576055b8336dd4a9006dd2a47615f76f2f8c30ab12b1b1c039d99e834583523f"}, ] pytest-mypy = [ {file = "pytest-mypy-0.7.0.tar.gz", hash = "sha256:5a667d9a2b66bf98b3a494411f221923a6e2c3eafbe771104951aaec8985673d"}, @@ -1121,6 +1123,8 @@ pyyaml = [ {file = "PyYAML-5.3.1-cp37-cp37m-win_amd64.whl", hash = "sha256:73f099454b799e05e5ab51423c7bcf361c58d3206fa7b0d555426b1f4d9a3eaf"}, {file = "PyYAML-5.3.1-cp38-cp38-win32.whl", hash = "sha256:06a0d7ba600ce0b2d2fe2e78453a470b5a6e000a985dd4a4e54e436cc36b0e97"}, {file = "PyYAML-5.3.1-cp38-cp38-win_amd64.whl", hash = "sha256:95f71d2af0ff4227885f7a6605c37fd53d3a106fcab511b8860ecca9fcf400ee"}, + {file = "PyYAML-5.3.1-cp39-cp39-win32.whl", hash = "sha256:ad9c67312c84def58f3c04504727ca879cb0013b2517c85a9a253f0cb6380c0a"}, + {file = "PyYAML-5.3.1-cp39-cp39-win_amd64.whl", hash = "sha256:6034f55dab5fea9e53f436aa68fa3ace2634918e8b5994d82f3621c04ff5ed2e"}, {file = "PyYAML-5.3.1.tar.gz", hash = "sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d"}, ] regex = [ @@ -1170,8 +1174,8 @@ rope = [ {file = "rope-0.18.0.tar.gz", hash = "sha256:786b5c38c530d4846aa68a42604f61b4e69a493390e3ca11b88df0fbfdc3ed04"}, ] schema = [ - {file = "schema-0.7.3-py2.py3-none-any.whl", hash = "sha256:c331438b60f634cab5664ab720d3083cc444f924d55269530c36b33e3354276f"}, - {file = "schema-0.7.3.tar.gz", hash = "sha256:4cf529318cfd1e844ecbe02f41f7e5aa027463e7403666a52746f31f04f47a5e"}, + {file = "schema-0.7.2-py2.py3-none-any.whl", hash = "sha256:3a03c2e2b22e6a331ae73750ab1da46916da6ca861b16e6f073ac1d1eba43b71"}, + {file = "schema-0.7.2.tar.gz", hash = "sha256:b536f2375b49fdf56f36279addae98bd86a8afbd58b3c32ce363c464bed5fc1c"}, ] six = [ {file = "six-1.15.0-py2.py3-none-any.whl", hash = "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"}, @@ -1197,19 +1201,28 @@ typed-ast = [ {file = "typed_ast-1.4.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:269151951236b0f9a6f04015a9004084a5ab0d5f19b57de779f908621e7d8b75"}, {file = "typed_ast-1.4.1-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:24995c843eb0ad11a4527b026b4dde3da70e1f2d8806c99b7b4a7cf491612652"}, {file = "typed_ast-1.4.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:fe460b922ec15dd205595c9b5b99e2f056fd98ae8f9f56b888e7a17dc2b757e7"}, + {file = "typed_ast-1.4.1-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:fcf135e17cc74dbfbc05894ebca928ffeb23d9790b3167a674921db19082401f"}, {file = "typed_ast-1.4.1-cp36-cp36m-win32.whl", hash = "sha256:4e3e5da80ccbebfff202a67bf900d081906c358ccc3d5e3c8aea42fdfdfd51c1"}, {file = "typed_ast-1.4.1-cp36-cp36m-win_amd64.whl", hash = "sha256:249862707802d40f7f29f6e1aad8d84b5aa9e44552d2cc17384b209f091276aa"}, {file = "typed_ast-1.4.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8ce678dbaf790dbdb3eba24056d5364fb45944f33553dd5869b7580cdbb83614"}, {file = "typed_ast-1.4.1-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:c9e348e02e4d2b4a8b2eedb48210430658df6951fa484e59de33ff773fbd4b41"}, {file = "typed_ast-1.4.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:bcd3b13b56ea479b3650b82cabd6b5343a625b0ced5429e4ccad28a8973f301b"}, + {file = "typed_ast-1.4.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:f208eb7aff048f6bea9586e61af041ddf7f9ade7caed625742af423f6bae3298"}, {file = "typed_ast-1.4.1-cp37-cp37m-win32.whl", hash = "sha256:d5d33e9e7af3b34a40dc05f498939f0ebf187f07c385fd58d591c533ad8562fe"}, {file = "typed_ast-1.4.1-cp37-cp37m-win_amd64.whl", hash = "sha256:0666aa36131496aed8f7be0410ff974562ab7eeac11ef351def9ea6fa28f6355"}, {file = "typed_ast-1.4.1-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:d205b1b46085271b4e15f670058ce182bd1199e56b317bf2ec004b6a44f911f6"}, {file = "typed_ast-1.4.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:6daac9731f172c2a22ade6ed0c00197ee7cc1221aa84cfdf9c31defeb059a907"}, {file = "typed_ast-1.4.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:498b0f36cc7054c1fead3d7fc59d2150f4d5c6c56ba7fb150c013fbc683a8d2d"}, + {file = "typed_ast-1.4.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:7e4c9d7658aaa1fc80018593abdf8598bf91325af6af5cce4ce7c73bc45ea53d"}, {file = "typed_ast-1.4.1-cp38-cp38-win32.whl", hash = "sha256:715ff2f2df46121071622063fc7543d9b1fd19ebfc4f5c8895af64a77a8c852c"}, {file = "typed_ast-1.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:fc0fea399acb12edbf8a628ba8d2312f583bdbdb3335635db062fa98cf71fca4"}, {file = "typed_ast-1.4.1-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:d43943ef777f9a1c42bf4e552ba23ac77a6351de620aa9acf64ad54933ad4d34"}, + {file = "typed_ast-1.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:92c325624e304ebf0e025d1224b77dd4e6393f18aab8d829b5b7e04afe9b7a2c"}, + {file = "typed_ast-1.4.1-cp39-cp39-manylinux1_i686.whl", hash = "sha256:d648b8e3bf2fe648745c8ffcee3db3ff903d0817a01a12dd6a6ea7a8f4889072"}, + {file = "typed_ast-1.4.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:fac11badff8313e23717f3dada86a15389d0708275bddf766cca67a84ead3e91"}, + {file = "typed_ast-1.4.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:0d8110d78a5736e16e26213114a38ca35cb15b6515d535413b090bd50951556d"}, + {file = "typed_ast-1.4.1-cp39-cp39-win32.whl", hash = "sha256:b52ccf7cfe4ce2a1064b18594381bccf4179c2ecf7f513134ec2f993dd4ab395"}, + {file = "typed_ast-1.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:3742b32cf1c6ef124d57f95be609c473d7ec4c14d0090e5a5e05a15269fb4d0c"}, {file = "typed_ast-1.4.1.tar.gz", hash = "sha256:8c8aaad94455178e3187ab22c8b01a3837f8ee50e09cf31f1ba129eb293ec30b"}, ] typing-extensions = [ diff --git a/pytest-lisa/poetry.lock b/pytest-lisa/poetry.lock index 9c1579df82..a8e852d5e6 100644 --- a/pytest-lisa/poetry.lock +++ b/pytest-lisa/poetry.lock @@ -38,7 +38,7 @@ typecheck = ["mypy"] [[package]] name = "cffi" -version = "1.14.3" +version = "1.14.4" description = "Foreign Function Interface for Python calling C code." category = "main" optional = false @@ -100,18 +100,18 @@ testing = ["mock (>=2.0.0,<3.0)"] [[package]] name = "importlib-metadata" -version = "2.0.0" +version = "3.1.1" description = "Read metadata from Python packages" category = "main" optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" +python-versions = ">=3.6" [package.dependencies] zipp = ">=0.5" [package.extras] -docs = ["sphinx", "rst.linker"] -testing = ["packaging", "pep517", "importlib-resources (>=1.3)"] +docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"] +testing = ["pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake8", "pytest-cov", "jaraco.test (>=3.2.0)", "packaging", "pep517", "pyfakefs", "flufl.flake8", "pytest-black (>=0.3.7)", "pytest-mypy", "importlib-resources (>=1.3)"] [[package]] name = "iniconfig" @@ -131,7 +131,7 @@ python-versions = "*" [[package]] name = "packaging" -version = "20.4" +version = "20.7" description = "Core utilities for Python packages" category = "main" optional = false @@ -139,7 +139,6 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [package.dependencies] pyparsing = ">=2.0.2" -six = "*" [[package]] name = "paramiko" @@ -249,7 +248,7 @@ develop = true [package.dependencies] pytest = "^6.1.2" PyYAML = "^5.3.1" -schema = "^0.7.3" +schema = "0.7.2" [package.source] type = "directory" @@ -285,7 +284,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [[package]] name = "schema" -version = "0.7.3" +version = "0.7.2" description = "Simple data validation library" category = "main" optional = false @@ -339,7 +338,7 @@ testing = ["pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "bc55352e00a00a309afc084b0584068709a437c282532207c1f0d1ea623e7588" +content-hash = "333ba43473867f73d807a941a7dc7ec215c16470cb489bd6fa0033e8607fb532" [metadata.files] atomicwrites = [ @@ -360,42 +359,42 @@ bcrypt = [ {file = "bcrypt-3.2.0.tar.gz", hash = "sha256:5b93c1726e50a93a033c36e5ca7fdcd29a5c7395af50a6892f5d9e7c6cfbfb29"}, ] cffi = [ - {file = "cffi-1.14.3-2-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:3eeeb0405fd145e714f7633a5173318bd88d8bbfc3dd0a5751f8c4f70ae629bc"}, - {file = "cffi-1.14.3-2-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:cb763ceceae04803adcc4e2d80d611ef201c73da32d8f2722e9d0ab0c7f10768"}, - {file = "cffi-1.14.3-2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:44f60519595eaca110f248e5017363d751b12782a6f2bd6a7041cba275215f5d"}, - {file = "cffi-1.14.3-2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c53af463f4a40de78c58b8b2710ade243c81cbca641e34debf3396a9640d6ec1"}, - {file = "cffi-1.14.3-2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:33c6cdc071ba5cd6d96769c8969a0531be2d08c2628a0143a10a7dcffa9719ca"}, - {file = "cffi-1.14.3-2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c11579638288e53fc94ad60022ff1b67865363e730ee41ad5e6f0a17188b327a"}, - {file = "cffi-1.14.3-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:3cb3e1b9ec43256c4e0f8d2837267a70b0e1ca8c4f456685508ae6106b1f504c"}, - {file = "cffi-1.14.3-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:f0620511387790860b249b9241c2f13c3a80e21a73e0b861a2df24e9d6f56730"}, - {file = "cffi-1.14.3-cp27-cp27m-win32.whl", hash = "sha256:005f2bfe11b6745d726dbb07ace4d53f057de66e336ff92d61b8c7e9c8f4777d"}, - {file = "cffi-1.14.3-cp27-cp27m-win_amd64.whl", hash = "sha256:2f9674623ca39c9ebe38afa3da402e9326c245f0f5ceff0623dccdac15023e05"}, - {file = "cffi-1.14.3-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:09e96138280241bd355cd585148dec04dbbedb4f46128f340d696eaafc82dd7b"}, - {file = "cffi-1.14.3-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:3363e77a6176afb8823b6e06db78c46dbc4c7813b00a41300a4873b6ba63b171"}, - {file = "cffi-1.14.3-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:0ef488305fdce2580c8b2708f22d7785ae222d9825d3094ab073e22e93dfe51f"}, - {file = "cffi-1.14.3-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:0b1ad452cc824665ddc682400b62c9e4f5b64736a2ba99110712fdee5f2505c4"}, - {file = "cffi-1.14.3-cp35-cp35m-win32.whl", hash = "sha256:85ba797e1de5b48aa5a8427b6ba62cf69607c18c5d4eb747604b7302f1ec382d"}, - {file = "cffi-1.14.3-cp35-cp35m-win_amd64.whl", hash = "sha256:e66399cf0fc07de4dce4f588fc25bfe84a6d1285cc544e67987d22663393926d"}, - {file = "cffi-1.14.3-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:15f351bed09897fbda218e4db5a3d5c06328862f6198d4fb385f3e14e19decb3"}, - {file = "cffi-1.14.3-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:4d7c26bfc1ea9f92084a1d75e11999e97b62d63128bcc90c3624d07813c52808"}, - {file = "cffi-1.14.3-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:23e5d2040367322824605bc29ae8ee9175200b92cb5483ac7d466927a9b3d537"}, - {file = "cffi-1.14.3-cp36-cp36m-win32.whl", hash = "sha256:a624fae282e81ad2e4871bdb767e2c914d0539708c0f078b5b355258293c98b0"}, - {file = "cffi-1.14.3-cp36-cp36m-win_amd64.whl", hash = "sha256:de31b5164d44ef4943db155b3e8e17929707cac1e5bd2f363e67a56e3af4af6e"}, - {file = "cffi-1.14.3-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:f92cdecb618e5fa4658aeb97d5eb3d2f47aa94ac6477c6daf0f306c5a3b9e6b1"}, - {file = "cffi-1.14.3-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:22399ff4870fb4c7ef19fff6eeb20a8bbf15571913c181c78cb361024d574579"}, - {file = "cffi-1.14.3-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:f4eae045e6ab2bb54ca279733fe4eb85f1effda392666308250714e01907f394"}, - {file = "cffi-1.14.3-cp37-cp37m-win32.whl", hash = "sha256:b0358e6fefc74a16f745afa366acc89f979040e0cbc4eec55ab26ad1f6a9bfbc"}, - {file = "cffi-1.14.3-cp37-cp37m-win_amd64.whl", hash = "sha256:6642f15ad963b5092d65aed022d033c77763515fdc07095208f15d3563003869"}, - {file = "cffi-1.14.3-cp38-cp38-manylinux1_i686.whl", hash = "sha256:2791f68edc5749024b4722500e86303a10d342527e1e3bcac47f35fbd25b764e"}, - {file = "cffi-1.14.3-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:529c4ed2e10437c205f38f3691a68be66c39197d01062618c55f74294a4a4828"}, - {file = "cffi-1.14.3-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:8f0f1e499e4000c4c347a124fa6a27d37608ced4fe9f7d45070563b7c4c370c9"}, - {file = "cffi-1.14.3-cp38-cp38-win32.whl", hash = "sha256:3b8eaf915ddc0709779889c472e553f0d3e8b7bdf62dab764c8921b09bf94522"}, - {file = "cffi-1.14.3-cp38-cp38-win_amd64.whl", hash = "sha256:bbd2f4dfee1079f76943767fce837ade3087b578aeb9f69aec7857d5bf25db15"}, - {file = "cffi-1.14.3-cp39-cp39-manylinux1_i686.whl", hash = "sha256:cc75f58cdaf043fe6a7a6c04b3b5a0e694c6a9e24050967747251fb80d7bce0d"}, - {file = "cffi-1.14.3-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:bf39a9e19ce7298f1bd6a9758fa99707e9e5b1ebe5e90f2c3913a47bc548747c"}, - {file = "cffi-1.14.3-cp39-cp39-win32.whl", hash = "sha256:d80998ed59176e8cba74028762fbd9b9153b9afc71ea118e63bbf5d4d0f9552b"}, - {file = "cffi-1.14.3-cp39-cp39-win_amd64.whl", hash = "sha256:c150eaa3dadbb2b5339675b88d4573c1be3cb6f2c33a6c83387e10cc0bf05bd3"}, - {file = "cffi-1.14.3.tar.gz", hash = "sha256:f92f789e4f9241cd262ad7a555ca2c648a98178a953af117ef7fad46aa1d5591"}, + {file = "cffi-1.14.4-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:ebb253464a5d0482b191274f1c8bf00e33f7e0b9c66405fbffc61ed2c839c775"}, + {file = "cffi-1.14.4-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:2c24d61263f511551f740d1a065eb0212db1dbbbbd241db758f5244281590c06"}, + {file = "cffi-1.14.4-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:9f7a31251289b2ab6d4012f6e83e58bc3b96bd151f5b5262467f4bb6b34a7c26"}, + {file = "cffi-1.14.4-cp27-cp27m-win32.whl", hash = "sha256:5cf4be6c304ad0b6602f5c4e90e2f59b47653ac1ed9c662ed379fe48a8f26b0c"}, + {file = "cffi-1.14.4-cp27-cp27m-win_amd64.whl", hash = "sha256:f60567825f791c6f8a592f3c6e3bd93dd2934e3f9dac189308426bd76b00ef3b"}, + {file = "cffi-1.14.4-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:c6332685306b6417a91b1ff9fae889b3ba65c2292d64bd9245c093b1b284809d"}, + {file = "cffi-1.14.4-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:d9efd8b7a3ef378dd61a1e77367f1924375befc2eba06168b6ebfa903a5e59ca"}, + {file = "cffi-1.14.4-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:51a8b381b16ddd370178a65360ebe15fbc1c71cf6f584613a7ea08bfad946698"}, + {file = "cffi-1.14.4-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:1d2c4994f515e5b485fd6d3a73d05526aa0fcf248eb135996b088d25dfa1865b"}, + {file = "cffi-1.14.4-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:af5c59122a011049aad5dd87424b8e65a80e4a6477419c0c1015f73fb5ea0293"}, + {file = "cffi-1.14.4-cp35-cp35m-win32.whl", hash = "sha256:594234691ac0e9b770aee9fcdb8fa02c22e43e5c619456efd0d6c2bf276f3eb2"}, + {file = "cffi-1.14.4-cp35-cp35m-win_amd64.whl", hash = "sha256:64081b3f8f6f3c3de6191ec89d7dc6c86a8a43911f7ecb422c60e90c70be41c7"}, + {file = "cffi-1.14.4-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:f803eaa94c2fcda012c047e62bc7a51b0bdabda1cad7a92a522694ea2d76e49f"}, + {file = "cffi-1.14.4-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:105abaf8a6075dc96c1fe5ae7aae073f4696f2905fde6aeada4c9d2926752362"}, + {file = "cffi-1.14.4-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:0638c3ae1a0edfb77c6765d487fee624d2b1ee1bdfeffc1f0b58c64d149e7eec"}, + {file = "cffi-1.14.4-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:7c6b1dece89874d9541fc974917b631406233ea0440d0bdfbb8e03bf39a49b3b"}, + {file = "cffi-1.14.4-cp36-cp36m-win32.whl", hash = "sha256:155136b51fd733fa94e1c2ea5211dcd4c8879869008fc811648f16541bf99668"}, + {file = "cffi-1.14.4-cp36-cp36m-win_amd64.whl", hash = "sha256:6bc25fc545a6b3d57b5f8618e59fc13d3a3a68431e8ca5fd4c13241cd70d0009"}, + {file = "cffi-1.14.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a7711edca4dcef1a75257b50a2fbfe92a65187c47dab5a0f1b9b332c5919a3fb"}, + {file = "cffi-1.14.4-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:00e28066507bfc3fe865a31f325c8391a1ac2916219340f87dfad602c3e48e5d"}, + {file = "cffi-1.14.4-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:798caa2a2384b1cbe8a2a139d80734c9db54f9cc155c99d7cc92441a23871c03"}, + {file = "cffi-1.14.4-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:a5ed8c05548b54b998b9498753fb9cadbfd92ee88e884641377d8a8b291bcc01"}, + {file = "cffi-1.14.4-cp37-cp37m-win32.whl", hash = "sha256:00a1ba5e2e95684448de9b89888ccd02c98d512064b4cb987d48f4b40aa0421e"}, + {file = "cffi-1.14.4-cp37-cp37m-win_amd64.whl", hash = "sha256:9cc46bc107224ff5b6d04369e7c595acb700c3613ad7bcf2e2012f62ece80c35"}, + {file = "cffi-1.14.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:df5169c4396adc04f9b0a05f13c074df878b6052430e03f50e68adf3a57aa28d"}, + {file = "cffi-1.14.4-cp38-cp38-manylinux1_i686.whl", hash = "sha256:9ffb888f19d54a4d4dfd4b3f29bc2c16aa4972f1c2ab9c4ab09b8ab8685b9c2b"}, + {file = "cffi-1.14.4-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:8d6603078baf4e11edc4168a514c5ce5b3ba6e3e9c374298cb88437957960a53"}, + {file = "cffi-1.14.4-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:d5ff0621c88ce83a28a10d2ce719b2ee85635e85c515f12bac99a95306da4b2e"}, + {file = "cffi-1.14.4-cp38-cp38-win32.whl", hash = "sha256:b4e248d1087abf9f4c10f3c398896c87ce82a9856494a7155823eb45a892395d"}, + {file = "cffi-1.14.4-cp38-cp38-win_amd64.whl", hash = "sha256:ec80dc47f54e6e9a78181ce05feb71a0353854cc26999db963695f950b5fb375"}, + {file = "cffi-1.14.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:840793c68105fe031f34d6a086eaea153a0cd5c491cde82a74b420edd0a2b909"}, + {file = "cffi-1.14.4-cp39-cp39-manylinux1_i686.whl", hash = "sha256:b18e0a9ef57d2b41f5c68beefa32317d286c3d6ac0484efd10d6e07491bb95dd"}, + {file = "cffi-1.14.4-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:045d792900a75e8b1e1b0ab6787dd733a8190ffcf80e8c8ceb2fb10a29ff238a"}, + {file = "cffi-1.14.4-cp39-cp39-win32.whl", hash = "sha256:ba4e9e0ae13fc41c6b23299545e5ef73055213e466bd107953e4a013a5ddd7e3"}, + {file = "cffi-1.14.4-cp39-cp39-win_amd64.whl", hash = "sha256:f032b34669220030f905152045dfa27741ce1a6db3324a5bc0b96b6c7420c87b"}, + {file = "cffi-1.14.4.tar.gz", hash = "sha256:1a465cbe98a7fd391d47dce4b8f7e5b921e6cd805ef421d04f5f66ba8f06086c"}, ] colorama = [ {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"}, @@ -434,8 +433,8 @@ fabric = [ {file = "fabric-2.5.0.tar.gz", hash = "sha256:24842d7d51556adcabd885ac3cf5e1df73fc622a1708bf3667bf5927576cdfa6"}, ] importlib-metadata = [ - {file = "importlib_metadata-2.0.0-py2.py3-none-any.whl", hash = "sha256:cefa1a2f919b866c5beb7c9f7b0ebb4061f30a8a9bf16d609b000e2dfaceb9c3"}, - {file = "importlib_metadata-2.0.0.tar.gz", hash = "sha256:77a540690e24b0305878c37ffd421785a6f7e53c8b5720d211b211de8d0e95da"}, + {file = "importlib_metadata-3.1.1-py3-none-any.whl", hash = "sha256:6112e21359ef8f344e7178aa5b72dc6e62b38b0d008e6d3cb212c5b84df72013"}, + {file = "importlib_metadata-3.1.1.tar.gz", hash = "sha256:b0c2d3b226157ae4517d9625decf63591461c66b3a808c2666d538946519d170"}, ] iniconfig = [ {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, @@ -447,8 +446,8 @@ invoke = [ {file = "invoke-1.4.1.tar.gz", hash = "sha256:de3f23bfe669e3db1085789fd859eb8ca8e0c5d9c20811e2407fa042e8a5e15d"}, ] packaging = [ - {file = "packaging-20.4-py2.py3-none-any.whl", hash = "sha256:998416ba6962ae7fbd6596850b80e17859a5753ba17c32284f67bfff33784181"}, - {file = "packaging-20.4.tar.gz", hash = "sha256:4357f74f47b9c12db93624a82154e9b120fa8293699949152b22065d556079f8"}, + {file = "packaging-20.7-py2.py3-none-any.whl", hash = "sha256:eb41423378682dadb7166144a4926e443093863024de508ca5c9737d6bc08376"}, + {file = "packaging-20.7.tar.gz", hash = "sha256:05af3bb85d320377db281cf254ab050e1a7ebcbf5410685a9a407e18a1f81236"}, ] paramiko = [ {file = "paramiko-2.7.2-py2.py3-none-any.whl", hash = "sha256:4f3e316fef2ac628b05097a637af35685183111d4bc1b5979bd397c2ab7b5898"}, @@ -507,11 +506,13 @@ pyyaml = [ {file = "PyYAML-5.3.1-cp37-cp37m-win_amd64.whl", hash = "sha256:73f099454b799e05e5ab51423c7bcf361c58d3206fa7b0d555426b1f4d9a3eaf"}, {file = "PyYAML-5.3.1-cp38-cp38-win32.whl", hash = "sha256:06a0d7ba600ce0b2d2fe2e78453a470b5a6e000a985dd4a4e54e436cc36b0e97"}, {file = "PyYAML-5.3.1-cp38-cp38-win_amd64.whl", hash = "sha256:95f71d2af0ff4227885f7a6605c37fd53d3a106fcab511b8860ecca9fcf400ee"}, + {file = "PyYAML-5.3.1-cp39-cp39-win32.whl", hash = "sha256:ad9c67312c84def58f3c04504727ca879cb0013b2517c85a9a253f0cb6380c0a"}, + {file = "PyYAML-5.3.1-cp39-cp39-win_amd64.whl", hash = "sha256:6034f55dab5fea9e53f436aa68fa3ace2634918e8b5994d82f3621c04ff5ed2e"}, {file = "PyYAML-5.3.1.tar.gz", hash = "sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d"}, ] schema = [ - {file = "schema-0.7.3-py2.py3-none-any.whl", hash = "sha256:c331438b60f634cab5664ab720d3083cc444f924d55269530c36b33e3354276f"}, - {file = "schema-0.7.3.tar.gz", hash = "sha256:4cf529318cfd1e844ecbe02f41f7e5aa027463e7403666a52746f31f04f47a5e"}, + {file = "schema-0.7.2-py2.py3-none-any.whl", hash = "sha256:3a03c2e2b22e6a331ae73750ab1da46916da6ca861b16e6f073ac1d1eba43b71"}, + {file = "schema-0.7.2.tar.gz", hash = "sha256:b536f2375b49fdf56f36279addae98bd86a8afbd58b3c32ce363c464bed5fc1c"}, ] six = [ {file = "six-1.15.0-py2.py3-none-any.whl", hash = "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"}, diff --git a/pytest-lisa/pyproject.toml b/pytest-lisa/pyproject.toml index 97b61056a4..53dfad4e5a 100644 --- a/pytest-lisa/pyproject.toml +++ b/pytest-lisa/pyproject.toml @@ -12,7 +12,7 @@ python = "^3.7" pytest = "^6.1.2" pytest-playbook = {path = "../pytest-playbook", develop = true} pytest-target = {path = "../pytest-target", develop = true} -schema = "^0.7.3" +schema = "0.7.2" [tool.poetry.dev-dependencies] diff --git a/pytest-playbook/poetry.lock b/pytest-playbook/poetry.lock index 6f2d824232..c352a6fd10 100644 --- a/pytest-playbook/poetry.lock +++ b/pytest-playbook/poetry.lock @@ -38,18 +38,18 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "importlib-metadata" -version = "2.0.0" +version = "3.1.1" description = "Read metadata from Python packages" category = "main" optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" +python-versions = ">=3.6" [package.dependencies] zipp = ">=0.5" [package.extras] -docs = ["sphinx", "rst.linker"] -testing = ["packaging", "pep517", "importlib-resources (>=1.3)"] +docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"] +testing = ["pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake8", "pytest-cov", "jaraco.test (>=3.2.0)", "packaging", "pep517", "pyfakefs", "flufl.flake8", "pytest-black (>=0.3.7)", "pytest-mypy", "importlib-resources (>=1.3)"] [[package]] name = "iniconfig" @@ -61,7 +61,7 @@ python-versions = "*" [[package]] name = "packaging" -version = "20.4" +version = "20.7" description = "Core utilities for Python packages" category = "main" optional = false @@ -69,7 +69,6 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [package.dependencies] pyparsing = ">=2.0.2" -six = "*" [[package]] name = "pluggy" @@ -129,12 +128,12 @@ name = "pyyaml" version = "5.3.1" description = "YAML parser and emitter for Python" category = "main" -optional = true +optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [[package]] name = "schema" -version = "0.7.3" +version = "0.7.2" description = "Simple data validation library" category = "main" optional = false @@ -143,14 +142,6 @@ python-versions = "*" [package.dependencies] contextlib2 = ">=0.5.5" -[[package]] -name = "six" -version = "1.15.0" -description = "Python 2 and 3 compatibility utilities" -category = "main" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" - [[package]] name = "toml" version = "0.10.2" @@ -174,7 +165,7 @@ testing = ["pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "0365579f4be5c800fa2a7e73898fadf530f5c3c2808113744a73b10b5b7ccc7d" +content-hash = "42f6d26da539ab20702f2f8e6a8efae0a1f0cae1bee4d4f3896c98995cb8d3f5" [metadata.files] atomicwrites = [ @@ -194,16 +185,16 @@ contextlib2 = [ {file = "contextlib2-0.6.0.post1.tar.gz", hash = "sha256:01f490098c18b19d2bd5bb5dc445b2054d2fa97f09a4280ba2c5f3c394c8162e"}, ] importlib-metadata = [ - {file = "importlib_metadata-2.0.0-py2.py3-none-any.whl", hash = "sha256:cefa1a2f919b866c5beb7c9f7b0ebb4061f30a8a9bf16d609b000e2dfaceb9c3"}, - {file = "importlib_metadata-2.0.0.tar.gz", hash = "sha256:77a540690e24b0305878c37ffd421785a6f7e53c8b5720d211b211de8d0e95da"}, + {file = "importlib_metadata-3.1.1-py3-none-any.whl", hash = "sha256:6112e21359ef8f344e7178aa5b72dc6e62b38b0d008e6d3cb212c5b84df72013"}, + {file = "importlib_metadata-3.1.1.tar.gz", hash = "sha256:b0c2d3b226157ae4517d9625decf63591461c66b3a808c2666d538946519d170"}, ] iniconfig = [ {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, ] packaging = [ - {file = "packaging-20.4-py2.py3-none-any.whl", hash = "sha256:998416ba6962ae7fbd6596850b80e17859a5753ba17c32284f67bfff33784181"}, - {file = "packaging-20.4.tar.gz", hash = "sha256:4357f74f47b9c12db93624a82154e9b120fa8293699949152b22065d556079f8"}, + {file = "packaging-20.7-py2.py3-none-any.whl", hash = "sha256:eb41423378682dadb7166144a4926e443093863024de508ca5c9737d6bc08376"}, + {file = "packaging-20.7.tar.gz", hash = "sha256:05af3bb85d320377db281cf254ab050e1a7ebcbf5410685a9a407e18a1f81236"}, ] pluggy = [ {file = "pluggy-0.13.1-py2.py3-none-any.whl", hash = "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"}, @@ -232,15 +223,13 @@ pyyaml = [ {file = "PyYAML-5.3.1-cp37-cp37m-win_amd64.whl", hash = "sha256:73f099454b799e05e5ab51423c7bcf361c58d3206fa7b0d555426b1f4d9a3eaf"}, {file = "PyYAML-5.3.1-cp38-cp38-win32.whl", hash = "sha256:06a0d7ba600ce0b2d2fe2e78453a470b5a6e000a985dd4a4e54e436cc36b0e97"}, {file = "PyYAML-5.3.1-cp38-cp38-win_amd64.whl", hash = "sha256:95f71d2af0ff4227885f7a6605c37fd53d3a106fcab511b8860ecca9fcf400ee"}, + {file = "PyYAML-5.3.1-cp39-cp39-win32.whl", hash = "sha256:ad9c67312c84def58f3c04504727ca879cb0013b2517c85a9a253f0cb6380c0a"}, + {file = "PyYAML-5.3.1-cp39-cp39-win_amd64.whl", hash = "sha256:6034f55dab5fea9e53f436aa68fa3ace2634918e8b5994d82f3621c04ff5ed2e"}, {file = "PyYAML-5.3.1.tar.gz", hash = "sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d"}, ] schema = [ - {file = "schema-0.7.3-py2.py3-none-any.whl", hash = "sha256:c331438b60f634cab5664ab720d3083cc444f924d55269530c36b33e3354276f"}, - {file = "schema-0.7.3.tar.gz", hash = "sha256:4cf529318cfd1e844ecbe02f41f7e5aa027463e7403666a52746f31f04f47a5e"}, -] -six = [ - {file = "six-1.15.0-py2.py3-none-any.whl", hash = "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"}, - {file = "six-1.15.0.tar.gz", hash = "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259"}, + {file = "schema-0.7.2-py2.py3-none-any.whl", hash = "sha256:3a03c2e2b22e6a331ae73750ab1da46916da6ca861b16e6f073ac1d1eba43b71"}, + {file = "schema-0.7.2.tar.gz", hash = "sha256:b536f2375b49fdf56f36279addae98bd86a8afbd58b3c32ce363c464bed5fc1c"}, ] toml = [ {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, diff --git a/pytest-playbook/pyproject.toml b/pytest-playbook/pyproject.toml index 4ad1234d06..2d44a4bb73 100644 --- a/pytest-playbook/pyproject.toml +++ b/pytest-playbook/pyproject.toml @@ -10,7 +10,7 @@ packages = [{include = "playbook.py"}] [tool.poetry.dependencies] python = "^3.7" pytest = "^6.1.2" -schema = "^0.7.3" +schema = "0.7.2" PyYAML = "^5.3.1" [tool.poetry.dev-dependencies] diff --git a/pytest-target/poetry.lock b/pytest-target/poetry.lock index c6dd5bfa58..e209922037 100644 --- a/pytest-target/poetry.lock +++ b/pytest-target/poetry.lock @@ -248,7 +248,7 @@ develop = true [package.dependencies] pytest = "^6.1.2" PyYAML = "^5.3.1" -schema = "^0.7.3" +schema = "0.7.2" [package.source] type = "directory" @@ -264,7 +264,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [[package]] name = "schema" -version = "0.7.3" +version = "0.7.2" description = "Simple data validation library" category = "main" optional = false @@ -490,8 +490,8 @@ pyyaml = [ {file = "PyYAML-5.3.1.tar.gz", hash = "sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d"}, ] schema = [ - {file = "schema-0.7.3-py2.py3-none-any.whl", hash = "sha256:c331438b60f634cab5664ab720d3083cc444f924d55269530c36b33e3354276f"}, - {file = "schema-0.7.3.tar.gz", hash = "sha256:4cf529318cfd1e844ecbe02f41f7e5aa027463e7403666a52746f31f04f47a5e"}, + {file = "schema-0.7.2-py2.py3-none-any.whl", hash = "sha256:3a03c2e2b22e6a331ae73750ab1da46916da6ca861b16e6f073ac1d1eba43b71"}, + {file = "schema-0.7.2.tar.gz", hash = "sha256:b536f2375b49fdf56f36279addae98bd86a8afbd58b3c32ce363c464bed5fc1c"}, ] six = [ {file = "six-1.15.0-py2.py3-none-any.whl", hash = "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"}, diff --git a/testsuites/test_lis.py b/testsuites/test_lis.py index d902282cfa..55f0c2aed8 100644 --- a/testsuites/test_lis.py +++ b/testsuites/test_lis.py @@ -7,6 +7,7 @@ from target import Azure import pytest + from lisa import LISA diff --git a/testsuites/test_xdp.py b/testsuites/test_xdp.py index 8aafe577b4..9386de5bf7 100644 --- a/testsuites/test_xdp.py +++ b/testsuites/test_xdp.py @@ -7,6 +7,7 @@ from target import Azure import pytest + from lisa import LISA From fa75c67b6b13045c71257a318ed2bb0ce1187495 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Tue, 8 Dec 2020 17:05:42 -0800 Subject: [PATCH 108/194] Setup module and class target fixtures Basic implementation but allows a target to be used across a module or class easily. Also sets us up for multiple targets. --- pytest-target/target/plugin.py | 104 +++++++++++++++++++++++---------- 1 file changed, 73 insertions(+), 31 deletions(-) diff --git a/pytest-target/target/plugin.py b/pytest-target/target/plugin.py index 5c7629c110..439675e2bf 100644 --- a/pytest-target/target/plugin.py +++ b/pytest-target/target/plugin.py @@ -25,7 +25,7 @@ from target.target import Target if typing.TYPE_CHECKING: - from typing import Any, Dict, Iterator, List, Type + from typing import Any, Dict, Iterator, List, Set, Type from _pytest.config.argparsing import Parser from _pytest.fixtures import SubRequest @@ -84,35 +84,24 @@ def pool(request: SubRequest) -> Iterator[List[Target]]: t.delete() -@pytest.fixture -def target(pool: List[Target], request: SubRequest) -> Iterator[Target]: - """This fixture provides a connected target for each test. - - It is parametrized indirectly in `pytest_generate_tests`. +def get_target( + pool: List[Target], + platform: Type[Target], + parameters: Dict[str, Any], + features: Set[str], +) -> Target: + """This function gets or creates an appropriate `Target`. - In this fixture we can check if any existing target matches all - the requirements. If so, we can re-use that target, and if not, we - can deallocate the currently running target and allocate a new - one. When all tests are finished, the pool fixture above will - delete all created VMs. Coupled with performing discrete - optimization in the test collection phase and ordering the tests - such that the test(s) with the lowest common denominator - requirements are executed first, we have the two-layer scheduling - as asked. - - However, this feels like putting the cart before the horse to me. - It would be much simpler in terms of design, implementation, and - usage that features are specified upfront when the targets are - specified. Then all this goes away, and tests are skipped when the - feature is missing, which also leaves users in full control of - their environments. + First check if any existing target in the `pool` matches all the + `features` and other requirements. If so, we can re-use that + target, and if not, we can deallocate the currently running target + and allocate a new one. When all tests are finished, the pool + fixture above will delete all created VMs. We can achieve + two-layer scheduling by implementing a custom scheduler in + pytest-xdist via `pytest_xdist_make_scheduler` and sorting the + tests such that they're grouped by features. """ - platform: Type[Target] = platforms[request.param["platform"]] - parameters: Dict[str, Any] = request.param["parameters"] - marker = request.node.get_closest_marker("lisa") - features = set(marker.kwargs["features"]) - # TODO: If `t` is not already in use, deallocate the previous # target, and ensure the tests have been sorted (and so grouped) # by their requirements. @@ -124,14 +113,63 @@ def target(pool: List[Target], request: SubRequest) -> Iterator[Target]: and t.parameters == parameters and t.features >= features ): - yield t - break + pool.remove(t) + return t else: # TODO: Reimplement caching. t = platform(f"pytest-{uuid4()}", parameters, features) - pool.append(t) - yield t + return t + + +def cleanup_target(pool: List[Target], t: Target) -> None: + """This is called by fixtures after they're done with a target.""" t.conn.close() + pool.append(t) + + +@pytest.fixture +def target(pool: List[Target], request: SubRequest) -> Iterator[Target]: + """This fixture provides a connected target for each test. + + It is parametrized indirectly in `pytest_generate_tests`. + + TODO: Clean up the code duplication here across the fixtures. + + """ + platform = platforms[request.param["platform"]] + parameters = request.param["parameters"] + # TODO: Use a ‘target’ marker instead. + marker = request.node.get_closest_marker("lisa") + features = set(marker.kwargs["features"]) + t = get_target(pool, platform, parameters, features) + yield t + cleanup_target(pool, t) + + +@pytest.fixture(scope="class") +def c_target(pool: List[Target], request: SubRequest) -> Iterator[Target]: + """This fixture is the same as `target` but shared across a class.""" + platform = platforms[request.param["platform"]] + parameters = request.param["parameters"] + # TODO: Use a ‘target’ marker instead. + marker = request.node.get_closest_marker("lisa") + features = set(marker.kwargs["features"]) + t = get_target(pool, platform, parameters, features) + yield t + cleanup_target(pool, t) + + +@pytest.fixture(scope="module") +def m_target(pool: List[Target], request: SubRequest) -> Iterator[Target]: + """This fixture is the same as `target` but shared across a module.""" + platform = platforms[request.param["platform"]] + parameters = request.param["parameters"] + # TODO: Use a ‘target’ marker instead. + marker = request.node.get_closest_marker("lisa") + features = set(marker.kwargs["features"]) + t = get_target(pool, platform, parameters, features) + yield t + cleanup_target(pool, t) targets: List[Dict[str, Any]] = [] @@ -154,3 +192,7 @@ def pytest_generate_tests(metafunc: Metafunc) -> None: """ if "target" in metafunc.fixturenames: metafunc.parametrize("target", targets, True, target_ids) + if "m_target" in metafunc.fixturenames: + metafunc.parametrize("m_target", targets, True, target_ids) + if "c_target" in metafunc.fixturenames: + metafunc.parametrize("c_target", targets, True, target_ids) From 6f1bef4645d84eaf52839b926e6ff5eddb10eeb4 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Tue, 8 Dec 2020 17:06:17 -0800 Subject: [PATCH 109/194] Add second smoke test example using multiple small tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I had tried this previously, and at the time I liked it but it wasn’t quite ready. This is re-done with a module scoped target fixture, and with pytest-azurepipelines, this should display really nicely. The only issue is that these each count as a separate test. I’ve also modified smoke test B to use the `caplog` module so we can capture INFO and above logs for this test, but not universally. Since I don’t want to make the call on which is the “better” example, I’m leaving both for people to review. --- pytest.ini | 6 +- testsuites/test_smoke_a.py | 76 +++++++++++++++++++ testsuites/{test_smoke.py => test_smoke_b.py} | 30 +++++--- 3 files changed, 96 insertions(+), 16 deletions(-) create mode 100644 testsuites/test_smoke_a.py rename testsuites/{test_smoke.py => test_smoke_b.py} (75%) diff --git a/pytest.ini b/pytest.ini index 7da0eb70c2..35239b690d 100644 --- a/pytest.ini +++ b/pytest.ini @@ -5,10 +5,8 @@ addopts = --capture=tee-sys --tb=line --no-header -log_cli = true -log_cli_level = WARNING -log_cli_format = %(asctime)s %(levelname)s %(message)s -log_cli_date_format = %Y-%m-%d %H:%M:%S +log_format = %(asctime)s %(levelname)s %(message)s +log_date_format = %Y-%m-%d %H:%M:%S render_collapsed = true junit_logging = all timeout = 1200 diff --git a/testsuites/test_smoke_a.py b/testsuites/test_smoke_a.py new file mode 100644 index 0000000000..bc9491e228 --- /dev/null +++ b/testsuites/test_smoke_a.py @@ -0,0 +1,76 @@ +"""Check that an Azure Linux VM can be deployed and is responsive. + +This example uses multiple tests with a module-scoped target fixture. +It's a more Pythonic approach, and since Pytest automatically groups +tests by fixture scopes, these run for each parameter of the target in +order as we would expect. This results in the "smoke test" actually +being a module of multiple unit tests. Another similar alternative is +to use a class and the class-scoped target fixture. See +`test_smoke_b.py` for the single-test approach. + +""" +from __future__ import annotations + +import typing + +if typing.TYPE_CHECKING: + from target import Azure + +import socket +import time + +from invoke.runners import CommandTimedOut, UnexpectedExit # type: ignore +from paramiko import SSHException # type: ignore + +from lisa import LISA + +pytestmark = LISA( + platform="Azure", + category="Functional", + area="deploy", + priority=0, + sku="Standard_DS2_v2", +) + + +def test_first_ping(m_target: Azure) -> None: + """"Pinging before reboot...""" + assert m_target.ping(), f"Pinging {m_target.host} before reboot failed" + + +def test_first_ssh(m_target: Azure) -> None: + """SSHing before reboot...""" + assert m_target.conn.open(), f"SSH {m_target.host} before reboot failed" + + +def test_reboot(m_target: Azure) -> None: + """Rebooting...""" + reboot_exit = 0 + try: + # If this succeeds, we should expect the exit code to be -1 + reboot_exit = m_target.conn.sudo("reboot", timeout=5).exited + except (TimeoutError, CommandTimedOut, SSHException, socket.error) as e: + print(f"SSH failed, using platform to reboot: '{e}'") + m_target.platform_restart() + except UnexpectedExit: + # TODO: How do we differentiate reboot working and the SSH + # connection disconnecting for other reasons? + assert reboot_exit == -1, "While SSH worked, 'reboot' command failed" + finally: + print("Sleeping for 10 seconds after reboot...") + time.sleep(10) + + +def test_second_ping(m_target: Azure) -> None: + """Pinging after reboot...""" + assert m_target.ping(), f"Pinging {m_target.host} after reboot failed" + + +def test_second_ssh(m_target: Azure) -> None: + """SSHing after reboot...""" + assert m_target.conn.open(), f"SSH {m_target.host} after reboot failed" + + +def test_boot_diagnostics(m_target: Azure) -> None: + """Retrieving boot diagnostics...""" + m_target.get_boot_diagnostics() diff --git a/testsuites/test_smoke.py b/testsuites/test_smoke_b.py similarity index 75% rename from testsuites/test_smoke.py rename to testsuites/test_smoke_b.py index dc7b017fde..ab7f807106 100644 --- a/testsuites/test_smoke.py +++ b/testsuites/test_smoke_b.py @@ -1,19 +1,20 @@ -"""Runs a 'smoke' test for an Azure Linux VM deployment.""" from __future__ import annotations import typing if typing.TYPE_CHECKING: from target import Azure + from _pytest.logging import LogCaptureFixture import logging import socket import time from invoke.runners import CommandTimedOut, Result, UnexpectedExit # type: ignore -from lisa import LISA from paramiko import SSHException # type: ignore +from lisa import LISA + @LISA( platform="Azure", @@ -22,24 +23,27 @@ priority=0, sku="Standard_DS2_v2", ) -def test_smoke(target: Azure) -> None: - """Check that a VM can be deployed and is responsive. +def test_smoke(target: Azure, caplog: LogCaptureFixture) -> None: + """Check that an Azure Linux VM can be deployed and is responsive. + + This example uses exactly one function for the entire test, which + means we have to catch failures that don't fail the test, and + instead emit warnings. It works, and it's closer to how LISAv2 + would have implemented it, but it's less Pythonic. For a more + "modern" example, see `test_smoke_a.py`. - 1. Deploy the VM (via 'node' fixture) and log it. + 1. Deploy the VM (via `target` fixture). 2. Ping the VM. 3. Connect to the VM via SSH. 4. Attempt to reboot via SSH, otherwise use the platform. - 5. Fetch the serial console logs. - - For commands where we expect a possible non-zero exit code, we - pass 'warn=True' to prevent it from throwing 'UnexpectedExit' and - we instead check its result at the end. + 5. Fetch the serial console logs AKA boot diagnostics. SSH failures DO NOT fail this test. - TODO: Capture these logs without capturing all INFO logs. - """ + # Capture INFO and above logs for this test. + caplog.set_level(logging.INFO) + logging.info("Pinging before reboot...") ping1 = Result() try: @@ -69,6 +73,8 @@ def test_smoke(target: Azure) -> None: if reboot_exit != -1: logging.warning("While SSH worked, 'reboot' command failed") + # TODO: We should check something more concrete here instead of + # sleeping an arbitrary amount of time. logging.info("Sleeping for 10 seconds after reboot...") time.sleep(10) From 17854c57a9ccc3550e0ed12940feb528b711e73a Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Tue, 8 Dec 2020 17:16:05 -0800 Subject: [PATCH 110/194] Update playbooks for demo and add module as criteria field --- playbooks/demo.yaml | 13 +++++++++++++ playbooks/smoke.yaml | 23 ----------------------- pytest-lisa/lisa.py | 3 +++ 3 files changed, 16 insertions(+), 23 deletions(-) create mode 100644 playbooks/demo.yaml delete mode 100644 playbooks/smoke.yaml diff --git a/playbooks/demo.yaml b/playbooks/demo.yaml new file mode 100644 index 0000000000..dfd9f41f7a --- /dev/null +++ b/playbooks/demo.yaml @@ -0,0 +1,13 @@ +targets: + - name: Debian + platform: Azure + parameters: + image: credativ:Debian:9:9.0.201706190 + + - name: Ubuntu + platform: Azure + parameters: + image: UbuntuLTS + +criteria: + - module: smoke diff --git a/playbooks/smoke.yaml b/playbooks/smoke.yaml deleted file mode 100644 index ea1a15ab60..0000000000 --- a/playbooks/smoke.yaml +++ /dev/null @@ -1,23 +0,0 @@ -targets: - - name: Debian - platform: Azure - parameters: - image: credativ:Debian:9:9.0.201706190 - - - name: GitHub - platform: Azure - parameters: - image: github:github-enterprise:github-enterprise:latest - - - name: Citrix - platform: Azure - parameters: - image: citrix:netscalervpx-130:netscalerbyol:latest - - - name: AudioCodes - platform: Azure - parameters: - image: audiocodes:mediantsessionbordercontroller:mediantvirtualsbcazure:latest - -criteria: - - name: smoke diff --git a/pytest-lisa/lisa.py b/pytest-lisa/lisa.py index bd97f61601..2993f562c7 100644 --- a/pytest-lisa/lisa.py +++ b/pytest-lisa/lisa.py @@ -72,6 +72,7 @@ def pytest_playbook_schema(schema: Dict[Any, Any]) -> None: # TODO: Validate that these strings are valid regular # expressions if we change our matching logic. Optional("name", default=None): str, + Optional("module", default=None): str, Optional("area", default=None): str, Optional("category", default=None): str, Optional("priority", default=None): int, @@ -149,6 +150,8 @@ def select(item: Item, times: int, exclude: bool) -> None: if any( [ c["name"] and c["name"] in item.name, + # NOTE: `Item` does have a `module` field, though it’s untyped. + c["module"] and c["module"] in item.module.__name__, # type: ignore c["area"] and c["area"].casefold() == i["area"].casefold(), c["category"] and c["category"].casefold() == i["category"].casefold(), From 1fd4cb11738387f1719286cda7c9fd080c703754 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Wed, 9 Dec 2020 16:19:43 -0800 Subject: [PATCH 111/194] Implement custom `LisaScheduler` to scope tests by parameterization --- conftest.py | 2 +- pytest-lisa/lisa.py | 81 ++++++++++++++++++++++++++++++++++ pytest-playbook/playbook.py | 3 +- pytest-target/target/plugin.py | 4 ++ pytest-target/target/target.py | 13 +++++- 5 files changed, 100 insertions(+), 3 deletions(-) diff --git a/conftest.py b/conftest.py index c63fcdc909..35efa00987 100644 --- a/conftest.py +++ b/conftest.py @@ -14,5 +14,5 @@ def pytest_html_report_title(report: HTMLReport) -> None: - """Set HTML report title.""" + """pytest-html hook to set the HTML report title.""" report.title = "LISAv3 Results" diff --git a/pytest-lisa/lisa.py b/pytest-lisa/lisa.py index 2993f562c7..4f879c4f0f 100644 --- a/pytest-lisa/lisa.py +++ b/pytest-lisa/lisa.py @@ -27,12 +27,15 @@ """ from __future__ import annotations +import re import sys import typing import playbook +import py import pytest from schema import Optional, Or, Schema, SchemaMissingKeyError # type: ignore +from xdist.scheduler.loadscope import LoadScopeScheduling # type: ignore if typing.TYPE_CHECKING: from typing import Any, Dict, List @@ -67,6 +70,9 @@ def pytest_configure(config: Config) -> None: def pytest_playbook_schema(schema: Dict[Any, Any]) -> None: """pytest-playbook hook to update the playbook schema.""" + # TODO: We also want to support a criteria selection on each + # `target` in the playbook, which this top-level criteria being + # the default. criteria_schema = Schema( { # TODO: Validate that these strings are valid regular @@ -90,6 +96,8 @@ def pytest_playbook_schema(schema: Dict[Any, Any]) -> None: "category": Or("Functional", "Performance", "Stress", "Community", "Longhaul"), "area": str, "priority": Or(0, 1, 2, 3), + # TODO: Move `features` to `pytest.mark.target` and don’t + # allow extra keys. Optional("features", default=list): [str], Optional("tags", default=list): [str], Optional(object): object, @@ -164,3 +172,76 @@ def select(item: Item, times: int, exclude: bool) -> None: if not included: included = items items[:] = [i for i in included if i not in excluded] + + +class LISAScheduling(LoadScopeScheduling): + """Implement load scheduling across nodes, but grouping by parameter. + + This algorithm ensures that all tests which share the same set of + parameters (namely the target) will run on the same executor as a + single work-unit. + + TODO: This essentially confines the targets and one target won't + be spun up multiple times when run in parallel, so we should make + this scheduler optional, as an alternative scenario is to spin up + multiple near-identical instances of a target in order to run + tests in parallel. + + TODO: We could also add an expected prefix to the target + parameter, like 'Target=' and then only split on it instead + of all parameters. + + This is modeled after the built-in `LoadFileScheduling`, which + also simply subclasses `LoadScopeScheduling`. See `_split_scope` + for the important part. Note that we can extend this to implement + any kind of scheduling algorithm we want. + + """ + + def __init__(self, config: Config, log=None): # type: ignore + super().__init__(config, log) + if log is None: + self.log = py.log.Producer("lisasched") + else: + self.log = log.lisasched + + regex = re.compile(r"\[(\w+)\]") + + def _split_scope(self, nodeid: str) -> str: + """Determine the scope (grouping) of a nodeid. + + Example of a parameterized test's nodeid:: + + example/test_module.py::test_function[A] + example/test_module.py::test_function[B] + example/test_module.py::test_function_extra[A][B] + + `LoadScopeScheduling` uses `nodeid.rsplit("::", 1)[0]`, or the + first `::` from the right, to split by scope, such that + classes will be grouped, then modules. `LoadFileScheduling` + uses `nodeid.split("::", 1)[0]`, or the first `::` from the + left, to instead split only by modules (Python files). + + We opportunistically find all the parameters (strings within + square brackets) and join them with a slash to create the + scope. If the function is not parameterized, and so has no + square brackets, then we simply fallback to the algorithm of + `LoadScopeScheduling`. So the above would map into the scopes: + 'A', 'B', and 'A/B'. + + """ + if "[" in nodeid: + scope = "/".join(self.regex.findall(nodeid)) + if self.config.getoption("verbose"): + self.log(f"Split nodeid '{nodeid}' into scope '{scope}'") + return scope + return super()._split_scope(nodeid) # type: ignore + + +def pytest_xdist_make_scheduler(config: Config) -> LISAScheduling: + """pytest-xdist hook for implementing a custom scheduler. + + https://github.com/pytest-dev/pytest-xdist/blob/master/OVERVIEW.md + + """ + return LISAScheduling(config) diff --git a/pytest-playbook/playbook.py b/pytest-playbook/playbook.py index f2a840d315..ab03c42e23 100644 --- a/pytest-playbook/playbook.py +++ b/pytest-playbook/playbook.py @@ -37,7 +37,7 @@ from _pytest.config import Config, PytestPluginManager from _pytest.config.argparsing import Parser -# TODO: I’m not a fan of this name. +# TODO: I’m not a fan of this name. Maybe ‘params’ or ‘data’? playbook: Dict[Any, Any] = dict() @@ -76,6 +76,7 @@ def pytest_addoption(parser: Parser, pluginmanager: PytestPluginManager) -> None group.addoption("--playbook", type=Path, help="Path to playbook.") +# TODO: See if this works without ‘trylast’. @pytest.hookimpl(trylast=True) def pytest_configure(config: Config) -> None: """Pytest hook to configure our plugin. diff --git a/pytest-target/target/plugin.py b/pytest-target/target/plugin.py index 439675e2bf..9639f071d3 100644 --- a/pytest-target/target/plugin.py +++ b/pytest-target/target/plugin.py @@ -189,6 +189,10 @@ def pytest_generate_tests(metafunc: Metafunc) -> None: This hook is run for each test, so we gather the `targets` in `pytest_sessionstart`. + TODO: Handle `targets` being empty (probably a user-error). Also + consider how this may change if we want to selectively + parameterize tests. + """ if "target" in metafunc.fixturenames: metafunc.parametrize("target", targets, True, target_ids) diff --git a/pytest-target/target/target.py b/pytest-target/target/target.py index 11d5f5e4b1..319a3837d3 100644 --- a/pytest-target/target/target.py +++ b/pytest-target/target/target.py @@ -17,7 +17,18 @@ class Target(ABC): - """Extends 'fabric.Connection' with our own utilities.""" + """This class represents a remote Linux target. + + As a partially abstract base class, it is meant to be subclassed + to provide platform support. So `Target` as a class maps to the + concept of a Linux target machine reachable via SSH (through + `self.conn`, an instance of `Fabric.Connection`). Each subclass of + `Target` provides the necessary implementation to instantiate an + actual Linux target, by deploying it on that platform. Each + _instance_ of a platform-specific subclass of `Target` maps to an + actual Linux target that has been deployed on that platform. + + """ # Typed instance attributes, not class attributes. parameters: Mapping[str, str] From 7a3f3a72f90c5ff031c4c759df102cf86881a820 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Thu, 10 Dec 2020 00:55:59 -0800 Subject: [PATCH 112/194] Move pytest-xdist dependency to pytest-lisa --- poetry.lock | 45 +++++++++++-------------- pyproject.toml | 1 - pytest-lisa/poetry.lock | 69 +++++++++++++++++++++++++++++++++++++- pytest-lisa/pyproject.toml | 1 + 4 files changed, 88 insertions(+), 28 deletions(-) diff --git a/poetry.lock b/poetry.lock index 4513f0aea6..aeac6ece5a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -111,14 +111,14 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "cryptography" -version = "3.2.1" +version = "3.3.1" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." category = "main" optional = false -python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*" +python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*" [package.dependencies] -cffi = ">=1.8,<1.11.3 || >1.11.3" +cffi = ">=1.12" six = ">=1.4.1" [package.extras] @@ -548,6 +548,7 @@ develop = true pytest = "^6.1.2" pytest-playbook = "0.1.0" pytest-target = "0.1.0" +pytest-xdist = "^2.1.0" schema = "0.7.2" [package.source] @@ -818,7 +819,7 @@ testing = ["pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "2446fb23de7593ab9aed977563ed514f029bfed67f5f69e102c72ff994e2bed6" +content-hash = "292ddb6210c73e8071accf159db11489d8f5643a7d608ea59c835fdbcad19a85" [metadata.files] apipkg = [ @@ -900,28 +901,20 @@ contextlib2 = [ {file = "contextlib2-0.6.0.post1.tar.gz", hash = "sha256:01f490098c18b19d2bd5bb5dc445b2054d2fa97f09a4280ba2c5f3c394c8162e"}, ] cryptography = [ - {file = "cryptography-3.2.1-cp27-cp27m-macosx_10_10_x86_64.whl", hash = "sha256:6dc59630ecce8c1f558277ceb212c751d6730bd12c80ea96b4ac65637c4f55e7"}, - {file = "cryptography-3.2.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:75e8e6684cf0034f6bf2a97095cb95f81537b12b36a8fedf06e73050bb171c2d"}, - {file = "cryptography-3.2.1-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:4e7268a0ca14536fecfdf2b00297d4e407da904718658c1ff1961c713f90fd33"}, - {file = "cryptography-3.2.1-cp27-cp27m-win32.whl", hash = "sha256:7117319b44ed1842c617d0a452383a5a052ec6aa726dfbaffa8b94c910444297"}, - {file = "cryptography-3.2.1-cp27-cp27m-win_amd64.whl", hash = "sha256:a733671100cd26d816eed39507e585c156e4498293a907029969234e5e634bc4"}, - {file = "cryptography-3.2.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:a75f306a16d9f9afebfbedc41c8c2351d8e61e818ba6b4c40815e2b5740bb6b8"}, - {file = "cryptography-3.2.1-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:5849d59358547bf789ee7e0d7a9036b2d29e9a4ddf1ce5e06bb45634f995c53e"}, - {file = "cryptography-3.2.1-cp35-abi3-macosx_10_10_x86_64.whl", hash = "sha256:bd717aa029217b8ef94a7d21632a3bb5a4e7218a4513d2521c2a2fd63011e98b"}, - {file = "cryptography-3.2.1-cp35-abi3-manylinux1_x86_64.whl", hash = "sha256:efe15aca4f64f3a7ea0c09c87826490e50ed166ce67368a68f315ea0807a20df"}, - {file = "cryptography-3.2.1-cp35-abi3-manylinux2010_x86_64.whl", hash = "sha256:32434673d8505b42c0de4de86da8c1620651abd24afe91ae0335597683ed1b77"}, - {file = "cryptography-3.2.1-cp35-abi3-manylinux2014_aarch64.whl", hash = "sha256:7b8d9d8d3a9bd240f453342981f765346c87ade811519f98664519696f8e6ab7"}, - {file = "cryptography-3.2.1-cp35-cp35m-win32.whl", hash = "sha256:d3545829ab42a66b84a9aaabf216a4dce7f16dbc76eb69be5c302ed6b8f4a29b"}, - {file = "cryptography-3.2.1-cp35-cp35m-win_amd64.whl", hash = "sha256:a4e27ed0b2504195f855b52052eadcc9795c59909c9d84314c5408687f933fc7"}, - {file = "cryptography-3.2.1-cp36-abi3-win32.whl", hash = "sha256:13b88a0bd044b4eae1ef40e265d006e34dbcde0c2f1e15eb9896501b2d8f6c6f"}, - {file = "cryptography-3.2.1-cp36-abi3-win_amd64.whl", hash = "sha256:07ca431b788249af92764e3be9a488aa1d39a0bc3be313d826bbec690417e538"}, - {file = "cryptography-3.2.1-cp36-cp36m-win32.whl", hash = "sha256:a035a10686532b0587d58a606004aa20ad895c60c4d029afa245802347fab57b"}, - {file = "cryptography-3.2.1-cp36-cp36m-win_amd64.whl", hash = "sha256:d26a2557d8f9122f9bf445fc7034242f4375bd4e95ecda007667540270965b13"}, - {file = "cryptography-3.2.1-cp37-cp37m-win32.whl", hash = "sha256:545a8550782dda68f8cdc75a6e3bf252017aa8f75f19f5a9ca940772fc0cb56e"}, - {file = "cryptography-3.2.1-cp37-cp37m-win_amd64.whl", hash = "sha256:55d0b896631412b6f0c7de56e12eb3e261ac347fbaa5d5e705291a9016e5f8cb"}, - {file = "cryptography-3.2.1-cp38-cp38-win32.whl", hash = "sha256:3cd75a683b15576cfc822c7c5742b3276e50b21a06672dc3a800a2d5da4ecd1b"}, - {file = "cryptography-3.2.1-cp38-cp38-win_amd64.whl", hash = "sha256:d25cecbac20713a7c3bc544372d42d8eafa89799f492a43b79e1dfd650484851"}, - {file = "cryptography-3.2.1.tar.gz", hash = "sha256:d3d5e10be0cf2a12214ddee45c6bd203dab435e3d83b4560c03066eda600bfe3"}, + {file = "cryptography-3.3.1-cp27-cp27m-macosx_10_10_x86_64.whl", hash = "sha256:c366df0401d1ec4e548bebe8f91d55ebcc0ec3137900d214dd7aac8427ef3030"}, + {file = "cryptography-3.3.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:9f6b0492d111b43de5f70052e24c1f0951cb9e6022188ebcb1cc3a3d301469b0"}, + {file = "cryptography-3.3.1-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:a69bd3c68b98298f490e84519b954335154917eaab52cf582fa2c5c7efc6e812"}, + {file = "cryptography-3.3.1-cp27-cp27m-win32.whl", hash = "sha256:84ef7a0c10c24a7773163f917f1cb6b4444597efd505a8aed0a22e8c4780f27e"}, + {file = "cryptography-3.3.1-cp27-cp27m-win_amd64.whl", hash = "sha256:594a1db4511bc4d960571536abe21b4e5c3003e8750ab8365fafce71c5d86901"}, + {file = "cryptography-3.3.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:0003a52a123602e1acee177dc90dd201f9bb1e73f24a070db7d36c588e8f5c7d"}, + {file = "cryptography-3.3.1-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:83d9d2dfec70364a74f4e7c70ad04d3ca2e6a08b703606993407bf46b97868c5"}, + {file = "cryptography-3.3.1-cp36-abi3-macosx_10_10_x86_64.whl", hash = "sha256:dc42f645f8f3a489c3dd416730a514e7a91a59510ddaadc09d04224c098d3302"}, + {file = "cryptography-3.3.1-cp36-abi3-manylinux1_x86_64.whl", hash = "sha256:788a3c9942df5e4371c199d10383f44a105d67d401fb4304178020142f020244"}, + {file = "cryptography-3.3.1-cp36-abi3-manylinux2010_x86_64.whl", hash = "sha256:69e836c9e5ff4373ce6d3ab311c1a2eed274793083858d3cd4c7d12ce20d5f9c"}, + {file = "cryptography-3.3.1-cp36-abi3-manylinux2014_aarch64.whl", hash = "sha256:9e21301f7a1e7c03dbea73e8602905a4ebba641547a462b26dd03451e5769e7c"}, + {file = "cryptography-3.3.1-cp36-abi3-win32.whl", hash = "sha256:b4890d5fb9b7a23e3bf8abf5a8a7da8e228f1e97dc96b30b95685df840b6914a"}, + {file = "cryptography-3.3.1-cp36-abi3-win_amd64.whl", hash = "sha256:0e85aaae861d0485eb5a79d33226dd6248d2a9f133b81532c8f5aae37de10ff7"}, + {file = "cryptography-3.3.1.tar.gz", hash = "sha256:7e177e4bea2de937a584b13645cab32f25e3d96fc0bc4a4cf99c27dc77682be6"}, ] execnet = [ {file = "execnet-1.7.1-py2.py3-none-any.whl", hash = "sha256:d4efd397930c46415f62f8a31388d6be4f27a91d7550eb79bc64a756e0056547"}, diff --git a/pyproject.toml b/pyproject.toml index 65965600c4..934b550c2a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,6 @@ pytest = "^6.1.1" pytest-timeout = "^1.4.2" pytest-html = "^2.1.1" pytest-rerunfailures = "^9.1.1" -pytest-xdist = "^2.1.0" pytest-lisa = {path = "pytest-lisa", develop = true} [tool.poetry.dev-dependencies] diff --git a/pytest-lisa/poetry.lock b/pytest-lisa/poetry.lock index a8e852d5e6..6f479a0661 100644 --- a/pytest-lisa/poetry.lock +++ b/pytest-lisa/poetry.lock @@ -1,3 +1,11 @@ +[[package]] +name = "apipkg" +version = "1.5" +description = "apipkg: namespace control and lazy-import mechanism" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + [[package]] name = "atomicwrites" version = "1.4.0" @@ -82,6 +90,20 @@ pep8test = ["black", "flake8", "flake8-import-order", "pep8-naming"] ssh = ["bcrypt (>=3.1.5)"] test = ["pytest (>=3.6.0,!=3.9.0,!=3.9.1,!=3.9.2)", "pretend", "iso8601", "pytz", "hypothesis (>=1.11.4,!=3.79.2)"] +[[package]] +name = "execnet" +version = "1.7.1" +description = "execnet: rapid multi-Python deployment" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[package.dependencies] +apipkg = ">=1.4" + +[package.extras] +testing = ["pre-commit"] + [[package]] name = "fabric" version = "2.5.0" @@ -236,6 +258,18 @@ toml = "*" checkqa_mypy = ["mypy (==0.780)"] testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] +[[package]] +name = "pytest-forked" +version = "1.3.0" +description = "run tests in isolated forked subprocesses" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[package.dependencies] +py = "*" +pytest = ">=3.10" + [[package]] name = "pytest-playbook" version = "0.1.0" @@ -274,6 +308,23 @@ tenacity = "^6.2.0" type = "directory" url = "../pytest-target" +[[package]] +name = "pytest-xdist" +version = "2.1.0" +description = "pytest xdist plugin for distributed testing and loop-on-failing modes" +category = "main" +optional = false +python-versions = ">=3.5" + +[package.dependencies] +execnet = ">=1.1" +pytest = ">=6.0.0" +pytest-forked = "*" + +[package.extras] +psutil = ["psutil (>=3.0)"] +testing = ["filelock"] + [[package]] name = "pyyaml" version = "5.3.1" @@ -338,9 +389,13 @@ testing = ["pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "333ba43473867f73d807a941a7dc7ec215c16470cb489bd6fa0033e8607fb532" +content-hash = "876608c892824f5f43904ec3a7eca019f8d63e7862105a169cfb81ac225620bb" [metadata.files] +apipkg = [ + {file = "apipkg-1.5-py2.py3-none-any.whl", hash = "sha256:58587dd4dc3daefad0487f6d9ae32b4542b185e1c36db6993290e7c41ca2b47c"}, + {file = "apipkg-1.5.tar.gz", hash = "sha256:37228cda29411948b422fae072f57e31d3396d2ee1c9783775980ee9c9990af6"}, +] atomicwrites = [ {file = "atomicwrites-1.4.0-py2.py3-none-any.whl", hash = "sha256:6d1784dea7c0c8d4a5172b6c620f40b6e4cbfdf96d783691f2e1302a7b88e197"}, {file = "atomicwrites-1.4.0.tar.gz", hash = "sha256:ae70396ad1a434f9c7046fd2dd196fc04b12f9e91ffb859164193be8b6168a7a"}, @@ -428,6 +483,10 @@ cryptography = [ {file = "cryptography-3.2.1-cp38-cp38-win_amd64.whl", hash = "sha256:d25cecbac20713a7c3bc544372d42d8eafa89799f492a43b79e1dfd650484851"}, {file = "cryptography-3.2.1.tar.gz", hash = "sha256:d3d5e10be0cf2a12214ddee45c6bd203dab435e3d83b4560c03066eda600bfe3"}, ] +execnet = [ + {file = "execnet-1.7.1-py2.py3-none-any.whl", hash = "sha256:d4efd397930c46415f62f8a31388d6be4f27a91d7550eb79bc64a756e0056547"}, + {file = "execnet-1.7.1.tar.gz", hash = "sha256:cacb9df31c9680ec5f95553976c4da484d407e85e41c83cb812aa014f0eddc50"}, +] fabric = [ {file = "fabric-2.5.0-py2.py3-none-any.whl", hash = "sha256:160331934ea60036604928e792fa8e9f813266b098ef5562aa82b88527740389"}, {file = "fabric-2.5.0.tar.gz", hash = "sha256:24842d7d51556adcabd885ac3cf5e1df73fc622a1708bf3667bf5927576cdfa6"}, @@ -493,8 +552,16 @@ pytest = [ {file = "pytest-6.1.2-py3-none-any.whl", hash = "sha256:4288fed0d9153d9646bfcdf0c0428197dba1ecb27a33bb6e031d002fa88653fe"}, {file = "pytest-6.1.2.tar.gz", hash = "sha256:c0a7e94a8cdbc5422a51ccdad8e6f1024795939cc89159a0ae7f0b316ad3823e"}, ] +pytest-forked = [ + {file = "pytest-forked-1.3.0.tar.gz", hash = "sha256:6aa9ac7e00ad1a539c41bec6d21011332de671e938c7637378ec9710204e37ca"}, + {file = "pytest_forked-1.3.0-py2.py3-none-any.whl", hash = "sha256:dc4147784048e70ef5d437951728825a131b81714b398d5d52f17c7c144d8815"}, +] pytest-playbook = [] pytest-target = [] +pytest-xdist = [ + {file = "pytest-xdist-2.1.0.tar.gz", hash = "sha256:82d938f1a24186520e2d9d3a64ef7d9ac7ecdf1a0659e095d18e596b8cbd0672"}, + {file = "pytest_xdist-2.1.0-py3-none-any.whl", hash = "sha256:7c629016b3bb006b88ac68e2b31551e7becf173c76b977768848e2bbed594d90"}, +] pyyaml = [ {file = "PyYAML-5.3.1-cp27-cp27m-win32.whl", hash = "sha256:74809a57b329d6cc0fdccee6318f44b9b8649961fa73144a98735b0aaf029f1f"}, {file = "PyYAML-5.3.1-cp27-cp27m-win_amd64.whl", hash = "sha256:240097ff019d7c70a4922b6869d8a86407758333f02203e0fc6ff79c5dcede76"}, diff --git a/pytest-lisa/pyproject.toml b/pytest-lisa/pyproject.toml index 53dfad4e5a..3b73f0eab7 100644 --- a/pytest-lisa/pyproject.toml +++ b/pytest-lisa/pyproject.toml @@ -12,6 +12,7 @@ python = "^3.7" pytest = "^6.1.2" pytest-playbook = {path = "../pytest-playbook", develop = true} pytest-target = {path = "../pytest-target", develop = true} +pytest-xdist = "^2.1.0" schema = "0.7.2" [tool.poetry.dev-dependencies] From 3380c865886bf164e3241e604bc27e463b6bfcb6 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Wed, 9 Dec 2020 16:39:16 -0800 Subject: [PATCH 113/194] Use priority less likely to conflict in rules to allow ICMP on Azure --- pytest-target/target/azure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytest-target/target/azure.py b/pytest-target/target/azure.py index 0ee4649da9..adbce7b3c1 100644 --- a/pytest-target/target/azure.py +++ b/pytest-target/target/azure.py @@ -79,7 +79,7 @@ def allow_ping(self) -> None: self.local( f"az network nsg rule create " f"--name allow{d}ICMP --resource-group {self.name}-rg " - f"--nsg-name {self.name}NSG --priority 100 " + f"--nsg-name {self.name}NSG --priority 150 " f"--access Allow --direction '{d}' --protocol Icmp " "--source-port-ranges '*' --destination-port-ranges '*'" ) From d032bfac96befb1a7668057ab206c1c134382dfb Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Fri, 11 Dec 2020 16:13:26 -0800 Subject: [PATCH 114/194] Prefix target parameter with `Target=` --- pytest-target/target/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytest-target/target/plugin.py b/pytest-target/target/plugin.py index 9639f071d3..e4f21f9edb 100644 --- a/pytest-target/target/plugin.py +++ b/pytest-target/target/plugin.py @@ -180,7 +180,7 @@ def pytest_sessionstart() -> None: """Gather the `targets` from the playbook.""" for target in playbook.playbook.get("targets", []): targets.append(target) - target_ids.append(target["name"]) + target_ids.append("Target=" + target["name"]) def pytest_generate_tests(metafunc: Metafunc) -> None: From 46bcc26d61f34f5f2105f72c98a724d51c8cb7e5 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Fri, 11 Dec 2020 16:13:46 -0800 Subject: [PATCH 115/194] Adjust scheduler to split on target parameter, and add tests Run with `pytest --doctest-modules pytest-lisa/` --- pytest-lisa/lisa.py | 66 ++++++++++++++++++++++++++------------------- 1 file changed, 39 insertions(+), 27 deletions(-) diff --git a/pytest-lisa/lisa.py b/pytest-lisa/lisa.py index 4f879c4f0f..e3e790fe4d 100644 --- a/pytest-lisa/lisa.py +++ b/pytest-lisa/lisa.py @@ -187,10 +187,6 @@ class LISAScheduling(LoadScopeScheduling): multiple near-identical instances of a target in order to run tests in parallel. - TODO: We could also add an expected prefix to the target - parameter, like 'Target=' and then only split on it instead - of all parameters. - This is modeled after the built-in `LoadFileScheduling`, which also simply subclasses `LoadScopeScheduling`. See `_split_scope` for the important part. Note that we can extend this to implement @@ -205,33 +201,49 @@ def __init__(self, config: Config, log=None): # type: ignore else: self.log = log.lisasched - regex = re.compile(r"\[(\w+)\]") + regex = re.compile(r"\[Target=(\w+)\]") def _split_scope(self, nodeid: str) -> str: - """Determine the scope (grouping) of a nodeid. - - Example of a parameterized test's nodeid:: - - example/test_module.py::test_function[A] - example/test_module.py::test_function[B] - example/test_module.py::test_function_extra[A][B] - - `LoadScopeScheduling` uses `nodeid.rsplit("::", 1)[0]`, or the - first `::` from the right, to split by scope, such that - classes will be grouped, then modules. `LoadFileScheduling` - uses `nodeid.split("::", 1)[0]`, or the first `::` from the - left, to instead split only by modules (Python files). - - We opportunistically find all the parameters (strings within - square brackets) and join them with a slash to create the - scope. If the function is not parameterized, and so has no - square brackets, then we simply fallback to the algorithm of - `LoadScopeScheduling`. So the above would map into the scopes: - 'A', 'B', and 'A/B'. + """Determine the scope (grouping) of a `nodeid`. + + Example of a parameterized test's `nodeid`: + + * ``example/test_module.py::test_function[Target=A]`` + * ``example/test_module.py::test_function[A][Target=B]`` + * ``example/test_module.py::test_function_extra[A][B][Target=C]`` + + `LoadScopeScheduling` uses ``nodeid.rsplit("::", 1)[0]``, or + the first ``::`` from the right, to split by scope, such that + classes will be grouped, then modules. ``LoadFileScheduling`` + uses ``nodeid.split("::", 1)[0]``, or the first ``::`` from + the left, to instead split only by modules (Python files). + + We opportunistically find the "Target" parameter and use it as + the scope. If the target parameter is missing then we simply + fallback to the algorithm of `LoadScopeScheduling`. So the + above would map into the scopes: 'A', 'B', and 'C'. + + >>> class Config: + ... def getoption(self, option): + ... return False + ... def getvalue(self, value): + ... return ["popen"] + >>> s = LISAScheduling(Config()) + >>> s._split_scope("example/test_module.py::test_function[Target=A][B][C]") + 'A' + >>> s._split_scope("example/test_module.py::test_function[A][Target=B][C]") + 'B' + >>> s._split_scope("example/test_module.py::test_function[A][B][Target=C]") + 'C' + >>> s._split_scope("example/test_module.py::test_function") + 'example/test_module.py' + >>> s._split_scope("example/test_module.py::test_class::test_function") + 'example/test_module.py::test_class' """ - if "[" in nodeid: - scope = "/".join(self.regex.findall(nodeid)) + search = self.regex.search(nodeid) + if search: + scope = search.group(1) if self.config.getoption("verbose"): self.log(f"Split nodeid '{nodeid}' into scope '{scope}'") return scope From 1f4936536b815f662e6c923e225cb17422e38ac0 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Fri, 11 Dec 2020 17:15:42 -0800 Subject: [PATCH 116/194] Rename `parameters` to `params` because of its use frequency Also cleanup code duplication. --- playbooks/demo.yaml | 4 +-- pytest-target/target/azure.py | 9 +++--- pytest-target/target/plugin.py | 51 ++++++++++++---------------------- pytest-target/target/target.py | 6 ++-- 4 files changed, 28 insertions(+), 42 deletions(-) diff --git a/playbooks/demo.yaml b/playbooks/demo.yaml index dfd9f41f7a..a04709ccfb 100644 --- a/playbooks/demo.yaml +++ b/playbooks/demo.yaml @@ -1,12 +1,12 @@ targets: - name: Debian platform: Azure - parameters: + params: image: credativ:Debian:9:9.0.201706190 - name: Ubuntu platform: Azure - parameters: + params: image: UbuntuLTS criteria: diff --git a/pytest-target/target/azure.py b/pytest-target/target/azure.py index adbce7b3c1..7725b28827 100644 --- a/pytest-target/target/azure.py +++ b/pytest-target/target/azure.py @@ -28,6 +28,7 @@ def schema(cls) -> Schema: "image": str, Optional("sku", default="Standard_DS1_v2"): str, Optional("location", default="eastus2"): str, + # TODO: Remove or support this. Optional("networking", default=""): str, } ) @@ -88,10 +89,10 @@ def allow_ping(self) -> None: def deploy(self) -> str: """Given deployment info, deploy a new VM.""" - image = self.parameters["image"] - sku = self.parameters["sku"] - location = self.parameters["location"] - networking = self.parameters["networking"] + image = self.params["image"] + sku = self.params["sku"] + location = self.params["location"] + networking = self.params["networking"] Azure.check_az_cli() diff --git a/pytest-target/target/plugin.py b/pytest-target/target/plugin.py index e4f21f9edb..89d1e4db6f 100644 --- a/pytest-target/target/plugin.py +++ b/pytest-target/target/plugin.py @@ -46,10 +46,10 @@ def pytest_playbook_schema(schema: Dict[Any, Any]) -> None: The `platforms` global is a mapping of platform names (strings) to the implementing subclasses of `Target` where each subclass - defines its own `parameters` schema, `deploy` and `delete` + defines its own parameters `schema`, `deploy` and `delete` methods, and other platform-specific functionality. A `Target` subclass need only be defined in a file loaded by Pytest, so a - `contest.py` file works just fine. + `conftest.py` file works just fine. TODO: Add field annotations, friendly error reporting, automatic case transformations, etc. @@ -64,7 +64,7 @@ def pytest_playbook_schema(schema: Dict[Any, Any]) -> None: # TODO: What should we do when lacking parameters? Ideally we # use the platform’s defaults from its own schema, but that # means this value must be set, even if to an empty dict. - Optional("parameters", default=dict): Or( + Optional("params", default=dict): Or( *[cls.schema() for cls in platforms.values()] ), } @@ -79,16 +79,15 @@ def pool(request: SubRequest) -> Iterator[List[Target]]: targets: List[Target] = [] yield targets for t in targets: - print(f"Created target: {t.features} / {t.parameters}") + # TODO: Use proper logging? + print(f"Created target: {t.features} / {t.params}") if not request.config.getoption("keep_vms"): t.delete() def get_target( pool: List[Target], - platform: Type[Target], - parameters: Dict[str, Any], - features: Set[str], + request: SubRequest, ) -> Target: """This function gets or creates an appropriate `Target`. @@ -102,22 +101,25 @@ def get_target( tests such that they're grouped by features. """ + # Unpack the request. + platform: Type[Target] = platforms[request.param["platform"]] + params: Dict[str, Any] = request.param["params"] + # TODO: Use a ‘target’ marker instead. + marker = request.node.get_closest_marker("lisa") + features: Set[str] = set(marker.kwargs["features"]) + # TODO: If `t` is not already in use, deallocate the previous # target, and ensure the tests have been sorted (and so grouped) # by their requirements. for t in pool: # TODO: Implement full feature comparison, etc. and not just # proof-of-concept string set comparison. - if ( - isinstance(t, platform) - and t.parameters == parameters - and t.features >= features - ): + if isinstance(t, platform) and t.params == params and t.features >= features: pool.remove(t) return t else: # TODO: Reimplement caching. - t = platform(f"pytest-{uuid4()}", parameters, features) + t = platform(f"pytest-{uuid4()}", params, features) return t @@ -133,15 +135,8 @@ def target(pool: List[Target], request: SubRequest) -> Iterator[Target]: It is parametrized indirectly in `pytest_generate_tests`. - TODO: Clean up the code duplication here across the fixtures. - """ - platform = platforms[request.param["platform"]] - parameters = request.param["parameters"] - # TODO: Use a ‘target’ marker instead. - marker = request.node.get_closest_marker("lisa") - features = set(marker.kwargs["features"]) - t = get_target(pool, platform, parameters, features) + t = get_target(pool, request) yield t cleanup_target(pool, t) @@ -149,12 +144,7 @@ def target(pool: List[Target], request: SubRequest) -> Iterator[Target]: @pytest.fixture(scope="class") def c_target(pool: List[Target], request: SubRequest) -> Iterator[Target]: """This fixture is the same as `target` but shared across a class.""" - platform = platforms[request.param["platform"]] - parameters = request.param["parameters"] - # TODO: Use a ‘target’ marker instead. - marker = request.node.get_closest_marker("lisa") - features = set(marker.kwargs["features"]) - t = get_target(pool, platform, parameters, features) + t = get_target(pool, request) yield t cleanup_target(pool, t) @@ -162,12 +152,7 @@ def c_target(pool: List[Target], request: SubRequest) -> Iterator[Target]: @pytest.fixture(scope="module") def m_target(pool: List[Target], request: SubRequest) -> Iterator[Target]: """This fixture is the same as `target` but shared across a module.""" - platform = platforms[request.param["platform"]] - parameters = request.param["parameters"] - # TODO: Use a ‘target’ marker instead. - marker = request.node.get_closest_marker("lisa") - features = set(marker.kwargs["features"]) - t = get_target(pool, platform, parameters, features) + t = get_target(pool, request) yield t cleanup_target(pool, t) diff --git a/pytest-target/target/target.py b/pytest-target/target/target.py index 319a3837d3..94c85405aa 100644 --- a/pytest-target/target/target.py +++ b/pytest-target/target/target.py @@ -31,7 +31,7 @@ class Target(ABC): """ # Typed instance attributes, not class attributes. - parameters: Mapping[str, str] + params: Mapping[str, str] features: Set[str] name: str host: str @@ -55,7 +55,7 @@ class Target(ABC): def __init__( self, name: str, - parameters: Mapping[str, str], + params: Mapping[str, str], features: Set[str], ): """Requires a unique name. @@ -67,7 +67,7 @@ def __init__( """ self.name = name # TODO: Do we need to re-validate the parameters here? - self.parameters = parameters + self.params = params self.features = features # TODO: Review this thoroughly as currently it depends on From 13232eaab4325c0dd6eac8d443d075666ef8821a Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Fri, 11 Dec 2020 17:40:58 -0800 Subject: [PATCH 117/194] Rename `Local` platform to `SSH` and accept `host` param MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit So that it’s useful for testing against existing machines. --- playbooks/test.yaml | 2 +- pytest-target/target/__init__.py | 5 ++--- pytest-target/target/plugin.py | 2 +- pytest-target/target/target.py | 28 ++++++++++++++++++++++------ selftests/test_basic.py | 6 +++--- 5 files changed, 29 insertions(+), 14 deletions(-) diff --git a/playbooks/test.yaml b/playbooks/test.yaml index 0495002ab0..5c32b14290 100644 --- a/playbooks/test.yaml +++ b/playbooks/test.yaml @@ -1,5 +1,5 @@ targets: - name: Local Tests - platform: Local + platform: SSH - name: Setup Plan platform: Custom diff --git a/pytest-target/target/__init__.py b/pytest-target/target/__init__.py index 40f5a87afb..ec55c129bf 100644 --- a/pytest-target/target/__init__.py +++ b/pytest-target/target/__init__.py @@ -29,8 +29,7 @@ """ # Provide common types in the package's namespace. from target.azure import Azure -from target.plugin import pool, target -from target.target import Local, Target +from target.target import SSH, Target # NOTE: This is mostly to avoid “imported but not used.” -__all__ = ["Azure", "Target", "Local", "pool", "target"] +__all__ = ["Azure", "Target", "SSH"] diff --git a/pytest-target/target/plugin.py b/pytest-target/target/plugin.py index 89d1e4db6f..38a66e393e 100644 --- a/pytest-target/target/plugin.py +++ b/pytest-target/target/plugin.py @@ -69,7 +69,7 @@ def pytest_playbook_schema(schema: Dict[Any, Any]) -> None: ), } ) - default_targets = [{"name": "Default", "platform": "Local"}] + default_targets = [{"name": "Default", "platform": "SSH"}] schema[Optional("targets", default=default_targets)] = [target_schema] diff --git a/pytest-target/target/target.py b/pytest-target/target/target.py index 94c85405aa..0dd5335676 100644 --- a/pytest-target/target/target.py +++ b/pytest-target/target/target.py @@ -8,8 +8,8 @@ import fabric # type: ignore import invoke # type: ignore +import schema # type: ignore from invoke.runners import Result # type: ignore -from schema import Schema # type: ignore from tenacity import retry, stop_after_attempt, wait_exponential # type: ignore if typing.TYPE_CHECKING: @@ -90,7 +90,7 @@ def __init__( # in Python 3.9 and up. @classmethod @abstractmethod - def schema(cls) -> Schema: + def schema(cls) -> schema.Schema: """Must return a schema for expected instance parameters. TODO: This schema is used for each instance. We may want to @@ -130,13 +130,29 @@ def cat(self, path: str) -> str: return buf.getvalue().decode("utf-8").strip() -class Local(Target): +class SSH(Target): + """The `SSH` platform simply connects to existing targets. + + It does not deploy nor delete the target. The default ``host`` is + ``localhost`` so this can be used for testing against the user's + system (if SSH is enabled). + + """ + @classmethod - def schema(cls) -> Schema: - return Schema(None) + def schema(cls) -> schema.Schema: + return schema.Schema( + { + schema.Optional( + "host", + default="localhost", + description="The address of the destination target.", + ): str + } + ) def deploy(self) -> str: - return "localhost" + return self.params["host"] def delete(self) -> None: pass diff --git a/selftests/test_basic.py b/selftests/test_basic.py index f592b28cba..fa948a8472 100644 --- a/selftests/test_basic.py +++ b/selftests/test_basic.py @@ -4,12 +4,12 @@ import typing if typing.TYPE_CHECKING: - from target import Local + from target import SSH from lisa import LISA @LISA(platform="Local", category="Functional", area="self-test", priority=1) -def test_basic(target: Local) -> None: - """Basic test which creates a Node connection to 'localhost'.""" +def test_basic(target: SSH) -> None: + """Basic test which creates a `Target` connection to 'localhost'.""" target.local("echo Hello World") From 61fbce99c337993a8542d1ab34a57b96a961dcbc Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Fri, 11 Dec 2020 17:42:38 -0800 Subject: [PATCH 118/194] Setup assertion rewriting for `azure` and `target` modules --- pytest-target/target/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pytest-target/target/__init__.py b/pytest-target/target/__init__.py index ec55c129bf..c8aa2c5b69 100644 --- a/pytest-target/target/__init__.py +++ b/pytest-target/target/__init__.py @@ -27,9 +27,14 @@ deployed targets. """ +import pytest + # Provide common types in the package's namespace. from target.azure import Azure from target.target import SSH, Target # NOTE: This is mostly to avoid “imported but not used.” __all__ = ["Azure", "Target", "SSH"] + +# See https://docs.pytest.org/en/stable/writing_plugins.html#assertion-rewriting +pytest.register_assert_rewrite("pytest_target.azure", "pytest_target.target") From 700ae226fceb47091a9218aa7af01475fc8aac84 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Fri, 11 Dec 2020 17:52:36 -0800 Subject: [PATCH 119/194] Rename `playbook.playbook` to `playbook.data` And fix default data when no playbook is specified. --- pytest-lisa/lisa.py | 2 +- pytest-playbook/playbook.py | 30 ++++++++++++++++-------------- pytest-target/target/plugin.py | 2 +- 3 files changed, 18 insertions(+), 16 deletions(-) diff --git a/pytest-lisa/lisa.py b/pytest-lisa/lisa.py index e3e790fe4d..9677ab9f8e 100644 --- a/pytest-lisa/lisa.py +++ b/pytest-lisa/lisa.py @@ -147,7 +147,7 @@ def select(item: Item, times: int, exclude: bool) -> None: for _ in range(times - included.count(item)): included.append(item) - for c in playbook.playbook.get("criteria", []): + for c in playbook.data.get("criteria", []): for item in items: mark = item.get_closest_marker("lisa") if not mark: diff --git a/pytest-playbook/playbook.py b/pytest-playbook/playbook.py index ab03c42e23..862e1a1b28 100644 --- a/pytest-playbook/playbook.py +++ b/pytest-playbook/playbook.py @@ -37,8 +37,8 @@ from _pytest.config import Config, PytestPluginManager from _pytest.config.argparsing import Parser -# TODO: I’m not a fan of this name. Maybe ‘params’ or ‘data’? -playbook: Dict[Any, Any] = dict() +data: Dict[Any, Any] = dict() +"""This global is the data read from the given playbook.""" class Hooks: @@ -85,18 +85,20 @@ def pytest_configure(config: Config) -> None: loaded and defined their `pytest_playbook_schema` hooks. """ - path: Optional[Path] = config.getoption("playbook") - if not path or not path.is_file(): - # TODO: Log an appropriate warning. - return - schema: Dict[Any, Any] = dict() config.hook.pytest_playbook_schema(schema=schema, config=config) - global playbook - try: - with open(path) as f: - data = yaml.load(f, Loader=Loader) - playbook = Schema(schema).validate(data) - except (yaml.YAMLError, SchemaMissingKeyError, OSError) as e: - pytest.exit(f"Error loading playbook '{path}': {e}") + global data + + path: Optional[Path] = config.getoption("playbook") + if not path or not path.is_file(): + # TODO: Use proper logging? + print("No playbook was specified, using defaults...") + data = Schema(schema).validate({}) + else: + try: + with open(path) as f: + data = yaml.load(f, Loader=Loader) + data = Schema(schema).validate(data) + except (yaml.YAMLError, SchemaMissingKeyError, OSError) as e: + pytest.exit(f"Error loading playbook '{path}': {e}") diff --git a/pytest-target/target/plugin.py b/pytest-target/target/plugin.py index 38a66e393e..6e7cff4b35 100644 --- a/pytest-target/target/plugin.py +++ b/pytest-target/target/plugin.py @@ -163,7 +163,7 @@ def m_target(pool: List[Target], request: SubRequest) -> Iterator[Target]: def pytest_sessionstart() -> None: """Gather the `targets` from the playbook.""" - for target in playbook.playbook.get("targets", []): + for target in playbook.data.get("targets", []): targets.append(target) target_ids.append("Target=" + target["name"]) From 8d3005a301969463aab2b9014c436ff085547ab1 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Fri, 11 Dec 2020 19:21:33 -0800 Subject: [PATCH 120/194] Flatten the playbook schema MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit By merging the platforms’ schema dictionaries with the mandatory keys we can flatten the playbook schema entirely, eliminating the nested “params” key. We also don’t need each platform to return a `Schema` object, but just a mapping. --- playbooks/demo.yaml | 6 ++---- pytest-target/target/azure.py | 24 ++++++++++----------- pytest-target/target/plugin.py | 39 ++++++++++++++++++++-------------- pytest-target/target/target.py | 28 ++++++++++++------------ selftests/conftest.py | 12 ++++++++--- 5 files changed, 59 insertions(+), 50 deletions(-) diff --git a/playbooks/demo.yaml b/playbooks/demo.yaml index a04709ccfb..c398260457 100644 --- a/playbooks/demo.yaml +++ b/playbooks/demo.yaml @@ -1,13 +1,11 @@ targets: - name: Debian platform: Azure - params: - image: credativ:Debian:9:9.0.201706190 + image: credativ:Debian:9:9.0.201706190 - name: Ubuntu platform: Azure - params: - image: UbuntuLTS + image: UbuntuLTS criteria: - module: smoke diff --git a/pytest-target/target/azure.py b/pytest-target/target/azure.py index 7725b28827..d09809c9d7 100644 --- a/pytest-target/target/azure.py +++ b/pytest-target/target/azure.py @@ -6,12 +6,12 @@ import typing from invoke.runners import Result # type: ignore -from schema import Optional, Schema # type: ignore +from schema import Optional # type: ignore from target.target import Target from tenacity import retry, stop_after_attempt, wait_exponential # type: ignore if typing.TYPE_CHECKING: - from typing import Any + from typing import Any, Mapping class Azure(Target): @@ -21,17 +21,15 @@ class Azure(Target): internal_address: str @classmethod - def schema(cls) -> Schema: - return Schema( - { - # TODO: Maybe validate as URN or path etc. - "image": str, - Optional("sku", default="Standard_DS1_v2"): str, - Optional("location", default="eastus2"): str, - # TODO: Remove or support this. - Optional("networking", default=""): str, - } - ) + def schema(cls) -> Mapping[Any, Any]: + return { + # TODO: Maybe validate as URN or path etc. + "image": str, + Optional("sku", default="Standard_DS1_v2"): str, + Optional("location", default="eastus2"): str, + # TODO: Remove or support this. + Optional("networking", default=""): str, + } # A class attribute because it’s defined. az_ok = False diff --git a/pytest-target/target/plugin.py b/pytest-target/target/plugin.py index 6e7cff4b35..3fb9dfd2b1 100644 --- a/pytest-target/target/plugin.py +++ b/pytest-target/target/plugin.py @@ -21,8 +21,8 @@ import pytest # See https://pypi.org/project/schema/ -from schema import Optional, Or, Schema # type: ignore -from target.target import Target +from schema import Literal, Optional, Or, Schema # type: ignore +from target.target import SSH, Target if typing.TYPE_CHECKING: from typing import Any, Dict, Iterator, List, Set, Type @@ -57,19 +57,26 @@ def pytest_playbook_schema(schema: Dict[Any, Any]) -> None: """ global platforms platforms = {cls.__name__: cls for cls in Target.__subclasses__()} # type: ignore - target_schema = Schema( - { - "name": str, - "platform": Or(*[platform for platform in platforms.keys()]), - # TODO: What should we do when lacking parameters? Ideally we - # use the platform’s defaults from its own schema, but that - # means this value must be set, even if to an empty dict. - Optional("params", default=dict): Or( - *[cls.schema() for cls in platforms.values()] - ), - } + target_schema = Or( + # We’re unpacking a list of updated schema from each platform. + *( + { + # We’re adding ‘name’ and ‘platform’ keys to each + # platform’s schema. + Literal("name", description="A friendly name for the target."): str, + Literal( + "platform", + description="The literal class name of the platform implementation," + + " a subclass of `Target`.", + ): name, + **cls.schema(), + } + for name, cls in platforms.items() + ) ) - default_targets = [{"name": "Default", "platform": "SSH"}] + default_targets = [ + {"name": "Default", "platform": "SSH", **Schema(SSH.schema()).validate({})} + ] schema[Optional("targets", default=default_targets)] = [target_schema] @@ -102,8 +109,8 @@ def get_target( """ # Unpack the request. - platform: Type[Target] = platforms[request.param["platform"]] - params: Dict[str, Any] = request.param["params"] + params: Dict[str, Any] = request.param + platform: Type[Target] = platforms[params["platform"]] # TODO: Use a ‘target’ marker instead. marker = request.node.get_closest_marker("lisa") features: Set[str] = set(marker.kwargs["features"]) diff --git a/pytest-target/target/target.py b/pytest-target/target/target.py index 0dd5335676..71453196f4 100644 --- a/pytest-target/target/target.py +++ b/pytest-target/target/target.py @@ -90,11 +90,13 @@ def __init__( # in Python 3.9 and up. @classmethod @abstractmethod - def schema(cls) -> schema.Schema: - """Must return a schema for expected instance parameters. + def schema(cls) -> Mapping[Any, Any]: + """Must return a mapping for expected instance parameters. - TODO: This schema is used for each instance. We may want to - define platform-level shared schemata too. + The items in this mapping are added to the playbook schema, so + they may container objects from the `schema` library. Each + target in the playbook will have `name` and `platform` keys in + addition to those specified here (they're merged). """ ... @@ -140,16 +142,14 @@ class SSH(Target): """ @classmethod - def schema(cls) -> schema.Schema: - return schema.Schema( - { - schema.Optional( - "host", - default="localhost", - description="The address of the destination target.", - ): str - } - ) + def schema(cls) -> Mapping[Any, Any]: + return { + schema.Optional( + "host", + default="localhost", + description="The address of the destination target.", + ): str + } def deploy(self) -> str: return self.params["host"] diff --git a/selftests/conftest.py b/selftests/conftest.py index cbcfcd1b5e..94a42e6165 100644 --- a/selftests/conftest.py +++ b/selftests/conftest.py @@ -1,11 +1,17 @@ -from schema import Schema # type: ignore +from __future__ import annotations + +import typing + +if typing.TYPE_CHECKING: + from typing import Any, Mapping + from target import Target class Custom(Target): @classmethod - def schema(cls) -> Schema: - return Schema(None) + def schema(cls) -> Mapping[Any, Any]: + return {} def deploy(self) -> str: return "localhost" From 42e495360586d6f3932403de2c892314ce611ea4 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Fri, 11 Dec 2020 19:23:20 -0800 Subject: [PATCH 121/194] Fix scheduler to handle whitespace in target names --- pytest-lisa/lisa.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pytest-lisa/lisa.py b/pytest-lisa/lisa.py index 9677ab9f8e..8eb2bebdb8 100644 --- a/pytest-lisa/lisa.py +++ b/pytest-lisa/lisa.py @@ -201,7 +201,8 @@ def __init__(self, config: Config, log=None): # type: ignore else: self.log = log.lisasched - regex = re.compile(r"\[Target=(\w+)\]") + # NOTE: Needs to handle whitespace, so can’t be `\w+`. + regex = re.compile(r"\[Target=([^\[\]]+)\]") def _split_scope(self, nodeid: str) -> str: """Determine the scope (grouping) of a `nodeid`. From 51a29e33ad941101e309436787653cdbffd5663d Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Fri, 11 Dec 2020 19:24:10 -0800 Subject: [PATCH 122/194] Add basic schema printing option to playbook --- pytest-playbook/playbook.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/pytest-playbook/playbook.py b/pytest-playbook/playbook.py index 862e1a1b28..ae879b04ab 100644 --- a/pytest-playbook/playbook.py +++ b/pytest-playbook/playbook.py @@ -17,6 +17,7 @@ from __future__ import annotations +import json import typing from pathlib import Path @@ -74,6 +75,10 @@ def pytest_addoption(parser: Parser, pluginmanager: PytestPluginManager) -> None """Pytest hook to add our CLI options.""" group = parser.getgroup("playbook") group.addoption("--playbook", type=Path, help="Path to playbook.") + group.addoption( + "--print-schema", + help="Print the JSON schema of the playbook with the given ID.", + ) # TODO: See if this works without ‘trylast’. @@ -88,6 +93,11 @@ def pytest_configure(config: Config) -> None: schema: Dict[Any, Any] = dict() config.hook.pytest_playbook_schema(schema=schema, config=config) + json_schema = config.getoption("print_schema") + if json_schema: + print(json.dumps(Schema(schema).json_schema(json_schema), indent=2)) + pytest.exit("Printed schema!", pytest.ExitCode.OK) + global data path: Optional[Path] = config.getoption("playbook") @@ -101,4 +111,6 @@ def pytest_configure(config: Config) -> None: data = yaml.load(f, Loader=Loader) data = Schema(schema).validate(data) except (yaml.YAMLError, SchemaMissingKeyError, OSError) as e: - pytest.exit(f"Error loading playbook '{path}': {e}") + pytest.exit( + f"Error loading playbook '{path}': {e}", pytest.ExitCode.USAGE_ERROR + ) From 8f4b0429984550c00369776dff6056885d3c3c23 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Mon, 14 Dec 2020 14:19:56 -0800 Subject: [PATCH 123/194] Simplify logic for generating target schema --- pytest-target/target/plugin.py | 41 +++++++++++++++++----------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/pytest-target/target/plugin.py b/pytest-target/target/plugin.py index 3fb9dfd2b1..eb97409223 100644 --- a/pytest-target/target/plugin.py +++ b/pytest-target/target/plugin.py @@ -55,29 +55,30 @@ def pytest_playbook_schema(schema: Dict[Any, Any]) -> None: case transformations, etc. """ + # Map the subclasses of `Target` into name and class pairs, used + # by `get_target` to lookup the type based on the name. global platforms platforms = {cls.__name__: cls for cls in Target.__subclasses__()} # type: ignore - target_schema = Or( - # We’re unpacking a list of updated schema from each platform. - *( - { - # We’re adding ‘name’ and ‘platform’ keys to each - # platform’s schema. - Literal("name", description="A friendly name for the target."): str, - Literal( - "platform", - description="The literal class name of the platform implementation," - + " a subclass of `Target`.", - ): name, - **cls.schema(), - } - for name, cls in platforms.items() - ) - ) - default_targets = [ - {"name": "Default", "platform": "SSH", **Schema(SSH.schema()).validate({})} + platform_description = "The class name of the platform implementation." + + # The target schema is `anyOf`/`Or` the platform’s schemas. + target_schemas = [ + { + # We’re adding ‘name’ and ‘platform’ keys to each + # platform’s schema. + Literal("name", description="A friendly name for the target."): str, + Literal("platform", description=platform_description): platform, + **cls.schema(), + } + for platform, cls in platforms.items() ] - schema[Optional("targets", default=default_targets)] = [target_schema] + target_schema = Or(*target_schemas) + default_target = { + "name": "Default", + "platform": "SSH", + **Schema(SSH.schema()).validate({}), # Fill in the defaults + } + schema[Optional("targets", default=[default_target])] = [target_schema] @pytest.fixture(scope="session") From 915aef8fb2004013e769d56cebd16c8f4ead9840 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Mon, 14 Dec 2020 16:39:49 -0800 Subject: [PATCH 124/194] Enable mutable platform defaults in the playbook --- README.md | 2 +- pytest-lisa/lisa.py | 1 + pytest-target/target/plugin.py | 45 +++++++++++++++++++++++++--------- pytest-target/target/target.py | 32 +++++++++++++++++++----- 4 files changed, 61 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 583bc81b38..b6556bcd42 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ poetry shell lisa --playbook=playbooks/test.yml selftests/ # Run a demo which deployes Azure resources -lisa --playbooks/smoke.yaml +lisa --playbook=playbooks/smoke.yaml ``` #### Enable Azure: diff --git a/pytest-lisa/lisa.py b/pytest-lisa/lisa.py index 8eb2bebdb8..8fa6accc93 100644 --- a/pytest-lisa/lisa.py +++ b/pytest-lisa/lisa.py @@ -92,6 +92,7 @@ def pytest_playbook_schema(schema: Dict[Any, Any]) -> None: lisa_schema = Schema( { + # TODO: Move platform to `pytest.mark.target`. "platform": str, "category": Or("Functional", "Performance", "Stress", "Community", "Longhaul"), "area": str, diff --git a/pytest-target/target/plugin.py b/pytest-target/target/plugin.py index eb97409223..b3ce1ba984 100644 --- a/pytest-target/target/plugin.py +++ b/pytest-target/target/plugin.py @@ -1,7 +1,6 @@ """Provides and parameterizes the `pool` and `target` fixtures. -TODO -==== +# TODO * Provide a `targets` fixture for tests which use more than one target at a time. * Deallocate targets when switching to a new target. @@ -9,7 +8,6 @@ * Add `pytest.mark.target` instead of LISA mark for target requirements. * Reimplement caching of targets between runs. -* Improve schema with annotations, error messages, etc. """ from __future__ import annotations @@ -46,20 +44,37 @@ def pytest_playbook_schema(schema: Dict[Any, Any]) -> None: The `platforms` global is a mapping of platform names (strings) to the implementing subclasses of `Target` where each subclass - defines its own parameters `schema`, `deploy` and `delete` - methods, and other platform-specific functionality. A `Target` - subclass need only be defined in a file loaded by Pytest, so a - `conftest.py` file works just fine. - - TODO: Add field annotations, friendly error reporting, automatic - case transformations, etc. + defines its own parameters `schema`, optional `defaults`, + `deploy`, `delete` methods, and other platform-specific + functionality. A `Target` subclass need only be defined in a file + loaded by Pytest, so a `conftest.py` file works just fine. """ # Map the subclasses of `Target` into name and class pairs, used # by `get_target` to lookup the type based on the name. global platforms platforms = {cls.__name__: cls for cls in Target.__subclasses__()} # type: ignore + + # The platform schema is an optional mapping of the platform name + # to defaults for its provided schema. Setting the defaults here + # is a bit particular, as we need to have the schema library parse + # the given dict as a schema, and fill in the nested defaults. platform_description = "The class name of the platform implementation." + platform_schemas = { + Optional( + platform, + default=Schema(cls.defaults()).validate({}), + description=platform_description, + ): cls.defaults() + for platform, cls in platforms.items() + } + schema[ + Optional( + "platforms", + default=Schema(platform_schemas).validate({}), + description="Default values for each platform.", + ) + ] = platform_schemas # The target schema is `anyOf`/`Or` the platform’s schemas. target_schemas = [ @@ -68,6 +83,7 @@ def pytest_playbook_schema(schema: Dict[Any, Any]) -> None: # platform’s schema. Literal("name", description="A friendly name for the target."): str, Literal("platform", description=platform_description): platform, + # Unpack the rest of the schema’s items. **cls.schema(), } for platform, cls in platforms.items() @@ -171,8 +187,13 @@ def m_target(pool: List[Target], request: SubRequest) -> Iterator[Target]: def pytest_sessionstart() -> None: """Gather the `targets` from the playbook.""" - for target in playbook.data.get("targets", []): - targets.append(target) + platform_defaults = playbook.data.get("platforms") + for target in playbook.data.get("targets"): + # Get a copy of this platform’s defaults (which may not exist) + # and update them with this target’s specific parameters. + params = platform_defaults.get(target["platform"]).copy() + params.update(target) + targets.append(params) target_ids.append("Target=" + target["name"]) diff --git a/pytest-target/target/target.py b/pytest-target/target/target.py index 71453196f4..d171783acb 100644 --- a/pytest-target/target/target.py +++ b/pytest-target/target/target.py @@ -94,16 +94,30 @@ def schema(cls) -> Mapping[Any, Any]: """Must return a mapping for expected instance parameters. The items in this mapping are added to the playbook schema, so - they may container objects from the `schema` library. Each + they may contain objects from the `schema` library. Each target in the playbook will have `name` and `platform` keys in - addition to those specified here (they're merged). + addition to those specified here (they're merged). Parameters + should generally be `schema.Optional`. If the parameter should + have a shared but mutable default value, set it in `defaults`. """ ... + @classmethod + def defaults(cls) -> Mapping[Any, Any]: + """Can return a mapping for default parameters. + + If specified, it should contain only `schema.Optional` + elements, where the names and types match those in `schema`, + but with a set default value, and those in `schema` should not + contain default values. This is used a base for each target. + + """ + return {} + @abstractmethod def deploy(self) -> str: - """Must deploy the target resources and return hostname.""" + """Must deploy the target resources and return the hostname.""" ... @abstractmethod @@ -145,9 +159,15 @@ class SSH(Target): def schema(cls) -> Mapping[Any, Any]: return { schema.Optional( - "host", - default="localhost", - description="The address of the destination target.", + "host", description="The address of the destination target." + ): str + } + + @classmethod + def defaults(cls) -> Mapping[Any, Any]: + return { + schema.Optional( + "host", default="localhost", description="The default value for host." ): str } From 9937d37c4b0ec0fd5f6f445459f3ba738e427aab Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Mon, 14 Dec 2020 17:01:58 -0800 Subject: [PATCH 125/194] Add descriptions to pytest-lisa schemata --- pytest-lisa/lisa.py | 72 +++++++++++++++++++++++++++++++++------------ 1 file changed, 53 insertions(+), 19 deletions(-) diff --git a/pytest-lisa/lisa.py b/pytest-lisa/lisa.py index 8fa6accc93..c2504366a7 100644 --- a/pytest-lisa/lisa.py +++ b/pytest-lisa/lisa.py @@ -17,10 +17,8 @@ - area: xdp exclude: true -TODO -==== +# TODO: * Provide test metadata statistics via a command-line flag. -* Improve schemata with annotations, error messages, etc. * Assert every test has a LISA marker. * Remove 'features' from marker. @@ -34,7 +32,7 @@ import playbook import py import pytest -from schema import Optional, Or, Schema, SchemaMissingKeyError # type: ignore +from schema import Literal, Optional, Or, Schema, SchemaMissingKeyError # type: ignore from xdist.scheduler.loadscope import LoadScopeScheduling # type: ignore if typing.TYPE_CHECKING: @@ -75,16 +73,44 @@ def pytest_playbook_schema(schema: Dict[Any, Any]) -> None: # the default. criteria_schema = Schema( { - # TODO: Validate that these strings are valid regular - # expressions if we change our matching logic. - Optional("name", default=None): str, - Optional("module", default=None): str, - Optional("area", default=None): str, - Optional("category", default=None): str, - Optional("priority", default=None): int, - Optional("tags", default=list): [str], - Optional("times", default=1): int, - Optional("exclude", default=False): bool, + # TODO: Should any/all of the strings be regex comparisons? + Optional( + "name", description="Substring match of test name.", default=None + ): str, + Optional( + "module", + description="Substring match of test file (Python module).", + default=None, + ): str, + Optional( + "area", + description="Case-folded equality comparison of test's area.", + default=None, + ): str, + Optional( + "category", + description="Case-folded equality comparison of test's category.", + default=None, + ): str, + Optional( + "priority", + # TODO: Should this instead be a range comparison? + description="Equality comparison of test's priority.", + default=None, + ): int, + Optional( + "tags", description="Subset comparison of test's tags.", default=list + ): [str], + Optional( + "times", + description="Number of times to run the matched tests.", + default=1, + ): int, + Optional( + "exclude", + description="Exclude the matched tests instead.", + default=False, + ): bool, } ) schema[Optional("criteria", default=list)] = [criteria_schema] @@ -93,14 +119,22 @@ def pytest_playbook_schema(schema: Dict[Any, Any]) -> None: lisa_schema = Schema( { # TODO: Move platform to `pytest.mark.target`. - "platform": str, - "category": Or("Functional", "Performance", "Stress", "Community", "Longhaul"), - "area": str, - "priority": Or(0, 1, 2, 3), + Literal("platform", description="The test's intended platform."): str, + Literal("category", description="The kind of test this is."): Or( + "Functional", "Performance", "Stress", "Community", "Longhaul" + ), + Literal("area", description="The test's area (or 'feature')."): str, + Literal( + "priority", description="The test's priority with 0 being the highest." + ): Or(0, 1, 2, 3), # TODO: Move `features` to `pytest.mark.target` and don’t # allow extra keys. Optional("features", default=list): [str], - Optional("tags", default=list): [str], + Optional( + "tags", + description="An arbitrary set of tags used for selection.", + default=list, + ): [str], Optional(object): object, }, ignore_extra_keys=True, From d207d734541cf21e5ef7cd9ca63e85996dea427b Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Mon, 14 Dec 2020 17:52:16 -0800 Subject: [PATCH 126/194] Add `pytest.mark.target` to `pytest-target` to finish decoupling --- pytest-lisa/lisa.py | 3 +-- pytest-target/target/plugin.py | 19 +++++++++++++++---- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/pytest-lisa/lisa.py b/pytest-lisa/lisa.py index c2504366a7..4e197b1520 100644 --- a/pytest-lisa/lisa.py +++ b/pytest-lisa/lisa.py @@ -20,7 +20,6 @@ # TODO: * Provide test metadata statistics via a command-line flag. * Assert every test has a LISA marker. -* Remove 'features' from marker. """ from __future__ import annotations @@ -60,7 +59,7 @@ def pytest_configure(config: Config) -> None: config.addinivalue_line( "markers", ( - "lisa(platform, category, area, priority, features, tags): " + "lisa(platform, category, area, priority, tags, features): " "Annotate a test with metadata." ), ) diff --git a/pytest-target/target/plugin.py b/pytest-target/target/plugin.py index b3ce1ba984..619849db4b 100644 --- a/pytest-target/target/plugin.py +++ b/pytest-target/target/plugin.py @@ -5,8 +5,6 @@ at a time. * Deallocate targets when switching to a new target. * Use richer feature/requirements comparison for targets. -* Add `pytest.mark.target` instead of LISA mark for target - requirements. * Reimplement caching of targets between runs. """ @@ -25,6 +23,7 @@ if typing.TYPE_CHECKING: from typing import Any, Dict, Iterator, List, Set, Type + from _pytest.config import Config from _pytest.config.argparsing import Parser from _pytest.fixtures import SubRequest from _pytest.python import Metafunc @@ -36,6 +35,19 @@ def pytest_addoption(parser: Parser) -> None: group.addoption("--keep-vms", action="store_true", help="Keeps deployed VMs.") +def pytest_configure(config: Config) -> None: + """Pytest hook to perform initial configuration. + + We're registering our custom marker so that it passes + `--strict-markers`. + + """ + config.addinivalue_line( + "markers", + ("target(platform, features): " "Specify target requirements."), + ) + + platforms: Dict[str, Type[Target]] = dict() @@ -128,8 +140,7 @@ def get_target( # Unpack the request. params: Dict[str, Any] = request.param platform: Type[Target] = platforms[params["platform"]] - # TODO: Use a ‘target’ marker instead. - marker = request.node.get_closest_marker("lisa") + marker = request.node.get_closest_marker("target") features: Set[str] = set(marker.kwargs["features"]) # TODO: If `t` is not already in use, deallocate the previous From 0585ab6835bc0e4233c44d64c9fedc7ef5e13e0e Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Mon, 14 Dec 2020 18:29:02 -0800 Subject: [PATCH 127/194] Forward `pytest.mark.lisa(features=...)` to `pytest.mark.target` And stop allowing extra keys in the LISA schema. --- playbooks/demo.yaml | 3 +++ pytest-lisa/lisa.py | 42 +++++++++++++++++++++++--------------- testsuites/test_smoke_a.py | 1 - testsuites/test_smoke_b.py | 1 - 4 files changed, 28 insertions(+), 19 deletions(-) diff --git a/playbooks/demo.yaml b/playbooks/demo.yaml index c398260457..85d0e25906 100644 --- a/playbooks/demo.yaml +++ b/playbooks/demo.yaml @@ -1,3 +1,6 @@ +platforms: + Azure: + sku: Standard_DS2_v2 targets: - name: Debian platform: Azure diff --git a/pytest-lisa/lisa.py b/pytest-lisa/lisa.py index 4e197b1520..2ad50cfa9f 100644 --- a/pytest-lisa/lisa.py +++ b/pytest-lisa/lisa.py @@ -31,7 +31,7 @@ import playbook import py import pytest -from schema import Literal, Optional, Or, Schema, SchemaMissingKeyError # type: ignore +from schema import Literal, Optional, Or, Schema, SchemaError # type: ignore from xdist.scheduler.loadscope import LoadScopeScheduling # type: ignore if typing.TYPE_CHECKING: @@ -117,7 +117,6 @@ def pytest_playbook_schema(schema: Dict[Any, Any]) -> None: lisa_schema = Schema( { - # TODO: Move platform to `pytest.mark.target`. Literal("platform", description="The test's intended platform."): str, Literal("category", description="The kind of test this is."): Or( "Functional", "Performance", "Stress", "Community", "Longhaul" @@ -126,26 +125,23 @@ def pytest_playbook_schema(schema: Dict[Any, Any]) -> None: Literal( "priority", description="The test's priority with 0 being the highest." ): Or(0, 1, 2, 3), - # TODO: Move `features` to `pytest.mark.target` and don’t - # allow extra keys. - Optional("features", default=list): [str], Optional( "tags", description="An arbitrary set of tags used for selection.", - default=list, + default=[], ): [str], - Optional(object): object, - }, - ignore_extra_keys=True, + # TODO: Consider just making users set this manually. + Optional( + "features", + description="A set of required features, passed to `pytest.mark.target`.", + default=[], + ): [str], + } ) -def validate_mark(mark: typing.Optional[Mark]) -> None: +def validate_mark(mark: Mark) -> None: """Validate each test's LISA parameters.""" - if not mark: - # TODO: `assert mark, "LISA marker is missing!"` but not all - # tests will have it, such as static analysis tests. - return assert not mark.args, "LISA marker cannot have positional arguments!" mark.kwargs.update(lisa_schema.validate(mark.kwargs)) # type: ignore @@ -155,7 +151,11 @@ def pytest_collection_modifyitems( ) -> None: """Pytest hook for modifying the selected items (tests). - https://docs.pytest.org/en/latest/reference.html#pytest.hookspec.pytest_collection_modifyitems + First we validate all the `LISA` marks on the collected tests. + Then we parse the given `criteria` in the playbook to include or + exclude tests. We do not care if the `platform` mismatches because + we intend a multiplicative effect where all selected tests in a + playbook are run on all the targets. """ # TODO: The ‘Item’ object has a ‘user_properties’ attribute which @@ -165,8 +165,16 @@ def pytest_collection_modifyitems( # Validate all LISA marks. for item in items: try: - validate_mark(item.get_closest_marker("lisa")) - except (SchemaMissingKeyError, AssertionError) as e: + mark = item.get_closest_marker("lisa") + # TODO: `assert mark, "LISA marker is missing!"` but not + # all tests will have it, such as static analysis tests. + if not mark: + continue + validate_mark(mark) + # Forward `features` to `pytest.mark.target` so LISA users + # don’t need to use two marks, but keep them decoupled. + item.add_marker(pytest.mark.target(features=mark.kwargs["features"])) + except (SchemaError, AssertionError) as e: pytest.exit(f"Error validating test '{item.name}' metadata: {e}") # Optionally select tests based on a playbook. diff --git a/testsuites/test_smoke_a.py b/testsuites/test_smoke_a.py index bc9491e228..1a2fff0d89 100644 --- a/testsuites/test_smoke_a.py +++ b/testsuites/test_smoke_a.py @@ -29,7 +29,6 @@ category="Functional", area="deploy", priority=0, - sku="Standard_DS2_v2", ) diff --git a/testsuites/test_smoke_b.py b/testsuites/test_smoke_b.py index ab7f807106..c07c408791 100644 --- a/testsuites/test_smoke_b.py +++ b/testsuites/test_smoke_b.py @@ -21,7 +21,6 @@ category="Functional", area="deploy", priority=0, - sku="Standard_DS2_v2", ) def test_smoke(target: Azure, caplog: LogCaptureFixture) -> None: """Check that an Azure Linux VM can be deployed and is responsive. From 0cda8c37c9fa7d459c55870ab7e20ac25694f8e3 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Mon, 14 Dec 2020 18:30:36 -0800 Subject: [PATCH 128/194] Stop matching `Target.params` when searching for targets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As we intend to have looser requirements, based on features (plus any additional requirements comparison that gets implemented). We can no longer assert on the params set because it contains the playbook’s targets’ names, which mean every target would have to be unique, preventing reuse. Also cleanup parameterization with a loop. --- pytest-target/target/plugin.py | 40 +++++++++++++++++----------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/pytest-target/target/plugin.py b/pytest-target/target/plugin.py index 619849db4b..11556f73ef 100644 --- a/pytest-target/target/plugin.py +++ b/pytest-target/target/plugin.py @@ -21,7 +21,7 @@ from target.target import SSH, Target if typing.TYPE_CHECKING: - from typing import Any, Dict, Iterator, List, Set, Type + from typing import Any, Dict, Iterator, List, Type from _pytest.config import Config from _pytest.config.argparsing import Parser @@ -137,11 +137,12 @@ def get_target( tests such that they're grouped by features. """ - # Unpack the request. - params: Dict[str, Any] = request.param - platform: Type[Target] = platforms[params["platform"]] + # Get the intended class for this parameterization of `target`. + platform: Type[Target] = platforms[request.param["platform"]] + + # Get the required features for this test. marker = request.node.get_closest_marker("target") - features: Set[str] = set(marker.kwargs["features"]) + features = set(marker.kwargs["features"]) # TODO: If `t` is not already in use, deallocate the previous # target, and ensure the tests have been sorted (and so grouped) @@ -149,12 +150,12 @@ def get_target( for t in pool: # TODO: Implement full feature comparison, etc. and not just # proof-of-concept string set comparison. - if isinstance(t, platform) and t.params == params and t.features >= features: + if isinstance(t, platform) and t.features >= features: pool.remove(t) return t else: # TODO: Reimplement caching. - t = platform(f"pytest-{uuid4()}", params, features) + t = platform(f"pytest-{uuid4()}", request.param, features) return t @@ -197,11 +198,16 @@ def m_target(pool: List[Target], request: SubRequest) -> Iterator[Target]: def pytest_sessionstart() -> None: - """Gather the `targets` from the playbook.""" + """Gather the `targets` from the playbook. + + First collect any user supplied defaults from the `platforms` key + in the playbook, which will default to the given `defaults` + implemented for each platform. Copy the defaults and then + overwrite with the target's specific parameters. + + """ platform_defaults = playbook.data.get("platforms") for target in playbook.data.get("targets"): - # Get a copy of this platform’s defaults (which may not exist) - # and update them with this target’s specific parameters. params = platform_defaults.get(target["platform"]).copy() params.update(target) targets.append(params) @@ -214,14 +220,8 @@ def pytest_generate_tests(metafunc: Metafunc) -> None: This hook is run for each test, so we gather the `targets` in `pytest_sessionstart`. - TODO: Handle `targets` being empty (probably a user-error). Also - consider how this may change if we want to selectively - parameterize tests. - """ - if "target" in metafunc.fixturenames: - metafunc.parametrize("target", targets, True, target_ids) - if "m_target" in metafunc.fixturenames: - metafunc.parametrize("m_target", targets, True, target_ids) - if "c_target" in metafunc.fixturenames: - metafunc.parametrize("c_target", targets, True, target_ids) + assert targets, "This should not be empty!" + for target_fixture in "target", "m_target", "c_target": + if target_fixture in metafunc.fixturenames: + metafunc.parametrize(target_fixture, targets, True, target_ids) From 26a78f99ad95aab78b64109dbe224c51c6e0a5f7 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Mon, 14 Dec 2020 18:42:31 -0800 Subject: [PATCH 129/194] Add `reuse` parameter to `pytest.mark.target` --- pytest-lisa/lisa.py | 11 ++++++++++- pytest-target/target/plugin.py | 18 +++++++++++------- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/pytest-lisa/lisa.py b/pytest-lisa/lisa.py index 2ad50cfa9f..83a943cee9 100644 --- a/pytest-lisa/lisa.py +++ b/pytest-lisa/lisa.py @@ -136,6 +136,11 @@ def pytest_playbook_schema(schema: Dict[Any, Any]) -> None: description="A set of required features, passed to `pytest.mark.target`.", default=[], ): [str], + Optional( + "reuse", + description="Set to false if the target is made unusable.", + default=True, + ): bool, } ) @@ -173,7 +178,11 @@ def pytest_collection_modifyitems( validate_mark(mark) # Forward `features` to `pytest.mark.target` so LISA users # don’t need to use two marks, but keep them decoupled. - item.add_marker(pytest.mark.target(features=mark.kwargs["features"])) + item.add_marker( + pytest.mark.target( + features=mark.kwargs["features"], reuse=mark.kwargs["reuse"] + ) + ) except (SchemaError, AssertionError) as e: pytest.exit(f"Error validating test '{item.name}' metadata: {e}") diff --git a/pytest-target/target/plugin.py b/pytest-target/target/plugin.py index 11556f73ef..ddbb12513f 100644 --- a/pytest-target/target/plugin.py +++ b/pytest-target/target/plugin.py @@ -44,7 +44,7 @@ def pytest_configure(config: Config) -> None: """ config.addinivalue_line( "markers", - ("target(platform, features): " "Specify target requirements."), + ("target(platform, features, reuse): " "Specify target requirements."), ) @@ -142,7 +142,7 @@ def get_target( # Get the required features for this test. marker = request.node.get_closest_marker("target") - features = set(marker.kwargs["features"]) + features = set(marker.kwargs.get("features", [])) # TODO: If `t` is not already in use, deallocate the previous # target, and ensure the tests have been sorted (and so grouped) @@ -159,10 +159,14 @@ def get_target( return t -def cleanup_target(pool: List[Target], t: Target) -> None: +def cleanup_target(pool: List[Target], request: SubRequest, t: Target) -> None: """This is called by fixtures after they're done with a target.""" t.conn.close() - pool.append(t) + marker = request.node.get_closest_marker("target") + if marker.kwargs.get("reuse", True): + pool.append(t) + else: + t.delete() @pytest.fixture @@ -174,7 +178,7 @@ def target(pool: List[Target], request: SubRequest) -> Iterator[Target]: """ t = get_target(pool, request) yield t - cleanup_target(pool, t) + cleanup_target(pool, request, t) @pytest.fixture(scope="class") @@ -182,7 +186,7 @@ def c_target(pool: List[Target], request: SubRequest) -> Iterator[Target]: """This fixture is the same as `target` but shared across a class.""" t = get_target(pool, request) yield t - cleanup_target(pool, t) + cleanup_target(pool, request, t) @pytest.fixture(scope="module") @@ -190,7 +194,7 @@ def m_target(pool: List[Target], request: SubRequest) -> Iterator[Target]: """This fixture is the same as `target` but shared across a module.""" t = get_target(pool, request) yield t - cleanup_target(pool, t) + cleanup_target(pool, request, t) targets: List[Dict[str, Any]] = [] From d4fe7f92df041cc193b9b6149216f90aad4fc55a Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Mon, 14 Dec 2020 18:53:09 -0800 Subject: [PATCH 130/194] Add defaults to `get()` to make type checking happy --- pytest-target/target/plugin.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pytest-target/target/plugin.py b/pytest-target/target/plugin.py index ddbb12513f..06bb4b8c3d 100644 --- a/pytest-target/target/plugin.py +++ b/pytest-target/target/plugin.py @@ -210,9 +210,9 @@ def pytest_sessionstart() -> None: overwrite with the target's specific parameters. """ - platform_defaults = playbook.data.get("platforms") - for target in playbook.data.get("targets"): - params = platform_defaults.get(target["platform"]).copy() + platform_defaults = playbook.data.get("platforms", {}) + for target in playbook.data.get("targets", []): + params = platform_defaults.get(target["platform"], {}).copy() params.update(target) targets.append(params) target_ids.append("Target=" + target["name"]) From 86d37bfc2b7f7b6b10be508c1548900a2eb89ad4 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Tue, 15 Dec 2020 12:47:43 -0800 Subject: [PATCH 131/194] Reference nested schemata with definitions --- pytest-target/target/plugin.py | 39 +++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/pytest-target/target/plugin.py b/pytest-target/target/plugin.py index 06bb4b8c3d..d58767fdf4 100644 --- a/pytest-target/target/plugin.py +++ b/pytest-target/target/plugin.py @@ -77,9 +77,11 @@ def pytest_playbook_schema(schema: Dict[Any, Any]) -> None: platform, default=Schema(cls.defaults()).validate({}), description=platform_description, - ): cls.defaults() + ): Schema(cls.defaults(), name=f"{platform}_defaults", as_reference=True) for platform, cls in platforms.items() } + # TODO: Assert that the set of key names in each `defaults()` is a + # subset of the key names in the corresponding `schema()`. schema[ Optional( "platforms", @@ -88,18 +90,27 @@ def pytest_playbook_schema(schema: Dict[Any, Any]) -> None: ) ] = platform_schemas - # The target schema is `anyOf`/`Or` the platform’s schemas. + # The targets schema is a list of dicts where each dict is ‘any + # of’ (`Or`) the platforms’ schemata, with the addition of a + # required ‘platform’ key matching the class name, and a friendly + # name for the target itself (which should be unique). target_schemas = [ - { - # We’re adding ‘name’ and ‘platform’ keys to each - # platform’s schema. - Literal("name", description="A friendly name for the target."): str, - Literal("platform", description=platform_description): platform, - # Unpack the rest of the schema’s items. - **cls.schema(), - } + Schema( + { + # We’re adding ‘name’ and ‘platform’ keys to each + # platform’s schema. + Literal("name", description="A friendly name for the target."): str, + Literal("platform", description=platform_description): platform, + # Unpack the rest of the schema’s items. + **cls.schema(), + }, + name=f"{platform}_schema", + as_reference=True, + ) for platform, cls in platforms.items() ] + # TODO: Perhaps elevate ‘name’ to the key, with the nested schema + # as the value. target_schema = Or(*target_schemas) default_target = { "name": "Default", @@ -197,8 +208,7 @@ def m_target(pool: List[Target], request: SubRequest) -> Iterator[Target]: cleanup_target(pool, request, t) -targets: List[Dict[str, Any]] = [] -target_ids: List[str] = [] +targets: Dict[str, Dict[str, Any]] = {} def pytest_sessionstart() -> None: @@ -214,8 +224,7 @@ def pytest_sessionstart() -> None: for target in playbook.data.get("targets", []): params = platform_defaults.get(target["platform"], {}).copy() params.update(target) - targets.append(params) - target_ids.append("Target=" + target["name"]) + targets["Target=" + target["name"]] = params def pytest_generate_tests(metafunc: Metafunc) -> None: @@ -228,4 +237,4 @@ def pytest_generate_tests(metafunc: Metafunc) -> None: assert targets, "This should not be empty!" for target_fixture in "target", "m_target", "c_target": if target_fixture in metafunc.fixturenames: - metafunc.parametrize(target_fixture, targets, True, target_ids) + metafunc.parametrize(target_fixture, targets.values(), True, targets.keys()) From a05413610cd8e831569cb162da4f11dccb9a0bf4 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Tue, 15 Dec 2020 13:50:56 -0800 Subject: [PATCH 132/194] Add logging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now that we don’t always capture logs (but use `caplog` instead) we can use the logging module. --- pytest-lisa/lisa.py | 11 +++++++---- pytest-playbook/playbook.py | 8 ++++---- pytest-target/target/plugin.py | 5 +++-- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/pytest-lisa/lisa.py b/pytest-lisa/lisa.py index 83a943cee9..1115b322e4 100644 --- a/pytest-lisa/lisa.py +++ b/pytest-lisa/lisa.py @@ -24,6 +24,7 @@ """ from __future__ import annotations +import logging import re import sys import typing @@ -67,9 +68,8 @@ def pytest_configure(config: Config) -> None: def pytest_playbook_schema(schema: Dict[Any, Any]) -> None: """pytest-playbook hook to update the playbook schema.""" - # TODO: We also want to support a criteria selection on each - # `target` in the playbook, which this top-level criteria being - # the default. + # TODO: We also want to support a ‘targets’ list that confines a + # test selection to only the given targets. criteria_schema = Schema( { # TODO: Should any/all of the strings be regex comparisons? @@ -193,8 +193,10 @@ def pytest_collection_modifyitems( def select(item: Item, times: int, exclude: bool) -> None: """Includes or excludes the item as appropriate.""" if exclude: + logging.debug(f"Excluding '{item}'") excluded.append(item) else: + logging.debug(f"Including '{item}' {times} times") for _ in range(times - included.count(item)): included.append(item) @@ -214,7 +216,8 @@ def select(item: Item, times: int, exclude: bool) -> None: c["area"] and c["area"].casefold() == i["area"].casefold(), c["category"] and c["category"].casefold() == i["category"].casefold(), - c["priority"] and c["priority"] == i["priority"], + # Priority of 0 is falsy so explicitly check against None. + c["priority"] is not None and c["priority"] == i["priority"], c["tags"] and set(c["tags"]) <= set(i["tags"]), ] ): diff --git a/pytest-playbook/playbook.py b/pytest-playbook/playbook.py index ae879b04ab..b09d7d943d 100644 --- a/pytest-playbook/playbook.py +++ b/pytest-playbook/playbook.py @@ -19,10 +19,11 @@ import json import typing +import warnings from pathlib import Path import yaml # TODO: Optionally load yaml. -from schema import Schema, SchemaMissingKeyError # type: ignore +from schema import Schema, SchemaError # type: ignore # See https://pyyaml.org/wiki/PyYAMLDocumentation try: @@ -102,15 +103,14 @@ def pytest_configure(config: Config) -> None: path: Optional[Path] = config.getoption("playbook") if not path or not path.is_file(): - # TODO: Use proper logging? - print("No playbook was specified, using defaults...") + warnings.warn("No playbook was specified, using defaults...") data = Schema(schema).validate({}) else: try: with open(path) as f: data = yaml.load(f, Loader=Loader) data = Schema(schema).validate(data) - except (yaml.YAMLError, SchemaMissingKeyError, OSError) as e: + except (yaml.YAMLError, SchemaError, OSError) as e: pytest.exit( f"Error loading playbook '{path}': {e}", pytest.ExitCode.USAGE_ERROR ) diff --git a/pytest-target/target/plugin.py b/pytest-target/target/plugin.py index d58767fdf4..d8b5dc0391 100644 --- a/pytest-target/target/plugin.py +++ b/pytest-target/target/plugin.py @@ -10,6 +10,7 @@ """ from __future__ import annotations +import logging import typing from uuid import uuid4 @@ -126,9 +127,8 @@ def pool(request: SubRequest) -> Iterator[List[Target]]: targets: List[Target] = [] yield targets for t in targets: - # TODO: Use proper logging? - print(f"Created target: {t.features} / {t.params}") if not request.config.getoption("keep_vms"): + logging.debug(f"Deleting target '{t.name}'") t.delete() @@ -166,6 +166,7 @@ def get_target( return t else: # TODO: Reimplement caching. + logging.debug(f"Creating target '{request.param}' with features '{features}'") t = platform(f"pytest-{uuid4()}", request.param, features) return t From c6bd4230422b0fa0ee84e89f23224e25e50b3703 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Tue, 15 Dec 2020 14:35:17 -0800 Subject: [PATCH 133/194] Check that matching targets have the same parameterization name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We stopped checking `params` equality because it was too much, but that became too little. We do need to check that the name matches, too, or we’d schedule a test on a target that doesn’t match that test’s target parameter (which would be nonsensical). --- pytest-target/target/plugin.py | 16 +++++++++++----- pytest-target/target/target.py | 35 ++++++++++++++++++++++------------ 2 files changed, 34 insertions(+), 17 deletions(-) diff --git a/pytest-target/target/plugin.py b/pytest-target/target/plugin.py index d8b5dc0391..34f402081c 100644 --- a/pytest-target/target/plugin.py +++ b/pytest-target/target/plugin.py @@ -12,7 +12,6 @@ import logging import typing -from uuid import uuid4 import playbook import pytest @@ -148,8 +147,10 @@ def get_target( tests such that they're grouped by features. """ + # Alias because we use it a lot in this function. + params: Dict[Any, Any] = request.param # Get the intended class for this parameterization of `target`. - platform: Type[Target] = platforms[request.param["platform"]] + platform: Type[Target] = platforms[params["platform"]] # Get the required features for this test. marker = request.node.get_closest_marker("target") @@ -161,13 +162,18 @@ def get_target( for t in pool: # TODO: Implement full feature comparison, etc. and not just # proof-of-concept string set comparison. - if isinstance(t, platform) and t.features >= features: + if ( + isinstance(t, platform) + # NOTE: This is not the same as `t.name`! + and t.params["name"] == params["name"] + and t.features >= features + ): pool.remove(t) return t else: # TODO: Reimplement caching. - logging.debug(f"Creating target '{request.param}' with features '{features}'") - t = platform(f"pytest-{uuid4()}", request.param, features) + logging.debug(f"Creating target '{params}' with features '{features}'") + t = platform(None, params, features) return t diff --git a/pytest-target/target/target.py b/pytest-target/target/target.py index d171783acb..2663f0347e 100644 --- a/pytest-target/target/target.py +++ b/pytest-target/target/target.py @@ -5,6 +5,7 @@ import typing from abc import ABC, abstractmethod from io import BytesIO +from uuid import uuid4 import fabric # type: ignore import invoke # type: ignore @@ -13,7 +14,7 @@ from tenacity import retry, stop_after_attempt, wait_exponential # type: ignore if typing.TYPE_CHECKING: - from typing import Any, Mapping, Set + from typing import Any, Mapping, Optional, Set class Target(ABC): @@ -31,9 +32,9 @@ class Target(ABC): """ # Typed instance attributes, not class attributes. + name: str params: Mapping[str, str] features: Set[str] - name: str host: str conn: fabric.Connection @@ -54,24 +55,29 @@ class Target(ABC): def __init__( self, - name: str, + name: Optional[str], params: Mapping[str, str], features: Set[str], ): - """Requires a unique name. + """Creates and deploys an instance of `Target`. + + * `name` is a unique ID for the group of associated resources + * `params` is the input parameters conforming to `schema()` + * `features` is set of arbitrary feature requirements - Name is a unique identifier for the group of associated - resources. Features is a list of requirements such as sriov, - rdma, gpu, xdp. Parameters are used by `deploy()`. + Subclass implementations of `Target` do not need to (and + should not) override `__init__()` as it is setup such that all + platform-specific setup logic can be encoded in `deploy()` + instead, which this calls. """ - self.name = name - # TODO: Do we need to re-validate the parameters here? + if name: + self.name = name + else: + self.name = f"pytest-{uuid4()}" self.params = params self.features = features - # TODO: Review this thoroughly as currently it depends on - # parameters which is side-effecty. self.host = self.deploy() fabric_config = self.config.copy() @@ -117,7 +123,12 @@ def defaults(cls) -> Mapping[Any, Any]: @abstractmethod def deploy(self) -> str: - """Must deploy the target resources and return the hostname.""" + """Must deploy the target resources and return the hostname. + + Subclass implementations can treat this like `__init__` with + `schema()` defining the input `params`. + + """ ... @abstractmethod From 8b9e2898e3062bcbba8d2e5b74c5db9736cb8a69 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Tue, 15 Dec 2020 14:38:59 -0800 Subject: [PATCH 134/194] Suppress warning caused by earlier xdist import --- pytest-lisa/lisa.py | 3 +++ pytest.ini | 1 + 2 files changed, 4 insertions(+) diff --git a/pytest-lisa/lisa.py b/pytest-lisa/lisa.py index 1115b322e4..38ab733d69 100644 --- a/pytest-lisa/lisa.py +++ b/pytest-lisa/lisa.py @@ -33,6 +33,9 @@ import py import pytest from schema import Literal, Optional, Or, Schema, SchemaError # type: ignore + +# TODO: Importing `xdist` here causes a `PytestAssertRewriteWarning` +# to be thrown, which we ignore for now. from xdist.scheduler.loadscope import LoadScopeScheduling # type: ignore if typing.TYPE_CHECKING: diff --git a/pytest.ini b/pytest.ini index 35239b690d..8a835b10cc 100644 --- a/pytest.ini +++ b/pytest.ini @@ -12,3 +12,4 @@ junit_logging = all timeout = 1200 filterwarnings = ignore:the imp module is deprecated in favour of importlib:DeprecationWarning + ignore:Module already imported so cannot be rewritten From 61c7c24da40441cc83c14eb07324c7208672835f Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Tue, 15 Dec 2020 15:08:23 -0800 Subject: [PATCH 135/194] Add basic `targets` fixture which gets N target instances --- pytest-lisa/lisa.py | 14 +++++++++----- pytest-target/target/plugin.py | 35 ++++++++++++++++++++++------------ selftests/test_basic.py | 9 ++++++++- 3 files changed, 40 insertions(+), 18 deletions(-) diff --git a/pytest-lisa/lisa.py b/pytest-lisa/lisa.py index 38ab733d69..e6ec26059a 100644 --- a/pytest-lisa/lisa.py +++ b/pytest-lisa/lisa.py @@ -32,7 +32,7 @@ import playbook import py import pytest -from schema import Literal, Optional, Or, Schema, SchemaError # type: ignore +from schema import And, Literal, Optional, Or, Schema, SchemaError # type: ignore # TODO: Importing `xdist` here causes a `PytestAssertRewriteWarning` # to be thrown, which we ignore for now. @@ -63,7 +63,7 @@ def pytest_configure(config: Config) -> None: config.addinivalue_line( "markers", ( - "lisa(platform, category, area, priority, tags, features): " + "lisa(platform, category, area, priority, tags): " "Annotate a test with metadata." ), ) @@ -144,6 +144,9 @@ def pytest_playbook_schema(schema: Dict[Any, Any]) -> None: description="Set to false if the target is made unusable.", default=True, ): bool, + Optional( + "count", description="Number of targets this test needs.", default=1 + ): And(int, lambda n: 0 < n < 10), } ) @@ -179,11 +182,12 @@ def pytest_collection_modifyitems( if not mark: continue validate_mark(mark) - # Forward `features` to `pytest.mark.target` so LISA users - # don’t need to use two marks, but keep them decoupled. + # Forward args to `pytest.mark.target` so LISA users don’t + # need to use two marks, but keep them decoupled. + kw = mark.kwargs item.add_marker( pytest.mark.target( - features=mark.kwargs["features"], reuse=mark.kwargs["reuse"] + features=kw["features"], reuse=kw["reuse"], count=kw["count"] ) ) except (SchemaError, AssertionError) as e: diff --git a/pytest-target/target/plugin.py b/pytest-target/target/plugin.py index 34f402081c..c48e8392fe 100644 --- a/pytest-target/target/plugin.py +++ b/pytest-target/target/plugin.py @@ -1,8 +1,6 @@ """Provides and parameterizes the `pool` and `target` fixtures. # TODO -* Provide a `targets` fixture for tests which use more than one target - at a time. * Deallocate targets when switching to a new target. * Use richer feature/requirements comparison for targets. * Reimplement caching of targets between runs. @@ -44,7 +42,7 @@ def pytest_configure(config: Config) -> None: """ config.addinivalue_line( "markers", - ("target(platform, features, reuse): " "Specify target requirements."), + ("target(platform, features, reuse, count): " "Specify target requirements."), ) @@ -199,6 +197,19 @@ def target(pool: List[Target], request: SubRequest) -> Iterator[Target]: cleanup_target(pool, request, t) +@pytest.fixture +def targets(pool: List[Target], request: SubRequest) -> Iterator[List[Target]]: + """This fixture obtains N targets for a test.""" + marker = request.node.get_closest_marker("target") + count = marker.kwargs.get("count", 1) + # TODO: Support sharing a `name` across the targets such that + # they’re in the same logical group for any platform. + ts = [get_target(pool, request) for _ in range(count)] + yield ts + for t in ts: + cleanup_target(pool, request, t) + + @pytest.fixture(scope="class") def c_target(pool: List[Target], request: SubRequest) -> Iterator[Target]: """This fixture is the same as `target` but shared across a class.""" @@ -215,7 +226,7 @@ def m_target(pool: List[Target], request: SubRequest) -> Iterator[Target]: cleanup_target(pool, request, t) -targets: Dict[str, Dict[str, Any]] = {} +target_params: Dict[str, Dict[str, Any]] = {} def pytest_sessionstart() -> None: @@ -228,10 +239,10 @@ def pytest_sessionstart() -> None: """ platform_defaults = playbook.data.get("platforms", {}) - for target in playbook.data.get("targets", []): - params = platform_defaults.get(target["platform"], {}).copy() - params.update(target) - targets["Target=" + target["name"]] = params + for t in playbook.data.get("targets", []): + params = platform_defaults.get(t["platform"], {}).copy() + params.update(t) + target_params["Target=" + t["name"]] = params def pytest_generate_tests(metafunc: Metafunc) -> None: @@ -241,7 +252,7 @@ def pytest_generate_tests(metafunc: Metafunc) -> None: `pytest_sessionstart`. """ - assert targets, "This should not be empty!" - for target_fixture in "target", "m_target", "c_target": - if target_fixture in metafunc.fixturenames: - metafunc.parametrize(target_fixture, targets.values(), True, targets.keys()) + assert target_params, "This should not be empty!" + for f in "target", "targets", "m_target", "c_target": + if f in metafunc.fixturenames: + metafunc.parametrize(f, target_params.values(), True, target_params.keys()) diff --git a/selftests/test_basic.py b/selftests/test_basic.py index fa948a8472..ee85747841 100644 --- a/selftests/test_basic.py +++ b/selftests/test_basic.py @@ -5,11 +5,18 @@ if typing.TYPE_CHECKING: from target import SSH + from typing import List from lisa import LISA -@LISA(platform="Local", category="Functional", area="self-test", priority=1) +@LISA(platform="SSH", category="Functional", area="self-test", priority=1) def test_basic(target: SSH) -> None: """Basic test which creates a `Target` connection to 'localhost'.""" target.local("echo Hello World") + + +@LISA(platform="SSH", category="Functional", area="self-test", priority=1, count=3) +def test_basic_multiple(targets: List[SSH]) -> None: + """Basic test which asks for 3 unique targets.""" + assert len({target.name for target in targets}) == 3 From f98c11646dc22cc09c2d5117783a0f8fd3e4537e Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Tue, 15 Dec 2020 15:38:23 -0800 Subject: [PATCH 136/194] Rename `Azure` to `AzureCLI` in preparation for more implementations --- playbooks/demo.yaml | 7 ++++--- pytest-lisa/lisa.py | 4 ++++ pytest-target/target/__init__.py | 4 ++-- pytest-target/target/azure.py | 25 +++++++++++++++++++------ testsuites/test_lis.py | 4 ++-- testsuites/test_smoke_a.py | 30 +++++++++++++++++------------- testsuites/test_smoke_b.py | 4 ++-- testsuites/test_xdp.py | 4 ++-- 8 files changed, 52 insertions(+), 30 deletions(-) diff --git a/playbooks/demo.yaml b/playbooks/demo.yaml index 85d0e25906..e172d74185 100644 --- a/playbooks/demo.yaml +++ b/playbooks/demo.yaml @@ -1,13 +1,14 @@ platforms: - Azure: + AzureCLI: sku: Standard_DS2_v2 + targets: - name: Debian - platform: Azure + platform: AzureCLI image: credativ:Debian:9:9.0.201706190 - name: Ubuntu - platform: Azure + platform: AzureCLI image: UbuntuLTS criteria: diff --git a/pytest-lisa/lisa.py b/pytest-lisa/lisa.py index e6ec26059a..860d0733e0 100644 --- a/pytest-lisa/lisa.py +++ b/pytest-lisa/lisa.py @@ -184,6 +184,10 @@ def pytest_collection_modifyitems( validate_mark(mark) # Forward args to `pytest.mark.target` so LISA users don’t # need to use two marks, but keep them decoupled. + # + # NOTE: The module and class scoped target fixtures won’t + # work with this, because the mark will need to be applied + # at that scope, and this applies on the function. kw = mark.kwargs item.add_marker( pytest.mark.target( diff --git a/pytest-target/target/__init__.py b/pytest-target/target/__init__.py index c8aa2c5b69..4a3aeac7d3 100644 --- a/pytest-target/target/__init__.py +++ b/pytest-target/target/__init__.py @@ -30,11 +30,11 @@ import pytest # Provide common types in the package's namespace. -from target.azure import Azure +from target.azure import AzureCLI from target.target import SSH, Target # NOTE: This is mostly to avoid “imported but not used.” -__all__ = ["Azure", "Target", "SSH"] +__all__ = ["AzureCLI", "Target", "SSH"] # See https://docs.pytest.org/en/stable/writing_plugins.html#assertion-rewriting pytest.register_assert_rewrite("pytest_target.azure", "pytest_target.target") diff --git a/pytest-target/target/azure.py b/pytest-target/target/azure.py index d09809c9d7..ea6b7baf63 100644 --- a/pytest-target/target/azure.py +++ b/pytest-target/target/azure.py @@ -11,23 +11,36 @@ from tenacity import retry, stop_after_attempt, wait_exponential # type: ignore if typing.TYPE_CHECKING: - from typing import Any, Mapping + from typing import Any, Dict -class Azure(Target): - """Implements Azure-specific target methods.""" +class AzureCLI(Target): + """Implements Azure-specific target methods. + + This implementation uses the Azure CLI `az` to automate creating + VMs based on the given parameters. + + """ # Custom instance attribute(s). internal_address: str @classmethod - def schema(cls) -> Mapping[Any, Any]: + def schema(cls) -> Dict[Any, Any]: return { # TODO: Maybe validate as URN or path etc. "image": str, + Optional("sku"): str, + Optional("location"): str, + Optional("networking"): str, + } + + @classmethod + def defaults(cls) -> Dict[Any, Any]: + return { + Optional("image", default="UbuntuLTS"): str, Optional("sku", default="Standard_DS1_v2"): str, Optional("location", default="eastus2"): str, - # TODO: Remove or support this. Optional("networking", default=""): str, } @@ -92,7 +105,7 @@ def deploy(self) -> str: location = self.params["location"] networking = self.params["networking"] - Azure.check_az_cli() + AzureCLI.check_az_cli() logging.info( f"""Deploying VM... diff --git a/testsuites/test_lis.py b/testsuites/test_lis.py index 55f0c2aed8..401bb691f7 100644 --- a/testsuites/test_lis.py +++ b/testsuites/test_lis.py @@ -4,7 +4,7 @@ import typing if typing.TYPE_CHECKING: - from target import Azure + from target import AzureCLI import pytest @@ -13,7 +13,7 @@ @LISA(platform="Azure", category="Functional", priority=0, area="LIS_DEPLOY") @pytest.mark.skip(reason="Scripts missing") -def test_lis_driver_version(target: Azure) -> None: +def test_lis_driver_version(target: AzureCLI) -> None: """Checks that the installed drivers have the correct version.""" # TODO: Include “utils.sh” automatically? Or something... for f in ["utils.sh", "LIS-VERSION-CHECK.sh"]: diff --git a/testsuites/test_smoke_a.py b/testsuites/test_smoke_a.py index 1a2fff0d89..fd827b9e02 100644 --- a/testsuites/test_smoke_a.py +++ b/testsuites/test_smoke_a.py @@ -14,35 +14,39 @@ import typing if typing.TYPE_CHECKING: - from target import Azure + from target import AzureCLI import socket import time +import pytest from invoke.runners import CommandTimedOut, UnexpectedExit # type: ignore from paramiko import SSHException # type: ignore from lisa import LISA -pytestmark = LISA( - platform="Azure", - category="Functional", - area="deploy", - priority=0, -) +pytestmark = [ + LISA( + platform="Azure", + category="Functional", + area="deploy", + priority=0, + ), + pytest.mark.target, +] -def test_first_ping(m_target: Azure) -> None: +def test_first_ping(m_target: AzureCLI) -> None: """"Pinging before reboot...""" assert m_target.ping(), f"Pinging {m_target.host} before reboot failed" -def test_first_ssh(m_target: Azure) -> None: +def test_first_ssh(m_target: AzureCLI) -> None: """SSHing before reboot...""" assert m_target.conn.open(), f"SSH {m_target.host} before reboot failed" -def test_reboot(m_target: Azure) -> None: +def test_reboot(m_target: AzureCLI) -> None: """Rebooting...""" reboot_exit = 0 try: @@ -60,16 +64,16 @@ def test_reboot(m_target: Azure) -> None: time.sleep(10) -def test_second_ping(m_target: Azure) -> None: +def test_second_ping(m_target: AzureCLI) -> None: """Pinging after reboot...""" assert m_target.ping(), f"Pinging {m_target.host} after reboot failed" -def test_second_ssh(m_target: Azure) -> None: +def test_second_ssh(m_target: AzureCLI) -> None: """SSHing after reboot...""" assert m_target.conn.open(), f"SSH {m_target.host} after reboot failed" -def test_boot_diagnostics(m_target: Azure) -> None: +def test_boot_diagnostics(m_target: AzureCLI) -> None: """Retrieving boot diagnostics...""" m_target.get_boot_diagnostics() diff --git a/testsuites/test_smoke_b.py b/testsuites/test_smoke_b.py index c07c408791..404b76e7a1 100644 --- a/testsuites/test_smoke_b.py +++ b/testsuites/test_smoke_b.py @@ -3,7 +3,7 @@ import typing if typing.TYPE_CHECKING: - from target import Azure + from target import AzureCLI from _pytest.logging import LogCaptureFixture import logging @@ -22,7 +22,7 @@ area="deploy", priority=0, ) -def test_smoke(target: Azure, caplog: LogCaptureFixture) -> None: +def test_smoke(target: AzureCLI, caplog: LogCaptureFixture) -> None: """Check that an Azure Linux VM can be deployed and is responsive. This example uses exactly one function for the entire test, which diff --git a/testsuites/test_xdp.py b/testsuites/test_xdp.py index 9386de5bf7..c110bf0be9 100644 --- a/testsuites/test_xdp.py +++ b/testsuites/test_xdp.py @@ -4,7 +4,7 @@ import typing if typing.TYPE_CHECKING: - from target import Azure + from target import AzureCLI import pytest @@ -24,7 +24,7 @@ # vm_image="Canonical:0001-com-ubuntu-server-focal:20_04-lts:latest", # vm_size="Standard_DS4_v2", @pytest.mark.skip(reason="Not Finished") -def test_verify_xdp_compliance(target: Azure) -> None: +def test_verify_xdp_compliance(target: AzureCLI) -> None: for f in [ "utils.sh", "XDPDumpSetup.sh", From 89ef35b269a8e55593d11a465a0af9aecf432916 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Tue, 15 Dec 2020 15:51:11 -0800 Subject: [PATCH 137/194] Hide some `az` output --- pytest-target/target/azure.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/pytest-target/target/azure.py b/pytest-target/target/azure.py index ea6b7baf63..472282fd32 100644 --- a/pytest-target/target/azure.py +++ b/pytest-target/target/azure.py @@ -5,6 +5,7 @@ import logging import typing +import invoke # type: ignore from invoke.runners import Result # type: ignore from schema import Optional # type: ignore from target.target import Target @@ -44,6 +45,18 @@ def defaults(cls) -> Dict[Any, Any]: Optional("networking", default=""): str, } + @classmethod + def _local(cls, *args: Any, **kwargs: Any) -> Result: + """A quiet version of `local()`. + + TODO: Consider adding this to the superclass. + + """ + config = Target.config.copy() + config["run"]["hide"] = True + context = invoke.Context(config=invoke.Config(overrides=config)) + return context.run(*args, **kwargs) + # A class attribute because it’s defined. az_ok = False @@ -53,10 +66,10 @@ def check_az_cli(cls) -> None: if cls.az_ok: # Shortcut if we already checked. return # E.g. on Ubuntu: `curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash` - assert cls.local("az --version", warn=True), "Please install the `az` CLI!" + assert cls._local("az --version", warn=True), "Please install the `az` CLI!" # TODO: Login with service principal (az login) and set # default subscription (az account set -s) using secrets. - account: Result = cls.local("az account show") + account: Result = cls._local("az account show") assert account.ok, "Please `az login`!" sub = json.loads(account.stdout) assert sub["isDefault"], "Please `az account set -s `!" @@ -69,12 +82,12 @@ def create_boot_storage(self, location: str) -> str: """Create a separate resource group and storage account for boot diagnostics.""" account = "pytestbootdiag" # This command always exits with 0 but returns a string. - if self.local("az group exists -n pytest-lisa").stdout.strip() == "false": - self.local(f"az group create -n pytest-lisa --location {location}") - if not self.local( + if self._local("az group exists -n pytest-lisa").stdout.strip() == "false": + self._local(f"az group create -n pytest-lisa --location {location}") + if not self._local( f"az storage account show -g pytest-lisa -n {account}", warn=True ): - self.local(f"az storage account create -g pytest-lisa -n {account}") + self._local(f"az storage account create -g pytest-lisa -n {account}") return account def allow_ping(self) -> None: @@ -117,7 +130,7 @@ def deploy(self) -> str: boot_storage = self.create_boot_storage(location) - self.local(f"az group create -n {self.name}-rg --location {location}") + self._local(f"az group create -n {self.name}-rg --location {location}") # TODO: Accept EULA terms when necessary. Like: # # local.run(f"az vm image terms accept --urn {vm_image}") From b45e65876488a959a2980e31463b8ec58a12e1e7 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Wed, 16 Dec 2020 00:44:56 -0800 Subject: [PATCH 138/194] Clean up of `pytest_playbook_schema` by moving code to `Target` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This doesn’t change the schema, it just organizes code. --- pytest-target/target/plugin.py | 62 +++++++++------------------ pytest-target/target/target.py | 76 +++++++++++++++++++++++++++++++--- 2 files changed, 90 insertions(+), 48 deletions(-) diff --git a/pytest-target/target/plugin.py b/pytest-target/target/plugin.py index c48e8392fe..4fa4b864a3 100644 --- a/pytest-target/target/plugin.py +++ b/pytest-target/target/plugin.py @@ -15,7 +15,7 @@ import pytest # See https://pypi.org/project/schema/ -from schema import Literal, Optional, Or, Schema # type: ignore +from schema import Optional, Or, Schema # type: ignore from target.target import SSH, Target if typing.TYPE_CHECKING: @@ -65,57 +65,33 @@ def pytest_playbook_schema(schema: Dict[Any, Any]) -> None: global platforms platforms = {cls.__name__: cls for cls in Target.__subclasses__()} # type: ignore - # The platform schema is an optional mapping of the platform name - # to defaults for its provided schema. Setting the defaults here - # is a bit particular, as we need to have the schema library parse - # the given dict as a schema, and fill in the nested defaults. - platform_description = "The class name of the platform implementation." - platform_schemas = { - Optional( - platform, - default=Schema(cls.defaults()).validate({}), - description=platform_description, - ): Schema(cls.defaults(), name=f"{platform}_defaults", as_reference=True) - for platform, cls in platforms.items() - } - # TODO: Assert that the set of key names in each `defaults()` is a - # subset of the key names in the corresponding `schema()`. + # The platforms schema is a set of optional mappings of each + # platform’s name to defaults for its provided schema. + platforms_schema = dict(cls.get_defaults() for cls in platforms.values()) + default_platforms = Schema(platforms_schema).validate({}) schema[ Optional( "platforms", - default=Schema(platform_schemas).validate({}), - description="Default values for each platform.", + default=default_platforms, + description="A set of objects with default values for each platform.", ) - ] = platform_schemas - - # The targets schema is a list of dicts where each dict is ‘any - # of’ (`Or`) the platforms’ schemata, with the addition of a - # required ‘platform’ key matching the class name, and a friendly - # name for the target itself (which should be unique). - target_schemas = [ - Schema( - { - # We’re adding ‘name’ and ‘platform’ keys to each - # platform’s schema. - Literal("name", description="A friendly name for the target."): str, - Literal("platform", description=platform_description): platform, - # Unpack the rest of the schema’s items. - **cls.schema(), - }, - name=f"{platform}_schema", - as_reference=True, - ) - for platform, cls in platforms.items() - ] - # TODO: Perhaps elevate ‘name’ to the key, with the nested schema - # as the value. - target_schema = Or(*target_schemas) + ] = platforms_schema + + # The targets schema is a list of ‘any of’ the platforms’ + # reference schemata. + targets_schema = [Or(*(cls.get_schema() for cls in platforms.values()))] default_target = { "name": "Default", "platform": "SSH", **Schema(SSH.schema()).validate({}), # Fill in the defaults } - schema[Optional("targets", default=[default_target])] = [target_schema] + schema[ + Optional( + "targets", + default=[default_target], + description="A list of targets with which to parameterize the tests.", + ) + ] = targets_schema @pytest.fixture(scope="session") diff --git a/pytest-target/target/target.py b/pytest-target/target/target.py index 2663f0347e..05a4b85fec 100644 --- a/pytest-target/target/target.py +++ b/pytest-target/target/target.py @@ -11,10 +11,11 @@ import invoke # type: ignore import schema # type: ignore from invoke.runners import Result # type: ignore +from schema import Literal, Schema # type: ignore from tenacity import retry, stop_after_attempt, wait_exponential # type: ignore if typing.TYPE_CHECKING: - from typing import Any, Mapping, Optional, Set + from typing import Any, Mapping, Optional, Set, Tuple class Target(ABC): @@ -113,9 +114,9 @@ def schema(cls) -> Mapping[Any, Any]: def defaults(cls) -> Mapping[Any, Any]: """Can return a mapping for default parameters. - If specified, it should contain only `schema.Optional` - elements, where the names and types match those in `schema`, - but with a set default value, and those in `schema` should not + If specified, it must contain only `schema.Optional` elements, + where the names and types match those in `schema()`, but with + a set default value, and those in `schema()` should not contain default values. This is used a base for each target. """ @@ -136,7 +137,72 @@ def delete(self) -> None: """Must delete the target resources.""" ... - # A class attribute because it’s defined. + platform_description = "The class name of the platform implementation." + + @classmethod + def get_defaults(cls) -> Tuple[schema.Optional, Schema]: + """Returns a tuple of "platform key" / "defaults value" pairs. + + This is an internal detail, used when generating the + playbook's schema. Subclasses should not override this. + + The key is an optional literal, the name of the subclass for + the platform, with a default value of the validated + `defaults()` schema when given no input (hence they must all + be optional). The value is reference schema definition + generated from the `defaults()` dict. + + When generating the playbook's schema all the platforms' + tuples are mapped into a single dict. + + TODO: Assert that the set of key names in each `defaults()` is + a subset of the key names in the corresponding `schema()`. + + """ + return ( + schema.Optional( + cls.__name__, + default=Schema(cls.defaults()).validate({}), + description=cls.platform_description, + ), + Schema(cls.defaults(), name=f"{cls.__name__}_Defaults", as_reference=True), + ) + + @classmethod + def get_schema(cls) -> Schema: + """Returns a reference schema definition for the class parameters. + + This is an internal detail, used when generating the + playbook's schema. Subclasses should not override this. + + We generate the whole definition by combining the values of + `cls.schema()` (which is defined by each platform's + implementation) with two required keys: + + * name: A friendly name for the target. + * platform: The name of the subclass for the platform. + + When generating the playbook's schema all the platforms' + schemata are mapped into an 'any of' schema. + + TODO: Perhaps elevate ‘name’ to the key, with the nested + schema as the value. + + """ + return Schema( + { + # We’re adding ‘name’ and ‘platform’ keys. + Literal("name", description="A friendly name for the target."): str, + Literal("platform", description=cls.platform_description): cls.__name__, + # Unpack the rest of the schema’s items. + **cls.schema(), + }, + name=f"{cls.__name__}_Schema", + as_reference=True, + ) + + # Platform-agnostic functionality should be added here: + local_context = invoke.Context(config=invoke.Config(overrides=config)) @classmethod From 4e0faa0dd438a36749fa429ef505be02d814a6e3 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Wed, 16 Dec 2020 14:50:27 -0800 Subject: [PATCH 139/194] Make `name` for `Target.__init__` non-optional --- pytest-target/target/plugin.py | 3 ++- pytest-target/target/target.py | 27 ++++++++++----------------- 2 files changed, 12 insertions(+), 18 deletions(-) diff --git a/pytest-target/target/plugin.py b/pytest-target/target/plugin.py index 4fa4b864a3..b73315de5e 100644 --- a/pytest-target/target/plugin.py +++ b/pytest-target/target/plugin.py @@ -10,6 +10,7 @@ import logging import typing +from uuid import uuid4 import playbook import pytest @@ -147,7 +148,7 @@ def get_target( else: # TODO: Reimplement caching. logging.debug(f"Creating target '{params}' with features '{features}'") - t = platform(None, params, features) + t = platform(f"pytest-{uuid4()}", params, features) return t diff --git a/pytest-target/target/target.py b/pytest-target/target/target.py index 05a4b85fec..ff637b2762 100644 --- a/pytest-target/target/target.py +++ b/pytest-target/target/target.py @@ -5,17 +5,15 @@ import typing from abc import ABC, abstractmethod from io import BytesIO -from uuid import uuid4 import fabric # type: ignore import invoke # type: ignore -import schema # type: ignore from invoke.runners import Result # type: ignore -from schema import Literal, Schema # type: ignore +from schema import Literal, Optional, Schema # type: ignore from tenacity import retry, stop_after_attempt, wait_exponential # type: ignore if typing.TYPE_CHECKING: - from typing import Any, Mapping, Optional, Set, Tuple + from typing import Any, Dict, Mapping, Set, Tuple class Target(ABC): @@ -56,7 +54,7 @@ class Target(ABC): def __init__( self, - name: Optional[str], + name: str, params: Mapping[str, str], features: Set[str], ): @@ -72,10 +70,7 @@ def __init__( instead, which this calls. """ - if name: - self.name = name - else: - self.name = f"pytest-{uuid4()}" + self.name = name self.params = params self.features = features @@ -140,7 +135,7 @@ def delete(self) -> None: platform_description = "The class name of the platform implementation." @classmethod - def get_defaults(cls) -> Tuple[schema.Optional, Schema]: + def get_defaults(cls) -> Tuple[Optional, Schema]: """Returns a tuple of "platform key" / "defaults value" pairs. This is an internal detail, used when generating the @@ -160,7 +155,7 @@ def get_defaults(cls) -> Tuple[schema.Optional, Schema]: """ return ( - schema.Optional( + Optional( cls.__name__, default=Schema(cls.defaults()).validate({}), description=cls.platform_description, @@ -233,17 +228,15 @@ class SSH(Target): """ @classmethod - def schema(cls) -> Mapping[Any, Any]: + def schema(cls) -> Dict[Any, Any]: return { - schema.Optional( - "host", description="The address of the destination target." - ): str + Optional("host", description="The address of the destination target."): str } @classmethod - def defaults(cls) -> Mapping[Any, Any]: + def defaults(cls) -> Dict[Any, Any]: return { - schema.Optional( + Optional( "host", default="localhost", description="The default value for host." ): str } From e724522c350c5e5ef9b9bc696c03ed74f94bdafa Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Wed, 16 Dec 2020 15:13:38 -0800 Subject: [PATCH 140/194] Print schema to a file, not just stdout --- pytest-playbook/playbook.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/pytest-playbook/playbook.py b/pytest-playbook/playbook.py index b09d7d943d..7cc571a8a3 100644 --- a/pytest-playbook/playbook.py +++ b/pytest-playbook/playbook.py @@ -78,7 +78,8 @@ def pytest_addoption(parser: Parser, pluginmanager: PytestPluginManager) -> None group.addoption("--playbook", type=Path, help="Path to playbook.") group.addoption( "--print-schema", - help="Print the JSON schema of the playbook with the given ID.", + type=Path, + help="Print the JSON schema of the playbook to the given path.", ) @@ -91,25 +92,27 @@ def pytest_configure(config: Config) -> None: loaded and defined their `pytest_playbook_schema` hooks. """ - schema: Dict[Any, Any] = dict() - config.hook.pytest_playbook_schema(schema=schema, config=config) + schema_dict: Dict[Any, Any] = dict() + config.hook.pytest_playbook_schema(schema=schema_dict, config=config) + schema = Schema(schema_dict) - json_schema = config.getoption("print_schema") + json_schema: Optional[Path] = config.getoption("print_schema") if json_schema: - print(json.dumps(Schema(schema).json_schema(json_schema), indent=2)) - pytest.exit("Printed schema!", pytest.ExitCode.OK) + with json_schema.open("w") as f: + json.dump(schema.json_schema(json_schema.name), f, indent=2) + pytest.exit(f"Printed schema to {json_schema}!", pytest.ExitCode.OK) global data path: Optional[Path] = config.getoption("playbook") if not path or not path.is_file(): warnings.warn("No playbook was specified, using defaults...") - data = Schema(schema).validate({}) + data = schema.validate({}) else: try: - with open(path) as f: + with path.open() as f: data = yaml.load(f, Loader=Loader) - data = Schema(schema).validate(data) + data = schema.validate(data) except (yaml.YAMLError, SchemaError, OSError) as e: pytest.exit( f"Error loading playbook '{path}': {e}", pytest.ExitCode.USAGE_ERROR From ba5a8a95e6623bba7e7a11949585f4c1dce67c3a Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Wed, 16 Dec 2020 16:10:50 -0800 Subject: [PATCH 141/194] Document how to lint playbooks against JSON Schema --- README.md | 68 ++++++++++++++- playbooks/schema.json | 195 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 260 insertions(+), 3 deletions(-) create mode 100644 playbooks/schema.json diff --git a/README.md b/README.md index b6556bcd42..54718237d1 100644 --- a/README.md +++ b/README.md @@ -98,6 +98,35 @@ poetry env list --full-path Use it to configure your editor. +The registered playbook schema can be generated using `--print-schema`: + +```bash +lisa --print-schema=playbooks/schema.json +``` + +This will create a file `playbooks/schema.json` with the [JSON Schema][] for all +the registered schemata (including those added in any local plugins). Note that +this file is committed to the repo for the public schema, but you can generate +your own (or update it) with the above command. + +Using the LSP [yaml-language-server][] you can setup almost any editor to lint +the playbook files against the schema. Either add as a comment to the top of the +playbook file: + +```yaml +# yaml-language-server: $schema=file:///path/to/playbooks/schema.json +``` + +Or set `yaml.schemas` as appropriate for your editor. Also ensure that +`yaml-language-server` is installed, which you can do via: + +```bash +npm install -g yaml-language-server +``` + +[JSON Schema]: https://json-schema.org/ +[yaml-language-server]: https://github.com/redhat-developer/yaml-language-server + ### Editor Setup #### Visual Studio Code @@ -130,23 +159,56 @@ Make sure below settings are in root level of `.vscode/settings.json` #### Emacs -Use the [pyvenv](https://github.com/jorgenschaefer/pyvenv) package: +I recommend using the [pyvenv][] package to have Emacs automatically use the +correct Python venv (setup by Poetry), and the [eglot][] package to provide LSP +support. The below expects you already have an `init.el` with [use-package][]. + +[pyvenv]: https://github.com/jorgenschaefer/pyvenv +[eglot]: https://github.com/joaotavora/eglot +[use-package]: https://github.com/jwiegley/use-package ```emacs-lisp (use-package pyvenv :ensure t :hook (python-mode . pyvenv-tracking-mode)) + +(use-package eglot + :hook + (python-mode . eglot-ensure) + (yaml-mode . eglot-ensure) + :custom + (eglot-auto-display-help-buffer t) + (eglot-autoshutdown t) + (eglot-confirm-server-initiated-edits nil) + :config + (add-to-list 'eglot-server-programs '(yaml-mode . ("yaml-language-server" "--stdio"))) + (defun eglot-format-buffer-on-save () + (if (and (project-current) (eglot-managed-p)) + (add-hook 'before-save-hook #'eglot-format-buffer nil 'local) + (remove-hook 'before-save-hook #'eglot-format-buffer 'local))) + (add-hook 'eglot-managed-mode-hook #'eglot-format-buffer-on-save)) ``` Then run `M-x add-dir-local-variable RET python-mode RET pyvenv-activate RET ` where the value is the path given by the command above. -This will create a `.dir-locals.el` file which looks like this: +This will create a `.dir-locals.el` with this variable set for Python. + +Refer to this `.dir-locals.el` for a complete setup: ```emacs-lisp ;;; Directory Local Variables ;;; For more information see (info "(emacs) Directory Variables") -((nil . ((pyvenv-activate . "~/.cache/pypoetry/virtualenvs/")))) +((python-mode + . ((eglot-workspace-configuration ; an LSP implementation + ;; Use `flake8’ instead of the default, and disable noisy plugins. + . ((:pyls . (:configurationSources ["flake8"] :plugins (:pycodestyle (:enabled nil) :mccabe (:enabled nil)))))))) + (yaml-mode + . ((eglot-workspace-configuration + ;; Set the `playbooks/schema.json’ as the schema for playbooks. + . ((:yaml . (:schemas (:playbooks/schema.json "playbooks/*"))))))) + ;; Set `pyvenv’ to use the given venv for the whole project. + (nil . ((pyvenv-activate . "~/.cache/pypoetry/virtualenvs/")))) ``` ### Contributor License Agreement diff --git a/playbooks/schema.json b/playbooks/schema.json new file mode 100644 index 0000000000..630b374c79 --- /dev/null +++ b/playbooks/schema.json @@ -0,0 +1,195 @@ +{ + "type": "object", + "properties": { + "criteria": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "default": null + }, + "module": { + "type": "string", + "default": null + }, + "area": { + "type": "string", + "default": null + }, + "category": { + "type": "string", + "default": null + }, + "priority": { + "type": "integer", + "default": null + }, + "tags": { + "type": "array", + "items": { + "type": "string" + }, + "default": "" + }, + "times": { + "type": "integer", + "default": 1 + }, + "exclude": { + "type": "boolean", + "default": false + } + }, + "required": [], + "additionalProperties": false + }, + "default": "" + }, + "platforms": { + "type": "object", + "properties": { + "SSH": { + "$ref": "#/definitions/SSH_Defaults", + "default": { + "host": "localhost" + } + }, + "AzureCLI": { + "$ref": "#/definitions/AzureCLI_Defaults", + "default": { + "location": "eastus2", + "sku": "Standard_DS1_v2", + "networking": "", + "image": "UbuntuLTS" + } + } + }, + "required": [], + "additionalProperties": false, + "default": { + "AzureCLI": { + "location": "eastus2", + "sku": "Standard_DS1_v2", + "networking": "", + "image": "UbuntuLTS" + }, + "SSH": { + "host": "localhost" + } + } + }, + "targets": { + "type": "array", + "items": { + "anyOf": [ + { + "$ref": "#/definitions/SSH_Schema" + }, + { + "$ref": "#/definitions/AzureCLI_Schema" + } + ] + }, + "default": [ + { + "name": "Default", + "platform": "SSH" + } + ] + } + }, + "required": [], + "additionalProperties": false, + "$id": "schema.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "definitions": { + "SSH_Defaults": { + "type": "object", + "properties": { + "host": { + "type": "string", + "default": "localhost" + } + }, + "required": [], + "additionalProperties": false + }, + "AzureCLI_Defaults": { + "type": "object", + "properties": { + "image": { + "type": "string", + "default": "UbuntuLTS" + }, + "sku": { + "type": "string", + "default": "Standard_DS1_v2" + }, + "location": { + "type": "string", + "default": "eastus2" + }, + "networking": { + "type": "string", + "default": "" + } + }, + "required": [], + "additionalProperties": false + }, + "SSH_Schema": { + "type": "object", + "properties": { + "name": { + "description": "A friendly name for the target.", + "type": "string" + }, + "platform": { + "description": "The class name of the platform implementation.", + "const": "SSH" + }, + "host": { + "type": "string" + } + }, + "required": [ + "name", + "platform" + ], + "additionalProperties": false + }, + "AzureCLI_Schema": { + "type": "object", + "properties": { + "name": { + "description": "A friendly name for the target.", + "type": "string" + }, + "platform": { + "description": "The class name of the platform implementation.", + "const": "AzureCLI" + }, + "image": { + "type": "string" + }, + "sku": { + "type": "string" + }, + "location": { + "type": "string" + }, + "networking": { + "type": "string" + } + }, + "required": [ + "name", + "platform", + "image" + ], + "additionalProperties": false + } + } +} \ No newline at end of file From 4674d5ae8d84181ee989630357a0fa3a881c2d5f Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Thu, 17 Dec 2020 15:14:58 -0800 Subject: [PATCH 142/194] Add `self.data` as field to `Target` (used for cache) --- pytest-target/target/azure.py | 28 +++++++++++++++++----------- pytest-target/target/target.py | 12 ++++++++++-- 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/pytest-target/target/azure.py b/pytest-target/target/azure.py index 472282fd32..c065c74b8e 100644 --- a/pytest-target/target/azure.py +++ b/pytest-target/target/azure.py @@ -111,26 +111,34 @@ def allow_ping(self) -> None: except Exception as e: logging.warning(f"Failed to create ICMP allow rules in NSG due to '{e}'") + def parse_data(self) -> str: + self.internal_address = self.data["privateIpAddress"] + return typing.cast(str, self.data["publicIpAddress"]) + def deploy(self) -> str: """Given deployment info, deploy a new VM.""" + if self.data: # Shortcut if refreshing from cache. + return self.parse_data() + + AzureCLI.check_az_cli() + image = self.params["image"] sku = self.params["sku"] location = self.params["location"] networking = self.params["networking"] - AzureCLI.check_az_cli() - logging.info( - f"""Deploying VM... - Resource Group: '{self.name}-rg' - Region: '{location}' - Image: '{image}' - SKU: '{sku}'""" + "Deploying VM...\n" + f" Group: '{self.name}-rg'\n" + f" Region: '{location}'\n" + f" Image: '{image}'\n" + f" SKU: '{sku}'" ) boot_storage = self.create_boot_storage(location) self._local(f"az group create -n {self.name}-rg --location {location}") + # TODO: Accept EULA terms when necessary. Like: # # local.run(f"az vm image terms accept --urn {vm_image}") @@ -152,11 +160,9 @@ def deploy(self) -> str: vm_command.append("--accelerated-networking true") self.data = json.loads(self.local(" ".join(vm_command)).stdout) - hostname: str = self.data["publicIpAddress"] - self.internal_address = self.data["privateIpAddress"] self.allow_ping() # TODO: Enable auto-shutdown 4 hours from deployment. - return hostname + return self.parse_data() def delete(self) -> None: """Delete the entire allocated resource group. @@ -165,7 +171,7 @@ def delete(self) -> None: the entire resource group. """ - logging.info(f"Deleting resource group '{self.name}-rg'") + logging.debug(f"Deleting resource group '{self.name}-rg'") self.local(f"az group delete -n {self.name}-rg --yes --no-wait") @retry(reraise=True, wait=wait_exponential(), stop=stop_after_attempt(3)) diff --git a/pytest-target/target/target.py b/pytest-target/target/target.py index ff637b2762..a38d0f9504 100644 --- a/pytest-target/target/target.py +++ b/pytest-target/target/target.py @@ -34,6 +34,7 @@ class Target(ABC): name: str params: Mapping[str, str] features: Set[str] + data: Mapping[Any, Any] host: str conn: fabric.Connection @@ -55,8 +56,9 @@ class Target(ABC): def __init__( self, name: str, - params: Mapping[str, str], + params: Mapping[Any, Any], features: Set[str], + data: Mapping[Any, Any], ): """Creates and deploys an instance of `Target`. @@ -71,8 +73,9 @@ def __init__( """ self.name = name - self.params = params + self.params = self.get_schema().validate(params) self.features = features + self.data = data self.host = self.deploy() @@ -124,6 +127,11 @@ def deploy(self) -> str: Subclass implementations can treat this like `__init__` with `schema()` defining the input `params`. + Data which should be cached must be saved to `self.data`. + + If `self.data` is populated then implementations should assume + they're refreshing a cached target. + """ ... From cff10692524bf8517bf0432cff2921f93f95c1dd Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Thu, 17 Dec 2020 16:49:46 -0800 Subject: [PATCH 143/194] Implement caching of targets between runs --- pytest-target/target/plugin.py | 49 +++++++++++++++++++++++++++------- 1 file changed, 39 insertions(+), 10 deletions(-) diff --git a/pytest-target/target/plugin.py b/pytest-target/target/plugin.py index b73315de5e..981ac47184 100644 --- a/pytest-target/target/plugin.py +++ b/pytest-target/target/plugin.py @@ -1,9 +1,8 @@ """Provides and parameterizes the `pool` and `target` fixtures. -# TODO +# TODO: * Deallocate targets when switching to a new target. * Use richer feature/requirements comparison for targets. -* Reimplement caching of targets between runs. """ from __future__ import annotations @@ -22,6 +21,7 @@ if typing.TYPE_CHECKING: from typing import Any, Dict, Iterator, List, Type + from _pytest.cacheprovider import Cache from _pytest.config import Config from _pytest.config.argparsing import Parser from _pytest.fixtures import SubRequest @@ -31,7 +31,12 @@ def pytest_addoption(parser: Parser) -> None: """Pytest hook to add our CLI options.""" group = parser.getgroup("target") - group.addoption("--keep-vms", action="store_true", help="Keeps deployed VMs.") + group.addoption( + "--keep-targets", action="store_true", help="Keeps targets between runs." + ) + group.addoption( + "--delete-targets", action="store_true", help="Deletes all cached targets." + ) def pytest_configure(config: Config) -> None: @@ -100,10 +105,11 @@ def pool(request: SubRequest) -> Iterator[List[Target]]: """This fixture tracks all deployed target resources.""" targets: List[Target] = [] yield targets + # TODO: Catch interrupts and always delete targets: + # `UnexpectedExit`, `KeyboardInterrupt`, `SystemExit`. for t in targets: - if not request.config.getoption("keep_vms"): - logging.debug(f"Deleting target '{t.name}'") - t.delete() + if not request.config.getoption("keep_targets"): + delete_target(t, request.config.cache) def get_target( @@ -146,9 +152,23 @@ def get_target( pool.remove(t) return t else: - # TODO: Reimplement caching. - logging.debug(f"Creating target '{params}' with features '{features}'") - t = platform(f"pytest-{uuid4()}", params, features) + logging.info(f"Instantiating target: '{params}'...") + assert request.config.cache is not None + key = "target/" + params["name"] + cache = request.config.cache.get(key, {}) + if cache: + logging.debug("Cache hit...") + name = cache["name"] + assert cache["params"] == params, "Params changed!" + # TODO: Check `features`, but a set isn’t serializable. + data = cache["data"] + else: + logging.debug("Cache miss...") + name = f"pytest-{uuid4()}" + data = None + t = platform(name, params, features, data) + cache = {"name": name, "params": params, "data": t.data} + request.config.cache.set(key, cache) return t @@ -159,7 +179,16 @@ def cleanup_target(pool: List[Target], request: SubRequest, t: Target) -> None: if marker.kwargs.get("reuse", True): pool.append(t) else: - t.delete() + delete_target(t, request.config.cache) + + +def delete_target(t: Target, cache: Optional[Cache]) -> None: + """Deletes a `Target` and removes it from the cache.""" + name: str = t.params["name"] + logging.info(f"Deleting target '{name}'...") + t.delete() + assert cache is not None + cache.set("target/" + name, None) @pytest.fixture From 731def176e06d7ca218b5c04804bfb5181ebda31 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Thu, 17 Dec 2020 17:01:52 -0800 Subject: [PATCH 144/194] Update readme --- CONTRIBUTING.md | 225 ++++++++++++++++++++++++++++++++++++------------ README.md | 134 ++-------------------------- 2 files changed, 179 insertions(+), 180 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f74461187d..038c3945c9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -3,38 +3,62 @@ This document describes the existing developer tooling we have in place (and what to expect of it), as well as our design and development philosophy. -## Naming Conventions +See [learnxinyminutes.com](https://learnxinyminutes.com/docs/python/) for a +Python crash course. -Naming conventions are not automatically enforced, so please read the [naming -conventions](https://www.python.org/dev/peps/pep-0008/#naming-conventions) -section of PEP 8, which describes what each of the different styles means. A -short summary of the most important parts: +## Pytest -* Modules (and hence files) should have short, all-lowercase names. -* Class (and exception) names should normally use the `CapWords` convention - (also known as `CamelCase`). -* Function and variable names should be lowercase, with words separated by - underscores as necessary to improve readability (also known as `snake_case`). -* To avoid collisions with the standard library, an underscore can be appended, - such as `id_`. -* Always use `self` for the first argument to instance methods. -* Always use `cls` for the first argument to class methods. -* Use one leading underscore only for non-public methods and instance variables, - such as `_data`. Do not activate name mangling with `__` unless necessary. -* If there is a pair of `get_x` and `set_x` methods, they should instead be a - proper property, which is easy to do with the built-in `@property` decorator. -* Constants should be `CAPITALIZED_SNAKE_CASE`. -* When importing a function, try to avoid renaming it with `import as` because - it introduces cognitive overhead to track yet another name. -* When deriving another module’s class (such as `unittest.TestCase`), reuse the - class name to avoid confusion, such as `LisaTestCase`, instead of introducing - a different connotation like `TestSuite`. +Underneath the hood, `lisa` is just [Pytest][]! Refer to its +[documentation](https://docs.pytest.org/en/stable/contents.html). -When in doubt, adhere to existing conventions, or check the style guide. +[Pytest]: https://docs.pytest.org/en/stable/ + +Some useful CLI options are: + +* `--help` and `--verbose`, of course +* `--log-cli-level=DEBUG` to enable live output of all `DEBUG` and above logs +* `--collect-only` to show which tests would be selected (but don’t run them) +* `--setup-only` to show in which order fixtures and tests would be setup (but + don’t run them) +* `--flake8 --mypy -m "flake8 or mypy"` to run the semantic analysis tools + +## YAML Schema + +The registered playbook schema can be generated using `--print-schema`: + +```bash +lisa --print-schema=playbooks/schema.json +``` + +This will create a file `playbooks/schema.json` with the [JSON +Schema](https://json-schema.org/) for all the registered schemata (including +those added in any local plugins). Note that this file is committed to the repo +for the public schema, but you can generate your own (or update it) with the +above command. + +Using the LSP [yaml-language-server][] you can setup almost any editor to lint +the playbook files against the schema. Either add as a comment to the top of the +playbook file: + +```yaml +# yaml-language-server: $schema=file:///path/to/playbooks/schema.json +``` + +Or set `yaml.schemas` as appropriate for your editor. Also ensure that +`yaml-language-server` is installed, which you can do via: -## Automated Tooling +```bash +npm install -g yaml-language-server +``` -If you have ran pytest-lisa already, then you have installed and used the `poetry` +See [learnxinyminutes.com](https://learnxinyminutes.com/docs/yaml/) for a crash +course in YAML. + +[yaml-language-server]: https://github.com/redhat-developer/yaml-language-server + +## Poetry + +If you have ran `lisa` already, then you have installed and used the `poetry` tool. [Poetry][] is a [PEP 518][] compliant and cross-platform build system which handles our Python dependencies and environment. @@ -77,14 +101,20 @@ From the documentation: On Linux, your initial run of `poetry install` will cause Poetry to automatically setup a new [virtualenv][] using [pyenv][]. If you are developing -on Windows, you will want to setup your own, perhaps using [Conda][]. +on Windows, you may need to setup your own, perhaps using [Conda][]. + +The path to the virtualenv used by Poetry can found with this command: + +```bash +poetry env list --full-path +``` + +Use it to configure your editor. [virtualenv]: https://docs.python-guide.org/dev/virtualenvs/ [pyenv]: https://github.com/pyenv/pyenv [Conda]: https://docs.conda.io/en/latest/ -* python: We pinned Python to version 3.8 so everyone uses the same version. - ### Developer Dependencies Similar to the previous section, `tool.poetry.dev-dependencies` is where `poetry @@ -117,13 +147,96 @@ adhere to our coding standards. * [rope](https://github.com/python-rope/rope), to provide completions and renaming support to pyls. -With these packages installed and a correctly setup editor (see the readme and -feel free to reach out to us), your code should automatically follow all the +With these packages installed and a correctly setup editor (see below, and feel +free to reach out to us), your code should automatically follow all the standards which we could automate. -The final sections, `tool.black`, `tool.isort`, `build-system`, and the -`.flake8` file (Flake8 does not yet support `pyproject.toml`) configure the -tools per their recommendations. +The final sections, `tool.black`, `tool.isort`, `build-system`, and files +`mypy.ini` and `.flake8` configure the tools per their recommendations. + +## Editor Setup + +### Visual Studio Code + +First, click the Python version in the bottom left, then enter the path emitted +by the command above. This will point Code to the Poetry virtual environment. + +Make sure below settings are in root level of `.vscode/settings.json` + +```json +{ + "python.analysis.typeCheckingMode": "strict", + "python.formatting.provider": "black", + "python.linting.enabled": true, + "python.linting.flake8Enabled": true, + "python.linting.mypyEnabled": true, + "python.linting.pylintEnabled": false, + "editor.formatOnSave": true, + "python.linting.mypyArgs": [ + "--strict", + "--namespace-packages", + "--show-column-numbers", + ], + "python.sortImports.path": "isort", + "python.analysis.useLibraryCodeForTypes": false, + "python.analysis.autoImportCompletions": false, + "files.eol": "\n", +} +``` + +### Emacs + +I recommend using the [pyvenv][] package to have Emacs automatically use the +correct Python venv (setup by Poetry), and the [eglot][] package to provide LSP +support. The below expects you already have an `init.el` with [use-package][]. + +[pyvenv]: https://github.com/jorgenschaefer/pyvenv +[eglot]: https://github.com/joaotavora/eglot +[use-package]: https://github.com/jwiegley/use-package + +```emacs-lisp +(use-package pyvenv + :ensure t + :hook (python-mode . pyvenv-tracking-mode)) + +(use-package eglot + :hook + (python-mode . eglot-ensure) + (yaml-mode . eglot-ensure) + :custom + (eglot-auto-display-help-buffer t) + (eglot-autoshutdown t) + (eglot-confirm-server-initiated-edits nil) + :config + (add-to-list 'eglot-server-programs '(yaml-mode . ("yaml-language-server" "--stdio"))) + (defun eglot-format-buffer-on-save () + (if (and (project-current) (eglot-managed-p)) + (add-hook 'before-save-hook #'eglot-format-buffer nil 'local) + (remove-hook 'before-save-hook #'eglot-format-buffer 'local))) + (add-hook 'eglot-managed-mode-hook #'eglot-format-buffer-on-save)) +``` + +Then run `M-x add-dir-local-variable RET python-mode RET pyvenv-activate RET +` where the value is the path given by the command above. +This will create a `.dir-locals.el` with this variable set for Python. + +Refer to this `.dir-locals.el` for a complete setup: + +```emacs-lisp +;;; Directory Local Variables +;;; For more information see (info "(emacs) Directory Variables") + +((python-mode + . ((eglot-workspace-configuration ; an LSP implementation + ;; Use `flake8’ instead of the default, and disable noisy plugins. + . ((:pyls . (:configurationSources ["flake8"] :plugins (:pycodestyle (:enabled nil) :mccabe (:enabled nil)))))))) + (yaml-mode + . ((eglot-workspace-configuration + ;; Set the `playbooks/schema.json’ as the schema for playbooks. + . ((:yaml . (:schemas (:playbooks/schema.json "playbooks/*"))))))) + ;; Set `pyvenv’ to use the given venv for the whole project. + (nil . ((pyvenv-activate . "~/.cache/pypoetry/virtualenvs/")))) +``` ## Type Annotations @@ -146,13 +259,31 @@ mypy’s [cheat sheet][]. [intro]: https://kishstats.com/python/2019/01/07/python-type-hinting.html [cheat sheet]: https://mypy.readthedocs.io/en/latest/cheat_sheet_py3.html -## Runbook schema +## Python Naming Conventions -Some plugins like Platform need follow this section to extend runbook schema. Runbook is the configurations of LISA runs. Every LISA run need a runbook. +Naming conventions are not automatically enforced, so please read the [naming +conventions](https://www.python.org/dev/peps/pep-0008/#naming-conventions) +section of PEP 8, which describes what each of the different styles means. A +short summary of the most important parts: -The runbook uses [dataclass](https://docs.python.org/3/library/dataclasses.html) to define, [dataclass-json](https://github.com/lidatong/dataclasses-json/) to deserialize, and [marshmallow](https://marshmallow.readthedocs.io/en/3.0/api_reference.html) to validate the schema. +* Modules (and hence files) should have short, all-lowercase names. +* Class (and exception) names should normally use the `CapWords` convention + (also known as `CamelCase`). +* Function and variable names should be lowercase, with words separated by + underscores as necessary to improve readability (also known as `snake_case`). +* To avoid collisions with the standard library, an underscore can be appended, + such as `id_`. +* Always use `self` for the first argument to instance methods. +* Always use `cls` for the first argument to class methods. +* Use one leading underscore only for non-public methods and instance variables, + such as `_data`. Do not activate name mangling with `__` unless necessary. +* If there is a pair of `get_x` and `set_x` methods, they should instead be a + proper property, which is easy to do with the built-in `@property` decorator. +* Constants should be `CAPITALIZED_SNAKE_CASE`. +* When importing a function, try to avoid renaming it with `import as` because + it introduces cognitive overhead to track yet another name. -See more examples in [schema.py](lisa/schema.py), if you need to extend runbook schema. +When in doubt, adhere to existing conventions, or check the style guide. ## Committing Guidelines @@ -242,28 +373,12 @@ Python world. If you make it through even some of these guides, you will be well on your way to being a “Pythonista” (a Python developer) writing “Pythonic” (canonically correct Python) code left and right. -### Async IO - -With Python 3.4, the Async IO pattern found in languages such as C# and Go is -available through the keywords `async` and `await`, along with the Python module -`asyncio`. Please read [Async IO in Python: A Complete -Walkthrough](https://realpython.com/async-io-python/) to understand at a high -level how asynchronous programming works. As of Python 3.7, One major “gotcha” -is that `asyncio.run(...)` should be used [exactly once in -`main`](https://docs.python.org/3/library/asyncio-task.html), it starts the -event loop. Everything else should be a coroutine or task which the event loop -schedules. - ## Future Sections Just a collection of reminders for the author to expand on later. -* [unittest](https://docs.python.org/3/library/unittest.html) * [doctest](https://docs.python.org/3/library/doctest.html) * [subprocess](https://pymotw.com/3/subprocess/index.html) * [GitHub Actions](https://github.com/LIS/LISAv2/actions) * [ShellCheck](https://www.shellcheck.net/) * [Governance](https://opensource.guide/leadership-and-governance/) -* [Maintenance Cost](https://web.archive.org/web/20120313070806/http://users.jyu.fi/~koskinen/smcosts.htm) -* Parallelism and multi-plexing -* Versioned inputs and outputs diff --git a/README.md b/README.md index 54718237d1..c87fcbc1f2 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,10 @@ LISA is a Linux test automation framework with built-in test cases to verify the quality of Linux distributions on multiple platforms (such as Azure, Hyper-V, -and bare metal). +and bare metal). It is an opinionated collection of custom [Pytest][] plugins, +configurations, and tests. See the [design document](DESIGN.md) for details. + +[Pytest]: https://docs.pytest.org/en/stable/ ## Getting Started: @@ -49,7 +52,7 @@ $env:PATH += ";$env:USERPROFILE\.poetry\bin" ### Clone LISA and `cd` into the Git repo: ```bash -git clone -b main https://github.com/LIS/LISAv2.git lisa +git clone -b main https://github.com/microsoft/lisa.git cd lisa ``` @@ -69,7 +72,7 @@ poetry shell # Run some self-tests lisa --playbook=playbooks/test.yml selftests/ -# Run a demo which deployes Azure resources +# Run a demo which deploys Azure resources lisa --playbook=playbooks/smoke.yaml ``` @@ -77,6 +80,8 @@ lisa --playbook=playbooks/smoke.yaml To run the demo you’ll need the [Azure CLI][] tool installed and configured: +[Azure CLI]: https://docs.microsoft.com/en-us/cli/azure/ + ```bash # Install Azure CLI, make sure `az` is in your `PATH` curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash @@ -86,130 +91,9 @@ az login az account set -s ``` -See the [design document](DESIGN.md) for details. - ## Contributing -The path to the virtualenv used by Poetry can found with this command: - -```bash -poetry env list --full-path -``` - -Use it to configure your editor. - -The registered playbook schema can be generated using `--print-schema`: - -```bash -lisa --print-schema=playbooks/schema.json -``` - -This will create a file `playbooks/schema.json` with the [JSON Schema][] for all -the registered schemata (including those added in any local plugins). Note that -this file is committed to the repo for the public schema, but you can generate -your own (or update it) with the above command. - -Using the LSP [yaml-language-server][] you can setup almost any editor to lint -the playbook files against the schema. Either add as a comment to the top of the -playbook file: - -```yaml -# yaml-language-server: $schema=file:///path/to/playbooks/schema.json -``` - -Or set `yaml.schemas` as appropriate for your editor. Also ensure that -`yaml-language-server` is installed, which you can do via: - -```bash -npm install -g yaml-language-server -``` - -[JSON Schema]: https://json-schema.org/ -[yaml-language-server]: https://github.com/redhat-developer/yaml-language-server - -### Editor Setup - -#### Visual Studio Code - -First, click the Python version in the bottom left, then enter the path emitted -by the command above. This will point Code to the Poetry virtual environment. - -Make sure below settings are in root level of `.vscode/settings.json` - -```json -{ - "python.analysis.typeCheckingMode": "strict", - "python.formatting.provider": "black", - "python.linting.enabled": true, - "python.linting.flake8Enabled": true, - "python.linting.mypyEnabled": true, - "python.linting.pylintEnabled": false, - "editor.formatOnSave": true, - "python.linting.mypyArgs": [ - "--strict", - "--namespace-packages", - "--show-column-numbers", - ], - "python.sortImports.path": "isort", - "python.analysis.useLibraryCodeForTypes": false, - "python.analysis.autoImportCompletions": false, - "files.eol": "\n", -} -``` - -#### Emacs - -I recommend using the [pyvenv][] package to have Emacs automatically use the -correct Python venv (setup by Poetry), and the [eglot][] package to provide LSP -support. The below expects you already have an `init.el` with [use-package][]. - -[pyvenv]: https://github.com/jorgenschaefer/pyvenv -[eglot]: https://github.com/joaotavora/eglot -[use-package]: https://github.com/jwiegley/use-package - -```emacs-lisp -(use-package pyvenv - :ensure t - :hook (python-mode . pyvenv-tracking-mode)) - -(use-package eglot - :hook - (python-mode . eglot-ensure) - (yaml-mode . eglot-ensure) - :custom - (eglot-auto-display-help-buffer t) - (eglot-autoshutdown t) - (eglot-confirm-server-initiated-edits nil) - :config - (add-to-list 'eglot-server-programs '(yaml-mode . ("yaml-language-server" "--stdio"))) - (defun eglot-format-buffer-on-save () - (if (and (project-current) (eglot-managed-p)) - (add-hook 'before-save-hook #'eglot-format-buffer nil 'local) - (remove-hook 'before-save-hook #'eglot-format-buffer 'local))) - (add-hook 'eglot-managed-mode-hook #'eglot-format-buffer-on-save)) -``` - -Then run `M-x add-dir-local-variable RET python-mode RET pyvenv-activate RET -` where the value is the path given by the command above. -This will create a `.dir-locals.el` with this variable set for Python. - -Refer to this `.dir-locals.el` for a complete setup: - -```emacs-lisp -;;; Directory Local Variables -;;; For more information see (info "(emacs) Directory Variables") - -((python-mode - . ((eglot-workspace-configuration ; an LSP implementation - ;; Use `flake8’ instead of the default, and disable noisy plugins. - . ((:pyls . (:configurationSources ["flake8"] :plugins (:pycodestyle (:enabled nil) :mccabe (:enabled nil)))))))) - (yaml-mode - . ((eglot-workspace-configuration - ;; Set the `playbooks/schema.json’ as the schema for playbooks. - . ((:yaml . (:schemas (:playbooks/schema.json "playbooks/*"))))))) - ;; Set `pyvenv’ to use the given venv for the whole project. - (nil . ((pyvenv-activate . "~/.cache/pypoetry/virtualenvs/")))) -``` +See the [Contributing Guidelines](CONTRIBUTING.md) for developer information! ### Contributor License Agreement From 37a1e893c5b751ea43c4eba4508d1c54e8d41001 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Thu, 17 Dec 2020 17:27:00 -0800 Subject: [PATCH 145/194] Skip `targets` test until cache key is fixed --- pytest-lisa/lisa.py | 2 +- pytest-target/target/plugin.py | 37 ++++++++++++++++++++-------------- selftests/test_basic.py | 3 +++ 3 files changed, 26 insertions(+), 16 deletions(-) diff --git a/pytest-lisa/lisa.py b/pytest-lisa/lisa.py index 860d0733e0..aec85862ca 100644 --- a/pytest-lisa/lisa.py +++ b/pytest-lisa/lisa.py @@ -115,7 +115,7 @@ def pytest_playbook_schema(schema: Dict[Any, Any]) -> None: ): bool, } ) - schema[Optional("criteria", default=list)] = [criteria_schema] + schema.update({Optional("criteria", default=list): [criteria_schema]}) lisa_schema = Schema( diff --git a/pytest-target/target/plugin.py b/pytest-target/target/plugin.py index 981ac47184..d2e141f172 100644 --- a/pytest-target/target/plugin.py +++ b/pytest-target/target/plugin.py @@ -48,7 +48,7 @@ def pytest_configure(config: Config) -> None: """ config.addinivalue_line( "markers", - ("target(platform, features, reuse, count): " "Specify target requirements."), + "target(platform, features, reuse, count): Specify target requirements.", ) @@ -75,13 +75,15 @@ def pytest_playbook_schema(schema: Dict[Any, Any]) -> None: # platform’s name to defaults for its provided schema. platforms_schema = dict(cls.get_defaults() for cls in platforms.values()) default_platforms = Schema(platforms_schema).validate({}) - schema[ - Optional( - "platforms", - default=default_platforms, - description="A set of objects with default values for each platform.", - ) - ] = platforms_schema + schema.update( + { + Optional( + "platforms", + default=default_platforms, + description="A set of objects with default values for each platform.", + ): platforms_schema + } + ) # The targets schema is a list of ‘any of’ the platforms’ # reference schemata. @@ -91,13 +93,15 @@ def pytest_playbook_schema(schema: Dict[Any, Any]) -> None: "platform": "SSH", **Schema(SSH.schema()).validate({}), # Fill in the defaults } - schema[ - Optional( - "targets", - default=[default_target], - description="A list of targets with which to parameterize the tests.", - ) - ] = targets_schema + schema.update( + { + Optional( + "targets", + default=[default_target], + description="A list of targets with which to parameterize the tests.", + ): targets_schema + } + ) @pytest.fixture(scope="session") @@ -154,6 +158,9 @@ def get_target( else: logging.info(f"Instantiating target: '{params}'...") assert request.config.cache is not None + # TODO: Using this key breaks `targets`. We’ll probably need + # to store a list of the unique names for this key, and for + # each of those store the `data`. key = "target/" + params["name"] cache = request.config.cache.get(key, {}) if cache: diff --git a/selftests/test_basic.py b/selftests/test_basic.py index ee85747841..197f6e1089 100644 --- a/selftests/test_basic.py +++ b/selftests/test_basic.py @@ -3,6 +3,8 @@ import typing +import pytest + if typing.TYPE_CHECKING: from target import SSH from typing import List @@ -17,6 +19,7 @@ def test_basic(target: SSH) -> None: @LISA(platform="SSH", category="Functional", area="self-test", priority=1, count=3) +@pytest.mark.skip("Need to fix cache bug with `targets`") def test_basic_multiple(targets: List[SSH]) -> None: """Basic test which asks for 3 unique targets.""" assert len({target.name for target in targets}) == 3 From b9672ce3206deb9336e4ddac0e2c61b1249ea1e8 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Thu, 17 Dec 2020 21:51:28 -0800 Subject: [PATCH 146/194] Rename `Target.name` to `group` and add `number` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This was planned in the TSD, it just didn’t make sense until we had proper groups. It was always supposed to be a unique group ID. --- pytest-target/target/azure.py | 31 +++++++++++++++++++------------ pytest-target/target/plugin.py | 9 +++------ pytest-target/target/target.py | 20 +++++++++++++++----- selftests/test_basic.py | 2 +- 4 files changed, 38 insertions(+), 24 deletions(-) diff --git a/pytest-target/target/azure.py b/pytest-target/target/azure.py index c065c74b8e..305b72752f 100644 --- a/pytest-target/target/azure.py +++ b/pytest-target/target/azure.py @@ -25,6 +25,7 @@ class AzureCLI(Target): # Custom instance attribute(s). internal_address: str + name: str @classmethod def schema(cls) -> Dict[Any, Any]: @@ -103,13 +104,15 @@ def allow_ping(self) -> None: for d in ["Inbound", "Outbound"]: self.local( f"az network nsg rule create " - f"--name allow{d}ICMP --resource-group {self.name}-rg " + f"--name allow{d}ICMP --resource-group {self.group}-rg " f"--nsg-name {self.name}NSG --priority 150 " f"--access Allow --direction '{d}' --protocol Icmp " "--source-port-ranges '*' --destination-port-ranges '*'" ) except Exception as e: - logging.warning(f"Failed to create ICMP allow rules in NSG due to '{e}'") + logging.warning( + f"Failed creating ICMP allow rules in '{self.name}NSG': {e}" + ) def parse_data(self) -> str: self.internal_address = self.data["privateIpAddress"] @@ -117,6 +120,7 @@ def parse_data(self) -> str: def deploy(self) -> str: """Given deployment info, deploy a new VM.""" + self.name = f"{self.group}-{self.number}" if self.data: # Shortcut if refreshing from cache. return self.parse_data() @@ -129,15 +133,15 @@ def deploy(self) -> str: logging.info( "Deploying VM...\n" - f" Group: '{self.name}-rg'\n" - f" Region: '{location}'\n" + f" Group: '{self.group}-rg'\n" + f" Region: '{location}'\n" f" Image: '{image}'\n" f" SKU: '{sku}'" ) boot_storage = self.create_boot_storage(location) - self._local(f"az group create -n {self.name}-rg --location {location}") + self._local(f"az group create -n {self.group}-rg --location {location}") # TODO: Accept EULA terms when necessary. Like: # @@ -148,7 +152,7 @@ def deploy(self) -> str: vm_command = [ "az vm create", - f"-g {self.name}-rg", + f"-g {self.group}-rg", f"-n {self.name}", f"--image {image}", f"--size {sku}", @@ -167,12 +171,15 @@ def deploy(self) -> str: def delete(self) -> None: """Delete the entire allocated resource group. - TODO: Delete VM itself. Only if it was the last VM then delete - the entire resource group. + TODO: Delete VM '{self.name}'. Only if it was + the last VM then delete the entire resource group. """ - logging.debug(f"Deleting resource group '{self.name}-rg'") - self.local(f"az group delete -n {self.name}-rg --yes --no-wait") + logging.debug(f"Deleting resource group '{self.group}-rg'") + try: + self.local(f"az group delete -n {self.group}-rg --yes --no-wait") + except Exception as e: + logging.warning(f"Failed deleting resource group '{self.group}-rg': {e}") @retry(reraise=True, wait=wait_exponential(), stop=stop_after_attempt(3)) def get_boot_diagnostics(self, **kwargs: Any) -> Result: @@ -181,10 +188,10 @@ def get_boot_diagnostics(self, **kwargs: Any) -> Result: # their logs aren’t UTF-8 encoded. I’ve filed a bug: # https://github.com/Azure/azure-cli/issues/15590 return self.local( - f"az vm boot-diagnostics get-boot-log -n {self.name} -g {self.name}-rg", + f"az vm boot-diagnostics get-boot-log -n {self.name} -g {self.group}-rg", **kwargs, ) def platform_restart(self) -> Result: """TODO: Should this '--force' and redeploy?""" - return self.local(f"az vm restart -n {self.name} -g {self.name}-rg") + return self.local(f"az vm restart -n {self.name} -g {self.group}-rg") diff --git a/pytest-target/target/plugin.py b/pytest-target/target/plugin.py index d2e141f172..c5e6042be0 100644 --- a/pytest-target/target/plugin.py +++ b/pytest-target/target/plugin.py @@ -116,10 +116,7 @@ def pool(request: SubRequest) -> Iterator[List[Target]]: delete_target(t, request.config.cache) -def get_target( - pool: List[Target], - request: SubRequest, -) -> Target: +def get_target(pool: List[Target], request: SubRequest, number: int = 0) -> Target: """This function gets or creates an appropriate `Target`. First check if any existing target in the `pool` matches all the @@ -192,7 +189,7 @@ def cleanup_target(pool: List[Target], request: SubRequest, t: Target) -> None: def delete_target(t: Target, cache: Optional[Cache]) -> None: """Deletes a `Target` and removes it from the cache.""" name: str = t.params["name"] - logging.info(f"Deleting target '{name}'...") + logging.info(f"Deleting target '{name}': {t.group}/{t.number}...") t.delete() assert cache is not None cache.set("target/" + name, None) @@ -217,7 +214,7 @@ def targets(pool: List[Target], request: SubRequest) -> Iterator[List[Target]]: count = marker.kwargs.get("count", 1) # TODO: Support sharing a `name` across the targets such that # they’re in the same logical group for any platform. - ts = [get_target(pool, request) for _ in range(count)] + ts = [get_target(pool, request, i) for i in range(count)] yield ts for t in ts: cleanup_target(pool, request, t) diff --git a/pytest-target/target/target.py b/pytest-target/target/target.py index a38d0f9504..61e6392736 100644 --- a/pytest-target/target/target.py +++ b/pytest-target/target/target.py @@ -31,10 +31,11 @@ class Target(ABC): """ # Typed instance attributes, not class attributes. - name: str + group: str params: Mapping[str, str] features: Set[str] data: Mapping[Any, Any] + number: int host: str conn: fabric.Connection @@ -55,16 +56,19 @@ class Target(ABC): def __init__( self, - name: str, + group: str, params: Mapping[Any, Any], features: Set[str], data: Mapping[Any, Any], + number: int = 0, ): """Creates and deploys an instance of `Target`. - * `name` is a unique ID for the group of associated resources + * `group` is a unique ID for the group of associated resources * `params` is the input parameters conforming to `schema()` * `features` is set of arbitrary feature requirements + * `data` is the cached data for the target + * `number` is the numerical ID of this target in its group Subclass implementations of `Target` do not need to (and should not) override `__init__()` as it is setup such that all @@ -72,10 +76,11 @@ def __init__( instead, which this calls. """ - self.name = name + self.group = group self.params = self.get_schema().validate(params) self.features = features self.data = data + self.number = number self.host = self.deploy() @@ -137,7 +142,12 @@ def deploy(self) -> str: @abstractmethod def delete(self) -> None: - """Must delete the target resources.""" + """Must delete the target's resources. + + If this is the last target in its group then implementations + should delete the group resource too. + + """ ... platform_description = "The class name of the platform implementation." diff --git a/selftests/test_basic.py b/selftests/test_basic.py index 197f6e1089..34509ee3b2 100644 --- a/selftests/test_basic.py +++ b/selftests/test_basic.py @@ -22,4 +22,4 @@ def test_basic(target: SSH) -> None: @pytest.mark.skip("Need to fix cache bug with `targets`") def test_basic_multiple(targets: List[SSH]) -> None: """Basic test which asks for 3 unique targets.""" - assert len({target.name for target in targets}) == 3 + assert len({target.group for target in targets}) == 3 From be86c87fe138e22602e1542d8cba0b717668fa1d Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Mon, 21 Dec 2020 16:05:50 -0800 Subject: [PATCH 147/194] Add `free` property to `Target` to indicate in-use state --- pytest-target/target/target.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pytest-target/target/target.py b/pytest-target/target/target.py index 61e6392736..80b6b7a117 100644 --- a/pytest-target/target/target.py +++ b/pytest-target/target/target.py @@ -36,6 +36,7 @@ class Target(ABC): features: Set[str] data: Mapping[Any, Any] number: int + free: bool host: str conn: fabric.Connection @@ -61,6 +62,7 @@ def __init__( features: Set[str], data: Mapping[Any, Any], number: int = 0, + free: bool = False, ): """Creates and deploys an instance of `Target`. @@ -69,6 +71,7 @@ def __init__( * `features` is set of arbitrary feature requirements * `data` is the cached data for the target * `number` is the numerical ID of this target in its group + * `free` is the state of the target in this session Subclass implementations of `Target` do not need to (and should not) override `__init__()` as it is setup such that all @@ -81,6 +84,7 @@ def __init__( self.features = features self.data = data self.number = number + self.free = free self.host = self.deploy() From 4e76b06389e3b03477d9dabcebd4aaf3cf25edde Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Mon, 21 Dec 2020 16:06:32 -0800 Subject: [PATCH 148/194] Add basic JSON serialization to `Target` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This records a target’s properties (including an arbitrary `data` dict for platform implementations) to (and from) a JSON object. This also moves the `__subclasses__` lookup functionality into a static method on `Target` which means `Target.from_json(...)` can be used like a factory. --- pytest-target/target/target.py | 41 +++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/pytest-target/target/target.py b/pytest-target/target/target.py index 80b6b7a117..25b1859d74 100644 --- a/pytest-target/target/target.py +++ b/pytest-target/target/target.py @@ -13,7 +13,7 @@ from tenacity import retry, stop_after_attempt, wait_exponential # type: ignore if typing.TYPE_CHECKING: - from typing import Any, Dict, Mapping, Set, Tuple + from typing import Any, Dict, List, Mapping, Set, Tuple class Target(ABC): @@ -154,6 +154,8 @@ def delete(self) -> None: """ ... + # Internal details follow: + platform_description = "The class name of the platform implementation." @classmethod @@ -218,6 +220,43 @@ def get_schema(cls) -> Schema: as_reference=True, ) + def to_json(self) -> Dict[str, Any]: + """Returns a JSON-serializable representation of `self`. + + This is an internal detail, used when caching the target. + + """ + return { + "group": self.group, + "params": self.params, + "features": list(self.features), + "data": self.data, + "number": self.number, + "free": self.free, + } + + @staticmethod + def from_json( + group: str, + params: Mapping[Any, Any], + features: List[str], + data: Mapping[Any, Any], + number: int, + free: bool, + ) -> Target: + """Instantiates the correct subclass given the JSON representation. + + This is an internal detail, used when (re-)creating the target. + + """ + platform = params["platform"] + cls: typing.Optional[typing.Type[Target]] = next( + (x for x in Target.__subclasses__() if x.__name__ == platform), + None, + ) + assert cls, f"Platform implementation not found for '{platform}'" + return cls(group, params, set(features), data, number, free) + # Platform-agnostic functionality should be added here: local_context = invoke.Context(config=invoke.Config(overrides=config)) From 9ac98ea8576505de85b648df458ba9dac578b31c Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Mon, 21 Dec 2020 17:09:03 -0800 Subject: [PATCH 149/194] Implement caching of target groups --- pytest-target/target/plugin.py | 186 +++++++++++++++++---------------- selftests/test_basic.py | 8 +- 2 files changed, 100 insertions(+), 94 deletions(-) diff --git a/pytest-target/target/plugin.py b/pytest-target/target/plugin.py index c5e6042be0..84d044cf74 100644 --- a/pytest-target/target/plugin.py +++ b/pytest-target/target/plugin.py @@ -3,6 +3,8 @@ # TODO: * Deallocate targets when switching to a new target. * Use richer feature/requirements comparison for targets. +* Make cache compatible with pytest-xdist. +* Cleanup `targets/pool` cache with a context manager. """ from __future__ import annotations @@ -19,9 +21,8 @@ from target.target import SSH, Target if typing.TYPE_CHECKING: - from typing import Any, Dict, Iterator, List, Type + from typing import Any, Dict, Iterator, List - from _pytest.cacheprovider import Cache from _pytest.config import Config from _pytest.config.argparsing import Parser from _pytest.fixtures import SubRequest @@ -52,28 +53,24 @@ def pytest_configure(config: Config) -> None: ) -platforms: Dict[str, Type[Target]] = dict() - def pytest_playbook_schema(schema: Dict[Any, Any]) -> None: """pytest-playbook hook to update the playbook schema. - The `platforms` global is a mapping of platform names (strings) to - the implementing subclasses of `Target` where each subclass - defines its own parameters `schema`, optional `defaults`, - `deploy`, `delete` methods, and other platform-specific - functionality. A `Target` subclass need only be defined in a file - loaded by Pytest, so a `conftest.py` file works just fine. + This adds `platforms` and `targets` keys to the playbook schema, + with their nested schemata accumulated from each platform's + implementations of `defaults()` and `schema()`. We do this by + iterating over the subclasses of `Target`, a handy feature of + Python that lets us automatically discover users' implementations, + even if they're defined in a local `conftest.py` Pytest + configuration file. """ - # Map the subclasses of `Target` into name and class pairs, used - # by `get_target` to lookup the type based on the name. - global platforms - platforms = {cls.__name__: cls for cls in Target.__subclasses__()} # type: ignore + classes = Target.__subclasses__() # The platforms schema is a set of optional mappings of each # platform’s name to defaults for its provided schema. - platforms_schema = dict(cls.get_defaults() for cls in platforms.values()) + platforms_schema = dict(cls.get_defaults() for cls in classes) default_platforms = Schema(platforms_schema).validate({}) schema.update( { @@ -87,7 +84,7 @@ def pytest_playbook_schema(schema: Dict[Any, Any]) -> None: # The targets schema is a list of ‘any of’ the platforms’ # reference schemata. - targets_schema = [Or(*(cls.get_schema() for cls in platforms.values()))] + targets_schema = [Or(*(cls.get_schema() for cls in classes))] default_target = { "name": "Default", "platform": "SSH", @@ -111,88 +108,98 @@ def pool(request: SubRequest) -> Iterator[List[Target]]: yield targets # TODO: Catch interrupts and always delete targets: # `UnexpectedExit`, `KeyboardInterrupt`, `SystemExit`. - for t in targets: - if not request.config.getoption("keep_targets"): - delete_target(t, request.config.cache) + if not request.config.getoption("keep_targets"): + logging.info("Deleting targets! Pass `--keep-targets` to prevent this.") + for t in targets: + t.delete() + targets.clear() + assert request.config.cache is not None + request.config.cache.set("target/pool", []) + + +def get_target(pool: List[Target], request: SubRequest) -> Target: + """Common case of getting one target.""" + marker = request.node.get_closest_marker("target") + count = marker.kwargs.get("count", 1) + assert count == 1, "Use `targets` fixture with `count` instead!" + return get_targets(pool, request).pop() -def get_target(pool: List[Target], request: SubRequest, number: int = 0) -> Target: - """This function gets or creates an appropriate `Target`. +def get_targets(pool: List[Target], request: SubRequest) -> List[Target]: + """This function gets or creates an appropriate number of `Target`s. - First check if any existing target in the `pool` matches all the - `features` and other requirements. If so, we can re-use that - target, and if not, we can deallocate the currently running target - and allocate a new one. When all tests are finished, the pool - fixture above will delete all created VMs. We can achieve - two-layer scheduling by implementing a custom scheduler in - pytest-xdist via `pytest_xdist_make_scheduler` and sorting the - tests such that they're grouped by features. + 1. Update `pool` (list of targets) from the cache + 2. Unpack request into params, required features, and count + 3. Setup fitness criteria for target(s) + 4. Find or create necessary targets + 5. Update cache with modified `pool` + 6. Return targets """ - # Alias because we use it a lot in this function. + assert request.config.cache is not None + # TODO: Use a file lock to handle multi-processing, and handle + # edge case where cache plugin isn’t available. + key = "target/pool" + # NOTE: We’re explicitly modifying this argument. + pool[:] = [Target.from_json(**x) for x in request.config.cache.get(key, [])] + + # Get the required params for this test. params: Dict[Any, Any] = request.param - # Get the intended class for this parameterization of `target`. - platform: Type[Target] = platforms[params["platform"]] # Get the required features for this test. marker = request.node.get_closest_marker("target") - features = set(marker.kwargs.get("features", [])) + features = marker.kwargs.get("features", []) + count = marker.kwargs.get("count", 1) + + def fits(t: Target) -> bool: + # TODO: Implement full feature comparison, etc. and not just + # proof-of-concept string set comparison. + logging.debug(f"Checking fit of {t.to_json()}...") + return ( + t.free + and params == t.params + and set(features) <= t.features + and count <= sum(t.group == x.group for x in pool) + ) # TODO: If `t` is not already in use, deallocate the previous # target, and ensure the tests have been sorted (and so grouped) # by their requirements. - for t in pool: - # TODO: Implement full feature comparison, etc. and not just - # proof-of-concept string set comparison. - if ( - isinstance(t, platform) - # NOTE: This is not the same as `t.name`! - and t.params["name"] == params["name"] - and t.features >= features - ): - pool.remove(t) - return t + ts: List[Target] = [] + logging.debug(f"Looking for {count} target(s) which fit: {params}...") + for i in range(count): + for t in pool: + if fits(t): + logging.debug(f"Found fit target '{i}'!") + t.free = False + ts.append(t) + break + if count == len(ts): + break else: - logging.info(f"Instantiating target: '{params}'...") - assert request.config.cache is not None - # TODO: Using this key breaks `targets`. We’ll probably need - # to store a list of the unique names for this key, and for - # each of those store the `data`. - key = "target/" + params["name"] - cache = request.config.cache.get(key, {}) - if cache: - logging.debug("Cache hit...") - name = cache["name"] - assert cache["params"] == params, "Params changed!" - # TODO: Check `features`, but a set isn’t serializable. - data = cache["data"] - else: - logging.debug("Cache miss...") - name = f"pytest-{uuid4()}" - data = None - t = platform(name, params, features, data) - cache = {"name": name, "params": params, "data": t.data} - request.config.cache.set(key, cache) - return t - - -def cleanup_target(pool: List[Target], request: SubRequest, t: Target) -> None: + group = f"pytest-{uuid4()}" + for i in range(count): + logging.info(f"Instantiating target '{group}/{i}': {params}...") + t = Target.from_json(group, params, features, {}, i, False) + ts.append(t) + pool.append(t) + request.config.cache.set(key, [x.to_json() for x in pool]) + return ts + + +def cleanup_target(t: Target, pool: List[Target], request: SubRequest) -> None: """This is called by fixtures after they're done with a target.""" t.conn.close() marker = request.node.get_closest_marker("target") if marker.kwargs.get("reuse", True): - pool.append(t) + # Leave in pool and mark free for use. + t.free = True else: - delete_target(t, request.config.cache) - - -def delete_target(t: Target, cache: Optional[Cache]) -> None: - """Deletes a `Target` and removes it from the cache.""" - name: str = t.params["name"] - logging.info(f"Deleting target '{name}': {t.group}/{t.number}...") - t.delete() - assert cache is not None - cache.set("target/" + name, None) + logging.info(f"Deleting target '{t.group}/{t.number}'...") + t.delete() + pool.remove(t) + assert request.config.cache is not None + request.config.cache.set("target/pool", [x.to_json() for x in pool]) @pytest.fixture @@ -204,20 +211,21 @@ def target(pool: List[Target], request: SubRequest) -> Iterator[Target]: """ t = get_target(pool, request) yield t - cleanup_target(pool, request, t) + cleanup_target(t, pool, request) @pytest.fixture def targets(pool: List[Target], request: SubRequest) -> Iterator[List[Target]]: - """This fixture obtains N targets for a test.""" - marker = request.node.get_closest_marker("target") - count = marker.kwargs.get("count", 1) - # TODO: Support sharing a `name` across the targets such that - # they’re in the same logical group for any platform. - ts = [get_target(pool, request, i) for i in range(count)] + """This fixture is the same as `target` but gets a list of targets. + + For example, use `pytest.mark.target(count=2)` to get a list of + two targets with the same parameters, in the same group. + + """ + ts = get_targets(pool, request) yield ts for t in ts: - cleanup_target(pool, request, t) + cleanup_target(t, pool, request) @pytest.fixture(scope="class") @@ -225,7 +233,7 @@ def c_target(pool: List[Target], request: SubRequest) -> Iterator[Target]: """This fixture is the same as `target` but shared across a class.""" t = get_target(pool, request) yield t - cleanup_target(pool, request, t) + cleanup_target(t, pool, request) @pytest.fixture(scope="module") @@ -233,7 +241,7 @@ def m_target(pool: List[Target], request: SubRequest) -> Iterator[Target]: """This fixture is the same as `target` but shared across a module.""" t = get_target(pool, request) yield t - cleanup_target(pool, request, t) + cleanup_target(t, pool, request) target_params: Dict[str, Dict[str, Any]] = {} diff --git a/selftests/test_basic.py b/selftests/test_basic.py index 34509ee3b2..de768e15a9 100644 --- a/selftests/test_basic.py +++ b/selftests/test_basic.py @@ -3,8 +3,6 @@ import typing -import pytest - if typing.TYPE_CHECKING: from target import SSH from typing import List @@ -19,7 +17,7 @@ def test_basic(target: SSH) -> None: @LISA(platform="SSH", category="Functional", area="self-test", priority=1, count=3) -@pytest.mark.skip("Need to fix cache bug with `targets`") def test_basic_multiple(targets: List[SSH]) -> None: - """Basic test which asks for 3 unique targets.""" - assert len({target.group for target in targets}) == 3 + """Basic test which asks for 3 unique targets in 1 group.""" + assert len({t.group for t in targets}) == 1 + assert len({t.number for t in targets}) == 3 From 2be9b903b759b581bc5d35bc5f9980d4c77001bf Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Mon, 21 Dec 2020 17:09:07 -0800 Subject: [PATCH 150/194] Implement `--delete-targets` --- pytest-target/target/plugin.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pytest-target/target/plugin.py b/pytest-target/target/plugin.py index 84d044cf74..1633e2c5af 100644 --- a/pytest-target/target/plugin.py +++ b/pytest-target/target/plugin.py @@ -52,6 +52,15 @@ def pytest_configure(config: Config) -> None: "target(platform, features, reuse, count): Specify target requirements.", ) + if config.getoption("delete_targets"): + logging.info("Deleting all cached targets!") + assert config.cache is not None + targets = [Target.from_json(**x) for x in config.cache.get("target/pool", [])] + for t in targets: + t.delete() + targets.clear() + config.cache.set("target/pool", []) + pytest.exit("Deleted all cached targets!", pytest.ExitCode.OK) def pytest_playbook_schema(schema: Dict[Any, Any]) -> None: From 92a3d0abc09c4f63cdf646a05a90d706cedc37b4 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Mon, 21 Dec 2020 18:12:10 -0800 Subject: [PATCH 151/194] Refactor cache access into pair of functions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A context manager wasn’t quite right, as we only want to deserialize the cached targets to a `List[Target]` once per session. But this cleaned up well with a pair of functions for handling cache access and a rename of `pool` to `target_pool`. --- pytest-target/target/plugin.py | 120 ++++++++++++++++----------------- 1 file changed, 60 insertions(+), 60 deletions(-) diff --git a/pytest-target/target/plugin.py b/pytest-target/target/plugin.py index 1633e2c5af..70218a1aad 100644 --- a/pytest-target/target/plugin.py +++ b/pytest-target/target/plugin.py @@ -4,7 +4,6 @@ * Deallocate targets when switching to a new target. * Use richer feature/requirements comparison for targets. * Make cache compatible with pytest-xdist. -* Cleanup `targets/pool` cache with a context manager. """ from __future__ import annotations @@ -23,12 +22,26 @@ if typing.TYPE_CHECKING: from typing import Any, Dict, Iterator, List + from _pytest.cacheprovider import Cache from _pytest.config import Config from _pytest.config.argparsing import Parser from _pytest.fixtures import SubRequest + from _pytest.mark.structures import Mark from _pytest.python import Metafunc +def get_target_cache(cache: Optional[Cache]) -> List[Target]: + assert cache is not None + key = "target/pool" + return [Target.from_json(**t) for t in cache.get(key, [])] + + +def set_target_cache(cache: Optional[Cache], pool: List[Target]) -> None: + assert cache is not None + key = "target/pool" + cache.set(key, [t.to_json() for t in pool]) + + def pytest_addoption(parser: Parser) -> None: """Pytest hook to add our CLI options.""" group = parser.getgroup("target") @@ -54,12 +67,11 @@ def pytest_configure(config: Config) -> None: if config.getoption("delete_targets"): logging.info("Deleting all cached targets!") - assert config.cache is not None - targets = [Target.from_json(**x) for x in config.cache.get("target/pool", [])] - for t in targets: + pool = get_target_cache(config.cache) + for t in pool: t.delete() - targets.clear() - config.cache.set("target/pool", []) + pool.clear() + set_target_cache(config.cache, pool) pytest.exit("Deleted all cached targets!", pytest.ExitCode.OK) @@ -110,22 +122,6 @@ def pytest_playbook_schema(schema: Dict[Any, Any]) -> None: ) -@pytest.fixture(scope="session") -def pool(request: SubRequest) -> Iterator[List[Target]]: - """This fixture tracks all deployed target resources.""" - targets: List[Target] = [] - yield targets - # TODO: Catch interrupts and always delete targets: - # `UnexpectedExit`, `KeyboardInterrupt`, `SystemExit`. - if not request.config.getoption("keep_targets"): - logging.info("Deleting targets! Pass `--keep-targets` to prevent this.") - for t in targets: - t.delete() - targets.clear() - assert request.config.cache is not None - request.config.cache.set("target/pool", []) - - def get_target(pool: List[Target], request: SubRequest) -> Target: """Common case of getting one target.""" marker = request.node.get_closest_marker("target") @@ -137,28 +133,18 @@ def get_target(pool: List[Target], request: SubRequest) -> Target: def get_targets(pool: List[Target], request: SubRequest) -> List[Target]: """This function gets or creates an appropriate number of `Target`s. - 1. Update `pool` (list of targets) from the cache - 2. Unpack request into params, required features, and count - 3. Setup fitness criteria for target(s) - 4. Find or create necessary targets - 5. Update cache with modified `pool` - 6. Return targets + 1. Unpack request into params, required features, and count + 2. Setup fitness criteria for target(s) + 3. Find or create necessary targets + 4. Update cache with modified `pool` + 5. Return targets """ - assert request.config.cache is not None - # TODO: Use a file lock to handle multi-processing, and handle - # edge case where cache plugin isn’t available. - key = "target/pool" - # NOTE: We’re explicitly modifying this argument. - pool[:] = [Target.from_json(**x) for x in request.config.cache.get(key, [])] - - # Get the required params for this test. params: Dict[Any, Any] = request.param - - # Get the required features for this test. - marker = request.node.get_closest_marker("target") - features = marker.kwargs.get("features", []) - count = marker.kwargs.get("count", 1) + mark: Optional[Mark] = request.node.get_closest_marker("target") + assert mark is not None + features = mark.kwargs.get("features", []) + count = mark.kwargs.get("count", 1) def fits(t: Target) -> bool: # TODO: Implement full feature comparison, etc. and not just @@ -190,67 +176,81 @@ def fits(t: Target) -> bool: for i in range(count): logging.info(f"Instantiating target '{group}/{i}': {params}...") t = Target.from_json(group, params, features, {}, i, False) - ts.append(t) pool.append(t) - request.config.cache.set(key, [x.to_json() for x in pool]) + ts.append(t) + set_target_cache(request.config.cache, pool) return ts def cleanup_target(t: Target, pool: List[Target], request: SubRequest) -> None: """This is called by fixtures after they're done with a target.""" t.conn.close() - marker = request.node.get_closest_marker("target") - if marker.kwargs.get("reuse", True): - # Leave in pool and mark free for use. + mark: Optional[Mark] = request.node.get_closest_marker("target") + assert mark is not None + if mark.kwargs.get("reuse", True): t.free = True else: logging.info(f"Deleting target '{t.group}/{t.number}'...") t.delete() pool.remove(t) - assert request.config.cache is not None - request.config.cache.set("target/pool", [x.to_json() for x in pool]) + set_target_cache(request.config.cache, pool) + + +@pytest.fixture(scope="session") +def target_pool(request: SubRequest) -> Iterator[List[Target]]: + """This fixture tracks all deployed target resources.""" + pool = get_target_cache(request.config.cache) + yield pool + # TODO: Catch interrupts and always delete targets: + # `UnexpectedExit`, `KeyboardInterrupt`, `SystemExit`. + if not request.config.getoption("keep_targets"): + logging.info("Deleting targets! Pass `--keep-targets` to prevent this.") + for t in pool: + t.delete() + pool.clear() + set_target_cache(request.config.cache, pool) @pytest.fixture -def target(pool: List[Target], request: SubRequest) -> Iterator[Target]: +def target(target_pool: List[Target], request: SubRequest) -> Iterator[Target]: """This fixture provides a connected target for each test. It is parametrized indirectly in `pytest_generate_tests`. """ - t = get_target(pool, request) + t = get_target(target_pool, request) yield t - cleanup_target(t, pool, request) + cleanup_target(t, target_pool, request) @pytest.fixture -def targets(pool: List[Target], request: SubRequest) -> Iterator[List[Target]]: +def targets(target_pool: List[Target], request: SubRequest) -> Iterator[List[Target]]: """This fixture is the same as `target` but gets a list of targets. For example, use `pytest.mark.target(count=2)` to get a list of two targets with the same parameters, in the same group. """ - ts = get_targets(pool, request) + ts = get_targets(target_pool, request) yield ts for t in ts: - cleanup_target(t, pool, request) + cleanup_target(t, target_pool, request) @pytest.fixture(scope="class") -def c_target(pool: List[Target], request: SubRequest) -> Iterator[Target]: +def c_target(target_pool: List[Target], request: SubRequest) -> Iterator[Target]: """This fixture is the same as `target` but shared across a class.""" - t = get_target(pool, request) + t = get_target(target_pool, request) yield t - cleanup_target(t, pool, request) + cleanup_target(t, target_pool, request) @pytest.fixture(scope="module") -def m_target(pool: List[Target], request: SubRequest) -> Iterator[Target]: +def m_target(target_pool: List[Target], request: SubRequest) -> Iterator[Target]: """This fixture is the same as `target` but shared across a module.""" - t = get_target(pool, request) + t = get_target(target_pool, request) yield t - cleanup_target(t, pool, request) + cleanup_target(t, target_pool, request) target_params: Dict[str, Dict[str, Any]] = {} From 6116b0f043594b6fe51cef967f8e9e62c93c9000 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Mon, 21 Dec 2020 19:11:36 -0800 Subject: [PATCH 152/194] Add FileLock package to `pytest-target` --- pytest-lisa/poetry.lock | 13 +++++++++++++ pytest-target/poetry.lock | 14 +++++++++++++- pytest-target/pyproject.toml | 1 + 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/pytest-lisa/poetry.lock b/pytest-lisa/poetry.lock index 6f479a0661..3072594dd5 100644 --- a/pytest-lisa/poetry.lock +++ b/pytest-lisa/poetry.lock @@ -120,6 +120,14 @@ paramiko = ">=2.4" pytest = ["mock (>=2.0.0,<3.0)", "pytest (>=3.2.5,<4.0)"] testing = ["mock (>=2.0.0,<3.0)"] +[[package]] +name = "filelock" +version = "3.0.12" +description = "A platform independent file lock." +category = "main" +optional = false +python-versions = "*" + [[package]] name = "importlib-metadata" version = "3.1.1" @@ -299,6 +307,7 @@ develop = true [package.dependencies] fabric = "^2.5.0" +filelock = "^3.0.12" invoke = "^1.4.1" pytest = "^6.1.2" pytest-playbook = "0.1.0" @@ -491,6 +500,10 @@ fabric = [ {file = "fabric-2.5.0-py2.py3-none-any.whl", hash = "sha256:160331934ea60036604928e792fa8e9f813266b098ef5562aa82b88527740389"}, {file = "fabric-2.5.0.tar.gz", hash = "sha256:24842d7d51556adcabd885ac3cf5e1df73fc622a1708bf3667bf5927576cdfa6"}, ] +filelock = [ + {file = "filelock-3.0.12-py3-none-any.whl", hash = "sha256:929b7d63ec5b7d6b71b0fa5ac14e030b3f70b75747cef1b10da9b879fef15836"}, + {file = "filelock-3.0.12.tar.gz", hash = "sha256:18d82244ee114f543149c66a6e0c14e9c4f8a1044b5cdaadd0f82159d6a6ff59"}, +] importlib-metadata = [ {file = "importlib_metadata-3.1.1-py3-none-any.whl", hash = "sha256:6112e21359ef8f344e7178aa5b72dc6e62b38b0d008e6d3cb212c5b84df72013"}, {file = "importlib_metadata-3.1.1.tar.gz", hash = "sha256:b0c2d3b226157ae4517d9625decf63591461c66b3a808c2666d538946519d170"}, diff --git a/pytest-target/poetry.lock b/pytest-target/poetry.lock index e209922037..74a761b516 100644 --- a/pytest-target/poetry.lock +++ b/pytest-target/poetry.lock @@ -98,6 +98,14 @@ paramiko = ">=2.4" pytest = ["mock (>=2.0.0,<3.0)", "pytest (>=3.2.5,<4.0)"] testing = ["mock (>=2.0.0,<3.0)"] +[[package]] +name = "filelock" +version = "3.0.12" +description = "A platform independent file lock." +category = "main" +optional = false +python-versions = "*" + [[package]] name = "importlib-metadata" version = "3.1.1" @@ -318,7 +326,7 @@ testing = ["pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "d0e0d3cfad95b0ead424722cf46a2ad67bc1140e5d7c08ad2821855e58086c88" +content-hash = "db5b497c6697ab98df1b46727bd1d10fd3a6853b142a9fc9c11a626032d2ebd5" [metadata.files] atomicwrites = [ @@ -412,6 +420,10 @@ fabric = [ {file = "fabric-2.5.0-py2.py3-none-any.whl", hash = "sha256:160331934ea60036604928e792fa8e9f813266b098ef5562aa82b88527740389"}, {file = "fabric-2.5.0.tar.gz", hash = "sha256:24842d7d51556adcabd885ac3cf5e1df73fc622a1708bf3667bf5927576cdfa6"}, ] +filelock = [ + {file = "filelock-3.0.12-py3-none-any.whl", hash = "sha256:929b7d63ec5b7d6b71b0fa5ac14e030b3f70b75747cef1b10da9b879fef15836"}, + {file = "filelock-3.0.12.tar.gz", hash = "sha256:18d82244ee114f543149c66a6e0c14e9c4f8a1044b5cdaadd0f82159d6a6ff59"}, +] importlib-metadata = [ {file = "importlib_metadata-3.1.1-py3-none-any.whl", hash = "sha256:6112e21359ef8f344e7178aa5b72dc6e62b38b0d008e6d3cb212c5b84df72013"}, {file = "importlib_metadata-3.1.1.tar.gz", hash = "sha256:b0c2d3b226157ae4517d9625decf63591461c66b3a808c2666d538946519d170"}, diff --git a/pytest-target/pyproject.toml b/pytest-target/pyproject.toml index bb6a272586..c111e393ab 100644 --- a/pytest-target/pyproject.toml +++ b/pytest-target/pyproject.toml @@ -11,6 +11,7 @@ packages = [{include = "target"}] python = "^3.7" pytest = "^6.1.2" fabric = "^2.5.0" +filelock = "^3.0.12" invoke = "^1.4.1" tenacity = "^6.2.0" pytest-playbook = {path = "../pytest-playbook", develop = true} From 45dbbdf71964dae24f80de710112bce40522260a Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Tue, 22 Dec 2020 11:31:29 -0800 Subject: [PATCH 153/194] Rename `Target.free` to `Target.locked` As it makes more sense from a acquire/release standpoint. --- pytest-target/target/plugin.py | 48 +++++++++++++++++++--------------- pytest-target/target/target.py | 14 +++++----- 2 files changed, 34 insertions(+), 28 deletions(-) diff --git a/pytest-target/target/plugin.py b/pytest-target/target/plugin.py index 70218a1aad..4b80619bb4 100644 --- a/pytest-target/target/plugin.py +++ b/pytest-target/target/plugin.py @@ -3,17 +3,18 @@ # TODO: * Deallocate targets when switching to a new target. * Use richer feature/requirements comparison for targets. -* Make cache compatible with pytest-xdist. """ from __future__ import annotations import logging import typing +from pathlib import Path from uuid import uuid4 import playbook import pytest +from filelock import FileLock # type: ignore # See https://pypi.org/project/schema/ from schema import Optional, Or, Schema # type: ignore @@ -22,24 +23,27 @@ if typing.TYPE_CHECKING: from typing import Any, Dict, Iterator, List - from _pytest.cacheprovider import Cache from _pytest.config import Config from _pytest.config.argparsing import Parser from _pytest.fixtures import SubRequest from _pytest.mark.structures import Mark from _pytest.python import Metafunc + from pytest import Session -def get_target_cache(cache: Optional[Cache]) -> List[Target]: - assert cache is not None - key = "target/pool" - return [Target.from_json(**t) for t in cache.get(key, [])] +def get_target_cache(config: Config) -> List[Target]: + assert config.cache is not None + lock = Path(config.cache.makedir("target")) / "pool.lock" + with FileLock(str(lock)): + cache = config.cache.get("target/pool", []) + return [Target.from_json(**t) for t in cache] -def set_target_cache(cache: Optional[Cache], pool: List[Target]) -> None: - assert cache is not None - key = "target/pool" - cache.set(key, [t.to_json() for t in pool]) +def set_target_cache(config: Config, pool: List[Target]) -> None: + assert config.cache is not None + lock = Path(config.cache.makedir("target")) / "pool.lock" + with FileLock(str(lock)): + config.cache.set("target/pool", [t.to_json() for t in pool]) def pytest_addoption(parser: Parser) -> None: @@ -67,11 +71,11 @@ def pytest_configure(config: Config) -> None: if config.getoption("delete_targets"): logging.info("Deleting all cached targets!") - pool = get_target_cache(config.cache) + pool = get_target_cache(config) for t in pool: t.delete() pool.clear() - set_target_cache(config.cache, pool) + set_target_cache(config, pool) pytest.exit("Deleted all cached targets!", pytest.ExitCode.OK) @@ -151,7 +155,7 @@ def fits(t: Target) -> bool: # proof-of-concept string set comparison. logging.debug(f"Checking fit of {t.to_json()}...") return ( - t.free + not t.locked and params == t.params and set(features) <= t.features and count <= sum(t.group == x.group for x in pool) @@ -166,7 +170,7 @@ def fits(t: Target) -> bool: for t in pool: if fits(t): logging.debug(f"Found fit target '{i}'!") - t.free = False + t.locked = True ts.append(t) break if count == len(ts): @@ -178,7 +182,7 @@ def fits(t: Target) -> bool: t = Target.from_json(group, params, features, {}, i, False) pool.append(t) ts.append(t) - set_target_cache(request.config.cache, pool) + set_target_cache(request.config, pool) return ts @@ -188,18 +192,18 @@ def cleanup_target(t: Target, pool: List[Target], request: SubRequest) -> None: mark: Optional[Mark] = request.node.get_closest_marker("target") assert mark is not None if mark.kwargs.get("reuse", True): - t.free = True + t.locked = False else: logging.info(f"Deleting target '{t.group}/{t.number}'...") t.delete() pool.remove(t) - set_target_cache(request.config.cache, pool) + set_target_cache(request.config, pool) @pytest.fixture(scope="session") def target_pool(request: SubRequest) -> Iterator[List[Target]]: """This fixture tracks all deployed target resources.""" - pool = get_target_cache(request.config.cache) + pool = get_target_cache(request.config) yield pool # TODO: Catch interrupts and always delete targets: # `UnexpectedExit`, `KeyboardInterrupt`, `SystemExit`. @@ -208,7 +212,7 @@ def target_pool(request: SubRequest) -> Iterator[List[Target]]: for t in pool: t.delete() pool.clear() - set_target_cache(request.config.cache, pool) + set_target_cache(request.config, pool) @pytest.fixture @@ -256,8 +260,10 @@ def m_target(target_pool: List[Target], request: SubRequest) -> Iterator[Target] target_params: Dict[str, Dict[str, Any]] = {} -def pytest_sessionstart() -> None: - """Gather the `targets` from the playbook. +def pytest_sessionstart(session: Session) -> None: + """Pytest hook to setup the session. + + Gather the `targets` from the playbook. First collect any user supplied defaults from the `platforms` key in the playbook, which will default to the given `defaults` diff --git a/pytest-target/target/target.py b/pytest-target/target/target.py index 25b1859d74..d8516afc59 100644 --- a/pytest-target/target/target.py +++ b/pytest-target/target/target.py @@ -36,7 +36,7 @@ class Target(ABC): features: Set[str] data: Mapping[Any, Any] number: int - free: bool + locked: bool host: str conn: fabric.Connection @@ -62,7 +62,7 @@ def __init__( features: Set[str], data: Mapping[Any, Any], number: int = 0, - free: bool = False, + locked: bool = True, ): """Creates and deploys an instance of `Target`. @@ -71,7 +71,7 @@ def __init__( * `features` is set of arbitrary feature requirements * `data` is the cached data for the target * `number` is the numerical ID of this target in its group - * `free` is the state of the target in this session + * `locked` is the state of the target's availability Subclass implementations of `Target` do not need to (and should not) override `__init__()` as it is setup such that all @@ -84,7 +84,7 @@ def __init__( self.features = features self.data = data self.number = number - self.free = free + self.locked = locked self.host = self.deploy() @@ -232,7 +232,7 @@ def to_json(self) -> Dict[str, Any]: "features": list(self.features), "data": self.data, "number": self.number, - "free": self.free, + "locked": self.locked, } @staticmethod @@ -242,7 +242,7 @@ def from_json( features: List[str], data: Mapping[Any, Any], number: int, - free: bool, + locked: bool, ) -> Target: """Instantiates the correct subclass given the JSON representation. @@ -255,7 +255,7 @@ def from_json( None, ) assert cls, f"Platform implementation not found for '{platform}'" - return cls(group, params, set(features), data, number, free) + return cls(group, params, set(features), data, number, locked) # Platform-agnostic functionality should be added here: From d5239f25bd38e79eff27b3ac60ae27e469bffc11 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Tue, 22 Dec 2020 11:38:05 -0800 Subject: [PATCH 154/194] Move `name` to base `Target` class --- pytest-target/target/azure.py | 2 -- pytest-target/target/target.py | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pytest-target/target/azure.py b/pytest-target/target/azure.py index 305b72752f..d3b1eeffd6 100644 --- a/pytest-target/target/azure.py +++ b/pytest-target/target/azure.py @@ -25,7 +25,6 @@ class AzureCLI(Target): # Custom instance attribute(s). internal_address: str - name: str @classmethod def schema(cls) -> Dict[Any, Any]: @@ -120,7 +119,6 @@ def parse_data(self) -> str: def deploy(self) -> str: """Given deployment info, deploy a new VM.""" - self.name = f"{self.group}-{self.number}" if self.data: # Shortcut if refreshing from cache. return self.parse_data() diff --git a/pytest-target/target/target.py b/pytest-target/target/target.py index d8516afc59..0abca045d0 100644 --- a/pytest-target/target/target.py +++ b/pytest-target/target/target.py @@ -37,6 +37,7 @@ class Target(ABC): data: Mapping[Any, Any] number: int locked: bool + name: str host: str conn: fabric.Connection @@ -85,6 +86,7 @@ def __init__( self.data = data self.number = number self.locked = locked + self.name = f"{self.group}-{self.number}" self.host = self.deploy() From 6797fcf09bae63e43b89c41740832c572716083c Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Tue, 22 Dec 2020 14:44:39 -0800 Subject: [PATCH 155/194] Use dataclass as serializable container for target data --- pytest-target/target/target.py | 90 +++++++++++++++++----------------- 1 file changed, 46 insertions(+), 44 deletions(-) diff --git a/pytest-target/target/target.py b/pytest-target/target/target.py index 0abca045d0..eafabf1d2a 100644 --- a/pytest-target/target/target.py +++ b/pytest-target/target/target.py @@ -1,9 +1,10 @@ """Provides the abstract base `Target` class.""" from __future__ import annotations +import dataclasses import platform import typing -from abc import ABC, abstractmethod +from abc import ABCMeta, abstractmethod from io import BytesIO import fabric # type: ignore @@ -13,10 +14,43 @@ from tenacity import retry, stop_after_attempt, wait_exponential # type: ignore if typing.TYPE_CHECKING: - from typing import Any, Dict, List, Mapping, Set, Tuple + from typing import Any, Dict, List, Mapping, Tuple, Type -class Target(ABC): +@dataclasses.dataclass +class TargetData: + """This class holds serializable data for a `Target`. + + This is an internal detail. It is separated out so we can easily + serialize to and from JSON in order to enable caching. By + decoupling these we prevent users from having to understand the + semantics of a `dataclass`, and fields added to subclasses don't + interfere with serialization. + + TODO: Consider using more from `dataclasses`, such as `field()` + and `__post_init__()`. + + """ + + group: str + params: Dict[str, str] + features: List[str] + data: Dict[Any, Any] + number: int + locked: bool + + def to_json(self) -> Dict[str, Any]: + """Returns a JSON-serializable representation of `self`.""" + return dataclasses.asdict(self) + + @staticmethod + def from_json(json: Dict[str, Any]) -> Target: + """Instantiates the correct subclass given the JSON representation.""" + cls = Target.get_platform(json["params"]["platform"]) + return cls(**json) + + +class Target(TargetData, metaclass=ABCMeta): """This class represents a remote Linux target. As a partially abstract base class, it is meant to be subclassed @@ -30,13 +64,9 @@ class Target(ABC): """ - # Typed instance attributes, not class attributes. - group: str - params: Mapping[str, str] - features: Set[str] - data: Mapping[Any, Any] - number: int - locked: bool + # Typed instance attributes (not class attributes) in addition to + # those inherited from the dataclass `TargetData`. These exist + # here and not on the superclass because they shouldn’t be cached. name: str host: str conn: fabric.Connection @@ -59,9 +89,9 @@ class Target(ABC): def __init__( self, group: str, - params: Mapping[Any, Any], - features: Set[str], - data: Mapping[Any, Any], + params: Dict[Any, Any], + features: List[str], + data: Dict[Any, Any], number: int = 0, locked: bool = True, ): @@ -86,8 +116,8 @@ def __init__( self.data = data self.number = number self.locked = locked - self.name = f"{self.group}-{self.number}" + self.name = f"{self.group}-{self.number}" self.host = self.deploy() fabric_config = self.config.copy() @@ -222,42 +252,14 @@ def get_schema(cls) -> Schema: as_reference=True, ) - def to_json(self) -> Dict[str, Any]: - """Returns a JSON-serializable representation of `self`. - - This is an internal detail, used when caching the target. - - """ - return { - "group": self.group, - "params": self.params, - "features": list(self.features), - "data": self.data, - "number": self.number, - "locked": self.locked, - } - @staticmethod - def from_json( - group: str, - params: Mapping[Any, Any], - features: List[str], - data: Mapping[Any, Any], - number: int, - locked: bool, - ) -> Target: - """Instantiates the correct subclass given the JSON representation. - - This is an internal detail, used when (re-)creating the target. - - """ - platform = params["platform"] + def get_platform(platform: str) -> Type[Target]: cls: typing.Optional[typing.Type[Target]] = next( (x for x in Target.__subclasses__() if x.__name__ == platform), None, ) assert cls, f"Platform implementation not found for '{platform}'" - return cls(group, params, set(features), data, number, locked) + return cls # Platform-agnostic functionality should be added here: From f876c7cfb49d7e6dea38aeb25b9580ba48a60285 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Tue, 22 Dec 2020 16:00:09 -0800 Subject: [PATCH 156/194] Refactor to use locking context manager instead of session fixture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This uses a context manager with a file lock to safely access a shared pool cached to disk (via Pytest’s built-in cache). Instead of having a session-scoped `pool` fixture we actively create a `Target` instance each time a `target` fixture is used (so this is now 1-1). This means instances will be in a clean-state, and don’t need to persist across a session, but also means that reusable targets must be properly cached to disk (see the reference implementations). We handle deletion of targets in the `configure` and `unconfigure` hooks, which are safe to call regardless of the use of pytest-xdist, and don’t require a session. --- pytest-target/target/plugin.py | 258 +++++++++++++++++++-------------- 1 file changed, 146 insertions(+), 112 deletions(-) diff --git a/pytest-target/target/plugin.py b/pytest-target/target/plugin.py index 4b80619bb4..d09808e545 100644 --- a/pytest-target/target/plugin.py +++ b/pytest-target/target/plugin.py @@ -9,6 +9,8 @@ import logging import typing +import warnings +from contextlib import contextmanager from pathlib import Path from uuid import uuid4 @@ -18,10 +20,10 @@ # See https://pypi.org/project/schema/ from schema import Optional, Or, Schema # type: ignore -from target.target import SSH, Target +from target.target import SSH, Target, TargetData if typing.TYPE_CHECKING: - from typing import Any, Dict, Iterator, List + from typing import Any, Dict, Generator, Iterator, List from _pytest.config import Config from _pytest.config.argparsing import Parser @@ -31,21 +33,6 @@ from pytest import Session -def get_target_cache(config: Config) -> List[Target]: - assert config.cache is not None - lock = Path(config.cache.makedir("target")) / "pool.lock" - with FileLock(str(lock)): - cache = config.cache.get("target/pool", []) - return [Target.from_json(**t) for t in cache] - - -def set_target_cache(config: Config, pool: List[Target]) -> None: - assert config.cache is not None - lock = Path(config.cache.makedir("target")) / "pool.lock" - with FileLock(str(lock)): - config.cache.set("target/pool", [t.to_json() for t in pool]) - - def pytest_addoption(parser: Parser) -> None: """Pytest hook to add our CLI options.""" group = parser.getgroup("target") @@ -57,28 +44,6 @@ def pytest_addoption(parser: Parser) -> None: ) -def pytest_configure(config: Config) -> None: - """Pytest hook to perform initial configuration. - - We're registering our custom marker so that it passes - `--strict-markers`. - - """ - config.addinivalue_line( - "markers", - "target(platform, features, reuse, count): Specify target requirements.", - ) - - if config.getoption("delete_targets"): - logging.info("Deleting all cached targets!") - pool = get_target_cache(config) - for t in pool: - t.delete() - pool.clear() - set_target_cache(config, pool) - pytest.exit("Deleted all cached targets!", pytest.ExitCode.OK) - - def pytest_playbook_schema(schema: Dict[Any, Any]) -> None: """pytest-playbook hook to update the playbook schema. @@ -126,22 +91,85 @@ def pytest_playbook_schema(schema: Dict[Any, Any]) -> None: ) -def get_target(pool: List[Target], request: SubRequest) -> Target: +@contextmanager +def target_pool(config: Config) -> Generator[Dict[str, Any], None, None]: + """Exclusive access to the cached targets pool. + + This handles access to the Pytest cache of serialized targets. The + cache is a dict of ``{target.name: target.to_json()}``. We use a + file lock to provide exclusive access even if Pytest is being run + in parallel with xdist. Entries have a `locked` property and must + only be modified during a session when locked by that session. + Locking means setting `locked` to `True` and updating the entry + before exiting this context manager. + + """ + # TODO: Handle edge case where cache plugin is disabled. + assert config.cache is not None + lock = Path(config.cache.makedir("target")) / "pool.lock" + with FileLock(str(lock)): + pool = config.cache.get("target/pool", {}) + yield pool + config.cache.set("target/pool", pool) + + +def delete_targets(config: Config) -> None: + """Deletes all cached targets.""" + with target_pool(config) as pool: + for name, json in pool.items(): + try: + Target.from_json(json).delete() + except Exception as e: + warnings.warn(f"Failed to delete '{name}': {e}") + pool.clear() + + +def pytest_configure(config: Config) -> None: + """Pytest hook to perform initial configuration. + + https://docs.pytest.org/en/stable/reference.html#pytest.hookspec.pytest_configure + + We're registering our custom marker so that it passes + `--strict-markers`. + + """ + config.addinivalue_line( + "markers", + "target(platform, features, reuse, count): Specify target requirements.", + ) + + if config.getoption("delete_targets"): + logging.info("Deleting all cached targets!") + delete_targets(config) + pytest.exit("Deleted all cached targets!", pytest.ExitCode.OK) + + +def pytest_unconfigure(config: Config) -> None: + """Pytest hook to perform teardown. + + https://docs.pytest.org/en/stable/reference.html#pytest.hookspec.pytest_unconfigure + + """ + if not config.getoption("keep_targets"): + logging.info("Deleting targets! Pass `--keep-targets` to prevent this.") + delete_targets(config) + + +def get_target(request: SubRequest) -> Target: """Common case of getting one target.""" marker = request.node.get_closest_marker("target") count = marker.kwargs.get("count", 1) assert count == 1, "Use `targets` fixture with `count` instead!" - return get_targets(pool, request).pop() + return get_targets(request).pop() -def get_targets(pool: List[Target], request: SubRequest) -> List[Target]: +def get_targets(request: SubRequest) -> List[Target]: """This function gets or creates an appropriate number of `Target`s. 1. Unpack request into params, required features, and count 2. Setup fitness criteria for target(s) 3. Find or create necessary targets - 4. Update cache with modified `pool` - 5. Return targets + 4. Return targets """ params: Dict[Any, Any] = request.param @@ -150,111 +178,117 @@ def get_targets(pool: List[Target], request: SubRequest) -> List[Target]: features = mark.kwargs.get("features", []) count = mark.kwargs.get("count", 1) - def fits(t: Target) -> bool: - # TODO: Implement full feature comparison, etc. and not just - # proof-of-concept string set comparison. - logging.debug(f"Checking fit of {t.to_json()}...") - return ( - not t.locked - and params == t.params - and set(features) <= t.features - and count <= sum(t.group == x.group for x in pool) - ) - - # TODO: If `t` is not already in use, deallocate the previous - # target, and ensure the tests have been sorted (and so grouped) - # by their requirements. - ts: List[Target] = [] - logging.debug(f"Looking for {count} target(s) which fit: {params}...") - for i in range(count): - for t in pool: - if fits(t): - logging.debug(f"Found fit target '{i}'!") - t.locked = True - ts.append(t) - break - if count == len(ts): - break - else: - group = f"pytest-{uuid4()}" - for i in range(count): - logging.info(f"Instantiating target '{group}/{i}': {params}...") - t = Target.from_json(group, params, features, {}, i, False) - pool.append(t) - ts.append(t) - set_target_cache(request.config, pool) - return ts + targets: List[Target] = [] + with target_pool(request.config) as pool: + + def fits(t: TargetData) -> bool: + """Checks if a given Target fits the current search criteria. + + Converting the cached JSON to a `TargetData` instance is + cheap and lets us use typed fields here. + + TODO: Implement full feature comparison, etc. and not just + proof-of-concept string set comparison. + """ + logging.debug(f"Checking fit of {t}...") + return ( + not t.locked + and params == t.params + and set(features) <= set(t.features) + and count <= sum(t.group == x["group"] for x in pool.values()) + ) -def cleanup_target(t: Target, pool: List[Target], request: SubRequest) -> None: + # TODO: If `t` is not already in use, deallocate the previous + # target, and ensure the tests have been sorted (and so grouped) + # by their requirements. + logging.debug(f"Looking for {count} target(s) which fit: {params}...") + for i in range(count): + for name, json in pool.items(): + if fits(TargetData(**json)): + logging.debug(f"Found fit target '{i}'!") + # TODO: De-duplicate this logic and its + # counterpart below. + t = Target.from_json(json) + assert name == t.name # Sanity check. + t.locked = True + pool[t.name] = t.to_json() + targets.append(t) + break + if count == len(targets): + # TODO: This is a kludgy way to either use the found + # targets or give up and instantiate new ones instead. + # Theoretically the length here should either be zero + # or `count` because of the check in `fits`, but + # perhaps we should handle the erroneous case where + # it’s in-between. + break + else: + group = f"pytest-{uuid4()}" + for i in range(count): + logging.info(f"Instantiating target '{group}-{i}': {params}...") + cls = Target.get_platform(params["platform"]) + t = cls(group, params, features, {}, i) + pool[t.name] = t.to_json() + targets.append(t) + return targets + + +def cleanup_target(t: Target, request: SubRequest) -> None: """This is called by fixtures after they're done with a target.""" t.conn.close() mark: Optional[Mark] = request.node.get_closest_marker("target") assert mark is not None - if mark.kwargs.get("reuse", True): - t.locked = False - else: - logging.info(f"Deleting target '{t.group}/{t.number}'...") - t.delete() - pool.remove(t) - set_target_cache(request.config, pool) - - -@pytest.fixture(scope="session") -def target_pool(request: SubRequest) -> Iterator[List[Target]]: - """This fixture tracks all deployed target resources.""" - pool = get_target_cache(request.config) - yield pool - # TODO: Catch interrupts and always delete targets: - # `UnexpectedExit`, `KeyboardInterrupt`, `SystemExit`. - if not request.config.getoption("keep_targets"): - logging.info("Deleting targets! Pass `--keep-targets` to prevent this.") - for t in pool: + with target_pool(request.config) as pool: + if mark.kwargs.get("reuse", True): + t.locked = False + pool[t.name] = t.to_json() + else: + logging.info(f"Deleting target '{t.group}/{t.number}'...") t.delete() - pool.clear() - set_target_cache(request.config, pool) + del pool[t.name] @pytest.fixture -def target(target_pool: List[Target], request: SubRequest) -> Iterator[Target]: +def target(request: SubRequest) -> Iterator[Target]: """This fixture provides a connected target for each test. It is parametrized indirectly in `pytest_generate_tests`. """ - t = get_target(target_pool, request) + t = get_target(request) yield t - cleanup_target(t, target_pool, request) + cleanup_target(t, request) @pytest.fixture -def targets(target_pool: List[Target], request: SubRequest) -> Iterator[List[Target]]: +def targets(request: SubRequest) -> Iterator[List[Target]]: """This fixture is the same as `target` but gets a list of targets. For example, use `pytest.mark.target(count=2)` to get a list of two targets with the same parameters, in the same group. """ - ts = get_targets(target_pool, request) + ts = get_targets(request) yield ts for t in ts: - cleanup_target(t, target_pool, request) + cleanup_target(t, request) @pytest.fixture(scope="class") -def c_target(target_pool: List[Target], request: SubRequest) -> Iterator[Target]: +def c_target(request: SubRequest) -> Iterator[Target]: """This fixture is the same as `target` but shared across a class.""" - t = get_target(target_pool, request) + t = get_target(request) yield t - cleanup_target(t, target_pool, request) + cleanup_target(t, request) @pytest.fixture(scope="module") -def m_target(target_pool: List[Target], request: SubRequest) -> Iterator[Target]: +def m_target(request: SubRequest) -> Iterator[Target]: """This fixture is the same as `target` but shared across a module.""" - t = get_target(target_pool, request) + t = get_target(request) yield t - cleanup_target(t, target_pool, request) + cleanup_target(t, request) target_params: Dict[str, Dict[str, Any]] = {} From 6df423d941ca1f581eae5c7a119f2c71e9fb75b3 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Tue, 22 Dec 2020 16:18:05 -0800 Subject: [PATCH 157/194] Emit full flake8/mypy errors in CI --- .github/workflows/ci-workflow.yaml | 2 +- conftest.py | 7 +++++++ pytest.ini | 1 - 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-workflow.yaml b/.github/workflows/ci-workflow.yaml index 778aa4243b..81a9724e29 100644 --- a/.github/workflows/ci-workflow.yaml +++ b/.github/workflows/ci-workflow.yaml @@ -41,4 +41,4 @@ jobs: run: poetry run pytest --verbose --playbook=playbooks/test.yaml --setup-show selftests/ - name: Run semantic analysis - run: poetry run pytest --quiet --flake8 --mypy -m "flake8 or mypy" + run: poetry run pytest --tb=auto --flake8 --mypy -m "flake8 or mypy" diff --git a/conftest.py b/conftest.py index 35efa00987..642f7b25c8 100644 --- a/conftest.py +++ b/conftest.py @@ -7,6 +7,7 @@ """ from __future__ import annotations +import logging import typing if typing.TYPE_CHECKING: @@ -16,3 +17,9 @@ def pytest_html_report_title(report: HTMLReport) -> None: """pytest-html hook to set the HTML report title.""" report.title = "LISAv3 Results" + + +def pytest_configure() -> None: + """Pytest hook for plugin configuration.""" + # Flake8 is noisy so we make it quieter here. + logging.getLogger("flake8").setLevel(logging.WARNING) diff --git a/pytest.ini b/pytest.ini index 8a835b10cc..6b2ea51d0a 100644 --- a/pytest.ini +++ b/pytest.ini @@ -11,5 +11,4 @@ render_collapsed = true junit_logging = all timeout = 1200 filterwarnings = - ignore:the imp module is deprecated in favour of importlib:DeprecationWarning ignore:Module already imported so cannot be rewritten From 45bb5c5ea34a7cd1d1e16f70b554ce76a1e7cc74 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Tue, 5 Jan 2021 11:22:37 -0800 Subject: [PATCH 158/194] Properly report deselected items --- README.md | 2 +- pytest-lisa/lisa.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c87fcbc1f2..e60f1dd289 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ poetry shell lisa --playbook=playbooks/test.yml selftests/ # Run a demo which deploys Azure resources -lisa --playbook=playbooks/smoke.yaml +lisa --playbook=playbooks/demo.yaml ``` #### Enable Azure: diff --git a/pytest-lisa/lisa.py b/pytest-lisa/lisa.py index aec85862ca..b5473cf087 100644 --- a/pytest-lisa/lisa.py +++ b/pytest-lisa/lisa.py @@ -236,7 +236,11 @@ def select(item: Item, times: int, exclude: bool) -> None: # Handle edge case of no items selected for inclusion. if not included: included = items - items[:] = [i for i in included if i not in excluded] + # Properly report deselected items. + collected = [i for i in included if i not in excluded] + deselected = [i for i in items if i not in collected] + config.hook.pytest_deselected(items=deselected) + items[:] = collected class LISAScheduling(LoadScopeScheduling): From ec8e757f61aa4a051371d70179c5bca503335504 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Tue, 5 Jan 2021 16:26:35 -0800 Subject: [PATCH 159/194] Update Python packages --- poetry.lock | 137 ++++++++++++++++++------------------ pytest-lisa/poetry.lock | 107 ++++++++++++++-------------- pytest-playbook/poetry.lock | 45 +++++++----- pytest-target/poetry.lock | 101 +++++++++++++------------- 4 files changed, 207 insertions(+), 183 deletions(-) diff --git a/poetry.lock b/poetry.lock index aeac6ece5a..440c2e3b14 100644 --- a/poetry.lock +++ b/poetry.lock @@ -162,7 +162,7 @@ testing = ["mock (>=2.0.0,<3.0)"] name = "filelock" version = "3.0.12" description = "A platform independent file lock." -category = "dev" +category = "main" optional = false python-versions = "*" @@ -225,13 +225,14 @@ test = ["pytest (>=4.0.2,<6)", "toml"] [[package]] name = "importlib-metadata" -version = "3.1.1" +version = "3.3.0" description = "Read metadata from Python packages" category = "main" optional = false python-versions = ">=3.6" [package.dependencies] +typing-extensions = {version = ">=3.6.4", markers = "python_version < \"3.8\""} zipp = ">=0.5" [package.extras] @@ -248,7 +249,7 @@ python-versions = "*" [[package]] name = "invoke" -version = "1.4.1" +version = "1.5.0" description = "Pythonic task execution" category = "main" optional = false @@ -256,7 +257,7 @@ python-versions = "*" [[package]] name = "isort" -version = "5.6.4" +version = "5.7.0" description = "A Python utility / library to sort Python imports." category = "dev" optional = false @@ -316,7 +317,7 @@ python-versions = "*" [[package]] name = "packaging" -version = "20.7" +version = "20.8" description = "Core utilities for Python packages" category = "main" optional = false @@ -379,7 +380,7 @@ dev = ["pre-commit", "tox"] [[package]] name = "py" -version = "1.9.0" +version = "1.10.0" description = "library with cross-python path, ini-parsing, io, code, log facilities" category = "main" optional = false @@ -478,30 +479,29 @@ python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" [[package]] name = "pytest" -version = "6.1.2" +version = "6.2.1" description = "pytest: simple powerful testing with Python" category = "main" optional = false -python-versions = ">=3.5" +python-versions = ">=3.6" [package.dependencies] atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} -attrs = ">=17.4.0" +attrs = ">=19.2.0" colorama = {version = "*", markers = "sys_platform == \"win32\""} importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} iniconfig = "*" packaging = "*" -pluggy = ">=0.12,<1.0" +pluggy = ">=0.12,<1.0.0a1" py = ">=1.8.2" toml = "*" [package.extras] -checkqa_mypy = ["mypy (==0.780)"] testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] [[package]] name = "pytest-flake8" -version = "1.0.6" +version = "1.0.7" description = "pytest plugin to check FLAKE8 requirements" category = "dev" optional = false @@ -622,6 +622,7 @@ develop = true [package.dependencies] fabric = "^2.5.0" +filelock = "^3.0.12" invoke = "^1.4.1" pytest = "^6.1.2" pytest-playbook = "0.1.0" @@ -644,7 +645,7 @@ pytest = ">=3.6.0" [[package]] name = "pytest-xdist" -version = "2.1.0" +version = "2.2.0" description = "pytest xdist plugin for distributed testing and loop-on-failing modes" category = "main" optional = false @@ -747,7 +748,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" [[package]] name = "tenacity" -version = "6.2.0" +version = "6.3.1" description = "Retry code until it succeeds" category = "main" optional = false @@ -761,7 +762,7 @@ doc = ["reno", "sphinx", "tornado (>=4.5)"] [[package]] name = "testfixtures" -version = "6.15.0" +version = "6.17.0" description = "A collection of helpers and mock objects for unit tests and doc tests." category = "dev" optional = false @@ -782,7 +783,7 @@ python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" [[package]] name = "typed-ast" -version = "1.4.1" +version = "1.4.2" description = "a fork of Python 2 and 3 ast modules with type comment support" category = "dev" optional = false @@ -792,7 +793,7 @@ python-versions = "*" name = "typing-extensions" version = "3.7.4.3" description = "Backported and Experimental Type Hints for Python 3.5+" -category = "dev" +category = "main" optional = false python-versions = "*" @@ -944,21 +945,21 @@ flake8-isort = [ {file = "flake8_isort-4.0.0-py2.py3-none-any.whl", hash = "sha256:729cd6ef9ba3659512dee337687c05d79c78e1215fdf921ed67e5fe46cce2f3c"}, ] importlib-metadata = [ - {file = "importlib_metadata-3.1.1-py3-none-any.whl", hash = "sha256:6112e21359ef8f344e7178aa5b72dc6e62b38b0d008e6d3cb212c5b84df72013"}, - {file = "importlib_metadata-3.1.1.tar.gz", hash = "sha256:b0c2d3b226157ae4517d9625decf63591461c66b3a808c2666d538946519d170"}, + {file = "importlib_metadata-3.3.0-py3-none-any.whl", hash = "sha256:bf792d480abbd5eda85794e4afb09dd538393f7d6e6ffef6e9f03d2014cf9450"}, + {file = "importlib_metadata-3.3.0.tar.gz", hash = "sha256:5c5a2720817414a6c41f0a49993908068243ae02c1635a228126519b509c8aed"}, ] iniconfig = [ {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, ] invoke = [ - {file = "invoke-1.4.1-py2-none-any.whl", hash = "sha256:93e12876d88130c8e0d7fd6618dd5387d6b36da55ad541481dfa5e001656f134"}, - {file = "invoke-1.4.1-py3-none-any.whl", hash = "sha256:87b3ef9d72a1667e104f89b159eaf8a514dbf2f3576885b2bbdefe74c3fb2132"}, - {file = "invoke-1.4.1.tar.gz", hash = "sha256:de3f23bfe669e3db1085789fd859eb8ca8e0c5d9c20811e2407fa042e8a5e15d"}, + {file = "invoke-1.5.0-py2-none-any.whl", hash = "sha256:da7c2d0be71be83ffd6337e078ef9643f41240024d6b2659e7b46e0b251e339f"}, + {file = "invoke-1.5.0-py3-none-any.whl", hash = "sha256:7e44d98a7dc00c91c79bac9e3007276965d2c96884b3c22077a9f04042bd6d90"}, + {file = "invoke-1.5.0.tar.gz", hash = "sha256:f0c560075b5fb29ba14dad44a7185514e94970d1b9d57dcd3723bec5fed92650"}, ] isort = [ - {file = "isort-5.6.4-py3-none-any.whl", hash = "sha256:dcab1d98b469a12a1a624ead220584391648790275560e1a43e54c5dceae65e7"}, - {file = "isort-5.6.4.tar.gz", hash = "sha256:dcaeec1b5f0eca77faea2a35ab790b4f3680ff75590bfcb7145986905aab2f58"}, + {file = "isort-5.7.0-py3-none-any.whl", hash = "sha256:fff4f0c04e1825522ce6949973e83110a6e907750cd92d128b0d14aaaadbffdc"}, + {file = "isort-5.7.0.tar.gz", hash = "sha256:c729845434366216d320e936b8ad6f9d681aab72dc7cbc2d51bedc3582f3ad1e"}, ] jedi = [ {file = "jedi-0.17.2-py2.py3-none-any.whl", hash = "sha256:98cc583fa0f2f8304968199b01b6b4b94f469a1f4a74c1560506ca2a211378b5"}, @@ -989,8 +990,8 @@ mypy-extensions = [ {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"}, ] packaging = [ - {file = "packaging-20.7-py2.py3-none-any.whl", hash = "sha256:eb41423378682dadb7166144a4926e443093863024de508ca5c9737d6bc08376"}, - {file = "packaging-20.7.tar.gz", hash = "sha256:05af3bb85d320377db281cf254ab050e1a7ebcbf5410685a9a407e18a1f81236"}, + {file = "packaging-20.8-py2.py3-none-any.whl", hash = "sha256:24e0da08660a87484d1602c30bb4902d74816b6985b93de36926f5bc95741858"}, + {file = "packaging-20.8.tar.gz", hash = "sha256:78598185a7008a470d64526a8059de9aaa449238f280fc9eb6b13ba6c4109093"}, ] paramiko = [ {file = "paramiko-2.7.2-py2.py3-none-any.whl", hash = "sha256:4f3e316fef2ac628b05097a637af35685183111d4bc1b5979bd397c2ab7b5898"}, @@ -1009,8 +1010,8 @@ pluggy = [ {file = "pluggy-0.13.1.tar.gz", hash = "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0"}, ] py = [ - {file = "py-1.9.0-py2.py3-none-any.whl", hash = "sha256:366389d1db726cd2fcfc79732e75410e5fe4d31db13692115529d34069a043c2"}, - {file = "py-1.9.0.tar.gz", hash = "sha256:9ca6883ce56b4e8da7e79ac18787889fa5206c79dcc67fb065376cd2fe03f342"}, + {file = "py-1.10.0-py2.py3-none-any.whl", hash = "sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a"}, + {file = "py-1.10.0.tar.gz", hash = "sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3"}, ] pycodestyle = [ {file = "pycodestyle-2.6.0-py2.py3-none-any.whl", hash = "sha256:2295e7b2f6b5bd100585ebcb1f616591b652db8a741695b3d8f5d28bdc934367"}, @@ -1059,12 +1060,12 @@ pyparsing = [ {file = "pyparsing-2.4.7.tar.gz", hash = "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1"}, ] pytest = [ - {file = "pytest-6.1.2-py3-none-any.whl", hash = "sha256:4288fed0d9153d9646bfcdf0c0428197dba1ecb27a33bb6e031d002fa88653fe"}, - {file = "pytest-6.1.2.tar.gz", hash = "sha256:c0a7e94a8cdbc5422a51ccdad8e6f1024795939cc89159a0ae7f0b316ad3823e"}, + {file = "pytest-6.2.1-py3-none-any.whl", hash = "sha256:1969f797a1a0dbd8ccf0fecc80262312729afea9c17f1d70ebf85c5e76c6f7c8"}, + {file = "pytest-6.2.1.tar.gz", hash = "sha256:66e419b1899bc27346cb2c993e12c5e5e8daba9073c1fbce33b9807abc95c306"}, ] pytest-flake8 = [ - {file = "pytest-flake8-1.0.6.tar.gz", hash = "sha256:1b82bb58c88eb1db40524018d3fcfd0424575029703b4e2d8e3ee873f2b17027"}, - {file = "pytest_flake8-1.0.6-py2.py3-none-any.whl", hash = "sha256:2e91578ecd9b200066f99c1e1de0f510fbb85bcf43712d46ea29fe47607cc234"}, + {file = "pytest-flake8-1.0.7.tar.gz", hash = "sha256:f0259761a903563f33d6f099914afef339c085085e643bee8343eb323b32dd6b"}, + {file = "pytest_flake8-1.0.7-py2.py3-none-any.whl", hash = "sha256:c28cf23e7d359753c896745fd4ba859495d02e16c84bac36caa8b1eec58f5bc1"}, ] pytest-forked = [ {file = "pytest-forked-1.3.0.tar.gz", hash = "sha256:6aa9ac7e00ad1a539c41bec6d21011332de671e938c7637378ec9710204e37ca"}, @@ -1094,8 +1095,8 @@ pytest-timeout = [ {file = "pytest_timeout-1.4.2-py2.py3-none-any.whl", hash = "sha256:541d7aa19b9a6b4e475c759fd6073ef43d7cdc9a92d95644c260076eb257a063"}, ] pytest-xdist = [ - {file = "pytest-xdist-2.1.0.tar.gz", hash = "sha256:82d938f1a24186520e2d9d3a64ef7d9ac7ecdf1a0659e095d18e596b8cbd0672"}, - {file = "pytest_xdist-2.1.0-py3-none-any.whl", hash = "sha256:7c629016b3bb006b88ac68e2b31551e7becf173c76b977768848e2bbed594d90"}, + {file = "pytest-xdist-2.2.0.tar.gz", hash = "sha256:1d8edbb1a45e8e1f8e44b1260583107fc23f8bc8da6d18cb331ff61d41258ecf"}, + {file = "pytest_xdist-2.2.0-py3-none-any.whl", hash = "sha256:f127e11e84ad37cc1de1088cb2990f3c354630d428af3f71282de589c5bb779b"}, ] python-jsonrpc-server = [ {file = "python-jsonrpc-server-0.4.0.tar.gz", hash = "sha256:62c543e541f101ec5b57dc654efc212d2c2e3ea47ff6f54b2e7dcb36ecf20595"}, @@ -1175,48 +1176,48 @@ six = [ {file = "six-1.15.0.tar.gz", hash = "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259"}, ] tenacity = [ - {file = "tenacity-6.2.0-py2.py3-none-any.whl", hash = "sha256:5a5d3dcd46381abe8b4f82b5736b8726fd3160c6c7161f53f8af7f1eb9b82173"}, - {file = "tenacity-6.2.0.tar.gz", hash = "sha256:29ae90e7faf488a8628432154bb34ace1cca58244c6ea399fd33f066ac71339a"}, + {file = "tenacity-6.3.1-py2.py3-none-any.whl", hash = "sha256:baed357d9f35ec64264d8a4bbf004c35058fad8795c5b0d8a7dc77ecdcbb8f39"}, + {file = "tenacity-6.3.1.tar.gz", hash = "sha256:e14d191fb0a309b563904bbc336582efe2037de437e543b38da749769b544d7f"}, ] testfixtures = [ - {file = "testfixtures-6.15.0-py2.py3-none-any.whl", hash = "sha256:e17f4f526fc90b0ac9bc7f8ca62b7dec17d9faf3d721f56bda4f0fd94d02f85a"}, - {file = "testfixtures-6.15.0.tar.gz", hash = "sha256:409f77cfbdad822d12a8ce5c4aa8fb4d0bb38073f4a5444fede3702716a2cec2"}, + {file = "testfixtures-6.17.0-py2.py3-none-any.whl", hash = "sha256:ebcc3e024d47bb58a60cdc678604151baa0c920ae2814004c89ac9066de31b2c"}, + {file = "testfixtures-6.17.0.tar.gz", hash = "sha256:fa7c170df68ca6367eda061e9ec339ae3e6d3679c31e04033f83ef97a7d7d0ce"}, ] toml = [ {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, ] typed-ast = [ - {file = "typed_ast-1.4.1-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:73d785a950fc82dd2a25897d525d003f6378d1cb23ab305578394694202a58c3"}, - {file = "typed_ast-1.4.1-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:aaee9905aee35ba5905cfb3c62f3e83b3bec7b39413f0a7f19be4e547ea01ebb"}, - {file = "typed_ast-1.4.1-cp35-cp35m-win32.whl", hash = "sha256:0c2c07682d61a629b68433afb159376e24e5b2fd4641d35424e462169c0a7919"}, - {file = "typed_ast-1.4.1-cp35-cp35m-win_amd64.whl", hash = "sha256:4083861b0aa07990b619bd7ddc365eb7fa4b817e99cf5f8d9cf21a42780f6e01"}, - {file = "typed_ast-1.4.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:269151951236b0f9a6f04015a9004084a5ab0d5f19b57de779f908621e7d8b75"}, - {file = "typed_ast-1.4.1-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:24995c843eb0ad11a4527b026b4dde3da70e1f2d8806c99b7b4a7cf491612652"}, - {file = "typed_ast-1.4.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:fe460b922ec15dd205595c9b5b99e2f056fd98ae8f9f56b888e7a17dc2b757e7"}, - {file = "typed_ast-1.4.1-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:fcf135e17cc74dbfbc05894ebca928ffeb23d9790b3167a674921db19082401f"}, - {file = "typed_ast-1.4.1-cp36-cp36m-win32.whl", hash = "sha256:4e3e5da80ccbebfff202a67bf900d081906c358ccc3d5e3c8aea42fdfdfd51c1"}, - {file = "typed_ast-1.4.1-cp36-cp36m-win_amd64.whl", hash = "sha256:249862707802d40f7f29f6e1aad8d84b5aa9e44552d2cc17384b209f091276aa"}, - {file = "typed_ast-1.4.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8ce678dbaf790dbdb3eba24056d5364fb45944f33553dd5869b7580cdbb83614"}, - {file = "typed_ast-1.4.1-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:c9e348e02e4d2b4a8b2eedb48210430658df6951fa484e59de33ff773fbd4b41"}, - {file = "typed_ast-1.4.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:bcd3b13b56ea479b3650b82cabd6b5343a625b0ced5429e4ccad28a8973f301b"}, - {file = "typed_ast-1.4.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:f208eb7aff048f6bea9586e61af041ddf7f9ade7caed625742af423f6bae3298"}, - {file = "typed_ast-1.4.1-cp37-cp37m-win32.whl", hash = "sha256:d5d33e9e7af3b34a40dc05f498939f0ebf187f07c385fd58d591c533ad8562fe"}, - {file = "typed_ast-1.4.1-cp37-cp37m-win_amd64.whl", hash = "sha256:0666aa36131496aed8f7be0410ff974562ab7eeac11ef351def9ea6fa28f6355"}, - {file = "typed_ast-1.4.1-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:d205b1b46085271b4e15f670058ce182bd1199e56b317bf2ec004b6a44f911f6"}, - {file = "typed_ast-1.4.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:6daac9731f172c2a22ade6ed0c00197ee7cc1221aa84cfdf9c31defeb059a907"}, - {file = "typed_ast-1.4.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:498b0f36cc7054c1fead3d7fc59d2150f4d5c6c56ba7fb150c013fbc683a8d2d"}, - {file = "typed_ast-1.4.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:7e4c9d7658aaa1fc80018593abdf8598bf91325af6af5cce4ce7c73bc45ea53d"}, - {file = "typed_ast-1.4.1-cp38-cp38-win32.whl", hash = "sha256:715ff2f2df46121071622063fc7543d9b1fd19ebfc4f5c8895af64a77a8c852c"}, - {file = "typed_ast-1.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:fc0fea399acb12edbf8a628ba8d2312f583bdbdb3335635db062fa98cf71fca4"}, - {file = "typed_ast-1.4.1-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:d43943ef777f9a1c42bf4e552ba23ac77a6351de620aa9acf64ad54933ad4d34"}, - {file = "typed_ast-1.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:92c325624e304ebf0e025d1224b77dd4e6393f18aab8d829b5b7e04afe9b7a2c"}, - {file = "typed_ast-1.4.1-cp39-cp39-manylinux1_i686.whl", hash = "sha256:d648b8e3bf2fe648745c8ffcee3db3ff903d0817a01a12dd6a6ea7a8f4889072"}, - {file = "typed_ast-1.4.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:fac11badff8313e23717f3dada86a15389d0708275bddf766cca67a84ead3e91"}, - {file = "typed_ast-1.4.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:0d8110d78a5736e16e26213114a38ca35cb15b6515d535413b090bd50951556d"}, - {file = "typed_ast-1.4.1-cp39-cp39-win32.whl", hash = "sha256:b52ccf7cfe4ce2a1064b18594381bccf4179c2ecf7f513134ec2f993dd4ab395"}, - {file = "typed_ast-1.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:3742b32cf1c6ef124d57f95be609c473d7ec4c14d0090e5a5e05a15269fb4d0c"}, - {file = "typed_ast-1.4.1.tar.gz", hash = "sha256:8c8aaad94455178e3187ab22c8b01a3837f8ee50e09cf31f1ba129eb293ec30b"}, + {file = "typed_ast-1.4.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:7703620125e4fb79b64aa52427ec192822e9f45d37d4b6625ab37ef403e1df70"}, + {file = "typed_ast-1.4.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:c9aadc4924d4b5799112837b226160428524a9a45f830e0d0f184b19e4090487"}, + {file = "typed_ast-1.4.2-cp35-cp35m-manylinux2014_aarch64.whl", hash = "sha256:9ec45db0c766f196ae629e509f059ff05fc3148f9ffd28f3cfe75d4afb485412"}, + {file = "typed_ast-1.4.2-cp35-cp35m-win32.whl", hash = "sha256:85f95aa97a35bdb2f2f7d10ec5bbdac0aeb9dafdaf88e17492da0504de2e6400"}, + {file = "typed_ast-1.4.2-cp35-cp35m-win_amd64.whl", hash = "sha256:9044ef2df88d7f33692ae3f18d3be63dec69c4fb1b5a4a9ac950f9b4ba571606"}, + {file = "typed_ast-1.4.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:c1c876fd795b36126f773db9cbb393f19808edd2637e00fd6caba0e25f2c7b64"}, + {file = "typed_ast-1.4.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:5dcfc2e264bd8a1db8b11a892bd1647154ce03eeba94b461effe68790d8b8e07"}, + {file = "typed_ast-1.4.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:8db0e856712f79c45956da0c9a40ca4246abc3485ae0d7ecc86a20f5e4c09abc"}, + {file = "typed_ast-1.4.2-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:d003156bb6a59cda9050e983441b7fa2487f7800d76bdc065566b7d728b4581a"}, + {file = "typed_ast-1.4.2-cp36-cp36m-win32.whl", hash = "sha256:4c790331247081ea7c632a76d5b2a265e6d325ecd3179d06e9cf8d46d90dd151"}, + {file = "typed_ast-1.4.2-cp36-cp36m-win_amd64.whl", hash = "sha256:d175297e9533d8d37437abc14e8a83cbc68af93cc9c1c59c2c292ec59a0697a3"}, + {file = "typed_ast-1.4.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:cf54cfa843f297991b7388c281cb3855d911137223c6b6d2dd82a47ae5125a41"}, + {file = "typed_ast-1.4.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:b4fcdcfa302538f70929eb7b392f536a237cbe2ed9cba88e3bf5027b39f5f77f"}, + {file = "typed_ast-1.4.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:987f15737aba2ab5f3928c617ccf1ce412e2e321c77ab16ca5a293e7bbffd581"}, + {file = "typed_ast-1.4.2-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:37f48d46d733d57cc70fd5f30572d11ab8ed92da6e6b28e024e4a3edfb456e37"}, + {file = "typed_ast-1.4.2-cp37-cp37m-win32.whl", hash = "sha256:36d829b31ab67d6fcb30e185ec996e1f72b892255a745d3a82138c97d21ed1cd"}, + {file = "typed_ast-1.4.2-cp37-cp37m-win_amd64.whl", hash = "sha256:8368f83e93c7156ccd40e49a783a6a6850ca25b556c0fa0240ed0f659d2fe496"}, + {file = "typed_ast-1.4.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:963c80b583b0661918718b095e02303d8078950b26cc00b5e5ea9ababe0de1fc"}, + {file = "typed_ast-1.4.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:e683e409e5c45d5c9082dc1daf13f6374300806240719f95dc783d1fc942af10"}, + {file = "typed_ast-1.4.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:84aa6223d71012c68d577c83f4e7db50d11d6b1399a9c779046d75e24bed74ea"}, + {file = "typed_ast-1.4.2-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:a38878a223bdd37c9709d07cd357bb79f4c760b29210e14ad0fb395294583787"}, + {file = "typed_ast-1.4.2-cp38-cp38-win32.whl", hash = "sha256:a2c927c49f2029291fbabd673d51a2180038f8cd5a5b2f290f78c4516be48be2"}, + {file = "typed_ast-1.4.2-cp38-cp38-win_amd64.whl", hash = "sha256:c0c74e5579af4b977c8b932f40a5464764b2f86681327410aa028a22d2f54937"}, + {file = "typed_ast-1.4.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:07d49388d5bf7e863f7fa2f124b1b1d89d8aa0e2f7812faff0a5658c01c59aa1"}, + {file = "typed_ast-1.4.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:240296b27397e4e37874abb1df2a608a92df85cf3e2a04d0d4d61055c8305ba6"}, + {file = "typed_ast-1.4.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:d746a437cdbca200622385305aedd9aef68e8a645e385cc483bdc5e488f07166"}, + {file = "typed_ast-1.4.2-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:14bf1522cdee369e8f5581238edac09150c765ec1cb33615855889cf33dcb92d"}, + {file = "typed_ast-1.4.2-cp39-cp39-win32.whl", hash = "sha256:cc7b98bf58167b7f2db91a4327da24fb93368838eb84a44c472283778fc2446b"}, + {file = "typed_ast-1.4.2-cp39-cp39-win_amd64.whl", hash = "sha256:7147e2a76c75f0f64c4319886e7639e490fee87c9d25cb1d4faef1d8cf83a440"}, + {file = "typed_ast-1.4.2.tar.gz", hash = "sha256:9fc0b3cb5d1720e7141d103cf4819aea239f7d136acf9ee4a69b047b7986175a"}, ] typing-extensions = [ {file = "typing_extensions-3.7.4.3-py2-none-any.whl", hash = "sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f"}, diff --git a/pytest-lisa/poetry.lock b/pytest-lisa/poetry.lock index 3072594dd5..2517d1b1bd 100644 --- a/pytest-lisa/poetry.lock +++ b/pytest-lisa/poetry.lock @@ -73,14 +73,14 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "cryptography" -version = "3.2.1" +version = "3.3.1" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." category = "main" optional = false -python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*" +python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*" [package.dependencies] -cffi = ">=1.8,<1.11.3 || >1.11.3" +cffi = ">=1.12" six = ">=1.4.1" [package.extras] @@ -130,13 +130,14 @@ python-versions = "*" [[package]] name = "importlib-metadata" -version = "3.1.1" +version = "3.3.0" description = "Read metadata from Python packages" category = "main" optional = false python-versions = ">=3.6" [package.dependencies] +typing-extensions = {version = ">=3.6.4", markers = "python_version < \"3.8\""} zipp = ">=0.5" [package.extras] @@ -153,7 +154,7 @@ python-versions = "*" [[package]] name = "invoke" -version = "1.4.1" +version = "1.5.0" description = "Pythonic task execution" category = "main" optional = false @@ -161,7 +162,7 @@ python-versions = "*" [[package]] name = "packaging" -version = "20.7" +version = "20.8" description = "Core utilities for Python packages" category = "main" optional = false @@ -205,7 +206,7 @@ dev = ["pre-commit", "tox"] [[package]] name = "py" -version = "1.9.0" +version = "1.10.0" description = "library with cross-python path, ini-parsing, io, code, log facilities" category = "main" optional = false @@ -245,25 +246,24 @@ python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" [[package]] name = "pytest" -version = "6.1.2" +version = "6.2.1" description = "pytest: simple powerful testing with Python" category = "main" optional = false -python-versions = ">=3.5" +python-versions = ">=3.6" [package.dependencies] atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} -attrs = ">=17.4.0" +attrs = ">=19.2.0" colorama = {version = "*", markers = "sys_platform == \"win32\""} importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} iniconfig = "*" packaging = "*" -pluggy = ">=0.12,<1.0" +pluggy = ">=0.12,<1.0.0a1" py = ">=1.8.2" toml = "*" [package.extras] -checkqa_mypy = ["mypy (==0.780)"] testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] [[package]] @@ -319,7 +319,7 @@ url = "../pytest-target" [[package]] name = "pytest-xdist" -version = "2.1.0" +version = "2.2.0" description = "pytest xdist plugin for distributed testing and loop-on-failing modes" category = "main" optional = false @@ -363,7 +363,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" [[package]] name = "tenacity" -version = "6.2.0" +version = "6.3.1" description = "Retry code until it succeeds" category = "main" optional = false @@ -383,6 +383,14 @@ category = "main" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +[[package]] +name = "typing-extensions" +version = "3.7.4.3" +description = "Backported and Experimental Type Hints for Python 3.5+" +category = "main" +optional = false +python-versions = "*" + [[package]] name = "zipp" version = "3.4.0" @@ -469,28 +477,20 @@ contextlib2 = [ {file = "contextlib2-0.6.0.post1.tar.gz", hash = "sha256:01f490098c18b19d2bd5bb5dc445b2054d2fa97f09a4280ba2c5f3c394c8162e"}, ] cryptography = [ - {file = "cryptography-3.2.1-cp27-cp27m-macosx_10_10_x86_64.whl", hash = "sha256:6dc59630ecce8c1f558277ceb212c751d6730bd12c80ea96b4ac65637c4f55e7"}, - {file = "cryptography-3.2.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:75e8e6684cf0034f6bf2a97095cb95f81537b12b36a8fedf06e73050bb171c2d"}, - {file = "cryptography-3.2.1-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:4e7268a0ca14536fecfdf2b00297d4e407da904718658c1ff1961c713f90fd33"}, - {file = "cryptography-3.2.1-cp27-cp27m-win32.whl", hash = "sha256:7117319b44ed1842c617d0a452383a5a052ec6aa726dfbaffa8b94c910444297"}, - {file = "cryptography-3.2.1-cp27-cp27m-win_amd64.whl", hash = "sha256:a733671100cd26d816eed39507e585c156e4498293a907029969234e5e634bc4"}, - {file = "cryptography-3.2.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:a75f306a16d9f9afebfbedc41c8c2351d8e61e818ba6b4c40815e2b5740bb6b8"}, - {file = "cryptography-3.2.1-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:5849d59358547bf789ee7e0d7a9036b2d29e9a4ddf1ce5e06bb45634f995c53e"}, - {file = "cryptography-3.2.1-cp35-abi3-macosx_10_10_x86_64.whl", hash = "sha256:bd717aa029217b8ef94a7d21632a3bb5a4e7218a4513d2521c2a2fd63011e98b"}, - {file = "cryptography-3.2.1-cp35-abi3-manylinux1_x86_64.whl", hash = "sha256:efe15aca4f64f3a7ea0c09c87826490e50ed166ce67368a68f315ea0807a20df"}, - {file = "cryptography-3.2.1-cp35-abi3-manylinux2010_x86_64.whl", hash = "sha256:32434673d8505b42c0de4de86da8c1620651abd24afe91ae0335597683ed1b77"}, - {file = "cryptography-3.2.1-cp35-abi3-manylinux2014_aarch64.whl", hash = "sha256:7b8d9d8d3a9bd240f453342981f765346c87ade811519f98664519696f8e6ab7"}, - {file = "cryptography-3.2.1-cp35-cp35m-win32.whl", hash = "sha256:d3545829ab42a66b84a9aaabf216a4dce7f16dbc76eb69be5c302ed6b8f4a29b"}, - {file = "cryptography-3.2.1-cp35-cp35m-win_amd64.whl", hash = "sha256:a4e27ed0b2504195f855b52052eadcc9795c59909c9d84314c5408687f933fc7"}, - {file = "cryptography-3.2.1-cp36-abi3-win32.whl", hash = "sha256:13b88a0bd044b4eae1ef40e265d006e34dbcde0c2f1e15eb9896501b2d8f6c6f"}, - {file = "cryptography-3.2.1-cp36-abi3-win_amd64.whl", hash = "sha256:07ca431b788249af92764e3be9a488aa1d39a0bc3be313d826bbec690417e538"}, - {file = "cryptography-3.2.1-cp36-cp36m-win32.whl", hash = "sha256:a035a10686532b0587d58a606004aa20ad895c60c4d029afa245802347fab57b"}, - {file = "cryptography-3.2.1-cp36-cp36m-win_amd64.whl", hash = "sha256:d26a2557d8f9122f9bf445fc7034242f4375bd4e95ecda007667540270965b13"}, - {file = "cryptography-3.2.1-cp37-cp37m-win32.whl", hash = "sha256:545a8550782dda68f8cdc75a6e3bf252017aa8f75f19f5a9ca940772fc0cb56e"}, - {file = "cryptography-3.2.1-cp37-cp37m-win_amd64.whl", hash = "sha256:55d0b896631412b6f0c7de56e12eb3e261ac347fbaa5d5e705291a9016e5f8cb"}, - {file = "cryptography-3.2.1-cp38-cp38-win32.whl", hash = "sha256:3cd75a683b15576cfc822c7c5742b3276e50b21a06672dc3a800a2d5da4ecd1b"}, - {file = "cryptography-3.2.1-cp38-cp38-win_amd64.whl", hash = "sha256:d25cecbac20713a7c3bc544372d42d8eafa89799f492a43b79e1dfd650484851"}, - {file = "cryptography-3.2.1.tar.gz", hash = "sha256:d3d5e10be0cf2a12214ddee45c6bd203dab435e3d83b4560c03066eda600bfe3"}, + {file = "cryptography-3.3.1-cp27-cp27m-macosx_10_10_x86_64.whl", hash = "sha256:c366df0401d1ec4e548bebe8f91d55ebcc0ec3137900d214dd7aac8427ef3030"}, + {file = "cryptography-3.3.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:9f6b0492d111b43de5f70052e24c1f0951cb9e6022188ebcb1cc3a3d301469b0"}, + {file = "cryptography-3.3.1-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:a69bd3c68b98298f490e84519b954335154917eaab52cf582fa2c5c7efc6e812"}, + {file = "cryptography-3.3.1-cp27-cp27m-win32.whl", hash = "sha256:84ef7a0c10c24a7773163f917f1cb6b4444597efd505a8aed0a22e8c4780f27e"}, + {file = "cryptography-3.3.1-cp27-cp27m-win_amd64.whl", hash = "sha256:594a1db4511bc4d960571536abe21b4e5c3003e8750ab8365fafce71c5d86901"}, + {file = "cryptography-3.3.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:0003a52a123602e1acee177dc90dd201f9bb1e73f24a070db7d36c588e8f5c7d"}, + {file = "cryptography-3.3.1-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:83d9d2dfec70364a74f4e7c70ad04d3ca2e6a08b703606993407bf46b97868c5"}, + {file = "cryptography-3.3.1-cp36-abi3-macosx_10_10_x86_64.whl", hash = "sha256:dc42f645f8f3a489c3dd416730a514e7a91a59510ddaadc09d04224c098d3302"}, + {file = "cryptography-3.3.1-cp36-abi3-manylinux1_x86_64.whl", hash = "sha256:788a3c9942df5e4371c199d10383f44a105d67d401fb4304178020142f020244"}, + {file = "cryptography-3.3.1-cp36-abi3-manylinux2010_x86_64.whl", hash = "sha256:69e836c9e5ff4373ce6d3ab311c1a2eed274793083858d3cd4c7d12ce20d5f9c"}, + {file = "cryptography-3.3.1-cp36-abi3-manylinux2014_aarch64.whl", hash = "sha256:9e21301f7a1e7c03dbea73e8602905a4ebba641547a462b26dd03451e5769e7c"}, + {file = "cryptography-3.3.1-cp36-abi3-win32.whl", hash = "sha256:b4890d5fb9b7a23e3bf8abf5a8a7da8e228f1e97dc96b30b95685df840b6914a"}, + {file = "cryptography-3.3.1-cp36-abi3-win_amd64.whl", hash = "sha256:0e85aaae861d0485eb5a79d33226dd6248d2a9f133b81532c8f5aae37de10ff7"}, + {file = "cryptography-3.3.1.tar.gz", hash = "sha256:7e177e4bea2de937a584b13645cab32f25e3d96fc0bc4a4cf99c27dc77682be6"}, ] execnet = [ {file = "execnet-1.7.1-py2.py3-none-any.whl", hash = "sha256:d4efd397930c46415f62f8a31388d6be4f27a91d7550eb79bc64a756e0056547"}, @@ -505,21 +505,21 @@ filelock = [ {file = "filelock-3.0.12.tar.gz", hash = "sha256:18d82244ee114f543149c66a6e0c14e9c4f8a1044b5cdaadd0f82159d6a6ff59"}, ] importlib-metadata = [ - {file = "importlib_metadata-3.1.1-py3-none-any.whl", hash = "sha256:6112e21359ef8f344e7178aa5b72dc6e62b38b0d008e6d3cb212c5b84df72013"}, - {file = "importlib_metadata-3.1.1.tar.gz", hash = "sha256:b0c2d3b226157ae4517d9625decf63591461c66b3a808c2666d538946519d170"}, + {file = "importlib_metadata-3.3.0-py3-none-any.whl", hash = "sha256:bf792d480abbd5eda85794e4afb09dd538393f7d6e6ffef6e9f03d2014cf9450"}, + {file = "importlib_metadata-3.3.0.tar.gz", hash = "sha256:5c5a2720817414a6c41f0a49993908068243ae02c1635a228126519b509c8aed"}, ] iniconfig = [ {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, ] invoke = [ - {file = "invoke-1.4.1-py2-none-any.whl", hash = "sha256:93e12876d88130c8e0d7fd6618dd5387d6b36da55ad541481dfa5e001656f134"}, - {file = "invoke-1.4.1-py3-none-any.whl", hash = "sha256:87b3ef9d72a1667e104f89b159eaf8a514dbf2f3576885b2bbdefe74c3fb2132"}, - {file = "invoke-1.4.1.tar.gz", hash = "sha256:de3f23bfe669e3db1085789fd859eb8ca8e0c5d9c20811e2407fa042e8a5e15d"}, + {file = "invoke-1.5.0-py2-none-any.whl", hash = "sha256:da7c2d0be71be83ffd6337e078ef9643f41240024d6b2659e7b46e0b251e339f"}, + {file = "invoke-1.5.0-py3-none-any.whl", hash = "sha256:7e44d98a7dc00c91c79bac9e3007276965d2c96884b3c22077a9f04042bd6d90"}, + {file = "invoke-1.5.0.tar.gz", hash = "sha256:f0c560075b5fb29ba14dad44a7185514e94970d1b9d57dcd3723bec5fed92650"}, ] packaging = [ - {file = "packaging-20.7-py2.py3-none-any.whl", hash = "sha256:eb41423378682dadb7166144a4926e443093863024de508ca5c9737d6bc08376"}, - {file = "packaging-20.7.tar.gz", hash = "sha256:05af3bb85d320377db281cf254ab050e1a7ebcbf5410685a9a407e18a1f81236"}, + {file = "packaging-20.8-py2.py3-none-any.whl", hash = "sha256:24e0da08660a87484d1602c30bb4902d74816b6985b93de36926f5bc95741858"}, + {file = "packaging-20.8.tar.gz", hash = "sha256:78598185a7008a470d64526a8059de9aaa449238f280fc9eb6b13ba6c4109093"}, ] paramiko = [ {file = "paramiko-2.7.2-py2.py3-none-any.whl", hash = "sha256:4f3e316fef2ac628b05097a637af35685183111d4bc1b5979bd397c2ab7b5898"}, @@ -530,8 +530,8 @@ pluggy = [ {file = "pluggy-0.13.1.tar.gz", hash = "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0"}, ] py = [ - {file = "py-1.9.0-py2.py3-none-any.whl", hash = "sha256:366389d1db726cd2fcfc79732e75410e5fe4d31db13692115529d34069a043c2"}, - {file = "py-1.9.0.tar.gz", hash = "sha256:9ca6883ce56b4e8da7e79ac18787889fa5206c79dcc67fb065376cd2fe03f342"}, + {file = "py-1.10.0-py2.py3-none-any.whl", hash = "sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a"}, + {file = "py-1.10.0.tar.gz", hash = "sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3"}, ] pycparser = [ {file = "pycparser-2.20-py2.py3-none-any.whl", hash = "sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705"}, @@ -562,8 +562,8 @@ pyparsing = [ {file = "pyparsing-2.4.7.tar.gz", hash = "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1"}, ] pytest = [ - {file = "pytest-6.1.2-py3-none-any.whl", hash = "sha256:4288fed0d9153d9646bfcdf0c0428197dba1ecb27a33bb6e031d002fa88653fe"}, - {file = "pytest-6.1.2.tar.gz", hash = "sha256:c0a7e94a8cdbc5422a51ccdad8e6f1024795939cc89159a0ae7f0b316ad3823e"}, + {file = "pytest-6.2.1-py3-none-any.whl", hash = "sha256:1969f797a1a0dbd8ccf0fecc80262312729afea9c17f1d70ebf85c5e76c6f7c8"}, + {file = "pytest-6.2.1.tar.gz", hash = "sha256:66e419b1899bc27346cb2c993e12c5e5e8daba9073c1fbce33b9807abc95c306"}, ] pytest-forked = [ {file = "pytest-forked-1.3.0.tar.gz", hash = "sha256:6aa9ac7e00ad1a539c41bec6d21011332de671e938c7637378ec9710204e37ca"}, @@ -572,8 +572,8 @@ pytest-forked = [ pytest-playbook = [] pytest-target = [] pytest-xdist = [ - {file = "pytest-xdist-2.1.0.tar.gz", hash = "sha256:82d938f1a24186520e2d9d3a64ef7d9ac7ecdf1a0659e095d18e596b8cbd0672"}, - {file = "pytest_xdist-2.1.0-py3-none-any.whl", hash = "sha256:7c629016b3bb006b88ac68e2b31551e7becf173c76b977768848e2bbed594d90"}, + {file = "pytest-xdist-2.2.0.tar.gz", hash = "sha256:1d8edbb1a45e8e1f8e44b1260583107fc23f8bc8da6d18cb331ff61d41258ecf"}, + {file = "pytest_xdist-2.2.0-py3-none-any.whl", hash = "sha256:f127e11e84ad37cc1de1088cb2990f3c354630d428af3f71282de589c5bb779b"}, ] pyyaml = [ {file = "PyYAML-5.3.1-cp27-cp27m-win32.whl", hash = "sha256:74809a57b329d6cc0fdccee6318f44b9b8649961fa73144a98735b0aaf029f1f"}, @@ -599,13 +599,18 @@ six = [ {file = "six-1.15.0.tar.gz", hash = "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259"}, ] tenacity = [ - {file = "tenacity-6.2.0-py2.py3-none-any.whl", hash = "sha256:5a5d3dcd46381abe8b4f82b5736b8726fd3160c6c7161f53f8af7f1eb9b82173"}, - {file = "tenacity-6.2.0.tar.gz", hash = "sha256:29ae90e7faf488a8628432154bb34ace1cca58244c6ea399fd33f066ac71339a"}, + {file = "tenacity-6.3.1-py2.py3-none-any.whl", hash = "sha256:baed357d9f35ec64264d8a4bbf004c35058fad8795c5b0d8a7dc77ecdcbb8f39"}, + {file = "tenacity-6.3.1.tar.gz", hash = "sha256:e14d191fb0a309b563904bbc336582efe2037de437e543b38da749769b544d7f"}, ] toml = [ {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, ] +typing-extensions = [ + {file = "typing_extensions-3.7.4.3-py2-none-any.whl", hash = "sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f"}, + {file = "typing_extensions-3.7.4.3-py3-none-any.whl", hash = "sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918"}, + {file = "typing_extensions-3.7.4.3.tar.gz", hash = "sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c"}, +] zipp = [ {file = "zipp-3.4.0-py3-none-any.whl", hash = "sha256:102c24ef8f171fd729d46599845e95c7ab894a4cf45f5de11a44cc7444fb1108"}, {file = "zipp-3.4.0.tar.gz", hash = "sha256:ed5eee1974372595f9e416cc7bbeeb12335201d8081ca8a0743c954d4446e5cb"}, diff --git a/pytest-playbook/poetry.lock b/pytest-playbook/poetry.lock index c352a6fd10..597abb748e 100644 --- a/pytest-playbook/poetry.lock +++ b/pytest-playbook/poetry.lock @@ -38,13 +38,14 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "importlib-metadata" -version = "3.1.1" +version = "3.3.0" description = "Read metadata from Python packages" category = "main" optional = false python-versions = ">=3.6" [package.dependencies] +typing-extensions = {version = ">=3.6.4", markers = "python_version < \"3.8\""} zipp = ">=0.5" [package.extras] @@ -61,7 +62,7 @@ python-versions = "*" [[package]] name = "packaging" -version = "20.7" +version = "20.8" description = "Core utilities for Python packages" category = "main" optional = false @@ -86,7 +87,7 @@ dev = ["pre-commit", "tox"] [[package]] name = "py" -version = "1.9.0" +version = "1.10.0" description = "library with cross-python path, ini-parsing, io, code, log facilities" category = "main" optional = false @@ -102,25 +103,24 @@ python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" [[package]] name = "pytest" -version = "6.1.2" +version = "6.2.1" description = "pytest: simple powerful testing with Python" category = "main" optional = false -python-versions = ">=3.5" +python-versions = ">=3.6" [package.dependencies] atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} -attrs = ">=17.4.0" +attrs = ">=19.2.0" colorama = {version = "*", markers = "sys_platform == \"win32\""} importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} iniconfig = "*" packaging = "*" -pluggy = ">=0.12,<1.0" +pluggy = ">=0.12,<1.0.0a1" py = ">=1.8.2" toml = "*" [package.extras] -checkqa_mypy = ["mypy (==0.780)"] testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] [[package]] @@ -150,6 +150,14 @@ category = "main" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +[[package]] +name = "typing-extensions" +version = "3.7.4.3" +description = "Backported and Experimental Type Hints for Python 3.5+" +category = "main" +optional = false +python-versions = "*" + [[package]] name = "zipp" version = "3.4.0" @@ -185,32 +193,32 @@ contextlib2 = [ {file = "contextlib2-0.6.0.post1.tar.gz", hash = "sha256:01f490098c18b19d2bd5bb5dc445b2054d2fa97f09a4280ba2c5f3c394c8162e"}, ] importlib-metadata = [ - {file = "importlib_metadata-3.1.1-py3-none-any.whl", hash = "sha256:6112e21359ef8f344e7178aa5b72dc6e62b38b0d008e6d3cb212c5b84df72013"}, - {file = "importlib_metadata-3.1.1.tar.gz", hash = "sha256:b0c2d3b226157ae4517d9625decf63591461c66b3a808c2666d538946519d170"}, + {file = "importlib_metadata-3.3.0-py3-none-any.whl", hash = "sha256:bf792d480abbd5eda85794e4afb09dd538393f7d6e6ffef6e9f03d2014cf9450"}, + {file = "importlib_metadata-3.3.0.tar.gz", hash = "sha256:5c5a2720817414a6c41f0a49993908068243ae02c1635a228126519b509c8aed"}, ] iniconfig = [ {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, ] packaging = [ - {file = "packaging-20.7-py2.py3-none-any.whl", hash = "sha256:eb41423378682dadb7166144a4926e443093863024de508ca5c9737d6bc08376"}, - {file = "packaging-20.7.tar.gz", hash = "sha256:05af3bb85d320377db281cf254ab050e1a7ebcbf5410685a9a407e18a1f81236"}, + {file = "packaging-20.8-py2.py3-none-any.whl", hash = "sha256:24e0da08660a87484d1602c30bb4902d74816b6985b93de36926f5bc95741858"}, + {file = "packaging-20.8.tar.gz", hash = "sha256:78598185a7008a470d64526a8059de9aaa449238f280fc9eb6b13ba6c4109093"}, ] pluggy = [ {file = "pluggy-0.13.1-py2.py3-none-any.whl", hash = "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"}, {file = "pluggy-0.13.1.tar.gz", hash = "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0"}, ] py = [ - {file = "py-1.9.0-py2.py3-none-any.whl", hash = "sha256:366389d1db726cd2fcfc79732e75410e5fe4d31db13692115529d34069a043c2"}, - {file = "py-1.9.0.tar.gz", hash = "sha256:9ca6883ce56b4e8da7e79ac18787889fa5206c79dcc67fb065376cd2fe03f342"}, + {file = "py-1.10.0-py2.py3-none-any.whl", hash = "sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a"}, + {file = "py-1.10.0.tar.gz", hash = "sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3"}, ] pyparsing = [ {file = "pyparsing-2.4.7-py2.py3-none-any.whl", hash = "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"}, {file = "pyparsing-2.4.7.tar.gz", hash = "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1"}, ] pytest = [ - {file = "pytest-6.1.2-py3-none-any.whl", hash = "sha256:4288fed0d9153d9646bfcdf0c0428197dba1ecb27a33bb6e031d002fa88653fe"}, - {file = "pytest-6.1.2.tar.gz", hash = "sha256:c0a7e94a8cdbc5422a51ccdad8e6f1024795939cc89159a0ae7f0b316ad3823e"}, + {file = "pytest-6.2.1-py3-none-any.whl", hash = "sha256:1969f797a1a0dbd8ccf0fecc80262312729afea9c17f1d70ebf85c5e76c6f7c8"}, + {file = "pytest-6.2.1.tar.gz", hash = "sha256:66e419b1899bc27346cb2c993e12c5e5e8daba9073c1fbce33b9807abc95c306"}, ] pyyaml = [ {file = "PyYAML-5.3.1-cp27-cp27m-win32.whl", hash = "sha256:74809a57b329d6cc0fdccee6318f44b9b8649961fa73144a98735b0aaf029f1f"}, @@ -235,6 +243,11 @@ toml = [ {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, ] +typing-extensions = [ + {file = "typing_extensions-3.7.4.3-py2-none-any.whl", hash = "sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f"}, + {file = "typing_extensions-3.7.4.3-py3-none-any.whl", hash = "sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918"}, + {file = "typing_extensions-3.7.4.3.tar.gz", hash = "sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c"}, +] zipp = [ {file = "zipp-3.4.0-py3-none-any.whl", hash = "sha256:102c24ef8f171fd729d46599845e95c7ab894a4cf45f5de11a44cc7444fb1108"}, {file = "zipp-3.4.0.tar.gz", hash = "sha256:ed5eee1974372595f9e416cc7bbeeb12335201d8081ca8a0743c954d4446e5cb"}, diff --git a/pytest-target/poetry.lock b/pytest-target/poetry.lock index 74a761b516..2412ffdd21 100644 --- a/pytest-target/poetry.lock +++ b/pytest-target/poetry.lock @@ -65,14 +65,14 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "cryptography" -version = "3.2.1" +version = "3.3.1" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." category = "main" optional = false -python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*" +python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*" [package.dependencies] -cffi = ">=1.8,<1.11.3 || >1.11.3" +cffi = ">=1.12" six = ">=1.4.1" [package.extras] @@ -108,13 +108,14 @@ python-versions = "*" [[package]] name = "importlib-metadata" -version = "3.1.1" +version = "3.3.0" description = "Read metadata from Python packages" category = "main" optional = false python-versions = ">=3.6" [package.dependencies] +typing-extensions = {version = ">=3.6.4", markers = "python_version < \"3.8\""} zipp = ">=0.5" [package.extras] @@ -131,7 +132,7 @@ python-versions = "*" [[package]] name = "invoke" -version = "1.4.1" +version = "1.5.0" description = "Pythonic task execution" category = "main" optional = false @@ -139,7 +140,7 @@ python-versions = "*" [[package]] name = "packaging" -version = "20.7" +version = "20.8" description = "Core utilities for Python packages" category = "main" optional = false @@ -183,7 +184,7 @@ dev = ["pre-commit", "tox"] [[package]] name = "py" -version = "1.9.0" +version = "1.10.0" description = "library with cross-python path, ini-parsing, io, code, log facilities" category = "main" optional = false @@ -223,25 +224,24 @@ python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" [[package]] name = "pytest" -version = "6.1.2" +version = "6.2.1" description = "pytest: simple powerful testing with Python" category = "main" optional = false -python-versions = ">=3.5" +python-versions = ">=3.6" [package.dependencies] atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} -attrs = ">=17.4.0" +attrs = ">=19.2.0" colorama = {version = "*", markers = "sys_platform == \"win32\""} importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} iniconfig = "*" packaging = "*" -pluggy = ">=0.12,<1.0" +pluggy = ">=0.12,<1.0.0a1" py = ">=1.8.2" toml = "*" [package.extras] -checkqa_mypy = ["mypy (==0.780)"] testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] [[package]] @@ -291,7 +291,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" [[package]] name = "tenacity" -version = "6.2.0" +version = "6.3.1" description = "Retry code until it succeeds" category = "main" optional = false @@ -311,6 +311,14 @@ category = "main" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +[[package]] +name = "typing-extensions" +version = "3.7.4.3" +description = "Backported and Experimental Type Hints for Python 3.5+" +category = "main" +optional = false +python-versions = "*" + [[package]] name = "zipp" version = "3.4.0" @@ -393,28 +401,20 @@ contextlib2 = [ {file = "contextlib2-0.6.0.post1.tar.gz", hash = "sha256:01f490098c18b19d2bd5bb5dc445b2054d2fa97f09a4280ba2c5f3c394c8162e"}, ] cryptography = [ - {file = "cryptography-3.2.1-cp27-cp27m-macosx_10_10_x86_64.whl", hash = "sha256:6dc59630ecce8c1f558277ceb212c751d6730bd12c80ea96b4ac65637c4f55e7"}, - {file = "cryptography-3.2.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:75e8e6684cf0034f6bf2a97095cb95f81537b12b36a8fedf06e73050bb171c2d"}, - {file = "cryptography-3.2.1-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:4e7268a0ca14536fecfdf2b00297d4e407da904718658c1ff1961c713f90fd33"}, - {file = "cryptography-3.2.1-cp27-cp27m-win32.whl", hash = "sha256:7117319b44ed1842c617d0a452383a5a052ec6aa726dfbaffa8b94c910444297"}, - {file = "cryptography-3.2.1-cp27-cp27m-win_amd64.whl", hash = "sha256:a733671100cd26d816eed39507e585c156e4498293a907029969234e5e634bc4"}, - {file = "cryptography-3.2.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:a75f306a16d9f9afebfbedc41c8c2351d8e61e818ba6b4c40815e2b5740bb6b8"}, - {file = "cryptography-3.2.1-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:5849d59358547bf789ee7e0d7a9036b2d29e9a4ddf1ce5e06bb45634f995c53e"}, - {file = "cryptography-3.2.1-cp35-abi3-macosx_10_10_x86_64.whl", hash = "sha256:bd717aa029217b8ef94a7d21632a3bb5a4e7218a4513d2521c2a2fd63011e98b"}, - {file = "cryptography-3.2.1-cp35-abi3-manylinux1_x86_64.whl", hash = "sha256:efe15aca4f64f3a7ea0c09c87826490e50ed166ce67368a68f315ea0807a20df"}, - {file = "cryptography-3.2.1-cp35-abi3-manylinux2010_x86_64.whl", hash = "sha256:32434673d8505b42c0de4de86da8c1620651abd24afe91ae0335597683ed1b77"}, - {file = "cryptography-3.2.1-cp35-abi3-manylinux2014_aarch64.whl", hash = "sha256:7b8d9d8d3a9bd240f453342981f765346c87ade811519f98664519696f8e6ab7"}, - {file = "cryptography-3.2.1-cp35-cp35m-win32.whl", hash = "sha256:d3545829ab42a66b84a9aaabf216a4dce7f16dbc76eb69be5c302ed6b8f4a29b"}, - {file = "cryptography-3.2.1-cp35-cp35m-win_amd64.whl", hash = "sha256:a4e27ed0b2504195f855b52052eadcc9795c59909c9d84314c5408687f933fc7"}, - {file = "cryptography-3.2.1-cp36-abi3-win32.whl", hash = "sha256:13b88a0bd044b4eae1ef40e265d006e34dbcde0c2f1e15eb9896501b2d8f6c6f"}, - {file = "cryptography-3.2.1-cp36-abi3-win_amd64.whl", hash = "sha256:07ca431b788249af92764e3be9a488aa1d39a0bc3be313d826bbec690417e538"}, - {file = "cryptography-3.2.1-cp36-cp36m-win32.whl", hash = "sha256:a035a10686532b0587d58a606004aa20ad895c60c4d029afa245802347fab57b"}, - {file = "cryptography-3.2.1-cp36-cp36m-win_amd64.whl", hash = "sha256:d26a2557d8f9122f9bf445fc7034242f4375bd4e95ecda007667540270965b13"}, - {file = "cryptography-3.2.1-cp37-cp37m-win32.whl", hash = "sha256:545a8550782dda68f8cdc75a6e3bf252017aa8f75f19f5a9ca940772fc0cb56e"}, - {file = "cryptography-3.2.1-cp37-cp37m-win_amd64.whl", hash = "sha256:55d0b896631412b6f0c7de56e12eb3e261ac347fbaa5d5e705291a9016e5f8cb"}, - {file = "cryptography-3.2.1-cp38-cp38-win32.whl", hash = "sha256:3cd75a683b15576cfc822c7c5742b3276e50b21a06672dc3a800a2d5da4ecd1b"}, - {file = "cryptography-3.2.1-cp38-cp38-win_amd64.whl", hash = "sha256:d25cecbac20713a7c3bc544372d42d8eafa89799f492a43b79e1dfd650484851"}, - {file = "cryptography-3.2.1.tar.gz", hash = "sha256:d3d5e10be0cf2a12214ddee45c6bd203dab435e3d83b4560c03066eda600bfe3"}, + {file = "cryptography-3.3.1-cp27-cp27m-macosx_10_10_x86_64.whl", hash = "sha256:c366df0401d1ec4e548bebe8f91d55ebcc0ec3137900d214dd7aac8427ef3030"}, + {file = "cryptography-3.3.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:9f6b0492d111b43de5f70052e24c1f0951cb9e6022188ebcb1cc3a3d301469b0"}, + {file = "cryptography-3.3.1-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:a69bd3c68b98298f490e84519b954335154917eaab52cf582fa2c5c7efc6e812"}, + {file = "cryptography-3.3.1-cp27-cp27m-win32.whl", hash = "sha256:84ef7a0c10c24a7773163f917f1cb6b4444597efd505a8aed0a22e8c4780f27e"}, + {file = "cryptography-3.3.1-cp27-cp27m-win_amd64.whl", hash = "sha256:594a1db4511bc4d960571536abe21b4e5c3003e8750ab8365fafce71c5d86901"}, + {file = "cryptography-3.3.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:0003a52a123602e1acee177dc90dd201f9bb1e73f24a070db7d36c588e8f5c7d"}, + {file = "cryptography-3.3.1-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:83d9d2dfec70364a74f4e7c70ad04d3ca2e6a08b703606993407bf46b97868c5"}, + {file = "cryptography-3.3.1-cp36-abi3-macosx_10_10_x86_64.whl", hash = "sha256:dc42f645f8f3a489c3dd416730a514e7a91a59510ddaadc09d04224c098d3302"}, + {file = "cryptography-3.3.1-cp36-abi3-manylinux1_x86_64.whl", hash = "sha256:788a3c9942df5e4371c199d10383f44a105d67d401fb4304178020142f020244"}, + {file = "cryptography-3.3.1-cp36-abi3-manylinux2010_x86_64.whl", hash = "sha256:69e836c9e5ff4373ce6d3ab311c1a2eed274793083858d3cd4c7d12ce20d5f9c"}, + {file = "cryptography-3.3.1-cp36-abi3-manylinux2014_aarch64.whl", hash = "sha256:9e21301f7a1e7c03dbea73e8602905a4ebba641547a462b26dd03451e5769e7c"}, + {file = "cryptography-3.3.1-cp36-abi3-win32.whl", hash = "sha256:b4890d5fb9b7a23e3bf8abf5a8a7da8e228f1e97dc96b30b95685df840b6914a"}, + {file = "cryptography-3.3.1-cp36-abi3-win_amd64.whl", hash = "sha256:0e85aaae861d0485eb5a79d33226dd6248d2a9f133b81532c8f5aae37de10ff7"}, + {file = "cryptography-3.3.1.tar.gz", hash = "sha256:7e177e4bea2de937a584b13645cab32f25e3d96fc0bc4a4cf99c27dc77682be6"}, ] fabric = [ {file = "fabric-2.5.0-py2.py3-none-any.whl", hash = "sha256:160331934ea60036604928e792fa8e9f813266b098ef5562aa82b88527740389"}, @@ -425,21 +425,21 @@ filelock = [ {file = "filelock-3.0.12.tar.gz", hash = "sha256:18d82244ee114f543149c66a6e0c14e9c4f8a1044b5cdaadd0f82159d6a6ff59"}, ] importlib-metadata = [ - {file = "importlib_metadata-3.1.1-py3-none-any.whl", hash = "sha256:6112e21359ef8f344e7178aa5b72dc6e62b38b0d008e6d3cb212c5b84df72013"}, - {file = "importlib_metadata-3.1.1.tar.gz", hash = "sha256:b0c2d3b226157ae4517d9625decf63591461c66b3a808c2666d538946519d170"}, + {file = "importlib_metadata-3.3.0-py3-none-any.whl", hash = "sha256:bf792d480abbd5eda85794e4afb09dd538393f7d6e6ffef6e9f03d2014cf9450"}, + {file = "importlib_metadata-3.3.0.tar.gz", hash = "sha256:5c5a2720817414a6c41f0a49993908068243ae02c1635a228126519b509c8aed"}, ] iniconfig = [ {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, ] invoke = [ - {file = "invoke-1.4.1-py2-none-any.whl", hash = "sha256:93e12876d88130c8e0d7fd6618dd5387d6b36da55ad541481dfa5e001656f134"}, - {file = "invoke-1.4.1-py3-none-any.whl", hash = "sha256:87b3ef9d72a1667e104f89b159eaf8a514dbf2f3576885b2bbdefe74c3fb2132"}, - {file = "invoke-1.4.1.tar.gz", hash = "sha256:de3f23bfe669e3db1085789fd859eb8ca8e0c5d9c20811e2407fa042e8a5e15d"}, + {file = "invoke-1.5.0-py2-none-any.whl", hash = "sha256:da7c2d0be71be83ffd6337e078ef9643f41240024d6b2659e7b46e0b251e339f"}, + {file = "invoke-1.5.0-py3-none-any.whl", hash = "sha256:7e44d98a7dc00c91c79bac9e3007276965d2c96884b3c22077a9f04042bd6d90"}, + {file = "invoke-1.5.0.tar.gz", hash = "sha256:f0c560075b5fb29ba14dad44a7185514e94970d1b9d57dcd3723bec5fed92650"}, ] packaging = [ - {file = "packaging-20.7-py2.py3-none-any.whl", hash = "sha256:eb41423378682dadb7166144a4926e443093863024de508ca5c9737d6bc08376"}, - {file = "packaging-20.7.tar.gz", hash = "sha256:05af3bb85d320377db281cf254ab050e1a7ebcbf5410685a9a407e18a1f81236"}, + {file = "packaging-20.8-py2.py3-none-any.whl", hash = "sha256:24e0da08660a87484d1602c30bb4902d74816b6985b93de36926f5bc95741858"}, + {file = "packaging-20.8.tar.gz", hash = "sha256:78598185a7008a470d64526a8059de9aaa449238f280fc9eb6b13ba6c4109093"}, ] paramiko = [ {file = "paramiko-2.7.2-py2.py3-none-any.whl", hash = "sha256:4f3e316fef2ac628b05097a637af35685183111d4bc1b5979bd397c2ab7b5898"}, @@ -450,8 +450,8 @@ pluggy = [ {file = "pluggy-0.13.1.tar.gz", hash = "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0"}, ] py = [ - {file = "py-1.9.0-py2.py3-none-any.whl", hash = "sha256:366389d1db726cd2fcfc79732e75410e5fe4d31db13692115529d34069a043c2"}, - {file = "py-1.9.0.tar.gz", hash = "sha256:9ca6883ce56b4e8da7e79ac18787889fa5206c79dcc67fb065376cd2fe03f342"}, + {file = "py-1.10.0-py2.py3-none-any.whl", hash = "sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a"}, + {file = "py-1.10.0.tar.gz", hash = "sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3"}, ] pycparser = [ {file = "pycparser-2.20-py2.py3-none-any.whl", hash = "sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705"}, @@ -482,8 +482,8 @@ pyparsing = [ {file = "pyparsing-2.4.7.tar.gz", hash = "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1"}, ] pytest = [ - {file = "pytest-6.1.2-py3-none-any.whl", hash = "sha256:4288fed0d9153d9646bfcdf0c0428197dba1ecb27a33bb6e031d002fa88653fe"}, - {file = "pytest-6.1.2.tar.gz", hash = "sha256:c0a7e94a8cdbc5422a51ccdad8e6f1024795939cc89159a0ae7f0b316ad3823e"}, + {file = "pytest-6.2.1-py3-none-any.whl", hash = "sha256:1969f797a1a0dbd8ccf0fecc80262312729afea9c17f1d70ebf85c5e76c6f7c8"}, + {file = "pytest-6.2.1.tar.gz", hash = "sha256:66e419b1899bc27346cb2c993e12c5e5e8daba9073c1fbce33b9807abc95c306"}, ] pytest-playbook = [] pyyaml = [ @@ -510,13 +510,18 @@ six = [ {file = "six-1.15.0.tar.gz", hash = "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259"}, ] tenacity = [ - {file = "tenacity-6.2.0-py2.py3-none-any.whl", hash = "sha256:5a5d3dcd46381abe8b4f82b5736b8726fd3160c6c7161f53f8af7f1eb9b82173"}, - {file = "tenacity-6.2.0.tar.gz", hash = "sha256:29ae90e7faf488a8628432154bb34ace1cca58244c6ea399fd33f066ac71339a"}, + {file = "tenacity-6.3.1-py2.py3-none-any.whl", hash = "sha256:baed357d9f35ec64264d8a4bbf004c35058fad8795c5b0d8a7dc77ecdcbb8f39"}, + {file = "tenacity-6.3.1.tar.gz", hash = "sha256:e14d191fb0a309b563904bbc336582efe2037de437e543b38da749769b544d7f"}, ] toml = [ {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, ] +typing-extensions = [ + {file = "typing_extensions-3.7.4.3-py2-none-any.whl", hash = "sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f"}, + {file = "typing_extensions-3.7.4.3-py3-none-any.whl", hash = "sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918"}, + {file = "typing_extensions-3.7.4.3.tar.gz", hash = "sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c"}, +] zipp = [ {file = "zipp-3.4.0-py3-none-any.whl", hash = "sha256:102c24ef8f171fd729d46599845e95c7ab894a4cf45f5de11a44cc7444fb1108"}, {file = "zipp-3.4.0.tar.gz", hash = "sha256:ed5eee1974372595f9e416cc7bbeeb12335201d8081ca8a0743c954d4446e5cb"}, From 80b44a662fe4975d43b8aeb6b653f803d830a5e1 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Tue, 5 Jan 2021 16:28:44 -0800 Subject: [PATCH 160/194] Add Sphinx as developer dependency --- poetry.lock | 355 ++++++++++++++++++++++++++++++++++++++++++++++++- pyproject.toml | 1 + 2 files changed, 355 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index 440c2e3b14..e2ef2fbc09 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,3 +1,11 @@ +[[package]] +name = "alabaster" +version = "0.7.12" +description = "A configurable sidebar-enabled Sphinx theme" +category = "dev" +optional = false +python-versions = "*" + [[package]] name = "apipkg" version = "1.5" @@ -36,6 +44,17 @@ docs = ["furo", "sphinx", "zope.interface"] tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface"] tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six"] +[[package]] +name = "babel" +version = "2.9.0" +description = "Internationalization utilities" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[package.dependencies] +pytz = ">=2015.7" + [[package]] name = "bcrypt" version = "3.2.0" @@ -74,6 +93,14 @@ typing-extensions = ">=3.7.4" colorama = ["colorama (>=0.4.3)"] d = ["aiohttp (>=3.3.2)", "aiohttp-cors"] +[[package]] +name = "certifi" +version = "2020.12.5" +description = "Python package for providing Mozilla's CA Bundle." +category = "dev" +optional = false +python-versions = "*" + [[package]] name = "cffi" version = "1.14.4" @@ -85,6 +112,14 @@ python-versions = "*" [package.dependencies] pycparser = "*" +[[package]] +name = "chardet" +version = "4.0.0" +description = "Universal encoding detector for Python 2 and 3" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + [[package]] name = "click" version = "7.1.2" @@ -128,6 +163,14 @@ pep8test = ["black", "flake8", "flake8-import-order", "pep8-naming"] ssh = ["bcrypt (>=3.1.5)"] test = ["pytest (>=3.6.0,!=3.9.0,!=3.9.1,!=3.9.2)", "pretend", "iso8601", "pytz", "hypothesis (>=1.11.4,!=3.79.2)"] +[[package]] +name = "docutils" +version = "0.16" +description = "Docutils -- Python Documentation Utilities" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + [[package]] name = "execnet" version = "1.7.1" @@ -223,6 +266,22 @@ testfixtures = ">=6.8.0,<7" [package.extras] test = ["pytest (>=4.0.2,<6)", "toml"] +[[package]] +name = "idna" +version = "2.10" +description = "Internationalized Domain Names in Applications (IDNA)" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[[package]] +name = "imagesize" +version = "1.2.0" +description = "Getting image size from png/jpeg/jpeg2000/gif file" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + [[package]] name = "importlib-metadata" version = "3.3.0" @@ -283,6 +342,28 @@ parso = ">=0.7.0,<0.8.0" qa = ["flake8 (==3.7.9)"] testing = ["Django (<3.1)", "colorama", "docopt", "pytest (>=3.9.0,<5.0.0)"] +[[package]] +name = "jinja2" +version = "2.11.2" +description = "A very fast and expressive template engine." +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[package.dependencies] +MarkupSafe = ">=0.23" + +[package.extras] +i18n = ["Babel (>=0.8)"] + +[[package]] +name = "markupsafe" +version = "1.1.1" +description = "Safely add untrusted strings to HTML/XML markup." +category = "dev" +optional = false +python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*" + [[package]] name = "mccabe" version = "0.6.1" @@ -410,6 +491,14 @@ category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +[[package]] +name = "pygments" +version = "2.7.3" +description = "Pygments is a syntax highlighting package written in Python." +category = "dev" +optional = false +python-versions = ">=3.5" + [[package]] name = "pyls-black" version = "0.4.6" @@ -700,6 +789,14 @@ rope = ["rope (>0.10.5)"] test = ["versioneer", "pylint (>=2.5.0)", "pytest", "mock", "pytest-cov", "coverage", "numpy", "pandas", "matplotlib", "flaky", "pyqt5"] yapf = ["yapf"] +[[package]] +name = "pytz" +version = "2020.5" +description = "World timezone definitions, modern and historical" +category = "dev" +optional = false +python-versions = "*" + [[package]] name = "pyyaml" version = "5.3.1" @@ -716,6 +813,24 @@ category = "dev" optional = false python-versions = "*" +[[package]] +name = "requests" +version = "2.25.1" +description = "Python HTTP for Humans." +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[package.dependencies] +certifi = ">=2017.4.17" +chardet = ">=3.0.2,<5" +idna = ">=2.5,<3" +urllib3 = ">=1.21.1,<1.27" + +[package.extras] +security = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)"] +socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"] + [[package]] name = "rope" version = "0.18.0" @@ -746,6 +861,116 @@ category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +[[package]] +name = "snowballstemmer" +version = "2.0.0" +description = "This package provides 26 stemmers for 25 languages generated from Snowball algorithms." +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "sphinx" +version = "3.4.2" +description = "Python documentation generator" +category = "dev" +optional = false +python-versions = ">=3.5" + +[package.dependencies] +alabaster = ">=0.7,<0.8" +babel = ">=1.3" +colorama = {version = ">=0.3.5", markers = "sys_platform == \"win32\""} +docutils = ">=0.12" +imagesize = "*" +Jinja2 = ">=2.3" +packaging = "*" +Pygments = ">=2.0" +requests = ">=2.5.0" +snowballstemmer = ">=1.1" +sphinxcontrib-applehelp = "*" +sphinxcontrib-devhelp = "*" +sphinxcontrib-htmlhelp = "*" +sphinxcontrib-jsmath = "*" +sphinxcontrib-qthelp = "*" +sphinxcontrib-serializinghtml = "*" + +[package.extras] +docs = ["sphinxcontrib-websupport"] +lint = ["flake8 (>=3.5.0)", "isort", "mypy (>=0.790)", "docutils-stubs"] +test = ["pytest", "pytest-cov", "html5lib", "cython", "typed-ast"] + +[[package]] +name = "sphinxcontrib-applehelp" +version = "1.0.2" +description = "sphinxcontrib-applehelp is a sphinx extension which outputs Apple help books" +category = "dev" +optional = false +python-versions = ">=3.5" + +[package.extras] +lint = ["flake8", "mypy", "docutils-stubs"] +test = ["pytest"] + +[[package]] +name = "sphinxcontrib-devhelp" +version = "1.0.2" +description = "sphinxcontrib-devhelp is a sphinx extension which outputs Devhelp document." +category = "dev" +optional = false +python-versions = ">=3.5" + +[package.extras] +lint = ["flake8", "mypy", "docutils-stubs"] +test = ["pytest"] + +[[package]] +name = "sphinxcontrib-htmlhelp" +version = "1.0.3" +description = "sphinxcontrib-htmlhelp is a sphinx extension which renders HTML help files" +category = "dev" +optional = false +python-versions = ">=3.5" + +[package.extras] +lint = ["flake8", "mypy", "docutils-stubs"] +test = ["pytest", "html5lib"] + +[[package]] +name = "sphinxcontrib-jsmath" +version = "1.0.1" +description = "A sphinx extension which renders display math in HTML via JavaScript" +category = "dev" +optional = false +python-versions = ">=3.5" + +[package.extras] +test = ["pytest", "flake8", "mypy"] + +[[package]] +name = "sphinxcontrib-qthelp" +version = "1.0.3" +description = "sphinxcontrib-qthelp is a sphinx extension which outputs QtHelp document." +category = "dev" +optional = false +python-versions = ">=3.5" + +[package.extras] +lint = ["flake8", "mypy", "docutils-stubs"] +test = ["pytest"] + +[[package]] +name = "sphinxcontrib-serializinghtml" +version = "1.1.4" +description = "sphinxcontrib-serializinghtml is a sphinx extension which outputs \"serialized\" HTML files (json and pickle)." +category = "dev" +optional = false +python-versions = ">=3.5" + +[package.extras] +lint = ["flake8", "mypy", "docutils-stubs"] +test = ["pytest"] + [[package]] name = "tenacity" version = "6.3.1" @@ -805,6 +1030,19 @@ category = "dev" optional = false python-versions = ">=3.6" +[[package]] +name = "urllib3" +version = "1.26.2" +description = "HTTP library with thread-safe connection pooling, file post, and more." +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" + +[package.extras] +brotli = ["brotlipy (>=0.6.0)"] +secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"] +socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] + [[package]] name = "zipp" version = "3.4.0" @@ -820,9 +1058,13 @@ testing = ["pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "292ddb6210c73e8071accf159db11489d8f5643a7d608ea59c835fdbcad19a85" +content-hash = "f13079090b61506a3b5858bd93ac9ce0cc0957335c2285f7aef06d803517acd5" [metadata.files] +alabaster = [ + {file = "alabaster-0.7.12-py2.py3-none-any.whl", hash = "sha256:446438bdcca0e05bd45ea2de1668c1d9b032e1a9154c2c259092d77031ddd359"}, + {file = "alabaster-0.7.12.tar.gz", hash = "sha256:a661d72d58e6ea8a57f7a86e37d86716863ee5e92788398526d58b26a4e4dc02"}, +] apipkg = [ {file = "apipkg-1.5-py2.py3-none-any.whl", hash = "sha256:58587dd4dc3daefad0487f6d9ae32b4542b185e1c36db6993290e7c41ca2b47c"}, {file = "apipkg-1.5.tar.gz", hash = "sha256:37228cda29411948b422fae072f57e31d3396d2ee1c9783775980ee9c9990af6"}, @@ -839,6 +1081,10 @@ attrs = [ {file = "attrs-20.3.0-py2.py3-none-any.whl", hash = "sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6"}, {file = "attrs-20.3.0.tar.gz", hash = "sha256:832aa3cde19744e49938b91fea06d69ecb9e649c93ba974535d08ad92164f700"}, ] +babel = [ + {file = "Babel-2.9.0-py2.py3-none-any.whl", hash = "sha256:9d35c22fcc79893c3ecc85ac4a56cde1ecf3f19c540bba0922308a6c06ca6fa5"}, + {file = "Babel-2.9.0.tar.gz", hash = "sha256:da031ab54472314f210b0adcff1588ee5d1d1d0ba4dbd07b94dba82bde791e05"}, +] bcrypt = [ {file = "bcrypt-3.2.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:c95d4cbebffafcdd28bd28bb4e25b31c50f6da605c81ffd9ad8a3d1b2ab7b1b6"}, {file = "bcrypt-3.2.0-cp36-abi3-manylinux1_x86_64.whl", hash = "sha256:63d4e3ff96188e5898779b6057878fecf3f11cfe6ec3b313ea09955d587ec7a7"}, @@ -851,6 +1097,10 @@ bcrypt = [ black = [ {file = "black-20.8b1.tar.gz", hash = "sha256:1c02557aa099101b9d21496f8a914e9ed2222ef70336404eeeac8edba836fbea"}, ] +certifi = [ + {file = "certifi-2020.12.5-py2.py3-none-any.whl", hash = "sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830"}, + {file = "certifi-2020.12.5.tar.gz", hash = "sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c"}, +] cffi = [ {file = "cffi-1.14.4-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:ebb253464a5d0482b191274f1c8bf00e33f7e0b9c66405fbffc61ed2c839c775"}, {file = "cffi-1.14.4-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:2c24d61263f511551f740d1a065eb0212db1dbbbbd241db758f5244281590c06"}, @@ -889,6 +1139,10 @@ cffi = [ {file = "cffi-1.14.4-cp39-cp39-win_amd64.whl", hash = "sha256:f032b34669220030f905152045dfa27741ce1a6db3324a5bc0b96b6c7420c87b"}, {file = "cffi-1.14.4.tar.gz", hash = "sha256:1a465cbe98a7fd391d47dce4b8f7e5b921e6cd805ef421d04f5f66ba8f06086c"}, ] +chardet = [ + {file = "chardet-4.0.0-py2.py3-none-any.whl", hash = "sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5"}, + {file = "chardet-4.0.0.tar.gz", hash = "sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa"}, +] click = [ {file = "click-7.1.2-py2.py3-none-any.whl", hash = "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc"}, {file = "click-7.1.2.tar.gz", hash = "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a"}, @@ -917,6 +1171,10 @@ cryptography = [ {file = "cryptography-3.3.1-cp36-abi3-win_amd64.whl", hash = "sha256:0e85aaae861d0485eb5a79d33226dd6248d2a9f133b81532c8f5aae37de10ff7"}, {file = "cryptography-3.3.1.tar.gz", hash = "sha256:7e177e4bea2de937a584b13645cab32f25e3d96fc0bc4a4cf99c27dc77682be6"}, ] +docutils = [ + {file = "docutils-0.16-py2.py3-none-any.whl", hash = "sha256:0c5b78adfbf7762415433f5515cd5c9e762339e23369dbe8000d84a4bf4ab3af"}, + {file = "docutils-0.16.tar.gz", hash = "sha256:c2de3a60e9e7d07be26b7f2b00ca0309c207e06c100f9cc2a94931fc75a478fc"}, +] execnet = [ {file = "execnet-1.7.1-py2.py3-none-any.whl", hash = "sha256:d4efd397930c46415f62f8a31388d6be4f27a91d7550eb79bc64a756e0056547"}, {file = "execnet-1.7.1.tar.gz", hash = "sha256:cacb9df31c9680ec5f95553976c4da484d407e85e41c83cb812aa014f0eddc50"}, @@ -944,6 +1202,14 @@ flake8-isort = [ {file = "flake8-isort-4.0.0.tar.gz", hash = "sha256:2b91300f4f1926b396c2c90185844eb1a3d5ec39ea6138832d119da0a208f4d9"}, {file = "flake8_isort-4.0.0-py2.py3-none-any.whl", hash = "sha256:729cd6ef9ba3659512dee337687c05d79c78e1215fdf921ed67e5fe46cce2f3c"}, ] +idna = [ + {file = "idna-2.10-py2.py3-none-any.whl", hash = "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"}, + {file = "idna-2.10.tar.gz", hash = "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6"}, +] +imagesize = [ + {file = "imagesize-1.2.0-py2.py3-none-any.whl", hash = "sha256:6965f19a6a2039c7d48bca7dba2473069ff854c36ae6f19d2cde309d998228a1"}, + {file = "imagesize-1.2.0.tar.gz", hash = "sha256:b1f6b5a4eab1f73479a50fb79fcf729514a900c341d8503d62a62dbc4127a2b1"}, +] importlib-metadata = [ {file = "importlib_metadata-3.3.0-py3-none-any.whl", hash = "sha256:bf792d480abbd5eda85794e4afb09dd538393f7d6e6ffef6e9f03d2014cf9450"}, {file = "importlib_metadata-3.3.0.tar.gz", hash = "sha256:5c5a2720817414a6c41f0a49993908068243ae02c1635a228126519b509c8aed"}, @@ -965,6 +1231,45 @@ jedi = [ {file = "jedi-0.17.2-py2.py3-none-any.whl", hash = "sha256:98cc583fa0f2f8304968199b01b6b4b94f469a1f4a74c1560506ca2a211378b5"}, {file = "jedi-0.17.2.tar.gz", hash = "sha256:86ed7d9b750603e4ba582ea8edc678657fb4007894a12bcf6f4bb97892f31d20"}, ] +jinja2 = [ + {file = "Jinja2-2.11.2-py2.py3-none-any.whl", hash = "sha256:f0a4641d3cf955324a89c04f3d94663aa4d638abe8f733ecd3582848e1c37035"}, + {file = "Jinja2-2.11.2.tar.gz", hash = "sha256:89aab215427ef59c34ad58735269eb58b1a5808103067f7bb9d5836c651b3bb0"}, +] +markupsafe = [ + {file = "MarkupSafe-1.1.1-cp27-cp27m-macosx_10_6_intel.whl", hash = "sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161"}, + {file = "MarkupSafe-1.1.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7"}, + {file = "MarkupSafe-1.1.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:500d4957e52ddc3351cabf489e79c91c17f6e0899158447047588650b5e69183"}, + {file = "MarkupSafe-1.1.1-cp27-cp27m-win32.whl", hash = "sha256:b2051432115498d3562c084a49bba65d97cf251f5a331c64a12ee7e04dacc51b"}, + {file = "MarkupSafe-1.1.1-cp27-cp27m-win_amd64.whl", hash = "sha256:98c7086708b163d425c67c7a91bad6e466bb99d797aa64f965e9d25c12111a5e"}, + {file = "MarkupSafe-1.1.1-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:cd5df75523866410809ca100dc9681e301e3c27567cf498077e8551b6d20e42f"}, + {file = "MarkupSafe-1.1.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:43a55c2930bbc139570ac2452adf3d70cdbb3cfe5912c71cdce1c2c6bbd9c5d1"}, + {file = "MarkupSafe-1.1.1-cp34-cp34m-macosx_10_6_intel.whl", hash = "sha256:1027c282dad077d0bae18be6794e6b6b8c91d58ed8a8d89a89d59693b9131db5"}, + {file = "MarkupSafe-1.1.1-cp34-cp34m-manylinux1_i686.whl", hash = "sha256:62fe6c95e3ec8a7fad637b7f3d372c15ec1caa01ab47926cfdf7a75b40e0eac1"}, + {file = "MarkupSafe-1.1.1-cp34-cp34m-manylinux1_x86_64.whl", hash = "sha256:88e5fcfb52ee7b911e8bb6d6aa2fd21fbecc674eadd44118a9cc3863f938e735"}, + {file = "MarkupSafe-1.1.1-cp34-cp34m-win32.whl", hash = "sha256:ade5e387d2ad0d7ebf59146cc00c8044acbd863725f887353a10df825fc8ae21"}, + {file = "MarkupSafe-1.1.1-cp34-cp34m-win_amd64.whl", hash = "sha256:09c4b7f37d6c648cb13f9230d847adf22f8171b1ccc4d5682398e77f40309235"}, + {file = "MarkupSafe-1.1.1-cp35-cp35m-macosx_10_6_intel.whl", hash = "sha256:79855e1c5b8da654cf486b830bd42c06e8780cea587384cf6545b7d9ac013a0b"}, + {file = "MarkupSafe-1.1.1-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:c8716a48d94b06bb3b2524c2b77e055fb313aeb4ea620c8dd03a105574ba704f"}, + {file = "MarkupSafe-1.1.1-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:7c1699dfe0cf8ff607dbdcc1e9b9af1755371f92a68f706051cc8c37d447c905"}, + {file = "MarkupSafe-1.1.1-cp35-cp35m-win32.whl", hash = "sha256:6dd73240d2af64df90aa7c4e7481e23825ea70af4b4922f8ede5b9e35f78a3b1"}, + {file = "MarkupSafe-1.1.1-cp35-cp35m-win_amd64.whl", hash = "sha256:9add70b36c5666a2ed02b43b335fe19002ee5235efd4b8a89bfcf9005bebac0d"}, + {file = "MarkupSafe-1.1.1-cp36-cp36m-macosx_10_6_intel.whl", hash = "sha256:24982cc2533820871eba85ba648cd53d8623687ff11cbb805be4ff7b4c971aff"}, + {file = "MarkupSafe-1.1.1-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473"}, + {file = "MarkupSafe-1.1.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:717ba8fe3ae9cc0006d7c451f0bb265ee07739daf76355d06366154ee68d221e"}, + {file = "MarkupSafe-1.1.1-cp36-cp36m-win32.whl", hash = "sha256:535f6fc4d397c1563d08b88e485c3496cf5784e927af890fb3c3aac7f933ec66"}, + {file = "MarkupSafe-1.1.1-cp36-cp36m-win_amd64.whl", hash = "sha256:b1282f8c00509d99fef04d8ba936b156d419be841854fe901d8ae224c59f0be5"}, + {file = "MarkupSafe-1.1.1-cp37-cp37m-macosx_10_6_intel.whl", hash = "sha256:8defac2f2ccd6805ebf65f5eeb132adcf2ab57aa11fdf4c0dd5169a004710e7d"}, + {file = "MarkupSafe-1.1.1-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:46c99d2de99945ec5cb54f23c8cd5689f6d7177305ebff350a58ce5f8de1669e"}, + {file = "MarkupSafe-1.1.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6"}, + {file = "MarkupSafe-1.1.1-cp37-cp37m-win32.whl", hash = "sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2"}, + {file = "MarkupSafe-1.1.1-cp37-cp37m-win_amd64.whl", hash = "sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c"}, + {file = "MarkupSafe-1.1.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6788b695d50a51edb699cb55e35487e430fa21f1ed838122d722e0ff0ac5ba15"}, + {file = "MarkupSafe-1.1.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:cdb132fc825c38e1aeec2c8aa9338310d29d337bebbd7baa06889d09a60a1fa2"}, + {file = "MarkupSafe-1.1.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:13d3144e1e340870b25e7b10b98d779608c02016d5184cfb9927a9f10c689f42"}, + {file = "MarkupSafe-1.1.1-cp38-cp38-win32.whl", hash = "sha256:596510de112c685489095da617b5bcbbac7dd6384aeebeda4df6025d0256a81b"}, + {file = "MarkupSafe-1.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:e8313f01ba26fbbe36c7be1966a7b7424942f670f38e666995b88d012765b9be"}, + {file = "MarkupSafe-1.1.1.tar.gz", hash = "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b"}, +] mccabe = [ {file = "mccabe-0.6.1-py2.py3-none-any.whl", hash = "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42"}, {file = "mccabe-0.6.1.tar.gz", hash = "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"}, @@ -1025,6 +1330,10 @@ pyflakes = [ {file = "pyflakes-2.2.0-py2.py3-none-any.whl", hash = "sha256:0d94e0e05a19e57a99444b6ddcf9a6eb2e5c68d3ca1e98e90707af8152c90a92"}, {file = "pyflakes-2.2.0.tar.gz", hash = "sha256:35b2d75ee967ea93b55750aa9edbbf72813e06a66ba54438df2cfac9e3c27fc8"}, ] +pygments = [ + {file = "Pygments-2.7.3-py3-none-any.whl", hash = "sha256:f275b6c0909e5dafd2d6269a656aa90fa58ebf4a74f8fcf9053195d226b24a08"}, + {file = "Pygments-2.7.3.tar.gz", hash = "sha256:ccf3acacf3782cbed4a989426012f1c535c9a90d3a7fc3f16d231b9372d2b716"}, +] pyls-black = [ {file = "pyls-black-0.4.6.tar.gz", hash = "sha256:33700e5ed605636ea7ba39188a1362d2f8602f7301f8f2b8544773886f965663"}, {file = "pyls_black-0.4.6-py3-none-any.whl", hash = "sha256:8f5fb8fed503588c10435d2d48e2c3751437f1bdb8116134b05a4591c4899940"}, @@ -1106,6 +1415,10 @@ python-language-server = [ {file = "python-language-server-0.35.1.tar.gz", hash = "sha256:6e0c9a3b2ae98e0eb22e98ed6b3c4e190a6bf9e27af53efd2396da60cd92b221"}, {file = "python_language_server-0.35.1-py2.py3-none-any.whl", hash = "sha256:7051090259e3e81c0cdb140de8e32b8f11219808cda4427e6faf61f9ff9a3bf4"}, ] +pytz = [ + {file = "pytz-2020.5-py2.py3-none-any.whl", hash = "sha256:16962c5fb8db4a8f63a26646d8886e9d769b6c511543557bc84e9569fb9a9cb4"}, + {file = "pytz-2020.5.tar.gz", hash = "sha256:180befebb1927b16f6b57101720075a984c019ac16b1b7575673bea42c6c3da5"}, +] pyyaml = [ {file = "PyYAML-5.3.1-cp27-cp27m-win32.whl", hash = "sha256:74809a57b329d6cc0fdccee6318f44b9b8649961fa73144a98735b0aaf029f1f"}, {file = "PyYAML-5.3.1-cp27-cp27m-win_amd64.whl", hash = "sha256:240097ff019d7c70a4922b6869d8a86407758333f02203e0fc6ff79c5dcede76"}, @@ -1164,6 +1477,10 @@ regex = [ {file = "regex-2020.11.13-cp39-cp39-win_amd64.whl", hash = "sha256:a15f64ae3a027b64496a71ab1f722355e570c3fac5ba2801cafce846bf5af01d"}, {file = "regex-2020.11.13.tar.gz", hash = "sha256:83d6b356e116ca119db8e7c6fc2983289d87b27b3fac238cfe5dca529d884562"}, ] +requests = [ + {file = "requests-2.25.1-py2.py3-none-any.whl", hash = "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e"}, + {file = "requests-2.25.1.tar.gz", hash = "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804"}, +] rope = [ {file = "rope-0.18.0.tar.gz", hash = "sha256:786b5c38c530d4846aa68a42604f61b4e69a493390e3ca11b88df0fbfdc3ed04"}, ] @@ -1175,6 +1492,38 @@ six = [ {file = "six-1.15.0-py2.py3-none-any.whl", hash = "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"}, {file = "six-1.15.0.tar.gz", hash = "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259"}, ] +snowballstemmer = [ + {file = "snowballstemmer-2.0.0-py2.py3-none-any.whl", hash = "sha256:209f257d7533fdb3cb73bdbd24f436239ca3b2fa67d56f6ff88e86be08cc5ef0"}, + {file = "snowballstemmer-2.0.0.tar.gz", hash = "sha256:df3bac3df4c2c01363f3dd2cfa78cce2840a79b9f1c2d2de9ce8d31683992f52"}, +] +sphinx = [ + {file = "Sphinx-3.4.2-py3-none-any.whl", hash = "sha256:b8aa4eb5502c53d3b5ca13a07abeedacd887f7770c198952fd5b9530d973e767"}, + {file = "Sphinx-3.4.2.tar.gz", hash = "sha256:77dec5ac77ca46eee54f59cf477780f4fb23327b3339ef39c8471abb829c1285"}, +] +sphinxcontrib-applehelp = [ + {file = "sphinxcontrib-applehelp-1.0.2.tar.gz", hash = "sha256:a072735ec80e7675e3f432fcae8610ecf509c5f1869d17e2eecff44389cdbc58"}, + {file = "sphinxcontrib_applehelp-1.0.2-py2.py3-none-any.whl", hash = "sha256:806111e5e962be97c29ec4c1e7fe277bfd19e9652fb1a4392105b43e01af885a"}, +] +sphinxcontrib-devhelp = [ + {file = "sphinxcontrib-devhelp-1.0.2.tar.gz", hash = "sha256:ff7f1afa7b9642e7060379360a67e9c41e8f3121f2ce9164266f61b9f4b338e4"}, + {file = "sphinxcontrib_devhelp-1.0.2-py2.py3-none-any.whl", hash = "sha256:8165223f9a335cc1af7ffe1ed31d2871f325254c0423bc0c4c7cd1c1e4734a2e"}, +] +sphinxcontrib-htmlhelp = [ + {file = "sphinxcontrib-htmlhelp-1.0.3.tar.gz", hash = "sha256:e8f5bb7e31b2dbb25b9cc435c8ab7a79787ebf7f906155729338f3156d93659b"}, + {file = "sphinxcontrib_htmlhelp-1.0.3-py2.py3-none-any.whl", hash = "sha256:3c0bc24a2c41e340ac37c85ced6dafc879ab485c095b1d65d2461ac2f7cca86f"}, +] +sphinxcontrib-jsmath = [ + {file = "sphinxcontrib-jsmath-1.0.1.tar.gz", hash = "sha256:a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8"}, + {file = "sphinxcontrib_jsmath-1.0.1-py2.py3-none-any.whl", hash = "sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178"}, +] +sphinxcontrib-qthelp = [ + {file = "sphinxcontrib-qthelp-1.0.3.tar.gz", hash = "sha256:4c33767ee058b70dba89a6fc5c1892c0d57a54be67ddd3e7875a18d14cba5a72"}, + {file = "sphinxcontrib_qthelp-1.0.3-py2.py3-none-any.whl", hash = "sha256:bd9fc24bcb748a8d51fd4ecaade681350aa63009a347a8c14e637895444dfab6"}, +] +sphinxcontrib-serializinghtml = [ + {file = "sphinxcontrib-serializinghtml-1.1.4.tar.gz", hash = "sha256:eaa0eccc86e982a9b939b2b82d12cc5d013385ba5eadcc7e4fed23f4405f77bc"}, + {file = "sphinxcontrib_serializinghtml-1.1.4-py2.py3-none-any.whl", hash = "sha256:f242a81d423f59617a8e5cf16f5d4d74e28ee9a66f9e5b637a18082991db5a9a"}, +] tenacity = [ {file = "tenacity-6.3.1-py2.py3-none-any.whl", hash = "sha256:baed357d9f35ec64264d8a4bbf004c35058fad8795c5b0d8a7dc77ecdcbb8f39"}, {file = "tenacity-6.3.1.tar.gz", hash = "sha256:e14d191fb0a309b563904bbc336582efe2037de437e543b38da749769b544d7f"}, @@ -1247,6 +1596,10 @@ ujson = [ {file = "ujson-4.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:f8a60928737a9a47e692fcd661ef2b5d75ba22c7c930025bd95e338f2a6e15bc"}, {file = "ujson-4.0.1.tar.gz", hash = "sha256:26cf6241b36ff5ce4539ae687b6b02673109c5e3efc96148806a7873eaa229d3"}, ] +urllib3 = [ + {file = "urllib3-1.26.2-py2.py3-none-any.whl", hash = "sha256:d8ff90d979214d7b4f8ce956e80f4028fc6860e4431f731ea4a8c08f23f99473"}, + {file = "urllib3-1.26.2.tar.gz", hash = "sha256:19188f96923873c92ccb987120ec4acaa12f0461fa9ce5d3d0772bc965a39e08"}, +] zipp = [ {file = "zipp-3.4.0-py3-none-any.whl", hash = "sha256:102c24ef8f171fd729d46599845e95c7ab894a4cf45f5de11a44cc7444fb1108"}, {file = "zipp-3.4.0.tar.gz", hash = "sha256:ed5eee1974372595f9e416cc7bbeeb12335201d8081ca8a0743c954d4446e5cb"}, diff --git a/pyproject.toml b/pyproject.toml index 934b550c2a..0f0296ffb7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,7 @@ pyls-mypy = "^0.1.8" rope = "^0.18.0" pytest-flake8 = "^1.0.6" pytest-mypy = "^0.7.0" +Sphinx = "^3.4.2" [tool.black] line-length = 88 From aa38bbc27615ce79296e2d4c37eb961df5bd75db Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Tue, 5 Jan 2021 16:30:08 -0800 Subject: [PATCH 161/194] Add recommonmark as developer dependency --- poetry.lock | 34 +++++++++++++++++++++++++++++++++- pyproject.toml | 1 + 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index e2ef2fbc09..b4015773de 100644 --- a/poetry.lock +++ b/poetry.lock @@ -136,6 +136,17 @@ category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +[[package]] +name = "commonmark" +version = "0.9.1" +description = "Python parser for the CommonMark Markdown spec" +category = "dev" +optional = false +python-versions = "*" + +[package.extras] +test = ["flake8 (==3.7.8)", "hypothesis (==3.55.3)"] + [[package]] name = "contextlib2" version = "0.6.0.post1" @@ -805,6 +816,19 @@ category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +[[package]] +name = "recommonmark" +version = "0.7.1" +description = "A docutils-compatibility bridge to CommonMark, enabling you to write CommonMark inside of Docutils & Sphinx projects." +category = "dev" +optional = false +python-versions = "*" + +[package.dependencies] +commonmark = ">=0.8.1" +docutils = ">=0.11" +sphinx = ">=1.3.1" + [[package]] name = "regex" version = "2020.11.13" @@ -1058,7 +1082,7 @@ testing = ["pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "f13079090b61506a3b5858bd93ac9ce0cc0957335c2285f7aef06d803517acd5" +content-hash = "45121cd7b463c5e45581a29242620b2fa000aad85ffb4677888c520495c68847" [metadata.files] alabaster = [ @@ -1151,6 +1175,10 @@ colorama = [ {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"}, {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"}, ] +commonmark = [ + {file = "commonmark-0.9.1-py2.py3-none-any.whl", hash = "sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9"}, + {file = "commonmark-0.9.1.tar.gz", hash = "sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60"}, +] contextlib2 = [ {file = "contextlib2-0.6.0.post1-py2.py3-none-any.whl", hash = "sha256:3355078a159fbb44ee60ea80abd0d87b80b78c248643b49aa6d94673b413609b"}, {file = "contextlib2-0.6.0.post1.tar.gz", hash = "sha256:01f490098c18b19d2bd5bb5dc445b2054d2fa97f09a4280ba2c5f3c394c8162e"}, @@ -1434,6 +1462,10 @@ pyyaml = [ {file = "PyYAML-5.3.1-cp39-cp39-win_amd64.whl", hash = "sha256:6034f55dab5fea9e53f436aa68fa3ace2634918e8b5994d82f3621c04ff5ed2e"}, {file = "PyYAML-5.3.1.tar.gz", hash = "sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d"}, ] +recommonmark = [ + {file = "recommonmark-0.7.1-py2.py3-none-any.whl", hash = "sha256:1b1db69af0231efce3fa21b94ff627ea33dee7079a01dd0a7f8482c3da148b3f"}, + {file = "recommonmark-0.7.1.tar.gz", hash = "sha256:bdb4db649f2222dcd8d2d844f0006b958d627f732415d399791ee436a3686d67"}, +] regex = [ {file = "regex-2020.11.13-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:8b882a78c320478b12ff024e81dc7d43c1462aa4a3341c754ee65d857a521f85"}, {file = "regex-2020.11.13-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:a63f1a07932c9686d2d416fb295ec2c01ab246e89b4d58e5fa468089cab44b70"}, diff --git a/pyproject.toml b/pyproject.toml index 0f0296ffb7..e87e54272e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,6 +31,7 @@ rope = "^0.18.0" pytest-flake8 = "^1.0.6" pytest-mypy = "^0.7.0" Sphinx = "^3.4.2" +recommonmark = "^0.7.1" [tool.black] line-length = 88 From c7c4b6cd158c47101495d8c950fc87d194e24e02 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Tue, 5 Jan 2021 16:50:44 -0800 Subject: [PATCH 162/194] Add basic Sphinx configuration --- .gitignore | 1 + CONTRIBUTING.md | 6 ++++++ conf.py | 45 +++++++++++++++++++++++++++++++++++++++++++++ index.md | 1 + 4 files changed, 53 insertions(+) create mode 100644 conf.py create mode 120000 index.md diff --git a/.gitignore b/.gitignore index b2c361f63a..e4a3b043c3 100644 --- a/.gitignore +++ b/.gitignore @@ -6,5 +6,6 @@ /*.html /*.xml /assets/ +/_build/ __pycache__/ dist/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 038c3945c9..ab91c5e8ef 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -373,6 +373,12 @@ Python world. If you make it through even some of these guides, you will be well on your way to being a “Pythonista” (a Python developer) writing “Pythonic” (canonically correct Python) code left and right. +## Generating Documentation + +```bash +sphinx-build . _build +``` + ## Future Sections Just a collection of reminders for the author to expand on later. diff --git a/conf.py b/conf.py new file mode 100644 index 0000000000..c3e0a8a9ce --- /dev/null +++ b/conf.py @@ -0,0 +1,45 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Project information ----------------------------------------------------- +import importlib.metadata + +metadata = importlib.metadata.metadata("LISA") + +project = metadata["Name"] +project_copyright = "Microsoft" # TODO: Add year and verify. +author = metadata["Author"] +version = metadata["Version"] +release = version + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = ["recommonmark"] + + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ["_build", ".archive", ".pytest_cache", "dist"] + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = "alabaster" + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ["_static"] diff --git a/index.md b/index.md new file mode 120000 index 0000000000..42061c01a1 --- /dev/null +++ b/index.md @@ -0,0 +1 @@ +README.md \ No newline at end of file From c9faf5cd75574d2947877b3daf712b94c311103c Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Tue, 5 Jan 2021 17:04:55 -0800 Subject: [PATCH 163/194] Enable built-in extensions --- conf.py | 23 ++++++++++++++++++++++- pyproject.toml | 6 +++++- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/conf.py b/conf.py index c3e0a8a9ce..0d6a039570 100644 --- a/conf.py +++ b/conf.py @@ -20,7 +20,28 @@ # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ["recommonmark"] +extensions = [ + "recommonmark", + "sphinx.ext.autodoc", + "sphinx.ext.autosummary", + "sphinx.ext.githubpages", + "sphinx.ext.linkcode", + "sphinx.ext.viewcode", + "sphinx.ext.todo", +] + + +def linkcode_resolve(domain, info): + """Configure linkcode extension.""" + if domain != "py": + return None + if not info["module"]: + return None + filename = info["module"].replace(".", "/") + url = metadata["Project-Url"].split(", ")[1] + # TODO: Update this branch to `main` branch after PR is merged. + branch = "andschwa/pytest" + return f"{url}/blob/{branch}/{filename}.py" # Add any paths that contain templates here, relative to this directory. diff --git a/pyproject.toml b/pyproject.toml index e87e54272e..9a7bab326b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,8 +2,12 @@ name = "LISA" version = "0.1.0" description = "Linux Integration Services Automation (LISA)" -authors = ["Andrew Schwartzmeyer "] license = "MIT License" +authors = ["Andrew Schwartzmeyer "] +readme = "README.md" +homepage = "https://microsoft.github.io/lisa" +repository = "https://github.com/microsoft/lisa" + include = [".md", "playbooks/*.yaml"] packages = [{include = "*.py"}, {include = "testsuites"}] From c5dcd7cbb7e7e426f50321b78792301a26f76ed9 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Wed, 6 Jan 2021 15:30:42 -0800 Subject: [PATCH 164/194] Create Table of Contents --- conf.py | 2 +- index.md | 1 - index.rst | 9 +++++++++ 3 files changed, 10 insertions(+), 2 deletions(-) delete mode 120000 index.md create mode 100644 index.rst diff --git a/conf.py b/conf.py index 0d6a039570..b52158786c 100644 --- a/conf.py +++ b/conf.py @@ -9,7 +9,7 @@ metadata = importlib.metadata.metadata("LISA") -project = metadata["Name"] +project = metadata["Name"].upper() project_copyright = "Microsoft" # TODO: Add year and verify. author = metadata["Author"] version = metadata["Version"] diff --git a/index.md b/index.md deleted file mode 120000 index 42061c01a1..0000000000 --- a/index.md +++ /dev/null @@ -1 +0,0 @@ -README.md \ No newline at end of file diff --git a/index.rst b/index.rst new file mode 100644 index 0000000000..8d4d9a5bb6 --- /dev/null +++ b/index.rst @@ -0,0 +1,9 @@ +.. toctree:: + :caption: Table of Contents + :name: mastertoc + :maxdepth: 2 + + README + DESIGN + CONTRIBUTING + CODE_OF_CONDUCT From 4fdda972434759ad91f11db642e578ed048c82f9 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Wed, 6 Jan 2021 15:30:54 -0800 Subject: [PATCH 165/194] Documentation updates --- DESIGN.md | 2 +- README.md | 39 +++++++++++++++++++-------------------- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 151472711b..75bf589901 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -1,4 +1,4 @@ -# LISAv3 Technical Specification Document +# Technical Specification Document This document outlines the technical specifications for LISAv3. We are evaluating the feasibility of leveraging diff --git a/README.md b/README.md index e60f1dd289..a800ed02e1 100644 --- a/README.md +++ b/README.md @@ -1,30 +1,30 @@ -# Linux Integration Services Automation 3.0 (LISAv3) +# Linux Integration Services Automation -[![CI Workflow for LISAv3](https://github.com/LIS/LISAv2/workflows/CI%20Workflow%20for%20LISAv3/badge.svg?branch=main)](https://github.com/LIS/LISAv2/actions?query=workflow%3A%22CI+Workflow+for+LISAv3%22+event%3Apush+branch%3Amain) +[![LISA/Pytest CI Workflow](https://github.com/microsoft/lisa/workflows/LISA/Pytest%20CI%20Workflow/badge.svg?branch=andschwa%2Fpytest)](https://github.com/microsoft/lisa/actions?query=workflow%3A%22LISA%2FPytest+CI+Workflow%22) [![Code Style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) -[![GitHub License](https://img.shields.io/github/license/LIS/LISAv2)](https://github.com/LIS/LISAv2/blob/main/LICENSE.md) LISA is a Linux test automation framework with built-in test cases to verify the quality of Linux distributions on multiple platforms (such as Azure, Hyper-V, and bare metal). It is an opinionated collection of custom [Pytest][] plugins, -configurations, and tests. See the [design document](DESIGN.md) for details. +configurations, and tests. See the [technical specification document](DESIGN.md) +for details. [Pytest]: https://docs.pytest.org/en/stable/ -## Getting Started: +## Getting Started -### Install Python 3: +### Install Python 3 Install Python 3.7 or newer from your Linux distribution’s package repositories, or [python.org](https://www.python.org/). -### Install Poetry: +### Install Poetry [Poetry](https://python-poetry.org/docs/) is our preferred tool for Python dependency management and packaging. We’ll use it to automatically setup a ‘virtualenv’ and install everything we need. -#### On Linux (or WSL): +#### On Linux (or WSL) ```bash curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python - @@ -42,21 +42,21 @@ your `PATH` before the Windows version, or this error will appear: Adjust your `PATH` appropriately to fix it. -#### On Windows (in PowerShell): +#### On Windows (in PowerShell) ```powershell (Invoke-WebRequest -Uri https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py -UseBasicParsing).Content | python - $env:PATH += ";$env:USERPROFILE\.poetry\bin" ``` -### Clone LISA and `cd` into the Git repo: +### Clone LISA and `cd` into the Git repo ```bash -git clone -b main https://github.com/microsoft/lisa.git +git clone -b andschwa/pytest https://github.com/microsoft/lisa.git cd lisa ``` -### Install Python dependencies: +### Install Python dependencies ```bash # Install the Python packages @@ -66,7 +66,7 @@ poetry install poetry shell ``` -### Use LISA: +### Use LISA ```bash # Run some self-tests @@ -76,7 +76,7 @@ lisa --playbook=playbooks/test.yml selftests/ lisa --playbook=playbooks/demo.yaml ``` -#### Enable Azure: +#### Enable Azure To run the demo you’ll need the [Azure CLI][] tool installed and configured: @@ -93,14 +93,14 @@ az account set -s ## Contributing -See the [Contributing Guidelines](CONTRIBUTING.md) for developer information! +See the [contributing guidelines](CONTRIBUTING.md) for developer information! ### Contributor License Agreement This project welcomes contributions and suggestions. Most contributions require you to agree to a Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us the rights to use your contribution. For -details, visit https://cla.opensource.microsoft.com. +details, visit . When you submit a pull request, a CLA bot will automatically determine whether you need to provide a CLA and decorate the PR appropriately (e.g., status check, @@ -111,8 +111,7 @@ This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact -[opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional -questions or comments. + with any additional questions or comments. ## Legal Notices @@ -129,9 +128,9 @@ referenced in the documentation may be either trademarks or registered trademarks of Microsoft in the United States and/or other countries. The licenses for this project do not grant you rights to use any Microsoft names, logos, or trademarks. Microsoft's general trademark guidelines can be found at -http://go.microsoft.com/fwlink/?LinkID=254653. +. -Privacy information can be found at https://privacy.microsoft.com/en-us/ +Privacy information can be found at Microsoft and any contributors reserve all other rights, whether under their respective copyrights, patents, or trademarks, whether by implication, estoppel From a36530e49db02f4c22b3989267a02ccc47aaba32 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Wed, 6 Jan 2021 16:09:10 -0800 Subject: [PATCH 166/194] Convert readme to reStructuredText for proper documentation --- README.md | 95 +---------------------------- index.rst | 174 +++++++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 169 insertions(+), 100 deletions(-) diff --git a/README.md b/README.md index a800ed02e1..44f8f6ad79 100644 --- a/README.md +++ b/README.md @@ -1,101 +1,12 @@ # Linux Integration Services Automation -[![LISA/Pytest CI Workflow](https://github.com/microsoft/lisa/workflows/LISA/Pytest%20CI%20Workflow/badge.svg?branch=andschwa%2Fpytest)](https://github.com/microsoft/lisa/actions?query=workflow%3A%22LISA%2FPytest+CI+Workflow%22) -[![Code Style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) - LISA is a Linux test automation framework with built-in test cases to verify the quality of Linux distributions on multiple platforms (such as Azure, Hyper-V, -and bare metal). It is an opinionated collection of custom [Pytest][] plugins, -configurations, and tests. See the [technical specification document](DESIGN.md) -for details. - -[Pytest]: https://docs.pytest.org/en/stable/ - -## Getting Started - -### Install Python 3 - -Install Python 3.7 or newer from your Linux distribution’s package repositories, -or [python.org](https://www.python.org/). - -### Install Poetry - -[Poetry](https://python-poetry.org/docs/) is our preferred tool for Python -dependency management and packaging. We’ll use it to automatically setup a -‘virtualenv’ and install everything we need. - -#### On Linux (or WSL) - -```bash -curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python - -source $HOME/.poetry/env -``` - -If you are using WSL, installing Poetry on both Windows and Linux may cause both -platforms’ versions of Poetry to be on your path, as Windows binaries are mapped -into WSL’s `PATH`. This means that the Linux `poetry` binary _must_ appear in -your `PATH` before the Windows version, or this error will appear: - -``` -`/usr/bin/env: ‘python\r’: No such file or directory` -``` - -Adjust your `PATH` appropriately to fix it. - -#### On Windows (in PowerShell) - -```powershell -(Invoke-WebRequest -Uri https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py -UseBasicParsing).Content | python - -$env:PATH += ";$env:USERPROFILE\.poetry\bin" -``` - -### Clone LISA and `cd` into the Git repo - -```bash -git clone -b andschwa/pytest https://github.com/microsoft/lisa.git -cd lisa -``` - -### Install Python dependencies - -```bash -# Install the Python packages -poetry install - -# Enter the virtual environment -poetry shell -``` - -### Use LISA - -```bash -# Run some self-tests -lisa --playbook=playbooks/test.yml selftests/ - -# Run a demo which deploys Azure resources -lisa --playbook=playbooks/demo.yaml -``` - -#### Enable Azure - -To run the demo you’ll need the [Azure CLI][] tool installed and configured: - -[Azure CLI]: https://docs.microsoft.com/en-us/cli/azure/ - -```bash -# Install Azure CLI, make sure `az` is in your `PATH` -curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash - -# Login and set subscription -az login -az account set -s -``` - -## Contributing +and bare metal). -See the [contributing guidelines](CONTRIBUTING.md) for developer information! +See the [documentation](https://microsoft.github.io/lisa) for details. -### Contributor License Agreement +## Contributor License Agreement This project welcomes contributions and suggestions. Most contributions require you to agree to a Contributor License Agreement (CLA) declaring that you have diff --git a/index.rst b/index.rst index 8d4d9a5bb6..4fadbb9345 100644 --- a/index.rst +++ b/index.rst @@ -1,9 +1,167 @@ +Linux Integration Services Automation +===================================== + +|LISA/Pytest CI Workflow| |Code Style: black| + +LISA is a Linux test automation framework with built-in test cases to +verify the quality of Linux distributions on multiple platforms (such +as Azure, Hyper-V, and bare metal). It is an opinionated collection of +custom `Pytest `__ plugins, +configurations, and tests. See the :doc:`technical specification +document ` for details. + .. toctree:: - :caption: Table of Contents - :name: mastertoc - :maxdepth: 2 - - README - DESIGN - CONTRIBUTING - CODE_OF_CONDUCT + :maxdepth: 3 + :hidden: + + Design + Contributing + Code of Conduct + +Getting Started +--------------- + +Install Python 3 +~~~~~~~~~~~~~~~~ + +Install Python 3.7 or newer from your Linux distribution’s package +repositories, or `python.org `__. + +Install Poetry +~~~~~~~~~~~~~~ + +`Poetry `__ is our preferred tool for +Python dependency management and packaging. We’ll use it to +automatically setup a ‘virtualenv’ and install everything we need. + +On Linux (or WSL) +^^^^^^^^^^^^^^^^^ + +.. code:: bash + + curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python - + source $HOME/.poetry/env + +If you are using WSL, installing Poetry on both Windows and Linux may +cause both platforms’ versions of Poetry to be on your path, as Windows +binaries are mapped into WSL’s ``PATH``. This means that the Linux +``poetry`` binary *must* appear in your ``PATH`` before the Windows +version, or this error will appear: + +:: + + `/usr/bin/env: ‘python\r’: No such file or directory` + +Adjust your ``PATH`` appropriately to fix it. + +On Windows (in PowerShell) +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. code:: powershell + + (Invoke-WebRequest -Uri https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py -UseBasicParsing).Content | python - + $env:PATH += ";$env:USERPROFILE\.poetry\bin" + +Clone LISA and ``cd`` into the Git repo +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code:: bash + + git clone -b andschwa/pytest https://github.com/microsoft/lisa.git + cd lisa + +Install Python dependencies +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code:: bash + + # Install the Python packages + poetry install + + # Enter the virtual environment + poetry shell + +Use LISA +~~~~~~~~ + +.. code:: bash + + # Run some self-tests + lisa --playbook=playbooks/test.yml selftests/ + + # Run a demo which deploys Azure resources + lisa --playbook=playbooks/demo.yaml + +Enable Azure +^^^^^^^^^^^^ + +To run the demo you’ll need the `Azure +CLI `__ tool installed and +configured: + +.. code:: bash + + # Install Azure CLI, make sure `az` is in your `PATH` + curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash + + # Login and set subscription + az login + az account set -s + +Contributing +------------ + +See the :doc:`contributing guidelines ` for developer +information! + +Contributor License Agreement +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This project welcomes contributions and suggestions. Most contributions +require you to agree to a Contributor License Agreement (CLA) declaring +that you have the right to, and actually do, grant us the rights to use +your contribution. For details, visit +https://cla.opensource.microsoft.com. + +When you submit a pull request, a CLA bot will automatically determine +whether you need to provide a CLA and decorate the PR appropriately +(e.g., status check, comment). Simply follow the instructions provided +by the bot. You will only need to do this once across all repos using +our CLA. + +This project has adopted the `Microsoft Open Source Code of +Conduct `__. For more +information see the `Code of Conduct +FAQ `__ or contact +opencode@microsoft.com with any additional questions or comments. + +Legal Notices +------------- + +Microsoft and any contributors grant you a license to the Microsoft +documentation and other content in this repository under the `Creative +Commons Attribution 4.0 International Public License +`__, see the +:doc:`LICENSE-DOCS ` file, and grant you a license to +any code in the repository under the `MIT License +`__, see the :doc:`LICENSE +` file. + +Microsoft, Windows, Microsoft Azure and/or other Microsoft products and +services referenced in the documentation may be either trademarks or +registered trademarks of Microsoft in the United States and/or other +countries. The licenses for this project do not grant you rights to use +any Microsoft names, logos, or trademarks. Microsoft’s general trademark +guidelines can be found at +https://go.microsoft.com/fwlink/?LinkID=254653. + +Privacy information can be found at https://privacy.microsoft.com/en-us/ + +Microsoft and any contributors reserve all other rights, whether under +their respective copyrights, patents, or trademarks, whether by +implication, estoppel or otherwise. + +.. |LISA/Pytest CI Workflow| image:: https://github.com/microsoft/lisa/workflows/LISA/Pytest%20CI%20Workflow/badge.svg?branch=andschwa%2Fpytest + :target: https://github.com/microsoft/lisa/actions?query=workflow%3A%22LISA%2FPytest+CI+Workflow%22 +.. |Code Style: black| image:: https://img.shields.io/badge/code%20style-black-000000.svg + :target: https://github.com/psf/black From c0283cd3774d4f6e0109781ab3660e96d92f76d2 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Wed, 6 Jan 2021 16:37:31 -0800 Subject: [PATCH 167/194] Add CI workflow to build documentation --- .github/workflows/ci-workflow.yaml | 18 ++++++++++++++++++ conf.py | 1 + 2 files changed, 19 insertions(+) diff --git a/.github/workflows/ci-workflow.yaml b/.github/workflows/ci-workflow.yaml index 81a9724e29..a40cae5ffc 100644 --- a/.github/workflows/ci-workflow.yaml +++ b/.github/workflows/ci-workflow.yaml @@ -42,3 +42,21 @@ jobs: - name: Run semantic analysis run: poetry run pytest --tb=auto --flake8 --mypy -m "flake8 or mypy" + + - name: Build documentation website + if: runner.os == 'Linux' && matrix.python-version == '3.9' + run: | + rm -rf docs/ + poetry run sphinx-build . docs/ + git config --local user.email 'lisval@microsoft.com' + git config --local user.name 'GitHub Action' + git add docs/ + git commit -m "Generated documentation" + + - name: Push to GitHub Pages branch + if: runner.os == 'Linux' && matrix.python-version == '3.9' + uses: ad-m/github-push-action@master + with: + branch: gh-pages + force: true + github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/conf.py b/conf.py index b52158786c..53848bcbd3 100644 --- a/conf.py +++ b/conf.py @@ -1,3 +1,4 @@ +# type: ignore # Configuration file for the Sphinx documentation builder. # # This file only contains a selection of the most common options. For a full From 4536ba45ad3ace860995351e2ccd4d49340725d1 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Thu, 7 Jan 2021 12:26:02 -0800 Subject: [PATCH 168/194] Setup autosummary to generate API documentation --- .gitignore | 1 + conf.py | 4 ++++ index.rst | 17 +++++++++++++++++ 3 files changed, 22 insertions(+) diff --git a/.gitignore b/.gitignore index e4a3b043c3..4f80922946 100644 --- a/.gitignore +++ b/.gitignore @@ -7,5 +7,6 @@ /*.xml /assets/ /_build/ +/modules/ __pycache__/ dist/ diff --git a/conf.py b/conf.py index 53848bcbd3..0b3f5d2158 100644 --- a/conf.py +++ b/conf.py @@ -31,6 +31,10 @@ "sphinx.ext.todo", ] +# Scan all found documents for autosummary directives, and generate +# stub pages for each, instead of using `sphinx-autogen` directly. +autosummary_generate = True + def linkcode_resolve(domain, info): """Configure linkcode extension.""" diff --git a/index.rst b/index.rst index 4fadbb9345..d927d80321 100644 --- a/index.rst +++ b/index.rst @@ -12,6 +12,7 @@ document ` for details. .. toctree:: :maxdepth: 3 + :caption: Documentation :hidden: Design @@ -108,6 +109,22 @@ configured: az login az account set -s +Python Modules +-------------- + +See the :doc:`technical specification document ` for design +details, and see the below table for auto-generated API documentation +of the framework. + +.. autosummary:: + :toctree: modules + :caption: API + :recursive: + + lisa + target + playbook + Contributing ------------ From 7296e662ac6bf5caf83bd3100b097b2faa808010 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Thu, 7 Jan 2021 12:31:58 -0800 Subject: [PATCH 169/194] Note use of Sphinx for documentation generation --- CONTRIBUTING.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ab91c5e8ef..d30ffbfce4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -375,8 +375,12 @@ on your way to being a “Pythonista” (a Python developer) writing “Pythonic ## Generating Documentation +We use [Sphinx](https://www.sphinx-doc.org/en/master/index.html) to generate our +[documentation](https://microsoft.github.io/lisa/). To build it locally (and +check all the links), use: + ```bash -sphinx-build . _build +sphinx-build -b linkcheck . _build ``` ## Future Sections From 2d7519a9873c1d89138e75ec7b409eaceee1a2c4 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Thu, 7 Jan 2021 12:48:32 -0800 Subject: [PATCH 170/194] Fix links to source in documentation --- conf.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/conf.py b/conf.py index 0b3f5d2158..36c3242030 100644 --- a/conf.py +++ b/conf.py @@ -40,13 +40,25 @@ def linkcode_resolve(domain, info): """Configure linkcode extension.""" if domain != "py": return None - if not info["module"]: + module = info["module"] + if not module: return None - filename = info["module"].replace(".", "/") + + # Map to subfolders. + if module.startswith("lisa"): + folder = "pytest-lisa" + elif module.startswith("target"): + folder = "pytest-target" + elif module.startswith("playbook"): + folder = "pytest-playbook" + else: + folder = "" + + filename = module.replace(".", "/") url = metadata["Project-Url"].split(", ")[1] # TODO: Update this branch to `main` branch after PR is merged. branch = "andschwa/pytest" - return f"{url}/blob/{branch}/{filename}.py" + return f"{url}/blob/{branch}/{folder}/{filename}.py" # Add any paths that contain templates here, relative to this directory. From 626a21556458351db01752992d59bf98307a0cde Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Thu, 7 Jan 2021 12:58:03 -0800 Subject: [PATCH 171/194] Setup autodoc to document class members --- conf.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/conf.py b/conf.py index 36c3242030..6427b29c71 100644 --- a/conf.py +++ b/conf.py @@ -31,6 +31,18 @@ "sphinx.ext.todo", ] +# Setup autodoc default options. +autodoc_member_order = "bysource" +autodoc_default_options = { + "members": True, + "member-order": "bysource", + "private-members": False, + "special-members": "__init__", + "ignore-module-all": True, + "undoc-members": True, + "show-inheritance": True, +} + # Scan all found documents for autosummary directives, and generate # stub pages for each, instead of using `sphinx-autogen` directly. autosummary_generate = True From 0f988cfb80c1bf32fbedea5ab8ed7745bbd525d3 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Thu, 7 Jan 2021 13:00:40 -0800 Subject: [PATCH 172/194] Make a couple members private to avoid documentation generation --- pytest-target/target/azure.py | 7 ++++--- pytest-target/target/plugin.py | 2 +- pytest-target/target/target.py | 12 +++++++----- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/pytest-target/target/azure.py b/pytest-target/target/azure.py index d3b1eeffd6..cba0cc404d 100644 --- a/pytest-target/target/azure.py +++ b/pytest-target/target/azure.py @@ -25,6 +25,7 @@ class AzureCLI(Target): # Custom instance attribute(s). internal_address: str + """Internal IP address of this target.""" @classmethod def schema(cls) -> Dict[Any, Any]: @@ -58,12 +59,12 @@ def _local(cls, *args: Any, **kwargs: Any) -> Result: return context.run(*args, **kwargs) # A class attribute because it’s defined. - az_ok = False + _az_ok = False @classmethod def check_az_cli(cls) -> None: """Assert that the `az` CLI is installed and logged in.""" - if cls.az_ok: # Shortcut if we already checked. + if cls._az_ok: # Shortcut if we already checked. return # E.g. on Ubuntu: `curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash` assert cls._local("az --version", warn=True), "Please install the `az` CLI!" @@ -76,7 +77,7 @@ def check_az_cli(cls) -> None: logging.info( f"Using account '{sub['user']['name']}' with subscription '{sub['name']}'" ) - cls.az_ok = True + cls._az_ok = True def create_boot_storage(self, location: str) -> str: """Create a separate resource group and storage account for boot diagnostics.""" diff --git a/pytest-target/target/plugin.py b/pytest-target/target/plugin.py index d09808e545..1727752424 100644 --- a/pytest-target/target/plugin.py +++ b/pytest-target/target/plugin.py @@ -1,4 +1,4 @@ -"""Provides and parameterizes the `pool` and `target` fixtures. +"""Provides and parameterizes the `target` fixture(s). # TODO: * Deallocate targets when switching to a new target. diff --git a/pytest-target/target/target.py b/pytest-target/target/target.py index eafabf1d2a..eff3d962fa 100644 --- a/pytest-target/target/target.py +++ b/pytest-target/target/target.py @@ -188,7 +188,7 @@ def delete(self) -> None: # Internal details follow: - platform_description = "The class name of the platform implementation." + _platform_description = "The class name of the platform implementation." @classmethod def get_defaults(cls) -> Tuple[Optional, Schema]: @@ -214,7 +214,7 @@ def get_defaults(cls) -> Tuple[Optional, Schema]: Optional( cls.__name__, default=Schema(cls.defaults()).validate({}), - description=cls.platform_description, + description=cls._platform_description, ), Schema(cls.defaults(), name=f"{cls.__name__}_Defaults", as_reference=True), ) @@ -244,7 +244,9 @@ def get_schema(cls) -> Schema: { # We’re adding ‘name’ and ‘platform’ keys. Literal("name", description="A friendly name for the target."): str, - Literal("platform", description=cls.platform_description): cls.__name__, + Literal( + "platform", description=cls._platform_description + ): cls.__name__, # Unpack the rest of the schema’s items. **cls.schema(), }, @@ -263,12 +265,12 @@ def get_platform(platform: str) -> Type[Target]: # Platform-agnostic functionality should be added here: - local_context = invoke.Context(config=invoke.Config(overrides=config)) + _local_context = invoke.Context(config=invoke.Config(overrides=config)) @classmethod def local(cls, *args: Any, **kwargs: Any) -> Result: """This patches Fabric's 'local()' function to ignore SSH environment.""" - return Target.local_context.run(*args, **kwargs) + return Target._local_context.run(*args, **kwargs) @retry(reraise=True, wait=wait_exponential(), stop=stop_after_attempt(3)) def ping(self, **kwargs: Any) -> Result: From 0f84dca14545b91efa15eb844b48137a445c77f4 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Thu, 7 Jan 2021 15:28:21 -0800 Subject: [PATCH 173/194] Generate TODO lists in documentation --- conf.py | 3 +++ pytest-lisa/lisa.py | 42 +++++++++++++++++--------------- pytest-playbook/playbook.py | 40 +++++++++++++++--------------- pytest-target/target/__init__.py | 31 ++++++++++++----------- pytest-target/target/azure.py | 21 ++++++---------- pytest-target/target/plugin.py | 34 +++++++++++++------------- pytest-target/target/target.py | 25 ++++++++++++------- 7 files changed, 105 insertions(+), 91 deletions(-) diff --git a/conf.py b/conf.py index 6427b29c71..03dc676860 100644 --- a/conf.py +++ b/conf.py @@ -73,6 +73,9 @@ def linkcode_resolve(domain, info): return f"{url}/blob/{branch}/{folder}/{filename}.py" +# Generate TODO list. +todo_include_todos = True + # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] diff --git a/pytest-lisa/lisa.py b/pytest-lisa/lisa.py index b5473cf087..1dfcc0c1d0 100644 --- a/pytest-lisa/lisa.py +++ b/pytest-lisa/lisa.py @@ -1,25 +1,29 @@ """A plugin for organizing, analyzing, and selecting tests. -This plugin provides the mark `pytest.mark.lisa`, aliased as `LISA`, +This plugin provides the mark ``pytest.mark.lisa``, aliased as ``LISA``, for marking up tests metadata beyond that which Pytest provides by -default. See the `lisa_schema` for the expected metadata input. - -Tests can be selected through a `playbook.yaml` file using the -criteria schema. For example:: - - criteria: - # Select all Priority 0 tests. - - priority: 0 - # Run tests with 'smoke' in the name twice. - - name: smoke - times: 2 - # Exclude all tests in Area "xdp" - - area: xdp - exclude: true - -# TODO: -* Provide test metadata statistics via a command-line flag. -* Assert every test has a LISA marker. +default. See the ``lisa_schema`` for the expected metadata input. + +Tests can be selected through a ``playbook.yaml`` file using the +criteria schema. For example: + +.. code-block:: yaml + + criteria: + # Select all Priority 0 tests. + - priority: 0 + # Run tests with 'smoke' in the name twice. + - name: smoke + times: 2 + # Exclude all tests in Area "xdp" + - area: xdp + exclude: true + +.. TODO:: + + * Review the ``criteria`` schema. + * Provide test metadata statistics via a command-line flag. + * Assert every test has a LISA marker. """ from __future__ import annotations diff --git a/pytest-playbook/playbook.py b/pytest-playbook/playbook.py index 7cc571a8a3..474132993d 100644 --- a/pytest-playbook/playbook.py +++ b/pytest-playbook/playbook.py @@ -1,17 +1,17 @@ """A plugin for creating, validating, and reading a playbook. -Use the `pytest_playbook_schema` hook to modify the schema dictionary -representing the data expected to be read and validated from a -`playbook.yaml` file, the path to which is provided by the user with -the command-line flag `--playbook`. +Use the ``pytest_playbook_schema`` hook to modify the schema +dictionary representing the data expected to be read and validated +from a ``playbook.yaml`` file, the path to which is provided by the +user with the command-line flag ``--playbook``. -This module's `playbook` attribute will hold the read and validated -data after all `pytest_configure` hooks have run. See -`pytest_playbook_schema` for example usage. +This module's ``data`` attribute will hold the read and validated data +after all ``pytest_configure`` hooks have run. See +``pytest_playbook_schema`` for example usage. -Remember not to use `from playbook import playbook` because then the -attribute will not contain the shared data. Instead use `import -playbook` and reference `playbook.playbook`. +Remember not to use ``from playbook import data`` because then the +attribute will not contain the shared data. Instead use ``import +playbook`` and reference ``playbook.data``. """ @@ -51,17 +51,19 @@ def pytest_playbook_schema(self, schema: Dict[Any, Any], config: Config) -> None """Update the Playbook's schema dict. The 'schema' is a mutable dict, and 'config' is optional. - Example usage:: + Example usage: - import playbook - from schema import Schema + .. code-block:: python - def pytest_playbook_schema(schema): - schema["targets"] = Schema({"name": str, "platform": str, "cpus": int}) + import playbook + from schema import Schema - def pytest_sessionstart(session): - for target in playbook.playbook["targets"]: - print(target["name"]) + def pytest_playbook_schema(schema): + schema["targets"] = Schema({"name": str, "platform": str, "cpus": int}) + + def pytest_sessionstart(session): + for target in playbook.playbook["targets"]: + print(target["name"]) """ @@ -89,7 +91,7 @@ def pytest_configure(config: Config) -> None: """Pytest hook to configure our plugin. This is set to be tried last so that all other plugins have been - loaded and defined their `pytest_playbook_schema` hooks. + loaded and defined their ``pytest_playbook_schema`` hooks. """ schema_dict: Dict[Any, Any] = dict() diff --git a/pytest-target/target/__init__.py b/pytest-target/target/__init__.py index 4a3aeac7d3..017661b9a0 100644 --- a/pytest-target/target/__init__.py +++ b/pytest-target/target/__init__.py @@ -11,20 +11,23 @@ targets listed in a `playbook.yaml` file. The fixture is parameterized across the list of provided targets. For example: - targets: - - name: Debian - platform: Azure - image: Debian:debian-10:10:latest - - name: Ubuntu - platform: Azure - image: Canonical:UbuntuServer:18.04-LTS:latest - - name: OpenSUSE - platform: Azure - image: SUSE:openSUSE-Leap:42.3:latest - -Will run all selected tests against each target. The `pool` fixture is -session-scoped and used by the `target` fixture to efficiently re-use -deployed targets. +.. code-block:: yaml + + platforms: + AzureCLI: + sku: Standard_DS2_v2 + + targets: + - name: Debian + platform: AzureCLI + image: Debian:debian-10:10:latest + + - name: Ubuntu + platform: AzureCLI + image: Canonical:UbuntuServer:18.04-LTS:latest + +Will run all selected tests against each target. The pool of targets +can be cached between runs with ``--keep-targets``. """ import pytest diff --git a/pytest-target/target/azure.py b/pytest-target/target/azure.py index cba0cc404d..4bf6797b33 100644 --- a/pytest-target/target/azure.py +++ b/pytest-target/target/azure.py @@ -1,4 +1,4 @@ -"""Provides an `Azure(Target)` implementation using the Azure CLI.""" +"""Provides an ``Azure(Target)`` implementation using the Azure CLI.""" from __future__ import annotations import json @@ -48,11 +48,8 @@ def defaults(cls) -> Dict[Any, Any]: @classmethod def _local(cls, *args: Any, **kwargs: Any) -> Result: - """A quiet version of `local()`. - - TODO: Consider adding this to the superclass. - - """ + """A quiet version of `local()`.""" + # TODO: Consider adding this to the superclass. config = Target.config.copy() config["run"]["hide"] = True context = invoke.Context(config=invoke.Config(overrides=config)) @@ -81,6 +78,7 @@ def check_az_cli(cls) -> None: def create_boot_storage(self, location: str) -> str: """Create a separate resource group and storage account for boot diagnostics.""" + # TODO: Use a different account per user. account = "pytestbootdiag" # This command always exits with 0 but returns a string. if self._local("az group exists -n pytest-lisa").stdout.strip() == "false": @@ -168,12 +166,9 @@ def deploy(self) -> str: return self.parse_data() def delete(self) -> None: - """Delete the entire allocated resource group. - - TODO: Delete VM '{self.name}'. Only if it was - the last VM then delete the entire resource group. - - """ + """Delete the entire allocated resource group.""" + # TODO: Delete VM '{self.name}'. Only if it was + # the last VM then delete the entire resource group. logging.debug(f"Deleting resource group '{self.group}-rg'") try: self.local(f"az group delete -n {self.group}-rg --yes --no-wait") @@ -192,5 +187,5 @@ def get_boot_diagnostics(self, **kwargs: Any) -> Result: ) def platform_restart(self) -> Result: - """TODO: Should this '--force' and redeploy?""" + """Should this use `--force` and redeploy?""" return self.local(f"az vm restart -n {self.name} -g {self.group}-rg") diff --git a/pytest-target/target/plugin.py b/pytest-target/target/plugin.py index 1727752424..bf178b0039 100644 --- a/pytest-target/target/plugin.py +++ b/pytest-target/target/plugin.py @@ -1,8 +1,9 @@ -"""Provides and parameterizes the `target` fixture(s). +"""Provides and parameterizes the ``target`` fixture(s). -# TODO: -* Deallocate targets when switching to a new target. -* Use richer feature/requirements comparison for targets. +.. TODO:: + + * Deallocate targets when switching to a new target. + * Use richer feature/requirements comparison for targets. """ from __future__ import annotations @@ -184,13 +185,12 @@ def get_targets(request: SubRequest) -> List[Target]: def fits(t: TargetData) -> bool: """Checks if a given Target fits the current search criteria. - Converting the cached JSON to a `TargetData` instance is + Converting the cached JSON to a ``TargetData`` instance is cheap and lets us use typed fields here. - TODO: Implement full feature comparison, etc. and not just - proof-of-concept string set comparison. - """ + # TODO: Implement full feature comparison, etc. and not + # just proof-of-concept string set comparison. logging.debug(f"Checking fit of {t}...") return ( not t.locked @@ -253,7 +253,7 @@ def cleanup_target(t: Target, request: SubRequest) -> None: def target(request: SubRequest) -> Iterator[Target]: """This fixture provides a connected target for each test. - It is parametrized indirectly in `pytest_generate_tests`. + It is parametrized indirectly in ``pytest_generate_tests``. """ t = get_target(request) @@ -263,9 +263,9 @@ def target(request: SubRequest) -> Iterator[Target]: @pytest.fixture def targets(request: SubRequest) -> Iterator[List[Target]]: - """This fixture is the same as `target` but gets a list of targets. + """This fixture is the same as ``target`` but gets a list of targets. - For example, use `pytest.mark.target(count=2)` to get a list of + For example, use ``pytest.mark.target(count=2)`` to get a list of two targets with the same parameters, in the same group. """ @@ -277,7 +277,7 @@ def targets(request: SubRequest) -> Iterator[List[Target]]: @pytest.fixture(scope="class") def c_target(request: SubRequest) -> Iterator[Target]: - """This fixture is the same as `target` but shared across a class.""" + """This fixture is the same as ``target`` but shared across a class.""" t = get_target(request) yield t cleanup_target(t, request) @@ -285,7 +285,7 @@ def c_target(request: SubRequest) -> Iterator[Target]: @pytest.fixture(scope="module") def m_target(request: SubRequest) -> Iterator[Target]: - """This fixture is the same as `target` but shared across a module.""" + """This fixture is the same as ``target`` but shared across a module.""" t = get_target(request) yield t cleanup_target(t, request) @@ -297,7 +297,7 @@ def m_target(request: SubRequest) -> Iterator[Target]: def pytest_sessionstart(session: Session) -> None: """Pytest hook to setup the session. - Gather the `targets` from the playbook. + Gather the targets from the playbook. First collect any user supplied defaults from the `platforms` key in the playbook, which will default to the given `defaults` @@ -313,10 +313,10 @@ def pytest_sessionstart(session: Session) -> None: def pytest_generate_tests(metafunc: Metafunc) -> None: - """Indirectly parametrize the `target` fixture based on the playbook. + """Indirectly parametrize the ``target`` fixture based on the playbook. - This hook is run for each test, so we gather the `targets` in - `pytest_sessionstart`. + This hook is run for each test, so we gather the targets in + ``pytest_sessionstart``. """ assert target_params, "This should not be empty!" diff --git a/pytest-target/target/target.py b/pytest-target/target/target.py index eff3d962fa..85d117f43c 100644 --- a/pytest-target/target/target.py +++ b/pytest-target/target/target.py @@ -1,4 +1,4 @@ -"""Provides the abstract base `Target` class.""" +"""Provides the abstract base ``Target`` class.""" from __future__ import annotations import dataclasses @@ -19,7 +19,7 @@ @dataclasses.dataclass class TargetData: - """This class holds serializable data for a `Target`. + """This class holds serializable data for a ``Target``. This is an internal detail. It is separated out so we can easily serialize to and from JSON in order to enable caching. By @@ -27,8 +27,10 @@ class TargetData: semantics of a `dataclass`, and fields added to subclasses don't interfere with serialization. - TODO: Consider using more from `dataclasses`, such as `field()` - and `__post_init__()`. + .. TODO:: + + Consider using more from `dataclasses`, such as `field()` and + `__post_init__()`. """ @@ -206,8 +208,11 @@ def get_defaults(cls) -> Tuple[Optional, Schema]: When generating the playbook's schema all the platforms' tuples are mapped into a single dict. - TODO: Assert that the set of key names in each `defaults()` is - a subset of the key names in the corresponding `schema()`. + .. TODO:: + + Assert that the set of key names in each ``defaults()`` is + a subset of the key names in the corresponding + ``schema()``. """ return ( @@ -227,7 +232,7 @@ def get_schema(cls) -> Schema: playbook's schema. Subclasses should not override this. We generate the whole definition by combining the values of - `cls.schema()` (which is defined by each platform's + ``cls.schema()`` (which is defined by each platform's implementation) with two required keys: * name: A friendly name for the target. @@ -236,8 +241,10 @@ def get_schema(cls) -> Schema: When generating the playbook's schema all the platforms' schemata are mapped into an 'any of' schema. - TODO: Perhaps elevate ‘name’ to the key, with the nested - schema as the value. + .. TODO:: + + Perhaps elevate ‘name’ to the key, with the nested schema + as the value. """ return Schema( From 48f6ad85ba03820a78c35aa93fee415fa15d80e5 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Thu, 7 Jan 2021 15:29:33 -0800 Subject: [PATCH 174/194] Keep CLA and Legal notices only in readme --- index.rst | 52 ++++------------------------------------------------ 1 file changed, 4 insertions(+), 48 deletions(-) diff --git a/index.rst b/index.rst index d927d80321..7ca8cdd199 100644 --- a/index.rst +++ b/index.rst @@ -8,7 +8,10 @@ verify the quality of Linux distributions on multiple platforms (such as Azure, Hyper-V, and bare metal). It is an opinionated collection of custom `Pytest `__ plugins, configurations, and tests. See the :doc:`technical specification -document ` for details. +document ` for details, and the `GitHub repository`_ for +sources. + +.. _GitHub repository: https://github.com/microsoft/lisa/tree/andschwa/pytest .. toctree:: :maxdepth: 3 @@ -131,53 +134,6 @@ Contributing See the :doc:`contributing guidelines ` for developer information! -Contributor License Agreement -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -This project welcomes contributions and suggestions. Most contributions -require you to agree to a Contributor License Agreement (CLA) declaring -that you have the right to, and actually do, grant us the rights to use -your contribution. For details, visit -https://cla.opensource.microsoft.com. - -When you submit a pull request, a CLA bot will automatically determine -whether you need to provide a CLA and decorate the PR appropriately -(e.g., status check, comment). Simply follow the instructions provided -by the bot. You will only need to do this once across all repos using -our CLA. - -This project has adopted the `Microsoft Open Source Code of -Conduct `__. For more -information see the `Code of Conduct -FAQ `__ or contact -opencode@microsoft.com with any additional questions or comments. - -Legal Notices -------------- - -Microsoft and any contributors grant you a license to the Microsoft -documentation and other content in this repository under the `Creative -Commons Attribution 4.0 International Public License -`__, see the -:doc:`LICENSE-DOCS ` file, and grant you a license to -any code in the repository under the `MIT License -`__, see the :doc:`LICENSE -` file. - -Microsoft, Windows, Microsoft Azure and/or other Microsoft products and -services referenced in the documentation may be either trademarks or -registered trademarks of Microsoft in the United States and/or other -countries. The licenses for this project do not grant you rights to use -any Microsoft names, logos, or trademarks. Microsoft’s general trademark -guidelines can be found at -https://go.microsoft.com/fwlink/?LinkID=254653. - -Privacy information can be found at https://privacy.microsoft.com/en-us/ - -Microsoft and any contributors reserve all other rights, whether under -their respective copyrights, patents, or trademarks, whether by -implication, estoppel or otherwise. - .. |LISA/Pytest CI Workflow| image:: https://github.com/microsoft/lisa/workflows/LISA/Pytest%20CI%20Workflow/badge.svg?branch=andschwa%2Fpytest :target: https://github.com/microsoft/lisa/actions?query=workflow%3A%22LISA%2FPytest+CI+Workflow%22 .. |Code Style: black| image:: https://img.shields.io/badge/code%20style-black-000000.svg From 0fe3ea7537119b61de6c2de732dc1e37264bdfff Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Thu, 7 Jan 2021 16:43:02 -0800 Subject: [PATCH 175/194] Update `plugin.py` docstrings --- pytest-target/target/plugin.py | 84 ++++++++++++++++++++-------------- 1 file changed, 49 insertions(+), 35 deletions(-) diff --git a/pytest-target/target/plugin.py b/pytest-target/target/plugin.py index bf178b0039..35b397292e 100644 --- a/pytest-target/target/plugin.py +++ b/pytest-target/target/plugin.py @@ -1,4 +1,4 @@ -"""Provides and parameterizes the ``target`` fixture(s). +"""Provides and parameterizes the :py:func:`.target` fixture(s). .. TODO:: @@ -35,7 +35,11 @@ def pytest_addoption(parser: Parser) -> None: - """Pytest hook to add our CLI options.""" + """Pytest `addoption hook`_ to add our CLI options. + + .. _addoption hook: https://docs.pytest.org/en/stable/reference.html#pytest.hookspec.pytest_addoption + + """ group = parser.getgroup("target") group.addoption( "--keep-targets", action="store_true", help="Keeps targets between runs." @@ -46,15 +50,17 @@ def pytest_addoption(parser: Parser) -> None: def pytest_playbook_schema(schema: Dict[Any, Any]) -> None: - """pytest-playbook hook to update the playbook schema. - - This adds `platforms` and `targets` keys to the playbook schema, - with their nested schemata accumulated from each platform's - implementations of `defaults()` and `schema()`. We do this by - iterating over the subclasses of `Target`, a handy feature of - Python that lets us automatically discover users' implementations, - even if they're defined in a local `conftest.py` Pytest - configuration file. + """:py:mod:`playbook` hook to update the playbook schema. + + This adds ``platforms`` and ``targets`` keys to the playbook + schema, with their nested schemata accumulated from each + platform's implementations of + :py:meth:`~target.target.Target.defaults` and + :py:meth:`~target.target.Target.schema`. We do this by iterating + over the subclasses of :py:class:`~target.target.Target`, a handy + feature of Python that lets us automatically discover users' + implementations, even if they're defined in a local + ``conftest.py`` Pytest configuration file. """ classes = Target.__subclasses__() @@ -99,10 +105,12 @@ def target_pool(config: Config) -> Generator[Dict[str, Any], None, None]: This handles access to the Pytest cache of serialized targets. The cache is a dict of ``{target.name: target.to_json()}``. We use a file lock to provide exclusive access even if Pytest is being run - in parallel with xdist. Entries have a `locked` property and must - only be modified during a session when locked by that session. - Locking means setting `locked` to `True` and updating the entry - before exiting this context manager. + in parallel with `pytest-xdist`_. Entries have a ``locked`` + property and must only be modified during a session when locked by + that session. Locking means setting ``locked`` to ``True`` and + updating the entry before exiting this context manager. + + .. _pytest-xdist: https://github.com/pytest-dev/pytest-xdist """ # TODO: Handle edge case where cache plugin is disabled. @@ -126,12 +134,12 @@ def delete_targets(config: Config) -> None: def pytest_configure(config: Config) -> None: - """Pytest hook to perform initial configuration. + """Pytest `configure hook`_ to perform initial configuration. - https://docs.pytest.org/en/stable/reference.html#pytest.hookspec.pytest_configure + .. _configure hook: https://docs.pytest.org/en/stable/reference.html#pytest.hookspec.pytest_configure We're registering our custom marker so that it passes - `--strict-markers`. + ``--strict-markers``. """ config.addinivalue_line( @@ -146,9 +154,9 @@ def pytest_configure(config: Config) -> None: def pytest_unconfigure(config: Config) -> None: - """Pytest hook to perform teardown. + """Pytest `unconfigure hook`_ to perform teardown. - https://docs.pytest.org/en/stable/reference.html#pytest.hookspec.pytest_unconfigure + .. _unconfigure hook: https://docs.pytest.org/en/stable/reference.html#pytest.hookspec.pytest_unconfigure """ if not config.getoption("keep_targets"): @@ -157,7 +165,7 @@ def pytest_unconfigure(config: Config) -> None: def get_target(request: SubRequest) -> Target: - """Common case of getting one target.""" + """Common case of getting one ``Target``.""" marker = request.node.get_closest_marker("target") count = marker.kwargs.get("count", 1) assert count == 1, "Use `targets` fixture with `count` instead!" @@ -165,7 +173,7 @@ def get_target(request: SubRequest) -> Target: def get_targets(request: SubRequest) -> List[Target]: - """This function gets or creates an appropriate number of `Target`s. + """This function gets or creates N ``Target`` instances. 1. Unpack request into params, required features, and count 2. Setup fitness criteria for target(s) @@ -183,7 +191,7 @@ def get_targets(request: SubRequest) -> List[Target]: with target_pool(request.config) as pool: def fits(t: TargetData) -> bool: - """Checks if a given Target fits the current search criteria. + """Checks if a given ``Target`` fits the current search criteria. Converting the cached JSON to a ``TargetData`` instance is cheap and lets us use typed fields here. @@ -235,7 +243,7 @@ def fits(t: TargetData) -> bool: def cleanup_target(t: Target, request: SubRequest) -> None: - """This is called by fixtures after they're done with a target.""" + """This is called by fixtures after they're done with a ``Target``.""" t.conn.close() mark: Optional[Mark] = request.node.get_closest_marker("target") assert mark is not None @@ -251,9 +259,9 @@ def cleanup_target(t: Target, request: SubRequest) -> None: @pytest.fixture def target(request: SubRequest) -> Iterator[Target]: - """This fixture provides a connected target for each test. + """This fixture provides a connected ``Target`` for each test. - It is parametrized indirectly in ``pytest_generate_tests``. + It is parametrized indirectly in :py:func:`pytest_generate_tests`. """ t = get_target(request) @@ -263,7 +271,7 @@ def target(request: SubRequest) -> Iterator[Target]: @pytest.fixture def targets(request: SubRequest) -> Iterator[List[Target]]: - """This fixture is the same as ``target`` but gets a list of targets. + """This fixture is the same as :py:func:`.target` but gets a ``Target`` list. For example, use ``pytest.mark.target(count=2)`` to get a list of two targets with the same parameters, in the same group. @@ -277,7 +285,7 @@ def targets(request: SubRequest) -> Iterator[List[Target]]: @pytest.fixture(scope="class") def c_target(request: SubRequest) -> Iterator[Target]: - """This fixture is the same as ``target`` but shared across a class.""" + """This fixture is the same as :py:func:`.target` but shared across a class.""" t = get_target(request) yield t cleanup_target(t, request) @@ -285,7 +293,7 @@ def c_target(request: SubRequest) -> Iterator[Target]: @pytest.fixture(scope="module") def m_target(request: SubRequest) -> Iterator[Target]: - """This fixture is the same as ``target`` but shared across a module.""" + """This fixture is the same as :py:func:`.target` but shared across a module.""" t = get_target(request) yield t cleanup_target(t, request) @@ -295,12 +303,14 @@ def m_target(request: SubRequest) -> Iterator[Target]: def pytest_sessionstart(session: Session) -> None: - """Pytest hook to setup the session. + """Pytest `sessionstart hook`_ to setup the session. + + .. _sessionstart hook: https://docs.pytest.org/en/stable/reference.html#pytest.hookspec.pytest_sessionstart Gather the targets from the playbook. - First collect any user supplied defaults from the `platforms` key - in the playbook, which will default to the given `defaults` + First collect any user supplied defaults from the ``platforms`` + key in the playbook, which will default to the given ``defaults`` implemented for each platform. Copy the defaults and then overwrite with the target's specific parameters. @@ -313,10 +323,14 @@ def pytest_sessionstart(session: Session) -> None: def pytest_generate_tests(metafunc: Metafunc) -> None: - """Indirectly parametrize the ``target`` fixture based on the playbook. + """Pytest `generate_tests hook`_ to indirectly parameterize :py:func:`.target`. + + .. _generate_tests hook: https://docs.pytest.org/en/stable/reference.html#pytest.hookspec.pytest_generate_tests - This hook is run for each test, so we gather the targets in - ``pytest_sessionstart``. + This takes the given targets (probably from the playbook) and + transforms them into parameters for all the tests using the + :py:func:`.target` fixture. Since this hook is run for each test, + so we gather the targets in :py:func:`pytest_sessionstart`. """ assert target_params, "This should not be empty!" From 17f435855424a5c2dfd542377661eb87b800a7eb Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Thu, 7 Jan 2021 16:43:08 -0800 Subject: [PATCH 176/194] =?UTF-8?q?Ignore=20long=20lines=20(they=E2=80=99r?= =?UTF-8?q?e=20inevitable=20with=20links)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .flake8 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.flake8 b/.flake8 index f855799a35..78708c2676 100644 --- a/.flake8 +++ b/.flake8 @@ -2,4 +2,4 @@ max-line-length = 88 select = B,BLK,C90,E,F,I,W max-complexity = 15 -extend-ignore = E203 +extend-ignore = E203,E501 From 3dcb7c6e3677745640f4a9f1d85f6109e8460633 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Thu, 7 Jan 2021 17:00:50 -0800 Subject: [PATCH 177/194] Update `target.py` docstrings --- pytest-target/target/azure.py | 2 +- pytest-target/target/target.py | 108 ++++++++++++++++++--------------- 2 files changed, 60 insertions(+), 50 deletions(-) diff --git a/pytest-target/target/azure.py b/pytest-target/target/azure.py index 4bf6797b33..fc6803ae5d 100644 --- a/pytest-target/target/azure.py +++ b/pytest-target/target/azure.py @@ -50,7 +50,7 @@ def defaults(cls) -> Dict[Any, Any]: def _local(cls, *args: Any, **kwargs: Any) -> Result: """A quiet version of `local()`.""" # TODO: Consider adding this to the superclass. - config = Target.config.copy() + config = Target._config.copy() config["run"]["hide"] = True context = invoke.Context(config=invoke.Config(overrides=config)) return context.run(*args, **kwargs) diff --git a/pytest-target/target/target.py b/pytest-target/target/target.py index 85d117f43c..64b886be28 100644 --- a/pytest-target/target/target.py +++ b/pytest-target/target/target.py @@ -1,4 +1,4 @@ -"""Provides the abstract base ``Target`` class.""" +"""Provides the abstract base :py:class:`~target.target.Target` class.""" from __future__ import annotations import dataclasses @@ -19,18 +19,18 @@ @dataclasses.dataclass class TargetData: - """This class holds serializable data for a ``Target``. + """This class holds serializable data for a :py:class:`Target`. This is an internal detail. It is separated out so we can easily serialize to and from JSON in order to enable caching. By decoupling these we prevent users from having to understand the - semantics of a `dataclass`, and fields added to subclasses don't + semantics of a ``dataclass``, and fields added to subclasses don't interfere with serialization. .. TODO:: - Consider using more from `dataclasses`, such as `field()` and - `__post_init__()`. + Consider using more from ``dataclasses``, such as ``field()`` + and ``__post_init__()``. """ @@ -42,12 +42,12 @@ class TargetData: locked: bool def to_json(self) -> Dict[str, Any]: - """Returns a JSON-serializable representation of `self`.""" + """Returns a JSON-serializable representation of the ``Target``.""" return dataclasses.asdict(self) @staticmethod def from_json(json: Dict[str, Any]) -> Target: - """Instantiates the correct subclass given the JSON representation.""" + """Instantiates the correct ``Target`` subclass given the JSON representation.""" cls = Target.get_platform(json["params"]["platform"]) return cls(**json) @@ -56,13 +56,16 @@ class Target(TargetData, metaclass=ABCMeta): """This class represents a remote Linux target. As a partially abstract base class, it is meant to be subclassed - to provide platform support. So `Target` as a class maps to the + to provide platform support. So ``Target`` as a class maps to the concept of a Linux target machine reachable via SSH (through - `self.conn`, an instance of `Fabric.Connection`). Each subclass of - `Target` provides the necessary implementation to instantiate an - actual Linux target, by deploying it on that platform. Each - _instance_ of a platform-specific subclass of `Target` maps to an - actual Linux target that has been deployed on that platform. + :py:attr:`conn`, an instance of `Fabric.Connection`_). Each + subclass of ``Target`` provides the necessary implementation to + instantiate an actual Linux target, by deploying it on that + platform. Each _instance_ of a platform-specific subclass of + ``Target`` maps to an actual Linux target that has been deployed + on that platform. + + .. _Fabric.Connection: https://docs.fabfile.org/en/stable/api/connection.html """ @@ -76,7 +79,7 @@ class Target(TargetData, metaclass=ABCMeta): # Setup a sane configuration for local and remote commands. Note # that the defaults between Fabric and Invoke are different, so we # use their Config classes explicitly later. - config = { + _config = { "run": { # Show each command as its run. "echo": True, @@ -97,19 +100,19 @@ def __init__( number: int = 0, locked: bool = True, ): - """Creates and deploys an instance of `Target`. + """Creates and deploys an instance of :py:class:`Target`. - * `group` is a unique ID for the group of associated resources - * `params` is the input parameters conforming to `schema()` - * `features` is set of arbitrary feature requirements - * `data` is the cached data for the target - * `number` is the numerical ID of this target in its group - * `locked` is the state of the target's availability + :param group: is a unique ID for the group of associated resources + :param params: is the input parameters conforming to `schema()` + :param features: is set of arbitrary feature requirements + :param data: is the cached data for the target + :param number: is the numerical ID of this target in its group + :param locked: is the state of the target's availability - Subclass implementations of `Target` do not need to (and - should not) override `__init__()` as it is setup such that all - platform-specific setup logic can be encoded in `deploy()` - instead, which this calls. + Subclass implementations of ``Target`` do not need to (and + should not) override :py:meth:`__init__` as it is setup such + that all platform-specific setup logic can be encoded in + :py:meth:`deploy` instead, which this calls. """ self.group = group @@ -122,7 +125,7 @@ def __init__( self.name = f"{self.group}-{self.number}" self.host = self.deploy() - fabric_config = self.config.copy() + fabric_config = self._config.copy() fabric_config["run"]["env"] = { # type: ignore # Set PATH since it’s not a login shell. "PATH": "/sbin:/usr/sbin:/usr/local/sbin:/bin:/usr/bin:/usr/local/bin" @@ -142,11 +145,14 @@ def schema(cls) -> Mapping[Any, Any]: """Must return a mapping for expected instance parameters. The items in this mapping are added to the playbook schema, so - they may contain objects from the `schema` library. Each - target in the playbook will have `name` and `platform` keys in - addition to those specified here (they're merged). Parameters - should generally be `schema.Optional`. If the parameter should - have a shared but mutable default value, set it in `defaults`. + they may contain objects from the `schema`_ library. Each + target in the playbook will have ``name`` and ``platform`` + keys in addition to those specified here (they're merged). + Parameters should generally be ``schema.Optional``. If the + parameter should have a shared but mutable default value, set + it in ``defaults``. + + .. _schema: https://github.com/keleshev/schema """ ... @@ -155,10 +161,11 @@ def schema(cls) -> Mapping[Any, Any]: def defaults(cls) -> Mapping[Any, Any]: """Can return a mapping for default parameters. - If specified, it must contain only `schema.Optional` elements, - where the names and types match those in `schema()`, but with - a set default value, and those in `schema()` should not - contain default values. This is used a base for each target. + If specified, it must contain only ``schema.Optional`` + elements, where the names and types match those in + :py:meth:`schema`, but with a set default value, and those in + :py:meth:`schema` should not contain default values. This is + used a base for each target. """ return {} @@ -167,13 +174,13 @@ def defaults(cls) -> Mapping[Any, Any]: def deploy(self) -> str: """Must deploy the target resources and return the hostname. - Subclass implementations can treat this like `__init__` with - `schema()` defining the input `params`. + Subclass implementations can treat this like ``__init__`` with + :py:meth:`schema` defining the input ``params``. - Data which should be cached must be saved to `self.data`. + Data which should be cached must be saved to :py:attr:`data`. - If `self.data` is populated then implementations should assume - they're refreshing a cached target. + If :py:attr:`data` is populated then implementations should + assume they're refreshing a cached target. """ ... @@ -201,9 +208,9 @@ def get_defaults(cls) -> Tuple[Optional, Schema]: The key is an optional literal, the name of the subclass for the platform, with a default value of the validated - `defaults()` schema when given no input (hence they must all - be optional). The value is reference schema definition - generated from the `defaults()` dict. + :py:meth:`defaults` schema when given no input (hence they + must all be optional). The value is reference schema + definition generated from the :py:meth:`defaults` dict. When generating the playbook's schema all the platforms' tuples are mapped into a single dict. @@ -232,11 +239,11 @@ def get_schema(cls) -> Schema: playbook's schema. Subclasses should not override this. We generate the whole definition by combining the values of - ``cls.schema()`` (which is defined by each platform's + :py:meth:`schema` (which is defined by each platform's implementation) with two required keys: - * name: A friendly name for the target. - * platform: The name of the subclass for the platform. + * ``name``: A friendly name for the target. + * ``platform``: The name of the subclass for the platform. When generating the playbook's schema all the platforms' schemata are mapped into an 'any of' schema. @@ -263,6 +270,7 @@ def get_schema(cls) -> Schema: @staticmethod def get_platform(platform: str) -> Type[Target]: + """Returns the :py:class:`Target` subclass for the named platform.""" cls: typing.Optional[typing.Type[Target]] = next( (x for x in Target.__subclasses__() if x.__name__ == platform), None, @@ -272,11 +280,11 @@ def get_platform(platform: str) -> Type[Target]: # Platform-agnostic functionality should be added here: - _local_context = invoke.Context(config=invoke.Config(overrides=config)) + _local_context = invoke.Context(config=invoke.Config(overrides=_config)) @classmethod def local(cls, *args: Any, **kwargs: Any) -> Result: - """This patches Fabric's 'local()' function to ignore SSH environment.""" + """This patches Fabric's ``local()`` function to ignore SSH environment.""" return Target._local_context.run(*args, **kwargs) @retry(reraise=True, wait=wait_exponential(), stop=stop_after_attempt(3)) @@ -293,7 +301,7 @@ def cat(self, path: str) -> str: class SSH(Target): - """The `SSH` platform simply connects to existing targets. + """This platform simply connects to existing targets. It does not deploy nor delete the target. The default ``host`` is ``localhost`` so this can be used for testing against the user's @@ -303,12 +311,14 @@ class SSH(Target): @classmethod def schema(cls) -> Dict[Any, Any]: + """Takes a ``host`` parameter.""" return { Optional("host", description="The address of the destination target."): str } @classmethod def defaults(cls) -> Dict[Any, Any]: + """Defaults to ``localhost``.""" return { Optional( "host", default="localhost", description="The default value for host." From 9f037c000d2bd6bdb68ef9158496106b6b5a35f0 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Mon, 11 Jan 2021 11:23:24 -0800 Subject: [PATCH 178/194] Add copyright headers and notice to documentation website Added to real sources but not demonstration tests. --- conf.py | 17 ++++++++--------- conftest.py | 2 ++ pytest-lisa/lisa.py | 2 ++ pytest-playbook/playbook.py | 2 ++ pytest-target/target/__init__.py | 2 ++ pytest-target/target/azure.py | 2 ++ pytest-target/target/plugin.py | 2 ++ pytest-target/target/target.py | 2 ++ 8 files changed, 22 insertions(+), 9 deletions(-) diff --git a/conf.py b/conf.py index 03dc676860..6ffe89f157 100644 --- a/conf.py +++ b/conf.py @@ -1,23 +1,22 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + # type: ignore -# Configuration file for the Sphinx documentation builder. -# -# This file only contains a selection of the most common options. For a full -# list see the documentation: -# https://www.sphinx-doc.org/en/master/usage/configuration.html +"""Configuration file for the Sphinx documentation builder. -# -- Project information ----------------------------------------------------- +https://www.sphinx-doc.org/en/master/usage/configuration.html +""" import importlib.metadata +from datetime import datetime metadata = importlib.metadata.metadata("LISA") project = metadata["Name"].upper() -project_copyright = "Microsoft" # TODO: Add year and verify. +copyright = f"{datetime.now().year} Microsoft Corporation" author = metadata["Author"] version = metadata["Version"] release = version -# -- General configuration --------------------------------------------------- - # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. diff --git a/conftest.py b/conftest.py index 642f7b25c8..ba780e5725 100644 --- a/conftest.py +++ b/conftest.py @@ -1,3 +1,5 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. """LISA tests' specific configurations go here. This file is essentially the staging ground for contributions to diff --git a/pytest-lisa/lisa.py b/pytest-lisa/lisa.py index 1dfcc0c1d0..2c03f5bf7b 100644 --- a/pytest-lisa/lisa.py +++ b/pytest-lisa/lisa.py @@ -1,3 +1,5 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. """A plugin for organizing, analyzing, and selecting tests. This plugin provides the mark ``pytest.mark.lisa``, aliased as ``LISA``, diff --git a/pytest-playbook/playbook.py b/pytest-playbook/playbook.py index 474132993d..7b815f0ef5 100644 --- a/pytest-playbook/playbook.py +++ b/pytest-playbook/playbook.py @@ -1,3 +1,5 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. """A plugin for creating, validating, and reading a playbook. Use the ``pytest_playbook_schema`` hook to modify the schema diff --git a/pytest-target/target/__init__.py b/pytest-target/target/__init__.py index 017661b9a0..3e5d320ff2 100644 --- a/pytest-target/target/__init__.py +++ b/pytest-target/target/__init__.py @@ -1,3 +1,5 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. """A plugin for creating, using, and managing remote targets. The abstract base `Target` class provides an interface for adding diff --git a/pytest-target/target/azure.py b/pytest-target/target/azure.py index fc6803ae5d..541668b706 100644 --- a/pytest-target/target/azure.py +++ b/pytest-target/target/azure.py @@ -1,3 +1,5 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. """Provides an ``Azure(Target)`` implementation using the Azure CLI.""" from __future__ import annotations diff --git a/pytest-target/target/plugin.py b/pytest-target/target/plugin.py index 35b397292e..81ac4e41e3 100644 --- a/pytest-target/target/plugin.py +++ b/pytest-target/target/plugin.py @@ -1,3 +1,5 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. """Provides and parameterizes the :py:func:`.target` fixture(s). .. TODO:: diff --git a/pytest-target/target/target.py b/pytest-target/target/target.py index 64b886be28..cbaf7ea9e3 100644 --- a/pytest-target/target/target.py +++ b/pytest-target/target/target.py @@ -1,3 +1,5 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. """Provides the abstract base :py:class:`~target.target.Target` class.""" from __future__ import annotations From 856705ccf2583f62ea211c53bf3db15d96f2f5d7 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Mon, 11 Jan 2021 11:25:40 -0800 Subject: [PATCH 179/194] Update index --- index.rst | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/index.rst b/index.rst index 7ca8cdd199..fdf558563c 100644 --- a/index.rst +++ b/index.rst @@ -6,7 +6,7 @@ Linux Integration Services Automation LISA is a Linux test automation framework with built-in test cases to verify the quality of Linux distributions on multiple platforms (such as Azure, Hyper-V, and bare metal). It is an opinionated collection of -custom `Pytest `__ plugins, +custom `Pytest `_ plugins, configurations, and tests. See the :doc:`technical specification document ` for details, and the `GitHub repository`_ for sources. @@ -25,16 +25,29 @@ sources. Getting Started --------------- +LISA is supported on almost any Linux or Windows installation provided +Python 3.7 (released in 2018) or newer is available and SSH can be +used to connect to the remote targets under test. The local SSH +configuration is respected so ``ProxyJump`` can be used. + Install Python 3 ~~~~~~~~~~~~~~~~ Install Python 3.7 or newer from your Linux distribution’s package -repositories, or `python.org `__. +repositories, or `python.org `_. + +On Ubuntu 20.04 and up, just run ``apt install python-is-python3``. + +Below that Ubuntu version, the ``python3`` package is out-of-date, so +use something like a `PPA`_ or `pyenv`_. + +.. _PPA: https://launchpad.net/~deadsnakes/+archive/ubuntu/ppa +.. _pyenv: https://github.com/pyenv/pyenv Install Poetry ~~~~~~~~~~~~~~ -`Poetry `__ is our preferred tool for +`Poetry `_ is our preferred tool for Python dependency management and packaging. We’ll use it to automatically setup a ‘virtualenv’ and install everything we need. @@ -100,7 +113,7 @@ Enable Azure ^^^^^^^^^^^^ To run the demo you’ll need the `Azure -CLI `__ tool installed and +CLI `_ tool installed and configured: .. code:: bash From 12c7050f2ba69eb7e6b4ea08491c95c4aeefad88 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Mon, 11 Jan 2021 16:57:08 -0800 Subject: [PATCH 180/194] Convert design from markdown to reStructuredText --- DESIGN.md | 982 ----------------------------------------------- DESIGN.rst | 1080 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 1080 insertions(+), 982 deletions(-) delete mode 100644 DESIGN.md create mode 100644 DESIGN.rst diff --git a/DESIGN.md b/DESIGN.md deleted file mode 100644 index 75bf589901..0000000000 --- a/DESIGN.md +++ /dev/null @@ -1,982 +0,0 @@ -# Technical Specification Document - -This document outlines the technical specifications for LISAv3. We are -evaluating the feasibility of leveraging -[Pytest](https://docs.pytest.org/en/stable/) as our test runner. - -Please see [PR #1065](https://github.com/LIS/LISAv2/pull/1065) for a working, -proof-of-concept prototype. - -Authored by Andrew Schwartzmeyer (he/him), version 0.3.0. - -## Why Pytest? - -Pytest is an [incredibly popular](https://docs.pytest.org/en/stable/talks.html) -MIT licensed open source Python testing framework. It has a thriving community -and plugin framework, with over 750 -[plugins](https://plugincompat.herokuapp.com/). Instead of writing (and -therefore maintaining) yet another test framework, we would do more with less by -reusing Pytest and existing plugins. This will allow us to focus on our unique -problems: organizing and understanding our tests, deploying necessary resources -(such as Azure, Hyper-V, or bare metal machines, collectively known as -“targets”), and analyzing our results. - -In fact, most of Pytest itself is implemented via [built-in -plugins](https://docs.pytest.org/en/stable/plugins.html), providing us with many -useful and well-documented examples. Furthermore, when others were confronted -with a problem similar to our own they also chose to use Pytest. -[Labgrid](https://github.com/labgrid-project/labgrid) is an open source embedded -board control library that delegated the testing framework logic to Pytest in -their [design](https://labgrid.readthedocs.io/en/latest/design_decisions.html), -and [U-Boot](https://github.com/u-boot/u-boot), an embedded board boot loader, -similarly leveraged Pytest in their -[tests](https://github.com/u-boot/u-boot/tree/master/test/py). KernelCI and -Avocado were also evaluated by the Labgrid developers at an [Embedded Linux -Conference](https://youtu.be/S0EJJM5bVUY) and both ruled out for reasons similar -to our own before they settled on Pytest. - -The [fundamental features](https://youtu.be/CMuSn9cofbI) of Pytest match our -needs very well: - -* Automatic test discovery, no boiler-plate test code -* Useful information when a test fails (assertions are introspected) -* Test and fixture [parameterization][] -* Modular setup/teardown via [fixtures][] -* Incredibly customizable (as detailed above) - -So all the logic for describing, discovering, running, skipping and reporting -results of the tests, as well as enabling and importing users’ plugins is -already written and maintained by the open source community. This leaves us to -focus on our hard and specific problems: creating an abstraction to launch the -necessary targets, organizing and publishing our tests, and reporting test -results upstream. Using Pytest would also allow us the space to abstract other -commonalities in our specific tests. In this way, LISAv3 could solve the -difficulties we have at hand without creating yet another test framework. - -Finally, by leveraging such a popular framework and reducing the amount of code -we need to maintain, we drastically increase our chances of receiving pull -requests instead of bug reports from users. This is important because despite -our best efforts it is practically guaranteed that as adoption of LISAv3 -increases, users will want changes to be made, and we need to empower them to do -so themselves. - -## What are we maintaining? - -The current proof-of-concept implementation uses the top-level `conftest.py` -file to define our “plugin” functionality. This works, but it is not ideal. I -believe that we will want to publish two open source Pytest plugins as packages -on [PyPI](https://pypi.org/), the Python Package Index: `pytest-target` and -`pytest-lisa`. We will also maintain our set of public “LISA” tests, but these -should simply install and use our plugins. - -The `pytest-target` plugin should encapsulate all our logic for _how_ and _when_ -to deploy targets (local or cloud virtual machines, or bare metal machines, and -all the associated resources), run tests on the specified targets, and delete -the targets. This includes specifying which features and resources each test -needs and each given target provides (such as number of cores, amount of RAM, -and other hardware like a GPU etc.), how to deploy and delete each target based -on its platform, and parameterization of the `target` fixture based on CLI or -YAML file input. In fact, some tests (like networking) will require multiple -targets at once. This plugin will need to manage resources intelligently, being -able to optimize for both time and cost, and make it easy for tests to request -and use various resources. - -The `pytest-lisa` plugin should encapsulate all our logic for how to organize -and select tests, as well as our opinions on displaying test results. This -includes the user modes, test metadata and inventory, test selection based on -criteria against that metadata, required and pre-configured upstream plugins, -and result notifiers. It will similarly support both CLI and YAML file input. - -We should strive to keep these plugins from depending on each other in order to -keep their scope well-defined. In the “LISA” repository of tests we will depend -on the two plugins and maintain additional [fixtures][] for our tests’ unique -requirements. Similarly, we and others may have private test repositories which -build upon the above by defining new platform support and internal service -integrations. The built-in plugin discovery of Pytest (via `conftest.py` files) -enables us to satisfy one of our requirements to “support plugins to orchestrate -the test environment.” - -Finally, a third smaller utility plugin, `pytest-schema` may be written in order -to share the common functionality of registering component schemata (e.g. -platform and target parameters from `pytest-target` and selection criteria from -`pytest-lisa`). This is somewhat of an implementation detail, but would be a -third and lower-level library we can publish. - -## pytest-target - -### How are targets provided and accessed? - -First we need to define “target” as an instance of a system-under-test. That is, -given some environment requirements, such an Azure image (URN) and size (SKU), a -target would be a virtual machine deployed by `pytest-target` with SSH access -provided to the requesting test. A target could optionally be pre-deployed and -simply connected. Some tests may request multiple targets as well. - -Pytest uses [fixtures][], which are the primary way of setting up test -requirements. They replace less flexible alternatives like setup/teardown -functions. It is through fixtures that we implement remote target -setup/teardown. Our `target` fixture returns a `Target` instance, which -currently provides: - -* Remote shell access via SSH -* Data including hostname / IP address -* Cross-platform ping functionality with exponential back-off -* Uploading of local files to arbitrary remote destinations -* Downloading of remote file contents into local string variable -* Asynchronous remote command execution with promises - -The `Azure(Target)` subclass additionally provides: - -* Automatic provisioning of an Azure VM given URN and SKU -* Allowing ICMP ping via Azure firewall rules -* Azure platform forced reboot by API -* Downloading boot diagnostics (serial console log) from platform - -The prototype demonstrates how easy it is to quickly implement these features. -As we need more features, they can be readily added and shared among tests. - -The `Target` class leverages [Fabric](https://www.fabfile.org/) which is a -popular high-level Python library for executing shell commands on remote systems -over SSH. Underneath the covers Fabric uses -[paramiko](https://docs.paramiko.org/en/stable/), the most popular low-level -Python SSH library. Fabric does the heavy lifting of safely connecting and -disconnecting from the node, executing the shell command (synchronously or -asynchronously), reporting the exit status, gathering the stdout and stderr, -providing stdin (or interactive auto-responses, similar to `expect`), uploading -and downloading files, and much more. In fact, these APIs are all available and -implemented for the local machine by the underlying -[Inovke](https://www.pyinvoke.org/) library, which is essentially a Python -`subprocess` wrapper with “a powerful and clean feature set.” - -Other test specific requirements, such as installing software and daemons, -downloading files from remote storage, or checking the state of our Bash test -scripts, would similarly be implemented by methods on the `Target` class or via -additional fixtures and thus shared among tests. - -### How do we interact with Azure? - -For Azure, we currently use the [Azure CLI](https://aka.ms/azureclidocs) to -deploy a virtual machine. For Hyper-V (and other virtualization platforms), we -would like to use [libvirt](https://libvirt.org/python.html), and for embedded / -bare metal environments we are evaluating -[labgrid](https://github.com/labgrid-project/labgrid). - -If possible, we do not want to use the [Azure Python -APIs](https://aka.ms/azsdk/python/all) directly because they are more -complicated (and less documented) than the [Azure -CLI](https://aka.ms/azureclidocs). With Invoke (as discussed above), `az` -becomes incredibly easy to work with. The Azure CLI lead developer states that -they have [feature parity](https://stackoverflow.com/a/50005660/1028665) and -that the CLI is more straightforward to use. Considering our ease-of-maintenance -requirement, this seems the apt choice, especially since the Azure CLI supports -deploying resources with [ARM -templates](https://docs.microsoft.com/en-us/azure/azure-resource-manager/templates/deploy-cli). -If it later becomes necessary to use the Python APIs directly, that is, of -course, still doable (and we can reuse existing code doing it). - -On the topic of “servicing” the Azure CLI, its developers state that “at command -level, packages only upgrading the PATCH version guarantee backward -compatibility.” The tool is also intended to be used in scripts, so servicing -would amount to documenting the tested version and having the Azure class check -that it’s compatible before using it (or warning and then trying its best). - -### What’s the `Target` class? - -In version 0.1 of this design document we detailed a planned refactor of what -was then called the `Node` class. This has since been executed with just a few -modifications (one being the rename to `Target`, as `Node` was found to be an -overloaded term in the context of data centers). This class and its subclasses -are decoupled from Pytest, and are used via fixtures. It looks like this: - -```python -from abc import ABC, abstractmethod -from schema import Schema -import fabric - -class Target(ABC): - parameters: Mapping[str, str] - features: Set[str] - name: str - host: str - conn: fabric.Connection # Provides run, sudo, get, put etc. - - def __init__(...): - ... - self.host = self.deploy() - self.conn = fabric.Connection(self.host) - - @classmethod - @property - @abstractmethod - def schema(cls) -> Schema: - """Must return the parameters schema for setup.""" - ... - - @abstractmethod - def deploy(self) -> str: - """Must deploy the target resources and return hostname.""" - ... - - @abstractmethod - def delete(self) -> None: - """Must delete the target resources.""" - ... - - @classmethod - def local(...) -> Result: - """Runs a local shell command.""" - ... -``` - -#### How are platforms implemented? - -Platform support is implemented by subclassing `Target` and defining the -`schema` property, `deploy` method, `delete` method, and any platform-specific -methods. Using the `__subclasses__` attribute of `Target` the available -platforms and their parameter schemata are automatically gathered from users’ -own `conftest.py` files and other plugins. This enables the `target` fixture to -dynamically instantiate a target from the gathered requirements and parameters. - -For example, the `Azure(Target)` class defines its required parameters using the -[schema][] library like this: - -```python -from schema import Optional, Schema -from target import Target - -class Azure(Target): - ... - schema: Schema = Schema( - { - # TODO: Maybe validate as URN or path etc. - "image": str, - Optional("sku", default="Standard_DS1_v2"): str, - Optional("location", default="eastus2"): str, - Optional("networking", default=""): str, - } - ) -``` - -In the YAML playbook, a set of Azure targets can then be defined like this: - -```yaml -targets: - - name: Debian - platform: Azure - parameters: - image: credativ:Debian:9:9.0.201706190 - location: westus2 - - - name: Ubuntu - platform: Azure - parameters: - image: UbuntuLTS - sku: Standard_DS3_v2 -``` - -These targets are then used to parameterize the `target` fixture in the -[pytest_generate_tests][] hook (see below for more details). - -This demonstrated how we can have platforms define their own schema and register -that schema automatically. A pending update to this is to have two schemata per -`Target` subclass: target-level and platform-level (the former is what’s -demonstrated above, the latter would be common settings, such as subscription). - -#### How are requirements examined? - -The `features` attribute is currently a set of strings and (combined with the -parameters dictionary) was used to demonstrate how we can test if an existing -target instance (representing a deployed machine) met a test’s requirements. It -should be updated with a `Requirements` class that represents all physical -attributes of the target, and a `requires` Pytest mark should be added which -takes instances of this class. Two `Requirements` should be comparable to -determine if one set meets (or exceeds) the other set. Existing code that does -this can be reused here. - -#### How do we share common tasks? - -Common tasks for targets like rebooting and pinging should be implemented on the -`Target` class, and platform-specific tasks on the respective subclass. - -Methods available from `Connection` include `run()` and `sudo()` which are used -to easily run arbitrary commands, and `get()` and `put()` to download and upload -arbitrary files. - -The `cat()` method wraps `get()` and returns the file as data in a string. This -makes test code like this possible: - -```python -assert target.conn.cat("state.txt") == "TestCompleted" -``` - -A `reboot()` method should be added that first tries to use `sudo("reboot", -timeout=5)` (with a short timeout to avoid a hung SSH session). It should retry -with an exponential back-off to see if the machine has rebooted by checking -either `uptime` or the existence of a file created before the reboot. This is to -avoid having to `sleep()` and just guess the amount of time it takes to reboot. - -A `restart()` method should “power cycle” the machine using the platform’s API, -and thus is in abstract method. - -Other tools and shared logic should be implemented as necessary. A major area of -concern is the automatic and package-manager agnostic installation of necessary -tools, much of which has been implemented previously and can be reused. - -### How are targets requested and managed? - -We implement a pair of Pytest fixtures to provide targets. The first is the -`pool` fixture, which looks like: - -```python -@pytest.fixture(scope="session") -def pool(request: SubRequest) -> Iterator[List[Target]]: - """This fixture tracks all deployed target resources.""" - targets: List[Target] = [] - yield targets - for t in targets: - t.delete() -``` - -The `pool` fixture is setup once at the beginning of the test session, at which -point the `targets` list is then provided as input to every instance of the -`target` fixture. While currently a list, to support optimal scheduling we will -likely want to use a priority queue, where the priority of a target represents -its cost (whether in terms of time or money), allowing us to provide either the -fastest or the cheapest target to each request. Targets not in use will be -deallocated, and all targets will be automatically deleted after the tests are -finished (unless the user requested otherwise, in which case they’ll be cached). - -Note that cross-session [caching](https://docs.pytest.org/en/stable/cache.html) -is provided by Pytest, and very easy to work with. An early prototype -implemented a `--keep-vms` flag successfully, and this will be implemented again -with the updated design. - -The second is the `target` fixture, which looks like: - -```python -@pytest.fixture -def target(pool: List[Target], request: SubRequest) -> Iterator[Target]: - """This fixture provides a connected target for each test.""" - platform: Type[Target] = playbook.PLATFORMS[request.param["platform"]] - parameters: Dict[str, Any] = request.param["parameters"] - marker = request.node.get_closest_marker("lisa") - features = set(marker.kwargs["features"]) - - # TODO: If `t` is not already in use, deallocate the previous target. - for t in pool: - if isinstance(t, platform) and t.parameters == parameters and t.features >= features: - yield t - break - else: - t = platform(parameters, features) - pool.append(t) - yield t - t.connection.close() -``` - -This is obviously still an early implementation, but it is viable. By using the -[pytest_collection_modifyitems][] hook to sort (and so group) the tests by their -requirements, the tests would efficiently reuse targets. This fixture is -indirectly parameterized during setup with the [pytest_generate_tests][] hook. -Test and fixture [parameterization][] is a huge feature of Pytest. When we -parameterize the `target` fixture for multiple targets (e.g. “Ubuntu” and -“Debian”), Pytest automatically creates a set of tests for each target. So -`test_smoke` turns into `test_smoke[Ubuntu]` and `test_smoke[Debian]`. This -allows us to run a collection of tests against multiple targets with ease. These -targets are defined in a YAML file and validated against the parameters -collected from the previously described platform subclasses. - -The entire implementation looks like so: - -```python -TARGETS: List[Dict[str, Any]] = [] -TARGET_IDS: List[str] = [] - -def pytest_configure(config: Config) -> None: - book = get_playbook(config.getoption("--playbook")) - for t in book.get("targets", []): - TARGETS.append(t) - TARGET_IDS.append(t["name"]) - -def pytest_generate_tests(metafunc: Metafunc) -> None: - if "target" in metafunc.fixturenames: - assert TARGETS, "No targets specified!" - metafunc.parametrize("target", TARGETS, True, TARGET_IDS) -``` - -The function `get_playbook()` only imports the [PyYAML][] library, opens the -playbook file `f` within a context manager, and returns -`playbook.schema.validate(yaml.load(f))`. This is leveraging Pytest’s existing -parameterization technology to achieve one of our “test entrance” goals of -requesting environments with a YAML playbook, and one of our “test parameter -validation” goals of validating platforms before executing tests so that we can -fail fast if a target has insufficient information to be setup. Parsing the same -parameters from a CLI can also be implemented. - -Finally, once the `target` fixture has returned a working and sanity-checked -environment to the requesting test, the test is capable of examining any and all -attributes of the `Target` and quickly marking itself as skipped, expected to -fail, or failed before executing the body of the test. Our static type checking -enables developers to ensure that the platform they requested supports all -methods and fields they use by annotating the test’s `target` parameter with the -expected platform type (or types). Ensuring the effectiveness of this type -checking will require us to carefully update our platform implementations, and -not rely on arbitrary objects of data. (For example, add an `internal_address` -field to `Azure`, don’t just look up `data["internal_address"]`.) - -### How are tests executed in parallel? - -While our original list of goals stated that we want to run tests “in parallel” -we were not specific about what was meant, and the topic of parallelism and -concurrency is understandably complex. We certainly don’t mean running two tests -at once on the same target, as this would undoubtedly lead to flaky tests. - -Assuming that we care about a set of tests passing on a particular image and -size combination, but not necessarily on a particular deployed instance, then we -can run tests concurrently by deploying multiple “identical” targets and -splitting the tests across them. The tests would still run in isolation on each -target. This sounds hard, but actually it’s practically free with Pytest via -[pytest-xdist][]. - -The default `pytest-xdist` implementation simply takes the list of tests and -runs them in a round-robin fashion with the desired number of executors. We’ve -talked at length about being able to schedule groups of tests to run in -particular executors and using particular targets. While there are many paths -open to us, this plugin actually provides a hook, `pytest_xdist_make_scheduler` -that exists specifically to “implement custom tests distribution logic.” - -Figuring out the requirements of our test scheduler and designing the best -algorithm will require further discussion and design review. For the purposes of -moving forward, we are not blocked, as the eventual implementation can be -dropped in-place with minimal effort. - -## pytest-lisa - -### What are the user modes? - -Because Pytest is incredibly customizable, we want to provide a few sets of -reasonable default configurations for some common scenarios. We will add a flag -like `--lisa-mode=[dev,debug,ci,demo]` to change the default options and output -of Pytest. Doing so is readily supported by Pytest via the [pytest_addoption][] -and [pytest_configure][] hooks. We call these the provided “user modes.” Note -that by “output” we mean not just logging (because that implies the Python -`logger` module, which Pytest allows full control over) but also commands’ -stdout and stderr as well as Pytest-provided information. - -* The dev(eloper) mode is intended for use by test developers while writing a - new test. It is verbose, caches the deployed VMs between runs, and generates a - digestible [HTML](https://pypi.org/project/pytest-html/) report. - -* The debug mode is like dev mode but with all possible information shown, and - will open the Python debugger automatically on failures (which is provided by - Pytest with the `--pdb` flag). - -* The CI mode will be fairly quiet on the console, showing all test results, but - putting the full info output into the generated report file (HTML for sharing - with humans and - [JUnit](https://docs.pytest.org/en/stable/_modules/_pytest/junitxml.html) for - the associated CI environment, which presents as native test results). - -* The demo mode will show the “executive summary” (a lot like CI, but finely - tuned for demos). For example, what `make smoke` currently shows. - -### How are tests described? - -The built-in [pytest-mark](https://docs.pytest.org/en/stable/mark.html) plugin -already provides functionality for adding metadata to tests, where we -specifically want: - -* Platform: used to skip tests inapplicable to the current system-under-test -* Category: our high-level test organization -* Area: feature being tested -* Priority: self-explanatory -* Tags: optional additional metadata for test organization - -We simply reuse this with minimal logic to enforce our required metadata, with -sane defaults (perhaps setting the area to the name of the module), and to list -statistics about our test coverage. This is already included in the prototype. -It looks like this: - -```python -import pytest - -@pytest.mark.lisa(platform="Azure", category="Functional", priority=0, area="LIS_DEPLOY") -def test_lis_driver_version(target: Azure) -> None: - """Checks that the installed drivers have the correct version.""" - ... -``` - -This is a functional example, which takes zero implementation. With this simple -decorator, all test [collection hooks][] can introspect the metadata, enforce -required parameters and set defaults, select tests based on arbitrary criteria, -and list test coverage statistics (test inventory). Designing and implementing -the test inventory algorithm is still under development, but it’s tractable. - -Note that Pytest leverages Python’s docstrings for built-in documentation (and -can even run tests discovered in such strings, like doctest). Hence we do not -have a separate field for the test’s documentation. As long as we continue to -follow the practice of using docstrings for our modules, classes, and functions, -we can automatically use [pydoc](https://docs.python.org/3/library/pydoc.html) -to generate full documentation for each plugin and test. - -Being just Python code, this decorator need not be `@pytest.mark.lisa(...)` but -can trivially be provided as simply `@LISA(...)`. In fact, we provide this in -`lisa.py` with: - -```python -LISA = pytest.mark.lisa - -@LISA(...) -def test_something(...) -``` - -Currently we validate the parameters given to this mark during test collection, -by using the following code, which leverages the [schema][] library: - -```python -from schema import Optional, Or, Schema - -lisa_schema = Schema( - { - "platform": str, - "category": Or("Functional", "Performance", "Stress", "Community", "Longhaul"), - "area": str, - "priority": Or(0, 1, 2, 3), - Optional("tags", default=list): [str], - }, -) - -def validate(mark: Mark) -> None: - """Validate each test's LISA parameters.""" - assert not mark.args, "LISA marker cannot have positional arguments!" - mark.kwargs.update(lisa_schema.validate(mark.kwargs)) -``` - -In the future we could change `LISA` to be a function with these keyword -arguments so that IDE auto-completion is enabled. However, this is not mandatory -to move forward, and parameter validation is enabled succinctly with the above, -which satisfies one of our “test parameter validation” requirements. - -This mark also does need to be repeated for each test, as marks can be scoped to -a module, and so one line could describe defaults for every test in a file, with -individual tests overriding parameters as needed. - -In the current implementation, we also take a `features: List[str]` argument -that is used to prove the concept deploying (or reusing) a target based on the -test’s required and the target’s available sets of features. However, as we move -forward we should define a separate `requires` mark that takes well-defined -classes describing the minimal required resources for a test. This will be part -of the refactor into the two Pytest plugins mentioned above. Coupled with the -test’s requested `target` fixture being parameterized (see discussion in -`pytest-target`) this demonstrates at least one way we can satisfy our “test run -planner/scheduler” requirement. - -Furthermore, we have a prototype -[generator](https://github.com/LIS/LISAv2/tree/pytest/generator) which parses -LISAv2 XML test descriptions and generates stubs with this mark filled in -correctly. - -### How are tests selected? - -Pytest already allows a user to specify which exact tests to run: - -* Listing folders on the CLI (see below on where tests should live) -* Specifying a name expression on the CLI (e.g. `-k smoke and xdp`) -* Specifying a mark expression on the CLI (e.g. `-m functional and not slow`) - -We can also implement any other mechanism via the -[pytest_collection_modifyitems][] hook. The proof-of-concept supports gathering -selection criteria from a YAML file: - -```yaml -criteria: - # Select all Priority 0 tests. - - priority: 0 - # Run tests with 'smoke' in the name twice. - - name: smoke - times: 2 - # Exclude all tests in Area "xdp" - - area: xdp - exclude: true -``` - -This criteria is validated against the following [schema][]: - -```python -from schema import Schema, Optional - -criteria_schema = Schema( - { - # TODO: Validate that these strings are valid regular - # expressions if we change our matching logic. - Optional("name", default=None): str, - Optional("area", default=None): str, - Optional("category", default=None): str, - Optional("priority", default=None): int, - Optional("tags", default=list): [str], - Optional("times", default=1): int, - Optional("exclude", default=False): bool, - } -) -``` - -The test collection is then modified using the Pytest hook, -[pytest_collection_modifyitems][]: - -```python -def pytest_collection_modifyitems( - session: Session, config: Config, items: List[Item] -) -> None: - included: List[Item] = [] - excluded: List[Item] = [] - - def select(item: Item, times: int, exclude: bool) -> None: - if exclude: - excluded.append(item) - else: - for _ in range(times - included.count(item)): - included.append(item) - - for c in criteria: # Where `criteria` is from the schema. - for item in items: - marker = item.get_closest_marker("lisa") - if not marker: - # Not all tests will have the LISA marker, such as - # static analysis tests. - continue - i = marker.kwargs - if any( - [ - c["name"] and c["name"] in item.name, - c["area"] and c["area"].casefold() == i["area"].casefold(), - c["category"] - and c["category"].casefold() == i["category"].casefold(), - c["priority"] and c["priority"] == i["priority"], - c["tags"] and set(c["tags"]) <= set(i["tags"]), - ] - ): - select(item, c["times"], c["exclude"]) - items[:] = [i for i in included if i not in excluded] -``` - -Together, the CLI support and YAML playbook satisfy one of our “test entrance” -requirements. We can also generate our own binary called `lisa` which simply -delegates to Pytest, if we really want to do so. - -Because this is simply a Python list, we can also sort the tests according to -our needs, such as by priority. If the `python-targets` plugin has already -sorted by requirements, that’s just fine, Python’s `sorted()` built-in is -guaranteed to be stable (meaning we can sort in multiple passes). - -### How are results reported? - -Parsing the results of a large test suite can be difficult. Fortunately, because -Pytest is a testing framework, there already exists support for generating -excellent reports. For developers, the -[HTML](https://pypi.org/project/pytest-html/) report is easy to read: it is -self-contained, holds all the results and logs, and each test can be expanded -and collapsed. Tests which were rerun are recorded separately. For CI pipelines, -Pytest has integrated -[JUnit](https://docs.pytest.org/en/stable/_modules/_pytest/junitxml.html) XML -test report support. This is the standard method of reporting results to CI -servers like Jenkins and are natively parsed into the CI system’s built-in test -display page. Finally, Azure DevOps pipelines are even supported with a -community plugin -[pytest-azurepipelines](https://pypi.org/project/pytest-azurepipelines/) which -enhances the standard JUnit report for ADO. - -One of our requirements is to support the lookup of previous tests’ execution -metrics, such as recorded performance metrics and duration, so that performance -tests can check regressions. This is the perfect example of carrying a small -fixture which provides access to our internal database and is dynamically added -to our tests when run internally, and the tests can lookup and record whatever -they need through the fixture. - -However, we also have internal requirements to report test results throughout -the test life cycle to a database (the “result manager” and “progress tracker”) -to be consumed by other tools. In this sense, LISAv3 (the composition of our -published plugins, tests, and fixtures) is simply a producer, and the consumers -can parse the test results, send emails, archive the collected logs, update a -GUI display of test progress, etc. Our repository’s `conftest.py` can implement -the necessary logic using Pytest’s ample [test running hooks][]. In particular, -the hook [pytest_runtest_makereport][] is called for each of the setup, call and -teardown phases of a test. As such it can used for precisely this purpose. - -### How is setup, run, and cleanup handled? - -Pytest strives to require minimal boiler-plate code. Thus the classic -“xunit-style” of defining a class with setup and teardown functions in addition -to test functions is not recommended (nor necessary). Generally Pytest expects -[fixtures][] to be used for dependency injection (which is what setup/teardown -functions usually do). For users that really want the classic style, it is -nonetheless fully -[supported](https://docs.pytest.org/en/stable/xunit_setup.html) and documented -(and can be applied at the module, class, and method scopes). Thus our “test -runner” requirement is satisfied. - -### How are tests timed out? - -The [pytest-timeout](https://pypi.org/project/pytest-timeout/) plugin provides -integrated timeouts via `@pytest.mark.timeout()`, a configuration -file option, environment variable, and CLI flag. The Fabric library provides -timeouts in both the configuration and per-command usage. These are already used -to satisfaction in the prototype. Additionally, Pytest has built-in support for -measuring the duration of each fixture’s setup and teardown and each test (it’s -simply the `--durations` and `--durations-min` flags). - -### How are tests organized? - -That is, what does a folder of tests map to: a platform, feature, or owner? - -In my opinion it is likely to be both. Tests which are common to a platform and -written by our team are probably best placed in a folder like `tests/azure` -whereas tests for a particular scenario which limits their image and SKU -applicability should be in a folder like `tests/acc`. It’s going to depend on -how often the tests are run together. - -Because Pytest can run tests and `conftest.py` files from arbitrary folders, -maintaining sets of tests and plugins separately from the base LISA repository -is easy. Custom repositories with new tests, plugins, fixtures, -platform-specific support, etc. can simply be cloned anywhere, and provided on -the command-line to Pytest. - -Test authors should keep tests which share requirements and are otherwise -similar to a single module (Python file). Not only is this well-organized, but -because marks can be applied at the module level, setting all the tests to be -skipped or expected to fail (with the built-in `skip` and `xfail` Pytest marks) -becomes even easier. - -An open question is if we really want to bring every test from LISAv2 directly -over, or if we should carefully analyze our tests to craft a new set of -high-level scenarios. An interesting result of reorganizing and rewriting the -tests would be the ability to have test layers, where the result of a high-level -test dictates if the tests below it should be skipped. If it passes, it implies -the tests underneath it would pass, and so skips them; but if it fails, the next -test below it runs and so on until a passing layer is found. - -### How will we port LISAv2 tests? - -Given the above, we still must decide if we want to put the engineering effort -into porting _every_ LISAv2 test. However, the prototype started by porting the -`LIS-DRIVER-VERSION-CHECK` test, proving that tests which exclusively use Bash -scripts are trivially portable. Unfortunately, most tests use an associated -PowerShell script which is tightly coupled to the LISAv2 framework. - -We believe that it is _possible_ to port these tests without untoward -modifications. We would need to write a mock library that implements (or stubs -where appropriate) LISAv2 framework functionality such as -`Provision-VMsForLisa`, `Copy-RemoteFiles`, `Run-LinuxCmd`, etc., and provides -both the expected “global” objects and the test function parameters `AllVmData` -and `CurrentTestData`. - -This work needs to be done regardless of the approach we take with our framework -(leveraging Pytest or writing our own), and it is not inconsequential work. It -needs to be thoroughly planned and executed, and is certainly a ways off. - -### How are tests and functions retried? - -Testing remote targets is inherently flaky, so we take a two-pronged approach to -dealing with the flakiness. - -The [pytest-rerunfailures](https://pypi.org/project/pytest-rerunfailures/) -plugin will be used to easily mark a test itself as flaky. It has the nice -feature of recording each rerun in the produced report. It looks like this: - -```python -@pytest.mark.flaky(reruns=5) -def test_something_flaky(...): - """This fails most of the time.""" - ... -``` - -Note that there is an open -[bug](https://github.com/pytest-dev/pytest-rerunfailures/issues/51) in this -plugin which can cause issues with fixtures using scopes other than “function” -but it can be worked around. - -The [Tenacity](https://tenacity.readthedocs.io/en/latest/) library should be -used to retry flaky functions that are not tests, such as downloading boot -diagnostics or pinging a node. As the modern Python retry library it has -easy-to-use decorators to retry functions (and context managers to use within -functions), as well as excellent wait and timeout support. It looks like this: - -```python -from tenacity import retry, stop_after_attempt, wait_exponential - -class Node: - ... - @retry(reraise=True, wait=wait_exponential(), stop=stop_after_attempt(3)) - def ping(self, **kwargs): - """Ping the node from the local system in a cross-platform manner.""" - flag = "-c 1" if platform.system() == "Linux" else "-n 1" - return self.local(f"ping {flag} {self.host}", **kwargs) - ... -``` - -We can additionally list a test twice when modifying the items collection, as -implemented in the criteria proof-of-concept. However, given the above -abilities, this may not be desired. - -## What does the “flow” of Pytest look like? - -This is best described in Pythonic pseudo-code, where the context manager -encapsulates each scope and the for loop encapsulates processing: - -```python -pool_fixture: a session-scoped context manager -target_fixture: a function-scoped context manager -items: a collection of tests -targets: a collection of targets -criteria: a collection of test selection criteria - -def pytest_addoption(parser): - """Add CLI options etc.""" - parser.addoption("--playbook", type=Path) - -pytest_addoption(parser) # Pytest fills in parser. - -def pytest_configure(config): - """Setup the run's configuration.""" - targets = playbook.get_targets() - criteria = playbook.get_criteria() - -pytest_configure(config) # Pytest fills in config. - -# pytest_generate_tests(metafunc) does this: -for test_metafunc in metafuncs: - for target in targets: - # items is tests * targets in size - items.append(test_metafunc[target]) - -# pytest_collection_modifyitems(session, config, items) does this: -for test in items: - validate(test) - include_or_exclude(test, criteria) - -# finally, each executor/session does this: -session_items = items.split() # based on scheduler algorithm -with pool_fixture as pool: - # the fixture has setup a pool to track the deployed targets - for test_function in session_items: - with target_fixture as target: - # the fixture has found or deployed an appropriate target - test_function(target) -``` - -## What Else? - -There’s still a lot more to think about and design. A non-exhaustive list of -future topics (some touched on above): - -* Terminology table -* Tests inventory (generating statistics from metadata) -* Environment / multiple targets class design -* Feature/requirement requests (NICs in particular) -* Custom test scheduler algorithm -* Secret management - -## What alternatives were tried? - -These are notes from things tried that did not work out, and why. - -### Writing Another Framework - -I believe the above set of technical specifications clearly describes how we can -leverage Pytest for our needs. Furthermore, the existing prototype proves this -is a viable option. Therefore I do not think we should consider writing and -maintaining a _new_ Python testing framework. We should avoid falling for “not -invented here” syndrome. The alternative prototype which does implement a new -framework required over five thousand lines of code, the Pytest-based prototype -used less than two hundred, or less than three percent. We do not want to take -on the maintenance cost of yet another framework, the maintenance cost of LISAv2 -already caused this mess in the first place. I think the work of prototyping -said new framework was valuable, as it provided insight into the eventual -technical design of LISAv3. - -### Using Remote Capabilities of `pytest-xdist` - -With the [pytest-xdist][] plugin there already exists support for running a -folder of tests on an arbitrary remote host via SSH. - -The LISA tests could be written as Python code suitable for running on the -target test system, which means direct access to the system in the test code -itself (subprocesses are still available, without having to use SSH within the -test, but would become far less necessary), something that is not possible with -any current prototype. Where the `pytest-xdist` plugin copies the package of code -to the target node and runs it, the pytest-lisa plugin could instantiate that -node (boot the necessary image on a remote machine or launch a new Hyper-V or -Azure VM, etc.) for the tests. - -However, this use of pytest-dist requires full Python support on the target -machines, and drastically changes how developers write tests. Furthermore, it -would not support running local commands against the remote node (like ping) or -running the test across a reboot of the node. Thus we do not want to use this -functionality of `pytest-xdist`. That said, `pytest-xdist` will still be useful -for running tests concurrently, as described above. - -### Using Paramiko Instead of Fabric - -The Paramiko library is less complex (smaller library footprint) than Fabric, as -the latter wraps the former, but it is a bit more difficult to use, and doesn’t -support reading existing SSH config files, nor does it support “ProxyJump” which -we use heavily. Fabric instead provides a clean high-level interface for -existing shell commands, handling all the connection abstractions for us. - -Using Paramiko looked like this: - -```python -from pathlib import Path -from typing import List - -from paramiko import SSHClient - -import pytest - -@pytest.fixture -def node() -> SSHClient: - with SSHClient() as client: - client.load_system_host_keys() - client.connect(hostname="...") - yield client - - -def test_lis_version(node: SSHClient) -> None: - with node.open_sftp() as sftp: - for f in ["utils.sh", "LIS-VERSION-CHECK.sh"]: - sftp.put(LINUX_SCRIPTS / f, f) - _, stdout, stderr = node.exec_command("./LIS-VERSION-CHECK.sh") - sftp.get("state.txt", "state.txt") - with Path("state.txt").open as f: - assert f.readline() == "TestCompleted" -``` - -It is more verbose than necessary when compared to Fabric. - -### StringIO - -For `Node.cat()` it would seem we could use `StringIO` like so: - -```python -from io import StringIO - -with StringIO() as result: - node.get("state.txt", result) - assert result.getvalue().strip() == "TestCompleted" -``` - -However, the data returned by Paramiko is in bytes, which in Python 3 are not -equivalent to strings, hence the existing implementation which uses `BytesIO` -and decodes the bytes to a string. - -[PyYAML]: https://pyyaml.org/wiki/PyYAMLDocumentation -[collection hooks]: https://docs.pytest.org/en/latest/reference.html#collection-hooks -[fixtures]: https://docs.pytest.org/en/stable/fixture.html -[parameterization]: https://docs.pytest.org/en/stable/parametrize.html -[pytest-xdist]: https://github.com/pytest-dev/pytest-xdist -[pytest_addoption]: https://docs.pytest.org/en/latest/reference.html#pytest.hookspec.pytest_addoption -[pytest_collection_modifyitems]: https://docs.pytest.org/en/latest/reference.html#pytest.hookspec.pytest_collection_modifyitems -[pytest_configure]: https://docs.pytest.org/en/latest/reference.html#pytest.hookspec.pytest_configure -[pytest_generate_tests]: https://docs.pytest.org/en/latest/reference.html#pytest.hookspec.pytest_generate_tests -[pytest_runtest_makereport]: https://docs.pytest.org/en/latest/reference.html#pytest.hookspec.pytest_runtest_makereport -[schema]: https://pypi.org/project/schema/ -[test running hooks]: https://docs.pytest.org/en/latest/reference.html#test-running-runtest-hooks diff --git a/DESIGN.rst b/DESIGN.rst new file mode 100644 index 0000000000..838b931cf9 --- /dev/null +++ b/DESIGN.rst @@ -0,0 +1,1080 @@ +Technical Specification Document +================================ + +This document outlines the technical specifications for LISAv3. We are +evaluating the feasibility of leveraging +`Pytest `_ as our test runner. + +Please see `PR #1065 `_ for a +working, proof-of-concept prototype. + +Authored by Andrew Schwartzmeyer (he/him), version 0.3.0. + +Why Pytest? +----------- + +Pytest is an `incredibly +popular `_ MIT licensed +open source Python testing framework. It has a thriving community and +plugin framework, with over 750 +`plugins `_. Instead of writing +(and therefore maintaining) yet another test framework, we would do more +with less by reusing Pytest and existing plugins. This will allow us to +focus on our unique problems: organizing and understanding our tests, +deploying necessary resources (such as Azure, Hyper-V, or bare metal +machines, collectively known as “targets”), and analyzing our results. + +In fact, most of Pytest itself is implemented via `built-in +plugins `_, providing +us with many useful and well-documented examples. Furthermore, when +others were confronted with a problem similar to our own they also chose +to use Pytest. `Labgrid `_ +is an open source embedded board control library that delegated the +testing framework logic to Pytest in their +`design `_, +and `U-Boot `_, an embedded board +boot loader, similarly leveraged Pytest in their +`tests `_. +KernelCI and Avocado were also evaluated by the Labgrid developers at an +`Embedded Linux Conference `_ and both +ruled out for reasons similar to our own before they settled on Pytest. + +The `fundamental features `_ of Pytest +match our needs very well: + +- Automatic test discovery, no boiler-plate test code +- Useful information when a test fails (assertions are introspected) +- Test and fixture + `parameterization `_ +- Modular setup/teardown via + `fixtures `_ +- Incredibly customizable (as detailed above) + +So all the logic for describing, discovering, running, skipping and +reporting results of the tests, as well as enabling and importing users’ +plugins is already written and maintained by the open source community. +This leaves us to focus on our hard and specific problems: creating an +abstraction to launch the necessary targets, organizing and publishing +our tests, and reporting test results upstream. Using Pytest would also +allow us the space to abstract other commonalities in our specific +tests. In this way, LISAv3 could solve the difficulties we have at hand +without creating yet another test framework. + +Finally, by leveraging such a popular framework and reducing the amount +of code we need to maintain, we drastically increase our chances of +receiving pull requests instead of bug reports from users. This is +important because despite our best efforts it is practically guaranteed +that as adoption of LISAv3 increases, users will want changes to be +made, and we need to empower them to do so themselves. + +What are we maintaining? +------------------------ + +The current proof-of-concept implementation uses the top-level +``conftest.py`` file to define our “plugin” functionality. This works, +but it is not ideal. I believe that we will want to publish two open +source Pytest plugins as packages on `PyPI `_, the +Python Package Index: ``pytest-target`` and ``pytest-lisa``. We will +also maintain our set of public “LISA” tests, but these should simply +install and use our plugins. + +The ``pytest-target`` plugin should encapsulate all our logic for *how* +and *when* to deploy targets (local or cloud virtual machines, or bare +metal machines, and all the associated resources), run tests on the +specified targets, and delete the targets. This includes specifying +which features and resources each test needs and each given target +provides (such as number of cores, amount of RAM, and other hardware +like a GPU etc.), how to deploy and delete each target based on its +platform, and parameterization of the ``target`` fixture based on CLI or +YAML file input. In fact, some tests (like networking) will require +multiple targets at once. This plugin will need to manage resources +intelligently, being able to optimize for both time and cost, and make +it easy for tests to request and use various resources. + +The ``pytest-lisa`` plugin should encapsulate all our logic for how to +organize and select tests, as well as our opinions on displaying test +results. This includes the user modes, test metadata and inventory, test +selection based on criteria against that metadata, required and +pre-configured upstream plugins, and result notifiers. It will similarly +support both CLI and YAML file input. + +We should strive to keep these plugins from depending on each other in +order to keep their scope well-defined. In the “LISA” repository of +tests we will depend on the two plugins and maintain additional +`fixtures `_ for our +tests’ unique requirements. Similarly, we and others may have private +test repositories which build upon the above by defining new platform +support and internal service integrations. The built-in plugin discovery +of Pytest (via ``conftest.py`` files) enables us to satisfy one of our +requirements to “support plugins to orchestrate the test environment.” + +Finally, a third smaller utility plugin, ``pytest-schema`` may be +written in order to share the common functionality of registering +component schemata (e.g. platform and target parameters from +``pytest-target`` and selection criteria from ``pytest-lisa``). This is +somewhat of an implementation detail, but would be a third and +lower-level library we can publish. + +pytest-target +------------- + +How are targets provided and accessed? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +First we need to define “target” as an instance of a system-under-test. +That is, given some environment requirements, such an Azure image (URN) +and size (SKU), a target would be a virtual machine deployed by +``pytest-target`` with SSH access provided to the requesting test. A +target could optionally be pre-deployed and simply connected. Some tests +may request multiple targets as well. + +Pytest uses +`fixtures `_, which are +the primary way of setting up test requirements. They replace less +flexible alternatives like setup/teardown functions. It is through +fixtures that we implement remote target setup/teardown. Our ``target`` +fixture returns a ``Target`` instance, which currently provides: + +- Remote shell access via SSH +- Data including hostname / IP address +- Cross-platform ping functionality with exponential back-off +- Uploading of local files to arbitrary remote destinations +- Downloading of remote file contents into local string variable +- Asynchronous remote command execution with promises + +The ``Azure(Target)`` subclass additionally provides: + +- Automatic provisioning of an Azure VM given URN and SKU +- Allowing ICMP ping via Azure firewall rules +- Azure platform forced reboot by API +- Downloading boot diagnostics (serial console log) from platform + +The prototype demonstrates how easy it is to quickly implement these +features. As we need more features, they can be readily added and shared +among tests. + +The ``Target`` class leverages `Fabric `_ +which is a popular high-level Python library for executing shell +commands on remote systems over SSH. Underneath the covers Fabric uses +`paramiko `_, the most popular +low-level Python SSH library. Fabric does the heavy lifting of safely +connecting and disconnecting from the node, executing the shell command +(synchronously or asynchronously), reporting the exit status, gathering +the stdout and stderr, providing stdin (or interactive auto-responses, +similar to ``expect``), uploading and downloading files, and much more. +In fact, these APIs are all available and implemented for the local +machine by the underlying `Inovke `_ +library, which is essentially a Python ``subprocess`` wrapper with “a +powerful and clean feature set.” + +Other test specific requirements, such as installing software and +daemons, downloading files from remote storage, or checking the state of +our Bash test scripts, would similarly be implemented by methods on the +``Target`` class or via additional fixtures and thus shared among tests. + +How do we interact with Azure? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For Azure, we currently use the `Azure +CLI `_ to deploy a virtual machine. For +Hyper-V (and other virtualization platforms), we would like to use +`libvirt `_, and for embedded / bare +metal environments we are evaluating +`labgrid `_. + +If possible, we do not want to use the `Azure Python +APIs `_ directly because they are more +complicated (and less documented) than the `Azure +CLI `_. With Invoke (as discussed above), +``az`` becomes incredibly easy to work with. The Azure CLI lead +developer states that they have `feature +parity `_ and that the +CLI is more straightforward to use. Considering our ease-of-maintenance +requirement, this seems the apt choice, especially since the Azure CLI +supports deploying resources with `ARM +templates `_. +If it later becomes necessary to use the Python APIs directly, that is, +of course, still doable (and we can reuse existing code doing it). + +On the topic of “servicing” the Azure CLI, its developers state that “at +command level, packages only upgrading the PATCH version guarantee +backward compatibility.” The tool is also intended to be used in +scripts, so servicing would amount to documenting the tested version and +having the Azure class check that it’s compatible before using it (or +warning and then trying its best). + +What’s the ``Target`` class? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In version 0.1 of this design document we detailed a planned refactor of +what was then called the ``Node`` class. This has since been executed +with just a few modifications (one being the rename to ``Target``, as +``Node`` was found to be an overloaded term in the context of data +centers). This class and its subclasses are decoupled from Pytest, and +are used via fixtures. It looks like this: + +.. code:: python + + from abc import ABC, abstractmethod + from schema import Schema + import fabric + + class Target(ABC): + parameters: Mapping[str, str] + features: Set[str] + name: str + host: str + conn: fabric.Connection # Provides run, sudo, get, put etc. + + def __init__(...): + ... + self.host = self.deploy() + self.conn = fabric.Connection(self.host) + + @classmethod + @property + @abstractmethod + def schema(cls) -> Schema: + """Must return the parameters schema for setup.""" + ... + + @abstractmethod + def deploy(self) -> str: + """Must deploy the target resources and return hostname.""" + ... + + @abstractmethod + def delete(self) -> None: + """Must delete the target resources.""" + ... + + @classmethod + def local(...) -> Result: + """Runs a local shell command.""" + ... + +How are platforms implemented? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Platform support is implemented by subclassing ``Target`` and defining +the ``schema`` property, ``deploy`` method, ``delete`` method, and any +platform-specific methods. Using the ``__subclasses__`` attribute of +``Target`` the available platforms and their parameter schemata are +automatically gathered from users’ own ``conftest.py`` files and other +plugins. This enables the ``target`` fixture to dynamically instantiate +a target from the gathered requirements and parameters. + +For example, the ``Azure(Target)`` class defines its required parameters +using the `schema `_ library like +this: + +.. code:: python + + from schema import Optional, Schema + from target import Target + + class Azure(Target): + ... + schema: Schema = Schema( + { + # TODO: Maybe validate as URN or path etc. + "image": str, + Optional("sku", default="Standard_DS1_v2"): str, + Optional("location", default="eastus2"): str, + Optional("networking", default=""): str, + } + ) + +In the YAML playbook, a set of Azure targets can then be defined like +this: + +.. code:: yaml + + targets: + - name: Debian + platform: Azure + parameters: + image: credativ:Debian:9:9.0.201706190 + location: westus2 + + - name: Ubuntu + platform: Azure + parameters: + image: UbuntuLTS + sku: Standard_DS3_v2 + +These targets are then used to parameterize the ``target`` fixture in +the +`pytest_generate_tests `_ +hook (see below for more details). + +This demonstrated how we can have platforms define their own schema and +register that schema automatically. A pending update to this is to have +two schemata per ``Target`` subclass: target-level and platform-level +(the former is what’s demonstrated above, the latter would be common +settings, such as subscription). + +How are requirements examined? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The ``features`` attribute is currently a set of strings and (combined +with the parameters dictionary) was used to demonstrate how we can test +if an existing target instance (representing a deployed machine) met a +test’s requirements. It should be updated with a ``Requirements`` class +that represents all physical attributes of the target, and a +``requires`` Pytest mark should be added which takes instances of this +class. Two ``Requirements`` should be comparable to determine if one set +meets (or exceeds) the other set. Existing code that does this can be +reused here. + +How do we share common tasks? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Common tasks for targets like rebooting and pinging should be +implemented on the ``Target`` class, and platform-specific tasks on the +respective subclass. + +Methods available from ``Connection`` include ``run()`` and ``sudo()`` +which are used to easily run arbitrary commands, and ``get()`` and +``put()`` to download and upload arbitrary files. + +The ``cat()`` method wraps ``get()`` and returns the file as data in a +string. This makes test code like this possible: + +.. code:: python + + assert target.conn.cat("state.txt") == "TestCompleted" + +A ``reboot()`` method should be added that first tries to use +``sudo("reboot", timeout=5)`` (with a short timeout to avoid a hung SSH +session). It should retry with an exponential back-off to see if the +machine has rebooted by checking either ``uptime`` or the existence of a +file created before the reboot. This is to avoid having to ``sleep()`` +and just guess the amount of time it takes to reboot. + +A ``restart()`` method should “power cycle” the machine using the +platform’s API, and thus is in abstract method. + +Other tools and shared logic should be implemented as necessary. A major +area of concern is the automatic and package-manager agnostic +installation of necessary tools, much of which has been implemented +previously and can be reused. + +How are targets requested and managed? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +We implement a pair of Pytest fixtures to provide targets. The first is +the ``pool`` fixture, which looks like: + +.. code:: python + + @pytest.fixture(scope="session") + def pool(request: SubRequest) -> Iterator[List[Target]]: + """This fixture tracks all deployed target resources.""" + targets: List[Target] = [] + yield targets + for t in targets: + t.delete() + +The ``pool`` fixture is setup once at the beginning of the test session, +at which point the ``targets`` list is then provided as input to every +instance of the ``target`` fixture. While currently a list, to support +optimal scheduling we will likely want to use a priority queue, where +the priority of a target represents its cost (whether in terms of time +or money), allowing us to provide either the fastest or the cheapest +target to each request. Targets not in use will be deallocated, and all +targets will be automatically deleted after the tests are finished +(unless the user requested otherwise, in which case they’ll be cached). + +Note that cross-session +`caching `_ is provided +by Pytest, and very easy to work with. An early prototype implemented a +``--keep-vms`` flag successfully, and this will be implemented again +with the updated design. + +The second is the ``target`` fixture, which looks like: + +.. code:: python + + @pytest.fixture + def target(pool: List[Target], request: SubRequest) -> Iterator[Target]: + """This fixture provides a connected target for each test.""" + platform: Type[Target] = playbook.PLATFORMS[request.param["platform"]] + parameters: Dict[str, Any] = request.param["parameters"] + marker = request.node.get_closest_marker("lisa") + features = set(marker.kwargs["features"]) + + # TODO: If `t` is not already in use, deallocate the previous target. + for t in pool: + if isinstance(t, platform) and t.parameters == parameters and t.features >= features: + yield t + break + else: + t = platform(parameters, features) + pool.append(t) + yield t + t.connection.close() + +This is obviously still an early implementation, but it is viable. By +using the +`pytest_collection_modifyitems `_ +hook to sort (and so group) the tests by their requirements, the tests +would efficiently reuse targets. This fixture is indirectly +parameterized during setup with the +`pytest_generate_tests `_ +hook. Test and fixture +`parameterization `_ +is a huge feature of Pytest. When we parameterize the ``target`` fixture +for multiple targets (e.g. “Ubuntu” and “Debian”), Pytest automatically +creates a set of tests for each target. So ``test_smoke`` turns into +``test_smoke[Ubuntu]`` and ``test_smoke[Debian]``. This allows us to run +a collection of tests against multiple targets with ease. These targets +are defined in a YAML file and validated against the parameters +collected from the previously described platform subclasses. + +The entire implementation looks like so: + +.. code:: python + + TARGETS: List[Dict[str, Any]] = [] + TARGET_IDS: List[str] = [] + + def pytest_configure(config: Config) -> None: + book = get_playbook(config.getoption("--playbook")) + for t in book.get("targets", []): + TARGETS.append(t) + TARGET_IDS.append(t["name"]) + + def pytest_generate_tests(metafunc: Metafunc) -> None: + if "target" in metafunc.fixturenames: + assert TARGETS, "No targets specified!" + metafunc.parametrize("target", TARGETS, True, TARGET_IDS) + +The function ``get_playbook()`` only imports the +`PyYAML `_ library, opens +the playbook file ``f`` within a context manager, and returns +``playbook.schema.validate(yaml.load(f))``. This is leveraging Pytest’s +existing parameterization technology to achieve one of our “test +entrance” goals of requesting environments with a YAML playbook, and one +of our “test parameter validation” goals of validating platforms before +executing tests so that we can fail fast if a target has insufficient +information to be setup. Parsing the same parameters from a CLI can also +be implemented. + +Finally, once the ``target`` fixture has returned a working and +sanity-checked environment to the requesting test, the test is capable +of examining any and all attributes of the ``Target`` and quickly +marking itself as skipped, expected to fail, or failed before executing +the body of the test. Our static type checking enables developers to +ensure that the platform they requested supports all methods and fields +they use by annotating the test’s ``target`` parameter with the expected +platform type (or types). Ensuring the effectiveness of this type +checking will require us to carefully update our platform +implementations, and not rely on arbitrary objects of data. (For +example, add an ``internal_address`` field to ``Azure``, don’t just look +up ``data["internal_address"]``.) + +How are tests executed in parallel? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +While our original list of goals stated that we want to run tests “in +parallel” we were not specific about what was meant, and the topic of +parallelism and concurrency is understandably complex. We certainly +don’t mean running two tests at once on the same target, as this would +undoubtedly lead to flaky tests. + +Assuming that we care about a set of tests passing on a particular image +and size combination, but not necessarily on a particular deployed +instance, then we can run tests concurrently by deploying multiple +“identical” targets and splitting the tests across them. The tests would +still run in isolation on each target. This sounds hard, but actually +it’s practically free with Pytest via +`pytest-xdist `_. + +The default ``pytest-xdist`` implementation simply takes the list of +tests and runs them in a round-robin fashion with the desired number of +executors. We’ve talked at length about being able to schedule groups of +tests to run in particular executors and using particular targets. While +there are many paths open to us, this plugin actually provides a hook, +``pytest_xdist_make_scheduler`` that exists specifically to “implement +custom tests distribution logic.” + +Figuring out the requirements of our test scheduler and designing the +best algorithm will require further discussion and design review. For +the purposes of moving forward, we are not blocked, as the eventual +implementation can be dropped in-place with minimal effort. + +pytest-lisa +----------- + +What are the user modes? +~~~~~~~~~~~~~~~~~~~~~~~~ + +Because Pytest is incredibly customizable, we want to provide a few sets +of reasonable default configurations for some common scenarios. We will +add a flag like ``--lisa-mode=[dev,debug,ci,demo]`` to change the +default options and output of Pytest. Doing so is readily supported by +Pytest via the +`pytest_addoption `_ +and +`pytest_configure `_ +hooks. We call these the provided “user modes.” Note that by “output” we +mean not just logging (because that implies the Python ``logger`` +module, which Pytest allows full control over) but also commands’ stdout +and stderr as well as Pytest-provided information. + +- The dev(eloper) mode is intended for use by test developers while + writing a new test. It is verbose, caches the deployed VMs between + runs, and generates a digestible + `HTML `_ report. + +- The debug mode is like dev mode but with all possible information + shown, and will open the Python debugger automatically on failures + (which is provided by Pytest with the ``--pdb`` flag). + +- The CI mode will be fairly quiet on the console, showing all test + results, but putting the full info output into the generated report + file (HTML for sharing with humans and + `JUnit `_ + for the associated CI environment, which presents as native test + results). + +- The demo mode will show the “executive summary” (a lot like CI, but + finely tuned for demos). For example, what ``make smoke`` currently + shows. + +How are tests described? +~~~~~~~~~~~~~~~~~~~~~~~~ + +The built-in +`pytest-mark `_ plugin +already provides functionality for adding metadata to tests, where we +specifically want: + +- Platform: used to skip tests inapplicable to the current + system-under-test +- Category: our high-level test organization +- Area: feature being tested +- Priority: self-explanatory +- Tags: optional additional metadata for test organization + +We simply reuse this with minimal logic to enforce our required +metadata, with sane defaults (perhaps setting the area to the name of +the module), and to list statistics about our test coverage. This is +already included in the prototype. It looks like this: + +.. code:: python + + import pytest + + @pytest.mark.lisa(platform="Azure", category="Functional", priority=0, area="LIS_DEPLOY") + def test_lis_driver_version(target: Azure) -> None: + """Checks that the installed drivers have the correct version.""" + ... + +This is a functional example, which takes zero implementation. With this +simple decorator, all test `collection +hooks `_ +can introspect the metadata, enforce required parameters and set +defaults, select tests based on arbitrary criteria, and list test +coverage statistics (test inventory). Designing and implementing the +test inventory algorithm is still under development, but it’s tractable. + +Note that Pytest leverages Python’s docstrings for built-in +documentation (and can even run tests discovered in such strings, like +doctest). Hence we do not have a separate field for the test’s +documentation. As long as we continue to follow the practice of using +docstrings for our modules, classes, and functions, we can automatically +use `pydoc `_ to generate +full documentation for each plugin and test. + +Being just Python code, this decorator need not be +``@pytest.mark.lisa(...)`` but can trivially be provided as simply +``@LISA(...)``. In fact, we provide this in ``lisa.py`` with: + +.. code:: python + + LISA = pytest.mark.lisa + + @LISA(...) + def test_something(...) + +Currently we validate the parameters given to this mark during test +collection, by using the following code, which leverages the +`schema `_ library: + +.. code:: python + + from schema import Optional, Or, Schema + + lisa_schema = Schema( + { + "platform": str, + "category": Or("Functional", "Performance", "Stress", "Community", "Longhaul"), + "area": str, + "priority": Or(0, 1, 2, 3), + Optional("tags", default=list): [str], + }, + ) + + def validate(mark: Mark) -> None: + """Validate each test's LISA parameters.""" + assert not mark.args, "LISA marker cannot have positional arguments!" + mark.kwargs.update(lisa_schema.validate(mark.kwargs)) + +In the future we could change ``LISA`` to be a function with these +keyword arguments so that IDE auto-completion is enabled. However, this +is not mandatory to move forward, and parameter validation is enabled +succinctly with the above, which satisfies one of our “test parameter +validation” requirements. + +This mark also does need to be repeated for each test, as marks can be +scoped to a module, and so one line could describe defaults for every +test in a file, with individual tests overriding parameters as needed. + +In the current implementation, we also take a ``features: List[str]`` +argument that is used to prove the concept deploying (or reusing) a +target based on the test’s required and the target’s available sets of +features. However, as we move forward we should define a separate +``requires`` mark that takes well-defined classes describing the minimal +required resources for a test. This will be part of the refactor into +the two Pytest plugins mentioned above. Coupled with the test’s +requested ``target`` fixture being parameterized (see discussion in +``pytest-target``) this demonstrates at least one way we can satisfy our +“test run planner/scheduler” requirement. + +Furthermore, we have a prototype +`generator `_ +which parses LISAv2 XML test descriptions and generates stubs with this +mark filled in correctly. + +How are tests selected? +~~~~~~~~~~~~~~~~~~~~~~~ + +Pytest already allows a user to specify which exact tests to run: + +- Listing folders on the CLI (see below on where tests should live) +- Specifying a name expression on the CLI (e.g. ``-k smoke and xdp``) +- Specifying a mark expression on the CLI + (e.g. ``-m functional and not slow``) + +We can also implement any other mechanism via the +`pytest_collection_modifyitems `_ +hook. The proof-of-concept supports gathering selection criteria from a +YAML file: + +.. code:: yaml + + criteria: + # Select all Priority 0 tests. + - priority: 0 + # Run tests with 'smoke' in the name twice. + - name: smoke + times: 2 + # Exclude all tests in Area "xdp" + - area: xdp + exclude: true + +This criteria is validated against the following +`schema `_: + +.. code:: python + + from schema import Schema, Optional + + criteria_schema = Schema( + { + # TODO: Validate that these strings are valid regular + # expressions if we change our matching logic. + Optional("name", default=None): str, + Optional("area", default=None): str, + Optional("category", default=None): str, + Optional("priority", default=None): int, + Optional("tags", default=list): [str], + Optional("times", default=1): int, + Optional("exclude", default=False): bool, + } + ) + +The test collection is then modified using the Pytest hook, +`pytest_collection_modifyitems `_: + +.. code:: python + + def pytest_collection_modifyitems( + session: Session, config: Config, items: List[Item] + ) -> None: + included: List[Item] = [] + excluded: List[Item] = [] + + def select(item: Item, times: int, exclude: bool) -> None: + if exclude: + excluded.append(item) + else: + for _ in range(times - included.count(item)): + included.append(item) + + for c in criteria: # Where `criteria` is from the schema. + for item in items: + marker = item.get_closest_marker("lisa") + if not marker: + # Not all tests will have the LISA marker, such as + # static analysis tests. + continue + i = marker.kwargs + if any( + [ + c["name"] and c["name"] in item.name, + c["area"] and c["area"].casefold() == i["area"].casefold(), + c["category"] + and c["category"].casefold() == i["category"].casefold(), + c["priority"] and c["priority"] == i["priority"], + c["tags"] and set(c["tags"]) <= set(i["tags"]), + ] + ): + select(item, c["times"], c["exclude"]) + items[:] = [i for i in included if i not in excluded] + +Together, the CLI support and YAML playbook satisfy one of our “test +entrance” requirements. We can also generate our own binary called +``lisa`` which simply delegates to Pytest, if we really want to do so. + +Because this is simply a Python list, we can also sort the tests +according to our needs, such as by priority. If the ``python-targets`` +plugin has already sorted by requirements, that’s just fine, Python’s +``sorted()`` built-in is guaranteed to be stable (meaning we can sort in +multiple passes). + +How are results reported? +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Parsing the results of a large test suite can be difficult. Fortunately, +because Pytest is a testing framework, there already exists support for +generating excellent reports. For developers, the +`HTML `_ report is easy to read: +it is self-contained, holds all the results and logs, and each test can +be expanded and collapsed. Tests which were rerun are recorded +separately. For CI pipelines, Pytest has integrated +`JUnit `_ +XML test report support. This is the standard method of reporting +results to CI servers like Jenkins and are natively parsed into the CI +system’s built-in test display page. Finally, Azure DevOps pipelines are +even supported with a community plugin +`pytest-azurepipelines `_ +which enhances the standard JUnit report for ADO. + +One of our requirements is to support the lookup of previous tests’ +execution metrics, such as recorded performance metrics and duration, so +that performance tests can check regressions. This is the perfect +example of carrying a small fixture which provides access to our +internal database and is dynamically added to our tests when run +internally, and the tests can lookup and record whatever they need +through the fixture. + +However, we also have internal requirements to report test results +throughout the test life cycle to a database (the “result manager” and +“progress tracker”) to be consumed by other tools. In this sense, LISAv3 +(the composition of our published plugins, tests, and fixtures) is +simply a producer, and the consumers can parse the test results, send +emails, archive the collected logs, update a GUI display of test +progress, etc. Our repository’s ``conftest.py`` can implement the +necessary logic using Pytest’s ample `test running +hooks `_. +In particular, the hook +`pytest_runtest_makereport `_ +is called for each of the setup, call and teardown phases of a test. As +such it can used for precisely this purpose. + +How is setup, run, and cleanup handled? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Pytest strives to require minimal boiler-plate code. Thus the classic +“xunit-style” of defining a class with setup and teardown functions in +addition to test functions is not recommended (nor necessary). Generally +Pytest expects +`fixtures `_ to be used +for dependency injection (which is what setup/teardown functions usually +do). For users that really want the classic style, it is nonetheless +fully `supported `_ +and documented (and can be applied at the module, class, and method +scopes). Thus our “test runner” requirement is satisfied. + +How are tests timed out? +~~~~~~~~~~~~~~~~~~~~~~~~ + +The `pytest-timeout `_ plugin +provides integrated timeouts via ``@pytest.mark.timeout()``, +a configuration file option, environment variable, and CLI flag. The +Fabric library provides timeouts in both the configuration and +per-command usage. These are already used to satisfaction in the +prototype. Additionally, Pytest has built-in support for measuring the +duration of each fixture’s setup and teardown and each test (it’s simply +the ``--durations`` and ``--durations-min`` flags). + +How are tests organized? +~~~~~~~~~~~~~~~~~~~~~~~~ + +That is, what does a folder of tests map to: a platform, feature, or +owner? + +In my opinion it is likely to be both. Tests which are common to a +platform and written by our team are probably best placed in a folder +like ``tests/azure`` whereas tests for a particular scenario which +limits their image and SKU applicability should be in a folder like +``tests/acc``. It’s going to depend on how often the tests are run +together. + +Because Pytest can run tests and ``conftest.py`` files from arbitrary +folders, maintaining sets of tests and plugins separately from the base +LISA repository is easy. Custom repositories with new tests, plugins, +fixtures, platform-specific support, etc. can simply be cloned anywhere, +and provided on the command-line to Pytest. + +Test authors should keep tests which share requirements and are +otherwise similar to a single module (Python file). Not only is this +well-organized, but because marks can be applied at the module level, +setting all the tests to be skipped or expected to fail (with the +built-in ``skip`` and ``xfail`` Pytest marks) becomes even easier. + +An open question is if we really want to bring every test from LISAv2 +directly over, or if we should carefully analyze our tests to craft a +new set of high-level scenarios. An interesting result of reorganizing +and rewriting the tests would be the ability to have test layers, where +the result of a high-level test dictates if the tests below it should be +skipped. If it passes, it implies the tests underneath it would pass, +and so skips them; but if it fails, the next test below it runs and so +on until a passing layer is found. + +How will we port LISAv2 tests? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Given the above, we still must decide if we want to put the engineering +effort into porting *every* LISAv2 test. However, the prototype started +by porting the ``LIS-DRIVER-VERSION-CHECK`` test, proving that tests +which exclusively use Bash scripts are trivially portable. +Unfortunately, most tests use an associated PowerShell script which is +tightly coupled to the LISAv2 framework. + +We believe that it is *possible* to port these tests without untoward +modifications. We would need to write a mock library that implements (or +stubs where appropriate) LISAv2 framework functionality such as +``Provision-VMsForLisa``, ``Copy-RemoteFiles``, ``Run-LinuxCmd``, etc., +and provides both the expected “global” objects and the test function +parameters ``AllVmData`` and ``CurrentTestData``. + +This work needs to be done regardless of the approach we take with our +framework (leveraging Pytest or writing our own), and it is not +inconsequential work. It needs to be thoroughly planned and executed, +and is certainly a ways off. + +How are tests and functions retried? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Testing remote targets is inherently flaky, so we take a two-pronged +approach to dealing with the flakiness. + +The +`pytest-rerunfailures `_ +plugin will be used to easily mark a test itself as flaky. It has the +nice feature of recording each rerun in the produced report. It looks +like this: + +.. code:: python + + @pytest.mark.flaky(reruns=5) + def test_something_flaky(...): + """This fails most of the time.""" + ... + +Note that there is an open +`bug `_ +in this plugin which can cause issues with fixtures using scopes other +than “function” but it can be worked around. + +The `Tenacity `_ library +should be used to retry flaky functions that are not tests, such as +downloading boot diagnostics or pinging a node. As the modern Python +retry library it has easy-to-use decorators to retry functions (and +context managers to use within functions), as well as excellent wait and +timeout support. It looks like this: + +.. code:: python + + from tenacity import retry, stop_after_attempt, wait_exponential + + class Node: + ... + @retry(reraise=True, wait=wait_exponential(), stop=stop_after_attempt(3)) + def ping(self, **kwargs): + """Ping the node from the local system in a cross-platform manner.""" + flag = "-c 1" if platform.system() == "Linux" else "-n 1" + return self.local(f"ping {flag} {self.host}", **kwargs) + ... + +We can additionally list a test twice when modifying the items +collection, as implemented in the criteria proof-of-concept. However, +given the above abilities, this may not be desired. + +What does the “flow” of Pytest look like? +----------------------------------------- + +This is best described in Pythonic pseudo-code, where the context +manager encapsulates each scope and the for loop encapsulates +processing: + +.. code:: python + + pool_fixture: a session-scoped context manager + target_fixture: a function-scoped context manager + items: a collection of tests + targets: a collection of targets + criteria: a collection of test selection criteria + + def pytest_addoption(parser): + """Add CLI options etc.""" + parser.addoption("--playbook", type=Path) + + pytest_addoption(parser) # Pytest fills in parser. + + def pytest_configure(config): + """Setup the run's configuration.""" + targets = playbook.get_targets() + criteria = playbook.get_criteria() + + pytest_configure(config) # Pytest fills in config. + + # pytest_generate_tests(metafunc) does this: + for test_metafunc in metafuncs: + for target in targets: + # items is tests * targets in size + items.append(test_metafunc[target]) + + # pytest_collection_modifyitems(session, config, items) does this: + for test in items: + validate(test) + include_or_exclude(test, criteria) + + # finally, each executor/session does this: + session_items = items.split() # based on scheduler algorithm + with pool_fixture as pool: + # the fixture has setup a pool to track the deployed targets + for test_function in session_items: + with target_fixture as target: + # the fixture has found or deployed an appropriate target + test_function(target) + +What Else? +---------- + +There’s still a lot more to think about and design. A non-exhaustive +list of future topics (some touched on above): + +- Terminology table +- Tests inventory (generating statistics from metadata) +- Environment / multiple targets class design +- Feature/requirement requests (NICs in particular) +- Custom test scheduler algorithm +- Secret management + +What alternatives were tried? +----------------------------- + +These are notes from things tried that did not work out, and why. + +Writing Another Framework +~~~~~~~~~~~~~~~~~~~~~~~~~ + +I believe the above set of technical specifications clearly describes +how we can leverage Pytest for our needs. Furthermore, the existing +prototype proves this is a viable option. Therefore I do not think we +should consider writing and maintaining a *new* Python testing +framework. We should avoid falling for “not invented here” syndrome. The +alternative prototype which does implement a new framework required over +five thousand lines of code, the Pytest-based prototype used less than +two hundred, or less than three percent. We do not want to take on the +maintenance cost of yet another framework, the maintenance cost of +LISAv2 already caused this mess in the first place. I think the work of +prototyping said new framework was valuable, as it provided insight into +the eventual technical design of LISAv3. + +Using Remote Capabilities of ``pytest-xdist`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +With the `pytest-xdist `_ +plugin there already exists support for running a folder of tests on an +arbitrary remote host via SSH. + +The LISA tests could be written as Python code suitable for running on +the target test system, which means direct access to the system in the +test code itself (subprocesses are still available, without having to +use SSH within the test, but would become far less necessary), something +that is not possible with any current prototype. Where the +``pytest-xdist`` plugin copies the package of code to the target node +and runs it, the pytest-lisa plugin could instantiate that node (boot +the necessary image on a remote machine or launch a new Hyper-V or Azure +VM, etc.) for the tests. + +However, this use of pytest-dist requires full Python support on the +target machines, and drastically changes how developers write tests. +Furthermore, it would not support running local commands against the +remote node (like ping) or running the test across a reboot of the node. +Thus we do not want to use this functionality of ``pytest-xdist``. That +said, ``pytest-xdist`` will still be useful for running tests +concurrently, as described above. + +Using Paramiko Instead of Fabric +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The Paramiko library is less complex (smaller library footprint) than +Fabric, as the latter wraps the former, but it is a bit more difficult +to use, and doesn’t support reading existing SSH config files, nor does +it support “ProxyJump” which we use heavily. Fabric instead provides a +clean high-level interface for existing shell commands, handling all the +connection abstractions for us. + +Using Paramiko looked like this: + +.. code:: python + + from pathlib import Path + from typing import List + + from paramiko import SSHClient + + import pytest + + @pytest.fixture + def node() -> SSHClient: + with SSHClient() as client: + client.load_system_host_keys() + client.connect(hostname="...") + yield client + + + def test_lis_version(node: SSHClient) -> None: + with node.open_sftp() as sftp: + for f in ["utils.sh", "LIS-VERSION-CHECK.sh"]: + sftp.put(LINUX_SCRIPTS / f, f) + _, stdout, stderr = node.exec_command("./LIS-VERSION-CHECK.sh") + sftp.get("state.txt", "state.txt") + with Path("state.txt").open as f: + assert f.readline() == "TestCompleted" + +It is more verbose than necessary when compared to Fabric. + +StringIO +~~~~~~~~ + +For ``Node.cat()`` it would seem we could use ``StringIO`` like so: + +.. code:: python + + from io import StringIO + + with StringIO() as result: + node.get("state.txt", result) + assert result.getvalue().strip() == "TestCompleted" + +However, the data returned by Paramiko is in bytes, which in Python 3 +are not equivalent to strings, hence the existing implementation which +uses ``BytesIO`` and decodes the bytes to a string. From fe1f0cbcfd8fa5fbd3463f5a6792c151e6d6138b Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Mon, 11 Jan 2021 17:46:16 -0800 Subject: [PATCH 181/194] Move `pytest-target` dependency from `pytest-lisa` to `LISA` Since we want to keep these plugins as decoupled as possible, we do not actually need to introduce this dependency. Both plugins `pytest-target` and `pytest-lisa` depend on `pytest-playbook`, but not on each other, so we bring them both into the top-level set of `LISA` dependencies. --- poetry.lock | 19 ++- pyproject.toml | 1 + pytest-lisa/poetry.lock | 289 +------------------------------------ pytest-lisa/pyproject.toml | 1 - 4 files changed, 16 insertions(+), 294 deletions(-) diff --git a/poetry.lock b/poetry.lock index b4015773de..6dc075a0fa 100644 --- a/poetry.lock +++ b/poetry.lock @@ -295,7 +295,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [[package]] name = "importlib-metadata" -version = "3.3.0" +version = "3.4.0" description = "Read metadata from Python packages" category = "main" optional = false @@ -306,8 +306,8 @@ typing-extensions = {version = ">=3.6.4", markers = "python_version < \"3.8\""} zipp = ">=0.5" [package.extras] -docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"] -testing = ["pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake8", "pytest-cov", "jaraco.test (>=3.2.0)", "packaging", "pep517", "pyfakefs", "flufl.flake8", "pytest-black (>=0.3.7)", "pytest-mypy", "importlib-resources (>=1.3)"] +docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"] +testing = ["pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake8", "pytest-cov", "pytest-enabler", "packaging", "pep517", "pyfakefs", "flufl.flake8", "pytest-black (>=0.3.7)", "pytest-mypy", "importlib-resources (>=1.3)"] [[package]] name = "iniconfig" @@ -647,7 +647,6 @@ develop = true [package.dependencies] pytest = "^6.1.2" pytest-playbook = "0.1.0" -pytest-target = "0.1.0" pytest-xdist = "^2.1.0" schema = "0.7.2" @@ -895,7 +894,7 @@ python-versions = "*" [[package]] name = "sphinx" -version = "3.4.2" +version = "3.4.3" description = "Python documentation generator" category = "dev" optional = false @@ -1082,7 +1081,7 @@ testing = ["pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "45121cd7b463c5e45581a29242620b2fa000aad85ffb4677888c520495c68847" +content-hash = "4f4e21335d25310f82b81670134aa8a48ec8ae442bdcb5d4eb0db2642bfc6c64" [metadata.files] alabaster = [ @@ -1239,8 +1238,8 @@ imagesize = [ {file = "imagesize-1.2.0.tar.gz", hash = "sha256:b1f6b5a4eab1f73479a50fb79fcf729514a900c341d8503d62a62dbc4127a2b1"}, ] importlib-metadata = [ - {file = "importlib_metadata-3.3.0-py3-none-any.whl", hash = "sha256:bf792d480abbd5eda85794e4afb09dd538393f7d6e6ffef6e9f03d2014cf9450"}, - {file = "importlib_metadata-3.3.0.tar.gz", hash = "sha256:5c5a2720817414a6c41f0a49993908068243ae02c1635a228126519b509c8aed"}, + {file = "importlib_metadata-3.4.0-py3-none-any.whl", hash = "sha256:ace61d5fc652dc280e7b6b4ff732a9c2d40db2c0f92bc6cb74e07b73d53a1771"}, + {file = "importlib_metadata-3.4.0.tar.gz", hash = "sha256:fa5daa4477a7414ae34e95942e4dd07f62adf589143c875c133c1e53c4eff38d"}, ] iniconfig = [ {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, @@ -1529,8 +1528,8 @@ snowballstemmer = [ {file = "snowballstemmer-2.0.0.tar.gz", hash = "sha256:df3bac3df4c2c01363f3dd2cfa78cce2840a79b9f1c2d2de9ce8d31683992f52"}, ] sphinx = [ - {file = "Sphinx-3.4.2-py3-none-any.whl", hash = "sha256:b8aa4eb5502c53d3b5ca13a07abeedacd887f7770c198952fd5b9530d973e767"}, - {file = "Sphinx-3.4.2.tar.gz", hash = "sha256:77dec5ac77ca46eee54f59cf477780f4fb23327b3339ef39c8471abb829c1285"}, + {file = "Sphinx-3.4.3-py3-none-any.whl", hash = "sha256:c314c857e7cd47c856d2c5adff514ac2e6495f8b8e0f886a8a37e9305dfea0d8"}, + {file = "Sphinx-3.4.3.tar.gz", hash = "sha256:41cad293f954f7d37f803d97eb184158cfd90f51195131e94875bc07cd08b93c"}, ] sphinxcontrib-applehelp = [ {file = "sphinxcontrib-applehelp-1.0.2.tar.gz", hash = "sha256:a072735ec80e7675e3f432fcae8610ecf509c5f1869d17e2eecff44389cdbc58"}, diff --git a/pyproject.toml b/pyproject.toml index 9a7bab326b..5a964be802 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,7 @@ pytest = "^6.1.1" pytest-timeout = "^1.4.2" pytest-html = "^2.1.1" pytest-rerunfailures = "^9.1.1" +pytest-target = {path = "pytest-target", develop = true} pytest-lisa = {path = "pytest-lisa", develop = true} [tool.poetry.dev-dependencies] diff --git a/pytest-lisa/poetry.lock b/pytest-lisa/poetry.lock index 2517d1b1bd..6bc110e4b3 100644 --- a/pytest-lisa/poetry.lock +++ b/pytest-lisa/poetry.lock @@ -28,33 +28,6 @@ docs = ["furo", "sphinx", "zope.interface"] tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface"] tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six"] -[[package]] -name = "bcrypt" -version = "3.2.0" -description = "Modern password hashing for your software and your servers" -category = "main" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -cffi = ">=1.1" -six = ">=1.4.1" - -[package.extras] -tests = ["pytest (>=3.2.1,!=3.3.0)"] -typecheck = ["mypy"] - -[[package]] -name = "cffi" -version = "1.14.4" -description = "Foreign Function Interface for Python calling C code." -category = "main" -optional = false -python-versions = "*" - -[package.dependencies] -pycparser = "*" - [[package]] name = "colorama" version = "0.4.4" @@ -71,25 +44,6 @@ category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -[[package]] -name = "cryptography" -version = "3.3.1" -description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." -category = "main" -optional = false -python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*" - -[package.dependencies] -cffi = ">=1.12" -six = ">=1.4.1" - -[package.extras] -docs = ["sphinx (>=1.6.5,!=1.8.0,!=3.1.0,!=3.1.1)", "sphinx-rtd-theme"] -docstest = ["doc8", "pyenchant (>=1.6.11)", "twine (>=1.12.0)", "sphinxcontrib-spelling (>=4.0.1)"] -pep8test = ["black", "flake8", "flake8-import-order", "pep8-naming"] -ssh = ["bcrypt (>=3.1.5)"] -test = ["pytest (>=3.6.0,!=3.9.0,!=3.9.1,!=3.9.2)", "pretend", "iso8601", "pytz", "hypothesis (>=1.11.4,!=3.79.2)"] - [[package]] name = "execnet" version = "1.7.1" @@ -104,33 +58,9 @@ apipkg = ">=1.4" [package.extras] testing = ["pre-commit"] -[[package]] -name = "fabric" -version = "2.5.0" -description = "High level SSH command execution" -category = "main" -optional = false -python-versions = "*" - -[package.dependencies] -invoke = ">=1.3,<2.0" -paramiko = ">=2.4" - -[package.extras] -pytest = ["mock (>=2.0.0,<3.0)", "pytest (>=3.2.5,<4.0)"] -testing = ["mock (>=2.0.0,<3.0)"] - -[[package]] -name = "filelock" -version = "3.0.12" -description = "A platform independent file lock." -category = "main" -optional = false -python-versions = "*" - [[package]] name = "importlib-metadata" -version = "3.3.0" +version = "3.4.0" description = "Read metadata from Python packages" category = "main" optional = false @@ -141,8 +71,8 @@ typing-extensions = {version = ">=3.6.4", markers = "python_version < \"3.8\""} zipp = ">=0.5" [package.extras] -docs = ["sphinx", "jaraco.packaging (>=3.2)", "rst.linker (>=1.9)"] -testing = ["pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake8", "pytest-cov", "jaraco.test (>=3.2.0)", "packaging", "pep517", "pyfakefs", "flufl.flake8", "pytest-black (>=0.3.7)", "pytest-mypy", "importlib-resources (>=1.3)"] +docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"] +testing = ["pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake8", "pytest-cov", "pytest-enabler", "packaging", "pep517", "pyfakefs", "flufl.flake8", "pytest-black (>=0.3.7)", "pytest-mypy", "importlib-resources (>=1.3)"] [[package]] name = "iniconfig" @@ -152,14 +82,6 @@ category = "main" optional = false python-versions = "*" -[[package]] -name = "invoke" -version = "1.5.0" -description = "Pythonic task execution" -category = "main" -optional = false -python-versions = "*" - [[package]] name = "packaging" version = "20.8" @@ -171,25 +93,6 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" [package.dependencies] pyparsing = ">=2.0.2" -[[package]] -name = "paramiko" -version = "2.7.2" -description = "SSH2 protocol library" -category = "main" -optional = false -python-versions = "*" - -[package.dependencies] -bcrypt = ">=3.1.3" -cryptography = ">=2.5" -pynacl = ">=1.0.1" - -[package.extras] -all = ["pyasn1 (>=0.1.7)", "pynacl (>=1.0.1)", "bcrypt (>=3.1.3)", "invoke (>=1.3)", "gssapi (>=1.4.1)", "pywin32 (>=2.1.8)"] -ed25519 = ["pynacl (>=1.0.1)", "bcrypt (>=3.1.3)"] -gssapi = ["pyasn1 (>=0.1.7)", "gssapi (>=1.4.1)", "pywin32 (>=2.1.8)"] -invoke = ["invoke (>=1.3)"] - [[package]] name = "pluggy" version = "0.13.1" @@ -212,30 +115,6 @@ category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -[[package]] -name = "pycparser" -version = "2.20" -description = "C parser in Python" -category = "main" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" - -[[package]] -name = "pynacl" -version = "1.4.0" -description = "Python binding to the Networking and Cryptography (NaCl) library" -category = "main" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" - -[package.dependencies] -cffi = ">=1.4.1" -six = "*" - -[package.extras] -docs = ["sphinx (>=1.6.5)", "sphinx-rtd-theme"] -tests = ["pytest (>=3.2.1,!=3.3.0)", "hypothesis (>=3.27.0)"] - [[package]] name = "pyparsing" version = "2.4.7" @@ -296,27 +175,6 @@ schema = "0.7.2" type = "directory" url = "../pytest-playbook" -[[package]] -name = "pytest-target" -version = "0.1.0" -description = "Pytest plugin for remote target orchestration." -category = "main" -optional = false -python-versions = "^3.7" -develop = true - -[package.dependencies] -fabric = "^2.5.0" -filelock = "^3.0.12" -invoke = "^1.4.1" -pytest = "^6.1.2" -pytest-playbook = "0.1.0" -tenacity = "^6.2.0" - -[package.source] -type = "directory" -url = "../pytest-target" - [[package]] name = "pytest-xdist" version = "2.2.0" @@ -353,28 +211,6 @@ python-versions = "*" [package.dependencies] contextlib2 = ">=0.5.5" -[[package]] -name = "six" -version = "1.15.0" -description = "Python 2 and 3 compatibility utilities" -category = "main" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" - -[[package]] -name = "tenacity" -version = "6.3.1" -description = "Retry code until it succeeds" -category = "main" -optional = false -python-versions = "*" - -[package.dependencies] -six = ">=1.9.0" - -[package.extras] -doc = ["reno", "sphinx", "tornado (>=4.5)"] - [[package]] name = "toml" version = "0.10.2" @@ -406,7 +242,7 @@ testing = ["pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "876608c892824f5f43904ec3a7eca019f8d63e7862105a169cfb81ac225620bb" +content-hash = "35352fa0bb935ffb30d81fdf5ffa203e15a25d4a7fd0e1243570236a1105f647" [metadata.files] apipkg = [ @@ -421,53 +257,6 @@ attrs = [ {file = "attrs-20.3.0-py2.py3-none-any.whl", hash = "sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6"}, {file = "attrs-20.3.0.tar.gz", hash = "sha256:832aa3cde19744e49938b91fea06d69ecb9e649c93ba974535d08ad92164f700"}, ] -bcrypt = [ - {file = "bcrypt-3.2.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:c95d4cbebffafcdd28bd28bb4e25b31c50f6da605c81ffd9ad8a3d1b2ab7b1b6"}, - {file = "bcrypt-3.2.0-cp36-abi3-manylinux1_x86_64.whl", hash = "sha256:63d4e3ff96188e5898779b6057878fecf3f11cfe6ec3b313ea09955d587ec7a7"}, - {file = "bcrypt-3.2.0-cp36-abi3-manylinux2010_x86_64.whl", hash = "sha256:cd1ea2ff3038509ea95f687256c46b79f5fc382ad0aa3664d200047546d511d1"}, - {file = "bcrypt-3.2.0-cp36-abi3-manylinux2014_aarch64.whl", hash = "sha256:cdcdcb3972027f83fe24a48b1e90ea4b584d35f1cc279d76de6fc4b13376239d"}, - {file = "bcrypt-3.2.0-cp36-abi3-win32.whl", hash = "sha256:a67fb841b35c28a59cebed05fbd3e80eea26e6d75851f0574a9273c80f3e9b55"}, - {file = "bcrypt-3.2.0-cp36-abi3-win_amd64.whl", hash = "sha256:81fec756feff5b6818ea7ab031205e1d323d8943d237303baca2c5f9c7846f34"}, - {file = "bcrypt-3.2.0.tar.gz", hash = "sha256:5b93c1726e50a93a033c36e5ca7fdcd29a5c7395af50a6892f5d9e7c6cfbfb29"}, -] -cffi = [ - {file = "cffi-1.14.4-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:ebb253464a5d0482b191274f1c8bf00e33f7e0b9c66405fbffc61ed2c839c775"}, - {file = "cffi-1.14.4-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:2c24d61263f511551f740d1a065eb0212db1dbbbbd241db758f5244281590c06"}, - {file = "cffi-1.14.4-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:9f7a31251289b2ab6d4012f6e83e58bc3b96bd151f5b5262467f4bb6b34a7c26"}, - {file = "cffi-1.14.4-cp27-cp27m-win32.whl", hash = "sha256:5cf4be6c304ad0b6602f5c4e90e2f59b47653ac1ed9c662ed379fe48a8f26b0c"}, - {file = "cffi-1.14.4-cp27-cp27m-win_amd64.whl", hash = "sha256:f60567825f791c6f8a592f3c6e3bd93dd2934e3f9dac189308426bd76b00ef3b"}, - {file = "cffi-1.14.4-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:c6332685306b6417a91b1ff9fae889b3ba65c2292d64bd9245c093b1b284809d"}, - {file = "cffi-1.14.4-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:d9efd8b7a3ef378dd61a1e77367f1924375befc2eba06168b6ebfa903a5e59ca"}, - {file = "cffi-1.14.4-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:51a8b381b16ddd370178a65360ebe15fbc1c71cf6f584613a7ea08bfad946698"}, - {file = "cffi-1.14.4-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:1d2c4994f515e5b485fd6d3a73d05526aa0fcf248eb135996b088d25dfa1865b"}, - {file = "cffi-1.14.4-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:af5c59122a011049aad5dd87424b8e65a80e4a6477419c0c1015f73fb5ea0293"}, - {file = "cffi-1.14.4-cp35-cp35m-win32.whl", hash = "sha256:594234691ac0e9b770aee9fcdb8fa02c22e43e5c619456efd0d6c2bf276f3eb2"}, - {file = "cffi-1.14.4-cp35-cp35m-win_amd64.whl", hash = "sha256:64081b3f8f6f3c3de6191ec89d7dc6c86a8a43911f7ecb422c60e90c70be41c7"}, - {file = "cffi-1.14.4-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:f803eaa94c2fcda012c047e62bc7a51b0bdabda1cad7a92a522694ea2d76e49f"}, - {file = "cffi-1.14.4-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:105abaf8a6075dc96c1fe5ae7aae073f4696f2905fde6aeada4c9d2926752362"}, - {file = "cffi-1.14.4-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:0638c3ae1a0edfb77c6765d487fee624d2b1ee1bdfeffc1f0b58c64d149e7eec"}, - {file = "cffi-1.14.4-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:7c6b1dece89874d9541fc974917b631406233ea0440d0bdfbb8e03bf39a49b3b"}, - {file = "cffi-1.14.4-cp36-cp36m-win32.whl", hash = "sha256:155136b51fd733fa94e1c2ea5211dcd4c8879869008fc811648f16541bf99668"}, - {file = "cffi-1.14.4-cp36-cp36m-win_amd64.whl", hash = "sha256:6bc25fc545a6b3d57b5f8618e59fc13d3a3a68431e8ca5fd4c13241cd70d0009"}, - {file = "cffi-1.14.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a7711edca4dcef1a75257b50a2fbfe92a65187c47dab5a0f1b9b332c5919a3fb"}, - {file = "cffi-1.14.4-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:00e28066507bfc3fe865a31f325c8391a1ac2916219340f87dfad602c3e48e5d"}, - {file = "cffi-1.14.4-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:798caa2a2384b1cbe8a2a139d80734c9db54f9cc155c99d7cc92441a23871c03"}, - {file = "cffi-1.14.4-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:a5ed8c05548b54b998b9498753fb9cadbfd92ee88e884641377d8a8b291bcc01"}, - {file = "cffi-1.14.4-cp37-cp37m-win32.whl", hash = "sha256:00a1ba5e2e95684448de9b89888ccd02c98d512064b4cb987d48f4b40aa0421e"}, - {file = "cffi-1.14.4-cp37-cp37m-win_amd64.whl", hash = "sha256:9cc46bc107224ff5b6d04369e7c595acb700c3613ad7bcf2e2012f62ece80c35"}, - {file = "cffi-1.14.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:df5169c4396adc04f9b0a05f13c074df878b6052430e03f50e68adf3a57aa28d"}, - {file = "cffi-1.14.4-cp38-cp38-manylinux1_i686.whl", hash = "sha256:9ffb888f19d54a4d4dfd4b3f29bc2c16aa4972f1c2ab9c4ab09b8ab8685b9c2b"}, - {file = "cffi-1.14.4-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:8d6603078baf4e11edc4168a514c5ce5b3ba6e3e9c374298cb88437957960a53"}, - {file = "cffi-1.14.4-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:d5ff0621c88ce83a28a10d2ce719b2ee85635e85c515f12bac99a95306da4b2e"}, - {file = "cffi-1.14.4-cp38-cp38-win32.whl", hash = "sha256:b4e248d1087abf9f4c10f3c398896c87ce82a9856494a7155823eb45a892395d"}, - {file = "cffi-1.14.4-cp38-cp38-win_amd64.whl", hash = "sha256:ec80dc47f54e6e9a78181ce05feb71a0353854cc26999db963695f950b5fb375"}, - {file = "cffi-1.14.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:840793c68105fe031f34d6a086eaea153a0cd5c491cde82a74b420edd0a2b909"}, - {file = "cffi-1.14.4-cp39-cp39-manylinux1_i686.whl", hash = "sha256:b18e0a9ef57d2b41f5c68beefa32317d286c3d6ac0484efd10d6e07491bb95dd"}, - {file = "cffi-1.14.4-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:045d792900a75e8b1e1b0ab6787dd733a8190ffcf80e8c8ceb2fb10a29ff238a"}, - {file = "cffi-1.14.4-cp39-cp39-win32.whl", hash = "sha256:ba4e9e0ae13fc41c6b23299545e5ef73055213e466bd107953e4a013a5ddd7e3"}, - {file = "cffi-1.14.4-cp39-cp39-win_amd64.whl", hash = "sha256:f032b34669220030f905152045dfa27741ce1a6db3324a5bc0b96b6c7420c87b"}, - {file = "cffi-1.14.4.tar.gz", hash = "sha256:1a465cbe98a7fd391d47dce4b8f7e5b921e6cd805ef421d04f5f66ba8f06086c"}, -] colorama = [ {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"}, {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"}, @@ -476,55 +265,22 @@ contextlib2 = [ {file = "contextlib2-0.6.0.post1-py2.py3-none-any.whl", hash = "sha256:3355078a159fbb44ee60ea80abd0d87b80b78c248643b49aa6d94673b413609b"}, {file = "contextlib2-0.6.0.post1.tar.gz", hash = "sha256:01f490098c18b19d2bd5bb5dc445b2054d2fa97f09a4280ba2c5f3c394c8162e"}, ] -cryptography = [ - {file = "cryptography-3.3.1-cp27-cp27m-macosx_10_10_x86_64.whl", hash = "sha256:c366df0401d1ec4e548bebe8f91d55ebcc0ec3137900d214dd7aac8427ef3030"}, - {file = "cryptography-3.3.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:9f6b0492d111b43de5f70052e24c1f0951cb9e6022188ebcb1cc3a3d301469b0"}, - {file = "cryptography-3.3.1-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:a69bd3c68b98298f490e84519b954335154917eaab52cf582fa2c5c7efc6e812"}, - {file = "cryptography-3.3.1-cp27-cp27m-win32.whl", hash = "sha256:84ef7a0c10c24a7773163f917f1cb6b4444597efd505a8aed0a22e8c4780f27e"}, - {file = "cryptography-3.3.1-cp27-cp27m-win_amd64.whl", hash = "sha256:594a1db4511bc4d960571536abe21b4e5c3003e8750ab8365fafce71c5d86901"}, - {file = "cryptography-3.3.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:0003a52a123602e1acee177dc90dd201f9bb1e73f24a070db7d36c588e8f5c7d"}, - {file = "cryptography-3.3.1-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:83d9d2dfec70364a74f4e7c70ad04d3ca2e6a08b703606993407bf46b97868c5"}, - {file = "cryptography-3.3.1-cp36-abi3-macosx_10_10_x86_64.whl", hash = "sha256:dc42f645f8f3a489c3dd416730a514e7a91a59510ddaadc09d04224c098d3302"}, - {file = "cryptography-3.3.1-cp36-abi3-manylinux1_x86_64.whl", hash = "sha256:788a3c9942df5e4371c199d10383f44a105d67d401fb4304178020142f020244"}, - {file = "cryptography-3.3.1-cp36-abi3-manylinux2010_x86_64.whl", hash = "sha256:69e836c9e5ff4373ce6d3ab311c1a2eed274793083858d3cd4c7d12ce20d5f9c"}, - {file = "cryptography-3.3.1-cp36-abi3-manylinux2014_aarch64.whl", hash = "sha256:9e21301f7a1e7c03dbea73e8602905a4ebba641547a462b26dd03451e5769e7c"}, - {file = "cryptography-3.3.1-cp36-abi3-win32.whl", hash = "sha256:b4890d5fb9b7a23e3bf8abf5a8a7da8e228f1e97dc96b30b95685df840b6914a"}, - {file = "cryptography-3.3.1-cp36-abi3-win_amd64.whl", hash = "sha256:0e85aaae861d0485eb5a79d33226dd6248d2a9f133b81532c8f5aae37de10ff7"}, - {file = "cryptography-3.3.1.tar.gz", hash = "sha256:7e177e4bea2de937a584b13645cab32f25e3d96fc0bc4a4cf99c27dc77682be6"}, -] execnet = [ {file = "execnet-1.7.1-py2.py3-none-any.whl", hash = "sha256:d4efd397930c46415f62f8a31388d6be4f27a91d7550eb79bc64a756e0056547"}, {file = "execnet-1.7.1.tar.gz", hash = "sha256:cacb9df31c9680ec5f95553976c4da484d407e85e41c83cb812aa014f0eddc50"}, ] -fabric = [ - {file = "fabric-2.5.0-py2.py3-none-any.whl", hash = "sha256:160331934ea60036604928e792fa8e9f813266b098ef5562aa82b88527740389"}, - {file = "fabric-2.5.0.tar.gz", hash = "sha256:24842d7d51556adcabd885ac3cf5e1df73fc622a1708bf3667bf5927576cdfa6"}, -] -filelock = [ - {file = "filelock-3.0.12-py3-none-any.whl", hash = "sha256:929b7d63ec5b7d6b71b0fa5ac14e030b3f70b75747cef1b10da9b879fef15836"}, - {file = "filelock-3.0.12.tar.gz", hash = "sha256:18d82244ee114f543149c66a6e0c14e9c4f8a1044b5cdaadd0f82159d6a6ff59"}, -] importlib-metadata = [ - {file = "importlib_metadata-3.3.0-py3-none-any.whl", hash = "sha256:bf792d480abbd5eda85794e4afb09dd538393f7d6e6ffef6e9f03d2014cf9450"}, - {file = "importlib_metadata-3.3.0.tar.gz", hash = "sha256:5c5a2720817414a6c41f0a49993908068243ae02c1635a228126519b509c8aed"}, + {file = "importlib_metadata-3.4.0-py3-none-any.whl", hash = "sha256:ace61d5fc652dc280e7b6b4ff732a9c2d40db2c0f92bc6cb74e07b73d53a1771"}, + {file = "importlib_metadata-3.4.0.tar.gz", hash = "sha256:fa5daa4477a7414ae34e95942e4dd07f62adf589143c875c133c1e53c4eff38d"}, ] iniconfig = [ {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, ] -invoke = [ - {file = "invoke-1.5.0-py2-none-any.whl", hash = "sha256:da7c2d0be71be83ffd6337e078ef9643f41240024d6b2659e7b46e0b251e339f"}, - {file = "invoke-1.5.0-py3-none-any.whl", hash = "sha256:7e44d98a7dc00c91c79bac9e3007276965d2c96884b3c22077a9f04042bd6d90"}, - {file = "invoke-1.5.0.tar.gz", hash = "sha256:f0c560075b5fb29ba14dad44a7185514e94970d1b9d57dcd3723bec5fed92650"}, -] packaging = [ {file = "packaging-20.8-py2.py3-none-any.whl", hash = "sha256:24e0da08660a87484d1602c30bb4902d74816b6985b93de36926f5bc95741858"}, {file = "packaging-20.8.tar.gz", hash = "sha256:78598185a7008a470d64526a8059de9aaa449238f280fc9eb6b13ba6c4109093"}, ] -paramiko = [ - {file = "paramiko-2.7.2-py2.py3-none-any.whl", hash = "sha256:4f3e316fef2ac628b05097a637af35685183111d4bc1b5979bd397c2ab7b5898"}, - {file = "paramiko-2.7.2.tar.gz", hash = "sha256:7f36f4ba2c0d81d219f4595e35f70d56cc94f9ac40a6acdf51d6ca210ce65035"}, -] pluggy = [ {file = "pluggy-0.13.1-py2.py3-none-any.whl", hash = "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"}, {file = "pluggy-0.13.1.tar.gz", hash = "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0"}, @@ -533,30 +289,6 @@ py = [ {file = "py-1.10.0-py2.py3-none-any.whl", hash = "sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a"}, {file = "py-1.10.0.tar.gz", hash = "sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3"}, ] -pycparser = [ - {file = "pycparser-2.20-py2.py3-none-any.whl", hash = "sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705"}, - {file = "pycparser-2.20.tar.gz", hash = "sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0"}, -] -pynacl = [ - {file = "PyNaCl-1.4.0-cp27-cp27m-macosx_10_10_x86_64.whl", hash = "sha256:ea6841bc3a76fa4942ce00f3bda7d436fda21e2d91602b9e21b7ca9ecab8f3ff"}, - {file = "PyNaCl-1.4.0-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:d452a6746f0a7e11121e64625109bc4468fc3100452817001dbe018bb8b08514"}, - {file = "PyNaCl-1.4.0-cp27-cp27m-win32.whl", hash = "sha256:2fe0fc5a2480361dcaf4e6e7cea00e078fcda07ba45f811b167e3f99e8cff574"}, - {file = "PyNaCl-1.4.0-cp27-cp27m-win_amd64.whl", hash = "sha256:f8851ab9041756003119368c1e6cd0b9c631f46d686b3904b18c0139f4419f80"}, - {file = "PyNaCl-1.4.0-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:7757ae33dae81c300487591c68790dfb5145c7d03324000433d9a2c141f82af7"}, - {file = "PyNaCl-1.4.0-cp35-abi3-macosx_10_10_x86_64.whl", hash = "sha256:757250ddb3bff1eecd7e41e65f7f833a8405fede0194319f87899690624f2122"}, - {file = "PyNaCl-1.4.0-cp35-abi3-manylinux1_x86_64.whl", hash = "sha256:30f9b96db44e09b3304f9ea95079b1b7316b2b4f3744fe3aaecccd95d547063d"}, - {file = "PyNaCl-1.4.0-cp35-abi3-win32.whl", hash = "sha256:4e10569f8cbed81cb7526ae137049759d2a8d57726d52c1a000a3ce366779634"}, - {file = "PyNaCl-1.4.0-cp35-abi3-win_amd64.whl", hash = "sha256:c914f78da4953b33d4685e3cdc7ce63401247a21425c16a39760e282075ac4a6"}, - {file = "PyNaCl-1.4.0-cp35-cp35m-win32.whl", hash = "sha256:06cbb4d9b2c4bd3c8dc0d267416aaed79906e7b33f114ddbf0911969794b1cc4"}, - {file = "PyNaCl-1.4.0-cp35-cp35m-win_amd64.whl", hash = "sha256:511d269ee845037b95c9781aa702f90ccc36036f95d0f31373a6a79bd8242e25"}, - {file = "PyNaCl-1.4.0-cp36-cp36m-win32.whl", hash = "sha256:11335f09060af52c97137d4ac54285bcb7df0cef29014a1a4efe64ac065434c4"}, - {file = "PyNaCl-1.4.0-cp36-cp36m-win_amd64.whl", hash = "sha256:cd401ccbc2a249a47a3a1724c2918fcd04be1f7b54eb2a5a71ff915db0ac51c6"}, - {file = "PyNaCl-1.4.0-cp37-cp37m-win32.whl", hash = "sha256:8122ba5f2a2169ca5da936b2e5a511740ffb73979381b4229d9188f6dcb22f1f"}, - {file = "PyNaCl-1.4.0-cp37-cp37m-win_amd64.whl", hash = "sha256:537a7ccbea22905a0ab36ea58577b39d1fa9b1884869d173b5cf111f006f689f"}, - {file = "PyNaCl-1.4.0-cp38-cp38-win32.whl", hash = "sha256:9c4a7ea4fb81536c1b1f5cc44d54a296f96ae78c1ebd2311bd0b60be45a48d96"}, - {file = "PyNaCl-1.4.0-cp38-cp38-win_amd64.whl", hash = "sha256:7c6092102219f59ff29788860ccb021e80fffd953920c4a8653889c029b2d420"}, - {file = "PyNaCl-1.4.0.tar.gz", hash = "sha256:54e9a2c849c742006516ad56a88f5c74bf2ce92c9f67435187c3c5953b346505"}, -] pyparsing = [ {file = "pyparsing-2.4.7-py2.py3-none-any.whl", hash = "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"}, {file = "pyparsing-2.4.7.tar.gz", hash = "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1"}, @@ -570,7 +302,6 @@ pytest-forked = [ {file = "pytest_forked-1.3.0-py2.py3-none-any.whl", hash = "sha256:dc4147784048e70ef5d437951728825a131b81714b398d5d52f17c7c144d8815"}, ] pytest-playbook = [] -pytest-target = [] pytest-xdist = [ {file = "pytest-xdist-2.2.0.tar.gz", hash = "sha256:1d8edbb1a45e8e1f8e44b1260583107fc23f8bc8da6d18cb331ff61d41258ecf"}, {file = "pytest_xdist-2.2.0-py3-none-any.whl", hash = "sha256:f127e11e84ad37cc1de1088cb2990f3c354630d428af3f71282de589c5bb779b"}, @@ -594,14 +325,6 @@ schema = [ {file = "schema-0.7.2-py2.py3-none-any.whl", hash = "sha256:3a03c2e2b22e6a331ae73750ab1da46916da6ca861b16e6f073ac1d1eba43b71"}, {file = "schema-0.7.2.tar.gz", hash = "sha256:b536f2375b49fdf56f36279addae98bd86a8afbd58b3c32ce363c464bed5fc1c"}, ] -six = [ - {file = "six-1.15.0-py2.py3-none-any.whl", hash = "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"}, - {file = "six-1.15.0.tar.gz", hash = "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259"}, -] -tenacity = [ - {file = "tenacity-6.3.1-py2.py3-none-any.whl", hash = "sha256:baed357d9f35ec64264d8a4bbf004c35058fad8795c5b0d8a7dc77ecdcbb8f39"}, - {file = "tenacity-6.3.1.tar.gz", hash = "sha256:e14d191fb0a309b563904bbc336582efe2037de437e543b38da749769b544d7f"}, -] toml = [ {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, diff --git a/pytest-lisa/pyproject.toml b/pytest-lisa/pyproject.toml index 3b73f0eab7..22b5db8e3f 100644 --- a/pytest-lisa/pyproject.toml +++ b/pytest-lisa/pyproject.toml @@ -11,7 +11,6 @@ packages = [{include = "lisa.py"}] python = "^3.7" pytest = "^6.1.2" pytest-playbook = {path = "../pytest-playbook", develop = true} -pytest-target = {path = "../pytest-target", develop = true} pytest-xdist = "^2.1.0" schema = "0.7.2" From df1084f954a5c1aaf4629c9ece9524be0dedb8d5 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Wed, 13 Jan 2021 11:46:15 -0800 Subject: [PATCH 182/194] Update TSD to 0.4 --- DESIGN.rst | 1336 +++++++++++++++--------------- pytest-lisa/lisa.py | 5 +- pytest-target/target/__init__.py | 11 +- pytest-target/target/plugin.py | 4 +- pytest-target/target/target.py | 10 +- testsuites/test_smoke_b.py | 7 +- 6 files changed, 684 insertions(+), 689 deletions(-) diff --git a/DESIGN.rst b/DESIGN.rst index 838b931cf9..119dc906d1 100644 --- a/DESIGN.rst +++ b/DESIGN.rst @@ -1,119 +1,141 @@ Technical Specification Document ================================ -This document outlines the technical specifications for LISAv3. We are -evaluating the feasibility of leveraging -`Pytest `_ as our test runner. +This document outlines the technical specifications and design for +LISAv3 leveraging `Pytest `_ as +the test runner. -Please see `PR #1065 `_ for a -working, proof-of-concept prototype. +Please see `PR #1107 `_ +for a working implementation and see the `documentation`_. -Authored by Andrew Schwartzmeyer (he/him), version 0.3.0. +.. _documentation: https://microsoft.github.io/lisa/. + +:Author: Andrew Schwartzmeyer (he/him) +:Version: 0.4 Why Pytest? ----------- -Pytest is an `incredibly -popular `_ MIT licensed -open source Python testing framework. It has a thriving community and -plugin framework, with over 750 -`plugins `_. Instead of writing -(and therefore maintaining) yet another test framework, we would do more -with less by reusing Pytest and existing plugins. This will allow us to +Pytest is an `incredibly popular +`_ MIT licensed open +source Python testing framework. It has a thriving community and +plugin framework, with over 750 `plugins +`_. Instead of writing (and +therefore maintaining) yet another test framework, we will do more +with less by reusing Pytest and existing plugins. This allows us to focus on our unique problems: organizing and understanding our tests, deploying necessary resources (such as Azure, Hyper-V, or bare metal machines, collectively known as “targets”), and analyzing our results. -In fact, most of Pytest itself is implemented via `built-in -plugins `_, providing -us with many useful and well-documented examples. Furthermore, when -others were confronted with a problem similar to our own they also chose -to use Pytest. `Labgrid `_ -is an open source embedded board control library that delegated the -testing framework logic to Pytest in their -`design `_, +Most of Pytest itself is implemented via `built-in plugins +`_, providing us with +many useful and well-documented examples. Furthermore, when others +were confronted with a problem similar to our own they also chose to +use Pytest. + +`Labgrid`_ is an open source embedded board control library that +delegated the testing framework logic to Pytest in their `design +`_, and `U-Boot `_, an embedded board -boot loader, similarly leveraged Pytest in their -`tests `_. -KernelCI and Avocado were also evaluated by the Labgrid developers at an -`Embedded Linux Conference `_ and both -ruled out for reasons similar to our own before they settled on Pytest. +boot loader, similarly leveraged Pytest in their `tests +`_. KernelCI and +Avocado were also evaluated by the Labgrid developers at an `Embedded +Linux Conference `_ and both ruled out +for reasons similar to our own before they settled on Pytest. + +.. _Labgrid: https://github.com/labgrid-project/labgrid The `fundamental features `_ of Pytest match our needs very well: -- Automatic test discovery, no boiler-plate test code -- Useful information when a test fails (assertions are introspected) -- Test and fixture - `parameterization `_ -- Modular setup/teardown via - `fixtures `_ -- Incredibly customizable (as detailed above) - -So all the logic for describing, discovering, running, skipping and -reporting results of the tests, as well as enabling and importing users’ -plugins is already written and maintained by the open source community. -This leaves us to focus on our hard and specific problems: creating an -abstraction to launch the necessary targets, organizing and publishing -our tests, and reporting test results upstream. Using Pytest would also -allow us the space to abstract other commonalities in our specific -tests. In this way, LISAv3 could solve the difficulties we have at hand -without creating yet another test framework. - -Finally, by leveraging such a popular framework and reducing the amount -of code we need to maintain, we drastically increase our chances of -receiving pull requests instead of bug reports from users. This is -important because despite our best efforts it is practically guaranteed -that as adoption of LISAv3 increases, users will want changes to be -made, and we need to empower them to do so themselves. +- Automatic test discovery, no boiler-plate test code +- Useful information when a test fails (assertions are introspected) +- Test and fixture `parameterization`_ +- Modular setup/teardown via `fixtures`_ +- Incredibly customizable (as detailed above) + +.. _parameterization: https://docs.pytest.org/en/stable/parametrize.html +.. _fixtures: https://docs.pytest.org/en/stable/fixture.html + +All the logic for describing, discovering, running, skipping and +reporting results of the tests, as well as enabling and importing +users’ plugins is already written and maintained by the open source +community. This leaves us to focus on our hard and specific problems: +creating an abstraction to launch the necessary targets, organizing +and publishing our tests, and reporting test results upstream. Using +Pytest also allows us the space to abstract other commonalities in our +specific tests. In this way, LISAv3 could solve the difficulties we +have at hand without creating yet another test framework. + +By leveraging such a popular framework we maximize the ease of +adoption for developers to write tests, as they are likely already +familiar with Pytest, and if not, have a wealth of examples and +`resources`_ from which to draw. The environment will be one of +instant familiarity, thus providing developers a running start. + +.. _resources: https://docs.pytest.org/en/stable/example/index.html + +Finally, by reducing the amount of code we maintain, we drastically +increase our chances of receiving pull requests instead of bug reports +from users. This is important because despite our best efforts it is +practically guaranteed that as adoption of LISAv3 increases, users +will want changes to be made, and we need to empower them to do so +themselves. Using Pytest gives us the best chances for users to +understand and extend the framework, plugins, etc. with ease. What are we maintaining? ------------------------ -The current proof-of-concept implementation uses the top-level -``conftest.py`` file to define our “plugin” functionality. This works, -but it is not ideal. I believe that we will want to publish two open -source Pytest plugins as packages on `PyPI `_, the -Python Package Index: ``pytest-target`` and ``pytest-lisa``. We will -also maintain our set of public “LISA” tests, but these should simply -install and use our plugins. +We have three Pytest plugins, soon to be published on `PyPI +`_, supporting the framework: + +- `pytest-target`_ +- `pytest-lisa`_ +- `pytest-playbook`_ -The ``pytest-target`` plugin should encapsulate all our logic for *how* -and *when* to deploy targets (local or cloud virtual machines, or bare +We will also maintain our set of public “LISA” tests, but these are +decoupled from the plugins and packages. + +The `pytest-target`_ plugin encapsulates all our logic for *how* and +*when* to deploy targets (local or cloud virtual machines, or bare metal machines, and all the associated resources), run tests on the specified targets, and delete the targets. This includes specifying which features and resources each test needs and each given target provides (such as number of cores, amount of RAM, and other hardware like a GPU etc.), how to deploy and delete each target based on its -platform, and parameterization of the ``target`` fixture based on CLI or -YAML file input. In fact, some tests (like networking) will require -multiple targets at once. This plugin will need to manage resources +platform, and `parameterization`_ of the +:py:func:`~target.plugin.target` fixture based on YAML file input (the +“playbook”). In fact, some tests (like networking) require multiple +targets at once. This plugin will need to manage resources intelligently, being able to optimize for both time and cost, and make it easy for tests to request and use various resources. -The ``pytest-lisa`` plugin should encapsulate all our logic for how to +The `pytest-lisa`_ plugin encapsulates all our logic for how to organize and select tests, as well as our opinions on displaying test -results. This includes the user modes, test metadata and inventory, test -selection based on criteria against that metadata, required and -pre-configured upstream plugins, and result notifiers. It will similarly -support both CLI and YAML file input. - -We should strive to keep these plugins from depending on each other in -order to keep their scope well-defined. In the “LISA” repository of -tests we will depend on the two plugins and maintain additional -`fixtures `_ for our -tests’ unique requirements. Similarly, we and others may have private -test repositories which build upon the above by defining new platform -support and internal service integrations. The built-in plugin discovery -of Pytest (via ``conftest.py`` files) enables us to satisfy one of our -requirements to “support plugins to orchestrate the test environment.” - -Finally, a third smaller utility plugin, ``pytest-schema`` may be -written in order to share the common functionality of registering -component schemata (e.g. platform and target parameters from -``pytest-target`` and selection criteria from ``pytest-lisa``). This is -somewhat of an implementation detail, but would be a third and -lower-level library we can publish. +results. This includes the user modes, test metadata and inventory, +test selection based on criteria against that metadata, required and +pre-configured upstream plugins, and result notifiers. It will +similarly support YAML file playbook input. + +The `pytest-playbook`_ plugin encapsulates the shared common +functionality of registering component schemata (e.g. platform and +target parameters from `pytest-target`_ and selection criteria from +`pytest-lisa`_). It uses the `schema`_ library. + +.. _schema: https://pypi.org/project/schema/ + +We have striven to keep `pytest-lisa`_ and `pytest-target`_ from +depending on each other in order to keep their scope well-defined. +They both depend on `pytest-playbook`_, and the “LISA” project depends +on them both, but they are independent plugins. + +In the “LISA” repository of tests we may also maintain additional +`fixtures`_ for our tests’ unique requirements. Similarly, we and +others may have private test repositories which build upon the above +by defining new platform support and internal service integrations. +The built-in plugin discovery of Pytest (via ``conftest.py`` files) +enables us to satisfy one of our requirements to “support plugins to +orchestrate the test environment.” pytest-target ------------- @@ -121,97 +143,72 @@ pytest-target How are targets provided and accessed? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -First we need to define “target” as an instance of a system-under-test. -That is, given some environment requirements, such an Azure image (URN) -and size (SKU), a target would be a virtual machine deployed by -``pytest-target`` with SSH access provided to the requesting test. A -target could optionally be pre-deployed and simply connected. Some tests -may request multiple targets as well. - -Pytest uses -`fixtures `_, which are -the primary way of setting up test requirements. They replace less -flexible alternatives like setup/teardown functions. It is through -fixtures that we implement remote target setup/teardown. Our ``target`` -fixture returns a ``Target`` instance, which currently provides: - -- Remote shell access via SSH -- Data including hostname / IP address -- Cross-platform ping functionality with exponential back-off -- Uploading of local files to arbitrary remote destinations -- Downloading of remote file contents into local string variable -- Asynchronous remote command execution with promises - -The ``Azure(Target)`` subclass additionally provides: - -- Automatic provisioning of an Azure VM given URN and SKU -- Allowing ICMP ping via Azure firewall rules -- Azure platform forced reboot by API -- Downloading boot diagnostics (serial console log) from platform - -The prototype demonstrates how easy it is to quickly implement these -features. As we need more features, they can be readily added and shared -among tests. - -The ``Target`` class leverages `Fabric `_ -which is a popular high-level Python library for executing shell -commands on remote systems over SSH. Underneath the covers Fabric uses -`paramiko `_, the most popular -low-level Python SSH library. Fabric does the heavy lifting of safely -connecting and disconnecting from the node, executing the shell command -(synchronously or asynchronously), reporting the exit status, gathering -the stdout and stderr, providing stdin (or interactive auto-responses, -similar to ``expect``), uploading and downloading files, and much more. -In fact, these APIs are all available and implemented for the local -machine by the underlying `Inovke `_ -library, which is essentially a Python ``subprocess`` wrapper with “a -powerful and clean feature set.” +First we need to define “target” as an instance of a +system-under-test. That is, given some environment requirements, such +an Azure image (URN) and size (SKU), a target would be a virtual +machine deployed by `pytest-target`_ with SSH access provided to the +requesting test. A target could optionally be pre-deployed and simply +connected. Some tests may request multiple targets as well. + +Pytest uses `fixtures`_, which are the primary way of setting up test +requirements. They replace less flexible alternatives like +setup/teardown functions. It is through fixtures that we implement +remote target setup/teardown. Our :py:func:`~target.plugin.target` +fixture returns a :py:class:`~target.target.Target` instance, which +currently provides: + +- Remote shell access via SSH using `Fabric`_ +- Data including hostname / IP address +- Cross-platform ping functionality with exponential back-off +- Uploading of local files to arbitrary remote destinations +- Downloading of remote file contents into local string variable +- Asynchronous remote command execution with promises + +.. _Fabric: https://www.fabfile.org/ + +The :py:class:`~target.azure.AzureCLI` subclass additionally provides: + +- An example of a working platform implementation +- Automatic provisioning of a parameterized Azure VM +- Allowing ICMP ping via Azure firewall rules +- Azure platform forced reboot by API +- Downloading boot diagnostics (serial console log) + +The :py:class:`~target.target.SSH` subclass is a simple implementation +which only connects to a given host. + +The :py:class:`~target.target.Target` class leverages `Fabric`_ which +is a popular high-level Python library for executing shell commands on +remote systems over SSH. Underneath the covers Fabric uses +`Paramiko`_, the most popular low-level Python SSH library. Fabric +does the heavy lifting of safely connecting and disconnecting from the +node, executing the shell command (synchronously or asynchronously), +reporting the exit status, gathering the ``stdout`` and ``stderr``, +providing ``stdin`` (or interactive auto-responses, similar to +``expect``), uploading and downloading files, and much more. In fact, +these APIs are all available and implemented for the local machine by +the underlying `Invoke`_ library, which is essentially a Python +``subprocess`` wrapper with “a powerful and clean feature set.” + +.. _Paramiko: https://docs.paramiko.org/en/stable/ +.. _Invoke: https://www.pyinvoke.org/ Other test specific requirements, such as installing software and -daemons, downloading files from remote storage, or checking the state of -our Bash test scripts, would similarly be implemented by methods on the -``Target`` class or via additional fixtures and thus shared among tests. - -How do we interact with Azure? -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -For Azure, we currently use the `Azure -CLI `_ to deploy a virtual machine. For -Hyper-V (and other virtualization platforms), we would like to use -`libvirt `_, and for embedded / bare -metal environments we are evaluating -`labgrid `_. - -If possible, we do not want to use the `Azure Python -APIs `_ directly because they are more -complicated (and less documented) than the `Azure -CLI `_. With Invoke (as discussed above), -``az`` becomes incredibly easy to work with. The Azure CLI lead -developer states that they have `feature -parity `_ and that the -CLI is more straightforward to use. Considering our ease-of-maintenance -requirement, this seems the apt choice, especially since the Azure CLI -supports deploying resources with `ARM -templates `_. -If it later becomes necessary to use the Python APIs directly, that is, -of course, still doable (and we can reuse existing code doing it). - -On the topic of “servicing” the Azure CLI, its developers state that “at -command level, packages only upgrading the PATCH version guarantee -backward compatibility.” The tool is also intended to be used in -scripts, so servicing would amount to documenting the tested version and -having the Azure class check that it’s compatible before using it (or -warning and then trying its best). - -What’s the ``Target`` class? -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -In version 0.1 of this design document we detailed a planned refactor of -what was then called the ``Node`` class. This has since been executed -with just a few modifications (one being the rename to ``Target``, as -``Node`` was found to be an overloaded term in the context of data -centers). This class and its subclasses are decoupled from Pytest, and -are used via fixtures. It looks like this: +daemons, downloading files from remote storage, or checking the state +of our Bash test scripts, would similarly be implemented by methods on +:py:class:`~target.target.Target`, its subclasses, or via additional +fixtures and thus shared among tests. + +What’s the :py:class:`~target.target.Target` class? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In version 0.1 of this design document we detailed a planned refactor +of what was then called the ``Node`` class. This has since been +executed with just a few modifications (one being the rename to +:py:class:`~target.target.Target`, as ``Node`` was found to be an +overloaded term in the context of data centers). This class and its +subclasses are decoupled from Pytest, and are used via fixtures. Its +interface looks like this: .. code:: python @@ -220,130 +217,186 @@ are used via fixtures. It looks like this: import fabric class Target(ABC): - parameters: Mapping[str, str] - features: Set[str] + + group: str + params: Dict[str, str] + features: List[str] + data: Dict[Any, Any] + number: int + locked: bool name: str host: str conn: fabric.Connection # Provides run, sudo, get, put etc. def __init__(...): ... + self.params = self.get_schema().validate(params) + self.name = f"{self.group}-{self.number}" self.host = self.deploy() - self.conn = fabric.Connection(self.host) + self.conn = fabric.Connection(self.host, ...) @classmethod - @property @abstractmethod - def schema(cls) -> Schema: - """Must return the parameters schema for setup.""" + def schema(cls) -> Mapping[Any, Any]: + """Must return a mapping for expected instance parameters.""" + ... + + @classmethod + def defaults(cls) -> Mapping[Any, Any]: + """Can return a mapping for default parameters.""" ... @abstractmethod def deploy(self) -> str: - """Must deploy the target resources and return hostname.""" + """Must deploy the target resources and return the hostname.""" ... @abstractmethod def delete(self) -> None: - """Must delete the target resources.""" + """Must delete the target's resources.""" ... - @classmethod - def local(...) -> Result: - """Runs a local shell command.""" - ... +This class allows us to answer the next question. -How are platforms implemented? -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +How are new platforms supported? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Platform support is implemented by subclassing +:py:class:`~target.target.Target` and implementing the abstract +methods in the above interface: + +- :py:meth:`~target.target.Target.schema`: Define the schema for the platform’s parameters +- :py:meth:`~target.target.Target.defaults`: Define defaults for those parameters +- :py:meth:`~target.target.Target.deploy`: Create an instance resource +- :py:meth:`~target.target.Target.delete`: Delete the instance and its resources -Platform support is implemented by subclassing ``Target`` and defining -the ``schema`` property, ``deploy`` method, ``delete`` method, and any -platform-specific methods. Using the ``__subclasses__`` attribute of -``Target`` the available platforms and their parameter schemata are -automatically gathered from users’ own ``conftest.py`` files and other -plugins. This enables the ``target`` fixture to dynamically instantiate -a target from the gathered requirements and parameters. +Internally we use the ``__subclasses__`` attribute of +:py:class:`~target.target.Target` to automatically gather all the +available platforms and their parameter schemata from users’ own +``conftest.py`` files and other plugins. This enables the +:py:func:`~target.plugin.target` fixture to dynamically instantiate a +target from the gathered requirements and parameters. -For example, the ``Azure(Target)`` class defines its required parameters -using the `schema `_ library like -this: +For example, the :py:class:`~target.azure.AzureCLI` subclass defines +its required parameters using the `schema`_ library like this: .. code:: python from schema import Optional, Schema from target import Target - class Azure(Target): + class AzureCLI(Target): ... - schema: Schema = Schema( - { - # TODO: Maybe validate as URN or path etc. + @classmethod + def schema(cls) -> Dict[Any, Any]: + return { "image": str, - Optional("sku", default="Standard_DS1_v2"): str, - Optional("location", default="eastus2"): str, - Optional("networking", default=""): str, + Optional("sku"): str, + Optional("location"): str, } - ) -In the YAML playbook, a set of Azure targets can then be defined like -this: +Simply through defining this subclass the user can now specify a set +of parameterized YAML targets in a playbook like this: .. code:: yaml + platforms: + AzureCLI: + sku: Standard_DS2_v2 + targets: - name: Debian - platform: Azure - parameters: - image: credativ:Debian:9:9.0.201706190 - location: westus2 + platform: AzureCLI + image: Debian:debian-10:10:latest - name: Ubuntu - platform: Azure - parameters: - image: UbuntuLTS - sku: Standard_DS3_v2 - -These targets are then used to parameterize the ``target`` fixture in -the -`pytest_generate_tests `_ -hook (see below for more details). - -This demonstrated how we can have platforms define their own schema and -register that schema automatically. A pending update to this is to have -two schemata per ``Target`` subclass: target-level and platform-level -(the former is what’s demonstrated above, the latter would be common -settings, such as subscription). + platform: AzureCLI + image: Canonical:UbuntuServer:18.04-LTS:latest + + +These targets are then used to parameterize the +:py:func:`~target.plugin.target` fixture in the +:py:func:`~target.plugin.pytest_generate_tests` hook (see below for +more details). + +This demonstrated how we can have platforms define their own schema +and register that schema automatically. The ``platforms`` key allows a +playbook to override the defaults in the platform implementation, +which are then eclipsed for each named target in the ``targets`` key. +This is accomplished through internal details in `pytest-target`_’s +hook implementation :py:func:`~target.plugin.pytest_playbook_schema` +using the `pytest-playbook`_ plugin, but for the users, it just works. + +How do we interact with Azure? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For :py:class:`~target.azure.AzureCLI`, we use the `Azure CLI`_ to +deploy a virtual machine. For Hyper-V (and other virtualization +platforms), we would like to use `libvirt`_, and for embedded / bare +metal environments we are evaluating `Labgrid`_. + +.. _Azure CLI: https://aka.ms/azureclidocs +.. _libvirt: https://libvirt.org/python.html + +If possible, we do not want to use the `Azure Python APIs +`_ directly because they are more +complicated (and less documented) than the `Azure CLI`_. With +`Invoke`_ (as discussed above), ``az`` becomes incredibly easy to work +with. The Azure CLI lead developer states that they have `feature +parity `_ and that the +CLI is more straightforward to use. Considering our +ease-of-maintenance requirement, this seems the apt choice, especially +since the Azure CLI supports deploying resources with `ARM templates +`_. + +If it later becomes necessary to use the Python APIs directly, that +is, of course, still doable (and we can reuse existing code doing it). +This implementation can coexist as simply another class, ``AzureAPI``. + +On the topic of “servicing” the `Azure CLI`_, its developers state +that “at command level, packages only upgrading the PATCH version +guarantee backward compatibility.” The tool is also intended to be +used in scripts, so servicing would amount to documenting the tested +version and having the Azure class check that it’s compatible before +using it (or warning and then trying its best). How are requirements examined? ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The ``features`` attribute is currently a set of strings and (combined -with the parameters dictionary) was used to demonstrate how we can test -if an existing target instance (representing a deployed machine) met a -test’s requirements. It should be updated with a ``Requirements`` class -that represents all physical attributes of the target, and a -``requires`` Pytest mark should be added which takes instances of this -class. Two ``Requirements`` should be comparable to determine if one set -meets (or exceeds) the other set. Existing code that does this can be -reused here. +The :py:attr:`~target.target.TargetData.features` attribute is +currently a list of strings and (combined with the +:py:attr:`~target.target.TargetData.params` dictionary) is used to +demonstrate how we can test if an existing target instance +(representing a deployed machine) met a test’s requirements. It should +be updated with a ``Requirements`` class that represents all physical +attributes of the target. The :py:mod:`target.plugin` module defines a +``@pytest.mark.target`` `pytest-mark`_ which takes the features list +but should instead take instances of this ``Requirements`` class. Two +``Requirements`` should be comparable to determine if one set meets +(or exceeds) the other set. Existing code that does this can be reused +for this. + +.. _pytest-mark: https://docs.pytest.org/en/stable/mark.html How do we share common tasks? ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Common tasks for targets like rebooting and pinging should be -implemented on the ``Target`` class, and platform-specific tasks on the -respective subclass. +implemented on the :py:class:`~target.target.Target` class, and +platform-specific tasks on the respective subclass. -Methods available from ``Connection`` include ``run()`` and ``sudo()`` -which are used to easily run arbitrary commands, and ``get()`` and -``put()`` to download and upload arbitrary files. +Methods available from :py:attr:`~target.target.Target.conn` include +``run()`` and ``sudo()`` which are used to easily run arbitrary +commands, and ``get()`` and ``put()`` to download and upload arbitrary +files. -The ``cat()`` method wraps ``get()`` and returns the file as data in a -string. This makes test code like this possible: +The :py:meth:`~target.target.Target.cat` method wraps ``get()`` and +returns the file as data in a string. This makes test code like this +possible: .. code:: python - assert target.conn.cat("state.txt") == "TestCompleted" + assert target.cat("state.txt") == "TestCompleted" A ``reboot()`` method should be added that first tries to use ``sudo("reboot", timeout=5)`` (with a short timeout to avoid a hung SSH @@ -352,8 +405,9 @@ machine has rebooted by checking either ``uptime`` or the existence of a file created before the reboot. This is to avoid having to ``sleep()`` and just guess the amount of time it takes to reboot. -A ``restart()`` method should “power cycle” the machine using the -platform’s API, and thus is in abstract method. +The :py:meth:`~target.target.Target.restart` method should “power +cycle” the machine using the platform’s API, and thus is in abstract +method as each platform needs to implement it differently. Other tools and shared logic should be implemented as necessary. A major area of concern is the automatic and package-manager agnostic @@ -363,307 +417,190 @@ previously and can be reused. How are targets requested and managed? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -We implement a pair of Pytest fixtures to provide targets. The first is -the ``pool`` fixture, which looks like: +In version 0.3 of this design document we detailed how we used a +session-scoped ``pool()`` fixture to manage targets across an entire +test session. This has since been replaced with an enhanced disk-based +`cache`_, accessed through a context manager with an atomic file lock: -.. code:: python - - @pytest.fixture(scope="session") - def pool(request: SubRequest) -> Iterator[List[Target]]: - """This fixture tracks all deployed target resources.""" - targets: List[Target] = [] - yield targets - for t in targets: - t.delete() - -The ``pool`` fixture is setup once at the beginning of the test session, -at which point the ``targets`` list is then provided as input to every -instance of the ``target`` fixture. While currently a list, to support -optimal scheduling we will likely want to use a priority queue, where -the priority of a target represents its cost (whether in terms of time -or money), allowing us to provide either the fastest or the cheapest -target to each request. Targets not in use will be deallocated, and all -targets will be automatically deleted after the tests are finished -(unless the user requested otherwise, in which case they’ll be cached). - -Note that cross-session -`caching `_ is provided -by Pytest, and very easy to work with. An early prototype implemented a -``--keep-vms`` flag successfully, and this will be implemented again -with the updated design. - -The second is the ``target`` fixture, which looks like: +.. _cache: https://docs.pytest.org/en/stable/cache.html .. code:: python - @pytest.fixture - def target(pool: List[Target], request: SubRequest) -> Iterator[Target]: - """This fixture provides a connected target for each test.""" - platform: Type[Target] = playbook.PLATFORMS[request.param["platform"]] - parameters: Dict[str, Any] = request.param["parameters"] - marker = request.node.get_closest_marker("lisa") - features = set(marker.kwargs["features"]) - - # TODO: If `t` is not already in use, deallocate the previous target. - for t in pool: - if isinstance(t, platform) and t.parameters == parameters and t.features >= features: - yield t - break - else: - t = platform(parameters, features) - pool.append(t) - yield t - t.connection.close() - -This is obviously still an early implementation, but it is viable. By -using the -`pytest_collection_modifyitems `_ -hook to sort (and so group) the tests by their requirements, the tests -would efficiently reuse targets. This fixture is indirectly -parameterized during setup with the -`pytest_generate_tests `_ -hook. Test and fixture -`parameterization `_ -is a huge feature of Pytest. When we parameterize the ``target`` fixture -for multiple targets (e.g. “Ubuntu” and “Debian”), Pytest automatically -creates a set of tests for each target. So ``test_smoke`` turns into -``test_smoke[Ubuntu]`` and ``test_smoke[Debian]``. This allows us to run -a collection of tests against multiple targets with ease. These targets -are defined in a YAML file and validated against the parameters -collected from the previously described platform subclasses. - -The entire implementation looks like so: + from filelock import FileLock + + @contextmanager + def target_pool(config: Config) -> Generator[Dict[str, Any], None, None]: + """Exclusive access to the cached targets pool.""" + assert config.cache is not None + lock = Path(config.cache.makedir("target")) / "pool.lock" + with FileLock(str(lock)): + pool = config.cache.get("target/pool", {}) + yield pool + config.cache.set("target/pool", pool) + +Note that the cross-session `cache`_ is provided by Pytest, and very +easy to work with. The key maps to a file path, and the data stored +and read is JSON. So our targets are serializable: internally the +`data class`_ :py:attr:`~target.target.TargetData` implements the +methods :py:meth:`~target.target.TargetData.to_json` and +:py:meth:`~target.target.TargetData.from_json`, and the +:py:func:`~target.plugin.target` fixture creates new instances of +:py:class:`~target.target.Target` for the requesting test from either +a “fit” cached target (and so locks it) or deploys a new target. + +.. _data class: https://docs.python.org/3/library/dataclasses.html .. code:: python - TARGETS: List[Dict[str, Any]] = [] - TARGET_IDS: List[str] = [] - - def pytest_configure(config: Config) -> None: - book = get_playbook(config.getoption("--playbook")) - for t in book.get("targets", []): - TARGETS.append(t) - TARGET_IDS.append(t["name"]) - - def pytest_generate_tests(metafunc: Metafunc) -> None: - if "target" in metafunc.fixturenames: - assert TARGETS, "No targets specified!" - metafunc.parametrize("target", TARGETS, True, TARGET_IDS) - -The function ``get_playbook()`` only imports the -`PyYAML `_ library, opens -the playbook file ``f`` within a context manager, and returns -``playbook.schema.validate(yaml.load(f))``. This is leveraging Pytest’s -existing parameterization technology to achieve one of our “test -entrance” goals of requesting environments with a YAML playbook, and one -of our “test parameter validation” goals of validating platforms before -executing tests so that we can fail fast if a target has insufficient -information to be setup. Parsing the same parameters from a CLI can also -be implemented. - -Finally, once the ``target`` fixture has returned a working and -sanity-checked environment to the requesting test, the test is capable -of examining any and all attributes of the ``Target`` and quickly -marking itself as skipped, expected to fail, or failed before executing -the body of the test. Our static type checking enables developers to -ensure that the platform they requested supports all methods and fields -they use by annotating the test’s ``target`` parameter with the expected -platform type (or types). Ensuring the effectiveness of this type -checking will require us to carefully update our platform -implementations, and not rely on arbitrary objects of data. (For -example, add an ``internal_address`` field to ``Azure``, don’t just look -up ``data["internal_address"]``.) - -How are tests executed in parallel? -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -While our original list of goals stated that we want to run tests “in -parallel” we were not specific about what was meant, and the topic of -parallelism and concurrency is understandably complex. We certainly -don’t mean running two tests at once on the same target, as this would -undoubtedly lead to flaky tests. - -Assuming that we care about a set of tests passing on a particular image -and size combination, but not necessarily on a particular deployed -instance, then we can run tests concurrently by deploying multiple -“identical” targets and splitting the tests across them. The tests would -still run in isolation on each target. This sounds hard, but actually -it’s practically free with Pytest via -`pytest-xdist `_. - -The default ``pytest-xdist`` implementation simply takes the list of -tests and runs them in a round-robin fashion with the desired number of -executors. We’ve talked at length about being able to schedule groups of -tests to run in particular executors and using particular targets. While -there are many paths open to us, this plugin actually provides a hook, -``pytest_xdist_make_scheduler`` that exists specifically to “implement -custom tests distribution logic.” - -Figuring out the requirements of our test scheduler and designing the -best algorithm will require further discussion and design review. For -the purposes of moving forward, we are not blocked, as the eventual -implementation can be dropped in-place with minimal effort. + @pytest.fixture + def target(request: SubRequest) -> Iterator[Target]: + ... + with target_pool(request.config) as pool: + for name, json in pool.items(): + if fits(TargetData(**json)): + t = Target.from_json(json) + t.locked = True + pool[t.name] = t.to_json() + # Or... + cls = Target.get_platform(params["platform"]) + t = cls(group, params, features, {}, i) + pool[t.name] = t.to_json() + +Because all access to the cache (and so the target pool) is within the +scope of the context manager, the access is locked in such a way that +this works with multiple Pytest processes, as used by `pytest-xdist`_ +and as necessary for parallel CPU-bound tasks (like testing multiple +targets) given Python’s `Global Interpreter Lock`_. Platform +implementations can save arbitrary JSON-serializable data to the +class’s :py:attr:`~target.target.TargetData.data` attribute and it +will be returned when recreated from the cache. + +.. _Global Interpreter Lock: https://wiki.python.org/moin/GlobalInterpreterLock + +While currently an unordered dictionary, to support optimal scheduling +we will likely want to use a priority queue, where the priority of a +target represents its cost (whether in terms of time or money), +allowing us to provide either the fastest or the cheapest target to +each request. By using the `pytest_collection_modifyitems`_ hook to +sort (and so group) the tests by their requirements, the tests would +efficiently reuse targets. Except for the most recently used target, +targets not in use (unlocked) should be deallocated. + +.. _pytest_collection_modifyitems: https://docs.pytest.org/en/latest/reference.html#pytest.hookspec.pytest_collection_modifyitems + +With the ``--keep-targets`` CLI flag the targets won’t be deleted at +the end of a run, and without it they will be automatically deleted. +Regardless, they will always be cached to disk when they are created +so that the CLI flag ``--delete-targets`` can delete *all* allocated +targets, even after a test session is interrupted. + +The fixture is indirectly parameterized during setup with the +:py:func:`~target.plugin.pytest_generate_tests` hook. Test and fixture +`parameterization`_ is a huge feature of Pytest. When we parameterize +the :py:func:`~target.plugin.target` fixture for multiple targets +(e.g. “Ubuntu” and “Debian”), Pytest automatically creates a set of +tests for each target. So ``test_smoke`` turns into +``test_smoke[Ubuntu]`` and ``test_smoke[Debian]``. This allows us to +run a collection of tests against multiple targets with ease. These +targets are defined in a YAML file (thanks to `pytest-playbook`_) and +validated against the parameters collected from the previously +described platform subclasses. + +Finally, once the :py:func:`~target.plugin.target` fixture has +returned a working and sanity-checked environment to the requesting +test, the test is capable of examining any and all attributes of the +:py:class:`~target.target.Target` and quickly marking itself as +skipped, expected to fail, or failed before executing the body of the +test. Our static type checking enables developers to ensure that the +platform they requested supports all methods and fields they use by +annotating the test’s ``target`` parameter with the expected platform +type (or types). Ensuring the effectiveness of this type checking will +require us to carefully update our platform implementations, and not +rely on arbitrary objects of data. (For example, add an +``internal_address`` field to ``AzureCLI``, don’t just look up +``data["internal_address"]``.) pytest-lisa ----------- -What are the user modes? -~~~~~~~~~~~~~~~~~~~~~~~~ - -Because Pytest is incredibly customizable, we want to provide a few sets -of reasonable default configurations for some common scenarios. We will -add a flag like ``--lisa-mode=[dev,debug,ci,demo]`` to change the -default options and output of Pytest. Doing so is readily supported by -Pytest via the -`pytest_addoption `_ -and -`pytest_configure `_ -hooks. We call these the provided “user modes.” Note that by “output” we -mean not just logging (because that implies the Python ``logger`` -module, which Pytest allows full control over) but also commands’ stdout -and stderr as well as Pytest-provided information. - -- The dev(eloper) mode is intended for use by test developers while - writing a new test. It is verbose, caches the deployed VMs between - runs, and generates a digestible - `HTML `_ report. - -- The debug mode is like dev mode but with all possible information - shown, and will open the Python debugger automatically on failures - (which is provided by Pytest with the ``--pdb`` flag). - -- The CI mode will be fairly quiet on the console, showing all test - results, but putting the full info output into the generated report - file (HTML for sharing with humans and - `JUnit `_ - for the associated CI environment, which presents as native test - results). - -- The demo mode will show the “executive summary” (a lot like CI, but - finely tuned for demos). For example, what ``make smoke`` currently - shows. - How are tests described? ~~~~~~~~~~~~~~~~~~~~~~~~ -The built-in -`pytest-mark `_ plugin -already provides functionality for adding metadata to tests, where we -specifically want: +The built-in `pytest-mark`_ plugin already provides functionality for +adding metadata to tests, where we specifically want (and describe +using `schema`_ :py:data:`~lisa.lisa_schema`): -- Platform: used to skip tests inapplicable to the current - system-under-test -- Category: our high-level test organization -- Area: feature being tested -- Priority: self-explanatory -- Tags: optional additional metadata for test organization +- Platform: used to skip tests inapplicable to the current + system-under-test +- Category: our high-level test organization +- Area: feature being tested +- Priority: self-explanatory +- Tags: optional additional metadata for test organization +- Features: a set of required features (like “GPU”) +- Reuse: a boolean to indicate if a target is reusable after the test +- Count: number of targets the test needs We simply reuse this with minimal logic to enforce our required -metadata, with sane defaults (perhaps setting the area to the name of -the module), and to list statistics about our test coverage. This is -already included in the prototype. It looks like this: +metadata, with sane defaults , and to list statistics about our test +coverage. It looks like this: .. code:: python - import pytest + from lisa import LISA - @pytest.mark.lisa(platform="Azure", category="Functional", priority=0, area="LIS_DEPLOY") - def test_lis_driver_version(target: Azure) -> None: - """Checks that the installed drivers have the correct version.""" - ... + @LISA(platform="Azure", category="Functional", area="deploy", priority=0) + def test_smoke(target: AzureCLI, caplog: LogCaptureFixture) -> None: + """Check that an Azure Linux VM can be deployed and is responsive. + +This is a functional example. With this simple decorator, all test +`collection hooks`_ can introspect the metadata, enforce required +parameters and set defaults, select tests based on arbitrary criteria, +and list test coverage statistics (test inventory). We validate the +metadata in :py:func:`lisa.pytest_collection_modifyitems`. -This is a functional example, which takes zero implementation. With this -simple decorator, all test `collection -hooks `_ -can introspect the metadata, enforce required parameters and set -defaults, select tests based on arbitrary criteria, and list test -coverage statistics (test inventory). Designing and implementing the -test inventory algorithm is still under development, but it’s tractable. +.. _collection hooks: https://docs.pytest.org/en/latest/reference.html#collection-hooks Note that Pytest leverages Python’s docstrings for built-in documentation (and can even run tests discovered in such strings, like doctest). Hence we do not have a separate field for the test’s -documentation. As long as we continue to follow the practice of using -docstrings for our modules, classes, and functions, we can automatically -use `pydoc `_ to generate -full documentation for each plugin and test. - -Being just Python code, this decorator need not be -``@pytest.mark.lisa(...)`` but can trivially be provided as simply -``@LISA(...)``. In fact, we provide this in ``lisa.py`` with: - -.. code:: python - - LISA = pytest.mark.lisa - - @LISA(...) - def test_something(...) - -Currently we validate the parameters given to this mark during test -collection, by using the following code, which leverages the -`schema `_ library: - -.. code:: python - - from schema import Optional, Or, Schema - - lisa_schema = Schema( - { - "platform": str, - "category": Or("Functional", "Performance", "Stress", "Community", "Longhaul"), - "area": str, - "priority": Or(0, 1, 2, 3), - Optional("tags", default=list): [str], - }, - ) - - def validate(mark: Mark) -> None: - """Validate each test's LISA parameters.""" - assert not mark.args, "LISA marker cannot have positional arguments!" - mark.kwargs.update(lisa_schema.validate(mark.kwargs)) - -In the future we could change ``LISA`` to be a function with these -keyword arguments so that IDE auto-completion is enabled. However, this -is not mandatory to move forward, and parameter validation is enabled -succinctly with the above, which satisfies one of our “test parameter -validation” requirements. +documentation. By following the best practice of using docstrings for +our modules, classes, and functions, we can automatically to generate +full `documentation`_ for each plugin and test (which you are likely +currently reading). This mark also does need to be repeated for each test, as marks can be scoped to a module, and so one line could describe defaults for every test in a file, with individual tests overriding parameters as needed. -In the current implementation, we also take a ``features: List[str]`` +In the current implementation, we take a ``features: List[str]`` argument that is used to prove the concept deploying (or reusing) a target based on the test’s required and the target’s available sets of -features. However, as we move forward we should define a separate -``requires`` mark that takes well-defined classes describing the minimal -required resources for a test. This will be part of the refactor into -the two Pytest plugins mentioned above. Coupled with the test’s -requested ``target`` fixture being parameterized (see discussion in -``pytest-target``) this demonstrates at least one way we can satisfy our -“test run planner/scheduler” requirement. - -Furthermore, we have a prototype -`generator `_ -which parses LISAv2 XML test descriptions and generates stubs with this -mark filled in correctly. +features, and it is passed to ``@pytest.mark.target``. See `How are +requirements examined?`_ for more. Coupled with the test’s requested +:py:func:`~target.plugin.target` fixture being parameterized (see +discussion in `pytest-target`_) this demonstrates at least one way we +can satisfy our “test run planner/scheduler” requirement. + +Furthermore, we have a prototype `generator +`_ which parses +LISAv2 XML test descriptions and generates stubs with this mark filled +in correctly. How are tests selected? ~~~~~~~~~~~~~~~~~~~~~~~ Pytest already allows a user to specify which exact tests to run: -- Listing folders on the CLI (see below on where tests should live) -- Specifying a name expression on the CLI (e.g. ``-k smoke and xdp``) -- Specifying a mark expression on the CLI - (e.g. ``-m functional and not slow``) +- Listing folders on the CLI (see below on where tests should live) +- Specifying a name expression on the CLI (e.g. ``-k smoke and xdp``) +- Specifying a mark expression on the CLI (e.g. ``-m functional and + not slow``) We can also implement any other mechanism via the -`pytest_collection_modifyitems `_ -hook. The proof-of-concept supports gathering selection criteria from a -YAML file: +`pytest_collection_modifyitems`_ hook. The existing implementation in :py:mod:`lisa` +supports gathering selection criteria from a YAML file: -.. code:: yaml +.. code-block:: yaml criteria: # Select all Priority 0 tests. @@ -675,97 +612,43 @@ YAML file: - area: xdp exclude: true -This criteria is validated against the following -`schema `_: - -.. code:: python - - from schema import Schema, Optional - - criteria_schema = Schema( - { - # TODO: Validate that these strings are valid regular - # expressions if we change our matching logic. - Optional("name", default=None): str, - Optional("area", default=None): str, - Optional("category", default=None): str, - Optional("priority", default=None): int, - Optional("tags", default=list): [str], - Optional("times", default=1): int, - Optional("exclude", default=False): bool, - } - ) - -The test collection is then modified using the Pytest hook, -`pytest_collection_modifyitems `_: - -.. code:: python +This criteria is validated against following `schema`_ defined in +:py:func:`lisa.pytest_playbook_schema`. - def pytest_collection_modifyitems( - session: Session, config: Config, items: List[Item] - ) -> None: - included: List[Item] = [] - excluded: List[Item] = [] - - def select(item: Item, times: int, exclude: bool) -> None: - if exclude: - excluded.append(item) - else: - for _ in range(times - included.count(item)): - included.append(item) - - for c in criteria: # Where `criteria` is from the schema. - for item in items: - marker = item.get_closest_marker("lisa") - if not marker: - # Not all tests will have the LISA marker, such as - # static analysis tests. - continue - i = marker.kwargs - if any( - [ - c["name"] and c["name"] in item.name, - c["area"] and c["area"].casefold() == i["area"].casefold(), - c["category"] - and c["category"].casefold() == i["category"].casefold(), - c["priority"] and c["priority"] == i["priority"], - c["tags"] and set(c["tags"]) <= set(i["tags"]), - ] - ): - select(item, c["times"], c["exclude"]) - items[:] = [i for i in included if i not in excluded] +The test collection is then modified using the Pytest hook in +:py:func:`lisa.pytest_collection_modifyitems`. Because this is simply +a Python list, we can also sort the tests according to our needs, such +as by priority. If the `pytest-target`_ plugin has already sorted by +requirements, that’s just fine, Python’s ``sorted()`` built-in is +guaranteed to be stable (meaning we can sort in multiple passes). Together, the CLI support and YAML playbook satisfy one of our “test -entrance” requirements. We can also generate our own binary called -``lisa`` which simply delegates to Pytest, if we really want to do so. - -Because this is simply a Python list, we can also sort the tests -according to our needs, such as by priority. If the ``python-targets`` -plugin has already sorted by requirements, that’s just fine, Python’s -``sorted()`` built-in is guaranteed to be stable (meaning we can sort in -multiple passes). +entrance” requirements. We also generate our own binary called +``lisa`` which simply delegates to Pytest. How are results reported? ~~~~~~~~~~~~~~~~~~~~~~~~~ -Parsing the results of a large test suite can be difficult. Fortunately, -because Pytest is a testing framework, there already exists support for -generating excellent reports. For developers, the -`HTML `_ report is easy to read: -it is self-contained, holds all the results and logs, and each test can -be expanded and collapsed. Tests which were rerun are recorded -separately. For CI pipelines, Pytest has integrated -`JUnit `_ +Parsing the results of a large test suite can be difficult. +Fortunately, because Pytest is a testing framework, there already +exists support for generating excellent reports. For developers, the +`HTML report`_ is easy to read: it is self-contained, holds all the +results and logs, and each test can be expanded and collapsed. Tests +which were rerun are recorded separately. For CI pipelines, Pytest has +integrated `JUnit +`_ XML test report support. This is the standard method of reporting results to CI servers like Jenkins and are natively parsed into the CI -system’s built-in test display page. Finally, Azure DevOps pipelines are -even supported with a community plugin -`pytest-azurepipelines `_ -which enhances the standard JUnit report for ADO. +system’s built-in test display page. Finally, Azure DevOps pipelines +are even supported with a community plugin `pytest-azurepipelines +`_ which enhances the +standard JUnit report for ADO. + +.. _HTML report: https://pypi.org/project/pytest-html/ One of our requirements is to support the lookup of previous tests’ -execution metrics, such as recorded performance metrics and duration, so -that performance tests can check regressions. This is the perfect +execution metrics, such as recorded performance metrics and duration, +so that performance tests can check regressions. This is the perfect example of carrying a small fixture which provides access to our internal database and is dynamically added to our tests when run internally, and the tests can lookup and record whatever they need @@ -773,43 +656,43 @@ through the fixture. However, we also have internal requirements to report test results throughout the test life cycle to a database (the “result manager” and -“progress tracker”) to be consumed by other tools. In this sense, LISAv3 -(the composition of our published plugins, tests, and fixtures) is -simply a producer, and the consumers can parse the test results, send -emails, archive the collected logs, update a GUI display of test +“progress tracker”) to be consumed by other tools. In this sense, +LISAv3 (the composition of our published plugins, tests, and fixtures) +is simply a producer, and the consumers can parse the test results, +send emails, archive the collected logs, update a GUI display of test progress, etc. Our repository’s ``conftest.py`` can implement the -necessary logic using Pytest’s ample `test running -hooks `_. -In particular, the hook -`pytest_runtest_makereport `_ -is called for each of the setup, call and teardown phases of a test. As -such it can used for precisely this purpose. +necessary logic using Pytest’s ample `test running hooks +`_. +In particular, the hook `pytest_runtest_makereport +`_ +is called for each of the setup, call and teardown phases of a test. +As such it can used for precisely this purpose. How is setup, run, and cleanup handled? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Pytest strives to require minimal boiler-plate code. Thus the classic “xunit-style” of defining a class with setup and teardown functions in -addition to test functions is not recommended (nor necessary). Generally -Pytest expects -`fixtures `_ to be used -for dependency injection (which is what setup/teardown functions usually -do). For users that really want the classic style, it is nonetheless -fully `supported `_ -and documented (and can be applied at the module, class, and method +addition to test functions is not recommended (nor necessary). +Generally Pytest expects `fixtures`_ to be used for dependency +injection (which is what setup/teardown functions usually do). For +users that really want the classic style, it is nonetheless fully +`supported `_ and +documented (and can be applied at the module, class, and method scopes). Thus our “test runner” requirement is satisfied. How are tests timed out? ~~~~~~~~~~~~~~~~~~~~~~~~ -The `pytest-timeout `_ plugin -provides integrated timeouts via ``@pytest.mark.timeout()``, -a configuration file option, environment variable, and CLI flag. The -Fabric library provides timeouts in both the configuration and -per-command usage. These are already used to satisfaction in the -prototype. Additionally, Pytest has built-in support for measuring the -duration of each fixture’s setup and teardown and each test (it’s simply -the ``--durations`` and ``--durations-min`` flags). +The `pytest-timeout `_ +plugin provides integrated timeouts via ``@pytest.mark.timeout()``, a configuration file option, environment variable, and +CLI flag. The Fabric library provides timeouts in both the +configuration and per-command usage. These are already used to +satisfaction in the prototype. Additionally, Pytest has built-in +support for measuring the duration of each fixture’s setup and +teardown and each test (it’s simply the ``--durations`` and +``--durations-min`` flags). How are tests organized? ~~~~~~~~~~~~~~~~~~~~~~~~ @@ -817,18 +700,18 @@ How are tests organized? That is, what does a folder of tests map to: a platform, feature, or owner? -In my opinion it is likely to be both. Tests which are common to a -platform and written by our team are probably best placed in a folder -like ``tests/azure`` whereas tests for a particular scenario which -limits their image and SKU applicability should be in a folder like -``tests/acc``. It’s going to depend on how often the tests are run -together. +In the author’s opinion it is likely to be both. Tests which are +common to a platform and written by our team are probably best placed +in a folder like ``tests/azure`` whereas tests for a particular +scenario which limits their image and SKU applicability should be in a +folder like ``tests/acc``. It’s going to depend on how often the tests +are run together. Because Pytest can run tests and ``conftest.py`` files from arbitrary -folders, maintaining sets of tests and plugins separately from the base -LISA repository is easy. Custom repositories with new tests, plugins, -fixtures, platform-specific support, etc. can simply be cloned anywhere, -and provided on the command-line to Pytest. +folders, maintaining sets of tests and plugins separately from the +base LISA repository is easy. Custom repositories with new tests, +plugins, fixtures, platform-specific support, etc. can simply be +cloned anywhere, and provided on the command-line to Pytest. Test authors should keep tests which share requirements and are otherwise similar to a single module (Python file). Not only is this @@ -839,33 +722,37 @@ built-in ``skip`` and ``xfail`` Pytest marks) becomes even easier. An open question is if we really want to bring every test from LISAv2 directly over, or if we should carefully analyze our tests to craft a new set of high-level scenarios. An interesting result of reorganizing -and rewriting the tests would be the ability to have test layers, where -the result of a high-level test dictates if the tests below it should be -skipped. If it passes, it implies the tests underneath it would pass, -and so skips them; but if it fails, the next test below it runs and so -on until a passing layer is found. +and rewriting the tests would be the ability to have test layers, +where the result of a high-level test dictates if the tests below it +should be skipped. If it passes, it implies the tests underneath it +would pass, and so skips them; but if it fails, the next test below it +runs and so on until a passing layer is found. How will we port LISAv2 tests? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Given the above, we still must decide if we want to put the engineering -effort into porting *every* LISAv2 test. However, the prototype started -by porting the ``LIS-DRIVER-VERSION-CHECK`` test, proving that tests -which exclusively use Bash scripts are trivially portable. -Unfortunately, most tests use an associated PowerShell script which is -tightly coupled to the LISAv2 framework. +Given the above, we still must decide if we want to put the +engineering effort into porting *every* LISAv2 test. However, the +prototype started by porting the ``LIS-DRIVER-VERSION-CHECK`` test, +proving that tests which exclusively use Bash scripts are trivially +portable. Unfortunately, most tests use an associated PowerShell +script which is tightly coupled to the LISAv2 framework. We believe that it is *possible* to port these tests without untoward -modifications. We would need to write a mock library that implements (or -stubs where appropriate) LISAv2 framework functionality such as -``Provision-VMsForLisa``, ``Copy-RemoteFiles``, ``Run-LinuxCmd``, etc., -and provides both the expected “global” objects and the test function -parameters ``AllVmData`` and ``CurrentTestData``. +modifications. We would need to write a mock library that implements +(or stubs where appropriate) LISAv2 framework functionality such as +``Provision-VMsForLisa``, ``Copy-RemoteFiles``, ``Run-LinuxCmd``, +etc., and provides both the expected “global” objects and the test +function parameters ``AllVmData`` and ``CurrentTestData``. But it +wouldn’t be great. This work needs to be done regardless of the approach we take with our framework (leveraging Pytest or writing our own), and it is not inconsequential work. It needs to be thoroughly planned and executed, -and is certainly a ways off. +and is certainly a ways off. The author’s personal opinion is that we +won’t want to port most LISAv2 tests, and instead create a new set of +well-documented, comprehensive, layered tests that cover our current +needs, instead of bringing along all these historical tests. How are tests and functions retried? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -873,11 +760,11 @@ How are tests and functions retried? Testing remote targets is inherently flaky, so we take a two-pronged approach to dealing with the flakiness. -The -`pytest-rerunfailures `_ -plugin will be used to easily mark a test itself as flaky. It has the -nice feature of recording each rerun in the produced report. It looks -like this: +The `pytest-rerunfailures`_ plugin can be used to easily mark a test +itself as flaky. It has the nice feature of recording each rerun in +the produced report. It looks like this: + +.. _pytest-rerunfailures: https://pypi.org/project/pytest-rerunfailures/ .. code:: python @@ -886,35 +773,133 @@ like this: """This fails most of the time.""" ... -Note that there is an open -`bug `_ -in this plugin which can cause issues with fixtures using scopes other -than “function” but it can be worked around. +Note that there is an open `bug +`_ in +this plugin which can cause issues with fixtures using scopes other +than “function” but it can be worked around (and we mostly use +“function” scope anyway). -The `Tenacity `_ library -should be used to retry flaky functions that are not tests, such as -downloading boot diagnostics or pinging a node. As the modern Python -retry library it has easy-to-use decorators to retry functions (and -context managers to use within functions), as well as excellent wait and -timeout support. It looks like this: +The `Tenacity`_ library is used to retry flaky functions that are not +tests, such as downloading boot diagnostics or pinging a node. As the +“modern Python retry library” it has easy-to-use decorators to retry +functions (and context managers to use within functions), as well as +excellent wait and timeout support. The +:py:meth:`~target.target.Target.ping` method looks like this: + +.. _Tenacity: https://tenacity.readthedocs.io/en/latest/ .. code:: python from tenacity import retry, stop_after_attempt, wait_exponential - class Node: + class Target: ... @retry(reraise=True, wait=wait_exponential(), stop=stop_after_attempt(3)) - def ping(self, **kwargs): + def ping(self, **kwargs: Any) -> Result: """Ping the node from the local system in a cross-platform manner.""" flag = "-c 1" if platform.system() == "Linux" else "-n 1" return self.local(f"ping {flag} {self.host}", **kwargs) - ... We can additionally list a test twice when modifying the items collection, as implemented in the criteria proof-of-concept. However, given the above abilities, this may not be desired. +How are tests executed in parallel? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +While our original list of goals stated that we want to run tests “in +parallel” we were not specific about what was meant, and the topic of +parallelism and concurrency is understandably complex. We certainly +don’t mean running two tests at once on the same target, as this would +undoubtedly lead to flaky tests. + +Assuming that we care about a set of tests passing on a particular +image and size combination, but not necessarily on a particular +deployed instance, then we can run tests concurrently by deploying +multiple “identical” targets and splitting the tests across them. The +tests would still run in isolation on each target. This sounds hard, +but actually it’s practically free with Pytest via `pytest-xdist`_. + +.. _pytest-xdist: https://github.com/pytest-dev/pytest-xdist + +The default `pytest-xdist`_ implementation simply takes the list of +tests and runs them in a round-robin fashion with the desired number +of executors. We’ve talked at length about being able to schedule +groups of tests to run in particular executors and using particular +targets. While there are many paths open to us, this plugin actually +provides a hook, `pytest_xdist_make_scheduler +`_ +that exists specifically to “implement custom tests distribution +logic.” We used this to create the :py:class:`~lisa.LISAScheduling` +custom scheduler. + +Figuring out the requirements of our test scheduler and designing the +best algorithm will require further discussion and design review. For +the purposes of moving forward, we are not blocked, as the eventual +implementation can be dropped in-place with minimal effort. + +What are the user modes? +~~~~~~~~~~~~~~~~~~~~~~~~ + +Because Pytest is incredibly `customizable`_, we may want to provide a +few sets of reasonable default configurations for some common +scenarios. We should add a flag like +``--lisa-mode=[dev,debug,ci,demo]`` to change the default options and +output of Pytest. Doing so is readily supported by Pytest via the +`pytest_addoption`_ and `pytest_configure`_ hooks. We call these the +provided “user modes.” Note that by “output” we mean not just logging +(because that implies the Python ``logger`` module, which Pytest +allows full control over) but also commands’ ``stdout`` and ``stderr`` +as well as Pytest-provided information. + +As the current implementation stands, we just have sane defaults in +our repository’s ``pytest.ini``, and users who install and use our +plugins or tests can edit their own ``pytests.ini`` + +.. _customizable: https://docs.pytest.org/en/stable/customize.html +.. _pytest_addoption: https://docs.pytest.org/en/latest/reference.html#pytest.hookspec.pytest_addoption +.. _pytest_configure: https://docs.pytest.org/en/latest/reference.html#pytest.hookspec.pytest_configure + +- The dev(eloper) mode is intended for use by test developers while + writing a new test. It is verbose, caches the deployed VMs between + runs, and generates a digestible `HTML report`_ report. + +- The debug mode is like dev mode but with all possible information + shown, and will open the Python debugger automatically on failures + (which is provided by Pytest with the ``--pdb`` flag). + +- The CI mode will be fairly quiet on the console, showing all test + results, but putting the full info output into the generated report + file (HTML for sharing with humans and `JUnit + `_ + for the associated CI environment, which presents as native test + results). + +- The demo mode will show the “executive summary” (a lot like CI, but + finely tuned for demos). + +pytest-playbook +--------------- + +This plugin is simple, but exciting. The module :py:mod:`playbook` +defines a hook :py:meth:`playbook.Hooks.pytest_playbook_schema` which +other plugins (as discussed above) can use to add schemata to the +final playbook. In :py:meth:`playbook.pytest_configure`, all the +schemata are gathered and then the file given by ``--playbook=`` +is read, validated, and made available at :py:data:`playbook.data`. It +uses the `PyYAML `_ +library, but can be extended to support other formats. Also “YAML +Schema” section in :doc:`contributing guidelines ` on +how to generate the `JSON Schema `_ for use +with editors or for manual review. + +This is leveraging Pytest’s existing parameterization technology to +achieve one of our “test entrance” goals of requesting environments +with a YAML playbook, and one of our “test parameter validation” goals +of validating platforms before executing tests so that we can fail +fast if a target has insufficient information to be setup. Parsing the +same parameters from a CLI can also be implemented. + What does the “flow” of Pytest look like? ----------------------------------------- @@ -925,7 +910,7 @@ processing: .. code:: python pool_fixture: a session-scoped context manager - target_fixture: a function-scoped context manager + target_fixture: a function-scoped context manager items: a collection of tests targets: a collection of targets criteria: a collection of test selection criteria @@ -969,12 +954,12 @@ What Else? There’s still a lot more to think about and design. A non-exhaustive list of future topics (some touched on above): -- Terminology table -- Tests inventory (generating statistics from metadata) -- Environment / multiple targets class design -- Feature/requirement requests (NICs in particular) -- Custom test scheduler algorithm -- Secret management +- Terminology table +- Tests inventory (generating statistics from metadata) +- Environment / multiple targets class design +- Feature/requirement requests (NICs in particular) +- Custom test scheduler algorithm +- Secret management What alternatives were tried? ----------------------------- @@ -984,18 +969,23 @@ These are notes from things tried that did not work out, and why. Writing Another Framework ~~~~~~~~~~~~~~~~~~~~~~~~~ -I believe the above set of technical specifications clearly describes -how we can leverage Pytest for our needs. Furthermore, the existing -prototype proves this is a viable option. Therefore I do not think we -should consider writing and maintaining a *new* Python testing -framework. We should avoid falling for “not invented here” syndrome. The -alternative prototype which does implement a new framework required over -five thousand lines of code, the Pytest-based prototype used less than -two hundred, or less than three percent. We do not want to take on the -maintenance cost of yet another framework, the maintenance cost of -LISAv2 already caused this mess in the first place. I think the work of -prototyping said new framework was valuable, as it provided insight into -the eventual technical design of LISAv3. +The author believes the above set of technical specifications clearly +describes how we can leverage Pytest for our needs. Furthermore, the +existing implementation proves this is a viable option. Therefore he +does not think we should write and maintain a *new* Python testing +framework. We should avoid falling for “not invented here” syndrome. +The alternative prototype which implements a whole new testing +framework required over five thousand lines of code, and the +Pytest-based prototype used less than two hundred (now barely six +hundred as a full fledged implementation with three separate Pytest +plugins, even after extensive feature additions and refactors), or +less than three percent. + +We do not want to take on the maintenance cost of yet another +framework, the maintenance cost of LISAv2 already caused this mess in +the first place. The work of prototyping said new framework was +valuable, as it provided insight into the eventual technical design of +LISAv3, as laid out in this document. Using Remote Capabilities of ``pytest-xdist`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/pytest-lisa/lisa.py b/pytest-lisa/lisa.py index 2c03f5bf7b..4c3ac39fd0 100644 --- a/pytest-lisa/lisa.py +++ b/pytest-lisa/lisa.py @@ -155,6 +155,7 @@ def pytest_playbook_schema(schema: Dict[Any, Any]) -> None: ): And(int, lambda n: 0 < n < 10), } ) +"""Schema used to validate a test's metadata.""" def validate_mark(mark: Mark) -> None: @@ -277,7 +278,7 @@ def __init__(self, config: Config, log=None): # type: ignore self.log = log.lisasched # NOTE: Needs to handle whitespace, so can’t be `\w+`. - regex = re.compile(r"\[Target=([^\[\]]+)\]") + _regex = re.compile(r"\[Target=([^\[\]]+)\]") def _split_scope(self, nodeid: str) -> str: """Determine the scope (grouping) of a `nodeid`. @@ -317,7 +318,7 @@ def _split_scope(self, nodeid: str) -> str: 'example/test_module.py::test_class' """ - search = self.regex.search(nodeid) + search = self._regex.search(nodeid) if search: scope = search.group(1) if self.config.getoption("verbose"): diff --git a/pytest-target/target/__init__.py b/pytest-target/target/__init__.py index 3e5d320ff2..6e28cad54a 100644 --- a/pytest-target/target/__init__.py +++ b/pytest-target/target/__init__.py @@ -2,11 +2,12 @@ # Licensed under the MIT License. """A plugin for creating, using, and managing remote targets. -The abstract base `Target` class provides an interface for adding -platform-specific support through sub-classes. A usable reference -implementation is the `Azure` class. A class for testing on the local -system is the `Local` class. Sub-classes can be implemented in a -`conftest.py` file and will be found automatically. +The abstract base :py:class:`~target.target.Target` class provides an +interface for adding platform-specific support through sub-classes. A +usable reference implementation is the +:py:class:`~target.azure.AzureCLI` class. A class for just connecting +over SSH is the :py:class:`~target.target.SSH` Sub-classes can be +implemented in a ``conftest.py`` file and will be found automatically. Tests can request access to a target through the function-scoped `target` Pytest fixture, which returns an instance based on the diff --git a/pytest-target/target/plugin.py b/pytest-target/target/plugin.py index 81ac4e41e3..c0adf050c3 100644 --- a/pytest-target/target/plugin.py +++ b/pytest-target/target/plugin.py @@ -245,7 +245,7 @@ def fits(t: TargetData) -> bool: def cleanup_target(t: Target, request: SubRequest) -> None: - """This is called by fixtures after they're done with a ``Target``.""" + """This is called by fixtures after they're done with a :py:class:`~target.target.Target`.""" t.conn.close() mark: Optional[Mark] = request.node.get_closest_marker("target") assert mark is not None @@ -261,7 +261,7 @@ def cleanup_target(t: Target, request: SubRequest) -> None: @pytest.fixture def target(request: SubRequest) -> Iterator[Target]: - """This fixture provides a connected ``Target`` for each test. + """This fixture provides a connected :py:class:`~target.target.Target` for each test. It is parametrized indirectly in :py:func:`pytest_generate_tests`. diff --git a/pytest-target/target/target.py b/pytest-target/target/target.py index cbaf7ea9e3..ec38c8eb20 100644 --- a/pytest-target/target/target.py +++ b/pytest-target/target/target.py @@ -1,6 +1,13 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -"""Provides the abstract base :py:class:`~target.target.Target` class.""" +"""Provides the abstract base :py:class:`~target.target.Target` class. + +This abstract base class provides the building blocks for any platform +support that could be required. Users simply define a subclass with +the abstract methods implemented to deploy or delete the target +appropriately. + +""" from __future__ import annotations import dataclasses @@ -77,6 +84,7 @@ class Target(TargetData, metaclass=ABCMeta): name: str host: str conn: fabric.Connection + """Used for SSH access, see `Fabric.Connection`_""" # Setup a sane configuration for local and remote commands. Note # that the defaults between Fabric and Invoke are different, so we diff --git a/testsuites/test_smoke_b.py b/testsuites/test_smoke_b.py index 404b76e7a1..e6fcdce832 100644 --- a/testsuites/test_smoke_b.py +++ b/testsuites/test_smoke_b.py @@ -16,12 +16,7 @@ from lisa import LISA -@LISA( - platform="Azure", - category="Functional", - area="deploy", - priority=0, -) +@LISA(platform="Azure", category="Functional", area="deploy", priority=0) def test_smoke(target: AzureCLI, caplog: LogCaptureFixture) -> None: """Check that an Azure Linux VM can be deployed and is responsive. From da6fce7ff4b3ef67e9a7830dcd77a9a4c792d5e5 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Thu, 14 Jan 2021 14:32:05 -0800 Subject: [PATCH 183/194] Make deploy and cache logic more robust MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When deploying new targets, we should only lock the cache afterwards so the lock isn’t waiting on the deployment itself. We should also catch any exception in the deployment, so we can still delete whatever may have succeeded. This also makes the `get_targets()` logic more clear. Searching for cached targets (and creating instances from cached data) is fast enough that I’m not worried about locking the cache around it. --- pytest-target/target/plugin.py | 37 +++++++++++++++++----------------- pytest-target/target/target.py | 8 ++++++-- 2 files changed, 25 insertions(+), 20 deletions(-) diff --git a/pytest-target/target/plugin.py b/pytest-target/target/plugin.py index c0adf050c3..a4eed3b4e4 100644 --- a/pytest-target/target/plugin.py +++ b/pytest-target/target/plugin.py @@ -217,30 +217,31 @@ def fits(t: TargetData) -> bool: for name, json in pool.items(): if fits(TargetData(**json)): logging.debug(f"Found fit target '{i}'!") - # TODO: De-duplicate this logic and its - # counterpart below. t = Target.from_json(json) assert name == t.name # Sanity check. t.locked = True pool[t.name] = t.to_json() targets.append(t) - break - if count == len(targets): - # TODO: This is a kludgy way to either use the found - # targets or give up and instantiate new ones instead. - # Theoretically the length here should either be zero - # or `count` because of the check in `fits`, but - # perhaps we should handle the erroneous case where - # it’s in-between. - break - else: - group = f"pytest-{uuid4()}" - for i in range(count): - logging.info(f"Instantiating target '{group}-{i}': {params}...") - cls = Target.get_platform(params["platform"]) - t = cls(group, params, features, {}, i) + break # Continue outer counting loop... + if targets: + assert len(targets) == count + else: + group = f"pytest-{uuid4()}" + for i in range(count): + logging.info(f"Instantiating target '{group}-{i}': {params}...") + t = Target.from_json( + { + "group": group, + "params": params, + "features": features, + "data": {}, + "number": i, + "locked": True, + } + ) + with target_pool(request.config) as pool: pool[t.name] = t.to_json() - targets.append(t) + targets.append(t) return targets diff --git a/pytest-target/target/target.py b/pytest-target/target/target.py index ec38c8eb20..43484b230c 100644 --- a/pytest-target/target/target.py +++ b/pytest-target/target/target.py @@ -13,6 +13,7 @@ import dataclasses import platform import typing +import warnings from abc import ABCMeta, abstractmethod from io import BytesIO @@ -131,9 +132,12 @@ def __init__( self.data = data self.number = number self.locked = locked - self.name = f"{self.group}-{self.number}" - self.host = self.deploy() + + try: + self.host = self.deploy() + except Exception as e: + warnings.warn(f"Failed to deploy '{self.name}': {e}") fabric_config = self._config.copy() fabric_config["run"]["env"] = { # type: ignore From 6559c4d481d1e83039ab2ccf0e73cbbab3297d36 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Fri, 15 Jan 2021 12:04:12 -0800 Subject: [PATCH 184/194] Fix up smoke test retry logic --- playbooks/demo.yaml | 2 +- pytest-target/target/target.py | 29 +++++++++++++++++++++++++---- pytest.ini | 3 +++ testsuites/test_smoke_b.py | 18 +++++------------- 4 files changed, 34 insertions(+), 18 deletions(-) diff --git a/playbooks/demo.yaml b/playbooks/demo.yaml index e172d74185..7331b9c460 100644 --- a/playbooks/demo.yaml +++ b/playbooks/demo.yaml @@ -12,4 +12,4 @@ targets: image: UbuntuLTS criteria: - - module: smoke + - module: test_smoke_b diff --git a/pytest-target/target/target.py b/pytest-target/target/target.py index 43484b230c..abe2448f1d 100644 --- a/pytest-target/target/target.py +++ b/pytest-target/target/target.py @@ -21,7 +21,12 @@ import invoke # type: ignore from invoke.runners import Result # type: ignore from schema import Literal, Optional, Schema # type: ignore -from tenacity import retry, stop_after_attempt, wait_exponential # type: ignore +from tenacity import ( # type: ignore + retry, + retry_if_result, + stop_after_attempt, + wait_exponential, +) if typing.TYPE_CHECKING: from typing import Any, Dict, List, Mapping, Tuple, Type @@ -301,11 +306,27 @@ def local(cls, *args: Any, **kwargs: Any) -> Result: """This patches Fabric's ``local()`` function to ignore SSH environment.""" return Target._local_context.run(*args, **kwargs) - @retry(reraise=True, wait=wait_exponential(), stop=stop_after_attempt(3)) + @retry( + retry=retry_if_result(lambda result: result.failed), + retry_error_callback=(lambda retry_state: retry_state.outcome.result()), + wait=wait_exponential(), + stop=stop_after_attempt(5), + ) def ping(self, **kwargs: Any) -> Result: - """Ping the node from the local system in a cross-platform manner.""" + """Ping the node from the local system in a cross-platform manner. + + This is setup such that it retries five times when the exit + code is nonzero, with an exponential backoff. Since we want to + return the command's result regardless of failure, we suppress + `Invoke`_'s exception with ``warn=True`` and `Tenacity`_'s exception + with ``retry_error_callback=...``. + + .. _Invoke: https://www.pyinvoke.org/ + .. _Tenacity: https://tenacity.readthedocs.io/en/latest/ + + """ flag = "-c 1" if platform.system() == "Linux" else "-n 1" - return self.local(f"ping {flag} {self.host}", **kwargs) + return self.local(f"ping {flag} {self.host}", warn=True, **kwargs) def cat(self, path: str) -> str: """Gets the value of a remote file without a temporary file.""" diff --git a/pytest.ini b/pytest.ini index 6b2ea51d0a..3b4caa889b 100644 --- a/pytest.ini +++ b/pytest.ini @@ -5,10 +5,13 @@ addopts = --capture=tee-sys --tb=line --no-header +log_level = WARNING log_format = %(asctime)s %(levelname)s %(message)s log_date_format = %Y-%m-%d %H:%M:%S render_collapsed = true junit_logging = all +junit_suite_name = LISAv3 +junit_duration_report = call timeout = 1200 filterwarnings = ignore:Module already imported so cannot be rewritten diff --git a/testsuites/test_smoke_b.py b/testsuites/test_smoke_b.py index e6fcdce832..0e43921a6f 100644 --- a/testsuites/test_smoke_b.py +++ b/testsuites/test_smoke_b.py @@ -10,7 +10,7 @@ import socket import time -from invoke.runners import CommandTimedOut, Result, UnexpectedExit # type: ignore +from invoke.runners import CommandTimedOut, UnexpectedExit # type: ignore from paramiko import SSHException # type: ignore from lisa import LISA @@ -39,11 +39,7 @@ def test_smoke(target: AzureCLI, caplog: LogCaptureFixture) -> None: caplog.set_level(logging.INFO) logging.info("Pinging before reboot...") - ping1 = Result() - try: - ping1 = target.ping() - except UnexpectedExit: - logging.warning(f"Pinging {target.host} before reboot failed") + ping1 = target.ping() ssh_errors = (TimeoutError, CommandTimedOut, SSHException, socket.error) @@ -73,11 +69,7 @@ def test_smoke(target: AzureCLI, caplog: LogCaptureFixture) -> None: time.sleep(10) logging.info("Pinging after reboot...") - ping2 = Result() - try: - ping2 = target.ping() - except UnexpectedExit: - logging.warning(f"Pinging {target.host} after reboot failed") + ping2 = target.ping() try: logging.info("SSHing after reboot...") @@ -94,5 +86,5 @@ def test_smoke(target: AzureCLI, caplog: LogCaptureFixture) -> None: logging.info("See full report for boot diagnostics.") # NOTE: The test criteria is to fail only if ping fails. - assert ping1.ok - assert ping2.ok + assert ping1.ok, f"Pinging {target.host} before reboot failed" + assert ping2.ok, f"Pinging {target.host} after reboot failed" From 44b6cac41ef074e446709cedb469b9ab963f72b2 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Fri, 15 Jan 2021 17:03:33 -0800 Subject: [PATCH 185/194] Add README files and info to plugins --- pyproject.toml | 16 ++++++++++++---- pytest-lisa/README.md | 9 +++++++++ pytest-lisa/pyproject.toml | 17 ++++++++++++++--- pytest-playbook/README.md | 9 +++++++++ pytest-playbook/pyproject.toml | 15 +++++++++++++-- pytest-target/README.md | 9 +++++++++ pytest-target/pyproject.toml | 15 +++++++++++++-- 7 files changed, 79 insertions(+), 11 deletions(-) create mode 100644 pytest-lisa/README.md create mode 100644 pytest-playbook/README.md create mode 100644 pytest-target/README.md diff --git a/pyproject.toml b/pyproject.toml index 5a964be802..16e344665d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,13 +2,21 @@ name = "LISA" version = "0.1.0" description = "Linux Integration Services Automation (LISA)" -license = "MIT License" +license = "MIT" authors = ["Andrew Schwartzmeyer "] readme = "README.md" homepage = "https://microsoft.github.io/lisa" -repository = "https://github.com/microsoft/lisa" - -include = [".md", "playbooks/*.yaml"] +repository = "https://github.com/microsoft/lisa/tree/andschwa/pytest" +classifiers = [ + "Development Status :: 3 - Alpha", + "Framework :: Pytest", + "Intended Audience :: Developers", + "Operating System :: Microsoft :: Windows", + "Operating System :: POSIX", + "Topic :: Software Development :: Testing", + "Topic :: Utilities" +] +include = [".md", "playbooks/*.yaml", "playbooks/schema.json"] packages = [{include = "*.py"}, {include = "testsuites"}] [tool.poetry.dependencies] diff --git a/pytest-lisa/README.md b/pytest-lisa/README.md new file mode 100644 index 0000000000..8d9261dfbd --- /dev/null +++ b/pytest-lisa/README.md @@ -0,0 +1,9 @@ +# pytest-lisa + +Pytest plugin for organizing tests. Supports the LISAv3 framework. + +See the [documentation][] for more information, and the LISAv3 [`README.md`][] +for notices. + +[documentation]: https://microsoft.github.io/lisa/modules/lisa.html +[`README.md`]: https://github.com/microsoft/lisa/blob/andschwa/pytest/README.md diff --git a/pytest-lisa/pyproject.toml b/pytest-lisa/pyproject.toml index 22b5db8e3f..1d8951c14c 100644 --- a/pytest-lisa/pyproject.toml +++ b/pytest-lisa/pyproject.toml @@ -1,10 +1,21 @@ [tool.poetry] name = "pytest-lisa" version = "0.1.0" -description = "Pytest plugin for Linux Integration Services Automation (LISA)." -authors = ["Andrew Schwartzmeyer "] +description = "Pytest plugin for organizing tests." license = "MIT" -classifiers = ["Framework :: Pytest"] +authors = ["Andrew Schwartzmeyer "] +readme = "README.md" +homepage = "https://microsoft.github.io/lisa" +repository = "https://github.com/microsoft/lisa/tree/andschwa/pytest/pytest-lisa" +classifiers = [ + "Development Status :: 3 - Alpha", + "Framework :: Pytest", + "Intended Audience :: Developers", + "Operating System :: Microsoft :: Windows", + "Operating System :: POSIX", + "Topic :: Software Development :: Testing", + "Topic :: Utilities" +] packages = [{include = "lisa.py"}] [tool.poetry.dependencies] diff --git a/pytest-playbook/README.md b/pytest-playbook/README.md new file mode 100644 index 0000000000..542954ea40 --- /dev/null +++ b/pytest-playbook/README.md @@ -0,0 +1,9 @@ +# pytest-playbook + +Pytest plugin for reading playbooks. Supports the LISAv3 framework. + +See the [documentation][] for more information, and the LISAv3 [`README.md`][] +for notices. + +[documentation]: https://microsoft.github.io/lisa/modules/playbook.html +[README]: https://github.com/microsoft/lisa/blob/andschwa/pytest/README.md diff --git a/pytest-playbook/pyproject.toml b/pytest-playbook/pyproject.toml index 2d44a4bb73..29022e6883 100644 --- a/pytest-playbook/pyproject.toml +++ b/pytest-playbook/pyproject.toml @@ -2,9 +2,20 @@ name = "pytest-playbook" version = "0.1.0" description = "Pytest plugin for reading playbooks." -authors = ["Andrew Schwartzmeyer "] license = "MIT" -classifiers = ["Framework :: Pytest"] +authors = ["Andrew Schwartzmeyer "] +readme = "README.md" +homepage = "https://microsoft.github.io/lisa" +repository = "https://github.com/microsoft/lisa/tree/andschwa/pytest/pytest-playbook" +classifiers = [ + "Development Status :: 3 - Alpha", + "Framework :: Pytest", + "Intended Audience :: Developers", + "Operating System :: Microsoft :: Windows", + "Operating System :: POSIX", + "Topic :: Software Development :: Testing", + "Topic :: Utilities" +] packages = [{include = "playbook.py"}] [tool.poetry.dependencies] diff --git a/pytest-target/README.md b/pytest-target/README.md new file mode 100644 index 0000000000..3acaa76ec2 --- /dev/null +++ b/pytest-target/README.md @@ -0,0 +1,9 @@ +# pytest-target + +Pytest plugin for remote target orchestration. Supports the LISAv3 framework. + +See the [documentation][] for more information, and the LISAv3 [`README.md`][] +for notices. + +[documentation]: https://microsoft.github.io/lisa/modules/target.html +[`README.md`]: https://github.com/microsoft/lisa/blob/andschwa/pytest/README.md diff --git a/pytest-target/pyproject.toml b/pytest-target/pyproject.toml index c111e393ab..34cf2809b4 100644 --- a/pytest-target/pyproject.toml +++ b/pytest-target/pyproject.toml @@ -2,9 +2,20 @@ name = "pytest-target" version = "0.1.0" description = "Pytest plugin for remote target orchestration." -authors = ["Andrew Schwartzmeyer "] license = "MIT" -classifiers = ["Framework :: Pytest"] +authors = ["Andrew Schwartzmeyer "] +readme = "README.md" +homepage = "https://microsoft.github.io/lisa" +repository = "https://github.com/microsoft/lisa/tree/andschwa/pytest/pytest-target" +classifiers = [ + "Development Status :: 3 - Alpha", + "Framework :: Pytest", + "Intended Audience :: Developers", + "Operating System :: Microsoft :: Windows", + "Operating System :: POSIX", + "Topic :: Software Development :: Testing", + "Topic :: Utilities" +] packages = [{include = "target"}] [tool.poetry.dependencies] From 3b225788f461ef466e62ff6fc712ac442ddc7f26 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Fri, 15 Jan 2021 17:50:10 -0800 Subject: [PATCH 186/194] Update `pytest-playbook` and `pytest-lisa` documentation --- conf.py | 3 +- pytest-lisa/lisa.py | 64 ++++++++++++++++++------------- pytest-playbook/README.md | 2 +- pytest-playbook/playbook.py | 75 +++++++++++++++++++++++-------------- 4 files changed, 87 insertions(+), 57 deletions(-) diff --git a/conf.py b/conf.py index 6ffe89f157..bc8872f14b 100644 --- a/conf.py +++ b/conf.py @@ -66,7 +66,8 @@ def linkcode_resolve(domain, info): folder = "" filename = module.replace(".", "/") - url = metadata["Project-Url"].split(", ")[1] + # NOTE: Note `metadata["Project-Url"]` because we need the base. + url = "https://github.com/microsoft/lisa" # TODO: Update this branch to `main` branch after PR is merged. branch = "andschwa/pytest" return f"{url}/blob/{branch}/{folder}/{filename}.py" diff --git a/pytest-lisa/lisa.py b/pytest-lisa/lisa.py index 4c3ac39fd0..7aa4ed3d0d 100644 --- a/pytest-lisa/lisa.py +++ b/pytest-lisa/lisa.py @@ -2,12 +2,15 @@ # Licensed under the MIT License. """A plugin for organizing, analyzing, and selecting tests. -This plugin provides the mark ``pytest.mark.lisa``, aliased as ``LISA``, -for marking up tests metadata beyond that which Pytest provides by -default. See the ``lisa_schema`` for the expected metadata input. +This plugin provides the mark ``pytest.mark.lisa``, aliased as +:py:func:`LISA`, for marking up tests metadata beyond that which +Pytest provides by default. See the :py:data:`lisa_schema` for the +expected metadata input. Tests can be selected through a ``playbook.yaml`` file using the -criteria schema. For example: +criteria schema (using the `pytest-playbook`_ plugin). For example: + +.. _pytest-playbook: https://microsoft.github.io/lisa/modules/playbook.html .. code-block:: yaml @@ -52,18 +55,21 @@ from pytest import Item, Session LISA = pytest.mark.lisa +"""Alias for the Pytest mark ``lisa``.""" def main() -> None: - """Wrapper function so we can have a `lisa` binary.""" + """Wrapper function so we can have a ``lisa`` binary.""" sys.exit(pytest.main()) def pytest_configure(config: Config) -> None: - """Pytest hook to perform initial configuration. + """Pytest `configure hook`_ to perform initial configuration. + + .. _configure hook: https://docs.pytest.org/en/stable/reference.html#pytest.hookspec.pytest_configure We're registering our custom marker so that it passes - `--strict-markers`. + ``--strict-markers``. """ config.addinivalue_line( @@ -76,7 +82,7 @@ def pytest_configure(config: Config) -> None: def pytest_playbook_schema(schema: Dict[Any, Any]) -> None: - """pytest-playbook hook to update the playbook schema.""" + """:py:meth:`~playbook.Hooks.pytest_playbook_schema` hook to update the playbook schema.""" # TODO: We also want to support a ‘targets’ list that confines a # test selection to only the given targets. criteria_schema = Schema( @@ -159,7 +165,7 @@ def pytest_playbook_schema(schema: Dict[Any, Any]) -> None: def validate_mark(mark: Mark) -> None: - """Validate each test's LISA parameters.""" + """Validate each test's :py:func:`LISA` parameters.""" assert not mark.args, "LISA marker cannot have positional arguments!" mark.kwargs.update(lisa_schema.validate(mark.kwargs)) # type: ignore @@ -167,13 +173,15 @@ def validate_mark(mark: Mark) -> None: def pytest_collection_modifyitems( session: Session, config: Config, items: List[Item] ) -> None: - """Pytest hook for modifying the selected items (tests). + """Pytest `collection modifyitems hook`_ for modifying the selected items (tests). - First we validate all the `LISA` marks on the collected tests. - Then we parse the given `criteria` in the playbook to include or - exclude tests. We do not care if the `platform` mismatches because - we intend a multiplicative effect where all selected tests in a - playbook are run on all the targets. + .. _collection modifyitems hook: https://docs.pytest.org/en/latest/reference.html#pytest.hookspec.pytest_collection_modifyitems + + First we validate all the :py:func:`LISA` marks on the collected + tests. Then we parse the given ``criteria`` in the playbook to + include or exclude tests. We do not care if the ``platform`` + mismatches because we intend a multiplicative effect where all + selected tests in a playbook are run on all the targets. """ # TODO: The ‘Item’ object has a ‘user_properties’ attribute which @@ -251,22 +259,24 @@ def select(item: Item, times: int, exclude: bool) -> None: class LISAScheduling(LoadScopeScheduling): - """Implement load scheduling across nodes, but grouping by parameter. + """Implement load scheduling across nodes, but grouping by target parameter. This algorithm ensures that all tests which share the same set of parameters (namely the target) will run on the same executor as a single work-unit. - TODO: This essentially confines the targets and one target won't - be spun up multiple times when run in parallel, so we should make - this scheduler optional, as an alternative scenario is to spin up - multiple near-identical instances of a target in order to run - tests in parallel. + .. TODO:: + + This essentially confines the targets and one target won't be + spun up multiple times when run in parallel, so we should make + this scheduler optional, as an alternative scenario is to spin + up multiple near-identical instances of a target in order to + run tests in parallel. - This is modeled after the built-in `LoadFileScheduling`, which - also simply subclasses `LoadScopeScheduling`. See `_split_scope` - for the important part. Note that we can extend this to implement - any kind of scheduling algorithm we want. + This is modeled after the built-in ``LoadFileScheduling``, which + also simply subclasses ``LoadScopeScheduling``. See + ``_split_scope`` for the important part. Note that we can extend + this to implement any kind of scheduling algorithm we want. """ @@ -328,9 +338,9 @@ def _split_scope(self, nodeid: str) -> str: def pytest_xdist_make_scheduler(config: Config) -> LISAScheduling: - """pytest-xdist hook for implementing a custom scheduler. + """pytest-xdist `make scheduler hook`_ for implementing a custom scheduler. - https://github.com/pytest-dev/pytest-xdist/blob/master/OVERVIEW.md + .. _make scheduler hook: https://github.com/pytest-dev/pytest-xdist/blob/master/OVERVIEW.md """ return LISAScheduling(config) diff --git a/pytest-playbook/README.md b/pytest-playbook/README.md index 542954ea40..8f921de114 100644 --- a/pytest-playbook/README.md +++ b/pytest-playbook/README.md @@ -6,4 +6,4 @@ See the [documentation][] for more information, and the LISAv3 [`README.md`][] for notices. [documentation]: https://microsoft.github.io/lisa/modules/playbook.html -[README]: https://github.com/microsoft/lisa/blob/andschwa/pytest/README.md +[`README.md`]: https://github.com/microsoft/lisa/blob/andschwa/pytest/README.md diff --git a/pytest-playbook/playbook.py b/pytest-playbook/playbook.py index 7b815f0ef5..ecd7269f32 100644 --- a/pytest-playbook/playbook.py +++ b/pytest-playbook/playbook.py @@ -2,18 +2,39 @@ # Licensed under the MIT License. """A plugin for creating, validating, and reading a playbook. -Use the ``pytest_playbook_schema`` hook to modify the schema -dictionary representing the data expected to be read and validated -from a ``playbook.yaml`` file, the path to which is provided by the -user with the command-line flag ``--playbook``. +Use the :py:meth:`~Hooks.pytest_playbook_schema` hook to modify the +`schema`_ dictionary representing the data expected to be read and +validated from a ``playbook.yaml`` `YAML`_ file, the path to which is +provided by the user with the command-line flag +``--playbook=``. -This module's ``data`` attribute will hold the read and validated data -after all ``pytest_configure`` hooks have run. See -``pytest_playbook_schema`` for example usage. +.. _schema: https://github.com/keleshev/schema +.. _YAML: https://pyyaml.org/wiki/PyYAMLDocumentation + +This module's :py:data:`data` attribute will hold the read and +validated data after all :py:func:`pytest_configure` hooks have run. +Use it in your ``conftest.py`` (or Pytest plugin) like so: + +.. code-block:: python + + import playbook + from schema import Schema + + def pytest_playbook_schema(schema): + schema["targets"] = Schema({"name": str, "platform": str, "cpus": int}) + + def pytest_sessionstart(session): + for target in playbook.data["targets"]: + print(target["name"]) Remember not to use ``from playbook import data`` because then the attribute will not contain the shared data. Instead use ``import -playbook`` and reference ``playbook.data``. +playbook`` and reference :py:data:`playbook.data`. + +All registered schema can be printed to a `JSON Schema`_ file with +``--print-schema=``. + +.. _JSON Schema: https://json-schema.org/ """ @@ -27,7 +48,6 @@ import yaml # TODO: Optionally load yaml. from schema import Schema, SchemaError # type: ignore -# See https://pyyaml.org/wiki/PyYAMLDocumentation try: from yaml import CLoader as Loader except ImportError: @@ -52,32 +72,28 @@ class Hooks: def pytest_playbook_schema(self, schema: Dict[Any, Any], config: Config) -> None: """Update the Playbook's schema dict. - The 'schema' is a mutable dict, and 'config' is optional. - Example usage: - - .. code-block:: python - - import playbook - from schema import Schema - - def pytest_playbook_schema(schema): - schema["targets"] = Schema({"name": str, "platform": str, "cpus": int}) - - def pytest_sessionstart(session): - for target in playbook.playbook["targets"]: - print(target["name"]) + :param schema: mutable dict passed to ``schema.validate()`` in :py:func:`pytest_configure`. + :param config: optional, allows access to Pytest ``Config`` object if given. """ # Now provide the hook implementations. def pytest_addhooks(pluginmanager: PytestPluginManager) -> None: - """Pytest hook to register our hooks.""" + """Pytest `addhooks hook`_ to register our hooks. + + .. _addhooks hook: https://docs.pytest.org/en/stable/reference.html#pytest.hookspec.pytest_addhooks + + """ pluginmanager.add_hookspecs(Hooks) def pytest_addoption(parser: Parser, pluginmanager: PytestPluginManager) -> None: - """Pytest hook to add our CLI options.""" + """Pytest `addoption hook`_ to add our CLI options. + + .. _addoption hook: https://docs.pytest.org/en/stable/reference.html#pytest.hookspec.pytest_addoption + + """ group = parser.getgroup("playbook") group.addoption("--playbook", type=Path, help="Path to playbook.") group.addoption( @@ -90,10 +106,13 @@ def pytest_addoption(parser: Parser, pluginmanager: PytestPluginManager) -> None # TODO: See if this works without ‘trylast’. @pytest.hookimpl(trylast=True) def pytest_configure(config: Config) -> None: - """Pytest hook to configure our plugin. + """Pytest `configure hook`_ to configure our plugin. + + .. _configure hook: https://docs.pytest.org/en/stable/reference.html#pytest.hookspec.pytest_configure - This is set to be tried last so that all other plugins have been - loaded and defined their ``pytest_playbook_schema`` hooks. + This is set to be tried last so that all other plugins and + ``conftest.py`` files have been loaded and defined their + :py:meth:`~Hooks.pytest_playbook_schema` hooks. """ schema_dict: Dict[Any, Any] = dict() From 2d10e24c597e0abe1754275c1230862eb8b0969c Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Wed, 20 Jan 2021 15:52:05 -0800 Subject: [PATCH 187/194] Use `tmp_path` fixture in smoke test for boot diagnostics --- testsuites/test_smoke_b.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/testsuites/test_smoke_b.py b/testsuites/test_smoke_b.py index 0e43921a6f..dd72bfc34a 100644 --- a/testsuites/test_smoke_b.py +++ b/testsuites/test_smoke_b.py @@ -9,6 +9,7 @@ import logging import socket import time +from pathlib import Path from invoke.runners import CommandTimedOut, UnexpectedExit # type: ignore from paramiko import SSHException # type: ignore @@ -17,7 +18,7 @@ @LISA(platform="Azure", category="Functional", area="deploy", priority=0) -def test_smoke(target: AzureCLI, caplog: LogCaptureFixture) -> None: +def test_smoke(target: AzureCLI, caplog: LogCaptureFixture, tmp_path: Path) -> None: """Check that an Azure Linux VM can be deployed and is responsive. This example uses exactly one function for the entire test, which @@ -78,12 +79,14 @@ def test_smoke(target: AzureCLI, caplog: LogCaptureFixture) -> None: logging.warning(f"SSH after reboot failed: '{e}'") logging.info("Retrieving boot diagnostics...") + path = tmp_path / "diagnostics.txt" try: - target.get_boot_diagnostics() + diagnostics = target.get_boot_diagnostics(hide=True) + path.write_text(diagnostics.stdout) except UnexpectedExit: logging.warning("Retrieving boot diagnostics failed.") else: - logging.info("See full report for boot diagnostics.") + logging.info(f"See '{path}' for boot diagnostics.") # NOTE: The test criteria is to fail only if ping fails. assert ping1.ok, f"Pinging {target.host} before reboot failed" From d552019f8cff076621a8c4798e511ffd25c5a570 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Wed, 20 Jan 2021 15:52:21 -0800 Subject: [PATCH 188/194] =?UTF-8?q?Hide=20output=20of=20commands=20to=20al?= =?UTF-8?q?low=20ping=20in=20the=20AzureCLI=E2=80=99s=20NSG?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pytest-target/target/azure.py | 3 ++- pytest-target/target/plugin.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/pytest-target/target/azure.py b/pytest-target/target/azure.py index 541668b706..0a4231b2b9 100644 --- a/pytest-target/target/azure.py +++ b/pytest-target/target/azure.py @@ -107,7 +107,8 @@ def allow_ping(self) -> None: f"--name allow{d}ICMP --resource-group {self.group}-rg " f"--nsg-name {self.name}NSG --priority 150 " f"--access Allow --direction '{d}' --protocol Icmp " - "--source-port-ranges '*' --destination-port-ranges '*'" + "--source-port-ranges '*' --destination-port-ranges '*'", + hide=True, ) except Exception as e: logging.warning( diff --git a/pytest-target/target/plugin.py b/pytest-target/target/plugin.py index a4eed3b4e4..aa196116f9 100644 --- a/pytest-target/target/plugin.py +++ b/pytest-target/target/plugin.py @@ -162,6 +162,7 @@ def pytest_unconfigure(config: Config) -> None: """ if not config.getoption("keep_targets"): + # TODO: Ignore `--help` or other times tests weren’t run. logging.info("Deleting targets! Pass `--keep-targets` to prevent this.") delete_targets(config) From 50fe160624fd2805ac0a410fbcb9b56c4b3e86b9 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Wed, 20 Jan 2021 15:52:43 -0800 Subject: [PATCH 189/194] Display summary of all tests Also remove the set log level, as this is the default. --- pytest.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index 3b4caa889b..faef4e85e7 100644 --- a/pytest.ini +++ b/pytest.ini @@ -5,7 +5,7 @@ addopts = --capture=tee-sys --tb=line --no-header -log_level = WARNING + -rA log_format = %(asctime)s %(levelname)s %(message)s log_date_format = %Y-%m-%d %H:%M:%S render_collapsed = true From 56739e9583ecaea9ded8d2c280f3f87fea46fbc5 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Wed, 20 Jan 2021 15:53:05 -0800 Subject: [PATCH 190/194] Write usage document --- USAGE.rst | 349 +++++++++++++++++++++++++++++++++ index.rst | 102 +--------- playbooks/demo.yaml | 2 +- pytest-target/target/target.py | 2 +- 4 files changed, 354 insertions(+), 101 deletions(-) create mode 100644 USAGE.rst diff --git a/USAGE.rst b/USAGE.rst new file mode 100644 index 0000000000..69aa324295 --- /dev/null +++ b/USAGE.rst @@ -0,0 +1,349 @@ +How to Use Pytest and LISA +========================== + +LISA is supported on almost any Linux or Windows installation provided +Python 3.7 (released in 2018) or newer is available and SSH can be +used to connect to the remote targets under test. The local SSH +configuration is respected so ``ProxyJump`` can be used. + +Install Python 3.7+ +------------------- + +Install Python 3.7 or newer from your Linux distribution’s package +repositories, or `python.org `_. + +On Ubuntu 20.04 and up, just run ``apt install python-is-python3``. + +Below that Ubuntu version, the ``python3`` package is out-of-date, so +use something like a `PPA`_ or `pyenv`_. + +.. _PPA: https://launchpad.net/~deadsnakes/+archive/ubuntu/ppa +.. _pyenv: https://github.com/pyenv/pyenv + +Install Poetry +-------------- + +`Poetry `_ is our preferred tool for +Python dependency management and packaging. We’ll use it to +automatically setup a virtual environment and install everything. + +On Linux (or WSL) +~~~~~~~~~~~~~~~~~ + +.. code:: bash + + curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python - + source $HOME/.poetry/env + +If you are using WSL, installing Poetry on both Windows and Linux may +cause both platforms’ versions of Poetry to be on your path, as Windows +binaries are mapped into WSL’s ``PATH``. This means that the Linux +``poetry`` binary *must* appear in your ``PATH`` before the Windows +version, or this error will appear: + +:: + + `/usr/bin/env: ‘python\r’: No such file or directory` + +Adjust your ``PATH`` appropriately to fix it. + +On Windows (in PowerShell) +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code:: powershell + + (Invoke-WebRequest -Uri https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py -UseBasicParsing).Content | python - + $env:PATH += ";$env:USERPROFILE\.poetry\bin" + +Clone LISA and ``cd`` into the Git repo +--------------------------------------- + +.. code:: bash + + git clone -b andschwa/pytest https://github.com/microsoft/lisa.git + cd lisa + +Install Python dependencies +--------------------------- + +Now we’ll use ``poetry`` to install all the necessary packages. Note +that we have a number of developer dependencies specified to make your +life easier when :doc:`contributing `, but you can +exclude these (and their potential additional requirements) with the +flag ``--no-dev``. Once installed, we use ``poetry shell`` to enter a +sub-shell with the Python virtual environment setup. + +.. code:: bash + + # Install the Python packages + poetry install + + # Enter the virtual environment + poetry shell + +Use LISA +-------- + +Under the covers ``lisa`` is just ``pytest``! Run ``lisa --help`` for +all available options, and refer to the Pytest `usage`_ documentation. +LISA is generally run with a :py:mod:`playbook` which is a `YAML file +`_ specifying a list of +remote targets, their parameters, and optionally a set of test +selection criteria. The ``demo.yaml`` looks like: + +.. _usage: https://docs.pytest.org/en/stable/usage.html + +.. code:: yaml + + platforms: + AzureCLI: + sku: Standard_DS2_v2 + + targets: + - name: Debian + platform: AzureCLI + image: credativ:Debian:9:9.0.201706190 + + - name: Ubuntu + platform: AzureCLI + image: Canonical:UbuntuServer:18.04-LTS:latest + + criteria: + - module: test_smoke_b + +The ``platforms`` key is used to set default parameters for +targets using that platform; in this case, the SKU is set to +``Standard_DS2_v2``. + +The ``targets`` key defines a number of targets on which the selected +tests will run. Here we’re asking for two targets using the same +:py:class:`~target.azure.AzureCLI` platform, both will use the same +default for the SKU, but different images. The ``name`` is just a +user-provided friendly name that is appended to the parameterized +tests and will show up in test results. + +The ``criteria`` key can be used to select a tests instead of using +Pytest’s CLI test selection interface. In this case we’re selecting +all tests from the module (Python file) named ``test_smoke_b``, one of +the examples of an Azure VM smoke test. + +Enable Azure +~~~~~~~~~~~~ + +Before running this demo, we will need to set up the `Azure CLI +`_ because this platform uses it. Install +it if you do not already have, then ensure it is logged in with your +choice of authentication, and set a default subscription, which will +be used to deploy the resources. + +.. code:: bash + + # Install Azure CLI, make sure `az` is in your `PATH` + curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash + + # Login and set subscription + az login + az account set -s + +Run the Demo +~~~~~~~~~~~~ + +Now we can run the demo! + +.. code:: bash + + # Run a demo which deploys Azure resources + lisa --playbook=playbooks/demo.yaml --keep-targets --html=demo.html + +This will sequentially deploy the two requested targets and run the +smoke test against them, printing the stdout, stderr, and logging of +all tests after they complete (see below for how to change this +behavior). The ``--keep-targets`` flag comes from the :py:mod:`target` +plugin and instructs it to cache the deployed targets between test +runs. Delete them by running ``lisa --delete-targets``. The +``--html=demo.html`` flag will cause an easy-to-read HTML report to be +written to ``demo.html``. + +It should look very similar to this slightly redacted example: + +:: + + $ lisa --playbook=playbooks/demo.yaml --keep-targets --html=demo.html + =========================== test session starts ========================== + collected 40 items / 38 deselected / 2 selected + + testsuites/test_smoke_b.py F. [100%] + + ================================ FAILURES ================================ + _______________________ test_smoke[Target=Debian] ________________________ + testsuites/test_smoke_b.py:93: in test_smoke + assert ping2.ok, f"Pinging {target.host} after reboot failed" + E AssertionError: Pinging 40.123.27.161 after reboot failed + E assert False + E + where False = .ok + ------------------------- Captured stdout setup -------------------------- + az vm create -g pytest-d6056453-a28c-4fec-8225-7c7aab02c84a-rg -n pytest-d6056453-a28c-4fec-8225-7c7aab02c84a-0 --image credativ:Debian:9:9.0.201706190 --size Standard_DS2_v2 --boot-diagnostics-storage pytestbootdiag --generate-ssh-keys + { + "fqdns": "", + "id": "/subscriptions/<...>/resourceGroups/pytest-d6056453-a28c-4fec-8225-7c7aab02c84a-rg/providers/Microsoft.Compute/virtualMachines/pytest-d6056453-a28c-4fec-8225-7c7aab02c84a-0", + "location": "eastus2", + "macAddress": "00-0D-3A-DE-07-17", + "powerState": "VM running", + "privateIpAddress": "10.0.0.4", + "publicIpAddress": "<...>", + "resourceGroup": "pytest-d6056453-a28c-4fec-8225-7c7aab02c84a-rg", + "zones": "" + } + -------------------------- Captured stdout call -------------------------- + ping -c 1 40.123.27.161 + PING 40.123.27.161 (40.123.27.161) 56(84) bytes of data. + + --- 40.123.27.161 ping statistics --- + 1 packets transmitted, 0 received, 100% packet loss, time 0ms + ... + ping -c 1 40.123.27.161 + PING 40.123.27.161 (40.123.27.161) 56(84) bytes of data. + 64 bytes from 40.123.27.161: icmp_seq=1 ttl=43 time=85.6 ms + + --- 40.123.27.161 ping statistics --- + 1 packets transmitted, 1 received, 0% packet loss, time 0ms + rtt min/avg/max/mdev = 85.562/85.562/85.562/0.000 ms + ping -c 1 40.123.27.161 + PING 40.123.27.161 (40.123.27.161) 56(84) bytes of data. + + --- 40.123.27.161 ping statistics --- + 1 packets transmitted, 0 received, 100% packet loss, time 0ms + ... + ping -c 1 40.123.27.161 + PING 40.123.27.161 (40.123.27.161) 56(84) bytes of data. + + --- 40.123.27.161 ping statistics --- + 1 packets transmitted, 0 received, 100% packet loss, time 0ms + + --------------------------- Captured log call ---------------------------- + 2021-01-20 17:14:56 INFO Pinging before reboot... + 2021-01-20 17:15:51 INFO SSHing before reboot... + 2021-01-20 17:15:52 INFO Connected (version 2.0, client OpenSSH_7.4p1) + 2021-01-20 17:15:53 INFO Authentication (publickey) successful! + 2021-01-20 17:15:53 INFO Rebooting... + 2021-01-20 17:15:53 WARNING While SSH worked, 'reboot' command failed + 2021-01-20 17:15:53 INFO Sleeping for 10 seconds after reboot... + 2021-01-20 17:16:03 INFO Pinging after reboot... + 2021-01-20 17:17:08 INFO SSHing after reboot... + 2021-01-20 17:19:16 ERROR Secsh channel 1 open FAILED: Connection timed out: Connect failed + 2021-01-20 17:19:16 WARNING SSH after reboot failed: 'ChannelException(2, 'Connect failed')' + 2021-01-20 17:19:16 INFO Retrieving boot diagnostics... + 2021-01-20 17:19:20 INFO See '/tmp/pytest-of-andschwa/pytest-181/test_smoke_Target_Debian_0/diagnostics.txt' for boot diagnostics. + ================================= PASSES ================================= + _______________________ test_smoke[Target=Ubuntu] ________________________ + ------------------------- Captured stdout setup -------------------------- + az vm create -g pytest-8f173841-d702-432e-bd32-f09a984bd3ab-rg -n pytest-8f173841-d702-432e-bd32-f09a984bd3ab-0 --image Canonical:UbuntuServer:18.04-LTS:latest --size Standard_DS2_v2 --boot-diagnostics-storage pytestbootdiag --generate-ssh-keys + { + "fqdns": "", + "id": "/subscriptions/<..>/resourceGroups/pytest-8f173841-d702-432e-bd32-f09a984bd3ab-rg/providers/Microsoft.Compute/virtualMachines/pytest-8f173841-d702-432e-bd32-f09a984bd3ab-0", + "location": "eastus2", + "macAddress": "00-0D-3A-7C-85-59", + "powerState": "VM running", + "privateIpAddress": "10.0.0.4", + "publicIpAddress": "<...>", + "resourceGroup": "pytest-8f173841-d702-432e-bd32-f09a984bd3ab-rg", + "zones": "" + } + -------------------------- Captured stdout call -------------------------- + ping -c 1 137.116.51.62 + PING 137.116.51.62 (137.116.51.62) 56(84) bytes of data. + + --- 137.116.51.62 ping statistics --- + 1 packets transmitted, 0 received, 100% packet loss, time 0ms + ... + ping -c 1 137.116.51.62 + PING 137.116.51.62 (137.116.51.62) 56(84) bytes of data. + 64 bytes from 137.116.51.62: icmp_seq=1 ttl=42 time=84.0 ms + + --- 137.116.51.62 ping statistics --- + 1 packets transmitted, 1 received, 0% packet loss, time 0ms + rtt min/avg/max/mdev = 84.004/84.004/84.004/0.000 ms + --------------------------- Captured log call ---------------------------- + 2021-01-20 17:20:26 INFO Pinging before reboot... + 2021-01-20 17:21:21 INFO SSHing before reboot... + 2021-01-20 17:21:21 INFO Connected (version 2.0, client OpenSSH_7.6p1) + 2021-01-20 17:21:22 INFO Authentication (publickey) successful! + 2021-01-20 17:21:22 INFO Rebooting... + 2021-01-20 17:21:24 WARNING While SSH worked, 'reboot' command failed + 2021-01-20 17:21:24 INFO Sleeping for 10 seconds after reboot... + 2021-01-20 17:21:34 INFO Pinging after reboot... + 2021-01-20 17:21:45 INFO SSHing after reboot... + 2021-01-20 17:21:46 INFO Connected (version 2.0, client OpenSSH_7.6p1) + 2021-01-20 17:21:46 INFO Authentication (publickey) successful! + 2021-01-20 17:21:46 INFO Retrieving boot diagnostics... + 2021-01-20 17:21:50 INFO See '/tmp/pytest-of-andschwa/pytest-181/test_smoke_Target_Ubuntu_0/diagnostics.txt' for boot diagnostics. + ----- generated html file: file:///home/andschwa/src/lisa/demo.html ------ + ======================== short test summary info ========================= + PASSED testsuites/test_smoke_b.py::test_smoke[Target=Ubuntu] + FAILED testsuites/test_smoke_b.py::test_smoke[Target=Debian] - AssertionError: Pinging 40.123.27.161 after reboot failed + ========= 1 failed, 1 passed, 38 deselected in 541.93s (0:09:01) ========= + +Settings +-------- + +Our opinionated `usage`_ settings are in ``pytest.ini``. Adjust them +(or override them on the CLI) as you see fit! They include: + +``--no-header`` + + For more succinct display, we suppress the default Pytest header + with the platform, root directory, plugins, and timeout + information. + +``--tb=short`` + + Since we’re generally testing commands on remote systems, we don’t + care about the full Python trace when a test fails, so we set the + `traceback printing + `_ + to be short. + +``-rA`` + + We want the status (and captured logs) of *all* tests printed in + the final summary, but Pytest defaults to failed and errored tests + with ``fE``, hence our use of ``A``. + +``timeout = 1200`` + + Since we run our tests on remote machines which may hang, we use + `pytest-timeout `_ to + cancel any tests that exceed 20 minutes. Note that the + :py:class:`target` class also has a “timeout” configuration for + individual commands using `Invoke `_. + +Suggestions +~~~~~~~~~~~ + +Test developers may wish to run with the flags: + +``--capture=tee-sys`` + + This will `capture + `_ all writes to + ``sys.stdout`` and ``sys.stderr``, but also pass them to ``sys`` + such that they’re printed *live* (useful when writing tests, but + annoying when running tests). + +``log_cli=true`` + + Pytest can emit captured `logs + `_ live too. Add + this to ``pytest.ini`` (and adjust the level and format as + desired). + +``--tb=auto`` + + To show the full traceback instead of just a line. + +``--html=path/to/report.html`` + + We include `pytest-html + `_ as a dependency + so users can generate HTML reports with all captured stdout, + stderr, traceback, and logs. diff --git a/index.rst b/index.rst index fdf558563c..fc0994f38e 100644 --- a/index.rst +++ b/index.rst @@ -18,6 +18,7 @@ sources. :caption: Documentation :hidden: + Usage Design Contributing Code of Conduct @@ -25,105 +26,8 @@ sources. Getting Started --------------- -LISA is supported on almost any Linux or Windows installation provided -Python 3.7 (released in 2018) or newer is available and SSH can be -used to connect to the remote targets under test. The local SSH -configuration is respected so ``ProxyJump`` can be used. - -Install Python 3 -~~~~~~~~~~~~~~~~ - -Install Python 3.7 or newer from your Linux distribution’s package -repositories, or `python.org `_. - -On Ubuntu 20.04 and up, just run ``apt install python-is-python3``. - -Below that Ubuntu version, the ``python3`` package is out-of-date, so -use something like a `PPA`_ or `pyenv`_. - -.. _PPA: https://launchpad.net/~deadsnakes/+archive/ubuntu/ppa -.. _pyenv: https://github.com/pyenv/pyenv - -Install Poetry -~~~~~~~~~~~~~~ - -`Poetry `_ is our preferred tool for -Python dependency management and packaging. We’ll use it to -automatically setup a ‘virtualenv’ and install everything we need. - -On Linux (or WSL) -^^^^^^^^^^^^^^^^^ - -.. code:: bash - - curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python - - source $HOME/.poetry/env - -If you are using WSL, installing Poetry on both Windows and Linux may -cause both platforms’ versions of Poetry to be on your path, as Windows -binaries are mapped into WSL’s ``PATH``. This means that the Linux -``poetry`` binary *must* appear in your ``PATH`` before the Windows -version, or this error will appear: - -:: - - `/usr/bin/env: ‘python\r’: No such file or directory` - -Adjust your ``PATH`` appropriately to fix it. - -On Windows (in PowerShell) -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code:: powershell - - (Invoke-WebRequest -Uri https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py -UseBasicParsing).Content | python - - $env:PATH += ";$env:USERPROFILE\.poetry\bin" - -Clone LISA and ``cd`` into the Git repo -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. code:: bash - - git clone -b andschwa/pytest https://github.com/microsoft/lisa.git - cd lisa - -Install Python dependencies -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. code:: bash - - # Install the Python packages - poetry install - - # Enter the virtual environment - poetry shell - -Use LISA -~~~~~~~~ - -.. code:: bash - - # Run some self-tests - lisa --playbook=playbooks/test.yml selftests/ - - # Run a demo which deploys Azure resources - lisa --playbook=playbooks/demo.yaml - -Enable Azure -^^^^^^^^^^^^ - -To run the demo you’ll need the `Azure -CLI `_ tool installed and -configured: - -.. code:: bash - - # Install Azure CLI, make sure `az` is in your `PATH` - curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash - - # Login and set subscription - az login - az account set -s +See the :doc:`usage document ` for how to setup the +requirements, run tests, and write new tests. Python Modules -------------- diff --git a/playbooks/demo.yaml b/playbooks/demo.yaml index 7331b9c460..74eca3e6f3 100644 --- a/playbooks/demo.yaml +++ b/playbooks/demo.yaml @@ -9,7 +9,7 @@ targets: - name: Ubuntu platform: AzureCLI - image: UbuntuLTS + image: Canonical:UbuntuServer:18.04-LTS:latest criteria: - module: test_smoke_b diff --git a/pytest-target/target/target.py b/pytest-target/target/target.py index abe2448f1d..dc2f39646a 100644 --- a/pytest-target/target/target.py +++ b/pytest-target/target/target.py @@ -101,7 +101,7 @@ class Target(TargetData, metaclass=ABCMeta): "echo": True, # Disable stdin forwarding. "in_stream": False, - # Don’t let remote commands take longer than five minutes + # Don’t let remote commands take longer than twenty minutes # (unless later overridden). This is to prevent hangs. "command_timeout": 1200, } From 6ab5edc5d374172179672b564a7f4e235ecb7e7b Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Wed, 20 Jan 2021 16:41:01 -0800 Subject: [PATCH 191/194] Use default capture mode and short traceback --- pytest.ini | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pytest.ini b/pytest.ini index faef4e85e7..e87ecdec1f 100644 --- a/pytest.ini +++ b/pytest.ini @@ -2,9 +2,8 @@ addopts = --strict-markers --self-contained-html - --capture=tee-sys - --tb=line --no-header + --tb=short -rA log_format = %(asctime)s %(levelname)s %(message)s log_date_format = %Y-%m-%d %H:%M:%S From 3a4e95d6281d73bd32b0de289d6e9171a67bbb39 Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Wed, 20 Jan 2021 18:05:21 -0800 Subject: [PATCH 192/194] Setup plugins to depend on pytest-playbook from PyPI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Otherwise they can’t be published themselves. This we should override the dependency at the top-level so we can install it from the local source in editable mode. --- poetry.lock | 2 +- pyproject.toml | 1 + pytest-lisa/poetry.lock | 18 ++++++++---------- pytest-lisa/pyproject.toml | 2 +- pytest-target/poetry.lock | 18 ++++++++---------- pytest-target/pyproject.toml | 2 +- 6 files changed, 20 insertions(+), 23 deletions(-) diff --git a/poetry.lock b/poetry.lock index 6dc075a0fa..43548f04e5 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1081,7 +1081,7 @@ testing = ["pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "4f4e21335d25310f82b81670134aa8a48ec8ae442bdcb5d4eb0db2642bfc6c64" +content-hash = "01f1cdbe9414f75f0366106f0b049986cde6d756a9e9614835b56c0abec3ee0f" [metadata.files] alabaster = [ diff --git a/pyproject.toml b/pyproject.toml index 16e344665d..0e08e2f332 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,7 @@ pytest = "^6.1.1" pytest-timeout = "^1.4.2" pytest-html = "^2.1.1" pytest-rerunfailures = "^9.1.1" +pytest-playbook = {path = "pytest-playbook", develop = true} pytest-target = {path = "pytest-target", develop = true} pytest-lisa = {path = "pytest-lisa", develop = true} diff --git a/pytest-lisa/poetry.lock b/pytest-lisa/poetry.lock index 6bc110e4b3..4a3d0ed6e5 100644 --- a/pytest-lisa/poetry.lock +++ b/pytest-lisa/poetry.lock @@ -163,18 +163,13 @@ version = "0.1.0" description = "Pytest plugin for reading playbooks." category = "main" optional = false -python-versions = "^3.7" -develop = true +python-versions = ">=3.7,<4.0" [package.dependencies] -pytest = "^6.1.2" -PyYAML = "^5.3.1" +pytest = ">=6.1.2,<7.0.0" +PyYAML = ">=5.3.1,<6.0.0" schema = "0.7.2" -[package.source] -type = "directory" -url = "../pytest-playbook" - [[package]] name = "pytest-xdist" version = "2.2.0" @@ -242,7 +237,7 @@ testing = ["pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "35352fa0bb935ffb30d81fdf5ffa203e15a25d4a7fd0e1243570236a1105f647" +content-hash = "f265436364a83d94f8fb710e4d992f88e631a07004e6bce037888eb0991288bc" [metadata.files] apipkg = [ @@ -301,7 +296,10 @@ pytest-forked = [ {file = "pytest-forked-1.3.0.tar.gz", hash = "sha256:6aa9ac7e00ad1a539c41bec6d21011332de671e938c7637378ec9710204e37ca"}, {file = "pytest_forked-1.3.0-py2.py3-none-any.whl", hash = "sha256:dc4147784048e70ef5d437951728825a131b81714b398d5d52f17c7c144d8815"}, ] -pytest-playbook = [] +pytest-playbook = [ + {file = "pytest-playbook-0.1.0.tar.gz", hash = "sha256:c9ce7446e9301646ee3fdb74a3d7cbdee4a83e775126b9062d0cbd6178ef36ec"}, + {file = "pytest_playbook-0.1.0-py3-none-any.whl", hash = "sha256:c2fb714d4347d89d7b031822eed279c956275f41f34927cb94685f04670bd01d"}, +] pytest-xdist = [ {file = "pytest-xdist-2.2.0.tar.gz", hash = "sha256:1d8edbb1a45e8e1f8e44b1260583107fc23f8bc8da6d18cb331ff61d41258ecf"}, {file = "pytest_xdist-2.2.0-py3-none-any.whl", hash = "sha256:f127e11e84ad37cc1de1088cb2990f3c354630d428af3f71282de589c5bb779b"}, diff --git a/pytest-lisa/pyproject.toml b/pytest-lisa/pyproject.toml index 1d8951c14c..33269571b0 100644 --- a/pytest-lisa/pyproject.toml +++ b/pytest-lisa/pyproject.toml @@ -21,7 +21,7 @@ packages = [{include = "lisa.py"}] [tool.poetry.dependencies] python = "^3.7" pytest = "^6.1.2" -pytest-playbook = {path = "../pytest-playbook", develop = true} +pytest-playbook = "^0.1.0" pytest-xdist = "^2.1.0" schema = "0.7.2" diff --git a/pytest-target/poetry.lock b/pytest-target/poetry.lock index 2412ffdd21..0990516cb3 100644 --- a/pytest-target/poetry.lock +++ b/pytest-target/poetry.lock @@ -250,18 +250,13 @@ version = "0.1.0" description = "Pytest plugin for reading playbooks." category = "main" optional = false -python-versions = "^3.7" -develop = true +python-versions = ">=3.7,<4.0" [package.dependencies] -pytest = "^6.1.2" -PyYAML = "^5.3.1" +pytest = ">=6.1.2,<7.0.0" +PyYAML = ">=5.3.1,<6.0.0" schema = "0.7.2" -[package.source] -type = "directory" -url = "../pytest-playbook" - [[package]] name = "pyyaml" version = "5.3.1" @@ -334,7 +329,7 @@ testing = ["pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "db5b497c6697ab98df1b46727bd1d10fd3a6853b142a9fc9c11a626032d2ebd5" +content-hash = "2cfd10a84e947de5a1483d7ae41f1213f0bff74c1318688aafaa2751aa1313c6" [metadata.files] atomicwrites = [ @@ -485,7 +480,10 @@ pytest = [ {file = "pytest-6.2.1-py3-none-any.whl", hash = "sha256:1969f797a1a0dbd8ccf0fecc80262312729afea9c17f1d70ebf85c5e76c6f7c8"}, {file = "pytest-6.2.1.tar.gz", hash = "sha256:66e419b1899bc27346cb2c993e12c5e5e8daba9073c1fbce33b9807abc95c306"}, ] -pytest-playbook = [] +pytest-playbook = [ + {file = "pytest-playbook-0.1.0.tar.gz", hash = "sha256:c9ce7446e9301646ee3fdb74a3d7cbdee4a83e775126b9062d0cbd6178ef36ec"}, + {file = "pytest_playbook-0.1.0-py3-none-any.whl", hash = "sha256:c2fb714d4347d89d7b031822eed279c956275f41f34927cb94685f04670bd01d"}, +] pyyaml = [ {file = "PyYAML-5.3.1-cp27-cp27m-win32.whl", hash = "sha256:74809a57b329d6cc0fdccee6318f44b9b8649961fa73144a98735b0aaf029f1f"}, {file = "PyYAML-5.3.1-cp27-cp27m-win_amd64.whl", hash = "sha256:240097ff019d7c70a4922b6869d8a86407758333f02203e0fc6ff79c5dcede76"}, diff --git a/pytest-target/pyproject.toml b/pytest-target/pyproject.toml index 34cf2809b4..b971971540 100644 --- a/pytest-target/pyproject.toml +++ b/pytest-target/pyproject.toml @@ -25,7 +25,7 @@ fabric = "^2.5.0" filelock = "^3.0.12" invoke = "^1.4.1" tenacity = "^6.2.0" -pytest-playbook = {path = "../pytest-playbook", develop = true} +pytest-playbook = "^0.1.0" [tool.poetry.dev-dependencies] From b9a67b9f09877b7601079175becb044def3849ba Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Wed, 20 Jan 2021 18:10:32 -0800 Subject: [PATCH 193/194] Document steps to publish packages --- CONTRIBUTING.md | 18 ++++++++++++++++++ pytest-playbook/pyproject.toml | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d30ffbfce4..a079aff1bc 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -383,6 +383,24 @@ check all the links), use: sphinx-build -b linkcheck . _build ``` +## Publishing Packages + +Setup [PyPI](https://pypi.org/) with Poetry’s +[repositories](https://python-poetry.org/docs/repositories/): + +```bash +poetry config pypi-token.pypi my-token +``` + +Then for each package, build and publish it! Don’t forget to increment the +version number afterwards. You should carefully test the package on the [Test +PyPI](https://test.pypi.org/) instance first, as you cannot overwrite uploaded +packages. You can also bump the patch version if you made a small mistake. + +```bash +poetry publish --build +``` + ## Future Sections Just a collection of reminders for the author to expand on later. diff --git a/pytest-playbook/pyproject.toml b/pytest-playbook/pyproject.toml index 29022e6883..45dddc9ea0 100644 --- a/pytest-playbook/pyproject.toml +++ b/pytest-playbook/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pytest-playbook" -version = "0.1.0" +version = "0.1.1" description = "Pytest plugin for reading playbooks." license = "MIT" authors = ["Andrew Schwartzmeyer "] From 7bbdbdaa1ace30e3c61c4eb139b244576f6aabaf Mon Sep 17 00:00:00 2001 From: Andrew Schwartzmeyer Date: Thu, 21 Jan 2021 10:08:36 -0800 Subject: [PATCH 194/194] Add smoke test code as example to usage document --- USAGE.rst | 102 ++++++++++++++++++++++++++++++++++++- testsuites/test_smoke_b.py | 7 ++- 2 files changed, 106 insertions(+), 3 deletions(-) diff --git a/USAGE.rst b/USAGE.rst index 69aa324295..eef66afe70 100644 --- a/USAGE.rst +++ b/USAGE.rst @@ -125,7 +125,107 @@ tests and will show up in test results. The ``criteria`` key can be used to select a tests instead of using Pytest’s CLI test selection interface. In this case we’re selecting all tests from the module (Python file) named ``test_smoke_b``, one of -the examples of an Azure VM smoke test. +the examples of an Azure VM smoke test, and it looks like this: + +.. code:: python + + from __future__ import annotations # For type checking. + + import typing + + if typing.TYPE_CHECKING: + from target import AzureCLI + from _pytest.logging import LogCaptureFixture + from pathlib import Path + + import logging + import socket + import time + + from invoke.runners import CommandTimedOut, UnexpectedExit # type: ignore + from paramiko import SSHException # type: ignore + + from lisa import LISA + + + @LISA(platform="Azure", category="Functional", area="deploy", priority=0) + def test_smoke(target: AzureCLI, caplog: LogCaptureFixture, tmp_path: Path) -> None: + """Check that an Azure Linux VM can be deployed and is responsive. + + This example uses exactly one function for the entire test, which + means we have to catch failures that don't fail the test, and + instead emit warnings. It works, and it's closer to how LISAv2 + would have implemented it, but it's less Pythonic. For a more + "modern" example, see `test_smoke_a.py`. + + 1. Deploy the VM (via `target` fixture). + 2. Ping the VM. + 3. Connect to the VM via SSH. + 4. Attempt to reboot via SSH, otherwise use the platform. + 5. Fetch the serial console logs AKA boot diagnostics. + + SSH failures DO NOT fail this test. + + """ + # Capture INFO and above logs for this test. + caplog.set_level(logging.INFO) + + logging.info("Pinging before reboot...") + ping1 = target.ping() + + ssh_errors = (TimeoutError, CommandTimedOut, SSHException, socket.error) + + try: + logging.info("SSHing before reboot...") + target.conn.open() + except ssh_errors as e: + logging.warning(f"SSH before reboot failed: '{e}'") + + reboot_exit = 0 + try: + logging.info("Rebooting...") + # If this succeeds, we should expect the exit code to be -1 + reboot_exit = target.conn.sudo("reboot", timeout=5).exited + except ssh_errors as e: + logging.warning(f"SSH failed, using platform to reboot: '{e}'") + target.platform_restart() + except UnexpectedExit: + # TODO: How do we differentiate reboot working and the SSH + # connection disconnecting for other reasons? + if reboot_exit != -1: + logging.warning("While SSH worked, 'reboot' command failed") + + # TODO: We should check something more concrete here instead of + # sleeping an arbitrary amount of time. + logging.info("Sleeping for 10 seconds after reboot...") + time.sleep(10) + + logging.info("Pinging after reboot...") + ping2 = target.ping() + + try: + logging.info("SSHing after reboot...") + target.conn.open() + except ssh_errors as e: + logging.warning(f"SSH after reboot failed: '{e}'") + + logging.info("Retrieving boot diagnostics...") + path = tmp_path / "diagnostics.txt" + try: + # NOTE: It’s actually more interesting to emit the downloaded + # boot diagnostics to `stdout` as they’re then captured in the + # HTML report, but this is to demo using `tmp_path`. + diagnostics = target.get_boot_diagnostics(hide=True) + path.write_text(diagnostics.stdout) + except UnexpectedExit: + logging.warning("Retrieving boot diagnostics failed.") + else: + logging.info(f"See '{path}' for boot diagnostics.") + + # NOTE: The test criteria is to fail only if ping fails. + assert ping1.ok, f"Pinging {target.host} before reboot failed" + assert ping2.ok, f"Pinging {target.host} after reboot failed" + Enable Azure ~~~~~~~~~~~~ diff --git a/testsuites/test_smoke_b.py b/testsuites/test_smoke_b.py index dd72bfc34a..7a6c5220c4 100644 --- a/testsuites/test_smoke_b.py +++ b/testsuites/test_smoke_b.py @@ -1,15 +1,15 @@ -from __future__ import annotations +from __future__ import annotations # For type checking. import typing if typing.TYPE_CHECKING: from target import AzureCLI from _pytest.logging import LogCaptureFixture + from pathlib import Path import logging import socket import time -from pathlib import Path from invoke.runners import CommandTimedOut, UnexpectedExit # type: ignore from paramiko import SSHException # type: ignore @@ -81,6 +81,9 @@ def test_smoke(target: AzureCLI, caplog: LogCaptureFixture, tmp_path: Path) -> N logging.info("Retrieving boot diagnostics...") path = tmp_path / "diagnostics.txt" try: + # NOTE: It’s actually more interesting to emit the downloaded + # boot diagnostics to `stdout` as they’re then captured in the + # HTML report, but this is to demo using `tmp_path`. diagnostics = target.get_boot_diagnostics(hide=True) path.write_text(diagnostics.stdout) except UnexpectedExit: