diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 3f2145828c..4df959ffa9 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -75,9 +75,20 @@ jobs: uses: actions/checkout@v4 with: # a test uses a submodule, and pants needs access to it to calculate deps. - submodules: 'true' + submodules: 'recursive' + # sadly, the submodule will only have fetch-depth=1, which is what we want + # for st2.git, but not for the submodules. We still want actions/checkout + # to do the initial checkout, however, so that it adds auth for fetching + # in the submodule. + + - name: Fetch repository submodules + run: | + git submodule status + git submodule foreach 'git fetch --all --tags' + git submodule foreach 'git tag' - name: 'Set up Python (${{ matrix.python-version }})' + id: python uses: actions/setup-python@v5 with: python-version: '${{ matrix.python-version }}' @@ -92,16 +103,42 @@ jobs: # To ignore a bad cache, bump the cache* integer. gha-cache-key: cache0-py${{ matrix.python-version }} - - name: Test - # We do not support running pytest everywhere yet. When we do it will be simply: - # pants test :: - # Until then, we need to manually adjust this command line to test what we can. + - name: Test pants-plugins + if: ${{ matrix.python-version-short == '3.9' }} run: | - pants test pylint_plugins/:: pants-plugins/:: + pants test pants-plugins/:: + + # We do not support running pytest everywhere yet. When we do it will be simply: + # pants test :: + # Until then, we need to manually adjust this command line to test what we can. + # So far, this includes unit, pack, and pants-plugins tests. + # TODO: run integration tests + + - name: Unit Tests + env: + # Github Actions uses the 'runner' user, so use that instead of stanley. + ST2TESTS_SYSTEM_USER: 'runner' + run: > + pants + --python-bootstrap-search-path=[] + --python-bootstrap-search-path=${{ steps.python.outputs.python-path }} + --tag=unit + test '::' + + - name: Pack Tests + env: + # Github Actions uses the 'runner' user, so use that instead of stanley. + ST2TESTS_SYSTEM_USER: 'runner' + run: > + pants + --python-bootstrap-search-path=[] + --python-bootstrap-search-path=${{ steps.python.outputs.python-path }} + --tag=pack + test '::' - name: Upload pants log uses: actions/upload-artifact@v4 with: name: pants-log-py${{ matrix.python-version }} path: .pants.d/pants.log - if: always() # We want the log even on failures. + if: ${{ always() }} # We want the log even on failures. diff --git a/BUILD b/BUILD index f8b1f03a4d..25c87e41c7 100644 --- a/BUILD +++ b/BUILD @@ -27,6 +27,36 @@ python_requirements( "st2auth/st2auth/backends/constants.py", ] ), + # make sure anything that uses st2-rbac-backend gets its deps + "st2-rbac-backend": dict( + dependencies=[ + # alphabetical order + "st2common/st2common/config.py", + "st2common/st2common/constants/keyvalue.py", + "st2common/st2common/constants/triggers.py", + "st2common/st2common/content/loader.py", + "st2common/st2common/exceptions/db.py", + "st2common/st2common/exceptions/rbac.py", + "st2common/st2common/log.py", + "st2common/st2common/models/api/rbac.py", + "st2common/st2common/models/db/action.py", + "st2common/st2common/models/db/auth.py", + "st2common/st2common/models/db/pack.py", + "st2common/st2common/models/db/rbac.py", + "st2common/st2common/models/db/webhook.py", + "st2common/st2common/models/system/common.py", + "st2common/st2common/persistence/auth.py", + "st2common/st2common/persistence/execution.py", + "st2common/st2common/persistence/rbac.py", + "st2common/st2common/rbac/backends/__init__.py", + "st2common/st2common/rbac/backends/base.py", + "st2common/st2common/rbac/types.py", + "st2common/st2common/script_setup.py", + "st2common/st2common/util/action_db.py", + "st2common/st2common/util/misc.py", + "st2common/st2common/util/uid.py", + ] + ), }, ) @@ -38,6 +68,13 @@ target( ], ) +target( + name="rbac_backends", + dependencies=[ + "//:reqs#st2-rbac-backend", + ], +) + python_test_utils( name="test_utils", skip_pylint=True, @@ -51,3 +88,31 @@ file( shell_sources( name="root", ) + +file( + name="logs_directory", + source="logs/.gitignore", +) + +files( + name="gitmodules", + sources=[ + ".gitmodules", + "**/.git", + ], +) + +shell_command( + name="capture_git_modules", + environment="in_repo_workspace", + command="cp -r .git/modules {chroot}/.git", + tools=["cp"], + # execution_dependencies allows pants to invalidate the output + # of this command if the .gitmodules file changes (for example: + # if a submodule gets updated to a different commit). + # Theoretically, nothing else should modify .git/modules/. + execution_dependencies=[":gitmodules"], + output_dependencies=[":gitmodules"], + output_directories=[".git/modules"], + workdir="/", +) diff --git a/BUILD.environment b/BUILD.environment new file mode 100644 index 0000000000..5c1f26cdd3 --- /dev/null +++ b/BUILD.environment @@ -0,0 +1,23 @@ +# Everything listed in pants.toml [evironments-preview.names] should be defined here. +# Relevant docs: +# - https://www.pantsbuild.org/stable/docs/using-pants/environments +# - https://www.pantsbuild.org/stable/reference/targets/experimental_workspace_environment +# - https://www.pantsbuild.org/stable/reference/targets/local_environment +# - https://www.pantsbuild.org/stable/reference/targets/docker_environment + +# This file MUST NOT use any macros. + +experimental_workspace_environment( + name="in_repo_workspace", + description=( + """ + This allows shell_command and similar to in the repo, instead of in a sandbox. + Only use this environment for commands or goals that are idempotent. + Ideally, such commands do NOT change anything in the repo. + + If you need to capture output, note that output gets captured from a temporary + sandbox, not from the repo root. So, you may need to copy output files into + the sandbox with something like `cp path/to/file {chroot}/path/to/file`. + """ + ), +) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8fbcebd310..b3c828b535 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -33,7 +33,7 @@ Added * Continue introducing `pants `_ to improve DX (Developer Experience) working on StackStorm, improve our security posture, and improve CI reliability thanks in part to pants' use of PEX lockfiles. This is not a user-facing addition. - #6118 #6141 #6133 #6120 #6181 #6183 #6200 #6237 + #6118 #6141 #6133 #6120 #6181 #6183 #6200 #6237 #6229 Contributed by @cognifloyd * Build of ST2 EL9 packages #6153 Contributed by @amanda11 diff --git a/contrib/chatops/BUILD b/contrib/chatops/BUILD index 1a74d30186..888be3a426 100644 --- a/contrib/chatops/BUILD +++ b/contrib/chatops/BUILD @@ -1,3 +1,5 @@ +__defaults__(all=dict(inject_pack_python_path=True)) + pack_metadata( name="metadata", ) diff --git a/contrib/chatops/actions/BUILD b/contrib/chatops/actions/BUILD index db46e8d6c9..4a1bef88f4 100644 --- a/contrib/chatops/actions/BUILD +++ b/contrib/chatops/actions/BUILD @@ -1 +1,7 @@ -python_sources() +python_sources( + overrides={ + "format_execution_result.py": dict( + dependencies=["./templates"], + ), + }, +) diff --git a/contrib/chatops/actions/templates/BUILD b/contrib/chatops/actions/templates/BUILD new file mode 100644 index 0000000000..96d5a456eb --- /dev/null +++ b/contrib/chatops/actions/templates/BUILD @@ -0,0 +1,3 @@ +resources( + sources=["*.j2"], +) diff --git a/contrib/chatops/tests/BUILD b/contrib/chatops/tests/BUILD index cd3fa380ae..ead8561daa 100644 --- a/contrib/chatops/tests/BUILD +++ b/contrib/chatops/tests/BUILD @@ -1,6 +1,11 @@ # tests can only be dependencies of other tests in this directory __dependents_rules__(("*", "/**", "!*")) +__defaults__( + {python_test: dict(tags=["pack"])}, + extend=True, +) + files( name="fixtures", sources=["fixtures/*.json"], @@ -8,6 +13,9 @@ files( python_tests( name="tests", - dependencies=[":fixtures"], + dependencies=[ + ":fixtures", + "contrib/chatops:metadata", + ], skip_pylint=True, ) diff --git a/contrib/core/BUILD b/contrib/core/BUILD index 7db2dc9d25..9df7a372c9 100644 --- a/contrib/core/BUILD +++ b/contrib/core/BUILD @@ -1,3 +1,5 @@ +__defaults__(all=dict(inject_pack_python_path=True)) + pack_metadata( name="metadata", ) @@ -8,5 +10,9 @@ python_requirements( ) python_sources( - dependencies=[":metadata"], + dependencies=[ + ":metadata", + "./actions", + "./actions/send_mail:send_mail_resources", + ], ) diff --git a/contrib/core/actions/send_mail/BUILD b/contrib/core/actions/send_mail/BUILD index f27e7c10ec..94280e6e49 100644 --- a/contrib/core/actions/send_mail/BUILD +++ b/contrib/core/actions/send_mail/BUILD @@ -1,4 +1,5 @@ -shell_source( - source="send_mail", +st2_shell_sources_and_resources( + name="send_mail", + sources=["send_mail"], skip_shellcheck=True, ) diff --git a/contrib/core/tests/BUILD b/contrib/core/tests/BUILD index 6f09c14528..144960edf3 100644 --- a/contrib/core/tests/BUILD +++ b/contrib/core/tests/BUILD @@ -1,8 +1,14 @@ # tests can only be dependencies of other tests in this directory __dependents_rules__(("*", "/**", "!*")) +__defaults__( + {python_test: dict(tags=["pack"])}, + extend=True, +) + python_tests( skip_pylint=True, + # st2_pack_dir="..", # directory containing pack.yaml overrides={ "test_action_sendmail.py": dict( dependencies=[ @@ -11,6 +17,7 @@ python_tests( # Use contrib/core as the canonical copy. "contrib/core:reqs#mail-parser", ], + uses=["mongo"], ), }, ) diff --git a/contrib/debug/BUILD b/contrib/debug/BUILD index 1a74d30186..888be3a426 100644 --- a/contrib/debug/BUILD +++ b/contrib/debug/BUILD @@ -1,3 +1,5 @@ +__defaults__(all=dict(inject_pack_python_path=True)) + pack_metadata( name="metadata", ) diff --git a/contrib/default/BUILD b/contrib/default/BUILD index 1a74d30186..888be3a426 100644 --- a/contrib/default/BUILD +++ b/contrib/default/BUILD @@ -1,3 +1,5 @@ +__defaults__(all=dict(inject_pack_python_path=True)) + pack_metadata( name="metadata", ) diff --git a/contrib/examples/BUILD b/contrib/examples/BUILD index de3b866405..ab10cd1c85 100644 --- a/contrib/examples/BUILD +++ b/contrib/examples/BUILD @@ -1,3 +1,5 @@ +__defaults__(all=dict(inject_pack_python_path=True)) + pack_metadata( name="metadata", ) diff --git a/contrib/examples/actions/pythonactions/isprime.py b/contrib/examples/actions/pythonactions/isprime.py index 5116831a37..65294a5619 100644 --- a/contrib/examples/actions/pythonactions/isprime.py +++ b/contrib/examples/actions/pythonactions/isprime.py @@ -15,7 +15,8 @@ import math -from environ import get_environ +# TODO: extend pants and pants-plugins/pack_metadata to add lib dirs extra_sys_path for pylint +from environ import get_environ # pylint: disable=E0401 from st2common.runners.base_action import Action diff --git a/contrib/examples/tests/BUILD b/contrib/examples/tests/BUILD index 0f0af81da5..25a2e7cc4b 100644 --- a/contrib/examples/tests/BUILD +++ b/contrib/examples/tests/BUILD @@ -1,6 +1,11 @@ # tests can only be dependencies of other tests in this directory __dependents_rules__(("*", "/**", "!*")) +__defaults__( + {python_test: dict(tags=["pack"])}, + extend=True, +) + python_tests( skip_pylint=True, ) diff --git a/contrib/hello_st2/BUILD b/contrib/hello_st2/BUILD index 1a74d30186..888be3a426 100644 --- a/contrib/hello_st2/BUILD +++ b/contrib/hello_st2/BUILD @@ -1,3 +1,5 @@ +__defaults__(all=dict(inject_pack_python_path=True)) + pack_metadata( name="metadata", ) diff --git a/contrib/linux/BUILD b/contrib/linux/BUILD index 8a73ff391a..201435eecc 100644 --- a/contrib/linux/BUILD +++ b/contrib/linux/BUILD @@ -1,3 +1,5 @@ +__defaults__(all=dict(inject_pack_python_path=True)) + pack_metadata( name="metadata", ) diff --git a/contrib/linux/tests/BUILD b/contrib/linux/tests/BUILD index 0f0af81da5..25a2e7cc4b 100644 --- a/contrib/linux/tests/BUILD +++ b/contrib/linux/tests/BUILD @@ -1,6 +1,11 @@ # tests can only be dependencies of other tests in this directory __dependents_rules__(("*", "/**", "!*")) +__defaults__( + {python_test: dict(tags=["pack"])}, + extend=True, +) + python_tests( skip_pylint=True, ) diff --git a/contrib/packs/BUILD b/contrib/packs/BUILD index 1a74d30186..888be3a426 100644 --- a/contrib/packs/BUILD +++ b/contrib/packs/BUILD @@ -1,3 +1,5 @@ +__defaults__(all=dict(inject_pack_python_path=True)) + pack_metadata( name="metadata", ) diff --git a/contrib/packs/tests/BUILD b/contrib/packs/tests/BUILD index 0f0af81da5..c8265214ca 100644 --- a/contrib/packs/tests/BUILD +++ b/contrib/packs/tests/BUILD @@ -1,6 +1,29 @@ # tests can only be dependencies of other tests in this directory __dependents_rules__(("*", "/**", "!*")) +__defaults__( + {python_test: dict(tags=["pack"])}, + extend=True, +) + python_tests( skip_pylint=True, + overrides={ + "test_action_aliases.py": dict( + dependencies=[ + # test needs the pack and aliases metadata + "contrib/packs:metadata", + ], + ), + "test_action_unload.py": dict( + stevedore_namespaces=[ + "st2common.metrics.driver", + ], + entry_point_dependencies={ + "contrib/runners/http_runner": ["st2common.runners.runner"], + "contrib/runners/local_runner": ["st2common.runners.runner"], + "contrib/runners/python_runner": ["st2common.runners.runner"], + }, + ), + }, ) diff --git a/contrib/runners/action_chain_runner/tests/BUILD b/contrib/runners/action_chain_runner/tests/BUILD index 3280583e0c..d19247d547 100644 --- a/contrib/runners/action_chain_runner/tests/BUILD +++ b/contrib/runners/action_chain_runner/tests/BUILD @@ -1,6 +1,9 @@ __defaults__( all=dict( skip_pylint=True, + entry_point_dependencies={ + "contrib/runners/action_chain_runner": ["st2common.runners.runner"], + }, ) ) diff --git a/contrib/runners/action_chain_runner/tests/unit/BUILD b/contrib/runners/action_chain_runner/tests/unit/BUILD index 9a24dba70a..47c65e48d3 100644 --- a/contrib/runners/action_chain_runner/tests/unit/BUILD +++ b/contrib/runners/action_chain_runner/tests/unit/BUILD @@ -5,4 +5,11 @@ __defaults__( python_tests( name="tests", + stevedore_namespaces=[ + "st2common.rbac.backend", + "st2common.metrics.driver", + # the core pack uses all runners. + "st2common.runners.runner", + ], + uses=["mongo"], ) diff --git a/contrib/runners/announcement_runner/tests/BUILD b/contrib/runners/announcement_runner/tests/BUILD index 3280583e0c..850b4aa71a 100644 --- a/contrib/runners/announcement_runner/tests/BUILD +++ b/contrib/runners/announcement_runner/tests/BUILD @@ -1,6 +1,9 @@ __defaults__( all=dict( skip_pylint=True, + entry_point_dependencies={ + "contrib/runners/announcement_runner": ["st2common.runners.runner"], + }, ) ) diff --git a/contrib/runners/http_runner/tests/BUILD b/contrib/runners/http_runner/tests/BUILD index 3280583e0c..eccc29b3ee 100644 --- a/contrib/runners/http_runner/tests/BUILD +++ b/contrib/runners/http_runner/tests/BUILD @@ -1,6 +1,9 @@ __defaults__( all=dict( skip_pylint=True, + entry_point_dependencies={ + "contrib/runners/http_runner": ["st2common.runners.runner"], + }, ) ) diff --git a/contrib/runners/inquirer_runner/tests/BUILD b/contrib/runners/inquirer_runner/tests/BUILD index 3280583e0c..36b90b1e2a 100644 --- a/contrib/runners/inquirer_runner/tests/BUILD +++ b/contrib/runners/inquirer_runner/tests/BUILD @@ -1,6 +1,9 @@ __defaults__( all=dict( skip_pylint=True, + entry_point_dependencies={ + "contrib/runners/inquirer_runner": ["st2common.runners.runner"], + }, ) ) diff --git a/contrib/runners/local_runner/tests/BUILD b/contrib/runners/local_runner/tests/BUILD index 3280583e0c..d692891e08 100644 --- a/contrib/runners/local_runner/tests/BUILD +++ b/contrib/runners/local_runner/tests/BUILD @@ -1,6 +1,9 @@ __defaults__( all=dict( skip_pylint=True, + entry_point_dependencies={ + "contrib/runners/local_runner": ["st2common.runners.runner"], + }, ) ) diff --git a/contrib/runners/local_runner/tests/integration/test_localrunner.py b/contrib/runners/local_runner/tests/integration/test_localrunner.py index 1ca4f4275f..fca7fccb90 100644 --- a/contrib/runners/local_runner/tests/integration/test_localrunner.py +++ b/contrib/runners/local_runner/tests/integration/test_localrunner.py @@ -297,6 +297,7 @@ def test_action_stdout_and_stderr_is_stored_in_the_db_short_running_action( self.assertEqual(output_dbs[db_index_1].data, mock_stderr[0]) self.assertEqual(output_dbs[db_index_2].data, mock_stderr[1]) + # FIXME: This test assumes passwordless sudo (so sudo won't capture stdin) def test_shell_command_sudo_password_is_passed_to_sudo_binary(self): # Verify that sudo password is correctly passed to sudo binary via stdin models = self.fixtures_loader.load_models( @@ -355,6 +356,7 @@ def test_shell_command_sudo_password_is_passed_to_sudo_binary(self): self.assertEqual(index, len(sudo_passwords)) + # FIXME: This test assumes passwordless sudo (so sudo won't capture stdin) def test_shell_command_invalid_stdout_password(self): # Simulate message printed to stderr by sudo when invalid sudo password is provided models = self.fixtures_loader.load_models( diff --git a/contrib/runners/noop_runner/tests/BUILD b/contrib/runners/noop_runner/tests/BUILD index 3280583e0c..208d20406f 100644 --- a/contrib/runners/noop_runner/tests/BUILD +++ b/contrib/runners/noop_runner/tests/BUILD @@ -1,6 +1,9 @@ __defaults__( all=dict( skip_pylint=True, + entry_point_dependencies={ + "contrib/runners/noop_runner": ["st2common.runners.runner"], + }, ) ) diff --git a/contrib/runners/orquesta_runner/tests/BUILD b/contrib/runners/orquesta_runner/tests/BUILD index 3280583e0c..ea19a75243 100644 --- a/contrib/runners/orquesta_runner/tests/BUILD +++ b/contrib/runners/orquesta_runner/tests/BUILD @@ -1,6 +1,12 @@ __defaults__( all=dict( skip_pylint=True, + entry_point_dependencies={ + "contrib/runners/orquesta_runner": [ + "st2common.runners.runner", + "orquesta.expressions.functions", + ], + }, ) ) diff --git a/contrib/runners/orquesta_runner/tests/unit/BUILD b/contrib/runners/orquesta_runner/tests/unit/BUILD index 1daa501cd5..c0435fb3db 100644 --- a/contrib/runners/orquesta_runner/tests/unit/BUILD +++ b/contrib/runners/orquesta_runner/tests/unit/BUILD @@ -5,6 +5,18 @@ __defaults__( python_tests( name="tests", + stevedore_namespaces=[ + "st2common.metrics.driver", + "st2common.rbac.backend", + # the core pack uses all runners. + "st2common.runners.runner", + "orquesta.expressions.functions", + ], + overrides={ + "test_policies.py": dict( + dependencies=["st2actions/st2actions/policies/retry.py"] + ) + }, ) python_test_utils( diff --git a/contrib/runners/orquesta_runner/tests/unit/test_basic.py b/contrib/runners/orquesta_runner/tests/unit/test_basic.py index 7c9351423a..9b84c94b3a 100644 --- a/contrib/runners/orquesta_runner/tests/unit/test_basic.py +++ b/contrib/runners/orquesta_runner/tests/unit/test_basic.py @@ -21,6 +21,7 @@ import six from orquesta import statuses as wf_statuses +from oslo_config import cfg import st2tests @@ -108,7 +109,7 @@ def get_runner_class(cls, runner_name): runners_utils, "invoke_post_run", mock.MagicMock(return_value=None) ) def test_run_workflow(self): - username = "stanley" + username = cfg.CONF.system_user.user wf_meta = base.get_wf_fixture_meta_data(TEST_PACK_PATH, "sequential.yaml") wf_input = {"who": "Thanos"} lv_ac_db = lv_db_models.LiveActionDB( diff --git a/contrib/runners/orquesta_runner/tests/unit/test_context.py b/contrib/runners/orquesta_runner/tests/unit/test_context.py index d9e726d9a1..983614a041 100644 --- a/contrib/runners/orquesta_runner/tests/unit/test_context.py +++ b/contrib/runners/orquesta_runner/tests/unit/test_context.py @@ -19,6 +19,7 @@ import mock from orquesta import statuses as wf_statuses +from oslo_config import cfg import st2tests @@ -125,7 +126,7 @@ def test_runtime_context(self): expected_st2_ctx = { "action_execution_id": str(ac_ex_db.id), "api_url": "http://127.0.0.1/v1", - "user": "stanley", + "user": cfg.CONF.system_user.user, "pack": "orquesta_tests", "action": "orquesta_tests.runtime-context", "runner": "orquesta", @@ -208,9 +209,10 @@ def test_action_context_sys_user(self): self.assertEqual(wf_ex_db.status, wf_statuses.SUCCEEDED) self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) + user = cfg.CONF.system_user.user # Check result. expected_result = { - "output": {"msg": "stanley, All your base are belong to us!"} + "output": {"msg": f"{user}, All your base are belong to us!"} } self.assertDictEqual(lv_ac_db.result, expected_result) diff --git a/contrib/runners/orquesta_runner/tests/unit/test_error_handling.py b/contrib/runners/orquesta_runner/tests/unit/test_error_handling.py index 9a4dd1cd5b..9aafac018f 100644 --- a/contrib/runners/orquesta_runner/tests/unit/test_error_handling.py +++ b/contrib/runners/orquesta_runner/tests/unit/test_error_handling.py @@ -18,6 +18,7 @@ import mock from orquesta import statuses as wf_statuses +from oslo_config import cfg import st2tests @@ -954,7 +955,7 @@ def test_fail_manually_with_recovery_failure(self): mock.MagicMock(side_effect=[RUNNER_RESULT_FAILED]), ) def test_include_result_to_error_log(self): - username = "stanley" + username = cfg.CONF.system_user.user wf_meta = base.get_wf_fixture_meta_data(TEST_PACK_PATH, "sequential.yaml") wf_input = {"who": "Thanos"} lv_ac_db = lv_db_models.LiveActionDB( diff --git a/contrib/runners/python_runner/tests/BUILD b/contrib/runners/python_runner/tests/BUILD index 3280583e0c..dfaf857926 100644 --- a/contrib/runners/python_runner/tests/BUILD +++ b/contrib/runners/python_runner/tests/BUILD @@ -1,6 +1,9 @@ __defaults__( all=dict( skip_pylint=True, + entry_point_dependencies={ + "contrib/runners/python_runner": ["st2common.runners.runner"], + }, ) ) diff --git a/contrib/runners/python_runner/tests/integration/BUILD b/contrib/runners/python_runner/tests/integration/BUILD index 2d782aaea0..5fdf71eed3 100644 --- a/contrib/runners/python_runner/tests/integration/BUILD +++ b/contrib/runners/python_runner/tests/integration/BUILD @@ -5,4 +5,9 @@ __defaults__( python_tests( name="tests", + overrides={ + "test_python_action_process_wrapper.py": dict( + dependencies=["contrib/examples/actions/noop.py"] + ) + }, ) diff --git a/contrib/runners/python_runner/tests/unit/BUILD b/contrib/runners/python_runner/tests/unit/BUILD index 9a24dba70a..656e54f328 100644 --- a/contrib/runners/python_runner/tests/unit/BUILD +++ b/contrib/runners/python_runner/tests/unit/BUILD @@ -5,4 +5,22 @@ __defaults__( python_tests( name="tests", + overrides={ + "test_output_schema.py": dict( + dependencies=[ + "st2tests/st2tests/resources/packs/pythonactions/actions/pascal_row.py", + "//:capture_git_modules", + ], + ), + "test_pythonrunner.py": dict( + dependencies=[ + "st2tests/st2tests/resources/packs/pythonactions/actions", + "//:capture_git_modules", + ], + stevedore_namespaces=[ + "st2common.metrics.driver", + "st2common.rbac.backend", + ], + ), + }, ) diff --git a/contrib/runners/python_runner/tests/unit/test_output_schema.py b/contrib/runners/python_runner/tests/unit/test_output_schema.py index e997d8b60b..c5fd0d00e2 100644 --- a/contrib/runners/python_runner/tests/unit/test_output_schema.py +++ b/contrib/runners/python_runner/tests/unit/test_output_schema.py @@ -66,7 +66,7 @@ def setUpClass(cls): assert_submodules_are_checked_out() def test_adherence_to_output_schema(self): - config = self.loader(os.path.join(BASE_DIR, "../../runner.yaml")) + config = self.loader(os.path.join(BASE_DIR, "../../python_runner/runner.yaml")) runner = self._get_mock_runner_obj() runner.entry_point = PASCAL_ROW_ACTION_PATH runner.pre_run() diff --git a/contrib/runners/python_runner/tests/unit/test_pythonrunner.py b/contrib/runners/python_runner/tests/unit/test_pythonrunner.py index a178bd20ff..22d9e14b36 100644 --- a/contrib/runners/python_runner/tests/unit/test_pythonrunner.py +++ b/contrib/runners/python_runner/tests/unit/test_pythonrunner.py @@ -36,13 +36,15 @@ from st2common.constants.action import LIVEACTION_STATUS_TIMED_OUT from st2common.constants.action import MAX_PARAM_LENGTH from st2common.constants.pack import COMMON_LIB_DIR -from st2common.constants.pack import SYSTEM_PACK_NAME +from st2common.constants.pack import SYSTEM_PACK_NAMES from st2common.persistence.execution import ActionExecutionOutput from python_runner.python_action_wrapper import PythonActionWrapper from st2tests.base import RunnerTestCase from st2tests.base import CleanDbTestCase from st2tests.base import blocking_eventlet_spawn from st2tests.base import make_mock_stream_readline +from st2tests.fixtures.packs.dummy_pack_1.fixture import PACK_NAME as DUMMY_PACK_1 +from st2tests.fixtures.packs.dummy_pack_5.fixture import PACK_NAME as DUMMY_PACK_5 from st2tests.fixtures.packs.dummy_pack_9.fixture import PACK_PATH as DUMMY_PACK_9_PATH from st2tests.fixtures.packs.test_content_version_fixture.fixture import ( PACK_NAME as TEST_CONTENT_VERSION, @@ -100,6 +102,10 @@ MOCK_EXECUTION.id = "598dbf0c0640fd54bffc688b" +# Use DUMMY_PACK_1 instead of depending on everything in the core (SYSTEM_PACK_NAME) pack. +@mock.patch( + "st2common.util.sandboxing.SYSTEM_PACK_NAMES", [DUMMY_PACK_1, *SYSTEM_PACK_NAMES] +) @mock.patch("python_runner.python_runner.sys", mock_sys) class PythonRunnerTestCase(RunnerTestCase, CleanDbTestCase): register_packs = True @@ -229,12 +235,10 @@ def test_simple_action_no_status_backward_compatibility(self): self.assertEqual(output["result"], [1, 2]) def test_simple_action_config_value_provided_overriden_in_datastore(self): - pack = "dummy_pack_5" user = "joe" # No values provided in the datastore - runner = self._get_mock_runner_obj_from_container(pack=pack, user=user) - + runner = self._get_mock_runner_obj_from_container(pack=DUMMY_PACK_5, user=user) self.assertEqual(runner._config["api_key"], "some_api_key") # static value self.assertEqual(runner._config["regions"], ["us-west-1"]) # static value self.assertEqual(runner._config["api_secret"], None) @@ -242,19 +246,19 @@ def test_simple_action_config_value_provided_overriden_in_datastore(self): # api_secret overriden in the datastore (user scoped value) config_service.set_datastore_value_for_config_key( - pack_name="dummy_pack_5", + pack_name=DUMMY_PACK_5, key_name="api_secret", - user="joe", + user=user, value="foosecret", secret=True, ) # private_key_path overriden in the datastore (global / non-user scoped value) config_service.set_datastore_value_for_config_key( - pack_name="dummy_pack_5", key_name="private_key_path", value="foopath" + pack_name=DUMMY_PACK_5, key_name="private_key_path", value="foopath" ) - runner = self._get_mock_runner_obj_from_container(pack=pack, user=user) + runner = self._get_mock_runner_obj_from_container(pack=DUMMY_PACK_5, user=user) self.assertEqual(runner._config["api_key"], "some_api_key") # static value self.assertEqual(runner._config["regions"], ["us-west-1"]) # static value self.assertEqual(runner._config["api_secret"], "foosecret") @@ -603,7 +607,7 @@ def test_pythonpath_env_var_contains_common_libs_config_enabled(self, mock_popen _, call_kwargs = mock_popen.call_args actual_env = call_kwargs["env"] - pack_common_lib_path = "fixtures/packs/core/lib" + pack_common_lib_path = f"fixtures/packs/{DUMMY_PACK_1}/lib" self.assertIn("PYTHONPATH", actual_env) self.assertIn(pack_common_lib_path, actual_env["PYTHONPATH"]) @@ -626,7 +630,7 @@ def test_pythonpath_env_var_not_contains_common_libs_config_disabled( _, call_kwargs = mock_popen.call_args actual_env = call_kwargs["env"] pack_common_lib_path = ( - "/mnt/src/storm/st2/st2tests/st2tests/fixtures/packs/core/lib" + f"/mnt/src/storm/st2/st2tests/st2tests/fixtures/packs/{DUMMY_PACK_1}/lib" ) self.assertIn("PYTHONPATH", actual_env) self.assertNotIn(pack_common_lib_path, actual_env["PYTHONPATH"]) @@ -714,7 +718,7 @@ def test_python_action_wrapper_script_doesnt_get_added_to_sys_path(self): def test_python_action_wrapper_action_script_file_doesnt_exist_friendly_error(self): # File in a directory which is not a Python package wrapper = PythonActionWrapper( - pack="dummy_pack_5", file_path="/tmp/doesnt.exist", user="joe" + pack=DUMMY_PACK_5, file_path="/tmp/doesnt.exist", user="joe" ) expected_msg = ( @@ -724,7 +728,7 @@ def test_python_action_wrapper_action_script_file_doesnt_exist_friendly_error(se # File in a directory which is a Python package wrapper = PythonActionWrapper( - pack="dummy_pack_5", file_path=ACTION_1_PATH, user="joe" + pack=DUMMY_PACK_5, file_path=ACTION_1_PATH, user="joe" ) expected_msg = ( @@ -738,7 +742,7 @@ def test_python_action_wrapper_action_script_file_contains_invalid_syntax_friend self, ): wrapper = PythonActionWrapper( - pack="dummy_pack_5", file_path=ACTION_2_PATH, user="joe" + pack=DUMMY_PACK_5, file_path=ACTION_2_PATH, user="joe" ) expected_msg = ( r'Failed to load action class from file ".*?invalid_syntax.py" ' @@ -994,7 +998,7 @@ def test_content_version_old_git_version(self, mock_run_command): runner.runner_parameters = {"content_version": "v0.10.0"} expected_msg = ( - r'Failed to create git worktree for pack "core": Installed git version ' + rf'Failed to create git worktree for pack "{DUMMY_PACK_1}": Installed git version ' "doesn't support git worktree command. To be able to utilize this " "functionality you need to use git >= 2.5.0." ) @@ -1015,7 +1019,7 @@ def test_content_version_pack_repo_not_git_repository(self, mock_run_command): runner.runner_parameters = {"content_version": "v0.10.0"} expected_msg = ( - r'Failed to create git worktree for pack "core": Pack directory ' + rf'Failed to create git worktree for pack "{DUMMY_PACK_1}": Pack directory ' '".*" is not a ' "git repository. To utilize this functionality, pack directory needs to " "be a git repository." @@ -1036,7 +1040,7 @@ def test_content_version_invalid_git_revision(self, mock_run_command): runner.runner_parameters = {"content_version": "vinvalid"} expected_msg = ( - r'Failed to create git worktree for pack "core": Invalid content_version ' + rf'Failed to create git worktree for pack "{DUMMY_PACK_1}": Invalid content_version ' '"vinvalid" provided. Make sure that git repository is up ' "to date and contains that revision." ) @@ -1052,7 +1056,9 @@ def test_missing_config_item_user_friendly_error(self): self.assertIsNotNone(output) self.assertIn("{}", output["stdout"]) self.assertIn("default_value", output["stdout"]) - self.assertIn('Config for pack "core" is missing key "key"', output["stderr"]) + self.assertIn( + f'Config for pack "{DUMMY_PACK_1}" is missing key "key"', output["stderr"] + ) self.assertIn( 'make sure you run "st2ctl reload --register-configs"', output["stderr"] ) @@ -1107,7 +1113,7 @@ def _get_mock_action_obj(self): """ action = mock.Mock() action.ref = "dummy.action" - action.pack = SYSTEM_PACK_NAME + action.pack = DUMMY_PACK_1 action.entry_point = "foo.py" action.runner_type = {"name": "python-script"} return action diff --git a/contrib/runners/remote_runner/tests/BUILD b/contrib/runners/remote_runner/tests/BUILD index 3280583e0c..3391ed1f72 100644 --- a/contrib/runners/remote_runner/tests/BUILD +++ b/contrib/runners/remote_runner/tests/BUILD @@ -1,6 +1,9 @@ __defaults__( all=dict( skip_pylint=True, + entry_point_dependencies={ + "contrib/runners/remote_runner": ["st2common.runners.runner"], + }, ) ) diff --git a/contrib/runners/winrm_runner/tests/BUILD b/contrib/runners/winrm_runner/tests/BUILD index 3280583e0c..9c9ad37ef4 100644 --- a/contrib/runners/winrm_runner/tests/BUILD +++ b/contrib/runners/winrm_runner/tests/BUILD @@ -1,6 +1,9 @@ __defaults__( all=dict( skip_pylint=True, + entry_point_dependencies={ + "contrib/runners/winrm_runner": ["st2common.runners.runner"], + }, ) ) diff --git a/lockfiles/pants-plugins.lock b/lockfiles/pants-plugins.lock index 3d8a7e4f15..4d9a43a8aa 100644 --- a/lockfiles/pants-plugins.lock +++ b/lockfiles/pants-plugins.lock @@ -9,8 +9,8 @@ // "CPython==3.9.*" // ], // "generated_with_requirements": [ -// "pantsbuild.pants.testutil==2.22.0", -// "pantsbuild.pants==2.22.0" +// "pantsbuild.pants.testutil==2.23.0a0", +// "pantsbuild.pants==2.23.0a0" // ], // "manylinux": "manylinux2014", // "requirement_constraints": [], @@ -25,6 +25,7 @@ "allow_wheels": true, "build_isolation": true, "constraints": [], + "excluded": [], "locked_resolves": [ { "locked_requirements": [ @@ -241,6 +242,66 @@ "requires_python": ">=3.7", "version": "2.0.0" }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "d024f44059a853b4b852cfc04fec33e346659d851371e46fc8e7c19de24d3da9", + "url": "https://files.pythonhosted.org/packages/71/da/16307f14b47f761235050076e1d2954fc7de9346f1410ba8c67a54a9f40e/libcst-1.4.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl" + }, + { + "algorithm": "sha256", + "hash": "b8ecdba8934632b4dadacb666cd3816627a6ead831b806336972ccc4ba7ca0e9", + "url": "https://files.pythonhosted.org/packages/7b/b1/8476fe4fa1061062855459d519ffe2115a891638c230ee3465c69fdbfd7a/libcst-1.4.0-cp39-cp39-macosx_10_9_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "8e54c777b8d27339b70f304d16fc8bc8674ef1bd34ed05ea874bf4921eb5a313", + "url": "https://files.pythonhosted.org/packages/7e/0d/89516795ff2a11be10c060c539895b3781793d46cb7c9b0b7b3c4fa3fbc1/libcst-1.4.0-cp39-cp39-macosx_11_0_arm64.whl" + }, + { + "algorithm": "sha256", + "hash": "bb0abf627ee14903d05d0ad9b2c6865f1b21eb4081e2c7bea1033f85db2b8bae", + "url": "https://files.pythonhosted.org/packages/95/cf/a2be91d53e4068d4def8b5cc475f20e1c1a7d32c85634ee7d6b3ea2e3c9b/libcst-1.4.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "061d6855ef30efe38b8a292b7e5d57c8e820e71fc9ec9846678b60a934b53bbb", + "url": "https://files.pythonhosted.org/packages/c0/c8/15ca337e5f5604aabed899609ba08abbc0e7815ffdfca37802da52d4d0bf/libcst-1.4.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" + }, + { + "algorithm": "sha256", + "hash": "449e0b16604f054fa7f27c3ffe86ea7ef6c409836fe68fe4e752a1894175db00", + "url": "https://files.pythonhosted.org/packages/e4/bd/ff41d7a8efc4f60a61d903c3f9823565006f44f2b8b11c99701f552b0851/libcst-1.4.0.tar.gz" + } + ], + "project_name": "libcst", + "requires_dists": [ + "Sphinx>=5.1.1; extra == \"dev\"", + "black==23.12.1; extra == \"dev\"", + "build>=0.10.0; extra == \"dev\"", + "coverage>=4.5.4; extra == \"dev\"", + "fixit==2.1.0; extra == \"dev\"", + "flake8==7.0.0; extra == \"dev\"", + "hypothesis>=4.36.0; extra == \"dev\"", + "hypothesmith>=0.0.4; extra == \"dev\"", + "jinja2==3.1.4; extra == \"dev\"", + "jupyter>=1.0.0; extra == \"dev\"", + "maturin<1.6,>=0.8.3; extra == \"dev\"", + "nbsphinx>=0.4.2; extra == \"dev\"", + "prompt-toolkit>=2.0.9; extra == \"dev\"", + "pyre-check==0.9.18; platform_system != \"Windows\" and extra == \"dev\"", + "pyyaml>=5.2", + "setuptools-rust>=1.5.2; extra == \"dev\"", + "setuptools-scm>=6.0.1; extra == \"dev\"", + "slotscheck>=0.7.1; extra == \"dev\"", + "sphinx-rtd-theme>=0.4.3; extra == \"dev\"", + "ufmt==2.6.0; extra == \"dev\"", + "usort==1.0.8.post1; extra == \"dev\"" + ], + "requires_python": ">=3.9", + "version": "1.4.0" + }, { "artifacts": [ { @@ -285,23 +346,23 @@ "artifacts": [ { "algorithm": "sha256", - "hash": "edfcecc959eebfb0c42602b29d7301d96b92d3bbde030ba2c4ce3e775801cbd3", - "url": "https://github.com/pantsbuild/pants/releases/download/release_2.22.0/pantsbuild.pants-2.22.0-cp39-cp39-manylinux2014_x86_64.whl" + "hash": "f7104cf619c928752041acfe36966742dec5309b171aeef921239d4595ee4161", + "url": "https://github.com/pantsbuild/pants/releases/download/release_2.23.0a0/pantsbuild.pants-2.23.0a0-cp39-cp39-manylinux2014_x86_64.whl" }, { "algorithm": "sha256", - "hash": "a4b2a095c0d77afa6605ed591dfa334ebbbc2858676a1397dbe1bbc7bf7a1e68", - "url": "https://github.com/pantsbuild/pants/releases/download/release_2.22.0/pantsbuild.pants-2.22.0-cp39-cp39-macosx_10_15_x86_64.whl" + "hash": "d74b12dd7c4dd4cc9a7c81e55126db298577830c62962a6f8cbb4d875930f9ed", + "url": "https://github.com/pantsbuild/pants/releases/download/release_2.23.0a0/pantsbuild.pants-2.23.0a0-cp39-cp39-macosx_10_15_x86_64.whl" }, { "algorithm": "sha256", - "hash": "72ff3f0351389688fd031c8cbb77beffd5e1234ea139da24940522e5de6c14f8", - "url": "https://github.com/pantsbuild/pants/releases/download/release_2.22.0/pantsbuild.pants-2.22.0-cp39-cp39-macosx_11_0_arm64.whl" + "hash": "3afee18ce33b16cb3147ed18e190f0e37d4f3561d58354ee1203f7c66cfe1c5f", + "url": "https://github.com/pantsbuild/pants/releases/download/release_2.23.0a0/pantsbuild.pants-2.23.0a0-cp39-cp39-macosx_11_0_arm64.whl" }, { "algorithm": "sha256", - "hash": "d68791c067bc8902d2fceef8226c3ac443ec3349ab6463fcfe081d33647c4055", - "url": "https://github.com/pantsbuild/pants/releases/download/release_2.22.0/pantsbuild.pants-2.22.0-cp39-cp39-manylinux2014_aarch64.whl" + "hash": "6e47e4076e8321005b15afa4bd63f1444e32446de2634043caeafa35853a279c", + "url": "https://github.com/pantsbuild/pants/releases/download/release_2.23.0a0/pantsbuild.pants-2.23.0a0-cp39-cp39-manylinux2014_aarch64.whl" } ], "project_name": "pantsbuild-pants", @@ -311,9 +372,10 @@ "chevron==0.14.0", "fasteners==0.16.3", "ijson==3.2.3", + "libcst==1.4.0", "node-semver==0.9.0", "packaging==21.3", - "pex==2.3.1", + "pex==2.16.2", "psutil==5.9.8", "python-lsp-jsonrpc==1.0.0", "setproctitle==1.3.2", @@ -322,46 +384,46 @@ "types-PyYAML==6.0.3", "types-setuptools==62.6.1", "types-toml==0.10.8", - "typing-extensions==4.3.0" + "typing-extensions~=4.12" ], "requires_python": "==3.9.*", - "version": "2.22.0" + "version": "2.23.0a0" }, { "artifacts": [ { "algorithm": "sha256", - "hash": "4c4a0319e98f4892581887a713a78a2ee37ace5cd1535e5e73164942c59e632a", - "url": "https://github.com/pantsbuild/pants/releases/download/release_2.22.0/pantsbuild.pants.testutil-2.22.0-py3-none-any.whl" + "hash": "f74af1d1cbac2f8c17e441e2e6c96588fc1816828ecc2665b535dd4ccfbaa6c7", + "url": "https://github.com/pantsbuild/pants/releases/download/release_2.23.0a0/pantsbuild.pants.testutil-2.23.0a0-py3-none-any.whl" } ], "project_name": "pantsbuild-pants-testutil", "requires_dists": [ - "pantsbuild.pants==2.22.0", + "pantsbuild.pants==2.23.0a0", "pytest<7.1.0,>=6.2.4" ], "requires_python": "==3.9.*", - "version": "2.22.0" + "version": "2.23.0a0" }, { "artifacts": [ { "algorithm": "sha256", - "hash": "64692a5bf6f298403aab930d22f0d836ae4736c5bc820e262e9092fe8c56f830", - "url": "https://files.pythonhosted.org/packages/e7/d0/fbda2a4d41d62d86ce53f5ae4fbaaee8c34070f75bb7ca009090510ae874/pex-2.3.1-py2.py3-none-any.whl" + "hash": "8610b5bf7731c98d871421ff21e769e8fcf42ea56aa4ac7f8a271f2405733f24", + "url": "https://files.pythonhosted.org/packages/de/45/94497d22a1517b2462394f641ea272e7ec624823f223c01a5f0d7e6f571d/pex-2.16.2-py2.py3-none-any.whl" }, { "algorithm": "sha256", - "hash": "d1264c91161c21139b454744c8053e25b8aad2d15da89232181b4f38f3f54575", - "url": "https://files.pythonhosted.org/packages/83/4b/1855a9cd872a5eca4cd385e0f66078845f3561d359fb976be52a2a68b9d1/pex-2.3.1.tar.gz" + "hash": "feb2f1e9819a741915759fc221ee6119447acdfc3e0aaa5bbe5800c39fa10003", + "url": "https://files.pythonhosted.org/packages/87/cf/a39ace2db568e3bce48c79fd462aa289608208fe4509ff746349162e5196/pex-2.16.2.tar.gz" } ], "project_name": "pex", "requires_dists": [ "subprocess32>=3.2.7; python_version < \"3\" and extra == \"subprocess\"" ], - "requires_python": "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,<3.13,>=2.7", - "version": "2.3.1" + "requires_python": "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,<3.14,>=2.7", + "version": "2.16.2" }, { "artifacts": [ @@ -828,19 +890,19 @@ "artifacts": [ { "algorithm": "sha256", - "hash": "25642c956049920a5aa49edcdd6ab1e06d7e5d467fc00e0506c44ac86fbfca02", - "url": "https://files.pythonhosted.org/packages/ed/d6/2afc375a8d55b8be879d6b4986d4f69f01115e795e36827fd3a40166028b/typing_extensions-4.3.0-py3-none-any.whl" + "hash": "04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d", + "url": "https://files.pythonhosted.org/packages/26/9f/ad63fc0248c5379346306f8668cda6e2e2e9c95e01216d2b8ffd9ff037d0/typing_extensions-4.12.2-py3-none-any.whl" }, { "algorithm": "sha256", - "hash": "e6d2677a32f47fc7eb2795db1dd15c1f34eff616bcaf2cfb5e997f854fa1c4a6", - "url": "https://files.pythonhosted.org/packages/9e/1d/d128169ff58c501059330f1ad96ed62b79114a2eb30b8238af63a2e27f70/typing_extensions-4.3.0.tar.gz" + "hash": "1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8", + "url": "https://files.pythonhosted.org/packages/df/db/f35a00659bc03fec321ba8bce9420de607a1d37f8342eee1863174c69557/typing_extensions-4.12.2.tar.gz" } ], "project_name": "typing-extensions", "requires_dists": [], - "requires_python": ">=3.7", - "version": "4.3.0" + "requires_python": ">=3.8", + "version": "4.12.2" }, { "artifacts": [ @@ -926,13 +988,14 @@ ], "only_builds": [], "only_wheels": [], + "overridden": [], "path_mappings": {}, - "pex_version": "2.3.1", + "pex_version": "2.16.2", "pip_version": "24.0", "prefer_older_binary": false, "requirements": [ - "pantsbuild.pants.testutil==2.22.0", - "pantsbuild.pants==2.22.0" + "pantsbuild.pants.testutil==2.23.0a0", + "pantsbuild.pants==2.23.0a0" ], "requires_python": [ "==3.9.*" diff --git a/pants-plugins/macros.py b/pants-plugins/macros.py index bc346a057b..11131f20ee 100644 --- a/pants-plugins/macros.py +++ b/pants-plugins/macros.py @@ -125,3 +125,34 @@ def st2_shell_sources_and_resources(**kwargs): kwargs["name"] += "_resources" resources(**kwargs) # noqa: F821 + + +# these are referenced by the logging.*.conf files. +_st2common_logging_deps = ( + "//st2common/st2common/log.py", + "//st2common/st2common/logging/formatters.py", +) + + +def st2_logging_conf_files(**kwargs): + """This creates a files target with logging dependencies.""" + deps = kwargs.pop("dependencies", []) or [] + deps = list(deps) + list(_st2common_logging_deps) + kwargs["dependencies"] = tuple(deps) + files(**kwargs) # noqa: F821 + + +def st2_logging_conf_file(**kwargs): + """This creates a file target with logging dependencies.""" + deps = kwargs.pop("dependencies", []) or [] + deps = list(deps) + list(_st2common_logging_deps) + kwargs["dependencies"] = tuple(deps) + file(**kwargs) # noqa: F821 + + +def st2_logging_conf_resources(**kwargs): + """This creates a resources target with logging dependencies.""" + deps = kwargs.pop("dependencies", []) or [] + deps = list(deps) + list(_st2common_logging_deps) + kwargs["dependencies"] = tuple(deps) + resources(**kwargs) # noqa: F821 diff --git a/pants-plugins/pack_metadata/python_rules/BUILD b/pants-plugins/pack_metadata/python_rules/BUILD new file mode 100644 index 0000000000..a172051977 --- /dev/null +++ b/pants-plugins/pack_metadata/python_rules/BUILD @@ -0,0 +1,9 @@ +python_sources() + +python_tests( + name="tests", +) + +python_test_utils( + name="test_utils", +) diff --git a/pants-plugins/pack_metadata/python_rules/__init__.py b/pants-plugins/pack_metadata/python_rules/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/pants-plugins/pack_metadata/python_rules/conftest.py b/pants-plugins/pack_metadata/python_rules/conftest.py new file mode 100644 index 0000000000..51104af2e8 --- /dev/null +++ b/pants-plugins/pack_metadata/python_rules/conftest.py @@ -0,0 +1,284 @@ +# Copyright 2024 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from textwrap import dedent + +import pytest +from pants.backend.python.dependency_inference.module_mapper import ( + FirstPartyPythonMappingImpl, +) +from pants.backend.python.goals.pytest_runner import PytestPluginSetup +from pants.backend.python.target_types import ( + PythonSourceTarget, + PythonSourcesGeneratorTarget, + PythonTestTarget, + PythonTestsGeneratorTarget, +) +from pants.backend.python.target_types_rules import rules as python_target_types_rules +from pants.engine.rules import QueryRule +from pants.testutil.python_rule_runner import PythonRuleRunner +from pants.testutil.rule_runner import RuleRunner + +from pack_metadata.python_rules import ( + python_module_mapper, + python_pack_content, + python_path_rules, +) +from pack_metadata.python_rules.python_module_mapper import ( + St2PythonPackContentMappingMarker, +) +from pack_metadata.python_rules.python_pack_content import ( + PackContentPythonEntryPoints, + PackContentPythonEntryPointsRequest, + PackContentResourceTargetsOfType, + PackContentResourceTargetsOfTypeRequest, + PackPythonLibs, + PackPythonLibsRequest, +) +from pack_metadata.python_rules.python_path_rules import ( + PackPythonPath, + PackPythonPathRequest, + PytestPackTestRequest, +) +from pack_metadata.target_types import ( + InjectPackPythonPathField, + PackContentResourceTarget, + PackMetadata, +) + +# some random pack names +packs = ( + "foo", # imports between actions + "dr_seuss", # imports from /actions/lib + "shards", # imports from /lib + "metals", # imports the action from a subdirectory +) + + +@pytest.fixture +def pack_names() -> tuple[str, ...]: + return packs + + +def write_test_files(rule_runner: RuleRunner): + for pack in packs: + rule_runner.write_files( + { + f"packs/{pack}/BUILD": dedent( + """ + __defaults__(all=dict(inject_pack_python_path=True)) + pack_metadata(name="metadata") + """ + ), + f"packs/{pack}/pack.yaml": dedent( + f""" + --- + name: {pack} + version: 1.0.0 + author: StackStorm + email: info@stackstorm.com + """ + ), + f"packs/{pack}/config.schema.yaml": "", + f"packs/{pack}/config.yaml.example": "", + f"packs/{pack}/icon.png": "", + f"packs/{pack}/README.md": f"# Pack {pack} README", + } + ) + + def action_metadata_file(action: str, entry_point: str = "") -> str: + entry_point = entry_point or f"{action}.py" + return dedent( + f""" + --- + name: {action} + runner_type: python-script + entry_point: {entry_point} + """ + ) + + def test_file(module: str, _object: str) -> str: + return dedent( + f""" + from {module} import {_object} + def test_{module.replace(".", "_")}() -> None: + pass + """ + ) + + rule_runner.write_files( + { + "packs/foo/actions/BUILD": "python_sources()", + "packs/foo/actions/get_bar.yaml": action_metadata_file("get_bar"), + "packs/foo/actions/get_bar.py": dedent( + """ + RESPONSE_CONSTANT = "foobar_key" + class BarAction: + def run(self): + return {RESPONSE_CONSTANT: "bar"} + """ + ), + "packs/foo/actions/get_baz.yaml": action_metadata_file("get_baz"), + "packs/foo/actions/get_baz.py": dedent( + """ + from get_bar import RESPONSE_CONSTANT + class BazAction: + def run(self): + return {RESPONSE_CONSTANT: "baz"} + """ + ), + "packs/foo/tests/BUILD": "python_tests()", + "packs/foo/tests/test_get_bar_action.py": test_file("get_bar", "BarAction"), + "packs/foo/tests/test_get_baz_action.py": test_file("get_baz", "BazAction"), + "packs/dr_seuss/actions/lib/seuss/BUILD": "python_sources()", + "packs/dr_seuss/actions/lib/seuss/__init__.py": "", + "packs/dr_seuss/actions/lib/seuss/things.py": dedent( + """ + THING1 = "thing one" + THING2 = "thing two" + """ + ), + "packs/dr_seuss/actions/BUILD": "python_sources()", + "packs/dr_seuss/actions/get_from_actions_lib.yaml": action_metadata_file( + "get_from_actions_lib" + ), + "packs/dr_seuss/actions/get_from_actions_lib.py": dedent( + """ + from seuss.things import THING1, THING2 + class GetFromActionsLibAction: + def run(self): + return {"things": (THING1, THING2)} + """ + ), + "packs/dr_seuss/tests/BUILD": "python_tests()", + "packs/dr_seuss/tests/test_get_from_actions_lib_action.py": test_file( + "get_from_actions_lib", "GetFromActionsLibAction" + ), + "packs/shards/lib/stormlight_archive/BUILD": "python_sources()", + "packs/shards/lib/stormlight_archive/__init__.py": "", + "packs/shards/lib/stormlight_archive/things.py": dedent( + """ + STORM_LIGHT = "Honor" + VOID_LIGHT = "Odium" + LIFE_LIGHT = "Cultivation" + """ + ), + "packs/shards/actions/BUILD": "python_sources()", + "packs/shards/actions/get_from_pack_lib.yaml": action_metadata_file( + "get_from_pack_lib" + ), + "packs/shards/actions/get_from_pack_lib.py": dedent( + """ + from stormlight_archive.things import STORM_LIGHT, VOID_LIGHT, LIFE_LIGHT + class GetFromPackLibAction: + def run(self): + return {"light_sources": (STORM_LIGHT, VOID_LIGHT, LIFE_LIGHT)} + """ + ), + "packs/shards/sensors/BUILD": "python_sources()", + "packs/shards/sensors/horn_eater.yaml": dedent( + """ + --- + name: horn_eater + entry_point: horn_eater.py + class_name: HornEaterSensor + trigger_types: [{name: horn_eater.saw.spren, payload_schema: {type: object}}] + """ + ), + "packs/shards/sensors/horn_eater.py": dedent( + """ + from st2reactor.sensor.base import PollingSensor + from stormlight_archive.things import STORM_LIGHT + class HornEaterSensor(PollingSensor): + def setup(self): pass + def poll(self): + if STORM_LIGHT in self.config: + self.sensor_service.dispatch( + trigger="horn_eater.saw.spren", payload={"spren_type": STORM_LIGHT} + ) + def cleanup(self): pass + def add_trigger(self): pass + def update_trigger(self): pass + def remove_trigger(self): pass + """ + ), + "packs/shards/tests/BUILD": "python_tests()", + "packs/shards/tests/test_get_from_pack_lib_action.py": test_file( + "get_from_pack_lib", "GetFromPackLibAction" + ), + "packs/shards/tests/test_horn_eater_sensor.py": test_file( + "horn_eater", "HornEaterSensor" + ), + "packs/metals/actions/fly.yaml": action_metadata_file( + "fly", "mist_born/fly.py" + ), + "packs/metals/actions/mist_born/BUILD": "python_sources()", + "packs/metals/actions/mist_born/__init__.py": "", + "packs/metals/actions/mist_born/fly.py": dedent( + """ + class FlyAction: + def run(self): + return {"metals": ("steel", "iron")} + """ + ), + "packs/metals/tests/BUILD": "python_tests()", + "packs/metals/tests/test_fly_action.py": test_file( + "mist_born.fly", "FlyAction" + ), + } + ) + + +@pytest.fixture +def rule_runner() -> RuleRunner: + rule_runner = PythonRuleRunner( + rules=[ + PythonTestsGeneratorTarget.register_plugin_field( + InjectPackPythonPathField, as_moved_field=True + ), + PythonTestTarget.register_plugin_field(InjectPackPythonPathField), + *python_target_types_rules(), + # TODO: not sure if we need a QueryRule for every rule... + *python_pack_content.rules(), + QueryRule( + PackContentResourceTargetsOfType, + (PackContentResourceTargetsOfTypeRequest,), + ), + QueryRule( + PackContentPythonEntryPoints, (PackContentPythonEntryPointsRequest,) + ), + QueryRule(PackPythonLibs, (PackPythonLibsRequest,)), + *python_module_mapper.rules(), + QueryRule( + FirstPartyPythonMappingImpl, (St2PythonPackContentMappingMarker,) + ), + *python_path_rules.rules(), + QueryRule(PackPythonPath, (PackPythonPathRequest,)), + QueryRule(PytestPluginSetup, (PytestPackTestRequest,)), + ], + target_types=[ + PackContentResourceTarget, + PackMetadata, + PythonSourceTarget, + PythonSourcesGeneratorTarget, + PythonTestTarget, + PythonTestsGeneratorTarget, + ], + ) + write_test_files(rule_runner) + args = [ + "--source-root-patterns=packs/*", + ] + rule_runner.set_options(args, env_inherit={"PATH", "PYENV_ROOT", "HOME"}) + return rule_runner diff --git a/pants-plugins/pack_metadata/python_rules/python_module_mapper.py b/pants-plugins/pack_metadata/python_rules/python_module_mapper.py new file mode 100644 index 0000000000..b00333e03a --- /dev/null +++ b/pants-plugins/pack_metadata/python_rules/python_module_mapper.py @@ -0,0 +1,81 @@ +# Copyright 2024 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from collections import defaultdict +from typing import DefaultDict + +from pants.backend.python.dependency_inference.module_mapper import ( + FirstPartyPythonMappingImpl, + FirstPartyPythonMappingImplMarker, + ModuleProvider, + ModuleProviderType, + ResolveName, +) +from pants.engine.rules import Get, MultiGet, collect_rules, rule +from pants.engine.unions import UnionRule +from pants.util.logging import LogLevel + +from pack_metadata.target_types import PackMetadata +from pack_metadata.python_rules.python_pack_content import ( + PackContentPythonEntryPoints, + PackContentPythonEntryPointsRequest, + PackPythonLibs, + PackPythonLibsRequest, +) + + +# This is only used to register our implementation with the plugin hook via unions. +class St2PythonPackContentMappingMarker(FirstPartyPythonMappingImplMarker): + pass + + +@rule( + desc=f"Creating map of `{PackMetadata.alias}` targets to Python modules in pack content", + level=LogLevel.DEBUG, +) +async def map_pack_content_to_python_modules( + _: St2PythonPackContentMappingMarker, +) -> FirstPartyPythonMappingImpl: + resolves_to_modules_to_providers: DefaultDict[ + ResolveName, DefaultDict[str, list[ModuleProvider]] + ] = defaultdict(lambda: defaultdict(list)) + + pack_content_python_entry_points, pack_python_libs = await MultiGet( + Get(PackContentPythonEntryPoints, PackContentPythonEntryPointsRequest()), + Get(PackPythonLibs, PackPythonLibsRequest()), + ) + + for pack_content in pack_content_python_entry_points: + for module in pack_content.get_possible_modules(): + resolves_to_modules_to_providers[pack_content.resolve][module].append( + ModuleProvider(pack_content.python_address, ModuleProviderType.IMPL) + ) + + for pack_lib in pack_python_libs: + provider_type = ( + ModuleProviderType.TYPE_STUB + if pack_lib.relative_to_lib.suffix == ".pyi" + else ModuleProviderType.IMPL + ) + resolves_to_modules_to_providers[pack_lib.resolve][pack_lib.module].append( + ModuleProvider(pack_lib.python_address, provider_type) + ) + + return FirstPartyPythonMappingImpl.create(resolves_to_modules_to_providers) + + +def rules(): + return ( + *collect_rules(), + UnionRule(FirstPartyPythonMappingImplMarker, St2PythonPackContentMappingMarker), + ) diff --git a/pants-plugins/pack_metadata/python_rules/python_module_mapper_test.py b/pants-plugins/pack_metadata/python_rules/python_module_mapper_test.py new file mode 100644 index 0000000000..d1d6d779fb --- /dev/null +++ b/pants-plugins/pack_metadata/python_rules/python_module_mapper_test.py @@ -0,0 +1,73 @@ +# Copyright 2024 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pants.backend.python.dependency_inference.module_mapper import ( + FirstPartyPythonMappingImpl, + ModuleProvider, + ModuleProviderType, +) +from pants.engine.internals.native_engine import Address +from pants.testutil.rule_runner import RuleRunner +from pants.util.frozendict import FrozenDict + +from pack_metadata.python_rules.python_module_mapper import ( + St2PythonPackContentMappingMarker, +) + + +def test_map_pack_content_to_python_modules(rule_runner: RuleRunner) -> None: + result = rule_runner.request( + FirstPartyPythonMappingImpl, + (St2PythonPackContentMappingMarker(),), + ) + + def module_provider(spec_path: str, relative_file_path: str) -> ModuleProvider: + return ModuleProvider( + Address(spec_path=spec_path, relative_file_path=relative_file_path), + ModuleProviderType.IMPL, + ) + + expected = { + "": { + "get_bar": (module_provider("packs/foo/actions", "get_bar.py"),), + "get_baz": (module_provider("packs/foo/actions", "get_baz.py"),), + "seuss": ( + module_provider("packs/dr_seuss/actions/lib/seuss", "__init__.py"), + ), + "seuss.things": ( + module_provider("packs/dr_seuss/actions/lib/seuss", "things.py"), + ), + "get_from_actions_lib": ( + module_provider("packs/dr_seuss/actions", "get_from_actions_lib.py"), + ), + "stormlight_archive": ( + module_provider("packs/shards/lib/stormlight_archive", "__init__.py"), + ), + "stormlight_archive.things": ( + module_provider("packs/shards/lib/stormlight_archive", "things.py"), + ), + "get_from_pack_lib": ( + module_provider("packs/shards/actions", "get_from_pack_lib.py"), + ), + "horn_eater": (module_provider("packs/shards/sensors", "horn_eater.py"),), + "fly": (module_provider("packs/metals/actions/mist_born", "fly.py"),), + "mist_born.fly": ( + module_provider("packs/metals/actions/mist_born", "fly.py"), + ), + } + } + assert isinstance(result, FrozenDict) + assert all(isinstance(value, FrozenDict) for value in result.values()) + # pytest reports dict differences better than FrozenDict + assert {resolve: dict(value) for resolve, value in result.items()} == expected diff --git a/pants-plugins/pack_metadata/python_rules/python_pack_content.py b/pants-plugins/pack_metadata/python_rules/python_pack_content.py new file mode 100644 index 0000000000..058270290b --- /dev/null +++ b/pants-plugins/pack_metadata/python_rules/python_pack_content.py @@ -0,0 +1,364 @@ +# Copyright 2024 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import yaml +from collections import defaultdict +from dataclasses import dataclass +from pathlib import PurePath +from typing import DefaultDict + +from pants.backend.python.dependency_inference.module_mapper import ( + module_from_stripped_path, +) +from pants.backend.python.subsystems.setup import PythonSetup +from pants.backend.python.target_types import PythonResolveField, PythonSourceField +from pants.base.glob_match_error_behavior import GlobMatchErrorBehavior +from pants.base.specs import FileLiteralSpec, RawSpecs, RecursiveGlobSpec +from pants.engine.collection import Collection +from pants.engine.fs import DigestContents +from pants.engine.internals.native_engine import Address, Digest +from pants.engine.rules import Get, MultiGet, collect_rules, rule +from pants.engine.target import ( + AllTargets, + AllUnexpandedTargets, + HydrateSourcesRequest, + HydratedSources, + Target, + Targets, +) +from pants.util.dirutil import fast_relpath +from pants.util.logging import LogLevel + +from pack_metadata.target_types import ( + PackContentResourceSourceField, + PackContentResourceTypeField, + PackContentResourceTypes, + PackMetadata, + PackMetadataSourcesField, +) + + +# Implementation Notes: +# +# With pants, we can rely on dependency inference for all the +# st2 components, runners, and other venv bits (st2 venv and pack venv). +# In ST2, all of that goes at the end of PYTHONPATH. +# +# Actions: +# At runtime, the python_runner creates a PYTHONPATH that includes: +# [pack/lib:]pack_venv/lib/python3.x:pack_venv/lib/python3.x/site-packages:pack/actions/lib:st2_pythonpath +# python_runner runs python_action_wrapper which: +# - injects the action's entry_point's directory in sys.path +# - and then imports the action module and runs it. +# +# Sensors: +# At runtime, ProcessSensorContainer creates PYTHONPATH that includes: +# [pack/lib:]st2_pythonpath +# Then the container runs the sensor via sensor_wrapper which +# process_container runs sensor_wrapper which: +# - injects the sensor's entry_point's directory in sys.path +# (effectively always "sensors/" as a split("/") assumes only one dir) +# - and then imports the class_name from sensor module and runs it. +# +# For actions, this pants plugin need to add this to PEX_EXTRA_SYS_PATH: +# pack/actions/path_to_entry_point:[pack/lib:]pack/actions/lib +# For sensors, this pants plugin need to add this to PEX_EXTRA_SYS_PATH: +# pack/sensors:[pack/lib:] +# +# This rules in this file are used by: +# python_module_mapper.py: +# Dependency inference uses pack_metadata's module_mapper to detect any +# python imports that require one of these PYTHONPATH modifications, +# resolving those imports to modules in lib/, actions/, or sensors/. +# python_path_rules.py: +# Then get the relevant python imports from dependencies and +# add their parent directory to a generated PEX_EXTRA_SYS_PATH. + + +@dataclass(frozen=True) +class PackContentResourceTargetsOfTypeRequest: + types: tuple[PackContentResourceTypes, ...] + + +class PackContentResourceTargetsOfType(Targets): + pass + + +@rule( + desc=f"Find all `{PackMetadata.alias}` targets in project filtered by content type", + level=LogLevel.DEBUG, +) +async def find_pack_metadata_targets_of_types( + request: PackContentResourceTargetsOfTypeRequest, targets: AllTargets +) -> PackContentResourceTargetsOfType: + return PackContentResourceTargetsOfType( + tgt + for tgt in targets + if tgt.has_field(PackContentResourceSourceField) + and ( + not request.types + or tgt[PackContentResourceTypeField].value in request.types + ) + ) + + +@dataclass(frozen=True) +class PackContentPythonEntryPoint: + metadata_address: Address + content_type: PackContentResourceTypes + entry_point: str + python_address: Address + resolve: str + + @property + def python_file_path(self) -> PurePath: + return PurePath(self.python_address.filename) + + @staticmethod + def _split_pack_content_path(path: PurePath) -> tuple[PurePath, PurePath]: + content_types = ("actions", "sensors") # only content_types with python content + pack_content_dir = path.parent + while pack_content_dir.name not in content_types: + pack_content_dir = pack_content_dir.parent + relative_to_pack_content_dir = path.relative_to(pack_content_dir) + return pack_content_dir, relative_to_pack_content_dir + + def get_possible_modules(self) -> tuple[str, ...]: + """Get module names that could be imported. Mirrors get_possible_paths logic.""" + path = self.python_file_path + + # st2 adds the parent dir of the python file to sys.path at runtime. + module = path.stem if path.suffix == ".py" else path.name + modules = [module] + + # By convention, however, just actions/ is on sys.path during tests. + # so, also construct the module name from actions/ to support tests. + _, relative_to_pack_content_dir = self._split_pack_content_path(path) + module = module_from_stripped_path(relative_to_pack_content_dir) + if module not in modules: + modules.append(module) + + return tuple(modules) + + def get_possible_paths(self) -> tuple[str, ...]: + """Get paths to add to PYTHONPATH and PEX_EXTRA_SYS_PATH. Mirrors get_possible_modules logic.""" + path = self.python_file_path + + # st2 adds the parent dir of the python file to sys.path at runtime. + paths = [path.parent.as_posix()] + + # By convention, however, just actions/ is on sys.path during tests. + # so, also construct the module name from actions/ to support tests. + pack_content_dir, _ = self._split_pack_content_path(path) + if path.parent != pack_content_dir: + paths.append(pack_content_dir.as_posix()) + + return tuple(paths) + + +class PackContentPythonEntryPoints(Collection[PackContentPythonEntryPoint]): + pass + + +class PackContentPythonEntryPointsRequest: + pass + + +@rule(desc="Find all Pack Content entry_points that are python", level=LogLevel.DEBUG) +async def find_pack_content_python_entry_points( + python_setup: PythonSetup, _: PackContentPythonEntryPointsRequest +) -> PackContentPythonEntryPoints: + action_or_sensor = ( + PackContentResourceTypes.action_metadata, + PackContentResourceTypes.sensor_metadata, + ) + + action_and_sensor_metadata_targets = await Get( + PackContentResourceTargetsOfType, + PackContentResourceTargetsOfTypeRequest(action_or_sensor), + ) + action_and_sensor_metadata_sources = await MultiGet( + Get(HydratedSources, HydrateSourcesRequest(tgt[PackContentResourceSourceField])) + for tgt in action_and_sensor_metadata_targets + ) + action_and_sensor_metadata_contents = await MultiGet( + Get(DigestContents, Digest, source.snapshot.digest) + for source in action_and_sensor_metadata_sources + ) + + # python file path -> list of info about metadata files that refer to it + pack_content_entry_points_by_spec: DefaultDict[ + str, list[tuple[Address, PackContentResourceTypes, str]] + ] = defaultdict(list) + + tgt: Target + contents: DigestContents + for tgt, contents in zip( + action_and_sensor_metadata_targets, action_and_sensor_metadata_contents + ): + content_type = tgt[PackContentResourceTypeField].value + if content_type not in action_or_sensor: + continue + assert len(contents) == 1 + try: + metadata = yaml.safe_load(contents[0].content) or {} + except yaml.YAMLError: + continue + if content_type == PackContentResourceTypes.action_metadata: + runner_type = metadata.get("runner_type", "") or "" + if runner_type != "python-script": + # only python-script has special PYTHONPATH rules + continue + # get the entry_point to find subdirectory that contains the module + entry_point = metadata.get("entry_point", "") or "" + if entry_point: + # address.filename is basically f"{spec_path}/{relative_file_path}" + path = PurePath(tgt.address.filename).parent / entry_point + pack_content_entry_points_by_spec[str(path)].append( + (tgt.address, content_type, entry_point) + ) + + python_targets = await Get( + Targets, + RawSpecs( + file_literals=tuple( + FileLiteralSpec(spec_path) + for spec_path in pack_content_entry_points_by_spec + ), + unmatched_glob_behavior=GlobMatchErrorBehavior.ignore, + description_of_origin="pack_metadata python module mapper", + ), + ) + + pack_content_entry_points: list[PackContentPythonEntryPoint] = [] + for tgt in python_targets: + if not tgt.has_field(PythonResolveField): + # this is unexpected + continue + for ( + metadata_address, + content_type, + entry_point, + ) in pack_content_entry_points_by_spec[tgt.address.filename]: + resolve = tgt[PythonResolveField].normalized_value(python_setup) + + pack_content_entry_points.append( + PackContentPythonEntryPoint( + metadata_address=metadata_address, + content_type=content_type, + entry_point=entry_point, + python_address=tgt.address, + resolve=resolve, + ) + ) + + return PackContentPythonEntryPoints(pack_content_entry_points) + + +@dataclass(frozen=True) +class PackPythonLib: + pack_path: PurePath + lib_dir: str + relative_to_lib: PurePath + python_address: Address + resolve: str + + @property + def module(self) -> str: + return module_from_stripped_path(self.relative_to_lib) + + @property + def lib_path(self) -> PurePath: + return self.pack_path / self.lib_dir + + +class PackPythonLibs(Collection[PackPythonLib]): + pass + + +class PackPythonLibsRequest: + pass + + +@rule(desc="Find all Pack lib directory python targets", level=LogLevel.DEBUG) +async def find_python_in_pack_lib_directories( + python_setup: PythonSetup, + all_unexpanded_targets: AllUnexpandedTargets, + _: PackPythonLibsRequest, +) -> PackPythonLibs: + pack_metadata_paths = [ + PurePath(tgt.address.spec_path) + for tgt in all_unexpanded_targets + if tgt.has_field(PackMetadataSourcesField) + ] + pack_lib_directory_targets = await MultiGet( + Get( + Targets, + RawSpecs( + recursive_globs=( + RecursiveGlobSpec(str(path / "lib")), + RecursiveGlobSpec(str(path / "actions" / "lib")), + ), + unmatched_glob_behavior=GlobMatchErrorBehavior.ignore, + description_of_origin="pack_metadata lib directory lookup", + ), + ) + for path in pack_metadata_paths + ) + + # Maybe this should use this to take codegen into account. + # Get(PythonSourceFiles, PythonSourceFilesRequest(targets=lib_directory_targets, include_resources=False) + # For now, just take the targets as they are. + + pack_python_libs: list[PackPythonLib] = [] + + pack_path: PurePath + lib_directory_targets: Targets + for pack_path, lib_directory_targets in zip( + pack_metadata_paths, pack_lib_directory_targets + ): + for tgt in lib_directory_targets: + if not tgt.has_field(PythonSourceField): + # only python targets matter here. + continue + + relative_to_pack = PurePath( + fast_relpath(tgt[PythonSourceField].file_path, str(pack_path)) + ) + if relative_to_pack.parts[0] == "lib": + lib_dir = "lib" + elif relative_to_pack.parts[:2] == ("actions", "lib"): + lib_dir = "actions/lib" + else: + # This should not happen as it is not in the requested glob. + # Use this to tell linters that lib_dir is defined below here. + continue + relative_to_lib = relative_to_pack.relative_to(lib_dir) + + resolve = tgt[PythonResolveField].normalized_value(python_setup) + + pack_python_libs.append( + PackPythonLib( + pack_path=pack_path, + lib_dir=lib_dir, + relative_to_lib=relative_to_lib, + python_address=tgt.address, + resolve=resolve, + ) + ) + + return PackPythonLibs(pack_python_libs) + + +def rules(): + return (*collect_rules(),) diff --git a/pants-plugins/pack_metadata/python_rules/python_pack_content_test.py b/pants-plugins/pack_metadata/python_rules/python_pack_content_test.py new file mode 100644 index 0000000000..33c1389bb3 --- /dev/null +++ b/pants-plugins/pack_metadata/python_rules/python_pack_content_test.py @@ -0,0 +1,158 @@ +# Copyright 2024 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from pants.engine.addresses import Address +from pants.testutil.rule_runner import RuleRunner + +from pack_metadata.python_rules.python_pack_content import ( + PackContentPythonEntryPoints, + PackContentPythonEntryPointsRequest, + PackContentResourceTargetsOfType, + PackContentResourceTargetsOfTypeRequest, + PackPythonLibs, + PackPythonLibsRequest, +) +from pack_metadata.target_types import PackContentResourceTypes + + +@pytest.mark.parametrize( + "requested_types,expected_count,expected_file_name", + ( + # one content type + ((PackContentResourceTypes.pack_metadata,), 4, "pack.yaml"), + ((PackContentResourceTypes.pack_config_schema,), 4, "config.schema.yaml"), + ((PackContentResourceTypes.pack_config_example,), 4, "config.yaml.example"), + ((PackContentResourceTypes.pack_icon,), 4, "icon.png"), + ((PackContentResourceTypes.action_metadata,), 5, ".yaml"), + ((PackContentResourceTypes.sensor_metadata,), 1, ".yaml"), + ((PackContentResourceTypes.rule_metadata,), 0, ""), + ((PackContentResourceTypes.policy_metadata,), 0, ""), + ((PackContentResourceTypes.unknown,), 0, ""), + # all content types + ((), 22, ""), + # some content types + ( + ( + PackContentResourceTypes.action_metadata, + PackContentResourceTypes.sensor_metadata, + ), + 6, + "", + ), + ( + ( + PackContentResourceTypes.pack_metadata, + PackContentResourceTypes.pack_config_schema, + PackContentResourceTypes.pack_config_example, + ), + 12, + "", + ), + ), +) +def test_find_pack_metadata_targets_of_types( + rule_runner: RuleRunner, + requested_types: tuple[PackContentResourceTypes, ...], + expected_count: int, + expected_file_name: str, +) -> None: + result = rule_runner.request( + PackContentResourceTargetsOfType, + (PackContentResourceTargetsOfTypeRequest(requested_types),), + ) + assert len(result) == expected_count + if expected_file_name: + for tgt in result: + tgt.address.relative_file_path.endswith(expected_file_name) + + +def test_find_pack_content_python_entry_points(rule_runner: RuleRunner) -> None: + result = rule_runner.request( + PackContentPythonEntryPoints, + (PackContentPythonEntryPointsRequest(),), + ) + assert len(result) == 6 # 5 actions + 1 sensor + assert {res.metadata_address for res in result} == { + Address( + "packs/foo", + relative_file_path="actions/get_bar.yaml", + target_name="metadata", + ), + Address( + "packs/foo", + relative_file_path="actions/get_baz.yaml", + target_name="metadata", + ), + Address( + "packs/dr_seuss", + relative_file_path="actions/get_from_actions_lib.yaml", + target_name="metadata", + ), + Address( + "packs/shards", + relative_file_path="actions/get_from_pack_lib.yaml", + target_name="metadata", + ), + Address( + "packs/shards", + relative_file_path="sensors/horn_eater.yaml", + target_name="metadata", + ), + Address( + "packs/metals", + relative_file_path="actions/fly.yaml", + target_name="metadata", + ), + } + assert {(res.content_type, res.entry_point) for res in result} == { + (PackContentResourceTypes.action_metadata, "get_bar.py"), + (PackContentResourceTypes.action_metadata, "get_baz.py"), + (PackContentResourceTypes.action_metadata, "get_from_actions_lib.py"), + (PackContentResourceTypes.action_metadata, "get_from_pack_lib.py"), + (PackContentResourceTypes.sensor_metadata, "horn_eater.py"), + (PackContentResourceTypes.action_metadata, "mist_born/fly.py"), + } + assert {res.python_address for res in result} == { + Address("packs/foo/actions", relative_file_path="get_bar.py"), + Address("packs/foo/actions", relative_file_path="get_baz.py"), + Address("packs/dr_seuss/actions", relative_file_path="get_from_actions_lib.py"), + Address("packs/shards/actions", relative_file_path="get_from_pack_lib.py"), + Address("packs/shards/sensors", relative_file_path="horn_eater.py"), + Address("packs/metals/actions/mist_born", relative_file_path="fly.py"), + } + + +def test_find_python_in_pack_lib_directories(rule_runner: RuleRunner) -> None: + result = rule_runner.request(PackPythonLibs, (PackPythonLibsRequest(),)) + assert len(result) == 4 + assert {(str(res.pack_path), res.lib_dir) for res in result} == { + ("packs/dr_seuss", "actions/lib"), + ("packs/shards", "lib"), + } + assert {res.python_address for res in result} == { + Address("packs/dr_seuss/actions/lib/seuss", relative_file_path="__init__.py"), + Address("packs/dr_seuss/actions/lib/seuss", relative_file_path="things.py"), + Address( + "packs/shards/lib/stormlight_archive", relative_file_path="__init__.py" + ), + Address("packs/shards/lib/stormlight_archive", relative_file_path="things.py"), + } + assert {res.module for res in result} == { + "seuss", + "seuss.things", + "stormlight_archive", + "stormlight_archive.things", + } diff --git a/pants-plugins/pack_metadata/python_rules/python_path_rules.py b/pants-plugins/pack_metadata/python_rules/python_path_rules.py new file mode 100644 index 0000000000..314ecd7bf9 --- /dev/null +++ b/pants-plugins/pack_metadata/python_rules/python_path_rules.py @@ -0,0 +1,131 @@ +# Copyright 2024 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from typing import Set + +from pants.backend.python.goals.pytest_runner import ( + PytestPluginSetupRequest, + PytestPluginSetup, +) +from pants.engine.internals.native_engine import Address +from pants.engine.rules import collect_rules, Get, MultiGet, rule +from pants.engine.target import Target, TransitiveTargets, TransitiveTargetsRequest +from pants.engine.unions import UnionRule +from pants.util.logging import LogLevel +from pants.util.ordered_set import OrderedSet + +from pack_metadata.python_rules.python_pack_content import ( + PackContentPythonEntryPoints, + PackContentPythonEntryPointsRequest, + PackPythonLibs, + PackPythonLibsRequest, +) +from pack_metadata.target_types import InjectPackPythonPathField + + +@dataclass(frozen=True) +class PackPythonPath: + entries: tuple[str, ...] = () + + +@dataclass(frozen=True) +class PackPythonPathRequest: + address: Address + + +@rule( + desc="Get pack paths that should be added to PYTHONPATH/PEX_EXTRA_SYS_PATH for a target.", + level=LogLevel.DEBUG, +) +async def get_extra_sys_path_for_pack_dependencies( + request: PackPythonPathRequest, +) -> PackPythonPath: + transitive_targets = await Get( + TransitiveTargets, TransitiveTargetsRequest((request.address,)) + ) + + dependency_addresses: Set[Address] = { + tgt.address for tgt in transitive_targets.closure + } + if not dependency_addresses: + return PackPythonPath() + + pack_content_python_entry_points, pack_python_libs = await MultiGet( + Get(PackContentPythonEntryPoints, PackContentPythonEntryPointsRequest()), + Get(PackPythonLibs, PackPythonLibsRequest()), + ) + + # only use addresses of actual dependencies + pack_python_content_addresses: Set[Address] = dependency_addresses & { + pack_content.python_address for pack_content in pack_content_python_entry_points + } + pack_python_lib_addresses: Set[Address] = dependency_addresses & { + pack_lib.python_address for pack_lib in pack_python_libs + } + + if not (pack_python_content_addresses or pack_python_lib_addresses): + return PackPythonPath() + + # filter pack_content_python_entry_points and pack_python_libs + pack_content_python_entry_points = ( + pack_content + for pack_content in pack_content_python_entry_points + if pack_content.python_address in pack_python_content_addresses + ) + pack_python_libs = ( + pack_lib + for pack_lib in pack_python_libs + if pack_lib.python_address in pack_python_lib_addresses + ) + + extra_sys_path_entries = OrderedSet() + for pack_content in pack_content_python_entry_points: + for path_entry in pack_content.get_possible_paths(): + extra_sys_path_entries.add(path_entry) + for pack_lib in pack_python_libs: + extra_sys_path_entries.add(pack_lib.lib_path.as_posix()) + + return PackPythonPath(tuple(extra_sys_path_entries)) + + +class PytestPackTestRequest(PytestPluginSetupRequest): + @classmethod + def is_applicable(cls, target: Target) -> bool: + if not target.has_field(InjectPackPythonPathField): + return False + return bool(target.get(InjectPackPythonPathField).value) + + +@rule( + desc="Inject pack paths in PYTHONPATH/PEX_EXTRA_SYS_PATH for python tests.", + level=LogLevel.DEBUG, +) +async def inject_extra_sys_path_for_pack_tests( + request: PytestPackTestRequest, +) -> PytestPluginSetup: + pack_python_path = await Get( + PackPythonPath, PackPythonPathRequest(request.target.address) + ) + return PytestPluginSetup( + # digest=EMPTY_DIGEST, + extra_sys_path=pack_python_path.entries, + ) + + +def rules(): + return [ + *collect_rules(), + UnionRule(PytestPluginSetupRequest, PytestPackTestRequest), + ] diff --git a/pants-plugins/pack_metadata/python_rules/python_path_rules_test.py b/pants-plugins/pack_metadata/python_rules/python_path_rules_test.py new file mode 100644 index 0000000000..74ff010b40 --- /dev/null +++ b/pants-plugins/pack_metadata/python_rules/python_path_rules_test.py @@ -0,0 +1,169 @@ +# Copyright 2024 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from pants.backend.python.goals.pytest_runner import PytestPluginSetup +from pants.engine.internals.native_engine import Address, EMPTY_DIGEST +from pants.testutil.rule_runner import RuleRunner + +from pack_metadata.python_rules.python_path_rules import ( + PackPythonPath, + PackPythonPathRequest, + PytestPackTestRequest, +) + + +@pytest.mark.parametrize( + "address,expected", + ( + ( + Address("packs/foo/actions", relative_file_path="get_bar.py"), + ("packs/foo/actions",), + ), + ( + Address("packs/foo/actions", relative_file_path="get_baz.py"), + ("packs/foo/actions",), + ), + ( + Address("packs/foo/tests", relative_file_path="test_get_bar_action.py"), + ("packs/foo/actions",), + ), + ( + Address("packs/foo/tests", relative_file_path="test_get_baz_action.py"), + ("packs/foo/actions",), + ), + ( + Address( + "packs/dr_seuss/actions/lib/seuss", relative_file_path="__init__.py" + ), + ("packs/dr_seuss/actions/lib",), + ), + ( + Address("packs/dr_seuss/actions/lib/seuss", relative_file_path="things.py"), + ("packs/dr_seuss/actions/lib",), + ), + ( + Address( + "packs/dr_seuss/actions", relative_file_path="get_from_actions_lib.py" + ), + ("packs/dr_seuss/actions", "packs/dr_seuss/actions/lib"), + ), + ( + Address( + "packs/dr_seuss/tests", + relative_file_path="test_get_from_actions_lib_action.py", + ), + ("packs/dr_seuss/actions", "packs/dr_seuss/actions/lib"), + ), + ( + Address( + "packs/shards/lib/stormlight_archive", relative_file_path="__init__.py" + ), + ("packs/shards/lib",), + ), + ( + Address( + "packs/shards/lib/stormlight_archive", relative_file_path="things.py" + ), + ("packs/shards/lib",), + ), + ( + Address("packs/shards/actions", relative_file_path="get_from_pack_lib.py"), + ("packs/shards/actions", "packs/shards/lib"), + ), + ( + Address("packs/shards/sensors", relative_file_path="horn_eater.py"), + ("packs/shards/sensors", "packs/shards/lib"), + ), + ( + Address( + "packs/shards/tests", + relative_file_path="test_get_from_pack_lib_action.py", + ), + ("packs/shards/actions", "packs/shards/lib"), + ), + ( + Address( + "packs/shards/tests", relative_file_path="test_horn_eater_sensor.py" + ), + ("packs/shards/sensors", "packs/shards/lib"), + ), + ( + Address("packs/metals/actions/mist_born", relative_file_path="__init__.py"), + (), # there are no dependencies, and this is not an action entry point. + ), + ( + Address("packs/metals/actions/mist_born", relative_file_path="fly.py"), + ("packs/metals/actions/mist_born", "packs/metals/actions"), + ), + ( + Address("packs/metals/tests", relative_file_path="test_fly_action.py"), + ("packs/metals/actions/mist_born", "packs/metals/actions"), + ), + ), +) +def test_get_extra_sys_path_for_pack_dependencies( + rule_runner: RuleRunner, address: Address, expected: tuple[str, ...] +) -> None: + pack_python_path = rule_runner.request( + PackPythonPath, (PackPythonPathRequest(address),) + ) + assert pack_python_path.entries == expected + + +@pytest.mark.xfail(raises=AttributeError, reason="Not implemented in pants yet.") +@pytest.mark.parametrize( + "address,expected", + ( + ( + Address("packs/foo/tests", relative_file_path="test_get_bar_action.py"), + ("packs/foo/actions",), + ), + ( + Address("packs/foo/tests", relative_file_path="test_get_baz_action.py"), + ("packs/foo/actions",), + ), + ( + Address( + "packs/dr_seuss/tests", + relative_file_path="test_get_from_actions_lib_action.py", + ), + ("packs/dr_seuss/actions", "packs/dr_seuss/actions/lib"), + ), + ( + Address( + "packs/shards/tests", + relative_file_path="test_get_from_pack_lib_action.py", + ), + ("packs/shards/actions", "packs/shards/lib"), + ), + ( + Address( + "packs/shards/tests", relative_file_path="test_horn_eater_sensor.py" + ), + ("packs/shards/sensors", "packs/shards/lib"), + ), + ( + Address("packs/metals/tests", relative_file_path="test_fly_action.py"), + ("packs/metals/actions/mist_born", "packs/metals/actions"), + ), + ), +) +def test_inject_extra_sys_path_for_pack_tests( + rule_runner: RuleRunner, address: Address, expected: tuple[str, ...] +) -> None: + target = rule_runner.get_target(address) + result = rule_runner.request(PytestPluginSetup, (PytestPackTestRequest(target),)) + assert result.digest == EMPTY_DIGEST + assert result.extra_sys_path == expected diff --git a/pants-plugins/pack_metadata/register.py b/pants-plugins/pack_metadata/register.py index 36c11079d9..6cdd7c9f8d 100644 --- a/pants-plugins/pack_metadata/register.py +++ b/pants-plugins/pack_metadata/register.py @@ -11,8 +11,21 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from pants.backend.python.target_types import ( + PythonTestTarget, + PythonTestsGeneratorTarget, +) + from pack_metadata import tailor, target_types_rules +from pack_metadata.python_rules import ( + python_module_mapper, + python_pack_content, + python_path_rules, +) from pack_metadata.target_types import ( + InjectPackPythonPathField, + PackContentResourceTarget, PackMetadata, PackMetadataInGitSubmodule, PacksGlob, @@ -21,13 +34,21 @@ def rules(): return [ + PythonTestsGeneratorTarget.register_plugin_field( + InjectPackPythonPathField, as_moved_field=True + ), + PythonTestTarget.register_plugin_field(InjectPackPythonPathField), *tailor.rules(), *target_types_rules.rules(), + *python_pack_content.rules(), + *python_module_mapper.rules(), + *python_path_rules.rules(), ] def target_types(): return [ + PackContentResourceTarget, PackMetadata, PackMetadataInGitSubmodule, PacksGlob, diff --git a/pants-plugins/pack_metadata/target_types.py b/pants-plugins/pack_metadata/target_types.py index 4c7c2c854f..659eb312df 100644 --- a/pants-plugins/pack_metadata/target_types.py +++ b/pants-plugins/pack_metadata/target_types.py @@ -11,12 +11,24 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Sequence - -from pants.engine.target import COMMON_TARGET_FIELDS, Dependencies +from enum import Enum +from pathlib import PurePath +from typing import Optional, Sequence, Tuple + +from pants.engine.internals.native_engine import Address +from pants.engine.target import ( + BoolField, + COMMON_TARGET_FIELDS, + Dependencies, + StringField, +) from pants.core.target_types import ( + ResourceDependenciesField, ResourcesGeneratingSourcesField, ResourcesGeneratorTarget, + ResourcesOverridesField, + ResourceSourceField, + ResourceTarget, GenericTarget, ) @@ -25,6 +37,81 @@ class UnmatchedGlobsError(Exception): """Error thrown when a required set of globs didn't match.""" +class PackContentResourceTypes(Enum): + # in root of pack + pack_metadata = "pack_metadata" + pack_config_schema = "pack_config_schema" + pack_config_example = "pack_config_example" + pack_icon = "pack_icon" + # in subdirectory (see _content_type_by_path_parts below + action_metadata = "action_metadata" + action_chain_workflow = "action_chain_workflow" + orquesta_workflow = "orquesta_workflow" + alias_metadata = "alias_metadata" + policy_metadata = "policy_metadata" + rule_metadata = "rule_metadata" + sensor_metadata = "sensor_metadata" + trigger_metadata = "trigger_metadata" + # other + unknown = "unknown" + + +_content_type_by_path_parts: dict[Tuple[str, ...], PackContentResourceTypes] = { + ("actions",): PackContentResourceTypes.action_metadata, + ("actions", "chains"): PackContentResourceTypes.action_chain_workflow, + ("actions", "workflows"): PackContentResourceTypes.orquesta_workflow, + ("aliases",): PackContentResourceTypes.alias_metadata, + ("policies",): PackContentResourceTypes.policy_metadata, + ("rules",): PackContentResourceTypes.rule_metadata, + ("sensors",): PackContentResourceTypes.sensor_metadata, + ("triggers",): PackContentResourceTypes.trigger_metadata, +} + + +class PackContentResourceTypeField(StringField): + alias = "type" + help = ( + "The content type of the resource." + "\nDo not use this field in BUILD files. It is calculated automatically" + "based on the conventional location of files in the st2 pack." + ) + valid_choices = PackContentResourceTypes + value: PackContentResourceTypes + + @classmethod + def compute_value( + cls, raw_value: Optional[str], address: Address + ) -> PackContentResourceTypes: + value = super().compute_value(raw_value, address) + if value is not None: + return PackContentResourceTypes(value) + path = PurePath(address.relative_file_path) + _yaml_suffixes = (".yaml", ".yml") + if len(path.parent.parts) == 0: + # in the pack root + if path.stem == "pack" and path.suffix in _yaml_suffixes: + return PackContentResourceTypes.pack_metadata + if path.stem == "config.schema" and path.suffix in _yaml_suffixes: + return PackContentResourceTypes.pack_config_schema + if ( + path.stem.startswith("config.") + and path.suffixes[0] in _yaml_suffixes + and path.suffix == ".example" + ): + return PackContentResourceTypes.pack_config_example + if path.name == "icon.png": + return PackContentResourceTypes.pack_icon + return PackContentResourceTypes.unknown + resource_type = _content_type_by_path_parts.get(path.parent.parts, None) + if resource_type is not None: + return resource_type + return PackContentResourceTypes.unknown + + +class PackContentResourceSourceField(ResourceSourceField): + pass + + class PackMetadataSourcesField(ResourcesGeneratingSourcesField): required = False default = ( @@ -58,9 +145,27 @@ def validate_resolved_files(self, files: Sequence[str]) -> None: super().validate_resolved_files(files) +class PackContentResourceTarget(ResourceTarget): + alias = "pack_content_resource" + core_fields = ( + *COMMON_TARGET_FIELDS, + ResourceDependenciesField, + PackContentResourceSourceField, + PackContentResourceTypeField, + # TODO: implicit depenedency on runner entry point. If a python action, the python file should (?) get that dep too. + ) + help = "A single pack content resource file (mostly for metadata files)." + + class PackMetadata(ResourcesGeneratorTarget): alias = "pack_metadata" - core_fields = (*COMMON_TARGET_FIELDS, Dependencies, PackMetadataSourcesField) + core_fields = ( + *COMMON_TARGET_FIELDS, + PackMetadataSourcesField, + ResourcesOverridesField, + ) + moved_fields = (ResourceDependenciesField,) + generated_target_cls = PackContentResourceTarget help = ( "Loose pack metadata files.\n\n" "Pack metadata includes top-level files (pack.yaml, .yaml.example, " @@ -73,9 +178,11 @@ class PackMetadataInGitSubmodule(PackMetadata): alias = "pack_metadata_in_git_submodule" core_fields = ( *COMMON_TARGET_FIELDS, - Dependencies, PackMetadataInGitSubmoduleSources, + ResourcesOverridesField, ) + moved_fields = (ResourceDependenciesField,) + generated_target_cls = PackContentResourceTarget help = PackMetadata.help + ( "\npack_metadata_in_git_submodule variant errors if the sources field " "has unmatched globs. It prints instructions on how to checkout git " @@ -96,3 +203,14 @@ class PacksGlob(GenericTarget): "subdirectories (packs) except those listed with ! in dependencies. " "This is unfortunately needed by tests that use a glob to load pack fixtures." ) + + +class InjectPackPythonPathField(BoolField): + alias = "inject_pack_python_path" + help = ( + "For pack tests, set this to true to make sure /lib or actions/ dirs get " + "added to PYTHONPATH (actually PEX_EXTRA_SYS_PATH). Use `__defaults__` to enable " + "this in the BUILD file where you define pack_metadata, like this: " + "`__defaults__(all=dict(inject_pack_python_path=True))`" + ) + default = False diff --git a/pants-plugins/uses_services/register.py b/pants-plugins/uses_services/register.py index 83ab32b3a7..24dc8037df 100644 --- a/pants-plugins/uses_services/register.py +++ b/pants-plugins/uses_services/register.py @@ -22,10 +22,13 @@ def rules(): return [ - PythonTestsGeneratorTarget.register_plugin_field(UsesServicesField), + PythonTestsGeneratorTarget.register_plugin_field( + UsesServicesField, as_moved_field=True + ), PythonTestTarget.register_plugin_field(UsesServicesField), *platform_rules.rules(), *mongo_rules.rules(), *rabbitmq_rules.rules(), *redis_rules.rules(), + # TODO: Add check that checks that system user is present (suggest setting ST2TESTS_SYSTEM_USER env var if not, or even just default it to the current user) ] diff --git a/pants.toml b/pants.toml index 38e9fee47a..e4a673e2b8 100644 --- a/pants.toml +++ b/pants.toml @@ -6,7 +6,7 @@ enabled = false repo_id = "de0dea7a-9f6a-4c6e-aa20-6ba5ad969b8a" [GLOBAL] -pants_version = "2.22.0" +pants_version = "2.23.0a0" pythonpath = ["%(buildroot)s/pants-plugins"] build_file_prelude_globs = ["pants-plugins/macros.py"] backend_packages = [ @@ -40,6 +40,7 @@ backend_packages = [ pants_ignore.add = [ # TODO: remove these once we start building wheels with pants. "dist_utils.py", + "test_dist_utils.py", "setup.py", # keep tailor from using legacy requirements files (not for pants) "contrib/examples/requirements.txt", @@ -89,9 +90,6 @@ root_patterns = [ "/contrib/packs", "/st2tests/testpacks/checks", "/st2tests/testpacks/errorcheck", - # pack common lib directories that ST2 adds to the PATH for actions/sensors - "/contrib/*/lib", - "/contrib/*/actions/lib", # other special-cased pack directories "/contrib/examples/actions/ubuntu_pkg_info", # python script runs via shell expecting cwd in PYTHONPATH # lint plugins @@ -159,7 +157,7 @@ py_editable_in_resolve = ["st2"] py_resolve_format = "mutable_virtualenv" # By default, pex modifies script shebangs to add '-sE'. # This breaks nosetest and anything that needs PYTHONPATH. -py_hermetic_scripts = false +py_non_hermetic_scripts_in_resolve = ["st2"] # If any targets generate sources/files, include them in the exported venv. py_generated_sources_in_resolve = ["st2"] @@ -238,8 +236,19 @@ config = "@lint-configs/regex-lint.yaml" [setuptools] install_from_resolve = "st2" +[test] +extra_env_vars = [ + # Use this so that the test system does not require the stanley user. + # For example: export ST2TESTS_SYSTEM_USER=${USER} + "ST2TESTS_SYSTEM_USER", +] + [twine] install_from_resolve = "twine" +[environments-preview.names] +# https://www.pantsbuild.org/stable/docs/using-pants/environments +in_repo_workspace = "//:in_repo_workspace" + [cli.alias] --all-changed = "--changed-since=HEAD --changed-dependents=transitive" diff --git a/pylint_plugins/BUILD b/pylint_plugins/BUILD index 1f7bfde6c6..9705181eec 100644 --- a/pylint_plugins/BUILD +++ b/pylint_plugins/BUILD @@ -8,6 +8,7 @@ python_sources() python_tests( name="tests", + tags=["unit"], dependencies=[ "./fixtures", "!//conftest.py:test_utils", diff --git a/st2actions/conf/BUILD b/st2actions/conf/BUILD index 42e57e0d5a..9d1bf2cc2b 100644 --- a/st2actions/conf/BUILD +++ b/st2actions/conf/BUILD @@ -1,21 +1,23 @@ -file( +st2_logging_conf_file( name="logging_console", source="console.conf", ) -files( +st2_logging_conf_files( name="logging", sources=["logging*.conf"], + dependencies=["//:logs_directory"], overrides={ "logging.conf": dict( dependencies=[ + "//:logs_directory", "//:reqs#python-json-logger", ], ), }, ) -files( +st2_logging_conf_files( name="logging_syslog", sources=["syslog*.conf"], ) diff --git a/st2actions/tests/unit/BUILD b/st2actions/tests/unit/BUILD index 9a24dba70a..7bb8fa73e5 100644 --- a/st2actions/tests/unit/BUILD +++ b/st2actions/tests/unit/BUILD @@ -5,4 +5,23 @@ __defaults__( python_tests( name="tests", + uses=["mongo"], + overrides={ + ( + "test_execution*.py", + "test_notifier.py", + "test_output_schema.py", + "test_policies.py", + "test_queue_consumers.py", + "test_runner_container.py", + "test_scheduler*.py", + "test_worker.py", + "test_workflow_engine.py", + ): dict( + stevedore_namespaces=[ + "st2common.runners.runner", + "st2common.metrics.driver", + ], + ), + }, ) diff --git a/st2actions/tests/unit/policies/BUILD b/st2actions/tests/unit/policies/BUILD index 57341b1358..66a22040d1 100644 --- a/st2actions/tests/unit/policies/BUILD +++ b/st2actions/tests/unit/policies/BUILD @@ -1,3 +1,8 @@ python_tests( name="tests", + stevedore_namespaces=[ + "st2common.runners.runner", + "st2common.metrics.driver", + ], + uses=["mongo"], ) diff --git a/st2actions/tests/unit/policies/test_base.py b/st2actions/tests/unit/policies/test_base.py index fb475fbf66..1b345d2b7a 100644 --- a/st2actions/tests/unit/policies/test_base.py +++ b/st2actions/tests/unit/policies/test_base.py @@ -16,9 +16,8 @@ from __future__ import absolute_import import mock -from st2tests import config as test_config - -test_config.parse_args() +# This import must be early for import-time side-effects. +from st2tests.base import CleanDbTestCase, DbTestCase import st2common from st2common.bootstrap.policiesregistrar import register_policy_types @@ -28,8 +27,6 @@ from st2common.services import action as action_service from st2common.services import policies as policy_service from st2common.bootstrap import runnersregistrar as runners_registrar -from st2tests.base import DbTestCase -from st2tests.base import CleanDbTestCase from st2tests.fixtures.generic.fixture import PACK_NAME as PACK from st2tests.fixturesloader import FixturesLoader diff --git a/st2actions/tests/unit/policies/test_concurrency.py b/st2actions/tests/unit/policies/test_concurrency.py index 1be4b86da3..7612bd5396 100644 --- a/st2actions/tests/unit/policies/test_concurrency.py +++ b/st2actions/tests/unit/policies/test_concurrency.py @@ -19,10 +19,9 @@ from mock import call from six.moves import range +# This import must be early for import-time side-effects. # Importing st2actions.scheduler relies on config being parsed :/ -import st2tests.config as tests_config - -tests_config.parse_args() +from st2tests import DbTestCase, EventletTestCase, ExecutionDbTestCase import st2common from st2actions.scheduler import handler as scheduling_queue @@ -38,8 +37,6 @@ from st2common.transport.liveaction import LiveActionPublisher from st2common.transport.publishers import CUDPublisher from st2common.bootstrap import runnersregistrar as runners_registrar -from st2tests import DbTestCase, EventletTestCase -from st2tests import ExecutionDbTestCase import st2tests.config as tests_config from st2tests.fixtures.generic.fixture import PACK_NAME as PACK from st2tests.fixturesloader import FixturesLoader diff --git a/st2actions/tests/unit/policies/test_concurrency_by_attr.py b/st2actions/tests/unit/policies/test_concurrency_by_attr.py index 937a4149ef..2edcdb4af7 100644 --- a/st2actions/tests/unit/policies/test_concurrency_by_attr.py +++ b/st2actions/tests/unit/policies/test_concurrency_by_attr.py @@ -18,10 +18,9 @@ import mock from mock import call +# This import must be early for import-time side-effects. # Importing st2actions.scheduler relies on config being parsed :/ -import st2tests.config as tests_config - -tests_config.parse_args() +from st2tests import ExecutionDbTestCase, EventletTestCase import st2common from st2actions.scheduler import handler as scheduling_queue @@ -36,7 +35,6 @@ from st2common.transport.liveaction import LiveActionPublisher from st2common.transport.publishers import CUDPublisher from st2common.bootstrap import runnersregistrar as runners_registrar -from st2tests import ExecutionDbTestCase, EventletTestCase import st2tests.config as tests_config from st2tests.fixtures.generic.fixture import PACK_NAME as PACK from st2tests.fixturesloader import FixturesLoader diff --git a/st2actions/tests/unit/test_action_runner_worker.py b/st2actions/tests/unit/test_action_runner_worker.py index 96e049b179..8477281b97 100644 --- a/st2actions/tests/unit/test_action_runner_worker.py +++ b/st2actions/tests/unit/test_action_runner_worker.py @@ -20,14 +20,17 @@ from st2common.transport.consumers import ActionsQueueConsumer from st2common.models.db.liveaction import LiveActionDB -from st2tests import config as test_config - -test_config.parse_args() +from st2tests import config as tests_config __all__ = ["ActionsQueueConsumerTestCase"] class ActionsQueueConsumerTestCase(TestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + tests_config.parse_args() + def test_process_right_dispatcher_is_used(self): handler = Mock() handler.message_type = LiveActionDB diff --git a/st2actions/tests/unit/test_async_runner.py b/st2actions/tests/unit/test_async_runner.py deleted file mode 100644 index 31258fae4e..0000000000 --- a/st2actions/tests/unit/test_async_runner.py +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright 2020 The StackStorm Authors. -# Copyright 2019 Extreme Networks, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import absolute_import - -try: - import simplejson as json -except: - import json - -from st2common.runners.base import AsyncActionRunner -from st2common.constants.action import LIVEACTION_STATUS_RUNNING - -RAISE_PROPERTY = "raise" - - -def get_runner(): - return AsyncTestRunner() - - -class AsyncTestRunner(AsyncActionRunner): - def __init__(self): - super(AsyncTestRunner, self).__init__(runner_id="1") - self.pre_run_called = False - self.run_called = False - self.post_run_called = False - - def pre_run(self): - self.pre_run_called = True - - def run(self, action_params): - self.run_called = True - result = {} - if self.runner_parameters.get(RAISE_PROPERTY, False): - raise Exception("Raise required.") - else: - result = {"ran": True, "action_params": action_params} - - return (LIVEACTION_STATUS_RUNNING, json.dumps(result), {"id": "foo"}) - - def post_run(self, status, result): - self.post_run_called = True diff --git a/st2actions/tests/unit/test_execution_cancellation.py b/st2actions/tests/unit/test_execution_cancellation.py index 96eacc1a89..539e930d25 100644 --- a/st2actions/tests/unit/test_execution_cancellation.py +++ b/st2actions/tests/unit/test_execution_cancellation.py @@ -20,10 +20,8 @@ from oslo_config import cfg -# XXX: actionsensor import depends on config being setup. -import st2tests.config as tests_config - -tests_config.parse_args() +# This import must be early for import-time side-effects. +from st2tests import ExecutionDbTestCase from st2common.constants import action as action_constants from st2common.models.api.action import ActionAPI @@ -36,7 +34,6 @@ from st2common.services import trace as trace_service from st2common.transport.liveaction import LiveActionPublisher from st2common.transport.publishers import CUDPublisher -from st2tests import ExecutionDbTestCase from st2tests.fixtures.generic.fixture import PACK_NAME as PACK from st2tests.fixturesloader import FixturesLoader from st2tests.mocks.execution import MockExecutionPublisher diff --git a/st2actions/tests/unit/test_executions.py b/st2actions/tests/unit/test_executions.py index 1c95a51061..dcb50c70f5 100644 --- a/st2actions/tests/unit/test_executions.py +++ b/st2actions/tests/unit/test_executions.py @@ -18,10 +18,8 @@ import mock -# XXX: actionsensor import depends on config being setup. -import st2tests.config as tests_config - -tests_config.parse_args() +# This import must be early for import-time side-effects. +from st2tests import ExecutionDbTestCase import st2common.bootstrap.runnersregistrar as runners_registrar from st2common.constants import action as action_constants @@ -46,7 +44,6 @@ from local_runner.local_shell_command_runner import LocalShellCommandRunner from st2tests.fixtures.packs import executions as fixture -from st2tests import ExecutionDbTestCase from st2tests.mocks.liveaction import MockLiveActionPublisher diff --git a/st2actions/tests/unit/test_notifier.py b/st2actions/tests/unit/test_notifier.py index b648d7fad3..f599cea08a 100644 --- a/st2actions/tests/unit/test_notifier.py +++ b/st2actions/tests/unit/test_notifier.py @@ -19,9 +19,8 @@ import bson import mock -import st2tests.config as tests_config - -tests_config.parse_args() +# This import must be early for import-time side-effects. +from st2tests.base import CleanDbTestCase from st2actions.notifier.notifier import Notifier from st2common.constants.action import LIVEACTION_COMPLETED_STATES @@ -40,7 +39,6 @@ from st2common.models.system.common import ResourceReference from st2common.util import date as date_utils from st2common.util import isotime -from st2tests.base import CleanDbTestCase ACTION_TRIGGER_TYPE = INTERNAL_TRIGGER_TYPES["action"][0] NOTIFY_TRIGGER_TYPE = INTERNAL_TRIGGER_TYPES["action"][1] diff --git a/st2actions/tests/unit/test_output_schema.py b/st2actions/tests/unit/test_output_schema.py index a66f9ffb12..d4ae6bd9cc 100644 --- a/st2actions/tests/unit/test_output_schema.py +++ b/st2actions/tests/unit/test_output_schema.py @@ -20,12 +20,9 @@ from python_runner import python_runner from orquesta_runner import orquesta_runner +# This import must be early for import-time side-effects. import st2tests -import st2tests.config as tests_config - -tests_config.parse_args() - from st2common.bootstrap import actionsregistrar from st2common.bootstrap import runnersregistrar from st2common.constants import action as ac_const diff --git a/st2actions/tests/unit/test_parallel_ssh.py b/st2actions/tests/unit/test_parallel_ssh.py index c1ef2e998a..70c9b79b68 100644 --- a/st2actions/tests/unit/test_parallel_ssh.py +++ b/st2actions/tests/unit/test_parallel_ssh.py @@ -25,8 +25,6 @@ from st2common.runners.paramiko_ssh import SSHCommandTimeoutError import st2tests.config as tests_config -tests_config.parse_args() - MOCK_STDERR_SUDO_PASSWORD_ERROR = """ [sudo] password for bar: Sorry, try again.\n [sudo] password for bar:' Sorry, try again.\n @@ -36,6 +34,11 @@ class ParallelSSHTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + tests_config.parse_args() + @patch("paramiko.SSHClient", Mock) @patch.object( ParamikoSSHClient, diff --git a/st2actions/tests/unit/test_paramiko_remote_script_runner.py b/st2actions/tests/unit/test_paramiko_remote_script_runner.py index 27495463ff..726456d11a 100644 --- a/st2actions/tests/unit/test_paramiko_remote_script_runner.py +++ b/st2actions/tests/unit/test_paramiko_remote_script_runner.py @@ -22,8 +22,6 @@ # before importing remote_script_runner classes. import st2tests.config as tests_config -tests_config.parse_args() - from st2common.util import jsonify from st2common.models.db.action import ActionDB from st2common.runners.parallel_ssh import ParallelSSHClient @@ -48,6 +46,11 @@ class ParamikoScriptRunnerTestCase(unittest.TestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + tests_config.parse_args() + @patch("st2common.runners.parallel_ssh.ParallelSSHClient", Mock) @patch.object(jsonify, "json_loads", MagicMock(return_value={})) @patch.object(ParallelSSHClient, "run", MagicMock(return_value={})) diff --git a/st2actions/tests/unit/test_paramiko_ssh.py b/st2actions/tests/unit/test_paramiko_ssh.py index 1ccdc110a2..d60c227b1d 100644 --- a/st2actions/tests/unit/test_paramiko_ssh.py +++ b/st2actions/tests/unit/test_paramiko_ssh.py @@ -29,12 +29,15 @@ from st2tests.fixturesloader import get_resources_base_path import st2tests.config as tests_config -tests_config.parse_args() - __all__ = ["ParamikoSSHClientTestCase"] class ParamikoSSHClientTestCase(unittest.TestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + tests_config.parse_args() + @patch("paramiko.SSHClient", Mock) def setUp(self): """ diff --git a/st2actions/tests/unit/test_paramiko_ssh_runner.py b/st2actions/tests/unit/test_paramiko_ssh_runner.py index 6700bb0347..116ea4eced 100644 --- a/st2actions/tests/unit/test_paramiko_ssh_runner.py +++ b/st2actions/tests/unit/test_paramiko_ssh_runner.py @@ -30,8 +30,6 @@ import st2tests.config as tests_config from st2tests.fixturesloader import get_resources_base_path -tests_config.parse_args() - class Runner(BaseParallelSSHRunner): def run(self): @@ -39,6 +37,11 @@ def run(self): class ParamikoSSHRunnerTestCase(unittest.TestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + tests_config.parse_args() + @mock.patch("st2common.runners.paramiko_ssh_runner.ParallelSSHClient") def test_pre_run(self, mock_client): # Test case which verifies that ParamikoSSHClient is instantiated with the correct arguments diff --git a/st2actions/tests/unit/test_polling_async_runner.py b/st2actions/tests/unit/test_polling_async_runner.py deleted file mode 100644 index c48bb9aa67..0000000000 --- a/st2actions/tests/unit/test_polling_async_runner.py +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright 2020 The StackStorm Authors. -# Copyright 2019 Extreme Networks, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import absolute_import - -try: - import simplejson as json -except: - import json - -from st2common.runners.base import PollingAsyncActionRunner -from st2common.constants.action import LIVEACTION_STATUS_RUNNING - -RAISE_PROPERTY = "raise" - - -def get_runner(): - return PollingAsyncTestRunner() - - -class PollingAsyncTestRunner(PollingAsyncActionRunner): - def __init__(self): - super(PollingAsyncTestRunner, self).__init__(runner_id="1") - self.pre_run_called = False - self.run_called = False - self.post_run_called = False - - def pre_run(self): - self.pre_run_called = True - - def run(self, action_params): - self.run_called = True - result = {} - if self.runner_parameters.get(RAISE_PROPERTY, False): - raise Exception("Raise required.") - else: - result = {"ran": True, "action_params": action_params} - - return (LIVEACTION_STATUS_RUNNING, json.dumps(result), {"id": "foo"}) - - def post_run(self, status, result): - self.post_run_called = True diff --git a/st2actions/tests/unit/test_queue_consumers.py b/st2actions/tests/unit/test_queue_consumers.py index 0ddfaea164..594ecac401 100644 --- a/st2actions/tests/unit/test_queue_consumers.py +++ b/st2actions/tests/unit/test_queue_consumers.py @@ -15,13 +15,12 @@ from __future__ import absolute_import -import st2tests.config as tests_config - -tests_config.parse_args() - import mock from kombu.message import Message +# This import must be early for import-time side-effects. +from st2tests.base import ExecutionDbTestCase + from st2actions import worker from st2actions.scheduler import entrypoint as scheduling from st2actions.scheduler import handler as scheduling_queue @@ -35,7 +34,6 @@ from st2common.transport.publishers import PoolPublisher from st2common.util import action_db as action_utils from st2common.util import date as date_utils -from st2tests.base import ExecutionDbTestCase from st2tests.fixtures.packs.core.fixture import PACK_PATH as CORE_PACK_PATH diff --git a/st2actions/tests/unit/test_remote_runners.py b/st2actions/tests/unit/test_remote_runners.py index 19f5cb40f1..06f9058fa9 100644 --- a/st2actions/tests/unit/test_remote_runners.py +++ b/st2actions/tests/unit/test_remote_runners.py @@ -13,18 +13,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -# XXX: FabricRunner import depends on config being setup. -from __future__ import absolute_import -import st2tests.config as tests_config - -tests_config.parse_args() - from unittest import TestCase +# This import must be early for import-time side-effects. +import st2tests.config as tests_config + from st2common.models.system.action import RemoteScriptAction class RemoteScriptActionTestCase(TestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + tests_config.parse_args() + def test_parameter_formatting(self): # Only named args named_args = { diff --git a/st2actions/tests/unit/test_runner_container.py b/st2actions/tests/unit/test_runner_container.py index 1134e176bf..e154a36534 100644 --- a/st2actions/tests/unit/test_runner_container.py +++ b/st2actions/tests/unit/test_runner_container.py @@ -16,9 +16,13 @@ from __future__ import absolute_import import mock +import os from oslo_config import cfg +# This import must be early for import-time side-effects. +from st2tests.base import DbTestCase + from st2common.constants import action as action_constants from st2common.runners.base import get_runner from st2common.exceptions.actionrunner import ( @@ -35,10 +39,6 @@ from st2common.util import date as date_utils from st2common.transport.publishers import PoolPublisher -from st2tests.base import DbTestCase -import st2tests.config as tests_config - -tests_config.parse_args() from st2tests.fixtures.generic.fixture import PACK_NAME as FIXTURES_PACK from st2tests.fixturesloader import FixturesLoader @@ -297,7 +297,8 @@ def test_dispatch(self): self.assertTrue(result.get("action_params").get("actionstr") == "bar") # Assert that context is written correctly. - context = {"user": "stanley", "third_party_system": {"ref_id": "1234"}} + system_user = os.environ.get("ST2TESTS_SYSTEM_USER", "") or "stanley" + context = {"user": system_user, "third_party_system": {"ref_id": "1234"}} self.assertDictEqual(liveaction_db.context, context) diff --git a/st2actions/tests/unit/test_scheduler.py b/st2actions/tests/unit/test_scheduler.py index 7556c7b036..65a59bd869 100644 --- a/st2actions/tests/unit/test_scheduler.py +++ b/st2actions/tests/unit/test_scheduler.py @@ -19,10 +19,6 @@ import mock import eventlet -from st2tests import config as test_config - -test_config.parse_args() - import st2common from st2tests import ExecutionDbTestCase from st2tests.fixtures.generic.fixture import PACK_NAME as PACK diff --git a/st2actions/tests/unit/test_scheduler_entrypoint.py b/st2actions/tests/unit/test_scheduler_entrypoint.py index 2bc535d99d..2862ba2b3c 100644 --- a/st2actions/tests/unit/test_scheduler_entrypoint.py +++ b/st2actions/tests/unit/test_scheduler_entrypoint.py @@ -16,10 +16,6 @@ import eventlet import mock -from st2tests import config as test_config - -test_config.parse_args() - from st2actions.cmd.scheduler import _run_scheduler from st2actions.scheduler.handler import ActionExecutionSchedulingQueueHandler from st2actions.scheduler.entrypoint import SchedulerEntrypoint diff --git a/st2actions/tests/unit/test_scheduler_retry.py b/st2actions/tests/unit/test_scheduler_retry.py index ad1f221df1..d975964bc6 100644 --- a/st2actions/tests/unit/test_scheduler_retry.py +++ b/st2actions/tests/unit/test_scheduler_retry.py @@ -13,19 +13,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +# This import must be first for import-time side-effects. +from st2tests.base import CleanDbTestCase + import eventlet import mock import pymongo import uuid -from st2tests import config as test_config - -test_config.parse_args() - from st2actions.scheduler import handler from st2common.models.db import execution_queue as ex_q_db from st2common.persistence import execution_queue as ex_q_db_access -from st2tests.base import CleanDbTestCase __all__ = ["SchedulerHandlerRetryTestCase"] diff --git a/st2actions/tests/unit/test_worker.py b/st2actions/tests/unit/test_worker.py index ca2bf172dc..b335b6f2be 100644 --- a/st2actions/tests/unit/test_worker.py +++ b/st2actions/tests/unit/test_worker.py @@ -14,6 +14,7 @@ # limitations under the License. from __future__ import absolute_import + from bson.errors import InvalidStringData import eventlet import mock @@ -21,6 +22,9 @@ from oslo_config import cfg import tempfile +# This import must be early for import-time side-effects. +from st2tests.base import DbTestCase + import st2actions.worker as actions_worker from st2common.constants import action as action_constants from st2common.models.db.liveaction import LiveActionDB @@ -33,14 +37,10 @@ from st2common.bootstrap import runnersregistrar as runners_registrar from local_runner.local_shell_command_runner import LocalShellCommandRunner -from st2tests.base import DbTestCase from st2tests.fixtures.generic.fixture import PACK_NAME as FIXTURES_PACK from st2tests.fixturesloader import FixturesLoader -import st2tests.config as tests_config from six.moves import range -tests_config.parse_args() - TEST_FIXTURES = {"actions": ["local.yaml"]} NON_UTF8_RESULT = { diff --git a/st2actions/tests/unit/test_workflow_engine.py b/st2actions/tests/unit/test_workflow_engine.py index 955f7ca2f0..68d68a3b2f 100644 --- a/st2actions/tests/unit/test_workflow_engine.py +++ b/st2actions/tests/unit/test_workflow_engine.py @@ -18,17 +18,13 @@ import eventlet import mock +# This import must be early for import-time side-effects. import st2tests from orquesta import statuses as wf_statuses from oslo_config import cfg from tooz import coordination -# XXX: actionsensor import depends on config being setup. -import st2tests.config as tests_config - -tests_config.parse_args() - from st2actions.workflows import workflows from st2common.bootstrap import actionsregistrar from st2common.bootstrap import runnersregistrar diff --git a/st2api/conf/BUILD b/st2api/conf/BUILD index 9c3668885c..5fc903fd3b 100644 --- a/st2api/conf/BUILD +++ b/st2api/conf/BUILD @@ -1,19 +1,21 @@ -file( +st2_logging_conf_file( name="logging_console", source="console.conf", ) -file( +st2_logging_conf_file( name="logging", source="logging.conf", + dependencies=["//:logs_directory"], ) -file( +st2_logging_conf_file( name="logging_gunicorn", source="logging.gunicorn.conf", + dependencies=["//:logs_directory"], ) -file( +st2_logging_conf_file( name="logging_syslog", source="syslog.conf", ) diff --git a/st2api/tests/unit/BUILD b/st2api/tests/unit/BUILD index 9a24dba70a..d649809607 100644 --- a/st2api/tests/unit/BUILD +++ b/st2api/tests/unit/BUILD @@ -5,4 +5,9 @@ __defaults__( python_tests( name="tests", + dependencies=["//:rbac_backends"], + stevedore_namespaces=[ + "st2common.rbac.backend", + ], + uses=["mongo"], ) diff --git a/st2api/tests/unit/controllers/BUILD b/st2api/tests/unit/controllers/BUILD index 57341b1358..e32c67ef43 100644 --- a/st2api/tests/unit/controllers/BUILD +++ b/st2api/tests/unit/controllers/BUILD @@ -1,3 +1,7 @@ python_tests( name="tests", + stevedore_namespaces=[ + "st2common.metrics.driver", + ], + uses=["mongo"], ) diff --git a/st2api/tests/unit/controllers/v1/BUILD b/st2api/tests/unit/controllers/v1/BUILD index 57341b1358..5a37a11f3e 100644 --- a/st2api/tests/unit/controllers/v1/BUILD +++ b/st2api/tests/unit/controllers/v1/BUILD @@ -1,3 +1,16 @@ python_tests( name="tests", + stevedore_namespaces=[ + "st2common.runners.runner", + "st2common.rbac.backend", + "st2common.metrics.driver", + ], + uses=["mongo"], + overrides={ + "test_webhooks.py": dict( + dependencies=[ + "st2common/st2common/models/api/webhook.py", + ], + ), + }, ) diff --git a/st2api/tests/unit/controllers/v1/test_alias_execution.py b/st2api/tests/unit/controllers/v1/test_alias_execution.py index 44261fde3f..c64b121023 100644 --- a/st2api/tests/unit/controllers/v1/test_alias_execution.py +++ b/st2api/tests/unit/controllers/v1/test_alias_execution.py @@ -14,8 +14,8 @@ # limitations under the License. import copy - import mock +import os from st2common.constants.action import LIVEACTION_STATUS_SUCCEEDED from st2common.models.db.execution import ActionExecutionDB @@ -50,6 +50,8 @@ __all__ = ["AliasExecutionTestCase"] +SYSTEM_USER = os.environ.get("ST2TESTS_SYSTEM_USER", "") or "stanley" + class AliasExecutionTestCase(FunctionalTest): @@ -241,7 +243,7 @@ def test_match_and_execute_matches_one(self, mock_request): self.assertIn("source_channel", mock_request.call_args[0][0].context.keys()) self.assertEqual(actual_context["source_channel"], "chat-channel") self.assertEqual(actual_context["api_user"], "chat-user") - self.assertEqual(actual_context["user"], "stanley") + self.assertEqual(actual_context["user"], SYSTEM_USER) @mock.patch.object(action_service, "request", return_value=(None, EXECUTION)) def test_match_and_execute_matches_one_multiple_match(self, mock_request): @@ -398,7 +400,7 @@ def _do_post( "name": alias_execution.name, "format": format_str, "command": command, - "user": "stanley", + "user": SYSTEM_USER, "source_channel": "test", "notification_route": "test", } diff --git a/st2api/tests/unit/controllers/v1/test_auth.py b/st2api/tests/unit/controllers/v1/test_auth.py index 695d490faa..1086db34e1 100644 --- a/st2api/tests/unit/controllers/v1/test_auth.py +++ b/st2api/tests/unit/controllers/v1/test_auth.py @@ -15,6 +15,7 @@ import uuid import datetime +import os import bson import mock @@ -29,7 +30,7 @@ from st2tests.fixturesloader import FixturesLoader OBJ_ID = bson.ObjectId() -USER = "stanley" +USER = os.environ.get("ST2TESTS_SYSTEM_USER", "") or "stanley" USER_DB = UserDB(name=USER) TOKEN = uuid.uuid4().hex NOW = date_utils.get_datetime_utc_now() diff --git a/st2api/tests/unit/controllers/v1/test_executions.py b/st2api/tests/unit/controllers/v1/test_executions.py index eb3face2ba..08c07f0fc3 100644 --- a/st2api/tests/unit/controllers/v1/test_executions.py +++ b/st2api/tests/unit/controllers/v1/test_executions.py @@ -15,6 +15,7 @@ import copy import mock +import os try: import simplejson as json @@ -60,6 +61,7 @@ "ActionExecutionOutputControllerTestCase", ] +SYSTEM_USER = os.environ.get("ST2TESTS_SYSTEM_USER", "") or "stanley" ACTION_1 = { "name": "st2.dummy.action1", @@ -686,7 +688,7 @@ def test_post_delete(self): delete_resp = self._do_delete(self._get_actionexecution_id(post_resp)) self.assertEqual(delete_resp.status_int, 200) self.assertEqual(delete_resp.json["status"], "canceled") - expected_result = {"message": "Action canceled by user.", "user": "stanley"} + expected_result = {"message": "Action canceled by user.", "user": SYSTEM_USER} self.assertDictEqual(delete_resp.json["result"], expected_result) def test_post_delete_duplicate(self): @@ -702,7 +704,10 @@ def test_post_delete_duplicate(self): delete_resp = self._do_delete(self._get_actionexecution_id(post_resp)) self.assertEqual(delete_resp.status_int, 200) self.assertEqual(delete_resp.json["status"], "canceled") - expected_result = {"message": "Action canceled by user.", "user": "stanley"} + expected_result = { + "message": "Action canceled by user.", + "user": SYSTEM_USER, + } self.assertDictEqual(delete_resp.json["result"], expected_result) def test_post_delete_trace(self): @@ -976,7 +981,7 @@ def test_template_encrypted_params(self): ), }, { - "name": "stanley:secret", + "name": f"{SYSTEM_USER}:secret", "secret": True, "scope": FULL_USER_SCOPE, "value": crypto_utils.symmetric_encrypt( @@ -994,18 +999,18 @@ def test_template_encrypted_params(self): ] kvps = [KeyValuePair.add_or_update(KeyValuePairDB(**x)) for x in register_items] - # By default, encrypt_user_param will be read from stanley's scope + # By default, encrypt_user_param will be read from system_user's scope # 1. parameters are not marked as secret resp = self._do_post(LIVE_ACTION_DEFAULT_ENCRYPT) self.assertEqual(resp.status_int, 201) - self.assertEqual(resp.json["context"]["user"], "stanley") + self.assertEqual(resp.json["context"]["user"], SYSTEM_USER) self.assertEqual(resp.json["parameters"]["encrypted_param"], "foo") self.assertEqual(resp.json["parameters"]["encrypted_user_param"], "bar") # 2. parameters are marked as secret resp = self._do_post(LIVE_ACTION_DEFAULT_ENCRYPT_SECRET_PARAM) self.assertEqual(resp.status_int, 201) - self.assertEqual(resp.json["context"]["user"], "stanley") + self.assertEqual(resp.json["context"]["user"], SYSTEM_USER) self.assertEqual( resp.json["parameters"]["encrypted_param"], MASKED_ATTRIBUTE_VALUE ) @@ -1077,7 +1082,7 @@ def test_re_run_workflow_success(self): ) expected_context = { - "user": "stanley", + "user": SYSTEM_USER, "pack": "starterpack", "re-run": {"ref": execution_id}, "trace_context": {"id_": str(trace.id)}, @@ -1106,7 +1111,7 @@ def test_re_run_workflow_task_success(self): expected_context = { "pack": "starterpack", - "user": "stanley", + "user": SYSTEM_USER, "re-run": {"ref": execution_id, "tasks": data["tasks"]}, "trace_context": {"id_": str(trace.id)}, } @@ -1134,7 +1139,7 @@ def test_re_run_workflow_tasks_success(self): expected_context = { "pack": "starterpack", - "user": "stanley", + "user": SYSTEM_USER, "re-run": {"ref": execution_id, "tasks": data["tasks"]}, "trace_context": {"id_": str(trace.id)}, } @@ -1162,7 +1167,7 @@ def test_re_run_workflow_tasks_reset_success(self): expected_context = { "pack": "starterpack", - "user": "stanley", + "user": SYSTEM_USER, "re-run": { "ref": execution_id, "tasks": data["tasks"], diff --git a/st2api/tests/unit/controllers/v1/test_packs.py b/st2api/tests/unit/controllers/v1/test_packs.py index c665b7d062..a238ed0f66 100644 --- a/st2api/tests/unit/controllers/v1/test_packs.py +++ b/st2api/tests/unit/controllers/v1/test_packs.py @@ -34,6 +34,9 @@ PACK_NAME as DUMMY_PACK_1, PACK_PATH as DUMMY_PACK_1_PATH, ) +from st2tests.fixtures.packs.dummy_pack_2.fixture import ( + PACK_NAME as DUMMY_PACK_2, +) from st2tests.fixtures.packs.dummy_pack_10.fixture import ( PACK_DIR_NAME as DUMMY_PACK_10, PACK_PATH as DUMMY_PACK_10_PATH, @@ -589,7 +592,7 @@ def test_packs_register_endpoint(self, mock_get_packs): resp = self.app.post_json( "/v1/packs/register", { - "packs": ["dummy_pack_2"], + "packs": [DUMMY_PACK_2], "fail_on_failure": False, "types": ["policies"], }, diff --git a/st2api/tests/unit/controllers/v1/test_packs_views.py b/st2api/tests/unit/controllers/v1/test_packs_views.py index e2e3f93783..97381e5e34 100644 --- a/st2api/tests/unit/controllers/v1/test_packs_views.py +++ b/st2api/tests/unit/controllers/v1/test_packs_views.py @@ -20,6 +20,13 @@ from st2common.persistence.pack import Pack from st2tests.api import FunctionalTest +from st2tests.fixtures.packs.dummy_pack_1.fixture import ( + PACK_NAME as DUMMY_PACK_1, +) +from st2tests.fixtures.packs.dummy_pack_16.fixture import ( + PACK_NAME as DUMMY_PACK_16, +) + @mock.patch("st2common.bootstrap.base.REGISTERED_PACKS_CACHE", {}) class PacksViewsControllerTestCase(FunctionalTest): @@ -31,7 +38,7 @@ def setUpClass(cls): actions_registrar.register_actions(use_pack_cache=False) def test_get_pack_files_success(self): - resp = self.app.get("/v1/packs/views/files/dummy_pack_1") + resp = self.app.get(f"/v1/packs/views/files/{DUMMY_PACK_1}") self.assertEqual(resp.status_int, http_client.OK) self.assertTrue(len(resp.json) > 1) item = [_item for _item in resp.json if _item["file_path"] == "pack.yaml"][0] @@ -53,12 +60,12 @@ def test_get_pack_files_binary_files_are_excluded(self): "etc/generate_new_token.png", ] - pack_db = Pack.get_by_ref("dummy_pack_1") + pack_db = Pack.get_by_ref(DUMMY_PACK_1) all_files_count = len(pack_db.files) non_binary_files_count = all_files_count - len(binary_files) - resp = self.app.get("/v1/packs/views/files/dummy_pack_1") + resp = self.app.get(f"/v1/packs/views/files/{DUMMY_PACK_1}") self.assertEqual(resp.status_int, http_client.OK) self.assertEqual(len(resp.json), non_binary_files_count) @@ -71,7 +78,7 @@ def test_get_pack_files_binary_files_are_excluded(self): self.assertFalse(item) def test_get_pack_file_success(self): - resp = self.app.get("/v1/packs/views/file/dummy_pack_1/pack.yaml") + resp = self.app.get(f"/v1/packs/views/file/{DUMMY_PACK_1}/pack.yaml") self.assertEqual(resp.status_int, http_client.OK) self.assertIn(b"name : dummy_pack_1", resp.body) @@ -84,46 +91,46 @@ def test_get_pack_file_pack_doesnt_exist(self): @mock.patch("st2api.controllers.v1.pack_views.MAX_FILE_SIZE", 1) def test_pack_file_file_larger_then_maximum_size(self): resp = self.app.get( - "/v1/packs/views/file/dummy_pack_1/pack.yaml", expect_errors=True + f"/v1/packs/views/file/{DUMMY_PACK_1}/pack.yaml", expect_errors=True ) self.assertEqual(resp.status_int, http_client.BAD_REQUEST) self.assertIn("File pack.yaml exceeds maximum allowed file size", resp) def test_headers_get_pack_file(self): - resp = self.app.get("/v1/packs/views/file/dummy_pack_1/pack.yaml") + resp = self.app.get(f"/v1/packs/views/file/{DUMMY_PACK_1}/pack.yaml") self.assertEqual(resp.status_int, http_client.OK) self.assertIn(b"name : dummy_pack_1", resp.body) self.assertIsNotNone(resp.headers["ETag"]) self.assertIsNotNone(resp.headers["Last-Modified"]) def test_no_change_get_pack_file(self): - resp = self.app.get("/v1/packs/views/file/dummy_pack_1/pack.yaml") + resp = self.app.get(f"/v1/packs/views/file/{DUMMY_PACK_1}/pack.yaml") self.assertEqual(resp.status_int, http_client.OK) self.assertIn(b"name : dummy_pack_1", resp.body) # Confirm NOT_MODIFIED resp = self.app.get( - "/v1/packs/views/file/dummy_pack_1/pack.yaml", + f"/v1/packs/views/file/{DUMMY_PACK_1}/pack.yaml", headers={"If-None-Match": resp.headers["ETag"]}, ) self.assertEqual(resp.status_code, http_client.NOT_MODIFIED) resp = self.app.get( - "/v1/packs/views/file/dummy_pack_1/pack.yaml", + f"/v1/packs/views/file/{DUMMY_PACK_1}/pack.yaml", headers={"If-Modified-Since": resp.headers["Last-Modified"]}, ) self.assertEqual(resp.status_code, http_client.NOT_MODIFIED) # Confirm value is returned if header do not match resp = self.app.get( - "/v1/packs/views/file/dummy_pack_1/pack.yaml", + f"/v1/packs/views/file/{DUMMY_PACK_1}/pack.yaml", headers={"If-None-Match": "ETAG"}, ) self.assertEqual(resp.status_code, http_client.OK) self.assertIn(b"name : dummy_pack_1", resp.body) resp = self.app.get( - "/v1/packs/views/file/dummy_pack_1/pack.yaml", + f"/v1/packs/views/file/{DUMMY_PACK_1}/pack.yaml", headers={"If-Modified-Since": "Last-Modified"}, ) self.assertEqual(resp.status_code, http_client.OK) @@ -131,11 +138,13 @@ def test_no_change_get_pack_file(self): def test_get_pack_files_and_pack_file_ref_doesnt_equal_pack_name(self): # Ref is not equal to the name, controller should still work - resp = self.app.get("/v1/packs/views/files/dummy_pack_16") + resp = self.app.get(f"/v1/packs/views/files/{DUMMY_PACK_16}") self.assertEqual(resp.status_int, http_client.OK) - self.assertEqual(len(resp.json), 4) + # 4 if running in workspace (BUILD file present) + # 3 if pants is running it (no BUILD file present) + self.assertIn(len(resp.json), [4, 3]) self.assertIn("pack.yaml", [f["file_path"] for f in resp.json]) - resp = self.app.get("/v1/packs/views/file/dummy_pack_16/pack.yaml") + resp = self.app.get(f"/v1/packs/views/file/{DUMMY_PACK_16}/pack.yaml") self.assertEqual(resp.status_int, http_client.OK) self.assertIn(b"ref: dummy_pack_16", resp.body) diff --git a/st2api/tests/unit/controllers/v1/test_sensortypes.py b/st2api/tests/unit/controllers/v1/test_sensortypes.py index c59a1c28e2..ea32f0e904 100644 --- a/st2api/tests/unit/controllers/v1/test_sensortypes.py +++ b/st2api/tests/unit/controllers/v1/test_sensortypes.py @@ -23,6 +23,10 @@ from st2tests.api import FunctionalTest from st2tests.api import APIControllerWithIncludeAndExcludeFilterTestCase +from st2tests.fixtures.packs.dummy_pack_1.fixture import ( + PACK_NAME as DUMMY_PACK_1, +) + http_client = six.moves.http_client __all__ = ["SensorTypeControllerTestCase"] @@ -75,7 +79,7 @@ def test_get_all_filters(self): resp = self.app.get("/v1/sensortypes?name=SampleSensor2") self.assertEqual(len(resp.json), 1) self.assertEqual(resp.json[0]["name"], "SampleSensor2") - self.assertEqual(resp.json[0]["ref"], "dummy_pack_1.SampleSensor2") + self.assertEqual(resp.json[0]["ref"], f"{DUMMY_PACK_1}.SampleSensor2") resp = self.app.get("/v1/sensortypes?name=SampleSensor3") self.assertEqual(len(resp.json), 1) @@ -85,7 +89,7 @@ def test_get_all_filters(self): resp = self.app.get("/v1/sensortypes?pack=foobar") self.assertEqual(len(resp.json), 0) - resp = self.app.get("/v1/sensortypes?pack=dummy_pack_1") + resp = self.app.get(f"/v1/sensortypes?pack={DUMMY_PACK_1}") self.assertEqual(len(resp.json), 3) # ?enabled filter @@ -99,20 +103,20 @@ def test_get_all_filters(self): self.assertEqual(resp.json[1]["enabled"], True) # ?trigger filter - resp = self.app.get("/v1/sensortypes?trigger=dummy_pack_1.event3") + resp = self.app.get(f"/v1/sensortypes?trigger={DUMMY_PACK_1}.event3") self.assertEqual(len(resp.json), 1) - self.assertEqual(resp.json[0]["trigger_types"], ["dummy_pack_1.event3"]) + self.assertEqual(resp.json[0]["trigger_types"], [f"{DUMMY_PACK_1}.event3"]) - resp = self.app.get("/v1/sensortypes?trigger=dummy_pack_1.event") + resp = self.app.get(f"/v1/sensortypes?trigger={DUMMY_PACK_1}.event") self.assertEqual(len(resp.json), 2) - self.assertEqual(resp.json[0]["trigger_types"], ["dummy_pack_1.event"]) - self.assertEqual(resp.json[1]["trigger_types"], ["dummy_pack_1.event"]) + self.assertEqual(resp.json[0]["trigger_types"], [f"{DUMMY_PACK_1}.event"]) + self.assertEqual(resp.json[1]["trigger_types"], [f"{DUMMY_PACK_1}.event"]) def test_get_one_success(self): - resp = self.app.get("/v1/sensortypes/dummy_pack_1.SampleSensor") + resp = self.app.get(f"/v1/sensortypes/{DUMMY_PACK_1}.SampleSensor") self.assertEqual(resp.status_int, http_client.OK) self.assertEqual(resp.json["name"], "SampleSensor") - self.assertEqual(resp.json["ref"], "dummy_pack_1.SampleSensor") + self.assertEqual(resp.json["ref"], f"{DUMMY_PACK_1}.SampleSensor") def test_get_one_doesnt_exist(self): resp = self.app.get("/v1/sensortypes/1", expect_errors=True) @@ -120,7 +124,7 @@ def test_get_one_doesnt_exist(self): def test_disable_and_enable_sensor(self): # Verify initial state - resp = self.app.get("/v1/sensortypes/dummy_pack_1.SampleSensor") + resp = self.app.get(f"/v1/sensortypes/{DUMMY_PACK_1}.SampleSensor") self.assertEqual(resp.status_int, http_client.OK) self.assertTrue(resp.json["enabled"]) @@ -129,24 +133,28 @@ def test_disable_and_enable_sensor(self): # Disable sensor data = copy.deepcopy(sensor_data) data["enabled"] = False - put_resp = self.app.put_json("/v1/sensortypes/dummy_pack_1.SampleSensor", data) + put_resp = self.app.put_json( + f"/v1/sensortypes/{DUMMY_PACK_1}.SampleSensor", data + ) self.assertEqual(put_resp.status_int, http_client.OK) - self.assertEqual(put_resp.json["ref"], "dummy_pack_1.SampleSensor") + self.assertEqual(put_resp.json["ref"], f"{DUMMY_PACK_1}.SampleSensor") self.assertFalse(put_resp.json["enabled"]) # Verify sensor has been disabled - resp = self.app.get("/v1/sensortypes/dummy_pack_1.SampleSensor") + resp = self.app.get(f"/v1/sensortypes/{DUMMY_PACK_1}.SampleSensor") self.assertEqual(resp.status_int, http_client.OK) self.assertFalse(resp.json["enabled"]) # Enable sensor data = copy.deepcopy(sensor_data) data["enabled"] = True - put_resp = self.app.put_json("/v1/sensortypes/dummy_pack_1.SampleSensor", data) + put_resp = self.app.put_json( + f"/v1/sensortypes/{DUMMY_PACK_1}.SampleSensor", data + ) self.assertEqual(put_resp.status_int, http_client.OK) self.assertTrue(put_resp.json["enabled"]) # Verify sensor has been enabled - resp = self.app.get("/v1/sensortypes/dummy_pack_1.SampleSensor") + resp = self.app.get(f"/v1/sensortypes/{DUMMY_PACK_1}.SampleSensor") self.assertEqual(resp.status_int, http_client.OK) self.assertTrue(resp.json["enabled"]) diff --git a/st2auth/conf/BUILD b/st2auth/conf/BUILD index 8fd094d9fa..3300d41753 100644 --- a/st2auth/conf/BUILD +++ b/st2auth/conf/BUILD @@ -8,22 +8,24 @@ file( source="htpasswd_dev", ) -file( +st2_logging_conf_file( name="logging_console", source="console.conf", ) -file( +st2_logging_conf_file( name="logging", source="logging.conf", + dependencies=["//:logs_directory"], ) -file( +st2_logging_conf_file( name="logging_gunicorn", source="logging.gunicorn.conf", + dependencies=["//:logs_directory"], ) -file( +st2_logging_conf_file( name="logging_syslog", source="syslog.conf", ) diff --git a/st2auth/tests/unit/BUILD b/st2auth/tests/unit/BUILD index 9a24dba70a..c128c7b4af 100644 --- a/st2auth/tests/unit/BUILD +++ b/st2auth/tests/unit/BUILD @@ -5,4 +5,6 @@ __defaults__( python_tests( name="tests", + dependencies=["//:auth_backends"], + uses=["mongo"], ) diff --git a/st2auth/tests/unit/controllers/v1/BUILD b/st2auth/tests/unit/controllers/v1/BUILD index 57341b1358..66f5d44271 100644 --- a/st2auth/tests/unit/controllers/v1/BUILD +++ b/st2auth/tests/unit/controllers/v1/BUILD @@ -1,3 +1,8 @@ python_tests( name="tests", + stevedore_namespaces=[ + "st2auth.sso.backends", + "st2common.metrics.driver", + ], + uses=["mongo"], ) diff --git a/st2common/st2common/content/loader.py b/st2common/st2common/content/loader.py index 7e57ef6ec0..582834a45d 100644 --- a/st2common/st2common/content/loader.py +++ b/st2common/st2common/content/loader.py @@ -149,6 +149,8 @@ def get_content_from_pack(self, pack_dir, content_type): def _get_packs_from_dir(self, base_dir): result = {} for pack_name in os.listdir(base_dir): + if pack_name == "__pycache__": + continue pack_dir = os.path.join(base_dir, pack_name) pack_manifest_file = os.path.join(pack_dir, MANIFEST_FILE_NAME) @@ -160,6 +162,8 @@ def _get_packs_from_dir(self, base_dir): def _get_content_from_dir(self, base_dir, content_type): content = {} for pack in os.listdir(base_dir): + if pack == "__pycache__": + continue # TODO: Use function from util which escapes the name pack_dir = os.path.join(base_dir, pack) diff --git a/st2common/tests/fixtures/local_runner/BUILD b/st2common/tests/fixtures/local_runner/BUILD index db46e8d6c9..25c5073d83 100644 --- a/st2common/tests/fixtures/local_runner/BUILD +++ b/st2common/tests/fixtures/local_runner/BUILD @@ -1 +1,8 @@ -python_sources() +resources( + name="command_strings", + sources=["escaping_test_command_*.txt"], +) + +python_sources( + dependencies=[":command_strings"], +) diff --git a/st2common/tests/resources/BUILD b/st2common/tests/resources/BUILD new file mode 100644 index 0000000000..a514672a5c --- /dev/null +++ b/st2common/tests/resources/BUILD @@ -0,0 +1,8 @@ +resource( + name="logging", + source="logging.conf", +) + +python_sources( + dependencies=[":logging"], +) diff --git a/st2common/tests/resources/fixture.py b/st2common/tests/resources/fixture.py new file mode 100644 index 0000000000..f6d0d3e940 --- /dev/null +++ b/st2common/tests/resources/fixture.py @@ -0,0 +1,16 @@ +# Copyright 2024 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from st2tests import fixturesloader + +FIXTURE_NAME, FIXTURE_PATH = fixturesloader.get_fixture_name_and_path(__file__) diff --git a/st2common/tests/resources/loadableplugin/BUILD b/st2common/tests/resources/loadableplugin/BUILD new file mode 100644 index 0000000000..bd7354bb91 --- /dev/null +++ b/st2common/tests/resources/loadableplugin/BUILD @@ -0,0 +1,11 @@ +resources( + name="metadata", + sources=["*.yaml"], +) + +python_sources( + dependencies=[ + ":metadata", + "./plugin", + ], +) diff --git a/st2common/tests/resources/loadableplugin/fixture.py b/st2common/tests/resources/loadableplugin/fixture.py new file mode 100644 index 0000000000..f6d0d3e940 --- /dev/null +++ b/st2common/tests/resources/loadableplugin/fixture.py @@ -0,0 +1,16 @@ +# Copyright 2024 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from st2tests import fixturesloader + +FIXTURE_NAME, FIXTURE_PATH = fixturesloader.get_fixture_name_and_path(__file__) diff --git a/st2common/tests/resources/packs/BUILD b/st2common/tests/resources/packs/BUILD new file mode 100644 index 0000000000..8280f255bd --- /dev/null +++ b/st2common/tests/resources/packs/BUILD @@ -0,0 +1,8 @@ +resources( + name="packs_directories", + sources=["**/.gitignore"], +) + +python_sources( + dependencies=[":packs_directories"], +) diff --git a/st2common/tests/resources/packs/fixture.py b/st2common/tests/resources/packs/fixture.py new file mode 100644 index 0000000000..995f0ea33f --- /dev/null +++ b/st2common/tests/resources/packs/fixture.py @@ -0,0 +1,16 @@ +# Copyright 2024 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from st2tests import fixturesloader + +_, PACKS_BASE_PATH = fixturesloader.get_fixture_name_and_path(__file__) diff --git a/st2common/tests/resources/packs2/BUILD b/st2common/tests/resources/packs2/BUILD new file mode 100644 index 0000000000..8280f255bd --- /dev/null +++ b/st2common/tests/resources/packs2/BUILD @@ -0,0 +1,8 @@ +resources( + name="packs_directories", + sources=["**/.gitignore"], +) + +python_sources( + dependencies=[":packs_directories"], +) diff --git a/st2common/tests/resources/packs2/fixture.py b/st2common/tests/resources/packs2/fixture.py new file mode 100644 index 0000000000..995f0ea33f --- /dev/null +++ b/st2common/tests/resources/packs2/fixture.py @@ -0,0 +1,16 @@ +# Copyright 2024 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from st2tests import fixturesloader + +_, PACKS_BASE_PATH = fixturesloader.get_fixture_name_and_path(__file__) diff --git a/st2common/tests/resources/packs3/BUILD b/st2common/tests/resources/packs3/BUILD new file mode 100644 index 0000000000..eaed03e50e --- /dev/null +++ b/st2common/tests/resources/packs3/BUILD @@ -0,0 +1,8 @@ +resources( + name="packs_overrides", + sources=["**/*.yaml"], +) + +python_sources( + dependencies=[":packs_overrides"], +) diff --git a/st2common/tests/resources/packs3/fixture.py b/st2common/tests/resources/packs3/fixture.py new file mode 100644 index 0000000000..995f0ea33f --- /dev/null +++ b/st2common/tests/resources/packs3/fixture.py @@ -0,0 +1,16 @@ +# Copyright 2024 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from st2tests import fixturesloader + +_, PACKS_BASE_PATH = fixturesloader.get_fixture_name_and_path(__file__) diff --git a/st2common/tests/resources/overrides/_global.yaml b/st2common/tests/resources/packs3/overrides/_global.yaml similarity index 100% rename from st2common/tests/resources/overrides/_global.yaml rename to st2common/tests/resources/packs3/overrides/_global.yaml diff --git a/st2common/tests/resources/overrides/overpack1.yaml b/st2common/tests/resources/packs3/overrides/overpack1.yaml similarity index 100% rename from st2common/tests/resources/overrides/overpack1.yaml rename to st2common/tests/resources/packs3/overrides/overpack1.yaml diff --git a/st2common/tests/resources/overrides/overpack2.yaml b/st2common/tests/resources/packs3/overrides/overpack2.yaml similarity index 100% rename from st2common/tests/resources/overrides/overpack2.yaml rename to st2common/tests/resources/packs3/overrides/overpack2.yaml diff --git a/st2common/tests/resources/overrides/overpack3.yaml b/st2common/tests/resources/packs3/overrides/overpack3.yaml similarity index 100% rename from st2common/tests/resources/overrides/overpack3.yaml rename to st2common/tests/resources/packs3/overrides/overpack3.yaml diff --git a/st2common/tests/resources/overrides/overpack4.yaml b/st2common/tests/resources/packs3/overrides/overpack4.yaml similarity index 100% rename from st2common/tests/resources/overrides/overpack4.yaml rename to st2common/tests/resources/packs3/overrides/overpack4.yaml diff --git a/st2common/tests/unit/BUILD b/st2common/tests/unit/BUILD index a1da37051b..2652b81de6 100644 --- a/st2common/tests/unit/BUILD +++ b/st2common/tests/unit/BUILD @@ -13,7 +13,7 @@ python_tests( ], uses=["mongo", "rabbitmq"], overrides={ - "test_util_file_system.py": dict( + ("test_util_file_system.py", "test_policies_registrar.py",): dict( dependencies=[ "st2tests/st2tests/policies", "st2tests/st2tests/policies/meta", diff --git a/st2common/tests/unit/services/BUILD b/st2common/tests/unit/services/BUILD index a30ca92a77..ae93633567 100644 --- a/st2common/tests/unit/services/BUILD +++ b/st2common/tests/unit/services/BUILD @@ -7,4 +7,12 @@ python_tests( "st2common.metrics.driver", ], uses=["mongo", "rabbitmq"], + overrides={ + "test_packs.py": dict( + # use the fixture to resolve ambiguous import (ambiguous due to symlink of core pack) + dependencies=[ + "st2tests/st2tests/fixtures/packs/core/actions/inject_trigger.py" + ], + ), + }, ) diff --git a/st2common/tests/unit/services/test_packs.py b/st2common/tests/unit/services/test_packs.py index 69152a8398..668137e9cb 100644 --- a/st2common/tests/unit/services/test_packs.py +++ b/st2common/tests/unit/services/test_packs.py @@ -43,6 +43,7 @@ from st2tests.fixtures.packs.orquesta_tests.fixture import ( PACK_NAME as TEST_SOURCE_WORKFLOW_PACK, ) +import st2tests.config as tests_config SOURCE_ACTION_WITH_PYTHON_SCRIPT_RUNNER = { "description": "Action which injects a new trigger in the system.", @@ -416,6 +417,8 @@ def test_exception_to_remove_resource_metadata_file(self, remove): class CloneActionDBAndFilesTestCase(unittest.TestCase): @classmethod def setUpClass(cls): + super().setUpClass() + tests_config.parse_args() action_files_path = os.path.join(TEST_DEST_PACK_PATH, "actions") workflow_files_path = os.path.join(action_files_path, "workflows") if not os.path.isdir(action_files_path): @@ -588,6 +591,11 @@ def test_workflows_directory_created_if_does_not_exist(self): class CloneActionFilesBackupTestCase(unittest.TestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + tests_config.parse_args() + @classmethod def tearDownClass(cls): action_files_path = os.path.join(TEST_DEST_PACK_PATH, "actions") diff --git a/st2common/tests/unit/services/test_policy.py b/st2common/tests/unit/services/test_policy.py index a590322452..5e891e9c65 100644 --- a/st2common/tests/unit/services/test_policy.py +++ b/st2common/tests/unit/services/test_policy.py @@ -15,9 +15,8 @@ from __future__ import absolute_import -import st2tests.config as tests_config - -tests_config.parse_args() +# This import must be early for import-time side-effects. +import st2tests import st2common @@ -29,7 +28,6 @@ from st2common.services import action as action_service from st2common.services import policies as policy_service -import st2tests from st2tests.fixtures.generic.fixture import PACK_NAME as PACK from st2tests import fixturesloader as fixtures diff --git a/st2common/tests/unit/services/test_workflow.py b/st2common/tests/unit/services/test_workflow.py index bcce91af00..d4897ff322 100644 --- a/st2common/tests/unit/services/test_workflow.py +++ b/st2common/tests/unit/services/test_workflow.py @@ -24,10 +24,6 @@ import st2tests -import st2tests.config as tests_config - -tests_config.parse_args() - from st2common.bootstrap import actionsregistrar from st2common.bootstrap import runnersregistrar from st2common.exceptions import action as action_exc diff --git a/st2common/tests/unit/services/test_workflow_cancellation.py b/st2common/tests/unit/services/test_workflow_cancellation.py index d8b7b2206f..88e81e4fbf 100644 --- a/st2common/tests/unit/services/test_workflow_cancellation.py +++ b/st2common/tests/unit/services/test_workflow_cancellation.py @@ -21,10 +21,6 @@ import st2tests -import st2tests.config as tests_config - -tests_config.parse_args() - from st2common.bootstrap import actionsregistrar from st2common.bootstrap import runnersregistrar from st2common.models.db import liveaction as lv_db_models diff --git a/st2common/tests/unit/services/test_workflow_identify_orphans.py b/st2common/tests/unit/services/test_workflow_identify_orphans.py index 7110b509c9..ba1df395a8 100644 --- a/st2common/tests/unit/services/test_workflow_identify_orphans.py +++ b/st2common/tests/unit/services/test_workflow_identify_orphans.py @@ -22,11 +22,6 @@ import st2tests -# XXX: actionsensor import depends on config being setup. -import st2tests.config as tests_config - -tests_config.parse_args() - from st2common.bootstrap import actionsregistrar from st2common.bootstrap import runnersregistrar from st2common.constants import action as ac_const diff --git a/st2common/tests/unit/services/test_workflow_rerun.py b/st2common/tests/unit/services/test_workflow_rerun.py index b1bcb7417c..c91e140d6c 100644 --- a/st2common/tests/unit/services/test_workflow_rerun.py +++ b/st2common/tests/unit/services/test_workflow_rerun.py @@ -15,6 +15,11 @@ from __future__ import absolute_import +from st2common.util.monkey_patch import monkey_patch + +monkey_patch() + + import mock import uuid @@ -23,10 +28,6 @@ import st2tests -import st2tests.config as tests_config - -tests_config.parse_args() - from local_runner import local_shell_command_runner from st2common.bootstrap import actionsregistrar from st2common.bootstrap import runnersregistrar diff --git a/st2common/tests/unit/services/test_workflow_service_retries.py b/st2common/tests/unit/services/test_workflow_service_retries.py index 0e322fe573..ca2fab6f9f 100644 --- a/st2common/tests/unit/services/test_workflow_service_retries.py +++ b/st2common/tests/unit/services/test_workflow_service_retries.py @@ -30,11 +30,6 @@ import st2tests -# XXX: actionsensor import depends on config being setup. -import st2tests.config as tests_config - -tests_config.parse_args() - from st2common.bootstrap import actionsregistrar from st2common.bootstrap import runnersregistrar from st2common.constants import action as ac_const diff --git a/st2common/tests/unit/test_action_alias_utils.py b/st2common/tests/unit/test_action_alias_utils.py index ec3251070a..1fc54359b0 100644 --- a/st2common/tests/unit/test_action_alias_utils.py +++ b/st2common/tests/unit/test_action_alias_utils.py @@ -30,6 +30,7 @@ search_regex_tokens, inject_immutable_parameters, ) +import st2tests.config as tests_config class TestActionAliasParser(TestCase): @@ -357,6 +358,11 @@ def test_subpatterns(self): class TestInjectImmutableParameters(TestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + tests_config.parse_args() + def test_immutable_parameters_are_injected(self): action_alias_db = Mock() action_alias_db.immutable_parameters = {"env": "dev"} diff --git a/st2common/tests/unit/test_content_loader.py b/st2common/tests/unit/test_content_loader.py index 6edba192d4..700cd19d30 100644 --- a/st2common/tests/unit/test_content_loader.py +++ b/st2common/tests/unit/test_content_loader.py @@ -37,39 +37,36 @@ from st2common.constants.meta import yaml_safe_load from st2tests import config -CURRENT_DIR = os.path.dirname(os.path.realpath(__file__)) -RESOURCES_DIR = os.path.abspath(os.path.join(CURRENT_DIR, "../resources")) +from tests.resources.packs.fixture import PACKS_BASE_PATH as PACKS_BASE_PATH_1 +from tests.resources.packs2.fixture import PACKS_BASE_PATH as PACKS_BASE_PATH_2 +from tests.resources.packs3.fixture import PACKS_BASE_PATH as PACKS_BASE_PATH_3 class ContentLoaderTest(unittest.TestCase): def test_get_sensors(self): - packs_base_path = os.path.join(RESOURCES_DIR, "packs/") loader = ContentPackLoader() pack_sensors = loader.get_content( - base_dirs=[packs_base_path], content_type="sensors" + base_dirs=[PACKS_BASE_PATH_1], content_type="sensors" ) self.assertIsNotNone(pack_sensors.get("pack1", None)) def test_get_sensors_pack_missing_sensors(self): loader = ContentPackLoader() - fail_pack_path = os.path.join(RESOURCES_DIR, "packs/pack2") + fail_pack_path = os.path.join(PACKS_BASE_PATH_1, "pack2") self.assertTrue(os.path.exists(fail_pack_path)) self.assertEqual(loader._get_sensors(fail_pack_path), None) def test_invalid_content_type(self): - packs_base_path = os.path.join(RESOURCES_DIR, "packs/") loader = ContentPackLoader() self.assertRaises( ValueError, loader.get_content, - base_dirs=[packs_base_path], + base_dirs=[PACKS_BASE_PATH_1], content_type="stuff", ) def test_get_content_multiple_directories(self): - packs_base_path_1 = os.path.join(RESOURCES_DIR, "packs/") - packs_base_path_2 = os.path.join(RESOURCES_DIR, "packs2/") - base_dirs = [packs_base_path_1, packs_base_path_2] + base_dirs = [PACKS_BASE_PATH_1, PACKS_BASE_PATH_2] LOG.warning = Mock() @@ -81,14 +78,14 @@ def test_get_content_multiple_directories(self): # Assert that a warning is emitted when a duplicated pack is found expected_msg = ( 'Pack "pack1" already found in ' - '"%s/packs/", ignoring content from ' - '"%s/packs2/"' % (RESOURCES_DIR, RESOURCES_DIR) + f'"{PACKS_BASE_PATH_1}", ignoring content from ' + f'"{PACKS_BASE_PATH_2}"' ) LOG.warning.assert_called_once_with(expected_msg) def test_get_content_from_pack_success(self): loader = ContentPackLoader() - pack_path = os.path.join(RESOURCES_DIR, "packs/pack1") + pack_path = os.path.join(PACKS_BASE_PATH_1, "pack1") sensors = loader.get_content_from_pack( pack_dir=pack_path, content_type="sensors" @@ -97,7 +94,7 @@ def test_get_content_from_pack_success(self): def test_get_content_from_pack_directory_doesnt_exist(self): loader = ContentPackLoader() - pack_path = os.path.join(RESOURCES_DIR, "packs/pack100") + pack_path = os.path.join(PACKS_BASE_PATH_1, "pack100") message_regex = "Directory .*? doesn't exist" self.assertRaisesRegex( @@ -110,7 +107,7 @@ def test_get_content_from_pack_directory_doesnt_exist(self): def test_get_content_from_pack_no_sensors(self): loader = ContentPackLoader() - pack_path = os.path.join(RESOURCES_DIR, "packs/pack2") + pack_path = os.path.join(PACKS_BASE_PATH_1, "pack2") result = loader.get_content_from_pack( pack_dir=pack_path, content_type="sensors" @@ -119,7 +116,9 @@ def test_get_content_from_pack_no_sensors(self): def test_get_override_action_from_default(self): config.parse_args() - cfg.CONF.set_override(name="base_path", override=RESOURCES_DIR, group="system") + cfg.CONF.set_override( + name="base_path", override=PACKS_BASE_PATH_3, group="system" + ) loader = OverrideLoader() content = {"name": "action1", "enabled": True} self.assertTrue(loader.override("overpack1", "actions", content)) @@ -130,7 +129,9 @@ def test_get_override_action_from_default(self): def test_get_override_action_from_exception(self): config.parse_args() - cfg.CONF.set_override(name="base_path", override=RESOURCES_DIR, group="system") + cfg.CONF.set_override( + name="base_path", override=PACKS_BASE_PATH_3, group="system" + ) loader = OverrideLoader() content = {"name": "action2", "enabled": True} self.assertFalse(loader.override("overpack1", "actions", content)) @@ -141,7 +142,9 @@ def test_get_override_action_from_exception(self): def test_get_override_action_from_default_no_exceptions(self): config.parse_args() - cfg.CONF.set_override(name="base_path", override=RESOURCES_DIR, group="system") + cfg.CONF.set_override( + name="base_path", override=PACKS_BASE_PATH_3, group="system" + ) loader = OverrideLoader() content = {"name": "action1", "enabled": True} self.assertTrue(loader.override("overpack4", "actions", content)) @@ -152,7 +155,9 @@ def test_get_override_action_from_default_no_exceptions(self): def test_get_override_action_from_global_default_no_exceptions(self): config.parse_args() - cfg.CONF.set_override(name="base_path", override=RESOURCES_DIR, group="system") + cfg.CONF.set_override( + name="base_path", override=PACKS_BASE_PATH_3, group="system" + ) loader = OverrideLoader() content = {"class_name": "sensor1", "enabled": True} self.assertTrue(loader.override("overpack1", "sensors", content)) @@ -160,7 +165,9 @@ def test_get_override_action_from_global_default_no_exceptions(self): def test_get_override_action_from_global_overridden_by_pack(self): config.parse_args() - cfg.CONF.set_override(name="base_path", override=RESOURCES_DIR, group="system") + cfg.CONF.set_override( + name="base_path", override=PACKS_BASE_PATH_3, group="system" + ) loader = OverrideLoader() content = {"class_name": "sensor1", "enabled": True} self.assertFalse(loader.override("overpack2", "sensors", content)) @@ -168,7 +175,9 @@ def test_get_override_action_from_global_overridden_by_pack(self): def test_get_override_action_from_global_overridden_by_pack_exception(self): config.parse_args() - cfg.CONF.set_override(name="base_path", override=RESOURCES_DIR, group="system") + cfg.CONF.set_override( + name="base_path", override=PACKS_BASE_PATH_3, group="system" + ) loader = OverrideLoader() content = {"class_name": "sensor1", "enabled": True} self.assertFalse(loader.override("overpack3", "sensors", content)) @@ -176,7 +185,9 @@ def test_get_override_action_from_global_overridden_by_pack_exception(self): def test_get_override_invalid_type(self): config.parse_args() - cfg.CONF.set_override(name="base_path", override=RESOURCES_DIR, group="system") + cfg.CONF.set_override( + name="base_path", override=PACKS_BASE_PATH_3, group="system" + ) loader = OverrideLoader() content = {"name": "action2", "enabled": True} self.assertRaises( @@ -189,7 +200,9 @@ def test_get_override_invalid_type(self): def test_get_override_invalid_default_key(self): config.parse_args() - cfg.CONF.set_override(name="base_path", override=RESOURCES_DIR, group="system") + cfg.CONF.set_override( + name="base_path", override=PACKS_BASE_PATH_3, group="system" + ) loader = OverrideLoader() content = {"name": "action1", "enabled": True} self.assertRaises( @@ -202,7 +215,9 @@ def test_get_override_invalid_default_key(self): def test_get_override_invalid_exceptions_key(self): config.parse_args() - cfg.CONF.set_override(name="base_path", override=RESOURCES_DIR, group="system") + cfg.CONF.set_override( + name="base_path", override=PACKS_BASE_PATH_3, group="system" + ) loader = OverrideLoader() content = {"name": "action1", "enabled": True} loader.override("overpack1", "actions", content) diff --git a/st2common/tests/unit/test_dist_utils.py b/st2common/tests/unit/test_dist_utils.py index 9b6060b480..95991e64c5 100644 --- a/st2common/tests/unit/test_dist_utils.py +++ b/st2common/tests/unit/test_dist_utils.py @@ -13,6 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +# NB: Pantsbuild ignores this file and any dist_utils.py files. +# TODO: delete this file when deleting all dist_utils.py files. + import os import sys diff --git a/st2common/tests/unit/test_executions_util.py b/st2common/tests/unit/test_executions_util.py index 00b89c7433..0776ef57cc 100644 --- a/st2common/tests/unit/test_executions_util.py +++ b/st2common/tests/unit/test_executions_util.py @@ -17,6 +17,9 @@ import mock import six +# This import must be early for import-time side-effects. +from st2tests.base import CleanDbTestCase + from st2common.constants import action as action_constants from st2common.models.api.action import RunnerTypeAPI, ActionAPI, LiveActionAPI from st2common.models.api.trigger import TriggerTypeAPI, TriggerAPI, TriggerInstanceAPI @@ -30,15 +33,12 @@ import st2common.util.action_db as action_utils import st2common.util.date as date_utils -from st2tests.base import CleanDbTestCase from st2tests.fixtures.generic.fixture import PACK_NAME as FIXTURES_PACK from st2tests.fixtures.descendants.fixture import PACK_NAME as DESCENDANTS_PACK from st2tests.fixturesloader import FixturesLoader -import st2tests.config as tests_config from six.moves import range -tests_config.parse_args() TEST_FIXTURES = { "liveactions": [ diff --git a/st2common/tests/unit/test_jinja_render_data_filters.py b/st2common/tests/unit/test_jinja_render_data_filters.py index 8db175cac2..bc9ad1c02c 100644 --- a/st2common/tests/unit/test_jinja_render_data_filters.py +++ b/st2common/tests/unit/test_jinja_render_data_filters.py @@ -21,9 +21,15 @@ from st2common.constants.keyvalue import FULL_SYSTEM_SCOPE from st2common.util import jinja as jinja_utils from st2common.services.keyvalues import KeyValueLookup +import st2tests.config as tests_config class JinjaUtilsDataFilterTestCase(unittest.TestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + tests_config.parse_args() + def test_filter_from_json_string(self): env = jinja_utils.get_jinja_environment() expected_obj = {"a": "b", "c": {"d": "e", "f": 1, "g": True}} diff --git a/st2common/tests/unit/test_jsonify.py b/st2common/tests/unit/test_jsonify.py index b4a375be69..906a548d4c 100644 --- a/st2common/tests/unit/test_jsonify.py +++ b/st2common/tests/unit/test_jsonify.py @@ -22,14 +22,13 @@ import st2tests.config as tests_config -tests_config.parse_args() - import st2common.util.jsonify as jsonify class JsonifyTests(unittest.TestCase): @classmethod def setUpClass(cls): + tests_config.parse_args() jsonify.DEFAULT_JSON_LIBRARY = "orjson" @classmethod diff --git a/st2common/tests/unit/test_logger.py b/st2common/tests/unit/test_logger.py index e2126208eb..bd43be9f96 100644 --- a/st2common/tests/unit/test_logger.py +++ b/st2common/tests/unit/test_logger.py @@ -35,9 +35,9 @@ from st2common.models.db.execution import ActionExecutionDB import st2tests.config as tests_config -CURRENT_DIR = os.path.dirname(os.path.realpath(__file__)) -RESOURCES_DIR = os.path.abspath(os.path.join(CURRENT_DIR, "../resources")) -CONFIG_FILE_PATH = os.path.join(RESOURCES_DIR, "logging.conf") +from tests.resources.fixture import FIXTURE_PATH + +CONFIG_FILE_PATH = os.path.join(FIXTURE_PATH, "logging.conf") MOCK_MASKED_ATTRIBUTES_BLACKLIST = [ "blacklisted_1", diff --git a/st2common/tests/unit/test_logging_middleware.py b/st2common/tests/unit/test_logging_middleware.py index 2da4bb5875..8521a622e6 100644 --- a/st2common/tests/unit/test_logging_middleware.py +++ b/st2common/tests/unit/test_logging_middleware.py @@ -20,11 +20,17 @@ from st2common.middleware.logging import LoggingMiddleware from st2common.constants.secrets import MASKED_ATTRIBUTE_VALUE +import st2tests.config as tests_config __all__ = ["LoggingMiddlewareTestCase"] class LoggingMiddlewareTestCase(unittest.TestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + tests_config.parse_args() + @mock.patch("st2common.middleware.logging.LOG") @mock.patch("st2common.middleware.logging.Request") def test_secret_parameters_are_masked_in_log_message(self, mock_request, mock_log): diff --git a/st2common/tests/unit/test_operators.py b/st2common/tests/unit/test_operators.py index 9bb6161f91..39b2cc7a8c 100644 --- a/st2common/tests/unit/test_operators.py +++ b/st2common/tests/unit/test_operators.py @@ -18,6 +18,7 @@ from st2common import operators from st2common.util import date as date_utils +import st2tests.config as tests_config def list_of_dicts_strict_equal(lofd1, lofd2): @@ -157,6 +158,11 @@ def test_less_simple_dicts(self): class SearchOperatorTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + tests_config.parse_args() + # The search command extends the rules engine into being a recursive descent # parser. As such, its tests are much more complex than other commands, so we # pull its tests out into their own test case. diff --git a/st2common/tests/unit/test_plugin_loader.py b/st2common/tests/unit/test_plugin_loader.py index e5a2822406..8e594ecb95 100644 --- a/st2common/tests/unit/test_plugin_loader.py +++ b/st2common/tests/unit/test_plugin_loader.py @@ -24,9 +24,7 @@ import st2common.util.loader as plugin_loader -PLUGIN_FOLDER = "loadableplugin" -SRC_RELATIVE = os.path.join("../resources", PLUGIN_FOLDER) -SRC_ROOT = os.path.join(os.path.abspath(os.path.dirname(__file__)), SRC_RELATIVE) +from tests.resources.loadableplugin.fixture import FIXTURE_PATH as SRC_ROOT class LoaderTest(unittest.TestCase): diff --git a/st2common/tests/unit/test_purge_rule_enforcement.py b/st2common/tests/unit/test_purge_rule_enforcement.py index 90b4d23799..1b00228fa3 100644 --- a/st2common/tests/unit/test_purge_rule_enforcement.py +++ b/st2common/tests/unit/test_purge_rule_enforcement.py @@ -13,6 +13,12 @@ # limitations under the License. from __future__ import absolute_import + +# pytest: make sure monkey_patching happens before importing mongoengine +from st2common.util.monkey_patch import monkey_patch + +monkey_patch() + from datetime import timedelta import bson diff --git a/st2common/tests/unit/test_purge_task_executions.py b/st2common/tests/unit/test_purge_task_executions.py index b0c7cd8bc2..b5c2dc19fe 100644 --- a/st2common/tests/unit/test_purge_task_executions.py +++ b/st2common/tests/unit/test_purge_task_executions.py @@ -13,6 +13,12 @@ # limitations under the License. from __future__ import absolute_import + +# pytest: make sure monkey_patching happens before importing mongoengine +from st2common.util.monkey_patch import monkey_patch + +monkey_patch() + from datetime import timedelta from st2common import log as logging diff --git a/st2common/tests/unit/test_purge_token.py b/st2common/tests/unit/test_purge_token.py index 75c24e62cd..1bad08d097 100644 --- a/st2common/tests/unit/test_purge_token.py +++ b/st2common/tests/unit/test_purge_token.py @@ -13,6 +13,12 @@ # limitations under the License. from __future__ import absolute_import + +# pytest: make sure monkey_patching happens before importing mongoengine +from st2common.util.monkey_patch import monkey_patch + +monkey_patch() + from datetime import timedelta import bson diff --git a/st2common/tests/unit/test_purge_trace.py b/st2common/tests/unit/test_purge_trace.py index 9a819b4018..7dde63f9f1 100644 --- a/st2common/tests/unit/test_purge_trace.py +++ b/st2common/tests/unit/test_purge_trace.py @@ -13,6 +13,12 @@ # limitations under the License. from __future__ import absolute_import + +# pytest: make sure monkey_patching happens before importing mongoengine +from st2common.util.monkey_patch import monkey_patch + +monkey_patch() + from datetime import timedelta import bson diff --git a/st2common/tests/unit/test_purge_worklows.py b/st2common/tests/unit/test_purge_worklows.py index 2975c504cf..383030b1e3 100644 --- a/st2common/tests/unit/test_purge_worklows.py +++ b/st2common/tests/unit/test_purge_worklows.py @@ -13,6 +13,12 @@ # limitations under the License. from __future__ import absolute_import + +# pytest: make sure monkey_patching happens before importing mongoengine +from st2common.util.monkey_patch import monkey_patch + +monkey_patch() + from datetime import timedelta from st2common import log as logging diff --git a/st2common/tests/unit/test_runners_utils.py b/st2common/tests/unit/test_runners_utils.py index 773fa9cc39..b6f61bd61c 100644 --- a/st2common/tests/unit/test_runners_utils.py +++ b/st2common/tests/unit/test_runners_utils.py @@ -15,26 +15,18 @@ from __future__ import absolute_import -# pytest: make sure monkey_patching happens before importing mongoengine -from st2common.util.monkey_patch import monkey_patch - -monkey_patch() +# This import must be early for import-time side-effects. +from st2tests import base import mock from st2common.runners import utils from st2common.services import executions as exe_svc from st2common.util import action_db as action_db_utils -from st2tests import base from st2tests import fixturesloader from st2tests.fixtures.generic.fixture import PACK_NAME as FIXTURES_PACK -from st2tests import config as tests_config - -tests_config.parse_args() - - TEST_FIXTURES = { "liveactions": ["liveaction1.yaml"], "actions": ["local.yaml"], diff --git a/st2common/tests/unit/test_util_file_system.py b/st2common/tests/unit/test_util_file_system.py index f45498702b..58f7bf6473 100644 --- a/st2common/tests/unit/test_util_file_system.py +++ b/st2common/tests/unit/test_util_file_system.py @@ -27,21 +27,21 @@ class FileSystemUtilsTestCase(unittest.TestCase): def test_get_file_list(self): + # NB: Make sure to exclude BUILD files as pants will not include them in the sandbox, + # but the BUILD files will be present if you directly run the tests. + basic_excludes = ["*.pyc", "__pycache__", "*BUILD"] + # Standard exclude pattern directory = os.path.join(ST2TESTS_DIR, "policies") expected = [ - "BUILD", "mock_exception.py", "concurrency.py", "__init__.py", - "meta/BUILD", "meta/mock_exception.yaml", "meta/concurrency.yaml", "meta/__init__.py", ] - result = get_file_list( - directory=directory, exclude_patterns=["*.pyc", "__pycache__"] - ) + result = get_file_list(directory=directory, exclude_patterns=basic_excludes) # directory listings are sorted because the item order must be exact for assert # to validate equivalence. Directory item order doesn't matter in general and may # even change on different platforms or locales. @@ -55,7 +55,7 @@ def test_get_file_list(self): "meta/__init__.py", ] result = get_file_list( - directory=directory, exclude_patterns=["*.pyc", "*.yaml", "*BUILD"] + directory=directory, exclude_patterns=["*.yaml"] + basic_excludes ) # directory listings are sorted because the item order must be exact for assert # to validate equivalence. Directory item order doesn't matter in general and may diff --git a/st2common/tests/unit/test_util_payload.py b/st2common/tests/unit/test_util_payload.py index d6629d6d57..715786e2f0 100644 --- a/st2common/tests/unit/test_util_payload.py +++ b/st2common/tests/unit/test_util_payload.py @@ -18,6 +18,7 @@ import unittest from st2common.util.payload import PayloadLookup +import st2tests.config as tests_config __all__ = ["PayloadLookupTestCase"] @@ -32,6 +33,7 @@ def setUpClass(cls): } ) super(PayloadLookupTestCase, cls).setUpClass() + tests_config.parse_args() def test_get_key(self): self.assertEqual(self.payload.get_value("trigger.pikachu"), ["Has no ears"]) diff --git a/st2reactor/conf/BUILD b/st2reactor/conf/BUILD index 57246fe48d..4f07917387 100644 --- a/st2reactor/conf/BUILD +++ b/st2reactor/conf/BUILD @@ -1,14 +1,15 @@ -file( +st2_logging_conf_file( name="logging_console", source="console.conf", ) -files( +st2_logging_conf_files( name="logging", sources=["logging*.conf"], + dependencies=["//:logs_directory"], ) -files( +st2_logging_conf_files( name="logging_syslog", sources=["syslog*.conf"], ) diff --git a/st2reactor/tests/resources/BUILD b/st2reactor/tests/resources/BUILD index 57341b1358..4cb4ff555e 100644 --- a/st2reactor/tests/resources/BUILD +++ b/st2reactor/tests/resources/BUILD @@ -1,3 +1,10 @@ -python_tests( - name="tests", +python_sources( + name="fixture_sensors", + # Override the default sources to include test_sensor*.py + # which are fixtures, not actual test files. + sources=["test_sensor*.py"], +) + +python_sources( + dependencies=[":fixture_sensors"], ) diff --git a/st2reactor/tests/resources/fixture.py b/st2reactor/tests/resources/fixture.py new file mode 100644 index 0000000000..f6d0d3e940 --- /dev/null +++ b/st2reactor/tests/resources/fixture.py @@ -0,0 +1,16 @@ +# Copyright 2024 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from st2tests import fixturesloader + +FIXTURE_NAME, FIXTURE_PATH = fixturesloader.get_fixture_name_and_path(__file__) diff --git a/st2reactor/tests/unit/BUILD b/st2reactor/tests/unit/BUILD index 9a24dba70a..abd3115705 100644 --- a/st2reactor/tests/unit/BUILD +++ b/st2reactor/tests/unit/BUILD @@ -5,4 +5,20 @@ __defaults__( python_tests( name="tests", + uses=["mongo"], + overrides={ + "test_enforce.py": dict( + stevedore_namespaces=[ + "st2common.rbac.backend", + "st2common.runners.runner", + "st2common.metrics.driver", + ], + ), + "test_rule_engine.py": dict( + stevedore_namespaces=[ + "st2common.runners.runner", + "st2common.metrics.driver", + ], + ), + }, ) diff --git a/st2reactor/tests/unit/test_garbage_collector.py b/st2reactor/tests/unit/test_garbage_collector.py index 93de6b25d0..f9ca515d0a 100644 --- a/st2reactor/tests/unit/test_garbage_collector.py +++ b/st2reactor/tests/unit/test_garbage_collector.py @@ -20,14 +20,18 @@ from oslo_config import cfg +# This import must be early for import-time side-effects. import st2tests.config as tests_config -tests_config.parse_args() - from st2reactor.garbage_collector import base as garbage_collector class GarbageCollectorServiceTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + tests_config.parse_args() + def tearDown(self): # Reset gc_max_idle_sec with a value of 1 to reenable for other tests. cfg.CONF.set_override("gc_max_idle_sec", 1, group="workflow_engine") diff --git a/st2reactor/tests/unit/test_process_container.py b/st2reactor/tests/unit/test_process_container.py index b05175b805..747618a4f0 100644 --- a/st2reactor/tests/unit/test_process_container.py +++ b/st2reactor/tests/unit/test_process_container.py @@ -27,8 +27,6 @@ import st2tests.config as tests_config -tests_config.parse_args() - MOCK_PACK_DB = PackDB( ref="wolfpack", name="wolf pack", @@ -38,6 +36,11 @@ class ProcessContainerTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + tests_config.parse_args() + def test_no_sensors_dont_quit(self): process_container = ProcessSensorContainer(None, poll_interval=0.1) process_container_thread = concurrency.spawn(process_container.run) diff --git a/st2reactor/tests/unit/test_sensor_wrapper.py b/st2reactor/tests/unit/test_sensor_wrapper.py index 3a9d3b9ced..36b8c04f10 100644 --- a/st2reactor/tests/unit/test_sensor_wrapper.py +++ b/st2reactor/tests/unit/test_sensor_wrapper.py @@ -33,8 +33,7 @@ from st2reactor.container.sensor_wrapper import SensorWrapper from st2reactor.sensor.base import Sensor, PollingSensor -CURRENT_DIR = os.path.abspath(os.path.dirname(__file__)) -RESOURCES_DIR = os.path.abspath(os.path.join(CURRENT_DIR, "../resources")) +from tests.resources.fixture import FIXTURE_PATH as RESOURCES_DIR __all__ = ["SensorWrapperTestCase"] diff --git a/st2stream/conf/BUILD b/st2stream/conf/BUILD index 9c3668885c..5fc903fd3b 100644 --- a/st2stream/conf/BUILD +++ b/st2stream/conf/BUILD @@ -1,19 +1,21 @@ -file( +st2_logging_conf_file( name="logging_console", source="console.conf", ) -file( +st2_logging_conf_file( name="logging", source="logging.conf", + dependencies=["//:logs_directory"], ) -file( +st2_logging_conf_file( name="logging_gunicorn", source="logging.gunicorn.conf", + dependencies=["//:logs_directory"], ) -file( +st2_logging_conf_file( name="logging_syslog", source="syslog.conf", ) diff --git a/st2stream/tests/unit/controllers/v1/BUILD b/st2stream/tests/unit/controllers/v1/BUILD index a1ca2f641e..982e61ec88 100644 --- a/st2stream/tests/unit/controllers/v1/BUILD +++ b/st2stream/tests/unit/controllers/v1/BUILD @@ -1,5 +1,11 @@ python_tests( name="tests", + stevedore_namespaces=[ + "st2common.rbac.backend", + "st2common.runners.runner", + "st2common.metrics.driver", + ], + uses=["mongo"], ) python_test_utils( diff --git a/st2tests/conf/BUILD b/st2tests/conf/BUILD index 72352b12b1..23af9ef7c9 100644 --- a/st2tests/conf/BUILD +++ b/st2tests/conf/BUILD @@ -8,12 +8,12 @@ file( source="st2_kvstore_tests.crypto.key.json", ) -file( +st2_logging_conf_file( name="logging.conf", source="logging.conf", ) -files( +st2_logging_conf_files( name="other_logging_conf", sources=["logging.*.conf"], ) diff --git a/st2tests/st2tests/config.py b/st2tests/st2tests/config.py index 351456a261..bef5197bcf 100644 --- a/st2tests/st2tests/config.py +++ b/st2tests/st2tests/config.py @@ -104,6 +104,9 @@ def _override_common_opts(): CONF.set_override(name="api_url", override="http://127.0.0.1", group="auth") CONF.set_override(name="mask_secrets", override=True, group="log") CONF.set_override(name="stream_output", override=False, group="actionrunner") + system_user = os.environ.get("ST2TESTS_SYSTEM_USER", "") + if system_user: + CONF.set_override(name="user", override=system_user, group="system_user") def _override_api_opts(): diff --git a/st2tests/st2tests/fixtures/conf/BUILD b/st2tests/st2tests/fixtures/conf/BUILD index 5674cea485..545508803f 100644 --- a/st2tests/st2tests/fixtures/conf/BUILD +++ b/st2tests/st2tests/fixtures/conf/BUILD @@ -1,10 +1,15 @@ +st2_logging_conf_resources( + name="logging", + sources=["logging.*.conf"], +) + resources( name="st2.tests.conf", sources=[ "st2.tests*.conf", - "logging.*.conf", # used by st2.tests*.conf ], dependencies=[ + ":logging", # used by st2.tests*.conf "st2tests/conf:other_logging_conf", # depending on st2auth from st2tests is not nice. "st2auth/conf:htpasswd", diff --git a/st2tests/st2tests/fixtures/generic/BUILD b/st2tests/st2tests/fixtures/generic/BUILD index 48fcf06310..5658638e23 100644 --- a/st2tests/st2tests/fixtures/generic/BUILD +++ b/st2tests/st2tests/fixtures/generic/BUILD @@ -3,6 +3,12 @@ pack_metadata( dependencies=[ "./actions:shell", "./actions:shell_resources", + # policytypes/fake_policy_type_1.py needs: + "//st2tests/st2tests/policies/concurrency.py", + # policytypes/fake_policy_type_2.py needs: + "//st2tests/st2tests/policies/mock_exception.py", + # policytypes/fake_policy_type_3.py needs: + "//st2actions/st2actions/policies/concurrency_by_attr.py", ], ) diff --git a/st2tests/st2tests/fixtures/packs/BUILD b/st2tests/st2tests/fixtures/packs/BUILD index 025cf82aac..a5006ef9d1 100644 --- a/st2tests/st2tests/fixtures/packs/BUILD +++ b/st2tests/st2tests/fixtures/packs/BUILD @@ -11,6 +11,7 @@ pack_metadata_in_git_submodule( "test_content_version/icon.png", "test_content_version/requirements.txt", ], + # NOTE: If you need the git metadata, make sure to depend on //:capture_git_modules ) st2_shell_sources_and_resources( diff --git a/st2tests/st2tests/fixtures/packs/action_chain_tests/actions/chains/test_pause_resume_with_error.yaml b/st2tests/st2tests/fixtures/packs/action_chain_tests/actions/chains/test_pause_resume_with_error.yaml index a4d46be55e..c0869f7b64 100644 --- a/st2tests/st2tests/fixtures/packs/action_chain_tests/actions/chains/test_pause_resume_with_error.yaml +++ b/st2tests/st2tests/fixtures/packs/action_chain_tests/actions/chains/test_pause_resume_with_error.yaml @@ -3,7 +3,7 @@ chain: name: task1 ref: core.local params: - cmd: "while [ -e '{{tempfile}}' ]; do sleep 0.1; exit 1" + cmd: "while [ -e '{{tempfile}}' ]; do sleep 0.1; done; exit 1" timeout: 180 on-failure: task2 - diff --git a/st2tests/st2tests/policies/meta/BUILD b/st2tests/st2tests/policies/meta/BUILD index 0700f34cd5..53560669a3 100644 --- a/st2tests/st2tests/policies/meta/BUILD +++ b/st2tests/st2tests/policies/meta/BUILD @@ -1,3 +1,9 @@ resources( - sources=["*.yaml"], + sources=[ + "*.yaml", + # pants ignores empty __init__.py files. + # However, the tests for st2common.util.file_system need it to be present, + # so we treat it as a resource instead of a python source file. + "__init__.py", + ], )