From bce119488ecb9dbd5fd6e1faf27a3eddca822be6 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Tue, 25 Jun 2024 14:10:01 -0500 Subject: [PATCH 01/98] Upgrade to pants 2.23.0a0 Lockfile diff: lockfiles/pants-plugins.lock [pants-plugins] == Upgraded dependencies == pantsbuild-pants 2.22.0 --> 2.23.0a0 pantsbuild-pants-testutil 2.22.0 --> 2.23.0a0 pex 2.3.1 --> 2.16.2 typing-extensions 4.3.0 --> 4.12.2 == Added dependencies == libcst 1.4.0 --- lockfiles/pants-plugins.lock | 127 ++++++++++++++++++++++++++--------- pants.toml | 2 +- 2 files changed, 96 insertions(+), 33 deletions(-) diff --git a/lockfiles/pants-plugins.lock b/lockfiles/pants-plugins.lock index 3d8a7e4f15..4d9a43a8aa 100644 --- a/lockfiles/pants-plugins.lock +++ b/lockfiles/pants-plugins.lock @@ -9,8 +9,8 @@ // "CPython==3.9.*" // ], // "generated_with_requirements": [ -// "pantsbuild.pants.testutil==2.22.0", -// "pantsbuild.pants==2.22.0" +// "pantsbuild.pants.testutil==2.23.0a0", +// "pantsbuild.pants==2.23.0a0" // ], // "manylinux": "manylinux2014", // "requirement_constraints": [], @@ -25,6 +25,7 @@ "allow_wheels": true, "build_isolation": true, "constraints": [], + "excluded": [], "locked_resolves": [ { "locked_requirements": [ @@ -241,6 +242,66 @@ "requires_python": ">=3.7", "version": "2.0.0" }, + { + "artifacts": [ + { + "algorithm": "sha256", + "hash": "d024f44059a853b4b852cfc04fec33e346659d851371e46fc8e7c19de24d3da9", + "url": "https://files.pythonhosted.org/packages/71/da/16307f14b47f761235050076e1d2954fc7de9346f1410ba8c67a54a9f40e/libcst-1.4.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl" + }, + { + "algorithm": "sha256", + "hash": "b8ecdba8934632b4dadacb666cd3816627a6ead831b806336972ccc4ba7ca0e9", + "url": "https://files.pythonhosted.org/packages/7b/b1/8476fe4fa1061062855459d519ffe2115a891638c230ee3465c69fdbfd7a/libcst-1.4.0-cp39-cp39-macosx_10_9_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "8e54c777b8d27339b70f304d16fc8bc8674ef1bd34ed05ea874bf4921eb5a313", + "url": "https://files.pythonhosted.org/packages/7e/0d/89516795ff2a11be10c060c539895b3781793d46cb7c9b0b7b3c4fa3fbc1/libcst-1.4.0-cp39-cp39-macosx_11_0_arm64.whl" + }, + { + "algorithm": "sha256", + "hash": "bb0abf627ee14903d05d0ad9b2c6865f1b21eb4081e2c7bea1033f85db2b8bae", + "url": "https://files.pythonhosted.org/packages/95/cf/a2be91d53e4068d4def8b5cc475f20e1c1a7d32c85634ee7d6b3ea2e3c9b/libcst-1.4.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + }, + { + "algorithm": "sha256", + "hash": "061d6855ef30efe38b8a292b7e5d57c8e820e71fc9ec9846678b60a934b53bbb", + "url": "https://files.pythonhosted.org/packages/c0/c8/15ca337e5f5604aabed899609ba08abbc0e7815ffdfca37802da52d4d0bf/libcst-1.4.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" + }, + { + "algorithm": "sha256", + "hash": "449e0b16604f054fa7f27c3ffe86ea7ef6c409836fe68fe4e752a1894175db00", + "url": "https://files.pythonhosted.org/packages/e4/bd/ff41d7a8efc4f60a61d903c3f9823565006f44f2b8b11c99701f552b0851/libcst-1.4.0.tar.gz" + } + ], + "project_name": "libcst", + "requires_dists": [ + "Sphinx>=5.1.1; extra == \"dev\"", + "black==23.12.1; extra == \"dev\"", + "build>=0.10.0; extra == \"dev\"", + "coverage>=4.5.4; extra == \"dev\"", + "fixit==2.1.0; extra == \"dev\"", + "flake8==7.0.0; extra == \"dev\"", + "hypothesis>=4.36.0; extra == \"dev\"", + "hypothesmith>=0.0.4; extra == \"dev\"", + "jinja2==3.1.4; extra == \"dev\"", + "jupyter>=1.0.0; extra == \"dev\"", + "maturin<1.6,>=0.8.3; extra == \"dev\"", + "nbsphinx>=0.4.2; extra == \"dev\"", + "prompt-toolkit>=2.0.9; extra == \"dev\"", + "pyre-check==0.9.18; platform_system != \"Windows\" and extra == \"dev\"", + "pyyaml>=5.2", + "setuptools-rust>=1.5.2; extra == \"dev\"", + "setuptools-scm>=6.0.1; extra == \"dev\"", + "slotscheck>=0.7.1; extra == \"dev\"", + "sphinx-rtd-theme>=0.4.3; extra == \"dev\"", + "ufmt==2.6.0; extra == \"dev\"", + "usort==1.0.8.post1; extra == \"dev\"" + ], + "requires_python": ">=3.9", + "version": "1.4.0" + }, { "artifacts": [ { @@ -285,23 +346,23 @@ "artifacts": [ { "algorithm": "sha256", - "hash": "edfcecc959eebfb0c42602b29d7301d96b92d3bbde030ba2c4ce3e775801cbd3", - "url": "https://github.com/pantsbuild/pants/releases/download/release_2.22.0/pantsbuild.pants-2.22.0-cp39-cp39-manylinux2014_x86_64.whl" + "hash": "f7104cf619c928752041acfe36966742dec5309b171aeef921239d4595ee4161", + "url": "https://github.com/pantsbuild/pants/releases/download/release_2.23.0a0/pantsbuild.pants-2.23.0a0-cp39-cp39-manylinux2014_x86_64.whl" }, { "algorithm": "sha256", - "hash": "a4b2a095c0d77afa6605ed591dfa334ebbbc2858676a1397dbe1bbc7bf7a1e68", - "url": "https://github.com/pantsbuild/pants/releases/download/release_2.22.0/pantsbuild.pants-2.22.0-cp39-cp39-macosx_10_15_x86_64.whl" + "hash": "d74b12dd7c4dd4cc9a7c81e55126db298577830c62962a6f8cbb4d875930f9ed", + "url": "https://github.com/pantsbuild/pants/releases/download/release_2.23.0a0/pantsbuild.pants-2.23.0a0-cp39-cp39-macosx_10_15_x86_64.whl" }, { "algorithm": "sha256", - "hash": "72ff3f0351389688fd031c8cbb77beffd5e1234ea139da24940522e5de6c14f8", - "url": "https://github.com/pantsbuild/pants/releases/download/release_2.22.0/pantsbuild.pants-2.22.0-cp39-cp39-macosx_11_0_arm64.whl" + "hash": "3afee18ce33b16cb3147ed18e190f0e37d4f3561d58354ee1203f7c66cfe1c5f", + "url": "https://github.com/pantsbuild/pants/releases/download/release_2.23.0a0/pantsbuild.pants-2.23.0a0-cp39-cp39-macosx_11_0_arm64.whl" }, { "algorithm": "sha256", - "hash": "d68791c067bc8902d2fceef8226c3ac443ec3349ab6463fcfe081d33647c4055", - "url": "https://github.com/pantsbuild/pants/releases/download/release_2.22.0/pantsbuild.pants-2.22.0-cp39-cp39-manylinux2014_aarch64.whl" + "hash": "6e47e4076e8321005b15afa4bd63f1444e32446de2634043caeafa35853a279c", + "url": "https://github.com/pantsbuild/pants/releases/download/release_2.23.0a0/pantsbuild.pants-2.23.0a0-cp39-cp39-manylinux2014_aarch64.whl" } ], "project_name": "pantsbuild-pants", @@ -311,9 +372,10 @@ "chevron==0.14.0", "fasteners==0.16.3", "ijson==3.2.3", + "libcst==1.4.0", "node-semver==0.9.0", "packaging==21.3", - "pex==2.3.1", + "pex==2.16.2", "psutil==5.9.8", "python-lsp-jsonrpc==1.0.0", "setproctitle==1.3.2", @@ -322,46 +384,46 @@ "types-PyYAML==6.0.3", "types-setuptools==62.6.1", "types-toml==0.10.8", - "typing-extensions==4.3.0" + "typing-extensions~=4.12" ], "requires_python": "==3.9.*", - "version": "2.22.0" + "version": "2.23.0a0" }, { "artifacts": [ { "algorithm": "sha256", - "hash": "4c4a0319e98f4892581887a713a78a2ee37ace5cd1535e5e73164942c59e632a", - "url": "https://github.com/pantsbuild/pants/releases/download/release_2.22.0/pantsbuild.pants.testutil-2.22.0-py3-none-any.whl" + "hash": "f74af1d1cbac2f8c17e441e2e6c96588fc1816828ecc2665b535dd4ccfbaa6c7", + "url": "https://github.com/pantsbuild/pants/releases/download/release_2.23.0a0/pantsbuild.pants.testutil-2.23.0a0-py3-none-any.whl" } ], "project_name": "pantsbuild-pants-testutil", "requires_dists": [ - "pantsbuild.pants==2.22.0", + "pantsbuild.pants==2.23.0a0", "pytest<7.1.0,>=6.2.4" ], "requires_python": "==3.9.*", - "version": "2.22.0" + "version": "2.23.0a0" }, { "artifacts": [ { "algorithm": "sha256", - "hash": "64692a5bf6f298403aab930d22f0d836ae4736c5bc820e262e9092fe8c56f830", - "url": "https://files.pythonhosted.org/packages/e7/d0/fbda2a4d41d62d86ce53f5ae4fbaaee8c34070f75bb7ca009090510ae874/pex-2.3.1-py2.py3-none-any.whl" + "hash": "8610b5bf7731c98d871421ff21e769e8fcf42ea56aa4ac7f8a271f2405733f24", + "url": "https://files.pythonhosted.org/packages/de/45/94497d22a1517b2462394f641ea272e7ec624823f223c01a5f0d7e6f571d/pex-2.16.2-py2.py3-none-any.whl" }, { "algorithm": "sha256", - "hash": "d1264c91161c21139b454744c8053e25b8aad2d15da89232181b4f38f3f54575", - "url": "https://files.pythonhosted.org/packages/83/4b/1855a9cd872a5eca4cd385e0f66078845f3561d359fb976be52a2a68b9d1/pex-2.3.1.tar.gz" + "hash": "feb2f1e9819a741915759fc221ee6119447acdfc3e0aaa5bbe5800c39fa10003", + "url": "https://files.pythonhosted.org/packages/87/cf/a39ace2db568e3bce48c79fd462aa289608208fe4509ff746349162e5196/pex-2.16.2.tar.gz" } ], "project_name": "pex", "requires_dists": [ "subprocess32>=3.2.7; python_version < \"3\" and extra == \"subprocess\"" ], - "requires_python": "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,<3.13,>=2.7", - "version": "2.3.1" + "requires_python": "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,<3.14,>=2.7", + "version": "2.16.2" }, { "artifacts": [ @@ -828,19 +890,19 @@ "artifacts": [ { "algorithm": "sha256", - "hash": "25642c956049920a5aa49edcdd6ab1e06d7e5d467fc00e0506c44ac86fbfca02", - "url": "https://files.pythonhosted.org/packages/ed/d6/2afc375a8d55b8be879d6b4986d4f69f01115e795e36827fd3a40166028b/typing_extensions-4.3.0-py3-none-any.whl" + "hash": "04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d", + "url": "https://files.pythonhosted.org/packages/26/9f/ad63fc0248c5379346306f8668cda6e2e2e9c95e01216d2b8ffd9ff037d0/typing_extensions-4.12.2-py3-none-any.whl" }, { "algorithm": "sha256", - "hash": "e6d2677a32f47fc7eb2795db1dd15c1f34eff616bcaf2cfb5e997f854fa1c4a6", - "url": "https://files.pythonhosted.org/packages/9e/1d/d128169ff58c501059330f1ad96ed62b79114a2eb30b8238af63a2e27f70/typing_extensions-4.3.0.tar.gz" + "hash": "1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8", + "url": "https://files.pythonhosted.org/packages/df/db/f35a00659bc03fec321ba8bce9420de607a1d37f8342eee1863174c69557/typing_extensions-4.12.2.tar.gz" } ], "project_name": "typing-extensions", "requires_dists": [], - "requires_python": ">=3.7", - "version": "4.3.0" + "requires_python": ">=3.8", + "version": "4.12.2" }, { "artifacts": [ @@ -926,13 +988,14 @@ ], "only_builds": [], "only_wheels": [], + "overridden": [], "path_mappings": {}, - "pex_version": "2.3.1", + "pex_version": "2.16.2", "pip_version": "24.0", "prefer_older_binary": false, "requirements": [ - "pantsbuild.pants.testutil==2.22.0", - "pantsbuild.pants==2.22.0" + "pantsbuild.pants.testutil==2.23.0a0", + "pantsbuild.pants==2.23.0a0" ], "requires_python": [ "==3.9.*" diff --git a/pants.toml b/pants.toml index 38e9fee47a..b33d1d9f09 100644 --- a/pants.toml +++ b/pants.toml @@ -6,7 +6,7 @@ enabled = false repo_id = "de0dea7a-9f6a-4c6e-aa20-6ba5ad969b8a" [GLOBAL] -pants_version = "2.22.0" +pants_version = "2.23.0a0" pythonpath = ["%(buildroot)s/pants-plugins"] build_file_prelude_globs = ["pants-plugins/macros.py"] backend_packages = [ From 004664520be1a4fdf2dc025577231d96c9e17bc3 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 5 Aug 2024 13:03:12 -0500 Subject: [PATCH 02/98] pants: rename [export].py_hermetic_scripts option It is now a per-resolve option: [export].py_non_hermetic_scripts_in_resolve = ["st2"] --- pants.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pants.toml b/pants.toml index b33d1d9f09..d1e9ba0611 100644 --- a/pants.toml +++ b/pants.toml @@ -159,7 +159,7 @@ py_editable_in_resolve = ["st2"] py_resolve_format = "mutable_virtualenv" # By default, pex modifies script shebangs to add '-sE'. # This breaks nosetest and anything that needs PYTHONPATH. -py_hermetic_scripts = false +py_non_hermetic_scripts_in_resolve = ["st2"] # If any targets generate sources/files, include them in the exported venv. py_generated_sources_in_resolve = ["st2"] From 1e91b624fb9645fef2d5ef9da397a05847e0124c Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 12 Sep 2024 13:53:44 -0500 Subject: [PATCH 03/98] update changelog entry --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8fbcebd310..b3c828b535 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -33,7 +33,7 @@ Added * Continue introducing `pants `_ to improve DX (Developer Experience) working on StackStorm, improve our security posture, and improve CI reliability thanks in part to pants' use of PEX lockfiles. This is not a user-facing addition. - #6118 #6141 #6133 #6120 #6181 #6183 #6200 #6237 + #6118 #6141 #6133 #6120 #6181 #6183 #6200 #6237 #6229 Contributed by @cognifloyd * Build of ST2 EL9 packages #6153 Contributed by @amanda11 From dc9f98be28107a05120b061e241201cef40516ae Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 22 May 2024 16:37:09 -0500 Subject: [PATCH 04/98] pants: test w/ correct python and only test pants-plugins under py3.9 --- .github/workflows/test.yaml | 18 ++++++++++++++---- pylint_plugins/BUILD | 1 + 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 3f2145828c..7e20bd1a27 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -78,6 +78,7 @@ jobs: submodules: 'true' - name: 'Set up Python (${{ matrix.python-version }})' + id: python uses: actions/setup-python@v5 with: python-version: '${{ matrix.python-version }}' @@ -92,16 +93,25 @@ jobs: # To ignore a bad cache, bump the cache* integer. gha-cache-key: cache0-py${{ matrix.python-version }} - - name: Test + - name: Test pants-plugins + if: ${{ matrix.python-version-short == '3.9' }} + run: | + pants test pants-plugins/:: + + - name: Unit Tests # We do not support running pytest everywhere yet. When we do it will be simply: # pants test :: # Until then, we need to manually adjust this command line to test what we can. - run: | - pants test pylint_plugins/:: pants-plugins/:: + run: > + pants test + --python-bootstrap-search-path=[] + --python-bootstrap-search-path=${{ steps.python.outputs.python-path }} + --tags=unit + pylint_plugins/:: - name: Upload pants log uses: actions/upload-artifact@v4 with: name: pants-log-py${{ matrix.python-version }} path: .pants.d/pants.log - if: always() # We want the log even on failures. + if: ${{ always() }} # We want the log even on failures. diff --git a/pylint_plugins/BUILD b/pylint_plugins/BUILD index 1f7bfde6c6..9705181eec 100644 --- a/pylint_plugins/BUILD +++ b/pylint_plugins/BUILD @@ -8,6 +8,7 @@ python_sources() python_tests( name="tests", + tags=["unit"], dependencies=[ "./fixtures", "!//conftest.py:test_utils", From d85903a13fd2f5d40e9558cfb226ec0a8e4bd4fa Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 22 May 2024 16:38:21 -0500 Subject: [PATCH 05/98] pants: test st2client/:: unit tests --- .github/workflows/test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 7e20bd1a27..ad20962751 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -108,6 +108,7 @@ jobs: --python-bootstrap-search-path=${{ steps.python.outputs.python-path }} --tags=unit pylint_plugins/:: + st2client/:: - name: Upload pants log uses: actions/upload-artifact@v4 From 60b5814ca2c073dfc41b79a08d12258756d181eb Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 22 May 2024 16:43:11 -0500 Subject: [PATCH 06/98] correct order of pants args --- .github/workflows/test.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index ad20962751..f8966d1009 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -103,10 +103,11 @@ jobs: # pants test :: # Until then, we need to manually adjust this command line to test what we can. run: > - pants test + pants --python-bootstrap-search-path=[] --python-bootstrap-search-path=${{ steps.python.outputs.python-path }} - --tags=unit + --tag=unit + test pylint_plugins/:: st2client/:: From 7228e108f481c93aa521324e6d5bb61a729c7262 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 22 May 2024 23:24:47 -0500 Subject: [PATCH 07/98] use fixtures imports for test_content_loader First, add metadata so pants can load the packs base path fixtures from st2common/resources for the content laoder tests. Second, add a fixture.py file in each base path, similar to the fixtures we have in st2tests, so that pants dependency inferrence can see where these fixtures are used. Third, the fixture file ended up creating a __pycache__ directory in the packs base paths which made tests fail due to an unexpected extra "pack" named "__pycache__". So, I excluded that as a valid pack name. --- st2common/st2common/content/loader.py | 4 ++ st2common/tests/resources/packs/BUILD | 8 +++ st2common/tests/resources/packs/fixture.py | 16 +++++ st2common/tests/resources/packs2/BUILD | 8 +++ st2common/tests/resources/packs2/fixture.py | 16 +++++ st2common/tests/resources/packs3/BUILD | 8 +++ st2common/tests/resources/packs3/fixture.py | 16 +++++ .../{ => packs3}/overrides/_global.yaml | 0 .../{ => packs3}/overrides/overpack1.yaml | 0 .../{ => packs3}/overrides/overpack2.yaml | 0 .../{ => packs3}/overrides/overpack3.yaml | 0 .../{ => packs3}/overrides/overpack4.yaml | 0 st2common/tests/unit/test_content_loader.py | 63 ++++++++++++------- 13 files changed, 115 insertions(+), 24 deletions(-) create mode 100644 st2common/tests/resources/packs/BUILD create mode 100644 st2common/tests/resources/packs/fixture.py create mode 100644 st2common/tests/resources/packs2/BUILD create mode 100644 st2common/tests/resources/packs2/fixture.py create mode 100644 st2common/tests/resources/packs3/BUILD create mode 100644 st2common/tests/resources/packs3/fixture.py rename st2common/tests/resources/{ => packs3}/overrides/_global.yaml (100%) rename st2common/tests/resources/{ => packs3}/overrides/overpack1.yaml (100%) rename st2common/tests/resources/{ => packs3}/overrides/overpack2.yaml (100%) rename st2common/tests/resources/{ => packs3}/overrides/overpack3.yaml (100%) rename st2common/tests/resources/{ => packs3}/overrides/overpack4.yaml (100%) diff --git a/st2common/st2common/content/loader.py b/st2common/st2common/content/loader.py index 7e57ef6ec0..582834a45d 100644 --- a/st2common/st2common/content/loader.py +++ b/st2common/st2common/content/loader.py @@ -149,6 +149,8 @@ def get_content_from_pack(self, pack_dir, content_type): def _get_packs_from_dir(self, base_dir): result = {} for pack_name in os.listdir(base_dir): + if pack_name == "__pycache__": + continue pack_dir = os.path.join(base_dir, pack_name) pack_manifest_file = os.path.join(pack_dir, MANIFEST_FILE_NAME) @@ -160,6 +162,8 @@ def _get_packs_from_dir(self, base_dir): def _get_content_from_dir(self, base_dir, content_type): content = {} for pack in os.listdir(base_dir): + if pack == "__pycache__": + continue # TODO: Use function from util which escapes the name pack_dir = os.path.join(base_dir, pack) diff --git a/st2common/tests/resources/packs/BUILD b/st2common/tests/resources/packs/BUILD new file mode 100644 index 0000000000..8280f255bd --- /dev/null +++ b/st2common/tests/resources/packs/BUILD @@ -0,0 +1,8 @@ +resources( + name="packs_directories", + sources=["**/.gitignore"], +) + +python_sources( + dependencies=[":packs_directories"], +) diff --git a/st2common/tests/resources/packs/fixture.py b/st2common/tests/resources/packs/fixture.py new file mode 100644 index 0000000000..995f0ea33f --- /dev/null +++ b/st2common/tests/resources/packs/fixture.py @@ -0,0 +1,16 @@ +# Copyright 2024 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from st2tests import fixturesloader + +_, PACKS_BASE_PATH = fixturesloader.get_fixture_name_and_path(__file__) diff --git a/st2common/tests/resources/packs2/BUILD b/st2common/tests/resources/packs2/BUILD new file mode 100644 index 0000000000..8280f255bd --- /dev/null +++ b/st2common/tests/resources/packs2/BUILD @@ -0,0 +1,8 @@ +resources( + name="packs_directories", + sources=["**/.gitignore"], +) + +python_sources( + dependencies=[":packs_directories"], +) diff --git a/st2common/tests/resources/packs2/fixture.py b/st2common/tests/resources/packs2/fixture.py new file mode 100644 index 0000000000..995f0ea33f --- /dev/null +++ b/st2common/tests/resources/packs2/fixture.py @@ -0,0 +1,16 @@ +# Copyright 2024 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from st2tests import fixturesloader + +_, PACKS_BASE_PATH = fixturesloader.get_fixture_name_and_path(__file__) diff --git a/st2common/tests/resources/packs3/BUILD b/st2common/tests/resources/packs3/BUILD new file mode 100644 index 0000000000..eaed03e50e --- /dev/null +++ b/st2common/tests/resources/packs3/BUILD @@ -0,0 +1,8 @@ +resources( + name="packs_overrides", + sources=["**/*.yaml"], +) + +python_sources( + dependencies=[":packs_overrides"], +) diff --git a/st2common/tests/resources/packs3/fixture.py b/st2common/tests/resources/packs3/fixture.py new file mode 100644 index 0000000000..995f0ea33f --- /dev/null +++ b/st2common/tests/resources/packs3/fixture.py @@ -0,0 +1,16 @@ +# Copyright 2024 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from st2tests import fixturesloader + +_, PACKS_BASE_PATH = fixturesloader.get_fixture_name_and_path(__file__) diff --git a/st2common/tests/resources/overrides/_global.yaml b/st2common/tests/resources/packs3/overrides/_global.yaml similarity index 100% rename from st2common/tests/resources/overrides/_global.yaml rename to st2common/tests/resources/packs3/overrides/_global.yaml diff --git a/st2common/tests/resources/overrides/overpack1.yaml b/st2common/tests/resources/packs3/overrides/overpack1.yaml similarity index 100% rename from st2common/tests/resources/overrides/overpack1.yaml rename to st2common/tests/resources/packs3/overrides/overpack1.yaml diff --git a/st2common/tests/resources/overrides/overpack2.yaml b/st2common/tests/resources/packs3/overrides/overpack2.yaml similarity index 100% rename from st2common/tests/resources/overrides/overpack2.yaml rename to st2common/tests/resources/packs3/overrides/overpack2.yaml diff --git a/st2common/tests/resources/overrides/overpack3.yaml b/st2common/tests/resources/packs3/overrides/overpack3.yaml similarity index 100% rename from st2common/tests/resources/overrides/overpack3.yaml rename to st2common/tests/resources/packs3/overrides/overpack3.yaml diff --git a/st2common/tests/resources/overrides/overpack4.yaml b/st2common/tests/resources/packs3/overrides/overpack4.yaml similarity index 100% rename from st2common/tests/resources/overrides/overpack4.yaml rename to st2common/tests/resources/packs3/overrides/overpack4.yaml diff --git a/st2common/tests/unit/test_content_loader.py b/st2common/tests/unit/test_content_loader.py index 6edba192d4..700cd19d30 100644 --- a/st2common/tests/unit/test_content_loader.py +++ b/st2common/tests/unit/test_content_loader.py @@ -37,39 +37,36 @@ from st2common.constants.meta import yaml_safe_load from st2tests import config -CURRENT_DIR = os.path.dirname(os.path.realpath(__file__)) -RESOURCES_DIR = os.path.abspath(os.path.join(CURRENT_DIR, "../resources")) +from tests.resources.packs.fixture import PACKS_BASE_PATH as PACKS_BASE_PATH_1 +from tests.resources.packs2.fixture import PACKS_BASE_PATH as PACKS_BASE_PATH_2 +from tests.resources.packs3.fixture import PACKS_BASE_PATH as PACKS_BASE_PATH_3 class ContentLoaderTest(unittest.TestCase): def test_get_sensors(self): - packs_base_path = os.path.join(RESOURCES_DIR, "packs/") loader = ContentPackLoader() pack_sensors = loader.get_content( - base_dirs=[packs_base_path], content_type="sensors" + base_dirs=[PACKS_BASE_PATH_1], content_type="sensors" ) self.assertIsNotNone(pack_sensors.get("pack1", None)) def test_get_sensors_pack_missing_sensors(self): loader = ContentPackLoader() - fail_pack_path = os.path.join(RESOURCES_DIR, "packs/pack2") + fail_pack_path = os.path.join(PACKS_BASE_PATH_1, "pack2") self.assertTrue(os.path.exists(fail_pack_path)) self.assertEqual(loader._get_sensors(fail_pack_path), None) def test_invalid_content_type(self): - packs_base_path = os.path.join(RESOURCES_DIR, "packs/") loader = ContentPackLoader() self.assertRaises( ValueError, loader.get_content, - base_dirs=[packs_base_path], + base_dirs=[PACKS_BASE_PATH_1], content_type="stuff", ) def test_get_content_multiple_directories(self): - packs_base_path_1 = os.path.join(RESOURCES_DIR, "packs/") - packs_base_path_2 = os.path.join(RESOURCES_DIR, "packs2/") - base_dirs = [packs_base_path_1, packs_base_path_2] + base_dirs = [PACKS_BASE_PATH_1, PACKS_BASE_PATH_2] LOG.warning = Mock() @@ -81,14 +78,14 @@ def test_get_content_multiple_directories(self): # Assert that a warning is emitted when a duplicated pack is found expected_msg = ( 'Pack "pack1" already found in ' - '"%s/packs/", ignoring content from ' - '"%s/packs2/"' % (RESOURCES_DIR, RESOURCES_DIR) + f'"{PACKS_BASE_PATH_1}", ignoring content from ' + f'"{PACKS_BASE_PATH_2}"' ) LOG.warning.assert_called_once_with(expected_msg) def test_get_content_from_pack_success(self): loader = ContentPackLoader() - pack_path = os.path.join(RESOURCES_DIR, "packs/pack1") + pack_path = os.path.join(PACKS_BASE_PATH_1, "pack1") sensors = loader.get_content_from_pack( pack_dir=pack_path, content_type="sensors" @@ -97,7 +94,7 @@ def test_get_content_from_pack_success(self): def test_get_content_from_pack_directory_doesnt_exist(self): loader = ContentPackLoader() - pack_path = os.path.join(RESOURCES_DIR, "packs/pack100") + pack_path = os.path.join(PACKS_BASE_PATH_1, "pack100") message_regex = "Directory .*? doesn't exist" self.assertRaisesRegex( @@ -110,7 +107,7 @@ def test_get_content_from_pack_directory_doesnt_exist(self): def test_get_content_from_pack_no_sensors(self): loader = ContentPackLoader() - pack_path = os.path.join(RESOURCES_DIR, "packs/pack2") + pack_path = os.path.join(PACKS_BASE_PATH_1, "pack2") result = loader.get_content_from_pack( pack_dir=pack_path, content_type="sensors" @@ -119,7 +116,9 @@ def test_get_content_from_pack_no_sensors(self): def test_get_override_action_from_default(self): config.parse_args() - cfg.CONF.set_override(name="base_path", override=RESOURCES_DIR, group="system") + cfg.CONF.set_override( + name="base_path", override=PACKS_BASE_PATH_3, group="system" + ) loader = OverrideLoader() content = {"name": "action1", "enabled": True} self.assertTrue(loader.override("overpack1", "actions", content)) @@ -130,7 +129,9 @@ def test_get_override_action_from_default(self): def test_get_override_action_from_exception(self): config.parse_args() - cfg.CONF.set_override(name="base_path", override=RESOURCES_DIR, group="system") + cfg.CONF.set_override( + name="base_path", override=PACKS_BASE_PATH_3, group="system" + ) loader = OverrideLoader() content = {"name": "action2", "enabled": True} self.assertFalse(loader.override("overpack1", "actions", content)) @@ -141,7 +142,9 @@ def test_get_override_action_from_exception(self): def test_get_override_action_from_default_no_exceptions(self): config.parse_args() - cfg.CONF.set_override(name="base_path", override=RESOURCES_DIR, group="system") + cfg.CONF.set_override( + name="base_path", override=PACKS_BASE_PATH_3, group="system" + ) loader = OverrideLoader() content = {"name": "action1", "enabled": True} self.assertTrue(loader.override("overpack4", "actions", content)) @@ -152,7 +155,9 @@ def test_get_override_action_from_default_no_exceptions(self): def test_get_override_action_from_global_default_no_exceptions(self): config.parse_args() - cfg.CONF.set_override(name="base_path", override=RESOURCES_DIR, group="system") + cfg.CONF.set_override( + name="base_path", override=PACKS_BASE_PATH_3, group="system" + ) loader = OverrideLoader() content = {"class_name": "sensor1", "enabled": True} self.assertTrue(loader.override("overpack1", "sensors", content)) @@ -160,7 +165,9 @@ def test_get_override_action_from_global_default_no_exceptions(self): def test_get_override_action_from_global_overridden_by_pack(self): config.parse_args() - cfg.CONF.set_override(name="base_path", override=RESOURCES_DIR, group="system") + cfg.CONF.set_override( + name="base_path", override=PACKS_BASE_PATH_3, group="system" + ) loader = OverrideLoader() content = {"class_name": "sensor1", "enabled": True} self.assertFalse(loader.override("overpack2", "sensors", content)) @@ -168,7 +175,9 @@ def test_get_override_action_from_global_overridden_by_pack(self): def test_get_override_action_from_global_overridden_by_pack_exception(self): config.parse_args() - cfg.CONF.set_override(name="base_path", override=RESOURCES_DIR, group="system") + cfg.CONF.set_override( + name="base_path", override=PACKS_BASE_PATH_3, group="system" + ) loader = OverrideLoader() content = {"class_name": "sensor1", "enabled": True} self.assertFalse(loader.override("overpack3", "sensors", content)) @@ -176,7 +185,9 @@ def test_get_override_action_from_global_overridden_by_pack_exception(self): def test_get_override_invalid_type(self): config.parse_args() - cfg.CONF.set_override(name="base_path", override=RESOURCES_DIR, group="system") + cfg.CONF.set_override( + name="base_path", override=PACKS_BASE_PATH_3, group="system" + ) loader = OverrideLoader() content = {"name": "action2", "enabled": True} self.assertRaises( @@ -189,7 +200,9 @@ def test_get_override_invalid_type(self): def test_get_override_invalid_default_key(self): config.parse_args() - cfg.CONF.set_override(name="base_path", override=RESOURCES_DIR, group="system") + cfg.CONF.set_override( + name="base_path", override=PACKS_BASE_PATH_3, group="system" + ) loader = OverrideLoader() content = {"name": "action1", "enabled": True} self.assertRaises( @@ -202,7 +215,9 @@ def test_get_override_invalid_default_key(self): def test_get_override_invalid_exceptions_key(self): config.parse_args() - cfg.CONF.set_override(name="base_path", override=RESOURCES_DIR, group="system") + cfg.CONF.set_override( + name="base_path", override=PACKS_BASE_PATH_3, group="system" + ) loader = OverrideLoader() content = {"name": "action1", "enabled": True} loader.override("overpack1", "actions", content) From 4aefa9d55c2630360f691b4befed771de35ff783 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 23 May 2024 10:51:02 -0500 Subject: [PATCH 08/98] use fixtures imports for test_logger First, add metadata so pants can load the conf fixture from st2common/tests/resources for the logger test. Second, add a fixture.py file, similar to the fixtures we have in st2tests, so that pants dependency inferrence can see where these fixtures are used. --- st2common/tests/resources/BUILD | 8 ++++++++ st2common/tests/resources/fixture.py | 16 ++++++++++++++++ st2common/tests/unit/test_logger.py | 6 +++--- 3 files changed, 27 insertions(+), 3 deletions(-) create mode 100644 st2common/tests/resources/BUILD create mode 100644 st2common/tests/resources/fixture.py diff --git a/st2common/tests/resources/BUILD b/st2common/tests/resources/BUILD new file mode 100644 index 0000000000..a514672a5c --- /dev/null +++ b/st2common/tests/resources/BUILD @@ -0,0 +1,8 @@ +resource( + name="logging", + source="logging.conf", +) + +python_sources( + dependencies=[":logging"], +) diff --git a/st2common/tests/resources/fixture.py b/st2common/tests/resources/fixture.py new file mode 100644 index 0000000000..f6d0d3e940 --- /dev/null +++ b/st2common/tests/resources/fixture.py @@ -0,0 +1,16 @@ +# Copyright 2024 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from st2tests import fixturesloader + +FIXTURE_NAME, FIXTURE_PATH = fixturesloader.get_fixture_name_and_path(__file__) diff --git a/st2common/tests/unit/test_logger.py b/st2common/tests/unit/test_logger.py index e2126208eb..bd43be9f96 100644 --- a/st2common/tests/unit/test_logger.py +++ b/st2common/tests/unit/test_logger.py @@ -35,9 +35,9 @@ from st2common.models.db.execution import ActionExecutionDB import st2tests.config as tests_config -CURRENT_DIR = os.path.dirname(os.path.realpath(__file__)) -RESOURCES_DIR = os.path.abspath(os.path.join(CURRENT_DIR, "../resources")) -CONFIG_FILE_PATH = os.path.join(RESOURCES_DIR, "logging.conf") +from tests.resources.fixture import FIXTURE_PATH + +CONFIG_FILE_PATH = os.path.join(FIXTURE_PATH, "logging.conf") MOCK_MASKED_ATTRIBUTES_BLACKLIST = [ "blacklisted_1", From 9fddea6480787793661f606a87e5e4d936f3b09f Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 23 May 2024 11:00:35 -0500 Subject: [PATCH 09/98] use fixture import for test_plugin_loader First, add metadata so pants can load the loadableplugin fixture from st2common/tests/resources for the plugin loader test. Second, add a fixture.py file, similar to the fixtures we have in st2tests, so that pants dependency inferrence can see where these fixtures are used. --- st2common/tests/resources/loadableplugin/BUILD | 11 +++++++++++ .../tests/resources/loadableplugin/fixture.py | 16 ++++++++++++++++ st2common/tests/unit/test_plugin_loader.py | 5 ++--- 3 files changed, 29 insertions(+), 3 deletions(-) create mode 100644 st2common/tests/resources/loadableplugin/BUILD create mode 100644 st2common/tests/resources/loadableplugin/fixture.py diff --git a/st2common/tests/resources/loadableplugin/BUILD b/st2common/tests/resources/loadableplugin/BUILD new file mode 100644 index 0000000000..bd7354bb91 --- /dev/null +++ b/st2common/tests/resources/loadableplugin/BUILD @@ -0,0 +1,11 @@ +resources( + name="metadata", + sources=["*.yaml"], +) + +python_sources( + dependencies=[ + ":metadata", + "./plugin", + ], +) diff --git a/st2common/tests/resources/loadableplugin/fixture.py b/st2common/tests/resources/loadableplugin/fixture.py new file mode 100644 index 0000000000..f6d0d3e940 --- /dev/null +++ b/st2common/tests/resources/loadableplugin/fixture.py @@ -0,0 +1,16 @@ +# Copyright 2024 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from st2tests import fixturesloader + +FIXTURE_NAME, FIXTURE_PATH = fixturesloader.get_fixture_name_and_path(__file__) diff --git a/st2common/tests/unit/test_plugin_loader.py b/st2common/tests/unit/test_plugin_loader.py index e5a2822406..b2be514aca 100644 --- a/st2common/tests/unit/test_plugin_loader.py +++ b/st2common/tests/unit/test_plugin_loader.py @@ -24,9 +24,8 @@ import st2common.util.loader as plugin_loader -PLUGIN_FOLDER = "loadableplugin" -SRC_RELATIVE = os.path.join("../resources", PLUGIN_FOLDER) -SRC_ROOT = os.path.join(os.path.abspath(os.path.dirname(__file__)), SRC_RELATIVE) +from tests.resources.loadableplugin.fixture import FIXTURE_NAME as PLUGIN_FOLDER +from tests.resources.loadableplugin.fixture import FIXTURE_PATH as SRC_ROOT class LoaderTest(unittest.TestCase): From 717e2d0aee15f46ebc9ac8b1701cae143cf33b66 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 23 May 2024 12:44:51 -0500 Subject: [PATCH 10/98] pants: ignore test_dist_utils.py since we already ignore dist_utils.py Once pants can build our packages, all of the dist_utils bits should be deleted. --- pants.toml | 1 + st2common/tests/unit/test_dist_utils.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/pants.toml b/pants.toml index d1e9ba0611..41085db3a9 100644 --- a/pants.toml +++ b/pants.toml @@ -40,6 +40,7 @@ backend_packages = [ pants_ignore.add = [ # TODO: remove these once we start building wheels with pants. "dist_utils.py", + "test_dist_utils.py", "setup.py", # keep tailor from using legacy requirements files (not for pants) "contrib/examples/requirements.txt", diff --git a/st2common/tests/unit/test_dist_utils.py b/st2common/tests/unit/test_dist_utils.py index 9b6060b480..95991e64c5 100644 --- a/st2common/tests/unit/test_dist_utils.py +++ b/st2common/tests/unit/test_dist_utils.py @@ -13,6 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +# NB: Pantsbuild ignores this file and any dist_utils.py files. +# TODO: delete this file when deleting all dist_utils.py files. + import os import sys From 17079238555cdf32db2538db31210b3c22638332 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 23 May 2024 13:42:25 -0500 Subject: [PATCH 11/98] pants: fix running st2common/tests/unit/test_policies.py by adding dependency metadata --- st2tests/st2tests/fixtures/generic/BUILD | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/st2tests/st2tests/fixtures/generic/BUILD b/st2tests/st2tests/fixtures/generic/BUILD index 48fcf06310..5658638e23 100644 --- a/st2tests/st2tests/fixtures/generic/BUILD +++ b/st2tests/st2tests/fixtures/generic/BUILD @@ -3,6 +3,12 @@ pack_metadata( dependencies=[ "./actions:shell", "./actions:shell_resources", + # policytypes/fake_policy_type_1.py needs: + "//st2tests/st2tests/policies/concurrency.py", + # policytypes/fake_policy_type_2.py needs: + "//st2tests/st2tests/policies/mock_exception.py", + # policytypes/fake_policy_type_3.py needs: + "//st2actions/st2actions/policies/concurrency_by_attr.py", ], ) From c90fd8b716719057b96e3e197e3b65a55a3ddf03 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 23 May 2024 14:32:38 -0500 Subject: [PATCH 12/98] pants: update deps so pants can run test_policies_registrar.py tests --- st2common/tests/unit/BUILD | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/st2common/tests/unit/BUILD b/st2common/tests/unit/BUILD index a1da37051b..d83cbc6442 100644 --- a/st2common/tests/unit/BUILD +++ b/st2common/tests/unit/BUILD @@ -13,7 +13,10 @@ python_tests( ], uses=["mongo", "rabbitmq"], overrides={ - "test_util_file_system.py": dict( + ( + "test_util_file_system.py", + "test_policies_registrar.py", + ): dict( dependencies=[ "st2tests/st2tests/policies", "st2tests/st2tests/policies/meta", From 81bf32e512c6ce0a43cdd22a1ecceb5cea90e4e9 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 23 May 2024 15:24:39 -0500 Subject: [PATCH 13/98] Ensure tests that use logging.conf files have the required st2common code --- pants-plugins/macros.py | 31 +++++++++++++++++++++++++++ st2actions/conf/BUILD | 6 +++--- st2api/conf/BUILD | 8 +++---- st2auth/conf/BUILD | 8 +++---- st2reactor/conf/BUILD | 6 +++--- st2stream/conf/BUILD | 8 +++---- st2tests/conf/BUILD | 4 ++-- st2tests/st2tests/fixtures/conf/BUILD | 7 +++++- 8 files changed, 57 insertions(+), 21 deletions(-) diff --git a/pants-plugins/macros.py b/pants-plugins/macros.py index bc346a057b..c41d9b1f33 100644 --- a/pants-plugins/macros.py +++ b/pants-plugins/macros.py @@ -125,3 +125,34 @@ def st2_shell_sources_and_resources(**kwargs): kwargs["name"] += "_resources" resources(**kwargs) # noqa: F821 + + +# these are referenced by the logging.*.conf files. +_st2common_logging_deps = ( + "//st2common/st2common/log.py", + "//st2common/st2common/logging/formatters.py", +) + + +def st2_logging_conf_files(**kwargs): + """This creates a files target with logging dependencies.""" + deps = kwargs.pop("dependencies", []) or [] + deps = list(deps) + list(_st2common_logging_deps) + kwargs["dependencies"] = tuple(deps) + files(**kwargs) + + +def st2_logging_conf_file(**kwargs): + """This creates a file target with logging dependencies.""" + deps = kwargs.pop("dependencies", []) or [] + deps = list(deps) + list(_st2common_logging_deps) + kwargs["dependencies"] = tuple(deps) + file(**kwargs) + + +def st2_logging_conf_resources(**kwargs): + """This creates a resources target with logging dependencies.""" + deps = kwargs.pop("dependencies", []) or [] + deps = list(deps) + list(_st2common_logging_deps) + kwargs["dependencies"] = tuple(deps) + resources(**kwargs) diff --git a/st2actions/conf/BUILD b/st2actions/conf/BUILD index 42e57e0d5a..ee9257725e 100644 --- a/st2actions/conf/BUILD +++ b/st2actions/conf/BUILD @@ -1,9 +1,9 @@ -file( +st2_logging_conf_file( name="logging_console", source="console.conf", ) -files( +st2_logging_conf_files( name="logging", sources=["logging*.conf"], overrides={ @@ -15,7 +15,7 @@ files( }, ) -files( +st2_logging_conf_files( name="logging_syslog", sources=["syslog*.conf"], ) diff --git a/st2api/conf/BUILD b/st2api/conf/BUILD index 9c3668885c..df77688843 100644 --- a/st2api/conf/BUILD +++ b/st2api/conf/BUILD @@ -1,19 +1,19 @@ -file( +st2_logging_conf_file( name="logging_console", source="console.conf", ) -file( +st2_logging_conf_file( name="logging", source="logging.conf", ) -file( +st2_logging_conf_file( name="logging_gunicorn", source="logging.gunicorn.conf", ) -file( +st2_logging_conf_file( name="logging_syslog", source="syslog.conf", ) diff --git a/st2auth/conf/BUILD b/st2auth/conf/BUILD index 8fd094d9fa..c1adc35ceb 100644 --- a/st2auth/conf/BUILD +++ b/st2auth/conf/BUILD @@ -8,22 +8,22 @@ file( source="htpasswd_dev", ) -file( +st2_logging_conf_file( name="logging_console", source="console.conf", ) -file( +st2_logging_conf_file( name="logging", source="logging.conf", ) -file( +st2_logging_conf_file( name="logging_gunicorn", source="logging.gunicorn.conf", ) -file( +st2_logging_conf_file( name="logging_syslog", source="syslog.conf", ) diff --git a/st2reactor/conf/BUILD b/st2reactor/conf/BUILD index 57246fe48d..8418a76e23 100644 --- a/st2reactor/conf/BUILD +++ b/st2reactor/conf/BUILD @@ -1,14 +1,14 @@ -file( +st2_logging_conf_file( name="logging_console", source="console.conf", ) -files( +st2_logging_conf_files( name="logging", sources=["logging*.conf"], ) -files( +st2_logging_conf_files( name="logging_syslog", sources=["syslog*.conf"], ) diff --git a/st2stream/conf/BUILD b/st2stream/conf/BUILD index 9c3668885c..df77688843 100644 --- a/st2stream/conf/BUILD +++ b/st2stream/conf/BUILD @@ -1,19 +1,19 @@ -file( +st2_logging_conf_file( name="logging_console", source="console.conf", ) -file( +st2_logging_conf_file( name="logging", source="logging.conf", ) -file( +st2_logging_conf_file( name="logging_gunicorn", source="logging.gunicorn.conf", ) -file( +st2_logging_conf_file( name="logging_syslog", source="syslog.conf", ) diff --git a/st2tests/conf/BUILD b/st2tests/conf/BUILD index 72352b12b1..23af9ef7c9 100644 --- a/st2tests/conf/BUILD +++ b/st2tests/conf/BUILD @@ -8,12 +8,12 @@ file( source="st2_kvstore_tests.crypto.key.json", ) -file( +st2_logging_conf_file( name="logging.conf", source="logging.conf", ) -files( +st2_logging_conf_files( name="other_logging_conf", sources=["logging.*.conf"], ) diff --git a/st2tests/st2tests/fixtures/conf/BUILD b/st2tests/st2tests/fixtures/conf/BUILD index 5674cea485..545508803f 100644 --- a/st2tests/st2tests/fixtures/conf/BUILD +++ b/st2tests/st2tests/fixtures/conf/BUILD @@ -1,10 +1,15 @@ +st2_logging_conf_resources( + name="logging", + sources=["logging.*.conf"], +) + resources( name="st2.tests.conf", sources=[ "st2.tests*.conf", - "logging.*.conf", # used by st2.tests*.conf ], dependencies=[ + ":logging", # used by st2.tests*.conf "st2tests/conf:other_logging_conf", # depending on st2auth from st2tests is not nice. "st2auth/conf:htpasswd", From 49f7253187a79bae5db338b5eb8774c96139e18a Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 22 May 2024 22:16:05 -0500 Subject: [PATCH 14/98] tests: Ensure tests can run in isolation Various tests were relying on the side effects of tests that nosetest runs before they ran. These include: - 3 in test_action_alias_utils.py::TestInjectImmutableParameters - 2 in test_jinja_render_data_filters.py - 1 in test_logging_middleware.py - 9 in test_operators.py::SearchOperatorTest - 3 in test_util_payload.py In particular, the oslo config initialization from the tests in st2common/tests/unit/services/ happens before these test ran, obscuring their dependence on this initialization. Pants runs each test file separately for fine-grained caching. --- st2common/tests/unit/test_action_alias_utils.py | 6 ++++++ st2common/tests/unit/test_jinja_render_data_filters.py | 6 ++++++ st2common/tests/unit/test_logging_middleware.py | 6 ++++++ st2common/tests/unit/test_operators.py | 6 ++++++ st2common/tests/unit/test_util_payload.py | 2 ++ 5 files changed, 26 insertions(+) diff --git a/st2common/tests/unit/test_action_alias_utils.py b/st2common/tests/unit/test_action_alias_utils.py index ec3251070a..1fc54359b0 100644 --- a/st2common/tests/unit/test_action_alias_utils.py +++ b/st2common/tests/unit/test_action_alias_utils.py @@ -30,6 +30,7 @@ search_regex_tokens, inject_immutable_parameters, ) +import st2tests.config as tests_config class TestActionAliasParser(TestCase): @@ -357,6 +358,11 @@ def test_subpatterns(self): class TestInjectImmutableParameters(TestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + tests_config.parse_args() + def test_immutable_parameters_are_injected(self): action_alias_db = Mock() action_alias_db.immutable_parameters = {"env": "dev"} diff --git a/st2common/tests/unit/test_jinja_render_data_filters.py b/st2common/tests/unit/test_jinja_render_data_filters.py index 8db175cac2..bc9ad1c02c 100644 --- a/st2common/tests/unit/test_jinja_render_data_filters.py +++ b/st2common/tests/unit/test_jinja_render_data_filters.py @@ -21,9 +21,15 @@ from st2common.constants.keyvalue import FULL_SYSTEM_SCOPE from st2common.util import jinja as jinja_utils from st2common.services.keyvalues import KeyValueLookup +import st2tests.config as tests_config class JinjaUtilsDataFilterTestCase(unittest.TestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + tests_config.parse_args() + def test_filter_from_json_string(self): env = jinja_utils.get_jinja_environment() expected_obj = {"a": "b", "c": {"d": "e", "f": 1, "g": True}} diff --git a/st2common/tests/unit/test_logging_middleware.py b/st2common/tests/unit/test_logging_middleware.py index 2da4bb5875..8521a622e6 100644 --- a/st2common/tests/unit/test_logging_middleware.py +++ b/st2common/tests/unit/test_logging_middleware.py @@ -20,11 +20,17 @@ from st2common.middleware.logging import LoggingMiddleware from st2common.constants.secrets import MASKED_ATTRIBUTE_VALUE +import st2tests.config as tests_config __all__ = ["LoggingMiddlewareTestCase"] class LoggingMiddlewareTestCase(unittest.TestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + tests_config.parse_args() + @mock.patch("st2common.middleware.logging.LOG") @mock.patch("st2common.middleware.logging.Request") def test_secret_parameters_are_masked_in_log_message(self, mock_request, mock_log): diff --git a/st2common/tests/unit/test_operators.py b/st2common/tests/unit/test_operators.py index 9bb6161f91..39b2cc7a8c 100644 --- a/st2common/tests/unit/test_operators.py +++ b/st2common/tests/unit/test_operators.py @@ -18,6 +18,7 @@ from st2common import operators from st2common.util import date as date_utils +import st2tests.config as tests_config def list_of_dicts_strict_equal(lofd1, lofd2): @@ -157,6 +158,11 @@ def test_less_simple_dicts(self): class SearchOperatorTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + tests_config.parse_args() + # The search command extends the rules engine into being a recursive descent # parser. As such, its tests are much more complex than other commands, so we # pull its tests out into their own test case. diff --git a/st2common/tests/unit/test_util_payload.py b/st2common/tests/unit/test_util_payload.py index d6629d6d57..715786e2f0 100644 --- a/st2common/tests/unit/test_util_payload.py +++ b/st2common/tests/unit/test_util_payload.py @@ -18,6 +18,7 @@ import unittest from st2common.util.payload import PayloadLookup +import st2tests.config as tests_config __all__ = ["PayloadLookupTestCase"] @@ -32,6 +33,7 @@ def setUpClass(cls): } ) super(PayloadLookupTestCase, cls).setUpClass() + tests_config.parse_args() def test_get_key(self): self.assertEqual(self.payload.get_value("trigger.pikachu"), ["Has no ears"]) From a7d435c2fa339711f0ae74b866e640d3007e3c3a Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 23 May 2024 15:57:46 -0500 Subject: [PATCH 15/98] pants: fix running test_util_file_system test --- st2common/tests/unit/test_util_file_system.py | 12 ++++++------ st2tests/st2tests/policies/meta/BUILD | 8 +++++++- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/st2common/tests/unit/test_util_file_system.py b/st2common/tests/unit/test_util_file_system.py index f45498702b..58f7bf6473 100644 --- a/st2common/tests/unit/test_util_file_system.py +++ b/st2common/tests/unit/test_util_file_system.py @@ -27,21 +27,21 @@ class FileSystemUtilsTestCase(unittest.TestCase): def test_get_file_list(self): + # NB: Make sure to exclude BUILD files as pants will not include them in the sandbox, + # but the BUILD files will be present if you directly run the tests. + basic_excludes = ["*.pyc", "__pycache__", "*BUILD"] + # Standard exclude pattern directory = os.path.join(ST2TESTS_DIR, "policies") expected = [ - "BUILD", "mock_exception.py", "concurrency.py", "__init__.py", - "meta/BUILD", "meta/mock_exception.yaml", "meta/concurrency.yaml", "meta/__init__.py", ] - result = get_file_list( - directory=directory, exclude_patterns=["*.pyc", "__pycache__"] - ) + result = get_file_list(directory=directory, exclude_patterns=basic_excludes) # directory listings are sorted because the item order must be exact for assert # to validate equivalence. Directory item order doesn't matter in general and may # even change on different platforms or locales. @@ -55,7 +55,7 @@ def test_get_file_list(self): "meta/__init__.py", ] result = get_file_list( - directory=directory, exclude_patterns=["*.pyc", "*.yaml", "*BUILD"] + directory=directory, exclude_patterns=["*.yaml"] + basic_excludes ) # directory listings are sorted because the item order must be exact for assert # to validate equivalence. Directory item order doesn't matter in general and may diff --git a/st2tests/st2tests/policies/meta/BUILD b/st2tests/st2tests/policies/meta/BUILD index 0700f34cd5..53560669a3 100644 --- a/st2tests/st2tests/policies/meta/BUILD +++ b/st2tests/st2tests/policies/meta/BUILD @@ -1,3 +1,9 @@ resources( - sources=["*.yaml"], + sources=[ + "*.yaml", + # pants ignores empty __init__.py files. + # However, the tests for st2common.util.file_system need it to be present, + # so we treat it as a resource instead of a python source file. + "__init__.py", + ], ) From c54dafc5411fa28731ceab8958e1b26a2b90f036 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 23 May 2024 16:21:23 -0500 Subject: [PATCH 16/98] pants: adjust deps to fix running test_shell_action_system_model tests --- st2common/tests/fixtures/local_runner/BUILD | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/st2common/tests/fixtures/local_runner/BUILD b/st2common/tests/fixtures/local_runner/BUILD index db46e8d6c9..25c5073d83 100644 --- a/st2common/tests/fixtures/local_runner/BUILD +++ b/st2common/tests/fixtures/local_runner/BUILD @@ -1 +1,8 @@ -python_sources() +resources( + name="command_strings", + sources=["escaping_test_command_*.txt"], +) + +python_sources( + dependencies=[":command_strings"], +) From 1f1ae2e931b091e7da47fc5d8899916e3a330183 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 23 May 2024 16:22:03 -0500 Subject: [PATCH 17/98] pants: monkey patch in each test_purge* test file so pytest can run them --- st2common/tests/unit/test_purge_rule_enforcement.py | 6 ++++++ st2common/tests/unit/test_purge_task_executions.py | 6 ++++++ st2common/tests/unit/test_purge_token.py | 6 ++++++ st2common/tests/unit/test_purge_trace.py | 6 ++++++ st2common/tests/unit/test_purge_worklows.py | 6 ++++++ 5 files changed, 30 insertions(+) diff --git a/st2common/tests/unit/test_purge_rule_enforcement.py b/st2common/tests/unit/test_purge_rule_enforcement.py index 90b4d23799..1b00228fa3 100644 --- a/st2common/tests/unit/test_purge_rule_enforcement.py +++ b/st2common/tests/unit/test_purge_rule_enforcement.py @@ -13,6 +13,12 @@ # limitations under the License. from __future__ import absolute_import + +# pytest: make sure monkey_patching happens before importing mongoengine +from st2common.util.monkey_patch import monkey_patch + +monkey_patch() + from datetime import timedelta import bson diff --git a/st2common/tests/unit/test_purge_task_executions.py b/st2common/tests/unit/test_purge_task_executions.py index b0c7cd8bc2..b5c2dc19fe 100644 --- a/st2common/tests/unit/test_purge_task_executions.py +++ b/st2common/tests/unit/test_purge_task_executions.py @@ -13,6 +13,12 @@ # limitations under the License. from __future__ import absolute_import + +# pytest: make sure monkey_patching happens before importing mongoengine +from st2common.util.monkey_patch import monkey_patch + +monkey_patch() + from datetime import timedelta from st2common import log as logging diff --git a/st2common/tests/unit/test_purge_token.py b/st2common/tests/unit/test_purge_token.py index 75c24e62cd..1bad08d097 100644 --- a/st2common/tests/unit/test_purge_token.py +++ b/st2common/tests/unit/test_purge_token.py @@ -13,6 +13,12 @@ # limitations under the License. from __future__ import absolute_import + +# pytest: make sure monkey_patching happens before importing mongoengine +from st2common.util.monkey_patch import monkey_patch + +monkey_patch() + from datetime import timedelta import bson diff --git a/st2common/tests/unit/test_purge_trace.py b/st2common/tests/unit/test_purge_trace.py index 9a819b4018..7dde63f9f1 100644 --- a/st2common/tests/unit/test_purge_trace.py +++ b/st2common/tests/unit/test_purge_trace.py @@ -13,6 +13,12 @@ # limitations under the License. from __future__ import absolute_import + +# pytest: make sure monkey_patching happens before importing mongoengine +from st2common.util.monkey_patch import monkey_patch + +monkey_patch() + from datetime import timedelta import bson diff --git a/st2common/tests/unit/test_purge_worklows.py b/st2common/tests/unit/test_purge_worklows.py index 2975c504cf..383030b1e3 100644 --- a/st2common/tests/unit/test_purge_worklows.py +++ b/st2common/tests/unit/test_purge_worklows.py @@ -13,6 +13,12 @@ # limitations under the License. from __future__ import absolute_import + +# pytest: make sure monkey_patching happens before importing mongoengine +from st2common.util.monkey_patch import monkey_patch + +monkey_patch() + from datetime import timedelta from st2common import log as logging From 8a2f489ec5de6fbfd0b31963d61d67c05834127f Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 23 May 2024 22:21:23 -0500 Subject: [PATCH 18/98] pants: fix running services/test_packs tests --- contrib/core/BUILD | 6 +++++- contrib/core/actions/send_mail/BUILD | 5 +++-- st2common/tests/unit/services/test_packs.py | 9 +++++++++ 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/contrib/core/BUILD b/contrib/core/BUILD index 7db2dc9d25..59673bd746 100644 --- a/contrib/core/BUILD +++ b/contrib/core/BUILD @@ -8,5 +8,9 @@ python_requirements( ) python_sources( - dependencies=[":metadata"], + dependencies=[ + ":metadata", + "./actions", + "./actions/send_mail:send_mail_resources", + ], ) diff --git a/contrib/core/actions/send_mail/BUILD b/contrib/core/actions/send_mail/BUILD index f27e7c10ec..94280e6e49 100644 --- a/contrib/core/actions/send_mail/BUILD +++ b/contrib/core/actions/send_mail/BUILD @@ -1,4 +1,5 @@ -shell_source( - source="send_mail", +st2_shell_sources_and_resources( + name="send_mail", + sources=["send_mail"], skip_shellcheck=True, ) diff --git a/st2common/tests/unit/services/test_packs.py b/st2common/tests/unit/services/test_packs.py index 69152a8398..9472f920b8 100644 --- a/st2common/tests/unit/services/test_packs.py +++ b/st2common/tests/unit/services/test_packs.py @@ -43,6 +43,7 @@ from st2tests.fixtures.packs.orquesta_tests.fixture import ( PACK_NAME as TEST_SOURCE_WORKFLOW_PACK, ) +import st2tests.config as tests_config SOURCE_ACTION_WITH_PYTHON_SCRIPT_RUNNER = { "description": "Action which injects a new trigger in the system.", @@ -416,6 +417,8 @@ def test_exception_to_remove_resource_metadata_file(self, remove): class CloneActionDBAndFilesTestCase(unittest.TestCase): @classmethod def setUpClass(cls): + super().setUpClass() + tests_config.parse_args() action_files_path = os.path.join(TEST_DEST_PACK_PATH, "actions") workflow_files_path = os.path.join(action_files_path, "workflows") if not os.path.isdir(action_files_path): @@ -423,6 +426,7 @@ def setUpClass(cls): if not os.path.isdir(workflow_files_path): os.mkdir(workflow_files_path) + @classmethod def tearDownClass(cls): action_files_path = os.path.join(TEST_DEST_PACK_PATH, "actions") @@ -588,6 +592,11 @@ def test_workflows_directory_created_if_does_not_exist(self): class CloneActionFilesBackupTestCase(unittest.TestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + tests_config.parse_args() + @classmethod def tearDownClass(cls): action_files_path = os.path.join(TEST_DEST_PACK_PATH, "actions") From 785add88a87779a4ad2356595a3e4e9b733d6772 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 23 May 2024 23:30:36 -0500 Subject: [PATCH 19/98] tests: drop unnecessary calls to tests_config.parse_args() This looks like copy pasta as some of these files have a comment saying that running this before importing something else is required. However, by importing st2tests, that already implicitly happens in st2tests/st2tests/base.py. Then tests_config.parse_args() gets called again in the class init. Plus, I reviewed all the other imports, and none of them have import time side effects that matter for oslo config bits. So, these calls are not necessary, and the comments about them are wrong. --- st2common/tests/unit/services/test_workflow.py | 4 ---- st2common/tests/unit/services/test_workflow_cancellation.py | 4 ---- .../tests/unit/services/test_workflow_identify_orphans.py | 5 ----- st2common/tests/unit/services/test_workflow_rerun.py | 4 ---- .../tests/unit/services/test_workflow_service_retries.py | 5 ----- 5 files changed, 22 deletions(-) diff --git a/st2common/tests/unit/services/test_workflow.py b/st2common/tests/unit/services/test_workflow.py index bcce91af00..d4897ff322 100644 --- a/st2common/tests/unit/services/test_workflow.py +++ b/st2common/tests/unit/services/test_workflow.py @@ -24,10 +24,6 @@ import st2tests -import st2tests.config as tests_config - -tests_config.parse_args() - from st2common.bootstrap import actionsregistrar from st2common.bootstrap import runnersregistrar from st2common.exceptions import action as action_exc diff --git a/st2common/tests/unit/services/test_workflow_cancellation.py b/st2common/tests/unit/services/test_workflow_cancellation.py index d8b7b2206f..88e81e4fbf 100644 --- a/st2common/tests/unit/services/test_workflow_cancellation.py +++ b/st2common/tests/unit/services/test_workflow_cancellation.py @@ -21,10 +21,6 @@ import st2tests -import st2tests.config as tests_config - -tests_config.parse_args() - from st2common.bootstrap import actionsregistrar from st2common.bootstrap import runnersregistrar from st2common.models.db import liveaction as lv_db_models diff --git a/st2common/tests/unit/services/test_workflow_identify_orphans.py b/st2common/tests/unit/services/test_workflow_identify_orphans.py index 7110b509c9..ba1df395a8 100644 --- a/st2common/tests/unit/services/test_workflow_identify_orphans.py +++ b/st2common/tests/unit/services/test_workflow_identify_orphans.py @@ -22,11 +22,6 @@ import st2tests -# XXX: actionsensor import depends on config being setup. -import st2tests.config as tests_config - -tests_config.parse_args() - from st2common.bootstrap import actionsregistrar from st2common.bootstrap import runnersregistrar from st2common.constants import action as ac_const diff --git a/st2common/tests/unit/services/test_workflow_rerun.py b/st2common/tests/unit/services/test_workflow_rerun.py index b1bcb7417c..bb3d1595ba 100644 --- a/st2common/tests/unit/services/test_workflow_rerun.py +++ b/st2common/tests/unit/services/test_workflow_rerun.py @@ -23,10 +23,6 @@ import st2tests -import st2tests.config as tests_config - -tests_config.parse_args() - from local_runner import local_shell_command_runner from st2common.bootstrap import actionsregistrar from st2common.bootstrap import runnersregistrar diff --git a/st2common/tests/unit/services/test_workflow_service_retries.py b/st2common/tests/unit/services/test_workflow_service_retries.py index 0e322fe573..ca2fab6f9f 100644 --- a/st2common/tests/unit/services/test_workflow_service_retries.py +++ b/st2common/tests/unit/services/test_workflow_service_retries.py @@ -30,11 +30,6 @@ import st2tests -# XXX: actionsensor import depends on config being setup. -import st2tests.config as tests_config - -tests_config.parse_args() - from st2common.bootstrap import actionsregistrar from st2common.bootstrap import runnersregistrar from st2common.constants import action as ac_const From 88a5d8850df97c593aeb7b87fa9e286ace6accc0 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 23 May 2024 23:32:20 -0500 Subject: [PATCH 20/98] tests: add missing monkey_patch for isolated test support pants runs each test file separately. test_workflow_rerun only worked under nosetest because earlier files already did the monkey_patch. Without this, running this file in isolation, with either nosetest or pytest, hangs. --- st2common/tests/unit/services/test_workflow_rerun.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/st2common/tests/unit/services/test_workflow_rerun.py b/st2common/tests/unit/services/test_workflow_rerun.py index bb3d1595ba..c91e140d6c 100644 --- a/st2common/tests/unit/services/test_workflow_rerun.py +++ b/st2common/tests/unit/services/test_workflow_rerun.py @@ -15,6 +15,11 @@ from __future__ import absolute_import +from st2common.util.monkey_patch import monkey_patch + +monkey_patch() + + import mock import uuid From cf93d961d70786eba95637a23c90f6ac4072ee57 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 23 May 2024 23:34:26 -0500 Subject: [PATCH 21/98] tests: allow overriding system_user.user via env vars in tests This way we do not need to patch the conf files in so many places. --- .github/workflows/test.yaml | 3 +++ pants.toml | 7 +++++++ st2tests/st2tests/config.py | 3 +++ 3 files changed, 13 insertions(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index f8966d1009..7d52781be4 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -99,6 +99,9 @@ jobs: pants test pants-plugins/:: - name: Unit Tests + env: + # Github Actions uses the 'runner' user, so use that instead of stanley. + ST2TESTS_SYSTEM_USER: 'runner' # We do not support running pytest everywhere yet. When we do it will be simply: # pants test :: # Until then, we need to manually adjust this command line to test what we can. diff --git a/pants.toml b/pants.toml index 41085db3a9..8cda497564 100644 --- a/pants.toml +++ b/pants.toml @@ -239,6 +239,13 @@ config = "@lint-configs/regex-lint.yaml" [setuptools] install_from_resolve = "st2" +[test] +extra_env_vars = [ + # Use this so that the test system does not require the stanley user. + # For example: export ST2TESTS_SYSTEM_USER=${USER} + "ST2TESTS_SYSTEM_USER", +] + [twine] install_from_resolve = "twine" diff --git a/st2tests/st2tests/config.py b/st2tests/st2tests/config.py index 351456a261..bef5197bcf 100644 --- a/st2tests/st2tests/config.py +++ b/st2tests/st2tests/config.py @@ -104,6 +104,9 @@ def _override_common_opts(): CONF.set_override(name="api_url", override="http://127.0.0.1", group="auth") CONF.set_override(name="mask_secrets", override=True, group="log") CONF.set_override(name="stream_output", override=False, group="actionrunner") + system_user = os.environ.get("ST2TESTS_SYSTEM_USER", "") + if system_user: + CONF.set_override(name="user", override=system_user, group="system_user") def _override_api_opts(): From 8869245fcb837eca349e7669a10b643010bd6153 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 23 May 2024 23:35:33 -0500 Subject: [PATCH 22/98] pants ci: run st2common unit tests all are passing locally now --- .github/workflows/test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 7d52781be4..f0be482057 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -113,6 +113,7 @@ jobs: test pylint_plugins/:: st2client/:: + st2common/:: - name: Upload pants log uses: actions/upload-artifact@v4 From 6a0e971bdade2a3d0c7e8afcad72dd60a3bf4682 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 23 May 2024 23:39:55 -0500 Subject: [PATCH 23/98] fmt with black --- st2common/tests/unit/services/test_packs.py | 1 - 1 file changed, 1 deletion(-) diff --git a/st2common/tests/unit/services/test_packs.py b/st2common/tests/unit/services/test_packs.py index 9472f920b8..668137e9cb 100644 --- a/st2common/tests/unit/services/test_packs.py +++ b/st2common/tests/unit/services/test_packs.py @@ -426,7 +426,6 @@ def setUpClass(cls): if not os.path.isdir(workflow_files_path): os.mkdir(workflow_files_path) - @classmethod def tearDownClass(cls): action_files_path = os.path.join(TEST_DEST_PACK_PATH, "actions") From 69ef3db35018ff16e958d9529ad38eae3227c1e9 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 23 May 2024 23:41:48 -0500 Subject: [PATCH 24/98] satisfy pylint --- pants-plugins/macros.py | 6 +++--- st2common/tests/unit/test_plugin_loader.py | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/pants-plugins/macros.py b/pants-plugins/macros.py index c41d9b1f33..11131f20ee 100644 --- a/pants-plugins/macros.py +++ b/pants-plugins/macros.py @@ -139,7 +139,7 @@ def st2_logging_conf_files(**kwargs): deps = kwargs.pop("dependencies", []) or [] deps = list(deps) + list(_st2common_logging_deps) kwargs["dependencies"] = tuple(deps) - files(**kwargs) + files(**kwargs) # noqa: F821 def st2_logging_conf_file(**kwargs): @@ -147,7 +147,7 @@ def st2_logging_conf_file(**kwargs): deps = kwargs.pop("dependencies", []) or [] deps = list(deps) + list(_st2common_logging_deps) kwargs["dependencies"] = tuple(deps) - file(**kwargs) + file(**kwargs) # noqa: F821 def st2_logging_conf_resources(**kwargs): @@ -155,4 +155,4 @@ def st2_logging_conf_resources(**kwargs): deps = kwargs.pop("dependencies", []) or [] deps = list(deps) + list(_st2common_logging_deps) kwargs["dependencies"] = tuple(deps) - resources(**kwargs) + resources(**kwargs) # noqa: F821 diff --git a/st2common/tests/unit/test_plugin_loader.py b/st2common/tests/unit/test_plugin_loader.py index b2be514aca..8e594ecb95 100644 --- a/st2common/tests/unit/test_plugin_loader.py +++ b/st2common/tests/unit/test_plugin_loader.py @@ -24,7 +24,6 @@ import st2common.util.loader as plugin_loader -from tests.resources.loadableplugin.fixture import FIXTURE_NAME as PLUGIN_FOLDER from tests.resources.loadableplugin.fixture import FIXTURE_PATH as SRC_ROOT From d4a943f75575dc3d6ba3f75de3ace937f63d203e Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 23 May 2024 23:43:33 -0500 Subject: [PATCH 25/98] pants update-build-files :: to fmt BUILD w/ black --- st2common/tests/unit/BUILD | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/st2common/tests/unit/BUILD b/st2common/tests/unit/BUILD index d83cbc6442..2652b81de6 100644 --- a/st2common/tests/unit/BUILD +++ b/st2common/tests/unit/BUILD @@ -13,10 +13,7 @@ python_tests( ], uses=["mongo", "rabbitmq"], overrides={ - ( - "test_util_file_system.py", - "test_policies_registrar.py", - ): dict( + ("test_util_file_system.py", "test_policies_registrar.py",): dict( dependencies=[ "st2tests/st2tests/policies", "st2tests/st2tests/policies/meta", From 65cae40dedf1fcf11ccc9189c374836c954147a1 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 24 May 2024 10:13:23 -0500 Subject: [PATCH 26/98] pants: register dependency on stevedore namespaces in st2actions/test/unit/policies --- st2actions/tests/unit/policies/BUILD | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/st2actions/tests/unit/policies/BUILD b/st2actions/tests/unit/policies/BUILD index 57341b1358..66a22040d1 100644 --- a/st2actions/tests/unit/policies/BUILD +++ b/st2actions/tests/unit/policies/BUILD @@ -1,3 +1,8 @@ python_tests( name="tests", + stevedore_namespaces=[ + "st2common.runners.runner", + "st2common.metrics.driver", + ], + uses=["mongo"], ) From 8a17a7bc581b95a2ca7cc30befb6e5cb747141d1 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 24 May 2024 10:23:01 -0500 Subject: [PATCH 27/98] Remove duplicate test fixture files These are not test files: - st2actions/tests/unit/test_async_runner.py - st2actions/tests/unit/test_polling_async_runner.py It looks like they were copied to st2tests/st2tests/mocks/runners/ at some point. Nothing imports from or uses the copies in st2actions, so just delete them. --- st2actions/tests/unit/test_async_runner.py | 54 ------------------- .../tests/unit/test_polling_async_runner.py | 54 ------------------- 2 files changed, 108 deletions(-) delete mode 100644 st2actions/tests/unit/test_async_runner.py delete mode 100644 st2actions/tests/unit/test_polling_async_runner.py diff --git a/st2actions/tests/unit/test_async_runner.py b/st2actions/tests/unit/test_async_runner.py deleted file mode 100644 index 31258fae4e..0000000000 --- a/st2actions/tests/unit/test_async_runner.py +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright 2020 The StackStorm Authors. -# Copyright 2019 Extreme Networks, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import absolute_import - -try: - import simplejson as json -except: - import json - -from st2common.runners.base import AsyncActionRunner -from st2common.constants.action import LIVEACTION_STATUS_RUNNING - -RAISE_PROPERTY = "raise" - - -def get_runner(): - return AsyncTestRunner() - - -class AsyncTestRunner(AsyncActionRunner): - def __init__(self): - super(AsyncTestRunner, self).__init__(runner_id="1") - self.pre_run_called = False - self.run_called = False - self.post_run_called = False - - def pre_run(self): - self.pre_run_called = True - - def run(self, action_params): - self.run_called = True - result = {} - if self.runner_parameters.get(RAISE_PROPERTY, False): - raise Exception("Raise required.") - else: - result = {"ran": True, "action_params": action_params} - - return (LIVEACTION_STATUS_RUNNING, json.dumps(result), {"id": "foo"}) - - def post_run(self, status, result): - self.post_run_called = True diff --git a/st2actions/tests/unit/test_polling_async_runner.py b/st2actions/tests/unit/test_polling_async_runner.py deleted file mode 100644 index c48bb9aa67..0000000000 --- a/st2actions/tests/unit/test_polling_async_runner.py +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright 2020 The StackStorm Authors. -# Copyright 2019 Extreme Networks, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import absolute_import - -try: - import simplejson as json -except: - import json - -from st2common.runners.base import PollingAsyncActionRunner -from st2common.constants.action import LIVEACTION_STATUS_RUNNING - -RAISE_PROPERTY = "raise" - - -def get_runner(): - return PollingAsyncTestRunner() - - -class PollingAsyncTestRunner(PollingAsyncActionRunner): - def __init__(self): - super(PollingAsyncTestRunner, self).__init__(runner_id="1") - self.pre_run_called = False - self.run_called = False - self.post_run_called = False - - def pre_run(self): - self.pre_run_called = True - - def run(self, action_params): - self.run_called = True - result = {} - if self.runner_parameters.get(RAISE_PROPERTY, False): - raise Exception("Raise required.") - else: - result = {"ran": True, "action_params": action_params} - - return (LIVEACTION_STATUS_RUNNING, json.dumps(result), {"id": "foo"}) - - def post_run(self, status, result): - self.post_run_called = True From 63610897013c098461091b5339df39a748e0c49f Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 24 May 2024 10:36:19 -0500 Subject: [PATCH 28/98] test: do not hardcode stanley user in tests --- st2actions/tests/unit/test_runner_container.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/st2actions/tests/unit/test_runner_container.py b/st2actions/tests/unit/test_runner_container.py index 1134e176bf..b6ea1f41aa 100644 --- a/st2actions/tests/unit/test_runner_container.py +++ b/st2actions/tests/unit/test_runner_container.py @@ -16,6 +16,7 @@ from __future__ import absolute_import import mock +import os from oslo_config import cfg @@ -297,7 +298,8 @@ def test_dispatch(self): self.assertTrue(result.get("action_params").get("actionstr") == "bar") # Assert that context is written correctly. - context = {"user": "stanley", "third_party_system": {"ref_id": "1234"}} + system_user = os.environ.get("ST2TESTS_SYSTEM_USER", "") or "stanley" + context = {"user": system_user, "third_party_system": {"ref_id": "1234"}} self.assertDictEqual(liveaction_db.context, context) From 656ae8e7604dbff2c739fd0bdada43681338aa97 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 24 May 2024 11:40:17 -0500 Subject: [PATCH 29/98] tests: reorder st2tests imports in st2actions tests for import side-effects importing anything form st2tests already handles running st2tests.config.parse_args() on import before loading the files from st2common that need those side effects. So, rely on that, and on the db test case base classes for running parse_args() where appropriate. The import side-effects are unfortunate, but this reduces how many places are making those changes. --- st2actions/tests/unit/policies/test_base.py | 7 ++----- st2actions/tests/unit/policies/test_concurrency.py | 7 ++----- .../unit/policies/test_concurrency_by_attr.py | 6 ++---- st2actions/tests/unit/test_action_runner_worker.py | 9 ++++++--- .../tests/unit/test_execution_cancellation.py | 7 ++----- st2actions/tests/unit/test_executions.py | 7 ++----- st2actions/tests/unit/test_notifier.py | 6 ++---- st2actions/tests/unit/test_output_schema.py | 5 +---- st2actions/tests/unit/test_parallel_ssh.py | 7 +++++-- .../unit/test_paramiko_remote_script_runner.py | 7 +++++-- st2actions/tests/unit/test_paramiko_ssh.py | 7 +++++-- st2actions/tests/unit/test_paramiko_ssh_runner.py | 7 +++++-- st2actions/tests/unit/test_queue_consumers.py | 8 +++----- st2actions/tests/unit/test_remote_runners.py | 14 ++++++++------ st2actions/tests/unit/test_runner_container.py | 7 +++---- st2actions/tests/unit/test_scheduler.py | 4 ---- st2actions/tests/unit/test_scheduler_entrypoint.py | 4 ---- st2actions/tests/unit/test_scheduler_retry.py | 8 +++----- st2actions/tests/unit/test_worker.py | 8 ++++---- st2actions/tests/unit/test_workflow_engine.py | 6 +----- 20 files changed, 61 insertions(+), 80 deletions(-) diff --git a/st2actions/tests/unit/policies/test_base.py b/st2actions/tests/unit/policies/test_base.py index fb475fbf66..1b345d2b7a 100644 --- a/st2actions/tests/unit/policies/test_base.py +++ b/st2actions/tests/unit/policies/test_base.py @@ -16,9 +16,8 @@ from __future__ import absolute_import import mock -from st2tests import config as test_config - -test_config.parse_args() +# This import must be early for import-time side-effects. +from st2tests.base import CleanDbTestCase, DbTestCase import st2common from st2common.bootstrap.policiesregistrar import register_policy_types @@ -28,8 +27,6 @@ from st2common.services import action as action_service from st2common.services import policies as policy_service from st2common.bootstrap import runnersregistrar as runners_registrar -from st2tests.base import DbTestCase -from st2tests.base import CleanDbTestCase from st2tests.fixtures.generic.fixture import PACK_NAME as PACK from st2tests.fixturesloader import FixturesLoader diff --git a/st2actions/tests/unit/policies/test_concurrency.py b/st2actions/tests/unit/policies/test_concurrency.py index 1be4b86da3..7612bd5396 100644 --- a/st2actions/tests/unit/policies/test_concurrency.py +++ b/st2actions/tests/unit/policies/test_concurrency.py @@ -19,10 +19,9 @@ from mock import call from six.moves import range +# This import must be early for import-time side-effects. # Importing st2actions.scheduler relies on config being parsed :/ -import st2tests.config as tests_config - -tests_config.parse_args() +from st2tests import DbTestCase, EventletTestCase, ExecutionDbTestCase import st2common from st2actions.scheduler import handler as scheduling_queue @@ -38,8 +37,6 @@ from st2common.transport.liveaction import LiveActionPublisher from st2common.transport.publishers import CUDPublisher from st2common.bootstrap import runnersregistrar as runners_registrar -from st2tests import DbTestCase, EventletTestCase -from st2tests import ExecutionDbTestCase import st2tests.config as tests_config from st2tests.fixtures.generic.fixture import PACK_NAME as PACK from st2tests.fixturesloader import FixturesLoader diff --git a/st2actions/tests/unit/policies/test_concurrency_by_attr.py b/st2actions/tests/unit/policies/test_concurrency_by_attr.py index 937a4149ef..2edcdb4af7 100644 --- a/st2actions/tests/unit/policies/test_concurrency_by_attr.py +++ b/st2actions/tests/unit/policies/test_concurrency_by_attr.py @@ -18,10 +18,9 @@ import mock from mock import call +# This import must be early for import-time side-effects. # Importing st2actions.scheduler relies on config being parsed :/ -import st2tests.config as tests_config - -tests_config.parse_args() +from st2tests import ExecutionDbTestCase, EventletTestCase import st2common from st2actions.scheduler import handler as scheduling_queue @@ -36,7 +35,6 @@ from st2common.transport.liveaction import LiveActionPublisher from st2common.transport.publishers import CUDPublisher from st2common.bootstrap import runnersregistrar as runners_registrar -from st2tests import ExecutionDbTestCase, EventletTestCase import st2tests.config as tests_config from st2tests.fixtures.generic.fixture import PACK_NAME as PACK from st2tests.fixturesloader import FixturesLoader diff --git a/st2actions/tests/unit/test_action_runner_worker.py b/st2actions/tests/unit/test_action_runner_worker.py index 96e049b179..8477281b97 100644 --- a/st2actions/tests/unit/test_action_runner_worker.py +++ b/st2actions/tests/unit/test_action_runner_worker.py @@ -20,14 +20,17 @@ from st2common.transport.consumers import ActionsQueueConsumer from st2common.models.db.liveaction import LiveActionDB -from st2tests import config as test_config - -test_config.parse_args() +from st2tests import config as tests_config __all__ = ["ActionsQueueConsumerTestCase"] class ActionsQueueConsumerTestCase(TestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + tests_config.parse_args() + def test_process_right_dispatcher_is_used(self): handler = Mock() handler.message_type = LiveActionDB diff --git a/st2actions/tests/unit/test_execution_cancellation.py b/st2actions/tests/unit/test_execution_cancellation.py index 96eacc1a89..539e930d25 100644 --- a/st2actions/tests/unit/test_execution_cancellation.py +++ b/st2actions/tests/unit/test_execution_cancellation.py @@ -20,10 +20,8 @@ from oslo_config import cfg -# XXX: actionsensor import depends on config being setup. -import st2tests.config as tests_config - -tests_config.parse_args() +# This import must be early for import-time side-effects. +from st2tests import ExecutionDbTestCase from st2common.constants import action as action_constants from st2common.models.api.action import ActionAPI @@ -36,7 +34,6 @@ from st2common.services import trace as trace_service from st2common.transport.liveaction import LiveActionPublisher from st2common.transport.publishers import CUDPublisher -from st2tests import ExecutionDbTestCase from st2tests.fixtures.generic.fixture import PACK_NAME as PACK from st2tests.fixturesloader import FixturesLoader from st2tests.mocks.execution import MockExecutionPublisher diff --git a/st2actions/tests/unit/test_executions.py b/st2actions/tests/unit/test_executions.py index 1c95a51061..dcb50c70f5 100644 --- a/st2actions/tests/unit/test_executions.py +++ b/st2actions/tests/unit/test_executions.py @@ -18,10 +18,8 @@ import mock -# XXX: actionsensor import depends on config being setup. -import st2tests.config as tests_config - -tests_config.parse_args() +# This import must be early for import-time side-effects. +from st2tests import ExecutionDbTestCase import st2common.bootstrap.runnersregistrar as runners_registrar from st2common.constants import action as action_constants @@ -46,7 +44,6 @@ from local_runner.local_shell_command_runner import LocalShellCommandRunner from st2tests.fixtures.packs import executions as fixture -from st2tests import ExecutionDbTestCase from st2tests.mocks.liveaction import MockLiveActionPublisher diff --git a/st2actions/tests/unit/test_notifier.py b/st2actions/tests/unit/test_notifier.py index b648d7fad3..f599cea08a 100644 --- a/st2actions/tests/unit/test_notifier.py +++ b/st2actions/tests/unit/test_notifier.py @@ -19,9 +19,8 @@ import bson import mock -import st2tests.config as tests_config - -tests_config.parse_args() +# This import must be early for import-time side-effects. +from st2tests.base import CleanDbTestCase from st2actions.notifier.notifier import Notifier from st2common.constants.action import LIVEACTION_COMPLETED_STATES @@ -40,7 +39,6 @@ from st2common.models.system.common import ResourceReference from st2common.util import date as date_utils from st2common.util import isotime -from st2tests.base import CleanDbTestCase ACTION_TRIGGER_TYPE = INTERNAL_TRIGGER_TYPES["action"][0] NOTIFY_TRIGGER_TYPE = INTERNAL_TRIGGER_TYPES["action"][1] diff --git a/st2actions/tests/unit/test_output_schema.py b/st2actions/tests/unit/test_output_schema.py index a66f9ffb12..d4ae6bd9cc 100644 --- a/st2actions/tests/unit/test_output_schema.py +++ b/st2actions/tests/unit/test_output_schema.py @@ -20,12 +20,9 @@ from python_runner import python_runner from orquesta_runner import orquesta_runner +# This import must be early for import-time side-effects. import st2tests -import st2tests.config as tests_config - -tests_config.parse_args() - from st2common.bootstrap import actionsregistrar from st2common.bootstrap import runnersregistrar from st2common.constants import action as ac_const diff --git a/st2actions/tests/unit/test_parallel_ssh.py b/st2actions/tests/unit/test_parallel_ssh.py index c1ef2e998a..70c9b79b68 100644 --- a/st2actions/tests/unit/test_parallel_ssh.py +++ b/st2actions/tests/unit/test_parallel_ssh.py @@ -25,8 +25,6 @@ from st2common.runners.paramiko_ssh import SSHCommandTimeoutError import st2tests.config as tests_config -tests_config.parse_args() - MOCK_STDERR_SUDO_PASSWORD_ERROR = """ [sudo] password for bar: Sorry, try again.\n [sudo] password for bar:' Sorry, try again.\n @@ -36,6 +34,11 @@ class ParallelSSHTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + tests_config.parse_args() + @patch("paramiko.SSHClient", Mock) @patch.object( ParamikoSSHClient, diff --git a/st2actions/tests/unit/test_paramiko_remote_script_runner.py b/st2actions/tests/unit/test_paramiko_remote_script_runner.py index 27495463ff..726456d11a 100644 --- a/st2actions/tests/unit/test_paramiko_remote_script_runner.py +++ b/st2actions/tests/unit/test_paramiko_remote_script_runner.py @@ -22,8 +22,6 @@ # before importing remote_script_runner classes. import st2tests.config as tests_config -tests_config.parse_args() - from st2common.util import jsonify from st2common.models.db.action import ActionDB from st2common.runners.parallel_ssh import ParallelSSHClient @@ -48,6 +46,11 @@ class ParamikoScriptRunnerTestCase(unittest.TestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + tests_config.parse_args() + @patch("st2common.runners.parallel_ssh.ParallelSSHClient", Mock) @patch.object(jsonify, "json_loads", MagicMock(return_value={})) @patch.object(ParallelSSHClient, "run", MagicMock(return_value={})) diff --git a/st2actions/tests/unit/test_paramiko_ssh.py b/st2actions/tests/unit/test_paramiko_ssh.py index 1ccdc110a2..d60c227b1d 100644 --- a/st2actions/tests/unit/test_paramiko_ssh.py +++ b/st2actions/tests/unit/test_paramiko_ssh.py @@ -29,12 +29,15 @@ from st2tests.fixturesloader import get_resources_base_path import st2tests.config as tests_config -tests_config.parse_args() - __all__ = ["ParamikoSSHClientTestCase"] class ParamikoSSHClientTestCase(unittest.TestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + tests_config.parse_args() + @patch("paramiko.SSHClient", Mock) def setUp(self): """ diff --git a/st2actions/tests/unit/test_paramiko_ssh_runner.py b/st2actions/tests/unit/test_paramiko_ssh_runner.py index 6700bb0347..116ea4eced 100644 --- a/st2actions/tests/unit/test_paramiko_ssh_runner.py +++ b/st2actions/tests/unit/test_paramiko_ssh_runner.py @@ -30,8 +30,6 @@ import st2tests.config as tests_config from st2tests.fixturesloader import get_resources_base_path -tests_config.parse_args() - class Runner(BaseParallelSSHRunner): def run(self): @@ -39,6 +37,11 @@ def run(self): class ParamikoSSHRunnerTestCase(unittest.TestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + tests_config.parse_args() + @mock.patch("st2common.runners.paramiko_ssh_runner.ParallelSSHClient") def test_pre_run(self, mock_client): # Test case which verifies that ParamikoSSHClient is instantiated with the correct arguments diff --git a/st2actions/tests/unit/test_queue_consumers.py b/st2actions/tests/unit/test_queue_consumers.py index 0ddfaea164..594ecac401 100644 --- a/st2actions/tests/unit/test_queue_consumers.py +++ b/st2actions/tests/unit/test_queue_consumers.py @@ -15,13 +15,12 @@ from __future__ import absolute_import -import st2tests.config as tests_config - -tests_config.parse_args() - import mock from kombu.message import Message +# This import must be early for import-time side-effects. +from st2tests.base import ExecutionDbTestCase + from st2actions import worker from st2actions.scheduler import entrypoint as scheduling from st2actions.scheduler import handler as scheduling_queue @@ -35,7 +34,6 @@ from st2common.transport.publishers import PoolPublisher from st2common.util import action_db as action_utils from st2common.util import date as date_utils -from st2tests.base import ExecutionDbTestCase from st2tests.fixtures.packs.core.fixture import PACK_PATH as CORE_PACK_PATH diff --git a/st2actions/tests/unit/test_remote_runners.py b/st2actions/tests/unit/test_remote_runners.py index 19f5cb40f1..06f9058fa9 100644 --- a/st2actions/tests/unit/test_remote_runners.py +++ b/st2actions/tests/unit/test_remote_runners.py @@ -13,18 +13,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -# XXX: FabricRunner import depends on config being setup. -from __future__ import absolute_import -import st2tests.config as tests_config - -tests_config.parse_args() - from unittest import TestCase +# This import must be early for import-time side-effects. +import st2tests.config as tests_config + from st2common.models.system.action import RemoteScriptAction class RemoteScriptActionTestCase(TestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + tests_config.parse_args() + def test_parameter_formatting(self): # Only named args named_args = { diff --git a/st2actions/tests/unit/test_runner_container.py b/st2actions/tests/unit/test_runner_container.py index b6ea1f41aa..e154a36534 100644 --- a/st2actions/tests/unit/test_runner_container.py +++ b/st2actions/tests/unit/test_runner_container.py @@ -20,6 +20,9 @@ from oslo_config import cfg +# This import must be early for import-time side-effects. +from st2tests.base import DbTestCase + from st2common.constants import action as action_constants from st2common.runners.base import get_runner from st2common.exceptions.actionrunner import ( @@ -36,10 +39,6 @@ from st2common.util import date as date_utils from st2common.transport.publishers import PoolPublisher -from st2tests.base import DbTestCase -import st2tests.config as tests_config - -tests_config.parse_args() from st2tests.fixtures.generic.fixture import PACK_NAME as FIXTURES_PACK from st2tests.fixturesloader import FixturesLoader diff --git a/st2actions/tests/unit/test_scheduler.py b/st2actions/tests/unit/test_scheduler.py index 7556c7b036..65a59bd869 100644 --- a/st2actions/tests/unit/test_scheduler.py +++ b/st2actions/tests/unit/test_scheduler.py @@ -19,10 +19,6 @@ import mock import eventlet -from st2tests import config as test_config - -test_config.parse_args() - import st2common from st2tests import ExecutionDbTestCase from st2tests.fixtures.generic.fixture import PACK_NAME as PACK diff --git a/st2actions/tests/unit/test_scheduler_entrypoint.py b/st2actions/tests/unit/test_scheduler_entrypoint.py index 2bc535d99d..2862ba2b3c 100644 --- a/st2actions/tests/unit/test_scheduler_entrypoint.py +++ b/st2actions/tests/unit/test_scheduler_entrypoint.py @@ -16,10 +16,6 @@ import eventlet import mock -from st2tests import config as test_config - -test_config.parse_args() - from st2actions.cmd.scheduler import _run_scheduler from st2actions.scheduler.handler import ActionExecutionSchedulingQueueHandler from st2actions.scheduler.entrypoint import SchedulerEntrypoint diff --git a/st2actions/tests/unit/test_scheduler_retry.py b/st2actions/tests/unit/test_scheduler_retry.py index ad1f221df1..d975964bc6 100644 --- a/st2actions/tests/unit/test_scheduler_retry.py +++ b/st2actions/tests/unit/test_scheduler_retry.py @@ -13,19 +13,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +# This import must be first for import-time side-effects. +from st2tests.base import CleanDbTestCase + import eventlet import mock import pymongo import uuid -from st2tests import config as test_config - -test_config.parse_args() - from st2actions.scheduler import handler from st2common.models.db import execution_queue as ex_q_db from st2common.persistence import execution_queue as ex_q_db_access -from st2tests.base import CleanDbTestCase __all__ = ["SchedulerHandlerRetryTestCase"] diff --git a/st2actions/tests/unit/test_worker.py b/st2actions/tests/unit/test_worker.py index ca2bf172dc..b335b6f2be 100644 --- a/st2actions/tests/unit/test_worker.py +++ b/st2actions/tests/unit/test_worker.py @@ -14,6 +14,7 @@ # limitations under the License. from __future__ import absolute_import + from bson.errors import InvalidStringData import eventlet import mock @@ -21,6 +22,9 @@ from oslo_config import cfg import tempfile +# This import must be early for import-time side-effects. +from st2tests.base import DbTestCase + import st2actions.worker as actions_worker from st2common.constants import action as action_constants from st2common.models.db.liveaction import LiveActionDB @@ -33,14 +37,10 @@ from st2common.bootstrap import runnersregistrar as runners_registrar from local_runner.local_shell_command_runner import LocalShellCommandRunner -from st2tests.base import DbTestCase from st2tests.fixtures.generic.fixture import PACK_NAME as FIXTURES_PACK from st2tests.fixturesloader import FixturesLoader -import st2tests.config as tests_config from six.moves import range -tests_config.parse_args() - TEST_FIXTURES = {"actions": ["local.yaml"]} NON_UTF8_RESULT = { diff --git a/st2actions/tests/unit/test_workflow_engine.py b/st2actions/tests/unit/test_workflow_engine.py index 955f7ca2f0..68d68a3b2f 100644 --- a/st2actions/tests/unit/test_workflow_engine.py +++ b/st2actions/tests/unit/test_workflow_engine.py @@ -18,17 +18,13 @@ import eventlet import mock +# This import must be early for import-time side-effects. import st2tests from orquesta import statuses as wf_statuses from oslo_config import cfg from tooz import coordination -# XXX: actionsensor import depends on config being setup. -import st2tests.config as tests_config - -tests_config.parse_args() - from st2actions.workflows import workflows from st2common.bootstrap import actionsregistrar from st2common.bootstrap import runnersregistrar From c6346c212bf51229ec6336d836c3197242c7ad50 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 24 May 2024 11:40:49 -0500 Subject: [PATCH 30/98] pants: record st2actions/tests/unit deps on runners --- st2actions/tests/unit/BUILD | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/st2actions/tests/unit/BUILD b/st2actions/tests/unit/BUILD index 9a24dba70a..7bb8fa73e5 100644 --- a/st2actions/tests/unit/BUILD +++ b/st2actions/tests/unit/BUILD @@ -5,4 +5,23 @@ __defaults__( python_tests( name="tests", + uses=["mongo"], + overrides={ + ( + "test_execution*.py", + "test_notifier.py", + "test_output_schema.py", + "test_policies.py", + "test_queue_consumers.py", + "test_runner_container.py", + "test_scheduler*.py", + "test_worker.py", + "test_workflow_engine.py", + ): dict( + stevedore_namespaces=[ + "st2common.runners.runner", + "st2common.metrics.driver", + ], + ), + }, ) From ddca590c321d9ce904d15b50c3f9352958fb0b8d Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 24 May 2024 11:55:33 -0500 Subject: [PATCH 31/98] tests: reorder st2tests imports in st2common tests for import side-effects importing anything form st2tests already handles running st2tests.config.parse_args() on import before loading the files from st2common that need those side effects. So, rely on that, and on the db test case base classes for running parse_args() where appropriate. The import side-effects are unfortunate, but this reduces how many places are making those changes. --- st2common/tests/unit/services/test_policy.py | 6 ++---- st2common/tests/unit/test_executions_util.py | 6 +++--- st2common/tests/unit/test_jsonify.py | 3 +-- st2common/tests/unit/test_runners_utils.py | 11 ++--------- 4 files changed, 8 insertions(+), 18 deletions(-) diff --git a/st2common/tests/unit/services/test_policy.py b/st2common/tests/unit/services/test_policy.py index a590322452..5e891e9c65 100644 --- a/st2common/tests/unit/services/test_policy.py +++ b/st2common/tests/unit/services/test_policy.py @@ -15,9 +15,8 @@ from __future__ import absolute_import -import st2tests.config as tests_config - -tests_config.parse_args() +# This import must be early for import-time side-effects. +import st2tests import st2common @@ -29,7 +28,6 @@ from st2common.services import action as action_service from st2common.services import policies as policy_service -import st2tests from st2tests.fixtures.generic.fixture import PACK_NAME as PACK from st2tests import fixturesloader as fixtures diff --git a/st2common/tests/unit/test_executions_util.py b/st2common/tests/unit/test_executions_util.py index 00b89c7433..0776ef57cc 100644 --- a/st2common/tests/unit/test_executions_util.py +++ b/st2common/tests/unit/test_executions_util.py @@ -17,6 +17,9 @@ import mock import six +# This import must be early for import-time side-effects. +from st2tests.base import CleanDbTestCase + from st2common.constants import action as action_constants from st2common.models.api.action import RunnerTypeAPI, ActionAPI, LiveActionAPI from st2common.models.api.trigger import TriggerTypeAPI, TriggerAPI, TriggerInstanceAPI @@ -30,15 +33,12 @@ import st2common.util.action_db as action_utils import st2common.util.date as date_utils -from st2tests.base import CleanDbTestCase from st2tests.fixtures.generic.fixture import PACK_NAME as FIXTURES_PACK from st2tests.fixtures.descendants.fixture import PACK_NAME as DESCENDANTS_PACK from st2tests.fixturesloader import FixturesLoader -import st2tests.config as tests_config from six.moves import range -tests_config.parse_args() TEST_FIXTURES = { "liveactions": [ diff --git a/st2common/tests/unit/test_jsonify.py b/st2common/tests/unit/test_jsonify.py index b4a375be69..906a548d4c 100644 --- a/st2common/tests/unit/test_jsonify.py +++ b/st2common/tests/unit/test_jsonify.py @@ -22,14 +22,13 @@ import st2tests.config as tests_config -tests_config.parse_args() - import st2common.util.jsonify as jsonify class JsonifyTests(unittest.TestCase): @classmethod def setUpClass(cls): + tests_config.parse_args() jsonify.DEFAULT_JSON_LIBRARY = "orjson" @classmethod diff --git a/st2common/tests/unit/test_runners_utils.py b/st2common/tests/unit/test_runners_utils.py index 773fa9cc39..a7ed4c40d3 100644 --- a/st2common/tests/unit/test_runners_utils.py +++ b/st2common/tests/unit/test_runners_utils.py @@ -15,10 +15,8 @@ from __future__ import absolute_import -# pytest: make sure monkey_patching happens before importing mongoengine -from st2common.util.monkey_patch import monkey_patch - -monkey_patch() +# This import must be early for import-time side-effects. +from st2tests import base import mock @@ -30,11 +28,6 @@ from st2tests.fixtures.generic.fixture import PACK_NAME as FIXTURES_PACK -from st2tests import config as tests_config - -tests_config.parse_args() - - TEST_FIXTURES = { "liveactions": ["liveaction1.yaml"], "actions": ["local.yaml"], From c25eba5a1c97a3b8748f844a6bb629d1c7359e11 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 24 May 2024 12:29:57 -0500 Subject: [PATCH 32/98] pants: record logging conf (dev) dep on logs/ directory --- BUILD | 5 +++++ st2actions/conf/BUILD | 2 ++ st2api/conf/BUILD | 2 ++ st2auth/conf/BUILD | 2 ++ st2reactor/conf/BUILD | 1 + st2stream/conf/BUILD | 2 ++ 6 files changed, 14 insertions(+) diff --git a/BUILD b/BUILD index f8b1f03a4d..b2d06d15f9 100644 --- a/BUILD +++ b/BUILD @@ -51,3 +51,8 @@ file( shell_sources( name="root", ) + +file( + name="logs_directory", + source="logs/.gitignore", +) diff --git a/st2actions/conf/BUILD b/st2actions/conf/BUILD index ee9257725e..9d1bf2cc2b 100644 --- a/st2actions/conf/BUILD +++ b/st2actions/conf/BUILD @@ -6,9 +6,11 @@ st2_logging_conf_file( st2_logging_conf_files( name="logging", sources=["logging*.conf"], + dependencies=["//:logs_directory"], overrides={ "logging.conf": dict( dependencies=[ + "//:logs_directory", "//:reqs#python-json-logger", ], ), diff --git a/st2api/conf/BUILD b/st2api/conf/BUILD index df77688843..5fc903fd3b 100644 --- a/st2api/conf/BUILD +++ b/st2api/conf/BUILD @@ -6,11 +6,13 @@ st2_logging_conf_file( st2_logging_conf_file( name="logging", source="logging.conf", + dependencies=["//:logs_directory"], ) st2_logging_conf_file( name="logging_gunicorn", source="logging.gunicorn.conf", + dependencies=["//:logs_directory"], ) st2_logging_conf_file( diff --git a/st2auth/conf/BUILD b/st2auth/conf/BUILD index c1adc35ceb..3300d41753 100644 --- a/st2auth/conf/BUILD +++ b/st2auth/conf/BUILD @@ -16,11 +16,13 @@ st2_logging_conf_file( st2_logging_conf_file( name="logging", source="logging.conf", + dependencies=["//:logs_directory"], ) st2_logging_conf_file( name="logging_gunicorn", source="logging.gunicorn.conf", + dependencies=["//:logs_directory"], ) st2_logging_conf_file( diff --git a/st2reactor/conf/BUILD b/st2reactor/conf/BUILD index 8418a76e23..4f07917387 100644 --- a/st2reactor/conf/BUILD +++ b/st2reactor/conf/BUILD @@ -6,6 +6,7 @@ st2_logging_conf_file( st2_logging_conf_files( name="logging", sources=["logging*.conf"], + dependencies=["//:logs_directory"], ) st2_logging_conf_files( diff --git a/st2stream/conf/BUILD b/st2stream/conf/BUILD index df77688843..5fc903fd3b 100644 --- a/st2stream/conf/BUILD +++ b/st2stream/conf/BUILD @@ -6,11 +6,13 @@ st2_logging_conf_file( st2_logging_conf_file( name="logging", source="logging.conf", + dependencies=["//:logs_directory"], ) st2_logging_conf_file( name="logging_gunicorn", source="logging.gunicorn.conf", + dependencies=["//:logs_directory"], ) st2_logging_conf_file( From fc65de20ef97fb6cc0b990b8e061ebadb46586df Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 24 May 2024 12:31:11 -0500 Subject: [PATCH 33/98] tests: move st2reactor test side-effects into setUpClass --- st2reactor/tests/unit/test_garbage_collector.py | 8 ++++++-- st2reactor/tests/unit/test_process_container.py | 7 +++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/st2reactor/tests/unit/test_garbage_collector.py b/st2reactor/tests/unit/test_garbage_collector.py index 93de6b25d0..f9ca515d0a 100644 --- a/st2reactor/tests/unit/test_garbage_collector.py +++ b/st2reactor/tests/unit/test_garbage_collector.py @@ -20,14 +20,18 @@ from oslo_config import cfg +# This import must be early for import-time side-effects. import st2tests.config as tests_config -tests_config.parse_args() - from st2reactor.garbage_collector import base as garbage_collector class GarbageCollectorServiceTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + tests_config.parse_args() + def tearDown(self): # Reset gc_max_idle_sec with a value of 1 to reenable for other tests. cfg.CONF.set_override("gc_max_idle_sec", 1, group="workflow_engine") diff --git a/st2reactor/tests/unit/test_process_container.py b/st2reactor/tests/unit/test_process_container.py index b05175b805..747618a4f0 100644 --- a/st2reactor/tests/unit/test_process_container.py +++ b/st2reactor/tests/unit/test_process_container.py @@ -27,8 +27,6 @@ import st2tests.config as tests_config -tests_config.parse_args() - MOCK_PACK_DB = PackDB( ref="wolfpack", name="wolf pack", @@ -38,6 +36,11 @@ class ProcessContainerTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + tests_config.parse_args() + def test_no_sensors_dont_quit(self): process_container = ProcessSensorContainer(None, poll_interval=0.1) process_container_thread = concurrency.spawn(process_container.run) From 2b54d842504c507b6511675b1138099b1582aa95 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 24 May 2024 12:48:04 -0500 Subject: [PATCH 34/98] use fixture import for st2reactor/tests/resources --- st2reactor/tests/resources/BUILD | 11 +++++++++-- st2reactor/tests/resources/fixture.py | 16 ++++++++++++++++ st2reactor/tests/unit/test_sensor_wrapper.py | 3 +-- 3 files changed, 26 insertions(+), 4 deletions(-) create mode 100644 st2reactor/tests/resources/fixture.py diff --git a/st2reactor/tests/resources/BUILD b/st2reactor/tests/resources/BUILD index 57341b1358..4cb4ff555e 100644 --- a/st2reactor/tests/resources/BUILD +++ b/st2reactor/tests/resources/BUILD @@ -1,3 +1,10 @@ -python_tests( - name="tests", +python_sources( + name="fixture_sensors", + # Override the default sources to include test_sensor*.py + # which are fixtures, not actual test files. + sources=["test_sensor*.py"], +) + +python_sources( + dependencies=[":fixture_sensors"], ) diff --git a/st2reactor/tests/resources/fixture.py b/st2reactor/tests/resources/fixture.py new file mode 100644 index 0000000000..f6d0d3e940 --- /dev/null +++ b/st2reactor/tests/resources/fixture.py @@ -0,0 +1,16 @@ +# Copyright 2024 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from st2tests import fixturesloader + +FIXTURE_NAME, FIXTURE_PATH = fixturesloader.get_fixture_name_and_path(__file__) diff --git a/st2reactor/tests/unit/test_sensor_wrapper.py b/st2reactor/tests/unit/test_sensor_wrapper.py index 3a9d3b9ced..36b8c04f10 100644 --- a/st2reactor/tests/unit/test_sensor_wrapper.py +++ b/st2reactor/tests/unit/test_sensor_wrapper.py @@ -33,8 +33,7 @@ from st2reactor.container.sensor_wrapper import SensorWrapper from st2reactor.sensor.base import Sensor, PollingSensor -CURRENT_DIR = os.path.abspath(os.path.dirname(__file__)) -RESOURCES_DIR = os.path.abspath(os.path.join(CURRENT_DIR, "../resources")) +from tests.resources.fixture import FIXTURE_PATH as RESOURCES_DIR __all__ = ["SensorWrapperTestCase"] From b2db72db790b980db9ff39aec856f0ff008c1b9a Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 24 May 2024 12:48:44 -0500 Subject: [PATCH 35/98] pants: add deps on stevedore_namespaces for st2reactor tests --- st2reactor/tests/unit/BUILD | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/st2reactor/tests/unit/BUILD b/st2reactor/tests/unit/BUILD index 9a24dba70a..abd3115705 100644 --- a/st2reactor/tests/unit/BUILD +++ b/st2reactor/tests/unit/BUILD @@ -5,4 +5,20 @@ __defaults__( python_tests( name="tests", + uses=["mongo"], + overrides={ + "test_enforce.py": dict( + stevedore_namespaces=[ + "st2common.rbac.backend", + "st2common.runners.runner", + "st2common.metrics.driver", + ], + ), + "test_rule_engine.py": dict( + stevedore_namespaces=[ + "st2common.runners.runner", + "st2common.metrics.driver", + ], + ), + }, ) From f4654552188ec33f06c7328afccf46212926f20b Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 24 May 2024 12:49:51 -0500 Subject: [PATCH 36/98] pants: run unit tests for st2tests/ st2actions/ st2reactor/ Actually there are no unit tests in st2tests/, so this just ensures no one will add one without pants running it. --- .github/workflows/test.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index f0be482057..705a9a95f3 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -112,8 +112,11 @@ jobs: --tag=unit test pylint_plugins/:: + st2tests/:: st2client/:: st2common/:: + st2actions/:: + st2reactor/:: - name: Upload pants log uses: actions/upload-artifact@v4 From d26da96dae327d4d3061b0c11c5a1d769622415c Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 24 May 2024 13:00:13 -0500 Subject: [PATCH 37/98] pants: add deps on stevedore_namespaces for st2stream tests --- st2stream/tests/unit/controllers/v1/BUILD | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/st2stream/tests/unit/controllers/v1/BUILD b/st2stream/tests/unit/controllers/v1/BUILD index a1ca2f641e..982e61ec88 100644 --- a/st2stream/tests/unit/controllers/v1/BUILD +++ b/st2stream/tests/unit/controllers/v1/BUILD @@ -1,5 +1,11 @@ python_tests( name="tests", + stevedore_namespaces=[ + "st2common.rbac.backend", + "st2common.runners.runner", + "st2common.metrics.driver", + ], + uses=["mongo"], ) python_test_utils( From ae3c06d5600b9307445753a577462bfc705a4baa Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 24 May 2024 13:13:40 -0500 Subject: [PATCH 38/98] pants: add deps so pants can run st2auth unit tests --- st2auth/tests/unit/BUILD | 2 ++ st2auth/tests/unit/controllers/v1/BUILD | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/st2auth/tests/unit/BUILD b/st2auth/tests/unit/BUILD index 9a24dba70a..c128c7b4af 100644 --- a/st2auth/tests/unit/BUILD +++ b/st2auth/tests/unit/BUILD @@ -5,4 +5,6 @@ __defaults__( python_tests( name="tests", + dependencies=["//:auth_backends"], + uses=["mongo"], ) diff --git a/st2auth/tests/unit/controllers/v1/BUILD b/st2auth/tests/unit/controllers/v1/BUILD index 57341b1358..66f5d44271 100644 --- a/st2auth/tests/unit/controllers/v1/BUILD +++ b/st2auth/tests/unit/controllers/v1/BUILD @@ -1,3 +1,8 @@ python_tests( name="tests", + stevedore_namespaces=[ + "st2auth.sso.backends", + "st2common.metrics.driver", + ], + uses=["mongo"], ) From 1bb600fe0d9e7642bc36b56a81f5227fc80dbfa8 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 24 May 2024 13:50:11 -0500 Subject: [PATCH 39/98] tests: respect ST2TESTS_SYSTEM_USER in st2api tests --- .../controllers/v1/test_alias_execution.py | 8 ++++--- st2api/tests/unit/controllers/v1/test_auth.py | 3 ++- .../unit/controllers/v1/test_executions.py | 22 ++++++++++--------- 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/st2api/tests/unit/controllers/v1/test_alias_execution.py b/st2api/tests/unit/controllers/v1/test_alias_execution.py index 44261fde3f..c64b121023 100644 --- a/st2api/tests/unit/controllers/v1/test_alias_execution.py +++ b/st2api/tests/unit/controllers/v1/test_alias_execution.py @@ -14,8 +14,8 @@ # limitations under the License. import copy - import mock +import os from st2common.constants.action import LIVEACTION_STATUS_SUCCEEDED from st2common.models.db.execution import ActionExecutionDB @@ -50,6 +50,8 @@ __all__ = ["AliasExecutionTestCase"] +SYSTEM_USER = os.environ.get("ST2TESTS_SYSTEM_USER", "") or "stanley" + class AliasExecutionTestCase(FunctionalTest): @@ -241,7 +243,7 @@ def test_match_and_execute_matches_one(self, mock_request): self.assertIn("source_channel", mock_request.call_args[0][0].context.keys()) self.assertEqual(actual_context["source_channel"], "chat-channel") self.assertEqual(actual_context["api_user"], "chat-user") - self.assertEqual(actual_context["user"], "stanley") + self.assertEqual(actual_context["user"], SYSTEM_USER) @mock.patch.object(action_service, "request", return_value=(None, EXECUTION)) def test_match_and_execute_matches_one_multiple_match(self, mock_request): @@ -398,7 +400,7 @@ def _do_post( "name": alias_execution.name, "format": format_str, "command": command, - "user": "stanley", + "user": SYSTEM_USER, "source_channel": "test", "notification_route": "test", } diff --git a/st2api/tests/unit/controllers/v1/test_auth.py b/st2api/tests/unit/controllers/v1/test_auth.py index 695d490faa..1086db34e1 100644 --- a/st2api/tests/unit/controllers/v1/test_auth.py +++ b/st2api/tests/unit/controllers/v1/test_auth.py @@ -15,6 +15,7 @@ import uuid import datetime +import os import bson import mock @@ -29,7 +30,7 @@ from st2tests.fixturesloader import FixturesLoader OBJ_ID = bson.ObjectId() -USER = "stanley" +USER = os.environ.get("ST2TESTS_SYSTEM_USER", "") or "stanley" USER_DB = UserDB(name=USER) TOKEN = uuid.uuid4().hex NOW = date_utils.get_datetime_utc_now() diff --git a/st2api/tests/unit/controllers/v1/test_executions.py b/st2api/tests/unit/controllers/v1/test_executions.py index eb3face2ba..0e941568a6 100644 --- a/st2api/tests/unit/controllers/v1/test_executions.py +++ b/st2api/tests/unit/controllers/v1/test_executions.py @@ -15,6 +15,7 @@ import copy import mock +import os try: import simplejson as json @@ -60,6 +61,7 @@ "ActionExecutionOutputControllerTestCase", ] +SYSTEM_USER = os.environ.get("ST2TESTS_SYSTEM_USER", "") or "stanley" ACTION_1 = { "name": "st2.dummy.action1", @@ -686,7 +688,7 @@ def test_post_delete(self): delete_resp = self._do_delete(self._get_actionexecution_id(post_resp)) self.assertEqual(delete_resp.status_int, 200) self.assertEqual(delete_resp.json["status"], "canceled") - expected_result = {"message": "Action canceled by user.", "user": "stanley"} + expected_result = {"message": "Action canceled by user.", "user": SYSTEM_USER} self.assertDictEqual(delete_resp.json["result"], expected_result) def test_post_delete_duplicate(self): @@ -702,7 +704,7 @@ def test_post_delete_duplicate(self): delete_resp = self._do_delete(self._get_actionexecution_id(post_resp)) self.assertEqual(delete_resp.status_int, 200) self.assertEqual(delete_resp.json["status"], "canceled") - expected_result = {"message": "Action canceled by user.", "user": "stanley"} + expected_result = {"message": "Action canceled by user.", "user": SYSTEM_USER} self.assertDictEqual(delete_resp.json["result"], expected_result) def test_post_delete_trace(self): @@ -976,7 +978,7 @@ def test_template_encrypted_params(self): ), }, { - "name": "stanley:secret", + "name": f"{SYSTEM_USER}:secret", "secret": True, "scope": FULL_USER_SCOPE, "value": crypto_utils.symmetric_encrypt( @@ -994,18 +996,18 @@ def test_template_encrypted_params(self): ] kvps = [KeyValuePair.add_or_update(KeyValuePairDB(**x)) for x in register_items] - # By default, encrypt_user_param will be read from stanley's scope + # By default, encrypt_user_param will be read from system_user's scope # 1. parameters are not marked as secret resp = self._do_post(LIVE_ACTION_DEFAULT_ENCRYPT) self.assertEqual(resp.status_int, 201) - self.assertEqual(resp.json["context"]["user"], "stanley") + self.assertEqual(resp.json["context"]["user"], SYSTEM_USER) self.assertEqual(resp.json["parameters"]["encrypted_param"], "foo") self.assertEqual(resp.json["parameters"]["encrypted_user_param"], "bar") # 2. parameters are marked as secret resp = self._do_post(LIVE_ACTION_DEFAULT_ENCRYPT_SECRET_PARAM) self.assertEqual(resp.status_int, 201) - self.assertEqual(resp.json["context"]["user"], "stanley") + self.assertEqual(resp.json["context"]["user"], SYSTEM_USER) self.assertEqual( resp.json["parameters"]["encrypted_param"], MASKED_ATTRIBUTE_VALUE ) @@ -1077,7 +1079,7 @@ def test_re_run_workflow_success(self): ) expected_context = { - "user": "stanley", + "user": SYSTEM_USER, "pack": "starterpack", "re-run": {"ref": execution_id}, "trace_context": {"id_": str(trace.id)}, @@ -1106,7 +1108,7 @@ def test_re_run_workflow_task_success(self): expected_context = { "pack": "starterpack", - "user": "stanley", + "user": SYSTEM_USER, "re-run": {"ref": execution_id, "tasks": data["tasks"]}, "trace_context": {"id_": str(trace.id)}, } @@ -1134,7 +1136,7 @@ def test_re_run_workflow_tasks_success(self): expected_context = { "pack": "starterpack", - "user": "stanley", + "user": SYSTEM_USER, "re-run": {"ref": execution_id, "tasks": data["tasks"]}, "trace_context": {"id_": str(trace.id)}, } @@ -1162,7 +1164,7 @@ def test_re_run_workflow_tasks_reset_success(self): expected_context = { "pack": "starterpack", - "user": "stanley", + "user": SYSTEM_USER, "re-run": { "ref": execution_id, "tasks": data["tasks"], From 6b8c84d4c069813c352e6160eb06d1cf4edbfb35 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 24 May 2024 14:02:52 -0500 Subject: [PATCH 40/98] test: use imports to signal fixture deps to pants --- .../tests/unit/controllers/v1/test_packs.py | 5 ++- .../unit/controllers/v1/test_packs_views.py | 37 ++++++++++++------- .../unit/controllers/v1/test_sensortypes.py | 34 +++++++++-------- 3 files changed, 46 insertions(+), 30 deletions(-) diff --git a/st2api/tests/unit/controllers/v1/test_packs.py b/st2api/tests/unit/controllers/v1/test_packs.py index c665b7d062..a238ed0f66 100644 --- a/st2api/tests/unit/controllers/v1/test_packs.py +++ b/st2api/tests/unit/controllers/v1/test_packs.py @@ -34,6 +34,9 @@ PACK_NAME as DUMMY_PACK_1, PACK_PATH as DUMMY_PACK_1_PATH, ) +from st2tests.fixtures.packs.dummy_pack_2.fixture import ( + PACK_NAME as DUMMY_PACK_2, +) from st2tests.fixtures.packs.dummy_pack_10.fixture import ( PACK_DIR_NAME as DUMMY_PACK_10, PACK_PATH as DUMMY_PACK_10_PATH, @@ -589,7 +592,7 @@ def test_packs_register_endpoint(self, mock_get_packs): resp = self.app.post_json( "/v1/packs/register", { - "packs": ["dummy_pack_2"], + "packs": [DUMMY_PACK_2], "fail_on_failure": False, "types": ["policies"], }, diff --git a/st2api/tests/unit/controllers/v1/test_packs_views.py b/st2api/tests/unit/controllers/v1/test_packs_views.py index e2e3f93783..97381e5e34 100644 --- a/st2api/tests/unit/controllers/v1/test_packs_views.py +++ b/st2api/tests/unit/controllers/v1/test_packs_views.py @@ -20,6 +20,13 @@ from st2common.persistence.pack import Pack from st2tests.api import FunctionalTest +from st2tests.fixtures.packs.dummy_pack_1.fixture import ( + PACK_NAME as DUMMY_PACK_1, +) +from st2tests.fixtures.packs.dummy_pack_16.fixture import ( + PACK_NAME as DUMMY_PACK_16, +) + @mock.patch("st2common.bootstrap.base.REGISTERED_PACKS_CACHE", {}) class PacksViewsControllerTestCase(FunctionalTest): @@ -31,7 +38,7 @@ def setUpClass(cls): actions_registrar.register_actions(use_pack_cache=False) def test_get_pack_files_success(self): - resp = self.app.get("/v1/packs/views/files/dummy_pack_1") + resp = self.app.get(f"/v1/packs/views/files/{DUMMY_PACK_1}") self.assertEqual(resp.status_int, http_client.OK) self.assertTrue(len(resp.json) > 1) item = [_item for _item in resp.json if _item["file_path"] == "pack.yaml"][0] @@ -53,12 +60,12 @@ def test_get_pack_files_binary_files_are_excluded(self): "etc/generate_new_token.png", ] - pack_db = Pack.get_by_ref("dummy_pack_1") + pack_db = Pack.get_by_ref(DUMMY_PACK_1) all_files_count = len(pack_db.files) non_binary_files_count = all_files_count - len(binary_files) - resp = self.app.get("/v1/packs/views/files/dummy_pack_1") + resp = self.app.get(f"/v1/packs/views/files/{DUMMY_PACK_1}") self.assertEqual(resp.status_int, http_client.OK) self.assertEqual(len(resp.json), non_binary_files_count) @@ -71,7 +78,7 @@ def test_get_pack_files_binary_files_are_excluded(self): self.assertFalse(item) def test_get_pack_file_success(self): - resp = self.app.get("/v1/packs/views/file/dummy_pack_1/pack.yaml") + resp = self.app.get(f"/v1/packs/views/file/{DUMMY_PACK_1}/pack.yaml") self.assertEqual(resp.status_int, http_client.OK) self.assertIn(b"name : dummy_pack_1", resp.body) @@ -84,46 +91,46 @@ def test_get_pack_file_pack_doesnt_exist(self): @mock.patch("st2api.controllers.v1.pack_views.MAX_FILE_SIZE", 1) def test_pack_file_file_larger_then_maximum_size(self): resp = self.app.get( - "/v1/packs/views/file/dummy_pack_1/pack.yaml", expect_errors=True + f"/v1/packs/views/file/{DUMMY_PACK_1}/pack.yaml", expect_errors=True ) self.assertEqual(resp.status_int, http_client.BAD_REQUEST) self.assertIn("File pack.yaml exceeds maximum allowed file size", resp) def test_headers_get_pack_file(self): - resp = self.app.get("/v1/packs/views/file/dummy_pack_1/pack.yaml") + resp = self.app.get(f"/v1/packs/views/file/{DUMMY_PACK_1}/pack.yaml") self.assertEqual(resp.status_int, http_client.OK) self.assertIn(b"name : dummy_pack_1", resp.body) self.assertIsNotNone(resp.headers["ETag"]) self.assertIsNotNone(resp.headers["Last-Modified"]) def test_no_change_get_pack_file(self): - resp = self.app.get("/v1/packs/views/file/dummy_pack_1/pack.yaml") + resp = self.app.get(f"/v1/packs/views/file/{DUMMY_PACK_1}/pack.yaml") self.assertEqual(resp.status_int, http_client.OK) self.assertIn(b"name : dummy_pack_1", resp.body) # Confirm NOT_MODIFIED resp = self.app.get( - "/v1/packs/views/file/dummy_pack_1/pack.yaml", + f"/v1/packs/views/file/{DUMMY_PACK_1}/pack.yaml", headers={"If-None-Match": resp.headers["ETag"]}, ) self.assertEqual(resp.status_code, http_client.NOT_MODIFIED) resp = self.app.get( - "/v1/packs/views/file/dummy_pack_1/pack.yaml", + f"/v1/packs/views/file/{DUMMY_PACK_1}/pack.yaml", headers={"If-Modified-Since": resp.headers["Last-Modified"]}, ) self.assertEqual(resp.status_code, http_client.NOT_MODIFIED) # Confirm value is returned if header do not match resp = self.app.get( - "/v1/packs/views/file/dummy_pack_1/pack.yaml", + f"/v1/packs/views/file/{DUMMY_PACK_1}/pack.yaml", headers={"If-None-Match": "ETAG"}, ) self.assertEqual(resp.status_code, http_client.OK) self.assertIn(b"name : dummy_pack_1", resp.body) resp = self.app.get( - "/v1/packs/views/file/dummy_pack_1/pack.yaml", + f"/v1/packs/views/file/{DUMMY_PACK_1}/pack.yaml", headers={"If-Modified-Since": "Last-Modified"}, ) self.assertEqual(resp.status_code, http_client.OK) @@ -131,11 +138,13 @@ def test_no_change_get_pack_file(self): def test_get_pack_files_and_pack_file_ref_doesnt_equal_pack_name(self): # Ref is not equal to the name, controller should still work - resp = self.app.get("/v1/packs/views/files/dummy_pack_16") + resp = self.app.get(f"/v1/packs/views/files/{DUMMY_PACK_16}") self.assertEqual(resp.status_int, http_client.OK) - self.assertEqual(len(resp.json), 4) + # 4 if running in workspace (BUILD file present) + # 3 if pants is running it (no BUILD file present) + self.assertIn(len(resp.json), [4, 3]) self.assertIn("pack.yaml", [f["file_path"] for f in resp.json]) - resp = self.app.get("/v1/packs/views/file/dummy_pack_16/pack.yaml") + resp = self.app.get(f"/v1/packs/views/file/{DUMMY_PACK_16}/pack.yaml") self.assertEqual(resp.status_int, http_client.OK) self.assertIn(b"ref: dummy_pack_16", resp.body) diff --git a/st2api/tests/unit/controllers/v1/test_sensortypes.py b/st2api/tests/unit/controllers/v1/test_sensortypes.py index c59a1c28e2..ac22a778cb 100644 --- a/st2api/tests/unit/controllers/v1/test_sensortypes.py +++ b/st2api/tests/unit/controllers/v1/test_sensortypes.py @@ -23,6 +23,10 @@ from st2tests.api import FunctionalTest from st2tests.api import APIControllerWithIncludeAndExcludeFilterTestCase +from st2tests.fixtures.packs.dummy_pack_1.fixture import ( + PACK_NAME as DUMMY_PACK_1, +) + http_client = six.moves.http_client __all__ = ["SensorTypeControllerTestCase"] @@ -75,7 +79,7 @@ def test_get_all_filters(self): resp = self.app.get("/v1/sensortypes?name=SampleSensor2") self.assertEqual(len(resp.json), 1) self.assertEqual(resp.json[0]["name"], "SampleSensor2") - self.assertEqual(resp.json[0]["ref"], "dummy_pack_1.SampleSensor2") + self.assertEqual(resp.json[0]["ref"], f"{DUMMY_PACK_1}.SampleSensor2") resp = self.app.get("/v1/sensortypes?name=SampleSensor3") self.assertEqual(len(resp.json), 1) @@ -85,7 +89,7 @@ def test_get_all_filters(self): resp = self.app.get("/v1/sensortypes?pack=foobar") self.assertEqual(len(resp.json), 0) - resp = self.app.get("/v1/sensortypes?pack=dummy_pack_1") + resp = self.app.get(f"/v1/sensortypes?pack={DUMMY_PACK_1}") self.assertEqual(len(resp.json), 3) # ?enabled filter @@ -99,20 +103,20 @@ def test_get_all_filters(self): self.assertEqual(resp.json[1]["enabled"], True) # ?trigger filter - resp = self.app.get("/v1/sensortypes?trigger=dummy_pack_1.event3") + resp = self.app.get(f"/v1/sensortypes?trigger={DUMMY_PACK_1}.event3") self.assertEqual(len(resp.json), 1) - self.assertEqual(resp.json[0]["trigger_types"], ["dummy_pack_1.event3"]) + self.assertEqual(resp.json[0]["trigger_types"], [f"{DUMMY_PACK_1}.event3"]) - resp = self.app.get("/v1/sensortypes?trigger=dummy_pack_1.event") + resp = self.app.get(f"/v1/sensortypes?trigger={DUMMY_PACK_1}.event") self.assertEqual(len(resp.json), 2) - self.assertEqual(resp.json[0]["trigger_types"], ["dummy_pack_1.event"]) - self.assertEqual(resp.json[1]["trigger_types"], ["dummy_pack_1.event"]) + self.assertEqual(resp.json[0]["trigger_types"], [f"{DUMMY_PACK_1}.event"]) + self.assertEqual(resp.json[1]["trigger_types"], [f"{DUMMY_PACK_1}.event"]) def test_get_one_success(self): - resp = self.app.get("/v1/sensortypes/dummy_pack_1.SampleSensor") + resp = self.app.get(f"/v1/sensortypes/{DUMMY_PACK_1}.SampleSensor") self.assertEqual(resp.status_int, http_client.OK) self.assertEqual(resp.json["name"], "SampleSensor") - self.assertEqual(resp.json["ref"], "dummy_pack_1.SampleSensor") + self.assertEqual(resp.json["ref"], f"{DUMMY_PACK_1}.SampleSensor") def test_get_one_doesnt_exist(self): resp = self.app.get("/v1/sensortypes/1", expect_errors=True) @@ -120,7 +124,7 @@ def test_get_one_doesnt_exist(self): def test_disable_and_enable_sensor(self): # Verify initial state - resp = self.app.get("/v1/sensortypes/dummy_pack_1.SampleSensor") + resp = self.app.get(f"/v1/sensortypes/{DUMMY_PACK_1}.SampleSensor") self.assertEqual(resp.status_int, http_client.OK) self.assertTrue(resp.json["enabled"]) @@ -129,24 +133,24 @@ def test_disable_and_enable_sensor(self): # Disable sensor data = copy.deepcopy(sensor_data) data["enabled"] = False - put_resp = self.app.put_json("/v1/sensortypes/dummy_pack_1.SampleSensor", data) + put_resp = self.app.put_json(f"/v1/sensortypes/{DUMMY_PACK_1}.SampleSensor", data) self.assertEqual(put_resp.status_int, http_client.OK) - self.assertEqual(put_resp.json["ref"], "dummy_pack_1.SampleSensor") + self.assertEqual(put_resp.json["ref"], f"{DUMMY_PACK_1}.SampleSensor") self.assertFalse(put_resp.json["enabled"]) # Verify sensor has been disabled - resp = self.app.get("/v1/sensortypes/dummy_pack_1.SampleSensor") + resp = self.app.get(f"/v1/sensortypes/{DUMMY_PACK_1}.SampleSensor") self.assertEqual(resp.status_int, http_client.OK) self.assertFalse(resp.json["enabled"]) # Enable sensor data = copy.deepcopy(sensor_data) data["enabled"] = True - put_resp = self.app.put_json("/v1/sensortypes/dummy_pack_1.SampleSensor", data) + put_resp = self.app.put_json(f"/v1/sensortypes/{DUMMY_PACK_1}.SampleSensor", data) self.assertEqual(put_resp.status_int, http_client.OK) self.assertTrue(put_resp.json["enabled"]) # Verify sensor has been enabled - resp = self.app.get("/v1/sensortypes/dummy_pack_1.SampleSensor") + resp = self.app.get(f"/v1/sensortypes/{DUMMY_PACK_1}.SampleSensor") self.assertEqual(resp.status_int, http_client.OK) self.assertTrue(resp.json["enabled"]) From 530221190985d0a5b0cee9c0f629397c3e42f1d5 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 24 May 2024 14:12:50 -0500 Subject: [PATCH 41/98] pants: add BUILD deps to run st2api unit tests --- BUILD | 37 ++++++++++++++++++++++++++ st2api/tests/unit/BUILD | 5 ++++ st2api/tests/unit/controllers/BUILD | 4 +++ st2api/tests/unit/controllers/v1/BUILD | 13 +++++++++ 4 files changed, 59 insertions(+) diff --git a/BUILD b/BUILD index b2d06d15f9..a56fcf6b6f 100644 --- a/BUILD +++ b/BUILD @@ -27,6 +27,36 @@ python_requirements( "st2auth/st2auth/backends/constants.py", ] ), + # make sure anything that uses st2-rbac-backend gets its deps + "st2-rbac-backend": dict( + dependencies=[ + # alphabetical order + "st2common/st2common/config.py", + "st2common/st2common/constants/keyvalue.py", + "st2common/st2common/constants/triggers.py", + "st2common/st2common/content/loader.py", + "st2common/st2common/exceptions/db.py", + "st2common/st2common/exceptions/rbac.py", + "st2common/st2common/log.py", + "st2common/st2common/models/api/rbac.py", + "st2common/st2common/models/db/action.py", + "st2common/st2common/models/db/auth.py", + "st2common/st2common/models/db/pack.py", + "st2common/st2common/models/db/rbac.py", + "st2common/st2common/models/db/webhook.py", + "st2common/st2common/models/system/common.py", + "st2common/st2common/persistence/auth.py", + "st2common/st2common/persistence/execution.py", + "st2common/st2common/persistence/rbac.py", + "st2common/st2common/rbac/backends/__init__.py", + "st2common/st2common/rbac/backends/base.py", + "st2common/st2common/rbac/types.py", + "st2common/st2common/script_setup.py", + "st2common/st2common/util/action_db.py", + "st2common/st2common/util/misc.py", + "st2common/st2common/util/uid.py", + ] + ), }, ) @@ -38,6 +68,13 @@ target( ], ) +target( + name="rbac_backends", + dependencies=[ + "//:reqs#st2-rbac-backend", + ], +) + python_test_utils( name="test_utils", skip_pylint=True, diff --git a/st2api/tests/unit/BUILD b/st2api/tests/unit/BUILD index 9a24dba70a..d649809607 100644 --- a/st2api/tests/unit/BUILD +++ b/st2api/tests/unit/BUILD @@ -5,4 +5,9 @@ __defaults__( python_tests( name="tests", + dependencies=["//:rbac_backends"], + stevedore_namespaces=[ + "st2common.rbac.backend", + ], + uses=["mongo"], ) diff --git a/st2api/tests/unit/controllers/BUILD b/st2api/tests/unit/controllers/BUILD index 57341b1358..e32c67ef43 100644 --- a/st2api/tests/unit/controllers/BUILD +++ b/st2api/tests/unit/controllers/BUILD @@ -1,3 +1,7 @@ python_tests( name="tests", + stevedore_namespaces=[ + "st2common.metrics.driver", + ], + uses=["mongo"], ) diff --git a/st2api/tests/unit/controllers/v1/BUILD b/st2api/tests/unit/controllers/v1/BUILD index 57341b1358..5a37a11f3e 100644 --- a/st2api/tests/unit/controllers/v1/BUILD +++ b/st2api/tests/unit/controllers/v1/BUILD @@ -1,3 +1,16 @@ python_tests( name="tests", + stevedore_namespaces=[ + "st2common.runners.runner", + "st2common.rbac.backend", + "st2common.metrics.driver", + ], + uses=["mongo"], + overrides={ + "test_webhooks.py": dict( + dependencies=[ + "st2common/st2common/models/api/webhook.py", + ], + ), + }, ) From 6bba55b7fca273c7144a81896203c55c944f1f1c Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 24 May 2024 14:13:25 -0500 Subject: [PATCH 42/98] pants: test st2*/ unit tests in CI --- .github/workflows/test.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 705a9a95f3..cac8aa7d35 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -117,6 +117,9 @@ jobs: st2common/:: st2actions/:: st2reactor/:: + st2stream/:: + st2auth/:: + st2api/:: - name: Upload pants log uses: actions/upload-artifact@v4 From b4c71552706210f2d503cea66adf6a8ca9384659 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Tue, 11 Jun 2024 10:40:05 -0500 Subject: [PATCH 43/98] pants CI: run pack and some runner tests --- .github/workflows/test.yaml | 18 ++++++++++++++++++ contrib/chatops/tests/BUILD | 5 +++++ contrib/core/tests/BUILD | 5 +++++ contrib/examples/tests/BUILD | 5 +++++ contrib/linux/tests/BUILD | 5 +++++ contrib/packs/tests/BUILD | 5 +++++ 6 files changed, 43 insertions(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index cac8aa7d35..7a0a6b85bd 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -120,6 +120,24 @@ jobs: st2stream/:: st2auth/:: st2api/:: + contrib/debug/:: + contrib/default/:: + contrib/hello_st2/:: + contrib/runners/announcement_runner/:: + contrib/runners/http_runner/:: + contrib/runners/inquirer_runner/:: + contrib/runners/local_runner/:: + contrib/runners/noop_runner/:: + contrib/runners/remote_runner/:: + contrib/runners/winrm_runner/:: + contrib/chatops/:: + contrib/core/:: + contrib/examples/:: + contrib/linux/:: + contrib/packs/:: +# contrib/runners/action_chain_runner/:: +# contrib/runners/orquesta_runner/:: +# contrib/runners/python_runner/:: - name: Upload pants log uses: actions/upload-artifact@v4 diff --git a/contrib/chatops/tests/BUILD b/contrib/chatops/tests/BUILD index cd3fa380ae..6925caed4b 100644 --- a/contrib/chatops/tests/BUILD +++ b/contrib/chatops/tests/BUILD @@ -1,6 +1,11 @@ # tests can only be dependencies of other tests in this directory __dependents_rules__(("*", "/**", "!*")) +__defaults__( + {(python_test, python_tests): dict(tags=["unit", "pack"])}, + extend=True, +) + files( name="fixtures", sources=["fixtures/*.json"], diff --git a/contrib/core/tests/BUILD b/contrib/core/tests/BUILD index 6f09c14528..e6278d7e9a 100644 --- a/contrib/core/tests/BUILD +++ b/contrib/core/tests/BUILD @@ -1,6 +1,11 @@ # tests can only be dependencies of other tests in this directory __dependents_rules__(("*", "/**", "!*")) +__defaults__( + {(python_test, python_tests): dict(tags=["unit", "pack"])}, + extend=True, +) + python_tests( skip_pylint=True, overrides={ diff --git a/contrib/examples/tests/BUILD b/contrib/examples/tests/BUILD index 0f0af81da5..4583892bc4 100644 --- a/contrib/examples/tests/BUILD +++ b/contrib/examples/tests/BUILD @@ -1,6 +1,11 @@ # tests can only be dependencies of other tests in this directory __dependents_rules__(("*", "/**", "!*")) +__defaults__( + {(python_test, python_tests): dict(tags=["unit", "pack"])}, + extend=True, +) + python_tests( skip_pylint=True, ) diff --git a/contrib/linux/tests/BUILD b/contrib/linux/tests/BUILD index 0f0af81da5..4583892bc4 100644 --- a/contrib/linux/tests/BUILD +++ b/contrib/linux/tests/BUILD @@ -1,6 +1,11 @@ # tests can only be dependencies of other tests in this directory __dependents_rules__(("*", "/**", "!*")) +__defaults__( + {(python_test, python_tests): dict(tags=["unit", "pack"])}, + extend=True, +) + python_tests( skip_pylint=True, ) diff --git a/contrib/packs/tests/BUILD b/contrib/packs/tests/BUILD index 0f0af81da5..4583892bc4 100644 --- a/contrib/packs/tests/BUILD +++ b/contrib/packs/tests/BUILD @@ -1,6 +1,11 @@ # tests can only be dependencies of other tests in this directory __dependents_rules__(("*", "/**", "!*")) +__defaults__( + {(python_test, python_tests): dict(tags=["unit", "pack"])}, + extend=True, +) + python_tests( skip_pylint=True, ) From 5bf1332dd0b00303846917b48e7a49a386e3724a Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Tue, 11 Jun 2024 10:40:56 -0500 Subject: [PATCH 44/98] wip support for running action_chain_runner tests via pants+pytest --- contrib/runners/action_chain_runner/tests/unit/BUILD | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/contrib/runners/action_chain_runner/tests/unit/BUILD b/contrib/runners/action_chain_runner/tests/unit/BUILD index 9a24dba70a..e4bc611756 100644 --- a/contrib/runners/action_chain_runner/tests/unit/BUILD +++ b/contrib/runners/action_chain_runner/tests/unit/BUILD @@ -5,4 +5,15 @@ __defaults__( python_tests( name="tests", + stevedore_namespaces=[ + "st2common.rbac.backend", + "st2common.metrics.driver", + # TODO: we only need THIS runner, but this pulls in all. + "st2common.runners.runner", + ], + uses=["mongo"], + # TODO: need st2common.runners.runner stevedore namespace, but only this runner! + #dependencies=[ + # "contrib/runners/action_chain_runner", + #], ) From 5bc97c0c32213f5a4c99c1bf85962c6ffda07ac3 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 29 Jul 2024 21:59:40 -0500 Subject: [PATCH 45/98] fmt --- st2api/tests/unit/controllers/v1/test_executions.py | 5 ++++- st2api/tests/unit/controllers/v1/test_sensortypes.py | 8 ++++++-- st2common/tests/unit/test_runners_utils.py | 1 - 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/st2api/tests/unit/controllers/v1/test_executions.py b/st2api/tests/unit/controllers/v1/test_executions.py index 0e941568a6..08c07f0fc3 100644 --- a/st2api/tests/unit/controllers/v1/test_executions.py +++ b/st2api/tests/unit/controllers/v1/test_executions.py @@ -704,7 +704,10 @@ def test_post_delete_duplicate(self): delete_resp = self._do_delete(self._get_actionexecution_id(post_resp)) self.assertEqual(delete_resp.status_int, 200) self.assertEqual(delete_resp.json["status"], "canceled") - expected_result = {"message": "Action canceled by user.", "user": SYSTEM_USER} + expected_result = { + "message": "Action canceled by user.", + "user": SYSTEM_USER, + } self.assertDictEqual(delete_resp.json["result"], expected_result) def test_post_delete_trace(self): diff --git a/st2api/tests/unit/controllers/v1/test_sensortypes.py b/st2api/tests/unit/controllers/v1/test_sensortypes.py index ac22a778cb..ea32f0e904 100644 --- a/st2api/tests/unit/controllers/v1/test_sensortypes.py +++ b/st2api/tests/unit/controllers/v1/test_sensortypes.py @@ -133,7 +133,9 @@ def test_disable_and_enable_sensor(self): # Disable sensor data = copy.deepcopy(sensor_data) data["enabled"] = False - put_resp = self.app.put_json(f"/v1/sensortypes/{DUMMY_PACK_1}.SampleSensor", data) + put_resp = self.app.put_json( + f"/v1/sensortypes/{DUMMY_PACK_1}.SampleSensor", data + ) self.assertEqual(put_resp.status_int, http_client.OK) self.assertEqual(put_resp.json["ref"], f"{DUMMY_PACK_1}.SampleSensor") self.assertFalse(put_resp.json["enabled"]) @@ -146,7 +148,9 @@ def test_disable_and_enable_sensor(self): # Enable sensor data = copy.deepcopy(sensor_data) data["enabled"] = True - put_resp = self.app.put_json(f"/v1/sensortypes/{DUMMY_PACK_1}.SampleSensor", data) + put_resp = self.app.put_json( + f"/v1/sensortypes/{DUMMY_PACK_1}.SampleSensor", data + ) self.assertEqual(put_resp.status_int, http_client.OK) self.assertTrue(put_resp.json["enabled"]) diff --git a/st2common/tests/unit/test_runners_utils.py b/st2common/tests/unit/test_runners_utils.py index a7ed4c40d3..b6f61bd61c 100644 --- a/st2common/tests/unit/test_runners_utils.py +++ b/st2common/tests/unit/test_runners_utils.py @@ -23,7 +23,6 @@ from st2common.runners import utils from st2common.services import executions as exe_svc from st2common.util import action_db as action_db_utils -from st2tests import base from st2tests import fixturesloader from st2tests.fixtures.generic.fixture import PACK_NAME as FIXTURES_PACK From 4a48fade99246a13d4873e4dd941483443cf10c3 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 29 Jul 2024 22:25:17 -0500 Subject: [PATCH 46/98] pants: add note about upcoming pants feature for action runner tests --- contrib/runners/action_chain_runner/tests/unit/BUILD | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/contrib/runners/action_chain_runner/tests/unit/BUILD b/contrib/runners/action_chain_runner/tests/unit/BUILD index e4bc611756..459639e1b7 100644 --- a/contrib/runners/action_chain_runner/tests/unit/BUILD +++ b/contrib/runners/action_chain_runner/tests/unit/BUILD @@ -12,8 +12,9 @@ python_tests( "st2common.runners.runner", ], uses=["mongo"], - # TODO: need st2common.runners.runner stevedore namespace, but only this runner! - #dependencies=[ - # "contrib/runners/action_chain_runner", - #], + # TODO: this will allow us to depend only on this runner once we upgrade to pants 2.23 + # https://github.com/pantsbuild/pants/pull/21062 + #entry_point_dependencies={ + # "contrib/runners/action_chain_runner": ["st2common.runners.runner"], + #}, ) From 68d817c18ffb4d750f4a65b38530fb8be507df00 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 1 Aug 2024 13:52:44 -0500 Subject: [PATCH 47/98] pants-plugins/pack_metadata: add pack_content_resource target type Now pack_metadata targets will generate pack_content_resource instead of just resource. pack_content_resource is still a resource, but this setup allows us to find the generated resource targets more simply. This also harmonizes the implementation of pack_metadata to follow the fields definition of resources (esp moving dependencies into moved_fields instead of core_fields). --- pants-plugins/pack_metadata/register.py | 2 ++ pants-plugins/pack_metadata/target_types.py | 28 +++++++++++++++++++-- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/pants-plugins/pack_metadata/register.py b/pants-plugins/pack_metadata/register.py index 36c11079d9..789ba35ffb 100644 --- a/pants-plugins/pack_metadata/register.py +++ b/pants-plugins/pack_metadata/register.py @@ -13,6 +13,7 @@ # limitations under the License. from pack_metadata import tailor, target_types_rules from pack_metadata.target_types import ( + PackContentResourceTarget, PackMetadata, PackMetadataInGitSubmodule, PacksGlob, @@ -28,6 +29,7 @@ def rules(): def target_types(): return [ + PackContentResourceTarget, PackMetadata, PackMetadataInGitSubmodule, PacksGlob, diff --git a/pants-plugins/pack_metadata/target_types.py b/pants-plugins/pack_metadata/target_types.py index 4c7c2c854f..e9a1076637 100644 --- a/pants-plugins/pack_metadata/target_types.py +++ b/pants-plugins/pack_metadata/target_types.py @@ -15,8 +15,12 @@ from pants.engine.target import COMMON_TARGET_FIELDS, Dependencies from pants.core.target_types import ( + ResourceDependenciesField, ResourcesGeneratingSourcesField, ResourcesGeneratorTarget, + ResourcesOverridesField, + ResourceSourceField, + ResourceTarget, GenericTarget, ) @@ -25,6 +29,10 @@ class UnmatchedGlobsError(Exception): """Error thrown when a required set of globs didn't match.""" +class PackContentResourceSourceField(ResourceSourceField): + pass + + class PackMetadataSourcesField(ResourcesGeneratingSourcesField): required = False default = ( @@ -58,9 +66,23 @@ def validate_resolved_files(self, files: Sequence[str]) -> None: super().validate_resolved_files(files) +class PackContentResourceTarget(ResourceTarget): + alias = "pack_content_resource" + core_fields = ( + *COMMON_TARGET_FIELDS, + ResourceDependenciesField, + PackContentResourceSourceField, + ) + help = ( + "A single pack content resource file (mostly for metadata files)." + ) + + class PackMetadata(ResourcesGeneratorTarget): alias = "pack_metadata" - core_fields = (*COMMON_TARGET_FIELDS, Dependencies, PackMetadataSourcesField) + core_fields = (*COMMON_TARGET_FIELDS, PackMetadataSourcesField, ResourcesOverridesField) + moved_fields = (ResourceDependenciesField,) + generated_target_cls = PackContentResourceTarget help = ( "Loose pack metadata files.\n\n" "Pack metadata includes top-level files (pack.yaml, .yaml.example, " @@ -73,9 +95,11 @@ class PackMetadataInGitSubmodule(PackMetadata): alias = "pack_metadata_in_git_submodule" core_fields = ( *COMMON_TARGET_FIELDS, - Dependencies, PackMetadataInGitSubmoduleSources, + ResourcesOverridesField, ) + moved_fields = (ResourceDependenciesField,) + generated_target_cls = PackContentResourceTarget help = PackMetadata.help + ( "\npack_metadata_in_git_submodule variant errors if the sources field " "has unmatched globs. It prints instructions on how to checkout git " From f6d028e49576cabd0a2fc1807af7aabf45c5104e Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 1 Aug 2024 13:58:13 -0500 Subject: [PATCH 48/98] pants-plugins/pack_metadata: classify metadata type of pack_content_resource targets This will allow rules to look up just action and sensor metadata (for example). --- .../pack_metadata/python_module_mapper.py | 53 ++++++++++++++ pants-plugins/pack_metadata/register.py | 3 +- pants-plugins/pack_metadata/target_types.py | 73 ++++++++++++++++++- 3 files changed, 126 insertions(+), 3 deletions(-) create mode 100644 pants-plugins/pack_metadata/python_module_mapper.py diff --git a/pants-plugins/pack_metadata/python_module_mapper.py b/pants-plugins/pack_metadata/python_module_mapper.py new file mode 100644 index 0000000000..bba29285fa --- /dev/null +++ b/pants-plugins/pack_metadata/python_module_mapper.py @@ -0,0 +1,53 @@ +# Copyright 2024 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from dataclasses import dataclass + +from pants.engine.rules import collect_rules, rule +from pants.engine.target import ( + AllTargets, + Targets, +) +from pants.util.logging import LogLevel + +from pack_metadata.target_types import ( + PackContentResourceSourceField, + PackContentResourceTypeField, + PackContentResourceTypes +) + + +@dataclass(frozen=True) +class PackContentResourceTargetsOfTypeRequest: + types: tuple[PackContentResourceTypes, ...] + + +class PackContentResourceTargetsOfType(Targets): + pass + + +@rule(desc="Find all PackMetadata targets in project filtered by content type", level=LogLevel.DEBUG) +async def find_pack_metadata_targets_of_types( + request: PackContentResourceTargetsOfTypeRequest, targets: AllTargets +) -> PackContentResourceTargetsOfType: + return PackContentResourceTargetsOfType( + tgt for tgt in targets + if tgt.has_field(PackContentResourceSourceField) + and (not request.types or tgt[PackContentResourceTypeField].value in request.types) + ) + + +def rules(): + return ( + *collect_rules(), + ) diff --git a/pants-plugins/pack_metadata/register.py b/pants-plugins/pack_metadata/register.py index 789ba35ffb..7a93b3ba6c 100644 --- a/pants-plugins/pack_metadata/register.py +++ b/pants-plugins/pack_metadata/register.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from pack_metadata import tailor, target_types_rules +from pack_metadata import python_module_mapper, tailor, target_types_rules from pack_metadata.target_types import ( PackContentResourceTarget, PackMetadata, @@ -24,6 +24,7 @@ def rules(): return [ *tailor.rules(), *target_types_rules.rules(), + *python_module_mapper.rules(), ] diff --git a/pants-plugins/pack_metadata/target_types.py b/pants-plugins/pack_metadata/target_types.py index e9a1076637..0e405770ea 100644 --- a/pants-plugins/pack_metadata/target_types.py +++ b/pants-plugins/pack_metadata/target_types.py @@ -11,9 +11,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Sequence +from enum import Enum +from pathlib import PurePath +from typing import Optional, Sequence, Tuple -from pants.engine.target import COMMON_TARGET_FIELDS, Dependencies +from pants.engine.internals.native_engine import Address +from pants.engine.target import COMMON_TARGET_FIELDS, Dependencies, StringField from pants.core.target_types import ( ResourceDependenciesField, ResourcesGeneratingSourcesField, @@ -29,6 +32,71 @@ class UnmatchedGlobsError(Exception): """Error thrown when a required set of globs didn't match.""" +class PackContentResourceTypes(Enum): + # in root of pack + pack_metadata = "pack_metadata" + pack_config_schema = "pack_config_schema" + pack_config_example = "pack_config_example" + pack_icon = "pack_icon" + # in subdirectory (see _content_type_by_path_parts below + action_metadata = "action_metadata" + action_chain_workflow = "action_chain_workflow" + orquesta_workflow = "orquesta_workflow" + alias_metadata = "alias_metadata" + policy_metadata = "policy_metadata" + rule_metadata = "rule_metadata" + sensor_metadata = "sensor_metadata" + trigger_metadata = "trigger_metadata" + # other + unknown = "unknown" + + +_content_type_by_path_parts: dict[Tuple[str, ...], PackContentResourceTypes] = { + ("actions",): PackContentResourceTypes.action_metadata, + ("actions", "chains"): PackContentResourceTypes.action_chain_workflow, + ("actions", "workflows"): PackContentResourceTypes.orquesta_workflow, + ("aliases",): PackContentResourceTypes.alias_metadata, + ("policies",): PackContentResourceTypes.policy_metadata, + ("rules",): PackContentResourceTypes.rule_metadata, + ("sensors",): PackContentResourceTypes.sensor_metadata, + ("triggers",): PackContentResourceTypes.trigger_metadata, +} + + +class PackContentResourceTypeField(StringField): + alias = "type" + help = ( + "The content type of the resource." + "\nDo not use this field in BUILD files. It is calculated automatically" + "based on the conventional location of files in the st2 pack." + ) + valid_choices = PackContentResourceTypes + value: PackContentResourceTypes + + @classmethod + def compute_value(cls, raw_value: Optional[str], address: Address) -> PackContentResourceTypes: + value = super().compute_value(raw_value, address) + if value is not None: + return PackContentResourceTypes(value) + path = PurePath(address.relative_file_path) + _yaml_suffixes = ("yaml", "yml") + if len(path.parent.parts) == 0: + # in the pack root + if path.name == "pack.yaml": + return PackContentResourceTypes.pack_metadata + if path.stem == "pack.schema" and path.suffix in _yaml_suffixes: + return PackContentResourceTypes.pack_config_schema + if path.suffix == "example" and path.suffixes[0] in _yaml_suffixes: + return PackContentResourceTypes.pack_config_example + if path.name == "icon.png": + return PackContentResourceTypes.pack_config_example + return PackContentResourceTypes.unknown + resource_type = _content_type_by_path_parts.get(path.parent.parts, None) + if resource_type is not None: + return resource_type + return PackContentResourceTypes.unknown + + class PackContentResourceSourceField(ResourceSourceField): pass @@ -72,6 +140,7 @@ class PackContentResourceTarget(ResourceTarget): *COMMON_TARGET_FIELDS, ResourceDependenciesField, PackContentResourceSourceField, + PackContentResourceTypeField, ) help = ( "A single pack content resource file (mostly for metadata files)." From 371db6c92e5e823014f1f193075010dcbc41ca92 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 1 Aug 2024 14:28:01 -0500 Subject: [PATCH 49/98] pants-plugins/pack_metadata: register actions/sensors in pants python module mapping Only handles the actual action/sensor python files. It does not yet handle: - /lib - /actions/lib --- .../pack_metadata/python_module_mapper.py | 168 +++++++++++++++++- 1 file changed, 167 insertions(+), 1 deletion(-) diff --git a/pants-plugins/pack_metadata/python_module_mapper.py b/pants-plugins/pack_metadata/python_module_mapper.py index bba29285fa..e53d402827 100644 --- a/pants-plugins/pack_metadata/python_module_mapper.py +++ b/pants-plugins/pack_metadata/python_module_mapper.py @@ -11,13 +11,36 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from collections import defaultdict from dataclasses import dataclass +from pathlib import PurePath +from typing import DefaultDict -from pants.engine.rules import collect_rules, rule +import yaml + +from pants.backend.python.dependency_inference.module_mapper import ( + FirstPartyPythonMappingImpl, + FirstPartyPythonMappingImplMarker, + ModuleProvider, + ModuleProviderType, + ResolveName, +) +from pants.backend.python.subsystems.setup import PythonSetup +from pants.backend.python.target_types import PythonResolveField +from pants.base.glob_match_error_behavior import GlobMatchErrorBehavior +from pants.base.specs import FileLiteralSpec, RawSpecs +from pants.engine.collection import Collection +from pants.engine.fs import DigestContents +from pants.engine.internals.native_engine import Address, Digest +from pants.engine.rules import Get, MultiGet, collect_rules, rule from pants.engine.target import ( AllTargets, + HydrateSourcesRequest, + HydratedSources, + Target, Targets, ) +from pants.engine.unions import UnionRule from pants.util.logging import LogLevel from pack_metadata.target_types import ( @@ -47,7 +70,150 @@ async def find_pack_metadata_targets_of_types( ) +@dataclass(frozen=True) +class PackContentPythonEntryPoint: + metadata_address: Address + content_type: PackContentResourceTypes + entry_point: str + python_address: Address + resolve: str + module: str + + +class PackContentPythonEntryPoints(Collection[PackContentPythonEntryPoint]): + pass + + +class PackContentPythonEntryPointsRequest: + pass + + +def get_possible_modules(path: PurePath) -> list[str]: + module = path.stem if path.suffix == ".py" else path.name + modules = [module] + + try: + start = path.parent.parts.index("actions") + 1 + except ValueError: + start = path.parent.parts.index("sensors") + 1 + + # st2 adds the parent dir of the python file to sys.path at runtime. + # by convention, however, just actions/ is on sys.path during tests. + # so, also construct the module name from actions/ to support tests. + if start < len(path.parent.parts): + modules.append(".".join((*path.parent.parts[start:], module))) + return modules + + +@rule(desc="Find all Pack Content entry_points that are python", level=LogLevel.DEBUG) +async def find_pack_content_python_entry_points( + python_setup: PythonSetup, + _: PackContentPythonEntryPointsRequest +) -> PackContentPythonEntryPoints: + action_or_sensor = ( + PackContentResourceTypes.action_metadata, + PackContentResourceTypes.sensor_metadata, + ) + + action_and_sensor_metadata_targets = await Get( + PackContentResourceTargetsOfType, + PackContentResourceTargetsOfTypeRequest(action_or_sensor), + ) + action_and_sensor_metadata_sources = await MultiGet( + Get(HydratedSources, HydrateSourcesRequest(tgt[PackContentResourceSourceField])) + for tgt in action_and_sensor_metadata_targets + ) + action_and_sensor_metadata_contents = await MultiGet( + Get(DigestContents, Digest, source.snapshot.digest) + for source in action_and_sensor_metadata_sources + ) + + # python file path -> list of info about metadata files that refer to it + pack_content_entry_points_by_spec: DefaultDict[ + str, list[tuple[Address, PackContentResourceTypes, str]] + ] = defaultdict(list) + + tgt: Target + contents: DigestContents + for tgt, contents in zip(action_and_sensor_metadata_targets, action_and_sensor_metadata_contents): + content_type = tgt[PackContentResourceTypeField].value + if content_type not in action_or_sensor: + continue + assert len(contents) == 1 + try: + metadata = yaml.safe_load(contents[0].content) or {} + except yaml.YAMLError: + continue + if content_type == PackContentResourceTypes.action_metadata: + runner_type = metadata.get("runner_type", "") or "" + if runner_type != "python-script": + # only python-script has special PYTHONPATH rules + continue + # get the entry_point to find subdirectory that contains the module + entry_point = metadata.get("entry_point", "") or "" + if entry_point: + # address.filename is basically f"{spec_path}/{relative_file_path}" + path = PurePath(tgt.address.filename).parent / entry_point + pack_content_entry_points_by_spec[str(path)].append((tgt.address, content_type, entry_point)) + + python_targets = await Get( + Targets, + RawSpecs( + file_literals=tuple(FileLiteralSpec(spec_path) for spec_path in pack_content_entry_points_by_spec), + unmatched_glob_behavior=GlobMatchErrorBehavior.ignore, + description_of_origin="pack_metadata python module mapper", + ) + ) + + pack_content_entry_points: list[PackContentPythonEntryPoint] = [] + for tgt in python_targets: + if not tgt.has_field(PythonResolveField): + # this is unexpected + continue + for metadata_address, content_type, entry_point in pack_content_entry_points_by_spec[tgt.address.filename]: + resolve = tgt[PythonResolveField].normalized_value(python_setup) + + for module in get_possible_modules(PurePath(tgt.address.filename)): + pack_content_entry_points.append(PackContentPythonEntryPoint( + metadata_address=metadata_address, + content_type=content_type, + entry_point=entry_point, + python_address=tgt.address, + resolve=resolve, + module=module, + )) + + return PackContentPythonEntryPoints(pack_content_entry_points) + + +# This is only used to register our implementation with the plugin hook via unions. +class St2PythonPackContentMappingMarker(FirstPartyPythonMappingImplMarker): + pass + + +@rule(desc="Creating map of pack_metadata targets to Python modules in pack content", level=LogLevel.DEBUG) +async def map_pack_content_to_python_modules( + _: St2PythonPackContentMappingMarker, +) -> FirstPartyPythonMappingImpl: + resolves_to_modules_to_providers: DefaultDict[ + ResolveName, DefaultDict[str, list[ModuleProvider]] + ] = defaultdict(lambda: defaultdict(list)) + + pack_content_python_entry_points = await Get( + PackContentPythonEntryPoints, + PackContentPythonEntryPointsRequest(), + ) + + for pack_content in pack_content_python_entry_points: + resolves_to_modules_to_providers[pack_content.resolve][pack_content.module].append( + ModuleProvider(pack_content.python_address, ModuleProviderType.IMPL) + ) + + return FirstPartyPythonMappingImpl.create(resolves_to_modules_to_providers) + + def rules(): return ( *collect_rules(), + UnionRule(FirstPartyPythonMappingImplMarker, St2PythonPackContentMappingMarker), ) From b655e84bc8b2d202da30f7a78a1b1f881c8b2470 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 1 Aug 2024 17:45:10 -0500 Subject: [PATCH 50/98] pants-plugins/pack_metadata: register lib and actions/lib in pants python module mapping This makes dependency inference aware of these which may be on the PYTHONPATH. - /lib - /actions/lib --- .../pack_metadata/python_module_mapper.py | 109 +++++++++++++++++- 1 file changed, 103 insertions(+), 6 deletions(-) diff --git a/pants-plugins/pack_metadata/python_module_mapper.py b/pants-plugins/pack_metadata/python_module_mapper.py index e53d402827..ac2dc61418 100644 --- a/pants-plugins/pack_metadata/python_module_mapper.py +++ b/pants-plugins/pack_metadata/python_module_mapper.py @@ -24,29 +24,33 @@ ModuleProvider, ModuleProviderType, ResolveName, + module_from_stripped_path, ) from pants.backend.python.subsystems.setup import PythonSetup -from pants.backend.python.target_types import PythonResolveField +from pants.backend.python.target_types import PythonResolveField, PythonSourceField from pants.base.glob_match_error_behavior import GlobMatchErrorBehavior -from pants.base.specs import FileLiteralSpec, RawSpecs +from pants.base.specs import FileLiteralSpec, RawSpecs, RecursiveGlobSpec from pants.engine.collection import Collection from pants.engine.fs import DigestContents from pants.engine.internals.native_engine import Address, Digest from pants.engine.rules import Get, MultiGet, collect_rules, rule from pants.engine.target import ( AllTargets, + AllUnexpandedTargets, HydrateSourcesRequest, HydratedSources, Target, Targets, ) from pants.engine.unions import UnionRule +from pants.util.dirutil import fast_relpath from pants.util.logging import LogLevel from pack_metadata.target_types import ( PackContentResourceSourceField, PackContentResourceTypeField, - PackContentResourceTypes + PackContentResourceTypes, + PackMetadataSourcesField, ) @@ -89,6 +93,8 @@ class PackContentPythonEntryPointsRequest: def get_possible_modules(path: PurePath) -> list[str]: + if path.name in ("__init__.py", "__init__.pyi"): + path = path.parent module = path.stem if path.suffix == ".py" else path.name modules = [module] @@ -186,6 +192,89 @@ async def find_pack_content_python_entry_points( return PackContentPythonEntryPoints(pack_content_entry_points) +@dataclass(frozen=True) +class PackPythonLib: + pack_path: PurePath + lib_dir: str + relative_to_lib: PurePath + python_address: Address + resolve: str + module: str + + +class PackPythonLibs(Collection[PackPythonLib]): + pass + + +class PackPythonLibsRequest: + pass + + +@rule(desc="Find all Pack lib directory python targets", level=LogLevel.DEBUG) +async def find_python_in_pack_lib_directories( + python_setup: PythonSetup, + all_unexpanded_targets: AllUnexpandedTargets, + _: PackPythonLibsRequest, +) -> PackPythonLibs: + pack_metadata_paths = [ + PurePath(tgt.address.spec_path) for tgt in all_unexpanded_targets + if tgt.has_field(PackMetadataSourcesField) + ] + pack_lib_directory_targets = await MultiGet( + Get( + Targets, + RawSpecs( + recursive_globs=( + RecursiveGlobSpec(str(path / "lib")), + RecursiveGlobSpec(str(path / "actions" / "lib")), + ), + unmatched_glob_behavior=GlobMatchErrorBehavior.ignore, + description_of_origin="pack_metadata lib directory lookup", + ) + ) + for path in pack_metadata_paths + ) + + # Maybe this should use this to take codegen into account. + # Get(PythonSourceFiles, PythonSourceFilesRequest(targets=lib_directory_targets, include_resources=False) + # For now, just take the targets as they are. + + pack_python_libs: list[PackPythonLib] = [] + + pack_path: PurePath + lib_directory_targets: Targets + for pack_path, lib_directory_targets in zip(pack_metadata_paths, pack_lib_directory_targets): + for tgt in lib_directory_targets: + if not tgt.has_field(PythonSourceField): + # only python targets matter here. + continue + + relative_to_pack = PurePath(fast_relpath(tgt[PythonSourceField].file_path, str(pack_path))) + if relative_to_pack.parts[0] == "lib": + lib_dir = "lib" + elif relative_to_pack.parts[:2] == ("actions", "lib"): + lib_dir = "actions/lib" + else: + # This should not happen as it is not in the requested glob. + # Use this to tell linters that lib_dir is defined below here. + continue + relative_to_lib = relative_to_pack.relative_to(lib_dir) + + resolve = tgt[PythonResolveField].normalized_value(python_setup) + module = module_from_stripped_path(relative_to_lib) + + pack_python_libs.append(PackPythonLib( + pack_path=pack_path, + lib_dir=lib_dir, + relative_to_lib=relative_to_lib, + python_address=tgt.address, + resolve=resolve, + module=module, + )) + + return PackPythonLibs(pack_python_libs) + + # This is only used to register our implementation with the plugin hook via unions. class St2PythonPackContentMappingMarker(FirstPartyPythonMappingImplMarker): pass @@ -199,9 +288,9 @@ async def map_pack_content_to_python_modules( ResolveName, DefaultDict[str, list[ModuleProvider]] ] = defaultdict(lambda: defaultdict(list)) - pack_content_python_entry_points = await Get( - PackContentPythonEntryPoints, - PackContentPythonEntryPointsRequest(), + pack_content_python_entry_points, pack_python_libs = await MultiGet( + Get(PackContentPythonEntryPoints, PackContentPythonEntryPointsRequest()), + Get(PackPythonLibs, PackPythonLibsRequest()), ) for pack_content in pack_content_python_entry_points: @@ -209,6 +298,14 @@ async def map_pack_content_to_python_modules( ModuleProvider(pack_content.python_address, ModuleProviderType.IMPL) ) + for pack_lib in pack_python_libs: + provider_type = ( + ModuleProviderType.TYPE_STUB if pack_lib.relative_to_lib.suffix == ".pyi" else ModuleProviderType.IMPL + ) + resolves_to_modules_to_providers[pack_lib.resolve][pack_lib.module].append( + ModuleProvider(pack_lib.python_address, provider_type) + ) + return FirstPartyPythonMappingImpl.create(resolves_to_modules_to_providers) From b3ca250011c23a45434fd2e2d2fec45ac50fab6d Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 1 Aug 2024 17:49:52 -0500 Subject: [PATCH 51/98] pants: Remove /lib and /actions/lib from source roots The pack_metadata plugin now handles identifying these imports for dep inference. Next step, modify the PYTHONPATH as well. --- pants.toml | 3 --- 1 file changed, 3 deletions(-) diff --git a/pants.toml b/pants.toml index 8cda497564..0bd79c9050 100644 --- a/pants.toml +++ b/pants.toml @@ -90,9 +90,6 @@ root_patterns = [ "/contrib/packs", "/st2tests/testpacks/checks", "/st2tests/testpacks/errorcheck", - # pack common lib directories that ST2 adds to the PATH for actions/sensors - "/contrib/*/lib", - "/contrib/*/actions/lib", # other special-cased pack directories "/contrib/examples/actions/ubuntu_pkg_info", # python script runs via shell expecting cwd in PYTHONPATH # lint plugins From 433e9011f92752b062948f3856d053a0de314481 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 1 Aug 2024 17:52:00 -0500 Subject: [PATCH 52/98] pants: Resolve ambiguous imports The pack_metadata plugin now handles identifying these imports for dep inference, so there are more opportunities for the core pack to be imported ambiguously thanks to its symlink in st2tests/st2tests/fixtures/packs --- st2common/tests/unit/services/BUILD | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/st2common/tests/unit/services/BUILD b/st2common/tests/unit/services/BUILD index a30ca92a77..f1342ac1b4 100644 --- a/st2common/tests/unit/services/BUILD +++ b/st2common/tests/unit/services/BUILD @@ -7,4 +7,10 @@ python_tests( "st2common.metrics.driver", ], uses=["mongo", "rabbitmq"], + overrides={ + "test_packs.py": dict( + # use the fixture to resolve ambiguous import (ambiguous due to symlink of core pack) + dependencies=["st2tests/st2tests/fixtures/packs/core/actions/inject_trigger.py"], + ), + } ) From b6ca68fc4c4e2621dbec75cf95f71b43def618c9 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 1 Aug 2024 17:52:23 -0500 Subject: [PATCH 53/98] pants: note that a test needs mongo --- contrib/core/tests/BUILD | 2 ++ 1 file changed, 2 insertions(+) diff --git a/contrib/core/tests/BUILD b/contrib/core/tests/BUILD index e6278d7e9a..205c90c328 100644 --- a/contrib/core/tests/BUILD +++ b/contrib/core/tests/BUILD @@ -8,6 +8,7 @@ __defaults__( python_tests( skip_pylint=True, + # st2_pack_dir="..", # directory containing pack.yaml overrides={ "test_action_sendmail.py": dict( dependencies=[ @@ -16,6 +17,7 @@ python_tests( # Use contrib/core as the canonical copy. "contrib/core:reqs#mail-parser", ], + uses=["mongo"], ), }, ) From 0577b2bbd35f0de9a04474cd6240155bd32f55f1 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 1 Aug 2024 18:58:48 -0500 Subject: [PATCH 54/98] Rename file --- .../{python_module_mapper.py => python_content_rules.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename pants-plugins/pack_metadata/{python_module_mapper.py => python_content_rules.py} (100%) diff --git a/pants-plugins/pack_metadata/python_module_mapper.py b/pants-plugins/pack_metadata/python_content_rules.py similarity index 100% rename from pants-plugins/pack_metadata/python_module_mapper.py rename to pants-plugins/pack_metadata/python_content_rules.py From 76e249aac08887e9de9cda6db27664b1c91029bb Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 1 Aug 2024 19:07:33 -0500 Subject: [PATCH 55/98] pants-plugins/pack_metadata: Extract python_module_mapper rule from python_content_rules --- .../pack_metadata/python_content_rules.py | 45 +---------- .../pack_metadata/python_module_mapper.py | 74 +++++++++++++++++++ pants-plugins/pack_metadata/register.py | 3 +- 3 files changed, 77 insertions(+), 45 deletions(-) create mode 100644 pants-plugins/pack_metadata/python_module_mapper.py diff --git a/pants-plugins/pack_metadata/python_content_rules.py b/pants-plugins/pack_metadata/python_content_rules.py index ac2dc61418..c711acbd44 100644 --- a/pants-plugins/pack_metadata/python_content_rules.py +++ b/pants-plugins/pack_metadata/python_content_rules.py @@ -18,14 +18,7 @@ import yaml -from pants.backend.python.dependency_inference.module_mapper import ( - FirstPartyPythonMappingImpl, - FirstPartyPythonMappingImplMarker, - ModuleProvider, - ModuleProviderType, - ResolveName, - module_from_stripped_path, -) +from pants.backend.python.dependency_inference.module_mapper import module_from_stripped_path from pants.backend.python.subsystems.setup import PythonSetup from pants.backend.python.target_types import PythonResolveField, PythonSourceField from pants.base.glob_match_error_behavior import GlobMatchErrorBehavior @@ -42,7 +35,6 @@ Target, Targets, ) -from pants.engine.unions import UnionRule from pants.util.dirutil import fast_relpath from pants.util.logging import LogLevel @@ -275,42 +267,7 @@ async def find_python_in_pack_lib_directories( return PackPythonLibs(pack_python_libs) -# This is only used to register our implementation with the plugin hook via unions. -class St2PythonPackContentMappingMarker(FirstPartyPythonMappingImplMarker): - pass - - -@rule(desc="Creating map of pack_metadata targets to Python modules in pack content", level=LogLevel.DEBUG) -async def map_pack_content_to_python_modules( - _: St2PythonPackContentMappingMarker, -) -> FirstPartyPythonMappingImpl: - resolves_to_modules_to_providers: DefaultDict[ - ResolveName, DefaultDict[str, list[ModuleProvider]] - ] = defaultdict(lambda: defaultdict(list)) - - pack_content_python_entry_points, pack_python_libs = await MultiGet( - Get(PackContentPythonEntryPoints, PackContentPythonEntryPointsRequest()), - Get(PackPythonLibs, PackPythonLibsRequest()), - ) - - for pack_content in pack_content_python_entry_points: - resolves_to_modules_to_providers[pack_content.resolve][pack_content.module].append( - ModuleProvider(pack_content.python_address, ModuleProviderType.IMPL) - ) - - for pack_lib in pack_python_libs: - provider_type = ( - ModuleProviderType.TYPE_STUB if pack_lib.relative_to_lib.suffix == ".pyi" else ModuleProviderType.IMPL - ) - resolves_to_modules_to_providers[pack_lib.resolve][pack_lib.module].append( - ModuleProvider(pack_lib.python_address, provider_type) - ) - - return FirstPartyPythonMappingImpl.create(resolves_to_modules_to_providers) - - def rules(): return ( *collect_rules(), - UnionRule(FirstPartyPythonMappingImplMarker, St2PythonPackContentMappingMarker), ) diff --git a/pants-plugins/pack_metadata/python_module_mapper.py b/pants-plugins/pack_metadata/python_module_mapper.py new file mode 100644 index 0000000000..c4b6646181 --- /dev/null +++ b/pants-plugins/pack_metadata/python_module_mapper.py @@ -0,0 +1,74 @@ +# Copyright 2024 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from collections import defaultdict +from typing import DefaultDict + +from pants.backend.python.dependency_inference.module_mapper import ( + FirstPartyPythonMappingImpl, + FirstPartyPythonMappingImplMarker, + ModuleProvider, + ModuleProviderType, + ResolveName, +) +from pants.engine.rules import Get, MultiGet, collect_rules, rule +from pants.engine.unions import UnionRule +from pants.util.logging import LogLevel + +from pack_metadata.python_content_rules import ( + PackContentPythonEntryPoints, + PackContentPythonEntryPointsRequest, + PackPythonLibs, + PackPythonLibsRequest, +) + + +# This is only used to register our implementation with the plugin hook via unions. +class St2PythonPackContentMappingMarker(FirstPartyPythonMappingImplMarker): + pass + + +@rule(desc="Creating map of pack_metadata targets to Python modules in pack content", level=LogLevel.DEBUG) +async def map_pack_content_to_python_modules( + _: St2PythonPackContentMappingMarker, +) -> FirstPartyPythonMappingImpl: + resolves_to_modules_to_providers: DefaultDict[ + ResolveName, DefaultDict[str, list[ModuleProvider]] + ] = defaultdict(lambda: defaultdict(list)) + + pack_content_python_entry_points, pack_python_libs = await MultiGet( + Get(PackContentPythonEntryPoints, PackContentPythonEntryPointsRequest()), + Get(PackPythonLibs, PackPythonLibsRequest()), + ) + + for pack_content in pack_content_python_entry_points: + resolves_to_modules_to_providers[pack_content.resolve][pack_content.module].append( + ModuleProvider(pack_content.python_address, ModuleProviderType.IMPL) + ) + + for pack_lib in pack_python_libs: + provider_type = ( + ModuleProviderType.TYPE_STUB if pack_lib.relative_to_lib.suffix == ".pyi" else ModuleProviderType.IMPL + ) + resolves_to_modules_to_providers[pack_lib.resolve][pack_lib.module].append( + ModuleProvider(pack_lib.python_address, provider_type) + ) + + return FirstPartyPythonMappingImpl.create(resolves_to_modules_to_providers) + + +def rules(): + return ( + *collect_rules(), + UnionRule(FirstPartyPythonMappingImplMarker, St2PythonPackContentMappingMarker), + ) diff --git a/pants-plugins/pack_metadata/register.py b/pants-plugins/pack_metadata/register.py index 7a93b3ba6c..f466edce2e 100644 --- a/pants-plugins/pack_metadata/register.py +++ b/pants-plugins/pack_metadata/register.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from pack_metadata import python_module_mapper, tailor, target_types_rules +from pack_metadata import python_content_rules, python_module_mapper, tailor, target_types_rules from pack_metadata.target_types import ( PackContentResourceTarget, PackMetadata, @@ -24,6 +24,7 @@ def rules(): return [ *tailor.rules(), *target_types_rules.rules(), + *python_content_rules.rules(), *python_module_mapper.rules(), ] From 13c15000b89913b401da91e31eb99c558d5a47f5 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 1 Aug 2024 19:13:14 -0500 Subject: [PATCH 56/98] pants-plugins/pack_metadata: Rename python_content_rules to python_pack_content --- pants-plugins/pack_metadata/python_module_mapper.py | 2 +- .../{python_content_rules.py => python_pack_content.py} | 0 pants-plugins/pack_metadata/register.py | 4 ++-- 3 files changed, 3 insertions(+), 3 deletions(-) rename pants-plugins/pack_metadata/{python_content_rules.py => python_pack_content.py} (100%) diff --git a/pants-plugins/pack_metadata/python_module_mapper.py b/pants-plugins/pack_metadata/python_module_mapper.py index c4b6646181..14fa535a6d 100644 --- a/pants-plugins/pack_metadata/python_module_mapper.py +++ b/pants-plugins/pack_metadata/python_module_mapper.py @@ -25,7 +25,7 @@ from pants.engine.unions import UnionRule from pants.util.logging import LogLevel -from pack_metadata.python_content_rules import ( +from pack_metadata.python_pack_content import ( PackContentPythonEntryPoints, PackContentPythonEntryPointsRequest, PackPythonLibs, diff --git a/pants-plugins/pack_metadata/python_content_rules.py b/pants-plugins/pack_metadata/python_pack_content.py similarity index 100% rename from pants-plugins/pack_metadata/python_content_rules.py rename to pants-plugins/pack_metadata/python_pack_content.py diff --git a/pants-plugins/pack_metadata/register.py b/pants-plugins/pack_metadata/register.py index f466edce2e..d48d70456c 100644 --- a/pants-plugins/pack_metadata/register.py +++ b/pants-plugins/pack_metadata/register.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from pack_metadata import python_content_rules, python_module_mapper, tailor, target_types_rules +from pack_metadata import python_pack_content, python_module_mapper, tailor, target_types_rules from pack_metadata.target_types import ( PackContentResourceTarget, PackMetadata, @@ -24,7 +24,7 @@ def rules(): return [ *tailor.rules(), *target_types_rules.rules(), - *python_content_rules.rules(), + *python_pack_content.rules(), *python_module_mapper.rules(), ] From ca2237c97e88c9f1fd4e07297d98d50dcf478478 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 2 Aug 2024 12:19:55 -0500 Subject: [PATCH 57/98] pants-plugins/pack_metadata: Reorder imports --- pants-plugins/pack_metadata/python_pack_content.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pants-plugins/pack_metadata/python_pack_content.py b/pants-plugins/pack_metadata/python_pack_content.py index c711acbd44..402a653706 100644 --- a/pants-plugins/pack_metadata/python_pack_content.py +++ b/pants-plugins/pack_metadata/python_pack_content.py @@ -11,13 +11,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import yaml from collections import defaultdict from dataclasses import dataclass from pathlib import PurePath from typing import DefaultDict -import yaml - from pants.backend.python.dependency_inference.module_mapper import module_from_stripped_path from pants.backend.python.subsystems.setup import PythonSetup from pants.backend.python.target_types import PythonResolveField, PythonSourceField From 3bd815a16e513650187e6d1ef51e65eef93c9faf Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 2 Aug 2024 12:23:51 -0500 Subject: [PATCH 58/98] pants-plugins/pack_metadata: Move some logic into PackContent dataclasses --- .../pack_metadata/python_module_mapper.py | 7 +- .../pack_metadata/python_pack_content.py | 73 +++++++++++-------- 2 files changed, 45 insertions(+), 35 deletions(-) diff --git a/pants-plugins/pack_metadata/python_module_mapper.py b/pants-plugins/pack_metadata/python_module_mapper.py index 14fa535a6d..c1c54cb97e 100644 --- a/pants-plugins/pack_metadata/python_module_mapper.py +++ b/pants-plugins/pack_metadata/python_module_mapper.py @@ -52,9 +52,10 @@ async def map_pack_content_to_python_modules( ) for pack_content in pack_content_python_entry_points: - resolves_to_modules_to_providers[pack_content.resolve][pack_content.module].append( - ModuleProvider(pack_content.python_address, ModuleProviderType.IMPL) - ) + for module in pack_content.get_possible_modules(): + resolves_to_modules_to_providers[pack_content.resolve][module].append( + ModuleProvider(pack_content.python_address, ModuleProviderType.IMPL) + ) for pack_lib in pack_python_libs: provider_type = ( diff --git a/pants-plugins/pack_metadata/python_pack_content.py b/pants-plugins/pack_metadata/python_pack_content.py index 402a653706..ff7651c596 100644 --- a/pants-plugins/pack_metadata/python_pack_content.py +++ b/pants-plugins/pack_metadata/python_pack_content.py @@ -72,34 +72,44 @@ class PackContentPythonEntryPoint: entry_point: str python_address: Address resolve: str - module: str + @property + def python_file_path(self) -> PurePath: + return PurePath(self.python_address.filename) -class PackContentPythonEntryPoints(Collection[PackContentPythonEntryPoint]): - pass + @staticmethod + def _split_pack_content_path(path: PurePath) -> tuple[PurePath, PurePath]: + content_types = ("actions", "sensors") # the only content_types with python content + pack_content_dir = path.parent + while pack_content_dir.name not in content_types: + pack_content_dir = pack_content_dir.parent + relative_to_pack_content_dir = path.relative_to(pack_content_dir) + return pack_content_dir, relative_to_pack_content_dir + def get_possible_modules(self) -> tuple[str, ...]: + """Get module names that could be imported. Mirrors get_possible_paths logic.""" + path = self.python_file_path -class PackContentPythonEntryPointsRequest: - pass + # st2 adds the parent dir of the python file to sys.path at runtime. + module = path.stem if path.suffix == ".py" else path.name + modules = [module] + # By convention, however, just actions/ is on sys.path during tests. + # so, also construct the module name from actions/ to support tests. + _, relative_to_pack_content_dir = self._split_pack_content_path(path) + module = module_from_stripped_path(relative_to_pack_content_dir) + if module not in modules: + modules.append(module) -def get_possible_modules(path: PurePath) -> list[str]: - if path.name in ("__init__.py", "__init__.pyi"): - path = path.parent - module = path.stem if path.suffix == ".py" else path.name - modules = [module] + return tuple(modules) + + +class PackContentPythonEntryPoints(Collection[PackContentPythonEntryPoint]): + pass - try: - start = path.parent.parts.index("actions") + 1 - except ValueError: - start = path.parent.parts.index("sensors") + 1 - # st2 adds the parent dir of the python file to sys.path at runtime. - # by convention, however, just actions/ is on sys.path during tests. - # so, also construct the module name from actions/ to support tests. - if start < len(path.parent.parts): - modules.append(".".join((*path.parent.parts[start:], module))) - return modules +class PackContentPythonEntryPointsRequest: + pass @rule(desc="Find all Pack Content entry_points that are python", level=LogLevel.DEBUG) @@ -170,15 +180,13 @@ async def find_pack_content_python_entry_points( for metadata_address, content_type, entry_point in pack_content_entry_points_by_spec[tgt.address.filename]: resolve = tgt[PythonResolveField].normalized_value(python_setup) - for module in get_possible_modules(PurePath(tgt.address.filename)): - pack_content_entry_points.append(PackContentPythonEntryPoint( - metadata_address=metadata_address, - content_type=content_type, - entry_point=entry_point, - python_address=tgt.address, - resolve=resolve, - module=module, - )) + pack_content_entry_points.append(PackContentPythonEntryPoint( + metadata_address=metadata_address, + content_type=content_type, + entry_point=entry_point, + python_address=tgt.address, + resolve=resolve, + )) return PackContentPythonEntryPoints(pack_content_entry_points) @@ -190,7 +198,10 @@ class PackPythonLib: relative_to_lib: PurePath python_address: Address resolve: str - module: str + + @property + def module(self) -> str: + return module_from_stripped_path(self.relative_to_lib) class PackPythonLibs(Collection[PackPythonLib]): @@ -252,7 +263,6 @@ async def find_python_in_pack_lib_directories( relative_to_lib = relative_to_pack.relative_to(lib_dir) resolve = tgt[PythonResolveField].normalized_value(python_setup) - module = module_from_stripped_path(relative_to_lib) pack_python_libs.append(PackPythonLib( pack_path=pack_path, @@ -260,7 +270,6 @@ async def find_python_in_pack_lib_directories( relative_to_lib=relative_to_lib, python_address=tgt.address, resolve=resolve, - module=module, )) return PackPythonLibs(pack_python_libs) From 31b7e6417315e0f242fca437261d9a207d021589 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 2 Aug 2024 12:36:17 -0500 Subject: [PATCH 59/98] pants-plugins/pack_metadata: Move python_pack_content into util_rules --- pants-plugins/pack_metadata/python_module_mapper.py | 2 +- pants-plugins/pack_metadata/register.py | 4 +++- pants-plugins/pack_metadata/util_rules/BUILD | 1 + pants-plugins/pack_metadata/util_rules/__init__.py | 0 .../pack_metadata/{ => util_rules}/python_pack_content.py | 0 5 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 pants-plugins/pack_metadata/util_rules/BUILD create mode 100644 pants-plugins/pack_metadata/util_rules/__init__.py rename pants-plugins/pack_metadata/{ => util_rules}/python_pack_content.py (100%) diff --git a/pants-plugins/pack_metadata/python_module_mapper.py b/pants-plugins/pack_metadata/python_module_mapper.py index c1c54cb97e..f2ed834b6a 100644 --- a/pants-plugins/pack_metadata/python_module_mapper.py +++ b/pants-plugins/pack_metadata/python_module_mapper.py @@ -25,7 +25,7 @@ from pants.engine.unions import UnionRule from pants.util.logging import LogLevel -from pack_metadata.python_pack_content import ( +from pack_metadata.util_rules.python_pack_content import ( PackContentPythonEntryPoints, PackContentPythonEntryPointsRequest, PackPythonLibs, diff --git a/pants-plugins/pack_metadata/register.py b/pants-plugins/pack_metadata/register.py index d48d70456c..fe4fbf4f8e 100644 --- a/pants-plugins/pack_metadata/register.py +++ b/pants-plugins/pack_metadata/register.py @@ -11,7 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from pack_metadata import python_pack_content, python_module_mapper, tailor, target_types_rules + +from pack_metadata import python_module_mapper, tailor, target_types_rules +from pack_metadata.util_rules import python_pack_content from pack_metadata.target_types import ( PackContentResourceTarget, PackMetadata, diff --git a/pants-plugins/pack_metadata/util_rules/BUILD b/pants-plugins/pack_metadata/util_rules/BUILD new file mode 100644 index 0000000000..db46e8d6c9 --- /dev/null +++ b/pants-plugins/pack_metadata/util_rules/BUILD @@ -0,0 +1 @@ +python_sources() diff --git a/pants-plugins/pack_metadata/util_rules/__init__.py b/pants-plugins/pack_metadata/util_rules/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/pants-plugins/pack_metadata/python_pack_content.py b/pants-plugins/pack_metadata/util_rules/python_pack_content.py similarity index 100% rename from pants-plugins/pack_metadata/python_pack_content.py rename to pants-plugins/pack_metadata/util_rules/python_pack_content.py From ffa1f5e12a5399fd57b5ff93ff8bfde04472b69a Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 2 Aug 2024 12:41:16 -0500 Subject: [PATCH 60/98] pants-plugins/pack_metadata: Add implementation notes for python_pack_content and related rules --- .../util_rules/python_pack_content.py | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/pants-plugins/pack_metadata/util_rules/python_pack_content.py b/pants-plugins/pack_metadata/util_rules/python_pack_content.py index ff7651c596..93c2e73556 100644 --- a/pants-plugins/pack_metadata/util_rules/python_pack_content.py +++ b/pants-plugins/pack_metadata/util_rules/python_pack_content.py @@ -45,6 +45,43 @@ ) +# Implementation Notes: +# +# With pants, we can rely on dependency inference for all the +# st2 components, runners, and other venv bits (st2 venv and pack venv). +# In ST2, all of that goes at the end of PYTHONPATH. +# +# Actions: +# At runtime, the python_runner creates a PYTHONPATH that includes: +# [pack/lib:]pack_venv/lib/python3.x:pack_venv/lib/python3.x/site-packages:pack/actions/lib:st2_pythonpath +# python_runner runs python_action_wrapper which: +# - injects the action's entry_point's directory in sys.path +# - and then imports the action module and runs it. +# +# Sensors: +# At runtime, ProcessSensorContainer creates PYTHONPATH that includes: +# [pack/lib:]st2_pythonpath +# Then the container runs the sensor via sensor_wrapper which +# process_container runs sensor_wrapper which: +# - injects the sensor's entry_point's directory in sys.path +# (effectively always "sensors/" as a split("/") assumes only one dir) +# - and then imports the class_name from sensor module and runs it. +# +# For actions, this pants plugin need to add this to PEX_EXTRA_SYS_PATH: +# pack/actions/path_to_entry_point:[pack/lib:]pack/actions/lib +# For sensors, this pants plugin need to add this to PEX_EXTRA_SYS_PATH: +# pack/sensors:[pack/lib:] +# +# This rules in this file are used by: +# python_module_mapper.py: +# Dependency inference uses pack_metadata's module_mapper to detect any +# python imports that require one of these PYTHONPATH modifications, +# resolving those imports to modules in lib/, actions/, or sensors/. +# python_path_rules.py: +# Then get the relevant python imports from dependencies and +# add their parent directory to a generated PEX_EXTRA_SYS_PATH. + + @dataclass(frozen=True) class PackContentResourceTargetsOfTypeRequest: types: tuple[PackContentResourceTypes, ...] From 3a82a29be4640c44af04be358fd1ae607611c805 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 2 Aug 2024 12:48:41 -0500 Subject: [PATCH 61/98] pants-plugins/pack_metadata: Add python_path_rules to generate PEX_EXTRA_SYS_PATH for tests This won't work until pants gets support for injecting path entries. --- .../pack_metadata/python_path_rules.py | 116 ++++++++++++++++++ pants-plugins/pack_metadata/register.py | 8 +- pants-plugins/pack_metadata/target_types.py | 13 +- .../util_rules/python_pack_content.py | 19 +++ 4 files changed, 154 insertions(+), 2 deletions(-) create mode 100644 pants-plugins/pack_metadata/python_path_rules.py diff --git a/pants-plugins/pack_metadata/python_path_rules.py b/pants-plugins/pack_metadata/python_path_rules.py new file mode 100644 index 0000000000..d0d903da81 --- /dev/null +++ b/pants-plugins/pack_metadata/python_path_rules.py @@ -0,0 +1,116 @@ +# Copyright 2024 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from typing import Set + +from pants.backend.python.goals.pytest_runner import ( + PytestPluginSetupRequest, + PytestPluginSetup, +) +from pants.engine.internals.native_engine import Address +from pants.engine.rules import collect_rules, Get, MultiGet, rule +from pants.engine.target import Target, TransitiveTargets, TransitiveTargetsRequest +from pants.engine.unions import UnionRule +from pants.util.ordered_set import OrderedSet + +from pack_metadata.util_rules.python_pack_content import ( + PackContentPythonEntryPoints, + PackContentPythonEntryPointsRequest, + PackPythonLibs, + PackPythonLibsRequest, +) +from pack_metadata.target_types import InjectPackPythonPathField + + +@dataclass +class PackPythonPath: + entries: tuple[str, ...] = () + + +@dataclass +class PackPythonPathRequest: + address: Address + + +@rule +async def get_extra_sys_path_for_pack_dependencies(request: PackPythonPathRequest) -> PackPythonPath: + transitive_targets = await Get(TransitiveTargets, TransitiveTargetsRequest((request.address,))) + + dependency_addresses: Set[Address] = { + tgt.address for tgt in transitive_targets.closure + } + if not dependency_addresses: + return PackPythonPath() + + pack_content_python_entry_points, pack_python_libs = await MultiGet( + Get(PackContentPythonEntryPoints, PackContentPythonEntryPointsRequest()), + Get(PackPythonLibs, PackPythonLibsRequest()), + ) + + # only use addresses of actual dependencies + pack_python_content_addresses: Set[Address] = dependency_addresses & { + pack_content.python_address for pack_content in pack_content_python_entry_points + } + pack_python_lib_addresses: Set[Address] = dependency_addresses & { + pack_lib.python_address for pack_lib in pack_python_libs + } + + if not (pack_python_content_addresses or pack_python_lib_addresses): + return PackPythonPath() + + # filter pack_content_python_entry_points and pack_python_libs + pack_content_python_entry_points = ( + pack_content for pack_content in pack_content_python_entry_points + if pack_content.python_address in pack_python_content_addresses + ) + pack_python_libs = ( + pack_lib for pack_lib in pack_python_libs + if pack_lib.python_address in pack_python_lib_addresses + ) + + extra_sys_path_entries = OrderedSet() + for pack_content in pack_content_python_entry_points: + for path_entry in pack_content.get_possible_paths(): + extra_sys_path_entries.add(path_entry) + for pack_lib in pack_python_libs: + extra_sys_path_entries.add(pack_lib.lib_path.as_posix()) + + return PackPythonPath(tuple(extra_sys_path_entries)) + + +class PytestPackTestRequest(PytestPluginSetupRequest): + @classmethod + def is_applicable(cls, target: Target) -> bool: + if not target.has_field(InjectPackPythonPathField): + return False + return bool(target.get(InjectPackPythonPathField).value) + + +@rule +async def inject_extra_sys_path_for_pack_tests( + request: PytestPackTestRequest, +) -> PytestPluginSetup: + pack_python_path = await Get(PackPythonPath, PackPythonPathRequest(request.target.address)) + return PytestPluginSetup( + # digest=EMPTY_DIGEST, + # extra_sys_path=pack_python_path, # TODO: make pants support this + ) + + +def rules(): + return [ + *collect_rules(), + UnionRule(PytestPluginSetupRequest, PytestPackTestRequest), + ] diff --git a/pants-plugins/pack_metadata/register.py b/pants-plugins/pack_metadata/register.py index fe4fbf4f8e..e324e8c033 100644 --- a/pants-plugins/pack_metadata/register.py +++ b/pants-plugins/pack_metadata/register.py @@ -12,9 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -from pack_metadata import python_module_mapper, tailor, target_types_rules +from pants.backend.python.target_types import PythonTestTarget, PythonTestsGeneratorTarget + +from pack_metadata import python_module_mapper, python_path_rules, tailor, target_types_rules from pack_metadata.util_rules import python_pack_content from pack_metadata.target_types import ( + InjectPackPythonPathField, PackContentResourceTarget, PackMetadata, PackMetadataInGitSubmodule, @@ -24,10 +27,13 @@ def rules(): return [ + PythonTestsGeneratorTarget.register_plugin_field(InjectPackPythonPathField, as_moved_field=True), + PythonTestTarget.register_plugin_field(InjectPackPythonPathField), *tailor.rules(), *target_types_rules.rules(), *python_pack_content.rules(), *python_module_mapper.rules(), + *python_path_rules.rules(), ] diff --git a/pants-plugins/pack_metadata/target_types.py b/pants-plugins/pack_metadata/target_types.py index 0e405770ea..ca019a4f55 100644 --- a/pants-plugins/pack_metadata/target_types.py +++ b/pants-plugins/pack_metadata/target_types.py @@ -16,7 +16,7 @@ from typing import Optional, Sequence, Tuple from pants.engine.internals.native_engine import Address -from pants.engine.target import COMMON_TARGET_FIELDS, Dependencies, StringField +from pants.engine.target import BoolField, COMMON_TARGET_FIELDS, Dependencies, StringField from pants.core.target_types import ( ResourceDependenciesField, ResourcesGeneratingSourcesField, @@ -189,3 +189,14 @@ class PacksGlob(GenericTarget): "subdirectories (packs) except those listed with ! in dependencies. " "This is unfortunately needed by tests that use a glob to load pack fixtures." ) + + +class InjectPackPythonPathField(BoolField): + alias = "inject_pack_python_path" + help = ( + "For pack tests, set this to true to make sure /lib or actions/ dirs get " + "added to PYTHONPATH (actually PEX_EXTRA_SYS_PATH). Use `__defaults__` to enable " + "this in the BUILD file where you define pack_metadata, like this: " + "`__defaults__(all=dict(inject_pack_python_path=True))`" + ) + default = False diff --git a/pants-plugins/pack_metadata/util_rules/python_pack_content.py b/pants-plugins/pack_metadata/util_rules/python_pack_content.py index 93c2e73556..ef83d57ada 100644 --- a/pants-plugins/pack_metadata/util_rules/python_pack_content.py +++ b/pants-plugins/pack_metadata/util_rules/python_pack_content.py @@ -140,6 +140,21 @@ def get_possible_modules(self) -> tuple[str, ...]: return tuple(modules) + def get_possible_paths(self) -> tuple[str, ...]: + """Get paths to add to PYTHONPATH and PEX_EXTRA_SYS_PATH. Mirrors get_possible_modules logic.""" + path = self.python_file_path + + # st2 adds the parent dir of the python file to sys.path at runtime. + paths = [path.parent.as_posix()] + + # By convention, however, just actions/ is on sys.path during tests. + # so, also construct the module name from actions/ to support tests. + pack_content_dir, _ = self._split_pack_content_path(path) + if path.parent != pack_content_dir: + paths.append(pack_content_dir.as_posix()) + + return tuple(paths) + class PackContentPythonEntryPoints(Collection[PackContentPythonEntryPoint]): pass @@ -240,6 +255,10 @@ class PackPythonLib: def module(self) -> str: return module_from_stripped_path(self.relative_to_lib) + @property + def lib_path(self) -> PurePath: + return self.pack_path / self.lib_dir + class PackPythonLibs(Collection[PackPythonLib]): pass From df71152aa0c55c914a245472f65047e7bae392e6 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 2 Aug 2024 13:17:48 -0500 Subject: [PATCH 62/98] pants: enable pack python injection for pack tests --- contrib/core/BUILD | 2 ++ contrib/debug/BUILD | 2 ++ contrib/default/BUILD | 2 ++ contrib/examples/BUILD | 2 ++ contrib/hello_st2/BUILD | 2 ++ contrib/linux/BUILD | 2 ++ contrib/packs/BUILD | 2 ++ 7 files changed, 14 insertions(+) diff --git a/contrib/core/BUILD b/contrib/core/BUILD index 59673bd746..9df7a372c9 100644 --- a/contrib/core/BUILD +++ b/contrib/core/BUILD @@ -1,3 +1,5 @@ +__defaults__(all=dict(inject_pack_python_path=True)) + pack_metadata( name="metadata", ) diff --git a/contrib/debug/BUILD b/contrib/debug/BUILD index 1a74d30186..888be3a426 100644 --- a/contrib/debug/BUILD +++ b/contrib/debug/BUILD @@ -1,3 +1,5 @@ +__defaults__(all=dict(inject_pack_python_path=True)) + pack_metadata( name="metadata", ) diff --git a/contrib/default/BUILD b/contrib/default/BUILD index 1a74d30186..888be3a426 100644 --- a/contrib/default/BUILD +++ b/contrib/default/BUILD @@ -1,3 +1,5 @@ +__defaults__(all=dict(inject_pack_python_path=True)) + pack_metadata( name="metadata", ) diff --git a/contrib/examples/BUILD b/contrib/examples/BUILD index de3b866405..ab10cd1c85 100644 --- a/contrib/examples/BUILD +++ b/contrib/examples/BUILD @@ -1,3 +1,5 @@ +__defaults__(all=dict(inject_pack_python_path=True)) + pack_metadata( name="metadata", ) diff --git a/contrib/hello_st2/BUILD b/contrib/hello_st2/BUILD index 1a74d30186..888be3a426 100644 --- a/contrib/hello_st2/BUILD +++ b/contrib/hello_st2/BUILD @@ -1,3 +1,5 @@ +__defaults__(all=dict(inject_pack_python_path=True)) + pack_metadata( name="metadata", ) diff --git a/contrib/linux/BUILD b/contrib/linux/BUILD index 8a73ff391a..201435eecc 100644 --- a/contrib/linux/BUILD +++ b/contrib/linux/BUILD @@ -1,3 +1,5 @@ +__defaults__(all=dict(inject_pack_python_path=True)) + pack_metadata( name="metadata", ) diff --git a/contrib/packs/BUILD b/contrib/packs/BUILD index 1a74d30186..888be3a426 100644 --- a/contrib/packs/BUILD +++ b/contrib/packs/BUILD @@ -1,3 +1,5 @@ +__defaults__(all=dict(inject_pack_python_path=True)) + pack_metadata( name="metadata", ) From bf34e01978395e56d23e2fc5265c092625204751 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 2 Aug 2024 13:46:37 -0500 Subject: [PATCH 63/98] pants-plugins/uses_services: switch python_targets(uses=...) to a moved field --- pants-plugins/uses_services/register.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pants-plugins/uses_services/register.py b/pants-plugins/uses_services/register.py index 83ab32b3a7..56b4d7ff6c 100644 --- a/pants-plugins/uses_services/register.py +++ b/pants-plugins/uses_services/register.py @@ -22,7 +22,7 @@ def rules(): return [ - PythonTestsGeneratorTarget.register_plugin_field(UsesServicesField), + PythonTestsGeneratorTarget.register_plugin_field(UsesServicesField, as_moved_field=True), PythonTestTarget.register_plugin_field(UsesServicesField), *platform_rules.rules(), *mongo_rules.rules(), From 4a4dcbc0312136d4e9b4015e9332462ce03f3716 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 2 Aug 2024 13:48:47 -0500 Subject: [PATCH 64/98] pants-plugins/pack_metadata: mark some dataclasses frozen to satisfy pants reqs --- pants-plugins/pack_metadata/python_path_rules.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pants-plugins/pack_metadata/python_path_rules.py b/pants-plugins/pack_metadata/python_path_rules.py index d0d903da81..4c2d8f150f 100644 --- a/pants-plugins/pack_metadata/python_path_rules.py +++ b/pants-plugins/pack_metadata/python_path_rules.py @@ -34,12 +34,12 @@ from pack_metadata.target_types import InjectPackPythonPathField -@dataclass +@dataclass(frozen=True) class PackPythonPath: entries: tuple[str, ...] = () -@dataclass +@dataclass(frozen=True) class PackPythonPathRequest: address: Address From 8354129e4bdf6bb72477c5f24443a7f1bcddd437 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 2 Aug 2024 14:41:46 -0500 Subject: [PATCH 65/98] pants-plugins/pack_metadata: stub tests for new rules --- .../python_module_mapper_test.py | 16 +++++++++++++ .../pack_metadata/python_path_rules_test.py | 20 ++++++++++++++++ pants-plugins/pack_metadata/util_rules/BUILD | 4 ++++ .../util_rules/python_pack_content_test.py | 24 +++++++++++++++++++ 4 files changed, 64 insertions(+) create mode 100644 pants-plugins/pack_metadata/python_module_mapper_test.py create mode 100644 pants-plugins/pack_metadata/python_path_rules_test.py create mode 100644 pants-plugins/pack_metadata/util_rules/python_pack_content_test.py diff --git a/pants-plugins/pack_metadata/python_module_mapper_test.py b/pants-plugins/pack_metadata/python_module_mapper_test.py new file mode 100644 index 0000000000..76e35800aa --- /dev/null +++ b/pants-plugins/pack_metadata/python_module_mapper_test.py @@ -0,0 +1,16 @@ +# Copyright 2024 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +def test_map_pack_content_to_python_modules() -> None: + pass diff --git a/pants-plugins/pack_metadata/python_path_rules_test.py b/pants-plugins/pack_metadata/python_path_rules_test.py new file mode 100644 index 0000000000..5d7febd6b6 --- /dev/null +++ b/pants-plugins/pack_metadata/python_path_rules_test.py @@ -0,0 +1,20 @@ +# Copyright 2024 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +def test_get_extra_sys_path_for_pack_dependencies() -> None: + pass + + +def test_inject_extra_sys_path_for_pack_tests() -> None: + pass diff --git a/pants-plugins/pack_metadata/util_rules/BUILD b/pants-plugins/pack_metadata/util_rules/BUILD index db46e8d6c9..0eea8b1cf1 100644 --- a/pants-plugins/pack_metadata/util_rules/BUILD +++ b/pants-plugins/pack_metadata/util_rules/BUILD @@ -1 +1,5 @@ python_sources() + +python_tests( + name="tests", +) diff --git a/pants-plugins/pack_metadata/util_rules/python_pack_content_test.py b/pants-plugins/pack_metadata/util_rules/python_pack_content_test.py new file mode 100644 index 0000000000..70c09e32ef --- /dev/null +++ b/pants-plugins/pack_metadata/util_rules/python_pack_content_test.py @@ -0,0 +1,24 @@ +# Copyright 2024 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +def test_find_pack_metadata_targets_of_types() -> None: + pass + + +def test_find_pack_content_python_entry_points() -> None: + pass + + +def test_find_python_in_pack_lib_directories() -> None: + pass From fb4861259e5267885211b39fb2b775b42b3f3cdb Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 2 Aug 2024 14:46:33 -0500 Subject: [PATCH 66/98] pants-plugins/pack_metadata: Add description to new rules --- pants-plugins/pack_metadata/python_module_mapper.py | 3 ++- pants-plugins/pack_metadata/python_path_rules.py | 8 ++++++-- .../pack_metadata/util_rules/python_pack_content.py | 3 ++- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/pants-plugins/pack_metadata/python_module_mapper.py b/pants-plugins/pack_metadata/python_module_mapper.py index f2ed834b6a..66147817ed 100644 --- a/pants-plugins/pack_metadata/python_module_mapper.py +++ b/pants-plugins/pack_metadata/python_module_mapper.py @@ -25,6 +25,7 @@ from pants.engine.unions import UnionRule from pants.util.logging import LogLevel +from pack_metadata.target_types import PackMetadata from pack_metadata.util_rules.python_pack_content import ( PackContentPythonEntryPoints, PackContentPythonEntryPointsRequest, @@ -38,7 +39,7 @@ class St2PythonPackContentMappingMarker(FirstPartyPythonMappingImplMarker): pass -@rule(desc="Creating map of pack_metadata targets to Python modules in pack content", level=LogLevel.DEBUG) +@rule(desc=f"Creating map of `{PackMetadata.alias}` targets to Python modules in pack content", level=LogLevel.DEBUG) async def map_pack_content_to_python_modules( _: St2PythonPackContentMappingMarker, ) -> FirstPartyPythonMappingImpl: diff --git a/pants-plugins/pack_metadata/python_path_rules.py b/pants-plugins/pack_metadata/python_path_rules.py index 4c2d8f150f..44843aa5e9 100644 --- a/pants-plugins/pack_metadata/python_path_rules.py +++ b/pants-plugins/pack_metadata/python_path_rules.py @@ -23,6 +23,7 @@ from pants.engine.rules import collect_rules, Get, MultiGet, rule from pants.engine.target import Target, TransitiveTargets, TransitiveTargetsRequest from pants.engine.unions import UnionRule +from pants.util.logging import LogLevel from pants.util.ordered_set import OrderedSet from pack_metadata.util_rules.python_pack_content import ( @@ -44,7 +45,10 @@ class PackPythonPathRequest: address: Address -@rule +@rule( + desc="Get pack paths that should be added to PYTHONPATH/PEX_EXTRA_SYS_PATH for a target.", + level=LogLevel.DEBUG +) async def get_extra_sys_path_for_pack_dependencies(request: PackPythonPathRequest) -> PackPythonPath: transitive_targets = await Get(TransitiveTargets, TransitiveTargetsRequest((request.address,))) @@ -98,7 +102,7 @@ def is_applicable(cls, target: Target) -> bool: return bool(target.get(InjectPackPythonPathField).value) -@rule +@rule(desc="Inject pack paths in PYTHONPATH/PEX_EXTRA_SYS_PATH for python tests.", level=LogLevel.DEBUG) async def inject_extra_sys_path_for_pack_tests( request: PytestPackTestRequest, ) -> PytestPluginSetup: diff --git a/pants-plugins/pack_metadata/util_rules/python_pack_content.py b/pants-plugins/pack_metadata/util_rules/python_pack_content.py index ef83d57ada..c67a3f9c35 100644 --- a/pants-plugins/pack_metadata/util_rules/python_pack_content.py +++ b/pants-plugins/pack_metadata/util_rules/python_pack_content.py @@ -41,6 +41,7 @@ PackContentResourceSourceField, PackContentResourceTypeField, PackContentResourceTypes, + PackMetadata, PackMetadataSourcesField, ) @@ -91,7 +92,7 @@ class PackContentResourceTargetsOfType(Targets): pass -@rule(desc="Find all PackMetadata targets in project filtered by content type", level=LogLevel.DEBUG) +@rule(desc=f"Find all `{PackMetadata.alias}` targets in project filtered by content type", level=LogLevel.DEBUG) async def find_pack_metadata_targets_of_types( request: PackContentResourceTargetsOfTypeRequest, targets: AllTargets ) -> PackContentResourceTargetsOfType: From 09af97d041e8037f75a2fa516095b3b66c645999 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 2 Aug 2024 15:45:33 -0500 Subject: [PATCH 67/98] pants-plugins/pack_metadata: rename util_rules to python_rules --- pants-plugins/pack_metadata/python_module_mapper.py | 2 +- pants-plugins/pack_metadata/python_path_rules.py | 2 +- pants-plugins/pack_metadata/{util_rules => python_rules}/BUILD | 0 .../pack_metadata/{util_rules => python_rules}/__init__.py | 0 .../{util_rules => python_rules}/python_pack_content.py | 0 .../{util_rules => python_rules}/python_pack_content_test.py | 0 pants-plugins/pack_metadata/register.py | 2 +- 7 files changed, 3 insertions(+), 3 deletions(-) rename pants-plugins/pack_metadata/{util_rules => python_rules}/BUILD (100%) rename pants-plugins/pack_metadata/{util_rules => python_rules}/__init__.py (100%) rename pants-plugins/pack_metadata/{util_rules => python_rules}/python_pack_content.py (100%) rename pants-plugins/pack_metadata/{util_rules => python_rules}/python_pack_content_test.py (100%) diff --git a/pants-plugins/pack_metadata/python_module_mapper.py b/pants-plugins/pack_metadata/python_module_mapper.py index 66147817ed..c7e46e2f23 100644 --- a/pants-plugins/pack_metadata/python_module_mapper.py +++ b/pants-plugins/pack_metadata/python_module_mapper.py @@ -26,7 +26,7 @@ from pants.util.logging import LogLevel from pack_metadata.target_types import PackMetadata -from pack_metadata.util_rules.python_pack_content import ( +from pack_metadata.python_rules.python_pack_content import ( PackContentPythonEntryPoints, PackContentPythonEntryPointsRequest, PackPythonLibs, diff --git a/pants-plugins/pack_metadata/python_path_rules.py b/pants-plugins/pack_metadata/python_path_rules.py index 44843aa5e9..49406c39a2 100644 --- a/pants-plugins/pack_metadata/python_path_rules.py +++ b/pants-plugins/pack_metadata/python_path_rules.py @@ -26,7 +26,7 @@ from pants.util.logging import LogLevel from pants.util.ordered_set import OrderedSet -from pack_metadata.util_rules.python_pack_content import ( +from pack_metadata.python_rules.python_pack_content import ( PackContentPythonEntryPoints, PackContentPythonEntryPointsRequest, PackPythonLibs, diff --git a/pants-plugins/pack_metadata/util_rules/BUILD b/pants-plugins/pack_metadata/python_rules/BUILD similarity index 100% rename from pants-plugins/pack_metadata/util_rules/BUILD rename to pants-plugins/pack_metadata/python_rules/BUILD diff --git a/pants-plugins/pack_metadata/util_rules/__init__.py b/pants-plugins/pack_metadata/python_rules/__init__.py similarity index 100% rename from pants-plugins/pack_metadata/util_rules/__init__.py rename to pants-plugins/pack_metadata/python_rules/__init__.py diff --git a/pants-plugins/pack_metadata/util_rules/python_pack_content.py b/pants-plugins/pack_metadata/python_rules/python_pack_content.py similarity index 100% rename from pants-plugins/pack_metadata/util_rules/python_pack_content.py rename to pants-plugins/pack_metadata/python_rules/python_pack_content.py diff --git a/pants-plugins/pack_metadata/util_rules/python_pack_content_test.py b/pants-plugins/pack_metadata/python_rules/python_pack_content_test.py similarity index 100% rename from pants-plugins/pack_metadata/util_rules/python_pack_content_test.py rename to pants-plugins/pack_metadata/python_rules/python_pack_content_test.py diff --git a/pants-plugins/pack_metadata/register.py b/pants-plugins/pack_metadata/register.py index e324e8c033..a8dd7f89cb 100644 --- a/pants-plugins/pack_metadata/register.py +++ b/pants-plugins/pack_metadata/register.py @@ -15,7 +15,7 @@ from pants.backend.python.target_types import PythonTestTarget, PythonTestsGeneratorTarget from pack_metadata import python_module_mapper, python_path_rules, tailor, target_types_rules -from pack_metadata.util_rules import python_pack_content +from pack_metadata.python_rules import python_pack_content from pack_metadata.target_types import ( InjectPackPythonPathField, PackContentResourceTarget, From 1de71ae00d8747dd2742512138d6db917273e4db Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 2 Aug 2024 15:51:46 -0500 Subject: [PATCH 68/98] pants-plugins/pack_metadata: move python_* to python_rules --- .../pack_metadata/{ => python_rules}/python_module_mapper.py | 0 .../{ => python_rules}/python_module_mapper_test.py | 0 .../pack_metadata/{ => python_rules}/python_path_rules.py | 0 .../{ => python_rules}/python_path_rules_test.py | 0 pants-plugins/pack_metadata/register.py | 4 ++-- 5 files changed, 2 insertions(+), 2 deletions(-) rename pants-plugins/pack_metadata/{ => python_rules}/python_module_mapper.py (100%) rename pants-plugins/pack_metadata/{ => python_rules}/python_module_mapper_test.py (100%) rename pants-plugins/pack_metadata/{ => python_rules}/python_path_rules.py (100%) rename pants-plugins/pack_metadata/{ => python_rules}/python_path_rules_test.py (100%) diff --git a/pants-plugins/pack_metadata/python_module_mapper.py b/pants-plugins/pack_metadata/python_rules/python_module_mapper.py similarity index 100% rename from pants-plugins/pack_metadata/python_module_mapper.py rename to pants-plugins/pack_metadata/python_rules/python_module_mapper.py diff --git a/pants-plugins/pack_metadata/python_module_mapper_test.py b/pants-plugins/pack_metadata/python_rules/python_module_mapper_test.py similarity index 100% rename from pants-plugins/pack_metadata/python_module_mapper_test.py rename to pants-plugins/pack_metadata/python_rules/python_module_mapper_test.py diff --git a/pants-plugins/pack_metadata/python_path_rules.py b/pants-plugins/pack_metadata/python_rules/python_path_rules.py similarity index 100% rename from pants-plugins/pack_metadata/python_path_rules.py rename to pants-plugins/pack_metadata/python_rules/python_path_rules.py diff --git a/pants-plugins/pack_metadata/python_path_rules_test.py b/pants-plugins/pack_metadata/python_rules/python_path_rules_test.py similarity index 100% rename from pants-plugins/pack_metadata/python_path_rules_test.py rename to pants-plugins/pack_metadata/python_rules/python_path_rules_test.py diff --git a/pants-plugins/pack_metadata/register.py b/pants-plugins/pack_metadata/register.py index a8dd7f89cb..9ea3ad5aa2 100644 --- a/pants-plugins/pack_metadata/register.py +++ b/pants-plugins/pack_metadata/register.py @@ -14,8 +14,8 @@ from pants.backend.python.target_types import PythonTestTarget, PythonTestsGeneratorTarget -from pack_metadata import python_module_mapper, python_path_rules, tailor, target_types_rules -from pack_metadata.python_rules import python_pack_content +from pack_metadata import tailor, target_types_rules +from pack_metadata.python_rules import python_module_mapper, python_pack_content, python_path_rules from pack_metadata.target_types import ( InjectPackPythonPathField, PackContentResourceTarget, From 5494a2a7c3cd3eec017ecaf88d2fbf26cfdc535f Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 2 Aug 2024 18:29:15 -0500 Subject: [PATCH 69/98] pants-plugins/pack_metadata: add python_rules.conftest.rule_runner fixture Writes various test scenarios in the test sandbox. This should avoid repeating the setup in every test. --- .../pack_metadata/python_rules/BUILD | 4 + .../pack_metadata/python_rules/conftest.py | 249 ++++++++++++++++++ .../python_rules/python_module_mapper_test.py | 5 +- .../python_rules/python_pack_content_test.py | 9 +- .../python_rules/python_path_rules_test.py | 7 +- 5 files changed, 268 insertions(+), 6 deletions(-) create mode 100644 pants-plugins/pack_metadata/python_rules/conftest.py diff --git a/pants-plugins/pack_metadata/python_rules/BUILD b/pants-plugins/pack_metadata/python_rules/BUILD index 0eea8b1cf1..a172051977 100644 --- a/pants-plugins/pack_metadata/python_rules/BUILD +++ b/pants-plugins/pack_metadata/python_rules/BUILD @@ -3,3 +3,7 @@ python_sources() python_tests( name="tests", ) + +python_test_utils( + name="test_utils", +) diff --git a/pants-plugins/pack_metadata/python_rules/conftest.py b/pants-plugins/pack_metadata/python_rules/conftest.py new file mode 100644 index 0000000000..782d709412 --- /dev/null +++ b/pants-plugins/pack_metadata/python_rules/conftest.py @@ -0,0 +1,249 @@ +# Copyright 2024 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from textwrap import dedent + +import pytest +from pants.backend.python.dependency_inference.module_mapper import FirstPartyPythonMappingImpl +from pants.backend.python.goals.pytest_runner import PytestPluginSetup +from pants.backend.python.target_types import ( + PythonSourceTarget, + PythonSourcesGeneratorTarget, + PythonTestTarget, + PythonTestsGeneratorTarget, +) +from pants.backend.python.target_types_rules import rules as python_target_types_rules +from pants.engine.rules import QueryRule +from pants.testutil.python_rule_runner import PythonRuleRunner +from pants.testutil.rule_runner import RuleRunner + +from pack_metadata.python_rules import python_module_mapper, python_pack_content, python_path_rules +from pack_metadata.python_rules.python_module_mapper import St2PythonPackContentMappingMarker +from pack_metadata.python_rules.python_pack_content import ( + PackContentPythonEntryPoints, + PackContentPythonEntryPointsRequest, + PackContentResourceTargetsOfType, + PackContentResourceTargetsOfTypeRequest, + PackPythonLibs, + PackPythonLibsRequest +) +from pack_metadata.python_rules.python_path_rules import PackPythonPath, PackPythonPathRequest, PytestPackTestRequest +from pack_metadata.target_types import InjectPackPythonPathField, PackContentResourceTarget, PackMetadata + +# some random pack names +packs = ( + "foo", # imports between actions + "dr_seuss", # imports from /actions/lib + "shards", # imports from /lib + "metals", # imports the action from a subdirectory +) + + +@pytest.fixture +def pack_names() -> tuple[str, ...]: + return packs + + +def write_test_files(rule_runner: RuleRunner): + for pack in packs: + rule_runner.write_files( + { + f"packs/{pack}/BUILD": dedent( + f""" + __defaults__(all=dict(inject_pack_python_path=True)) + pack_metadata(name="metadata") + """ + ), + f"packs/{pack}/pack.yaml": dedent( + f""" + --- + name: {pack} + version: 1.0.0 + author: StackStorm + email: info@stackstorm.com + """ + ), + f"packs/{pack}/config.schema.yaml": "", + f"packs/{pack}/{pack}.yaml.example": "", + f"packs/{pack}/icon.png": "", + f"packs/{pack}/README.md": f"# Pack {pack} README", + } + ) + + def action_metadata_file(action: str, entry_point: str = "") -> str: + entry_point = entry_point or f"{action}.py" + return dedent( + f""" + --- + name: {action} + runner_type: python-script + entry_point: {entry_point} + """ + ) + + def test_file(module: str, _object: str) -> str: + return dedent( + f""" + from {module} import {_object} + def test_{module.replace(".", "_")}() -> None: + pass + """ + ) + + rule_runner.write_files( + { + "packs/foo/actions/BUILD": "python_sources()", + "packs/foo/actions/get_bar.yaml": action_metadata_file("get_bar"), + "packs/foo/actions/get_bar.py": dedent( + """ + RESPONSE_CONSTANT = "foobar_key" + class BarAction: + def run(self): + return {RESPONSE_CONSTANT: "bar"} + """ + ), + "packs/foo/actions/get_baz.yaml": action_metadata_file("get_baz"), + "packs/foo/actions/get_baz.py": dedent( + """ + from get_bar import RESPONSE_CONSTANT + class BazAction: + def run(self): + return {RESPONSE_CONSTANT: "baz"} + """ + ), + "packs/foo/tests/BUILD": "python_tests()", + "packs/foo/tests/test_get_bar_action.py": test_file("get_bar", "BarAction"), + "packs/foo/tests/test_get_baz_action.py": test_file("get_baz", "BazAction"), + "packs/dr_seuss/actions/lib/BUILD": "python_sources()", + "packs/dr_seuss/actions/lib/seuss/__init__.py": "", + "packs/dr_seuss/actions/lib/seuss/things.py": dedent( + """ + THING1 = "thing one" + THING2 = "thing two" + """ + ), + "packs/dr_seuss/actions/BUILD": "python_sources()", + "packs/dr_seuss/actions/get_from_actions_lib.yaml": action_metadata_file("get_from_actions_lib"), + "packs/dr_seuss/actions/get_from_actions_lib.py": dedent( + """ + from seuss.things import THING1, THING2 + class GetFromActionsLibAction: + def run(self): + return {"things": (THING1, THING2)} + """ + ), + "packs/dr_seuss/tests/BUILD": "python_tests()", + "packs/dr_seuss/tests/test_get_from_actions_lib_action.py": test_file( + "get_from_actions_lib", "GetFromActionsLibAction" + ), + "packs/shards/lib/stormlight_archive/BUILD": "python_sources()", + "packs/shards/lib/stormlight_archive/__init__.py": "", + "packs/shards/lib/stormlight_archive/things.py": dedent( + """ + STORM_LIGHT = "Honor" + VOID_LIGHT = "Odium" + LIFE_LIGHT = "Cultivation" + """ + ), + "packs/shards/actions/BUILD": "python_sources()", + "packs/shards/actions/get_from_pack_lib.yaml": action_metadata_file("get_from_pack_lib"), + "packs/shards/actions/get_from_pack_lib.py": dedent( + """ + from stormlight_archive.things import STORM_LIGHT, VOID_LIGHT, LIFE_LIGHT + class GetFromPackLibAction: + def run(self): + return {"light_sources": (STORM_LIGHT, VOID_LIGHT, LIFE_LIGHT)} + """ + ), + "packs/shards/sensors/BUILD": "python_sources()", + "packs/shards/sensors/horn_eater.yaml": dedent( + """ + --- + name: horn_eater + entry_point: horn_eater.py + class_name: HornEaterSensor + trigger_types: [{name: horn_eater.saw.spren, payload_schema: {type: object}}] + """ + ), + "packs/shards/sensors/horn_eater.py": dedent( + """ + from st2reactor.sensor.base import PollingSensor + from stormlight_archive.things import STORM_LIGHT + class HornEaterSensor(PollingSensor): + def setup(self): pass + def poll(self): + if STORM_LIGHT in self.config: + self.sensor_service.dispatch( + trigger="horn_eater.saw.spren", payload={"spren_type": STORM_LIGHT} + ) + def cleanup(self): pass + def add_trigger(self): pass + def update_trigger(self): pass + def remove_trigger(self): pass + """ + ), + "packs/shards/tests/BUILD": "python_tests()", + "packs/shards/tests/test_get_from_pack_lib_action.py": test_file( + "get_from_pack_lib", "GetFromPackLibAction" + ), + "packs/shards/tests/test_horn_eater_sensor.py": test_file("horn_eater", "HornEaterSensor"), + "packs/metals/actions/fly.yaml": action_metadata_file("fly", "mist_born/fly.py"), + "packs/metals/actions/mist_born/BUILD": "python_sources()", + "packs/metals/actions/mist_born/__init__.py": "", + "packs/metals/actions/mist_born/fly.py": dedent( + """ + class FlyAction: + def run(self): + return {"metals": ("steel", "iron")} + """ + ), + "packs/metals/tests/BUILD": "python_tests()", + "packs/metals/tests/test_fly_action.py": test_file("mist_born.fly", "FlyAction"), + } + ) + + +@pytest.fixture +def rule_runner() -> RuleRunner: + rule_runner = PythonRuleRunner( + rules=[ + PythonTestsGeneratorTarget.register_plugin_field(InjectPackPythonPathField, as_moved_field=True), + PythonTestTarget.register_plugin_field(InjectPackPythonPathField), + *python_target_types_rules(), + # TODO: not sure if we need a QueryRule for every rule... + *python_pack_content.rules(), + QueryRule(PackContentResourceTargetsOfType, (PackContentResourceTargetsOfTypeRequest,)), + QueryRule(PackContentPythonEntryPoints, (PackContentPythonEntryPointsRequest,)), + QueryRule(PackPythonLibs, (PackPythonLibsRequest,)), + *python_module_mapper.rules(), + QueryRule(FirstPartyPythonMappingImpl, (St2PythonPackContentMappingMarker,)), + *python_path_rules.rules(), + QueryRule(PackPythonPath, (PackPythonPathRequest,)), + QueryRule(PytestPluginSetup, (PytestPackTestRequest,)), + ], + target_types=[ + PackContentResourceTarget, + PackMetadata, + PythonSourceTarget, + PythonSourcesGeneratorTarget, + PythonTestTarget, + PythonTestsGeneratorTarget, + ], + ) + write_test_files(rule_runner) + args = [ + "--source-root-patterns=packs/*" + ] + rule_runner.set_options(args, env_inherit={"PATH", "PYENV_ROOT", "HOME"}) + return rule_runner diff --git a/pants-plugins/pack_metadata/python_rules/python_module_mapper_test.py b/pants-plugins/pack_metadata/python_rules/python_module_mapper_test.py index 76e35800aa..0bf6854651 100644 --- a/pants-plugins/pack_metadata/python_rules/python_module_mapper_test.py +++ b/pants-plugins/pack_metadata/python_rules/python_module_mapper_test.py @@ -12,5 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -def test_map_pack_content_to_python_modules() -> None: +from pants.testutil.rule_runner import RuleRunner + + +def test_map_pack_content_to_python_modules(rule_runner: RuleRunner) -> None: pass diff --git a/pants-plugins/pack_metadata/python_rules/python_pack_content_test.py b/pants-plugins/pack_metadata/python_rules/python_pack_content_test.py index 70c09e32ef..9f7006e225 100644 --- a/pants-plugins/pack_metadata/python_rules/python_pack_content_test.py +++ b/pants-plugins/pack_metadata/python_rules/python_pack_content_test.py @@ -12,13 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -def test_find_pack_metadata_targets_of_types() -> None: +from pants.testutil.rule_runner import RuleRunner + + +def test_find_pack_metadata_targets_of_types(rule_runner: RuleRunner) -> None: pass -def test_find_pack_content_python_entry_points() -> None: +def test_find_pack_content_python_entry_points(rule_runner: RuleRunner) -> None: pass -def test_find_python_in_pack_lib_directories() -> None: +def test_find_python_in_pack_lib_directories(rule_runner: RuleRunner) -> None: pass diff --git a/pants-plugins/pack_metadata/python_rules/python_path_rules_test.py b/pants-plugins/pack_metadata/python_rules/python_path_rules_test.py index 5d7febd6b6..865e6fa443 100644 --- a/pants-plugins/pack_metadata/python_rules/python_path_rules_test.py +++ b/pants-plugins/pack_metadata/python_rules/python_path_rules_test.py @@ -12,9 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -def test_get_extra_sys_path_for_pack_dependencies() -> None: +from pants.testutil.rule_runner import RuleRunner + + +def test_get_extra_sys_path_for_pack_dependencies(rule_runner: RuleRunner) -> None: pass -def test_inject_extra_sys_path_for_pack_tests() -> None: +def test_inject_extra_sys_path_for_pack_tests(rule_runner: RuleRunner) -> None: pass From 97f72267310eea79ed79e2133b691f44a0bd788d Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 2 Aug 2024 18:34:37 -0500 Subject: [PATCH 70/98] pants-plugins: fmt w/ black --- .../pack_metadata/python_rules/conftest.py | 69 ++++++++++---- .../python_rules/python_module_mapper.py | 9 +- .../python_rules/python_pack_content.py | 92 ++++++++++++------- .../python_rules/python_path_rules.py | 25 +++-- pants-plugins/pack_metadata/register.py | 15 ++- pants-plugins/pack_metadata/target_types.py | 21 +++-- pants-plugins/uses_services/register.py | 4 +- 7 files changed, 166 insertions(+), 69 deletions(-) diff --git a/pants-plugins/pack_metadata/python_rules/conftest.py b/pants-plugins/pack_metadata/python_rules/conftest.py index 782d709412..5e0437fd6b 100644 --- a/pants-plugins/pack_metadata/python_rules/conftest.py +++ b/pants-plugins/pack_metadata/python_rules/conftest.py @@ -15,7 +15,9 @@ from textwrap import dedent import pytest -from pants.backend.python.dependency_inference.module_mapper import FirstPartyPythonMappingImpl +from pants.backend.python.dependency_inference.module_mapper import ( + FirstPartyPythonMappingImpl, +) from pants.backend.python.goals.pytest_runner import PytestPluginSetup from pants.backend.python.target_types import ( PythonSourceTarget, @@ -28,18 +30,32 @@ from pants.testutil.python_rule_runner import PythonRuleRunner from pants.testutil.rule_runner import RuleRunner -from pack_metadata.python_rules import python_module_mapper, python_pack_content, python_path_rules -from pack_metadata.python_rules.python_module_mapper import St2PythonPackContentMappingMarker +from pack_metadata.python_rules import ( + python_module_mapper, + python_pack_content, + python_path_rules, +) +from pack_metadata.python_rules.python_module_mapper import ( + St2PythonPackContentMappingMarker, +) from pack_metadata.python_rules.python_pack_content import ( PackContentPythonEntryPoints, PackContentPythonEntryPointsRequest, PackContentResourceTargetsOfType, PackContentResourceTargetsOfTypeRequest, PackPythonLibs, - PackPythonLibsRequest + PackPythonLibsRequest, +) +from pack_metadata.python_rules.python_path_rules import ( + PackPythonPath, + PackPythonPathRequest, + PytestPackTestRequest, +) +from pack_metadata.target_types import ( + InjectPackPythonPathField, + PackContentResourceTarget, + PackMetadata, ) -from pack_metadata.python_rules.python_path_rules import PackPythonPath, PackPythonPathRequest, PytestPackTestRequest -from pack_metadata.target_types import InjectPackPythonPathField, PackContentResourceTarget, PackMetadata # some random pack names packs = ( @@ -60,7 +76,7 @@ def write_test_files(rule_runner: RuleRunner): rule_runner.write_files( { f"packs/{pack}/BUILD": dedent( - f""" + """ __defaults__(all=dict(inject_pack_python_path=True)) pack_metadata(name="metadata") """ @@ -134,7 +150,9 @@ def run(self): """ ), "packs/dr_seuss/actions/BUILD": "python_sources()", - "packs/dr_seuss/actions/get_from_actions_lib.yaml": action_metadata_file("get_from_actions_lib"), + "packs/dr_seuss/actions/get_from_actions_lib.yaml": action_metadata_file( + "get_from_actions_lib" + ), "packs/dr_seuss/actions/get_from_actions_lib.py": dedent( """ from seuss.things import THING1, THING2 @@ -157,7 +175,9 @@ def run(self): """ ), "packs/shards/actions/BUILD": "python_sources()", - "packs/shards/actions/get_from_pack_lib.yaml": action_metadata_file("get_from_pack_lib"), + "packs/shards/actions/get_from_pack_lib.yaml": action_metadata_file( + "get_from_pack_lib" + ), "packs/shards/actions/get_from_pack_lib.py": dedent( """ from stormlight_archive.things import STORM_LIGHT, VOID_LIGHT, LIFE_LIGHT @@ -197,8 +217,12 @@ def remove_trigger(self): pass "packs/shards/tests/test_get_from_pack_lib_action.py": test_file( "get_from_pack_lib", "GetFromPackLibAction" ), - "packs/shards/tests/test_horn_eater_sensor.py": test_file("horn_eater", "HornEaterSensor"), - "packs/metals/actions/fly.yaml": action_metadata_file("fly", "mist_born/fly.py"), + "packs/shards/tests/test_horn_eater_sensor.py": test_file( + "horn_eater", "HornEaterSensor" + ), + "packs/metals/actions/fly.yaml": action_metadata_file( + "fly", "mist_born/fly.py" + ), "packs/metals/actions/mist_born/BUILD": "python_sources()", "packs/metals/actions/mist_born/__init__.py": "", "packs/metals/actions/mist_born/fly.py": dedent( @@ -209,7 +233,9 @@ def run(self): """ ), "packs/metals/tests/BUILD": "python_tests()", - "packs/metals/tests/test_fly_action.py": test_file("mist_born.fly", "FlyAction"), + "packs/metals/tests/test_fly_action.py": test_file( + "mist_born.fly", "FlyAction" + ), } ) @@ -218,16 +244,25 @@ def run(self): def rule_runner() -> RuleRunner: rule_runner = PythonRuleRunner( rules=[ - PythonTestsGeneratorTarget.register_plugin_field(InjectPackPythonPathField, as_moved_field=True), + PythonTestsGeneratorTarget.register_plugin_field( + InjectPackPythonPathField, as_moved_field=True + ), PythonTestTarget.register_plugin_field(InjectPackPythonPathField), *python_target_types_rules(), # TODO: not sure if we need a QueryRule for every rule... *python_pack_content.rules(), - QueryRule(PackContentResourceTargetsOfType, (PackContentResourceTargetsOfTypeRequest,)), - QueryRule(PackContentPythonEntryPoints, (PackContentPythonEntryPointsRequest,)), + QueryRule( + PackContentResourceTargetsOfType, + (PackContentResourceTargetsOfTypeRequest,), + ), + QueryRule( + PackContentPythonEntryPoints, (PackContentPythonEntryPointsRequest,) + ), QueryRule(PackPythonLibs, (PackPythonLibsRequest,)), *python_module_mapper.rules(), - QueryRule(FirstPartyPythonMappingImpl, (St2PythonPackContentMappingMarker,)), + QueryRule( + FirstPartyPythonMappingImpl, (St2PythonPackContentMappingMarker,) + ), *python_path_rules.rules(), QueryRule(PackPythonPath, (PackPythonPathRequest,)), QueryRule(PytestPluginSetup, (PytestPackTestRequest,)), @@ -243,7 +278,7 @@ def rule_runner() -> RuleRunner: ) write_test_files(rule_runner) args = [ - "--source-root-patterns=packs/*" + "--source-root-patterns=packs/*", ] rule_runner.set_options(args, env_inherit={"PATH", "PYENV_ROOT", "HOME"}) return rule_runner diff --git a/pants-plugins/pack_metadata/python_rules/python_module_mapper.py b/pants-plugins/pack_metadata/python_rules/python_module_mapper.py index c7e46e2f23..b00333e03a 100644 --- a/pants-plugins/pack_metadata/python_rules/python_module_mapper.py +++ b/pants-plugins/pack_metadata/python_rules/python_module_mapper.py @@ -39,7 +39,10 @@ class St2PythonPackContentMappingMarker(FirstPartyPythonMappingImplMarker): pass -@rule(desc=f"Creating map of `{PackMetadata.alias}` targets to Python modules in pack content", level=LogLevel.DEBUG) +@rule( + desc=f"Creating map of `{PackMetadata.alias}` targets to Python modules in pack content", + level=LogLevel.DEBUG, +) async def map_pack_content_to_python_modules( _: St2PythonPackContentMappingMarker, ) -> FirstPartyPythonMappingImpl: @@ -60,7 +63,9 @@ async def map_pack_content_to_python_modules( for pack_lib in pack_python_libs: provider_type = ( - ModuleProviderType.TYPE_STUB if pack_lib.relative_to_lib.suffix == ".pyi" else ModuleProviderType.IMPL + ModuleProviderType.TYPE_STUB + if pack_lib.relative_to_lib.suffix == ".pyi" + else ModuleProviderType.IMPL ) resolves_to_modules_to_providers[pack_lib.resolve][pack_lib.module].append( ModuleProvider(pack_lib.python_address, provider_type) diff --git a/pants-plugins/pack_metadata/python_rules/python_pack_content.py b/pants-plugins/pack_metadata/python_rules/python_pack_content.py index c67a3f9c35..058270290b 100644 --- a/pants-plugins/pack_metadata/python_rules/python_pack_content.py +++ b/pants-plugins/pack_metadata/python_rules/python_pack_content.py @@ -17,7 +17,9 @@ from pathlib import PurePath from typing import DefaultDict -from pants.backend.python.dependency_inference.module_mapper import module_from_stripped_path +from pants.backend.python.dependency_inference.module_mapper import ( + module_from_stripped_path, +) from pants.backend.python.subsystems.setup import PythonSetup from pants.backend.python.target_types import PythonResolveField, PythonSourceField from pants.base.glob_match_error_behavior import GlobMatchErrorBehavior @@ -92,14 +94,21 @@ class PackContentResourceTargetsOfType(Targets): pass -@rule(desc=f"Find all `{PackMetadata.alias}` targets in project filtered by content type", level=LogLevel.DEBUG) +@rule( + desc=f"Find all `{PackMetadata.alias}` targets in project filtered by content type", + level=LogLevel.DEBUG, +) async def find_pack_metadata_targets_of_types( request: PackContentResourceTargetsOfTypeRequest, targets: AllTargets ) -> PackContentResourceTargetsOfType: return PackContentResourceTargetsOfType( - tgt for tgt in targets + tgt + for tgt in targets if tgt.has_field(PackContentResourceSourceField) - and (not request.types or tgt[PackContentResourceTypeField].value in request.types) + and ( + not request.types + or tgt[PackContentResourceTypeField].value in request.types + ) ) @@ -117,7 +126,7 @@ def python_file_path(self) -> PurePath: @staticmethod def _split_pack_content_path(path: PurePath) -> tuple[PurePath, PurePath]: - content_types = ("actions", "sensors") # the only content_types with python content + content_types = ("actions", "sensors") # only content_types with python content pack_content_dir = path.parent while pack_content_dir.name not in content_types: pack_content_dir = pack_content_dir.parent @@ -167,8 +176,7 @@ class PackContentPythonEntryPointsRequest: @rule(desc="Find all Pack Content entry_points that are python", level=LogLevel.DEBUG) async def find_pack_content_python_entry_points( - python_setup: PythonSetup, - _: PackContentPythonEntryPointsRequest + python_setup: PythonSetup, _: PackContentPythonEntryPointsRequest ) -> PackContentPythonEntryPoints: action_or_sensor = ( PackContentResourceTypes.action_metadata, @@ -195,7 +203,9 @@ async def find_pack_content_python_entry_points( tgt: Target contents: DigestContents - for tgt, contents in zip(action_and_sensor_metadata_targets, action_and_sensor_metadata_contents): + for tgt, contents in zip( + action_and_sensor_metadata_targets, action_and_sensor_metadata_contents + ): content_type = tgt[PackContentResourceTypeField].value if content_type not in action_or_sensor: continue @@ -214,15 +224,20 @@ async def find_pack_content_python_entry_points( if entry_point: # address.filename is basically f"{spec_path}/{relative_file_path}" path = PurePath(tgt.address.filename).parent / entry_point - pack_content_entry_points_by_spec[str(path)].append((tgt.address, content_type, entry_point)) + pack_content_entry_points_by_spec[str(path)].append( + (tgt.address, content_type, entry_point) + ) python_targets = await Get( Targets, RawSpecs( - file_literals=tuple(FileLiteralSpec(spec_path) for spec_path in pack_content_entry_points_by_spec), + file_literals=tuple( + FileLiteralSpec(spec_path) + for spec_path in pack_content_entry_points_by_spec + ), unmatched_glob_behavior=GlobMatchErrorBehavior.ignore, description_of_origin="pack_metadata python module mapper", - ) + ), ) pack_content_entry_points: list[PackContentPythonEntryPoint] = [] @@ -230,16 +245,22 @@ async def find_pack_content_python_entry_points( if not tgt.has_field(PythonResolveField): # this is unexpected continue - for metadata_address, content_type, entry_point in pack_content_entry_points_by_spec[tgt.address.filename]: + for ( + metadata_address, + content_type, + entry_point, + ) in pack_content_entry_points_by_spec[tgt.address.filename]: resolve = tgt[PythonResolveField].normalized_value(python_setup) - pack_content_entry_points.append(PackContentPythonEntryPoint( - metadata_address=metadata_address, - content_type=content_type, - entry_point=entry_point, - python_address=tgt.address, - resolve=resolve, - )) + pack_content_entry_points.append( + PackContentPythonEntryPoint( + metadata_address=metadata_address, + content_type=content_type, + entry_point=entry_point, + python_address=tgt.address, + resolve=resolve, + ) + ) return PackContentPythonEntryPoints(pack_content_entry_points) @@ -276,7 +297,8 @@ async def find_python_in_pack_lib_directories( _: PackPythonLibsRequest, ) -> PackPythonLibs: pack_metadata_paths = [ - PurePath(tgt.address.spec_path) for tgt in all_unexpanded_targets + PurePath(tgt.address.spec_path) + for tgt in all_unexpanded_targets if tgt.has_field(PackMetadataSourcesField) ] pack_lib_directory_targets = await MultiGet( @@ -289,7 +311,7 @@ async def find_python_in_pack_lib_directories( ), unmatched_glob_behavior=GlobMatchErrorBehavior.ignore, description_of_origin="pack_metadata lib directory lookup", - ) + ), ) for path in pack_metadata_paths ) @@ -302,13 +324,17 @@ async def find_python_in_pack_lib_directories( pack_path: PurePath lib_directory_targets: Targets - for pack_path, lib_directory_targets in zip(pack_metadata_paths, pack_lib_directory_targets): + for pack_path, lib_directory_targets in zip( + pack_metadata_paths, pack_lib_directory_targets + ): for tgt in lib_directory_targets: if not tgt.has_field(PythonSourceField): # only python targets matter here. continue - relative_to_pack = PurePath(fast_relpath(tgt[PythonSourceField].file_path, str(pack_path))) + relative_to_pack = PurePath( + fast_relpath(tgt[PythonSourceField].file_path, str(pack_path)) + ) if relative_to_pack.parts[0] == "lib": lib_dir = "lib" elif relative_to_pack.parts[:2] == ("actions", "lib"): @@ -321,18 +347,18 @@ async def find_python_in_pack_lib_directories( resolve = tgt[PythonResolveField].normalized_value(python_setup) - pack_python_libs.append(PackPythonLib( - pack_path=pack_path, - lib_dir=lib_dir, - relative_to_lib=relative_to_lib, - python_address=tgt.address, - resolve=resolve, - )) + pack_python_libs.append( + PackPythonLib( + pack_path=pack_path, + lib_dir=lib_dir, + relative_to_lib=relative_to_lib, + python_address=tgt.address, + resolve=resolve, + ) + ) return PackPythonLibs(pack_python_libs) def rules(): - return ( - *collect_rules(), - ) + return (*collect_rules(),) diff --git a/pants-plugins/pack_metadata/python_rules/python_path_rules.py b/pants-plugins/pack_metadata/python_rules/python_path_rules.py index 49406c39a2..e4074ad469 100644 --- a/pants-plugins/pack_metadata/python_rules/python_path_rules.py +++ b/pants-plugins/pack_metadata/python_rules/python_path_rules.py @@ -47,10 +47,14 @@ class PackPythonPathRequest: @rule( desc="Get pack paths that should be added to PYTHONPATH/PEX_EXTRA_SYS_PATH for a target.", - level=LogLevel.DEBUG + level=LogLevel.DEBUG, ) -async def get_extra_sys_path_for_pack_dependencies(request: PackPythonPathRequest) -> PackPythonPath: - transitive_targets = await Get(TransitiveTargets, TransitiveTargetsRequest((request.address,))) +async def get_extra_sys_path_for_pack_dependencies( + request: PackPythonPathRequest, +) -> PackPythonPath: + transitive_targets = await Get( + TransitiveTargets, TransitiveTargetsRequest((request.address,)) + ) dependency_addresses: Set[Address] = { tgt.address for tgt in transitive_targets.closure @@ -76,11 +80,13 @@ async def get_extra_sys_path_for_pack_dependencies(request: PackPythonPathReques # filter pack_content_python_entry_points and pack_python_libs pack_content_python_entry_points = ( - pack_content for pack_content in pack_content_python_entry_points + pack_content + for pack_content in pack_content_python_entry_points if pack_content.python_address in pack_python_content_addresses ) pack_python_libs = ( - pack_lib for pack_lib in pack_python_libs + pack_lib + for pack_lib in pack_python_libs if pack_lib.python_address in pack_python_lib_addresses ) @@ -102,11 +108,16 @@ def is_applicable(cls, target: Target) -> bool: return bool(target.get(InjectPackPythonPathField).value) -@rule(desc="Inject pack paths in PYTHONPATH/PEX_EXTRA_SYS_PATH for python tests.", level=LogLevel.DEBUG) +@rule( + desc="Inject pack paths in PYTHONPATH/PEX_EXTRA_SYS_PATH for python tests.", + level=LogLevel.DEBUG, +) async def inject_extra_sys_path_for_pack_tests( request: PytestPackTestRequest, ) -> PytestPluginSetup: - pack_python_path = await Get(PackPythonPath, PackPythonPathRequest(request.target.address)) + pack_python_path = await Get( + PackPythonPath, PackPythonPathRequest(request.target.address) + ) return PytestPluginSetup( # digest=EMPTY_DIGEST, # extra_sys_path=pack_python_path, # TODO: make pants support this diff --git a/pants-plugins/pack_metadata/register.py b/pants-plugins/pack_metadata/register.py index 9ea3ad5aa2..6cdd7c9f8d 100644 --- a/pants-plugins/pack_metadata/register.py +++ b/pants-plugins/pack_metadata/register.py @@ -12,10 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -from pants.backend.python.target_types import PythonTestTarget, PythonTestsGeneratorTarget +from pants.backend.python.target_types import ( + PythonTestTarget, + PythonTestsGeneratorTarget, +) from pack_metadata import tailor, target_types_rules -from pack_metadata.python_rules import python_module_mapper, python_pack_content, python_path_rules +from pack_metadata.python_rules import ( + python_module_mapper, + python_pack_content, + python_path_rules, +) from pack_metadata.target_types import ( InjectPackPythonPathField, PackContentResourceTarget, @@ -27,7 +34,9 @@ def rules(): return [ - PythonTestsGeneratorTarget.register_plugin_field(InjectPackPythonPathField, as_moved_field=True), + PythonTestsGeneratorTarget.register_plugin_field( + InjectPackPythonPathField, as_moved_field=True + ), PythonTestTarget.register_plugin_field(InjectPackPythonPathField), *tailor.rules(), *target_types_rules.rules(), diff --git a/pants-plugins/pack_metadata/target_types.py b/pants-plugins/pack_metadata/target_types.py index ca019a4f55..8e3b67fc33 100644 --- a/pants-plugins/pack_metadata/target_types.py +++ b/pants-plugins/pack_metadata/target_types.py @@ -16,7 +16,12 @@ from typing import Optional, Sequence, Tuple from pants.engine.internals.native_engine import Address -from pants.engine.target import BoolField, COMMON_TARGET_FIELDS, Dependencies, StringField +from pants.engine.target import ( + BoolField, + COMMON_TARGET_FIELDS, + Dependencies, + StringField, +) from pants.core.target_types import ( ResourceDependenciesField, ResourcesGeneratingSourcesField, @@ -74,7 +79,9 @@ class PackContentResourceTypeField(StringField): value: PackContentResourceTypes @classmethod - def compute_value(cls, raw_value: Optional[str], address: Address) -> PackContentResourceTypes: + def compute_value( + cls, raw_value: Optional[str], address: Address + ) -> PackContentResourceTypes: value = super().compute_value(raw_value, address) if value is not None: return PackContentResourceTypes(value) @@ -142,14 +149,16 @@ class PackContentResourceTarget(ResourceTarget): PackContentResourceSourceField, PackContentResourceTypeField, ) - help = ( - "A single pack content resource file (mostly for metadata files)." - ) + help = "A single pack content resource file (mostly for metadata files)." class PackMetadata(ResourcesGeneratorTarget): alias = "pack_metadata" - core_fields = (*COMMON_TARGET_FIELDS, PackMetadataSourcesField, ResourcesOverridesField) + core_fields = ( + *COMMON_TARGET_FIELDS, + PackMetadataSourcesField, + ResourcesOverridesField, + ) moved_fields = (ResourceDependenciesField,) generated_target_cls = PackContentResourceTarget help = ( diff --git a/pants-plugins/uses_services/register.py b/pants-plugins/uses_services/register.py index 56b4d7ff6c..6693b9b12d 100644 --- a/pants-plugins/uses_services/register.py +++ b/pants-plugins/uses_services/register.py @@ -22,7 +22,9 @@ def rules(): return [ - PythonTestsGeneratorTarget.register_plugin_field(UsesServicesField, as_moved_field=True), + PythonTestsGeneratorTarget.register_plugin_field( + UsesServicesField, as_moved_field=True + ), PythonTestTarget.register_plugin_field(UsesServicesField), *platform_rules.rules(), *mongo_rules.rules(), From b069a83724dcdef7362cdd2f69b76989f8e4a29a Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 2 Aug 2024 19:37:27 -0500 Subject: [PATCH 71/98] pants-plugins/pack_metadata: add test for pack content type detection And fix the identified issues. --- .../pack_metadata/python_rules/conftest.py | 2 +- .../python_rules/python_pack_content_test.py | 59 ++++++++++++++++++- pants-plugins/pack_metadata/target_types.py | 10 ++-- 3 files changed, 63 insertions(+), 8 deletions(-) diff --git a/pants-plugins/pack_metadata/python_rules/conftest.py b/pants-plugins/pack_metadata/python_rules/conftest.py index 5e0437fd6b..105ddbc7d1 100644 --- a/pants-plugins/pack_metadata/python_rules/conftest.py +++ b/pants-plugins/pack_metadata/python_rules/conftest.py @@ -91,7 +91,7 @@ def write_test_files(rule_runner: RuleRunner): """ ), f"packs/{pack}/config.schema.yaml": "", - f"packs/{pack}/{pack}.yaml.example": "", + f"packs/{pack}/config.yaml.example": "", f"packs/{pack}/icon.png": "", f"packs/{pack}/README.md": f"# Pack {pack} README", } diff --git a/pants-plugins/pack_metadata/python_rules/python_pack_content_test.py b/pants-plugins/pack_metadata/python_rules/python_pack_content_test.py index 9f7006e225..a9a7026c5f 100644 --- a/pants-plugins/pack_metadata/python_rules/python_pack_content_test.py +++ b/pants-plugins/pack_metadata/python_rules/python_pack_content_test.py @@ -12,11 +12,66 @@ # See the License for the specific language governing permissions and # limitations under the License. +import pytest + from pants.testutil.rule_runner import RuleRunner +from pack_metadata.python_rules.python_pack_content import ( + PackContentResourceTargetsOfType, + PackContentResourceTargetsOfTypeRequest, +) +from pack_metadata.target_types import PackContentResourceTypes -def test_find_pack_metadata_targets_of_types(rule_runner: RuleRunner) -> None: - pass + +@pytest.mark.parametrize( + "requested_types,expected_count,expected_file_name", + ( + # one content type + ((PackContentResourceTypes.pack_metadata,), 4, "pack.yaml"), + ((PackContentResourceTypes.pack_config_schema,), 4, "config.schema.yaml"), + ((PackContentResourceTypes.pack_config_example,), 4, "config.yaml.example"), + ((PackContentResourceTypes.pack_icon,), 4, "icon.png"), + ((PackContentResourceTypes.action_metadata,), 5, ".yaml"), + ((PackContentResourceTypes.sensor_metadata,), 1, ".yaml"), + ((PackContentResourceTypes.rule_metadata,), 0, ""), + ((PackContentResourceTypes.policy_metadata,), 0, ""), + ((PackContentResourceTypes.unknown,), 0, ""), + # all content types + ((), 22, ""), + # some content types + ( + ( + PackContentResourceTypes.action_metadata, + PackContentResourceTypes.sensor_metadata, + ), + 6, + "", + ), + ( + ( + PackContentResourceTypes.pack_metadata, + PackContentResourceTypes.pack_config_schema, + PackContentResourceTypes.pack_config_example, + ), + 12, + "", + ), + ), +) +def test_find_pack_metadata_targets_of_types( + rule_runner: RuleRunner, + requested_types: tuple[PackContentResourceTypes, ...], + expected_count: int, + expected_file_name: str, +) -> None: + result = rule_runner.request( + PackContentResourceTargetsOfType, + (PackContentResourceTargetsOfTypeRequest(requested_types),), + ) + assert len(result) == expected_count + if expected_file_name: + for tgt in result: + tgt.address.relative_file_path.endswith(expected_file_name) def test_find_pack_content_python_entry_points(rule_runner: RuleRunner) -> None: diff --git a/pants-plugins/pack_metadata/target_types.py b/pants-plugins/pack_metadata/target_types.py index 8e3b67fc33..8128cabf2c 100644 --- a/pants-plugins/pack_metadata/target_types.py +++ b/pants-plugins/pack_metadata/target_types.py @@ -86,17 +86,17 @@ def compute_value( if value is not None: return PackContentResourceTypes(value) path = PurePath(address.relative_file_path) - _yaml_suffixes = ("yaml", "yml") + _yaml_suffixes = (".yaml", ".yml") if len(path.parent.parts) == 0: # in the pack root - if path.name == "pack.yaml": + if path.stem == "pack" and path.suffix in _yaml_suffixes: return PackContentResourceTypes.pack_metadata - if path.stem == "pack.schema" and path.suffix in _yaml_suffixes: + if path.stem == "config.schema" and path.suffix in _yaml_suffixes: return PackContentResourceTypes.pack_config_schema - if path.suffix == "example" and path.suffixes[0] in _yaml_suffixes: + if path.stem.startswith("config.") and path.suffixes[0] in _yaml_suffixes and path.suffix == ".example": return PackContentResourceTypes.pack_config_example if path.name == "icon.png": - return PackContentResourceTypes.pack_config_example + return PackContentResourceTypes.pack_icon return PackContentResourceTypes.unknown resource_type = _content_type_by_path_parts.get(path.parent.parts, None) if resource_type is not None: From 8ff5e936a29d9651f69e99cc8badabb442521913 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Sat, 3 Aug 2024 22:17:38 -0500 Subject: [PATCH 72/98] pants-plugins/pack_metadata: add tests for entry_point and pack_lib rules And fix the identified mistake in conftest. --- .../pack_metadata/python_rules/conftest.py | 2 +- .../python_rules/python_pack_content_test.py | 80 ++++++++++++++++++- 2 files changed, 79 insertions(+), 3 deletions(-) diff --git a/pants-plugins/pack_metadata/python_rules/conftest.py b/pants-plugins/pack_metadata/python_rules/conftest.py index 105ddbc7d1..51104af2e8 100644 --- a/pants-plugins/pack_metadata/python_rules/conftest.py +++ b/pants-plugins/pack_metadata/python_rules/conftest.py @@ -141,7 +141,7 @@ def run(self): "packs/foo/tests/BUILD": "python_tests()", "packs/foo/tests/test_get_bar_action.py": test_file("get_bar", "BarAction"), "packs/foo/tests/test_get_baz_action.py": test_file("get_baz", "BazAction"), - "packs/dr_seuss/actions/lib/BUILD": "python_sources()", + "packs/dr_seuss/actions/lib/seuss/BUILD": "python_sources()", "packs/dr_seuss/actions/lib/seuss/__init__.py": "", "packs/dr_seuss/actions/lib/seuss/things.py": dedent( """ diff --git a/pants-plugins/pack_metadata/python_rules/python_pack_content_test.py b/pants-plugins/pack_metadata/python_rules/python_pack_content_test.py index a9a7026c5f..33c1389bb3 100644 --- a/pants-plugins/pack_metadata/python_rules/python_pack_content_test.py +++ b/pants-plugins/pack_metadata/python_rules/python_pack_content_test.py @@ -14,11 +14,16 @@ import pytest +from pants.engine.addresses import Address from pants.testutil.rule_runner import RuleRunner from pack_metadata.python_rules.python_pack_content import ( + PackContentPythonEntryPoints, + PackContentPythonEntryPointsRequest, PackContentResourceTargetsOfType, PackContentResourceTargetsOfTypeRequest, + PackPythonLibs, + PackPythonLibsRequest, ) from pack_metadata.target_types import PackContentResourceTypes @@ -75,8 +80,79 @@ def test_find_pack_metadata_targets_of_types( def test_find_pack_content_python_entry_points(rule_runner: RuleRunner) -> None: - pass + result = rule_runner.request( + PackContentPythonEntryPoints, + (PackContentPythonEntryPointsRequest(),), + ) + assert len(result) == 6 # 5 actions + 1 sensor + assert {res.metadata_address for res in result} == { + Address( + "packs/foo", + relative_file_path="actions/get_bar.yaml", + target_name="metadata", + ), + Address( + "packs/foo", + relative_file_path="actions/get_baz.yaml", + target_name="metadata", + ), + Address( + "packs/dr_seuss", + relative_file_path="actions/get_from_actions_lib.yaml", + target_name="metadata", + ), + Address( + "packs/shards", + relative_file_path="actions/get_from_pack_lib.yaml", + target_name="metadata", + ), + Address( + "packs/shards", + relative_file_path="sensors/horn_eater.yaml", + target_name="metadata", + ), + Address( + "packs/metals", + relative_file_path="actions/fly.yaml", + target_name="metadata", + ), + } + assert {(res.content_type, res.entry_point) for res in result} == { + (PackContentResourceTypes.action_metadata, "get_bar.py"), + (PackContentResourceTypes.action_metadata, "get_baz.py"), + (PackContentResourceTypes.action_metadata, "get_from_actions_lib.py"), + (PackContentResourceTypes.action_metadata, "get_from_pack_lib.py"), + (PackContentResourceTypes.sensor_metadata, "horn_eater.py"), + (PackContentResourceTypes.action_metadata, "mist_born/fly.py"), + } + assert {res.python_address for res in result} == { + Address("packs/foo/actions", relative_file_path="get_bar.py"), + Address("packs/foo/actions", relative_file_path="get_baz.py"), + Address("packs/dr_seuss/actions", relative_file_path="get_from_actions_lib.py"), + Address("packs/shards/actions", relative_file_path="get_from_pack_lib.py"), + Address("packs/shards/sensors", relative_file_path="horn_eater.py"), + Address("packs/metals/actions/mist_born", relative_file_path="fly.py"), + } def test_find_python_in_pack_lib_directories(rule_runner: RuleRunner) -> None: - pass + result = rule_runner.request(PackPythonLibs, (PackPythonLibsRequest(),)) + assert len(result) == 4 + assert {(str(res.pack_path), res.lib_dir) for res in result} == { + ("packs/dr_seuss", "actions/lib"), + ("packs/shards", "lib"), + } + assert {res.python_address for res in result} == { + Address("packs/dr_seuss/actions/lib/seuss", relative_file_path="__init__.py"), + Address("packs/dr_seuss/actions/lib/seuss", relative_file_path="things.py"), + Address( + "packs/shards/lib/stormlight_archive", relative_file_path="__init__.py" + ), + Address("packs/shards/lib/stormlight_archive", relative_file_path="things.py"), + } + assert {res.module for res in result} == { + "seuss", + "seuss.things", + "stormlight_archive", + "stormlight_archive.things", + } From d3b06a902788811de8906d10287da2d819e565e5 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Sat, 3 Aug 2024 22:18:13 -0500 Subject: [PATCH 73/98] pants-plugins/pack_metadata: fmt --- pants-plugins/pack_metadata/target_types.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pants-plugins/pack_metadata/target_types.py b/pants-plugins/pack_metadata/target_types.py index 8128cabf2c..01d80c24ad 100644 --- a/pants-plugins/pack_metadata/target_types.py +++ b/pants-plugins/pack_metadata/target_types.py @@ -93,7 +93,11 @@ def compute_value( return PackContentResourceTypes.pack_metadata if path.stem == "config.schema" and path.suffix in _yaml_suffixes: return PackContentResourceTypes.pack_config_schema - if path.stem.startswith("config.") and path.suffixes[0] in _yaml_suffixes and path.suffix == ".example": + if ( + path.stem.startswith("config.") + and path.suffixes[0] in _yaml_suffixes + and path.suffix == ".example" + ): return PackContentResourceTypes.pack_config_example if path.name == "icon.png": return PackContentResourceTypes.pack_icon From 1c37c6866e95e0094105baf230eefccf6ee8a25f Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 5 Aug 2024 11:45:48 -0500 Subject: [PATCH 74/98] pants-plugins/pack_metadata: add test for python module mapper rule --- .../python_rules/python_module_mapper_test.py | 56 ++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/pants-plugins/pack_metadata/python_rules/python_module_mapper_test.py b/pants-plugins/pack_metadata/python_rules/python_module_mapper_test.py index 0bf6854651..d1d6d779fb 100644 --- a/pants-plugins/pack_metadata/python_rules/python_module_mapper_test.py +++ b/pants-plugins/pack_metadata/python_rules/python_module_mapper_test.py @@ -12,8 +12,62 @@ # See the License for the specific language governing permissions and # limitations under the License. +from pants.backend.python.dependency_inference.module_mapper import ( + FirstPartyPythonMappingImpl, + ModuleProvider, + ModuleProviderType, +) +from pants.engine.internals.native_engine import Address from pants.testutil.rule_runner import RuleRunner +from pants.util.frozendict import FrozenDict + +from pack_metadata.python_rules.python_module_mapper import ( + St2PythonPackContentMappingMarker, +) def test_map_pack_content_to_python_modules(rule_runner: RuleRunner) -> None: - pass + result = rule_runner.request( + FirstPartyPythonMappingImpl, + (St2PythonPackContentMappingMarker(),), + ) + + def module_provider(spec_path: str, relative_file_path: str) -> ModuleProvider: + return ModuleProvider( + Address(spec_path=spec_path, relative_file_path=relative_file_path), + ModuleProviderType.IMPL, + ) + + expected = { + "": { + "get_bar": (module_provider("packs/foo/actions", "get_bar.py"),), + "get_baz": (module_provider("packs/foo/actions", "get_baz.py"),), + "seuss": ( + module_provider("packs/dr_seuss/actions/lib/seuss", "__init__.py"), + ), + "seuss.things": ( + module_provider("packs/dr_seuss/actions/lib/seuss", "things.py"), + ), + "get_from_actions_lib": ( + module_provider("packs/dr_seuss/actions", "get_from_actions_lib.py"), + ), + "stormlight_archive": ( + module_provider("packs/shards/lib/stormlight_archive", "__init__.py"), + ), + "stormlight_archive.things": ( + module_provider("packs/shards/lib/stormlight_archive", "things.py"), + ), + "get_from_pack_lib": ( + module_provider("packs/shards/actions", "get_from_pack_lib.py"), + ), + "horn_eater": (module_provider("packs/shards/sensors", "horn_eater.py"),), + "fly": (module_provider("packs/metals/actions/mist_born", "fly.py"),), + "mist_born.fly": ( + module_provider("packs/metals/actions/mist_born", "fly.py"), + ), + } + } + assert isinstance(result, FrozenDict) + assert all(isinstance(value, FrozenDict) for value in result.values()) + # pytest reports dict differences better than FrozenDict + assert {resolve: dict(value) for resolve, value in result.items()} == expected From 8b990253e00b61b8a6782d519d7f002f5e2ca11a Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 5 Aug 2024 12:10:22 -0500 Subject: [PATCH 75/98] pants-plugins/pack_metadata: add test for get_extra_sys_path_for_pack_dependencies rule --- .../python_rules/python_path_rules_test.py | 105 +++++++++++++++++- 1 file changed, 103 insertions(+), 2 deletions(-) diff --git a/pants-plugins/pack_metadata/python_rules/python_path_rules_test.py b/pants-plugins/pack_metadata/python_rules/python_path_rules_test.py index 865e6fa443..06bc72e787 100644 --- a/pants-plugins/pack_metadata/python_rules/python_path_rules_test.py +++ b/pants-plugins/pack_metadata/python_rules/python_path_rules_test.py @@ -12,11 +12,112 @@ # See the License for the specific language governing permissions and # limitations under the License. +import pytest +from pants.engine.internals.native_engine import Address from pants.testutil.rule_runner import RuleRunner +from pack_metadata.python_rules.python_path_rules import ( + PackPythonPath, + PackPythonPathRequest, +) -def test_get_extra_sys_path_for_pack_dependencies(rule_runner: RuleRunner) -> None: - pass + +@pytest.mark.parametrize( + "address,expected", + ( + ( + Address("packs/foo/actions", relative_file_path="get_bar.py"), + ("packs/foo/actions",), + ), + ( + Address("packs/foo/actions", relative_file_path="get_baz.py"), + ("packs/foo/actions",), + ), + ( + Address("packs/foo/tests", relative_file_path="test_get_bar_action.py"), + ("packs/foo/actions",), + ), + ( + Address("packs/foo/tests", relative_file_path="test_get_baz_action.py"), + ("packs/foo/actions",), + ), + ( + Address( + "packs/dr_seuss/actions/lib/seuss", relative_file_path="__init__.py" + ), + ("packs/dr_seuss/actions/lib",), + ), + ( + Address("packs/dr_seuss/actions/lib/seuss", relative_file_path="things.py"), + ("packs/dr_seuss/actions/lib",), + ), + ( + Address( + "packs/dr_seuss/actions", relative_file_path="get_from_actions_lib.py" + ), + ("packs/dr_seuss/actions", "packs/dr_seuss/actions/lib"), + ), + ( + Address( + "packs/dr_seuss/tests", + relative_file_path="test_get_from_actions_lib_action.py", + ), + ("packs/dr_seuss/actions", "packs/dr_seuss/actions/lib"), + ), + ( + Address( + "packs/shards/lib/stormlight_archive", relative_file_path="__init__.py" + ), + ("packs/shards/lib",), + ), + ( + Address( + "packs/shards/lib/stormlight_archive", relative_file_path="things.py" + ), + ("packs/shards/lib",), + ), + ( + Address("packs/shards/actions", relative_file_path="get_from_pack_lib.py"), + ("packs/shards/actions", "packs/shards/lib"), + ), + ( + Address("packs/shards/sensors", relative_file_path="horn_eater.py"), + ("packs/shards/sensors", "packs/shards/lib"), + ), + ( + Address( + "packs/shards/tests", + relative_file_path="test_get_from_pack_lib_action.py", + ), + ("packs/shards/actions", "packs/shards/lib"), + ), + ( + Address( + "packs/shards/tests", relative_file_path="test_horn_eater_sensor.py" + ), + ("packs/shards/sensors", "packs/shards/lib"), + ), + ( + Address("packs/metals/actions/mist_born", relative_file_path="__init__.py"), + (), # there are no dependencies, and this is not an action entry point. + ), + ( + Address("packs/metals/actions/mist_born", relative_file_path="fly.py"), + ("packs/metals/actions/mist_born", "packs/metals/actions"), + ), + ( + Address("packs/metals/tests", relative_file_path="test_fly_action.py"), + ("packs/metals/actions/mist_born", "packs/metals/actions"), + ), + ), +) +def test_get_extra_sys_path_for_pack_dependencies( + rule_runner: RuleRunner, address: Address, expected: tuple[str, ...] +) -> None: + pack_python_path = rule_runner.request( + PackPythonPath, (PackPythonPathRequest(address),) + ) + assert pack_python_path.entries == expected def test_inject_extra_sys_path_for_pack_tests(rule_runner: RuleRunner) -> None: From d81ff6b42dc43054db8440c7469bd066a761dba0 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 5 Aug 2024 12:22:18 -0500 Subject: [PATCH 76/98] pants-plugins/pack_metadata: add test for inject_extra_sys_path_for_pack_tests rule --- .../python_rules/python_path_rules.py | 2 +- .../python_rules/python_path_rules_test.py | 51 +++++++++++++++++-- 2 files changed, 49 insertions(+), 4 deletions(-) diff --git a/pants-plugins/pack_metadata/python_rules/python_path_rules.py b/pants-plugins/pack_metadata/python_rules/python_path_rules.py index e4074ad469..3436c43286 100644 --- a/pants-plugins/pack_metadata/python_rules/python_path_rules.py +++ b/pants-plugins/pack_metadata/python_rules/python_path_rules.py @@ -120,7 +120,7 @@ async def inject_extra_sys_path_for_pack_tests( ) return PytestPluginSetup( # digest=EMPTY_DIGEST, - # extra_sys_path=pack_python_path, # TODO: make pants support this + # extra_sys_path=pack_python_path.entries, # TODO: make pants support this ) diff --git a/pants-plugins/pack_metadata/python_rules/python_path_rules_test.py b/pants-plugins/pack_metadata/python_rules/python_path_rules_test.py index 06bc72e787..a7514c4aaf 100644 --- a/pants-plugins/pack_metadata/python_rules/python_path_rules_test.py +++ b/pants-plugins/pack_metadata/python_rules/python_path_rules_test.py @@ -13,12 +13,14 @@ # limitations under the License. import pytest -from pants.engine.internals.native_engine import Address +from pants.backend.python.goals.pytest_runner import PytestPluginSetup +from pants.engine.internals.native_engine import Address, EMPTY_DIGEST from pants.testutil.rule_runner import RuleRunner from pack_metadata.python_rules.python_path_rules import ( PackPythonPath, PackPythonPathRequest, + PytestPackTestRequest, ) @@ -120,5 +122,48 @@ def test_get_extra_sys_path_for_pack_dependencies( assert pack_python_path.entries == expected -def test_inject_extra_sys_path_for_pack_tests(rule_runner: RuleRunner) -> None: - pass +@pytest.mark.xfail(raises=AttributeError, reason="Not implemented in pants yet.") +@pytest.mark.parametrize( + "address,expected", + ( + ( + Address("packs/foo/tests", relative_file_path="test_get_bar_action.py"), + ("packs/foo/actions",), + ), + ( + Address("packs/foo/tests", relative_file_path="test_get_baz_action.py"), + ("packs/foo/actions",), + ), + ( + Address( + "packs/dr_seuss/tests", + relative_file_path="test_get_from_actions_lib_action.py", + ), + ("packs/dr_seuss/actions", "packs/dr_seuss/actions/lib"), + ), + ( + Address( + "packs/shards/tests", + relative_file_path="test_get_from_pack_lib_action.py", + ), + ("packs/shards/actions", "packs/shards/lib"), + ), + ( + Address( + "packs/shards/tests", relative_file_path="test_horn_eater_sensor.py" + ), + ("packs/shards/sensors", "packs/shards/lib"), + ), + ( + Address("packs/metals/tests", relative_file_path="test_fly_action.py"), + ("packs/metals/actions/mist_born", "packs/metals/actions"), + ), + ), +) +def test_inject_extra_sys_path_for_pack_tests( + rule_runner: RuleRunner, address: Address, expected: tuple[str, ...] +) -> None: + target = rule_runner.get_target(address) + result = rule_runner.request(PytestPluginSetup, (PytestPackTestRequest(target),)) + assert result.digest == EMPTY_DIGEST + assert result.extra_sys_path == expected # TODO: pants does not have this attribute yet. From 33d5085311ef103061b752b3ab7d8ad92d72a716 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Tue, 10 Sep 2024 11:07:02 -0500 Subject: [PATCH 77/98] pants: record python_tests(entry_point_dependencies=...) for runner tests --- contrib/runners/action_chain_runner/tests/BUILD | 3 +++ contrib/runners/action_chain_runner/tests/unit/BUILD | 7 +------ contrib/runners/announcement_runner/tests/BUILD | 3 +++ contrib/runners/http_runner/tests/BUILD | 3 +++ contrib/runners/inquirer_runner/tests/BUILD | 3 +++ contrib/runners/local_runner/tests/BUILD | 3 +++ contrib/runners/noop_runner/tests/BUILD | 3 +++ contrib/runners/orquesta_runner/tests/BUILD | 3 +++ contrib/runners/python_runner/tests/BUILD | 3 +++ contrib/runners/remote_runner/tests/BUILD | 3 +++ contrib/runners/winrm_runner/tests/BUILD | 3 +++ st2common/tests/unit/services/BUILD | 6 ++++-- 12 files changed, 35 insertions(+), 8 deletions(-) diff --git a/contrib/runners/action_chain_runner/tests/BUILD b/contrib/runners/action_chain_runner/tests/BUILD index 3280583e0c..d19247d547 100644 --- a/contrib/runners/action_chain_runner/tests/BUILD +++ b/contrib/runners/action_chain_runner/tests/BUILD @@ -1,6 +1,9 @@ __defaults__( all=dict( skip_pylint=True, + entry_point_dependencies={ + "contrib/runners/action_chain_runner": ["st2common.runners.runner"], + }, ) ) diff --git a/contrib/runners/action_chain_runner/tests/unit/BUILD b/contrib/runners/action_chain_runner/tests/unit/BUILD index 459639e1b7..47c65e48d3 100644 --- a/contrib/runners/action_chain_runner/tests/unit/BUILD +++ b/contrib/runners/action_chain_runner/tests/unit/BUILD @@ -8,13 +8,8 @@ python_tests( stevedore_namespaces=[ "st2common.rbac.backend", "st2common.metrics.driver", - # TODO: we only need THIS runner, but this pulls in all. + # the core pack uses all runners. "st2common.runners.runner", ], uses=["mongo"], - # TODO: this will allow us to depend only on this runner once we upgrade to pants 2.23 - # https://github.com/pantsbuild/pants/pull/21062 - #entry_point_dependencies={ - # "contrib/runners/action_chain_runner": ["st2common.runners.runner"], - #}, ) diff --git a/contrib/runners/announcement_runner/tests/BUILD b/contrib/runners/announcement_runner/tests/BUILD index 3280583e0c..850b4aa71a 100644 --- a/contrib/runners/announcement_runner/tests/BUILD +++ b/contrib/runners/announcement_runner/tests/BUILD @@ -1,6 +1,9 @@ __defaults__( all=dict( skip_pylint=True, + entry_point_dependencies={ + "contrib/runners/announcement_runner": ["st2common.runners.runner"], + }, ) ) diff --git a/contrib/runners/http_runner/tests/BUILD b/contrib/runners/http_runner/tests/BUILD index 3280583e0c..eccc29b3ee 100644 --- a/contrib/runners/http_runner/tests/BUILD +++ b/contrib/runners/http_runner/tests/BUILD @@ -1,6 +1,9 @@ __defaults__( all=dict( skip_pylint=True, + entry_point_dependencies={ + "contrib/runners/http_runner": ["st2common.runners.runner"], + }, ) ) diff --git a/contrib/runners/inquirer_runner/tests/BUILD b/contrib/runners/inquirer_runner/tests/BUILD index 3280583e0c..36b90b1e2a 100644 --- a/contrib/runners/inquirer_runner/tests/BUILD +++ b/contrib/runners/inquirer_runner/tests/BUILD @@ -1,6 +1,9 @@ __defaults__( all=dict( skip_pylint=True, + entry_point_dependencies={ + "contrib/runners/inquirer_runner": ["st2common.runners.runner"], + }, ) ) diff --git a/contrib/runners/local_runner/tests/BUILD b/contrib/runners/local_runner/tests/BUILD index 3280583e0c..d692891e08 100644 --- a/contrib/runners/local_runner/tests/BUILD +++ b/contrib/runners/local_runner/tests/BUILD @@ -1,6 +1,9 @@ __defaults__( all=dict( skip_pylint=True, + entry_point_dependencies={ + "contrib/runners/local_runner": ["st2common.runners.runner"], + }, ) ) diff --git a/contrib/runners/noop_runner/tests/BUILD b/contrib/runners/noop_runner/tests/BUILD index 3280583e0c..208d20406f 100644 --- a/contrib/runners/noop_runner/tests/BUILD +++ b/contrib/runners/noop_runner/tests/BUILD @@ -1,6 +1,9 @@ __defaults__( all=dict( skip_pylint=True, + entry_point_dependencies={ + "contrib/runners/noop_runner": ["st2common.runners.runner"], + }, ) ) diff --git a/contrib/runners/orquesta_runner/tests/BUILD b/contrib/runners/orquesta_runner/tests/BUILD index 3280583e0c..e0764b1850 100644 --- a/contrib/runners/orquesta_runner/tests/BUILD +++ b/contrib/runners/orquesta_runner/tests/BUILD @@ -1,6 +1,9 @@ __defaults__( all=dict( skip_pylint=True, + entry_point_dependencies={ + "contrib/runners/orquesta_runner": ["st2common.runners.runner"], + }, ) ) diff --git a/contrib/runners/python_runner/tests/BUILD b/contrib/runners/python_runner/tests/BUILD index 3280583e0c..dfaf857926 100644 --- a/contrib/runners/python_runner/tests/BUILD +++ b/contrib/runners/python_runner/tests/BUILD @@ -1,6 +1,9 @@ __defaults__( all=dict( skip_pylint=True, + entry_point_dependencies={ + "contrib/runners/python_runner": ["st2common.runners.runner"], + }, ) ) diff --git a/contrib/runners/remote_runner/tests/BUILD b/contrib/runners/remote_runner/tests/BUILD index 3280583e0c..3391ed1f72 100644 --- a/contrib/runners/remote_runner/tests/BUILD +++ b/contrib/runners/remote_runner/tests/BUILD @@ -1,6 +1,9 @@ __defaults__( all=dict( skip_pylint=True, + entry_point_dependencies={ + "contrib/runners/remote_runner": ["st2common.runners.runner"], + }, ) ) diff --git a/contrib/runners/winrm_runner/tests/BUILD b/contrib/runners/winrm_runner/tests/BUILD index 3280583e0c..9c9ad37ef4 100644 --- a/contrib/runners/winrm_runner/tests/BUILD +++ b/contrib/runners/winrm_runner/tests/BUILD @@ -1,6 +1,9 @@ __defaults__( all=dict( skip_pylint=True, + entry_point_dependencies={ + "contrib/runners/winrm_runner": ["st2common.runners.runner"], + }, ) ) diff --git a/st2common/tests/unit/services/BUILD b/st2common/tests/unit/services/BUILD index f1342ac1b4..ae93633567 100644 --- a/st2common/tests/unit/services/BUILD +++ b/st2common/tests/unit/services/BUILD @@ -10,7 +10,9 @@ python_tests( overrides={ "test_packs.py": dict( # use the fixture to resolve ambiguous import (ambiguous due to symlink of core pack) - dependencies=["st2tests/st2tests/fixtures/packs/core/actions/inject_trigger.py"], + dependencies=[ + "st2tests/st2tests/fixtures/packs/core/actions/inject_trigger.py" + ], ), - } + }, ) From 7b39bffeaca83c606061047e493219b29d0274dc Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Tue, 10 Sep 2024 19:11:54 -0500 Subject: [PATCH 78/98] fix shell syntax in action chain test I do not know why this worked before. Perhaps it relied on quirks from old versions of /bin/sh. Without this fix the action chain test test_chain_pause_resume_with_error fails. --- .../actions/chains/test_pause_resume_with_error.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/st2tests/st2tests/fixtures/packs/action_chain_tests/actions/chains/test_pause_resume_with_error.yaml b/st2tests/st2tests/fixtures/packs/action_chain_tests/actions/chains/test_pause_resume_with_error.yaml index a4d46be55e..c0869f7b64 100644 --- a/st2tests/st2tests/fixtures/packs/action_chain_tests/actions/chains/test_pause_resume_with_error.yaml +++ b/st2tests/st2tests/fixtures/packs/action_chain_tests/actions/chains/test_pause_resume_with_error.yaml @@ -3,7 +3,7 @@ chain: name: task1 ref: core.local params: - cmd: "while [ -e '{{tempfile}}' ]; do sleep 0.1; exit 1" + cmd: "while [ -e '{{tempfile}}' ]; do sleep 0.1; done; exit 1" timeout: 180 on-failure: task2 - From f841444294e0223eb710ec76ef6915e1ccc7d4de Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Tue, 10 Sep 2024 20:22:06 -0500 Subject: [PATCH 79/98] pants: record test dependency that was not inferred --- contrib/runners/python_runner/tests/integration/BUILD | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/contrib/runners/python_runner/tests/integration/BUILD b/contrib/runners/python_runner/tests/integration/BUILD index 2d782aaea0..5fdf71eed3 100644 --- a/contrib/runners/python_runner/tests/integration/BUILD +++ b/contrib/runners/python_runner/tests/integration/BUILD @@ -5,4 +5,9 @@ __defaults__( python_tests( name="tests", + overrides={ + "test_python_action_process_wrapper.py": dict( + dependencies=["contrib/examples/actions/noop.py"] + ) + }, ) From 9dacdf314ec9dcb267a6830a6fc022ea81a62737 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Tue, 10 Sep 2024 20:46:58 -0500 Subject: [PATCH 80/98] do not rely on the runner.yaml symlink in the test The symlink is not registered in pants. So far, this is the first time something has tried to access that file. So, just use the actual file instead of the symlink --- contrib/runners/python_runner/tests/unit/test_output_schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/runners/python_runner/tests/unit/test_output_schema.py b/contrib/runners/python_runner/tests/unit/test_output_schema.py index e997d8b60b..c5fd0d00e2 100644 --- a/contrib/runners/python_runner/tests/unit/test_output_schema.py +++ b/contrib/runners/python_runner/tests/unit/test_output_schema.py @@ -66,7 +66,7 @@ def setUpClass(cls): assert_submodules_are_checked_out() def test_adherence_to_output_schema(self): - config = self.loader(os.path.join(BASE_DIR, "../../runner.yaml")) + config = self.loader(os.path.join(BASE_DIR, "../../python_runner/runner.yaml")) runner = self._get_mock_runner_obj() runner.entry_point = PASCAL_ROW_ACTION_PATH runner.pre_run() From 1079b9c624e883c3a907cfc854d3a9608cdd20ef Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Tue, 10 Sep 2024 20:47:23 -0500 Subject: [PATCH 81/98] pants: record test dependencies that could not easily be inferred --- contrib/runners/python_runner/tests/unit/BUILD | 12 ++++++++++++ st2tests/st2tests/fixtures/packs/BUILD | 2 ++ 2 files changed, 14 insertions(+) diff --git a/contrib/runners/python_runner/tests/unit/BUILD b/contrib/runners/python_runner/tests/unit/BUILD index 9a24dba70a..c1479c4b4d 100644 --- a/contrib/runners/python_runner/tests/unit/BUILD +++ b/contrib/runners/python_runner/tests/unit/BUILD @@ -5,4 +5,16 @@ __defaults__( python_tests( name="tests", + overrides={ + "test_output_schema.py": dict( + dependencies=[ + "st2tests/st2tests/resources/packs/pythonactions/actions/pascal_row.py", + ], + ), + "test_pythonrunner.py": dict( + dependencies=[ + "st2tests/st2tests/resources/packs/pythonactions/actions", + ], + ), + }, ) diff --git a/st2tests/st2tests/fixtures/packs/BUILD b/st2tests/st2tests/fixtures/packs/BUILD index 025cf82aac..71ed30b00c 100644 --- a/st2tests/st2tests/fixtures/packs/BUILD +++ b/st2tests/st2tests/fixtures/packs/BUILD @@ -8,8 +8,10 @@ pack_metadata_in_git_submodule( sources=[ "test_content_version/pack.yaml", "test_content_version/**/*.yaml", + "!test_content_version/.github/workflows/*.yaml", "test_content_version/icon.png", "test_content_version/requirements.txt", + "test_content_version/.git", # file that is git ignored, but used by the tests ], ) From 4b3f8cb5bc99b47d88dbbd3dc5435667bbe59833 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 11 Sep 2024 11:17:32 -0500 Subject: [PATCH 82/98] pants: record test dependency on st2common.metrics.driver --- contrib/runners/python_runner/tests/unit/BUILD | 3 +++ 1 file changed, 3 insertions(+) diff --git a/contrib/runners/python_runner/tests/unit/BUILD b/contrib/runners/python_runner/tests/unit/BUILD index c1479c4b4d..2a106afcbb 100644 --- a/contrib/runners/python_runner/tests/unit/BUILD +++ b/contrib/runners/python_runner/tests/unit/BUILD @@ -15,6 +15,9 @@ python_tests( dependencies=[ "st2tests/st2tests/resources/packs/pythonactions/actions", ], + stevedore_namespaces=[ + "st2common.metrics.driver", + ], ), }, ) From 6471f25995b8f224ff72273e60392696e50092a8 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 11 Sep 2024 11:21:22 -0500 Subject: [PATCH 83/98] python_runner test: use dummy_pack_1 instead of core --- .../tests/unit/test_pythonrunner.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/contrib/runners/python_runner/tests/unit/test_pythonrunner.py b/contrib/runners/python_runner/tests/unit/test_pythonrunner.py index a178bd20ff..7c5e2e7b29 100644 --- a/contrib/runners/python_runner/tests/unit/test_pythonrunner.py +++ b/contrib/runners/python_runner/tests/unit/test_pythonrunner.py @@ -36,13 +36,14 @@ from st2common.constants.action import LIVEACTION_STATUS_TIMED_OUT from st2common.constants.action import MAX_PARAM_LENGTH from st2common.constants.pack import COMMON_LIB_DIR -from st2common.constants.pack import SYSTEM_PACK_NAME +from st2common.constants.pack import SYSTEM_PACK_NAMES from st2common.persistence.execution import ActionExecutionOutput from python_runner.python_action_wrapper import PythonActionWrapper from st2tests.base import RunnerTestCase from st2tests.base import CleanDbTestCase from st2tests.base import blocking_eventlet_spawn from st2tests.base import make_mock_stream_readline +from st2tests.fixtures.packs.dummy_pack_1.fixture import PACK_NAME as DUMMY_PACK_1 from st2tests.fixtures.packs.dummy_pack_9.fixture import PACK_PATH as DUMMY_PACK_9_PATH from st2tests.fixtures.packs.test_content_version_fixture.fixture import ( PACK_NAME as TEST_CONTENT_VERSION, @@ -100,6 +101,8 @@ MOCK_EXECUTION.id = "598dbf0c0640fd54bffc688b" +# Use DUMMY_PACK_1 instead of depending on everything in the core (SYSTEM_PACK_NAME) pack. +@mock.patch("st2common.util.sandboxing.SYSTEM_PACK_NAMES", [DUMMY_PACK_1, *SYSTEM_PACK_NAMES]) @mock.patch("python_runner.python_runner.sys", mock_sys) class PythonRunnerTestCase(RunnerTestCase, CleanDbTestCase): register_packs = True @@ -603,7 +606,7 @@ def test_pythonpath_env_var_contains_common_libs_config_enabled(self, mock_popen _, call_kwargs = mock_popen.call_args actual_env = call_kwargs["env"] - pack_common_lib_path = "fixtures/packs/core/lib" + pack_common_lib_path = f"fixtures/packs/{DUMMY_PACK_1}/lib" self.assertIn("PYTHONPATH", actual_env) self.assertIn(pack_common_lib_path, actual_env["PYTHONPATH"]) @@ -626,7 +629,7 @@ def test_pythonpath_env_var_not_contains_common_libs_config_disabled( _, call_kwargs = mock_popen.call_args actual_env = call_kwargs["env"] pack_common_lib_path = ( - "/mnt/src/storm/st2/st2tests/st2tests/fixtures/packs/core/lib" + f"/mnt/src/storm/st2/st2tests/st2tests/fixtures/packs/{DUMMY_PACK_1}/lib" ) self.assertIn("PYTHONPATH", actual_env) self.assertNotIn(pack_common_lib_path, actual_env["PYTHONPATH"]) @@ -994,7 +997,7 @@ def test_content_version_old_git_version(self, mock_run_command): runner.runner_parameters = {"content_version": "v0.10.0"} expected_msg = ( - r'Failed to create git worktree for pack "core": Installed git version ' + fr'Failed to create git worktree for pack "{DUMMY_PACK_1}": Installed git version ' "doesn't support git worktree command. To be able to utilize this " "functionality you need to use git >= 2.5.0." ) @@ -1015,7 +1018,7 @@ def test_content_version_pack_repo_not_git_repository(self, mock_run_command): runner.runner_parameters = {"content_version": "v0.10.0"} expected_msg = ( - r'Failed to create git worktree for pack "core": Pack directory ' + fr'Failed to create git worktree for pack "{DUMMY_PACK_1}": Pack directory ' '".*" is not a ' "git repository. To utilize this functionality, pack directory needs to " "be a git repository." @@ -1036,7 +1039,7 @@ def test_content_version_invalid_git_revision(self, mock_run_command): runner.runner_parameters = {"content_version": "vinvalid"} expected_msg = ( - r'Failed to create git worktree for pack "core": Invalid content_version ' + fr'Failed to create git worktree for pack "{DUMMY_PACK_1}": Invalid content_version ' '"vinvalid" provided. Make sure that git repository is up ' "to date and contains that revision." ) @@ -1052,7 +1055,7 @@ def test_missing_config_item_user_friendly_error(self): self.assertIsNotNone(output) self.assertIn("{}", output["stdout"]) self.assertIn("default_value", output["stdout"]) - self.assertIn('Config for pack "core" is missing key "key"', output["stderr"]) + self.assertIn(f'Config for pack "{DUMMY_PACK_1}" is missing key "key"', output["stderr"]) self.assertIn( 'make sure you run "st2ctl reload --register-configs"', output["stderr"] ) @@ -1107,7 +1110,7 @@ def _get_mock_action_obj(self): """ action = mock.Mock() action.ref = "dummy.action" - action.pack = SYSTEM_PACK_NAME + action.pack = DUMMY_PACK_1 action.entry_point = "foo.py" action.runner_type = {"name": "python-script"} return action From 45f6ceeef8ed5ef3c72a217c70addda4b45086cd Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 11 Sep 2024 14:36:22 -0500 Subject: [PATCH 84/98] python runner test depends on dummy_pack_5 and rbac backend --- .../runners/python_runner/tests/unit/BUILD | 1 + .../tests/unit/test_pythonrunner.py | 19 +++++++++---------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/contrib/runners/python_runner/tests/unit/BUILD b/contrib/runners/python_runner/tests/unit/BUILD index 2a106afcbb..39ad860aa4 100644 --- a/contrib/runners/python_runner/tests/unit/BUILD +++ b/contrib/runners/python_runner/tests/unit/BUILD @@ -17,6 +17,7 @@ python_tests( ], stevedore_namespaces=[ "st2common.metrics.driver", + "st2common.rbac.backend", ], ), }, diff --git a/contrib/runners/python_runner/tests/unit/test_pythonrunner.py b/contrib/runners/python_runner/tests/unit/test_pythonrunner.py index 7c5e2e7b29..0cd8550fb0 100644 --- a/contrib/runners/python_runner/tests/unit/test_pythonrunner.py +++ b/contrib/runners/python_runner/tests/unit/test_pythonrunner.py @@ -44,6 +44,7 @@ from st2tests.base import blocking_eventlet_spawn from st2tests.base import make_mock_stream_readline from st2tests.fixtures.packs.dummy_pack_1.fixture import PACK_NAME as DUMMY_PACK_1 +from st2tests.fixtures.packs.dummy_pack_5.fixture import PACK_NAME as DUMMY_PACK_5 from st2tests.fixtures.packs.dummy_pack_9.fixture import PACK_PATH as DUMMY_PACK_9_PATH from st2tests.fixtures.packs.test_content_version_fixture.fixture import ( PACK_NAME as TEST_CONTENT_VERSION, @@ -232,12 +233,10 @@ def test_simple_action_no_status_backward_compatibility(self): self.assertEqual(output["result"], [1, 2]) def test_simple_action_config_value_provided_overriden_in_datastore(self): - pack = "dummy_pack_5" user = "joe" # No values provided in the datastore - runner = self._get_mock_runner_obj_from_container(pack=pack, user=user) - + runner = self._get_mock_runner_obj_from_container(pack=DUMMY_PACK_5, user=user) self.assertEqual(runner._config["api_key"], "some_api_key") # static value self.assertEqual(runner._config["regions"], ["us-west-1"]) # static value self.assertEqual(runner._config["api_secret"], None) @@ -245,19 +244,19 @@ def test_simple_action_config_value_provided_overriden_in_datastore(self): # api_secret overriden in the datastore (user scoped value) config_service.set_datastore_value_for_config_key( - pack_name="dummy_pack_5", + pack_name=DUMMY_PACK_5, key_name="api_secret", - user="joe", + user=user, value="foosecret", secret=True, ) # private_key_path overriden in the datastore (global / non-user scoped value) config_service.set_datastore_value_for_config_key( - pack_name="dummy_pack_5", key_name="private_key_path", value="foopath" + pack_name=DUMMY_PACK_5, key_name="private_key_path", value="foopath" ) - runner = self._get_mock_runner_obj_from_container(pack=pack, user=user) + runner = self._get_mock_runner_obj_from_container(pack=DUMMY_PACK_5, user=user) self.assertEqual(runner._config["api_key"], "some_api_key") # static value self.assertEqual(runner._config["regions"], ["us-west-1"]) # static value self.assertEqual(runner._config["api_secret"], "foosecret") @@ -717,7 +716,7 @@ def test_python_action_wrapper_script_doesnt_get_added_to_sys_path(self): def test_python_action_wrapper_action_script_file_doesnt_exist_friendly_error(self): # File in a directory which is not a Python package wrapper = PythonActionWrapper( - pack="dummy_pack_5", file_path="/tmp/doesnt.exist", user="joe" + pack=DUMMY_PACK_5, file_path="/tmp/doesnt.exist", user="joe" ) expected_msg = ( @@ -727,7 +726,7 @@ def test_python_action_wrapper_action_script_file_doesnt_exist_friendly_error(se # File in a directory which is a Python package wrapper = PythonActionWrapper( - pack="dummy_pack_5", file_path=ACTION_1_PATH, user="joe" + pack=DUMMY_PACK_5, file_path=ACTION_1_PATH, user="joe" ) expected_msg = ( @@ -741,7 +740,7 @@ def test_python_action_wrapper_action_script_file_contains_invalid_syntax_friend self, ): wrapper = PythonActionWrapper( - pack="dummy_pack_5", file_path=ACTION_2_PATH, user="joe" + pack=DUMMY_PACK_5, file_path=ACTION_2_PATH, user="joe" ) expected_msg = ( r'Failed to load action class from file ".*?invalid_syntax.py" ' From cb1c6120b96618ec12bdfbda1940e9e9bf0d869e Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 11 Sep 2024 15:27:28 -0500 Subject: [PATCH 85/98] remove hard-coded references to "stanley" user in tests --- contrib/runners/orquesta_runner/tests/unit/test_basic.py | 3 ++- contrib/runners/orquesta_runner/tests/unit/test_context.py | 6 ++++-- .../orquesta_runner/tests/unit/test_error_handling.py | 3 ++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/contrib/runners/orquesta_runner/tests/unit/test_basic.py b/contrib/runners/orquesta_runner/tests/unit/test_basic.py index 7c9351423a..9b84c94b3a 100644 --- a/contrib/runners/orquesta_runner/tests/unit/test_basic.py +++ b/contrib/runners/orquesta_runner/tests/unit/test_basic.py @@ -21,6 +21,7 @@ import six from orquesta import statuses as wf_statuses +from oslo_config import cfg import st2tests @@ -108,7 +109,7 @@ def get_runner_class(cls, runner_name): runners_utils, "invoke_post_run", mock.MagicMock(return_value=None) ) def test_run_workflow(self): - username = "stanley" + username = cfg.CONF.system_user.user wf_meta = base.get_wf_fixture_meta_data(TEST_PACK_PATH, "sequential.yaml") wf_input = {"who": "Thanos"} lv_ac_db = lv_db_models.LiveActionDB( diff --git a/contrib/runners/orquesta_runner/tests/unit/test_context.py b/contrib/runners/orquesta_runner/tests/unit/test_context.py index d9e726d9a1..983614a041 100644 --- a/contrib/runners/orquesta_runner/tests/unit/test_context.py +++ b/contrib/runners/orquesta_runner/tests/unit/test_context.py @@ -19,6 +19,7 @@ import mock from orquesta import statuses as wf_statuses +from oslo_config import cfg import st2tests @@ -125,7 +126,7 @@ def test_runtime_context(self): expected_st2_ctx = { "action_execution_id": str(ac_ex_db.id), "api_url": "http://127.0.0.1/v1", - "user": "stanley", + "user": cfg.CONF.system_user.user, "pack": "orquesta_tests", "action": "orquesta_tests.runtime-context", "runner": "orquesta", @@ -208,9 +209,10 @@ def test_action_context_sys_user(self): self.assertEqual(wf_ex_db.status, wf_statuses.SUCCEEDED) self.assertEqual(lv_ac_db.status, ac_const.LIVEACTION_STATUS_SUCCEEDED) + user = cfg.CONF.system_user.user # Check result. expected_result = { - "output": {"msg": "stanley, All your base are belong to us!"} + "output": {"msg": f"{user}, All your base are belong to us!"} } self.assertDictEqual(lv_ac_db.result, expected_result) diff --git a/contrib/runners/orquesta_runner/tests/unit/test_error_handling.py b/contrib/runners/orquesta_runner/tests/unit/test_error_handling.py index 9a4dd1cd5b..9aafac018f 100644 --- a/contrib/runners/orquesta_runner/tests/unit/test_error_handling.py +++ b/contrib/runners/orquesta_runner/tests/unit/test_error_handling.py @@ -18,6 +18,7 @@ import mock from orquesta import statuses as wf_statuses +from oslo_config import cfg import st2tests @@ -954,7 +955,7 @@ def test_fail_manually_with_recovery_failure(self): mock.MagicMock(side_effect=[RUNNER_RESULT_FAILED]), ) def test_include_result_to_error_log(self): - username = "stanley" + username = cfg.CONF.system_user.user wf_meta = base.get_wf_fixture_meta_data(TEST_PACK_PATH, "sequential.yaml") wf_input = {"who": "Thanos"} lv_ac_db = lv_db_models.LiveActionDB( From 63e5c219016d8026f118ac05649ff6f962bd9326 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 11 Sep 2024 15:28:13 -0500 Subject: [PATCH 86/98] pants: add dpeendency metadata for orquesta unit tests --- contrib/runners/orquesta_runner/tests/BUILD | 5 ++++- contrib/runners/orquesta_runner/tests/unit/BUILD | 12 ++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/contrib/runners/orquesta_runner/tests/BUILD b/contrib/runners/orquesta_runner/tests/BUILD index e0764b1850..ea19a75243 100644 --- a/contrib/runners/orquesta_runner/tests/BUILD +++ b/contrib/runners/orquesta_runner/tests/BUILD @@ -2,7 +2,10 @@ __defaults__( all=dict( skip_pylint=True, entry_point_dependencies={ - "contrib/runners/orquesta_runner": ["st2common.runners.runner"], + "contrib/runners/orquesta_runner": [ + "st2common.runners.runner", + "orquesta.expressions.functions", + ], }, ) ) diff --git a/contrib/runners/orquesta_runner/tests/unit/BUILD b/contrib/runners/orquesta_runner/tests/unit/BUILD index 1daa501cd5..ef8ca05700 100644 --- a/contrib/runners/orquesta_runner/tests/unit/BUILD +++ b/contrib/runners/orquesta_runner/tests/unit/BUILD @@ -5,6 +5,18 @@ __defaults__( python_tests( name="tests", + stevedore_namespaces=[ + "st2common.metrics.driver", + "st2common.rbac.backend", + # the core pack uses all runners. + "st2common.runners.runner", + "orquesta.expressions.functions", + ], + overrides={ + "test_policies.py": dict( + dependencies=["st2actions/st2actions/policies/retry.py"] + ) + } ) python_test_utils( From 542e120d1b3d10563b326674ed421109f95fe39f Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 11 Sep 2024 15:30:34 -0500 Subject: [PATCH 87/98] add/update TODOs --- .../runners/local_runner/tests/integration/test_localrunner.py | 2 ++ pants-plugins/pack_metadata/python_rules/python_path_rules.py | 2 +- pants-plugins/pack_metadata/target_types.py | 1 + pants-plugins/uses_services/register.py | 1 + 4 files changed, 5 insertions(+), 1 deletion(-) diff --git a/contrib/runners/local_runner/tests/integration/test_localrunner.py b/contrib/runners/local_runner/tests/integration/test_localrunner.py index 1ca4f4275f..fca7fccb90 100644 --- a/contrib/runners/local_runner/tests/integration/test_localrunner.py +++ b/contrib/runners/local_runner/tests/integration/test_localrunner.py @@ -297,6 +297,7 @@ def test_action_stdout_and_stderr_is_stored_in_the_db_short_running_action( self.assertEqual(output_dbs[db_index_1].data, mock_stderr[0]) self.assertEqual(output_dbs[db_index_2].data, mock_stderr[1]) + # FIXME: This test assumes passwordless sudo (so sudo won't capture stdin) def test_shell_command_sudo_password_is_passed_to_sudo_binary(self): # Verify that sudo password is correctly passed to sudo binary via stdin models = self.fixtures_loader.load_models( @@ -355,6 +356,7 @@ def test_shell_command_sudo_password_is_passed_to_sudo_binary(self): self.assertEqual(index, len(sudo_passwords)) + # FIXME: This test assumes passwordless sudo (so sudo won't capture stdin) def test_shell_command_invalid_stdout_password(self): # Simulate message printed to stderr by sudo when invalid sudo password is provided models = self.fixtures_loader.load_models( diff --git a/pants-plugins/pack_metadata/python_rules/python_path_rules.py b/pants-plugins/pack_metadata/python_rules/python_path_rules.py index 3436c43286..30a92a0d55 100644 --- a/pants-plugins/pack_metadata/python_rules/python_path_rules.py +++ b/pants-plugins/pack_metadata/python_rules/python_path_rules.py @@ -120,7 +120,7 @@ async def inject_extra_sys_path_for_pack_tests( ) return PytestPluginSetup( # digest=EMPTY_DIGEST, - # extra_sys_path=pack_python_path.entries, # TODO: make pants support this + # extra_sys_path=pack_python_path.entries, # TODO: requires pants >2.23.0dev6 ) diff --git a/pants-plugins/pack_metadata/target_types.py b/pants-plugins/pack_metadata/target_types.py index 01d80c24ad..659eb312df 100644 --- a/pants-plugins/pack_metadata/target_types.py +++ b/pants-plugins/pack_metadata/target_types.py @@ -152,6 +152,7 @@ class PackContentResourceTarget(ResourceTarget): ResourceDependenciesField, PackContentResourceSourceField, PackContentResourceTypeField, + # TODO: implicit depenedency on runner entry point. If a python action, the python file should (?) get that dep too. ) help = "A single pack content resource file (mostly for metadata files)." diff --git a/pants-plugins/uses_services/register.py b/pants-plugins/uses_services/register.py index 6693b9b12d..24dc8037df 100644 --- a/pants-plugins/uses_services/register.py +++ b/pants-plugins/uses_services/register.py @@ -30,4 +30,5 @@ def rules(): *mongo_rules.rules(), *rabbitmq_rules.rules(), *redis_rules.rules(), + # TODO: Add check that checks that system user is present (suggest setting ST2TESTS_SYSTEM_USER env var if not, or even just default it to the current user) ] From 4ea924503697f950d0fda436b9d5005acc507c26 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 11 Sep 2024 15:31:18 -0500 Subject: [PATCH 88/98] pants: test more runners now, but temporarily skip pack tests --- .github/workflows/test.yaml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 7a0a6b85bd..4f6c461204 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -123,21 +123,22 @@ jobs: contrib/debug/:: contrib/default/:: contrib/hello_st2/:: + contrib/runners/action_chain_runner/:: contrib/runners/announcement_runner/:: contrib/runners/http_runner/:: contrib/runners/inquirer_runner/:: contrib/runners/local_runner/:: contrib/runners/noop_runner/:: + contrib/runners/orquesta_runner/:: contrib/runners/remote_runner/:: contrib/runners/winrm_runner/:: - contrib/chatops/:: - contrib/core/:: - contrib/examples/:: - contrib/linux/:: - contrib/packs/:: -# contrib/runners/action_chain_runner/:: -# contrib/runners/orquesta_runner/:: -# contrib/runners/python_runner/:: + # pack tests will not work until pants >2.23.0dev6 to get the PytestPluginSetup(extra_sys_path=...) field. + #contrib/chatops/:: + #contrib/core/:: + #contrib/examples/:: + #contrib/linux/:: + #contrib/packs/:: +# contrib/runners/python_runner/:: # TODO: figure out how to put .git metadata for test_content_version pack in sandbox - name: Upload pants log uses: actions/upload-artifact@v4 From a80e091f31a8932902fa693367f440a07be17994 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 11 Sep 2024 15:44:28 -0500 Subject: [PATCH 89/98] fmt w/ black --- contrib/runners/orquesta_runner/tests/unit/BUILD | 2 +- .../python_runner/tests/unit/test_pythonrunner.py | 14 +++++++++----- .../python_rules/python_path_rules_test.py | 4 +++- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/contrib/runners/orquesta_runner/tests/unit/BUILD b/contrib/runners/orquesta_runner/tests/unit/BUILD index ef8ca05700..c0435fb3db 100644 --- a/contrib/runners/orquesta_runner/tests/unit/BUILD +++ b/contrib/runners/orquesta_runner/tests/unit/BUILD @@ -16,7 +16,7 @@ python_tests( "test_policies.py": dict( dependencies=["st2actions/st2actions/policies/retry.py"] ) - } + }, ) python_test_utils( diff --git a/contrib/runners/python_runner/tests/unit/test_pythonrunner.py b/contrib/runners/python_runner/tests/unit/test_pythonrunner.py index 0cd8550fb0..22d9e14b36 100644 --- a/contrib/runners/python_runner/tests/unit/test_pythonrunner.py +++ b/contrib/runners/python_runner/tests/unit/test_pythonrunner.py @@ -103,7 +103,9 @@ # Use DUMMY_PACK_1 instead of depending on everything in the core (SYSTEM_PACK_NAME) pack. -@mock.patch("st2common.util.sandboxing.SYSTEM_PACK_NAMES", [DUMMY_PACK_1, *SYSTEM_PACK_NAMES]) +@mock.patch( + "st2common.util.sandboxing.SYSTEM_PACK_NAMES", [DUMMY_PACK_1, *SYSTEM_PACK_NAMES] +) @mock.patch("python_runner.python_runner.sys", mock_sys) class PythonRunnerTestCase(RunnerTestCase, CleanDbTestCase): register_packs = True @@ -996,7 +998,7 @@ def test_content_version_old_git_version(self, mock_run_command): runner.runner_parameters = {"content_version": "v0.10.0"} expected_msg = ( - fr'Failed to create git worktree for pack "{DUMMY_PACK_1}": Installed git version ' + rf'Failed to create git worktree for pack "{DUMMY_PACK_1}": Installed git version ' "doesn't support git worktree command. To be able to utilize this " "functionality you need to use git >= 2.5.0." ) @@ -1017,7 +1019,7 @@ def test_content_version_pack_repo_not_git_repository(self, mock_run_command): runner.runner_parameters = {"content_version": "v0.10.0"} expected_msg = ( - fr'Failed to create git worktree for pack "{DUMMY_PACK_1}": Pack directory ' + rf'Failed to create git worktree for pack "{DUMMY_PACK_1}": Pack directory ' '".*" is not a ' "git repository. To utilize this functionality, pack directory needs to " "be a git repository." @@ -1038,7 +1040,7 @@ def test_content_version_invalid_git_revision(self, mock_run_command): runner.runner_parameters = {"content_version": "vinvalid"} expected_msg = ( - fr'Failed to create git worktree for pack "{DUMMY_PACK_1}": Invalid content_version ' + rf'Failed to create git worktree for pack "{DUMMY_PACK_1}": Invalid content_version ' '"vinvalid" provided. Make sure that git repository is up ' "to date and contains that revision." ) @@ -1054,7 +1056,9 @@ def test_missing_config_item_user_friendly_error(self): self.assertIsNotNone(output) self.assertIn("{}", output["stdout"]) self.assertIn("default_value", output["stdout"]) - self.assertIn(f'Config for pack "{DUMMY_PACK_1}" is missing key "key"', output["stderr"]) + self.assertIn( + f'Config for pack "{DUMMY_PACK_1}" is missing key "key"', output["stderr"] + ) self.assertIn( 'make sure you run "st2ctl reload --register-configs"', output["stderr"] ) diff --git a/pants-plugins/pack_metadata/python_rules/python_path_rules_test.py b/pants-plugins/pack_metadata/python_rules/python_path_rules_test.py index a7514c4aaf..3fb3214fd6 100644 --- a/pants-plugins/pack_metadata/python_rules/python_path_rules_test.py +++ b/pants-plugins/pack_metadata/python_rules/python_path_rules_test.py @@ -166,4 +166,6 @@ def test_inject_extra_sys_path_for_pack_tests( target = rule_runner.get_target(address) result = rule_runner.request(PytestPluginSetup, (PytestPackTestRequest(target),)) assert result.digest == EMPTY_DIGEST - assert result.extra_sys_path == expected # TODO: pants does not have this attribute yet. + assert ( + result.extra_sys_path == expected + ) # TODO: pants does not have this attribute yet. From a14721f6579f062f96970e87bd5c48406bedede7 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 12 Sep 2024 13:23:48 -0500 Subject: [PATCH 90/98] pants: capture .git/modules so pants can run pythonrunner tests --- .github/workflows/test.yaml | 2 +- BUILD | 23 +++++++++++++++++++ BUILD.environment | 23 +++++++++++++++++++ .../runners/python_runner/tests/unit/BUILD | 2 ++ pants.toml | 4 ++++ st2tests/st2tests/fixtures/packs/BUILD | 3 +-- 6 files changed, 54 insertions(+), 3 deletions(-) create mode 100644 BUILD.environment diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 4f6c461204..bb9e779749 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -130,6 +130,7 @@ jobs: contrib/runners/local_runner/:: contrib/runners/noop_runner/:: contrib/runners/orquesta_runner/:: + contrib/runners/python_runner/:: contrib/runners/remote_runner/:: contrib/runners/winrm_runner/:: # pack tests will not work until pants >2.23.0dev6 to get the PytestPluginSetup(extra_sys_path=...) field. @@ -138,7 +139,6 @@ jobs: #contrib/examples/:: #contrib/linux/:: #contrib/packs/:: -# contrib/runners/python_runner/:: # TODO: figure out how to put .git metadata for test_content_version pack in sandbox - name: Upload pants log uses: actions/upload-artifact@v4 diff --git a/BUILD b/BUILD index a56fcf6b6f..25c87e41c7 100644 --- a/BUILD +++ b/BUILD @@ -93,3 +93,26 @@ file( name="logs_directory", source="logs/.gitignore", ) + +files( + name="gitmodules", + sources=[ + ".gitmodules", + "**/.git", + ], +) + +shell_command( + name="capture_git_modules", + environment="in_repo_workspace", + command="cp -r .git/modules {chroot}/.git", + tools=["cp"], + # execution_dependencies allows pants to invalidate the output + # of this command if the .gitmodules file changes (for example: + # if a submodule gets updated to a different commit). + # Theoretically, nothing else should modify .git/modules/. + execution_dependencies=[":gitmodules"], + output_dependencies=[":gitmodules"], + output_directories=[".git/modules"], + workdir="/", +) diff --git a/BUILD.environment b/BUILD.environment new file mode 100644 index 0000000000..5c1f26cdd3 --- /dev/null +++ b/BUILD.environment @@ -0,0 +1,23 @@ +# Everything listed in pants.toml [evironments-preview.names] should be defined here. +# Relevant docs: +# - https://www.pantsbuild.org/stable/docs/using-pants/environments +# - https://www.pantsbuild.org/stable/reference/targets/experimental_workspace_environment +# - https://www.pantsbuild.org/stable/reference/targets/local_environment +# - https://www.pantsbuild.org/stable/reference/targets/docker_environment + +# This file MUST NOT use any macros. + +experimental_workspace_environment( + name="in_repo_workspace", + description=( + """ + This allows shell_command and similar to in the repo, instead of in a sandbox. + Only use this environment for commands or goals that are idempotent. + Ideally, such commands do NOT change anything in the repo. + + If you need to capture output, note that output gets captured from a temporary + sandbox, not from the repo root. So, you may need to copy output files into + the sandbox with something like `cp path/to/file {chroot}/path/to/file`. + """ + ), +) diff --git a/contrib/runners/python_runner/tests/unit/BUILD b/contrib/runners/python_runner/tests/unit/BUILD index 39ad860aa4..656e54f328 100644 --- a/contrib/runners/python_runner/tests/unit/BUILD +++ b/contrib/runners/python_runner/tests/unit/BUILD @@ -9,11 +9,13 @@ python_tests( "test_output_schema.py": dict( dependencies=[ "st2tests/st2tests/resources/packs/pythonactions/actions/pascal_row.py", + "//:capture_git_modules", ], ), "test_pythonrunner.py": dict( dependencies=[ "st2tests/st2tests/resources/packs/pythonactions/actions", + "//:capture_git_modules", ], stevedore_namespaces=[ "st2common.metrics.driver", diff --git a/pants.toml b/pants.toml index 0bd79c9050..e4a673e2b8 100644 --- a/pants.toml +++ b/pants.toml @@ -246,5 +246,9 @@ extra_env_vars = [ [twine] install_from_resolve = "twine" +[environments-preview.names] +# https://www.pantsbuild.org/stable/docs/using-pants/environments +in_repo_workspace = "//:in_repo_workspace" + [cli.alias] --all-changed = "--changed-since=HEAD --changed-dependents=transitive" diff --git a/st2tests/st2tests/fixtures/packs/BUILD b/st2tests/st2tests/fixtures/packs/BUILD index 71ed30b00c..a5006ef9d1 100644 --- a/st2tests/st2tests/fixtures/packs/BUILD +++ b/st2tests/st2tests/fixtures/packs/BUILD @@ -8,11 +8,10 @@ pack_metadata_in_git_submodule( sources=[ "test_content_version/pack.yaml", "test_content_version/**/*.yaml", - "!test_content_version/.github/workflows/*.yaml", "test_content_version/icon.png", "test_content_version/requirements.txt", - "test_content_version/.git", # file that is git ignored, but used by the tests ], + # NOTE: If you need the git metadata, make sure to depend on //:capture_git_modules ) st2_shell_sources_and_resources( From e90c2c0ca35355e09156af541223851bf1026516 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 12 Sep 2024 15:01:20 -0500 Subject: [PATCH 91/98] pants: use new 2.23 features --- contrib/chatops/BUILD | 2 ++ pants-plugins/pack_metadata/python_rules/python_path_rules.py | 2 +- .../pack_metadata/python_rules/python_path_rules_test.py | 4 +--- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/contrib/chatops/BUILD b/contrib/chatops/BUILD index 1a74d30186..888be3a426 100644 --- a/contrib/chatops/BUILD +++ b/contrib/chatops/BUILD @@ -1,3 +1,5 @@ +__defaults__(all=dict(inject_pack_python_path=True)) + pack_metadata( name="metadata", ) diff --git a/pants-plugins/pack_metadata/python_rules/python_path_rules.py b/pants-plugins/pack_metadata/python_rules/python_path_rules.py index 30a92a0d55..314ecd7bf9 100644 --- a/pants-plugins/pack_metadata/python_rules/python_path_rules.py +++ b/pants-plugins/pack_metadata/python_rules/python_path_rules.py @@ -120,7 +120,7 @@ async def inject_extra_sys_path_for_pack_tests( ) return PytestPluginSetup( # digest=EMPTY_DIGEST, - # extra_sys_path=pack_python_path.entries, # TODO: requires pants >2.23.0dev6 + extra_sys_path=pack_python_path.entries, ) diff --git a/pants-plugins/pack_metadata/python_rules/python_path_rules_test.py b/pants-plugins/pack_metadata/python_rules/python_path_rules_test.py index 3fb3214fd6..74ff010b40 100644 --- a/pants-plugins/pack_metadata/python_rules/python_path_rules_test.py +++ b/pants-plugins/pack_metadata/python_rules/python_path_rules_test.py @@ -166,6 +166,4 @@ def test_inject_extra_sys_path_for_pack_tests( target = rule_runner.get_target(address) result = rule_runner.request(PytestPluginSetup, (PytestPackTestRequest(target),)) assert result.digest == EMPTY_DIGEST - assert ( - result.extra_sys_path == expected - ) # TODO: pants does not have this attribute yet. + assert result.extra_sys_path == expected From de25a79705c89e79dfef42ec7ae079f754f886f0 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 12 Sep 2024 16:01:46 -0500 Subject: [PATCH 92/98] pants: reenable testing pack tests --- .github/workflows/test.yaml | 11 +++++------ contrib/chatops/actions/BUILD | 8 +++++++- contrib/chatops/actions/templates/BUILD | 3 +++ contrib/chatops/tests/BUILD | 5 ++++- contrib/packs/tests/BUILD | 18 ++++++++++++++++++ 5 files changed, 37 insertions(+), 8 deletions(-) create mode 100644 contrib/chatops/actions/templates/BUILD diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index bb9e779749..c7e487dc81 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -120,9 +120,14 @@ jobs: st2stream/:: st2auth/:: st2api/:: + contrib/chatops/:: + contrib/core/:: contrib/debug/:: contrib/default/:: + contrib/examples/:: contrib/hello_st2/:: + contrib/linux/:: + contrib/packs/:: contrib/runners/action_chain_runner/:: contrib/runners/announcement_runner/:: contrib/runners/http_runner/:: @@ -133,12 +138,6 @@ jobs: contrib/runners/python_runner/:: contrib/runners/remote_runner/:: contrib/runners/winrm_runner/:: - # pack tests will not work until pants >2.23.0dev6 to get the PytestPluginSetup(extra_sys_path=...) field. - #contrib/chatops/:: - #contrib/core/:: - #contrib/examples/:: - #contrib/linux/:: - #contrib/packs/:: - name: Upload pants log uses: actions/upload-artifact@v4 diff --git a/contrib/chatops/actions/BUILD b/contrib/chatops/actions/BUILD index db46e8d6c9..4a1bef88f4 100644 --- a/contrib/chatops/actions/BUILD +++ b/contrib/chatops/actions/BUILD @@ -1 +1,7 @@ -python_sources() +python_sources( + overrides={ + "format_execution_result.py": dict( + dependencies=["./templates"], + ), + }, +) diff --git a/contrib/chatops/actions/templates/BUILD b/contrib/chatops/actions/templates/BUILD new file mode 100644 index 0000000000..96d5a456eb --- /dev/null +++ b/contrib/chatops/actions/templates/BUILD @@ -0,0 +1,3 @@ +resources( + sources=["*.j2"], +) diff --git a/contrib/chatops/tests/BUILD b/contrib/chatops/tests/BUILD index 6925caed4b..b6c7dac80d 100644 --- a/contrib/chatops/tests/BUILD +++ b/contrib/chatops/tests/BUILD @@ -13,6 +13,9 @@ files( python_tests( name="tests", - dependencies=[":fixtures"], + dependencies=[ + ":fixtures", + "contrib/chatops:metadata", + ], skip_pylint=True, ) diff --git a/contrib/packs/tests/BUILD b/contrib/packs/tests/BUILD index 4583892bc4..ca0f765ed7 100644 --- a/contrib/packs/tests/BUILD +++ b/contrib/packs/tests/BUILD @@ -8,4 +8,22 @@ __defaults__( python_tests( skip_pylint=True, + overrides={ + "test_action_aliases.py": dict( + dependencies=[ + # test needs the pack and aliases metadata + "contrib/packs:metadata", + ], + ), + "test_action_unload.py": dict( + stevedore_namespaces=[ + "st2common.metrics.driver", + ], + entry_point_dependencies={ + "contrib/runners/http_runner": ["st2common.runners.runner"], + "contrib/runners/local_runner": ["st2common.runners.runner"], + "contrib/runners/python_runner": ["st2common.runners.runner"], + }, + ), + }, ) From b5545a8191525a010011ea6f80fb51e7a949b260 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 12 Sep 2024 16:06:36 -0500 Subject: [PATCH 93/98] pants: separate pack from unit tests --- .github/workflows/test.yaml | 49 +++++++++++++----------------------- contrib/chatops/tests/BUILD | 2 +- contrib/core/tests/BUILD | 2 +- contrib/examples/tests/BUILD | 2 +- contrib/linux/tests/BUILD | 2 +- contrib/packs/tests/BUILD | 2 +- 6 files changed, 23 insertions(+), 36 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index c7e487dc81..f8c92e2ba0 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -98,46 +98,33 @@ jobs: run: | pants test pants-plugins/:: + # We do not support running pytest everywhere yet. When we do it will be simply: + # pants test :: + # Until then, we need to manually adjust this command line to test what we can. + # So far, this includes unit, pack, and pants-plugins tests. + # TODO: run integration tests + - name: Unit Tests env: # Github Actions uses the 'runner' user, so use that instead of stanley. ST2TESTS_SYSTEM_USER: 'runner' - # We do not support running pytest everywhere yet. When we do it will be simply: - # pants test :: - # Until then, we need to manually adjust this command line to test what we can. run: > pants --python-bootstrap-search-path=[] --python-bootstrap-search-path=${{ steps.python.outputs.python-path }} --tag=unit - test - pylint_plugins/:: - st2tests/:: - st2client/:: - st2common/:: - st2actions/:: - st2reactor/:: - st2stream/:: - st2auth/:: - st2api/:: - contrib/chatops/:: - contrib/core/:: - contrib/debug/:: - contrib/default/:: - contrib/examples/:: - contrib/hello_st2/:: - contrib/linux/:: - contrib/packs/:: - contrib/runners/action_chain_runner/:: - contrib/runners/announcement_runner/:: - contrib/runners/http_runner/:: - contrib/runners/inquirer_runner/:: - contrib/runners/local_runner/:: - contrib/runners/noop_runner/:: - contrib/runners/orquesta_runner/:: - contrib/runners/python_runner/:: - contrib/runners/remote_runner/:: - contrib/runners/winrm_runner/:: + test '::' + + - name: Pack Tests + env: + # Github Actions uses the 'runner' user, so use that instead of stanley. + ST2TESTS_SYSTEM_USER: 'runner' + run: > + pants + --python-bootstrap-search-path=[] + --python-bootstrap-search-path=${{ steps.python.outputs.python-path }} + --tag=pack + test '::' - name: Upload pants log uses: actions/upload-artifact@v4 diff --git a/contrib/chatops/tests/BUILD b/contrib/chatops/tests/BUILD index b6c7dac80d..ead8561daa 100644 --- a/contrib/chatops/tests/BUILD +++ b/contrib/chatops/tests/BUILD @@ -2,7 +2,7 @@ __dependents_rules__(("*", "/**", "!*")) __defaults__( - {(python_test, python_tests): dict(tags=["unit", "pack"])}, + {python_test: dict(tags=["pack"])}, extend=True, ) diff --git a/contrib/core/tests/BUILD b/contrib/core/tests/BUILD index 205c90c328..144960edf3 100644 --- a/contrib/core/tests/BUILD +++ b/contrib/core/tests/BUILD @@ -2,7 +2,7 @@ __dependents_rules__(("*", "/**", "!*")) __defaults__( - {(python_test, python_tests): dict(tags=["unit", "pack"])}, + {python_test: dict(tags=["pack"])}, extend=True, ) diff --git a/contrib/examples/tests/BUILD b/contrib/examples/tests/BUILD index 4583892bc4..25a2e7cc4b 100644 --- a/contrib/examples/tests/BUILD +++ b/contrib/examples/tests/BUILD @@ -2,7 +2,7 @@ __dependents_rules__(("*", "/**", "!*")) __defaults__( - {(python_test, python_tests): dict(tags=["unit", "pack"])}, + {python_test: dict(tags=["pack"])}, extend=True, ) diff --git a/contrib/linux/tests/BUILD b/contrib/linux/tests/BUILD index 4583892bc4..25a2e7cc4b 100644 --- a/contrib/linux/tests/BUILD +++ b/contrib/linux/tests/BUILD @@ -2,7 +2,7 @@ __dependents_rules__(("*", "/**", "!*")) __defaults__( - {(python_test, python_tests): dict(tags=["unit", "pack"])}, + {python_test: dict(tags=["pack"])}, extend=True, ) diff --git a/contrib/packs/tests/BUILD b/contrib/packs/tests/BUILD index ca0f765ed7..c8265214ca 100644 --- a/contrib/packs/tests/BUILD +++ b/contrib/packs/tests/BUILD @@ -2,7 +2,7 @@ __dependents_rules__(("*", "/**", "!*")) __defaults__( - {(python_test, python_tests): dict(tags=["unit", "pack"])}, + {python_test: dict(tags=["pack"])}, extend=True, ) From 1a8a911e025a9a6f7b2bd6157b42e7400468376c Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 12 Sep 2024 16:35:10 -0500 Subject: [PATCH 94/98] pants: workaround GHA using fetch-depth=1 for submodules I could change fetch-depth, but that would change it for both submodules and for the st2.git checkout. --- BUILD | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/BUILD b/BUILD index 25c87e41c7..26d3fe9998 100644 --- a/BUILD +++ b/BUILD @@ -102,6 +102,11 @@ files( ], ) +run_shell_command( + name="git_submodules_fetch", + command="git submodule foreach 'git fetch --all'", +) + shell_command( name="capture_git_modules", environment="in_repo_workspace", @@ -111,7 +116,10 @@ shell_command( # of this command if the .gitmodules file changes (for example: # if a submodule gets updated to a different commit). # Theoretically, nothing else should modify .git/modules/. - execution_dependencies=[":gitmodules"], + execution_dependencies=[ + ":gitmodules", + ":git_submodules_fetch", + ], output_dependencies=[":gitmodules"], output_directories=[".git/modules"], workdir="/", From 1e13d28ef5227af56de4bc56ec32ed8d7ef3c996 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 12 Sep 2024 17:00:12 -0500 Subject: [PATCH 95/98] pants: use GHA task to fetch submodules instead of pants --- .github/workflows/test.yaml | 7 +++++++ BUILD | 10 +--------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index f8c92e2ba0..31756b0498 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -76,6 +76,13 @@ jobs: with: # a test uses a submodule, and pants needs access to it to calculate deps. submodules: 'true' + # sadly, the submodule will only have fetch-depth=1, which is what we want + # for st2.git, but not for the submodules. We still want actions/checkout + # to do the initial checkout, however, so that it adds auth for fetching + # in the submodule. + + - name: Fetch repository submodules + run: git submodule update --init --recursive --remote - name: 'Set up Python (${{ matrix.python-version }})' id: python diff --git a/BUILD b/BUILD index 26d3fe9998..25c87e41c7 100644 --- a/BUILD +++ b/BUILD @@ -102,11 +102,6 @@ files( ], ) -run_shell_command( - name="git_submodules_fetch", - command="git submodule foreach 'git fetch --all'", -) - shell_command( name="capture_git_modules", environment="in_repo_workspace", @@ -116,10 +111,7 @@ shell_command( # of this command if the .gitmodules file changes (for example: # if a submodule gets updated to a different commit). # Theoretically, nothing else should modify .git/modules/. - execution_dependencies=[ - ":gitmodules", - ":git_submodules_fetch", - ], + execution_dependencies=[":gitmodules"], output_dependencies=[":gitmodules"], output_directories=[".git/modules"], workdir="/", From d9a115ddbc0fdc57dd0499a5e9ced0672b53a50e Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 12 Sep 2024 17:41:51 -0500 Subject: [PATCH 96/98] pants: try again to get git submodules working --- .github/workflows/test.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 31756b0498..59380e6f40 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -75,14 +75,17 @@ jobs: uses: actions/checkout@v4 with: # a test uses a submodule, and pants needs access to it to calculate deps. - submodules: 'true' + submodules: 'recursive' # sadly, the submodule will only have fetch-depth=1, which is what we want # for st2.git, but not for the submodules. We still want actions/checkout # to do the initial checkout, however, so that it adds auth for fetching # in the submodule. - name: Fetch repository submodules - run: git submodule update --init --recursive --remote + run: | + git submodule update --init --recursive --remote + git submodule status + git submodule foreach 'git tag' - name: 'Set up Python (${{ matrix.python-version }})' id: python From 3ff52e8223bda4e915c58e71c626000a227c84ea Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 12 Sep 2024 17:49:49 -0500 Subject: [PATCH 97/98] pants: try again to get git submodules working --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 59380e6f40..4df959ffa9 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -83,8 +83,8 @@ jobs: - name: Fetch repository submodules run: | - git submodule update --init --recursive --remote git submodule status + git submodule foreach 'git fetch --all --tags' git submodule foreach 'git tag' - name: 'Set up Python (${{ matrix.python-version }})' From a026915f91d83c0894b5b8534ad14cce4ce93a30 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 12 Sep 2024 18:08:45 -0500 Subject: [PATCH 98/98] pants: ignore pylint error --- contrib/examples/actions/pythonactions/isprime.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/contrib/examples/actions/pythonactions/isprime.py b/contrib/examples/actions/pythonactions/isprime.py index 5116831a37..65294a5619 100644 --- a/contrib/examples/actions/pythonactions/isprime.py +++ b/contrib/examples/actions/pythonactions/isprime.py @@ -15,7 +15,8 @@ import math -from environ import get_environ +# TODO: extend pants and pants-plugins/pack_metadata to add lib dirs extra_sys_path for pylint +from environ import get_environ # pylint: disable=E0401 from st2common.runners.base_action import Action