Run more tests via Pants + pytest

.github/workflows/test.yaml

@@ -75,9 +75,20 @@ jobs:
         uses: actions/checkout@v4
         with:
           # a test uses a submodule, and pants needs access to it to calculate deps.
-          submodules: 'true'
+          submodules: 'recursive'
+          # sadly, the submodule will only have fetch-depth=1, which is what we want
+          # for st2.git, but not for the submodules. We still want actions/checkout
+          # to do the initial checkout, however, so that it adds auth for fetching
+          # in the submodule.
+
+      - name: Fetch repository submodules
+        run: |
+          git submodule status
+          git submodule foreach 'git fetch --all --tags'
+          git submodule foreach 'git tag'
 
       - name: 'Set up Python (${{ matrix.python-version }})'
+        id: python
         uses: actions/setup-python@v5
         with:
           python-version: '${{ matrix.python-version }}'
@@ -92,16 +103,42 @@ jobs:
           # To ignore a bad cache, bump the cache* integer.
           gha-cache-key: cache0-py${{ matrix.python-version }}
 
-      - name: Test
-        # We do not support running pytest everywhere yet. When we do it will be simply:
-        #   pants test ::
-        # Until then, we need to manually adjust this command line to test what we can.
+      - name: Test pants-plugins
+        if: ${{ matrix.python-version-short == '3.9' }}
         run: |
-          pants test pylint_plugins/:: pants-plugins/::
+          pants test pants-plugins/::
+
+      # We do not support running pytest everywhere yet. When we do it will be simply:
+      #   pants test ::
+      # Until then, we need to manually adjust this command line to test what we can.
+      # So far, this includes unit, pack, and pants-plugins tests.
+      # TODO: run integration tests
+
+      - name: Unit Tests
+        env:
+          # Github Actions uses the 'runner' user, so use that instead of stanley.
+          ST2TESTS_SYSTEM_USER: 'runner'
+        run: >
+          pants
+          --python-bootstrap-search-path=[]
+          --python-bootstrap-search-path=${{ steps.python.outputs.python-path }}
+          --tag=unit
+          test '::'
+
+      - name: Pack Tests
+        env:
+          # Github Actions uses the 'runner' user, so use that instead of stanley.
+          ST2TESTS_SYSTEM_USER: 'runner'
+        run: >
+          pants
+          --python-bootstrap-search-path=[]
+          --python-bootstrap-search-path=${{ steps.python.outputs.python-path }}
+          --tag=pack
+          test '::'
 
       - name: Upload pants log
         uses: actions/upload-artifact@v4
         with:
           name: pants-log-py${{ matrix.python-version }}
           path: .pants.d/pants.log
-        if: always()  # We want the log even on failures.
+        if: ${{ always() }}  # We want the log even on failures.

BUILD

@@ -27,6 +27,36 @@ python_requirements(
                 "st2auth/st2auth/backends/constants.py",
             ]
         ),
+        # make sure anything that uses st2-rbac-backend gets its deps
+        "st2-rbac-backend": dict(
+            dependencies=[
+                # alphabetical order
+                "st2common/st2common/config.py",
+                "st2common/st2common/constants/keyvalue.py",
+                "st2common/st2common/constants/triggers.py",
+                "st2common/st2common/content/loader.py",
+                "st2common/st2common/exceptions/db.py",
+                "st2common/st2common/exceptions/rbac.py",
+                "st2common/st2common/log.py",
+                "st2common/st2common/models/api/rbac.py",
+                "st2common/st2common/models/db/action.py",
+                "st2common/st2common/models/db/auth.py",
+                "st2common/st2common/models/db/pack.py",
+                "st2common/st2common/models/db/rbac.py",
+                "st2common/st2common/models/db/webhook.py",
+                "st2common/st2common/models/system/common.py",
+                "st2common/st2common/persistence/auth.py",
+                "st2common/st2common/persistence/execution.py",
+                "st2common/st2common/persistence/rbac.py",
+                "st2common/st2common/rbac/backends/__init__.py",
+                "st2common/st2common/rbac/backends/base.py",
+                "st2common/st2common/rbac/types.py",
+                "st2common/st2common/script_setup.py",
+                "st2common/st2common/util/action_db.py",
+                "st2common/st2common/util/misc.py",
+                "st2common/st2common/util/uid.py",
+            ]
+        ),
     },
 )
 
@@ -38,6 +68,13 @@ target(
     ],
 )
 
+target(
+    name="rbac_backends",
+    dependencies=[
+        "//:reqs#st2-rbac-backend",
+    ],
+)
+
 python_test_utils(
     name="test_utils",
     skip_pylint=True,
@@ -51,3 +88,31 @@ file(
 shell_sources(
     name="root",
 )
+
+file(
+    name="logs_directory",
+    source="logs/.gitignore",
+)
+
+files(
+    name="gitmodules",
+    sources=[
+        ".gitmodules",
+        "**/.git",
+    ],
+)
+
+shell_command(
+    name="capture_git_modules",
+    environment="in_repo_workspace",
+    command="cp -r .git/modules {chroot}/.git",
+    tools=["cp"],
+    # execution_dependencies allows pants to invalidate the output
+    # of this command if the .gitmodules file changes (for example:
+    # if a submodule gets updated to a different commit).
+    # Theoretically, nothing else should modify .git/modules/.
+    execution_dependencies=[":gitmodules"],
+    output_dependencies=[":gitmodules"],
+    output_directories=[".git/modules"],
+    workdir="/",
+)

BUILD.environment

@@ -0,0 +1,23 @@
+# Everything listed in pants.toml [evironments-preview.names] should be defined here.
+# Relevant docs:
+# - https://www.pantsbuild.org/stable/docs/using-pants/environments
+# - https://www.pantsbuild.org/stable/reference/targets/experimental_workspace_environment
+# - https://www.pantsbuild.org/stable/reference/targets/local_environment
+# - https://www.pantsbuild.org/stable/reference/targets/docker_environment
+
+# This file MUST NOT use any macros.
+
+experimental_workspace_environment(
+    name="in_repo_workspace",
+    description=(
+        """
+        This allows shell_command and similar to in the repo, instead of in a sandbox.
+        Only use this environment for commands or goals that are idempotent.
+        Ideally, such commands do NOT change anything in the repo.
+
+        If you need to capture output, note that output gets captured from a temporary
+        sandbox, not from the repo root. So, you may need to copy output files into
+        the sandbox with something like `cp path/to/file {chroot}/path/to/file`.
+        """
+    ),
+)

CHANGELOG.rst

@@ -33,7 +33,7 @@ Added
 * Continue introducing `pants <https://www.pantsbuild.org/docs>`_ to improve DX (Developer Experience)
   working on StackStorm, improve our security posture, and improve CI reliability thanks in part
   to pants' use of PEX lockfiles. This is not a user-facing addition.
-  #6118 #6141 #6133 #6120 #6181 #6183 #6200 #6237
+  #6118 #6141 #6133 #6120 #6181 #6183 #6200 #6237 #6229
   Contributed by @cognifloyd
 * Build of ST2 EL9 packages #6153
   Contributed by @amanda11

contrib/chatops/BUILD

@@ -1,3 +1,5 @@
+__defaults__(all=dict(inject_pack_python_path=True))
+
 pack_metadata(
     name="metadata",
 )

contrib/chatops/actions/BUILD

@@ -1 +1,7 @@
-python_sources()
+python_sources(
+    overrides={
+        "format_execution_result.py": dict(
+            dependencies=["./templates"],
+        ),
+    },
+)

contrib/chatops/actions/templates/BUILD

@@ -0,0 +1,3 @@
+resources(
+    sources=["*.j2"],
+)

contrib/chatops/tests/BUILD

@@ -1,13 +1,21 @@
 # tests can only be dependencies of other tests in this directory
 __dependents_rules__(("*", "/**", "!*"))
 
+__defaults__(
+    {python_test: dict(tags=["pack"])},
+    extend=True,
+)
+
 files(
     name="fixtures",
     sources=["fixtures/*.json"],
 )
 
 python_tests(
     name="tests",
-    dependencies=[":fixtures"],
+    dependencies=[
+        ":fixtures",
+        "contrib/chatops:metadata",
+    ],
     skip_pylint=True,
 )

contrib/core/BUILD

@@ -1,3 +1,5 @@
+__defaults__(all=dict(inject_pack_python_path=True))
+
 pack_metadata(
     name="metadata",
 )
@@ -8,5 +10,9 @@ python_requirements(
 )
 
 python_sources(
-    dependencies=[":metadata"],
+    dependencies=[
+        ":metadata",
+        "./actions",
+        "./actions/send_mail:send_mail_resources",
+    ],
 )

contrib/core/actions/send_mail/BUILD

@@ -1,4 +1,5 @@
-shell_source(
-    source="send_mail",
+st2_shell_sources_and_resources(
+    name="send_mail",
+    sources=["send_mail"],
     skip_shellcheck=True,
 )

contrib/core/tests/BUILD

@@ -1,8 +1,14 @@
 # tests can only be dependencies of other tests in this directory
 __dependents_rules__(("*", "/**", "!*"))
 
+__defaults__(
+    {python_test: dict(tags=["pack"])},
+    extend=True,
+)
+
 python_tests(
     skip_pylint=True,
+    # st2_pack_dir="..",  # directory containing pack.yaml
     overrides={
         "test_action_sendmail.py": dict(
             dependencies=[
@@ -11,6 +17,7 @@ python_tests(
                 # Use contrib/core as the canonical copy.
                 "contrib/core:reqs#mail-parser",
             ],
+            uses=["mongo"],
         ),
     },
 )

contrib/debug/BUILD

@@ -1,3 +1,5 @@
+__defaults__(all=dict(inject_pack_python_path=True))
+
 pack_metadata(
     name="metadata",
 )

contrib/default/BUILD

@@ -1,3 +1,5 @@
+__defaults__(all=dict(inject_pack_python_path=True))
+
 pack_metadata(
     name="metadata",
 )

contrib/examples/BUILD

@@ -1,3 +1,5 @@
+__defaults__(all=dict(inject_pack_python_path=True))
+
 pack_metadata(
     name="metadata",
 )

contrib/examples/actions/pythonactions/isprime.py

@@ -15,7 +15,8 @@
 
 import math
 
-from environ import get_environ
+# TODO: extend pants and pants-plugins/pack_metadata to add lib dirs extra_sys_path for pylint
+from environ import get_environ  # pylint: disable=E0401
 from st2common.runners.base_action import Action
 
 

contrib/examples/tests/BUILD

@@ -1,6 +1,11 @@
 # tests can only be dependencies of other tests in this directory
 __dependents_rules__(("*", "/**", "!*"))
 
+__defaults__(
+    {python_test: dict(tags=["pack"])},
+    extend=True,
+)
+
 python_tests(
     skip_pylint=True,
 )

contrib/hello_st2/BUILD

@@ -1,3 +1,5 @@
+__defaults__(all=dict(inject_pack_python_path=True))
+
 pack_metadata(
     name="metadata",
 )

contrib/linux/BUILD

@@ -1,3 +1,5 @@
+__defaults__(all=dict(inject_pack_python_path=True))
+
 pack_metadata(
     name="metadata",
 )

contrib/linux/tests/BUILD

@@ -1,6 +1,11 @@
 # tests can only be dependencies of other tests in this directory
 __dependents_rules__(("*", "/**", "!*"))
 
+__defaults__(
+    {python_test: dict(tags=["pack"])},
+    extend=True,
+)
+
 python_tests(
     skip_pylint=True,
 )

contrib/packs/BUILD

@@ -1,3 +1,5 @@
+__defaults__(all=dict(inject_pack_python_path=True))
+
 pack_metadata(
     name="metadata",
 )

contrib/packs/tests/BUILD

@@ -1,6 +1,29 @@
 # tests can only be dependencies of other tests in this directory
 __dependents_rules__(("*", "/**", "!*"))
 
+__defaults__(
+    {python_test: dict(tags=["pack"])},
+    extend=True,
+)
+
 python_tests(
     skip_pylint=True,
+    overrides={
+        "test_action_aliases.py": dict(
+            dependencies=[
+                # test needs the pack and aliases metadata
+                "contrib/packs:metadata",
+            ],
+        ),
+        "test_action_unload.py": dict(
+            stevedore_namespaces=[
+                "st2common.metrics.driver",
+            ],
+            entry_point_dependencies={
+                "contrib/runners/http_runner": ["st2common.runners.runner"],
+                "contrib/runners/local_runner": ["st2common.runners.runner"],
+                "contrib/runners/python_runner": ["st2common.runners.runner"],
+            },
+        ),
+    },
 )

contrib/runners/action_chain_runner/tests/BUILD

@@ -1,6 +1,9 @@
 __defaults__(
     all=dict(
         skip_pylint=True,
+        entry_point_dependencies={
+            "contrib/runners/action_chain_runner": ["st2common.runners.runner"],
+        },
     )
 )
 

contrib/runners/action_chain_runner/tests/unit/BUILD

@@ -5,4 +5,11 @@ __defaults__(
 
 python_tests(
     name="tests",
+    stevedore_namespaces=[
+        "st2common.rbac.backend",
+        "st2common.metrics.driver",
+        # the core pack uses all runners.
+        "st2common.runners.runner",
+    ],
+    uses=["mongo"],
 )