Packaging: Add pex_binary BUILD metadata for building st2 venv

CHANGELOG.rst

@@ -78,7 +78,7 @@ Added
   working on StackStorm, improve our security posture, and improve CI reliability thanks in part
   to pants' use of PEX lockfiles. This is not a user-facing addition.
   #6118 #6141 #6133 #6120 #6181 #6183 #6200 #6237 #6229 #6240 #6241 #6244 #6251 #6253
-  #6254 #6258 #6259 #6260 #6269 #6275 #6279 #6278 #6282 #6283 #6273 #6287 #6306
+  #6254 #6258 #6259 #6260 #6269 #6275 #6279 #6278 #6282 #6283 #6273 #6287 #6306 #6307
   Contributed by @cognifloyd
 * Build of ST2 EL9 packages #6153
   Contributed by @amanda11

packaging/BUILD.venv

@@ -0,0 +1,73 @@
+# rules on what packaging can depend on
+__dependencies_rules__(
+    (
+        # All python sources in this directory
+        "<python_*>[/*]",
+        (
+            # may depend on 3rd party dependencies,
+            "//reqs#*",
+            # and on anything in this diretory,
+            "/**",
+            # but nothing else (eg not st2common, st2*, runners, ...).
+            "!*",
+        ),
+    ),
+    # other targets (not python in this directory) may depend on anything.
+    ("*", "*"),
+)
+
+python_sources()
+
+# We use st2-py*.pex to quickly build a venv (like /opt/stackstorm/st2)
+# that includes all requirements and our wheels.
+
+
+def _pex_py3(minor: str, constraint: str = ""):
+    if not constraint:
+        constraint = f"CPython==3.{minor}.*"
+    return parametrize(
+        f"py3{minor}",
+        output_path=f"${{spec_path_normalized}}/st2-py3{minor}.pex",
+        interpreter_constraints=[constraint],
+    )
+
+
+pex_binary(
+    name="st2.pex",
+    dependencies=[
+        # this should depend on all python_distribution targets
+        "//st2actions",
+        "//st2api",
+        "//st2auth",
+        "//st2client",
+        "//st2common",
+        "//st2reactor",
+        "//st2stream",
+        "//st2tests",
+        "//contrib/runners/action_chain_runner",
+        "//contrib/runners/announcement_runner",
+        "//contrib/runners/http_runner",
+        "//contrib/runners/inquirer_runner",
+        "//contrib/runners/local_runner",
+        "//contrib/runners/noop_runner",
+        "//contrib/runners/orquesta_runner",
+        "//contrib/runners/python_runner",
+        "//contrib/runners/remote_runner",
+        "//contrib/runners/winrm_runner",
+    ],
+    executable="build_st2_venv.py",  # included by dependency inferrence
+    execution_mode="venv",
+    layout="zipapp",  # zipapp creates a single file, loose and packed create directories
+    sh_boot=True,  # faster startup time (only relevant for unpacking the pex)
+    include_tools=True,  # include pex.tools to populate a venv from the pex
+    # TODO: To improve docker layer caching, we could break this into 2 pexes
+    #       one w/ include_requirements=False and the other w/ include_requirements=True.
+    include_requirements=True,  # include third party requirements
+    include_sources=False,  # already includes our wheels, skipping wheel-owned sources
+    venv_hermetic_scripts=False,  # do not add -sE to script shebangs
+    # 1 parametrize group per python minor version in [DEFAULT].st2_interpreter_constraints in pants.toml
+    **_pex_py3("8", constraint="CPython>=3.8.1,<3.9"),
+    **_pex_py3("9"),
+    **_pex_py3("10"),
+    **_pex_py3("11"),
+)

packaging/build_st2_venv.py

@@ -0,0 +1,124 @@
+# Copyright 2025 The StackStorm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# NOTE: In this script, all 3rd party deps are available thanks to pex.
+# Do not import any st2 code to avoid polluting the pex with extra files
+# (Pants uses dependency inference to add sources beyond our wheels).
+
+import os
+import sys
+import subprocess
+
+from pathlib import Path
+from typing import List, Optional
+
+from oslo_config import cfg
+
+
+def get_pex_path() -> str:
+    return os.environ.get("PEX", sys.argv[0])
+
+
+def get_st2_base_path(args: Optional[List[str]] = None) -> Path:
+    st2_config_path = (
+        os.environ.get("ST2_CONFIG_PATH", os.environ.get("ST2_CONF"))
+        or "/etc/st2/st2.conf"
+    )
+
+    cfg.CONF.register_opts(
+        [cfg.StrOpt("base_path", default="/opt/stackstorm")], group="system"
+    )
+
+    try:
+        cfg.CONF(args=args, default_config_files=[st2_config_path], use_env=False)
+    except cfg.ConfigFilesNotFoundError:
+        pass
+
+    st2_base_path = os.environ.get(
+        "ST2_SYSTEM__BASE_PATH", cfg.CONF["system"]["base_path"]
+    )
+    return Path(st2_base_path)
+
+
+def unpack_venv(st2_venv_path: Path) -> int:
+    if st2_venv_path.exists():
+        print(f"WARNING: This will overwrite {st2_venv_path}", file=sys.stderr)
+
+    env = {"PEX_TOOLS": "1"}
+    cmd = [
+        get_pex_path(),
+        "venv",
+        "--force",  # remove and replace the venv if it exists
+        "--non-hermetic-scripts",  # do not add -sE to python shebang
+        # st2-packages has a note about python symlinks breaking pack install.
+        # uncomment this if that proves to still be an issue.
+        # "--copies",  # pack install follows python symlinks to find bin dir
+        "--system-site-packages",
+        "--compile",  # pre-compile all pyc files
+        "--prompt=st2",
+        str(st2_venv_path),
+    ]
+    pretty_cmd = "".join(k + "=" + v + " " for k, v in env.items()) + " ".join(cmd)
+    print(f"Now running: {pretty_cmd}", file=sys.stderr)
+
+    result = subprocess.call(cmd, env=env)
+
+    if result == 0:
+        print(f"Successfully unpacked venv to {st2_venv_path}", file=sys.stderr)
+    else:
+        print(
+            f"Encountered an error unpacking venv to {st2_venv_path}", file=sys.stderr
+        )
+
+    return result
+
+
+def tidy_venv(st2_venv_path: Path) -> None:
+    """Clean up and remove this script from the venv.
+
+    Unfortunately, the way pants uses pex, this script ends up in the venv.
+    """
+    for path in (st2_venv_path / "lib").glob("python*"):
+        script_path = path / "site-packages" / "packaging" / "build_st2_venv.py"
+        if script_path.exists():
+            script_path.unlink()
+
+        script_path = path / "site-packages" / "__pex_executable__.py"
+        if script_path.exists():
+            script_path.unlink()
+
+    # and remove the reference to this script
+    main_path = st2_venv_path / "__main__.py"
+    main_path.write_text(main_path.read_text().replace("__pex_executable__", ""))
+
+
+def main() -> int:
+    st2_base_path = get_st2_base_path(sys.argv[1:])
+    st2_venv_path = st2_base_path / "st2"
+
+    if not os.access(st2_base_path, os.W_OK):
+        print(
+            f"ERROR: venv parent directory is not writable: {st2_base_path}",
+            file=sys.stderr,
+        )
+        return 1
+
+    venv_result = unpack_venv(st2_venv_path)
+    tidy_venv(st2_venv_path)
+
+    return venv_result
+
+
+if __name__ == "__main__":
+    sys.exit(main())

pants.toml

@@ -106,6 +106,7 @@ root_patterns = [
 # DEFAULT has values that we can reuse/interpolate below
 [DEFAULT]
 # This is the range of python versions that we support.
+# On update, make sure to also update parametrizations in packaging/BUILD*.
 st2_interpreter_constraints = "CPython>=3.8.1,<3.12"
 
 # This should match the pants interpreter_constraints: