From 039bf1e8e44bf3e9de770dec947884265ef46c8f Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 27 Mar 2023 20:40:17 -0500 Subject: [PATCH 01/11] add pex_binary for st2 venv installation --- BUILD | 32 ++++++++++++++++++++++++++++++++ pants-plugins/macros.py | 10 ++++++++++ pants.toml | 2 ++ 3 files changed, 44 insertions(+) diff --git a/BUILD b/BUILD index 3a01152242..78d4379df1 100644 --- a/BUILD +++ b/BUILD @@ -119,3 +119,35 @@ shell_command( output_directories=[".git/modules"], workdir="/", ) + +# this is used to quickly build a venv that includes all requirements and our wheels +for ic_name, ic in supported_python_interpreter_constraints().items(): + pex_binary( + name=f"st2-{ic_name}", + output_path=f"st2-{ic_name}.pex", + interpreter_constraints=ic, + dependencies=[ + # this should depend on all python_distribution targets + "st2actions", + "st2api", + "st2auth", + "st2client", + "st2common", + "st2reactor", + "st2stream", + "st2tests", + "contrib/runners/action_chain_runner", + "contrib/runners/announcement_runner", + "contrib/runners/http_runner", + "contrib/runners/inquirer_runner", + "contrib/runners/local_runner", + "contrib/runners/noop_runner", + "contrib/runners/orquesta_runner", + "contrib/runners/python_runner", + "contrib/runners/remote_runner", + "contrib/runners/winrm_runner", + ], + include_tools=True, # include pex.tools to populate a venv from the pex + include_sources=False, # always includes generated wheels, so transitive sources not needed + venv_hermetic_scripts=False, # do not add -sE to script shebangs + ) diff --git a/pants-plugins/macros.py b/pants-plugins/macros.py index 11131f20ee..d866a7f824 100644 --- a/pants-plugins/macros.py +++ b/pants-plugins/macros.py @@ -13,6 +13,16 @@ # limitations under the License. +# use this to parametrize targets as necessary. +# eg: interpreter_constraints=parametrize(**supported_python_interpreter_constraints) +def supported_python_interpreter_constraints(): + return dict( + py36=["CPython==3.6.*"], + py37=["CPython==3.7.*"], + py38=["CPython==3.8.*"], + ) + + def st2_publish_repos(): """Return the list of repos twine should publish to. diff --git a/pants.toml b/pants.toml index 86abb36f3a..b17fb19a63 100644 --- a/pants.toml +++ b/pants.toml @@ -106,6 +106,8 @@ root_patterns = [ # DEFAULT has values that we can reuse/interpolate below [DEFAULT] # This is the range of python versions that we support. +# Make sure to also update supported_interpreter_constraints in pants-plugins/macros.py +# (which is for use in parametrize() and similar ic-specific use-cases). st2_interpreter_constraints = "CPython>=3.8.1,<3.12" # This should match the pants interpreter_constraints: From 347d9ab1ed8a2b87e9e00c90fe7cd9b8cc9dba91 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 13 Apr 2023 22:08:53 -0500 Subject: [PATCH 02/11] move st2-py*.pex definition to packaging/BUILD --- BUILD | 32 -------------------------------- packaging/BUILD | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 32 deletions(-) create mode 100644 packaging/BUILD diff --git a/BUILD b/BUILD index 78d4379df1..3a01152242 100644 --- a/BUILD +++ b/BUILD @@ -119,35 +119,3 @@ shell_command( output_directories=[".git/modules"], workdir="/", ) - -# this is used to quickly build a venv that includes all requirements and our wheels -for ic_name, ic in supported_python_interpreter_constraints().items(): - pex_binary( - name=f"st2-{ic_name}", - output_path=f"st2-{ic_name}.pex", - interpreter_constraints=ic, - dependencies=[ - # this should depend on all python_distribution targets - "st2actions", - "st2api", - "st2auth", - "st2client", - "st2common", - "st2reactor", - "st2stream", - "st2tests", - "contrib/runners/action_chain_runner", - "contrib/runners/announcement_runner", - "contrib/runners/http_runner", - "contrib/runners/inquirer_runner", - "contrib/runners/local_runner", - "contrib/runners/noop_runner", - "contrib/runners/orquesta_runner", - "contrib/runners/python_runner", - "contrib/runners/remote_runner", - "contrib/runners/winrm_runner", - ], - include_tools=True, # include pex.tools to populate a venv from the pex - include_sources=False, # always includes generated wheels, so transitive sources not needed - venv_hermetic_scripts=False, # do not add -sE to script shebangs - ) diff --git a/packaging/BUILD b/packaging/BUILD new file mode 100644 index 0000000000..d68d654f05 --- /dev/null +++ b/packaging/BUILD @@ -0,0 +1,32 @@ +# We use st2-py*.pex to quickly build a venv (like /opt/stackstorm/st2) +# that includes all requirements and our wheels. +for ic_name, ic in supported_python_interpreter_constraints().items(): + pex_binary( + name=f"st2-{ic_name}.pex", + output_path=f"st2-{ic_name}.pex", + interpreter_constraints=ic, + dependencies=[ + # this should depend on all python_distribution targets + "//st2actions", + "//st2api", + "//st2auth", + "//st2client", + "//st2common", + "//st2reactor", + "//st2stream", + "//st2tests", + "//contrib/runners/action_chain_runner", + "//contrib/runners/announcement_runner", + "//contrib/runners/http_runner", + "//contrib/runners/inquirer_runner", + "//contrib/runners/local_runner", + "//contrib/runners/noop_runner", + "//contrib/runners/orquesta_runner", + "//contrib/runners/python_runner", + "//contrib/runners/remote_runner", + "//contrib/runners/winrm_runner", + ], + include_tools=True, # include pex.tools to populate a venv from the pex + include_sources=False, # always includes generated wheels, so transitive sources not needed + venv_hermetic_scripts=False, # do not add -sE to script shebangs + ) From b2f5184aca4deb9d874acb073196b34aefdbbfe5 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 19 Dec 2024 18:44:32 -0600 Subject: [PATCH 03/11] Pants: Use **parametrize(...) groups in packaging/BUILD --- packaging/BUILD | 78 +++++++++++++++++++++++++---------------- pants-plugins/macros.py | 10 ------ pants.toml | 3 +- 3 files changed, 49 insertions(+), 42 deletions(-) diff --git a/packaging/BUILD b/packaging/BUILD index d68d654f05..22af4dd342 100644 --- a/packaging/BUILD +++ b/packaging/BUILD @@ -1,32 +1,50 @@ # We use st2-py*.pex to quickly build a venv (like /opt/stackstorm/st2) # that includes all requirements and our wheels. -for ic_name, ic in supported_python_interpreter_constraints().items(): - pex_binary( - name=f"st2-{ic_name}.pex", - output_path=f"st2-{ic_name}.pex", - interpreter_constraints=ic, - dependencies=[ - # this should depend on all python_distribution targets - "//st2actions", - "//st2api", - "//st2auth", - "//st2client", - "//st2common", - "//st2reactor", - "//st2stream", - "//st2tests", - "//contrib/runners/action_chain_runner", - "//contrib/runners/announcement_runner", - "//contrib/runners/http_runner", - "//contrib/runners/inquirer_runner", - "//contrib/runners/local_runner", - "//contrib/runners/noop_runner", - "//contrib/runners/orquesta_runner", - "//contrib/runners/python_runner", - "//contrib/runners/remote_runner", - "//contrib/runners/winrm_runner", - ], - include_tools=True, # include pex.tools to populate a venv from the pex - include_sources=False, # always includes generated wheels, so transitive sources not needed - venv_hermetic_scripts=False, # do not add -sE to script shebangs - ) +pex_binary( + name="st2.pex", + dependencies=[ + # this should depend on all python_distribution targets + "//st2actions", + "//st2api", + "//st2auth", + "//st2client", + "//st2common", + "//st2reactor", + "//st2stream", + "//st2tests", + "//contrib/runners/action_chain_runner", + "//contrib/runners/announcement_runner", + "//contrib/runners/http_runner", + "//contrib/runners/inquirer_runner", + "//contrib/runners/local_runner", + "//contrib/runners/noop_runner", + "//contrib/runners/orquesta_runner", + "//contrib/runners/python_runner", + "//contrib/runners/remote_runner", + "//contrib/runners/winrm_runner", + ], + include_tools=True, # include pex.tools to populate a venv from the pex + include_sources=False, # always includes generated wheels, so transitive sources not needed + venv_hermetic_scripts=False, # do not add -sE to script shebangs + # 1 parametrize group per python minor version in [DEFAULT].st2_interpreter_constraints in pants.toml + **parametrize( + "py38", + output_path="st2-py38.pex", + interpreter_constraints=["CPython==3.8.*"], + ), + **parametrize( + "py39", + output_path="st2-py39.pex", + interpreter_constraints=["CPython==3.9.*"], + ), + **parametrize( + "py310", + output_path="st2-py310.pex", + interpreter_constraints=["CPython==3.10.*"], + ), + **parametrize( + "py311", + output_path="st2-py311.pex", + interpreter_constraints=["CPython==3.11.*"], + ), +) diff --git a/pants-plugins/macros.py b/pants-plugins/macros.py index d866a7f824..11131f20ee 100644 --- a/pants-plugins/macros.py +++ b/pants-plugins/macros.py @@ -13,16 +13,6 @@ # limitations under the License. -# use this to parametrize targets as necessary. -# eg: interpreter_constraints=parametrize(**supported_python_interpreter_constraints) -def supported_python_interpreter_constraints(): - return dict( - py36=["CPython==3.6.*"], - py37=["CPython==3.7.*"], - py38=["CPython==3.8.*"], - ) - - def st2_publish_repos(): """Return the list of repos twine should publish to. diff --git a/pants.toml b/pants.toml index b17fb19a63..b88cf53458 100644 --- a/pants.toml +++ b/pants.toml @@ -106,8 +106,7 @@ root_patterns = [ # DEFAULT has values that we can reuse/interpolate below [DEFAULT] # This is the range of python versions that we support. -# Make sure to also update supported_interpreter_constraints in pants-plugins/macros.py -# (which is for use in parametrize() and similar ic-specific use-cases). +# On update, make sure to also update parametrizations in packaging/BUILD. st2_interpreter_constraints = "CPython>=3.8.1,<3.12" # This should match the pants interpreter_constraints: From 45eaf72ffebbb1cd02d4aab20b9209944dc1cee3 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 30 Dec 2024 14:58:48 -0600 Subject: [PATCH 04/11] Packaging: Build /opt/stackstorm/st2 when running ./st2.pex Now we can treat st2.pex as a self-extracting venv installer. Use a preamble file, which pex executes before its bootstrap code, to make the pex just build /opt/stackstorm/st2 instead of teaching all the installers what a pex-tools are. --- packaging/BUILD | 12 ++++++++- packaging/pex_preamble.py | 51 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 packaging/pex_preamble.py diff --git a/packaging/BUILD b/packaging/BUILD index 22af4dd342..2ed0298dab 100644 --- a/packaging/BUILD +++ b/packaging/BUILD @@ -1,8 +1,15 @@ +python_sources() + # We use st2-py*.pex to quickly build a venv (like /opt/stackstorm/st2) # that includes all requirements and our wheels. pex_binary( name="st2.pex", + extra_build_args=[ + "--preamble-file", + f"source_files/{build_file_dir()}/pex_preamble.py", # preamble gets executed before the pex bootstrap + ], dependencies=[ + "./pex_preamble.py", # this should depend on all python_distribution targets "//st2actions", "//st2api", @@ -23,8 +30,11 @@ pex_binary( "//contrib/runners/remote_runner", "//contrib/runners/winrm_runner", ], + execution_mode="venv", + layout="zipapp", # zipapp creates a single file, loose and packed create directories + sh_boot=True, # faster startup time (only relevant for unpacking the pex) include_tools=True, # include pex.tools to populate a venv from the pex - include_sources=False, # always includes generated wheels, so transitive sources not needed + include_sources=True, # include pex_preamble.py (already includes generated wheels, skipping wheel-owned sources) venv_hermetic_scripts=False, # do not add -sE to script shebangs # 1 parametrize group per python minor version in [DEFAULT].st2_interpreter_constraints in pants.toml **parametrize( diff --git a/packaging/pex_preamble.py b/packaging/pex_preamble.py new file mode 100644 index 0000000000..5add993183 --- /dev/null +++ b/packaging/pex_preamble.py @@ -0,0 +1,51 @@ +# Copyright 2024 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys + +# This check makes re-exec safe by ensuring we modify env+argv once. +if os.environ.get("ST2_PEX_EXTRACT", "0") not in ("1", "skip"): + os.environ["ST2_PEX_EXTRACT"] = "1" + + st2_config_path = ( + os.environ.get("ST2_CONFIG_PATH", os.environ.get("ST2_CONF")) + or "/etc/st2/st2.conf" + ) + + # late import to minimize re-exec overhead (oslo_config is not available yet, so use stdlib here) + import configparser + + conf = configparser.ConfigParser() + conf.read_dict({"system": {"base_path": "/opt/stackstorm"}}) + conf.read(st2_config_path) + st2_base_path = conf.get("system", "base_path") + st2_base_path = os.environ.pop("ST2_SYSTEM__BASE_PATH", st2_base_path) + + st2_venv = os.path.join(st2_base_path, "st2") + if os.path.exists(st2_venv): + print(f"WARNING: This will overwrite {st2_venv}", file=sys.stderr) + + # This env var and sys.argv will create a venv in the st2_venv dir. + os.environ["PEX_TOOLS"] = "1" + sys.argv[1:1] = ( + "venv", + "--force", # remove and replace the venv if it exists + "--non-hermetic-scripts", # do not add -sE to python shebang + "--system-site-packages", + "--prompt=st2", + st2_venv, + ) + +# The standard PEX bootstrap code is below this line. From 5dda82b5db4bf93456beb05b0527f94bcdcc4da9 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 30 Dec 2024 16:14:28 -0600 Subject: [PATCH 05/11] Pants: Move venv packaging targets to separate BUILD file Other packaging metadata will go in packaging/BUILD later. --- packaging/{BUILD => BUILD.venv} | 0 pants.toml | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename packaging/{BUILD => BUILD.venv} (100%) diff --git a/packaging/BUILD b/packaging/BUILD.venv similarity index 100% rename from packaging/BUILD rename to packaging/BUILD.venv diff --git a/pants.toml b/pants.toml index b88cf53458..a2ae99b718 100644 --- a/pants.toml +++ b/pants.toml @@ -106,7 +106,7 @@ root_patterns = [ # DEFAULT has values that we can reuse/interpolate below [DEFAULT] # This is the range of python versions that we support. -# On update, make sure to also update parametrizations in packaging/BUILD. +# On update, make sure to also update parametrizations in packaging/BUILD*. st2_interpreter_constraints = "CPython>=3.8.1,<3.12" # This should match the pants interpreter_constraints: From c75129456ed7101d42e12dac2eb845baad10f9a2 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 2 Jan 2025 18:28:01 -0600 Subject: [PATCH 06/11] pants: Dry parametrization in packaging/BUILD.venv --- packaging/BUILD.venv | 34 ++++++++++++++-------------------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/packaging/BUILD.venv b/packaging/BUILD.venv index 2ed0298dab..ed50d587fc 100644 --- a/packaging/BUILD.venv +++ b/packaging/BUILD.venv @@ -2,6 +2,16 @@ python_sources() # We use st2-py*.pex to quickly build a venv (like /opt/stackstorm/st2) # that includes all requirements and our wheels. + + +def _pex_py3(minor: str): + return parametrize( + f"py3{minor}", + output_path=f"${{spec_path_normalized}}/st2-py3{minor}.pex", + interpreter_constraints=[f"CPython==3.{minor}.*"], + ) + + pex_binary( name="st2.pex", extra_build_args=[ @@ -37,24 +47,8 @@ pex_binary( include_sources=True, # include pex_preamble.py (already includes generated wheels, skipping wheel-owned sources) venv_hermetic_scripts=False, # do not add -sE to script shebangs # 1 parametrize group per python minor version in [DEFAULT].st2_interpreter_constraints in pants.toml - **parametrize( - "py38", - output_path="st2-py38.pex", - interpreter_constraints=["CPython==3.8.*"], - ), - **parametrize( - "py39", - output_path="st2-py39.pex", - interpreter_constraints=["CPython==3.9.*"], - ), - **parametrize( - "py310", - output_path="st2-py310.pex", - interpreter_constraints=["CPython==3.10.*"], - ), - **parametrize( - "py311", - output_path="st2-py311.pex", - interpreter_constraints=["CPython==3.11.*"], - ), + **_pex_py3("8"), + **_pex_py3("9"), + **_pex_py3("10"), + **_pex_py3("11"), ) From acf2c9fbac5beeaf0b6034c27d3c1c61beebc120 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Tue, 25 Feb 2025 22:13:17 -0600 Subject: [PATCH 07/11] pants: handle 3.8 parametrization better --- packaging/BUILD.venv | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packaging/BUILD.venv b/packaging/BUILD.venv index ed50d587fc..309e57acc7 100644 --- a/packaging/BUILD.venv +++ b/packaging/BUILD.venv @@ -4,11 +4,13 @@ python_sources() # that includes all requirements and our wheels. -def _pex_py3(minor: str): +def _pex_py3(minor: str, constraint: str = ""): + if not constraint: + constraint = f"CPython==3.{minor}.*" return parametrize( f"py3{minor}", output_path=f"${{spec_path_normalized}}/st2-py3{minor}.pex", - interpreter_constraints=[f"CPython==3.{minor}.*"], + interpreter_constraints=[constraint], ) @@ -47,7 +49,7 @@ pex_binary( include_sources=True, # include pex_preamble.py (already includes generated wheels, skipping wheel-owned sources) venv_hermetic_scripts=False, # do not add -sE to script shebangs # 1 parametrize group per python minor version in [DEFAULT].st2_interpreter_constraints in pants.toml - **_pex_py3("8"), + **_pex_py3("8", constraint="CPython>=3.8.1,<3.9"), **_pex_py3("9"), **_pex_py3("10"), **_pex_py3("11"), From 083ddf4e2728b8177b93851fe53744d1dee24700 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 26 Feb 2025 15:27:08 -0600 Subject: [PATCH 08/11] packaging: Use pex_binary(executable=...) instead of pex premable The preamble method was hacky and had some uncomfortable sharp edges. The pex cache does not know about the preamble which can break running the pex to unpack itself in some cases (like if you use PEX_TOOLS to inspect the pex before unpacking it). So, switch to a python script that will run when the pex is executed. This script has access to the full venv, thanks to pex, before we have even unpacked it. So, we can use deps like oslo.config safely. However, using a pex entry_point/script/executable causes the script to be included in the extracted venv. If we import any st2 code, then that can make pants add other sources or conf files that we do not want in the pex (other than via the wheels of our sources). So, I left a note warning against importing st2 code. --- packaging/BUILD.venv | 8 +-- packaging/build_st2_venv.py | 124 ++++++++++++++++++++++++++++++++++++ packaging/pex_preamble.py | 51 --------------- 3 files changed, 126 insertions(+), 57 deletions(-) create mode 100644 packaging/build_st2_venv.py delete mode 100644 packaging/pex_preamble.py diff --git a/packaging/BUILD.venv b/packaging/BUILD.venv index 309e57acc7..e1bd7453e4 100644 --- a/packaging/BUILD.venv +++ b/packaging/BUILD.venv @@ -16,12 +16,7 @@ def _pex_py3(minor: str, constraint: str = ""): pex_binary( name="st2.pex", - extra_build_args=[ - "--preamble-file", - f"source_files/{build_file_dir()}/pex_preamble.py", # preamble gets executed before the pex bootstrap - ], dependencies=[ - "./pex_preamble.py", # this should depend on all python_distribution targets "//st2actions", "//st2api", @@ -42,11 +37,12 @@ pex_binary( "//contrib/runners/remote_runner", "//contrib/runners/winrm_runner", ], + executable="build_st2_venv.py", # included by dependency inferrence execution_mode="venv", layout="zipapp", # zipapp creates a single file, loose and packed create directories sh_boot=True, # faster startup time (only relevant for unpacking the pex) include_tools=True, # include pex.tools to populate a venv from the pex - include_sources=True, # include pex_preamble.py (already includes generated wheels, skipping wheel-owned sources) + include_sources=False, # already includes our wheels, skipping wheel-owned sources venv_hermetic_scripts=False, # do not add -sE to script shebangs # 1 parametrize group per python minor version in [DEFAULT].st2_interpreter_constraints in pants.toml **_pex_py3("8", constraint="CPython>=3.8.1,<3.9"), diff --git a/packaging/build_st2_venv.py b/packaging/build_st2_venv.py new file mode 100644 index 0000000000..f40720de87 --- /dev/null +++ b/packaging/build_st2_venv.py @@ -0,0 +1,124 @@ +# Copyright 2025 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# NOTE: In this script, all 3rd party deps are available thanks to pex. +# Do not import any st2 code to avoid polluting the pex with extra files +# (Pants uses dependency inference to add sources beyond our wheels). + +import os +import sys +import subprocess + +from pathlib import Path +from typing import List, Optional + +from oslo_config import cfg + + +def get_pex_path() -> str: + return os.environ.get("PEX", sys.argv[0]) + + +def get_st2_base_path(args: Optional[List[str]] = None) -> Path: + st2_config_path = ( + os.environ.get("ST2_CONFIG_PATH", os.environ.get("ST2_CONF")) + or "/etc/st2/st2.conf" + ) + + cfg.CONF.register_opts( + [cfg.StrOpt("base_path", default="/opt/stackstorm")], group="system" + ) + + try: + cfg.CONF(args=args, default_config_files=[st2_config_path], use_env=False) + except cfg.ConfigFilesNotFoundError: + pass + + st2_base_path = os.environ.get( + "ST2_SYSTEM__BASE_PATH", cfg.CONF["system"]["base_path"] + ) + return Path(st2_base_path) + + +def unpack_venv(st2_venv_path: Path) -> int: + if st2_venv_path.exists(): + print(f"WARNING: This will overwrite {st2_venv_path}", file=sys.stderr) + + env = {"PEX_TOOLS": "1"} + cmd = [ + get_pex_path(), + "venv", + "--force", # remove and replace the venv if it exists + "--non-hermetic-scripts", # do not add -sE to python shebang + # st2-packages has a note about python symlinks breaking pack install. + # uncomment this if that proves to still be an issue. + # "--copies", # pack install follows python symlinks to find bin dir + "--system-site-packages", + "--compile", # pre-compile all pyc files + "--prompt=st2", + str(st2_venv_path), + ] + pretty_cmd = "".join(k + "=" + v + " " for k, v in env.items()) + " ".join(cmd) + print(f"Now running: {pretty_cmd}", file=sys.stderr) + + result = subprocess.call(cmd, env=env) + + if result == 0: + print(f"Successfully unpacked venv to {st2_venv_path}", file=sys.stderr) + else: + print( + f"Encountered an error unpacking venv to {st2_venv_path}", file=sys.stderr + ) + + return result + + +def tidy_venv(st2_venv_path: Path) -> None: + """Clean up and remove this script from the venv. + + Unfortunately, the way pants uses pex, this script ends up in the venv. + """ + for path in (st2_venv_path / "lib").glob("python*"): + script_path = path / "site-packages" / "packaging" / "build_st2_venv.py" + if script_path.exists(): + script_path.unlink() + + script_path = path / "site-packages" / "__pex_executable__.py" + if script_path.exists(): + script_path.unlink() + + # and remove the reference to this script + main_path = st2_venv_path / "__main__.py" + main_path.write_text(main_path.read_text().replace("__pex_executable__", "")) + + +def main() -> int: + st2_base_path = get_st2_base_path(sys.argv[1:]) + st2_venv_path = st2_base_path / "st2" + + if not os.access(st2_base_path, os.W_OK): + print( + f"ERROR: venv parent directory is not writable: {st2_base_path}", + file=sys.stderr, + ) + return 1 + + venv_result = unpack_venv(st2_venv_path) + tidy_venv(st2_venv_path) + + return venv_result + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/packaging/pex_preamble.py b/packaging/pex_preamble.py deleted file mode 100644 index 5add993183..0000000000 --- a/packaging/pex_preamble.py +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright 2024 The StackStorm Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import sys - -# This check makes re-exec safe by ensuring we modify env+argv once. -if os.environ.get("ST2_PEX_EXTRACT", "0") not in ("1", "skip"): - os.environ["ST2_PEX_EXTRACT"] = "1" - - st2_config_path = ( - os.environ.get("ST2_CONFIG_PATH", os.environ.get("ST2_CONF")) - or "/etc/st2/st2.conf" - ) - - # late import to minimize re-exec overhead (oslo_config is not available yet, so use stdlib here) - import configparser - - conf = configparser.ConfigParser() - conf.read_dict({"system": {"base_path": "/opt/stackstorm"}}) - conf.read(st2_config_path) - st2_base_path = conf.get("system", "base_path") - st2_base_path = os.environ.pop("ST2_SYSTEM__BASE_PATH", st2_base_path) - - st2_venv = os.path.join(st2_base_path, "st2") - if os.path.exists(st2_venv): - print(f"WARNING: This will overwrite {st2_venv}", file=sys.stderr) - - # This env var and sys.argv will create a venv in the st2_venv dir. - os.environ["PEX_TOOLS"] = "1" - sys.argv[1:1] = ( - "venv", - "--force", # remove and replace the venv if it exists - "--non-hermetic-scripts", # do not add -sE to python shebang - "--system-site-packages", - "--prompt=st2", - st2_venv, - ) - -# The standard PEX bootstrap code is below this line. From 18f470a3c4c511ae5f5ab0043127eed988e92dcc Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 26 Feb 2025 15:38:34 -0600 Subject: [PATCH 09/11] packaging: add note about pex_binary(include_requirements=...) --- packaging/BUILD.venv | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packaging/BUILD.venv b/packaging/BUILD.venv index e1bd7453e4..65d09d51c0 100644 --- a/packaging/BUILD.venv +++ b/packaging/BUILD.venv @@ -42,6 +42,9 @@ pex_binary( layout="zipapp", # zipapp creates a single file, loose and packed create directories sh_boot=True, # faster startup time (only relevant for unpacking the pex) include_tools=True, # include pex.tools to populate a venv from the pex + # TODO: To improve docker layer caching, we could break this into 2 pexes + # one w/ include_requirements=False and the other w/ include_requirements=True. + include_requirements=True, # include third party requirements include_sources=False, # already includes our wheels, skipping wheel-owned sources venv_hermetic_scripts=False, # do not add -sE to script shebangs # 1 parametrize group per python minor version in [DEFAULT].st2_interpreter_constraints in pants.toml From c7183fdacac7023ef46af5c31d5db6e2929ed6e5 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 27 Feb 2025 10:36:59 -0600 Subject: [PATCH 10/11] BUILD: add dependency rule to prevent st2 deps in install script --- packaging/BUILD.venv | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/packaging/BUILD.venv b/packaging/BUILD.venv index 65d09d51c0..2c665ec5f5 100644 --- a/packaging/BUILD.venv +++ b/packaging/BUILD.venv @@ -1,3 +1,21 @@ +# rules on what packaging can depend on +__dependencies_rules__( + ( + # All python sources in this directory + "[/*]", + ( + # may depend on 3rd party dependencies, + "//reqs#*", + # and on anything in this diretory, + "/**", + # but nothing else (eg not st2common, st2*, runners, ...). + "!*", + ), + ), + # other targets (not python in this directory) may depend on anything. + ("*", "*"), +) + python_sources() # We use st2-py*.pex to quickly build a venv (like /opt/stackstorm/st2) From 8a81662415b229a8122ccd552b7e96bccffa93d6 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 26 Feb 2025 21:58:32 -0600 Subject: [PATCH 11/11] add merge conflict magnet --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 25a164fd9c..aa78a80b33 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -78,7 +78,7 @@ Added working on StackStorm, improve our security posture, and improve CI reliability thanks in part to pants' use of PEX lockfiles. This is not a user-facing addition. #6118 #6141 #6133 #6120 #6181 #6183 #6200 #6237 #6229 #6240 #6241 #6244 #6251 #6253 - #6254 #6258 #6259 #6260 #6269 #6275 #6279 #6278 #6282 #6283 #6273 #6287 #6306 + #6254 #6258 #6259 #6260 #6269 #6275 #6279 #6278 #6282 #6283 #6273 #6287 #6306 #6307 Contributed by @cognifloyd * Build of ST2 EL9 packages #6153 Contributed by @amanda11