diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 25a164fd9c..aa78a80b33 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -78,7 +78,7 @@ Added working on StackStorm, improve our security posture, and improve CI reliability thanks in part to pants' use of PEX lockfiles. This is not a user-facing addition. #6118 #6141 #6133 #6120 #6181 #6183 #6200 #6237 #6229 #6240 #6241 #6244 #6251 #6253 - #6254 #6258 #6259 #6260 #6269 #6275 #6279 #6278 #6282 #6283 #6273 #6287 #6306 + #6254 #6258 #6259 #6260 #6269 #6275 #6279 #6278 #6282 #6283 #6273 #6287 #6306 #6307 Contributed by @cognifloyd * Build of ST2 EL9 packages #6153 Contributed by @amanda11 diff --git a/packaging/BUILD.venv b/packaging/BUILD.venv new file mode 100644 index 0000000000..2c665ec5f5 --- /dev/null +++ b/packaging/BUILD.venv @@ -0,0 +1,73 @@ +# rules on what packaging can depend on +__dependencies_rules__( + ( + # All python sources in this directory + "[/*]", + ( + # may depend on 3rd party dependencies, + "//reqs#*", + # and on anything in this diretory, + "/**", + # but nothing else (eg not st2common, st2*, runners, ...). + "!*", + ), + ), + # other targets (not python in this directory) may depend on anything. + ("*", "*"), +) + +python_sources() + +# We use st2-py*.pex to quickly build a venv (like /opt/stackstorm/st2) +# that includes all requirements and our wheels. + + +def _pex_py3(minor: str, constraint: str = ""): + if not constraint: + constraint = f"CPython==3.{minor}.*" + return parametrize( + f"py3{minor}", + output_path=f"${{spec_path_normalized}}/st2-py3{minor}.pex", + interpreter_constraints=[constraint], + ) + + +pex_binary( + name="st2.pex", + dependencies=[ + # this should depend on all python_distribution targets + "//st2actions", + "//st2api", + "//st2auth", + "//st2client", + "//st2common", + "//st2reactor", + "//st2stream", + "//st2tests", + "//contrib/runners/action_chain_runner", + "//contrib/runners/announcement_runner", + "//contrib/runners/http_runner", + "//contrib/runners/inquirer_runner", + "//contrib/runners/local_runner", + "//contrib/runners/noop_runner", + "//contrib/runners/orquesta_runner", + "//contrib/runners/python_runner", + "//contrib/runners/remote_runner", + "//contrib/runners/winrm_runner", + ], + executable="build_st2_venv.py", # included by dependency inferrence + execution_mode="venv", + layout="zipapp", # zipapp creates a single file, loose and packed create directories + sh_boot=True, # faster startup time (only relevant for unpacking the pex) + include_tools=True, # include pex.tools to populate a venv from the pex + # TODO: To improve docker layer caching, we could break this into 2 pexes + # one w/ include_requirements=False and the other w/ include_requirements=True. + include_requirements=True, # include third party requirements + include_sources=False, # already includes our wheels, skipping wheel-owned sources + venv_hermetic_scripts=False, # do not add -sE to script shebangs + # 1 parametrize group per python minor version in [DEFAULT].st2_interpreter_constraints in pants.toml + **_pex_py3("8", constraint="CPython>=3.8.1,<3.9"), + **_pex_py3("9"), + **_pex_py3("10"), + **_pex_py3("11"), +) diff --git a/packaging/build_st2_venv.py b/packaging/build_st2_venv.py new file mode 100644 index 0000000000..f40720de87 --- /dev/null +++ b/packaging/build_st2_venv.py @@ -0,0 +1,124 @@ +# Copyright 2025 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# NOTE: In this script, all 3rd party deps are available thanks to pex. +# Do not import any st2 code to avoid polluting the pex with extra files +# (Pants uses dependency inference to add sources beyond our wheels). + +import os +import sys +import subprocess + +from pathlib import Path +from typing import List, Optional + +from oslo_config import cfg + + +def get_pex_path() -> str: + return os.environ.get("PEX", sys.argv[0]) + + +def get_st2_base_path(args: Optional[List[str]] = None) -> Path: + st2_config_path = ( + os.environ.get("ST2_CONFIG_PATH", os.environ.get("ST2_CONF")) + or "/etc/st2/st2.conf" + ) + + cfg.CONF.register_opts( + [cfg.StrOpt("base_path", default="/opt/stackstorm")], group="system" + ) + + try: + cfg.CONF(args=args, default_config_files=[st2_config_path], use_env=False) + except cfg.ConfigFilesNotFoundError: + pass + + st2_base_path = os.environ.get( + "ST2_SYSTEM__BASE_PATH", cfg.CONF["system"]["base_path"] + ) + return Path(st2_base_path) + + +def unpack_venv(st2_venv_path: Path) -> int: + if st2_venv_path.exists(): + print(f"WARNING: This will overwrite {st2_venv_path}", file=sys.stderr) + + env = {"PEX_TOOLS": "1"} + cmd = [ + get_pex_path(), + "venv", + "--force", # remove and replace the venv if it exists + "--non-hermetic-scripts", # do not add -sE to python shebang + # st2-packages has a note about python symlinks breaking pack install. + # uncomment this if that proves to still be an issue. + # "--copies", # pack install follows python symlinks to find bin dir + "--system-site-packages", + "--compile", # pre-compile all pyc files + "--prompt=st2", + str(st2_venv_path), + ] + pretty_cmd = "".join(k + "=" + v + " " for k, v in env.items()) + " ".join(cmd) + print(f"Now running: {pretty_cmd}", file=sys.stderr) + + result = subprocess.call(cmd, env=env) + + if result == 0: + print(f"Successfully unpacked venv to {st2_venv_path}", file=sys.stderr) + else: + print( + f"Encountered an error unpacking venv to {st2_venv_path}", file=sys.stderr + ) + + return result + + +def tidy_venv(st2_venv_path: Path) -> None: + """Clean up and remove this script from the venv. + + Unfortunately, the way pants uses pex, this script ends up in the venv. + """ + for path in (st2_venv_path / "lib").glob("python*"): + script_path = path / "site-packages" / "packaging" / "build_st2_venv.py" + if script_path.exists(): + script_path.unlink() + + script_path = path / "site-packages" / "__pex_executable__.py" + if script_path.exists(): + script_path.unlink() + + # and remove the reference to this script + main_path = st2_venv_path / "__main__.py" + main_path.write_text(main_path.read_text().replace("__pex_executable__", "")) + + +def main() -> int: + st2_base_path = get_st2_base_path(sys.argv[1:]) + st2_venv_path = st2_base_path / "st2" + + if not os.access(st2_base_path, os.W_OK): + print( + f"ERROR: venv parent directory is not writable: {st2_base_path}", + file=sys.stderr, + ) + return 1 + + venv_result = unpack_venv(st2_venv_path) + tidy_venv(st2_venv_path) + + return venv_result + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/pants.toml b/pants.toml index 86abb36f3a..a2ae99b718 100644 --- a/pants.toml +++ b/pants.toml @@ -106,6 +106,7 @@ root_patterns = [ # DEFAULT has values that we can reuse/interpolate below [DEFAULT] # This is the range of python versions that we support. +# On update, make sure to also update parametrizations in packaging/BUILD*. st2_interpreter_constraints = "CPython>=3.8.1,<3.12" # This should match the pants interpreter_constraints: