From 161c64179a8338acd4adc2789d05e1c521e6bdc1 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 2 Jan 2025 18:36:00 -0600 Subject: [PATCH 1/6] Packaging: Add self-install archives for each pack to /opt/stackstorm/install --- contrib/chatops/BUILD | 8 ++++ contrib/chatops/tests/BUILD | 2 +- contrib/core/BUILD | 11 +++++ contrib/core/tests/BUILD | 2 +- contrib/debug/BUILD | 7 +++ contrib/default/BUILD | 6 +++ contrib/examples/BUILD | 26 +++++++++++ contrib/examples/actions/windows/BUILD | 3 ++ contrib/examples/tests/BUILD | 2 +- contrib/hello_st2/BUILD | 16 +++++++ contrib/linux/BUILD | 12 +++++ contrib/linux/tests/BUILD | 2 +- contrib/packs/BUILD | 9 ++++ contrib/packs/tests/BUILD | 2 +- pants-plugins/macros.py | 61 ++++++++++++++++++++++++++ pants.toml | 5 ++- 16 files changed, 167 insertions(+), 7 deletions(-) create mode 100644 contrib/examples/actions/windows/BUILD diff --git a/contrib/chatops/BUILD b/contrib/chatops/BUILD index 888be3a426..c9f2a2d183 100644 --- a/contrib/chatops/BUILD +++ b/contrib/chatops/BUILD @@ -3,3 +3,11 @@ __defaults__(all=dict(inject_pack_python_path=True)) pack_metadata( name="metadata", ) + +st2_pack_archive( + dependencies=[ + ":metadata", + "./actions", + "./tests", + ], +) diff --git a/contrib/chatops/tests/BUILD b/contrib/chatops/tests/BUILD index ead8561daa..63c5ba6b03 100644 --- a/contrib/chatops/tests/BUILD +++ b/contrib/chatops/tests/BUILD @@ -1,5 +1,5 @@ # tests can only be dependencies of other tests in this directory -__dependents_rules__(("*", "/**", "!*")) +__dependents_rules__(("*", "/**", f"//{build_file_dir().parent}:files", "!*")) __defaults__( {python_test: dict(tags=["pack"])}, diff --git a/contrib/core/BUILD b/contrib/core/BUILD index 9df7a372c9..e175664166 100644 --- a/contrib/core/BUILD +++ b/contrib/core/BUILD @@ -10,9 +10,20 @@ python_requirements( ) python_sources( + # this is for fixture.py dependencies=[ ":metadata", "./actions", "./actions/send_mail:send_mail_resources", ], ) + +st2_pack_archive( + dependencies=[ + ":metadata", + ":reqs", + "./actions", + "./actions/send_mail:send_mail_resources", + "./tests", + ], +) diff --git a/contrib/core/tests/BUILD b/contrib/core/tests/BUILD index 1a4583beb6..c84e095e16 100644 --- a/contrib/core/tests/BUILD +++ b/contrib/core/tests/BUILD @@ -1,5 +1,5 @@ # tests can only be dependencies of other tests in this directory -__dependents_rules__(("*", "/**", "!*")) +__dependents_rules__(("*", "/**", f"//{build_file_dir().parent}:files", "!*")) __defaults__( {python_test: dict(tags=["pack"])}, diff --git a/contrib/debug/BUILD b/contrib/debug/BUILD index 888be3a426..4713a9b1cc 100644 --- a/contrib/debug/BUILD +++ b/contrib/debug/BUILD @@ -3,3 +3,10 @@ __defaults__(all=dict(inject_pack_python_path=True)) pack_metadata( name="metadata", ) + +st2_pack_archive( + dependencies=[ + ":metadata", + "./actions", + ], +) diff --git a/contrib/default/BUILD b/contrib/default/BUILD index 888be3a426..fc15b27bea 100644 --- a/contrib/default/BUILD +++ b/contrib/default/BUILD @@ -3,3 +3,9 @@ __defaults__(all=dict(inject_pack_python_path=True)) pack_metadata( name="metadata", ) + +st2_pack_archive( + dependencies=[ + ":metadata", + ], +) diff --git a/contrib/examples/BUILD b/contrib/examples/BUILD index ab10cd1c85..22d0401478 100644 --- a/contrib/examples/BUILD +++ b/contrib/examples/BUILD @@ -18,3 +18,29 @@ python_requirement( name="flask", requirements=["flask"], ) + +# Also capture the requirements file for distribution in the pack archive. +files( + name="reqs", + sources=["requirements*.txt"], +) + +st2_pack_archive( + # we need to list targets of all files because transitive dep targets are NOT included + dependencies=[ + ":metadata", + ":reqs", + "./actions", + "./actions:shell", + "./actions/bash_exit_code", + "./actions/bash_ping", + "./actions/bash_random", + "./actions/pythonactions", + "./actions/ubuntu_pkg_info", + "./actions/ubuntu_pkg_info/lib", + "./actions/windows", + "./lib", + "./sensors", + "./tests", + ], +) diff --git a/contrib/examples/actions/windows/BUILD b/contrib/examples/actions/windows/BUILD new file mode 100644 index 0000000000..47dc872204 --- /dev/null +++ b/contrib/examples/actions/windows/BUILD @@ -0,0 +1,3 @@ +files( + sources=["*.ps1"], +) diff --git a/contrib/examples/tests/BUILD b/contrib/examples/tests/BUILD index 25a2e7cc4b..0ddb50de63 100644 --- a/contrib/examples/tests/BUILD +++ b/contrib/examples/tests/BUILD @@ -1,5 +1,5 @@ # tests can only be dependencies of other tests in this directory -__dependents_rules__(("*", "/**", "!*")) +__dependents_rules__(("*", "/**", f"//{build_file_dir().parent}:files", "!*")) __defaults__( {python_test: dict(tags=["pack"])}, diff --git a/contrib/hello_st2/BUILD b/contrib/hello_st2/BUILD index 888be3a426..6938233485 100644 --- a/contrib/hello_st2/BUILD +++ b/contrib/hello_st2/BUILD @@ -3,3 +3,19 @@ __defaults__(all=dict(inject_pack_python_path=True)) pack_metadata( name="metadata", ) + +# Capture the requirements file for distribution in the pack archive; +# we do not need to use `python_requirements()` for this sample file. +files( + name="reqs", + sources=["requirements*.txt"], +) + +st2_pack_archive( + dependencies=[ + ":metadata", + ":reqs", + "./actions", + "./sensors", + ], +) diff --git a/contrib/linux/BUILD b/contrib/linux/BUILD index 201435eecc..e4619b05cb 100644 --- a/contrib/linux/BUILD +++ b/contrib/linux/BUILD @@ -7,3 +7,15 @@ pack_metadata( python_requirements( name="reqs", ) + +st2_pack_archive( + dependencies=[ + ":metadata", + ":reqs", + "./actions", + "./actions/checks", + # "./actions/lib", # nothing in dir + "./sensors", + "./tests", + ], +) diff --git a/contrib/linux/tests/BUILD b/contrib/linux/tests/BUILD index 25a2e7cc4b..0ddb50de63 100644 --- a/contrib/linux/tests/BUILD +++ b/contrib/linux/tests/BUILD @@ -1,5 +1,5 @@ # tests can only be dependencies of other tests in this directory -__dependents_rules__(("*", "/**", "!*")) +__dependents_rules__(("*", "/**", f"//{build_file_dir().parent}:files", "!*")) __defaults__( {python_test: dict(tags=["pack"])}, diff --git a/contrib/packs/BUILD b/contrib/packs/BUILD index 888be3a426..cb69238387 100644 --- a/contrib/packs/BUILD +++ b/contrib/packs/BUILD @@ -3,3 +3,12 @@ __defaults__(all=dict(inject_pack_python_path=True)) pack_metadata( name="metadata", ) + +st2_pack_archive( + dependencies=[ + ":metadata", + "./actions", + "./actions/pack_mgmt", + "./tests", + ], +) diff --git a/contrib/packs/tests/BUILD b/contrib/packs/tests/BUILD index c8265214ca..6326aced98 100644 --- a/contrib/packs/tests/BUILD +++ b/contrib/packs/tests/BUILD @@ -1,5 +1,5 @@ # tests can only be dependencies of other tests in this directory -__dependents_rules__(("*", "/**", "!*")) +__dependents_rules__(("*", "/**", f"//{build_file_dir().parent}:files", "!*")) __defaults__( {python_test: dict(tags=["pack"])}, diff --git a/pants-plugins/macros.py b/pants-plugins/macros.py index 11131f20ee..19b26d8546 100644 --- a/pants-plugins/macros.py +++ b/pants-plugins/macros.py @@ -112,6 +112,67 @@ def st2_component_python_distribution(**kwargs): python_distribution(**kwargs) # noqa: F821 +# Default copied from PEX (which uses zipfile standard MS-DOS epoch). +# https://github.com/pex-tool/pex/blob/v2.1.137/pex/common.py#L39-L45 +MTIME = "1980-01-01T00:00:00Z" + + +def st2_pack_archive(**kwargs): + """Create a makeself_archive using files from the given dependencies. + + This macro should be used in the same BUILD file as the pack_metadata target. + """ + build_file_path = build_file_dir() # noqa: F821 + if "st2tests" == build_file_path.parts[0]: + # avoid creating duplicate archive for the core pack + # which is also located under st2tests/st2tests/fixtures/packs + return + pack_name = build_file_path.name # noqa: F821 + + dependencies = kwargs.pop("dependencies", []) + if ":metadata" not in dependencies: + dependencies = [":metadata", *dependencies] + + # This is basically a "wrap_as_files" target (which does not exist yet) + shell_command( # noqa: F821 + name="files", + execution_dependencies=dependencies, + command="true", + output_directories=["."], + root_output_directory=".", + ) + + makeself_archive( # noqa: F821 + name="archive", + label=f"{pack_name} StackStorm pack", + files=[ + ":files", # archive contents + "//:license", # LICENSE file included in archive header, excluded from contents + ], + # startup_script=["echo", "pack-archive"], + args=( + # Makeself expects '--arg value' (space) not '--arg=value' (equals) for cmdline + "--license", + "__archive/LICENSE", + "--target", + f"/opt/stackstorm/packs/{pack_name}", + # reproducibility flags: + "--tar-extra", # extra tar args: '--arg=value' (equals delimited) space separated + f"--owner=root:0 --group=root:0 --mtime={MTIME} --exclude=LICENSE", # TODO: include LICENSE file? + "--packaging-date", + MTIME, # TODO: maybe use release date instead of an epoch date? + # compression/encryption flags: + # "--gzip", # gzip is the default compressor + # "--complevel", "9", # 9 is the default compression level + # "--gpg-encrypt", # gpg (encrypt only) handles compression if selected + # "--gpg-asymmetric-encrypt-sign", # gpg (encrypt and sign) handles compression if selected + # "--gpg-extra", "...", # if using gpg, pass extra gpg args here + # "--nocomp", # maybe use no compression to use rpm/deb's compression instead of gzip (default)? + ), + output_path=f"packaging/packs/{pack_name}.tgz.run", + ) + + def st2_shell_sources_and_resources(**kwargs): """This creates a shell_sources and a resources target. diff --git a/pants.toml b/pants.toml index a2ae99b718..092dc72028 100644 --- a/pants.toml +++ b/pants.toml @@ -27,6 +27,9 @@ backend_packages = [ "pants.backend.shell", "pants.backend.shell.lint.shellcheck", + # packaging + "pants.backend.experimental.makeself", + # internal plugins in pants-plugins/ "pants.backend.plugin_development", "api_spec", @@ -43,8 +46,6 @@ pants_ignore.add = [ "test_dist_utils.py", "setup.py", # keep tailor from using legacy requirements files (not for pants) - "contrib/examples/requirements.txt", - "contrib/hello_st2/requirements.txt", "contrib/runners/*/in-requirements.txt", "contrib/runners/*/requirements.txt", "st2*/in-requirements.txt", From e6da5699e85cdbdfecdc63dd3e4e1a7ec431158b Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 2 Jan 2025 18:39:18 -0600 Subject: [PATCH 2/6] pants-plugins/pack_metadata: include Markdown files --- .../pack_metadata/python_rules/python_pack_content_test.py | 1 + pants-plugins/pack_metadata/target_types.py | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/pants-plugins/pack_metadata/python_rules/python_pack_content_test.py b/pants-plugins/pack_metadata/python_rules/python_pack_content_test.py index 33c1389bb3..cf25605b71 100644 --- a/pants-plugins/pack_metadata/python_rules/python_pack_content_test.py +++ b/pants-plugins/pack_metadata/python_rules/python_pack_content_test.py @@ -36,6 +36,7 @@ ((PackContentResourceTypes.pack_config_schema,), 4, "config.schema.yaml"), ((PackContentResourceTypes.pack_config_example,), 4, "config.yaml.example"), ((PackContentResourceTypes.pack_icon,), 4, "icon.png"), + ((PackContentResourceTypes.pack_doc,), 1, ".md"), ((PackContentResourceTypes.action_metadata,), 5, ".yaml"), ((PackContentResourceTypes.sensor_metadata,), 1, ".yaml"), ((PackContentResourceTypes.rule_metadata,), 0, ""), diff --git a/pants-plugins/pack_metadata/target_types.py b/pants-plugins/pack_metadata/target_types.py index 01d80c24ad..8fa0c8843b 100644 --- a/pants-plugins/pack_metadata/target_types.py +++ b/pants-plugins/pack_metadata/target_types.py @@ -43,6 +43,7 @@ class PackContentResourceTypes(Enum): pack_config_schema = "pack_config_schema" pack_config_example = "pack_config_example" pack_icon = "pack_icon" + pack_doc = "pack_doc" # in subdirectory (see _content_type_by_path_parts below action_metadata = "action_metadata" action_chain_workflow = "action_chain_workflow" @@ -86,6 +87,8 @@ def compute_value( if value is not None: return PackContentResourceTypes(value) path = PurePath(address.relative_file_path) + if path.suffix == ".md": + return PackContentResourceTypes.pack_doc _yaml_suffixes = (".yaml", ".yml") if len(path.parent.parts) == 0: # in the pack root @@ -123,8 +126,7 @@ class PackMetadataSourcesField(ResourcesGeneratingSourcesField): "**/*.yml", "icon.png", # used in st2web ui # "requirements*.txt", # including this causes target conflicts - # "README.md", - # "HISTORY.md", + "**/*.md", # including README.md, HISTORY.md # exclude yaml files under tests "!tests/**/*.yml", "!tests/**/*.yaml", From 940fb13fcf1f03435fcd6a81f8d4fa6b3bddb802 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 5 Mar 2025 14:09:09 -0600 Subject: [PATCH 3/6] pants: cleanup makself_archive def and add docs links --- pants-plugins/macros.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/pants-plugins/macros.py b/pants-plugins/macros.py index 19b26d8546..70a688751d 100644 --- a/pants-plugins/macros.py +++ b/pants-plugins/macros.py @@ -142,6 +142,8 @@ def st2_pack_archive(**kwargs): root_output_directory=".", ) + # https://www.pantsbuild.org/stable/docs/shell/self-extractable-archives + # https://www.pantsbuild.org/stable/reference/targets/makeself_archive makeself_archive( # noqa: F821 name="archive", label=f"{pack_name} StackStorm pack", @@ -149,8 +151,7 @@ def st2_pack_archive(**kwargs): ":files", # archive contents "//:license", # LICENSE file included in archive header, excluded from contents ], - # startup_script=["echo", "pack-archive"], - args=( + args=( # see: https://makeself.io/#usage # Makeself expects '--arg value' (space) not '--arg=value' (equals) for cmdline "--license", "__archive/LICENSE", @@ -158,16 +159,9 @@ def st2_pack_archive(**kwargs): f"/opt/stackstorm/packs/{pack_name}", # reproducibility flags: "--tar-extra", # extra tar args: '--arg=value' (equals delimited) space separated - f"--owner=root:0 --group=root:0 --mtime={MTIME} --exclude=LICENSE", # TODO: include LICENSE file? + f"--owner=root:0 --group=root:0 --mtime={MTIME} --exclude=LICENSE", "--packaging-date", - MTIME, # TODO: maybe use release date instead of an epoch date? - # compression/encryption flags: - # "--gzip", # gzip is the default compressor - # "--complevel", "9", # 9 is the default compression level - # "--gpg-encrypt", # gpg (encrypt only) handles compression if selected - # "--gpg-asymmetric-encrypt-sign", # gpg (encrypt and sign) handles compression if selected - # "--gpg-extra", "...", # if using gpg, pass extra gpg args here - # "--nocomp", # maybe use no compression to use rpm/deb's compression instead of gzip (default)? + MTIME, ), output_path=f"packaging/packs/{pack_name}.tgz.run", ) From fb276866a12d4c5f15300d1a2a040d63897f55f0 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 5 Mar 2025 13:15:49 -0600 Subject: [PATCH 4/6] add merge conflict magnet --- CHANGELOG.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index aa78a80b33..2b25f64e1e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -79,6 +79,7 @@ Added to pants' use of PEX lockfiles. This is not a user-facing addition. #6118 #6141 #6133 #6120 #6181 #6183 #6200 #6237 #6229 #6240 #6241 #6244 #6251 #6253 #6254 #6258 #6259 #6260 #6269 #6275 #6279 #6278 #6282 #6283 #6273 #6287 #6306 #6307 + #6311 Contributed by @cognifloyd * Build of ST2 EL9 packages #6153 Contributed by @amanda11 From 1a66cefec63d792a84bcb973512ec189daf73317 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 3 Jan 2025 16:46:58 -0600 Subject: [PATCH 5/6] Packaging+pants: Add BUILD macros: ST2_PACKS_GROUP and ST2_SVC_USER There is some legacy bits in st2-packages.git that attempt to pull the packs group and system user from /etc/st2/st2.conf. But, that code is not in use. Effectively, the user/group names have been hard-coded. Rather than preserve an unused install feature, the pants+nfpm based system package build will use hard-coded group/user names. --- pants-plugins/macros.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pants-plugins/macros.py b/pants-plugins/macros.py index 70a688751d..0279408d08 100644 --- a/pants-plugins/macros.py +++ b/pants-plugins/macros.py @@ -116,6 +116,10 @@ def st2_component_python_distribution(**kwargs): # https://github.com/pex-tool/pex/blob/v2.1.137/pex/common.py#L39-L45 MTIME = "1980-01-01T00:00:00Z" +# These are used for system packages (rpm/deb) +ST2_PACKS_GROUP = "st2packs" +ST2_SVC_USER = "st2" + def st2_pack_archive(**kwargs): """Create a makeself_archive using files from the given dependencies. @@ -159,7 +163,7 @@ def st2_pack_archive(**kwargs): f"/opt/stackstorm/packs/{pack_name}", # reproducibility flags: "--tar-extra", # extra tar args: '--arg=value' (equals delimited) space separated - f"--owner=root:0 --group=root:0 --mtime={MTIME} --exclude=LICENSE", + f"--owner=root --group={ST2_PACKS_GROUP} --mtime={MTIME} --exclude=LICENSE", "--packaging-date", MTIME, ), From bba75f7fba38668b07561ff376a48dc38b3d16ee Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 5 Mar 2025 19:58:29 -0600 Subject: [PATCH 6/6] pants-plugins/pack_metadata: include md files in test resource counts --- .../pack_metadata/python_rules/python_pack_content_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pants-plugins/pack_metadata/python_rules/python_pack_content_test.py b/pants-plugins/pack_metadata/python_rules/python_pack_content_test.py index cf25605b71..97bcb0ccfc 100644 --- a/pants-plugins/pack_metadata/python_rules/python_pack_content_test.py +++ b/pants-plugins/pack_metadata/python_rules/python_pack_content_test.py @@ -36,14 +36,14 @@ ((PackContentResourceTypes.pack_config_schema,), 4, "config.schema.yaml"), ((PackContentResourceTypes.pack_config_example,), 4, "config.yaml.example"), ((PackContentResourceTypes.pack_icon,), 4, "icon.png"), - ((PackContentResourceTypes.pack_doc,), 1, ".md"), + ((PackContentResourceTypes.pack_doc,), 4, ".md"), ((PackContentResourceTypes.action_metadata,), 5, ".yaml"), ((PackContentResourceTypes.sensor_metadata,), 1, ".yaml"), ((PackContentResourceTypes.rule_metadata,), 0, ""), ((PackContentResourceTypes.policy_metadata,), 0, ""), ((PackContentResourceTypes.unknown,), 0, ""), # all content types - ((), 22, ""), + ((), 26, ""), # some content types ( (