From 6f3264b2e0fe0c4ee3e482341cf3de928d75f2e4 Mon Sep 17 00:00:00 2001 From: blag Date: Fri, 13 Sep 2019 02:48:39 -0700 Subject: [PATCH 1/2] Reformat long commands in the Makefile --- .circle/Makefile | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/.circle/Makefile b/.circle/Makefile index 40870af6..dd820255 100644 --- a/.circle/Makefile +++ b/.circle/Makefile @@ -98,21 +98,35 @@ compile: @echo @echo "==================== packs-resource-register ====================" @echo - . $(VIRTUALENV_DIR)/bin/activate; if [ ! "${CHANGED_FILES}" ]; then echo "No files have changed, skipping run..."; else st2-check-register-pack-resources /tmp/packs/$(PACK_NAME) || exit 1; fi; + . $(VIRTUALENV_DIR)/bin/activate; \ + if [ ! "${CHANGED_FILES}" ]; then \ + echo "No files have changed, skipping run..."; \ + else \ + st2-check-register-pack-resources /tmp/packs/$(PACK_NAME) || exit 1; \ + fi; .PHONY: .packs-tests .packs-tests: @echo @echo "==================== packs-tests ====================" @echo - . $(VIRTUALENV_DIR)/bin/activate; if [ ! "${CHANGED_FILES}" ]; then echo "No files have changed, skipping run..."; else ($(ST2_REPO_PATH)/st2common/bin/st2-run-pack-tests -c -t -x -j -p $(ROOT_DIR) || exit 1); fi; + . $(VIRTUALENV_DIR)/bin/activate; \ + if [ ! "${CHANGED_FILES}" ]; then \ + echo "No files have changed, skipping run..."; \ + else \ + $(ST2_REPO_PATH)/st2common/bin/st2-run-pack-tests -c -t -x -j -p $(ROOT_DIR) || exit 1; \ + fi; .PHONY: .packs-missing-tests .packs-missing-tests: @echo @echo "==================== pack-missing-tests ====================" @echo - if [ ! "${CHANGED_FILES}" ]; then echo "No files have changed, skipping run..."; else (st2-check-print-pack-tests-coverage $(ROOT_DIR) || exit 1); fi; + if [ ! "${CHANGED_FILES}" ]; then \ + echo "No files have changed, skipping run..."; \ + else \ + st2-check-print-pack-tests-coverage $(ROOT_DIR) || exit 1; \ + fi; .PHONY: .clone_st2_repo .clone_st2_repo: /tmp/st2 From 5b818a947df15d34c8b1aef79c22721c83d3d1aa Mon Sep 17 00:00:00 2001 From: blag Date: Tue, 10 Sep 2019 23:00:43 -0600 Subject: [PATCH 2/2] Update Makefile to handle a number of actions that is larger than the Bash maximum arguments count --- .circle/Makefile | 85 +++++++++++++++++++++++++++++++++++++---- utils/git-changes-files | 7 ---- utils/git-changes-json | 7 ---- utils/git-changes-packs | 2 +- utils/git-changes-py | 7 ---- utils/git-changes-yaml | 7 ---- 6 files changed, 79 insertions(+), 36 deletions(-) diff --git a/.circle/Makefile b/.circle/Makefile index dd820255..1a6dd8e3 100644 --- a/.circle/Makefile +++ b/.circle/Makefile @@ -1,9 +1,11 @@ ROOT_DIR ?= $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) +ifndef FORCE_CHECK_ALL_FILES CHANGED_FILES := $(shell $(CI_DIR)/utils/git-changes files) -CHANGED_DIRECTORIES := $(shell $(CI_DIR)/utils/git-changes directories) CHANGED_PY := $(shell ${CI_DIR}/utils/git-changes py) CHANGED_YAML := $(shell $(CI_DIR)/utils/git-changes yaml) CHANGED_JSON := $(shell $(CI_DIR)/utils/git-changes json) +endif +CHANGED_DIRECTORIES := $(shell $(CI_DIR)/utils/git-changes directories) VIRTUALENV_DIR ?= virtualenv ST2_REPO_PATH ?= /tmp/st2 ST2_REPO_BRANCH ?= master @@ -65,33 +67,102 @@ compile: @echo @echo "==================== flake8 ====================" @echo - . $(VIRTUALENV_DIR)/bin/activate; if [ ! "${CHANGED_PY}" ]; then echo "No files have changed, skipping run..."; fi; for file in ${CHANGED_PY}; do if [ -n "$$file" ]; then flake8 --config=$(CI_DIR)/lint-configs/python/.flake8 $$file || exit 1; fi; done + . $(VIRTUALENV_DIR)/bin/activate; \ + if [ "$${FORCE_CHECK_ALL_FILES}" ]; then \ + find ./* -name "*.py" | while read py_file; do \ + flake8 --config=$(CI_DIR)/lint-configs/python/.flake8 $$py_file || exit 1; \ + done; \ + elif [ -n "${CHANGED_PY}" ]; then \ + for file in ${CHANGED_PY}; do \ + if [ -n "$$file" ]; then \ + flake8 --config=$(CI_DIR)/lint-configs/python/.flake8 $$file || exit 1; \ + fi; \ + done; \ + else \ + echo "No files have changed, skipping run..."; \ + fi; .PHONY: .pylint .pylint: @echo @echo "==================== pylint ====================" @echo - . $(VIRTUALENV_DIR)/bin/activate; if [ ! "${CHANGED_PY}" ]; then echo "No files have changed, skipping run..."; else (REQUIREMENTS_DIR=$(CI_DIR)/.circle/ CONFIG_DIR=$(CI_DIR)/lint-configs/ st2-check-pylint-pack $(ROOT_DIR) || exit 1); fi; + . $(VIRTUALENV_DIR)/bin/activate; \ + if [ "$${FORCE_CHECK_ALL_FILES}" ] || [ -n "${CHANGED_PY}" ]; then \ + REQUIREMENTS_DIR=$(CI_DIR)/.circle/ \ + CONFIG_DIR=$(CI_DIR)/lint-configs/ \ + st2-check-pylint-pack $(ROOT_DIR) || exit 1; \ + else \ + echo "No files have changed, skipping run..."; \ + fi; .PHONY: .configs-check .configs-check: @echo @echo "==================== configs-check ====================" @echo - . $(VIRTUALENV_DIR)/bin/activate; if [ ! "${CHANGED_YAML}" ]; then echo "No files have changed, skipping run..."; fi; for file in $(CHANGED_YAML); do if [ -n "$$file" ]; then st2-check-validate-yaml-file $$file || exit 1 ; fi; done - . $(VIRTUALENV_DIR)/bin/activate; if [ ! "${CHANGED_JSON}" ]; then echo "No files have changed, skipping run..."; fi; for file in $(CHANGED_JSON); do if [ -n "$$file" ]; then st2-check-validate-json-file $$file || exit 1 ; fi; done + @# The number of changed files in the AWS pack exceeds the limits of Bash, + @# leading to CI failures like this: + @# https://circleci.com/gh/StackStorm-Exchange/stackstorm-aws/320 + @# Instead of passing the entire list into a Bash for loop, we convert the + @# make variable to a Bash string, convert that to a Bash array, and then + @# iterate through each element of the array + . $(VIRTUALENV_DIR)/bin/activate; \ + if [ "$${FORCE_CHECK_ALL_FILES}" = "true" ]; then \ + find $(CI_DIR)/* -name "*.yaml" -o -name "*.yml" | while read yaml_file; do \ + st2-check-validate-yaml-file "$$yaml_file" || exit 1 ; \ + done; \ + elif [ -n "${CHANGED_YAML}" ]; then \ + for file in $(CHANGED_YAML); do \ + if [ -n "$$file" ]; then \ + st2-check-validate-yaml-file $$file || exit 1 ; \ + fi; \ + done; \ + else \ + echo "No files have changed, skipping run..."; \ + fi + @# + . $(VIRTUALENV_DIR)/bin/activate; \ + if [ "$${FORCE_CHECK_ALL_FILES}" = "true" ]; then \ + find $(CI_DIR)/* -name "*.json" | while read json_file; do \ + st2-check-validate-json-file "$$json_file" || exit 1 ; \ + done; \ + elif [ -n "${CHANGED_JSON}" ]; then \ + for file in $(CHANGED_JSON); do \ + if [ -n "$$file" ]; then \ + echo "file: $$file"; \ + st2-check-validate-json-file $$file || exit 1 ; \ + fi; \ + done; \ + else \ + echo "No files have changed, skipping run..."; \ + fi + @# @echo @echo "==================== example config check ====================" @echo - . $(VIRTUALENV_DIR)/bin/activate; if [ ! "${CHANGED_FILES}" ]; then echo "No files have changed, skipping run..."; else st2-check-validate-pack-example-config /tmp/packs/$(PACK_NAME) || exit 1; fi; + . $(VIRTUALENV_DIR)/bin/activate; \ + if [ "$${FORCE_CHECK_ALL_FILES}" = "true" ]; then \ + find ./* | while read file; do \ + st2-check-validate-pack-example-config $$file || exit 1; \ + done; \ + elif [ -n "${CHANGED_FILES}" ]; then \ + st2-check-validate-pack-example-config /tmp/packs/$(PACK_NAME) || exit 1; \ + else \ + echo "No files have changed, skipping run..."; \ + fi; .PHONY: .metadata-check .metadata-check: @echo @echo "==================== metadata-check ====================" @echo - . $(VIRTUALENV_DIR)/bin/activate; if [ ! "${CHANGED_YAML}" ]; then echo "No files have changed, skipping run..."; else (st2-check-validate-pack-metadata-exists $(ROOT_DIR) || exit 1); fi; + . $(VIRTUALENV_DIR)/bin/activate; \ + if [ "$${FORCE_CHECK_ALL_FILES}" = "true" ] || [ -n "${CHANGED_YAML}" ]; then \ + st2-check-validate-pack-metadata-exists $(ROOT_DIR) || exit 1; \ + else \ + echo "No files have changed, skipping run..."; \ + fi; .PHONY: .packs-resource-register .packs-resource-register: diff --git a/utils/git-changes-files b/utils/git-changes-files index 78334b14..a1e37dd4 100755 --- a/utils/git-changes-files +++ b/utils/git-changes-files @@ -2,13 +2,6 @@ ROOT_DIRECTORY=$1 BASE_BRANCH="origin/master" -# If this environment variable if specified we will return all the files so the -# checks run on all the files not only changed ones. -if [ "${FORCE_CHECK_ALL_FILES}" = "true" ]; then - echo $(find ${ROOT_DIRECTORY}/*) - exit 0 -fi - cd ${ROOT_DIRECTORY} # Note: We include all but deleted files ("D" flag) echo $(git diff --relative --diff-filter=ACMRTUXB --name-only ${BASE_BRANCH}) diff --git a/utils/git-changes-json b/utils/git-changes-json index 9cf94246..f13606ec 100755 --- a/utils/git-changes-json +++ b/utils/git-changes-json @@ -2,13 +2,6 @@ ROOT_DIRECTORY=$1 BASE_BRANCH="origin/master" -# If this environment variable if specified we will return all the files so the -# checks run on all the files not only changed ones. -if [ "${FORCE_CHECK_ALL_FILES}" = "true" ]; then - echo $(find ${ROOT_DIRECTORY}/* -name "*.json") - exit 0 -fi - cd ${ROOT_DIRECTORY} # Note: We include all but deleted files ("D" flag) echo $(git diff --relative --diff-filter=ACMRTUXB --name-only ${BASE_BRANCH} -- '*.json') diff --git a/utils/git-changes-packs b/utils/git-changes-packs index 0da125d7..2f1a8bb0 100755 --- a/utils/git-changes-packs +++ b/utils/git-changes-packs @@ -5,7 +5,7 @@ BASE_BRANCH="origin/master" # If this environment variable if specified we will return all the files so the # checks run on all the files not only changed ones. if [ "${FORCE_CHECK_ALL_FILES}" = "true" ]; then - echo $(find ${ROOT_DIRECTORY}/* -maxdepth 0 -type d | grep -v linux | grep -v openstack) + find ${ROOT_DIRECTORY}/* -maxdepth 0 -type d | grep -v linux | grep -v openstack exit 0 fi diff --git a/utils/git-changes-py b/utils/git-changes-py index dd845a44..3e34a12c 100755 --- a/utils/git-changes-py +++ b/utils/git-changes-py @@ -2,13 +2,6 @@ ROOT_DIRECTORY=$1 BASE_BRANCH="origin/master" -# If this environment variable if specified we will return all the files so the -# checks run on all the files not only changed ones. -if [ "${FORCE_CHECK_ALL_FILES}" = "true" ]; then - echo $(find ${ROOT_DIRECTORY}/* -name "*.py") - exit 0 -fi - cd ${ROOT_DIRECTORY} # Note: We include all but deleted files ("D" flag) echo $(git diff --relative --diff-filter=ACMRTUXB --name-only ${BASE_BRANCH} -- '*.py') diff --git a/utils/git-changes-yaml b/utils/git-changes-yaml index 3bb5503b..4abd1a9b 100755 --- a/utils/git-changes-yaml +++ b/utils/git-changes-yaml @@ -2,13 +2,6 @@ ROOT_DIRECTORY=$1 BASE_BRANCH="origin/master" -# If this environment variable if specified we will return all the files so the -# checks run on all the files not only changed ones. -if [ "${FORCE_CHECK_ALL_FILES}" = "true" ]; then - echo $(find ${ROOT_DIRECTORY}/* -name "*.yaml") - exit 0 -fi - cd ${ROOT_DIRECTORY} # Note: We include all but deleted files ("D" flag) echo $(git diff --relative --diff-filter=ACMRTUXB --name-only ${BASE_BRANCH} -- '*.yaml' '*.yml')