Change Makefile to handle reeeeeally long lists of files

.circle/Makefile

@@ -1,9 +1,11 @@
 ROOT_DIR ?= $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
+ifndef FORCE_CHECK_ALL_FILES
 CHANGED_FILES := $(shell $(CI_DIR)/utils/git-changes files)
-CHANGED_DIRECTORIES := $(shell $(CI_DIR)/utils/git-changes directories)
 CHANGED_PY := $(shell ${CI_DIR}/utils/git-changes py)
 CHANGED_YAML := $(shell $(CI_DIR)/utils/git-changes yaml)
 CHANGED_JSON := $(shell $(CI_DIR)/utils/git-changes json)
+endif
+CHANGED_DIRECTORIES := $(shell $(CI_DIR)/utils/git-changes directories)
 VIRTUALENV_DIR ?= virtualenv
 ST2_REPO_PATH ?= /tmp/st2
 ST2_REPO_BRANCH ?= master
@@ -65,54 +67,137 @@ compile:
 	@echo
 	@echo "==================== flake8 ===================="
 	@echo
-	. $(VIRTUALENV_DIR)/bin/activate; if [ ! "${CHANGED_PY}" ]; then echo "No files have changed, skipping run..."; fi; for file in ${CHANGED_PY}; do if [ -n "$$file" ]; then flake8 --config=$(CI_DIR)/lint-configs/python/.flake8 $$file || exit 1; fi; done
+	. $(VIRTUALENV_DIR)/bin/activate; \
+	if [ "$${FORCE_CHECK_ALL_FILES}" ]; then \
+		find ./* -name "*.py" | while read py_file; do \
+			flake8 --config=$(CI_DIR)/lint-configs/python/.flake8 $$py_file || exit 1; \
+		done; \
+	elif [ -n "${CHANGED_PY}" ]; then \
+		for file in ${CHANGED_PY}; do \
+			if [ -n "$$file" ]; then \
+				flake8 --config=$(CI_DIR)/lint-configs/python/.flake8 $$file || exit 1; \
+			fi; \
+		done; \
+	else \
+		echo "No files have changed, skipping run..."; \
+	fi;
 
 .PHONY: .pylint
 .pylint:
 	@echo
 	@echo "==================== pylint ===================="
 	@echo
-	. $(VIRTUALENV_DIR)/bin/activate; if [ ! "${CHANGED_PY}" ]; then echo "No files have changed, skipping run..."; else (REQUIREMENTS_DIR=$(CI_DIR)/.circle/ CONFIG_DIR=$(CI_DIR)/lint-configs/ st2-check-pylint-pack $(ROOT_DIR) || exit 1); fi;
+	. $(VIRTUALENV_DIR)/bin/activate; \
+	if [ "$${FORCE_CHECK_ALL_FILES}" ] || [ -n "${CHANGED_PY}" ]; then \
+		REQUIREMENTS_DIR=$(CI_DIR)/.circle/ \
+		CONFIG_DIR=$(CI_DIR)/lint-configs/ \
+		st2-check-pylint-pack $(ROOT_DIR) || exit 1; \
+	else \
+		echo "No files have changed, skipping run..."; \
+	fi;
 
 .PHONY: .configs-check
 .configs-check:
 	@echo
 	@echo "==================== configs-check ===================="
 	@echo
-	. $(VIRTUALENV_DIR)/bin/activate; if [ ! "${CHANGED_YAML}" ]; then echo "No files have changed, skipping run..."; fi; for file in $(CHANGED_YAML); do if [ -n "$$file" ]; then st2-check-validate-yaml-file $$file || exit 1 ; fi; done
-	. $(VIRTUALENV_DIR)/bin/activate; if [ ! "${CHANGED_JSON}" ]; then echo "No files have changed, skipping run..."; fi; for file in $(CHANGED_JSON); do if [ -n "$$file" ]; then st2-check-validate-json-file $$file || exit 1 ; fi; done
+	@# The number of changed files in the AWS pack exceeds the limits of Bash,
+	@# leading to CI failures like this:
+	@# https://circleci.com/gh/StackStorm-Exchange/stackstorm-aws/320
+	@# Instead of passing the entire list into a Bash for loop, we convert the
+	@# make variable to a Bash string, convert that to a Bash array, and then
+	@# iterate through each element of the array
+	. $(VIRTUALENV_DIR)/bin/activate; \
+	if [ "$${FORCE_CHECK_ALL_FILES}" = "true" ]; then \
+		find $(CI_DIR)/* -name "*.yaml" -o -name "*.yml" | while read yaml_file; do \
+			st2-check-validate-yaml-file "$$yaml_file" || exit 1 ; \
+		done; \
+	elif [ -n "${CHANGED_YAML}" ]; then \
+		for file in $(CHANGED_YAML); do \
+			if [ -n "$$file" ]; then \
+				st2-check-validate-yaml-file $$file || exit 1 ; \
+			fi; \
+		done; \
+	else \
+		echo "No files have changed, skipping run..."; \
+	fi
+	@#
+	. $(VIRTUALENV_DIR)/bin/activate; \
+	if [ "$${FORCE_CHECK_ALL_FILES}" = "true" ]; then \
+		find $(CI_DIR)/* -name "*.json" | while read json_file; do \
+			st2-check-validate-json-file "$$json_file" || exit 1 ; \
+		done; \
+	elif [ -n "${CHANGED_JSON}" ]; then \
+		for file in $(CHANGED_JSON); do \
+			if [ -n "$$file" ]; then \
+				echo "file: $$file"; \
+				st2-check-validate-json-file $$file || exit 1 ; \
+			fi; \
+		done; \
+	else \
+		echo "No files have changed, skipping run..."; \
+	fi
+	@#
 	@echo
 	@echo "==================== example config check ===================="
 	@echo
-	. $(VIRTUALENV_DIR)/bin/activate; if [ ! "${CHANGED_FILES}" ]; then echo "No files have changed, skipping run..."; else st2-check-validate-pack-example-config /tmp/packs/$(PACK_NAME) || exit 1; fi;
+	. $(VIRTUALENV_DIR)/bin/activate; \
+	if [ "$${FORCE_CHECK_ALL_FILES}" = "true" ]; then \
+		find ./* | while read file; do \
+			st2-check-validate-pack-example-config $$file || exit 1; \
+		done; \
+	elif [ -n "${CHANGED_FILES}" ]; then \
+		st2-check-validate-pack-example-config /tmp/packs/$(PACK_NAME) || exit 1; \
+	else \
+		echo "No files have changed, skipping run..."; \
+	fi;
 
 .PHONY: .metadata-check
 .metadata-check:
 	@echo
 	@echo "==================== metadata-check ===================="
 	@echo
-	. $(VIRTUALENV_DIR)/bin/activate; if [ ! "${CHANGED_YAML}" ]; then echo "No files have changed, skipping run..."; else (st2-check-validate-pack-metadata-exists $(ROOT_DIR) || exit 1); fi;
+	. $(VIRTUALENV_DIR)/bin/activate; \
+	if [ "$${FORCE_CHECK_ALL_FILES}" = "true" ] || [ -n "${CHANGED_YAML}" ]; then \
+		st2-check-validate-pack-metadata-exists $(ROOT_DIR) || exit 1; \
+	else \
+		echo "No files have changed, skipping run..."; \
+	fi;
 
 .PHONY: .packs-resource-register
 .packs-resource-register:
 	@echo
 	@echo "==================== packs-resource-register ===================="
 	@echo
-	. $(VIRTUALENV_DIR)/bin/activate; if [ ! "${CHANGED_FILES}" ]; then echo "No files have changed, skipping run..."; else st2-check-register-pack-resources /tmp/packs/$(PACK_NAME) || exit 1; fi;
+	. $(VIRTUALENV_DIR)/bin/activate; \
+	if [ ! "${CHANGED_FILES}" ]; then \
+		echo "No files have changed, skipping run..."; \
+	else \
+		st2-check-register-pack-resources /tmp/packs/$(PACK_NAME) || exit 1; \
+	fi;
 
 .PHONY: .packs-tests
 .packs-tests:
 	@echo
 	@echo "==================== packs-tests ===================="
 	@echo
-	. $(VIRTUALENV_DIR)/bin/activate; if [ ! "${CHANGED_FILES}" ]; then echo "No files have changed, skipping run..."; else ($(ST2_REPO_PATH)/st2common/bin/st2-run-pack-tests -c -t -x -j -p $(ROOT_DIR) || exit 1); fi;
+	. $(VIRTUALENV_DIR)/bin/activate; \
+	if [ ! "${CHANGED_FILES}" ]; then \
+		echo "No files have changed, skipping run..."; \
+	else \
+		$(ST2_REPO_PATH)/st2common/bin/st2-run-pack-tests -c -t -x -j -p $(ROOT_DIR) || exit 1; \
+	fi;
 
 .PHONY: .packs-missing-tests
 .packs-missing-tests:
 	@echo
 	@echo "==================== pack-missing-tests ===================="
 	@echo
-	if [ ! "${CHANGED_FILES}" ]; then echo "No files have changed, skipping run..."; else (st2-check-print-pack-tests-coverage $(ROOT_DIR) || exit 1); fi;
+	if [ ! "${CHANGED_FILES}" ]; then \
+		echo "No files have changed, skipping run..."; \
+	else \
+		st2-check-print-pack-tests-coverage $(ROOT_DIR) || exit 1; \
+	fi;
 
 .PHONY: .clone_st2_repo
 .clone_st2_repo: /tmp/st2

utils/git-changes-files

@@ -2,13 +2,6 @@
 ROOT_DIRECTORY=$1
 BASE_BRANCH="origin/master"
 
-# If this environment variable if specified we will return all the files so the
-# checks run on all the files not only changed ones.
-if [ "${FORCE_CHECK_ALL_FILES}" = "true" ]; then
-    echo $(find ${ROOT_DIRECTORY}/*)
-    exit 0
-fi
-
 cd ${ROOT_DIRECTORY}
 # Note: We include all but deleted files ("D" flag)
 echo $(git diff --relative --diff-filter=ACMRTUXB --name-only ${BASE_BRANCH})

utils/git-changes-json

@@ -2,13 +2,6 @@
 ROOT_DIRECTORY=$1
 BASE_BRANCH="origin/master"
 
-# If this environment variable if specified we will return all the files so the
-# checks run on all the files not only changed ones.
-if [ "${FORCE_CHECK_ALL_FILES}" = "true" ]; then
-    echo $(find ${ROOT_DIRECTORY}/* -name "*.json")
-    exit 0
-fi
-
 cd ${ROOT_DIRECTORY}
 # Note: We include all but deleted files ("D" flag)
 echo $(git diff --relative --diff-filter=ACMRTUXB --name-only ${BASE_BRANCH} -- '*.json')

utils/git-changes-packs

@@ -5,7 +5,7 @@ BASE_BRANCH="origin/master"
 # If this environment variable if specified we will return all the files so the
 # checks run on all the files not only changed ones.
 if [ "${FORCE_CHECK_ALL_FILES}" = "true" ]; then
-    echo $(find ${ROOT_DIRECTORY}/* -maxdepth 0 -type d | grep -v linux | grep -v openstack)
+    find ${ROOT_DIRECTORY}/* -maxdepth 0 -type d | grep -v linux | grep -v openstack
     exit 0
 fi
 

utils/git-changes-py

@@ -2,13 +2,6 @@
 ROOT_DIRECTORY=$1
 BASE_BRANCH="origin/master"
 
-# If this environment variable if specified we will return all the files so the
-# checks run on all the files not only changed ones.
-if [ "${FORCE_CHECK_ALL_FILES}" = "true" ]; then
-    echo $(find ${ROOT_DIRECTORY}/* -name "*.py")
-    exit 0
-fi
-
 cd ${ROOT_DIRECTORY}
 # Note: We include all but deleted files ("D" flag)
 echo $(git diff --relative --diff-filter=ACMRTUXB --name-only ${BASE_BRANCH} -- '*.py')

utils/git-changes-yaml

@@ -2,13 +2,6 @@
 ROOT_DIRECTORY=$1
 BASE_BRANCH="origin/master"
 
-# If this environment variable if specified we will return all the files so the
-# checks run on all the files not only changed ones.
-if [ "${FORCE_CHECK_ALL_FILES}" = "true" ]; then
-    echo $(find ${ROOT_DIRECTORY}/* -name "*.yaml")
-    exit 0
-fi
-
 cd ${ROOT_DIRECTORY}
 # Note: We include all but deleted files ("D" flag)
 echo $(git diff --relative --diff-filter=ACMRTUXB --name-only ${BASE_BRANCH} -- '*.yaml' '*.yml')