Small cleanup

.circle/Makefile

@@ -1,9 +1,9 @@
 ROOT_DIR ?= $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
 ifndef FORCE_CHECK_ALL_FILES
-CHANGED_FILES := $(shell $(CI_DIR)/utils/git-changes files)
-CHANGED_PY := $(shell ${CI_DIR}/utils/git-changes py)
-CHANGED_YAML := $(shell $(CI_DIR)/utils/git-changes yaml)
-CHANGED_JSON := $(shell $(CI_DIR)/utils/git-changes json)
+NUM_CHANGED_FILES := $(shell $(CI_DIR)/utils/git-changes files | wc -w)
+NUM_CHANGED_PY := $(shell $(CI_DIR)/utils/git-changes py | wc -w)
+NUM_CHANGED_YAML := $(shell $(CI_DIR)/utils/git-changes yaml | wc -w)
+NUM_CHANGED_JSON := $(shell $(CI_DIR)/utils/git-changes json | wc -w)
 endif
 CHANGED_DIRECTORIES := $(shell $(CI_DIR)/utils/git-changes directories)
 VIRTUALENV_DIR ?= virtualenv
@@ -24,6 +24,14 @@ all: requirements lint packs-resource-register packs-tests
 .PHONY: all-ci
 all-ci: compile .license-check .flake8 .pylint .copy-pack-to-subdirectory .configs-check .metadata-check .packs-resource-register .packs-tests
 
+.PHONY: changed-files
+changed-files:
+	@echo "NUM_CHANGED_PY: $(NUM_CHANGED_PY)"
+	@echo "NUM_CHANGED_JSON: $(NUM_CHANGED_JSON)"
+	@echo "NUM_CHANGED_YAML: $(NUM_CHANGED_YAML)"
+	@echo "NUM_CHANGED_FILES: $(NUM_CHANGED_FILES)"
+	@echo "CHANGED_DIRECTORIES: $(CHANGED_DIRECTORIES)"
+
 .PHONY: lint
 lint: requirements flake8 pylint configs-check metadata-check
 
@@ -72,12 +80,15 @@ compile:
 		find ./* -name "*.py" | while read py_file; do \
 			flake8 --config=$(CI_DIR)/lint-configs/python/.flake8 $$py_file || exit 1; \
 		done; \
-	elif [ -n "${CHANGED_PY}" ]; then \
-		for file in ${CHANGED_PY}; do \
+	elif [ ${NUM_CHANGED_PY} -gt 0 ]; then \
+		echo "Checking ${NUM_CHANGED_PY} Python files"; \
+		$(CI_DIR)/utils/git-changes py >~/.git-changes-py; \
+		while read -r file; do \
 			if [ -n "$$file" ]; then \
 				flake8 --config=$(CI_DIR)/lint-configs/python/.flake8 $$file || exit 1; \
 			fi; \
-		done; \
+		done < ~/.git-changes-py; \
+		if [ -e ~/.git-changes-py ]; then rm ~/.git-changes-py; fi; \
 	else \
 		echo "No files have changed, skipping run..."; \
 	fi;
@@ -88,7 +99,7 @@ compile:
 	@echo "==================== pylint ===================="
 	@echo
 	. $(VIRTUALENV_DIR)/bin/activate; \
-	if [ "$${FORCE_CHECK_ALL_FILES}" = "true" ] || [ -n "${CHANGED_PY}" ]; then \
+	if [ "$${FORCE_CHECK_ALL_FILES}" = "true" ] || [ ${NUM_CHANGED_PY} -gt 0 ]; then \
 		REQUIREMENTS_DIR=$(CI_DIR)/.circle/ \
 		CONFIG_DIR=$(CI_DIR)/lint-configs/ \
 		st2-check-pylint-pack $(ROOT_DIR) || exit 1; \
@@ -112,12 +123,15 @@ compile:
 		find $(CI_DIR)/* -name "*.yaml" -o -name "*.yml" | while read yaml_file; do \
 			st2-check-validate-yaml-file "$$yaml_file" || exit 1 ; \
 		done; \
-	elif [ -n "${CHANGED_YAML}" ]; then \
-		for file in $(CHANGED_YAML); do \
+	elif [ ${NUM_CHANGED_YAML} -gt 0 ]; then \
+		echo "Checking $(NUM_CHANGED_YAML) YAML files"; \
+		$(CI_DIR)/utils/git-changes yaml >~/.git-changes-yaml; \
+		while read -r file; do \
 			if [ -n "$$file" ]; then \
 				st2-check-validate-yaml-file $$file || exit 1 ; \
 			fi; \
-		done; \
+		done < ~/.git-changes-yaml; \
+		if [ -e ~/.git-changes-yaml ]; then rm ~/.git-changes-yaml; fi; \
 	else \
 		echo "No files have changed, skipping run..."; \
 	fi
@@ -127,13 +141,16 @@ compile:
 		find $(CI_DIR)/* -name "*.json" | while read json_file; do \
 			st2-check-validate-json-file "$$json_file" || exit 1 ; \
 		done; \
-	elif [ -n "${CHANGED_JSON}" ]; then \
-		for file in $(CHANGED_JSON); do \
+	elif [ -n "${NUM_CHANGED_JSON}" ]; then \
+		echo "Checking $(NUM_CHANGED_JSON) JSON files"; \
+		$(CI_DIR)/utils/git-changes json >~/.git-changes-json; \
+		while read -r file; do \
 			if [ -n "$$file" ]; then \
 				echo "file: $$file"; \
 				st2-check-validate-json-file $$file || exit 1 ; \
 			fi; \
-		done; \
+		done < ~/.git-changes-json; \
+		if [ -e ~/.git-changes-json ]; then rm ~/.git-changes-json; fi; \
 	else \
 		echo "No files have changed, skipping run..."; \
 	fi
@@ -142,7 +159,7 @@ compile:
 	@echo "==================== example config check ===================="
 	@echo
 	. $(VIRTUALENV_DIR)/bin/activate; \
-	if [ "$${FORCE_CHECK_ALL_FILES}" = "true" ] || [ -n "${CHANGED_FILES}" ]; then \
+	if [ "$${FORCE_CHECK_ALL_FILES}" = "true" ] || [ ${NUM_CHANGED_FILES} -gt 0 ]; then \
 		st2-check-validate-pack-example-config /tmp/packs/$(PACK_NAME) || exit 1; \
 	else \
 		echo "No files have changed, skipping run..."; \
@@ -154,7 +171,7 @@ compile:
 	@echo "==================== metadata-check ===================="
 	@echo
 	. $(VIRTUALENV_DIR)/bin/activate; \
-	if [ "$${FORCE_CHECK_ALL_FILES}" = "true" ] || [ -n "${CHANGED_YAML}" ]; then \
+	if [ "$${FORCE_CHECK_ALL_FILES}" = "true" ] || [ ${NUM_CHANGED_YAML} -gt 0 ]; then \
 		st2-check-validate-pack-metadata-exists $(ROOT_DIR) || exit 1; \
 	else \
 		echo "No files have changed, skipping run..."; \
@@ -166,7 +183,7 @@ compile:
 	@echo "==================== packs-resource-register ===================="
 	@echo
 	. $(VIRTUALENV_DIR)/bin/activate; \
-	if [ -z "${CHANGED_FILES}" ]; then \
+	if [ ${NUM_CHANGED_FILES} -eq 0 ]; then \
 		echo "No files have changed, skipping run..."; \
 	else \
 		st2-check-register-pack-resources /tmp/packs/$(PACK_NAME) || exit 1; \
@@ -178,7 +195,7 @@ compile:
 	@echo "==================== packs-tests ===================="
 	@echo
 	. $(VIRTUALENV_DIR)/bin/activate; \
-	if [ -z "${CHANGED_FILES}" ]; then \
+	if [ ${NUM_CHANGED_FILES} -eq 0 ]; then \
 		echo "No files have changed, skipping run..."; \
 	else \
 		$(ST2_REPO_PATH)/st2common/bin/st2-run-pack-tests -c -t -x -j -p $(ROOT_DIR) || exit 1; \
@@ -189,7 +206,7 @@ compile:
 	@echo
 	@echo "==================== pack-missing-tests ===================="
 	@echo
-	if [ -z "${CHANGED_FILES}" ]; then \
+	if [ ${NUM_CHANGED_FILES} -eq 0 ]; then \
 		echo "No files have changed, skipping run..."; \
 	else \
 		st2-check-print-pack-tests-coverage $(ROOT_DIR) || exit 1; \
@@ -201,7 +218,7 @@ compile:
 	@echo
 	@echo "==================== license-check ===================="
 	@echo
-	if [ -z "${CHANGED_FILES}" ] && [ "$${FORCE_CHECK_ALL_FILES}" = "false" ]; then \
+	if [ ${NUM_CHANGED_FILES} -eq 0 ] && [ "$${FORCE_CHECK_ALL_FILES}" = "false" ]; then \
 		echo "No files have changed, skipping run..."; \
 	else \
 		if [ ! -f "$(ROOT_DIR)/LICENSE" ]; then \

.circle/dependencies

@@ -18,10 +18,10 @@ python --version
 pip --version
 
 # Clone st2 repo so other scripts can reference StackStorm Python code
-[ -d /tmp/st2 ] || git clone --depth 1 --single-branch --branch "${ST2_BRANCH}" https://github.com/StackStorm/st2.git /tmp/st2
+[[ -d /tmp/st2 ]] || git clone --depth 1 --single-branch --branch "${ST2_BRANCH}" https://github.com/StackStorm/st2.git /tmp/st2
 
 # Clone lint-configs
-[ -d ~/ci/lint-configs ] || git clone --depth 1 --single-branch --branch "${LINT_CONFIGS_BRANCH}" https://github.com/StackStorm/lint-configs.git ~/ci/lint-configs
+[[ -d ~/ci/lint-configs ]] || git clone --depth 1 --single-branch --branch "${LINT_CONFIGS_BRANCH}" https://github.com/StackStorm/lint-configs.git ~/ci/lint-configs
 
 # Create a directory for debian packages so we can cache it in Circle CI
 sudo rm -rf /var/cache/apt/archives && sudo ln -s ~/.apt-cache /var/cache/apt/archives && mkdir -p ~/.apt-cache/partial
@@ -43,7 +43,7 @@ echo "Installing StackStorm requirements from /tmp/st2/requirements.txt"
 # Copy over Makefile and  install StackStorm runners and register metrics drivers
 echo "Installing StackStorm runners and registering metrics drivers"
 
-if [ ! -z "${ROOT_DIR}" ]; then
+if [[ -n "${ROOT_DIR}" ]]; then
     PACK_REQUIREMENTS_FILE="${ROOT_DIR}/requirements.txt"
     PACK_TESTS_REQUIREMENTS_FILE="${ROOT_DIR}/requirements-tests.txt"
 
@@ -60,13 +60,13 @@ else
 fi
 
 # Install pack requirements
-if [ -f "${PACK_REQUIREMENTS_FILE}" ]; then
+if [[ -f "${PACK_REQUIREMENTS_FILE}" ]]; then
     echo "Installing pack requirements from ${PACK_REQUIREMENTS_FILE}"
     ~/virtualenv/bin/pip install -r "${PACK_REQUIREMENTS_FILE}"
 fi
 
 # Install pack tests requirements
-if [ -f "${PACK_TESTS_REQUIREMENTS_FILE}" ]; then
+if [[ -f "${PACK_TESTS_REQUIREMENTS_FILE}" ]]; then
     echo "Installing pack tests requirements from ${PACK_TESTS_REQUIREMENTS_FILE}"
     ~/virtualenv/bin/pip install -r "${PACK_TESTS_REQUIREMENTS_FILE}"
 fi

.circle/deployment

@@ -6,7 +6,7 @@ FORCE_REBUILD_INDEX="${FORCE_REBUILD_INDEX:-0}"
 
 export PYTHONPATH=/tmp/st2/st2common:${PYTHONPATH}
 
-if [ -z "${PACK_NAME}" ]; then
+if [[ -z "${PACK_NAME}" ]]; then
     export PACK_NAME=$(~/virtualenv/bin/python ~/ci/.circle/validate.py "${CIRCLE_PROJECT_REPONAME}" pack.yaml)
 else
     export PACK_NAME="${PACK_NAME}"
@@ -21,7 +21,7 @@ fi
 # TODO: figure out how to make deploy.py rebuild the index.
 # python ~/packs/.circle/deploy.py pack.yaml "${CIRCLE_PROJECT_REPONAME}"
 
-# Clean up so the script can be retries in case of failure (e.g. race)
+# Clean up so the script can be retried in case of failure (e.g. race)
 rm -rf ~/index
 git clone https://${MACHINE_USER}:${MACHINE_PASSWORD}@github.com/StackStorm-Exchange/index ~/index 2>/dev/null
 
@@ -63,9 +63,9 @@ git status
 ~/virtualenv/bin/python ~/ci/utils/pack_content.py --input . --output ~/index/v1/packs/"${PACK_NAME}"
 
 # Rebuild the index JSON
-if [ -n "$(git -C ~/index status -s)" ] || [ ${FORCE_REBUILD_INDEX} == "1" ];
+if [[ -n "$(git -C ~/index status -s)" ]] || [[ ${FORCE_REBUILD_INDEX} == "1" ]];
 then
-  if [ ${FORCE_REBUILD_INDEX} == "1" ]; then
+  if [[ ${FORCE_REBUILD_INDEX} == "1" ]]; then
       echo "Forcing index rebuild..."
   fi
   ~/virtualenv/bin/python ~/ci/.circle/index.py --glob "~/index/v1/packs/*/pack.yaml" --output "~/index/v1/"
@@ -75,7 +75,7 @@ fi
 
 # Check if an icon has been added or changed
 ICON_TARGET="${HOME}/index/v1/icons/${PACK_NAME}.png"
-if [ -f icon.png ] && { [ ! -f ${ICON_TARGET} ] || ! cmp -s icon.png ${ICON_TARGET}; }
+if [[ -f icon.png ]] && { [[ ! -f ${ICON_TARGET} ]] || ! cmp -s icon.png ${ICON_TARGET}; }
 then
   echo "Uploading the pack icon..."
   mkdir -p ~/index/v1/icons/

.circle/exit_on_py2_checks

@@ -2,7 +2,7 @@
 
 # Script which checks exit code of "test" script under Python 2.7 environment
 # and exists with an appropriate code
-if [ $# -lt 1 ] ; then
+if [[ $# -lt 1 ]] ; then
     echo "Usage: $0 <test script exit code>"
     echo "Example: $0 1"
     exit 2
@@ -15,7 +15,7 @@ echo "Original script exit code: ${TEST_EXIT_CODE}"
 # If pack doesn't declare Python 2 support, we don't treat failures as fatal
 SUPPORTS_PYTHON2=$(~/virtualenv/bin/python -c $'import yaml, sys\nresult = yaml.safe_load(open("pack.yaml", "r").read())\nif "2" in result.get("python_versions", []):\n    print("yes")')
 
-if [ "${SUPPORTS_PYTHON2}" != "yes" ] && [ ${TEST_EXIT_CODE} -ne 0 ] ; then
+if [[ "${SUPPORTS_PYTHON2}" != "yes" ]] && [[ ${TEST_EXIT_CODE} -ne 0 ]] ; then
     echo "Ignoring failures since pack doesn't declare Python 2 support in pack.yaml"
     exit 0
 fi

.circle/exit_on_py3_checks

@@ -2,7 +2,7 @@
 
 # Script which checks exit code of "test" script under Python 3.6 environment
 # and exists with an appropriate code
-if [ $# -lt 1 ] ; then
+if [[ $# -lt 1 ]] ; then
     echo "Usage: $0 <test script exit code>"
     echo "Example: $0 1"
     exit 2
@@ -15,7 +15,7 @@ echo "Original script exit code: ${TEST_EXIT_CODE}"
 # If pack doesn't declare Python 3 support, we don't treat failures as fatal
 SUPPORTS_PYTHON3=$(~/virtualenv/bin/python -c $'import yaml, sys\nresult = yaml.safe_load(open("pack.yaml", "r").read())\nif "3" in result.get("python_versions", []):\n    print("yes")')
 
-if [ "${SUPPORTS_PYTHON3}" != "yes" ] && [ ${TEST_EXIT_CODE} -ne 0 ] ; then
+if [[ "${SUPPORTS_PYTHON3}" != "yes" ]] && [[ ${TEST_EXIT_CODE} -ne 0 ]] ; then
     echo "Ignoring failures since pack doesn't declare Python 3 support in pack.yaml"
     exit 0
 fi

.circle/retry_on_failure.sh

@@ -5,17 +5,17 @@ MAX_RETRIES=5
 
 COMMAND=$1
 
-if [ $# -ne 1 ]; then
+if [[ $# -ne 1 ]]; then
     echo "Usage: ${0} <shell command>"
     exit 1
 fi
 
 COUNT=0
-while [  ${COUNT} -lt ${MAX_RETRIES} ]; do
+while [[  ${COUNT} -lt ${MAX_RETRIES} ]]; do
    ${COMMAND}
    EXIT_CODE=$?
 
-   if [ ${EXIT_CODE} -eq 0 ]; then
+   if [[ ${EXIT_CODE} -eq 0 ]]; then
       exit 0
    fi
 

.circle/test

@@ -7,7 +7,7 @@ set -e
 export CI_DIR=/home/circleci/ci
 export PYTHONPATH=/tmp/st2/st2common:${PYTHONPATH}
 
-if [ -z "${PACK_NAME}" ]; then
+if [[ -z "${PACK_NAME}" ]]; then
     export PACK_NAME=$(~/virtualenv/bin/python ~/ci/.circle/validate.py "${CIRCLE_PROJECT_REPONAME}" pack.yaml)
 else
     export PACK_NAME="${PACK_NAME}"
@@ -28,12 +28,12 @@ echo "Running on branch: ${GIT_BRANCH}"
 echo "Running with Python from $(which python)"
 echo "Python version: ${PYTHON_VERSION}"
 
-if [ "${GIT_BRANCH}" = "master" ]; then
+if [[ "${GIT_BRANCH}" = "master" ]]; then
     echo "Running on master branch, forcing check of all files"
-    export FORCE_CHECK_ALL_FILES="true"
+    export FORCE_CHECK_ALL_FILES="${FORCE_CHECK_ALL_FILES:-true}"
 fi
 
-if [ ! -z "${FORCE_CHECK_ALL_FILES}" ]; then
+if [[ -n "${FORCE_CHECK_ALL_FILES}" ]]; then
     export FORCE_CHECK_ALL_FILES="${FORCE_CHECK_ALL_FILES}"
 fi
 
@@ -43,8 +43,9 @@ export ST2_CONFIG_FILE=${CI_DIR}/conf/st2.tests.conf
 echo "ROOT_DIR=${ROOT_DIR}"
 echo "FORCE_CHECK_ALL_FILES=${FORCE_CHECK_ALL_FILES}"
 
-if [ ! -z "${ROOT_DIR}" ]; then
+if [[ -n "${ROOT_DIR}" ]]; then
     export ROOT_DIR="${ROOT_DIR}"
+    echo "ROOT_DIR: ${ROOT_DIR}"
     make -C ${ROOT_DIR} all-ci
 else
     make all-ci

.circleci/config.yml

@@ -42,8 +42,7 @@ jobs:
           name: Check Bash file syntax
           command: |
             find . -name '*.sh' | xargs bash -n
-            grep -lrE '^#!/bin/bash' | xargs bash -n
-            grep -lrE '^#!/usr/bin/env sh' . | xargs sh -n\
+            grep -lrE '^#!/bin/bash' . | xargs bash -n
       - run:
           name: Check Python files
           command: |
@@ -98,8 +97,7 @@ jobs:
           name: Check Bash file syntax
           command: |
             find . -name '*.sh' | xargs bash -n
-            grep -lrE '^#!/bin/bash' | xargs bash -n
-            grep -lrE '^#!/usr/bin/env sh' . | xargs sh -n\
+            grep -lrE '^#!/bin/bash' . | xargs bash -n
       - run:
           name: Check Python files
           command: |

tools/add_webhook_to_github_repo.sh

@@ -16,15 +16,15 @@ set -e
 SCRIPT_PATH=$(dirname "$(readlink -f "$0")")
 source "${SCRIPT_PATH}/common.sh"
 
-if [ ! $# -eq 1 ];
+if [[ ! $# -eq 1 ]];
 then
     echo "Usage: $0 <webhook url>"
     exit 1;
 fi
 
 WEBHOOK_URL="$1"
 
-if [ ! -z "${REPO_NAMES}" ]; then
+if [[ -n "${REPO_NAMES}" ]]; then
   OIFS=$IFS;
   IFS=" "
   REPO_NAMES=($REPO_NAMES)

tools/reset_github_user_token_and_update_circleci.sh

@@ -11,7 +11,7 @@
 # * CIRCLECI_TOKEN: a CircleCI token for the Exchange organization.
 #
 
-if [ ! $# -eq 1 ];
+if [[ ! $# -eq 1 ]];
 then
     echo "Usage: $0 <pack>"
     exit 1
@@ -29,7 +29,7 @@ curl -sS --fail -u "${USERNAME}:${PASSWORD}" -X POST --header "Content-Type: app
     -d '{"scopes": ["public_repo"], "note": "CircleCI: '"${REPO_NAME}"'"}' \
     "https://api.github.com/authorizations" | jq ".token" > "/tmp/${PACK}_user_token"
 
-if [ ! -s "/tmp/${PACK}_user_token" ];
+if [[ ! -s "/tmp/${PACK}_user_token" ]];
 then
     echo "Could not create a token."
     exit 1

tools/set_repo_description.sh

@@ -16,7 +16,7 @@ set -e
 SCRIPT_PATH=$(dirname "$(readlink -f "$0")")
 source "${SCRIPT_PATH}/common.sh"
 
-if [ ! -z "${REPO_NAMES}" ]; then
+if [[ -n "${REPO_NAMES}" ]]; then
   OIFS=$IFS;
   IFS=" "
   REPO_NAMES=($REPO_NAMES)
@@ -32,7 +32,7 @@ for REPO_NAME in ${REPO_NAMES[@]}; do
     PACK_YAML_URL="https://raw.githubusercontent.com/StackStorm-Exchange/${REPO_NAME}/master/pack.yaml"
     PACK_DESCRIPTION=$(curl -sS --fail -X GET "${PACK_YAML_URL}" | python -c 'import yaml,sys; c=yaml.safe_load(sys.stdin);print c["description"]')
 
-    if [ -z "${PACK_DESCRIPTION}" ]; then
+    if [[ -z "${PACK_DESCRIPTION}" ]]; then
         echo "Description not available for pack ${REPO_NAME}, skipping..."
     else
         curl -sS --fail -u "${USERNAME}:${PASSWORD}" -X PATCH --header "Content-Type: application/json" \