From 8f7d37a7a0935ced8e4737378bcb7c9b8897cbfa Mon Sep 17 00:00:00 2001 From: blag Date: Tue, 14 Apr 2020 22:29:59 -0700 Subject: [PATCH 01/16] Add make target to print out CHANGED_ variables --- .circle/Makefile | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.circle/Makefile b/.circle/Makefile index 428105c0..b1a9571b 100644 --- a/.circle/Makefile +++ b/.circle/Makefile @@ -24,6 +24,19 @@ all: requirements lint packs-resource-register packs-tests .PHONY: all-ci all-ci: compile .license-check .flake8 .pylint .copy-pack-to-subdirectory .configs-check .metadata-check .packs-resource-register .packs-tests +.PHONY: changed-files +changed-files: + @echo "CHANGED_PY:" + @echo "$(CHANGED_PY)" + @echo "CHANGED_JSON:" + @echo "$(CHANGED_JSON)" + @echo "CHANGED_DIRECTORIES:" + @echo "$(CHANGED_DIRECTORIES)" + @echo "CHANGED_YAML:" + @echo "$(CHANGED_YAML)" + @echo "CHANGED_FILES:" + @echo "$(CHANGED_FILES)" + .PHONY: lint lint: requirements flake8 pylint configs-check metadata-check From 9734aa6056b3f7820efacb7a57702a9b435b5b64 Mon Sep 17 00:00:00 2001 From: blag Date: Tue, 14 Apr 2020 22:30:27 -0700 Subject: [PATCH 02/16] Fix typo --- .circle/deployment | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circle/deployment b/.circle/deployment index 34ef9a6a..cc21397c 100755 --- a/.circle/deployment +++ b/.circle/deployment @@ -21,7 +21,7 @@ fi # TODO: figure out how to make deploy.py rebuild the index. # python ~/packs/.circle/deploy.py pack.yaml "${CIRCLE_PROJECT_REPONAME}" -# Clean up so the script can be retries in case of failure (e.g. race) +# Clean up so the script can be retried in case of failure (e.g. race) rm -rf ~/index git clone https://${MACHINE_USER}:${MACHINE_PASSWORD}@github.com/StackStorm-Exchange/index ~/index 2>/dev/null From 960ddc5524c949f9df70ed8e13df1ee19be5a2c9 Mon Sep 17 00:00:00 2001 From: blag Date: Tue, 14 Apr 2020 22:49:48 -0700 Subject: [PATCH 03/16] Use Bash's builtin [[ ]] operator instead of the [ executable --- .circle/Makefile | 34 +++++++++---------- .circle/dependencies | 10 +++--- .circle/deployment | 8 ++--- .circle/exit_on_py2_checks | 4 +-- .circle/exit_on_py3_checks | 4 +-- .circle/retry_on_failure.sh | 6 ++-- .circle/test | 8 ++--- tools/add_webhook_to_github_repo.sh | 4 +-- ...t_github_user_token_and_update_circleci.sh | 4 +-- tools/set_repo_description.sh | 4 +-- tools/split-packs.sh | 4 +-- .../trigger_circle_ci_builds_for_all_packs.sh | 6 ++-- utils/exchange-bootstrap.sh | 4 +-- utils/git-changes | 2 +- utils/git-changes-packs | 2 +- 15 files changed, 52 insertions(+), 52 deletions(-) diff --git a/.circle/Makefile b/.circle/Makefile index b1a9571b..96a50912 100644 --- a/.circle/Makefile +++ b/.circle/Makefile @@ -81,13 +81,13 @@ compile: @echo "==================== flake8 ====================" @echo . $(VIRTUALENV_DIR)/bin/activate; \ - if [ "$${FORCE_CHECK_ALL_FILES}" = "true" ]; then \ + if [[ "$${FORCE_CHECK_ALL_FILES}" = "true" ]]; then \ find ./* -name "*.py" | while read py_file; do \ flake8 --config=$(CI_DIR)/lint-configs/python/.flake8 $$py_file || exit 1; \ done; \ - elif [ -n "${CHANGED_PY}" ]; then \ + elif [[ -n "${CHANGED_PY}" ]]; then \ for file in ${CHANGED_PY}; do \ - if [ -n "$$file" ]; then \ + if [[ -n "$$file" ]]; then \ flake8 --config=$(CI_DIR)/lint-configs/python/.flake8 $$file || exit 1; \ fi; \ done; \ @@ -101,7 +101,7 @@ compile: @echo "==================== pylint ====================" @echo . $(VIRTUALENV_DIR)/bin/activate; \ - if [ "$${FORCE_CHECK_ALL_FILES}" = "true" ] || [ -n "${CHANGED_PY}" ]; then \ + if [[ "$${FORCE_CHECK_ALL_FILES}" = "true" ]] || [[ -n "${CHANGED_PY}" ]]; then \ REQUIREMENTS_DIR=$(CI_DIR)/.circle/ \ CONFIG_DIR=$(CI_DIR)/lint-configs/ \ st2-check-pylint-pack $(ROOT_DIR) || exit 1; \ @@ -121,13 +121,13 @@ compile: @# make variable to a Bash string, convert that to a Bash array, and then @# iterate through each element of the array . $(VIRTUALENV_DIR)/bin/activate; \ - if [ "$${FORCE_CHECK_ALL_FILES}" = "true" ]; then \ + if [[ "$${FORCE_CHECK_ALL_FILES}" = "true" ]]; then \ find $(CI_DIR)/* -name "*.yaml" -o -name "*.yml" | while read yaml_file; do \ st2-check-validate-yaml-file "$$yaml_file" || exit 1 ; \ done; \ - elif [ -n "${CHANGED_YAML}" ]; then \ + elif [[ -n "${CHANGED_YAML}" ]]; then \ for file in $(CHANGED_YAML); do \ - if [ -n "$$file" ]; then \ + if [[ -n "$$file" ]]; then \ st2-check-validate-yaml-file $$file || exit 1 ; \ fi; \ done; \ @@ -136,13 +136,13 @@ compile: fi @# . $(VIRTUALENV_DIR)/bin/activate; \ - if [ "$${FORCE_CHECK_ALL_FILES}" = "true" ]; then \ + if [[ "$${FORCE_CHECK_ALL_FILES}" = "true" ]]; then \ find $(CI_DIR)/* -name "*.json" | while read json_file; do \ st2-check-validate-json-file "$$json_file" || exit 1 ; \ done; \ - elif [ -n "${CHANGED_JSON}" ]; then \ + elif [[ -n "${CHANGED_JSON}" ]]; then \ for file in $(CHANGED_JSON); do \ - if [ -n "$$file" ]; then \ + if [[ -n "$$file" ]]; then \ echo "file: $$file"; \ st2-check-validate-json-file $$file || exit 1 ; \ fi; \ @@ -155,7 +155,7 @@ compile: @echo "==================== example config check ====================" @echo . $(VIRTUALENV_DIR)/bin/activate; \ - if [ "$${FORCE_CHECK_ALL_FILES}" = "true" ] || [ -n "${CHANGED_FILES}" ]; then \ + if [[ "$${FORCE_CHECK_ALL_FILES}" = "true" ]] || [[ -n "${CHANGED_FILES}" ]]; then \ st2-check-validate-pack-example-config /tmp/packs/$(PACK_NAME) || exit 1; \ else \ echo "No files have changed, skipping run..."; \ @@ -167,7 +167,7 @@ compile: @echo "==================== metadata-check ====================" @echo . $(VIRTUALENV_DIR)/bin/activate; \ - if [ "$${FORCE_CHECK_ALL_FILES}" = "true" ] || [ -n "${CHANGED_YAML}" ]; then \ + if [[ "$${FORCE_CHECK_ALL_FILES}" = "true" ]] || [[ -n "${CHANGED_YAML}" ]]; then \ st2-check-validate-pack-metadata-exists $(ROOT_DIR) || exit 1; \ else \ echo "No files have changed, skipping run..."; \ @@ -179,7 +179,7 @@ compile: @echo "==================== packs-resource-register ====================" @echo . $(VIRTUALENV_DIR)/bin/activate; \ - if [ -z "${CHANGED_FILES}" ]; then \ + if [[ -z "${CHANGED_FILES}" ]]; then \ echo "No files have changed, skipping run..."; \ else \ st2-check-register-pack-resources /tmp/packs/$(PACK_NAME) || exit 1; \ @@ -191,7 +191,7 @@ compile: @echo "==================== packs-tests ====================" @echo . $(VIRTUALENV_DIR)/bin/activate; \ - if [ -z "${CHANGED_FILES}" ]; then \ + if [[ -z "${CHANGED_FILES}" ]]; then \ echo "No files have changed, skipping run..."; \ else \ $(ST2_REPO_PATH)/st2common/bin/st2-run-pack-tests -c -t -x -j -p $(ROOT_DIR) || exit 1; \ @@ -202,7 +202,7 @@ compile: @echo @echo "==================== pack-missing-tests ====================" @echo - if [ -z "${CHANGED_FILES}" ]; then \ + if [[ -z "${CHANGED_FILES}" ]]; then \ echo "No files have changed, skipping run..."; \ else \ st2-check-print-pack-tests-coverage $(ROOT_DIR) || exit 1; \ @@ -214,10 +214,10 @@ compile: @echo @echo "==================== license-check ====================" @echo - if [ -z "${CHANGED_FILES}" ] && [ "$${FORCE_CHECK_ALL_FILES}" = "false" ]; then \ + if [[ -z "${CHANGED_FILES}" ]] && [[ "$${FORCE_CHECK_ALL_FILES}" = "false" ]]; then \ echo "No files have changed, skipping run..."; \ else \ - if [ ! -f "$(ROOT_DIR)/LICENSE" ]; then \ + if [[ ! -f "$(ROOT_DIR)/LICENSE" ]]; then \ echo "Missing LICENSE file in $(ROOT_DIR)"; \ exit 2;\ fi;\ diff --git a/.circle/dependencies b/.circle/dependencies index cd736973..bd33dfc8 100755 --- a/.circle/dependencies +++ b/.circle/dependencies @@ -18,10 +18,10 @@ python --version pip --version # Clone st2 repo so other scripts can reference StackStorm Python code -[ -d /tmp/st2 ] || git clone --depth 1 --single-branch --branch "${ST2_BRANCH}" https://github.com/StackStorm/st2.git /tmp/st2 +[[ -d /tmp/st2 ]] || git clone --depth 1 --single-branch --branch "${ST2_BRANCH}" https://github.com/StackStorm/st2.git /tmp/st2 # Clone lint-configs -[ -d ~/ci/lint-configs ] || git clone --depth 1 --single-branch --branch "${LINT_CONFIGS_BRANCH}" https://github.com/StackStorm/lint-configs.git ~/ci/lint-configs +[[ -d ~/ci/lint-configs ]] || git clone --depth 1 --single-branch --branch "${LINT_CONFIGS_BRANCH}" https://github.com/StackStorm/lint-configs.git ~/ci/lint-configs # Create a directory for debian packages so we can cache it in Circle CI sudo rm -rf /var/cache/apt/archives && sudo ln -s ~/.apt-cache /var/cache/apt/archives && mkdir -p ~/.apt-cache/partial @@ -43,7 +43,7 @@ echo "Installing StackStorm requirements from /tmp/st2/requirements.txt" # Copy over Makefile and install StackStorm runners and register metrics drivers echo "Installing StackStorm runners and registering metrics drivers" -if [ ! -z "${ROOT_DIR}" ]; then +if [[ ! -z "${ROOT_DIR}" ]]; then PACK_REQUIREMENTS_FILE="${ROOT_DIR}/requirements.txt" PACK_TESTS_REQUIREMENTS_FILE="${ROOT_DIR}/requirements-tests.txt" @@ -60,13 +60,13 @@ else fi # Install pack requirements -if [ -f "${PACK_REQUIREMENTS_FILE}" ]; then +if [[ -f "${PACK_REQUIREMENTS_FILE}" ]]; then echo "Installing pack requirements from ${PACK_REQUIREMENTS_FILE}" ~/virtualenv/bin/pip install -r "${PACK_REQUIREMENTS_FILE}" fi # Install pack tests requirements -if [ -f "${PACK_TESTS_REQUIREMENTS_FILE}" ]; then +if [[ -f "${PACK_TESTS_REQUIREMENTS_FILE}" ]]; then echo "Installing pack tests requirements from ${PACK_TESTS_REQUIREMENTS_FILE}" ~/virtualenv/bin/pip install -r "${PACK_TESTS_REQUIREMENTS_FILE}" fi diff --git a/.circle/deployment b/.circle/deployment index cc21397c..facfe8e7 100755 --- a/.circle/deployment +++ b/.circle/deployment @@ -6,7 +6,7 @@ FORCE_REBUILD_INDEX="${FORCE_REBUILD_INDEX:-0}" export PYTHONPATH=/tmp/st2/st2common:${PYTHONPATH} -if [ -z "${PACK_NAME}" ]; then +if [[ -z "${PACK_NAME}" ]]; then export PACK_NAME=$(~/virtualenv/bin/python ~/ci/.circle/validate.py "${CIRCLE_PROJECT_REPONAME}" pack.yaml) else export PACK_NAME="${PACK_NAME}" @@ -63,9 +63,9 @@ git status ~/virtualenv/bin/python ~/ci/utils/pack_content.py --input . --output ~/index/v1/packs/"${PACK_NAME}" # Rebuild the index JSON -if [ -n "$(git -C ~/index status -s)" ] || [ ${FORCE_REBUILD_INDEX} == "1" ]; +if [[ -n "$(git -C ~/index status -s)" ]] || [[ ${FORCE_REBUILD_INDEX} == "1" ]]; then - if [ ${FORCE_REBUILD_INDEX} == "1" ]; then + if [[ ${FORCE_REBUILD_INDEX} == "1" ]]; then echo "Forcing index rebuild..." fi ~/virtualenv/bin/python ~/ci/.circle/index.py --glob "~/index/v1/packs/*/pack.yaml" --output "~/index/v1/" @@ -75,7 +75,7 @@ fi # Check if an icon has been added or changed ICON_TARGET="${HOME}/index/v1/icons/${PACK_NAME}.png" -if [ -f icon.png ] && { [ ! -f ${ICON_TARGET} ] || ! cmp -s icon.png ${ICON_TARGET}; } +if [[ -f icon.png ]] && { [[ ! -f ${ICON_TARGET} ]] || ! cmp -s icon.png ${ICON_TARGET}; } then echo "Uploading the pack icon..." mkdir -p ~/index/v1/icons/ diff --git a/.circle/exit_on_py2_checks b/.circle/exit_on_py2_checks index 3527a571..6ff0793a 100755 --- a/.circle/exit_on_py2_checks +++ b/.circle/exit_on_py2_checks @@ -2,7 +2,7 @@ # Script which checks exit code of "test" script under Python 2.7 environment # and exists with an appropriate code -if [ $# -lt 1 ] ; then +if [[ $# -lt 1 ]] ; then echo "Usage: $0 " echo "Example: $0 1" exit 2 @@ -15,7 +15,7 @@ echo "Original script exit code: ${TEST_EXIT_CODE}" # If pack doesn't declare Python 2 support, we don't treat failures as fatal SUPPORTS_PYTHON2=$(~/virtualenv/bin/python -c $'import yaml, sys\nresult = yaml.safe_load(open("pack.yaml", "r").read())\nif "2" in result.get("python_versions", []):\n print("yes")') -if [ "${SUPPORTS_PYTHON2}" != "yes" ] && [ ${TEST_EXIT_CODE} -ne 0 ] ; then +if [[ "${SUPPORTS_PYTHON2}" != "yes" ]] && [[ ${TEST_EXIT_CODE} -ne 0 ]] ; then echo "Ignoring failures since pack doesn't declare Python 2 support in pack.yaml" exit 0 fi diff --git a/.circle/exit_on_py3_checks b/.circle/exit_on_py3_checks index 928654dc..85ab8b4c 100755 --- a/.circle/exit_on_py3_checks +++ b/.circle/exit_on_py3_checks @@ -2,7 +2,7 @@ # Script which checks exit code of "test" script under Python 3.6 environment # and exists with an appropriate code -if [ $# -lt 1 ] ; then +if [[ $# -lt 1 ]] ; then echo "Usage: $0 " echo "Example: $0 1" exit 2 @@ -15,7 +15,7 @@ echo "Original script exit code: ${TEST_EXIT_CODE}" # If pack doesn't declare Python 3 support, we don't treat failures as fatal SUPPORTS_PYTHON3=$(~/virtualenv/bin/python -c $'import yaml, sys\nresult = yaml.safe_load(open("pack.yaml", "r").read())\nif "3" in result.get("python_versions", []):\n print("yes")') -if [ "${SUPPORTS_PYTHON3}" != "yes" ] && [ ${TEST_EXIT_CODE} -ne 0 ] ; then +if [[ "${SUPPORTS_PYTHON3}" != "yes" ]] && [[ ${TEST_EXIT_CODE} -ne 0 ]] ; then echo "Ignoring failures since pack doesn't declare Python 3 support in pack.yaml" exit 0 fi diff --git a/.circle/retry_on_failure.sh b/.circle/retry_on_failure.sh index 4135d9d3..7c40f183 100755 --- a/.circle/retry_on_failure.sh +++ b/.circle/retry_on_failure.sh @@ -5,17 +5,17 @@ MAX_RETRIES=5 COMMAND=$1 -if [ $# -ne 1 ]; then +if [[ $# -ne 1 ]]; then echo "Usage: ${0} " exit 1 fi COUNT=0 -while [ ${COUNT} -lt ${MAX_RETRIES} ]; do +while [[ ${COUNT} -lt ${MAX_RETRIES} ]]; do ${COMMAND} EXIT_CODE=$? - if [ ${EXIT_CODE} -eq 0 ]; then + if [[ ${EXIT_CODE} -eq 0 ]]; then exit 0 fi diff --git a/.circle/test b/.circle/test index 2d443a97..8234842d 100755 --- a/.circle/test +++ b/.circle/test @@ -7,7 +7,7 @@ set -e export CI_DIR=/home/circleci/ci export PYTHONPATH=/tmp/st2/st2common:${PYTHONPATH} -if [ -z "${PACK_NAME}" ]; then +if [[ -z "${PACK_NAME}" ]]; then export PACK_NAME=$(~/virtualenv/bin/python ~/ci/.circle/validate.py "${CIRCLE_PROJECT_REPONAME}" pack.yaml) else export PACK_NAME="${PACK_NAME}" @@ -28,12 +28,12 @@ echo "Running on branch: ${GIT_BRANCH}" echo "Running with Python from $(which python)" echo "Python version: ${PYTHON_VERSION}" -if [ "${GIT_BRANCH}" = "master" ]; then +if [[ "${GIT_BRANCH}" = "master" ]]; then echo "Running on master branch, forcing check of all files" export FORCE_CHECK_ALL_FILES="true" fi -if [ ! -z "${FORCE_CHECK_ALL_FILES}" ]; then +if [[ ! -z "${FORCE_CHECK_ALL_FILES}" ]]; then export FORCE_CHECK_ALL_FILES="${FORCE_CHECK_ALL_FILES}" fi @@ -43,7 +43,7 @@ export ST2_CONFIG_FILE=${CI_DIR}/conf/st2.tests.conf echo "ROOT_DIR=${ROOT_DIR}" echo "FORCE_CHECK_ALL_FILES=${FORCE_CHECK_ALL_FILES}" -if [ ! -z "${ROOT_DIR}" ]; then +if [[ ! -z "${ROOT_DIR}" ]]; then export ROOT_DIR="${ROOT_DIR}" make -C ${ROOT_DIR} all-ci else diff --git a/tools/add_webhook_to_github_repo.sh b/tools/add_webhook_to_github_repo.sh index de012952..b06c71a7 100755 --- a/tools/add_webhook_to_github_repo.sh +++ b/tools/add_webhook_to_github_repo.sh @@ -16,7 +16,7 @@ set -e SCRIPT_PATH=$(dirname "$(readlink -f "$0")") source "${SCRIPT_PATH}/common.sh" -if [ ! $# -eq 1 ]; +if [[ ! $# -eq 1 ]]; then echo "Usage: $0 " exit 1; @@ -24,7 +24,7 @@ fi WEBHOOK_URL="$1" -if [ ! -z "${REPO_NAMES}" ]; then +if [[ ! -z "${REPO_NAMES}" ]]; then OIFS=$IFS; IFS=" " REPO_NAMES=($REPO_NAMES) diff --git a/tools/reset_github_user_token_and_update_circleci.sh b/tools/reset_github_user_token_and_update_circleci.sh index 01799b9f..0c2bd6e4 100755 --- a/tools/reset_github_user_token_and_update_circleci.sh +++ b/tools/reset_github_user_token_and_update_circleci.sh @@ -11,7 +11,7 @@ # * CIRCLECI_TOKEN: a CircleCI token for the Exchange organization. # -if [ ! $# -eq 1 ]; +if [[ ! $# -eq 1 ]]; then echo "Usage: $0 " exit 1 @@ -29,7 +29,7 @@ curl -sS --fail -u "${USERNAME}:${PASSWORD}" -X POST --header "Content-Type: app -d '{"scopes": ["public_repo"], "note": "CircleCI: '"${REPO_NAME}"'"}' \ "https://api.github.com/authorizations" | jq ".token" > "/tmp/${PACK}_user_token" -if [ ! -s "/tmp/${PACK}_user_token" ]; +if [[ ! -s "/tmp/${PACK}_user_token" ]]; then echo "Could not create a token." exit 1 diff --git a/tools/set_repo_description.sh b/tools/set_repo_description.sh index 319d5f93..29cee8e3 100755 --- a/tools/set_repo_description.sh +++ b/tools/set_repo_description.sh @@ -16,7 +16,7 @@ set -e SCRIPT_PATH=$(dirname "$(readlink -f "$0")") source "${SCRIPT_PATH}/common.sh" -if [ ! -z "${REPO_NAMES}" ]; then +if [[ ! -z "${REPO_NAMES}" ]]; then OIFS=$IFS; IFS=" " REPO_NAMES=($REPO_NAMES) @@ -32,7 +32,7 @@ for REPO_NAME in ${REPO_NAMES[@]}; do PACK_YAML_URL="https://raw.githubusercontent.com/StackStorm-Exchange/${REPO_NAME}/master/pack.yaml" PACK_DESCRIPTION=$(curl -sS --fail -X GET "${PACK_YAML_URL}" | python -c 'import yaml,sys; c=yaml.safe_load(sys.stdin);print c["description"]') - if [ -z "${PACK_DESCRIPTION}" ]; then + if [[ -z "${PACK_DESCRIPTION}" ]]; then echo "Description not available for pack ${REPO_NAME}, skipping..." else curl -sS --fail -u "${USERNAME}:${PASSWORD}" -X PATCH --header "Content-Type: application/json" \ diff --git a/tools/split-packs.sh b/tools/split-packs.sh index 0290186f..ecad4d46 100755 --- a/tools/split-packs.sh +++ b/tools/split-packs.sh @@ -11,13 +11,13 @@ set -e -if [ $# -lt 1 ] || [[ $1 =~ ^-+h(elp)?$ ]] ; then +if [[ $# -lt 1 ]] || [[ $1 =~ ^-+h(elp)?$ ]] ; then echo "Usage: $0 subtree_repo_url [new_org_url]" echo "Example: $0 https://github.com/example/mysubtreerepo.git git@github.com:example2" exit fi SUBTREE_REPO_URL=$1 -if [ $# -gt 1 ] ; then +if [[ $# -gt 1 ]] ; then EXCHANGE_ORG_URL=$2 else EXCHANGE_ORG_URL=$(dirname $1) diff --git a/tools/trigger_circle_ci_builds_for_all_packs.sh b/tools/trigger_circle_ci_builds_for_all_packs.sh index e3a634c7..63d896f4 100755 --- a/tools/trigger_circle_ci_builds_for_all_packs.sh +++ b/tools/trigger_circle_ci_builds_for_all_packs.sh @@ -19,12 +19,12 @@ source "${SCRIPT_PATH}/common.sh" FORCE_REBUILD_INDEX="${FORCE_REBUILD_INDEX:-0}" -if [ ! ${CIRCLECI_TOKEN} ]; then +if [[ ! ${CIRCLECI_TOKEN} ]]; then echo "CIRCLECI_TOKEN environment variable not set" exit 2 fi -if [ ! -z "${REPO_NAMES}" ]; then +if [[ ! -z "${REPO_NAMES}" ]]; then # Only force build for provided repos OIFS=$IFS; IFS=" " @@ -41,7 +41,7 @@ for REPO_NAME in ${REPO_NAMES[@]}; do CIRCLE_TRIGGER_BUILD_URL="https://circleci.com/api/v1/project/${EXCHANGE_ORG}/${REPO_NAME}/tree/master?circle-token=${CIRCLECI_TOKEN}" - if [ ${FORCE_REBUILD_INDEX} == "1" ]; then + if [[ ${FORCE_REBUILD_INDEX} == "1" ]]; then curl \ --header "Content-Type: application/json" \ --request POST \ diff --git a/utils/exchange-bootstrap.sh b/utils/exchange-bootstrap.sh index eaa47add..5a4d8811 100755 --- a/utils/exchange-bootstrap.sh +++ b/utils/exchange-bootstrap.sh @@ -23,7 +23,7 @@ set -e -if [ ! $# -eq 1 ]; +if [[ ! $# -eq 1 ]]; then echo "Usage: $0 " exit 1; @@ -84,7 +84,7 @@ curl -sS --fail -u "${USERNAME}:${PASSWORD}" -X POST --header "Content-Type: app -d '{"scopes": ["public_repo"], "note": "CircleCI: '"${REPO_NAME}"'"}' \ "https://api.github.com/authorizations" | jq ".token" > "/tmp/${PACK}_user_token" -if [ ! -s "/tmp/${PACK}_user_token" ]; +if [[ ! -s "/tmp/${PACK}_user_token" ]]; then echo "Could not create a token." exit 1 diff --git a/utils/git-changes b/utils/git-changes index d067b724..1e74cd1c 100755 --- a/utils/git-changes +++ b/utils/git-changes @@ -3,7 +3,7 @@ METHOD=$1 # Allow user to specify custom root dir when operating in scenarios with # multiple packs in a repo and similar -if [ -z "${ROOT_DIR}" ]; then +if [[ -z "${ROOT_DIR}" ]]; then ROOT_DIR=$(git rev-parse --show-toplevel) fi diff --git a/utils/git-changes-packs b/utils/git-changes-packs index 2f1a8bb0..64d16c66 100755 --- a/utils/git-changes-packs +++ b/utils/git-changes-packs @@ -4,7 +4,7 @@ BASE_BRANCH="origin/master" # If this environment variable if specified we will return all the files so the # checks run on all the files not only changed ones. -if [ "${FORCE_CHECK_ALL_FILES}" = "true" ]; then +if [[ "${FORCE_CHECK_ALL_FILES}" = "true" ]]; then find ${ROOT_DIRECTORY}/* -maxdepth 0 -type d | grep -v linux | grep -v openstack exit 0 fi From 39d808f2095f240d3279840c265eeb114a2a4716 Mon Sep 17 00:00:00 2001 From: blag Date: Tue, 14 Apr 2020 22:54:49 -0700 Subject: [PATCH 04/16] Use [[ -n ... ]] instead of [[ ! -z ... ]] --- .circle/dependencies | 2 +- .circle/test | 4 ++-- tools/add_webhook_to_github_repo.sh | 2 +- tools/set_repo_description.sh | 2 +- tools/trigger_circle_ci_builds_for_all_packs.sh | 2 +- utils/exchange-bootstrap.sh | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.circle/dependencies b/.circle/dependencies index bd33dfc8..dad95bd0 100755 --- a/.circle/dependencies +++ b/.circle/dependencies @@ -43,7 +43,7 @@ echo "Installing StackStorm requirements from /tmp/st2/requirements.txt" # Copy over Makefile and install StackStorm runners and register metrics drivers echo "Installing StackStorm runners and registering metrics drivers" -if [[ ! -z "${ROOT_DIR}" ]]; then +if [[ -n "${ROOT_DIR}" ]]; then PACK_REQUIREMENTS_FILE="${ROOT_DIR}/requirements.txt" PACK_TESTS_REQUIREMENTS_FILE="${ROOT_DIR}/requirements-tests.txt" diff --git a/.circle/test b/.circle/test index 8234842d..4df24251 100755 --- a/.circle/test +++ b/.circle/test @@ -33,7 +33,7 @@ if [[ "${GIT_BRANCH}" = "master" ]]; then export FORCE_CHECK_ALL_FILES="true" fi -if [[ ! -z "${FORCE_CHECK_ALL_FILES}" ]]; then +if [[ -n "${FORCE_CHECK_ALL_FILES}" ]]; then export FORCE_CHECK_ALL_FILES="${FORCE_CHECK_ALL_FILES}" fi @@ -43,7 +43,7 @@ export ST2_CONFIG_FILE=${CI_DIR}/conf/st2.tests.conf echo "ROOT_DIR=${ROOT_DIR}" echo "FORCE_CHECK_ALL_FILES=${FORCE_CHECK_ALL_FILES}" -if [[ ! -z "${ROOT_DIR}" ]]; then +if [[ -n "${ROOT_DIR}" ]]; then export ROOT_DIR="${ROOT_DIR}" make -C ${ROOT_DIR} all-ci else diff --git a/tools/add_webhook_to_github_repo.sh b/tools/add_webhook_to_github_repo.sh index b06c71a7..f7b6a115 100755 --- a/tools/add_webhook_to_github_repo.sh +++ b/tools/add_webhook_to_github_repo.sh @@ -24,7 +24,7 @@ fi WEBHOOK_URL="$1" -if [[ ! -z "${REPO_NAMES}" ]]; then +if [[ -n "${REPO_NAMES}" ]]; then OIFS=$IFS; IFS=" " REPO_NAMES=($REPO_NAMES) diff --git a/tools/set_repo_description.sh b/tools/set_repo_description.sh index 29cee8e3..88d7d95e 100755 --- a/tools/set_repo_description.sh +++ b/tools/set_repo_description.sh @@ -16,7 +16,7 @@ set -e SCRIPT_PATH=$(dirname "$(readlink -f "$0")") source "${SCRIPT_PATH}/common.sh" -if [[ ! -z "${REPO_NAMES}" ]]; then +if [[ -n "${REPO_NAMES}" ]]; then OIFS=$IFS; IFS=" " REPO_NAMES=($REPO_NAMES) diff --git a/tools/trigger_circle_ci_builds_for_all_packs.sh b/tools/trigger_circle_ci_builds_for_all_packs.sh index 63d896f4..91274139 100755 --- a/tools/trigger_circle_ci_builds_for_all_packs.sh +++ b/tools/trigger_circle_ci_builds_for_all_packs.sh @@ -24,7 +24,7 @@ if [[ ! ${CIRCLECI_TOKEN} ]]; then exit 2 fi -if [[ ! -z "${REPO_NAMES}" ]]; then +if [[ -n "${REPO_NAMES}" ]]; then # Only force build for provided repos OIFS=$IFS; IFS=" " diff --git a/utils/exchange-bootstrap.sh b/utils/exchange-bootstrap.sh index 5a4d8811..d570a926 100755 --- a/utils/exchange-bootstrap.sh +++ b/utils/exchange-bootstrap.sh @@ -111,7 +111,7 @@ curl -sS --fail -u "${USERNAME}:${PASSWORD}" -X POST --header "Content-Type: app "https://api.github.com/repos/${EXCHANGE_ORG}/${REPO_NAME}/hooks" # Github: If second Slack webhook URL set (e.g. for community), configure that to notify on changes -if [[ ! -z $SLACK_WEBHOOK_URL_COMMUNITY ]]; +if [[ -n $SLACK_WEBHOOK_URL_COMMUNITY ]]; then echo "Github: Configuring Github to send webhook notifications to our community Slack" curl -sS --fail -u "${USERNAME}:${PASSWORD}" -X POST --header "Content-Type: application/json" \ From 779a04010c3d829c6cd25df00fdbda5a0ae069a8 Mon Sep 17 00:00:00 2001 From: blag Date: Wed, 15 Apr 2020 01:03:15 -0700 Subject: [PATCH 05/16] Use bash in git-changes utility scripts --- utils/git-changes | 2 +- utils/git-changes-directories | 2 +- utils/git-changes-files | 2 +- utils/git-changes-json | 2 +- utils/git-changes-packs | 2 +- utils/git-changes-py | 2 +- utils/git-changes-yaml | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/utils/git-changes b/utils/git-changes index 1e74cd1c..3999186b 100755 --- a/utils/git-changes +++ b/utils/git-changes @@ -1,4 +1,4 @@ -#!/usr/bin/env sh +#!/bin/bash METHOD=$1 # Allow user to specify custom root dir when operating in scenarios with diff --git a/utils/git-changes-directories b/utils/git-changes-directories index 2dc3ac3c..1fa59ca7 100755 --- a/utils/git-changes-directories +++ b/utils/git-changes-directories @@ -1,4 +1,4 @@ -#!/usr/bin/env sh +#!/bin/bash ROOT_DIRECTORY=$1 BASE_BRANCH="origin/master" diff --git a/utils/git-changes-files b/utils/git-changes-files index a1e37dd4..4ae5d456 100755 --- a/utils/git-changes-files +++ b/utils/git-changes-files @@ -1,4 +1,4 @@ -#!/usr/bin/env sh +#!/bin/bash ROOT_DIRECTORY=$1 BASE_BRANCH="origin/master" diff --git a/utils/git-changes-json b/utils/git-changes-json index f13606ec..22281dfb 100755 --- a/utils/git-changes-json +++ b/utils/git-changes-json @@ -1,4 +1,4 @@ -#!/usr/bin/env sh +#!/bin/bash ROOT_DIRECTORY=$1 BASE_BRANCH="origin/master" diff --git a/utils/git-changes-packs b/utils/git-changes-packs index 64d16c66..aad46505 100755 --- a/utils/git-changes-packs +++ b/utils/git-changes-packs @@ -1,4 +1,4 @@ -#!/usr/bin/env sh +#!/bin/bash ROOT_DIRECTORY=$1 BASE_BRANCH="origin/master" diff --git a/utils/git-changes-py b/utils/git-changes-py index 3e34a12c..b1f8c7d1 100755 --- a/utils/git-changes-py +++ b/utils/git-changes-py @@ -1,4 +1,4 @@ -#!/usr/bin/env sh +#!/bin/bash ROOT_DIRECTORY=$1 BASE_BRANCH="origin/master" diff --git a/utils/git-changes-yaml b/utils/git-changes-yaml index 4abd1a9b..87b225f7 100755 --- a/utils/git-changes-yaml +++ b/utils/git-changes-yaml @@ -1,4 +1,4 @@ -#!/usr/bin/env sh +#!/bin/bash ROOT_DIRECTORY=$1 BASE_BRANCH="origin/master" From 869125f67a634d05fc178b0affc08dde16ebd77a Mon Sep 17 00:00:00 2001 From: blag Date: Wed, 15 Apr 2020 01:26:22 -0700 Subject: [PATCH 06/16] Allow BASE_BRANCH to be overridden --- utils/git-changes-directories | 2 +- utils/git-changes-files | 2 +- utils/git-changes-json | 2 +- utils/git-changes-packs | 2 +- utils/git-changes-py | 2 +- utils/git-changes-yaml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/utils/git-changes-directories b/utils/git-changes-directories index 1fa59ca7..dd084a5b 100755 --- a/utils/git-changes-directories +++ b/utils/git-changes-directories @@ -1,6 +1,6 @@ #!/bin/bash ROOT_DIRECTORY=$1 -BASE_BRANCH="origin/master" +BASE_BRANCH="${BASE_BRANCH:-origin/master}" cd ${ROOT_DIRECTORY} # Note: We include all but deleted files ("D" flag) diff --git a/utils/git-changes-files b/utils/git-changes-files index 4ae5d456..37d117ca 100755 --- a/utils/git-changes-files +++ b/utils/git-changes-files @@ -1,6 +1,6 @@ #!/bin/bash ROOT_DIRECTORY=$1 -BASE_BRANCH="origin/master" +BASE_BRANCH="${BASE_BRANCH:-origin/master}" cd ${ROOT_DIRECTORY} # Note: We include all but deleted files ("D" flag) diff --git a/utils/git-changes-json b/utils/git-changes-json index 22281dfb..4d4fd803 100755 --- a/utils/git-changes-json +++ b/utils/git-changes-json @@ -1,6 +1,6 @@ #!/bin/bash ROOT_DIRECTORY=$1 -BASE_BRANCH="origin/master" +BASE_BRANCH="${BASE_BRANCH:-origin/master}" cd ${ROOT_DIRECTORY} # Note: We include all but deleted files ("D" flag) diff --git a/utils/git-changes-packs b/utils/git-changes-packs index aad46505..c93e1c91 100755 --- a/utils/git-changes-packs +++ b/utils/git-changes-packs @@ -1,6 +1,6 @@ #!/bin/bash ROOT_DIRECTORY=$1 -BASE_BRANCH="origin/master" +BASE_BRANCH="${BASE_BRANCH:-origin/master}" # If this environment variable if specified we will return all the files so the # checks run on all the files not only changed ones. diff --git a/utils/git-changes-py b/utils/git-changes-py index b1f8c7d1..66e98528 100755 --- a/utils/git-changes-py +++ b/utils/git-changes-py @@ -1,6 +1,6 @@ #!/bin/bash ROOT_DIRECTORY=$1 -BASE_BRANCH="origin/master" +BASE_BRANCH="${BASE_BRANCH:-origin/master}" cd ${ROOT_DIRECTORY} # Note: We include all but deleted files ("D" flag) diff --git a/utils/git-changes-yaml b/utils/git-changes-yaml index 87b225f7..3b0fb78f 100755 --- a/utils/git-changes-yaml +++ b/utils/git-changes-yaml @@ -1,6 +1,6 @@ #!/bin/bash ROOT_DIRECTORY=$1 -BASE_BRANCH="origin/master" +BASE_BRANCH="${BASE_BRANCH:-origin/master}" cd ${ROOT_DIRECTORY} # Note: We include all but deleted files ("D" flag) From 4266697be10e761f5fa1c33df2d6917c5640d138 Mon Sep 17 00:00:00 2001 From: blag Date: Wed, 15 Apr 2020 01:26:54 -0700 Subject: [PATCH 07/16] Use the output of git diff directly instead of echoing it all into one line --- utils/git-changes-directories | 2 +- utils/git-changes-files | 2 +- utils/git-changes-json | 2 +- utils/git-changes-packs | 2 +- utils/git-changes-py | 2 +- utils/git-changes-yaml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/utils/git-changes-directories b/utils/git-changes-directories index dd084a5b..1f0d703b 100755 --- a/utils/git-changes-directories +++ b/utils/git-changes-directories @@ -4,4 +4,4 @@ BASE_BRANCH="${BASE_BRANCH:-origin/master}" cd ${ROOT_DIRECTORY} # Note: We include all but deleted files ("D" flag) -echo $(git diff --relative --diff-filter=ACMRTUXB --name-only ${BASE_BRANCH} | xargs -I FILENAME dirname FILENAME | awk '/^\./ {next} /.*/{print "./"$1}') +git diff --relative --diff-filter=ACMRTUXB --name-only "${BASE_BRANCH}" | xargs -I FILENAME dirname FILENAME | awk '/^\./ {next} /.*/{print "./"$1}' diff --git a/utils/git-changes-files b/utils/git-changes-files index 37d117ca..92d3d490 100755 --- a/utils/git-changes-files +++ b/utils/git-changes-files @@ -4,4 +4,4 @@ BASE_BRANCH="${BASE_BRANCH:-origin/master}" cd ${ROOT_DIRECTORY} # Note: We include all but deleted files ("D" flag) -echo $(git diff --relative --diff-filter=ACMRTUXB --name-only ${BASE_BRANCH}) +git diff --relative --diff-filter=ACMRTUXB --name-only "${BASE_BRANCH}" diff --git a/utils/git-changes-json b/utils/git-changes-json index 4d4fd803..1a8cec98 100755 --- a/utils/git-changes-json +++ b/utils/git-changes-json @@ -4,4 +4,4 @@ BASE_BRANCH="${BASE_BRANCH:-origin/master}" cd ${ROOT_DIRECTORY} # Note: We include all but deleted files ("D" flag) -echo $(git diff --relative --diff-filter=ACMRTUXB --name-only ${BASE_BRANCH} -- '*.json') +git diff --relative --diff-filter=ACMRTUXB --name-only "${BASE_BRANCH}" -- '*.json' diff --git a/utils/git-changes-packs b/utils/git-changes-packs index c93e1c91..b608c843 100755 --- a/utils/git-changes-packs +++ b/utils/git-changes-packs @@ -11,4 +11,4 @@ fi cd ${ROOT_DIRECTORY} # Note: We include all but deleted files ("D" flag) -echo $(git diff --relative --diff-filter=ACMRTUXB --name-only ${BASE_BRANCH} -- '*' | xargs -I FILENAME dirname FILENAME | awk -F'/' '/^\./ {next} /^[^\/]+$/ {next} /^Makefile/ {next} /.*/{print $1"/"$2"/"}' | uniq) +git diff --relative --diff-filter=ACMRTUXB --name-only "${BASE_BRANCH}" -- '*' | xargs -I FILENAME dirname FILENAME | awk -F'/' '/^\./ {next} /^[^\/]+$/ {next} /^Makefile/ {next} /.*/{print $1"/"$2"/"}' | uniq diff --git a/utils/git-changes-py b/utils/git-changes-py index 66e98528..955089e8 100755 --- a/utils/git-changes-py +++ b/utils/git-changes-py @@ -4,4 +4,4 @@ BASE_BRANCH="${BASE_BRANCH:-origin/master}" cd ${ROOT_DIRECTORY} # Note: We include all but deleted files ("D" flag) -echo $(git diff --relative --diff-filter=ACMRTUXB --name-only ${BASE_BRANCH} -- '*.py') +git diff --relative --diff-filter=ACMRTUXB --name-only "${BASE_BRANCH}" -- '*.py' diff --git a/utils/git-changes-yaml b/utils/git-changes-yaml index 3b0fb78f..f4edb419 100755 --- a/utils/git-changes-yaml +++ b/utils/git-changes-yaml @@ -4,4 +4,4 @@ BASE_BRANCH="${BASE_BRANCH:-origin/master}" cd ${ROOT_DIRECTORY} # Note: We include all but deleted files ("D" flag) -echo $(git diff --relative --diff-filter=ACMRTUXB --name-only ${BASE_BRANCH} -- '*.yaml' '*.yml') +git diff --relative --diff-filter=ACMRTUXB --name-only "${BASE_BRANCH}" -- '*.yaml' '*.yml' From a711f6577d4cd2386ecaeaf88f110547ebb62fc3 Mon Sep 17 00:00:00 2001 From: blag Date: Wed, 15 Apr 2020 01:27:15 -0700 Subject: [PATCH 08/16] Print out ROOT_DIR before makeing with it --- .circle/test | 1 + 1 file changed, 1 insertion(+) diff --git a/.circle/test b/.circle/test index 4df24251..3eb0cecd 100755 --- a/.circle/test +++ b/.circle/test @@ -45,6 +45,7 @@ echo "FORCE_CHECK_ALL_FILES=${FORCE_CHECK_ALL_FILES}" if [[ -n "${ROOT_DIR}" ]]; then export ROOT_DIR="${ROOT_DIR}" + echo "ROOT_DIR: ${ROOT_DIR}" make -C ${ROOT_DIR} all-ci else make all-ci From fa834883bbcfe7c9e0c6bde62789da91e92eab7f Mon Sep 17 00:00:00 2001 From: blag Date: Wed, 15 Apr 2020 01:56:51 -0700 Subject: [PATCH 09/16] Reformat the changed files debug information --- .circle/Makefile | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/.circle/Makefile b/.circle/Makefile index 96a50912..129b6fa8 100644 --- a/.circle/Makefile +++ b/.circle/Makefile @@ -26,16 +26,11 @@ all-ci: compile .license-check .flake8 .pylint .copy-pack-to-subdirectory .confi .PHONY: changed-files changed-files: - @echo "CHANGED_PY:" - @echo "$(CHANGED_PY)" - @echo "CHANGED_JSON:" - @echo "$(CHANGED_JSON)" - @echo "CHANGED_DIRECTORIES:" - @echo "$(CHANGED_DIRECTORIES)" - @echo "CHANGED_YAML:" - @echo "$(CHANGED_YAML)" - @echo "CHANGED_FILES:" - @echo "$(CHANGED_FILES)" + @echo "CHANGED_PY: $(CHANGED_PY)" + @echo "CHANGED_JSON: $(CHANGED_JSON)" + @echo "NUM_CHANGED_YAML: $(NUM_CHANGED_YAML)" + @echo "NUM_CHANGED_FILES: $(NUM_CHANGED_FILES)" + @echo "CHANGED_DIRECTORIES: $(CHANGED_DIRECTORIES)" .PHONY: lint lint: requirements flake8 pylint configs-check metadata-check From 937d904f875c308ef725d04140af4263e6749d40 Mon Sep 17 00:00:00 2001 From: blag Date: Wed, 15 Apr 2020 01:32:39 -0700 Subject: [PATCH 10/16] Only count the number of changed files, don't pull the full list of changed files since it can be too large for Bash to interpolate --- .circle/Makefile | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.circle/Makefile b/.circle/Makefile index 129b6fa8..936c5ca8 100644 --- a/.circle/Makefile +++ b/.circle/Makefile @@ -1,6 +1,6 @@ ROOT_DIR ?= $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) ifndef FORCE_CHECK_ALL_FILES -CHANGED_FILES := $(shell $(CI_DIR)/utils/git-changes files) +NUM_CHANGED_FILES := $(shell $(CI_DIR)/utils/git-changes files | wc -w) CHANGED_PY := $(shell ${CI_DIR}/utils/git-changes py) CHANGED_YAML := $(shell $(CI_DIR)/utils/git-changes yaml) CHANGED_JSON := $(shell $(CI_DIR)/utils/git-changes json) @@ -150,7 +150,7 @@ compile: @echo "==================== example config check ====================" @echo . $(VIRTUALENV_DIR)/bin/activate; \ - if [[ "$${FORCE_CHECK_ALL_FILES}" = "true" ]] || [[ -n "${CHANGED_FILES}" ]]; then \ + if [[ "$${FORCE_CHECK_ALL_FILES}" = "true" ]] || [[ ${NUM_CHANGED_FILES} -gt 0 ]]; then \ st2-check-validate-pack-example-config /tmp/packs/$(PACK_NAME) || exit 1; \ else \ echo "No files have changed, skipping run..."; \ @@ -162,7 +162,7 @@ compile: @echo "==================== metadata-check ====================" @echo . $(VIRTUALENV_DIR)/bin/activate; \ - if [[ "$${FORCE_CHECK_ALL_FILES}" = "true" ]] || [[ -n "${CHANGED_YAML}" ]]; then \ + if [[ "$${FORCE_CHECK_ALL_FILES}" = "true" ]] || [[ ${NUM_CHANGED_YAML} -gt 0 ]]; then \ st2-check-validate-pack-metadata-exists $(ROOT_DIR) || exit 1; \ else \ echo "No files have changed, skipping run..."; \ @@ -174,7 +174,7 @@ compile: @echo "==================== packs-resource-register ====================" @echo . $(VIRTUALENV_DIR)/bin/activate; \ - if [[ -z "${CHANGED_FILES}" ]]; then \ + if [[ ${NUM_CHANGED_FILES} -eq 0 ]]; then \ echo "No files have changed, skipping run..."; \ else \ st2-check-register-pack-resources /tmp/packs/$(PACK_NAME) || exit 1; \ @@ -186,7 +186,7 @@ compile: @echo "==================== packs-tests ====================" @echo . $(VIRTUALENV_DIR)/bin/activate; \ - if [[ -z "${CHANGED_FILES}" ]]; then \ + if [[ ${NUM_CHANGED_FILES} -eq 0 ]]; then \ echo "No files have changed, skipping run..."; \ else \ $(ST2_REPO_PATH)/st2common/bin/st2-run-pack-tests -c -t -x -j -p $(ROOT_DIR) || exit 1; \ @@ -197,7 +197,7 @@ compile: @echo @echo "==================== pack-missing-tests ====================" @echo - if [[ -z "${CHANGED_FILES}" ]]; then \ + if [[ ${NUM_CHANGED_FILES} -eq 0 ]]; then \ echo "No files have changed, skipping run..."; \ else \ st2-check-print-pack-tests-coverage $(ROOT_DIR) || exit 1; \ @@ -209,7 +209,7 @@ compile: @echo @echo "==================== license-check ====================" @echo - if [[ -z "${CHANGED_FILES}" ]] && [[ "$${FORCE_CHECK_ALL_FILES}" = "false" ]]; then \ + if [[ ${NUM_CHANGED_FILES} -eq 0 ]] && [[ "$${FORCE_CHECK_ALL_FILES}" = "false" ]]; then \ echo "No files have changed, skipping run..."; \ else \ if [[ ! -f "$(ROOT_DIR)/LICENSE" ]]; then \ From 5c032d401e53a9db9785569458d671344a3ce262 Mon Sep 17 00:00:00 2001 From: blag Date: Wed, 15 Apr 2020 01:41:50 -0700 Subject: [PATCH 11/16] Allow FORCE_CHECK_ALL_FILES to be overridden --- .circle/test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circle/test b/.circle/test index 3eb0cecd..e95e3d35 100755 --- a/.circle/test +++ b/.circle/test @@ -30,7 +30,7 @@ echo "Python version: ${PYTHON_VERSION}" if [[ "${GIT_BRANCH}" = "master" ]]; then echo "Running on master branch, forcing check of all files" - export FORCE_CHECK_ALL_FILES="true" + export FORCE_CHECK_ALL_FILES="${FORCE_CHECK_ALL_FILES:-true}" fi if [[ -n "${FORCE_CHECK_ALL_FILES}" ]]; then From de3e1efacc0255a7331491f5bfcdb3aad30f1abb Mon Sep 17 00:00:00 2001 From: blag Date: Wed, 15 Apr 2020 02:08:40 -0700 Subject: [PATCH 12/16] Only count the number of changed YAML files, and use a different mechanism to iterate over all of them --- .circle/Makefile | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.circle/Makefile b/.circle/Makefile index 936c5ca8..3fe7b3ad 100644 --- a/.circle/Makefile +++ b/.circle/Makefile @@ -2,7 +2,7 @@ ROOT_DIR ?= $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) ifndef FORCE_CHECK_ALL_FILES NUM_CHANGED_FILES := $(shell $(CI_DIR)/utils/git-changes files | wc -w) CHANGED_PY := $(shell ${CI_DIR}/utils/git-changes py) -CHANGED_YAML := $(shell $(CI_DIR)/utils/git-changes yaml) +NUM_CHANGED_YAML := $(shell $(CI_DIR)/utils/git-changes yaml | wc -w) CHANGED_JSON := $(shell $(CI_DIR)/utils/git-changes json) endif CHANGED_DIRECTORIES := $(shell $(CI_DIR)/utils/git-changes directories) @@ -120,12 +120,15 @@ compile: find $(CI_DIR)/* -name "*.yaml" -o -name "*.yml" | while read yaml_file; do \ st2-check-validate-yaml-file "$$yaml_file" || exit 1 ; \ done; \ - elif [[ -n "${CHANGED_YAML}" ]]; then \ - for file in $(CHANGED_YAML); do \ + elif [[ ${NUM_CHANGED_YAML} -gt 0 ]]; then \ + echo "Checking $(NUM_CHANGED_YAML) YAML files"; \ + $(CI_DIR)/utils/git-changes yaml >~/.git-changes-yaml; \ + while read -r file; do \ if [[ -n "$$file" ]]; then \ st2-check-validate-yaml-file $$file || exit 1 ; \ fi; \ - done; \ + done < ~/.git-changes-yaml; \ + if [[ -e ~/.git-changes-yaml ]]; then rm ~/.git-changes-yaml; fi; \ else \ echo "No files have changed, skipping run..."; \ fi From e615c5b3121c785088ffe3a53ce5633a5d8d3984 Mon Sep 17 00:00:00 2001 From: blag Date: Wed, 15 Apr 2020 02:25:37 -0700 Subject: [PATCH 13/16] Only count the number of changed JSON files, and use a different mechanism to iterate over all of them --- .circle/Makefile | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.circle/Makefile b/.circle/Makefile index 3fe7b3ad..68320cba 100644 --- a/.circle/Makefile +++ b/.circle/Makefile @@ -3,7 +3,7 @@ ifndef FORCE_CHECK_ALL_FILES NUM_CHANGED_FILES := $(shell $(CI_DIR)/utils/git-changes files | wc -w) CHANGED_PY := $(shell ${CI_DIR}/utils/git-changes py) NUM_CHANGED_YAML := $(shell $(CI_DIR)/utils/git-changes yaml | wc -w) -CHANGED_JSON := $(shell $(CI_DIR)/utils/git-changes json) +NUM_CHANGED_JSON := $(shell $(CI_DIR)/utils/git-changes json | wc -w) endif CHANGED_DIRECTORIES := $(shell $(CI_DIR)/utils/git-changes directories) VIRTUALENV_DIR ?= virtualenv @@ -27,7 +27,7 @@ all-ci: compile .license-check .flake8 .pylint .copy-pack-to-subdirectory .confi .PHONY: changed-files changed-files: @echo "CHANGED_PY: $(CHANGED_PY)" - @echo "CHANGED_JSON: $(CHANGED_JSON)" + @echo "NUM_CHANGED_JSON: $(NUM_CHANGED_JSON)" @echo "NUM_CHANGED_YAML: $(NUM_CHANGED_YAML)" @echo "NUM_CHANGED_FILES: $(NUM_CHANGED_FILES)" @echo "CHANGED_DIRECTORIES: $(CHANGED_DIRECTORIES)" @@ -138,13 +138,16 @@ compile: find $(CI_DIR)/* -name "*.json" | while read json_file; do \ st2-check-validate-json-file "$$json_file" || exit 1 ; \ done; \ - elif [[ -n "${CHANGED_JSON}" ]]; then \ - for file in $(CHANGED_JSON); do \ + elif [[ -n "${NUM_CHANGED_JSON}" ]]; then \ + echo "Checking $(NUM_CHANGED_JSON) JSON files"; \ + $(CI_DIR)/utils/git-changes json >~/.git-changes-json; \ + while read -r file; do \ if [[ -n "$$file" ]]; then \ echo "file: $$file"; \ st2-check-validate-json-file $$file || exit 1 ; \ fi; \ - done; \ + done < ~/.git-changes-json; \ + if [[ -e ~/.git-changes-json ]]; then rm ~/.git-changes-json; fi; \ else \ echo "No files have changed, skipping run..."; \ fi From 81553220e8bce14c114ef6b55b40da8097d0347a Mon Sep 17 00:00:00 2001 From: blag Date: Wed, 15 Apr 2020 02:31:47 -0700 Subject: [PATCH 14/16] Only count the number of changed Python files, and use a different mechanism to iterate over all of them --- .circle/Makefile | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/.circle/Makefile b/.circle/Makefile index 68320cba..e6259f52 100644 --- a/.circle/Makefile +++ b/.circle/Makefile @@ -1,7 +1,7 @@ ROOT_DIR ?= $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) ifndef FORCE_CHECK_ALL_FILES NUM_CHANGED_FILES := $(shell $(CI_DIR)/utils/git-changes files | wc -w) -CHANGED_PY := $(shell ${CI_DIR}/utils/git-changes py) +NUM_CHANGED_PY := $(shell $(CI_DIR)/utils/git-changes py | wc -w) NUM_CHANGED_YAML := $(shell $(CI_DIR)/utils/git-changes yaml | wc -w) NUM_CHANGED_JSON := $(shell $(CI_DIR)/utils/git-changes json | wc -w) endif @@ -26,7 +26,7 @@ all-ci: compile .license-check .flake8 .pylint .copy-pack-to-subdirectory .confi .PHONY: changed-files changed-files: - @echo "CHANGED_PY: $(CHANGED_PY)" + @echo "NUM_CHANGED_PY: $(NUM_CHANGED_PY)" @echo "NUM_CHANGED_JSON: $(NUM_CHANGED_JSON)" @echo "NUM_CHANGED_YAML: $(NUM_CHANGED_YAML)" @echo "NUM_CHANGED_FILES: $(NUM_CHANGED_FILES)" @@ -80,12 +80,15 @@ compile: find ./* -name "*.py" | while read py_file; do \ flake8 --config=$(CI_DIR)/lint-configs/python/.flake8 $$py_file || exit 1; \ done; \ - elif [[ -n "${CHANGED_PY}" ]]; then \ - for file in ${CHANGED_PY}; do \ + elif [[ ${NUM_CHANGED_PY} -gt 0 ]]; then \ + echo "Checking ${NUM_CHANGED_PY} Python files"; \ + $(CI_DIR)/utils/git-changes py >~/.git-changes-py; \ + while read -r file; do \ if [[ -n "$$file" ]]; then \ flake8 --config=$(CI_DIR)/lint-configs/python/.flake8 $$file || exit 1; \ fi; \ - done; \ + done < ~/.git-changes-py; \ + if [[ -e ~/.git-changes-py ]]; then rm ~/.git-changes-py; fi; \ else \ echo "No files have changed, skipping run..."; \ fi; @@ -96,7 +99,7 @@ compile: @echo "==================== pylint ====================" @echo . $(VIRTUALENV_DIR)/bin/activate; \ - if [[ "$${FORCE_CHECK_ALL_FILES}" = "true" ]] || [[ -n "${CHANGED_PY}" ]]; then \ + if [[ "$${FORCE_CHECK_ALL_FILES}" = "true" ]] || [[ ${NUM_CHANGED_PY} -gt 0 ]]; then \ REQUIREMENTS_DIR=$(CI_DIR)/.circle/ \ CONFIG_DIR=$(CI_DIR)/lint-configs/ \ st2-check-pylint-pack $(ROOT_DIR) || exit 1; \ From 7e913a5767d147fb1230a55d2441912d86bf27ed Mon Sep 17 00:00:00 2001 From: blag Date: Wed, 15 Apr 2020 02:56:24 -0700 Subject: [PATCH 15/16] Revert 960ddc5524c949f9df70ed8e13df1ee19be5a2c9 and use [ instead of [[ after all --- .circle/Makefile | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/.circle/Makefile b/.circle/Makefile index e6259f52..5b3f4899 100644 --- a/.circle/Makefile +++ b/.circle/Makefile @@ -76,19 +76,19 @@ compile: @echo "==================== flake8 ====================" @echo . $(VIRTUALENV_DIR)/bin/activate; \ - if [[ "$${FORCE_CHECK_ALL_FILES}" = "true" ]]; then \ + if [ "$${FORCE_CHECK_ALL_FILES}" = "true" ]; then \ find ./* -name "*.py" | while read py_file; do \ flake8 --config=$(CI_DIR)/lint-configs/python/.flake8 $$py_file || exit 1; \ done; \ - elif [[ ${NUM_CHANGED_PY} -gt 0 ]]; then \ + elif [ ${NUM_CHANGED_PY} -gt 0 ]; then \ echo "Checking ${NUM_CHANGED_PY} Python files"; \ $(CI_DIR)/utils/git-changes py >~/.git-changes-py; \ while read -r file; do \ - if [[ -n "$$file" ]]; then \ + if [ -n "$$file" ]; then \ flake8 --config=$(CI_DIR)/lint-configs/python/.flake8 $$file || exit 1; \ fi; \ done < ~/.git-changes-py; \ - if [[ -e ~/.git-changes-py ]]; then rm ~/.git-changes-py; fi; \ + if [ -e ~/.git-changes-py ]; then rm ~/.git-changes-py; fi; \ else \ echo "No files have changed, skipping run..."; \ fi; @@ -99,7 +99,7 @@ compile: @echo "==================== pylint ====================" @echo . $(VIRTUALENV_DIR)/bin/activate; \ - if [[ "$${FORCE_CHECK_ALL_FILES}" = "true" ]] || [[ ${NUM_CHANGED_PY} -gt 0 ]]; then \ + if [ "$${FORCE_CHECK_ALL_FILES}" = "true" ] || [ ${NUM_CHANGED_PY} -gt 0 ]; then \ REQUIREMENTS_DIR=$(CI_DIR)/.circle/ \ CONFIG_DIR=$(CI_DIR)/lint-configs/ \ st2-check-pylint-pack $(ROOT_DIR) || exit 1; \ @@ -119,38 +119,38 @@ compile: @# make variable to a Bash string, convert that to a Bash array, and then @# iterate through each element of the array . $(VIRTUALENV_DIR)/bin/activate; \ - if [[ "$${FORCE_CHECK_ALL_FILES}" = "true" ]]; then \ + if [ "$${FORCE_CHECK_ALL_FILES}" = "true" ]; then \ find $(CI_DIR)/* -name "*.yaml" -o -name "*.yml" | while read yaml_file; do \ st2-check-validate-yaml-file "$$yaml_file" || exit 1 ; \ done; \ - elif [[ ${NUM_CHANGED_YAML} -gt 0 ]]; then \ + elif [ ${NUM_CHANGED_YAML} -gt 0 ]; then \ echo "Checking $(NUM_CHANGED_YAML) YAML files"; \ $(CI_DIR)/utils/git-changes yaml >~/.git-changes-yaml; \ while read -r file; do \ - if [[ -n "$$file" ]]; then \ + if [ -n "$$file" ]; then \ st2-check-validate-yaml-file $$file || exit 1 ; \ fi; \ done < ~/.git-changes-yaml; \ - if [[ -e ~/.git-changes-yaml ]]; then rm ~/.git-changes-yaml; fi; \ + if [ -e ~/.git-changes-yaml ]; then rm ~/.git-changes-yaml; fi; \ else \ echo "No files have changed, skipping run..."; \ fi @# . $(VIRTUALENV_DIR)/bin/activate; \ - if [[ "$${FORCE_CHECK_ALL_FILES}" = "true" ]]; then \ + if [ "$${FORCE_CHECK_ALL_FILES}" = "true" ]; then \ find $(CI_DIR)/* -name "*.json" | while read json_file; do \ st2-check-validate-json-file "$$json_file" || exit 1 ; \ done; \ - elif [[ -n "${NUM_CHANGED_JSON}" ]]; then \ + elif [ -n "${NUM_CHANGED_JSON}" ]; then \ echo "Checking $(NUM_CHANGED_JSON) JSON files"; \ $(CI_DIR)/utils/git-changes json >~/.git-changes-json; \ while read -r file; do \ - if [[ -n "$$file" ]]; then \ + if [ -n "$$file" ]; then \ echo "file: $$file"; \ st2-check-validate-json-file $$file || exit 1 ; \ fi; \ done < ~/.git-changes-json; \ - if [[ -e ~/.git-changes-json ]]; then rm ~/.git-changes-json; fi; \ + if [ -e ~/.git-changes-json ]; then rm ~/.git-changes-json; fi; \ else \ echo "No files have changed, skipping run..."; \ fi @@ -159,7 +159,7 @@ compile: @echo "==================== example config check ====================" @echo . $(VIRTUALENV_DIR)/bin/activate; \ - if [[ "$${FORCE_CHECK_ALL_FILES}" = "true" ]] || [[ ${NUM_CHANGED_FILES} -gt 0 ]]; then \ + if [ "$${FORCE_CHECK_ALL_FILES}" = "true" ] || [ ${NUM_CHANGED_FILES} -gt 0 ]; then \ st2-check-validate-pack-example-config /tmp/packs/$(PACK_NAME) || exit 1; \ else \ echo "No files have changed, skipping run..."; \ @@ -171,7 +171,7 @@ compile: @echo "==================== metadata-check ====================" @echo . $(VIRTUALENV_DIR)/bin/activate; \ - if [[ "$${FORCE_CHECK_ALL_FILES}" = "true" ]] || [[ ${NUM_CHANGED_YAML} -gt 0 ]]; then \ + if [ "$${FORCE_CHECK_ALL_FILES}" = "true" ] || [ ${NUM_CHANGED_YAML} -gt 0 ]; then \ st2-check-validate-pack-metadata-exists $(ROOT_DIR) || exit 1; \ else \ echo "No files have changed, skipping run..."; \ @@ -183,7 +183,7 @@ compile: @echo "==================== packs-resource-register ====================" @echo . $(VIRTUALENV_DIR)/bin/activate; \ - if [[ ${NUM_CHANGED_FILES} -eq 0 ]]; then \ + if [ ${NUM_CHANGED_FILES} -eq 0 ]; then \ echo "No files have changed, skipping run..."; \ else \ st2-check-register-pack-resources /tmp/packs/$(PACK_NAME) || exit 1; \ @@ -195,7 +195,7 @@ compile: @echo "==================== packs-tests ====================" @echo . $(VIRTUALENV_DIR)/bin/activate; \ - if [[ ${NUM_CHANGED_FILES} -eq 0 ]]; then \ + if [ ${NUM_CHANGED_FILES} -eq 0 ]; then \ echo "No files have changed, skipping run..."; \ else \ $(ST2_REPO_PATH)/st2common/bin/st2-run-pack-tests -c -t -x -j -p $(ROOT_DIR) || exit 1; \ @@ -206,7 +206,7 @@ compile: @echo @echo "==================== pack-missing-tests ====================" @echo - if [[ ${NUM_CHANGED_FILES} -eq 0 ]]; then \ + if [ ${NUM_CHANGED_FILES} -eq 0 ]; then \ echo "No files have changed, skipping run..."; \ else \ st2-check-print-pack-tests-coverage $(ROOT_DIR) || exit 1; \ @@ -218,10 +218,10 @@ compile: @echo @echo "==================== license-check ====================" @echo - if [[ ${NUM_CHANGED_FILES} -eq 0 ]] && [[ "$${FORCE_CHECK_ALL_FILES}" = "false" ]]; then \ + if [ ${NUM_CHANGED_FILES} -eq 0 ] && [ "$${FORCE_CHECK_ALL_FILES}" = "false" ]; then \ echo "No files have changed, skipping run..."; \ else \ - if [[ ! -f "$(ROOT_DIR)/LICENSE" ]]; then \ + if [ ! -f "$(ROOT_DIR)/LICENSE" ]; then \ echo "Missing LICENSE file in $(ROOT_DIR)"; \ exit 2;\ fi;\ From 146b7230a9b556c2b064795531759189a1029c45 Mon Sep 17 00:00:00 2001 From: blag Date: Wed, 15 Apr 2020 03:16:01 -0700 Subject: [PATCH 16/16] Fix CI for ci repo --- .circleci/config.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 477724c3..edd32983 100755 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -42,8 +42,7 @@ jobs: name: Check Bash file syntax command: | find . -name '*.sh' | xargs bash -n - grep -lrE '^#!/bin/bash' | xargs bash -n - grep -lrE '^#!/usr/bin/env sh' . | xargs sh -n\ + grep -lrE '^#!/bin/bash' . | xargs bash -n - run: name: Check Python files command: | @@ -98,8 +97,7 @@ jobs: name: Check Bash file syntax command: | find . -name '*.sh' | xargs bash -n - grep -lrE '^#!/bin/bash' | xargs bash -n - grep -lrE '^#!/usr/bin/env sh' . | xargs sh -n\ + grep -lrE '^#!/bin/bash' . | xargs bash -n - run: name: Check Python files command: |