From af2169765be10c4e4dcb17d286a3f5c41dd41dfe Mon Sep 17 00:00:00 2001 From: armab Date: Sat, 23 May 2020 16:11:52 +0100 Subject: [PATCH 1/8] Pin pip to 20.0.2 consistently across the targets --- .travis.yml | 1 - Makefile | 23 ++++++++--------------- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/.travis.yml b/.travis.yml index b5bdfbe0d8..4d30cf5b82 100644 --- a/.travis.yml +++ b/.travis.yml @@ -82,7 +82,6 @@ cache: before_install: # If you update these versions, make sure you update the versions in the .st2client-install-check # and the requirements targets in the Makefile to match - - pip install --upgrade "pip==20.0.2" - sudo pip install --upgrade "virtualenv==16.6.0" install: diff --git a/Makefile b/Makefile index de39f43489..4cbaa1ac11 100644 --- a/Makefile +++ b/Makefile @@ -52,6 +52,9 @@ COVERAGE_GLOBS := .coverage.unit.* .coverage.integration.* .coverage.mistral.* COVERAGE_GLOBS_QUOTED := $(foreach glob,$(COVERAGE_GLOBS),'$(glob)') REQUIREMENTS := test-requirements.txt requirements.txt +# Pin common pip version here across all the targets +# Note! Periodic maintenance pip upgrades are required to be up-to-date with the latest pip security fixes and updates +PIP_VERSION ?= 20.0.2 PIP_OPTIONS := $(ST2_PIP_OPTIONS) ifndef PYLINT_CONCURRENCY @@ -216,8 +219,7 @@ check-python-packages: @echo "" @echo "================== CHECK PYTHON PACKAGES ====================" @echo "" - - test -f $(VIRTUALENV_COMPONENTS_DIR)/bin/activate || virtualenv --python=$(PYTHON_VERSION) $(VIRTUALENV_COMPONENTS_DIR) --no-download + test -f $(VIRTUALENV_COMPONENTS_DIR)/bin/activate || virtualenv --python=$(PYTHON_VERSION) $(VIRTUALENV_COMPONENTS_DIR) --pip $(PIP_VERSION) @for component in $(COMPONENTS_WITHOUT_ST2TESTS); do \ echo "==========================================================="; \ echo "Checking component:" $$component; \ @@ -233,7 +235,7 @@ check-python-packages-nightly: @echo "================== CHECK PYTHON PACKAGES ====================" @echo "" - test -f $(VIRTUALENV_COMPONENTS_DIR)/bin/activate || virtualenv --python=$(PYTHON_VERSION) $(VIRTUALENV_COMPONENTS_DIR) --no-download + test -f $(VIRTUALENV_COMPONENTS_DIR)/bin/activate || virtualenv --python=$(PYTHON_VERSION) $(VIRTUALENV_COMPONENTS_DIR) --pip $(PIP_VERSION) @for component in $(COMPONENTS_WITHOUT_ST2TESTS); do \ echo "==========================================================="; \ echo "Checking component:" $$component; \ @@ -365,7 +367,7 @@ flake8: requirements .flake8 @echo @echo "==================== st2client install check ====================" @echo - test -f $(VIRTUALENV_ST2CLIENT_DIR)/bin/activate || virtualenv --python=$(PYTHON_VERSION) $(VIRTUALENV_ST2CLIENT_DIR) --no-download + test -f $(VIRTUALENV_ST2CLIENT_DIR)/bin/activate || virtualenv --python=$(PYTHON_VERSION) $(VIRTUALENV_ST2CLIENT_DIR) --pip $(PIP_VERSION) # Setup PYTHONPATH in bash activate script... # Delete existing entries (if any) @@ -379,10 +381,6 @@ flake8: requirements .flake8 touch $(VIRTUALENV_ST2CLIENT_DIR)/bin/activate chmod +x $(VIRTUALENV_ST2CLIENT_DIR)/bin/activate - # If you update these versions, make sure you also update the versions in the - # requirements target and .travis.yml to match - # Make sure we use the latest version of pip - $(VIRTUALENV_ST2CLIENT_DIR)/bin/pip install --upgrade "pip==20.0.2" # NOTE We need to upgrade setuptools to avoid bug with dependency resolving in old versions # Setuptools 42 added support for python_requires, which is used by the configparser package, # which is required by the importlib-metadata package @@ -527,11 +525,8 @@ requirements: virtualenv .requirements .sdist-requirements install-runners @echo @echo "==================== requirements ====================" @echo - # If you update these versions, make sure you also update the versions in the - # .st2client-install-check target and .travis.yml to match - # Make sure we use latest version of pip + # If you update these versions, make sure you also update the versions in the .st2client-install-check target to match $(VIRTUALENV_DIR)/bin/pip --version - $(VIRTUALENV_DIR)/bin/pip install --upgrade "pip==20.0.2" # setuptools >= 41.0.1 is required for packs.install in dev envs # setuptools >= 42 is required so setup.py install respects dependencies' python_requires $(VIRTUALENV_DIR)/bin/pip install --upgrade "setuptools==44.1.0" @@ -586,9 +581,7 @@ virtualenv: @echo @echo "==================== virtualenv ====================" @echo - # Note: We pass --no-download flag to make sure version of pip which we install (9.0.1) is used - # instead of latest version being downloaded from PyPi - test -f $(VIRTUALENV_DIR)/bin/activate || virtualenv --python=$(PYTHON_VERSION) $(VIRTUALENV_DIR) --no-download + test -f $(VIRTUALENV_DIR)/bin/activate || virtualenv --python=$(PYTHON_VERSION) $(VIRTUALENV_DIR) --pip $(PIP_VERSION) # Setup PYTHONPATH in bash activate script... # Delete existing entries (if any) From eaa329cff7d38d92280ec68c410497d61b757b5f Mon Sep 17 00:00:00 2001 From: armab Date: Sat, 23 May 2020 16:42:09 +0100 Subject: [PATCH 2/8] Debug virtualenv options under py2 --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index 4cbaa1ac11..3f67785ea9 100644 --- a/Makefile +++ b/Makefile @@ -581,6 +581,8 @@ virtualenv: @echo @echo "==================== virtualenv ====================" @echo + virtualenv --version + virtualenv --help test -f $(VIRTUALENV_DIR)/bin/activate || virtualenv --python=$(PYTHON_VERSION) $(VIRTUALENV_DIR) --pip $(PIP_VERSION) # Setup PYTHONPATH in bash activate script... From 13b024ab9575c974e32abe8a932eb0fcb51dcb33 Mon Sep 17 00:00:00 2001 From: armab Date: Sat, 23 May 2020 17:25:31 +0100 Subject: [PATCH 3/8] Try to update virtualenv to latest version --- .travis.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4d30cf5b82..23adbb62b5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -80,9 +80,8 @@ cache: #- .tox/ before_install: - # If you update these versions, make sure you update the versions in the .st2client-install-check - # and the requirements targets in the Makefile to match - - sudo pip install --upgrade "virtualenv==16.6.0" + # Upgrade virtualenv to latest + - sudo pip install --upgrade "virtualenv" install: - ./scripts/travis/install-requirements.sh From 90ad5669a31b25e5daed40c1ea1e8542f1fa4b1b Mon Sep 17 00:00:00 2001 From: armab Date: Sat, 23 May 2020 17:34:43 +0100 Subject: [PATCH 4/8] Debug virtualenv options --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 3f67785ea9..26734a0872 100644 --- a/Makefile +++ b/Makefile @@ -581,8 +581,8 @@ virtualenv: @echo @echo "==================== virtualenv ====================" @echo - virtualenv --version - virtualenv --help + sudo virtualenv --version + sudo virtualenv --help test -f $(VIRTUALENV_DIR)/bin/activate || virtualenv --python=$(PYTHON_VERSION) $(VIRTUALENV_DIR) --pip $(PIP_VERSION) # Setup PYTHONPATH in bash activate script... From e94839889d23eb65634fa1907f66bd360e8a0ee9 Mon Sep 17 00:00:00 2001 From: armab Date: Sat, 23 May 2020 17:50:44 +0100 Subject: [PATCH 5/8] Fix virtualenv installation --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 23adbb62b5..647dcc8bbc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -81,7 +81,7 @@ cache: before_install: # Upgrade virtualenv to latest - - sudo pip install --upgrade "virtualenv" + - pip install --upgrade "virtualenv" install: - ./scripts/travis/install-requirements.sh From c1a6a383196b6d887b327fd24ff9b79c8d451fa7 Mon Sep 17 00:00:00 2001 From: armab Date: Sat, 23 May 2020 17:55:38 +0100 Subject: [PATCH 6/8] Debug virtualenv options --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 26734a0872..3f67785ea9 100644 --- a/Makefile +++ b/Makefile @@ -581,8 +581,8 @@ virtualenv: @echo @echo "==================== virtualenv ====================" @echo - sudo virtualenv --version - sudo virtualenv --help + virtualenv --version + virtualenv --help test -f $(VIRTUALENV_DIR)/bin/activate || virtualenv --python=$(PYTHON_VERSION) $(VIRTUALENV_DIR) --pip $(PIP_VERSION) # Setup PYTHONPATH in bash activate script... From 617f4ce2258bde5ee716b6fae352c6fd6dab714e Mon Sep 17 00:00:00 2001 From: armab Date: Sat, 23 May 2020 18:19:32 +0100 Subject: [PATCH 7/8] Debug virtualenv options --- Makefile | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index 3f67785ea9..f156b8be5b 100644 --- a/Makefile +++ b/Makefile @@ -219,7 +219,8 @@ check-python-packages: @echo "" @echo "================== CHECK PYTHON PACKAGES ====================" @echo "" - test -f $(VIRTUALENV_COMPONENTS_DIR)/bin/activate || virtualenv --python=$(PYTHON_VERSION) $(VIRTUALENV_COMPONENTS_DIR) --pip $(PIP_VERSION) + + test -f $(VIRTUALENV_COMPONENTS_DIR)/bin/activate || virtualenv --python=$(PYTHON_VERSION) $(VIRTUALENV_COMPONENTS_DIR) --no-download @for component in $(COMPONENTS_WITHOUT_ST2TESTS); do \ echo "==========================================================="; \ echo "Checking component:" $$component; \ @@ -235,7 +236,7 @@ check-python-packages-nightly: @echo "================== CHECK PYTHON PACKAGES ====================" @echo "" - test -f $(VIRTUALENV_COMPONENTS_DIR)/bin/activate || virtualenv --python=$(PYTHON_VERSION) $(VIRTUALENV_COMPONENTS_DIR) --pip $(PIP_VERSION) + test -f $(VIRTUALENV_COMPONENTS_DIR)/bin/activate || virtualenv --python=$(PYTHON_VERSION) $(VIRTUALENV_COMPONENTS_DIR) --no-download @for component in $(COMPONENTS_WITHOUT_ST2TESTS); do \ echo "==========================================================="; \ echo "Checking component:" $$component; \ @@ -321,7 +322,7 @@ lint-api-spec: requirements .lint-api-spec @echo @echo "================== Lint API spec ====================" @echo - . $(VIRTUALENV_DIR)/bin/activate; st2common/bin/st2-validate-api-spec --config-file conf/st2.dev.conf + . $(VIRTUALENV_DIR)/bin/activate; st2common/bin/st2-validate-api-spec --config-file conf/st2.dev.conf .PHONY: generate-api-spec generate-api-spec: requirements .generate-api-spec @@ -367,7 +368,7 @@ flake8: requirements .flake8 @echo @echo "==================== st2client install check ====================" @echo - test -f $(VIRTUALENV_ST2CLIENT_DIR)/bin/activate || virtualenv --python=$(PYTHON_VERSION) $(VIRTUALENV_ST2CLIENT_DIR) --pip $(PIP_VERSION) + test -f $(VIRTUALENV_ST2CLIENT_DIR)/bin/activate || virtualenv --python=$(PYTHON_VERSION) $(VIRTUALENV_ST2CLIENT_DIR) --no-download # Setup PYTHONPATH in bash activate script... # Delete existing entries (if any) @@ -381,6 +382,7 @@ flake8: requirements .flake8 touch $(VIRTUALENV_ST2CLIENT_DIR)/bin/activate chmod +x $(VIRTUALENV_ST2CLIENT_DIR)/bin/activate + $(VIRTUALENV_ST2CLIENT_DIR)/bin/pip install --upgrade "pip==$(PIP_VERSION)" # NOTE We need to upgrade setuptools to avoid bug with dependency resolving in old versions # Setuptools 42 added support for python_requires, which is used by the configparser package, # which is required by the importlib-metadata package @@ -503,7 +505,8 @@ distclean: clean .PHONY: .requirements .requirements: virtualenv - # Print out pip version so we can see what version was restored from the Travis cache + $(VIRTUALENV_DIR)/bin/pip install --upgrade "pip==$(PIP_VERSION)" + # Print out pip version $(VIRTUALENV_DIR)/bin/pip --version # Generate all requirements to support current CI pipeline. $(VIRTUALENV_DIR)/bin/python scripts/fixate-requirements.py --skip=virtualenv,virtualenv-osx -s st2*/in-requirements.txt contrib/runners/*/in-requirements.txt -f fixed-requirements.txt -o requirements.txt @@ -525,8 +528,6 @@ requirements: virtualenv .requirements .sdist-requirements install-runners @echo @echo "==================== requirements ====================" @echo - # If you update these versions, make sure you also update the versions in the .st2client-install-check target to match - $(VIRTUALENV_DIR)/bin/pip --version # setuptools >= 41.0.1 is required for packs.install in dev envs # setuptools >= 42 is required so setup.py install respects dependencies' python_requires $(VIRTUALENV_DIR)/bin/pip install --upgrade "setuptools==44.1.0" @@ -581,9 +582,7 @@ virtualenv: @echo @echo "==================== virtualenv ====================" @echo - virtualenv --version - virtualenv --help - test -f $(VIRTUALENV_DIR)/bin/activate || virtualenv --python=$(PYTHON_VERSION) $(VIRTUALENV_DIR) --pip $(PIP_VERSION) + test -f $(VIRTUALENV_DIR)/bin/activate || virtualenv --python=$(PYTHON_VERSION) $(VIRTUALENV_DIR) --no-download # Setup PYTHONPATH in bash activate script... # Delete existing entries (if any) From 027553a363a3eb1293f1b9dfbc8955ae52df519a Mon Sep 17 00:00:00 2001 From: armab Date: Sat, 23 May 2020 18:31:48 +0100 Subject: [PATCH 8/8] Remove virtualenv installation --- .travis.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 647dcc8bbc..b2c5b4e593 100644 --- a/.travis.yml +++ b/.travis.yml @@ -79,10 +79,6 @@ cache: # are updated so it's disabled #- .tox/ -before_install: - # Upgrade virtualenv to latest - - pip install --upgrade "virtualenv" - install: - ./scripts/travis/install-requirements.sh # prep a travis-specific dev conf file that uses travis instead of stanley