From 01211729514a55aeba6b257a71227d7530127d91 Mon Sep 17 00:00:00 2001 From: winem Date: Sun, 8 Nov 2020 23:54:31 +0100 Subject: [PATCH 01/47] check if ST2_VERSION refers to a released st2 version or a development branch and use extra tags for releases --- Makefile | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/Makefile b/Makefile index c6470a86..8791acd4 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,17 @@ ST2_VERSION ?= 3.4dev DOCKER_TAG ?= ${ST2_VERSION} +RELEASE_TAG_REGEX := [^dev]$$ SHELL := /bin/bash +ifneq ($(shell echo ${ST2_VERSION} | grep -E "${RELEASE_TAG_REGEX}"), ) + RELEASE_VERSION = true + MAJOR = $(word 1, $(subst ., ,${ST2_VERSION})) + MINOR = $(word 2, $(subst ., ,${ST2_VERSION})) + PATCH = $(word 3, $(subst ., ,${ST2_VERSION})) +else + RELEASE_VERSION = false +endif + # Build all required images (st2 base image plus st2 components) .PHONY: build build: @@ -20,6 +30,13 @@ build: $$component/; \ echo -e "\033[32mSuccessfully built \033[1mstackstorm/$$component:${DOCKER_TAG}\033[0m\033[32m Docker image for StackStorm version \033[1m${ST2_VERSION}\033[0m"; \ done +ifeq ($(RELEASE_VERSION), true) + for image in st2 st2*; do \ + docker tag stackstorm/$$image:${DOCKER_TAG} stackstorm/$$image:${MAJOR}; \ + docker tag stackstorm/$$image:${DOCKER_TAG} stackstorm/$$image:${MAJOR}.${MINOR}; \ + echo -e "\033[32mSuccessfully tagged \033[1mstackstorm/$$image:${DOCKER_TAG}\033[0m\033[32m with \033[1mstackstorm/$$image:${MAJOR}\033[0m\033[32m and \033[1mstackstorm/$$image:${MAJOR}.${MINOR}\033[0m"; \ + done +endif .PHONY: push push: @@ -30,3 +47,11 @@ push: docker push stackstorm/$$component:${DOCKER_TAG}; \ echo -e "\033[32mSuccessfully pushed \033[1mstackstorm/$$component:${DOCKER_TAG}\033[0m\033[32m Docker image for StackStorm version \033[1m${ST2_VERSION}\033[0m"; \ done +ifeq ($(RELEASE_VERSION), true) + for image in st2 st2*; do \ + docker push stackstorm/$$image:${MAJOR}; \ + echo -e "\033[32mSuccessfully pushed \033[1mstackstorm/$$image:${MAJOR}\033[0m\033[32m Docker image for StackStorm version \033[1m${ST2_VERSION}\033[0m"; \ + docker push stackstorm/$$image:${MAJOR}.${MINOR}; \ + echo -e "\033[32mSuccessfully pushed \033[1mstackstorm/$$image:${MAJOR}.${MINOR}\033[0m\033[32m Docker image for StackStorm version \033[1m${ST2_VERSION}\033[0m"; \ + done +endif \ No newline at end of file From 0fe866e9d81d29cd41e7eebacf75b4e8a66558a1 Mon Sep 17 00:00:00 2001 From: winem Date: Sat, 28 Nov 2020 23:47:43 +0100 Subject: [PATCH 02/47] add script to provide info about required tags --- get_latest_dockertag.sh | 69 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100755 get_latest_dockertag.sh diff --git a/get_latest_dockertag.sh b/get_latest_dockertag.sh new file mode 100755 index 00000000..9ebb1260 --- /dev/null +++ b/get_latest_dockertag.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash + +component=$1 +build_version=$2 +registry=registry.hub.docker.com +tmp_release_cache_file=/tmp/dockerhub_st2_releases + +if [[ ${build_version} =~ ^([0-9]+)\.([0-9]+).?([0-9]*)$ ]]; then + build_major=${BASH_REMATCH[1]} + build_minor=${BASH_REMATCH[2]} + build_patch=${BASH_REMATCH[3]} +else + echo "The provided version ${build_version} does not have the expected format." + exit 1 +fi + +# dockerhub lists the tags in ascending order. 1st object = lowest tag; last object = highest tag +curl -s https://${registry}/v1/repositories/stackstorm/${component}/tags | jq -r '.[] | select(.name | endswith("dev") | not).name' > ${tmp_release_cache_file} + +if [ -z ${build_patch} ]; then + build_patch=0 +fi + +readarray -t available_releases < $tmp_release_cache_file +#latest_release_array=(${available_releases[-1]//\./ }) +latest_release=${available_releases[-1]} +latest_release_array=(${latest_release//\./ }) + +latest_major=${latest_release_array[0]} +latest_minor=${latest_release_array[1]} +latest_patch=${latest_release_array[2]} + +# possible values of the tag_update flag: +# 0 = no additional tags to be set +# 1 = add the major.minor tag +# 2 = add the tags major and major.minor +# 3 = add the latest tag (for future use) + +tag_update_flag=0 + + +if [ ${build_version} == ${latest_release} ]; then + tag_update_flag=2 +else + if [ ${build_major} -lt ${latest_major} ]; then + readarray -t build_version_major_matching_releases < <(grep ${build_major} $tmp_release_cache_file) + latest_build_version_major_matching_release=${build_version_major_matching_releases[-1]} + latest_build_version_major_matching_release_array=(${latest_build_version_major_matching_release//\./ }) + latest_build_version_major_matching_minor=${latest_build_version_major_matching_release_array[1]} + latest_build_version_major_matching_patch=${latest_build_version_major_matching_release_array[2]} + if [ ${build_minor} -gt ${latest_build_version_major_matching_minor} ]; then + tag_update_flag=2 + elif [ ${build_minor} -eq ${latest_build_version_major_matching_minor} ] && [ ${build_patch} -ge ${latest_build_version_major_matching_patch} ]; then + tag_update_flag=2 + else + # building a release for a major version that is *not* the latest of the given major + readarray -t build_version_major_minor_matching_releases < <(grep ${build_major}.${build_minor} $tmp_release_cache_file) + latest_build_version_major_minor_matching_release=${build_version_major_matching_releases[-1]} + latest_build_version_major_minor_matching_release_array=(${latest_build_version_major_minor_matching_release//\./ }) + latest_build_version_major_minor_matching_patch=${latest_build_version_major_minor_matching_release_array[2]} + if [ ${build_patch} -ge ${latest_build_version_major_minor_matching_patch}]; then + tag_update_flag=1 + fi + fi + else + # building a release for a new major version + tag_update_flag=2 + fi +fi \ No newline at end of file From 1a4d02db69ae70a66d7acc72e3002bc264d2b8f8 Mon Sep 17 00:00:00 2001 From: winem Date: Sun, 29 Nov 2020 23:58:17 +0100 Subject: [PATCH 03/47] finish code required to determine the expected tags to be updated --- Makefile | 11 ++++++++++- get_latest_dockertag.sh => determine_needed_tags.sh | 9 ++++----- 2 files changed, 14 insertions(+), 6 deletions(-) rename get_latest_dockertag.sh => determine_needed_tags.sh (97%) diff --git a/Makefile b/Makefile index 8791acd4..a366aa97 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,10 @@ -ST2_VERSION ?= 3.4dev +ST2_VERSION ?= 3.3.0 DOCKER_TAG ?= ${ST2_VERSION} RELEASE_TAG_REGEX := [^dev]$$ SHELL := /bin/bash +TAG_UPDATE_FLAG = $(shell ./determine_needed_tags.sh st2 ${ST2_VERSION}) + ifneq ($(shell echo ${ST2_VERSION} | grep -E "${RELEASE_TAG_REGEX}"), ) RELEASE_VERSION = true MAJOR = $(word 1, $(subst ., ,${ST2_VERSION})) @@ -31,12 +33,19 @@ build: echo -e "\033[32mSuccessfully built \033[1mstackstorm/$$component:${DOCKER_TAG}\033[0m\033[32m Docker image for StackStorm version \033[1m${ST2_VERSION}\033[0m"; \ done ifeq ($(RELEASE_VERSION), true) +ifeq ($(TAG_UPDATE_FLAG), 1) + for image in st2 st2*; do \ + docker tag stackstorm/$$image:${DOCKER_TAG} stackstorm/$$image:${MAJOR}.${MINOR}; \ + echo -e "\033[32mSuccessfully tagged \033[1mstackstorm/$$image:${DOCKER_TAG}\033[0m\033[32m with \033[1mstackstorm/$$image:${MAJOR}.${MINOR}\033[0m"; \ + done +else ifeq ($(TAG_UPDATE_FLAG), 2) for image in st2 st2*; do \ docker tag stackstorm/$$image:${DOCKER_TAG} stackstorm/$$image:${MAJOR}; \ docker tag stackstorm/$$image:${DOCKER_TAG} stackstorm/$$image:${MAJOR}.${MINOR}; \ echo -e "\033[32mSuccessfully tagged \033[1mstackstorm/$$image:${DOCKER_TAG}\033[0m\033[32m with \033[1mstackstorm/$$image:${MAJOR}\033[0m\033[32m and \033[1mstackstorm/$$image:${MAJOR}.${MINOR}\033[0m"; \ done endif +endif .PHONY: push push: diff --git a/get_latest_dockertag.sh b/determine_needed_tags.sh similarity index 97% rename from get_latest_dockertag.sh rename to determine_needed_tags.sh index 9ebb1260..f8dd5dd4 100755 --- a/get_latest_dockertag.sh +++ b/determine_needed_tags.sh @@ -22,7 +22,6 @@ if [ -z ${build_patch} ]; then fi readarray -t available_releases < $tmp_release_cache_file -#latest_release_array=(${available_releases[-1]//\./ }) latest_release=${available_releases[-1]} latest_release_array=(${latest_release//\./ }) @@ -30,15 +29,13 @@ latest_major=${latest_release_array[0]} latest_minor=${latest_release_array[1]} latest_patch=${latest_release_array[2]} +tag_update_flag=0 # possible values of the tag_update flag: # 0 = no additional tags to be set # 1 = add the major.minor tag # 2 = add the tags major and major.minor # 3 = add the latest tag (for future use) -tag_update_flag=0 - - if [ ${build_version} == ${latest_release} ]; then tag_update_flag=2 else @@ -66,4 +63,6 @@ else # building a release for a new major version tag_update_flag=2 fi -fi \ No newline at end of file +fi + +echo $tag_update_flag From d3f09616467de9040e96cc422dc1e37885477f16 Mon Sep 17 00:00:00 2001 From: winem Date: Sun, 29 Nov 2020 23:59:32 +0100 Subject: [PATCH 04/47] Add comment with the supported values of TAG_UPDATE_FLAG --- Makefile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Makefile b/Makefile index a366aa97..ad341b33 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,11 @@ RELEASE_TAG_REGEX := [^dev]$$ SHELL := /bin/bash TAG_UPDATE_FLAG = $(shell ./determine_needed_tags.sh st2 ${ST2_VERSION}) +# possible values of the TAG_UPDATE_FLAG +# 0 = no additional tags to be set +# 1 = add the major.minor tag +# 2 = add the tags major and major.minor +# 3 = add the latest tag (for future use) ifneq ($(shell echo ${ST2_VERSION} | grep -E "${RELEASE_TAG_REGEX}"), ) RELEASE_VERSION = true From 05b025f06989c91c7d30e6a8c022005e8d7ff870 Mon Sep 17 00:00:00 2001 From: winem Date: Sun, 29 Nov 2020 23:59:41 +0100 Subject: [PATCH 05/47] Add comment with the supported values of TAG_UPDATE_FLAG --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ad341b33..7db79338 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ RELEASE_TAG_REGEX := [^dev]$$ SHELL := /bin/bash TAG_UPDATE_FLAG = $(shell ./determine_needed_tags.sh st2 ${ST2_VERSION}) -# possible values of the TAG_UPDATE_FLAG +# supported values of the TAG_UPDATE_FLAG # 0 = no additional tags to be set # 1 = add the major.minor tag # 2 = add the tags major and major.minor From 6442e6dbf1a65095f632a22728644642228989ce Mon Sep 17 00:00:00 2001 From: winem Date: Mon, 30 Nov 2020 00:02:15 +0100 Subject: [PATCH 06/47] Revert ST2_VERSION to 3.4dev --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 7db79338..55940f8e 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -ST2_VERSION ?= 3.3.0 +ST2_VERSION ?= 3.4dev DOCKER_TAG ?= ${ST2_VERSION} RELEASE_TAG_REGEX := [^dev]$$ SHELL := /bin/bash From f411442df7a942c9a01d467c18728b624adb2ee1 Mon Sep 17 00:00:00 2001 From: winem Date: Wed, 2 Dec 2020 21:00:21 +0100 Subject: [PATCH 07/47] Fix indentation according to the code style --- Makefile | 74 ++++++++++++++++++++-------------------- determine_needed_tags.sh | 58 +++++++++++++++---------------- 2 files changed, 66 insertions(+), 66 deletions(-) diff --git a/Makefile b/Makefile index 55940f8e..aab0c1ea 100644 --- a/Makefile +++ b/Makefile @@ -22,50 +22,50 @@ endif # Build all required images (st2 base image plus st2 components) .PHONY: build build: - @docker build \ - --pull \ - --no-cache \ - --build-arg ST2_VERSION=${ST2_VERSION} \ - -t stackstorm/st2:${DOCKER_TAG} base/ - @echo -e "\033[32mSuccessfully built \033[1mstackstorm/st2:${DOCKER_TAG}\033[0m\033[32m common Docker image with StackStorm version \033[1m${ST2_VERSION}\033[0m" - @set -e; \ - for component in st2*; do \ - docker build \ - --no-cache \ - --build-arg ST2_VERSION=${ST2_VERSION} \ - --tag stackstorm/$$component:${DOCKER_TAG} \ - $$component/; \ - echo -e "\033[32mSuccessfully built \033[1mstackstorm/$$component:${DOCKER_TAG}\033[0m\033[32m Docker image for StackStorm version \033[1m${ST2_VERSION}\033[0m"; \ - done + @docker build \ + --pull \ + --no-cache \ + --build-arg ST2_VERSION=${ST2_VERSION} \ + -t stackstorm/st2:${DOCKER_TAG} base/ + @echo -e "\033[32mSuccessfully built \033[1mstackstorm/st2:${DOCKER_TAG}\033[0m\033[32m common Docker image with StackStorm version \033[1m${ST2_VERSION}\033[0m" + @set -e; \ + for component in st2*; do \ + docker build \ + --no-cache \ + --build-arg ST2_VERSION=${ST2_VERSION} \ + --tag stackstorm/$$component:${DOCKER_TAG} \ + $$component/; \ + echo -e "\033[32mSuccessfully built \033[1mstackstorm/$$component:${DOCKER_TAG}\033[0m\033[32m Docker image for StackStorm version \033[1m${ST2_VERSION}\033[0m"; \ + done ifeq ($(RELEASE_VERSION), true) ifeq ($(TAG_UPDATE_FLAG), 1) - for image in st2 st2*; do \ - docker tag stackstorm/$$image:${DOCKER_TAG} stackstorm/$$image:${MAJOR}.${MINOR}; \ - echo -e "\033[32mSuccessfully tagged \033[1mstackstorm/$$image:${DOCKER_TAG}\033[0m\033[32m with \033[1mstackstorm/$$image:${MAJOR}.${MINOR}\033[0m"; \ - done + for image in st2 st2*; do \ + docker tag stackstorm/$$image:${DOCKER_TAG} stackstorm/$$image:${MAJOR}.${MINOR}; \ + echo -e "\033[32mSuccessfully tagged \033[1mstackstorm/$$image:${DOCKER_TAG}\033[0m\033[32m with \033[1mstackstorm/$$image:${MAJOR}.${MINOR}\033[0m"; \ + done else ifeq ($(TAG_UPDATE_FLAG), 2) - for image in st2 st2*; do \ - docker tag stackstorm/$$image:${DOCKER_TAG} stackstorm/$$image:${MAJOR}; \ - docker tag stackstorm/$$image:${DOCKER_TAG} stackstorm/$$image:${MAJOR}.${MINOR}; \ - echo -e "\033[32mSuccessfully tagged \033[1mstackstorm/$$image:${DOCKER_TAG}\033[0m\033[32m with \033[1mstackstorm/$$image:${MAJOR}\033[0m\033[32m and \033[1mstackstorm/$$image:${MAJOR}.${MINOR}\033[0m"; \ - done + for image in st2 st2*; do \ + docker tag stackstorm/$$image:${DOCKER_TAG} stackstorm/$$image:${MAJOR}; \ + docker tag stackstorm/$$image:${DOCKER_TAG} stackstorm/$$image:${MAJOR}.${MINOR}; \ + echo -e "\033[32mSuccessfully tagged \033[1mstackstorm/$$image:${DOCKER_TAG}\033[0m\033[32m with \033[1mstackstorm/$$image:${MAJOR}\033[0m\033[32m and \033[1mstackstorm/$$image:${MAJOR}.${MINOR}\033[0m"; \ + done endif endif .PHONY: push push: - docker push stackstorm/st2:${DOCKER_TAG}; - @echo -e "\033[32mSuccessfully pushed \033[1mstackstorm/st2:${DOCKER_TAG}\033[0m\033[32m Docker image for StackStorm version \033[1m${ST2_VERSION}\033[0m"; - @set -e; \ - for component in st2*; do \ - docker push stackstorm/$$component:${DOCKER_TAG}; \ - echo -e "\033[32mSuccessfully pushed \033[1mstackstorm/$$component:${DOCKER_TAG}\033[0m\033[32m Docker image for StackStorm version \033[1m${ST2_VERSION}\033[0m"; \ - done + docker push stackstorm/st2:${DOCKER_TAG}; + @echo -e "\033[32mSuccessfully pushed \033[1mstackstorm/st2:${DOCKER_TAG}\033[0m\033[32m Docker image for StackStorm version \033[1m${ST2_VERSION}\033[0m"; + @set -e; \ + for component in st2*; do \ + docker push stackstorm/$$component:${DOCKER_TAG}; \ + echo -e "\033[32mSuccessfully pushed \033[1mstackstorm/$$component:${DOCKER_TAG}\033[0m\033[32m Docker image for StackStorm version \033[1m${ST2_VERSION}\033[0m"; \ + done ifeq ($(RELEASE_VERSION), true) - for image in st2 st2*; do \ - docker push stackstorm/$$image:${MAJOR}; \ - echo -e "\033[32mSuccessfully pushed \033[1mstackstorm/$$image:${MAJOR}\033[0m\033[32m Docker image for StackStorm version \033[1m${ST2_VERSION}\033[0m"; \ - docker push stackstorm/$$image:${MAJOR}.${MINOR}; \ - echo -e "\033[32mSuccessfully pushed \033[1mstackstorm/$$image:${MAJOR}.${MINOR}\033[0m\033[32m Docker image for StackStorm version \033[1m${ST2_VERSION}\033[0m"; \ - done + for image in st2 st2*; do \ + docker push stackstorm/$$image:${MAJOR}; \ + echo -e "\033[32mSuccessfully pushed \033[1mstackstorm/$$image:${MAJOR}\033[0m\033[32m Docker image for StackStorm version \033[1m${ST2_VERSION}\033[0m"; \ + docker push stackstorm/$$image:${MAJOR}.${MINOR}; \ + echo -e "\033[32mSuccessfully pushed \033[1mstackstorm/$$image:${MAJOR}.${MINOR}\033[0m\033[32m Docker image for StackStorm version \033[1m${ST2_VERSION}\033[0m"; \ + done endif \ No newline at end of file diff --git a/determine_needed_tags.sh b/determine_needed_tags.sh index f8dd5dd4..feccb1c7 100755 --- a/determine_needed_tags.sh +++ b/determine_needed_tags.sh @@ -6,19 +6,19 @@ registry=registry.hub.docker.com tmp_release_cache_file=/tmp/dockerhub_st2_releases if [[ ${build_version} =~ ^([0-9]+)\.([0-9]+).?([0-9]*)$ ]]; then - build_major=${BASH_REMATCH[1]} + build_major=${BASH_REMATCH[1]} build_minor=${BASH_REMATCH[2]} build_patch=${BASH_REMATCH[3]} else - echo "The provided version ${build_version} does not have the expected format." - exit 1 + echo "The provided version ${build_version} does not have the expected format." + exit 1 fi # dockerhub lists the tags in ascending order. 1st object = lowest tag; last object = highest tag curl -s https://${registry}/v1/repositories/stackstorm/${component}/tags | jq -r '.[] | select(.name | endswith("dev") | not).name' > ${tmp_release_cache_file} if [ -z ${build_patch} ]; then - build_patch=0 + build_patch=0 fi readarray -t available_releases < $tmp_release_cache_file @@ -37,32 +37,32 @@ tag_update_flag=0 # 3 = add the latest tag (for future use) if [ ${build_version} == ${latest_release} ]; then - tag_update_flag=2 + tag_update_flag=2 else - if [ ${build_major} -lt ${latest_major} ]; then - readarray -t build_version_major_matching_releases < <(grep ${build_major} $tmp_release_cache_file) - latest_build_version_major_matching_release=${build_version_major_matching_releases[-1]} - latest_build_version_major_matching_release_array=(${latest_build_version_major_matching_release//\./ }) - latest_build_version_major_matching_minor=${latest_build_version_major_matching_release_array[1]} - latest_build_version_major_matching_patch=${latest_build_version_major_matching_release_array[2]} - if [ ${build_minor} -gt ${latest_build_version_major_matching_minor} ]; then - tag_update_flag=2 - elif [ ${build_minor} -eq ${latest_build_version_major_matching_minor} ] && [ ${build_patch} -ge ${latest_build_version_major_matching_patch} ]; then - tag_update_flag=2 - else - # building a release for a major version that is *not* the latest of the given major - readarray -t build_version_major_minor_matching_releases < <(grep ${build_major}.${build_minor} $tmp_release_cache_file) - latest_build_version_major_minor_matching_release=${build_version_major_matching_releases[-1]} - latest_build_version_major_minor_matching_release_array=(${latest_build_version_major_minor_matching_release//\./ }) - latest_build_version_major_minor_matching_patch=${latest_build_version_major_minor_matching_release_array[2]} - if [ ${build_patch} -ge ${latest_build_version_major_minor_matching_patch}]; then - tag_update_flag=1 - fi - fi - else - # building a release for a new major version - tag_update_flag=2 - fi + if [ ${build_major} -lt ${latest_major} ]; then + readarray -t build_version_major_matching_releases < <(grep ${build_major} $tmp_release_cache_file) + latest_build_version_major_matching_release=${build_version_major_matching_releases[-1]} + latest_build_version_major_matching_release_array=(${latest_build_version_major_matching_release//\./ }) + latest_build_version_major_matching_minor=${latest_build_version_major_matching_release_array[1]} + latest_build_version_major_matching_patch=${latest_build_version_major_matching_release_array[2]} + if [ ${build_minor} -gt ${latest_build_version_major_matching_minor} ]; then + tag_update_flag=2 + elif [ ${build_minor} -eq ${latest_build_version_major_matching_minor} ] && [ ${build_patch} -ge ${latest_build_version_major_matching_patch} ]; then + tag_update_flag=2 + else + # building a release for a major version that is *not* the latest of the given major + readarray -t build_version_major_minor_matching_releases < <(grep ${build_major}.${build_minor} $tmp_release_cache_file) + latest_build_version_major_minor_matching_release=${build_version_major_matching_releases[-1]} + latest_build_version_major_minor_matching_release_array=(${latest_build_version_major_minor_matching_release//\./ }) + latest_build_version_major_minor_matching_patch=${latest_build_version_major_minor_matching_release_array[2]} + if [ ${build_patch} -ge ${latest_build_version_major_minor_matching_patch}]; then + tag_update_flag=1 + fi + fi + else + # building a release for a new major version + tag_update_flag=2 + fi fi echo $tag_update_flag From 7a2a34fb4fdc56546f63cf9a02fa3436cb2f7d59 Mon Sep 17 00:00:00 2001 From: winem Date: Wed, 2 Dec 2020 21:14:15 +0100 Subject: [PATCH 08/47] Drop the registry variable as there's no need to have it configurable --- determine_needed_tags.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/determine_needed_tags.sh b/determine_needed_tags.sh index feccb1c7..e6003b50 100755 --- a/determine_needed_tags.sh +++ b/determine_needed_tags.sh @@ -2,7 +2,6 @@ component=$1 build_version=$2 -registry=registry.hub.docker.com tmp_release_cache_file=/tmp/dockerhub_st2_releases if [[ ${build_version} =~ ^([0-9]+)\.([0-9]+).?([0-9]*)$ ]]; then @@ -15,7 +14,7 @@ else fi # dockerhub lists the tags in ascending order. 1st object = lowest tag; last object = highest tag -curl -s https://${registry}/v1/repositories/stackstorm/${component}/tags | jq -r '.[] | select(.name | endswith("dev") | not).name' > ${tmp_release_cache_file} +curl -s https://registry.hub.docker.com/v1/repositories/stackstorm/${component}/tags | jq -r '.[] | select(.name | endswith("dev") | not).name' > ${tmp_release_cache_file} if [ -z ${build_patch} ]; then build_patch=0 From 06766e4afecf528b0497c13582c166c7785f3480 Mon Sep 17 00:00:00 2001 From: winem Date: Wed, 2 Dec 2020 21:24:03 +0100 Subject: [PATCH 09/47] write the available releases directly into an bash array and don't use a temporary file --- determine_needed_tags.sh | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/determine_needed_tags.sh b/determine_needed_tags.sh index e6003b50..24dc4bfa 100755 --- a/determine_needed_tags.sh +++ b/determine_needed_tags.sh @@ -2,7 +2,6 @@ component=$1 build_version=$2 -tmp_release_cache_file=/tmp/dockerhub_st2_releases if [[ ${build_version} =~ ^([0-9]+)\.([0-9]+).?([0-9]*)$ ]]; then build_major=${BASH_REMATCH[1]} @@ -13,14 +12,13 @@ else exit 1 fi -# dockerhub lists the tags in ascending order. 1st object = lowest tag; last object = highest tag -curl -s https://registry.hub.docker.com/v1/repositories/stackstorm/${component}/tags | jq -r '.[] | select(.name | endswith("dev") | not).name' > ${tmp_release_cache_file} - if [ -z ${build_patch} ]; then build_patch=0 fi -readarray -t available_releases < $tmp_release_cache_file +# dockerhub lists the tags in ascending order. 1st object = lowest tag; last object = highest tag +readarray -t available_releases < <(curl -s https://registry.hub.docker.com/v1/repositories/stackstorm/${component}/tags | jq -r '.[] | select(.name | endswith("dev") | not).name') + latest_release=${available_releases[-1]} latest_release_array=(${latest_release//\./ }) From 0f8547228d3e71d77d1f773272ab24917d7bdf2a Mon Sep 17 00:00:00 2001 From: winem Date: Thu, 3 Dec 2020 00:52:09 +0100 Subject: [PATCH 10/47] Simplify the logic to determine the correct tags to be updated --- determine_needed_tags.sh | 41 +++++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/determine_needed_tags.sh b/determine_needed_tags.sh index 24dc4bfa..200fa787 100755 --- a/determine_needed_tags.sh +++ b/determine_needed_tags.sh @@ -5,8 +5,8 @@ build_version=$2 if [[ ${build_version} =~ ^([0-9]+)\.([0-9]+).?([0-9]*)$ ]]; then build_major=${BASH_REMATCH[1]} - build_minor=${BASH_REMATCH[2]} - build_patch=${BASH_REMATCH[3]} + build_minor=${BASH_REMATCH[2]} + build_patch=${BASH_REMATCH[3]} else echo "The provided version ${build_version} does not have the expected format." exit 1 @@ -17,14 +17,14 @@ if [ -z ${build_patch} ]; then fi # dockerhub lists the tags in ascending order. 1st object = lowest tag; last object = highest tag -readarray -t available_releases < <(curl -s https://registry.hub.docker.com/v1/repositories/stackstorm/${component}/tags | jq -r '.[] | select(.name | endswith("dev") | not).name') +docker_tags_json=$(curl -s https://registry.hub.docker.com/v1/repositories/stackstorm/${component}/tags) +readarray -t available_releases < <(echo $docker_tags_json | jq -r '.[] | select(.name | endswith("dev") | not).name') latest_release=${available_releases[-1]} latest_release_array=(${latest_release//\./ }) latest_major=${latest_release_array[0]} latest_minor=${latest_release_array[1]} -latest_patch=${latest_release_array[2]} tag_update_flag=0 # possible values of the tag_update flag: @@ -34,27 +34,30 @@ tag_update_flag=0 # 3 = add the latest tag (for future use) if [ ${build_version} == ${latest_release} ]; then + # building a release of the latest st2 version tag_update_flag=2 else - if [ ${build_major} -lt ${latest_major} ]; then - readarray -t build_version_major_matching_releases < <(grep ${build_major} $tmp_release_cache_file) - latest_build_version_major_matching_release=${build_version_major_matching_releases[-1]} - latest_build_version_major_matching_release_array=(${latest_build_version_major_matching_release//\./ }) - latest_build_version_major_matching_minor=${latest_build_version_major_matching_release_array[1]} - latest_build_version_major_matching_patch=${latest_build_version_major_matching_release_array[2]} - if [ ${build_minor} -gt ${latest_build_version_major_matching_minor} ]; then - tag_update_flag=2 - elif [ ${build_minor} -eq ${latest_build_version_major_matching_minor} ] && [ ${build_patch} -ge ${latest_build_version_major_matching_patch} ]; then - tag_update_flag=2 - else - # building a release for a major version that is *not* the latest of the given major - readarray -t build_version_major_minor_matching_releases < <(grep ${build_major}.${build_minor} $tmp_release_cache_file) - latest_build_version_major_minor_matching_release=${build_version_major_matching_releases[-1]} + # building a release for a st2 version that does not match the latest version + if [ ${build_major} -le ${latest_major} ]; then + # building a release for an older major + readarray -t build_version_major_minor_matching_releases < <(echo $docker_tags_json | jq -r '.[] | select(.name | endswith("dev") | not) | select(.name | startswith("'"${build_major}.${build_minor}"'")).name') + if [ ${#build_version_major_minor_matching_releases[@]} -ge 1 ]; then + # at least one version matching the current builds major and minor version is available + latest_build_version_major_minor_matching_release=${build_version_major_minor_matching_releases[-1]} latest_build_version_major_minor_matching_release_array=(${latest_build_version_major_minor_matching_release//\./ }) + latest_build_version_major_minor_matching_major=${latest_build_version_major_minor_matching_release_array[0]} + latest_build_version_major_minor_matching_minor=${latest_build_version_major_minor_matching_release_array[1]} latest_build_version_major_minor_matching_patch=${latest_build_version_major_minor_matching_release_array[2]} - if [ ${build_patch} -ge ${latest_build_version_major_minor_matching_patch}]; then + if [ ${build_minor} -ge ${latest_build_version_major_minor_matching_patch} ] && [ ${build_patch} -ge ${latest_build_version_major_minor_matching_patch} ]; then + # building a release for a new or updated patch version of the current or a new minor version + tag_update_flag=2 + elif [ ${build_minor} -lt ${latest_build_version_major_minor_matching_minor} ] && [ ${build_patch} -ge ${latest_build_version_major_minor_matching_patch} ]; then + # building a patch release for an older minor version tag_update_flag=1 fi + else + # building a release for an old, unreleased major version of st2 + tag_update_flag=2 fi else # building a release for a new major version From 8f052f11b2e494fe064a62ee1994f4e40ed3166b Mon Sep 17 00:00:00 2001 From: winem Date: Thu, 3 Dec 2020 01:01:23 +0100 Subject: [PATCH 11/47] Use tabs in the Makefile to avoid Missing seperator errors --- Makefile | 84 ++++++++++++++++++++++++++++---------------------------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/Makefile b/Makefile index aab0c1ea..f125c7c0 100644 --- a/Makefile +++ b/Makefile @@ -11,61 +11,61 @@ TAG_UPDATE_FLAG = $(shell ./determine_needed_tags.sh st2 ${ST2_VERSION}) # 3 = add the latest tag (for future use) ifneq ($(shell echo ${ST2_VERSION} | grep -E "${RELEASE_TAG_REGEX}"), ) - RELEASE_VERSION = true - MAJOR = $(word 1, $(subst ., ,${ST2_VERSION})) - MINOR = $(word 2, $(subst ., ,${ST2_VERSION})) - PATCH = $(word 3, $(subst ., ,${ST2_VERSION})) + RELEASE_VERSION = true + MAJOR = $(word 1, $(subst ., ,${ST2_VERSION})) + MINOR = $(word 2, $(subst ., ,${ST2_VERSION})) + PATCH = $(word 3, $(subst ., ,${ST2_VERSION})) else - RELEASE_VERSION = false + RELEASE_VERSION = false endif # Build all required images (st2 base image plus st2 components) .PHONY: build build: - @docker build \ - --pull \ - --no-cache \ - --build-arg ST2_VERSION=${ST2_VERSION} \ - -t stackstorm/st2:${DOCKER_TAG} base/ - @echo -e "\033[32mSuccessfully built \033[1mstackstorm/st2:${DOCKER_TAG}\033[0m\033[32m common Docker image with StackStorm version \033[1m${ST2_VERSION}\033[0m" - @set -e; \ - for component in st2*; do \ - docker build \ - --no-cache \ - --build-arg ST2_VERSION=${ST2_VERSION} \ - --tag stackstorm/$$component:${DOCKER_TAG} \ - $$component/; \ - echo -e "\033[32mSuccessfully built \033[1mstackstorm/$$component:${DOCKER_TAG}\033[0m\033[32m Docker image for StackStorm version \033[1m${ST2_VERSION}\033[0m"; \ - done + @docker build \ + --pull \ + --no-cache \ + --build-arg ST2_VERSION=${ST2_VERSION} \ + -t stackstorm/st2:${DOCKER_TAG} base/ + @echo -e "\033[32mSuccessfully built \033[1mstackstorm/st2:${DOCKER_TAG}\033[0m\033[32m common Docker image with StackStorm version \033[1m${ST2_VERSION}\033[0m" + @set -e; \ + for component in st2*; do \ + docker build \ + --no-cache \ + --build-arg ST2_VERSION=${ST2_VERSION} \ + --tag stackstorm/$$component:${DOCKER_TAG} \ + $$component/; \ + echo -e "\033[32mSuccessfully built \033[1mstackstorm/$$component:${DOCKER_TAG}\033[0m\033[32m Docker image for StackStorm version \033[1m${ST2_VERSION}\033[0m"; \ + done ifeq ($(RELEASE_VERSION), true) ifeq ($(TAG_UPDATE_FLAG), 1) - for image in st2 st2*; do \ - docker tag stackstorm/$$image:${DOCKER_TAG} stackstorm/$$image:${MAJOR}.${MINOR}; \ - echo -e "\033[32mSuccessfully tagged \033[1mstackstorm/$$image:${DOCKER_TAG}\033[0m\033[32m with \033[1mstackstorm/$$image:${MAJOR}.${MINOR}\033[0m"; \ - done + for image in st2 st2*; do \ + docker tag stackstorm/$$image:${DOCKER_TAG} stackstorm/$$image:${MAJOR}.${MINOR}; \ + echo -e "\033[32mSuccessfully tagged \033[1mstackstorm/$$image:${DOCKER_TAG}\033[0m\033[32m with \033[1mstackstorm/$$image:${MAJOR}.${MINOR}\033[0m"; \ + done else ifeq ($(TAG_UPDATE_FLAG), 2) - for image in st2 st2*; do \ - docker tag stackstorm/$$image:${DOCKER_TAG} stackstorm/$$image:${MAJOR}; \ - docker tag stackstorm/$$image:${DOCKER_TAG} stackstorm/$$image:${MAJOR}.${MINOR}; \ - echo -e "\033[32mSuccessfully tagged \033[1mstackstorm/$$image:${DOCKER_TAG}\033[0m\033[32m with \033[1mstackstorm/$$image:${MAJOR}\033[0m\033[32m and \033[1mstackstorm/$$image:${MAJOR}.${MINOR}\033[0m"; \ - done + for image in st2 st2*; do \ + docker tag stackstorm/$$image:${DOCKER_TAG} stackstorm/$$image:${MAJOR}; \ + docker tag stackstorm/$$image:${DOCKER_TAG} stackstorm/$$image:${MAJOR}.${MINOR}; \ + echo -e "\033[32mSuccessfully tagged \033[1mstackstorm/$$image:${DOCKER_TAG}\033[0m\033[32m with \033[1mstackstorm/$$image:${MAJOR}\033[0m\033[32m and \033[1mstackstorm/$$image:${MAJOR}.${MINOR}\033[0m"; \ + done endif endif .PHONY: push push: - docker push stackstorm/st2:${DOCKER_TAG}; - @echo -e "\033[32mSuccessfully pushed \033[1mstackstorm/st2:${DOCKER_TAG}\033[0m\033[32m Docker image for StackStorm version \033[1m${ST2_VERSION}\033[0m"; - @set -e; \ - for component in st2*; do \ - docker push stackstorm/$$component:${DOCKER_TAG}; \ - echo -e "\033[32mSuccessfully pushed \033[1mstackstorm/$$component:${DOCKER_TAG}\033[0m\033[32m Docker image for StackStorm version \033[1m${ST2_VERSION}\033[0m"; \ - done + docker push stackstorm/st2:${DOCKER_TAG}; + @echo -e "\033[32mSuccessfully pushed \033[1mstackstorm/st2:${DOCKER_TAG}\033[0m\033[32m Docker image for StackStorm version \033[1m${ST2_VERSION}\033[0m"; + @set -e; \ + for component in st2*; do \ + docker push stackstorm/$$component:${DOCKER_TAG}; \ + echo -e "\033[32mSuccessfully pushed \033[1mstackstorm/$$component:${DOCKER_TAG}\033[0m\033[32m Docker image for StackStorm version \033[1m${ST2_VERSION}\033[0m"; \ + done ifeq ($(RELEASE_VERSION), true) - for image in st2 st2*; do \ - docker push stackstorm/$$image:${MAJOR}; \ - echo -e "\033[32mSuccessfully pushed \033[1mstackstorm/$$image:${MAJOR}\033[0m\033[32m Docker image for StackStorm version \033[1m${ST2_VERSION}\033[0m"; \ - docker push stackstorm/$$image:${MAJOR}.${MINOR}; \ - echo -e "\033[32mSuccessfully pushed \033[1mstackstorm/$$image:${MAJOR}.${MINOR}\033[0m\033[32m Docker image for StackStorm version \033[1m${ST2_VERSION}\033[0m"; \ - done + for image in st2 st2*; do \ + docker push stackstorm/$$image:${MAJOR}; \ + echo -e "\033[32mSuccessfully pushed \033[1mstackstorm/$$image:${MAJOR}\033[0m\033[32m Docker image for StackStorm version \033[1m${ST2_VERSION}\033[0m"; \ + docker push stackstorm/$$image:${MAJOR}.${MINOR}; \ + echo -e "\033[32mSuccessfully pushed \033[1mstackstorm/$$image:${MAJOR}.${MINOR}\033[0m\033[32m Docker image for StackStorm version \033[1m${ST2_VERSION}\033[0m"; \ + done endif \ No newline at end of file From f60ffd5c2d566d4e243059add16f446896fd520e Mon Sep 17 00:00:00 2001 From: winem Date: Sat, 5 Dec 2020 23:53:29 +0100 Subject: [PATCH 12/47] Don't rely on the dockerhub API and sort the list of returned tags --- determine_needed_tags.sh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/determine_needed_tags.sh b/determine_needed_tags.sh index 200fa787..251af949 100755 --- a/determine_needed_tags.sh +++ b/determine_needed_tags.sh @@ -1,6 +1,9 @@ #!/usr/bin/env bash +# the first argument must be the name of the container, i.e. st2, st2actionrunner, st2stream component=$1 +# the 2nd argument is the version of the current build and expects at least major.miinor to be provided +# i.e. 3.3.0, 3.3, 2.10.5 build_version=$2 if [[ ${build_version} =~ ^([0-9]+)\.([0-9]+).?([0-9]*)$ ]]; then @@ -16,10 +19,12 @@ if [ -z ${build_patch} ]; then build_patch=0 fi -# dockerhub lists the tags in ascending order. 1st object = lowest tag; last object = highest tag +# dockerhub lists the tags in ascending order. 1st object = lowest tag; last object = highest tag or latest docker_tags_json=$(curl -s https://registry.hub.docker.com/v1/repositories/stackstorm/${component}/tags) -readarray -t available_releases < <(echo $docker_tags_json | jq -r '.[] | select(.name | endswith("dev") | not).name') +readarray -t available_releases < <(echo $docker_tags_json | jq -r '.[] | select(.name | endswith("dev") | not).name' | sort) +# sort returns the list with the highest tag or "latest" as last item +# so change the value below to -2 when introducing the tag latest to get i.e. 3.3.0 instead of latest latest_release=${available_releases[-1]} latest_release_array=(${latest_release//\./ }) @@ -51,7 +56,7 @@ else if [ ${build_minor} -ge ${latest_build_version_major_minor_matching_patch} ] && [ ${build_patch} -ge ${latest_build_version_major_minor_matching_patch} ]; then # building a release for a new or updated patch version of the current or a new minor version tag_update_flag=2 - elif [ ${build_minor} -lt ${latest_build_version_major_minor_matching_minor} ] && [ ${build_patch} -ge ${latest_build_version_major_minor_matching_patch} ]; then + elif [ ${build_minor} -lt ${latest_build_version_major_minor_matching_minor} ] && [ ${build_patch} -ge ${latest_build_version_major_minor_matching_patch} ]; then # building a patch release for an older minor version tag_update_flag=1 fi From 410a70961d77a859536dcb36c5c3caba05e17990 Mon Sep 17 00:00:00 2001 From: winem Date: Sun, 6 Dec 2020 22:34:25 +0100 Subject: [PATCH 13/47] add usage info and fail on faulty user input --- determine_needed_tags.sh | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/determine_needed_tags.sh b/determine_needed_tags.sh index 251af949..d56622e2 100755 --- a/determine_needed_tags.sh +++ b/determine_needed_tags.sh @@ -6,6 +6,35 @@ component=$1 # i.e. 3.3.0, 3.3, 2.10.5 build_version=$2 +showHelp() { + echo "Use this script to determine the right tags to be attached to your new image." + echo + echo "Syntax: $0 st2component image_version" + echo "st2component = an valid image of an existing stackstorm component (i.e. st2, st2api, st2stream)" + echo "image_version = the st2 release used for your current build (i.e. 3.3.0)" + echo + echo "Possible output:" + echo "0 = no additional tags to be updated" + echo "1 = update the major.minor tag (i.e. 3.3)" + echo "2 = update the major and the major.minor tag (i.e. 3 and 3.3)" + echo + echo "Run $0 -h|--help to show this usage information." +} + +case $1 in + -h|--help) + showHelp + exit + ;; +esac + +if [ "$#" -ne 2 ]; then + echo "Error: Missing or unexpected number of positional arguments. Expected: 2" + echo + showHelp + exit 1 +fi + if [[ ${build_version} =~ ^([0-9]+)\.([0-9]+).?([0-9]*)$ ]]; then build_major=${BASH_REMATCH[1]} build_minor=${BASH_REMATCH[2]} @@ -70,4 +99,4 @@ else fi fi -echo $tag_update_flag +echo $tag_update_flag \ No newline at end of file From 4de5dc1d12d3787b12939badd12824f3f6518abe Mon Sep 17 00:00:00 2001 From: winem Date: Fri, 11 Dec 2020 21:22:18 +0100 Subject: [PATCH 14/47] Add newlines at the end of the Makefile and determine needed tag script --- Makefile | 2 +- determine_needed_tags.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index f125c7c0..f7facc8b 100644 --- a/Makefile +++ b/Makefile @@ -68,4 +68,4 @@ ifeq ($(RELEASE_VERSION), true) docker push stackstorm/$$image:${MAJOR}.${MINOR}; \ echo -e "\033[32mSuccessfully pushed \033[1mstackstorm/$$image:${MAJOR}.${MINOR}\033[0m\033[32m Docker image for StackStorm version \033[1m${ST2_VERSION}\033[0m"; \ done -endif \ No newline at end of file +endif diff --git a/determine_needed_tags.sh b/determine_needed_tags.sh index d56622e2..3839125b 100755 --- a/determine_needed_tags.sh +++ b/determine_needed_tags.sh @@ -99,4 +99,4 @@ else fi fi -echo $tag_update_flag \ No newline at end of file +echo $tag_update_flag From 624c07528d03a4f5f8dad64f2f2c5d7ecff4a808 Mon Sep 17 00:00:00 2001 From: winem Date: Fri, 11 Dec 2020 21:33:46 +0100 Subject: [PATCH 15/47] Support the latest tag as well --- determine_needed_tags.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/determine_needed_tags.sh b/determine_needed_tags.sh index 3839125b..7d7fc135 100755 --- a/determine_needed_tags.sh +++ b/determine_needed_tags.sh @@ -65,11 +65,11 @@ tag_update_flag=0 # 0 = no additional tags to be set # 1 = add the major.minor tag # 2 = add the tags major and major.minor -# 3 = add the latest tag (for future use) +# 3 = add the tags latest in addition to the tags from #2 if [ ${build_version} == ${latest_release} ]; then # building a release of the latest st2 version - tag_update_flag=2 + tag_update_flag=3 else # building a release for a st2 version that does not match the latest version if [ ${build_major} -le ${latest_major} ]; then @@ -84,7 +84,7 @@ else latest_build_version_major_minor_matching_patch=${latest_build_version_major_minor_matching_release_array[2]} if [ ${build_minor} -ge ${latest_build_version_major_minor_matching_patch} ] && [ ${build_patch} -ge ${latest_build_version_major_minor_matching_patch} ]; then # building a release for a new or updated patch version of the current or a new minor version - tag_update_flag=2 + tag_update_flag=3 elif [ ${build_minor} -lt ${latest_build_version_major_minor_matching_minor} ] && [ ${build_patch} -ge ${latest_build_version_major_minor_matching_patch} ]; then # building a patch release for an older minor version tag_update_flag=1 @@ -95,7 +95,7 @@ else fi else # building a release for a new major version - tag_update_flag=2 + tag_update_flag=3 fi fi From 31c46c3c31e9bd924821c36f9b0b6bdb79716a1d Mon Sep 17 00:00:00 2001 From: winem Date: Fri, 11 Dec 2020 21:36:28 +0100 Subject: [PATCH 16/47] fix minor comparison --- determine_needed_tags.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/determine_needed_tags.sh b/determine_needed_tags.sh index 7d7fc135..9f4fd551 100755 --- a/determine_needed_tags.sh +++ b/determine_needed_tags.sh @@ -82,7 +82,7 @@ else latest_build_version_major_minor_matching_major=${latest_build_version_major_minor_matching_release_array[0]} latest_build_version_major_minor_matching_minor=${latest_build_version_major_minor_matching_release_array[1]} latest_build_version_major_minor_matching_patch=${latest_build_version_major_minor_matching_release_array[2]} - if [ ${build_minor} -ge ${latest_build_version_major_minor_matching_patch} ] && [ ${build_patch} -ge ${latest_build_version_major_minor_matching_patch} ]; then + if [ ${build_minor} -ge ${latest_build_version_major_minor_matching_minor} ] && [ ${build_patch} -ge ${latest_build_version_major_minor_matching_patch} ]; then # building a release for a new or updated patch version of the current or a new minor version tag_update_flag=3 elif [ ${build_minor} -lt ${latest_build_version_major_minor_matching_minor} ] && [ ${build_patch} -ge ${latest_build_version_major_minor_matching_patch} ]; then From a768002f20b6c5f65a53e29eac42b00f44988322 Mon Sep 17 00:00:00 2001 From: winem Date: Fri, 11 Dec 2020 21:42:55 +0100 Subject: [PATCH 17/47] explain the logic of the script in additional comments --- determine_needed_tags.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/determine_needed_tags.sh b/determine_needed_tags.sh index 9f4fd551..394bcdd7 100755 --- a/determine_needed_tags.sh +++ b/determine_needed_tags.sh @@ -63,9 +63,13 @@ latest_minor=${latest_release_array[1]} tag_update_flag=0 # possible values of the tag_update flag: # 0 = no additional tags to be set +# (this applies just in case of builds for older releases while a newer minor or minor.patch is already available) # 1 = add the major.minor tag +# (this applies, if the build is for is just for a new patch version) # 2 = add the tags major and major.minor -# 3 = add the tags latest in addition to the tags from #2 +# (this applies if the build is for a version equal to or greater than the latest minor of a major) +# 3 = add the tags latest in addition to the tags from #2 +# (this applies if the build with the major, minor and patch version being equal to or greater than the currently latest version) if [ ${build_version} == ${latest_release} ]; then # building a release of the latest st2 version From 94f3b7a4979ea10ea0947c2c5cf5332e1f0a3947 Mon Sep 17 00:00:00 2001 From: winem Date: Fri, 11 Dec 2020 23:49:02 +0100 Subject: [PATCH 18/47] add comments to explain the tagging logic and fix the logic --- determine_needed_tags.sh | 62 +++++++++++++++++++++++++++++----------- 1 file changed, 46 insertions(+), 16 deletions(-) diff --git a/determine_needed_tags.sh b/determine_needed_tags.sh index 394bcdd7..2a703cc7 100755 --- a/determine_needed_tags.sh +++ b/determine_needed_tags.sh @@ -48,6 +48,31 @@ if [ -z ${build_patch} ]; then build_patch=0 fi +tag_update_flag=0 +# possible values of the tag_update flag: +# 0 = no additional tags to be set +# (this applies just in case of builds for older releases while a newer minor or minor.patch is already available) +# 1 = add the major.minor tag +# (this applies, if the build is for is just for a new patch version) +# 2 = add the tags major and major.minor +# (this applies if the build is for a version equal to or greater than the latest minor of a major) +# 3 = add the tags latest in addition to the tags from #2 +# (this applies if the build with the major, minor and patch version being equal to or greater than the currently latest version +# or if the build is for a completely new st2 component) + +# check if there are already images for the given component available at dockerhub +dockerhub_registry_status_code=$(curl -sfL -o /dev/null -w "%{http_code}" https://registry.hub.docker.com/v1/repositories/stackstorm/${component}/tags) + +if [ ${dockerhub_registry_status_code} -eq 404 ]; then + # there is no repository stackstorm/${component} available at dockerhub -> this build is for a new st2 component + tag_update_flag=3 + echo $tag_update_flag + exit +elif [ ${dockerhub_registry_status_code} -ne 200 ]; then + echo "Unexpected HTTP statuscode for https://registry.hub.docker.com/v1/repositories/stackstorm/${component}/tags: ${dockerhub_registry_status_code}" + exit 1 +fi + # dockerhub lists the tags in ascending order. 1st object = lowest tag; last object = highest tag or latest docker_tags_json=$(curl -s https://registry.hub.docker.com/v1/repositories/stackstorm/${component}/tags) readarray -t available_releases < <(echo $docker_tags_json | jq -r '.[] | select(.name | endswith("dev") | not).name' | sort) @@ -60,16 +85,6 @@ latest_release_array=(${latest_release//\./ }) latest_major=${latest_release_array[0]} latest_minor=${latest_release_array[1]} -tag_update_flag=0 -# possible values of the tag_update flag: -# 0 = no additional tags to be set -# (this applies just in case of builds for older releases while a newer minor or minor.patch is already available) -# 1 = add the major.minor tag -# (this applies, if the build is for is just for a new patch version) -# 2 = add the tags major and major.minor -# (this applies if the build is for a version equal to or greater than the latest minor of a major) -# 3 = add the tags latest in addition to the tags from #2 -# (this applies if the build with the major, minor and patch version being equal to or greater than the currently latest version) if [ ${build_version} == ${latest_release} ]; then # building a release of the latest st2 version @@ -86,16 +101,31 @@ else latest_build_version_major_minor_matching_major=${latest_build_version_major_minor_matching_release_array[0]} latest_build_version_major_minor_matching_minor=${latest_build_version_major_minor_matching_release_array[1]} latest_build_version_major_minor_matching_patch=${latest_build_version_major_minor_matching_release_array[2]} - if [ ${build_minor} -ge ${latest_build_version_major_minor_matching_minor} ] && [ ${build_patch} -ge ${latest_build_version_major_minor_matching_patch} ]; then - # building a release for a new or updated patch version of the current or a new minor version + if [ ${build_minor} -eq ${latest_minor} ] && \ + [ ${build_minor} -eq ${latest_build_version_major_minor_matching_minor} ] && \ + [ ${build_patch} -ge ${latest_build_version_major_minor_matching_patch} ]; then + # building a release for a new or updated patch version of the current major.minor version tag_update_flag=3 - elif [ ${build_minor} -lt ${latest_build_version_major_minor_matching_minor} ] && [ ${build_patch} -ge ${latest_build_version_major_minor_matching_patch} ]; then + elif [ ${build_minor} -eq ${latest_build_version_major_minor_matching_minor} ] && \ + [ ${build_patch} -ge ${latest_build_version_major_minor_matching_patch} ]; then + # building a release for a new or updated patch version of an old major.minor version + tag_update_flag=1 + elif [ ${build_minor} -lt ${latest_minor} ] && [ ${build_patch} -ge ${latest_build_version_major_minor_matching_patch} ]; then # building a patch release for an older minor version tag_update_flag=1 fi - else - # building a release for an old, unreleased major version of st2 - tag_update_flag=2 + elif [ ${build_minor} -gt ${latest_minor} ]; then + # building a release of a new minor version of the current or an old major version + if [ ${build_major} -eq ${latest_major} ]; then + # building a release of a new minor for the latest major version + tag_update_flag=3 + else + # building a release for a new minor of an old major version + tag_update_flag=2 + fi + #else + # # building a release for an old, unreleased major version of st2 + # tag_update_flag=2 fi else # building a release for a new major version From e8eee1efc8e1678c69134116f6b057e46c96cbae Mon Sep 17 00:00:00 2001 From: winem Date: Sat, 12 Dec 2020 22:26:55 +0100 Subject: [PATCH 19/47] Fix copy paste failure --- determine_needed_tags.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/determine_needed_tags.sh b/determine_needed_tags.sh index 2a703cc7..01bf3658 100755 --- a/determine_needed_tags.sh +++ b/determine_needed_tags.sh @@ -109,7 +109,7 @@ else elif [ ${build_minor} -eq ${latest_build_version_major_minor_matching_minor} ] && \ [ ${build_patch} -ge ${latest_build_version_major_minor_matching_patch} ]; then # building a release for a new or updated patch version of an old major.minor version - tag_update_flag=1 + tag_update_flag=2 elif [ ${build_minor} -lt ${latest_minor} ] && [ ${build_patch} -ge ${latest_build_version_major_minor_matching_patch} ]; then # building a patch release for an older minor version tag_update_flag=1 From 26bd208b947a4cae191f0d1c34957555a39ae380 Mon Sep 17 00:00:00 2001 From: winem Date: Sat, 12 Dec 2020 23:26:38 +0100 Subject: [PATCH 20/47] Add logic to tag the latest images and push the images with all applicable tags --- Makefile | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index f7facc8b..27f5ccdb 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -ST2_VERSION ?= 3.4dev +ST2_VERSION ?= 3.2.0 DOCKER_TAG ?= ${ST2_VERSION} RELEASE_TAG_REGEX := [^dev]$$ SHELL := /bin/bash @@ -49,6 +49,13 @@ else ifeq ($(TAG_UPDATE_FLAG), 2) docker tag stackstorm/$$image:${DOCKER_TAG} stackstorm/$$image:${MAJOR}.${MINOR}; \ echo -e "\033[32mSuccessfully tagged \033[1mstackstorm/$$image:${DOCKER_TAG}\033[0m\033[32m with \033[1mstackstorm/$$image:${MAJOR}\033[0m\033[32m and \033[1mstackstorm/$$image:${MAJOR}.${MINOR}\033[0m"; \ done +else ifeq ($(TAG_UPDATE_FLAG), 3) + for image in st2 st2*; do \ + docker tag stackstorm/$$image:${DOCKER_TAG} stackstorm/$$image:${MAJOR}; \ + docker tag stackstorm/$$image:${DOCKER_TAG} stackstorm/$$image:${MAJOR}.${MINOR}; \ + docker tag stackstorm/$$image:${DOCKER_TAG} stackstorm/$$image:latest; \ + echo -e "\033[32mSuccessfully tagged \033[1mstackstorm/$$image:${DOCKER_TAG}\033[0m\033[32m with \033[1mstackstorm/$$image:${MAJOR}\033[0m\033[32m, \033[1mstackstorm/$$image:${MAJOR}.${MINOR}\033[32m and \033[1mstackstorm/$$image:latest\033[0m"; \ + done endif endif @@ -62,10 +69,26 @@ push: echo -e "\033[32mSuccessfully pushed \033[1mstackstorm/$$component:${DOCKER_TAG}\033[0m\033[32m Docker image for StackStorm version \033[1m${ST2_VERSION}\033[0m"; \ done ifeq ($(RELEASE_VERSION), true) +ifeq ($(TAG_UPDATE_FLAG), 1) + for image in st2 st2*; do \ + docker push stackstorm/$$image:${MAJOR}.${MINOR}; \ + echo -e "\033[32mSuccessfully pushed \033[1mstackstorm/$$image:${MAJOR}.${MINOR}\033[0m\033[32m Docker image for StackStorm version \033[1m${ST2_VERSION}\033[0m"; \ + done +else ifeq ($(TAG_UPDATE_FLAG), 2) + for image in st2 st2*; do \ + docker push stackstorm/$$image:${MAJOR}; \ + echo -e "\033[32mSuccessfully pushed \033[1mstackstorm/$$image:${MAJOR}\033[0m\033[32m Docker image for StackStorm version \033[1m${ST2_VERSION}\033[0m"; \ + docker push stackstorm/$$image:${MAJOR}.${MINOR}; \ + echo -e "\033[32mSuccessfully pushed \033[1mstackstorm/$$image:${MAJOR}.${MINOR}\033[0m\033[32m Docker image for StackStorm version \033[1m${ST2_VERSION}\033[0m"; \ + done +else ifeq ($(TAG_UPDATE_FLAG), 3) for image in st2 st2*; do \ docker push stackstorm/$$image:${MAJOR}; \ echo -e "\033[32mSuccessfully pushed \033[1mstackstorm/$$image:${MAJOR}\033[0m\033[32m Docker image for StackStorm version \033[1m${ST2_VERSION}\033[0m"; \ docker push stackstorm/$$image:${MAJOR}.${MINOR}; \ echo -e "\033[32mSuccessfully pushed \033[1mstackstorm/$$image:${MAJOR}.${MINOR}\033[0m\033[32m Docker image for StackStorm version \033[1m${ST2_VERSION}\033[0m"; \ + docker push stackstorm/$$image:latest; \ + echo -e "\033[32mSuccessfully pushed \033[1mstackstorm/$$image:latest\033[0m\033[32m Docker image for StackStorm version \033[1m${ST2_VERSION}\033[0m"; \ done endif +endif From 5271e568f5c2cab7028a3b34501dd04248bdae5d Mon Sep 17 00:00:00 2001 From: winem Date: Sun, 13 Dec 2020 00:27:45 +0100 Subject: [PATCH 21/47] Add logic to distinguish between new patch releases or the latest and older minor versions of old releases --- determine_needed_tags.sh | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/determine_needed_tags.sh b/determine_needed_tags.sh index 01bf3658..44a97f14 100755 --- a/determine_needed_tags.sh +++ b/determine_needed_tags.sh @@ -109,7 +109,17 @@ else elif [ ${build_minor} -eq ${latest_build_version_major_minor_matching_minor} ] && \ [ ${build_patch} -ge ${latest_build_version_major_minor_matching_patch} ]; then # building a release for a new or updated patch version of an old major.minor version - tag_update_flag=2 + readarray -t build_version_major_matching_releases < <(echo $docker_tags_json | jq -r '.[] | select(.name | endswith("dev") | not) | select(.name | startswith("'"${build_major}"'")).name') + latest_build_version_major_matching_release=${build_version_major_matching_releases[-1]} + latest_build_version_major_matching_release_array=(${latest_build_version_major_matching_release//\./ }) + latest_build_version_major_matching_minor=${latest_build_version_major_matching_release_array[1]} + if [ ${build_minor} -ge ${latest_build_version_major_matching_minor} ]; then + # building a release for a new or updated patch version of the latest or a new minor version of the major release + tag_update_flag=2 + else + # building a release for an older minor version of the major release + tag_update_flag=1 + fi elif [ ${build_minor} -lt ${latest_minor} ] && [ ${build_patch} -ge ${latest_build_version_major_minor_matching_patch} ]; then # building a patch release for an older minor version tag_update_flag=1 From 031a4588e687662217c254b95a83af96f3ea6941 Mon Sep 17 00:00:00 2001 From: winem Date: Tue, 15 Dec 2020 14:44:49 +0100 Subject: [PATCH 22/47] Reset Makefile to 3.4dev --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 27f5ccdb..ab804319 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -ST2_VERSION ?= 3.2.0 +ST2_VERSION ?= 3.4dev DOCKER_TAG ?= ${ST2_VERSION} RELEASE_TAG_REGEX := [^dev]$$ SHELL := /bin/bash From 5f83010cac2061268da35f2628e1dcf57a8c050b Mon Sep 17 00:00:00 2001 From: winem Date: Tue, 9 Feb 2021 21:31:43 +0100 Subject: [PATCH 23/47] Update documentation and comments to reflect that the latest tag is also supported --- Makefile | 4 ++-- determine_needed_tags.sh | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index ab804319..5160803d 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -ST2_VERSION ?= 3.4dev +ST2_VERSION ?= 3.3.0 DOCKER_TAG ?= ${ST2_VERSION} RELEASE_TAG_REGEX := [^dev]$$ SHELL := /bin/bash @@ -8,7 +8,7 @@ TAG_UPDATE_FLAG = $(shell ./determine_needed_tags.sh st2 ${ST2_VERSION}) # 0 = no additional tags to be set # 1 = add the major.minor tag # 2 = add the tags major and major.minor -# 3 = add the latest tag (for future use) +# 3 = add the tags major, major.minor and latest ifneq ($(shell echo ${ST2_VERSION} | grep -E "${RELEASE_TAG_REGEX}"), ) RELEASE_VERSION = true diff --git a/determine_needed_tags.sh b/determine_needed_tags.sh index 44a97f14..e10889c1 100755 --- a/determine_needed_tags.sh +++ b/determine_needed_tags.sh @@ -17,6 +17,7 @@ showHelp() { echo "0 = no additional tags to be updated" echo "1 = update the major.minor tag (i.e. 3.3)" echo "2 = update the major and the major.minor tag (i.e. 3 and 3.3)" + echo "3 = update the major and the major.minor tag (i.e. 3 and 3.3) as well as the latest tag" echo echo "Run $0 -h|--help to show this usage information." } From 300f97f6dae8dac89b814af6ee1db7a076e5d611 Mon Sep 17 00:00:00 2001 From: armab Date: Thu, 11 Feb 2021 14:37:59 +0000 Subject: [PATCH 24/47] Test v3.2.0 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 5160803d..bfba6cc0 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -ST2_VERSION ?= 3.3.0 +ST2_VERSION ?= 3.2.0 DOCKER_TAG ?= ${ST2_VERSION} RELEASE_TAG_REGEX := [^dev]$$ SHELL := /bin/bash From 6d9265ec8f03a83b8aef6c02cca972f26b3d2ac9 Mon Sep 17 00:00:00 2001 From: armab Date: Thu, 11 Feb 2021 14:47:13 +0000 Subject: [PATCH 25/47] Push v3.3.0 and latest --- .circleci/config.yml | 8 ++++---- Makefile | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0e2915b7..9f8a85f6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -27,12 +27,12 @@ jobs: name: Deploy image to Docker Hub shell: /bin/bash -eo pipefail command: | - if [[ "${CIRCLE_BRANCH}" == "master" ]] || [[ "${CIRCLE_BRANCH}" =~ ^v[0-9]+\.[0-9]+$ ]]; then + #if [[ "${CIRCLE_BRANCH}" == "master" ]] || [[ "${CIRCLE_BRANCH}" =~ ^v[0-9]+\.[0-9]+$ ]]; then docker login --username ${DOCKER_USERNAME} --password ${DOCKER_PASSWORD} make push - else - echo "Skipping Docker push for '${CIRCLE_BRANCH}' branch..." - fi + #else + # echo "Skipping Docker push for '${CIRCLE_BRANCH}' branch..." + #fi workflows: version: 2 diff --git a/Makefile b/Makefile index bfba6cc0..5160803d 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -ST2_VERSION ?= 3.2.0 +ST2_VERSION ?= 3.3.0 DOCKER_TAG ?= ${ST2_VERSION} RELEASE_TAG_REGEX := [^dev]$$ SHELL := /bin/bash From 387bf4074899642ed9259d3c94bf691aaf515e77 Mon Sep 17 00:00:00 2001 From: armab Date: Thu, 11 Feb 2021 14:53:41 +0000 Subject: [PATCH 26/47] Push v3.3.0 and latest --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9f8a85f6..340fa351 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -28,8 +28,8 @@ jobs: shell: /bin/bash -eo pipefail command: | #if [[ "${CIRCLE_BRANCH}" == "master" ]] || [[ "${CIRCLE_BRANCH}" =~ ^v[0-9]+\.[0-9]+$ ]]; then - docker login --username ${DOCKER_USERNAME} --password ${DOCKER_PASSWORD} - make push + docker login --username ${DOCKER_USERNAME} --password ${DOCKER_PASSWORD} + make push #else # echo "Skipping Docker push for '${CIRCLE_BRANCH}' branch..." #fi From 55e4a220633212c14b82020ef27c06447c97e140 Mon Sep 17 00:00:00 2001 From: armab Date: Thu, 11 Feb 2021 16:17:56 +0000 Subject: [PATCH 27/47] Remove shell setting for deploy run --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 340fa351..84527306 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -25,7 +25,7 @@ jobs: command: docker images - run: name: Deploy image to Docker Hub - shell: /bin/bash -eo pipefail + #shell: /bin/bash -eo pipefail command: | #if [[ "${CIRCLE_BRANCH}" == "master" ]] || [[ "${CIRCLE_BRANCH}" =~ ^v[0-9]+\.[0-9]+$ ]]; then docker login --username ${DOCKER_USERNAME} --password ${DOCKER_PASSWORD} From e58e5c77e68debf31f93e8d9a9532ae708279168 Mon Sep 17 00:00:00 2001 From: armab Date: Thu, 11 Feb 2021 16:33:56 +0000 Subject: [PATCH 28/47] Fix Makefile variables and 'TAG_UPDATE_FLAG' shell executed several times --- Makefile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 5160803d..413a6cb9 100644 --- a/Makefile +++ b/Makefile @@ -3,20 +3,20 @@ DOCKER_TAG ?= ${ST2_VERSION} RELEASE_TAG_REGEX := [^dev]$$ SHELL := /bin/bash -TAG_UPDATE_FLAG = $(shell ./determine_needed_tags.sh st2 ${ST2_VERSION}) # supported values of the TAG_UPDATE_FLAG # 0 = no additional tags to be set # 1 = add the major.minor tag # 2 = add the tags major and major.minor # 3 = add the tags major, major.minor and latest +TAG_UPDATE_FLAG := $(shell ./determine_needed_tags.sh st2 ${ST2_VERSION}) ifneq ($(shell echo ${ST2_VERSION} | grep -E "${RELEASE_TAG_REGEX}"), ) - RELEASE_VERSION = true - MAJOR = $(word 1, $(subst ., ,${ST2_VERSION})) - MINOR = $(word 2, $(subst ., ,${ST2_VERSION})) - PATCH = $(word 3, $(subst ., ,${ST2_VERSION})) + RELEASE_VERSION := true + MAJOR := $(word 1, $(subst ., ,${ST2_VERSION})) + MINOR := $(word 2, $(subst ., ,${ST2_VERSION})) + PATCH := $(word 3, $(subst ., ,${ST2_VERSION})) else - RELEASE_VERSION = false + RELEASE_VERSION := false endif # Build all required images (st2 base image plus st2 components) From 5657921cf14d88cef6f576bd68c3454695d29110 Mon Sep 17 00:00:00 2001 From: armab Date: Thu, 11 Feb 2021 16:51:56 +0000 Subject: [PATCH 29/47] Test 3.2.0 --- .circleci/config.yml | 13 ++++++------- Makefile | 2 +- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 84527306..81c75a73 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -25,14 +25,13 @@ jobs: command: docker images - run: name: Deploy image to Docker Hub - #shell: /bin/bash -eo pipefail command: | - #if [[ "${CIRCLE_BRANCH}" == "master" ]] || [[ "${CIRCLE_BRANCH}" =~ ^v[0-9]+\.[0-9]+$ ]]; then - docker login --username ${DOCKER_USERNAME} --password ${DOCKER_PASSWORD} - make push - #else - # echo "Skipping Docker push for '${CIRCLE_BRANCH}' branch..." - #fi + if [[ "${CIRCLE_BRANCH}" == "master" ]] || [[ "${CIRCLE_BRANCH}" =~ ^v[0-9]+\.[0-9]+$ ]]; then + docker login --username ${DOCKER_USERNAME} --password ${DOCKER_PASSWORD} + make push + else + echo "Skipping Docker push for '${CIRCLE_BRANCH}' branch..." + fi workflows: version: 2 diff --git a/Makefile b/Makefile index 413a6cb9..652b8385 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -ST2_VERSION ?= 3.3.0 +ST2_VERSION ?= 3.2.0 DOCKER_TAG ?= ${ST2_VERSION} RELEASE_TAG_REGEX := [^dev]$$ SHELL := /bin/bash From 4e15921337c4cf706995e267eba58e2ce450d36b Mon Sep 17 00:00:00 2001 From: armab Date: Thu, 11 Feb 2021 16:58:07 +0000 Subject: [PATCH 30/47] Revert bash for Deploy stage --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 81c75a73..0e2915b7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -25,6 +25,7 @@ jobs: command: docker images - run: name: Deploy image to Docker Hub + shell: /bin/bash -eo pipefail command: | if [[ "${CIRCLE_BRANCH}" == "master" ]] || [[ "${CIRCLE_BRANCH}" =~ ^v[0-9]+\.[0-9]+$ ]]; then docker login --username ${DOCKER_USERNAME} --password ${DOCKER_PASSWORD} From 516d2a79040b2eb4a80863ea9ef91a6e10efe078 Mon Sep 17 00:00:00 2001 From: winem Date: Thu, 11 Feb 2021 21:10:06 +0100 Subject: [PATCH 31/47] Don't return latest as element in the array of available releases --- determine_needed_tags.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/determine_needed_tags.sh b/determine_needed_tags.sh index e10889c1..84ee744c 100755 --- a/determine_needed_tags.sh +++ b/determine_needed_tags.sh @@ -76,7 +76,7 @@ fi # dockerhub lists the tags in ascending order. 1st object = lowest tag; last object = highest tag or latest docker_tags_json=$(curl -s https://registry.hub.docker.com/v1/repositories/stackstorm/${component}/tags) -readarray -t available_releases < <(echo $docker_tags_json | jq -r '.[] | select(.name | endswith("dev") | not).name' | sort) +readarray -t available_releases < <(echo $docker_tags_json | jq -r '.[] | select((.name | endswith("dev") | not) and (.name=="latest" | not)).name' | sort) # sort returns the list with the highest tag or "latest" as last item # so change the value below to -2 when introducing the tag latest to get i.e. 3.3.0 instead of latest From 8a9b6664c6af0a9512943dd01d667e990dab5486 Mon Sep 17 00:00:00 2001 From: winem Date: Thu, 11 Feb 2021 21:34:24 +0100 Subject: [PATCH 32/47] Don't set the update flag to 3 if parsing of the available releases fails --- determine_needed_tags.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/determine_needed_tags.sh b/determine_needed_tags.sh index 84ee744c..e82f7efc 100755 --- a/determine_needed_tags.sh +++ b/determine_needed_tags.sh @@ -138,7 +138,7 @@ else # # building a release for an old, unreleased major version of st2 # tag_update_flag=2 fi - else + elif [ ${build_major} -gt ${latest_major} ]; then # building a release for a new major version tag_update_flag=3 fi From beaed9e9cd4f9db6ac0326d1c65a371ef6ca2624 Mon Sep 17 00:00:00 2001 From: winem Date: Sat, 13 Feb 2021 23:44:19 +0100 Subject: [PATCH 33/47] Add additional error handling and validate variables before referring to them --- determine_needed_tags.sh | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/determine_needed_tags.sh b/determine_needed_tags.sh index e82f7efc..5dc7053c 100755 --- a/determine_needed_tags.sh +++ b/determine_needed_tags.sh @@ -36,6 +36,19 @@ if [ "$#" -ne 2 ]; then exit 1 fi +# check for dependencies +missing_package="" +for dep in curl jq; do + if ! which $dep > /dev/null; then + missing_package+=" $dep" + fi +done + +if [ ! -z ${missing_package} ]; then + echo "Requirement(s) not satisfied: ${missing_package}" + exit 1 +fi + if [[ ${build_version} =~ ^([0-9]+)\.([0-9]+).?([0-9]*)$ ]]; then build_major=${BASH_REMATCH[1]} build_minor=${BASH_REMATCH[2]} @@ -67,7 +80,6 @@ dockerhub_registry_status_code=$(curl -sfL -o /dev/null -w "%{http_code}" https: if [ ${dockerhub_registry_status_code} -eq 404 ]; then # there is no repository stackstorm/${component} available at dockerhub -> this build is for a new st2 component tag_update_flag=3 - echo $tag_update_flag exit elif [ ${dockerhub_registry_status_code} -ne 200 ]; then echo "Unexpected HTTP statuscode for https://registry.hub.docker.com/v1/repositories/stackstorm/${component}/tags: ${dockerhub_registry_status_code}" @@ -86,6 +98,10 @@ latest_release_array=(${latest_release//\./ }) latest_major=${latest_release_array[0]} latest_minor=${latest_release_array[1]} +# validate the value stored as latest_release +if [[ ! ${latest_release} =~ ^([0-9]+)\.([0-9]+).?([0-9]*)$ ]]; then + echo "Unexpected error. The latest release ${latest_release} does not match the expected format." +fi if [ ${build_version} == ${latest_release} ]; then # building a release of the latest st2 version From fbf9dae7073f31bc78e5c4e26b44191474ff20fc Mon Sep 17 00:00:00 2001 From: winem Date: Mon, 15 Feb 2021 22:11:17 +0100 Subject: [PATCH 34/47] Add quotes to fix variable parsing when checking if missing_packages is empty or not --- determine_needed_tags.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/determine_needed_tags.sh b/determine_needed_tags.sh index 5dc7053c..3d8c0a22 100755 --- a/determine_needed_tags.sh +++ b/determine_needed_tags.sh @@ -37,15 +37,15 @@ if [ "$#" -ne 2 ]; then fi # check for dependencies -missing_package="" +missing_packages="" for dep in curl jq; do if ! which $dep > /dev/null; then - missing_package+=" $dep" + missing_packages+=" $dep" fi done -if [ ! -z ${missing_package} ]; then - echo "Requirement(s) not satisfied: ${missing_package}" +if [ ! -z "${missing_packages}" ]; then + echo "Requirement(s) not satisfied: ${missing_packages}" exit 1 fi From 1123837d63dd01f56f5369111afb0d46bb19284c Mon Sep 17 00:00:00 2001 From: winem Date: Mon, 15 Feb 2021 22:52:49 +0100 Subject: [PATCH 35/47] Log an error from the makefile if no tags can be added due to errors on determine needed tags script --- Makefile | 4 ++++ determine_needed_tags.sh | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 652b8385..0d04b843 100644 --- a/Makefile +++ b/Makefile @@ -56,6 +56,8 @@ else ifeq ($(TAG_UPDATE_FLAG), 3) docker tag stackstorm/$$image:${DOCKER_TAG} stackstorm/$$image:latest; \ echo -e "\033[32mSuccessfully tagged \033[1mstackstorm/$$image:${DOCKER_TAG}\033[0m\033[32m with \033[1mstackstorm/$$image:${MAJOR}\033[0m\033[32m, \033[1mstackstorm/$$image:${MAJOR}.${MINOR}\033[32m and \033[1mstackstorm/$$image:latest\033[0m"; \ done +else + echo -e "\033[31mNo images were tagged due to an error when determining the correct tags: ${TAG_UPDATE_FLAG}\033[0m" endif endif @@ -90,5 +92,7 @@ else ifeq ($(TAG_UPDATE_FLAG), 3) docker push stackstorm/$$image:latest; \ echo -e "\033[32mSuccessfully pushed \033[1mstackstorm/$$image:latest\033[0m\033[32m Docker image for StackStorm version \033[1m${ST2_VERSION}\033[0m"; \ done +else + echo -e "\033[31mNo images were tagged due to an error when determining the correct tags: ${TAG_UPDATE_FLAG}\033[0m" endif endif diff --git a/determine_needed_tags.sh b/determine_needed_tags.sh index 3d8c0a22..2deaf4db 100755 --- a/determine_needed_tags.sh +++ b/determine_needed_tags.sh @@ -31,7 +31,6 @@ esac if [ "$#" -ne 2 ]; then echo "Error: Missing or unexpected number of positional arguments. Expected: 2" - echo showHelp exit 1 fi From cfcfb973080f348352c3b84264ea559e3c34686f Mon Sep 17 00:00:00 2001 From: winem Date: Tue, 16 Feb 2021 22:06:11 +0100 Subject: [PATCH 36/47] Improve the error handling on the tags scirpt and prefix all errors with Error: --- Makefile | 20 +++++++++++++++----- determine_needed_tags.sh | 8 ++++---- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 0d04b843..0e9447f0 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -ST2_VERSION ?= 3.2.0 +ST2_VERSION ?= 3.4dev DOCKER_TAG ?= ${ST2_VERSION} RELEASE_TAG_REGEX := [^dev]$$ SHELL := /bin/bash @@ -22,6 +22,13 @@ endif # Build all required images (st2 base image plus st2 components) .PHONY: build build: +ifeq ($(RELEASE_VERSION), true) +ifneq ($(shell echo "${TAG_UPDATE_FLAG}" | grep -E "Error:"),) + @echo -e "Failed to identify the tags to be set." + @echo -e "\033[31mNo images were tagged due to an error when determining the correct tags: ${TAG_UPDATE_FLAG}\033[0m" +endif +endif + @docker build \ --pull \ --no-cache \ @@ -56,13 +63,18 @@ else ifeq ($(TAG_UPDATE_FLAG), 3) docker tag stackstorm/$$image:${DOCKER_TAG} stackstorm/$$image:latest; \ echo -e "\033[32mSuccessfully tagged \033[1mstackstorm/$$image:${DOCKER_TAG}\033[0m\033[32m with \033[1mstackstorm/$$image:${MAJOR}\033[0m\033[32m, \033[1mstackstorm/$$image:${MAJOR}.${MINOR}\033[32m and \033[1mstackstorm/$$image:latest\033[0m"; \ done -else - echo -e "\033[31mNo images were tagged due to an error when determining the correct tags: ${TAG_UPDATE_FLAG}\033[0m" endif endif .PHONY: push push: +ifeq ($(RELEASE_VERSION), true) +ifneq ($(shell echo "${TAG_UPDATE_FLAG}" | grep -E "Error:"),) + @echo -e "Failed to identify the tags to be set." + @echo -e "\033[31mNo images were tagged due to an error when determining the correct tags: ${TAG_UPDATE_FLAG}\033[0m" +endif +endif + docker push stackstorm/st2:${DOCKER_TAG}; @echo -e "\033[32mSuccessfully pushed \033[1mstackstorm/st2:${DOCKER_TAG}\033[0m\033[32m Docker image for StackStorm version \033[1m${ST2_VERSION}\033[0m"; @set -e; \ @@ -92,7 +104,5 @@ else ifeq ($(TAG_UPDATE_FLAG), 3) docker push stackstorm/$$image:latest; \ echo -e "\033[32mSuccessfully pushed \033[1mstackstorm/$$image:latest\033[0m\033[32m Docker image for StackStorm version \033[1m${ST2_VERSION}\033[0m"; \ done -else - echo -e "\033[31mNo images were tagged due to an error when determining the correct tags: ${TAG_UPDATE_FLAG}\033[0m" endif endif diff --git a/determine_needed_tags.sh b/determine_needed_tags.sh index 2deaf4db..a8917389 100755 --- a/determine_needed_tags.sh +++ b/determine_needed_tags.sh @@ -44,7 +44,7 @@ for dep in curl jq; do done if [ ! -z "${missing_packages}" ]; then - echo "Requirement(s) not satisfied: ${missing_packages}" + echo "Error: Requirement(s) not satisfied: ${missing_packages}" exit 1 fi @@ -53,7 +53,7 @@ if [[ ${build_version} =~ ^([0-9]+)\.([0-9]+).?([0-9]*)$ ]]; then build_minor=${BASH_REMATCH[2]} build_patch=${BASH_REMATCH[3]} else - echo "The provided version ${build_version} does not have the expected format." + echo "Error: The provided version ${build_version} does not have the expected format." exit 1 fi @@ -81,7 +81,7 @@ if [ ${dockerhub_registry_status_code} -eq 404 ]; then tag_update_flag=3 exit elif [ ${dockerhub_registry_status_code} -ne 200 ]; then - echo "Unexpected HTTP statuscode for https://registry.hub.docker.com/v1/repositories/stackstorm/${component}/tags: ${dockerhub_registry_status_code}" + echo "Error: Unexpected HTTP statuscode for https://registry.hub.docker.com/v1/repositories/stackstorm/${component}/tags: ${dockerhub_registry_status_code}" exit 1 fi @@ -99,7 +99,7 @@ latest_minor=${latest_release_array[1]} # validate the value stored as latest_release if [[ ! ${latest_release} =~ ^([0-9]+)\.([0-9]+).?([0-9]*)$ ]]; then - echo "Unexpected error. The latest release ${latest_release} does not match the expected format." + echo "Error: Unexpected error. The latest release ${latest_release} does not match the expected format." fi if [ ${build_version} == ${latest_release} ]; then From da156e95c878e6c9537189f8bdede90f2d41f6d2 Mon Sep 17 00:00:00 2001 From: winem Date: Tue, 16 Feb 2021 22:29:32 +0100 Subject: [PATCH 37/47] Commit to trigger a CI run for 3.3.0 (an actual release) --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 0e9447f0..df7b55ad 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -ST2_VERSION ?= 3.4dev +ST2_VERSION ?= 3.3.0 DOCKER_TAG ?= ${ST2_VERSION} RELEASE_TAG_REGEX := [^dev]$$ SHELL := /bin/bash From 570a3a821343e64d1b901faf30e4740cc8fda1a6 Mon Sep 17 00:00:00 2001 From: winem Date: Tue, 16 Feb 2021 22:53:05 +0100 Subject: [PATCH 38/47] Fail the make if determining tags fails --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index df7b55ad..255c9fef 100644 --- a/Makefile +++ b/Makefile @@ -26,6 +26,7 @@ ifeq ($(RELEASE_VERSION), true) ifneq ($(shell echo "${TAG_UPDATE_FLAG}" | grep -E "Error:"),) @echo -e "Failed to identify the tags to be set." @echo -e "\033[31mNo images were tagged due to an error when determining the correct tags: ${TAG_UPDATE_FLAG}\033[0m" + exit 1 endif endif @@ -72,6 +73,7 @@ ifeq ($(RELEASE_VERSION), true) ifneq ($(shell echo "${TAG_UPDATE_FLAG}" | grep -E "Error:"),) @echo -e "Failed to identify the tags to be set." @echo -e "\033[31mNo images were tagged due to an error when determining the correct tags: ${TAG_UPDATE_FLAG}\033[0m" + exit 1 endif endif From 5992a5e4cf2ee805853ce07b91157242ad81588b Mon Sep 17 00:00:00 2001 From: winem Date: Tue, 16 Feb 2021 22:55:50 +0100 Subject: [PATCH 39/47] Add curl and jq on the circleci node --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0e2915b7..364b3a98 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -16,7 +16,7 @@ jobs: docker_layer_caching: false - run: name: Install Dependencies - command: apk add make bash + command: apk add make bash curl jq - run: name: Build Docker images command: make build From bf6adac35bec19a8d4b66182a75a927632b30609 Mon Sep 17 00:00:00 2001 From: winem Date: Tue, 16 Feb 2021 23:01:47 +0100 Subject: [PATCH 40/47] Try to fix the curl relocating error --- .circleci/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 364b3a98..1b07cbb1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -17,6 +17,9 @@ jobs: - run: name: Install Dependencies command: apk add make bash curl jq + - run: + name: Update curl + command: apk add -U curl - run: name: Build Docker images command: make build From 49b20b6fba04b4ef35824c60935c4e916e8f2efb Mon Sep 17 00:00:00 2001 From: winem Date: Tue, 16 Feb 2021 23:03:35 +0100 Subject: [PATCH 41/47] Still trying to fix the curl relocating error --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1b07cbb1..c75d729a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -19,7 +19,7 @@ jobs: command: apk add make bash curl jq - run: name: Update curl - command: apk add -U curl + command: apk --upgrade --no-cache add curl - run: name: Build Docker images command: make build From 7c7574c55d3d03d60093ee75c1585c246e9a42da Mon Sep 17 00:00:00 2001 From: winem Date: Tue, 16 Feb 2021 23:10:05 +0100 Subject: [PATCH 42/47] Change the Makefile to 3.4dev again --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 255c9fef..ac000495 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -ST2_VERSION ?= 3.3.0 +ST2_VERSION ?= 3.4dev DOCKER_TAG ?= ${ST2_VERSION} RELEASE_TAG_REGEX := [^dev]$$ SHELL := /bin/bash From 4063f57558e867bc0af40abeb228374c312b4d21 Mon Sep 17 00:00:00 2001 From: winem Date: Tue, 16 Feb 2021 23:56:09 +0100 Subject: [PATCH 43/47] Make 3.3.0 again for a first test with the duplicate code being moved into a dedicated target --- Makefile | 31 ++++++++++++------------------- 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/Makefile b/Makefile index ac000495..7faf89c7 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -ST2_VERSION ?= 3.4dev +ST2_VERSION ?= 3.3.0 DOCKER_TAG ?= ${ST2_VERSION} RELEASE_TAG_REGEX := [^dev]$$ SHELL := /bin/bash @@ -21,15 +21,7 @@ endif # Build all required images (st2 base image plus st2 components) .PHONY: build -build: -ifeq ($(RELEASE_VERSION), true) -ifneq ($(shell echo "${TAG_UPDATE_FLAG}" | grep -E "Error:"),) - @echo -e "Failed to identify the tags to be set." - @echo -e "\033[31mNo images were tagged due to an error when determining the correct tags: ${TAG_UPDATE_FLAG}\033[0m" - exit 1 -endif -endif - +build: verify_tag_update_flag @docker build \ --pull \ --no-cache \ @@ -68,15 +60,7 @@ endif endif .PHONY: push -push: -ifeq ($(RELEASE_VERSION), true) -ifneq ($(shell echo "${TAG_UPDATE_FLAG}" | grep -E "Error:"),) - @echo -e "Failed to identify the tags to be set." - @echo -e "\033[31mNo images were tagged due to an error when determining the correct tags: ${TAG_UPDATE_FLAG}\033[0m" - exit 1 -endif -endif - +push: verify_tag_update_flag docker push stackstorm/st2:${DOCKER_TAG}; @echo -e "\033[32mSuccessfully pushed \033[1mstackstorm/st2:${DOCKER_TAG}\033[0m\033[32m Docker image for StackStorm version \033[1m${ST2_VERSION}\033[0m"; @set -e; \ @@ -108,3 +92,12 @@ else ifeq ($(TAG_UPDATE_FLAG), 3) done endif endif + +verify_tag_update_flag: +ifeq ($(RELEASE_VERSION), true) +ifneq ($(shell echo "${TAG_UPDATE_FLAG}" | grep -E "Error:"),) + @echo -e "Failed to identify the tags to be set." + @echo -e "\033[31mNo images were tagged due to an error when determining the correct tags: ${TAG_UPDATE_FLAG}\033[0m" + exit 1 +endif +endif \ No newline at end of file From f4dd7056a79610155f3939b2f11a8cbe91efc0ef Mon Sep 17 00:00:00 2001 From: winem Date: Wed, 17 Feb 2021 00:02:49 +0100 Subject: [PATCH 44/47] Build for 3.4dev again --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 7faf89c7..e0a2589d 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -ST2_VERSION ?= 3.3.0 +ST2_VERSION ?= 3.4dev DOCKER_TAG ?= ${ST2_VERSION} RELEASE_TAG_REGEX := [^dev]$$ SHELL := /bin/bash From f69ce159f5af64f98ed763618e2738f485919bc8 Mon Sep 17 00:00:00 2001 From: armab Date: Wed, 17 Feb 2021 20:25:56 +0000 Subject: [PATCH 45/47] Test st2 3.2.1 --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index e0a2589d..34142d1e 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -ST2_VERSION ?= 3.4dev +ST2_VERSION ?= 3.2.1 DOCKER_TAG ?= ${ST2_VERSION} RELEASE_TAG_REGEX := [^dev]$$ SHELL := /bin/bash @@ -100,4 +100,4 @@ ifneq ($(shell echo "${TAG_UPDATE_FLAG}" | grep -E "Error:"),) @echo -e "\033[31mNo images were tagged due to an error when determining the correct tags: ${TAG_UPDATE_FLAG}\033[0m" exit 1 endif -endif \ No newline at end of file +endif From 3567e4f2f9dc35620a7f0aecc94fafd9be5678c7 Mon Sep 17 00:00:00 2001 From: armab Date: Wed, 17 Feb 2021 20:34:05 +0000 Subject: [PATCH 46/47] Test st2 3.2.0 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 34142d1e..0674f81b 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -ST2_VERSION ?= 3.2.1 +ST2_VERSION ?= 3.2.0 DOCKER_TAG ?= ${ST2_VERSION} RELEASE_TAG_REGEX := [^dev]$$ SHELL := /bin/bash From 4fb24f9cfc543f4b753877da6531a7765d26cf9e Mon Sep 17 00:00:00 2001 From: armab Date: Wed, 17 Feb 2021 21:08:29 +0000 Subject: [PATCH 47/47] Revert to 3.4dev --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 0674f81b..5d112b13 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -ST2_VERSION ?= 3.2.0 +ST2_VERSION ?= 3.4dev DOCKER_TAG ?= ${ST2_VERSION} RELEASE_TAG_REGEX := [^dev]$$ SHELL := /bin/bash