diff --git a/.circle/docker-compose.sh b/.circle/docker-compose.sh index 9b87f978..6034c5a5 100755 --- a/.circle/docker-compose.sh +++ b/.circle/docker-compose.sh @@ -11,7 +11,7 @@ set -e # Source the build environment defintion (details in buildenv.sh) . ~/.buildenv -# Used for `RABBITMQHOST` `MONGODBHOST`, see docker-compose.override.yml +# Used for `RABBITMQHOST`, `MONGODBHOST`, and `REDISHOST` see docker-compose.override.yml HOST_IP=$(ifconfig docker0 | grep 'inet addr' | awk -F: '{print $2}' | awk '{print $1}') set -x @@ -26,6 +26,7 @@ case "$1" in -e ST2PKG_RELEASE=${ST2PKG_RELEASE} \ -e RABBITMQHOST=${HOST_IP} \ -e MONGODBHOST=${HOST_IP} \ + -e REDISHOST=${HOST_IP} \ -e ST2_CIRCLE_URL=${CIRCLE_BUILD_URL} \ $2 /bin/true ;; @@ -38,6 +39,7 @@ case "$1" in -e ST2PKG_RELEASE=${ST2PKG_RELEASE} \ -e RABBITMQHOST=${HOST_IP} \ -e MONGODBHOST=${HOST_IP} \ + -e REDISHOST=${HOST_IP} \ -e ST2_PACKAGES="${ST2_PACKAGES}" \ -e ST2_CIRCLE_URL=${CIRCLE_BUILD_URL} \ $2 build @@ -52,6 +54,7 @@ case "$1" in -e ST2PKG_RELEASE=${ST2PKG_RELEASE} \ -e RABBITMQHOST=${HOST_IP} \ -e MONGODBHOST=${HOST_IP} \ + -e REDISHOST=${HOST_IP} \ -e ST2_PACKAGES="${ST2_PACKAGES}" \ -e ST2_CIRCLE_URL=${CIRCLE_BUILD_URL} \ $2 test diff --git a/docker-compose.circle.yml b/docker-compose.circle.yml index 3f20e505..a942857c 100644 --- a/docker-compose.circle.yml +++ b/docker-compose.circle.yml @@ -24,6 +24,7 @@ bionic: - bionictest - rabbitmq - mongodb + - redis el8: image: quay.io/stackstorm/packagingrunner @@ -39,6 +40,7 @@ el8: - centos8test - rabbitmq - mongodb + - redis el7: image: quay.io/stackstorm/packagingrunner diff --git a/docker-compose.circle2.yml b/docker-compose.circle2.yml index 51977e24..47c27883 100644 --- a/docker-compose.circle2.yml +++ b/docker-compose.circle2.yml @@ -11,6 +11,7 @@ xenial: - xenialtest - rabbitmq - mongodb + - redis bionic: image: quay.io/stackstorm/packagingrunner @@ -26,6 +27,7 @@ bionic: - bionictest - rabbitmq - mongodb + - redis el8: image: quay.io/stackstorm/packagingrunner @@ -40,6 +42,7 @@ el8: - centos8test - rabbitmq - mongodb + - redis el7: image: quay.io/stackstorm/packagingrunner @@ -54,6 +57,7 @@ el7: - centos7test - rabbitmq - mongodb + - redis ## Package build nodes # diff --git a/docker-compose.override.yml b/docker-compose.override.yml index 1630c900..cb8a5e06 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -48,3 +48,8 @@ mongodb: image: mongo:4.0 ports: - "27017:27017" + +redis: + image: redis:latest + ports: + - "6379:6379" diff --git a/docker-compose.yml b/docker-compose.yml index 671e14ea..7c05469f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,6 +11,7 @@ xenial: - xenialtest - rabbitmq - mongodb + - redis bionic: image: quay.io/stackstorm/packagingrunner @@ -26,6 +27,7 @@ bionic: - bionictest - rabbitmq - mongodb + - redis el8: image: quay.io/stackstorm/packagingrunner @@ -41,7 +43,7 @@ el8: - centos8test - rabbitmq - mongodb - + - redis el7: image: quay.io/stackstorm/packagingrunner @@ -56,6 +58,7 @@ el7: - centos7test - rabbitmq - mongodb + - redis ## Package build nodes # diff --git a/rake/build/environment.rb b/rake/build/environment.rb index b9ec7600..5007731a 100644 --- a/rake/build/environment.rb +++ b/rake/build/environment.rb @@ -58,6 +58,7 @@ # Default hostnames of dependat services (the value can take an address also) envpass :rabbitmqhost, 'rabbitmq', proc: convert_to_ipaddr envpass :mongodbhost, 'mongodb', proc: convert_to_ipaddr + envpass :redishost, 'redis', proc: convert_to_ipaddr # upload_sources - a list of directories which should be propogated # to remote nodes. diff --git a/rake/spec/default/60-st2_all-services-ok_spec.rb b/rake/spec/default/60-st2_all-services-ok_spec.rb index a594a253..77ad1051 100644 --- a/rake/spec/default/60-st2_all-services-ok_spec.rb +++ b/rake/spec/default/60-st2_all-services-ok_spec.rb @@ -15,6 +15,11 @@ it { is_expected.to be_reachable.with :port => 27017, :timeout => 1 } end + describe 'redis' do + subject { host(spec[:redishost]) } + it { is_expected.to be_reachable.with :port => 6379, :timeout => 1 } + end + end describe 'start st2 components and services' do diff --git a/rake/spec/spec_helper.rb b/rake/spec/spec_helper.rb index fa4ea8c4..20c1b7aa 100644 --- a/rake/spec/spec_helper.rb +++ b/rake/spec/spec_helper.rb @@ -31,6 +31,7 @@ class ST2Spec package_list: pipeopts.packages, rabbitmqhost: pipeopts.rabbitmqhost, mongodbhost: pipeopts.mongodbhost, + redishost: pipeopts.redishost, loglines_to_show: 100, logdest_pattern: { st2actionrunner: 'st2actionrunner.{pid}' diff --git a/scripts/generate_st2_config.sh b/scripts/generate_st2_config.sh index 1147dc1f..27a12437 100644 --- a/scripts/generate_st2_config.sh +++ b/scripts/generate_st2_config.sh @@ -8,6 +8,7 @@ set -e # --- Go! MONGOHOST="${MONGODBHOST:-mongodb}" RABBITMQHOST="${RABBITMQHOST:-rabbitmq}" +REDISHOST="${REDISHOST:-redis}" CONF=/etc/st2/st2.conf AMQP="amqp://guest:guest@$RABBITMQHOST:5672/" @@ -20,6 +21,7 @@ EHD # Don't join into one cmd with previous, otherwise it becomes # non-interactive waiting ^D. MONGO=$(echo "$MONGO" | sed -r 's/^\s+//') +REDIS="redis://${REDISHOST}:6379" # Specify rabbitmq host sed -i "/\[messaging\]/,/\[.*\]\|url/ {n; s#url.*=.*#url = $AMQP#}" $CONF @@ -28,4 +30,7 @@ sed -i "/\[auth\]/,/\[.*\]\|enable/ {n; s#enable.*=.*#enable = False#}" $CONF # Create database section, st2.conf ships without it (grep "\[database\]" $CONF &>/dev/null) || echo "$MONGO" >> /etc/st2/st2.conf +# Specify redis host +sed -i "/\[coordination\]/,/\[.*\]\|url/ {n; s#url.*=.*#url = $REDIS#}" $CONF + echo "Resulting $CONF >>>" "$(cat $CONF)" diff --git a/scripts/includes/common.sh b/scripts/includes/common.sh index 24dd95e6..e5922a5c 100644 --- a/scripts/includes/common.sh +++ b/scripts/includes/common.sh @@ -83,9 +83,10 @@ check_st2_host_dependencies() { # CHECK 1: Determine which, if any, of the required ports are used by an existing process. # Abort the installation early if the following ports are being used by an existing process. - # nginx (80, 443), mongodb (27017), rabbitmq (4369, 5672, 25672), and st2 (9100-9102). + # nginx (80, 443), mongodb (27017), rabbitmq (4369, 5672, 25672), redis (6379) + # and st2 (9100-9102). - declare -a ports=("80" "443" "4369" "5672" "9100" "9101" "9102" "25672" "27017") + declare -a ports=("80" "443" "4369" "5672" "6379" "9100" "9101" "9102" "25672" "27017") declare -a used=() for i in "${ports[@]}" diff --git a/scripts/st2bootstrap-deb.sh b/scripts/st2bootstrap-deb.sh index 47177a89..8545f7bf 100644 --- a/scripts/st2bootstrap-deb.sh +++ b/scripts/st2bootstrap-deb.sh @@ -233,9 +233,10 @@ check_st2_host_dependencies() { # CHECK 1: Determine which, if any, of the required ports are used by an existing process. # Abort the installation early if the following ports are being used by an existing process. - # nginx (80, 443), mongodb (27017), rabbitmq (4369, 5672, 25672), and st2 (9100-9102). + # nginx (80, 443), mongodb (27017), rabbitmq (4369, 5672, 25672), redis (6379) + # and st2 (9100-9102). - declare -a ports=("80" "443" "4369" "5672" "9100" "9101" "9102" "25672" "27017") + declare -a ports=("80" "443" "4369" "5672" "6379" "9100" "9101" "9102" "25672" "27017") declare -a used=() for i in "${ports[@]}" @@ -548,6 +549,11 @@ EOF } +install_redis() { + # Install Redis Server. By default, redis only listen on localhost only. + sudo apt-get install -y redis-server +} + get_full_pkg_versions() { if [[ "$VERSION" != '' ]]; then @@ -609,6 +615,9 @@ install_st2() { # Configure [messaging] section in st2.conf (username password for RabbitMQ access) AMQP="amqp://stackstorm:$ST2_RABBITMQ_PASSWORD@127.0.0.1:5672" sudo crudini --set /etc/st2/st2.conf messaging url "${AMQP}" + + # Configure [coordination] section in st2.conf (url for Redis access) + sudo crudini --set /etc/st2/st2.conf coordination url "redis://127.0.0.1:6379" sudo st2ctl start sudo st2ctl reload --register-all @@ -712,6 +721,7 @@ STEP="Configure Proxy" && configure_proxy STEP="Install st2 dependencies" && install_st2_dependencies STEP="Install st2 dependencies (RabbitMQ)" && install_rabbitmq STEP="Install st2 dependencies (MongoDB)" && install_mongodb +STEP="Install st2 dependencies (Redis)" && install_redis STEP="Install st2" && install_st2 STEP="Configure st2 user" && configure_st2_user STEP="Configure st2 auth" && configure_st2_authentication diff --git a/scripts/st2bootstrap-deb.template.sh b/scripts/st2bootstrap-deb.template.sh index 1c0de357..84757acd 100644 --- a/scripts/st2bootstrap-deb.template.sh +++ b/scripts/st2bootstrap-deb.template.sh @@ -235,6 +235,11 @@ EOF } +install_redis() { + # Install Redis Server. By default, redis only listen on localhost only. + sudo apt-get install -y redis-server +} + get_full_pkg_versions() { if [[ "$VERSION" != '' ]]; then @@ -296,6 +301,9 @@ install_st2() { # Configure [messaging] section in st2.conf (username password for RabbitMQ access) AMQP="amqp://stackstorm:$ST2_RABBITMQ_PASSWORD@127.0.0.1:5672" sudo crudini --set /etc/st2/st2.conf messaging url "${AMQP}" + + # Configure [coordination] section in st2.conf (url for Redis access) + sudo crudini --set /etc/st2/st2.conf coordination url "redis://127.0.0.1:6379" sudo st2ctl start sudo st2ctl reload --register-all @@ -399,6 +407,7 @@ STEP="Configure Proxy" && configure_proxy STEP="Install st2 dependencies" && install_st2_dependencies STEP="Install st2 dependencies (RabbitMQ)" && install_rabbitmq STEP="Install st2 dependencies (MongoDB)" && install_mongodb +STEP="Install st2 dependencies (Redis)" && install_redis STEP="Install st2" && install_st2 STEP="Configure st2 user" && configure_st2_user STEP="Configure st2 auth" && configure_st2_authentication diff --git a/scripts/st2bootstrap-el7.sh b/scripts/st2bootstrap-el7.sh index 7c80567a..f5d972a0 100644 --- a/scripts/st2bootstrap-el7.sh +++ b/scripts/st2bootstrap-el7.sh @@ -194,9 +194,10 @@ check_st2_host_dependencies() { # CHECK 1: Determine which, if any, of the required ports are used by an existing process. # Abort the installation early if the following ports are being used by an existing process. - # nginx (80, 443), mongodb (27017), rabbitmq (4369, 5672, 25672), and st2 (9100-9102). + # nginx (80, 443), mongodb (27017), rabbitmq (4369, 5672, 25672), redis (6379) + # and st2 (9100-9102). - declare -a ports=("80" "443" "4369" "5672" "9100" "9101" "9102" "25672" "27017") + declare -a ports=("80" "443" "4369" "5672" "6379" "9100" "9101" "9102" "25672" "27017") declare -a used=() for i in "${ports[@]}" @@ -585,6 +586,13 @@ EOF sudo systemctl restart mongod } +install_redis() { + # Install Redis Server. By default, redis only listen on localhost only. + sudo yum install -y redis + sudo systemctl start redis + sudo systemctl enable redis +} + install_st2() { curl -sL https://packagecloud.io/install/repositories/StackStorm/${REPO_PREFIX}${RELEASE}/script.rpm.sh | sudo bash @@ -606,6 +614,9 @@ install_st2() { AMQP="amqp://stackstorm:$ST2_RABBITMQ_PASSWORD@127.0.0.1:5672" sudo crudini --set /etc/st2/st2.conf messaging url "${AMQP}" + # Configure [coordination] section in st2.conf (url for Redis access) + sudo crudini --set /etc/st2/st2.conf coordination url "redis://127.0.0.1:6379" + sudo st2ctl start sudo st2ctl reload --register-all } @@ -725,6 +736,7 @@ STEP="Generate random password" && generate_random_passwords STEP="Install st2 dependencies" && install_st2_dependencies STEP="Install st2 dependencies (RabbitMQ)" && install_rabbitmq STEP="Install st2 dependencies (MongoDB)" && install_mongodb +STEP="Install st2 dependencies (Redis)" && install_redis STEP="Install st2" && install_st2 STEP="Configure st2 user" && configure_st2_user STEP="Configure st2 auth" && configure_st2_authentication diff --git a/scripts/st2bootstrap-el7.template.sh b/scripts/st2bootstrap-el7.template.sh index 06908514..58a98ea2 100644 --- a/scripts/st2bootstrap-el7.template.sh +++ b/scripts/st2bootstrap-el7.template.sh @@ -222,6 +222,13 @@ EOF sudo systemctl restart mongod } +install_redis() { + # Install Redis Server. By default, redis only listen on localhost only. + sudo yum install -y redis + sudo systemctl start redis + sudo systemctl enable redis +} + install_st2() { curl -sL https://packagecloud.io/install/repositories/StackStorm/${REPO_PREFIX}${RELEASE}/script.rpm.sh | sudo bash @@ -243,6 +250,9 @@ install_st2() { AMQP="amqp://stackstorm:$ST2_RABBITMQ_PASSWORD@127.0.0.1:5672" sudo crudini --set /etc/st2/st2.conf messaging url "${AMQP}" + # Configure [coordination] section in st2.conf (url for Redis access) + sudo crudini --set /etc/st2/st2.conf coordination url "redis://127.0.0.1:6379" + sudo st2ctl start sudo st2ctl reload --register-all } @@ -362,6 +372,7 @@ STEP="Generate random password" && generate_random_passwords STEP="Install st2 dependencies" && install_st2_dependencies STEP="Install st2 dependencies (RabbitMQ)" && install_rabbitmq STEP="Install st2 dependencies (MongoDB)" && install_mongodb +STEP="Install st2 dependencies (Redis)" && install_redis STEP="Install st2" && install_st2 STEP="Configure st2 user" && configure_st2_user STEP="Configure st2 auth" && configure_st2_authentication diff --git a/scripts/st2bootstrap-el8.sh b/scripts/st2bootstrap-el8.sh index d25ac583..ef572725 100644 --- a/scripts/st2bootstrap-el8.sh +++ b/scripts/st2bootstrap-el8.sh @@ -198,9 +198,10 @@ check_st2_host_dependencies() { # CHECK 1: Determine which, if any, of the required ports are used by an existing process. # Abort the installation early if the following ports are being used by an existing process. - # nginx (80, 443), mongodb (27017), rabbitmq (4369, 5672, 25672), and st2 (9100-9102). + # nginx (80, 443), mongodb (27017), rabbitmq (4369, 5672, 25672), redis (6379) + # and st2 (9100-9102). - declare -a ports=("80" "443" "4369" "5672" "9100" "9101" "9102" "25672" "27017") + declare -a ports=("80" "443" "4369" "5672" "6379" "9100" "9101" "9102" "25672" "27017") declare -a used=() for i in "${ports[@]}" @@ -592,6 +593,13 @@ EOF sudo systemctl restart mongod } +install_redis() { + # Install Redis Server. By default, redis only listen on localhost only. + sudo yum install -y redis + sudo systemctl start redis + sudo systemctl enable redis +} + install_st2() { curl -sL https://packagecloud.io/install/repositories/StackStorm/${REPO_PREFIX}${RELEASE}/script.rpm.sh | sudo bash @@ -613,6 +621,9 @@ install_st2() { AMQP="amqp://stackstorm:$ST2_RABBITMQ_PASSWORD@127.0.0.1:5672" sudo crudini --set /etc/st2/st2.conf messaging url "${AMQP}" + # Configure [coordination] section in st2.conf (url for Redis access) + sudo crudini --set /etc/st2/st2.conf coordination url "redis://127.0.0.1:6379" + sudo st2ctl start sudo st2ctl reload --register-all } @@ -748,6 +759,7 @@ STEP="Generate random password" && generate_random_passwords STEP="Install st2 dependencies" && install_st2_dependencies STEP="Install st2 dependencies (RabbitMQ)" && install_rabbitmq STEP="Install st2 dependencies (MongoDB)" && install_mongodb +STEP="Install st2 dependencies (Redis)" && install_redis STEP="Install st2" && install_st2 STEP="Configure st2 user" && configure_st2_user STEP="Configure st2 auth" && configure_st2_authentication diff --git a/scripts/st2bootstrap-el8.template.sh b/scripts/st2bootstrap-el8.template.sh index e8618371..84d59f92 100644 --- a/scripts/st2bootstrap-el8.template.sh +++ b/scripts/st2bootstrap-el8.template.sh @@ -229,6 +229,13 @@ EOF sudo systemctl restart mongod } +install_redis() { + # Install Redis Server. By default, redis only listen on localhost only. + sudo yum install -y redis + sudo systemctl start redis + sudo systemctl enable redis +} + install_st2() { curl -sL https://packagecloud.io/install/repositories/StackStorm/${REPO_PREFIX}${RELEASE}/script.rpm.sh | sudo bash @@ -250,6 +257,9 @@ install_st2() { AMQP="amqp://stackstorm:$ST2_RABBITMQ_PASSWORD@127.0.0.1:5672" sudo crudini --set /etc/st2/st2.conf messaging url "${AMQP}" + # Configure [coordination] section in st2.conf (url for Redis access) + sudo crudini --set /etc/st2/st2.conf coordination url "redis://127.0.0.1:6379" + sudo st2ctl start sudo st2ctl reload --register-all } @@ -385,6 +395,7 @@ STEP="Generate random password" && generate_random_passwords STEP="Install st2 dependencies" && install_st2_dependencies STEP="Install st2 dependencies (RabbitMQ)" && install_rabbitmq STEP="Install st2 dependencies (MongoDB)" && install_mongodb +STEP="Install st2 dependencies (Redis)" && install_redis STEP="Install st2" && install_st2 STEP="Configure st2 user" && configure_st2_user STEP="Configure st2 auth" && configure_st2_authentication