From 6cc70fd6eac3b6b6521a8bde69c95f0b60c15f02 Mon Sep 17 00:00:00 2001 From: armab Date: Fri, 19 Jun 2020 18:36:05 +0100 Subject: [PATCH 01/32] Remove ALL-IN-ONE Docker deployment You can find the old deprecated version in `DEPRECATED/all-in-one` branch archive: https://github.com/StackStorm/st2-docker/tree/DEPRECATED/all-in-one Closes #22, closes #23, closes #26, closes #29, closes #34, closes #41, closes #43, closes #92, closes #112, closes #117, closes #125, closes #133, closes #141, closes #145, closes #151, closes #163, closes #187, closes #188, closes #189, closes #190 Closes #162, closes #138, closes #108, closes #102, closes #65 --- .circleci/config.yml | 135 --------- .travis.yml | 27 -- CHANGELOG.rst | 34 --- Makefile | 22 -- README.md | 259 ---------------- VERSIONING.md | 16 - bin/build.sh | 34 --- bin/common.sh | 76 ----- bin/deploy.sh | 45 --- bin/gatekeeper.sh | 21 -- bin/load.sh | 13 - bin/save.sh | 27 -- bin/write-env.sh | 55 ---- docker-compose.yml | 95 ------ docs/tutorial.md | 283 ------------------ images/stackstorm/Dockerfile | 144 --------- images/stackstorm/bin/entrypoint-1ppc.sh | 103 ------- images/stackstorm/bin/entrypoint.sh | 75 ----- images/stackstorm/bin/inject_env.py | 29 -- images/stackstorm/bin/install.sh | 66 ---- images/stackstorm/bin/st2.sh | 16 - images/stackstorm/bin/test.sh | 14 - images/stackstorm/config/init-fake.conf | 13 - images/stackstorm/config/local.conf | 9 - .../stackstorm/config/nginx.st2-1ppc.conf.tpl | 151 ---------- runtime/compose-1ppc/.gitignore | 1 - runtime/compose-1ppc/README.md | 119 -------- runtime/compose-1ppc/common.env | 5 - runtime/compose-1ppc/docker-compose.yml | 165 ---------- runtime/entrypoint.d/README.md | 10 - runtime/entrypoint.d/add-packs-dev.sh | 4 - runtime/st2.d/README.md | 9 - runtime/st2.d/reload.sh | 3 - runtime/st2.d/virtualenv.sh | 9 - tutorial/actions/hello-concurrency.yaml | 13 - tutorial/actions/hello.yaml | 13 - tutorial/actions/mistral-basic.yaml | 11 - tutorial/actions/workflows/mistral-basic.yaml | 15 - tutorial/policies/hello-concurrency.yaml | 8 - tutorial/rules/monitor_file.yaml | 17 -- 40 files changed, 2164 deletions(-) delete mode 100644 .circleci/config.yml delete mode 100644 .travis.yml delete mode 100644 CHANGELOG.rst delete mode 100644 Makefile delete mode 100644 README.md delete mode 100644 VERSIONING.md delete mode 100755 bin/build.sh delete mode 100644 bin/common.sh delete mode 100755 bin/deploy.sh delete mode 100755 bin/gatekeeper.sh delete mode 100755 bin/load.sh delete mode 100755 bin/save.sh delete mode 100755 bin/write-env.sh delete mode 100644 docker-compose.yml delete mode 100644 docs/tutorial.md delete mode 100644 images/stackstorm/Dockerfile delete mode 100755 images/stackstorm/bin/entrypoint-1ppc.sh delete mode 100755 images/stackstorm/bin/entrypoint.sh delete mode 100755 images/stackstorm/bin/inject_env.py delete mode 100755 images/stackstorm/bin/install.sh delete mode 100755 images/stackstorm/bin/st2.sh delete mode 100755 images/stackstorm/bin/test.sh delete mode 100644 images/stackstorm/config/init-fake.conf delete mode 100644 images/stackstorm/config/local.conf delete mode 100644 images/stackstorm/config/nginx.st2-1ppc.conf.tpl delete mode 100644 runtime/compose-1ppc/.gitignore delete mode 100644 runtime/compose-1ppc/README.md delete mode 100644 runtime/compose-1ppc/common.env delete mode 100644 runtime/compose-1ppc/docker-compose.yml delete mode 100644 runtime/entrypoint.d/README.md delete mode 100755 runtime/entrypoint.d/add-packs-dev.sh delete mode 100644 runtime/st2.d/README.md delete mode 100755 runtime/st2.d/reload.sh delete mode 100755 runtime/st2.d/virtualenv.sh delete mode 100644 tutorial/actions/hello-concurrency.yaml delete mode 100644 tutorial/actions/hello.yaml delete mode 100644 tutorial/actions/mistral-basic.yaml delete mode 100644 tutorial/actions/workflows/mistral-basic.yaml delete mode 100644 tutorial/policies/hello-concurrency.yaml delete mode 100644 tutorial/rules/monitor_file.yaml diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index aa66f4f3..00000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,135 +0,0 @@ -# -defaults: &defaults - working_directory: /build - docker: - - image: docker:18.06.0-ce-git - -version: 2 -jobs: - nightly-build: - <<: *defaults - steps: - - checkout - - setup_remote_docker: - docker_layer_caching: true - - run: - name: Install bash - command: apk add --no-cache bash coreutils - - run: - name: Build image - command: bin/build.sh - environment: - BUILD_DEV: true - - run: - name: Save image - command: bin/save.sh - environment: - BUILD_DEV: true - - persist_to_workspace: - root: /workspace - paths: - - tar - nightly-deploy: - <<: *defaults - steps: - - checkout - - attach_workspace: - at: /workspace - - setup_remote_docker: - docker_layer_caching: true - - run: - name: Install bash - command: apk add --no-cache bash coreutils - - run: - name: Load image - command: bin/load.sh - - run: - name: Deploy image to Docker Hub - command: | - docker login --username ${DOCKER_USERNAME} --password ${DOCKER_PASSWORD} - bin/deploy.sh - environment: - BUILD_DEV: true - build: - <<: *defaults - steps: - - checkout - - setup_remote_docker: - docker_layer_caching: true - - run: - name: Install bash - command: apk add --no-cache bash coreutils - - run: - name: Build image - command: bin/build.sh - - run: - name: Save image - command: bin/save.sh - - persist_to_workspace: - root: /workspace - paths: - - tar - deploy: - <<: *defaults - steps: - - attach_workspace: - at: /workspace - - checkout - - setup_remote_docker: - docker_layer_caching: true - - run: - name: Install bash - command: apk add --no-cache bash coreutils - - run: - name: Load image from workspace - command: bin/load.sh - - run: - name: Deploy image to Docker Hub - command: | - docker login --username ${DOCKER_USERNAME} --password ${DOCKER_PASSWORD} - bin/deploy.sh - -workflows: - version: 2 - nightly-workflow: - triggers: - - schedule: - cron: "0 9 * * *" - filters: - branches: - only: - - master - jobs: - - nightly-build: - filters: - tags: - only: - - /.*/ - - nightly-deploy: - requires: - - nightly-build - filters: - branches: - only: - - master - tags: - only: - - /^v.*/ - commit-workflow: - jobs: - - build: - filters: - tags: - only: - - /.*/ - - deploy: - requires: - - build - filters: - branches: - only: - - master - - feature/circleci - tags: - only: - - /^v.*/ diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 635bbc55..00000000 --- a/.travis.yml +++ /dev/null @@ -1,27 +0,0 @@ -dist: trusty -sudo: required - -language: python - -branches: - only: - - master - -services: - - docker - -before_install: - - docker build --build-arg ST2_REPO=unstable --build-arg CIRCLE_SHA1=${TRAVIS_COMMIT} --build-arg CIRCLE_PROJECT_USERNAME=stackstorm --build-arg CIRCLE_PROJECT_REPONAME=st2-docker -t stackstorm/stackstorm:dev images/stackstorm - - docker login -u ${DOCKER_USER} -p ${DOCKER_PASSWORD} - - docker push stackstorm/stackstorm:dev - -install: true - -script: true - -notifications: - email: - recipients: - - warren.van.winckel@gmail.com - on_success: never - on_failure: always diff --git a/CHANGELOG.rst b/CHANGELOG.rst deleted file mode 100644 index 0c5eac66..00000000 --- a/CHANGELOG.rst +++ /dev/null @@ -1,34 +0,0 @@ -Changelog -========= - -2018-06-28 ----------- - -Changed -~~~~~~~ - -* Add ``st2workflowengine`` to ``entrypoint-1ppc.sh`` and ``compose-1ppc/docker-compose.yml``. - -2018-06-18 ----------- - -Changed -~~~~~~~ - -* The ``TAG`` environment variable is replaced by ``ST2_IMAGE_TAG``. - -2018-02-27 ----------- - -Changed -~~~~~~~ - -* Pin DB's to specific, tested versions. - -2017-10-23 ----------- - -Changed -~~~~~~~ - -* Rename ``/entrypoint.d/`` to ``/st2-docker/entrypoint.d/``. diff --git a/Makefile b/Makefile deleted file mode 100644 index 18d2f423..00000000 --- a/Makefile +++ /dev/null @@ -1,22 +0,0 @@ -SHA := $(shell git describe --match=NeVeRmAtCh --always --abbrev=40 --dirty=*) - -build: - docker build --build-arg CIRCLE_SHA1="$(SHA)" -t stackstorm/stackstorm:latest images/stackstorm - -dev-build: - docker build --build-arg ST2_REPO=unstable --build-arg CIRCLE_SHA1="$(SHA)" -t stackstorm/stackstorm:local-dev images/stackstorm - -env: - bin/write-env.sh conf - -up: - docker-compose up -d - -down: - docker-compose down - -rmi: - docker rmi $$(docker images -f dangling=true -q) - -exec: - docker-compose exec stackstorm /bin/bash diff --git a/README.md b/README.md deleted file mode 100644 index b13c73da..00000000 --- a/README.md +++ /dev/null @@ -1,259 +0,0 @@ -# StackStorm in all-in-one Docker container - -> **DEPRECATED!** -> -> This all-in-one Docker demo deployment wasn't supported for a long time and is deprecated. -> Latest StackStorm release that supported all-in-one docker was `v3.1.0` based on outdated `Ubuntu Trusty` with `python 2`. - -[![Circle CI Build Status](https://circleci.com/gh/StackStorm/st2-docker/tree/master.svg?style=shield)](https://circleci.com/gh/StackStorm/st2-docker) -[![Go to Docker Hub](https://img.shields.io/badge/Docker%20Hub-%E2%86%92-blue.svg)](https://hub.docker.com/r/stackstorm/stackstorm/) - - -## READ FIRST!! - -- **Check the [CHANGELOG.rst](https://github.com/StackStorm/st2-docker/blob/master/CHANGELOG.rst)** - file for any potential changes that may require restarting containers. -- Be sure to use the latest `docker-compose.yml`. Run `git pull` in your `st2-docker` workspace! -- Run `st2ctl reload --register-all` to reload all services. -- **For information on how the stackstorm docker image is versioned, see - [VERSIONING.md](https://github.com/StackStorm/st2-docker/blob/master/VERSIONING.md)**. -- If a specific image version is required, it is always best to be explicit and specify the image - digest. See the example of setting `ST2_IMAGE_TAG` environment variable [below](#EnvVars). -- Kubernetes installation is available via Helm charts at https://docs.stackstorm.com/install/k8s_ha.html - and provides High Availability deployment for both StackStorm Community and Enterprise editions. - - -## TL;DR - -``` -git clone git@github.com:stackstorm/st2-docker -cd st2-docker -make env -docker-compose up -d -docker-compose exec stackstorm bash -``` - -Open `https://localhost` in your browser. StackStorm Username/Password can be found in: `cat conf/stackstorm.env` - - -## Usage - -### Prerequisites - -- Docker Engine 1.13.0+ - - -### Container configuration - -The default container configuration is as follows: - - - stackstorm (st2 + st2web + st2mistral) - - mongo - - rabbitmq - - postgres - - redis - - -### Step by step instructions - -We use Version 3 of the compose file format, so if you want to run docker-compose, you'll need to -ensure you're running Docker Engine release 1.13.0+. - -First, execute - - ``` - make env - ``` - -to create the environment files used by `docker-compose`. You may want to change the values of the -variables as necessary, but the defaults should be okay if you are not using any off-cluster -services (e.g. mongo, redis, postgres, rabbitmq). - -NOTE: `make env` only needs to be run once. - -As an example, if you want to change the username and password used by StackStorm, change the -`ST2_USER` and `ST2_PASSWORD` variables in `conf/stackstorm.env` prior to bringing up your docker -environment. - -Second, start the docker environment. execute - - ``` - docker-compose up -d - ``` - -This will pull the required images from docker hub, and then start them. - -To stop the docker environment, run: - - ``` - docker-compose down - ``` - -## Building the stackstorm image - -The pre-built `stackstorm/stackstorm` image may not meet your requirements. You may need to install -additional libraries, packages or files into the image. For example, if you want to install the -Ansible pack, you must first install the `libkrb5-dev` package. While the package could be installed -using a script in `/st2-docker/entrypoint.d`, this will increase the startup time of the container -and may result in containers that execute different code than others. - -Make any necessary changes to `images/stackstorm/Dockerfile`. For example, append `libkrb5-dev` to -the first `apt-get install` command. Next, run: - - ``` - REPO=stable - docker build --build-arg ST2_REPO=${REPO} -t stackstorm/stackstorm:${REPO} images/stackstorm - ``` - -where REPO is one of 'stable', 'unstable', 'staging-stable', 'staging-unstable'. Otherwise, -the following `docker-compose` command will download the specified image from docker hub. - - -### Getting started: Simple Tutorial Tour - -After you spin up the environment, you can play around with st2 *in container-ized environment* by following [this tutorial guide](./docs/tutorial.md). - - -## Data persistence - -It's designed to suffice the ordinary use case by default. If you need to customize it, check below and modify `docker-compose.yml` - -- The mongo, rabbitmq, postgres and redis containers store their data on persistent storage -- The stackstorm container persists the contents in following directories - - `/var/log` - - `/opt/stackstorm/packs` - - `/opt/stackstorm/virtualenvs` - - `/opt/stackstorm/configs` - -Since data directories may persist between invocations of `docker-compose`, you may see the following error: - -``` -2018-02-21 16:36:21.453 UTC [1] FATAL: database files are incompatible with server -2018-02-21 16:36:21.453 UTC [1] DETAIL: The data directory was initialized by PostgreSQL version 9.6, which is not compatible with this version 10.2 (Debian 10.2-1.pgdg90+1). -``` - -In `docker-compose.yml`, pin the postgres version to `9.6` and you will not see the error again. - -``` -- image: postgres:latest -+ image: postgres:9.6 -``` - -## Environment variables - -Below is the complete list of available options that can be used to customize your container. - -| Parameter | Description | -|-----------|-------------| -| `ST2_USER` | StackStorm account username | -| `ST2_PASSWORD` | StackStorm account password | -| `MONGO_HOST` | MongoDB server hostname | -| `MONGO_PORT` | MongoDB server port (typically `27017`) | -| `MONGO_DB` | *(Optional)* MongoDB dbname (will use `st2` if not specified) | -| `MONGO_USER` | *(Optional)* MongoDB username (will connect without credentials if this and `MONGO_PASS` are not specified) | -| `MONGO_PASS` | *(Optional)* MongoDB password | -| `RABBITMQ_HOST` | RabbitMQ server hostname | -| `RABBITMQ_PORT` | RabbitMQ server port (typically `5672`) | -| `RABBITMQ_DEFAULT_USER` | RabbitMQ username | -| `RABBITMQ_DEFAULT_PASS` | RabbitMQ password | -| `POSTGRES_HOST` | PostgreSQL server hostname | -| `POSTGRES_PORT` | PostgreSQL server port (typically `5432`) | -| `POSTGRES_DB` | PostgreSQL database | -| `POSTGRES_USER` | PostgreSQL username | -| `POSTGRES_PASSWORD` | PostgreSQL password | -| `REDIS_HOST` | Redis server hostname | -| `REDIS_PORT` | Redis server port | -| `REDIS_PASSWORD` | *(Optional)* Redis password | - -Also, you can export an additional variable to control which StackStorm version to run by specifying -the exact Docker image tag: - - ``` - export ST2_IMAGE_TAG="2.7.1@sha256:4920fd479c907149d9a062c939f158291f0f641fcd1730d9dd2df2696cad2dae" - docker-compose up -d - ``` - -## Running custom shell scripts on boot - -The `stackstorm` container supports running arbitrary shell scripts when the container launches: - -* Scripts located in `/st2-docker/entrypoint.d` are executed before the init process starts any -stackstorm services. -* Scripts located in `/st2-docker/st2.d` are executed after stackstorm services are running. - -NOTE: Only scripts with a suffix of `.sh` will be executed, and in alphabetical order of the file -name. - -### /st2-docker/entrypoint.d - -For example, if you want to modify `/etc/st2/st2.conf` to set `system_packs_base_path` parameter, -create `modify-st2-config.sh` with the follwing content: - - ``` - #/bin/bash - crudini --set /etc/st2/st2.conf content system_packs_base_path /opt/stackstorm/custom_packs - ``` - -Then bind mount it to `/st2-docker/entrypoint.d/modify-st2-config.sh` - -- via `docker run` - - ``` - docker run -it -d --privileged \ - -v /path/to/modify-st2-config.sh:/st2-docker/entrypoint.d/modify-st2-config.sh \ - stackstorm/stackstorm:latest - ``` - -- via `docker-compose.yml` - - ``` - services: - stackstorm: - image: stackstorm/stackstorm:${ST2_IMAGE_TAG:-latest} - : (snip) - volumes: - - /path/to/modify-st2-config.sh:/st2-docker/entrypoint.d/modify-st2-config.sh - ``` - -The above example shows just modifying st2 config but basically there is no limitation so you can -do almost anything. - -You can also bind mount a specific directory to `/st2-docker/entrypoint.d` then place scripts as -much as you want. - -### /st2-docker/st2.d - -Scripts in this directory can be used to register packs, reload or restart services, etc. -You can bind mount these scripts as mentioned in the previous section. - -NOTE: These scripts are currently not available when running in 1ppc mode. - - -## To enable chatops - -Chatops is installed in the `stackstorm` image, but not started by default. - -To enable chatops, delete the file `/etc/init/st2chatops.override` using a script in -`/st2-docker/entrypoint.d`. - - ``` - #!/bin/bash - - sudo rm /etc/init/st2chatops.override - ``` - -You also need to configure st2chatops, replace `/opt/stackstorm/chatops/st2chatops.env` with one -that is properly configured. The easiest way is to use bind-mount. - -See [st2chatops.env](https://github.com/StackStorm/st2chatops/blob/master/st2chatops.env) for the required variables. - -## packs.dev directory - -By default, `./packs.dev` directory is bind-mounted to `/opt/stackstorm/packs.dev` in `stackstorm` container and registered as a secondary pack location. This is done by the startup script at [./runtime/entrypoint.d/add-packs-dev.sh](./runtime/entrypoint.d/add-packs-dev.sh) - -This feature exists just for convenience, for testing and developing packs, and for [tutorial](./docs/tutorial.md). You can use it for arbitrary purpose, or ignore, or even disable it completely by removing corresponding entries and files. -Refer to the official StackStorm document for the list of available configuration parameters for `st2chatops`. - -## Advanced: using 1ppc image - -Official image now supports running in 1ppc mode: stands for *One Process Per Container*. Interested? Check [runtime/compose-1ppc](./runtime/compose-1ppc) diff --git a/VERSIONING.md b/VERSIONING.md deleted file mode 100644 index a7676a42..00000000 --- a/VERSIONING.md +++ /dev/null @@ -1,16 +0,0 @@ -# StackStorm Docker Image Versioning - -See https://github.com/StackStorm/st2-docker/issues/78 for more information. - -| Image:Tag | StackStorm Version | Description | -|-----------|--------------------|-------------| -| stackstorm:dev | 2.7dev | Latest 2.7dev, and most recent st2-docker changes from the st2-docker `master` branch. | -| stackstorm:latest | 2.6.0 (latest stable version of Stackstorm) | Changes merged to st2-docker `master` branch will result in a new image being deployed tagged 'latest'. | -| stackstorm:2.6 | 2.6.0 | Mutable. This tag is updated when there is a new 2.6.x release. | -| stackstorm:2.6.0 | 2.6.0 | Immutable, even if changes merged to st2-docker `master` branch | -| stackstorm:2.5 | 2.5.1 | Mutable. This tag is updated when there is a new 2.5.x release. | -| stackstorm:2.5.1 | 2.5.1 | Immutable, even if changes merged to st2-docker `master` branch | -| stackstorm:2.5.0 | 2.5.0 | Immutable, even if changes merged to st2-docker `master` branch | -| stackstorm:2.4 | 2.4.1 | Mutable. This tag is updated when there is a new 2.4.x release. | -| stackstorm:2.4.1 | 2.4.1 | Immutable, even if changes merged to st2-docker `master` branch | -| stackstorm:2.4.0 | 2.4.0 | Immutable, even if changes merged to st2-docker `master` branch | diff --git a/bin/build.sh b/bin/build.sh deleted file mode 100755 index 5eb9ffbb..00000000 --- a/bin/build.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash -# -# This script runs within the CircleCI environment to build stackstorm images. - -set -euo pipefail -IDS=$'\n\t' - -source bin/common.sh - -for name in stackstorm; do - if [ ! -z ${BUILD_DEV} ]; then - # Triggered to run nightly via ops-infra - # Build unstable, and tag as "dev". - - ${dry_run} docker build --no-cache --build-arg ST2_REPO=unstable --build-arg CIRCLE_SHA1=${CIRCLE_SHA1} \ - --build-arg CIRCLE_PROJECT_USERNAME=${CIRCLE_PROJECT_USERNAME:-} \ - --build-arg CIRCLE_PROJECT_REPONAME=${CIRCLE_PROJECT_REPONAME:-} \ - --build-arg CIRCLE_BUILD_URL=${CIRCLE_BUILD_URL:-} \ - -t stackstorm/${name}:dev images/${name} - - continue - fi - - # From this point on, not a dev build... - - name_tag="${name}:${tag}" - - # Build the ${name_tag} image using Dockerfile in images/${name} - ${dry_run} docker build --no-cache --build-arg ST2_TAG=${st2_tag} --build-arg CIRCLE_SHA1=${CIRCLE_SHA1} \ - --build-arg CIRCLE_PROJECT_USERNAME=${CIRCLE_PROJECT_USERNAME:-} \ - --build-arg CIRCLE_PROJECT_REPONAME=${CIRCLE_PROJECT_REPONAME:-} \ - --build-arg CIRCLE_BUILD_URL=${CIRCLE_BUILD_URL:-} \ - -t stackstorm/${name_tag} images/${name} -done diff --git a/bin/common.sh b/bin/common.sh deleted file mode 100644 index baffad46..00000000 --- a/bin/common.sh +++ /dev/null @@ -1,76 +0,0 @@ -# The following variables will be set for use by the calling script - -# latest - the highest version tagged in the repo (beginning with "v") -# st2_tag - the MAJOR.MINOR.PATCH version of st2 installed in the image -# short_tag - MAJOR.MINOR from ${st2_tag} -# latest_short - contains the highest version beginning with ${short_tag} -# tagged_build - true if build was tagged, else false -# tag - tag image with this value (if tagged_build st2_tag else latest) - -# Set debug to 'echo' to test -dry_run='' -if [ ${DRY_RUN:-} ]; then - dry_run='echo' - echo "Dry run mode enabled..." - sleep 2 -fi - -WORKSPACE=/workspace/tar -echo WORKSPACE=${WORKSPACE} - -CIRCLE_SHA1=${CIRCLE_SHA1:-} -echo CIRCLE_SHA1=${CIRCLE_SHA1} - -if [ -z ${CIRCLE_SHA1} ]; then - echo "ERROR: CIRCLE_SHA1 is not defined." - echo "To resolve, run:" - echo " $ export CIRCLE_SHA1=" - echo " $ $0" - exit 1 -fi - -CIRCLE_TAG=${CIRCLE_TAG:-} -echo CIRCLE_TAG=${CIRCLE_TAG} - -BUILD_DEV=${BUILD_DEV:-} -echo BUILD_DEV=${BUILD_DEV} - -# Get the highest tag prefixed with 'v' -# NOTE: We remove the 'v' prefix before returning -latest=`git tag -l "v*" | sort -rV | head -1 | cut -c 2-` - -if [ ! -z ${CIRCLE_TAG} ]; then - if [[ ${CIRCLE_TAG} =~ ^v([0-9]+)\.([0-9]+)\.([0-9]+).*$ ]]; then - CIRCLE_TAG_MAJOR=${BASH_REMATCH[1]} - CIRCLE_TAG_MINOR=${BASH_REMATCH[2]} - CIRCLE_TAG_PATCH=${BASH_REMATCH[3]} - else - echo "ERROR: CIRCLE_TAG must begin with format 'vMAJOR.MINOR.PATCH'" - exit 1 - fi -fi - -short_tag='' - -if [[ ${CIRCLE_TAG} =~ ^v(.+)$ ]]; then - # A tag was pushed, so we'll build an image using this specific release. - tagged_build=true - st2_tag=${BASH_REMATCH[1]} - tag=${st2_tag} - short_tag="${CIRCLE_TAG_MAJOR}.${CIRCLE_TAG_MINOR}" - latest_short=`git tag -l "v${short_tag}*" | sort -rV | head -1 | cut -c 2-` - echo latest_short=${latest_short} -else - # NOTE: A valid version tag was not pushed - # Build and tag an image using the highest StackStorm release - tagged_build=false - tag='latest' - st2_tag=${latest} -fi - -# These variables are available in calling scripts -echo latest=${latest} -echo short_tag=${short_tag} -echo st2_tag=${st2_tag} -echo tag=${tag} -echo tagged_build=${tagged_build} diff --git a/bin/deploy.sh b/bin/deploy.sh deleted file mode 100755 index e4098ad4..00000000 --- a/bin/deploy.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash -# -# This script runs within the CircleCI environment to deploy st2-docker images -# to Docker Hub. - -set -euo pipefail -IDS=$'\n\t' - -source bin/common.sh - -for name in stackstorm; do - if [ ! -z ${BUILD_DEV} ]; then - # Build unstable, and tag as "dev". - - # TODO: Potentially useful to prepend "dev" with revision of latest unstable - # release (e.g. "2.4dev") - ${dry_run} docker push stackstorm/${name}:dev - continue - fi - - # From this point on, not a dev build... - name_tag="${name}:${tag}" - - if ${tagged_build}; then - # gatekeeper.sh returns 'allow' on STDOUT if the images can be pushed - if [ `bin/gatekeeper.sh ${name} ${tag}` != 'allow' ]; then - echo "${name_tag} already exists on docker hub.. not pushing again!" - exit 1 - fi - fi - - ${dry_run} docker push stackstorm/${name}:${tag} - - if ${tagged_build}; then - if [ "${st2_tag}" == "${latest_short}" ]; then - ${dry_run} docker tag stackstorm/${name_tag} stackstorm/${name}:${short_tag} - ${dry_run} docker push stackstorm/${name}:${short_tag} - fi - - if [ "${st2_tag}" == "${latest}" ]; then - ${dry_run} docker tag stackstorm/${name_tag} stackstorm/${name}:latest - ${dry_run} docker push stackstorm/${name}:latest - fi - fi -done diff --git a/bin/gatekeeper.sh b/bin/gatekeeper.sh deleted file mode 100755 index c0e8c016..00000000 --- a/bin/gatekeeper.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -set -euo pipefail -IDS=$'\n\t' - -if [ $# -lt 2 ]; then - echo "Usage: $0 " - exit 1 -fi - -name=$1 -tag=$2 - -if [ ${tag} == 'latest' ]; then - echo 'allow' - exit 0 -fi - -wget -q https://registry.hub.docker.com/v1/repositories/stackstorm/${name}/tags -O - \ - | sed -e 's/[][]//g' -e 's/"//g' -e 's/ //g' | tr '}' '\n' | awk -F: '{print $3}' | grep ${tag} \ - || echo 'allow' diff --git a/bin/load.sh b/bin/load.sh deleted file mode 100755 index 00ba08ba..00000000 --- a/bin/load.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -# -# This script runs within the CircleCI environment to build stackstorm images. - -set -euo pipefail -IDS=$'\n\t' - -source bin/common.sh - -for name in stackstorm; do - # Load the tarball (tags are automatically loaded) - ${dry_run} docker load -i ${WORKSPACE}/${name}.tar -done diff --git a/bin/save.sh b/bin/save.sh deleted file mode 100755 index e27e99ea..00000000 --- a/bin/save.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -# -# This script runs within the CircleCI environment to build stackstorm images. - -set -euo pipefail -IDS=$'\n\t' - -source bin/common.sh - -${dry_run} mkdir -p ${WORKSPACE} - -for name in stackstorm; do - if [ ! -z ${BUILD_DEV} ]; then - ${dry_run} docker save -o ${WORKSPACE}/${name}.tar stackstorm/${name}:dev - - continue - fi - - # From this point on, not a dev build... - - name_tag="${name}:${tag}" - - # Save the image ${name} using tag ${tag} - tags="stackstorm/${name_tag}" - - ${dry_run} docker save -o ${WORKSPACE}/${name}.tar ${tags} -done diff --git a/bin/write-env.sh b/bin/write-env.sh deleted file mode 100755 index aca8906f..00000000 --- a/bin/write-env.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/bash - -# Create env files in the specified directory - -CONF_DIR=${1:-conf} - -# Create a random password of length specified by $1 -function randpwd() -{ - echo $(openssl rand -base64 $1 | tr '/' 'A') -} - -mkdir -p ${CONF_DIR} - -if [ ! -f ${CONF_DIR}/mongo.env ]; then - echo "MONGO_HOST=${MONGO_HOST:-mongo}" > ${CONF_DIR}/mongo.env - echo "MONGO_PORT=${MONGO_PORT:-27017}" >> ${CONF_DIR}/mongo.env - if [ -z ${MONGO_DB} ]; then - echo "#MONGO_DB=" >> ${CONF_DIR}/mongo.env - else - echo "MONGO_DB=${MONGO_DB}" >> ${CONF_DIR}/mongo.env - fi - if [ -z ${MONGO_USER} ]; then - echo "#MONGO_USER=" >> ${CONF_DIR}/mongo.env - else - echo "MONGO_USER=${MONGO_USER}" >> ${CONF_DIR}/mongo.env - fi - if [ -z ${MONGO_PASS} ]; then - echo "#MONGO_PASS=" >> ${CONF_DIR}/mongo.env - else - echo "MONGO_PASS=${MONGO_PASS}" >> ${CONF_DIR}/mongo.env - fi -fi -if [ ! -f ${CONF_DIR}/postgres.env ]; then - echo "POSTGRES_USER=${POSTGRES_USER:-mistral-user}" > ${CONF_DIR}/postgres.env - echo "POSTGRES_PASSWORD=${POSTGRES_PASS:-$(randpwd 18)}" >> ${CONF_DIR}/postgres.env - echo "POSTGRES_HOST=${POSTGRES_HOST:-postgres}" >> ${CONF_DIR}/postgres.env - echo "POSTGRES_PORT=${POSTGRES_PORT:-5432}" >> ${CONF_DIR}/postgres.env - echo "POSTGRES_DB=${POSTGRES_DB:-mistral}" >> ${CONF_DIR}/postgres.env -fi -if [ ! -f ${CONF_DIR}/rabbitmq.env ]; then - echo "RABBITMQ_DEFAULT_USER=${RABBITMQ_DEFAULT_USER:-admin}" > ${CONF_DIR}/rabbitmq.env - echo "RABBITMQ_DEFAULT_PASS=${RABBITMQ_DEFAULT_PASS:-$(randpwd 18)}" >> ${CONF_DIR}/rabbitmq.env - echo "RABBITMQ_HOST=${RABBITMQ_HOST:-rabbitmq}" >> ${CONF_DIR}/rabbitmq.env - echo "RABBITMQ_PORT=${RABBITMQ_PORT:-5672}" >> ${CONF_DIR}/rabbitmq.env -fi -if [ ! -f ${CONF_DIR}/redis.env ]; then - echo "REDIS_PASSWORD=${REDIS_PASSWORD:-$(randpwd 18)}" > ${CONF_DIR}/redis.env - echo "REDIS_HOST=${REDIS_HOST:-redis}" >> ${CONF_DIR}/redis.env - echo "REDIS_PORT=${REDIS_PORT:-6379}" >> ${CONF_DIR}/redis.env -fi -if [ ! -f ${CONF_DIR}/stackstorm.env ]; then - echo "ST2_USER=${ST2_USER:-st2admin}" > ${CONF_DIR}/stackstorm.env - echo "ST2_PASSWORD=${ST2_PASSWORD:-$(randpwd 6)}" >> ${CONF_DIR}/stackstorm.env -fi diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 33bb9ab2..00000000 --- a/docker-compose.yml +++ /dev/null @@ -1,95 +0,0 @@ -version: '3' - -services: - stackstorm: - image: stackstorm/stackstorm:${ST2_IMAGE_TAG:-latest} - env_file: - - conf/stackstorm.env - - conf/mongo.env - - conf/rabbitmq.env - - conf/postgres.env - - conf/redis.env - ports: - - "443:443" - depends_on: - - mongo - - postgres - - rabbitmq - - redis - networks: - - public - - private - volumes: - - stackstorm-packs-volume:/opt/stackstorm/packs - - stackstorm-virtualenvs-volume:/opt/stackstorm/virtualenvs - - stackstorm-configs-volume:/opt/stackstorm/configs - - stackstorm-log-volume:/var/log - - ./packs.dev:/opt/stackstorm/packs.dev - - ./runtime/entrypoint.d:/st2-docker/entrypoint.d - - ./runtime/st2.d:/st2-docker/st2.d - - ./conf/stackstorm.env:/st2-docker/env - dns_search: . - -### External Services - - mongo: - image: mongo:3.4 - env_file: - - conf/mongo.env - networks: - - private - volumes: - - mongo-volume:/data/db - - mongo-configdb-volume:/data/configdb - dns_search: . - rabbitmq: - image: rabbitmq:3.6-management - env_file: - - conf/rabbitmq.env - networks: - - private - volumes: - - rabbitmq-volume:/var/lib/rabbitmq - dns_search: . - postgres: - image: postgres:9.6 - env_file: - - conf/postgres.env - networks: - - private - volumes: - - postgres-volume:/var/lib/postgresql/data - dns_search: . - redis: - image: redis:4.0 - env_file: - - conf/redis.env - networks: - - private - volumes: - - redis-volume:/data - dns_search: . - command: [ - "bash", "-c", - ' - docker-entrypoint.sh - --requirepass "$$REDIS_PASSWORD" - ' - ] - -volumes: - mongo-volume: - mongo-configdb-volume: - postgres-volume: - rabbitmq-volume: - redis-volume: - stackstorm-packs-volume: - stackstorm-virtualenvs-volume: - stackstorm-configs-volume: - stackstorm-log-volume: - -networks: - public: - driver: bridge - private: - driver: bridge diff --git a/docs/tutorial.md b/docs/tutorial.md deleted file mode 100644 index 031c6551..00000000 --- a/docs/tutorial.md +++ /dev/null @@ -1,283 +0,0 @@ -# Simple Tutorial Tour - -## Adding a simple action - -We will add a simple action that runs a local shell command. -Run the following from your docker host. - -``` -sudo chown $USERNAME:$USERNAME packs.dev -mkdir -p packs.dev/tutorial/actions -sudo chown -R $USERNAME:$USERNAME packs.dev -cp tutorial/actions/hello.yaml packs.dev/tutorial/actions -``` - -Get a bash shell in the `stackstorm` container: - - ``` - docker-compose exec stackstorm /bin/bash - ``` - -Load the new action into StackStorm. Whenever you change the yaml file, you need -to run `st2ctl reload`. Within the container, run the following: - - ``` - root@aff39eda0bdd:/# st2ctl reload --register-all - - ... output trimmed ... - - ``` - -Now, let's run the action: - - ``` - root@aff39eda0bdd:/# st2 run tutorial.hello - . - id: 58f67dbf33a99300bdc4d618 - status: succeeded - parameters: None - result: - failed: false - return_code: 0 - stderr: '' - stdout: Hello human! - succeeded: true - ``` - -The action takes a single parameter `name`, which as we can see above, -defaults to 'human' if `name` is not specified. If we specify a value for -`name`, then as expected, the value is found in `result.stdout`. - - ``` - root@aff39eda0bdd:/# st2 run tutorial.hello name=Stanley - . - id: 58f67dc533a99300bdc4d61b - status: succeeded - parameters: - name: Stanley - result: - failed: false - return_code: 0 - stderr: '' - stdout: Hello Stanley! - succeeded: true - ``` - -Congratulations, you have created your first simple action! - -### A Slight Variation: Concurrency - -If you want to take advantage of concurrency, use a slight variation on the above. -On the host, run: - -``` -mkdir -p packs.dev/tutorial/policies -cp tutorial/actions/hello-concurrency.yaml packs.dev/tutorial/actions -cp tutorial/policies/hello-concurrency.yaml packs.dev/tutorial/policies -``` - -Inside the `stackstorm` container, run: - -``` -st2ctl reload --register-all -``` - -Open two terminals to the `stackstorm` container. In the first, type (but don't execute): - -``` -st2 run tutorial.hello-concurrency name=1 -``` - -In the second, type: - -``` -st2 run tutorial.hello-concurrency name=2 -``` - -Now, execute the command in the first terminal, wait 5 seconds and then execute the command in the -second terminal. After a second or so, you should see the following in the second terminal: - -``` -root@258b11849aa7:/# st2 run examples.hello-concurrency name=2 -. -id: 590cec228964ad01567f61e3 -status: delayed -parameters: - name: 2 -result: None -``` - -If you run `st2 execution list` before 10 seconds have elapsed, the status of the second action should -be "delayed". Between 10 and 20 seconds, the status of the second action should be "running". After -20 seconds, the status of the second action should be "succeeded". - -``` -root@ffc8bc7909c6:/# st2 execution list -+--------------------------+----------------------------+--------------+-------------------------+-------------------------------+-------------------------------+ -| id | action.ref | context.user | status | start_timestamp | end_timestamp | -+--------------------------+----------------------------+--------------+-------------------------+-------------------------------+-------------------------------+ - : -(snip) - : -| 5a366f07a1d7aa00ecfd3cef | tutorial.hello-concurrency | st2admin | succeeded (11s elapsed) | Sun, 17 Dec 2017 13:20:07 UTC | Sun, 17 Dec 2017 13:20:18 UTC | -| 5a366f0aa1d7aa00ecfd3cf2 | tutorial.hello-concurrency | st2admin | succeeded (18s elapsed) | Sun, 17 Dec 2017 13:20:10 UTC | Sun, 17 Dec 2017 13:20:28 UTC | -+--------------------------+----------------------------+--------------+-------------------------+-------------------------------+-------------------------------+ - -``` - -## Adding a rule - -To perform a very basic end-to-end test of StackStorm, let's create a simple rule. -Run the following from your docker host. - - ``` - mkdir packs.dev/tutorial/rules - cp tutorial/rules/monitor_file.yaml packs.dev/tutorial/rules - ``` - -Take a look at `monitor_file.yaml`. The `core.local` action is triggered when the -contents of `/tmp/watcher.log` change. - -Use `docker-compose exec` to connect to the `stackstorm` container: - - ``` - docker-compose exec stackstorm /bin/bash - ``` - -Run the following: - - ``` - st2ctl reload - ``` - -When we append to `/tmp/watcher.log`, the sensor will inject a trigger and the -action will be executed. Now let's append a line to the file in the container. - -``` -echo "hello" >> /tmp/watcher.log -``` - -You should see that the action was fired: - - ``` - root@ffc8bc7909c6:/# st2 execution list - +--------------------------+----------------------------+--------------+-------------------------+-------------------------------+-------------------------------+ - | id | action.ref | context.user | status | start_timestamp | end_timestamp | - +--------------------------+----------------------------+--------------+-------------------------+-------------------------------+-------------------------------+ - : - (snip) - : - | 5a36702fa1d7aa00373b785c | core.local | stanley | succeeded (0s elapsed) | Sun, 17 Dec 2017 13:25:03 UTC | Sun, 17 Dec 2017 13:25:03 UTC | - +--------------------------+----------------------------+--------------+-------------------------+-------------------------------+-------------------------------+ - root@ffc8bc7909c6:/# st2 execution get 5a36702fa1d7aa00373b785c - id: 5a36702fa1d7aa00373b785c - status: succeeded (0s elapsed) - parameters: - cmd: 'echo "{''file_name'': ''watcher.log'', ''line'': u''hello'', ''file_path'': ''/tmp/watcher.log''}"' - result: - failed: false - return_code: 0 - stderr: '' - stdout: '{''file_name'': ''watcher.log'', ''line'': u''hello'', ''file_path'': ''/tmp/watcher.log''}' - succeeded: true - ``` - -Congratulations, you have created your first rule! - -## Adding a python action - -As an example of how to create a new action, let's add a new action called `echo_action`. - -First, on the host, we create the metadata file `./packs.dev/tutorial/actions/my_echo_action.yaml`: - -```yaml ---- -name: "echo_action" -runner_type: "python-script" -description: "Print message to standard output." -enabled: true -entry_point: "my_echo_action.py" -parameters: - message: - type: "string" - description: "Message to print." - required: true - position: 0 -``` - -Then, add the action script at `./packs.dev/tutorial/actions/my_echo_action.py`. - -```python -import sys - -from st2common.runners.base_action import Action - -class MyEchoAction(Action): - def run(self, message): - print(message) - - if message == 'working': - return (True, message) - return (False, message) -``` - -When you rename, or create a new action, you must run `st2ctl reload` inside the `st2` -container. Next, to initialize the virtualenv, run: - -``` -st2 run packs.setup_virtualenv packs=tutorial -``` - -Then you can run your action using the following: - -``` -st2 run tutorial.echo_action message=working -``` - -You should see output similar to: - -``` -. -id: 58c0abcff4aa45009f42dca3 -status: succeeded -parameters: - message: working -result: - exit_code: 0 - result: working - stderr: '' - stdout: 'working - - ' -``` - -Congratulations! You have successfully added your first action! - -## Adding a simple mistral workflow - -To add a simple mistral workflow, run the following from your docker host. - - ``` - mkdir -p packs.dev/tutorial/actions/workflows - cp -R tutorial/actions/mistral-basic.yaml packs.dev/tutorial/actions/mistral-basic.yaml - cp -R tutorial/actions/workflows/mistral-basic.yaml packs.dev/tutorial/actions/workflows/mistral-basic.yaml - ``` - -Use `docker-compose exec` to connect to the `stackstorm` container: - - ``` - docker-compose exec stackstorm /bin/bash - ``` - -Within the container, run the following: - - ``` - st2 action create /opt/stackstorm/packs.dev/tutorial/actions/mistral-basic.yaml - st2 run examples.mistral-basic cmd=date -a - ``` - -The `st2 run` command should complete successfully. Please see -[mistral documentation](https://docs.stackstorm.com/mistral.html#basic-workflow) -for more details about this basic workflow. - -Congratulations, you have created your first mistral workflow! diff --git a/images/stackstorm/Dockerfile b/images/stackstorm/Dockerfile deleted file mode 100644 index f49a15a7..00000000 --- a/images/stackstorm/Dockerfile +++ /dev/null @@ -1,144 +0,0 @@ -FROM ubuntu:trusty - -# let Upstart know it's in a container -ENV container docker - -COPY config/init-fake.conf /etc/init/fake-container-events.conf - -# uuid-runtime is required, otherwise the last test in 'test_quickstart_key.yaml' fails -RUN DEBIAN_FRONTEND=noninteractive \ - apt-get update \ - && apt-get install -y \ - openssh-server \ - sudo \ - curl \ - gdebi-core \ - sshpass \ - cron \ - netcat \ - net-tools \ - crudini \ - uuid-runtime \ - apache2-utils \ - bash-completion - -# enable bash-completion -RUN dpkg-divert /etc/bash.bashrc && \ - sed -i '/^# enable bash completion/,/^# sudo hint/{//p;//d;s/^#//}' /etc/bash.bashrc - -RUN echo -e "#!/bin/sh\nexit 101\n" > /usr/sbin/policy-rc.d \ - && rm /sbin/initctl; dpkg-divert --rename --remove /sbin/initctl \ - && locale-gen en_US.UTF-8 - -ENV LANG=en_US.UTF-8 -ENV LANGUAGE en_US:en -ENV LC_ALL en_US.UTF-8 - -# remove some pointless services -RUN /usr/sbin/update-rc.d -f ondemand remove; \ - for f in \ - /etc/init/u*.conf \ - /etc/init/mounted-dev.conf \ - /etc/init/mounted-proc.conf \ - /etc/init/mounted-run.conf \ - /etc/init/mounted-tmp.conf \ - /etc/init/mounted-var.conf \ - /etc/init/hostname.conf \ - /etc/init/networking.conf \ - /etc/init/tty*.conf \ - /etc/init/plymouth*.conf \ - /etc/init/hwclock*.conf \ - /etc/init/module*.conf\ - ; do \ - dpkg-divert --local --rename --add "$f"; \ - done; \ - echo '# /lib/init/fstab: cleared out for bare-bones Docker' > /lib/init/fstab - -# Default value of ST2_REPO is "stable" -ARG ST2_REPO=stable - -# Configure system so that the "stable" ST2 packages -# are fetched from packagecloud.io -RUN curl -s https://packagecloud.io/install/repositories/StackStorm/${ST2_REPO}/script.deb.sh | sudo bash - -# The following variable is the most recent commit in -# the st2-docker repo used to fetch this Dockerfile. -ARG CIRCLE_SHA1 -ARG CIRCLE_BUILD_URL -ARG CIRCLE_PROJECT_USERNAME -ARG CIRCLE_PROJECT_REPONAME - -# Override these values if you want to specify different package versions -ARG ST2_TAG -ARG ST2_VERSION -ARG ST2WEB_VERSION -ARG ST2MISTRAL_VERSION - -COPY bin/install.sh /install.sh - -# It is not possible to dynamically set ARG's, so we do the needful in bin/install.sh -# Install st2, st2web, st2mistral and st2chatops -RUN /install.sh - -# Unless these lines are changed, the services are not started when runlevel -> 2 -# Call mistral-db-manage before mistral starts -RUN sed -i 's/start on filesystem and net-device-up IFACE!=lo/start on runlevel \[2345\]/' /etc/init/st2*.conf \ - && sed -i 's/stop on starting rc RUNLEVEL=\[016\]/stop on runlevel \[!2345\]/' /etc/init/st2*.conf \ - && sed -i 's/start on filesystem and net-device-up IFACE!=lo/start on runlevel \[2345\]/' /etc/init/mistral.conf \ - && sed -i 's/stop on starting rc RUNLEVEL=\[016\]/stop on runlevel \[!2345\]/' /etc/init/mistral.conf \ - && sed -i '/start mistral-api/i\ /opt/stackstorm/mistral/bin/mistral-db-manage --config-file /etc/mistral/mistral.conf upgrade head\n\ - /opt/stackstorm/mistral/bin/mistral-db-manage --config-file /etc/mistral/mistral.conf populate\n' /etc/init/mistral.conf - -# Setup symmetric crypto key for datastore -RUN mkdir -p /etc/st2/keys \ - && st2-generate-symmetric-crypto-key --key-path /etc/st2/keys/datastore_key.json \ - && usermod -a -G st2 st2 && chgrp st2 /etc/st2/keys && chmod o-r /etc/st2/keys \ - && chgrp st2 /etc/st2/keys/datastore_key.json && chmod o-r /etc/st2/keys/datastore_key.json \ - && crudini --set /etc/st2/st2.conf keyvalue encryption_key_path /etc/st2/keys/datastore_key.json \ - && crudini --set /etc/st2/st2.conf auth enable True - -# Install redis client library for coordination backend -# see: https://docs.stackstorm.com/latest/reference/policies.html -RUN bash -c 'source /opt/stackstorm/st2/bin/activate && pip install redis' - -# Setup SSH and SUDO access for stanley user -RUN mkdir -p /home/stanley/.ssh && chmod 0700 /home/stanley/.ssh \ - && ssh-keygen -f /home/stanley/.ssh/stanley_rsa -P "" \ - && cat /home/stanley/.ssh/stanley_rsa.pub >> /home/stanley/.ssh/authorized_keys \ - && chown -R stanley:stanley /home/stanley/.ssh \ - && echo "stanley ALL=(ALL) NOPASSWD: SETENV: ALL" >> /etc/sudoers.d/st2 \ - && chmod 0440 /etc/sudoers.d/st2 \ - && sed -i -r "s/^Defaults\s+\+?requiretty/# Defaults +requiretty/g" /etc/sudoers - -# Install and configure nginx -# Use hkp://...:80 explicitly to grab the GPG key for nginx because port 11371 -# is sometimes blocked by firewalls. See: -# https://github.com/StackStorm/st2-docker/issues/135#issuecomment-392186954 -RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys ABF5BD827BD9BF62 \ - && echo "deb http://nginx.org/packages/mainline/ubuntu/ trusty nginx" >> /etc/apt/sources.list \ - && echo "deb-src http://nginx.org/packages/mainline/ubuntu/ trusty nginx" >> /etc/apt/sources.list \ - && apt-get update \ - && apt-get install -y nginx \ - && cp /usr/share/doc/st2/conf/nginx/st2.conf /etc/nginx/conf.d/st2-base.cnf \ - && ( cd /etc/nginx/conf.d && ln -s st2-base.cnf st2.conf ) \ - && mkdir -p /etc/ssl/st2 \ - && mkdir /var/run/sshd \ - && openssl req -x509 -newkey rsa:2048 -keyout /etc/ssl/st2/st2.key -out /etc/ssl/st2/st2.crt -days 3650 -nodes -subj '/O=st2 self signed/CN=localhost' - -EXPOSE 22 443 - -COPY bin/entrypoint.sh /st2-docker/bin/entrypoint.sh -COPY bin/st2.sh /st2-docker/bin/st2.sh -COPY config/local.conf /etc/init/local.conf - -# 1ppc -RUN wget -O /dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64 \ - && chmod +x /dumb-init -COPY bin/entrypoint-1ppc.sh /st2-docker/bin/entrypoint-1ppc.sh -COPY bin/inject_env.py /st2-docker/bin/inject_env.py -COPY config/nginx.st2-1ppc.conf.tpl /etc/nginx/conf.d/st2-1ppc.conf.tpl - -# Default username/password is used unless overridden by supplying ST2_USER and/or ST2_PASSWORD -# environment variables to `docker run` after the name of the image: -# docker run -e ST2_USER... image -ENTRYPOINT ["/st2-docker/bin/entrypoint.sh"] diff --git a/images/stackstorm/bin/entrypoint-1ppc.sh b/images/stackstorm/bin/entrypoint-1ppc.sh deleted file mode 100755 index 759076a4..00000000 --- a/images/stackstorm/bin/entrypoint-1ppc.sh +++ /dev/null @@ -1,103 +0,0 @@ -#!/bin/bash - -ST2_CONF=/etc/st2/st2.conf -crudini --set ${ST2_CONF} auth api_url ${ST2_API_URL} -crudini --set ${ST2_CONF} mistral api_url ${ST2_API_URL} -crudini --set ${ST2_CONF} mistral v2_base_url ${ST2_MISTRAL_API_URL} - -# Configure CORS to accept any source -# st2api gunicorn process is directly exposed to clients in 1ppc mode -crudini --set ${ST2_CONF} api allow_origin '*' - -# Generate nginx config for st2web to support load balancing to st2api, st2auth and st2stream -/st2-docker/bin/inject_env.py \ - < /etc/nginx/conf.d/st2-1ppc.conf.tpl \ - > /etc/nginx/conf.d/st2.conf - -case "$ST2_SERVICE" in - "nop" ) - exec tail -f /dev/null - ;; - "st2api" ) - DAEMON_ARGS="-k eventlet -b 0.0.0.0:9101 --workers 1 --threads 1 --graceful-timeout 10 --timeout 30" - exec /opt/stackstorm/st2/bin/gunicorn st2api.wsgi:application $DAEMON_ARGS - ;; - "st2auth" ) - DAEMON_ARGS="-k eventlet -b 0.0.0.0:9100 --workers 1 --threads 1 --graceful-timeout 10 --timeout 30" - exec /opt/stackstorm/st2/bin/gunicorn st2auth.wsgi:application $DAEMON_ARGS - ;; - "st2stream" ) - DAEMON_ARGS="-k eventlet -b 0.0.0.0:9102 --workers 1 --threads 10 --graceful-timeout 10 --timeout 30" - exec /opt/stackstorm/st2/bin/gunicorn st2stream.wsgi:application $DAEMON_ARGS - ;; - "st2sensorcontainer" ) - DAEMON_ARGS="--config-file /etc/st2/st2.conf" - exec /opt/stackstorm/st2/bin/st2sensorcontainer ${DAEMON_ARGS} - ;; - "st2rulesengine" ) - DAEMON_ARGS="--config-file /etc/st2/st2.conf" - exec /opt/stackstorm/st2/bin/st2rulesengine ${DAEMON_ARGS} - ;; - "st2timersengine" ) - DAEMON_ARGS="--config-file /etc/st2/st2.conf" - exec /opt/stackstorm/st2/bin/st2timersengine ${DAEMON_ARGS} - ;; - "st2workflowengine" ) - DAEMON_ARGS="--config-file /etc/st2/st2.conf" - exec /opt/stackstorm/st2/bin/st2workflowengine ${DAEMON_ARGS} - ;; - "st2actionrunner" ) - DAEMON_ARGS="--config-file /etc/st2/st2.conf" - exec /opt/stackstorm/st2/bin/st2actionrunner ${DAEMON_ARGS} - ;; - "st2scheduler" ) - DAEMON_ARGS="--config-file /etc/st2/st2.conf" - exec /opt/stackstorm/st2/bin/st2scheduler ${DAEMON_ARGS} - ;; - "st2resultstracker" ) - DAEMON_ARGS="--config-file /etc/st2/st2.conf" - exec /opt/stackstorm/st2/bin/st2resultstracker ${DAEMON_ARGS} - ;; - "st2notifier" ) - DAEMON_ARGS="--config-file /etc/st2/st2.conf" - exec /opt/stackstorm/st2/bin/st2notifier ${DAEMON_ARGS} - ;; - "st2garbagecollector" ) - DAEMON_ARGS="--config-file /etc/st2/st2.conf" - exec /opt/stackstorm/st2/bin/st2garbagecollector ${DAEMON_ARGS} - ;; - "mistral-api" ) - set -e - /opt/stackstorm/mistral/bin/mistral-db-manage --config-file /etc/mistral/mistral.conf upgrade head - /opt/stackstorm/mistral/bin/mistral-db-manage --config-file /etc/mistral/mistral.conf populate - API_ARGS="--log-file /var/log/mistral/mistral-api.log -b 0.0.0.0:8989 -w 2 mistral.api.wsgi --graceful-timeout 10" - exec /opt/stackstorm/mistral/bin/gunicorn $API_ARGS - ;; - "mistral-server" ) - set -e - /opt/stackstorm/mistral/bin/mistral-db-manage --config-file /etc/mistral/mistral.conf upgrade head - /opt/stackstorm/mistral/bin/mistral-db-manage --config-file /etc/mistral/mistral.conf populate - SERVER_ARGS="--config-file /etc/mistral/mistral.conf --log-file /var/log/mistral/mistral-server.log" - exec /opt/stackstorm/mistral/bin/mistral-server --server engine,executor,notifier ${SERVER_ARGS} - ;; - "st2web" ) - exec /usr/sbin/nginx -g 'daemon off;' - ;; - "st2chatops" ) - set -e - export ST2_API=${ST2_API_URL} - cd /opt/stackstorm/chatops - exec bin/hubot $DAEMON_ARGS - ;; - "st2-register-content" ) - set -ex - PACKS=${PACKS:-"chatops core default linux packs"} - for PACK in ${PACKS}; do - st2-register-content --config-file /etc/st2/st2.conf \ - --register-all \ - --register-setup-virtualenvs \ - --register-pack /opt/stackstorm/packs/${PACK} - done - ;; - -esac diff --git a/images/stackstorm/bin/entrypoint.sh b/images/stackstorm/bin/entrypoint.sh deleted file mode 100755 index 320b2bf9..00000000 --- a/images/stackstorm/bin/entrypoint.sh +++ /dev/null @@ -1,75 +0,0 @@ -#!/bin/bash - -# Create htpasswd file and login to st2 using specified username/password -htpasswd -b /etc/st2/htpasswd ${ST2_USER} ${ST2_PASSWORD} - -mkdir -p /root/.st2 - -ROOT_CONF=/root/.st2/config - -touch ${ROOT_CONF} - -crudini --set ${ROOT_CONF} credentials username ${ST2_USER} -crudini --set ${ROOT_CONF} credentials password ${ST2_PASSWORD} - -ST2_CONF=/etc/st2/st2.conf - -ST2_API_URL=${ST2_API_URL:-http://127.0.0.1:9101} -MISTRAL_BASE_URL=${MISTRAL_BASE_URL:-http://127.0.0.1:8989/v2} - -crudini --set ${ST2_CONF} auth api_url ${ST2_API_URL} -crudini --set ${ST2_CONF} mistral api_url ${ST2_API_URL} -crudini --set ${ST2_CONF} mistral v2_base_url ${MISTRAL_BASE_URL} -crudini --set ${ST2_CONF} messaging url \ - amqp://${RABBITMQ_DEFAULT_USER}:${RABBITMQ_DEFAULT_PASS}@${RABBITMQ_HOST}:${RABBITMQ_PORT} -crudini --set ${ST2_CONF} coordination url \ - redis://:${REDIS_PASSWORD}@${REDIS_HOST}:${REDIS_PORT} -crudini --set ${ST2_CONF} database host ${MONGO_HOST} -crudini --set ${ST2_CONF} database port ${MONGO_PORT} -if [ ! -z ${MONGO_DB} ]; then - crudini --set ${ST2_CONF} database db_name ${MONGO_DB} -fi -if [ ! -z ${MONGO_USER} ]; then - crudini --set ${ST2_CONF} database username ${MONGO_USER} -fi -if [ ! -z ${MONGO_PASS} ]; then - crudini --set ${ST2_CONF} database password ${MONGO_PASS} -fi - -# NOTE: Only certain distros of MongoDB support SSL/TLS -# 1) enterprise versions -# 2) those built from source (https://github.com/mongodb/mongo/wiki/Build-Mongodb-From-Source) -# -#crudini --set ${ST2_CONF} database ssl True -#crudini --set ${ST2_CONF} database ssl_keyfile None -#crudini --set ${ST2_CONF} database ssl_certfile None -#crudini --set ${ST2_CONF} database ssl_cert_reqs None -#crudini --set ${ST2_CONF} database ssl_ca_certs None -#crudini --set ${ST2_CONF} database ssl_match_hostname True - -MISTRAL_CONF=/etc/mistral/mistral.conf - -crudini --set ${MISTRAL_CONF} DEFAULT transport_url \ - rabbit://${RABBITMQ_DEFAULT_USER}:${RABBITMQ_DEFAULT_PASS}@${RABBITMQ_HOST}:${RABBITMQ_PORT} -crudini --set ${MISTRAL_CONF} database connection \ - postgresql+psycopg2://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB} - -# Run custom init scripts -for f in /st2-docker/entrypoint.d/*; do - case "$f" in - *.sh) echo "$0: running $f"; . "$f" ;; - *) echo "$0: ignoring $f" ;; - esac - echo -done - -# 1ppc: launch entrypoint-1ppc.sh via dumb-init if $ST2_SERVICE is set -if [ ! -z ${ST2_SERVICE} ]; then - exec /dumb-init -- /st2-docker/bin/entrypoint-1ppc.sh -fi - -# Ensure the base st2 nginx config is used - -( cd /etc/nginx/conf.d && ln -sf st2-base.cnf st2.conf ) - -exec /sbin/init diff --git a/images/stackstorm/bin/inject_env.py b/images/stackstorm/bin/inject_env.py deleted file mode 100755 index 8c4c47b0..00000000 --- a/images/stackstorm/bin/inject_env.py +++ /dev/null @@ -1,29 +0,0 @@ -#!/opt/stackstorm/st2/bin/python - -""" - jinja2 template converter script - - This script will accept template input from STDIN, then render output to STDOUT - Within a template, you can access environment variables with `env['YOUR_ENVVAR']` - - Usage example: - env HOGE=fuga inject_env.py < template_file > output_file -""" - -import os -import sys -import jinja2 - -def striptrailingslash(value): - """ - custom filter that strips forwarding slashes - """ - return value.strip('/') - -# create jinja environment and add custom filters -environment = jinja2.Environment(loader=None) -environment.filters['striptrailingslash'] = striptrailingslash - -# load template string from STDIN, then render to STDOUT -template = environment.from_string(sys.stdin.read()) -sys.stdout.write(template.render(env=os.environ)) diff --git a/images/stackstorm/bin/install.sh b/images/stackstorm/bin/install.sh deleted file mode 100755 index 8b49b268..00000000 --- a/images/stackstorm/bin/install.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/bash - -set -euo pipefail -IDS=$'\n\t' - -# apt-cache may not have current package data without apt-get update -apt-get update - -declare -A vers=() -declare -A pkgs=( ["ST2_VERSION"]="st2" \ - ["ST2WEB_VERSION"]="st2web" \ - ["ST2MISTRAL_VERSION"]="st2mistral" \ - ["ST2CHATOPS_VERSION"]="st2chatops" ) - -# Expand keys of pkgs array. -for i in "${!pkgs[@]}" -do - # Save the newest available version of $pkgs[$i] - if [ -z ${!i:-} ]; then - vers["$i"]=$(apt-cache madison ${pkgs["$i"]} | cut -f 2 -d '|' | tr -d '[ \t]' | grep "^${ST2_TAG:-}" | head -1) - else - vers["$i"]=${!i} - fi -done - -# Install st2, st2web, and st2mistral -sudo apt-get install -y st2=${vers['ST2_VERSION']} st2web=${vers['ST2WEB_VERSION']} st2mistral=${vers['ST2MISTRAL_VERSION']} - -# Install st2chatops, but disable unless entrypoint.d file is present -# Using GNU sort's version comparison, this performs a descending sort on -# a two element list containing "2.10" and ${vers['ST2CHATOPS_VERSION']}. -# If the "2.10.0" element is the first element, then install node.js v10. -# Else, install node.js v6. -node_script="setup_6.x" -if [ $(printf "2.10.0\n${vers['ST2CHATOPS_VERSION']}\n" | sort -V | head -n 1) = "2.10.0" ]; then - node_script="setup_10.x" -fi - -curl -sL https://deb.nodesource.com/${node_script} \ - | sudo -E bash - && sudo apt-get install -y st2chatops=${vers['ST2CHATOPS_VERSION']} && echo manual \ - | sudo tee /etc/init/st2chatops.override - -MANIFEST="/st2-manifest.txt" - -echo "Image built at $(date)" > $MANIFEST - -if [[ "${CIRCLE_PROJECT_REPONAME:-}" != "" ]] && [[ "${CIRCLE_PROJECT_USERNAME:-}" != "" ]] && [[ "${CIRCLE_SHA1:-}" != "" ]]; then - echo "GitHub URL: https://github.com/${CIRCLE_PROJECT_USERNAME:-}/${CIRCLE_PROJECT_REPONAME:-}/commit/${CIRCLE_SHA1:-}" >> $MANIFEST -fi -if [[ "${CIRCLE_PROJECT_REPONAME:-}" == "" ]] && [[ "${CIRCLE_PROJECT_USERNAME:-}" == "" ]] && [[ "${CIRCLE_SHA1:-}" != "" ]]; then - echo "Commit SHA: ${CIRCLE_SHA1:-}" >> $MANIFEST -fi -if [[ "${CIRCLE_BUILD_URL:-}" != "" ]]; then - echo "Build URL: ${CIRCLE_BUILD_URL:-}" >> $MANIFEST -fi -if [[ "${ST2_TAG:-}" != "" ]]; then - echo "Tag: ${ST2_TAG:-}" >> $MANIFEST -fi - -echo "" >> $MANIFEST - -echo "Installed versions:" >> $MANIFEST -for i in "${!pkgs[@]}" -do - echo " - ${pkgs[$i]}-${vers[$i]}" >> $MANIFEST -done diff --git a/images/stackstorm/bin/st2.sh b/images/stackstorm/bin/st2.sh deleted file mode 100755 index 3f2492b8..00000000 --- a/images/stackstorm/bin/st2.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -ENV_FILE=/st2-docker/env - -if [ -f ${ENV_FILE} ]; then - source ${ENV_FILE} -fi - -# Run custom init scripts which require ST2 to be running -for f in /st2-docker/st2.d/*; do - case "$f" in - *.sh) echo "$0: running $f"; . "$f" ;; - *) echo "$0: ignoring $f" ;; - esac - echo -done diff --git a/images/stackstorm/bin/test.sh b/images/stackstorm/bin/test.sh deleted file mode 100755 index 2a968f49..00000000 --- a/images/stackstorm/bin/test.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -eux - -# Run Integration Tests via Inspec Infra testing framework -# https://www.inspec.io - -echo -e '\033[33mRunning Inspec Integration Tests ...\033[0m' -cd /st2-docker/test/integration -for dir in */; do - dir=$(basename $dir) - if [ -f "${dir}/inspec.yml" ]; then - echo -e "\nRunning tests for \033[1;36m${dir}\033[0m ..." - sudo inspec exec --show-progress ${dir} - fi -done diff --git a/images/stackstorm/config/init-fake.conf b/images/stackstorm/config/init-fake.conf deleted file mode 100644 index 46c39c99..00000000 --- a/images/stackstorm/config/init-fake.conf +++ /dev/null @@ -1,13 +0,0 @@ -# fake some events needed for correct startup other services - -description "In-Container Upstart Fake Events" - -start on startup - -script - rm -rf /var/run/*.pid - rm -rf /var/run/network/* - /sbin/initctl emit stopped JOB=udevtrigger --no-wait - /sbin/initctl emit started JOB=udev --no-wait - /sbin/initctl emit runlevel RUNLEVEL=3 --no-wait -end script diff --git a/images/stackstorm/config/local.conf b/images/stackstorm/config/local.conf deleted file mode 100644 index 0ad081ef..00000000 --- a/images/stackstorm/config/local.conf +++ /dev/null @@ -1,9 +0,0 @@ -description "Post ST2 API start scripts" -author "StackStorm" - -start on started st2api -stop on shutdown - -script -/st2-docker/bin/st2.sh -end script diff --git a/images/stackstorm/config/nginx.st2-1ppc.conf.tpl b/images/stackstorm/config/nginx.st2-1ppc.conf.tpl deleted file mode 100644 index e5cea707..00000000 --- a/images/stackstorm/config/nginx.st2-1ppc.conf.tpl +++ /dev/null @@ -1,151 +0,0 @@ -# -# nginx configuration to expose st2 webui, redirect HTTP->HTTPS, -# provide SSL termination, and reverse-proxy st2api and st2auth API endpoint. -# To enable: -# cp ${LOCATION}/st2.conf /etc/nginx/sites-available -# ln -l /etc/nginx/sites-available/st2.conf /etc/nginx/sites-enabled/st2.conf -# see https://docs.stackstorm.com/install.html for details - -server { - listen *:80 default_server; - - add_header Front-End-Https on; - add_header X-Content-Type-Options nosniff; - - if ($ssl_protocol = "") { - return 301 https://$host$request_uri; - } - - index index.html; - - access_log /var/log/nginx/st2webui.access.log combined; - error_log /var/log/nginx/st2webui.error.log; -} - -server { - listen *:443 ssl; - - ssl on; - - ssl_certificate /etc/ssl/st2/st2.crt; - ssl_certificate_key /etc/ssl/st2/st2.key; - ssl_session_cache shared:SSL:10m; - ssl_session_timeout 5m; - ssl_protocols TLSv1 TLSv1.1 TLSv1.2; - ssl_ciphers EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH:ECDHE-RSA-AES128-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA128:DHE-RSA-AES128-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES128-GCM-SHA128:ECDHE-RSA-AES128-SHA384:ECDHE-RSA-AES128-SHA128:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES128-SHA:DHE-RSA-AES128-SHA128:DHE-RSA-AES128-SHA128:DHE-RSA-AES128-SHA:DHE-RSA-AES128-SHA:AES128-GCM-SHA384:AES128-GCM-SHA128:AES128-SHA128:AES128-SHA128:AES128-SHA:AES128-SHA:HIGH:!aNULL:!eNULL:!EXPORT:!DES:!MD5:!PSK:!RC4; - ssl_prefer_server_ciphers on; - - index index.html; - - access_log /var/log/nginx/ssl-st2webui.access.log combined; - error_log /var/log/nginx/ssl-st2webui.error.log; - - add_header Front-End-Https on; - add_header X-Content-Type-Options nosniff; - - resolver {{ env['ST2WEB_DNS_RESOLVER'] | default('127.0.0.1') }} valid=10s ipv6=off; - - location @apiError { - add_header Content-Type application/json always; - return 503 '{ "faultstring": "Nginx is unable to reach st2api. Make sure service is running." }'; - } - - location /api/ { - error_page 502 = @apiError; - - set $st2_api_url {{ env['ST2_API_URL'] | striptrailingslash }}; - - rewrite ^/api/(.*) /$1 break; - - proxy_pass $st2_api_url$uri$is_args$args; - proxy_read_timeout 90; - proxy_connect_timeout 90; - proxy_redirect off; - - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - - proxy_set_header Connection ''; - chunked_transfer_encoding off; - proxy_buffering off; - proxy_cache off; - proxy_set_header Host $host; - } - - location @streamError { - add_header Content-Type text/event-stream; - return 200 "retry: 1000\n\n"; - } - - # For backward compatibility reasons, rewrite requests from "/api/stream" - # to "/stream/v1/stream" and "/api/v1/stream" to "/stream/v1/stream" - location /api/stream { - rewrite ^/api/stream/?(.*)$ /stream/v1/stream/$1 last; - } - location /api/v1/stream { - rewrite ^/api/v1/stream/?(.*)$ /stream/v1/stream/$1 last; - } - location /stream/ { - error_page 502 = @streamError; - - set $st2_stream_url {{ env['ST2_STREAM_URL'] | striptrailingslash }}; - - rewrite ^/stream/(.*) /$1 break; - - proxy_pass $st2_stream_url$uri$is_args$args; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - - sendfile on; - tcp_nopush on; - tcp_nodelay on; - - # Disable buffering and chunked encoding. - # In the stream case we want to receive the whole payload at once, we don't - # want multiple chunks. - proxy_set_header Connection ''; - chunked_transfer_encoding off; - proxy_buffering off; - proxy_cache off; - } - - location @authError { - add_header Content-Type application/json always; - return 503 '{ "faultstring": "Nginx is unable to reach st2auth. Make sure service is running." }'; - } - - location /auth/ { - error_page 502 = @authError; - - set $st2_auth_url {{ env['ST2_AUTH_URL'] | striptrailingslash }}; - - rewrite ^/auth/(.*) /$1 break; - - proxy_pass $st2_auth_url$uri$is_args$args; - proxy_read_timeout 90; - proxy_connect_timeout 90; - proxy_redirect off; - - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_pass_header Authorization; - - proxy_set_header Connection ''; - chunked_transfer_encoding off; - proxy_buffering off; - proxy_cache off; - proxy_set_header Host $host; - } - - location / { - root /opt/stackstorm/static/webui/; - index index.html; - - sendfile on; - tcp_nopush on; - tcp_nodelay on; - } -} diff --git a/runtime/compose-1ppc/.gitignore b/runtime/compose-1ppc/.gitignore deleted file mode 100644 index c9a5c234..00000000 --- a/runtime/compose-1ppc/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/conf diff --git a/runtime/compose-1ppc/README.md b/runtime/compose-1ppc/README.md deleted file mode 100644 index e7a206b7..00000000 --- a/runtime/compose-1ppc/README.md +++ /dev/null @@ -1,119 +0,0 @@ -# StackStorm 1ppc: One Process Per Container - -**What's this?** - -StackStorm Docker image that runs one st2 service per container. - -**Why we need this?** - -> Each container should have only one concern - -*Quote from [Best practices for writing Dockerfiles](https://docs.docker.com/engine/userguide/eng-image/dockerfile_best-practices/)* - -## Getting started - -A sample `docker-compose.yml` file is located under `runtime/compose-1ppc` directory. -Follow the instruction below to setup a running StackStorm instance which consists of -containers that each are running individual st2 service. - -1. Generate .env files - -``` -(cd ../../ && make env) -cp -r ../../conf . -``` - -2. Start containers - -``` -docker-compose up -d -``` - -Now you can access StackStorm Web UI. - -3. Register initial content - -``` -docker-compose exec st2actionrunner \ - st2-register-content --config-file /etc/st2/st2.conf \ - --register-all --register-setup-virtualenvs -``` - -Note: `/opt/stackstorm/virtualenvs` directory needs to be mounted as a shared volume on -the container that you run the above command. - -4. Run simple action - -``` -docker-compose exec st2client st2 run core.local cmd=date -``` - -5. Install examples - -``` -docker-compose exec st2client st2 pack install https://github.com/shusugmt/st2-pack-examples -``` - -6. Run mistral example - -``` -docker-compose exec st2client st2 run examples.mistral_examples -``` - -**FAQ** - -- Q: Fails to run mistral actions -- A: Restart `mistrap-api` or `mistral-server` container once by `docker-compose up -d --force-recreate mistral-api` - - This is caused by the conflict of `mistral-db-manage` command being invoked by both `mistral-api` and - `mistral-server`. When you first run `docker-compose up -d` the command runs in both containers almost at the - same time and simply fails to load the required content into database. Restarting one of them will re-run - the command again and populate postgres with a proper data. -- Q: I can login to the Web UI but when I click any link, I'm redirected back to login page -- A: Check you docker host clock - -### Scaling out - -``` -docker-compose up --scale st2actionrunner=3 -d -``` - -## Additional environment variables in 1ppc - -| Parameter | Description | -|-----------|-------------| -| `ST2WEB_DNS_RESOLVER` | *(Optional)* Hostname or address of the DNS resolver that nginx running in st2web container will use. Default is `127.0.0.1` which is suitable for sidecar pattern in Kubernetes. | - -### Sharing Content - -See [official document](https://docs.stackstorm.com/reference/ha.html#sharing-content) for details. - -- `/opt/stackstorm/packs` - - st2api - - st2actionrunner - - st2sensorcontainer -- `/opt/stackstorm/virtualenvs` - - st2actionrunner - - st2sensorcontainer - - -### Running st2chatops - -Add following service entry to `docker-compose.yml` - -``` - st2chatops: - <<: *base - environment: - - ST2_SERVICE=st2chatops - - HUBOT_ADAPTER=slack - - HUBOT_SLACK_TOKEN=xoxb-CHANGE-ME-PLEASE - - ST2_API_KEY=CHANGE-ME-PLEASE -``` - -See official docs and `/opt/stackstorm/chatops/st2chatops.env` for chatops configuration details. - - -### Notes - -- Currently all inter-service connections are done via plain http, which might be a problem in - production setup. diff --git a/runtime/compose-1ppc/common.env b/runtime/compose-1ppc/common.env deleted file mode 100644 index 0851c659..00000000 --- a/runtime/compose-1ppc/common.env +++ /dev/null @@ -1,5 +0,0 @@ -ST2_AUTH_URL=http://st2auth:9100/ -ST2_API_URL=http://st2api:9101/ -ST2_STREAM_URL=http://st2stream:9102/ - -ST2_MISTRAL_API_URL=http://mistral-api:8989/v2 diff --git a/runtime/compose-1ppc/docker-compose.yml b/runtime/compose-1ppc/docker-compose.yml deleted file mode 100644 index 12debd77..00000000 --- a/runtime/compose-1ppc/docker-compose.yml +++ /dev/null @@ -1,165 +0,0 @@ -version: '3' - -services: - st2client: &base - image: stackstorm/stackstorm:${ST2_IMAGE_TAG:-latest} - env_file: - - conf/stackstorm.env - - conf/mongo.env - - conf/rabbitmq.env - - conf/postgres.env - - conf/redis.env - - common.env - networks: - - private - restart: on-failure - environment: - - ST2_SERVICE=nop - dns_search: . - st2api: - <<: *base - environment: - - ST2_SERVICE=st2api - volumes: - - stackstorm-packs:/opt/stackstorm/packs - st2auth: - <<: *base - environment: - - ST2_SERVICE=st2auth - st2stream: - <<: *base - environment: - - ST2_SERVICE=st2stream - st2sensorcontainer: - <<: *base - environment: - - ST2_SERVICE=st2sensorcontainer - volumes: - - stackstorm-packs:/opt/stackstorm/packs - - stackstorm-virtualenvs:/opt/stackstorm/virtualenvs - st2rulesengine: - <<: *base - environment: - - ST2_SERVICE=st2rulesengine - st2timersengine: - <<: *base - environment: - - ST2_SERVICE=st2timersengine - st2workflowengine: - <<: *base - environment: - - ST2_SERVICE=st2workflowengine - st2actionrunner: - <<: *base - environment: - - ST2_SERVICE=st2actionrunner - - ST2_ACTION_AUTH_URL=http://st2auth:9100/ - volumes: - - stackstorm-packs:/opt/stackstorm/packs - - stackstorm-virtualenvs:/opt/stackstorm/virtualenvs - st2scheduler: - <<: *base - environment: - - ST2_SERVICE=st2scheduler - st2resultstracker: - <<: *base - environment: - - ST2_SERVICE=st2resultstracker - st2notifier: - <<: *base - environment: - - ST2_SERVICE=st2notifier - st2garbagecollector: - <<: *base - environment: - - ST2_SERVICE=st2garbagecollector - mistral-api: - <<: *base - environment: - - ST2_SERVICE=mistral-api - restart: on-failure - mistral-server: - <<: *base - environment: - - ST2_SERVICE=mistral-server - restart: on-failure - st2web: - <<: *base - environment: - - ST2_SERVICE=st2web - - ST2WEB_DNS_RESOLVER=st2web-dns - ports: - - "443:443" - networks: - - public - - private - - st2web-dns: - image: janeczku/go-dnsmasq:latest - environment: - - DNSMASQ_ENABLE_SEARCH=1 - networks: - - private - dns_search: . - -### External Services - - mongo: - image: mongo:3.4 - env_file: - - conf/mongo.env - networks: - - private - volumes: - - mongo-volume:/data/db - - mongo-configdb-volume:/data/configdb - dns_search: . - rabbitmq: - image: rabbitmq:management - env_file: - - conf/rabbitmq.env - networks: - - private - volumes: - - rabbitmq-volume:/var/lib/rabbitmq - dns_search: . - postgres: - image: postgres:latest - env_file: - - conf/postgres.env - networks: - - private - volumes: - - postgres-volume:/var/lib/postgresql/data - dns_search: . - redis: - image: redis:latest - env_file: - - conf/redis.env - networks: - - private - volumes: - - redis-volume:/data - dns_search: . - command: [ - "bash", "-c", - ' - docker-entrypoint.sh - --requirepass "$$REDIS_PASSWORD" - ' - ] - -volumes: - mongo-volume: - mongo-configdb-volume: - postgres-volume: - rabbitmq-volume: - redis-volume: - stackstorm-packs: - stackstorm-virtualenvs: - -networks: - public: - driver: bridge - private: - driver: bridge diff --git a/runtime/entrypoint.d/README.md b/runtime/entrypoint.d/README.md deleted file mode 100644 index d4ff4002..00000000 --- a/runtime/entrypoint.d/README.md +++ /dev/null @@ -1,10 +0,0 @@ -# Description - -As soon as the `stackstorm` container is started, and before StackStorm services are started -by init, shell scripts with suffix `.sh` located in this directory are executed in alphanumeric -order of the file name. - -Scripts in this directory can be used to write configuration files required by StackStorm. - -NOTE: The scripts must not rely on any StackStorm service being available. If you require -StackStorm to be running, then place the scripts in the `st2.d` directory instead. diff --git a/runtime/entrypoint.d/add-packs-dev.sh b/runtime/entrypoint.d/add-packs-dev.sh deleted file mode 100755 index 78901cdf..00000000 --- a/runtime/entrypoint.d/add-packs-dev.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -mkdir -p /opt/stackstorm/packs.dev -crudini --set /etc/st2/st2.conf content packs_base_paths /opt/stackstorm/packs.dev diff --git a/runtime/st2.d/README.md b/runtime/st2.d/README.md deleted file mode 100644 index 1ec8332a..00000000 --- a/runtime/st2.d/README.md +++ /dev/null @@ -1,9 +0,0 @@ -# Description - -As soon as the `st2api` service is running, scripts with suffix `.sh` located in this -directory are executed in alphanumeric order of the file name. - -Scripts in this directory can be used to interact with StackStorm. For example: - - - reload and/or restart StackStorm services, or - - perform any operation that requires StackStorm to be running. diff --git a/runtime/st2.d/reload.sh b/runtime/st2.d/reload.sh deleted file mode 100755 index e9effa5c..00000000 --- a/runtime/st2.d/reload.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -st2ctl reload --register-all diff --git a/runtime/st2.d/virtualenv.sh b/runtime/st2.d/virtualenv.sh deleted file mode 100755 index e88ab4c2..00000000 --- a/runtime/st2.d/virtualenv.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -EXAMPLES=/opt/stackstorm/packs/examples - -if [ ! -d "$EXAMPLES" ]; then - echo "Installing examples..." - cp -R /usr/share/doc/st2/examples /opt/stackstorm/packs - chgrp -R st2packs /opt/stackstorm/packs/examples - st2 run packs.setup_virtualenv packs=examples -fi diff --git a/tutorial/actions/hello-concurrency.yaml b/tutorial/actions/hello-concurrency.yaml deleted file mode 100644 index f0ae415f..00000000 --- a/tutorial/actions/hello-concurrency.yaml +++ /dev/null @@ -1,13 +0,0 @@ ---- -name: "hello-concurrency" -runner_type: "local-shell-cmd" -description: "Says hello with a shell command" -entry_point: "" -parameters: - cmd: - immutable: true - default: "sleep 10 && echo Hello {{name}}!" - name: - type: string - description: "A human name" - default: "human" diff --git a/tutorial/actions/hello.yaml b/tutorial/actions/hello.yaml deleted file mode 100644 index aac824d7..00000000 --- a/tutorial/actions/hello.yaml +++ /dev/null @@ -1,13 +0,0 @@ ---- -name: "hello" -runner_type: "local-shell-cmd" -description: "Says hello with a shell command" -entry_point: "" -parameters: - cmd: - immutable: true - default: "echo Hello {{name}}!" - name: - type: string - description: "A human name" - default: "human" diff --git a/tutorial/actions/mistral-basic.yaml b/tutorial/actions/mistral-basic.yaml deleted file mode 100644 index 3d2b9cd8..00000000 --- a/tutorial/actions/mistral-basic.yaml +++ /dev/null @@ -1,11 +0,0 @@ ---- -description: Run a local linux command -enabled: true -entry_point: workflows/mistral-basic.yaml -name: mistral-basic -pack: tutorial -parameters: - cmd: - required: true - type: string -runner_type: mistral-v2 diff --git a/tutorial/actions/workflows/mistral-basic.yaml b/tutorial/actions/workflows/mistral-basic.yaml deleted file mode 100644 index 12d8443d..00000000 --- a/tutorial/actions/workflows/mistral-basic.yaml +++ /dev/null @@ -1,15 +0,0 @@ -version: '2.0' - -examples.mistral-basic: - description: A basic workflow that runs an arbitrary linux command. - type: direct - input: - - cmd - output: - stdout: <% $.stdout %> - tasks: - task1: - action: core.local cmd=<% $.cmd %> - publish: - stdout: <% task(task1).result.stdout %> - stderr: <% task(task1).result.stderr %> diff --git a/tutorial/policies/hello-concurrency.yaml b/tutorial/policies/hello-concurrency.yaml deleted file mode 100644 index b60d2b03..00000000 --- a/tutorial/policies/hello-concurrency.yaml +++ /dev/null @@ -1,8 +0,0 @@ -name: hello-concurrency -description: Limits the concurrent executions for hello. -enabled: true -resource_ref: tutorial.hello-concurrency -policy_type: action.concurrency -parameters: - action: delay - threshold: 1 diff --git a/tutorial/rules/monitor_file.yaml b/tutorial/rules/monitor_file.yaml deleted file mode 100644 index 65000291..00000000 --- a/tutorial/rules/monitor_file.yaml +++ /dev/null @@ -1,17 +0,0 @@ ---- -name: monitor_file -pack: "tutorial" -description: "Monitor file for change" -enabled: true - -trigger: - parameters: - file_path: /tmp/watcher.log - type: linux.file_watch.line - -criteria: {} - -action: - parameters: - cmd: echo "{{trigger}}" - ref: core.local From 05ef98c4a59b2b42d2b89844a192b2cb418716c9 Mon Sep 17 00:00:00 2001 From: armab Date: Fri, 19 Jun 2020 18:40:11 +0100 Subject: [PATCH 02/32] Add 'StackStorm Authors' copyright note following the LF guidelines https://www.linuxfoundation.org/blog/2020/01/copyright-notices-in-open-source-software-projects/ --- LICENSE | 1 + 1 file changed, 1 insertion(+) diff --git a/LICENSE b/LICENSE index 50860225..123b1687 100644 --- a/LICENSE +++ b/LICENSE @@ -186,6 +186,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. + Copyright 2020 The StackStorm Authors. Copyright 2017 StackStorm, Inc. Licensed under the Apache License, Version 2.0 (the "License"); From c0a840bcea3f561ba02d0adc2898860deb098230 Mon Sep 17 00:00:00 2001 From: John Hogenmiller Date: Wed, 6 May 2020 20:25:51 +0000 Subject: [PATCH 03/32] feat: add docker-compose setup --- README.md | 305 +++++++++++++++++++++++++++++++++++++++++++++ docker-compose.yml | 194 ++++++++++++++++++++++++++++ htpasswd | 2 + st2-cli.conf | 3 + st2.docker.conf | 23 ++++ st2.user.conf | 6 + 6 files changed, 533 insertions(+) create mode 100644 README.md create mode 100644 docker-compose.yml create mode 100644 htpasswd create mode 100644 st2-cli.conf create mode 100644 st2.docker.conf create mode 100644 st2.user.conf diff --git a/README.md b/README.md new file mode 100644 index 00000000..8f758d3f --- /dev/null +++ b/README.md @@ -0,0 +1,305 @@ +# StackStorm in Docker Compose + +This docker-compose is provided as a way to allow someone to "get up and running" quickly with StackStorm using Docker. It is not designed to be used in production, but rather a way to test out StackStorm and facilitate pack development. + +## TL;DR + +```shell +git clone git@github.com:stackstorm/st2-dockerfiles +cd st2-docker/stackstorm-compose +docker-compose up -d +docker-compose run st2api st2ctl reload --register-all # you should only have to do this the first time +docker-compose run st2client bash # this gives you access to the st2 command line +``` + +Open `http://localhost:8000` in your browser. StackStorm Username/Password by default is: st2admin/Ch@ngeMe. + +## Usage + +### Prerequisites + +- Docker Engine 18.09+ +- Docker Compose 1.12+ + +### Compose Configuration + +The image version, exposed ports, and "packs.dev" directory is configurable with environment variables. + +- **ST2_VERSION** this is the tag at the end of the docker image (ie: stackstorm/st2api:v3.3dev) +- **ST2_IMAGE_REPO** The image or path to the images. Default is "stackstorm/". You may change this is using the Enterprise version or a private docker repository. +- **ST2_EXPOSE_HTTP** Port to expose st2web port 80 on. Default is `8000`, and you amy want to do `0.0.0.0:8000` to expose on all interfaces. +- **ST2_PACKS_DEV** Directory to development packs, absolute or relative to docker-compose.yml. This allows you to develop packs locally. Default is `./packs.dev`. When making a number of packs, it is recommended to make a directory outside of st2-dockerfiles, with each subdirectory underneath that being an independent git repo. Example: `ST2_PACKS_DEV=${HOME}/mypacks`, with `${HOME}/mypacks/st2-helloworld` being a git repo for the "helloworld" pack. + +### Credentials + +The `htpasswd` file is created with a default username of `st2admin` and a default password of `Ch@ngeMe`. This can be changed using the [htpasswd utility](https://httpd.apache.org/docs/2.4/programs/htpasswd.html). + +Another file (`st2-cli.conf`) contains default credentials and is mounted into the "st2client" container. If you change credentials in htpasswd, you will probably want to change them in st2-cli.conf. + +### Further configuration + +The base st2 docker images have a built-in `/etc/st2/st2.conf` configuration file. Each st2 Docker image will load: + +- /etc/st2/st2.conf +- /etc/st2/st2.docker.conf (values here will override st2.conf) +- /etc/st2/st2.user.conf (values here will override st2.docker.conf) + +Review `../base/files/st2.tmp.conf` and `st2.docker.conf` for currently set values, and it is recommended to place overrides in `st2.user.conf`. + +### Step by step first time instructions + +First, optionally set and export all the environment variables you want to change. You could make a .env file with customizations. + +Example: + +```shell +export ST2_PACKS_DEV=$HOME/projects/stackstorm-packs +export ST2_EXPOSE_HTTP=0.0.0.0:8000 +``` + +Secondly make any customizations to st2.user.conf, htpasswd, and st2-cli.conf. + +Example: + +To enable [sharing code between actions and sensors](https://docs.stackstorm.com/reference/sharing_code_sensors_actions.html), add these two lines to st2.user.conf: + +```ini +[packs] +enable_common_libs = True +``` + +Third, start the docker environment: + +```shell +docker-compose up -d +``` + +This will pull the required images from docker hub, and then start them. + +To stop the docker environment, run: + +```shell +docker-compose down +``` + +Fourth, if this is your first time running StackStorm in Docker Compose, you will need to force st2api to register everything. + +```shell +docker-compose run st2api st2ctl reload --register-all +``` + +## Regular Usage + +To run st2 commands, you can use the st2client service: + +```shell +docker-compose run st2client st2 +``` + +Example: + +```shell +$ docker-compose run st2client st2 run core.echo message=hello +. +id: 5eb30d77afe5aa8493f31187 +action.ref: core.echo +context.user: st2admin +parameters: + message: hello +status: succeeded +start_timestamp: Wed, 06 May 2020 19:18:15 UTC +end_timestamp: Wed, 06 May 2020 19:18:15 UTC +result: + failed: false + return_code: 0 + stderr: '' + stdout: hello + succeeded: true +``` + +Alternatively, you could run `docker-compose run st2client bash` to be dropped into a container with st2. At that point, you can just run `st2` commands. + +Example: + +```shell +$ docker-compose run st2client bash +Welcome to StackStorm HA v3.3dev (Ubuntu 16.04 LTS GNU/Linux x86_64) + * Documentation: https://docs.stackstorm.com/ + * Community: https://stackstorm.com/community-signup + * Forum: https://forum.stackstorm.com/ + * Enterprise: https://stackstorm.com/#product + + Warning! Do not edit configs, packs or any content inplace as they will be overridden. Modify Helm values.yaml instead! + It's recommended to use st2client container to work with StackStorm cluster. + +root@aaabd11745f0:/opt/stackstorm# st2 run core.echo message="from the inside" +. +id: 5eb310f571af8f57a4582430 +action.ref: core.echo +context.user: st2admin +parameters: + message: from the inside +status: succeeded +start_timestamp: Wed, 06 May 2020 19:33:09 UTC +end_timestamp: Wed, 06 May 2020 19:33:09 UTC +result: + failed: false + return_code: 0 + stderr: '' + stdout: from the inside + succeeded: true +``` + +## Pack Configuration + +Pack configs will be in /opt/stackstorm/configs/$PACKNAME, which is a docker volume shared between st2api, st2actionrunner, and st2sensorcontainer. You can use the `st2 pack config ` in the st2client container in order to configure a pack. + +### Use st2 pack config + +```shell +$ docker-compose run st2client st2 pack config git +repositories[0].url: https://github.com/StackStorm/st2-dockerfiles.git +repositories[0].branch [master]: +~~~ Would you like to add another item to "repositories" array / list? [y]: n +--- +Do you want to preview the config in an editor before saving? [y]: n +--- +Do you want me to save it? [y]: y ++----------+--------------------------------------------------------------+ +| Property | Value | ++----------+--------------------------------------------------------------+ +| id | 5eb3164f566aa824ea88f536 | +| pack | git | +| values | { | +| | "repositories": [ | +| | { | +| | "url": | +| | "https://github.com/StackStorm/st2-dockerfiles.git", | +| | "branch": "master" | +| | } | +| | ] | +| | } | ++----------+--------------------------------------------------------------+ +``` + +### Copy a config file into a container + +First, find the actual container name of st2api by running `docker-compose ps st2api`. + +```shell +$ docker-compose ps st2api + Name Command State Ports +-------------------------------------------------------------------- +compose_st2api_1 /opt/stackstorm/st2/bin/st ... Up 9101/tcp +``` + +Next, use `docker cp` to copy your file into place. + +```shell +docker cp git.yaml compose_st2api_1:/opt/stackstorm/configs/git.yaml +``` + +## Register the configs + +If you copied the file in or, you will need to register it. st2client does not need access to the configs directory, as it will talk to st2api. + +```shell +$ docker-compose run st2client st2 run packs.load packs=git register=configs +. +id: 5eb3171c566aa824ea88f538 +action.ref: packs.load +context.user: st2admin +parameters: + packs: + - git + register: configs +status: succeeded +start_timestamp: Wed, 06 May 2020 19:59:24 UTC +end_timestamp: Wed, 06 May 2020 19:59:25 UTC +result: + exit_code: 0 + result: + configs: 1 + stdout: '' +``` + +## Local Pack Development + +See [Create and Contribute a Pack](https://docs.stackstorm.com/reference/packs.html) for how to actually develop a pack. + +If you are working on a development pack, you will need to register it and install the virutalenv (if it's python). + +### packs.dev directory + +As mentioned above, your default `packs.dev` directory is relative to your `docker-compose.yml` file. However, if you start developing here, git will not like being inside another git directory. You will want to set `ST2_PACKS_DEV` to a directory outside of `st2-dockerfiles` and restart the docker-compose services. + +Example: We have a pack called helloworld in `packs.dev/helloworld`. The directory name has to match the pack name. So even if you have a git repo named "st2-helloworld", it should be cloned locally as "helloworld". + +For these examples, we will be operating inside the st2client container. + +### Register the pack + +Register the pack by running `st2 run packs.load packs=, register=all`. Alternatively you can specify different register option (like register=actions) to focus on the parts you need to (re)register. You will be running this command a lot as you develop actions, sensors, rules and workflows. + +```shell +root@aaabd11745f0:/opt/stackstorm# st2 run packs.load packs=helloworld register=all +. +id: 5eb3100f71af8f57a458241f +action.ref: packs.load +context.user: st2admin +parameters: + packs: + - helloworld + register: all +status: succeeded +start_timestamp: Wed, 06 May 2020 19:29:19 UTC +end_timestamp: Wed, 06 May 2020 19:29:21 UTC +result: + exit_code: 0 + result: + actions: 13 + aliases: 0 + configs: 0 + policies: 0 + policy_types: 3 + rule_types: 2 + rules: 0 + runners: 15 + sensors: 0 + triggers: 0 +``` + +### Create the Python Virtual Environment + +If you are using python-runners in your locally developed pack, you will need to create the virtual environment by hand. You should typically only have to run this if you have changed your requirements.txt. + +To setup the virtual environment: `st2 run packs.setup_virtualenv packs=,` + +```shell +root@aaabd11745f0:/opt/stackstorm# st2 run packs.setup_virtualenv packs=helloworld +.... +id: 5eb311f871af8f57a4582433 +action.ref: packs.setup_virtualenv +context.user: st2admin +parameters: + packs: + - helloworld +status: succeeded +start_timestamp: Wed, 06 May 2020 19:37:28 UTC +end_timestamp: Wed, 06 May 2020 19:37:36 UTC +result: + exit_code: 0 + result: 'Successfully set up virtualenv for the following packs: helloworld' + stderr: 'st2.actions.python.SetupVirtualEnvironmentAction: DEBUG Setting up virtualenv for pack "helloworld" (/opt/stackstorm/packs.dev/helloworld) + st2.actions.python.SetupVirtualEnvironmentAction: INFO Virtualenv path "/opt/stackstorm/virtualenvs/helloworld" doesn''t exist + st2.actions.python.SetupVirtualEnvironmentAction: DEBUG Creating virtualenv for pack "helloworld" in "/opt/stackstorm/virtualenvs/helloworld" + st2.actions.python.SetupVirtualEnvironmentAction: DEBUG Creating virtualenv in "/opt/stackstorm/virtualenvs/helloworld" using Python binary "/opt/stackstorm/st2/bin/python" + st2.actions.python.SetupVirtualEnvironmentAction: DEBUG Running command "/opt/stackstorm/st2/bin/virtualenv -p /opt/stackstorm/st2/bin/python --always-copy --no-download /opt/stackstorm/virtualenvs/helloworld" to create virtualenv. + st2.actions.python.SetupVirtualEnvironmentAction: DEBUG Installing base requirements + st2.actions.python.SetupVirtualEnvironmentAction: DEBUG Installing requirement six>=1.9.0,<2.0 with command /opt/stackstorm/virtualenvs/helloworld/bin/pip install six>=1.9.0,<2.0. + st2.actions.python.SetupVirtualEnvironmentAction: DEBUG Installing pack specific requirements from "/opt/stackstorm/packs.dev/helloworld/requirements.txt" + st2.actions.python.SetupVirtualEnvironmentAction: DEBUG Installing requirements from file /opt/stackstorm/packs.dev/helloworld/requirements.txt with command /opt/stackstorm/virtualenvs/helloworld/bin/pip install -U -r /opt/stackstorm/packs.dev/helloworld/requirements.txt. + st2.actions.python.SetupVirtualEnvironmentAction: DEBUG Virtualenv for pack "helloworld" successfully created in "/opt/stackstorm/virtualenvs/helloworld" + ' + stdout: '' +``` diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..688ec9e5 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,194 @@ +version: '3' + +services: + st2web: + image: ${ST2_IMAGE_REPO:-stackstorm/}st2web:${ST2_VERSION:-3.3dev} + environment: + ST2_AUTH_URL: ${ST2_AUTH_URL:-http://st2auth:9100/} + ST2_API_URL: ${ST2_API_URL:-http://st2api:9101/} + ST2_STREAM_URL: ${ST2_STREAM_URL:-http://st2stream:9102/} + ST2WEB_HTTPS: ${ST2WEB_HTTPS:-0} + depends_on: + - st2auth + - st2api + - st2stream + ports: + - "${ST2_EXPOSE_HTTP:-8000}:80" + # - "${ST2_EXPOSE_HTTPS:-8443}:443" + # more work would be needed with certificate generate to make https work. + networks: + - private + - public + dns_search: . + st2api: + image: ${ST2_IMAGE_REPO:-stackstorm/}st2api:${ST2_VERSION:-3.3dev} + depends_on: ["mongo", "rabbitmq"] + networks: + - private + environment: + ST2_AUTH_URL: ${ST2_AUTH_URL:-http://st2auth:9100/} + ST2_API_URL: ${ST2_API_URL:-http://st2api:9101/} + ST2_STREAM_URL: ${ST2_STREAM_URL:-http://st2stream:9102/} + volumes: + - ./st2.docker.conf:/etc/st2/st2.docker.conf:ro + - ./st2.user.conf:/etc/st2/st2.user.conf:ro + - stackstorm-packs-configs:/opt/stackstorm/configs:rw + - stackstorm-packs:/opt/stackstorm/packs:rw + - ${ST2_PACKS_DEV:-./packs.dev}:/opt/stackstorm/packs.dev:rw + dns_search: . + st2stream: + image: ${ST2_IMAGE_REPO:-stackstorm/}st2stream:${ST2_VERSION:-3.3dev} + depends_on: ["st2api"] + networks: + - private + volumes: + - ./st2.docker.conf:/etc/st2/st2.docker.conf:ro + - ./st2.user.conf:/etc/st2/st2.user.conf:ro + dns_search: . + st2scheduler: + image: ${ST2_IMAGE_REPO:-stackstorm/}st2scheduler:${ST2_VERSION:-3.3dev} + depends_on: ["st2api"] + networks: + - private + volumes: + - ./st2.docker.conf:/etc/st2/st2.docker.conf:ro + - ./st2.user.conf:/etc/st2/st2.user.conf:ro + dns_search: . + st2workflowengine: + image: ${ST2_IMAGE_REPO:-stackstorm/}st2workflowengine:${ST2_VERSION:-3.3dev} + depends_on: ["st2api"] + networks: + - private + volumes: + - ./st2.docker.conf:/etc/st2/st2.docker.conf:ro + - ./st2.user.conf:/etc/st2/st2.user.conf:ro + dns_search: . + st2auth: + image: ${ST2_IMAGE_REPO:-stackstorm/}st2auth:${ST2_VERSION:-3.3dev} + depends_on: ["st2api"] + networks: + - private + volumes: + - ./st2.docker.conf:/etc/st2/st2.docker.conf:ro + - ./st2.user.conf:/etc/st2/st2.user.conf:ro + - ./htpasswd:/etc/st2/htpasswd:ro + dns_search: . + st2actionrunner: + image: ${ST2_IMAGE_REPO:-stackstorm/}st2actionrunner:${ST2_VERSION:-3.3dev} + depends_on: ["st2api"] + networks: + - private + volumes: + - ./st2.docker.conf:/etc/st2/st2.docker.conf:ro + - ./st2.user.conf:/etc/st2/st2.user.conf:ro + - stackstorm-packs-configs:/opt/stackstorm/configs:rw + - stackstorm-packs:/opt/stackstorm/packs:rw + - ${ST2_PACKS_DEV:-./packs.dev}:/opt/stackstorm/packs.dev:rw + - stackstorm-virtualenvs:/opt/stackstorm/virtualenvs:rw + - stackstorm-ssh:/home/stanley.ssh + dns_search: . + st2garbagecollector: + image: ${ST2_IMAGE_REPO:-stackstorm/}st2garbagecollector:${ST2_VERSION:-3.3dev} + depends_on: ["st2api"] + networks: + - private + volumes: + - ./st2.docker.conf:/etc/st2/st2.docker.conf:ro + - ./st2.user.conf:/etc/st2/st2.user.conf:ro + dns_search: . + st2notifier: + image: ${ST2_IMAGE_REPO:-stackstorm/}st2garbagecollector:${ST2_VERSION:-3.3dev} + depends_on: ["st2api"] + networks: + - private + volumes: + - ./st2.docker.conf:/etc/st2/st2.docker.conf:ro + - ./st2.user.conf:/etc/st2/st2.user.conf:ro + dns_search: . + st2resultstracker: + image: ${ST2_IMAGE_REPO:-stackstorm/}st2resultstracker:${ST2_VERSION:-3.3dev} + depends_on: ["st2api"] + networks: + - private + volumes: + - ./st2.docker.conf:/etc/st2/st2.docker.conf:ro + - ./st2.user.conf:/etc/st2/st2.user.conf:ro + dns_search: . + st2rulesengine: + image: ${ST2_IMAGE_REPO:-stackstorm/}st2rulesengine:${ST2_VERSION:-3.3dev} + depends_on: ["st2api"] + networks: + - private + volumes: + - ./st2.docker.conf:/etc/st2/st2.docker.conf:ro + - ./st2.user.conf:/etc/st2/st2.user.conf:ro + dns_search: . + st2sensorcontainer: + image: ${ST2_IMAGE_REPO:-stackstorm/}st2sensorcontainer:${ST2_VERSION:-3.3dev} + depends_on: ["st2api"] + networks: + - private + dns_search: . + volumes: + - ./st2.docker.conf:/etc/st2/st2.docker.conf:ro + - ./st2.user.conf:/etc/st2/st2.user.conf:ro + - stackstorm-virtualenvs:/opt/stackstorm/virtualenvs:ro + - stackstorm-packs:/opt/stackstorm/packs:ro + - stackstorm-packs-configs:/opt/stackstorm/configs:ro + - ${ST2_PACKS_DEV:-./packs.dev}:/opt/stackstorm/packs.dev:ro + st2timersengine: + image: ${ST2_IMAGE_REPO:-stackstorm/}st2timersengine:${ST2_VERSION:-3.3dev} + depends_on: ["st2api"] + networks: + - private + dns_search: . + volumes: + - ./st2.docker.conf:/etc/st2/st2.docker.conf:ro + st2client: + image: ${ST2_IMAGE_REPO:-stackstorm/}st2:${ST2_VERSION:-3.3dev} + networks: + - private + environment: + ST2_AUTH_URL: ${ST2_AUTH_URL:-http://st2auth:9100/} + ST2_API_URL: ${ST2_API_URL:-http://st2api:9101/} + ST2_STREAM_URL: ${ST2_STREAM_URL:-http://st2stream:9102/} + volumes: + - ./st2-cli.conf:/root/.st2/config + # external services + mongo: + image: mongo:3.4 + networks: + - private + volumes: + - stackstorm-mongodb:/data/db + dns_search: . + rabbitmq: + image: rabbitmq:3.8 + networks: + - private + dns_search: . + volumes: + - stackstorm-rabbitmq:/var/lib/rabbitmq + redis: + image: redis:6.0 + networks: + - private + dns_search: . + volumes: + - stackstorm-redis:/data + +volumes: + stackstorm-mongodb: + stackstorm-rabbitmq: + stackstorm-redis: + stackstorm-postgres: + stackstorm-packs: + stackstorm-packs-configs: + stackstorm-virtualenvs: + stackstorm-ssh: + +networks: + public: + driver: bridge + private: + driver: bridge diff --git a/htpasswd b/htpasswd new file mode 100644 index 00000000..a9771452 --- /dev/null +++ b/htpasswd @@ -0,0 +1,2 @@ +# st2admin:Ch@ngeMe +st2admin:$apr1$GjA7KmRf$nmbcSB1XoYRPfQrL9ZBD5. diff --git a/st2-cli.conf b/st2-cli.conf new file mode 100644 index 00000000..94a9bba3 --- /dev/null +++ b/st2-cli.conf @@ -0,0 +1,3 @@ +[credentials] +username = st2admin +password = Ch@ngeMe diff --git a/st2.docker.conf b/st2.docker.conf new file mode 100644 index 00000000..25378f32 --- /dev/null +++ b/st2.docker.conf @@ -0,0 +1,23 @@ +# /etc/st2/st2.docker.conf +# +# This is override st2 config file which will be passed to any st2 command, like this: +# `--config-file /etc/st2/st2.conf --config-file /etc/st2/st2.docker.conf --config-file /etc/st2/st2.user.conf` +# making possible to keep custom st2 config directives in it, instead of modifying the original st2.conf every time. +# The order of merging: st2.conf < st2.docker.conf < st2.user.conf +[auth] +api_url = http://st2api:9101/ +[messaging] +url = amqp://guest:guest@rabbitmq:5672 +[database] +host = mongo +# st2.conf for kubernetes disables retry (relying on kubernetes), but +# for straight docker, we want to re-enable these. +# Connection retry total time (minutes). +connection_retry_max_delay_m = 1 +# Backoff multiplier (seconds). +connection_retry_backoff_mul = 1 +# Connection and server selection timeout (in ms). +connection_timeout = 3000 + +[content] +packs_base_paths=/opt/stackstorm/packs.dev diff --git a/st2.user.conf b/st2.user.conf new file mode 100644 index 00000000..268e0c95 --- /dev/null +++ b/st2.user.conf @@ -0,0 +1,6 @@ +# /etc/st2/st2.user.conf +# +# This is override st2 config file which will be passed to any st2 command, like this: +# `--config-file /etc/st2/st2.conf --config-file /etc/st2/st2.docker.conf --config-file /etc/st2/st2.user.conf` +# making possible to keep custom st2 config directives in it, instead of modifying the original st2.conf every time. +# The order of merging: st2.conf < st2.docker.conf < st2.user.conf From e2d23d7f7f07ed78d8cbc2b8c3baa1d440c26dc0 Mon Sep 17 00:00:00 2001 From: John Hogenmiller Date: Sun, 10 May 2020 00:58:00 +0000 Subject: [PATCH 04/32] localhost:8000 and document updates --- README.md | 4 ++-- docker-compose.yml | 4 ++-- st2.docker.conf | 7 ++++--- st2.user.conf | 6 +++--- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 8f758d3f..590d1d86 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ The image version, exposed ports, and "packs.dev" directory is configurable with - **ST2_VERSION** this is the tag at the end of the docker image (ie: stackstorm/st2api:v3.3dev) - **ST2_IMAGE_REPO** The image or path to the images. Default is "stackstorm/". You may change this is using the Enterprise version or a private docker repository. -- **ST2_EXPOSE_HTTP** Port to expose st2web port 80 on. Default is `8000`, and you amy want to do `0.0.0.0:8000` to expose on all interfaces. +- **ST2_EXPOSE_HTTP** Port to expose st2web port 80 on. Default is `127.0.0.1:8000`, and you may want to do `0.0.0.0:8000` to expose on all interfaces. - **ST2_PACKS_DEV** Directory to development packs, absolute or relative to docker-compose.yml. This allows you to develop packs locally. Default is `./packs.dev`. When making a number of packs, it is recommended to make a directory outside of st2-dockerfiles, with each subdirectory underneath that being an independent git repo. Example: `ST2_PACKS_DEV=${HOME}/mypacks`, with `${HOME}/mypacks/st2-helloworld` being a git repo for the "helloworld" pack. ### Credentials @@ -199,7 +199,7 @@ Next, use `docker cp` to copy your file into place. docker cp git.yaml compose_st2api_1:/opt/stackstorm/configs/git.yaml ``` -## Register the configs +## Register the pack config If you copied the file in or, you will need to register it. st2client does not need access to the configs directory, as it will talk to st2api. diff --git a/docker-compose.yml b/docker-compose.yml index 688ec9e5..cdb3ef05 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,8 +13,8 @@ services: - st2api - st2stream ports: - - "${ST2_EXPOSE_HTTP:-8000}:80" - # - "${ST2_EXPOSE_HTTPS:-8443}:443" + - "${ST2_EXPOSE_HTTP:-127.0.0.1:8000}:80" + # - "${ST2_EXPOSE_HTTPS:-127.0.0.1:8443}:443" # more work would be needed with certificate generate to make https work. networks: - private diff --git a/st2.docker.conf b/st2.docker.conf index 25378f32..7bd21893 100644 --- a/st2.docker.conf +++ b/st2.docker.conf @@ -1,8 +1,9 @@ # /etc/st2/st2.docker.conf # -# This is override st2 config file which will be passed to any st2 command, like this: -# `--config-file /etc/st2/st2.conf --config-file /etc/st2/st2.docker.conf --config-file /etc/st2/st2.user.conf` -# making possible to keep custom st2 config directives in it, instead of modifying the original st2.conf every time. +# This st2.docker.conf overrides st2.conf in the base image, and has been written to facilitate operation of +# stackstorm-compose/docker-compose.yml. It is recommended to not alter this file as an end-user, as this file +# is updated as necessary by the StackStorm st2-dockerfile maintainers. Instead, make local changes in the +# file "st2.user.conf". # The order of merging: st2.conf < st2.docker.conf < st2.user.conf [auth] api_url = http://st2api:9101/ diff --git a/st2.user.conf b/st2.user.conf index 268e0c95..9db6d427 100644 --- a/st2.user.conf +++ b/st2.user.conf @@ -1,6 +1,6 @@ # /etc/st2/st2.user.conf # -# This is override st2 config file which will be passed to any st2 command, like this: -# `--config-file /etc/st2/st2.conf --config-file /etc/st2/st2.docker.conf --config-file /etc/st2/st2.user.conf` -# making possible to keep custom st2 config directives in it, instead of modifying the original st2.conf every time. +# This file is provided with no custom overrides and should be utilized by end-users to make any +# desired configuration changes. No changes to this file should be checked into the st2-dockerfiles +# repository. # The order of merging: st2.conf < st2.docker.conf < st2.user.conf From bb13af0fc2429a0b6281be9382417911d0cc8166 Mon Sep 17 00:00:00 2001 From: John Hogenmiller Date: Sun, 10 May 2020 00:59:54 +0000 Subject: [PATCH 05/32] localhost and document updates --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 590d1d86..9697200b 100644 --- a/README.md +++ b/README.md @@ -201,7 +201,7 @@ docker cp git.yaml compose_st2api_1:/opt/stackstorm/configs/git.yaml ## Register the pack config -If you copied the file in or, you will need to register it. st2client does not need access to the configs directory, as it will talk to st2api. +If you used `docker cp` to copy the config in, you will need to manually load that configuration. The st2client service does not need access to the configs directory, as it will talk to st2api. ```shell $ docker-compose run st2client st2 run packs.load packs=git register=configs From d6f21da136b2c8e74e383f00f338b5ea2b0dac51 Mon Sep 17 00:00:00 2001 From: John Hogenmiller Date: Mon, 11 May 2020 18:53:29 +0000 Subject: [PATCH 06/32] add health check to st2web --- docker-compose.yml | 8 ++++++++ st2web-healthcheck.sh | 13 +++++++++++++ 2 files changed, 21 insertions(+) create mode 100755 st2web-healthcheck.sh diff --git a/docker-compose.yml b/docker-compose.yml index cdb3ef05..870ad736 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,6 +3,7 @@ version: '3' services: st2web: image: ${ST2_IMAGE_REPO:-stackstorm/}st2web:${ST2_VERSION:-3.3dev} + restart: always environment: ST2_AUTH_URL: ${ST2_AUTH_URL:-http://st2auth:9100/} ST2_API_URL: ${ST2_API_URL:-http://st2api:9101/} @@ -12,6 +13,13 @@ services: - st2auth - st2api - st2stream + healthcheck: + test: ["CMD", "/st2web-healthcheck.sh"] + interval: 30s + timeout: 1s + retries: 1 + volumes: + - ./st2web-healthcheck.sh:/st2web-healthcheck.sh ports: - "${ST2_EXPOSE_HTTP:-127.0.0.1:8000}:80" # - "${ST2_EXPOSE_HTTPS:-127.0.0.1:8443}:443" diff --git a/st2web-healthcheck.sh b/st2web-healthcheck.sh new file mode 100755 index 00000000..c8335c55 --- /dev/null +++ b/st2web-healthcheck.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Check each service through the nginx reverse proxy for a specific return code. If the curl request +# fails to work through nginx, a stop signal will be sent to nginx, causing the container to restart. +API_STATUS=$(curl --write-out "%{http_code}\n" --silent --output /dev/null http://localhost/api/v1/) +if [ "${API_STATUS}" != "401" ]; then echo "st2api failure"; nginx -s stop; fi + +STREAM_STATUS=$(curl --write-out "%{http_code}\n" --silent --output /dev/null http://localhost/stream/v1/stream) +if [ "${STREAM_STATUS}" != "401" ]; then echo "st2stream failure"; nginx -s stop; fi + +AUTH_STATUS=$(curl --write-out "%{http_code}\n" --silent --output /dev/null http://localhost/auth/v1/) +if [ "${AUTH_STATUS}" != "404" ]; then echo "st2auth failure"; nginx -s stop; fi + +exit 0 \ No newline at end of file From 1fd7768ff3bec7ae681bca2f45ba98a21df4ce87 Mon Sep 17 00:00:00 2001 From: John Hogenmiller Date: Thu, 14 May 2020 12:08:18 +0000 Subject: [PATCH 07/32] fix(stackstorm-compose): st2notifier image --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 870ad736..35d6f2bc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -105,7 +105,7 @@ services: - ./st2.user.conf:/etc/st2/st2.user.conf:ro dns_search: . st2notifier: - image: ${ST2_IMAGE_REPO:-stackstorm/}st2garbagecollector:${ST2_VERSION:-3.3dev} + image: ${ST2_IMAGE_REPO:-stackstorm/}st2notifier:${ST2_VERSION:-3.3dev} depends_on: ["st2api"] networks: - private From ee7393dc647f86c79c1f617e20efab43c11784e2 Mon Sep 17 00:00:00 2001 From: John Hogenmiller Date: Sun, 17 May 2020 12:04:49 +0000 Subject: [PATCH 08/32] set docker-compose to restart: on-failure --- docker-compose.yml | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 35d6f2bc..4516be79 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,7 +3,7 @@ version: '3' services: st2web: image: ${ST2_IMAGE_REPO:-stackstorm/}st2web:${ST2_VERSION:-3.3dev} - restart: always + restart: on-failure environment: ST2_AUTH_URL: ${ST2_AUTH_URL:-http://st2auth:9100/} ST2_API_URL: ${ST2_API_URL:-http://st2api:9101/} @@ -30,6 +30,7 @@ services: dns_search: . st2api: image: ${ST2_IMAGE_REPO:-stackstorm/}st2api:${ST2_VERSION:-3.3dev} + restart: on-failure depends_on: ["mongo", "rabbitmq"] networks: - private @@ -46,6 +47,7 @@ services: dns_search: . st2stream: image: ${ST2_IMAGE_REPO:-stackstorm/}st2stream:${ST2_VERSION:-3.3dev} + restart: on-failure depends_on: ["st2api"] networks: - private @@ -55,6 +57,7 @@ services: dns_search: . st2scheduler: image: ${ST2_IMAGE_REPO:-stackstorm/}st2scheduler:${ST2_VERSION:-3.3dev} + restart: on-failure depends_on: ["st2api"] networks: - private @@ -64,6 +67,7 @@ services: dns_search: . st2workflowengine: image: ${ST2_IMAGE_REPO:-stackstorm/}st2workflowengine:${ST2_VERSION:-3.3dev} + restart: on-failure depends_on: ["st2api"] networks: - private @@ -73,6 +77,7 @@ services: dns_search: . st2auth: image: ${ST2_IMAGE_REPO:-stackstorm/}st2auth:${ST2_VERSION:-3.3dev} + restart: on-failure depends_on: ["st2api"] networks: - private @@ -83,6 +88,7 @@ services: dns_search: . st2actionrunner: image: ${ST2_IMAGE_REPO:-stackstorm/}st2actionrunner:${ST2_VERSION:-3.3dev} + restart: on-failure depends_on: ["st2api"] networks: - private @@ -97,6 +103,7 @@ services: dns_search: . st2garbagecollector: image: ${ST2_IMAGE_REPO:-stackstorm/}st2garbagecollector:${ST2_VERSION:-3.3dev} + restart: on-failure depends_on: ["st2api"] networks: - private @@ -106,6 +113,7 @@ services: dns_search: . st2notifier: image: ${ST2_IMAGE_REPO:-stackstorm/}st2notifier:${ST2_VERSION:-3.3dev} + restart: on-failure depends_on: ["st2api"] networks: - private @@ -115,6 +123,7 @@ services: dns_search: . st2resultstracker: image: ${ST2_IMAGE_REPO:-stackstorm/}st2resultstracker:${ST2_VERSION:-3.3dev} + restart: on-failure depends_on: ["st2api"] networks: - private @@ -124,6 +133,7 @@ services: dns_search: . st2rulesengine: image: ${ST2_IMAGE_REPO:-stackstorm/}st2rulesengine:${ST2_VERSION:-3.3dev} + restart: on-failure depends_on: ["st2api"] networks: - private @@ -133,6 +143,7 @@ services: dns_search: . st2sensorcontainer: image: ${ST2_IMAGE_REPO:-stackstorm/}st2sensorcontainer:${ST2_VERSION:-3.3dev} + restart: on-failure depends_on: ["st2api"] networks: - private @@ -146,6 +157,7 @@ services: - ${ST2_PACKS_DEV:-./packs.dev}:/opt/stackstorm/packs.dev:ro st2timersengine: image: ${ST2_IMAGE_REPO:-stackstorm/}st2timersengine:${ST2_VERSION:-3.3dev} + restart: on-failure depends_on: ["st2api"] networks: - private @@ -154,6 +166,7 @@ services: - ./st2.docker.conf:/etc/st2/st2.docker.conf:ro st2client: image: ${ST2_IMAGE_REPO:-stackstorm/}st2:${ST2_VERSION:-3.3dev} + restart: on-failure networks: - private environment: @@ -165,6 +178,7 @@ services: # external services mongo: image: mongo:3.4 + restart: on-failure networks: - private volumes: @@ -172,6 +186,7 @@ services: dns_search: . rabbitmq: image: rabbitmq:3.8 + restart: on-failure networks: - private dns_search: . @@ -179,6 +194,7 @@ services: - stackstorm-rabbitmq:/var/lib/rabbitmq redis: image: redis:6.0 + restart: on-failure networks: - private dns_search: . From 7e22cde0c4af3c5b3b97643e20c2eb382d9212ee Mon Sep 17 00:00:00 2001 From: John Hogenmiller Date: Sun, 17 May 2020 12:07:28 +0000 Subject: [PATCH 09/32] change run to exec for running st2ctl --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9697200b..89d72384 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ This docker-compose is provided as a way to allow someone to "get up and running git clone git@github.com:stackstorm/st2-dockerfiles cd st2-docker/stackstorm-compose docker-compose up -d -docker-compose run st2api st2ctl reload --register-all # you should only have to do this the first time +docker-compose exec st2api st2ctl reload --register-all # you should only have to do this the first time docker-compose run st2client bash # this gives you access to the st2 command line ``` @@ -85,7 +85,7 @@ docker-compose down Fourth, if this is your first time running StackStorm in Docker Compose, you will need to force st2api to register everything. ```shell -docker-compose run st2api st2ctl reload --register-all +docker-compose exec st2api st2ctl reload --register-all ``` ## Regular Usage From f25533cf7c8363b71b2a30d5f66b92ec412ba6ff Mon Sep 17 00:00:00 2001 From: John Hogenmiller Date: Sun, 17 May 2020 12:12:57 +0000 Subject: [PATCH 10/32] set compose st2client to use st2actionrunner image --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 4516be79..716c784e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -165,7 +165,7 @@ services: volumes: - ./st2.docker.conf:/etc/st2/st2.docker.conf:ro st2client: - image: ${ST2_IMAGE_REPO:-stackstorm/}st2:${ST2_VERSION:-3.3dev} + image: ${ST2_IMAGE_REPO:-stackstorm/}st2actionrunner:${ST2_VERSION:-3.3dev} restart: on-failure networks: - private From a6b48202bb3466adb497ab46efedc3e9f177e65f Mon Sep 17 00:00:00 2001 From: John Hogenmiller Date: Sun, 17 May 2020 12:37:42 +0000 Subject: [PATCH 11/32] make st2client run always, matching stackstorm-ha --- README.md | 14 +++++++------- docker-compose.yml | 1 + 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 89d72384..09d57fa6 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ git clone git@github.com:stackstorm/st2-dockerfiles cd st2-docker/stackstorm-compose docker-compose up -d docker-compose exec st2api st2ctl reload --register-all # you should only have to do this the first time -docker-compose run st2client bash # this gives you access to the st2 command line +docker-compose exec st2client bash # this gives you access to the st2 command line ``` Open `http://localhost:8000` in your browser. StackStorm Username/Password by default is: st2admin/Ch@ngeMe. @@ -93,13 +93,13 @@ docker-compose exec st2api st2ctl reload --register-all To run st2 commands, you can use the st2client service: ```shell -docker-compose run st2client st2 +docker-compose exec st2client st2 ``` Example: ```shell -$ docker-compose run st2client st2 run core.echo message=hello +$ docker-compose exec st2client st2 run core.echo message=hello . id: 5eb30d77afe5aa8493f31187 action.ref: core.echo @@ -117,12 +117,12 @@ result: succeeded: true ``` -Alternatively, you could run `docker-compose run st2client bash` to be dropped into a container with st2. At that point, you can just run `st2` commands. +Alternatively, you could run `docker-compose exec st2client bash` to be dropped into a container with st2. At that point, you can just run `st2` commands. Example: ```shell -$ docker-compose run st2client bash +$ docker-compose exec st2client bash Welcome to StackStorm HA v3.3dev (Ubuntu 16.04 LTS GNU/Linux x86_64) * Documentation: https://docs.stackstorm.com/ * Community: https://stackstorm.com/community-signup @@ -157,7 +157,7 @@ Pack configs will be in /opt/stackstorm/configs/$PACKNAME, which is a docker vol ### Use st2 pack config ```shell -$ docker-compose run st2client st2 pack config git +$ docker-compose exec st2client st2 pack config git repositories[0].url: https://github.com/StackStorm/st2-dockerfiles.git repositories[0].branch [master]: ~~~ Would you like to add another item to "repositories" array / list? [y]: n @@ -204,7 +204,7 @@ docker cp git.yaml compose_st2api_1:/opt/stackstorm/configs/git.yaml If you used `docker cp` to copy the config in, you will need to manually load that configuration. The st2client service does not need access to the configs directory, as it will talk to st2api. ```shell -$ docker-compose run st2client st2 run packs.load packs=git register=configs +$ docker-compose exec st2client st2 run packs.load packs=git register=configs . id: 5eb3171c566aa824ea88f538 action.ref: packs.load diff --git a/docker-compose.yml b/docker-compose.yml index 716c784e..605c22b4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -167,6 +167,7 @@ services: st2client: image: ${ST2_IMAGE_REPO:-stackstorm/}st2actionrunner:${ST2_VERSION:-3.3dev} restart: on-failure + command: sleep infinity networks: - private environment: From df2502bed9de13d7d241cadcbfa7937c4b94b535 Mon Sep 17 00:00:00 2001 From: John Hogenmiller Date: Mon, 25 May 2020 12:49:57 +0000 Subject: [PATCH 12/32] Add secret keys on start for encrypted secrets --- docker-compose.yml | 14 +++++++++++++- makesecrets.sh | 11 +++++++++++ st2.docker.conf | 4 ++++ 3 files changed, 28 insertions(+), 1 deletion(-) create mode 100755 makesecrets.sh diff --git a/docker-compose.yml b/docker-compose.yml index 605c22b4..7a98fd10 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -28,10 +28,20 @@ services: - private - public dns_search: . + st2makesecrets: + image: ${ST2_IMAGE_REPO:-stackstorm/}st2actionrunner:${ST2_VERSION:-3.3dev} + restart: on-failure + networks: + - private + volumes: + - ./makesecrets.sh:/makesecrets.sh + - stackstorm-keys:/etc/st2/keys:rw + dns_search: . + command: /makesecrets.sh st2api: image: ${ST2_IMAGE_REPO:-stackstorm/}st2api:${ST2_VERSION:-3.3dev} restart: on-failure - depends_on: ["mongo", "rabbitmq"] + depends_on: ["mongo", "rabbitmq", "st2makesecrets"] networks: - private environment: @@ -41,6 +51,7 @@ services: volumes: - ./st2.docker.conf:/etc/st2/st2.docker.conf:ro - ./st2.user.conf:/etc/st2/st2.user.conf:ro + - stackstorm-keys:/etc/st2/keys:rw - stackstorm-packs-configs:/opt/stackstorm/configs:rw - stackstorm-packs:/opt/stackstorm/packs:rw - ${ST2_PACKS_DEV:-./packs.dev}:/opt/stackstorm/packs.dev:rw @@ -209,6 +220,7 @@ volumes: stackstorm-postgres: stackstorm-packs: stackstorm-packs-configs: + stackstorm-keys: stackstorm-virtualenvs: stackstorm-ssh: diff --git a/makesecrets.sh b/makesecrets.sh new file mode 100755 index 00000000..edc9c6ec --- /dev/null +++ b/makesecrets.sh @@ -0,0 +1,11 @@ +#!/bin/bash +# this needs to run as root, so can't be ran in the st2api container +KEYPATH=/etc/st2/keys/datastore_key.json +if [ ! -f "/etc/st2/keys/datastore_key.json" ] +then + echo "Generating ${KEYPATH}" + st2-generate-symmetric-crypto-key --key-path /etc/st2/keys/datastore_key.json + chown -R st2:st2 /etc/st2/keys + chmod -R 750 /etc/st2/keys +fi + diff --git a/st2.docker.conf b/st2.docker.conf index 7bd21893..f5ea3f6d 100644 --- a/st2.docker.conf +++ b/st2.docker.conf @@ -9,6 +9,10 @@ api_url = http://st2api:9101/ [messaging] url = amqp://guest:guest@rabbitmq:5672 + +[keyvalue] +encryption_key_path = /etc/st2/keys/datastore_key.json + [database] host = mongo # st2.conf for kubernetes disables retry (relying on kubernetes), but From ebc6d7a8b49dd4afedec0c29451ebdb92dbfa1f6 Mon Sep 17 00:00:00 2001 From: John Hogenmiller Date: Mon, 25 May 2020 13:10:49 +0000 Subject: [PATCH 13/32] better st2web healthchecks --- st2web-healthcheck.sh | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/st2web-healthcheck.sh b/st2web-healthcheck.sh index c8335c55..227f5629 100755 --- a/st2web-healthcheck.sh +++ b/st2web-healthcheck.sh @@ -1,13 +1,23 @@ #!/usr/bin/env bash +# check downstream services and mark container unhealthy if downstream is not unreachable +DOWNSTREAM_API_STATUS=$(curl --write-out "%{http_code}\n" --silent --output /dev/null $ST2_API_URL/v1) +if [ "${DOWNSTREAM_API_STATUS}" != "404" ]; then echo "st2api downstream failure"; exit 1; fi + +DOWNSTREAM_STREAM_STATUS=$(curl --write-out "%{http_code}\n" --silent --output /dev/null $ST2_STREAM_URL/v1/stream) +if [ "${DOWNSTREAM_STREAM_STATUS}" != "404" ]; then echo "st2stream downstream failure"; exit 1; fi + +DOWNSTREAM_AUTH_STATUS=$(curl --write-out "%{http_code}\n" --silent --output /dev/null $ST2_AUTH_URL/v1) +if [ "${DOWNSTREAM_AUTH_STATUS}" != "404" ]; then echo "st2auth downstream failure"; exit 1; fi + # Check each service through the nginx reverse proxy for a specific return code. If the curl request # fails to work through nginx, a stop signal will be sent to nginx, causing the container to restart. API_STATUS=$(curl --write-out "%{http_code}\n" --silent --output /dev/null http://localhost/api/v1/) -if [ "${API_STATUS}" != "401" ]; then echo "st2api failure"; nginx -s stop; fi +if [ "${API_STATUS}" != "401" ]; then echo "st2api nginx failure"; nginx -s stop; fi STREAM_STATUS=$(curl --write-out "%{http_code}\n" --silent --output /dev/null http://localhost/stream/v1/stream) -if [ "${STREAM_STATUS}" != "401" ]; then echo "st2stream failure"; nginx -s stop; fi +if [ "${STREAM_STATUS}" != "401" ]; then echo "st2stream nginx failure"; nginx -s stop; fi AUTH_STATUS=$(curl --write-out "%{http_code}\n" --silent --output /dev/null http://localhost/auth/v1/) -if [ "${AUTH_STATUS}" != "404" ]; then echo "st2auth failure"; nginx -s stop; fi +if [ "${AUTH_STATUS}" != "404" ]; then echo "st2auth nginx failure"; nginx -s stop; fi exit 0 \ No newline at end of file From 0bfdfb76334b63375fd5b77c8dd1c9959087bc8f Mon Sep 17 00:00:00 2001 From: John Hogenmiller Date: Mon, 25 May 2020 13:51:05 +0000 Subject: [PATCH 14/32] st2client startup will register actions --- README.md | 15 ++++++++------- docker-compose.yml | 8 +++++++- st2client-startup.sh | 19 +++++++++++++++++++ 3 files changed, 34 insertions(+), 8 deletions(-) create mode 100755 st2client-startup.sh diff --git a/README.md b/README.md index 09d57fa6..2b65caa7 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,6 @@ This docker-compose is provided as a way to allow someone to "get up and running git clone git@github.com:stackstorm/st2-dockerfiles cd st2-docker/stackstorm-compose docker-compose up -d -docker-compose exec st2api st2ctl reload --register-all # you should only have to do this the first time docker-compose exec st2client bash # this gives you access to the st2 command line ``` @@ -82,12 +81,6 @@ To stop the docker environment, run: docker-compose down ``` -Fourth, if this is your first time running StackStorm in Docker Compose, you will need to force st2api to register everything. - -```shell -docker-compose exec st2api st2ctl reload --register-all -``` - ## Regular Usage To run st2 commands, you can use the st2client service: @@ -303,3 +296,11 @@ result: ' stdout: '' ``` + +# Remove everything + +If you want to uninstall, or start from a "clean" installation, docker-compose can remove all the containers and volumes in one comamnd. + +```shell +docker-compose down --remove-orphans -v +``` \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 7a98fd10..2b7e07be 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -178,14 +178,20 @@ services: st2client: image: ${ST2_IMAGE_REPO:-stackstorm/}st2actionrunner:${ST2_VERSION:-3.3dev} restart: on-failure - command: sleep infinity + depends_on: + - st2auth + - st2api + - st2stream + command: /st2client-startup.sh networks: - private environment: + ST2CLIENT: 1 ST2_AUTH_URL: ${ST2_AUTH_URL:-http://st2auth:9100/} ST2_API_URL: ${ST2_API_URL:-http://st2api:9101/} ST2_STREAM_URL: ${ST2_STREAM_URL:-http://st2stream:9102/} volumes: + - ./st2client-startup.sh:/st2client-startup.sh - ./st2-cli.conf:/root/.st2/config # external services mongo: diff --git a/st2client-startup.sh b/st2client-startup.sh new file mode 100755 index 00000000..8676922b --- /dev/null +++ b/st2client-startup.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +#st2client startup and registration +while true +do +ACTIONS=$(st2 action list) +if [ "$?" -ne 0 ] +then + echo "unable to reach downstream, will try again" + sleep 5 +elif [ "$ACTIONS" == "No matching items found" ] +then + echo "No packs registered, will register" + st2 pack register +else + echo "actions found st2client ready" + sleep infinity +fi +done \ No newline at end of file From 003c22b0ec307f57ffeb65c6e8fd315fe4bcd7b9 Mon Sep 17 00:00:00 2001 From: John Hogenmiller Date: Tue, 26 May 2020 16:12:58 +0000 Subject: [PATCH 15/32] readme path fix --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2b65caa7..43e9cdfe 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ This docker-compose is provided as a way to allow someone to "get up and running ```shell git clone git@github.com:stackstorm/st2-dockerfiles -cd st2-docker/stackstorm-compose +cd st2-dockerfiles/stackstorm-compose docker-compose up -d docker-compose exec st2client bash # this gives you access to the st2 command line ``` From e1a11e08a970230d195379a9e8d3f50849802a7b Mon Sep 17 00:00:00 2001 From: armab Date: Sun, 21 Jun 2020 14:04:10 +0100 Subject: [PATCH 16/32] Add CIrcleCI config to verify ST2 in docker-compose --- .circleci/config.yml | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 00000000..77d68417 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,35 @@ +version: 2 +jobs: + # Run docker-compose install and verify st2 deployment + docker-compose-up: + working_directory: ~/docker + machine: + # Available images https://circleci.com/docs/2.0/configuration-reference/#available-machine-images + image: ubuntu-1604:201903-01 + steps: + - checkout + - name: Start ST2 in docker-compose + command: docker-compose up --detach + +workflows: + version: 2 + docker-compose: + jobs: + - docker-compose-up + # Nightly run docker-compose install and ensure no regressions for st2 deployment in 'master' branch + docker-compose-nightly: + jobs: + - docker-compose-up + triggers: + - schedule: + cron: "0 1 * * *" + filters: + branches: + only: + - master + +experimental: + notify: + branches: + only: + - master From 57d49dfa9a3b64f63a98b68997bd7a236f730260 Mon Sep 17 00:00:00 2001 From: armab Date: Sun, 21 Jun 2020 14:15:17 +0100 Subject: [PATCH 17/32] Adjust bash scripts to st2 code style --- makesecrets.sh | 16 +++++++++------- st2client-startup.sh | 21 ++++++++++----------- st2web-healthcheck.sh | 33 ++++++++++++++++++++++----------- 3 files changed, 41 insertions(+), 29 deletions(-) diff --git a/makesecrets.sh b/makesecrets.sh index edc9c6ec..4355b2bb 100755 --- a/makesecrets.sh +++ b/makesecrets.sh @@ -1,11 +1,13 @@ #!/bin/bash + +# Generate st2 datastore crypto key on st2 startup +# https://docs.stackstorm.com/datastore.html#securing-secrets-admin-only + # this needs to run as root, so can't be ran in the st2api container KEYPATH=/etc/st2/keys/datastore_key.json -if [ ! -f "/etc/st2/keys/datastore_key.json" ] -then - echo "Generating ${KEYPATH}" - st2-generate-symmetric-crypto-key --key-path /etc/st2/keys/datastore_key.json - chown -R st2:st2 /etc/st2/keys - chmod -R 750 /etc/st2/keys +if [ ! -f "${KEYPATH}" ]; then + echo "Generating ${KEYPATH}" + st2-generate-symmetric-crypto-key --key-path ${KEYPATH} + chown -R st2:st2 /etc/st2/keys + chmod -R 750 /etc/st2/keys fi - diff --git a/st2client-startup.sh b/st2client-startup.sh index 8676922b..4c47f6e1 100755 --- a/st2client-startup.sh +++ b/st2client-startup.sh @@ -1,19 +1,18 @@ #!/bin/bash -#st2client startup and registration -while true -do -ACTIONS=$(st2 action list) -if [ "$?" -ne 0 ] -then +# st2client startup and registration +while true; do + ACTIONS=$(st2 action list) + if [ "$?" -ne 0 ] + then echo "unable to reach downstream, will try again" sleep 5 -elif [ "$ACTIONS" == "No matching items found" ] -then + elif [ "$ACTIONS" == "No matching items found" ] + then echo "No packs registered, will register" st2 pack register -else + else echo "actions found st2client ready" sleep infinity -fi -done \ No newline at end of file + fi +done diff --git a/st2web-healthcheck.sh b/st2web-healthcheck.sh index 227f5629..327a1733 100755 --- a/st2web-healthcheck.sh +++ b/st2web-healthcheck.sh @@ -1,23 +1,34 @@ #!/usr/bin/env bash + +# A set of scripts to ensure st2web service liveness + # check downstream services and mark container unhealthy if downstream is not unreachable DOWNSTREAM_API_STATUS=$(curl --write-out "%{http_code}\n" --silent --output /dev/null $ST2_API_URL/v1) -if [ "${DOWNSTREAM_API_STATUS}" != "404" ]; then echo "st2api downstream failure"; exit 1; fi - +if [ "${DOWNSTREAM_API_STATUS}" != "404" ]; then + echo "st2api downstream failure"; exit 1; +fi DOWNSTREAM_STREAM_STATUS=$(curl --write-out "%{http_code}\n" --silent --output /dev/null $ST2_STREAM_URL/v1/stream) -if [ "${DOWNSTREAM_STREAM_STATUS}" != "404" ]; then echo "st2stream downstream failure"; exit 1; fi - +if [ "${DOWNSTREAM_STREAM_STATUS}" != "404" ]; then + echo "st2stream downstream failure"; exit 1; +fi DOWNSTREAM_AUTH_STATUS=$(curl --write-out "%{http_code}\n" --silent --output /dev/null $ST2_AUTH_URL/v1) -if [ "${DOWNSTREAM_AUTH_STATUS}" != "404" ]; then echo "st2auth downstream failure"; exit 1; fi +if [ "${DOWNSTREAM_AUTH_STATUS}" != "404" ]; then + echo "st2auth downstream failure"; exit 1; +fi # Check each service through the nginx reverse proxy for a specific return code. If the curl request # fails to work through nginx, a stop signal will be sent to nginx, causing the container to restart. API_STATUS=$(curl --write-out "%{http_code}\n" --silent --output /dev/null http://localhost/api/v1/) -if [ "${API_STATUS}" != "401" ]; then echo "st2api nginx failure"; nginx -s stop; fi - +if [ "${API_STATUS}" != "401" ]; then + echo "st2api nginx failure"; nginx -s stop; +fi STREAM_STATUS=$(curl --write-out "%{http_code}\n" --silent --output /dev/null http://localhost/stream/v1/stream) -if [ "${STREAM_STATUS}" != "401" ]; then echo "st2stream nginx failure"; nginx -s stop; fi - +if [ "${STREAM_STATUS}" != "401" ]; then + echo "st2stream nginx failure"; nginx -s stop; +fi AUTH_STATUS=$(curl --write-out "%{http_code}\n" --silent --output /dev/null http://localhost/auth/v1/) -if [ "${AUTH_STATUS}" != "404" ]; then echo "st2auth nginx failure"; nginx -s stop; fi +if [ "${AUTH_STATUS}" != "404" ]; then + echo "st2auth nginx failure"; nginx -s stop; +fi -exit 0 \ No newline at end of file +exit 0 From e546b303aad551331ebf65c9321661b2e91bd514 Mon Sep 17 00:00:00 2001 From: armab Date: Sun, 21 Jun 2020 14:17:35 +0100 Subject: [PATCH 18/32] Fix CircleCI config syntax --- .circleci/config.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 77d68417..5d3736d8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,8 +8,9 @@ jobs: image: ubuntu-1604:201903-01 steps: - checkout - - name: Start ST2 in docker-compose - command: docker-compose up --detach + - run: + name: Start ST2 in docker-compose + command: docker-compose up --detach workflows: version: 2 From 33b29fff205aa7cb5c1c9c849a5f847ff800c0a2 Mon Sep 17 00:00:00 2001 From: armab Date: Sun, 21 Jun 2020 14:22:40 +0100 Subject: [PATCH 19/32] Split pulling the images into a separated step --- .circleci/config.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5d3736d8..1b669bf4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -9,8 +9,15 @@ jobs: steps: - checkout - run: - name: Start ST2 in docker-compose + name: Pull StackStorm Docker Images + command: docker-compose pull + - run: + name: Start StackStorm in docker-compose command: docker-compose up --detach + - run: + when: always + name: Show created services + command: docker-compose ps --all workflows: version: 2 From 572171824f5f9373e05bf5397100db9601d3accb Mon Sep 17 00:00:00 2001 From: armab Date: Sun, 21 Jun 2020 14:26:03 +0100 Subject: [PATCH 20/32] Remove unsupported --all option due to outdated docker-compose --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1b669bf4..c50cbf9e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -17,7 +17,7 @@ jobs: - run: when: always name: Show created services - command: docker-compose ps --all + command: docker-compose ps workflows: version: 2 From 15508a67eeacb10c94319c58e04e9da3e26df219 Mon Sep 17 00:00:00 2001 From: armab Date: Sun, 21 Jun 2020 21:23:44 +0100 Subject: [PATCH 21/32] Add docker-compose lint check --- .circleci/config.yml | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c50cbf9e..664723b7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,5 +1,15 @@ -version: 2 +version: 2.1 jobs: + # Verify docker-compose.yml format for any immediate linting issues + docker-compose-lint: + docker: + - image: lachlanevenson/k8s-helm:v2.16.9 + steps: + - checkout + - run: + name: Docker-compose lint check + command: docker-compose config + # Run docker-compose install and verify st2 deployment docker-compose-up: working_directory: ~/docker @@ -9,25 +19,32 @@ jobs: steps: - checkout - run: - name: Pull StackStorm Docker Images + name: Pull Docker Images command: docker-compose pull - run: name: Start StackStorm in docker-compose command: docker-compose up --detach - run: when: always - name: Show created services + name: List created services command: docker-compose ps workflows: version: 2 docker-compose: jobs: + - docker-compose-lint - docker-compose-up + requires: + - docker-compose-lint + # Nightly run docker-compose install and ensure no regressions for st2 deployment in 'master' branch docker-compose-nightly: jobs: + - docker-compose-lint - docker-compose-up + requires: + - docker-compose-lint triggers: - schedule: cron: "0 1 * * *" From 0d602ab65af99df42052109284a329e09c08fbd2 Mon Sep 17 00:00:00 2001 From: armab Date: Sun, 21 Jun 2020 21:25:00 +0100 Subject: [PATCH 22/32] Add docker-compose lint check --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 664723b7..8836ef68 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -34,7 +34,7 @@ workflows: docker-compose: jobs: - docker-compose-lint - - docker-compose-up + - docker-compose-up: requires: - docker-compose-lint @@ -42,7 +42,7 @@ workflows: docker-compose-nightly: jobs: - docker-compose-lint - - docker-compose-up + - docker-compose-up: requires: - docker-compose-lint triggers: From 143b5fef689f0d63a1baf9052d6e1e2581371122 Mon Sep 17 00:00:00 2001 From: armab Date: Sun, 21 Jun 2020 21:37:05 +0100 Subject: [PATCH 23/32] Use latest docker-compose image --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8836ef68..4f7bf23d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3,7 +3,7 @@ jobs: # Verify docker-compose.yml format for any immediate linting issues docker-compose-lint: docker: - - image: lachlanevenson/k8s-helm:v2.16.9 + - image: docker/compose:latest steps: - checkout - run: From a533a31d7c7739a242f12c538df2d989bdf6066e Mon Sep 17 00:00:00 2001 From: armab Date: Tue, 23 Jun 2020 18:19:47 +0100 Subject: [PATCH 24/32] Add quick smoke tests to verify the ST2 cluster --- .circleci/config.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4f7bf23d..37c2185e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -26,8 +26,17 @@ jobs: command: docker-compose up --detach - run: when: always - name: List created services + name: List created Services command: docker-compose ps + - run: + name: Smoke-test StackStorm cluster + command: | + # TODO: Replace with more organized BATS tests + # Example: https://github.com/StackStorm/stackstorm-ha/blob/master/tests/st2tests.sh + docker-compose exec st2client st2 action list --pack=core + docker-compose exec st2client st2 run core.local cmd=date + docker-compose exec st2client st2 pack install github + docker-compose exec st2client st2 execution list workflows: version: 2 From 4642e138073110880676c4ef7561501b5bd99034 Mon Sep 17 00:00:00 2001 From: armab Date: Tue, 23 Jun 2020 19:08:15 +0100 Subject: [PATCH 25/32] Workaround Docker services startup wait before tests --- .circleci/config.yml | 5 ++++- st2client-startup.sh | 6 ++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 37c2185e..5f323cbd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -31,7 +31,10 @@ jobs: - run: name: Smoke-test StackStorm cluster command: | - # TODO: Replace with more organized BATS tests + # TODO: Replace with 'docker-compose run status' + # Examples: https://github.com/docker/compose/issues/374#issuecomment-310266246 + sleep 30 + # TODO: Replace with more organized BATS tests 'docker-compose run tests' # Example: https://github.com/StackStorm/stackstorm-ha/blob/master/tests/st2tests.sh docker-compose exec st2client st2 action list --pack=core docker-compose exec st2client st2 run core.local cmd=date diff --git a/st2client-startup.sh b/st2client-startup.sh index 4c47f6e1..1861ec06 100755 --- a/st2client-startup.sh +++ b/st2client-startup.sh @@ -3,12 +3,10 @@ # st2client startup and registration while true; do ACTIONS=$(st2 action list) - if [ "$?" -ne 0 ] - then + if [ "$?" -ne 0 ]; then echo "unable to reach downstream, will try again" sleep 5 - elif [ "$ACTIONS" == "No matching items found" ] - then + elif [ "$ACTIONS" == "No matching items found" ]; then echo "No packs registered, will register" st2 pack register else From 5bed965d0d9a48d7499fbc8df966aa57804a6706 Mon Sep 17 00:00:00 2001 From: armab Date: Tue, 23 Jun 2020 19:20:46 +0100 Subject: [PATCH 26/32] Workaround Docker services startup wait before tests --- .circleci/config.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5f323cbd..c55074ad 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -24,22 +24,22 @@ jobs: - run: name: Start StackStorm in docker-compose command: docker-compose up --detach - - run: - when: always - name: List created Services - command: docker-compose ps - run: name: Smoke-test StackStorm cluster command: | # TODO: Replace with 'docker-compose run status' # Examples: https://github.com/docker/compose/issues/374#issuecomment-310266246 - sleep 30 + sleep 45 # TODO: Replace with more organized BATS tests 'docker-compose run tests' # Example: https://github.com/StackStorm/stackstorm-ha/blob/master/tests/st2tests.sh docker-compose exec st2client st2 action list --pack=core docker-compose exec st2client st2 run core.local cmd=date docker-compose exec st2client st2 pack install github docker-compose exec st2client st2 execution list + - run: + when: always + name: List created Services + command: docker-compose ps workflows: version: 2 From 9a96385f5d38b1ea734889fa118b95eed09e110e Mon Sep 17 00:00:00 2001 From: armab Date: Tue, 23 Jun 2020 19:54:31 +0100 Subject: [PATCH 27/32] Move files/scripts into separated dirs --- .circleci/config.yml | 4 +- README.md | 34 +++++------ docker-compose.yml | 56 +++++++++---------- htpasswd => files/htpasswd | 1 + st2-cli.conf => files/st2-cli.conf | 1 + st2.docker.conf => files/st2.docker.conf | 6 +- st2.user.conf => files/st2.user.conf | 3 +- makesecrets.sh => scripts/makesecrets.sh | 0 .../st2client-startup.sh | 0 .../st2web-healthcheck.sh | 0 10 files changed, 53 insertions(+), 52 deletions(-) rename htpasswd => files/htpasswd (77%) rename st2-cli.conf => files/st2-cli.conf (72%) rename st2.docker.conf => files/st2.docker.conf (76%) rename st2.user.conf => files/st2.user.conf (62%) rename makesecrets.sh => scripts/makesecrets.sh (100%) rename st2client-startup.sh => scripts/st2client-startup.sh (100%) rename st2web-healthcheck.sh => scripts/st2web-healthcheck.sh (100%) diff --git a/.circleci/config.yml b/.circleci/config.yml index c55074ad..ca942c6b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -22,10 +22,10 @@ jobs: name: Pull Docker Images command: docker-compose pull - run: - name: Start StackStorm in docker-compose + name: Start st2 with docker-compose command: docker-compose up --detach - run: - name: Smoke-test StackStorm cluster + name: Run st2 Smoke-Tests command: | # TODO: Replace with 'docker-compose run status' # Examples: https://github.com/docker/compose/issues/374#issuecomment-310266246 diff --git a/README.md b/README.md index 43e9cdfe..2d7c698a 100644 --- a/README.md +++ b/README.md @@ -5,13 +5,11 @@ This docker-compose is provided as a way to allow someone to "get up and running ## TL;DR ```shell -git clone git@github.com:stackstorm/st2-dockerfiles -cd st2-dockerfiles/stackstorm-compose docker-compose up -d docker-compose exec st2client bash # this gives you access to the st2 command line ``` -Open `http://localhost:8000` in your browser. StackStorm Username/Password by default is: st2admin/Ch@ngeMe. +Open `http://localhost:8000/` in your browser. StackStorm Username/Password by default is: `st2admin/Ch@ngeMe`. ## Usage @@ -27,23 +25,23 @@ The image version, exposed ports, and "packs.dev" directory is configurable with - **ST2_VERSION** this is the tag at the end of the docker image (ie: stackstorm/st2api:v3.3dev) - **ST2_IMAGE_REPO** The image or path to the images. Default is "stackstorm/". You may change this is using the Enterprise version or a private docker repository. - **ST2_EXPOSE_HTTP** Port to expose st2web port 80 on. Default is `127.0.0.1:8000`, and you may want to do `0.0.0.0:8000` to expose on all interfaces. -- **ST2_PACKS_DEV** Directory to development packs, absolute or relative to docker-compose.yml. This allows you to develop packs locally. Default is `./packs.dev`. When making a number of packs, it is recommended to make a directory outside of st2-dockerfiles, with each subdirectory underneath that being an independent git repo. Example: `ST2_PACKS_DEV=${HOME}/mypacks`, with `${HOME}/mypacks/st2-helloworld` being a git repo for the "helloworld" pack. +- **ST2_PACKS_DEV** Directory to development packs, absolute or relative to docker-compose.yml. This allows you to develop packs locally. Default is `./packs.dev`. When making a number of packs, it is recommended to make a directory outside of st2-docker, with each subdirectory underneath that being an independent git repo. Example: `ST2_PACKS_DEV=${HOME}/mypacks`, with `${HOME}/mypacks/st2-helloworld` being a git repo for the "helloworld" pack. ### Credentials -The `htpasswd` file is created with a default username of `st2admin` and a default password of `Ch@ngeMe`. This can be changed using the [htpasswd utility](https://httpd.apache.org/docs/2.4/programs/htpasswd.html). +The `files/htpasswd` file is created with a default username of `st2admin` and a default password of `Ch@ngeMe`. This can be changed using the [htpasswd utility](https://httpd.apache.org/docs/2.4/programs/htpasswd.html). -Another file (`st2-cli.conf`) contains default credentials and is mounted into the "st2client" container. If you change credentials in htpasswd, you will probably want to change them in st2-cli.conf. +Another file (`files/st2-cli.conf`) contains default credentials and is mounted into the "st2client" container. If you change credentials in htpasswd, you will probably want to change them in st2-cli.conf. ### Further configuration The base st2 docker images have a built-in `/etc/st2/st2.conf` configuration file. Each st2 Docker image will load: -- /etc/st2/st2.conf +- /etc/st2/st2.conf (default [st2.conf](https://github.com/StackStorm/st2/blob/master/conf/st2.package.conf)) - /etc/st2/st2.docker.conf (values here will override st2.conf) - /etc/st2/st2.user.conf (values here will override st2.docker.conf) -Review `../base/files/st2.tmp.conf` and `st2.docker.conf` for currently set values, and it is recommended to place overrides in `st2.user.conf`. +Review `st2.docker.conf` for currently set values, and it is recommended to place overrides in `st2.user.conf`. ### Step by step first time instructions @@ -56,11 +54,11 @@ export ST2_PACKS_DEV=$HOME/projects/stackstorm-packs export ST2_EXPOSE_HTTP=0.0.0.0:8000 ``` -Secondly make any customizations to st2.user.conf, htpasswd, and st2-cli.conf. +Secondly make any customizations to `files/st2.user.conf`, `files/htpasswd`, and `files/st2-cli.conf`. Example: -To enable [sharing code between actions and sensors](https://docs.stackstorm.com/reference/sharing_code_sensors_actions.html), add these two lines to st2.user.conf: +To enable [sharing code between actions and sensors](https://docs.stackstorm.com/reference/sharing_code_sensors_actions.html), add these two lines to `files/st2.user.conf`: ```ini [packs] @@ -116,14 +114,16 @@ Example: ```shell $ docker-compose exec st2client bash -Welcome to StackStorm HA v3.3dev (Ubuntu 16.04 LTS GNU/Linux x86_64) +Welcome to StackStorm v3.2.0 (Ubuntu 18.04.4 LTS GNU/Linux x86_64) * Documentation: https://docs.stackstorm.com/ * Community: https://stackstorm.com/community-signup * Forum: https://forum.stackstorm.com/ - * Enterprise: https://stackstorm.com/#product - Warning! Do not edit configs, packs or any content inplace as they will be overridden. Modify Helm values.yaml instead! - It's recommended to use st2client container to work with StackStorm cluster. + Here you can use StackStorm CLI. Examples: + st2 action list --pack=core + st2 run core.local cmd=date + st2 run core.local_sudo cmd='apt-get update' --tail + st2 execution list root@aaabd11745f0:/opt/stackstorm# st2 run core.echo message="from the inside" . @@ -145,7 +145,7 @@ result: ## Pack Configuration -Pack configs will be in /opt/stackstorm/configs/$PACKNAME, which is a docker volume shared between st2api, st2actionrunner, and st2sensorcontainer. You can use the `st2 pack config ` in the st2client container in order to configure a pack. +Pack configs will be in `/opt/stackstorm/configs/$PACKNAME`, which is a docker volume shared between st2api, st2actionrunner, and st2sensorcontainer. You can use the `st2 pack config ` in the st2client container in order to configure a pack. ### Use st2 pack config @@ -224,7 +224,7 @@ If you are working on a development pack, you will need to register it and insta ### packs.dev directory -As mentioned above, your default `packs.dev` directory is relative to your `docker-compose.yml` file. However, if you start developing here, git will not like being inside another git directory. You will want to set `ST2_PACKS_DEV` to a directory outside of `st2-dockerfiles` and restart the docker-compose services. +As mentioned above, your default `packs.dev` directory is relative to your `docker-compose.yml` file. However, if you start developing here, git will not like being inside another git directory. You will want to set `ST2_PACKS_DEV` to a directory outside of `st2-docker` and restart the docker-compose services. Example: We have a pack called helloworld in `packs.dev/helloworld`. The directory name has to match the pack name. So even if you have a git repo named "st2-helloworld", it should be cloned locally as "helloworld". @@ -303,4 +303,4 @@ If you want to uninstall, or start from a "clean" installation, docker-compose c ```shell docker-compose down --remove-orphans -v -``` \ No newline at end of file +``` diff --git a/docker-compose.yml b/docker-compose.yml index 2b7e07be..3b6d8555 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,7 +19,7 @@ services: timeout: 1s retries: 1 volumes: - - ./st2web-healthcheck.sh:/st2web-healthcheck.sh + - ./scripts/st2web-healthcheck.sh:/st2web-healthcheck.sh ports: - "${ST2_EXPOSE_HTTP:-127.0.0.1:8000}:80" # - "${ST2_EXPOSE_HTTPS:-127.0.0.1:8443}:443" @@ -34,7 +34,7 @@ services: networks: - private volumes: - - ./makesecrets.sh:/makesecrets.sh + - ./scripts/makesecrets.sh:/makesecrets.sh - stackstorm-keys:/etc/st2/keys:rw dns_search: . command: /makesecrets.sh @@ -49,8 +49,8 @@ services: ST2_API_URL: ${ST2_API_URL:-http://st2api:9101/} ST2_STREAM_URL: ${ST2_STREAM_URL:-http://st2stream:9102/} volumes: - - ./st2.docker.conf:/etc/st2/st2.docker.conf:ro - - ./st2.user.conf:/etc/st2/st2.user.conf:ro + - ./files/st2.docker.conf:/etc/st2/st2.docker.conf:ro + - ./files/st2.user.conf:/etc/st2/st2.user.conf:ro - stackstorm-keys:/etc/st2/keys:rw - stackstorm-packs-configs:/opt/stackstorm/configs:rw - stackstorm-packs:/opt/stackstorm/packs:rw @@ -63,8 +63,8 @@ services: networks: - private volumes: - - ./st2.docker.conf:/etc/st2/st2.docker.conf:ro - - ./st2.user.conf:/etc/st2/st2.user.conf:ro + - ./files/st2.docker.conf:/etc/st2/st2.docker.conf:ro + - ./files/st2.user.conf:/etc/st2/st2.user.conf:ro dns_search: . st2scheduler: image: ${ST2_IMAGE_REPO:-stackstorm/}st2scheduler:${ST2_VERSION:-3.3dev} @@ -73,8 +73,8 @@ services: networks: - private volumes: - - ./st2.docker.conf:/etc/st2/st2.docker.conf:ro - - ./st2.user.conf:/etc/st2/st2.user.conf:ro + - ./files/st2.docker.conf:/etc/st2/st2.docker.conf:ro + - ./files/st2.user.conf:/etc/st2/st2.user.conf:ro dns_search: . st2workflowengine: image: ${ST2_IMAGE_REPO:-stackstorm/}st2workflowengine:${ST2_VERSION:-3.3dev} @@ -83,8 +83,8 @@ services: networks: - private volumes: - - ./st2.docker.conf:/etc/st2/st2.docker.conf:ro - - ./st2.user.conf:/etc/st2/st2.user.conf:ro + - ./files/st2.docker.conf:/etc/st2/st2.docker.conf:ro + - ./files/st2.user.conf:/etc/st2/st2.user.conf:ro dns_search: . st2auth: image: ${ST2_IMAGE_REPO:-stackstorm/}st2auth:${ST2_VERSION:-3.3dev} @@ -93,9 +93,9 @@ services: networks: - private volumes: - - ./st2.docker.conf:/etc/st2/st2.docker.conf:ro - - ./st2.user.conf:/etc/st2/st2.user.conf:ro - - ./htpasswd:/etc/st2/htpasswd:ro + - ./files/st2.docker.conf:/etc/st2/st2.docker.conf:ro + - ./files/st2.user.conf:/etc/st2/st2.user.conf:ro + - ./files/htpasswd:/etc/st2/htpasswd:ro dns_search: . st2actionrunner: image: ${ST2_IMAGE_REPO:-stackstorm/}st2actionrunner:${ST2_VERSION:-3.3dev} @@ -104,8 +104,8 @@ services: networks: - private volumes: - - ./st2.docker.conf:/etc/st2/st2.docker.conf:ro - - ./st2.user.conf:/etc/st2/st2.user.conf:ro + - ./files/st2.docker.conf:/etc/st2/st2.docker.conf:ro + - ./files/st2.user.conf:/etc/st2/st2.user.conf:ro - stackstorm-packs-configs:/opt/stackstorm/configs:rw - stackstorm-packs:/opt/stackstorm/packs:rw - ${ST2_PACKS_DEV:-./packs.dev}:/opt/stackstorm/packs.dev:rw @@ -119,8 +119,8 @@ services: networks: - private volumes: - - ./st2.docker.conf:/etc/st2/st2.docker.conf:ro - - ./st2.user.conf:/etc/st2/st2.user.conf:ro + - ./files/st2.docker.conf:/etc/st2/st2.docker.conf:ro + - ./files/st2.user.conf:/etc/st2/st2.user.conf:ro dns_search: . st2notifier: image: ${ST2_IMAGE_REPO:-stackstorm/}st2notifier:${ST2_VERSION:-3.3dev} @@ -129,8 +129,8 @@ services: networks: - private volumes: - - ./st2.docker.conf:/etc/st2/st2.docker.conf:ro - - ./st2.user.conf:/etc/st2/st2.user.conf:ro + - ./files/st2.docker.conf:/etc/st2/st2.docker.conf:ro + - ./files/st2.user.conf:/etc/st2/st2.user.conf:ro dns_search: . st2resultstracker: image: ${ST2_IMAGE_REPO:-stackstorm/}st2resultstracker:${ST2_VERSION:-3.3dev} @@ -139,8 +139,8 @@ services: networks: - private volumes: - - ./st2.docker.conf:/etc/st2/st2.docker.conf:ro - - ./st2.user.conf:/etc/st2/st2.user.conf:ro + - ./files/st2.docker.conf:/etc/st2/st2.docker.conf:ro + - ./files/st2.user.conf:/etc/st2/st2.user.conf:ro dns_search: . st2rulesengine: image: ${ST2_IMAGE_REPO:-stackstorm/}st2rulesengine:${ST2_VERSION:-3.3dev} @@ -149,8 +149,8 @@ services: networks: - private volumes: - - ./st2.docker.conf:/etc/st2/st2.docker.conf:ro - - ./st2.user.conf:/etc/st2/st2.user.conf:ro + - ./files/st2.docker.conf:/etc/st2/st2.docker.conf:ro + - ./files/st2.user.conf:/etc/st2/st2.user.conf:ro dns_search: . st2sensorcontainer: image: ${ST2_IMAGE_REPO:-stackstorm/}st2sensorcontainer:${ST2_VERSION:-3.3dev} @@ -160,8 +160,8 @@ services: - private dns_search: . volumes: - - ./st2.docker.conf:/etc/st2/st2.docker.conf:ro - - ./st2.user.conf:/etc/st2/st2.user.conf:ro + - ./files/st2.docker.conf:/etc/st2/st2.docker.conf:ro + - ./files/st2.user.conf:/etc/st2/st2.user.conf:ro - stackstorm-virtualenvs:/opt/stackstorm/virtualenvs:ro - stackstorm-packs:/opt/stackstorm/packs:ro - stackstorm-packs-configs:/opt/stackstorm/configs:ro @@ -174,7 +174,7 @@ services: - private dns_search: . volumes: - - ./st2.docker.conf:/etc/st2/st2.docker.conf:ro + - ./files/st2.docker.conf:/etc/st2/st2.docker.conf:ro st2client: image: ${ST2_IMAGE_REPO:-stackstorm/}st2actionrunner:${ST2_VERSION:-3.3dev} restart: on-failure @@ -191,8 +191,8 @@ services: ST2_API_URL: ${ST2_API_URL:-http://st2api:9101/} ST2_STREAM_URL: ${ST2_STREAM_URL:-http://st2stream:9102/} volumes: - - ./st2client-startup.sh:/st2client-startup.sh - - ./st2-cli.conf:/root/.st2/config + - ./scripts/st2client-startup.sh:/st2client-startup.sh + - ./files/st2-cli.conf:/root/.st2/config # external services mongo: image: mongo:3.4 diff --git a/htpasswd b/files/htpasswd similarity index 77% rename from htpasswd rename to files/htpasswd index a9771452..2773ede4 100644 --- a/htpasswd +++ b/files/htpasswd @@ -1,2 +1,3 @@ +# /etc/st2/htpasswd # st2admin:Ch@ngeMe st2admin:$apr1$GjA7KmRf$nmbcSB1XoYRPfQrL9ZBD5. diff --git a/st2-cli.conf b/files/st2-cli.conf similarity index 72% rename from st2-cli.conf rename to files/st2-cli.conf index 94a9bba3..030a80c4 100644 --- a/st2-cli.conf +++ b/files/st2-cli.conf @@ -1,3 +1,4 @@ +# /root/.st2/config [credentials] username = st2admin password = Ch@ngeMe diff --git a/st2.docker.conf b/files/st2.docker.conf similarity index 76% rename from st2.docker.conf rename to files/st2.docker.conf index f5ea3f6d..3318e380 100644 --- a/st2.docker.conf +++ b/files/st2.docker.conf @@ -1,9 +1,9 @@ # /etc/st2/st2.docker.conf # # This st2.docker.conf overrides st2.conf in the base image, and has been written to facilitate operation of -# stackstorm-compose/docker-compose.yml. It is recommended to not alter this file as an end-user, as this file -# is updated as necessary by the StackStorm st2-dockerfile maintainers. Instead, make local changes in the -# file "st2.user.conf". +# docker-compose.yml. It is recommended to not alter this file as an end-user, as this file +# is updated as necessary by the StackStorm st2-docker maintainers. +# Instead, make local changes in the file "st2.user.conf". # The order of merging: st2.conf < st2.docker.conf < st2.user.conf [auth] api_url = http://st2api:9101/ diff --git a/st2.user.conf b/files/st2.user.conf similarity index 62% rename from st2.user.conf rename to files/st2.user.conf index 9db6d427..9526c50b 100644 --- a/st2.user.conf +++ b/files/st2.user.conf @@ -1,6 +1,5 @@ # /etc/st2/st2.user.conf # # This file is provided with no custom overrides and should be utilized by end-users to make any -# desired configuration changes. No changes to this file should be checked into the st2-dockerfiles -# repository. +# desired configuration changes. No changes to this file should be checked into the st2-docker repository. # The order of merging: st2.conf < st2.docker.conf < st2.user.conf diff --git a/makesecrets.sh b/scripts/makesecrets.sh similarity index 100% rename from makesecrets.sh rename to scripts/makesecrets.sh diff --git a/st2client-startup.sh b/scripts/st2client-startup.sh similarity index 100% rename from st2client-startup.sh rename to scripts/st2client-startup.sh diff --git a/st2web-healthcheck.sh b/scripts/st2web-healthcheck.sh similarity index 100% rename from st2web-healthcheck.sh rename to scripts/st2web-healthcheck.sh From df91d5af003b51db300c43a7708afb7331dea501 Mon Sep 17 00:00:00 2001 From: armab Date: Tue, 23 Jun 2020 20:02:47 +0100 Subject: [PATCH 28/32] Switch to mongo 4.0 --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 3b6d8555..9c2ca40d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -195,7 +195,7 @@ services: - ./files/st2-cli.conf:/root/.st2/config # external services mongo: - image: mongo:3.4 + image: mongo:4.0 restart: on-failure networks: - private From f546ab8c913066271ccfaeb86fdf0c648be12f1b Mon Sep 17 00:00:00 2001 From: armab Date: Tue, 23 Jun 2020 20:10:31 +0100 Subject: [PATCH 29/32] Minor adjustments --- .github/ISSUE_TEMPLATE/bug_report.md | 2 +- README.md | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 1b850cdb..0b7cbc5c 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -11,7 +11,7 @@ A clear and concise description of what the problem is. - Host OS: [e.g. Ubuntu 16.04] - docker: [e.g. 18.03.1-ce] - docker-compose: [e.g. 1.21.1] - - stackstorm/stackstorm image: [e.g. v2.7.1] + - stackstorm version: [e.g. v3.2.0] **To Reproduce** If necessary, please specify steps to reproduce the behavior. diff --git a/README.md b/README.md index 2d7c698a..8d8595e7 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ # StackStorm in Docker Compose +[![Circle CI Build Status](https://circleci.com/gh/StackStorm/st2-docker/tree/master.svg?style=shield)](https://circleci.com/gh/StackStorm/st2-docker) This docker-compose is provided as a way to allow someone to "get up and running" quickly with StackStorm using Docker. It is not designed to be used in production, but rather a way to test out StackStorm and facilitate pack development. @@ -29,7 +30,7 @@ The image version, exposed ports, and "packs.dev" directory is configurable with ### Credentials -The `files/htpasswd` file is created with a default username of `st2admin` and a default password of `Ch@ngeMe`. This can be changed using the [htpasswd utility](https://httpd.apache.org/docs/2.4/programs/htpasswd.html). +The `files/htpasswd` file is provided with a default username of `st2admin` and a default password of `Ch@ngeMe`. This can be changed using the [htpasswd utility](https://httpd.apache.org/docs/2.4/programs/htpasswd.html). Another file (`files/st2-cli.conf`) contains default credentials and is mounted into the "st2client" container. If you change credentials in htpasswd, you will probably want to change them in st2-cli.conf. @@ -114,7 +115,7 @@ Example: ```shell $ docker-compose exec st2client bash -Welcome to StackStorm v3.2.0 (Ubuntu 18.04.4 LTS GNU/Linux x86_64) +Welcome to StackStorm v3.3dev (Ubuntu 18.04.4 LTS GNU/Linux x86_64) * Documentation: https://docs.stackstorm.com/ * Community: https://stackstorm.com/community-signup * Forum: https://forum.stackstorm.com/ From c5da6a975dc76e1fdbc030e596149c3d51a82310 Mon Sep 17 00:00:00 2001 From: armab Date: Fri, 17 Jul 2020 16:20:39 +0100 Subject: [PATCH 30/32] Add a Changelog record for the new docker-compose deployment --- CHANGELOG.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..d53cdedf --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,19 @@ +# Changelog + +## 2020-07-17 +* Replace docker-compose with a new deployment based on [stackstorm/st2-dockerfiles](https://github.com/StackStorm/st2-dockerfiles/) images relying on `Ubuntu Bionic` and `python 3` since st2 `v3.3dev` (#192) + +## 2020-05-26 +* Deprecate demo all-in-one docker-compose deployment based on outdated `Ubuntu Trusty` with `python 2`, unsupported since st2 `v3.1.0` (#191) + +## 2018-06-28 +* Add `st2workflowengine` to `entrypoint-1ppc.sh` and `compose-1ppc/docker-compose.yml`. + +## 2018-06-18 +* The `TAG` environment variable is replaced by `ST2_IMAGE_TAG`. + +## 2018-02-27 +* Pin DB's to specific, tested versions. + +## 2017-10-23 +* Rename `/entrypoint.d/` to `/st2-docker/entrypoint.d/`. From 05bd5b1fcbfc4f96994ba1993ee07ee280d2c91a Mon Sep 17 00:00:00 2001 From: armab Date: Fri, 17 Jul 2020 16:21:04 +0100 Subject: [PATCH 31/32] Minor adjustments --- .circleci/config.yml | 4 ++-- .github/ISSUE_TEMPLATE/bug_report.md | 6 +++--- README.md | 3 ++- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ca942c6b..5119f246 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -25,10 +25,10 @@ jobs: name: Start st2 with docker-compose command: docker-compose up --detach - run: - name: Run st2 Smoke-Tests + name: Run st2 smoke-tests command: | # TODO: Replace with 'docker-compose run status' - # Examples: https://github.com/docker/compose/issues/374#issuecomment-310266246 + # Example: https://github.com/docker/compose/issues/374#issuecomment-310266246 sleep 45 # TODO: Replace with more organized BATS tests 'docker-compose run tests' # Example: https://github.com/StackStorm/stackstorm-ha/blob/master/tests/st2tests.sh diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 0b7cbc5c..39f70e61 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -8,10 +8,10 @@ about: Create a report to help us improve the stackstorm docker image A clear and concise description of what the problem is. **Versions** - - Host OS: [e.g. Ubuntu 16.04] - - docker: [e.g. 18.03.1-ce] + - Host OS: [e.g. Ubuntu 18.04] + - docker: [e.g. 18.09.1-ce] - docker-compose: [e.g. 1.21.1] - - stackstorm version: [e.g. v3.2.0] + - stackstorm version: [e.g. v3.3.0] **To Reproduce** If necessary, please specify steps to reproduce the behavior. diff --git a/README.md b/README.md index 8d8595e7..5a5bec85 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,8 @@ # StackStorm in Docker Compose [![Circle CI Build Status](https://circleci.com/gh/StackStorm/st2-docker/tree/master.svg?style=shield)](https://circleci.com/gh/StackStorm/st2-docker) -This docker-compose is provided as a way to allow someone to "get up and running" quickly with StackStorm using Docker. It is not designed to be used in production, but rather a way to test out StackStorm and facilitate pack development. +This docker-compose is provided as a way to allow "get up and running" quickly with StackStorm using Docker (based on [st2-dockerfiles](https://github.com/stackstorm/st2-dockerfiles)). It is not designed to be used in production, but rather a way to test out StackStorm and facilitate pack development. +> If you need Highly Availability experience, there is Kubernetes installation available via Helm charts at https://docs.stackstorm.com/install/k8s_ha.html. ## TL;DR From 0a1b754c486263ab3b440fc26a7b9e60fc358966 Mon Sep 17 00:00:00 2001 From: armab Date: Fri, 17 Jul 2020 16:31:18 +0100 Subject: [PATCH 32/32] Switch st2web to localhost on default http port for better UX This also consistent with the old deployment configuration --- README.md | 6 +++--- docker-compose.yml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 5a5bec85..69940d28 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ docker-compose up -d docker-compose exec st2client bash # this gives you access to the st2 command line ``` -Open `http://localhost:8000/` in your browser. StackStorm Username/Password by default is: `st2admin/Ch@ngeMe`. +Open `http://localhost/` in your browser. StackStorm Username/Password by default is: `st2admin/Ch@ngeMe`. ## Usage @@ -26,7 +26,7 @@ The image version, exposed ports, and "packs.dev" directory is configurable with - **ST2_VERSION** this is the tag at the end of the docker image (ie: stackstorm/st2api:v3.3dev) - **ST2_IMAGE_REPO** The image or path to the images. Default is "stackstorm/". You may change this is using the Enterprise version or a private docker repository. -- **ST2_EXPOSE_HTTP** Port to expose st2web port 80 on. Default is `127.0.0.1:8000`, and you may want to do `0.0.0.0:8000` to expose on all interfaces. +- **ST2_EXPOSE_HTTP** Port to expose st2web port 80 on. Default is `127.0.0.1:80`, and you may want to do `0.0.0.0:80` to expose on all interfaces. - **ST2_PACKS_DEV** Directory to development packs, absolute or relative to docker-compose.yml. This allows you to develop packs locally. Default is `./packs.dev`. When making a number of packs, it is recommended to make a directory outside of st2-docker, with each subdirectory underneath that being an independent git repo. Example: `ST2_PACKS_DEV=${HOME}/mypacks`, with `${HOME}/mypacks/st2-helloworld` being a git repo for the "helloworld" pack. ### Credentials @@ -53,7 +53,7 @@ Example: ```shell export ST2_PACKS_DEV=$HOME/projects/stackstorm-packs -export ST2_EXPOSE_HTTP=0.0.0.0:8000 +export ST2_EXPOSE_HTTP=0.0.0.0:80 ``` Secondly make any customizations to `files/st2.user.conf`, `files/htpasswd`, and `files/st2-cli.conf`. diff --git a/docker-compose.yml b/docker-compose.yml index 9c2ca40d..59aa02c4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,8 +21,8 @@ services: volumes: - ./scripts/st2web-healthcheck.sh:/st2web-healthcheck.sh ports: - - "${ST2_EXPOSE_HTTP:-127.0.0.1:8000}:80" - # - "${ST2_EXPOSE_HTTPS:-127.0.0.1:8443}:443" + - "${ST2_EXPOSE_HTTP:-127.0.0.1:80}:80" + # - "${ST2_EXPOSE_HTTPS:-127.0.0.1:443}:443" # more work would be needed with certificate generate to make https work. networks: - private