[WIP] Add ubuntu xenial base image

docker-compose.override.yml

@@ -0,0 +1,7 @@
+version: '3'
+
+services:
+  stackstorm:
+    image: stackstorm/stackstorm:${ST2_IMAGE_TAG:-xenial}
+    environment:
+      - ST2_ENABLE_SSHD=1

images/stackstorm-xenial/.circleci/config.yml

@@ -0,0 +1,15 @@
+version: 2
+jobs:
+  build:
+    docker:
+      - image: docker/compose:1.23.1
+    steps:
+      - run: |
+          set -x
+          printenv | sort
+      - checkout
+      - setup_remote_docker:
+          docker_layer_caching: true
+      - run:
+          name: Install bash
+          command: apk add --no-cache bash

images/stackstorm-xenial/.circleci/docker-compose.yml

@@ -0,0 +1,74 @@
+version: '3'
+
+services:
+  stackstorm:
+    image: stackstorm/stackstorm:xenial
+    restart: always
+    environment:
+      - ST2_ENABLE_SSHD=1
+    env_file:
+      - conf/stackstorm.env
+      - conf/mongo.env
+      - conf/rabbitmq.env
+      - conf/postgres.env
+      - conf/redis.env
+    volumes:
+      - ./packs.dev:/opt/stackstorm/packs.dev
+      - ./runtime/entrypoint.d:/st2-docker/entrypoint.d
+      - ./runtime/st2.d:/st2-docker/st2.d
+      - ./conf/stackstorm.env:/st2-docker/env
+    ports:
+      - "443:443"
+    depends_on:
+      - mongo
+      - postgres
+      - rabbitmq
+      - redis
+    networks:
+      - public
+      - private
+    dns_search: .
+
+### External Services
+
+  mongo:
+    image: mongo:3.4
+    env_file:
+      - conf/mongo.env
+    networks:
+      - private
+    dns_search: .
+  rabbitmq:
+    image: rabbitmq:3.6-management
+    env_file:
+      - conf/rabbitmq.env
+    networks:
+      - private
+    dns_search: .
+  postgres:
+    image: postgres:9.6
+    env_file:
+      - conf/postgres.env
+    networks:
+      - private
+    dns_search: .
+  redis:
+    image: redis:4.0
+    env_file:
+      - conf/redis.env
+    networks:
+      - private
+    dns_search: .
+    command: [
+      "bash", "-c",
+      '
+       docker-entrypoint.sh
+       --requirepass "$$REDIS_PASSWORD"
+      '
+    ]
+
+networks:
+  public:
+    driver: bridge
+  private:
+    driver: bridge

images/stackstorm-xenial/Dockerfile

@@ -0,0 +1,110 @@
+FROM ubuntu:16.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+# Install essential packages
+RUN set -eux && \
+apt-get update && \
+apt-get -y install --no-install-recommends \
+      apache2-utils \
+      apt-transport-https \
+      ca-certificates \
+      crudini \
+      curl \
+      supervisor \
+      wget
+
+# Install nginx
+RUN set -eux && \
+curl -sSL https://nginx.org/keys/nginx_signing.key | apt-key add - && \
+echo "deb https://nginx.org/packages/mainline/ubuntu/ xenial nginx" >> /etc/apt/sources.list.d/nginx.list && \
+apt-get update && \
+apt-get -y install --no-install-recommends nginx
+
+# Set nodejs version to install
+# This can be overriden at build time by passing `--arg NODE_REPO=node_12.x`
+ARG NODE_REPO=node_10.x
+
+# Install nodejs
+RUN set -eux && \
+curl -sSL https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - && \
+echo "deb https://deb.nodesource.com/${NODE_REPO}/ xenial main" > /etc/apt/sources.list.d/nodesource.list && \
+apt-get update && \
+apt-get -y install --no-install-recommends nodejs
+
+# Set st2 repo/version to install
+# These can be overriden at build time
+# If ST2_REPO=stable and ST2_VERSION left blank, then it picks the latest stable version
+#  available at build time
+ARG ST2_REPO=stable
+ARG ST2_VERSION=
+
+# Install stackstorm: st2, st2chatops, st2web, st2mistral
+RUN set -eux && \
+curl -sSL https://packagecloud.io/StackStorm/${ST2_REPO}/gpgkey | apt-key add - && \
+echo "deb https://packagecloud.io/StackStorm/${ST2_REPO}/ubuntu/ xenial main" > /etc/apt/sources.list.d/st2.list && \
+apt-get update && \
+apt-get -y install --no-install-recommends \
+      st2=${ST2_VERSION}* \
+      st2chatops=${ST2_VERSION}* \
+      st2mistral=${ST2_VERSION}* \
+      st2web=${ST2_VERSION}*
+
+# Copy supervisord main config
+# It switches source file between stable/unstable based on ST2_REPO var
+#   so that it can have different supervisord configs among `latest` and `dev` image
+#   which is useful especially when introducing a new st2 component
+COPY ${ST2_REPO}/supervisord.conf /etc/supervisor/supervisord.conf
+
+# Copy supervisord config fragments
+# These files are dynamically copied into /etc/supervisor/conf.d based on env vars and
+#   become effective to supervisord during the startup process
+COPY supervisord.d/ /st2-docker/supervisord.d
+
+# Copy entrypoint shell scripts
+# - entrypoint.sh: Main entrypoint
+# - entrypoint-supervisord.sh: Starts supervisord. Called from entrypoint.sh
+# - entrypoint-1ppc.sh: Handles 1ppc mode. Called from entrypoint.sh instead of
+#     entrypoint-supervisord.sh if running 1ppc mode
+COPY bin/entrypoint.sh /st2-docker/bin/
+COPY bin/entrypoint-supervisord.sh /st2-docker/bin/
+COPY bin/entrypoint-1ppc.sh /st2-docker/bin/
+
+# Copy shell scripts that are launched by supervisord
+# - run-sshd: Starts sshd. It also does some initialization before starting daemon
+# - run-postinit.sh: Run shell scripts in /st2-docker/st2.d/ *AFTER* st2 become ready
+COPY bin/run-sshd.sh /st2-docker/bin/
+COPY bin/run-postinit.sh /st2-docker/bin/
+
+# Copy shell scripts that are luunched by supervisord: for all st2 components
+COPY bin/run-st2/ /st2-docker/bin/run-st2
+
+# Replace st2ctl command
+# This will become unnecessary when the modification is merged into upstream
+COPY st2ctl /opt/stackstorm/st2/bin/st2ctl
+
+# Copy insecure keys
+COPY insecure_keys/insecure_datastore_key.json /etc/st2/keys/datastore_key.json 
+COPY insecure_keys/insecure_ssl_st2.crt /etc/ssl/st2/st2.crt
+COPY insecure_keys/insecure_ssl_st2.key /etc/ssl/st2/st2.key
+COPY insecure_keys/insecure_stanley_rsa /home/stanley/.ssh/stanley_rsa
+COPY insecure_keys/insecure_stanley_rsa.pub /home/stanley/.ssh/stanley_rsa.pub
+
+# Do post-install setup
+RUN set -eux && \
+# Install redis client library for coordination backend
+# see: https://docs.stackstorm.com/latest/reference/policies.html
+bash -c 'source /opt/stackstorm/st2/bin/activate && pip install redis' && \
+# Enable authentication
+crudini --set /etc/st2/st2.conf auth enable True && \
+# Enable datastore (st2kv) encryption by specifying encryption key
+crudini --set /etc/st2/st2.conf keyvalue encryption_key_path /etc/st2/keys/datastore_key.json && \
+# Copy nginx config for st2web
+cp /usr/share/doc/st2/conf/nginx/st2.conf /etc/nginx/conf.d/st2.conf && \
+# Set default locale to C.UTF-8
+echo LANG=C.UTF-8 > /etc/default/locale
+
+# Set LANG to C.UTF-8
+ENV LANG=C.UTF-8
+
+ENTRYPOINT ["/st2-docker/bin/entrypoint.sh"]

images/stackstorm-xenial/README.md

@@ -0,0 +1,52 @@
+
+## Building images
+
+```
+docker build -t stackstorm/stackstorm:xenial .
+
+docker build -t stackstorm/stackstorm:xenial-2.9.1 --build-arg ST2_VERSION=2.9.1 .
+docker build -t stackstorm/stackstorm:xenial-2.9.0 --build-arg ST2_VERSION=2.9.0 .
+docker build -t stackstorm/stackstorm:xenial-2.8.1 --build-arg ST2_VERSION=2.8.1 .
+
+docker build -t stackstorm/stackstorm:xenial-dev --build-arg ST2_REPO=unstable --build-arg NODE_REPO=node_8.x .
+```
+
+## Running `st2-self-test`
+
+```
+apt-get -y install uuid-runtime
+export ST2_AUTH_TOKEN=$(st2 auth $ST2_USER -p $ST2_PASSWORD -t)
+/opt/stackstorm/st2/bin/st2-self-check
+```
+
+## Insecure keys
+
+Introduce "insecure" keys in following places.
+
+- st2 datastore key
+- SSL self-signed cert for nginx/st2web
+- ssh key for stanley user
+
+The idea is like "insecure ssh key" of Vagrant. We save the pre-generated keys in this repo and copy them into the image.
+
+This helps those keys to be consistent among images of different versions/builds. Previously, those are dynamically generated during the docker build process, and it depends on the environment wheter it changes or not because of docker layer caching. This unstableness is considered harmful in both dev/ci and production use case. Leaving the default keys in production also makes the installation vulnerable.
+
+Unlike vagrant, these insecure keys are not replaced at runtime by default. For dev/ci use case, the default insecure key should just work fine. But for production usage, it is strongly recommended to replace them by the one that user generated by their own. This should be mentioned in main README.md to warn users.
+
+## New envvars
+
+- `ST2_DISABLE_MISTRAL`
+    - Controls whether to enable Mistral or not. If any value is set, following steps will be skipped during the container startup.
+        - Modification of `/etc/mistral/mistral.conf` to set PostgreSQL/RabbitMQ connection configs
+        - Invocation of `mistral-db-manage` for schema migration/data population
+        - Configuration of mistral-api, mistral-server process under supervisord: this means that those processes can't be started with `supervisorctl start`, even not be listed at all with `supervisorctl list`
+    - default: (not set)
+
+- `ST2_ENABLE_ST2CHATOPS`
+    - Controls whether to start `st2chatops` or not. If any value is set, supervisord will launch st2chatops. If not, it will even skip the configuration of supervisord for `st2chatops`, so that it prevents from `st2ctl reload` or `st2ctl restart` or any similar command to accidentaly starts `st2chatops` process
+    - default: (not set)
+
+- `ST2_ENABLE_SSHD`
+    - Controls whether to start sshd daemon or not. If any value is set, supervisord will launch sshd. **Only meant for testing purpose.** Need to be enabled when you run `st2-self-check`, since it uses some `core.remote` actions during the tests
+    - default: (not set)
+

images/stackstorm-xenial/bin/entrypoint-1ppc.sh

@@ -0,0 +1,99 @@
+#!/bin/bash
+
+ST2_CONF=/etc/st2/st2.conf
+crudini --set ${ST2_CONF} auth    api_url     ${ST2_API_URL}
+crudini --set ${ST2_CONF} mistral api_url     ${ST2_API_URL}
+crudini --set ${ST2_CONF} mistral v2_base_url ${ST2_MISTRAL_API_URL}
+
+# Configure CORS to accept any source
+# st2api gunicorn process is directly exposed to clients in 1ppc mode
+crudini --set ${ST2_CONF} api allow_origin '*'
+
+# Generate nginx config for st2web to support load balancing to st2api, st2auth and st2stream
+/st2-docker/bin/inject_env.py \
+  < /etc/nginx/conf.d/st2-1ppc.conf.tpl \
+  > /etc/nginx/conf.d/st2.conf
+
+case "$ST2_SERVICE" in
+  "nop" )
+    exec tail -f /dev/null
+    ;;
+  "st2api" )
+    DAEMON_ARGS="-k eventlet -b 0.0.0.0:9101 --workers 1 --threads 1 --graceful-timeout 10 --timeout 30"
+    exec /opt/stackstorm/st2/bin/gunicorn st2api.wsgi:application $DAEMON_ARGS
+    ;;
+  "st2auth" )
+    DAEMON_ARGS="-k eventlet -b 0.0.0.0:9100 --workers 1 --threads 1 --graceful-timeout 10 --timeout 30"
+    exec /opt/stackstorm/st2/bin/gunicorn st2auth.wsgi:application $DAEMON_ARGS
+    ;;
+  "st2stream" )
+    DAEMON_ARGS="-k eventlet -b 0.0.0.0:9102 --workers 1 --threads 10 --graceful-timeout 10 --timeout 30"
+    exec /opt/stackstorm/st2/bin/gunicorn st2stream.wsgi:application $DAEMON_ARGS
+    ;;
+  "st2sensorcontainer" )
+    DAEMON_ARGS="--config-file /etc/st2/st2.conf"
+    exec /opt/stackstorm/st2/bin/st2sensorcontainer ${DAEMON_ARGS}
+    ;;
+  "st2rulesengine" )
+    DAEMON_ARGS="--config-file /etc/st2/st2.conf"
+    exec /opt/stackstorm/st2/bin/st2rulesengine ${DAEMON_ARGS}
+    ;;
+  "st2workflowengine" )
+    DAEMON_ARGS="--config-file /etc/st2/st2.conf"
+    exec /opt/stackstorm/st2/bin/st2workflowengine ${DAEMON_ARGS}
+    ;;
+  "st2actionrunner" )
+    DAEMON_ARGS="--config-file /etc/st2/st2.conf"
+    exec /opt/stackstorm/st2/bin/st2actionrunner ${DAEMON_ARGS}
+    ;;
+  "st2resultstracker" )
+    DAEMON_ARGS="--config-file /etc/st2/st2.conf"
+    exec /opt/stackstorm/st2/bin/st2resultstracker ${DAEMON_ARGS}
+    ;;
+  "st2notifier" )
+    DAEMON_ARGS="--config-file /etc/st2/st2.conf"
+    exec /opt/stackstorm/st2/bin/st2notifier ${DAEMON_ARGS}
+    ;;
+  "st2garbagecollector" )
+    DAEMON_ARGS="--config-file /etc/st2/st2.conf"
+    exec /opt/stackstorm/st2/bin/st2garbagecollector ${DAEMON_ARGS}
+    ;;
+  "st2timersengine" )
+    DAEMON_ARGS="--config-file /etc/st2/st2.conf"
+    exec /opt/stackstorm/st2/bin/st2timersengine ${DAEMON_ARGS}
+    ;;
+  "mistral-api" )
+    set -e
+    /opt/stackstorm/mistral/bin/mistral-db-manage --config-file /etc/mistral/mistral.conf upgrade head
+    /opt/stackstorm/mistral/bin/mistral-db-manage --config-file /etc/mistral/mistral.conf populate
+    API_ARGS="--log-file /var/log/mistral/mistral-api.log -b 0.0.0.0:8989 -w 2 mistral.api.wsgi --graceful-timeout 10"
+    exec /opt/stackstorm/mistral/bin/gunicorn $API_ARGS
+    ;;
+  "mistral-server" )
+    set -e
+    /opt/stackstorm/mistral/bin/mistral-db-manage --config-file /etc/mistral/mistral.conf upgrade head
+    /opt/stackstorm/mistral/bin/mistral-db-manage --config-file /etc/mistral/mistral.conf populate
+    SERVER_ARGS="--config-file /etc/mistral/mistral.conf --log-file /var/log/mistral/mistral-server.log"
+    exec /opt/stackstorm/mistral/bin/mistral-server --server engine,executor,notifier ${SERVER_ARGS}
+    ;;
+  "st2web" )
+    exec /usr/sbin/nginx -g 'daemon off;'
+    ;;
+  "st2chatops" )
+    set -e
+    export ST2_API=${ST2_API_URL}
+    cd /opt/stackstorm/chatops
+    exec bin/hubot $DAEMON_ARGS
+    ;;
+  "st2-register-content" )
+    set -ex
+    PACKS=${PACKS:-"chatops core default linux packs"}
+    for PACK in ${PACKS}; do
+      st2-register-content --config-file /etc/st2/st2.conf \
+        --register-all \
+        --register-setup-virtualenvs \
+        --register-pack /opt/stackstorm/packs/${PACK}
+    done
+    ;;
+
+esac

images/stackstorm-xenial/bin/entrypoint-supervisord.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+set -e
+
+# set default path
+export PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
+
+# set number of st2actionrunners to start
+ST2ACTIONRUNNER_WORKERS=$(/usr/bin/nproc 2>/dev/null)
+ST2ACTIONRUNNER_WORKERS=${ST2ACTIONRUNNER_WORKERS:-10}
+export ST2ACTIONRUNNER_WORKERS
+
+# Read configuration variable file if it is present
+set -o allexport
+[ -r /etc/default/supervisord ] && . /etc/default/supervisord
+set +o allexport
+
+# launch supervisord
+exec /usr/bin/supervisord -c /etc/supervisor/supervisord.conf