From 34db0b58c990e6dd0142c38fd8955e7de27f1885 Mon Sep 17 00:00:00 2001 From: geek Date: Fri, 16 Jun 2017 13:02:46 -0500 Subject: [PATCH] Support triton metrics --- Dockerfile | 32 ++++++++----- bin/prestart.sh | 25 ++++++++++ etc/containerpilot.json | 8 ++-- etc/prometheus.yml.ctmpl | 30 +++++++++++- examples/compose/README.md | 7 +++ .../compose/docker-compose.yml | 33 +++++++------ examples/triton/README.md | 20 ++++++++ examples/triton/docker-compose.yml | 46 +++++++++++++++++++ setup.sh => examples/triton/setup.sh | 17 ++++++- local-compose.yml | 21 --------- 10 files changed, 182 insertions(+), 57 deletions(-) create mode 100755 bin/prestart.sh create mode 100644 examples/compose/README.md rename docker-compose.yml => examples/compose/docker-compose.yml (51%) create mode 100644 examples/triton/README.md create mode 100644 examples/triton/docker-compose.yml rename setup.sh => examples/triton/setup.sh (81%) delete mode 100644 local-compose.yml diff --git a/Dockerfile b/Dockerfile index cc929cc..8c378a2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ FROM alpine:3.6 # artisanally hand-rolling curl and the rest of our stack we'll just use # Alpine so we can use `docker build`. -RUN apk add --update curl +RUN apk add --update curl bash # add Prometheus. alas, the Prometheus developers provide no checksum RUN export PROM_VERSION=1.7.1 \ @@ -22,22 +22,31 @@ RUN export PROM_VERSION=1.7.1 \ && ln -s /usr/share/prometheus/console_libraries /usr/share/prometheus/consoles/ /etc/prometheus/ \ && rm /tmp/prometheus-${PROM_VERSION}.linux-amd64.tar.gz -# get consul-template -RUN curl -Lso /tmp/consul-template_0.14.0_linux_amd64.zip https://releases.hashicorp.com/consul-template/0.14.0/consul-template_0.14.0_linux_amd64.zip \ - && echo "7c70ea5f230a70c809333e75fdcff2f6f1e838f29cfb872e1420a63cdf7f3a78 /tmp/consul-template_0.14.0_linux_amd64.zip" | sha256sum -c \ - && unzip /tmp/consul-template_0.14.0_linux_amd64.zip \ - && mv consul-template /bin \ - && rm /tmp/consul-template_0.14.0_linux_amd64.zip - -# get consul-agent -RUN export CONSUL_VERSION=0.7.0 \ - && export CONSUL_CHECKSUM=b350591af10d7d23514ebaa0565638539900cdb3aaa048f077217c4c46653dd8 \ +# Install Consul +# Releases at https://releases.hashicorp.com/consul +RUN set -ex \ + && export CONSUL_VERSION=0.7.5 \ + && export CONSUL_CHECKSUM=40ce7175535551882ecdff21fdd276cef6eaab96be8a8260e0599fadb6f1f5b8 \ && curl --retry 7 --fail -vo /tmp/consul.zip "https://releases.hashicorp.com/consul/${CONSUL_VERSION}/consul_${CONSUL_VERSION}_linux_amd64.zip" \ && echo "${CONSUL_CHECKSUM} /tmp/consul.zip" | sha256sum -c \ && unzip /tmp/consul -d /usr/local/bin \ && rm /tmp/consul.zip \ + # Create empty directories for Consul config and data \ + && mkdir -p /etc/consul \ + && mkdir -p /var/lib/consul \ && mkdir /config + +# Install Consul template +# Releases at https://releases.hashicorp.com/consul-template/ +RUN set -ex \ + && export CONSUL_TEMPLATE_VERSION=0.18.0 \ + && export CONSUL_TEMPLATE_CHECKSUM=f7adf1f879389e7f4e881d63ef3b84bce5bc6e073eb7a64940785d32c997bc4b \ + && curl --retry 7 --fail -Lso /tmp/consul-template.zip "https://releases.hashicorp.com/consul-template/${CONSUL_TEMPLATE_VERSION}/consul-template_${CONSUL_TEMPLATE_VERSION}_linux_amd64.zip" \ + && echo "${CONSUL_TEMPLATE_CHECKSUM} /tmp/consul-template.zip" | sha256sum -c \ + && unzip /tmp/consul-template.zip -d /usr/local/bin \ + && rm /tmp/consul-template.zip + # Add Containerpilot and set its configuration ENV CONTAINERPILOT_VERSION 3.0.0 ENV CONTAINERPILOT /etc/containerpilot.json @@ -58,6 +67,7 @@ ENV CONTAINERPILOT /etc/containerpilot.json # ref https://prometheus.io/docs/operating/configuration/ # for details on building your own config COPY etc/prometheus.yml.ctmpl /etc/prometheus/prometheus.yml.ctmpl +COPY bin /bin # Override the entrypoint to include Containerpilot WORKDIR /prometheus diff --git a/bin/prestart.sh b/bin/prestart.sh new file mode 100755 index 0000000..b47604a --- /dev/null +++ b/bin/prestart.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# Do we have env vars for Triton discovery? +# Copy creds from env vars to files on disk +if [ -n ${!TRITON_CREDS_PATH} ] \ + && [ -n ${!TRITON_CA} ] \ + && [ -n ${!TRITON_CERT} ] \ + && [ -n ${!TRITON_KEY} ] +then + mkdir -p ${TRITON_CREDS_PATH} + echo -e "${TRITON_CA}" | tr '#' '\n' > ${TRITON_CREDS_PATH}/ca.pem + echo -e "${TRITON_CERT}" | tr '#' '\n' > ${TRITON_CREDS_PATH}/cert.pem + echo -e "${TRITON_KEY}" | tr '#' '\n' > ${TRITON_CREDS_PATH}/key.pem +fi + +# Are we on Triton? Do we _not_ have a user-defined DC? +# Set the DC automatically from mdata +if [ -n ${TRITON_DC} ] \ + && [ -f "/native/usr/sbin/mdata-get" ] +then + export TRITON_DC=$(/native/usr/sbin/mdata-get sdc:datacenter_name) +fi + +# Create Prometheus config +consul-template -once -consul-addr ${CONSUL}:8500 -template /etc/prometheus/prometheus.yml.ctmpl:/etc/prometheus/prometheus.yml diff --git a/etc/containerpilot.json b/etc/containerpilot.json index f0a7238..e8bfd56 100644 --- a/etc/containerpilot.json +++ b/etc/containerpilot.json @@ -3,10 +3,8 @@ "jobs": [ { "name": "setup-config", - "exec": [ - "consul-template", "-once", "-consul", "{{ if .CONSUL_AGENT }}localhost{{ else }}{{ .CONSUL | default "consul" }}{{ end }}:8500", - "-template", "/etc/prometheus/prometheus.yml.ctmpl:/etc/prometheus/prometheus.yml" - ]{{ if .CONSUL_AGENT }}, + "exec": "/bin/prestart.sh" + {{ if .CONSUL_AGENT }}, "when": { "source": "consul-agent", "once": "healthy" @@ -23,7 +21,7 @@ "-web.console.templates=/etc/prometheus/consoles" ], "health": { - "exec": "curl -so /dev/null http://localhost:9090/metrics", + "exec": "curl -fso /dev/null http://localhost:9090/metrics", "interval": 10, "ttl": 25 }, diff --git a/etc/prometheus.yml.ctmpl b/etc/prometheus.yml.ctmpl index 302bf45..86b033b 100644 --- a/etc/prometheus.yml.ctmpl +++ b/etc/prometheus.yml.ctmpl @@ -1,7 +1,7 @@ # my global config global: scrape_interval: 15s # By default, scrape targets every 15 seconds. - evaluation_interval: 15s # By default, scrape targets every 15 seconds. + evaluation_interval: 15s # By default, evaluate rules every 15 seconds. # scrape_timeout is set to the global default (10s). # Attach these labels to any time series or alerts when communicating with @@ -26,11 +26,37 @@ scrape_configs: metrics_path: /metrics # scheme defaults to 'http'. + {{ $consul_agent := env "CONSUL_AGENT" }} + {{ $consul := env "CONSUL" }} consul_sd_configs: - - server: '{{ if .CONSUL_AGENT }}localhost{{ else }}{{ if .CONSUL }}{{ .CONSUL }}{{ else }}consul{{ end }}{{ end }}:8500' + - server: '{{ if $consul_agent }}localhost{{ else }}{{ if $consul }}{{ $consul }}{{ else }}consul{{ end }}{{ end }}:8500' services: ['containerpilot'] - job_name: 'prometheus' metrics_path: /metrics static_configs: - targets: ['localhost:9090'] + + {{ $triton_tls_configured := env "TRITON_CREDS_PATH" }} + {{ if $triton_tls_configured }} + - job_name: 'triton' + scheme: https + tls_config: + ca_file: '{{env "TRITON_CA_PATH"}}' + cert_file: '{{env "TRITON_CERT_PATH"}}' + key_file: '{{env "TRITON_KEY_PATH"}}' + insecure_skip_verify: true + triton_sd_configs: + - account: '{{env "TRITON_ACCOUNT"}}' + dns_suffix: 'cmon.{{env "TRITON_DC"}}.triton.zone' + endpoint: 'cmon.{{env "TRITON_DC"}}.triton.zone' + version: 1 + tls_config: + ca_file: '{{env "TRITON_CA_PATH"}}' + cert_file: '{{env "TRITON_CERT_PATH"}}' + key_file: '{{env "TRITON_KEY_PATH"}}' + insecure_skip_verify: true + relabel_configs: + - source_labels: [__meta_triton_machine_alias] + target_label: instance + {{ end }} diff --git a/examples/compose/README.md b/examples/compose/README.md new file mode 100644 index 0000000..187e78d --- /dev/null +++ b/examples/compose/README.md @@ -0,0 +1,7 @@ +# Autopilot Pattern Prometheus on local Docker + +To launch Prometheus locally (on Docker for Mac as an example): + +```bash +$ docker-compose -p prometheus up -d +``` diff --git a/docker-compose.yml b/examples/compose/docker-compose.yml similarity index 51% rename from docker-compose.yml rename to examples/compose/docker-compose.yml index b26dbf7..efdcb72 100644 --- a/docker-compose.yml +++ b/examples/compose/docker-compose.yml @@ -1,33 +1,32 @@ +version: '2.1' # Prometheus demonstration of the autopilot pattern -prometheus: - image: autopilotpattern/prometheus:0.17.0-r1 +services: + prometheus: + build: ../../ mem_limit: 1g restart: always - labels: - - triton.cns.services=prometheus + dns: + - 127.0.0.1 ports: - 9090 - env_file: _env + environment: + - CONSUL=consul + - CONSUL_AGENT=1 + links: + - consul:consul -# Start with a single host which will bootstrap the cluster. -# In production we'll want to use an HA cluster. -consul: + # Start with a single host which will bootstrap the cluster. + # In production we'll want to use an HA cluster. + consul: image: autopilotpattern/consul:0.7.2-r0.8 restart: always mem_limit: 128m - expose: - - 53 - - 8300 - - 8301 - - 8302 - - 8400 + ports: - 8500 dns: - - 127.0.0.1 - labels: - - triton.cns.services=consul + - 127.0.0.1 command: > /usr/local/bin/containerpilot /bin/consul agent -server diff --git a/examples/triton/README.md b/examples/triton/README.md new file mode 100644 index 0000000..900f26a --- /dev/null +++ b/examples/triton/README.md @@ -0,0 +1,20 @@ +# Autopilot Pattern Prometheus on Triton + +1. [Get a Joyent account](https://my.joyent.com/landing/signup/) and [add your SSH key](https://docs.joyent.com/public-cloud/getting-started). +2. Install [Docker](https://docs.docker.com/docker-for-mac/install/) on your laptop or other environment, as well as the [Joyent Triton CLI](https://www.joyent.com/blog/introducing-the-triton-command-line-tool). +3. Install the [Triton Docker CLI helper](https://github.com/joyent/triton-docker-cli). + +Check that everything is configured correctly by running the `setup.sh` script. This will check that your environment is setup correctly and create an `_env` file that includes environment variables with reasonable defaults, if not, run `eval "$(triton env)"`. + +```bash +$ setup.sh +$ vim _env +``` + +See the [README](../../README.md) for details on environment variables in `_env`. + +Start everything: + +```bash +triton-docker up -d +``` diff --git a/examples/triton/docker-compose.yml b/examples/triton/docker-compose.yml new file mode 100644 index 0000000..7770adf --- /dev/null +++ b/examples/triton/docker-compose.yml @@ -0,0 +1,46 @@ +version: '2.1' +# Prometheus demonstration of the autopilot pattern + +services: + prometheus: + image: autopilotpattern/prometheus:${TAG:-latest} + # Joyent recommends setting instances to always restart on Triton + restart: always + labels: + # This label sets the CNS name, Triton's automatic DNS + # Learn more at https://docs.joyent.com/public-cloud/network/cns + - triton.cns.services=prometheus + # This label selects the proper Joyent resource package + # https://www.joyent.com/blog/optimizing-docker-on-triton#ram-cpu-and-disk-resources-for-your-containers + - com.joyent.package=g4-highcpu-1G + network_mode: bridge + ports: + # You may not want these port declarations for production. Without them, Prometheus will only + # listen on the private network. This will also result in a public prometheus CNS record being created, + # in the triton.zone domain. + - 9090 + env_file: _env + environment: + - CONSUL_AGENT=1 + - CONSUL=pc.svc.${TRITON_CNS_SEARCH_DOMAIN_PRIVATE} + + + # Consul is the service catalog + consul: + image: autopilotpattern/consul:0.7.2-r0.8 + command: > + /usr/local/bin/containerpilot + /bin/consul agent -server + -bootstrap-expect 1 + -config-dir=/etc/consul + -ui-dir /ui + # Change "-bootstrap" to "-bootstrap-expect 3", then scale to 3 or more to + # turn this into an HA Consul raft. + restart: always + mem_limit: 128m + ports: + # As above, this port delcaration should not be made for production. + - 8500 + labels: + - triton.cns.services=pc + network_mode: bridge diff --git a/setup.sh b/examples/triton/setup.sh similarity index 81% rename from setup.sh rename to examples/triton/setup.sh index 390083d..30b312a 100755 --- a/setup.sh +++ b/examples/triton/setup.sh @@ -97,7 +97,22 @@ check() { exit 1 fi - echo CONSUL=consul.svc.${TRITON_ACCOUNT}.${TRITON_DC}.cns.joyent.com > _env + + echo '# Prometheus discovery for Triton' > _env + echo TRITON_ACCOUNT=${TRITON_ACCOUNT} >> _env + echo '#TRITON_DC= # Leave empty and Autopilot Pattern Prometheus will automatically detect the DC' >> _env + echo >> _env + + echo '# Prometheus authentication for Triton' >> _env + TRITON_CREDS_PATH=/root/.triton + echo TRITON_CREDS_PATH=${TRITON_CREDS_PATH} >> _env + echo TRITON_CA=$(cat "${DOCKER_CERT_PATH}"/ca.pem | tr '\n' '#') >> _env + echo TRITON_CA_PATH=${TRITON_CREDS_PATH}/ca.pem >> _env + echo TRITON_KEY=$(cat "${DOCKER_CERT_PATH}"/key.pem | tr '\n' '#') >> _env + echo TRITON_KEY_PATH=${TRITON_CREDS_PATH}/key.pem >> _env + echo TRITON_CERT=$(cat "${DOCKER_CERT_PATH}"/cert.pem | tr '\n' '#') >> _env + echo TRITON_CERT_PATH=${TRITON_CREDS_PATH}/cert.pem >> _env + echo >> _env } # --------------------------------------------------- diff --git a/local-compose.yml b/local-compose.yml deleted file mode 100644 index 255042c..0000000 --- a/local-compose.yml +++ /dev/null @@ -1,21 +0,0 @@ -prometheus: - extends: - file: docker-compose.yml - service: prometheus - build: . - mem_limit: 128g - environment: - - CONSUL=consul - links: - - consul:consul - ports: - - "9090:9090" - dns: - - 127.0.0.1 - -consul: - extends: - file: docker-compose.yml - service: consul - ports: - - "8500:8500"