From bd17829cb128005feec367664e4b8b6b5a424afc Mon Sep 17 00:00:00 2001 From: jnbnyc Date: Wed, 26 Oct 2016 11:07:26 -0400 Subject: [PATCH 01/37] Updated env-config/config-deploy.py to override default settings with environment variables for postgres settings --- env-config/config-deploy.py | 41 ++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/env-config/config-deploy.py b/env-config/config-deploy.py index d72448114..2d197e890 100644 --- a/env-config/config-deploy.py +++ b/env-config/config-deploy.py @@ -14,6 +14,36 @@ # Insert any config items here. # This will be fed into Flask/SQLAlchemy inside security_monkey/__init__.py +import os + +# '': '', +# Setting default settings +sm_config = { + 'postgres': { + 'database': 'secmonkey', + 'host': 'localhost', + 'password': 'securitymonkeypassword', + 'port': '5432', + 'user': 'postgres' + } +} + +if 'SECURITY_MONKEY_POSTGRES_HOST' in os.environ: + sm_config['postgres']['host'] = os.environ.get('SECURITY_MONKEY_POSTGRES_HOST') + +if 'SECURITY_MONKEY_POSTGRES_USER' in os.environ: + sm_config['postgres']['user'] = os.environ.get('SECURITY_MONKEY_POSTGRES_USER') + +if 'SECURITY_MONKEY_POSTGRES_PASSWORD' in os.environ: + sm_config['postgres']['password'] = os.environ.get('SECURITY_MONKEY_POSTGRES_PASSWORD') + +if 'SECURITY_MONKEY_POSTGRES_DATABASE' in os.environ: + sm_config['postgres']['database'] = os.environ.get('SECURITY_MONKEY_POSTGRES_DATABASE') + +if 'SECURITY_MONKEY_POSTGRES_PORT' in os.environ: + sm_config['postgres']['port'] = os.environ.get('SECURITY_MONKEY_POSTGRES_PORT') + + LOG_CFG = { 'version': 1, 'disable_existing_loggers': False, @@ -52,7 +82,16 @@ } } -SQLALCHEMY_DATABASE_URI = 'postgresql://postgres:securitymonkeypassword@localhost:5432/secmonkey' +SQLALCHEMY_DATABASE_URI = 'postgresql://%s:%s@%s:%s/%s' % ( + sm_config['postgres']['user'], + sm_config['postgres']['password'], + sm_config['postgres']['host'], + sm_config['postgres']['port'], + sm_config['postgres']['database'] +) + +# print sm_config['postgres'] +# print SQLALCHEMY_DATABASE_URI SQLALCHEMY_POOL_SIZE = 50 SQLALCHEMY_MAX_OVERFLOW = 15 From b5ee83e3ae947570182ad07f5ecdfb2eb239e38b Mon Sep 17 00:00:00 2001 From: jnbnyc Date: Wed, 26 Oct 2016 11:48:59 -0400 Subject: [PATCH 02/37] Adding the original Dockerfile from Netflix-Skunkworks/zerotodocker --- Dockerfile | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..fd118b79a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,35 @@ + +# Copyright 2014 Netflix, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM ubuntu:14.04 +MAINTAINER Netflix Open Source Development + +RUN apt-get update &&\ + apt-get -y -q install python-software-properties software-properties-common postgresql-9.3 postgresql-client-9.3 postgresql-contrib-9.3 curl &&\ + apt-get install -y python-pip python-dev python-psycopg2 libpq-dev git sudo swig python-m2crypto &&\ + cd /usr/local/src &&\ + git clone -b v0.3.4 --branch master https://github.com/Netflix/security_monkey.git &&\ + cd security_monkey &&\ + python setup.py install &&\ + /bin/mkdir -p /var/log/security_monkey/ + +ADD api-start.sh /usr/local/src/security_monkey/scripts/api-start.sh +ADD config-deploy.py /usr/local/src/security_monkey/env-config/config-deploy.py + +RUN chmod +x /usr/local/src/security_monkey/scripts/api-start.sh + +EXPOSE 5000 + +ENTRYPOINT ["/usr/local/src/security_monkey/scripts/api-start.sh"] \ No newline at end of file From 52c66a36efb1eb95218a1dd939839c3eef31db1c Mon Sep 17 00:00:00 2001 From: jnbnyc Date: Wed, 26 Oct 2016 12:06:31 -0400 Subject: [PATCH 03/37] Adding original entrypoint scripts from Netflix-Skunkworks/zerotodocker --- docker/api-init.sh | 11 +++++++++++ docker/api-start.sh | 5 +++++ docker/scheduler-start.sh | 8 ++++++++ 3 files changed, 24 insertions(+) create mode 100644 docker/api-init.sh create mode 100644 docker/api-start.sh create mode 100644 docker/scheduler-start.sh diff --git a/docker/api-init.sh b/docker/api-init.sh new file mode 100644 index 000000000..bb6d360ef --- /dev/null +++ b/docker/api-init.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +sudo -u postgres psql -h postgres --command "ALTER USER postgres with PASSWORD 'securitymonkeypassword';" +sudo -u postgres createdb -h postgres -O postgres secmonkey + +cd /usr/local/src/security_monkey +sed -i s/localhost/postgres/ env-config/config-deploy.py +export SECURITY_MONKEY_SETTINGS=`pwd`/env-config/config-deploy.py +mkdir -p /var/log/security_monkey/ +touch "/var/log/security_monkey/security_monkey-deploy.log" +python manage.py db upgrade diff --git a/docker/api-start.sh b/docker/api-start.sh new file mode 100644 index 000000000..9c52fef2f --- /dev/null +++ b/docker/api-start.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +cd /usr/local/src/security_monkey +export SECURITY_MONKEY_SETTINGS=`pwd`/env-config/config-deploy.py +python manage.py run_api_server -b 0.0.0.0:5000 diff --git a/docker/scheduler-start.sh b/docker/scheduler-start.sh new file mode 100644 index 000000000..532851a92 --- /dev/null +++ b/docker/scheduler-start.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +mkdir -p /var/log/security_monkey +touch /var/log/security_monkey/security_monkey-deploy.log + +cd /usr/local/src/security_monkey +export SECURITY_MONKEY_SETTINGS=`pwd`/env-config/config-deploy.py +python manage.py start_scheduler From e7b3f63c8aec4c92f9c772d82199f9921726c9c8 Mon Sep 17 00:00:00 2001 From: jnbnyc Date: Wed, 26 Oct 2016 17:34:34 -0400 Subject: [PATCH 04/37] Disable export of the SECURITY_MONKEY_SETTINGS variable in these entrypoints as this should be set before these entrypoints exist --- docker/api-init.sh | 4 ++-- docker/api-start.sh | 2 +- docker/scheduler-start.sh | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/api-init.sh b/docker/api-init.sh index bb6d360ef..5416d28ce 100644 --- a/docker/api-init.sh +++ b/docker/api-init.sh @@ -4,8 +4,8 @@ sudo -u postgres psql -h postgres --command "ALTER USER postgres with PASSWORD ' sudo -u postgres createdb -h postgres -O postgres secmonkey cd /usr/local/src/security_monkey -sed -i s/localhost/postgres/ env-config/config-deploy.py -export SECURITY_MONKEY_SETTINGS=`pwd`/env-config/config-deploy.py +# sed -i s/localhost/postgres/ env-config/config-deploy.py +# export SECURITY_MONKEY_SETTINGS=`pwd`/env-config/config-deploy.py mkdir -p /var/log/security_monkey/ touch "/var/log/security_monkey/security_monkey-deploy.log" python manage.py db upgrade diff --git a/docker/api-start.sh b/docker/api-start.sh index 9c52fef2f..c2787cd90 100644 --- a/docker/api-start.sh +++ b/docker/api-start.sh @@ -1,5 +1,5 @@ #!/bin/bash cd /usr/local/src/security_monkey -export SECURITY_MONKEY_SETTINGS=`pwd`/env-config/config-deploy.py +# export SECURITY_MONKEY_SETTINGS=`pwd`/env-config/config-deploy.py python manage.py run_api_server -b 0.0.0.0:5000 diff --git a/docker/scheduler-start.sh b/docker/scheduler-start.sh index 532851a92..d0124de4b 100644 --- a/docker/scheduler-start.sh +++ b/docker/scheduler-start.sh @@ -4,5 +4,5 @@ mkdir -p /var/log/security_monkey touch /var/log/security_monkey/security_monkey-deploy.log cd /usr/local/src/security_monkey -export SECURITY_MONKEY_SETTINGS=`pwd`/env-config/config-deploy.py +# export SECURITY_MONKEY_SETTINGS=`pwd`/env-config/config-deploy.py python manage.py start_scheduler From 668071247f4e275af9b2d0ec882df5e55de8247f Mon Sep 17 00:00:00 2001 From: jnbnyc Date: Wed, 26 Oct 2016 17:38:18 -0400 Subject: [PATCH 05/37] Remove unnecessary commented out lines --- docker/api-init.sh | 2 -- docker/api-start.sh | 1 - docker/scheduler-start.sh | 1 - 3 files changed, 4 deletions(-) diff --git a/docker/api-init.sh b/docker/api-init.sh index 5416d28ce..70507dd1b 100644 --- a/docker/api-init.sh +++ b/docker/api-init.sh @@ -4,8 +4,6 @@ sudo -u postgres psql -h postgres --command "ALTER USER postgres with PASSWORD ' sudo -u postgres createdb -h postgres -O postgres secmonkey cd /usr/local/src/security_monkey -# sed -i s/localhost/postgres/ env-config/config-deploy.py -# export SECURITY_MONKEY_SETTINGS=`pwd`/env-config/config-deploy.py mkdir -p /var/log/security_monkey/ touch "/var/log/security_monkey/security_monkey-deploy.log" python manage.py db upgrade diff --git a/docker/api-start.sh b/docker/api-start.sh index c2787cd90..c4a72d64f 100644 --- a/docker/api-start.sh +++ b/docker/api-start.sh @@ -1,5 +1,4 @@ #!/bin/bash cd /usr/local/src/security_monkey -# export SECURITY_MONKEY_SETTINGS=`pwd`/env-config/config-deploy.py python manage.py run_api_server -b 0.0.0.0:5000 diff --git a/docker/scheduler-start.sh b/docker/scheduler-start.sh index d0124de4b..0f70228f1 100644 --- a/docker/scheduler-start.sh +++ b/docker/scheduler-start.sh @@ -4,5 +4,4 @@ mkdir -p /var/log/security_monkey touch /var/log/security_monkey/security_monkey-deploy.log cd /usr/local/src/security_monkey -# export SECURITY_MONKEY_SETTINGS=`pwd`/env-config/config-deploy.py python manage.py start_scheduler From bc2fb0034a7cb1fdbc7ceab5b2b9605b15e77f8d Mon Sep 17 00:00:00 2001 From: jnbnyc Date: Thu, 27 Oct 2016 09:58:23 -0400 Subject: [PATCH 06/37] Organize order of operations in api-init --- docker/api-init.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/api-init.sh b/docker/api-init.sh index 70507dd1b..3a2850753 100644 --- a/docker/api-init.sh +++ b/docker/api-init.sh @@ -3,7 +3,8 @@ sudo -u postgres psql -h postgres --command "ALTER USER postgres with PASSWORD 'securitymonkeypassword';" sudo -u postgres createdb -h postgres -O postgres secmonkey -cd /usr/local/src/security_monkey mkdir -p /var/log/security_monkey/ touch "/var/log/security_monkey/security_monkey-deploy.log" + +cd /usr/local/src/security_monkey python manage.py db upgrade From 59518f7b441092e9dd8174a794bc82f0936549a1 Mon Sep 17 00:00:00 2001 From: jnbnyc Date: Thu, 27 Oct 2016 11:51:28 -0400 Subject: [PATCH 07/37] 1 - Add SECURITY_MONKEY_API_PORT environment variable to override default settings defined in SECURITY_MONKEY_SETTINGS 2 - Update entrypoints to use environment variables --- docker/api-init.sh | 9 +++++++-- docker/api-start.sh | 2 +- env-config/config-deploy.py | 7 ++++++- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/docker/api-init.sh b/docker/api-init.sh index 3a2850753..74bb8182f 100644 --- a/docker/api-init.sh +++ b/docker/api-init.sh @@ -1,7 +1,12 @@ #!/bin/bash -sudo -u postgres psql -h postgres --command "ALTER USER postgres with PASSWORD 'securitymonkeypassword';" -sudo -u postgres createdb -h postgres -O postgres secmonkey +sudo -u ${SECURITY_MONKEY_POSTGRES_USER:-postgres} psql\ + -h ${SECURITY_MONKEY_POSTGRES_HOST:-postgres} -p ${SECURITY_MONKEY_POSTGRES_PORT:-5432}\ + --command "ALTER USER ${SECURITY_MONKEY_POSTGRES_USER:-postgres} with PASSWORD '${SECURITY_MONKEY_POSTGRES_PASSWORD:-securitymonkeypassword}';" + +sudo -u ${SECURITY_MONKEY_POSTGRES_USER:-postgres} createdb\ + -h ${SECURITY_MONKEY_POSTGRES_HOST:-postgres} -p ${SECURITY_MONKEY_POSTGRES_PORT:-5432}\ + -O ${SECURITY_MONKEY_POSTGRES_USER:-postgres} ${SECURITY_MONKEY_POSTGRES_DATABASE:-secmonkey} mkdir -p /var/log/security_monkey/ touch "/var/log/security_monkey/security_monkey-deploy.log" diff --git a/docker/api-start.sh b/docker/api-start.sh index c4a72d64f..d02689d4a 100644 --- a/docker/api-start.sh +++ b/docker/api-start.sh @@ -1,4 +1,4 @@ #!/bin/bash cd /usr/local/src/security_monkey -python manage.py run_api_server -b 0.0.0.0:5000 +python manage.py run_api_server -b 0.0.0.0:${SECURITY_MONKEY_API_PORT:-5000} diff --git a/env-config/config-deploy.py b/env-config/config-deploy.py index 2d197e890..e553a8f48 100644 --- a/env-config/config-deploy.py +++ b/env-config/config-deploy.py @@ -25,6 +25,9 @@ 'password': 'securitymonkeypassword', 'port': '5432', 'user': 'postgres' + }, + 'api': { + 'port': 5000 } } @@ -43,6 +46,8 @@ if 'SECURITY_MONKEY_POSTGRES_PORT' in os.environ: sm_config['postgres']['port'] = os.environ.get('SECURITY_MONKEY_POSTGRES_PORT') +if 'SECURITY_MONKEY_API_PORT' in os.environ: + sm_config['api']['port'] = os.environ.get('SECURITY_MONKEY_API_PORT') LOG_CFG = { 'version': 1, @@ -98,7 +103,7 @@ ENVIRONMENT = 'ec2' USE_ROUTE53 = False FQDN = 'ec2-XX-XXX-XXX-XXX.compute-1.amazonaws.com' -API_PORT = '5000' +API_PORT = sm_config['api']['port'] WEB_PORT = '443' WEB_PATH = '/static/ui.html' FRONTED_BY_NGINX = True From 415d5877d34ad95b4e43bac9e746b2041f9d960b Mon Sep 17 00:00:00 2001 From: jnbnyc Date: Thu, 27 Oct 2016 12:06:32 -0400 Subject: [PATCH 08/37] Add original Dockerfile for nginx from Netflix-Skunkworks/zerotodocker --- Dockerfile.nginx | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 Dockerfile.nginx diff --git a/Dockerfile.nginx b/Dockerfile.nginx new file mode 100644 index 000000000..2827171ee --- /dev/null +++ b/Dockerfile.nginx @@ -0,0 +1,48 @@ +# Copyright 2014 Netflix, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM nginx:1.7.7 +MAINTAINER Netflix Open Source Development + +RUN apt-get update &&\ + apt-get install -y curl git sudo apt-transport-https &&\ + cd /usr/local/src &&\ + git clone -b v0.3.4 https://github.com/Netflix/security_monkey.git &&\ + cd security_monkey + +RUN curl https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - &&\ + curl https://storage.googleapis.com/download.dartlang.org/linux/debian/dart_stable.list > /etc/apt/sources.list.d/dart_stable.list && \ + apt-get update &&\ + apt-get install -y -q dart + +RUN cd /usr/local/src/security_monkey/dart &&\ + /usr/lib/dart/bin/pub build &&\ + /bin/mkdir -p /usr/local/src/security_monkey/security_monkey/static/ &&\ + /bin/cp -R /usr/local/src/security_monkey/dart/build/web/* /usr/local/src/security_monkey/security_monkey/static/ + +RUN /bin/mkdir -p /var/log/security_monkey/ &&\ + /usr/bin/touch /var/log/security_monkey/security_monkey.access.log &&\ + /usr/bin/touch /var/log/security_monkey/security_monkey.error.log + +EXPOSE 443 + +ADD securitymonkey.conf /etc/nginx/sites-available/securitymonkey.conf +COPY nginx.conf /etc/nginx/nginx.conf +ADD server.crt /etc/nginx/ssl/ +ADD server.key /etc/nginx/ssl/ + +RUN /bin/mkdir -p /etc/nginx/sites-enabled/ &&\ + ln -s /etc/nginx/sites-available/securitymonkey.conf /etc/nginx/sites-enabled/securitymonkey.conf + +ENTRYPOINT ["nginx", "-g", "daemon off;"] \ No newline at end of file From d0d5b5c283a5f03b83ceab5fbf76df43e1f532a3 Mon Sep 17 00:00:00 2001 From: jnbnyc Date: Thu, 27 Oct 2016 12:25:43 -0400 Subject: [PATCH 09/37] Add original securitymonkey.conf for nginx from Netflix-Skunkworks/zerotodocker --- etc/nginx/conf.d/securitymonkey.conf | 64 ++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 etc/nginx/conf.d/securitymonkey.conf diff --git a/etc/nginx/conf.d/securitymonkey.conf b/etc/nginx/conf.d/securitymonkey.conf new file mode 100644 index 000000000..582d34eae --- /dev/null +++ b/etc/nginx/conf.d/securitymonkey.conf @@ -0,0 +1,64 @@ +server { + listen 0.0.0.0:80 default; + listen 0.0.0.0:443 ssl default; + ssl_certificate /etc/nginx/ssl/server.crt; + ssl_certificate_key /etc/nginx/ssl/server.key; + access_log /var/log/security_monkey/security_monkey.access.log; + error_log /var/log/security_monkey/security_monkey.error.log; + + location /register { + proxy_read_timeout 120; + proxy_pass http://smapi:5000; + proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504; + proxy_redirect off; + proxy_buffering off; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + location /logout { + proxy_read_timeout 120; + proxy_pass http://smapi:5000; + proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504; + proxy_redirect off; + proxy_buffering off; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + location /login { + proxy_read_timeout 120; + proxy_pass http://smapi:5000; + proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504; + proxy_redirect off; + proxy_buffering off; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + location /api { + proxy_read_timeout 120; + proxy_pass http://smapi:5000; + proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504; + proxy_redirect off; + proxy_buffering off; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + location /static { + rewrite ^/static/(.*)$ /$1 break; + root /usr/local/src/security_monkey/security_monkey/static; + index ui.html; + } + + location / { + root /usr/local/src/security_monkey/security_monkey/static; + index ui.html; + } + +} \ No newline at end of file From 0d291fd1dfae574bf6006d1efa3a7f2f1091bdd1 Mon Sep 17 00:00:00 2001 From: jnbnyc Date: Thu, 27 Oct 2016 12:26:58 -0400 Subject: [PATCH 10/37] Add original insecure certs for tls provided by Netflix-Skunkworks. These are meant to act as a placeholder for the example. --- etc/nginx/ssl/server.crt | 25 +++++++++++++++++++++++++ etc/nginx/ssl/server.key | 27 +++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 etc/nginx/ssl/server.crt create mode 100644 etc/nginx/ssl/server.key diff --git a/etc/nginx/ssl/server.crt b/etc/nginx/ssl/server.crt new file mode 100644 index 000000000..f916f2452 --- /dev/null +++ b/etc/nginx/ssl/server.crt @@ -0,0 +1,25 @@ + +-----BEGIN CERTIFICATE----- +MIID8jCCAtoCCQCnvR3ajR8rsjANBgkqhkiG9w0BAQsFADCBujELMAkGA1UEBhMC +VVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEjAQBgNVBAcMCUxvcyBHYXRvczEnMCUG +A1UECgweU2VsZiBTaWduZWQgRG9ja2VyIENlcnRzIExtdGQuMR4wHAYDVQQLDBVT +ZWxmIFNpZ25lZCBEb2NrZXIgT1UxFTATBgNVBAMMDFNvbWVEb2NrZXJJUDEiMCAG +CSqGSIb3DQEJARYTcGtlbGxleUBuZXRmbGl4LmNvbTAeFw0xNDExMTAwMzQ0Mzda +Fw0xNTExMTAwMzQ0MzdaMIG6MQswCQYDVQQGEwJVUzETMBEGA1UECAwKQ2FsaWZv +cm5pYTESMBAGA1UEBwwJTG9zIEdhdG9zMScwJQYDVQQKDB5TZWxmIFNpZ25lZCBE +b2NrZXIgQ2VydHMgTG10ZC4xHjAcBgNVBAsMFVNlbGYgU2lnbmVkIERvY2tlciBP +VTEVMBMGA1UEAwwMU29tZURvY2tlcklQMSIwIAYJKoZIhvcNAQkBFhNwa2VsbGV5 +QG5ldGZsaXguY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvkeh +4rS7A2Irm8UAuEitneMRviSDPI04a3sbYIZISPMTsmn50ZXAewikL7YAVchDceiW +dhu4T+lah/lAmTjHLgleEUu2htsSO25pQBSC1mzpxU2RhY8EpW4pgjZdkzlHdsy5 +ZiYS4HAHIp7ZFr+DEDAoFgR2uIMK8W7jAwLXcjT1qs/q69Po1EAnOucLAwu6fiUS +MY/BAwGxhRDp9Q0dT7FiHwN8756MdQufapnuosHPcQoOj73IOhSN7EBqNXFWOjPc +nKKhmFxuFhKgIDcY8yJoAx5qWOWM7vlgACHd/41kE53RMd7k7hp3GXxofs7qHtaf +ZxtdMqiuPC/UUoMEzwIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQCckPS2VT4VZr8S +UsyuSy4RWtpwTg2sVGsddb27R3l9a/fRupHnSs1v+ZyrP7a/QcK16wNAGYm05Ajd +jXxaI8auNwi+Vix5r9pV3OG9bGoDp5m0+TeeV+RGs6nhIgnyJrYqDBV6St3UPwwS +U0mh4iFxwdR8NZYpLAu1R+t0t7G0do8yJKmHezs7zXy6J16Xl8txjFAVF5pHw2aN +h5Lj1FcdrOUCaeAFTmTK15ZcojDuIOoN/EdpNyHgelXqr0NQGcfrGJIGsn/8vXyV ++2B7QTr7w5gtqu5RQ5pem14wMoJl2tRNx02fP9CYmJoD8KAximnJDxL1PXEFIMxw +t7hR6j5w +-----END CERTIFICATE----- \ No newline at end of file diff --git a/etc/nginx/ssl/server.key b/etc/nginx/ssl/server.key new file mode 100644 index 000000000..75e6192b2 --- /dev/null +++ b/etc/nginx/ssl/server.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEAvkeh4rS7A2Irm8UAuEitneMRviSDPI04a3sbYIZISPMTsmn5 +0ZXAewikL7YAVchDceiWdhu4T+lah/lAmTjHLgleEUu2htsSO25pQBSC1mzpxU2R +hY8EpW4pgjZdkzlHdsy5ZiYS4HAHIp7ZFr+DEDAoFgR2uIMK8W7jAwLXcjT1qs/q +69Po1EAnOucLAwu6fiUSMY/BAwGxhRDp9Q0dT7FiHwN8756MdQufapnuosHPcQoO +j73IOhSN7EBqNXFWOjPcnKKhmFxuFhKgIDcY8yJoAx5qWOWM7vlgACHd/41kE53R +Md7k7hp3GXxofs7qHtafZxtdMqiuPC/UUoMEzwIDAQABAoIBAAShg6wbz+1eJ0eM +PN8/TZJpI1ZoVxVTLotXHKx1uC6mU9VkxOK37U5RQIW0vT9ZN2L/USIgvJSdTkm+ +5DaWbiqNqc9r805G6KOvpTodBa4oSmw13hPNDkTkcHgt0IszuqyGgQCebpktwM6z +5P6rvInZUjBu/WCyN8L2euno2Mt7ubYsHp6IZNeDF5UKBd3fcORCCN2oIjud/wo+ +7qzgc1Qz8yZLg5rLY5t1TXmWsYPnTgCo2xSUt+40VEPu2IeShg+TxyFzVTkfsxs9 +ZT6+HwW3wcnRLbUcdX/sYldaKHW+k4rDHnRtqzaxgwmL+Pd/RC/b1MRReY2IANkU +P1q5kikCgYEA7GuPfeX4SmvZqipRi6CujWHZcR5pHUAqURrA+PkGw7PVo3Z59aor +KbAUsPDpQLhLnsO/nTjxjnvz+uHYRJjsAypXtwp0cyvV5TUd8REwzg8ivRB1U6TX +aUTmqpdC7JL8bfN10puqOG2dE4/tiVpW02e2pSp+gBdjS0t0Rn/PagsCgYEAzgnU +7Fs+G0sKBkXF+/M/XFin5dGqlvYBKXwh49qeyyOZFHzGNs97qtvuRnKZ3xgSsbnV +KGa4PcSBcSH7ktiKMsjxs2rxkGrFvD+7TpM4FfNxOL04AupHLgkb0JOlCDsgPkwH +DoyOSomutL3hSXrIEbg+2pgPUm6kgMfKFw5rjs0CgYEAmZq4u/+ydgfkf97G5IUd +Y9ZzRD+R4+NQmylav2hssLIi1/Wd/7L0ID2688tSgS48U4ay9B+PMhfuyM37iXYh +wVy9aLtuNXYBns4IChw1LWxAEb6jvBiZrFeL1sI4RSSqxXUs1A03ZzowmAknN9pu +FySHUoBteCO56622eieIR0kCgYA0uB1MMPiOQUAaZMYI9q+ysFidnG1a3S1k8Qc2 +5xyUe9JShK6vHZ3WFRo92lr205EM+rDI+qeP3nUEfp6Bb3jFD9eQgf+3ZSqdRW1m +6JKEQ1soGnp3fHykEd6VGwIyPfa9GfUASwSpm9shEijQBWOl7Q/gHOJmxDrBXN2f +0qfAzQKBgQDL7gyh7hBCBlwFreKmmYyNQDYWp4XJvvO0mOhR4rpNMgFAWqBWPd/C +VBPCctp71Dj9MztNrIUBTOsU54DFHfC+UIicoyAtPIWUbaYlS0Wxf1DY6orvAp+T +3D+eNMsdnZqEPC+Q+Fb2DkF8CzrhYdH2nZEMN8rA50/L/r0OGkOiLg== +-----END RSA PRIVATE KEY----- \ No newline at end of file From ee4e4e2c947a9429ac91877fae12231754b28cf5 Mon Sep 17 00:00:00 2001 From: jnbnyc Date: Thu, 27 Oct 2016 12:45:33 -0400 Subject: [PATCH 11/37] Move nginx resources to docker-nginx for simpler docker build --- {etc/nginx/conf.d => docker-nginx}/securitymonkey.conf | 0 {etc/nginx/ssl => docker-nginx}/server.crt | 0 {etc/nginx/ssl => docker-nginx}/server.key | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename {etc/nginx/conf.d => docker-nginx}/securitymonkey.conf (100%) rename {etc/nginx/ssl => docker-nginx}/server.crt (100%) rename {etc/nginx/ssl => docker-nginx}/server.key (100%) diff --git a/etc/nginx/conf.d/securitymonkey.conf b/docker-nginx/securitymonkey.conf similarity index 100% rename from etc/nginx/conf.d/securitymonkey.conf rename to docker-nginx/securitymonkey.conf diff --git a/etc/nginx/ssl/server.crt b/docker-nginx/server.crt similarity index 100% rename from etc/nginx/ssl/server.crt rename to docker-nginx/server.crt diff --git a/etc/nginx/ssl/server.key b/docker-nginx/server.key similarity index 100% rename from etc/nginx/ssl/server.key rename to docker-nginx/server.key From 2af97ae6aaa798a29781c247a40f6f47e50ad7ff Mon Sep 17 00:00:00 2001 From: jnbnyc Date: Thu, 27 Oct 2016 15:26:54 -0400 Subject: [PATCH 12/37] Add items to .gitignore and .dockerignore to avoid committing secrets --- .dockerignore | 7 +++++++ .gitignore | 2 ++ 2 files changed, 9 insertions(+) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..33a7d08e0 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,7 @@ +secmonkey.env +boto.cfg +.travis.yml +#docs +supervisor +config-default.py +generate-docs.py diff --git a/.gitignore b/.gitignore index 18a7937b3..b41a95b6a 100644 --- a/.gitignore +++ b/.gitignore @@ -53,3 +53,5 @@ devlog/ venv/ .idea/ +boto.cfg +secmonkey.env From 79a6ecd31d4ac79f7e3028088456e9480493e48d Mon Sep 17 00:00:00 2001 From: jnbnyc Date: Thu, 27 Oct 2016 15:34:50 -0400 Subject: [PATCH 13/37] Update Dockerfiles to build and run SecurityMonkey from this directory, as opposed to checking out the repository directly, since this code has not yet been merged. This can be used to build and develop locally. --- Dockerfile | 26 +++++++++------ Dockerfile.nginx | 35 ++++++++++++-------- docker-compose.yml | 79 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 118 insertions(+), 22 deletions(-) create mode 100644 docker-compose.yml diff --git a/Dockerfile b/Dockerfile index fd118b79a..328cabdee 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,20 +16,28 @@ FROM ubuntu:14.04 MAINTAINER Netflix Open Source Development +ENV SECURITY_MONKEY_VERSION=v0.7.0 \ + SECURITY_MONKEY_SETTINGS=/usr/local/src/security_monkey/env-config/config-deploy.py + RUN apt-get update &&\ apt-get -y -q install python-software-properties software-properties-common postgresql-9.3 postgresql-client-9.3 postgresql-contrib-9.3 curl &&\ - apt-get install -y python-pip python-dev python-psycopg2 libpq-dev git sudo swig python-m2crypto &&\ - cd /usr/local/src &&\ - git clone -b v0.3.4 --branch master https://github.com/Netflix/security_monkey.git &&\ - cd security_monkey &&\ + apt-get install -y python-pip python-dev python-psycopg2 libffi-dev libpq-dev libyaml-dev libxml2-dev libxmlsec1-dev git sudo swig python-m2crypto &&\ + rm -rf /var/lib/apt/lists/* + +RUN cd /usr/local/src &&\ +# git clone --branch $SECURITY_MONKEY_VERSION https://github.com/Netflix/security_monkey.git + mkdir -p security_monkey +ADD . /usr/local/src/security_monkey + +RUN cd /usr/local/src/security_monkey &&\ python setup.py install &&\ /bin/mkdir -p /var/log/security_monkey/ -ADD api-start.sh /usr/local/src/security_monkey/scripts/api-start.sh -ADD config-deploy.py /usr/local/src/security_monkey/env-config/config-deploy.py - -RUN chmod +x /usr/local/src/security_monkey/scripts/api-start.sh +RUN chmod +x /usr/local/src/security_monkey/docker/*.sh &&\ + mkdir -pv /var/log/security_monkey &&\ + ln -s /dev/stdout /var/log/security_monkey/securitymonkey.log +WORKDIR /usr/local/src/security_monkey EXPOSE 5000 -ENTRYPOINT ["/usr/local/src/security_monkey/scripts/api-start.sh"] \ No newline at end of file +ENTRYPOINT ["/usr/local/src/security_monkey/docker/api-start.sh"] \ No newline at end of file diff --git a/Dockerfile.nginx b/Dockerfile.nginx index 2827171ee..fd2889f7f 100644 --- a/Dockerfile.nginx +++ b/Dockerfile.nginx @@ -12,37 +12,46 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM nginx:1.7.7 +FROM nginx:1.11.4 MAINTAINER Netflix Open Source Development +ENV SECURITY_MONKEY_VERSION=v0.7.0 RUN apt-get update &&\ apt-get install -y curl git sudo apt-transport-https &&\ - cd /usr/local/src &&\ - git clone -b v0.3.4 https://github.com/Netflix/security_monkey.git &&\ - cd security_monkey + rm -rf /var/lib/apt/lists/* + +RUN cd /usr/local/src &&\ +# git clone -b $SECURITY_MONKEY_VERSION https://github.com/Netflix/security_monkey.git + mkdir -p security_monkey +ADD . /usr/local/src/security_monkey RUN curl https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - &&\ curl https://storage.googleapis.com/download.dartlang.org/linux/debian/dart_stable.list > /etc/apt/sources.list.d/dart_stable.list && \ apt-get update &&\ - apt-get install -y -q dart + apt-get install -y -q dart &&\ + rm -rf /var/lib/apt/lists/* RUN cd /usr/local/src/security_monkey/dart &&\ + /usr/lib/dart/bin/pub get &&\ /usr/lib/dart/bin/pub build &&\ /bin/mkdir -p /usr/local/src/security_monkey/security_monkey/static/ &&\ /bin/cp -R /usr/local/src/security_monkey/dart/build/web/* /usr/local/src/security_monkey/security_monkey/static/ RUN /bin/mkdir -p /var/log/security_monkey/ &&\ - /usr/bin/touch /var/log/security_monkey/security_monkey.access.log &&\ - /usr/bin/touch /var/log/security_monkey/security_monkey.error.log + # /usr/bin/touch /var/log/security_monkey/security_monkey.access.log &&\ + # /usr/bin/touch /var/log/security_monkey/security_monkey.error.log + ln -s /dev/stdout /var/log/security_monkey/security_monkey.access.log &&\ + ln -s /dev/stderr /var/log/security_monkey/security_monkey.error.log +WORKDIR /etc/nginx EXPOSE 443 -ADD securitymonkey.conf /etc/nginx/sites-available/securitymonkey.conf -COPY nginx.conf /etc/nginx/nginx.conf -ADD server.crt /etc/nginx/ssl/ -ADD server.key /etc/nginx/ssl/ +ADD docker-nginx/securitymonkey.conf /etc/nginx/conf.d/securitymonkey.conf +# COPY nginx.conf /etc/nginx/nginx.conf +ADD docker-nginx/server.crt /etc/nginx/ssl/ +ADD docker-nginx/server.key /etc/nginx/ssl/ -RUN /bin/mkdir -p /etc/nginx/sites-enabled/ &&\ - ln -s /etc/nginx/sites-available/securitymonkey.conf /etc/nginx/sites-enabled/securitymonkey.conf +# RUN /bin/mkdir -p /etc/nginx/sites-enabled/ &&\ +# ln -s /etc/nginx/sites-available/securitymonkey.conf /etc/nginx/sites-enabled/securitymonkey.conf ENTRYPOINT ["nginx", "-g", "daemon off;"] \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..6128ce0e4 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,79 @@ +--- + +### +# +# Documentation: http://securitymonkey.readthedocs.io/en/latest/index.html +# +# shortcuts +# open https://$(docker-machine active | xargs docker-machine ip) +# find . -type f -name "*.sh" -exec chmod +x {} \; +# find . -type f -name "*.sh" -exec chmod -x {} \; +# +### + + +version: '2' +services: + postgres: + container_name: secmonkey-db + image: postgres:9 + # volumes: + # - ./postgres-data/:/var/lib/postgresql/data + + api: + container_name: secmonkey-api + build: . + volumes: + - ./docker:/usr/local/src/security_monkey/docker/ + depends_on: + - postgres + env_file: secmonkey.env + entrypoint: "/usr/local/src/security_monkey/docker/api-start.sh" + + scheduler: + container_name: secmonkey-scheduler + build: . + volumes: + - ./docker:/usr/local/src/security_monkey/docker/ + depends_on: + - api + env_file: secmonkey.env + entrypoint: "/usr/local/src/security_monkey/docker/scheduler-start.sh" + + nginx: + container_name: secmonkey-nginx + build: + context: ./ + dockerfile: ./Dockerfile.nginx + working_dir: /etc/nginx + depends_on: + - api + ports: + - 80:80 + - 443:443 + links: + - api:smapi + +# volumes: +# - postgres-data: {} + +### ### ### + ### ### ### + + scratch: + container_name: scratch + build: . + working_dir: /usr/local/src/security_monkey + volumes: + - ./data/aws_accounts.json:/usr/local/src/security_monkey/data/aws_accounts.json + - ./docker:/usr/local/src/security_monkey/docker/ + depends_on: + - postgres + env_file: secmonkey.env + # environment: + # - AWS_ACCESS_KEY_ID= + # - AWS_SECRET_ACCESS_KEY= + # - SECURITY_MONKEY_POSTGRES_HOST= + entrypoint: + - sleep + - 8h From cc0766bbff582f05a489b485bbd57dc392808712 Mon Sep 17 00:00:00 2001 From: jnbnyc Date: Fri, 28 Oct 2016 11:42:29 -0400 Subject: [PATCH 14/37] Update FQDN settings to use environment variables as override to default --- env-config/config-deploy.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/env-config/config-deploy.py b/env-config/config-deploy.py index e553a8f48..3be6b73c9 100644 --- a/env-config/config-deploy.py +++ b/env-config/config-deploy.py @@ -19,6 +19,7 @@ # '': '', # Setting default settings sm_config = { + 'fqdn': 'ec2-XX-XXX-XXX-XXX.compute-1.amazonaws.com' 'postgres': { 'database': 'secmonkey', 'host': 'localhost', @@ -49,6 +50,9 @@ if 'SECURITY_MONKEY_API_PORT' in os.environ: sm_config['api']['port'] = os.environ.get('SECURITY_MONKEY_API_PORT') +if 'SECURITY_MONKEY_FQDN' in os.environ: + sm_config['fqdn'] = os.environ.get('SECURITY_MONKEY_FQDN') + LOG_CFG = { 'version': 1, 'disable_existing_loggers': False, @@ -102,7 +106,7 @@ SQLALCHEMY_MAX_OVERFLOW = 15 ENVIRONMENT = 'ec2' USE_ROUTE53 = False -FQDN = 'ec2-XX-XXX-XXX-XXX.compute-1.amazonaws.com' +FQDN = sm_config['fqdn'] API_PORT = sm_config['api']['port'] WEB_PORT = '443' WEB_PATH = '/static/ui.html' From 06e96c2e55653f14fa952a242ef2f58153bdef1c Mon Sep 17 00:00:00 2001 From: jnbnyc Date: Fri, 28 Oct 2016 12:40:02 -0400 Subject: [PATCH 15/37] Update email settings to use environment variable overrides --- env-config/config-deploy.py | 46 +++++++++++++++++++++++++++++++------ 1 file changed, 39 insertions(+), 7 deletions(-) diff --git a/env-config/config-deploy.py b/env-config/config-deploy.py index 3be6b73c9..ed166221d 100644 --- a/env-config/config-deploy.py +++ b/env-config/config-deploy.py @@ -29,6 +29,15 @@ }, 'api': { 'port': 5000 + }, + 'email': { + 'security-team-email': [], + 'smtp': False, + 'ses-region': 'us-east-1', + 'default-sender': 'securitymonkey@example.com', + 'server': 'smtp.example.com', + 'username': 'username', + 'password': 'password' } } @@ -53,6 +62,29 @@ if 'SECURITY_MONKEY_FQDN' in os.environ: sm_config['fqdn'] = os.environ.get('SECURITY_MONKEY_FQDN') +if 'SECURITY_MONKEY_SECURITY_TEAM_EMAIL' in os.environ: + sm_config['email']['security-team-email'] = os.environ.get('SECURITY_MONKEY_SECURITY_TEAM_EMAIL') + +if 'SECURITY_MONKEY_SMTP' in os.environ: + # Must change String from environment variable into Boolean + if os.environ.get('SECURITY_MONKEY_SMTP') = 'True': + sm_config['email']['smtp'] = True + +if 'SECURITY_MONKEY_SES_REGION' in os.environ: + sm_config['email']['ses-region'] = os.environ.get('SECURITY_MONKEY_SES_REGION') + +if 'SECURITY_MONKEY_EMAIL_DEFAULT_SENDER' in os.environ: + sm_config['email']['default-sender'] = os.environ.get('SECURITY_MONKEY_EMAIL_DEFAULT_SENDER') + +if 'SECURITY_MONKEY_EMAIL_SERVER' in os.environ: + sm_config['email']['server'] = os.environ.get('SECURITY_MONKEY_EMAIL_SERVER') + +if 'SECURITY_MONKEY_EMAIL_USERNAME' in os.environ: + sm_config['email']['username'] = os.environ.get('SECURITY_MONKEY_EMAIL_USERNAME') + +if 'SECURITY_MONKEY_EMAIL_PASSWORD' in os.environ: + sm_config['email']['password'] = os.environ.get('SECURITY_MONKEY_EMAIL_PASSWORD') + LOG_CFG = { 'version': 1, 'disable_existing_loggers': False, @@ -116,7 +148,7 @@ SECRET_KEY = '' -MAIL_DEFAULT_SENDER = 'securitymonkey@example.com' +MAIL_DEFAULT_SENDER = sm_config['email']['default-sender'] SECURITY_REGISTERABLE = True SECURITY_CONFIRMABLE = False SECURITY_RECOVERABLE = False @@ -131,16 +163,16 @@ SECURITY_POST_CHANGE_VIEW = BASE_URL # This address gets all change notifications (i.e. 'securityteam@example.com') -SECURITY_TEAM_EMAIL = [] +SECURITY_TEAM_EMAIL = sm_config['email']['security-team-email'] # These are only required if using SMTP instead of SES -EMAILS_USE_SMTP = False # Otherwise, Use SES -SES_REGION = 'us-east-1' -MAIL_SERVER = 'smtp.example.com' +EMAILS_USE_SMTP = sm_config['email']['smtp'] # Otherwise, Use SES +SES_REGION = sm_config['email']['ses-region'] +MAIL_SERVER = sm_config['email']['server'] MAIL_PORT = 465 MAIL_USE_SSL = True -MAIL_USERNAME = 'username' -MAIL_PASSWORD = 'password' +MAIL_USERNAME = sm_config['email']['username'] +MAIL_PASSWORD = sm_config['email']['password'] WTF_CSRF_ENABLED = True WTF_CSRF_SSL_STRICT = True # Checks Referer Header. Set to False for API access. From 8b63c871693c40873cac1f949ea5b2112fcb956c Mon Sep 17 00:00:00 2001 From: jnbnyc Date: Fri, 28 Oct 2016 14:40:58 -0400 Subject: [PATCH 16/37] Add documentation for SecurityMonkey Docker --- docs/docker.rst | 51 +++++++++++++++++++++++++++++++++++++++++++++ docs/quickstart.rst | 2 ++ 2 files changed, 53 insertions(+) create mode 100644 docs/docker.rst diff --git a/docs/docker.rst b/docs/docker.rst new file mode 100644 index 000000000..b0fb6f2ea --- /dev/null +++ b/docs/docker.rst @@ -0,0 +1,51 @@ +Docker Instructions +=================== + +The docker-compose.yml file describes the SecurityMonkey environment. This is intended for local development with the intention of deploying SecurityMonkey containers with a Docker Orchestration tool like Kubernetes. + +The Dockerfile builds SecurityMonkey into a container with several different entrypoints. These are for the different responsibilities SecurityMonkey has. +Also, the Dockerfile.nginx file is used to build an NGINX container that will front the API, serve the static assets, and provide TLS. + +Quick Start: +------------ + Define your specific settings in **secmonkey.env** file. For example, this file will look like:: + + AWS_ACCESS_KEY_ID= + AWS_SECRET_ACCESS_KEY= + SECURITY_MONKEY_POSTGRES_HOST=postgres + SECURITY_MONKEY_FQDN=192.168.99.100 + + $ docker-compose build + ``this will locally build all the containers necessary`` + + $ docker-compose up -d postgres + ``this will start the database container`` + + $ docker-compose up -d scratch + ``this will start a container in which you canuse to setup the database, create users, and other manual configurations, see the below section for more info`` + + $ docker-compose up + ``this will bring up the remaining containers (scheduler and nginx)`` + +Commands: +--------- + + $ docker-compose build ``[api | scheduler | nginx | scratch]`` + + $ docker-compose up -d ``[postgres | api | scheduler | nginx | scratch]`` + +More Info: +---------- +:: + + $ docker-compose up -d scratch + +The scratch container is where the SecurityMonkey code is available for you to run manual configurations such as:: + + $ python manage.py create_user admin@example.com Admin + +and/or:: + + $ python manage.py add_account --number $account --name $name -r SecurityMonkey + +The scratch container provides a sandbox and is useful for local development. It is not required otherwise. diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 9bf174f3a..6a02937fe 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -8,6 +8,8 @@ Docker Images Before we start, consider following the `docker instructions `_ . Docker helps simplify the process to get up and running. The docker images are not currently ready for production use, but are good enough to get up and running with an instance of security_monkey. +Local `docker instructions <./docker.html>`_ + Not into the docker thing? Keep reading. Setup IAM Roles From 3d6c0a0ecf496a7fcf7668316346ce1f10b4e6c1 Mon Sep 17 00:00:00 2001 From: jnbnyc Date: Fri, 28 Oct 2016 14:59:29 -0400 Subject: [PATCH 17/37] Add a forgotten comma --- env-config/config-deploy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/env-config/config-deploy.py b/env-config/config-deploy.py index ed166221d..59dbb26a0 100644 --- a/env-config/config-deploy.py +++ b/env-config/config-deploy.py @@ -19,7 +19,7 @@ # '': '', # Setting default settings sm_config = { - 'fqdn': 'ec2-XX-XXX-XXX-XXX.compute-1.amazonaws.com' + 'fqdn': 'ec2-XX-XXX-XXX-XXX.compute-1.amazonaws.com', 'postgres': { 'database': 'secmonkey', 'host': 'localhost', From 01bd572c0d0b80283c41c398b24c10be687b8699 Mon Sep 17 00:00:00 2001 From: jnbnyc Date: Fri, 28 Oct 2016 15:00:32 -0400 Subject: [PATCH 18/37] Update entrypoints to work without chmod +x --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 6128ce0e4..d50814068 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -28,7 +28,7 @@ services: depends_on: - postgres env_file: secmonkey.env - entrypoint: "/usr/local/src/security_monkey/docker/api-start.sh" + entrypoint: ["/bin/bash", "/usr/local/src/security_monkey/docker/api-start.sh"] scheduler: container_name: secmonkey-scheduler @@ -38,7 +38,7 @@ services: depends_on: - api env_file: secmonkey.env - entrypoint: "/usr/local/src/security_monkey/docker/scheduler-start.sh" + entrypoint: ["/bin/bash", "/usr/local/src/security_monkey/docker/scheduler-start.sh"] nginx: container_name: secmonkey-nginx From 9e6635a4246fbeb0e2aafac99425ed805068807d Mon Sep 17 00:00:00 2001 From: jnbnyc Date: Fri, 28 Oct 2016 15:07:32 -0400 Subject: [PATCH 19/37] Add a missing equals sign '=' --- env-config/config-deploy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/env-config/config-deploy.py b/env-config/config-deploy.py index 59dbb26a0..66374c914 100644 --- a/env-config/config-deploy.py +++ b/env-config/config-deploy.py @@ -67,7 +67,7 @@ if 'SECURITY_MONKEY_SMTP' in os.environ: # Must change String from environment variable into Boolean - if os.environ.get('SECURITY_MONKEY_SMTP') = 'True': + if os.environ.get('SECURITY_MONKEY_SMTP') == 'True': sm_config['email']['smtp'] = True if 'SECURITY_MONKEY_SES_REGION' in os.environ: From c88d57e0a6f5945bfc3fe21cd84417eeead733f2 Mon Sep 17 00:00:00 2001 From: jnbnyc Date: Tue, 1 Nov 2016 08:18:36 -0400 Subject: [PATCH 20/37] Remove baked-in insecure certificates --- docker-nginx/server.crt | 25 ------------------------- docker-nginx/server.key | 27 --------------------------- 2 files changed, 52 deletions(-) delete mode 100644 docker-nginx/server.crt delete mode 100644 docker-nginx/server.key diff --git a/docker-nginx/server.crt b/docker-nginx/server.crt deleted file mode 100644 index f916f2452..000000000 --- a/docker-nginx/server.crt +++ /dev/null @@ -1,25 +0,0 @@ - ------BEGIN CERTIFICATE----- -MIID8jCCAtoCCQCnvR3ajR8rsjANBgkqhkiG9w0BAQsFADCBujELMAkGA1UEBhMC -VVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEjAQBgNVBAcMCUxvcyBHYXRvczEnMCUG -A1UECgweU2VsZiBTaWduZWQgRG9ja2VyIENlcnRzIExtdGQuMR4wHAYDVQQLDBVT -ZWxmIFNpZ25lZCBEb2NrZXIgT1UxFTATBgNVBAMMDFNvbWVEb2NrZXJJUDEiMCAG -CSqGSIb3DQEJARYTcGtlbGxleUBuZXRmbGl4LmNvbTAeFw0xNDExMTAwMzQ0Mzda -Fw0xNTExMTAwMzQ0MzdaMIG6MQswCQYDVQQGEwJVUzETMBEGA1UECAwKQ2FsaWZv -cm5pYTESMBAGA1UEBwwJTG9zIEdhdG9zMScwJQYDVQQKDB5TZWxmIFNpZ25lZCBE -b2NrZXIgQ2VydHMgTG10ZC4xHjAcBgNVBAsMFVNlbGYgU2lnbmVkIERvY2tlciBP -VTEVMBMGA1UEAwwMU29tZURvY2tlcklQMSIwIAYJKoZIhvcNAQkBFhNwa2VsbGV5 -QG5ldGZsaXguY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvkeh -4rS7A2Irm8UAuEitneMRviSDPI04a3sbYIZISPMTsmn50ZXAewikL7YAVchDceiW -dhu4T+lah/lAmTjHLgleEUu2htsSO25pQBSC1mzpxU2RhY8EpW4pgjZdkzlHdsy5 -ZiYS4HAHIp7ZFr+DEDAoFgR2uIMK8W7jAwLXcjT1qs/q69Po1EAnOucLAwu6fiUS -MY/BAwGxhRDp9Q0dT7FiHwN8756MdQufapnuosHPcQoOj73IOhSN7EBqNXFWOjPc -nKKhmFxuFhKgIDcY8yJoAx5qWOWM7vlgACHd/41kE53RMd7k7hp3GXxofs7qHtaf -ZxtdMqiuPC/UUoMEzwIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQCckPS2VT4VZr8S -UsyuSy4RWtpwTg2sVGsddb27R3l9a/fRupHnSs1v+ZyrP7a/QcK16wNAGYm05Ajd -jXxaI8auNwi+Vix5r9pV3OG9bGoDp5m0+TeeV+RGs6nhIgnyJrYqDBV6St3UPwwS -U0mh4iFxwdR8NZYpLAu1R+t0t7G0do8yJKmHezs7zXy6J16Xl8txjFAVF5pHw2aN -h5Lj1FcdrOUCaeAFTmTK15ZcojDuIOoN/EdpNyHgelXqr0NQGcfrGJIGsn/8vXyV -+2B7QTr7w5gtqu5RQ5pem14wMoJl2tRNx02fP9CYmJoD8KAximnJDxL1PXEFIMxw -t7hR6j5w ------END CERTIFICATE----- \ No newline at end of file diff --git a/docker-nginx/server.key b/docker-nginx/server.key deleted file mode 100644 index 75e6192b2..000000000 --- a/docker-nginx/server.key +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEpAIBAAKCAQEAvkeh4rS7A2Irm8UAuEitneMRviSDPI04a3sbYIZISPMTsmn5 -0ZXAewikL7YAVchDceiWdhu4T+lah/lAmTjHLgleEUu2htsSO25pQBSC1mzpxU2R -hY8EpW4pgjZdkzlHdsy5ZiYS4HAHIp7ZFr+DEDAoFgR2uIMK8W7jAwLXcjT1qs/q -69Po1EAnOucLAwu6fiUSMY/BAwGxhRDp9Q0dT7FiHwN8756MdQufapnuosHPcQoO -j73IOhSN7EBqNXFWOjPcnKKhmFxuFhKgIDcY8yJoAx5qWOWM7vlgACHd/41kE53R -Md7k7hp3GXxofs7qHtafZxtdMqiuPC/UUoMEzwIDAQABAoIBAAShg6wbz+1eJ0eM -PN8/TZJpI1ZoVxVTLotXHKx1uC6mU9VkxOK37U5RQIW0vT9ZN2L/USIgvJSdTkm+ -5DaWbiqNqc9r805G6KOvpTodBa4oSmw13hPNDkTkcHgt0IszuqyGgQCebpktwM6z -5P6rvInZUjBu/WCyN8L2euno2Mt7ubYsHp6IZNeDF5UKBd3fcORCCN2oIjud/wo+ -7qzgc1Qz8yZLg5rLY5t1TXmWsYPnTgCo2xSUt+40VEPu2IeShg+TxyFzVTkfsxs9 -ZT6+HwW3wcnRLbUcdX/sYldaKHW+k4rDHnRtqzaxgwmL+Pd/RC/b1MRReY2IANkU -P1q5kikCgYEA7GuPfeX4SmvZqipRi6CujWHZcR5pHUAqURrA+PkGw7PVo3Z59aor -KbAUsPDpQLhLnsO/nTjxjnvz+uHYRJjsAypXtwp0cyvV5TUd8REwzg8ivRB1U6TX -aUTmqpdC7JL8bfN10puqOG2dE4/tiVpW02e2pSp+gBdjS0t0Rn/PagsCgYEAzgnU -7Fs+G0sKBkXF+/M/XFin5dGqlvYBKXwh49qeyyOZFHzGNs97qtvuRnKZ3xgSsbnV -KGa4PcSBcSH7ktiKMsjxs2rxkGrFvD+7TpM4FfNxOL04AupHLgkb0JOlCDsgPkwH -DoyOSomutL3hSXrIEbg+2pgPUm6kgMfKFw5rjs0CgYEAmZq4u/+ydgfkf97G5IUd -Y9ZzRD+R4+NQmylav2hssLIi1/Wd/7L0ID2688tSgS48U4ay9B+PMhfuyM37iXYh -wVy9aLtuNXYBns4IChw1LWxAEb6jvBiZrFeL1sI4RSSqxXUs1A03ZzowmAknN9pu -FySHUoBteCO56622eieIR0kCgYA0uB1MMPiOQUAaZMYI9q+ysFidnG1a3S1k8Qc2 -5xyUe9JShK6vHZ3WFRo92lr205EM+rDI+qeP3nUEfp6Bb3jFD9eQgf+3ZSqdRW1m -6JKEQ1soGnp3fHykEd6VGwIyPfa9GfUASwSpm9shEijQBWOl7Q/gHOJmxDrBXN2f -0qfAzQKBgQDL7gyh7hBCBlwFreKmmYyNQDYWp4XJvvO0mOhR4rpNMgFAWqBWPd/C -VBPCctp71Dj9MztNrIUBTOsU54DFHfC+UIicoyAtPIWUbaYlS0Wxf1DY6orvAp+T -3D+eNMsdnZqEPC+Q+Fb2DkF8CzrhYdH2nZEMN8rA50/L/r0OGkOiLg== ------END RSA PRIVATE KEY----- \ No newline at end of file From 819f72f9a787eb15fe005b21a21ec379ae615767 Mon Sep 17 00:00:00 2001 From: jnbnyc Date: Tue, 1 Nov 2016 08:57:50 -0400 Subject: [PATCH 21/37] Add new entrypoint for nginx, disables ssl if cert:key pair are not found --- docker-nginx/start-nginx.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100755 docker-nginx/start-nginx.sh diff --git a/docker-nginx/start-nginx.sh b/docker-nginx/start-nginx.sh new file mode 100755 index 000000000..97ac28149 --- /dev/null +++ b/docker-nginx/start-nginx.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +SECURITY_MONKEY_SSL_CERT=${SECURITY_MONKEY_SSL_CERT:-/etc/nginx/ssl/server.crt} +SECURITY_MONKEY_SSL_KEY=${SECURITY_MONKEY_SSL_KEY:-/etc/nginx/ssl/server.key} +SECURITY_MONKEY_CONF=${SECURITY_MONKEY_CONF:-/etc/nginx/conf.d/securitymonkey.conf} + +if [ ! -f "$SECURITY_MONKEY_SSL_CERT" ] || [ ! -f "$SECURITY_MONKEY_SSL_KEY" ]; then + # comment out ssl configurations when cert and key are not found + sed -i.bak 's@^\s*listen.*ssl.*$@#&@' $SECURITY_MONKEY_CONF + sed -i.bak 's@^\s.*ssl_certificate.*$@#&@' $SECURITY_MONKEY_CONF +fi + +exec nginx From 629cbd094bc140ca13a5f55c3c0e9c4feb8951a9 Mon Sep 17 00:00:00 2001 From: jnbnyc Date: Tue, 1 Nov 2016 08:58:34 -0400 Subject: [PATCH 22/37] Add original nginx.conf for reference --- docker-nginx/nginx.conf | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 docker-nginx/nginx.conf diff --git a/docker-nginx/nginx.conf b/docker-nginx/nginx.conf new file mode 100644 index 000000000..e4bad8dbc --- /dev/null +++ b/docker-nginx/nginx.conf @@ -0,0 +1,32 @@ + +user nginx; +worker_processes 1; + +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + + +events { + worker_connections 1024; +} + + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + #tcp_nopush on; + + keepalive_timeout 65; + + #gzip on; + + include /etc/nginx/conf.d/*.conf; +} From 10baf33edc8599349111c596aa716bafe9fd6da9 Mon Sep 17 00:00:00 2001 From: jnbnyc Date: Tue, 1 Nov 2016 08:59:42 -0400 Subject: [PATCH 23/37] Turn daemon off in nginx.conf --- docker-nginx/nginx.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-nginx/nginx.conf b/docker-nginx/nginx.conf index e4bad8dbc..7e0e29ff1 100644 --- a/docker-nginx/nginx.conf +++ b/docker-nginx/nginx.conf @@ -1,6 +1,7 @@ user nginx; worker_processes 1; +daemon off; error_log /var/log/nginx/error.log warn; pid /var/run/nginx.pid; From 633337d73f84761b2f0f5bc9202d08cf4908da7a Mon Sep 17 00:00:00 2001 From: jnbnyc Date: Tue, 1 Nov 2016 09:06:27 -0400 Subject: [PATCH 24/37] Update nginx docker build and entrypoint --- Dockerfile.nginx | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/Dockerfile.nginx b/Dockerfile.nginx index fd2889f7f..e9d64c10b 100644 --- a/Dockerfile.nginx +++ b/Dockerfile.nginx @@ -37,7 +37,7 @@ RUN cd /usr/local/src/security_monkey/dart &&\ /bin/mkdir -p /usr/local/src/security_monkey/security_monkey/static/ &&\ /bin/cp -R /usr/local/src/security_monkey/dart/build/web/* /usr/local/src/security_monkey/security_monkey/static/ -RUN /bin/mkdir -p /var/log/security_monkey/ &&\ +RUN /bin/mkdir -p /var/log/security_monkey/ /etc/nginx/ssl/ &&\ # /usr/bin/touch /var/log/security_monkey/security_monkey.access.log &&\ # /usr/bin/touch /var/log/security_monkey/security_monkey.error.log ln -s /dev/stdout /var/log/security_monkey/security_monkey.access.log &&\ @@ -47,11 +47,7 @@ WORKDIR /etc/nginx EXPOSE 443 ADD docker-nginx/securitymonkey.conf /etc/nginx/conf.d/securitymonkey.conf -# COPY nginx.conf /etc/nginx/nginx.conf -ADD docker-nginx/server.crt /etc/nginx/ssl/ -ADD docker-nginx/server.key /etc/nginx/ssl/ +COPY docker-nginx/nginx.conf /etc/nginx/nginx.conf +# ADD docker-nginx/server.crt docker-nginx/server.key /etc/nginx/ssl/ -# RUN /bin/mkdir -p /etc/nginx/sites-enabled/ &&\ -# ln -s /etc/nginx/sites-available/securitymonkey.conf /etc/nginx/sites-enabled/securitymonkey.conf - -ENTRYPOINT ["nginx", "-g", "daemon off;"] \ No newline at end of file +ENTRYPOINT ["/usr/local/src/security_monkey/docker-nginx/start-nginx.sh"] From 479ae6a2b7428be38c50d1a4b936d8c3d5a060ac Mon Sep 17 00:00:00 2001 From: jnbnyc Date: Tue, 1 Nov 2016 09:28:18 -0400 Subject: [PATCH 25/37] Move NGINX Dockerfile to docker-nginx/ --- docker-compose.yml | 2 +- Dockerfile.nginx => docker-nginx/Dockerfile | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename Dockerfile.nginx => docker-nginx/Dockerfile (100%) diff --git a/docker-compose.yml b/docker-compose.yml index d50814068..579721a81 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -44,7 +44,7 @@ services: container_name: secmonkey-nginx build: context: ./ - dockerfile: ./Dockerfile.nginx + dockerfile: ./docker-nginx/Dockerfile working_dir: /etc/nginx depends_on: - api diff --git a/Dockerfile.nginx b/docker-nginx/Dockerfile similarity index 100% rename from Dockerfile.nginx rename to docker-nginx/Dockerfile From 4d3c309cf20e9178d6516ee09e1f89d30a16cd55 Mon Sep 17 00:00:00 2001 From: jnbnyc Date: Fri, 4 Nov 2016 10:51:02 -0400 Subject: [PATCH 26/37] Refactor for local docker development - Removed python-m2crypto from Dockerfile - Created env-config/config-docker.py for settings - Reverted env-config/config-deploy.py back to v0.7.0 original - Moved docker-nginx directory to docker/nginx - Entrypoints are executable - Added brief documentation in docker folder --- .dockerignore | 1 + .gitignore | 3 + Dockerfile | 9 +- docker-compose.yml | 27 +- docker-nginx/start-nginx.sh | 13 - docker/README.rst | 9 + docker/api-init.sh | 0 docker/api-start.sh | 0 {docker-nginx => docker/nginx}/Dockerfile | 8 +- {docker-nginx => docker/nginx}/nginx.conf | 0 .../nginx}/securitymonkey.conf | 0 docker/nginx/start-nginx.sh | 12 + docker/scheduler-start.sh | 0 env-config/config-deploy.py | 194 +---------- env-config/config-docker.py | 322 ++++++++++++++++++ 15 files changed, 382 insertions(+), 216 deletions(-) delete mode 100755 docker-nginx/start-nginx.sh create mode 100644 docker/README.rst mode change 100644 => 100755 docker/api-init.sh mode change 100644 => 100755 docker/api-start.sh rename {docker-nginx => docker/nginx}/Dockerfile (89%) rename {docker-nginx => docker/nginx}/nginx.conf (100%) rename {docker-nginx => docker/nginx}/securitymonkey.conf (100%) create mode 100755 docker/nginx/start-nginx.sh mode change 100644 => 100755 docker/scheduler-start.sh create mode 100644 env-config/config-docker.py diff --git a/.dockerignore b/.dockerignore index 33a7d08e0..1750cd364 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,4 @@ +.git secmonkey.env boto.cfg .travis.yml diff --git a/.gitignore b/.gitignore index b41a95b6a..2de573b06 100644 --- a/.gitignore +++ b/.gitignore @@ -55,3 +55,6 @@ venv/ boto.cfg secmonkey.env +*.crt +*.key + diff --git a/Dockerfile b/Dockerfile index 328cabdee..5a4806779 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,16 +17,16 @@ FROM ubuntu:14.04 MAINTAINER Netflix Open Source Development ENV SECURITY_MONKEY_VERSION=v0.7.0 \ - SECURITY_MONKEY_SETTINGS=/usr/local/src/security_monkey/env-config/config-deploy.py + SECURITY_MONKEY_SETTINGS=/usr/local/src/security_monkey/env-config/config-docker.py RUN apt-get update &&\ apt-get -y -q install python-software-properties software-properties-common postgresql-9.3 postgresql-client-9.3 postgresql-contrib-9.3 curl &&\ - apt-get install -y python-pip python-dev python-psycopg2 libffi-dev libpq-dev libyaml-dev libxml2-dev libxmlsec1-dev git sudo swig python-m2crypto &&\ + apt-get install -y python-pip python-dev python-psycopg2 libffi-dev libpq-dev libyaml-dev libxml2-dev libxmlsec1-dev git sudo swig &&\ rm -rf /var/lib/apt/lists/* RUN cd /usr/local/src &&\ # git clone --branch $SECURITY_MONKEY_VERSION https://github.com/Netflix/security_monkey.git - mkdir -p security_monkey + /bin/mkdir -p security_monkey ADD . /usr/local/src/security_monkey RUN cd /usr/local/src/security_monkey &&\ @@ -35,7 +35,8 @@ RUN cd /usr/local/src/security_monkey &&\ RUN chmod +x /usr/local/src/security_monkey/docker/*.sh &&\ mkdir -pv /var/log/security_monkey &&\ - ln -s /dev/stdout /var/log/security_monkey/securitymonkey.log + /usr/bin/touch /var/log/security_monkey/securitymonkey.log + # ln -s /dev/stdout /var/log/security_monkey/securitymonkey.log WORKDIR /usr/local/src/security_monkey EXPOSE 5000 diff --git a/docker-compose.yml b/docker-compose.yml index 579721a81..b447709e4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,11 +3,10 @@ ### # # Documentation: http://securitymonkey.readthedocs.io/en/latest/index.html +# http://securitymonkey.readthedocs.io/en/latest/docker.html # # shortcuts # open https://$(docker-machine active | xargs docker-machine ip) -# find . -type f -name "*.sh" -exec chmod +x {} \; -# find . -type f -name "*.sh" -exec chmod -x {} \; # ### @@ -23,29 +22,32 @@ services: api: container_name: secmonkey-api build: . - volumes: - - ./docker:/usr/local/src/security_monkey/docker/ + volumes_from: + - init depends_on: - postgres env_file: secmonkey.env - entrypoint: ["/bin/bash", "/usr/local/src/security_monkey/docker/api-start.sh"] + entrypoint: ["/usr/local/src/security_monkey/docker/api-start.sh"] scheduler: container_name: secmonkey-scheduler build: . - volumes: - - ./docker:/usr/local/src/security_monkey/docker/ + volumes_from: + - init depends_on: - api env_file: secmonkey.env - entrypoint: ["/bin/bash", "/usr/local/src/security_monkey/docker/scheduler-start.sh"] + entrypoint: ["/usr/local/src/security_monkey/docker/scheduler-start.sh"] nginx: container_name: secmonkey-nginx build: context: ./ - dockerfile: ./docker-nginx/Dockerfile + dockerfile: ./docker/nginx/Dockerfile working_dir: /etc/nginx + volumes: + - ./docker/nginx/server.crt:/etc/nginx/ssl/server.crt + - ./docker/nginx/server.key:/etc/nginx/ssl/server.key depends_on: - api ports: @@ -60,13 +62,14 @@ services: ### ### ### ### ### ### - scratch: - container_name: scratch + init: + container_name: init build: . working_dir: /usr/local/src/security_monkey volumes: - ./data/aws_accounts.json:/usr/local/src/security_monkey/data/aws_accounts.json - ./docker:/usr/local/src/security_monkey/docker/ + - ./env-config/config-docker.py:/usr/local/src/security_monkey/env-config/config-docker.py depends_on: - postgres env_file: secmonkey.env @@ -74,6 +77,6 @@ services: # - AWS_ACCESS_KEY_ID= # - AWS_SECRET_ACCESS_KEY= # - SECURITY_MONKEY_POSTGRES_HOST= - entrypoint: + entrypoint: # /usr/local/src/security_monkey/docker/api-init.sh - sleep - 8h diff --git a/docker-nginx/start-nginx.sh b/docker-nginx/start-nginx.sh deleted file mode 100755 index 97ac28149..000000000 --- a/docker-nginx/start-nginx.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash - -SECURITY_MONKEY_SSL_CERT=${SECURITY_MONKEY_SSL_CERT:-/etc/nginx/ssl/server.crt} -SECURITY_MONKEY_SSL_KEY=${SECURITY_MONKEY_SSL_KEY:-/etc/nginx/ssl/server.key} -SECURITY_MONKEY_CONF=${SECURITY_MONKEY_CONF:-/etc/nginx/conf.d/securitymonkey.conf} - -if [ ! -f "$SECURITY_MONKEY_SSL_CERT" ] || [ ! -f "$SECURITY_MONKEY_SSL_KEY" ]; then - # comment out ssl configurations when cert and key are not found - sed -i.bak 's@^\s*listen.*ssl.*$@#&@' $SECURITY_MONKEY_CONF - sed -i.bak 's@^\s.*ssl_certificate.*$@#&@' $SECURITY_MONKEY_CONF -fi - -exec nginx diff --git a/docker/README.rst b/docker/README.rst new file mode 100644 index 000000000..04d86b4e2 --- /dev/null +++ b/docker/README.rst @@ -0,0 +1,9 @@ +************************ +Docker local development +************************ + +Project resources +================= + +- `Docker documentation `_ +- `Development documentation `_ diff --git a/docker/api-init.sh b/docker/api-init.sh old mode 100644 new mode 100755 diff --git a/docker/api-start.sh b/docker/api-start.sh old mode 100644 new mode 100755 diff --git a/docker-nginx/Dockerfile b/docker/nginx/Dockerfile similarity index 89% rename from docker-nginx/Dockerfile rename to docker/nginx/Dockerfile index e9d64c10b..4487ccc61 100644 --- a/docker-nginx/Dockerfile +++ b/docker/nginx/Dockerfile @@ -46,8 +46,8 @@ RUN /bin/mkdir -p /var/log/security_monkey/ /etc/nginx/ssl/ &&\ WORKDIR /etc/nginx EXPOSE 443 -ADD docker-nginx/securitymonkey.conf /etc/nginx/conf.d/securitymonkey.conf -COPY docker-nginx/nginx.conf /etc/nginx/nginx.conf -# ADD docker-nginx/server.crt docker-nginx/server.key /etc/nginx/ssl/ +ADD docker/nginx/securitymonkey.conf /etc/nginx/conf.d/securitymonkey.conf +COPY docker/nginx/nginx.conf /etc/nginx/nginx.conf +# ADD docker/nginx/server.crt docker/nginx/server.key /etc/nginx/ssl/ -ENTRYPOINT ["/usr/local/src/security_monkey/docker-nginx/start-nginx.sh"] +ENTRYPOINT ["/usr/local/src/security_monkey/docker/nginx/start-nginx.sh"] diff --git a/docker-nginx/nginx.conf b/docker/nginx/nginx.conf similarity index 100% rename from docker-nginx/nginx.conf rename to docker/nginx/nginx.conf diff --git a/docker-nginx/securitymonkey.conf b/docker/nginx/securitymonkey.conf similarity index 100% rename from docker-nginx/securitymonkey.conf rename to docker/nginx/securitymonkey.conf diff --git a/docker/nginx/start-nginx.sh b/docker/nginx/start-nginx.sh new file mode 100755 index 000000000..3c2747336 --- /dev/null +++ b/docker/nginx/start-nginx.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +SECURITY_MONKEY_SSL_CERT=${SECURITY_MONKEY_SSL_CERT:-/etc/nginx/ssl/server.crt} +SECURITY_MONKEY_SSL_KEY=${SECURITY_MONKEY_SSL_KEY:-/etc/nginx/ssl/server.key} + +if [ ! -f "$SECURITY_MONKEY_SSL_CERT" ] || [ ! -f "$SECURITY_MONKEY_SSL_KEY" ]; then + # Fail if SSL is unavailable + echo "$(date) Error: Missing files required for SSL" + # exit 1 +fi + +exec nginx diff --git a/docker/scheduler-start.sh b/docker/scheduler-start.sh old mode 100644 new mode 100755 diff --git a/env-config/config-deploy.py b/env-config/config-deploy.py index 66374c914..f5c5fb0ba 100644 --- a/env-config/config-deploy.py +++ b/env-config/config-deploy.py @@ -14,77 +14,6 @@ # Insert any config items here. # This will be fed into Flask/SQLAlchemy inside security_monkey/__init__.py -import os - -# '': '', -# Setting default settings -sm_config = { - 'fqdn': 'ec2-XX-XXX-XXX-XXX.compute-1.amazonaws.com', - 'postgres': { - 'database': 'secmonkey', - 'host': 'localhost', - 'password': 'securitymonkeypassword', - 'port': '5432', - 'user': 'postgres' - }, - 'api': { - 'port': 5000 - }, - 'email': { - 'security-team-email': [], - 'smtp': False, - 'ses-region': 'us-east-1', - 'default-sender': 'securitymonkey@example.com', - 'server': 'smtp.example.com', - 'username': 'username', - 'password': 'password' - } -} - -if 'SECURITY_MONKEY_POSTGRES_HOST' in os.environ: - sm_config['postgres']['host'] = os.environ.get('SECURITY_MONKEY_POSTGRES_HOST') - -if 'SECURITY_MONKEY_POSTGRES_USER' in os.environ: - sm_config['postgres']['user'] = os.environ.get('SECURITY_MONKEY_POSTGRES_USER') - -if 'SECURITY_MONKEY_POSTGRES_PASSWORD' in os.environ: - sm_config['postgres']['password'] = os.environ.get('SECURITY_MONKEY_POSTGRES_PASSWORD') - -if 'SECURITY_MONKEY_POSTGRES_DATABASE' in os.environ: - sm_config['postgres']['database'] = os.environ.get('SECURITY_MONKEY_POSTGRES_DATABASE') - -if 'SECURITY_MONKEY_POSTGRES_PORT' in os.environ: - sm_config['postgres']['port'] = os.environ.get('SECURITY_MONKEY_POSTGRES_PORT') - -if 'SECURITY_MONKEY_API_PORT' in os.environ: - sm_config['api']['port'] = os.environ.get('SECURITY_MONKEY_API_PORT') - -if 'SECURITY_MONKEY_FQDN' in os.environ: - sm_config['fqdn'] = os.environ.get('SECURITY_MONKEY_FQDN') - -if 'SECURITY_MONKEY_SECURITY_TEAM_EMAIL' in os.environ: - sm_config['email']['security-team-email'] = os.environ.get('SECURITY_MONKEY_SECURITY_TEAM_EMAIL') - -if 'SECURITY_MONKEY_SMTP' in os.environ: - # Must change String from environment variable into Boolean - if os.environ.get('SECURITY_MONKEY_SMTP') == 'True': - sm_config['email']['smtp'] = True - -if 'SECURITY_MONKEY_SES_REGION' in os.environ: - sm_config['email']['ses-region'] = os.environ.get('SECURITY_MONKEY_SES_REGION') - -if 'SECURITY_MONKEY_EMAIL_DEFAULT_SENDER' in os.environ: - sm_config['email']['default-sender'] = os.environ.get('SECURITY_MONKEY_EMAIL_DEFAULT_SENDER') - -if 'SECURITY_MONKEY_EMAIL_SERVER' in os.environ: - sm_config['email']['server'] = os.environ.get('SECURITY_MONKEY_EMAIL_SERVER') - -if 'SECURITY_MONKEY_EMAIL_USERNAME' in os.environ: - sm_config['email']['username'] = os.environ.get('SECURITY_MONKEY_EMAIL_USERNAME') - -if 'SECURITY_MONKEY_EMAIL_PASSWORD' in os.environ: - sm_config['email']['password'] = os.environ.get('SECURITY_MONKEY_EMAIL_PASSWORD') - LOG_CFG = { 'version': 1, 'disable_existing_loggers': False, @@ -123,23 +52,14 @@ } } -SQLALCHEMY_DATABASE_URI = 'postgresql://%s:%s@%s:%s/%s' % ( - sm_config['postgres']['user'], - sm_config['postgres']['password'], - sm_config['postgres']['host'], - sm_config['postgres']['port'], - sm_config['postgres']['database'] -) - -# print sm_config['postgres'] -# print SQLALCHEMY_DATABASE_URI +SQLALCHEMY_DATABASE_URI = 'postgresql://postgres:securitymonkeypassword@localhost:5432/secmonkey' SQLALCHEMY_POOL_SIZE = 50 SQLALCHEMY_MAX_OVERFLOW = 15 ENVIRONMENT = 'ec2' USE_ROUTE53 = False -FQDN = sm_config['fqdn'] -API_PORT = sm_config['api']['port'] +FQDN = 'ec2-XX-XXX-XXX-XXX.compute-1.amazonaws.com' +API_PORT = '5000' WEB_PORT = '443' WEB_PATH = '/static/ui.html' FRONTED_BY_NGINX = True @@ -148,7 +68,7 @@ SECRET_KEY = '' -MAIL_DEFAULT_SENDER = sm_config['email']['default-sender'] +MAIL_DEFAULT_SENDER = 'securitymonkey@example.com' SECURITY_REGISTERABLE = True SECURITY_CONFIRMABLE = False SECURITY_RECOVERABLE = False @@ -163,16 +83,16 @@ SECURITY_POST_CHANGE_VIEW = BASE_URL # This address gets all change notifications (i.e. 'securityteam@example.com') -SECURITY_TEAM_EMAIL = sm_config['email']['security-team-email'] +SECURITY_TEAM_EMAIL = [] # These are only required if using SMTP instead of SES -EMAILS_USE_SMTP = sm_config['email']['smtp'] # Otherwise, Use SES -SES_REGION = sm_config['email']['ses-region'] -MAIL_SERVER = sm_config['email']['server'] +EMAILS_USE_SMTP = False # Otherwise, Use SES +SES_REGION = 'us-east-1' +MAIL_SERVER = 'smtp.example.com' MAIL_PORT = 465 MAIL_USE_SSL = True -MAIL_USERNAME = sm_config['email']['username'] -MAIL_PASSWORD = sm_config['email']['password'] +MAIL_USERNAME = 'username' +MAIL_PASSWORD = 'password' WTF_CSRF_ENABLED = True WTF_CSRF_SSL_STRICT = True # Checks Referer Header. Set to False for API access. @@ -187,7 +107,7 @@ MAX_THREADS = 30 # SSO SETTINGS: -ACTIVE_PROVIDERS = [] # "ping", "google" or "onelogin" +ACTIVE_PROVIDERS = [] # "ping" or "google" PING_NAME = '' # Use to override the Ping name in the UI. PING_REDIRECT_URI = "{BASE}api/1/auth/ping".format(BASE=BASE_URL) @@ -202,98 +122,6 @@ GOOGLE_AUTH_ENDPOINT = '' GOOGLE_SECRET = '' -ONELOGIN_APP_ID = '' # OneLogin App ID provider by your administrator -ONELOGIN_EMAIL_FIELD = 'User.email' # SAML attribute used to provide email address -ONELOGIN_DEFAULT_ROLE = 'View' # Default RBAC when user doesn't already exist -ONELOGIN_HTTPS = True # If using HTTPS strict mode will check the requests are HTTPS -ONELOGIN_SETTINGS = { - # If strict is True, then the Python Toolkit will reject unsigned - # or unencrypted messages if it expects them to be signed or encrypted. - # Also it will reject the messages if the SAML standard is not strictly - # followed. Destination, NameId, Conditions ... are validated too. - "strict": True, - - # Enable debug mode (outputs errors). - "debug": True, - - # Service Provider Data that we are deploying. - "sp": { - # Identifier of the SP entity (must be a URI) - "entityId": "{BASE}metadata/".format(BASE=BASE_URL), - # Specifies info about where and how the message MUST be - # returned to the requester, in this case our SP. - "assertionConsumerService": { - # URL Location where the from the IdP will be returned - "url": "{BASE}api/1/auth/onelogin?acs".format(BASE=BASE_URL), - # SAML protocol binding to be used when returning the - # message. OneLogin Toolkit supports this endpoint for the - # HTTP-POST binding only. - "binding": "urn:oasis:names:tc:SAML:2.0:bindings:HTTP-POST" - }, - # If you need to specify requested attributes, set a - # attributeConsumingService. nameFormat, attributeValue and - # friendlyName can be omitted - #"attributeConsumingService": { - # "ServiceName": "SP test", - # "serviceDescription": "Test Service", - # "requestedAttributes": [ - # { - # "name": "", - # "isRequired": False, - # "nameFormat": "", - # "friendlyName": "", - # "attributeValue": "" - # } - # ] - #}, - # Specifies info about where and how the message MUST be - # returned to the requester, in this case our SP. - "singleLogoutService": { - # URL Location where the from the IdP will be returned - "url": "{BASE}api/1/auth/onelogin?sls".format(BASE=BASE_URL), - # SAML protocol binding to be used when returning the - # message. OneLogin Toolkit supports the HTTP-Redirect binding - # only for this endpoint. - "binding": "urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect" - }, - # Specifies the constraints on the name identifier to be used to - # represent the requested subject. - # Take a look on src/onelogin/saml2/constants.py to see the NameIdFormat that are supported. - "NameIDFormat": "urn:oasis:names:tc:SAML:1.1:nameid-format:unspecified", - # Usually x509cert and privateKey of the SP are provided by files placed at - # the certs folder. But we can also provide them with the following parameters - "x509cert": "", - "privateKey": "" - }, - - # Identity Provider Data that we want connected with our SP. - "idp": { - # Identifier of the IdP entity (must be a URI) - "entityId": "https://app.onelogin.com/saml/metadata/{APP_ID}".format(APP_ID=ONELOGIN_APP_ID), - # SSO endpoint info of the IdP. (Authentication Request protocol) - "singleSignOnService": { - # URL Target of the IdP where the Authentication Request Message - # will be sent. - "url": "https://app.onelogin.com/trust/saml2/http-post/sso/{APP_ID}".format(APP_ID=ONELOGIN_APP_ID), - # SAML protocol binding to be used when returning the - # message. OneLogin Toolkit supports the HTTP-Redirect binding - # only for this endpoint. - "binding": "urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect" - }, - # SLO endpoint info of the IdP. - "singleLogoutService": { - # URL Location of the IdP where SLO Request will be sent. - "url": "https://app.onelogin.com/trust/saml2/http-redirect/slo/{APP_ID}".format(APP_ID=ONELOGIN_APP_ID), - # SAML protocol binding to be used when returning the - # message. OneLogin Toolkit supports the HTTP-Redirect binding - # only for this endpoint. - "binding": "urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect" - }, - # Public x509 certificate of the IdP - "x509cert": "" - } -} - from datetime import timedelta PERMANENT_SESSION_LIFETIME=timedelta(minutes=60) SESSION_REFRESH_EACH_REQUEST=True diff --git a/env-config/config-docker.py b/env-config/config-docker.py new file mode 100644 index 000000000..1bc51e520 --- /dev/null +++ b/env-config/config-docker.py @@ -0,0 +1,322 @@ +# Copyright 2014 Netflix, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Insert any config items here. +# This will be fed into Flask/SQLAlchemy inside security_monkey/__init__.py + +import os + +# '': '', +# Setting default settings +sm_config = { + 'fqdn': 'ec2-XX-XXX-XXX-XXX.compute-1.amazonaws.com', + 'postgres': { + 'database': 'secmonkey', + 'host': 'localhost', + 'password': 'securitymonkeypassword', + 'port': '5432', + 'user': 'postgres' + }, + 'api': { + 'port': 5000 + }, + 'email': { + 'security-team-email': [], + 'smtp': False, + 'ses-region': 'us-east-1', + 'default-sender': 'securitymonkey@example.com', + 'server': 'smtp.example.com', + 'username': 'username', + 'password': 'password' + } +} + +logcfg = { + 'console': { + 'level': 'DEBUG' + }, + 'apscheduler': { + 'level': 'INFO' + } +} + +if 'SECURITY_MONKEY_POSTGRES_HOST' in os.environ: + sm_config['postgres']['host'] = os.environ.get('SECURITY_MONKEY_POSTGRES_HOST') + +if 'SECURITY_MONKEY_POSTGRES_USER' in os.environ: + sm_config['postgres']['user'] = os.environ.get('SECURITY_MONKEY_POSTGRES_USER') + +if 'SECURITY_MONKEY_POSTGRES_PASSWORD' in os.environ: + sm_config['postgres']['password'] = os.environ.get('SECURITY_MONKEY_POSTGRES_PASSWORD') + +if 'SECURITY_MONKEY_POSTGRES_DATABASE' in os.environ: + sm_config['postgres']['database'] = os.environ.get('SECURITY_MONKEY_POSTGRES_DATABASE') + +if 'SECURITY_MONKEY_POSTGRES_PORT' in os.environ: + sm_config['postgres']['port'] = os.environ.get('SECURITY_MONKEY_POSTGRES_PORT') + +if 'SECURITY_MONKEY_API_PORT' in os.environ: + sm_config['api']['port'] = os.environ.get('SECURITY_MONKEY_API_PORT') + +if 'SECURITY_MONKEY_FQDN' in os.environ: + sm_config['fqdn'] = os.environ.get('SECURITY_MONKEY_FQDN') + +if 'SECURITY_MONKEY_SECURITY_TEAM_EMAIL' in os.environ: + sm_config['email']['security-team-email'] = os.environ.get('SECURITY_MONKEY_SECURITY_TEAM_EMAIL') + +if 'SECURITY_MONKEY_SMTP' in os.environ: + # Must change String from environment variable into Boolean + if os.environ.get('SECURITY_MONKEY_SMTP') == 'True': + sm_config['email']['smtp'] = True + +if 'SECURITY_MONKEY_SES_REGION' in os.environ: + sm_config['email']['ses-region'] = os.environ.get('SECURITY_MONKEY_SES_REGION') + +if 'SECURITY_MONKEY_EMAIL_DEFAULT_SENDER' in os.environ: + sm_config['email']['default-sender'] = os.environ.get('SECURITY_MONKEY_EMAIL_DEFAULT_SENDER') + +if 'SECURITY_MONKEY_EMAIL_SERVER' in os.environ: + sm_config['email']['server'] = os.environ.get('SECURITY_MONKEY_EMAIL_SERVER') + +if 'SECURITY_MONKEY_EMAIL_USERNAME' in os.environ: + sm_config['email']['username'] = os.environ.get('SECURITY_MONKEY_EMAIL_USERNAME') + +if 'SECURITY_MONKEY_EMAIL_PASSWORD' in os.environ: + sm_config['email']['password'] = os.environ.get('SECURITY_MONKEY_EMAIL_PASSWORD') + +if 'SM_CONSOLE_LOG_LEVEL' in os.environ: + logcfg['console']['level'] = os.environ.get('SM_CONSOLE_LOG_LEVEL') + +if 'SM_APSCHEDULER_LOG_LEVEL' in os.environ: + logcfg['apscheduler']['level'] = os.environ.get('SM_APPSCHEDULER_LOG_LEVEL') + + +LOG_CFG = { + 'version': 1, + 'disable_existing_loggers': False, + 'formatters': { + 'standard': { + 'format': '%(asctime)s %(levelname)s: %(message)s ' + '[in %(pathname)s:%(lineno)d]' + } + }, + 'handlers': { + 'file': { + 'class': 'logging.handlers.RotatingFileHandler', + 'level': 'DEBUG', + 'formatter': 'standard', + 'filename': '/var/log/security_monkey/securitymonkey.log', + 'maxBytes': 10485760, + 'backupCount': 100, + 'encoding': 'utf8' + }, + 'console': { + 'class': 'logging.StreamHandler', + 'level': 'DEBUG', + 'formatter': 'standard', + 'stream': 'ext://sys.stdout' + } + }, + 'loggers': { + 'security_monkey': { + 'handlers': ['console'], + 'level': logcfg['console']['level'] + }, + 'apscheduler': { + 'handlers': ['console'], + 'level': logcfg['apscheduler']['level'] + } + } +} + +SQLALCHEMY_DATABASE_URI = 'postgresql://%s:%s@%s:%s/%s' % ( + sm_config['postgres']['user'], + sm_config['postgres']['password'], + sm_config['postgres']['host'], + sm_config['postgres']['port'], + sm_config['postgres']['database'] +) + +# print sm_config['postgres'] +# print SQLALCHEMY_DATABASE_URI + +SQLALCHEMY_POOL_SIZE = 50 +SQLALCHEMY_MAX_OVERFLOW = 15 +ENVIRONMENT = 'ec2' +USE_ROUTE53 = False +FQDN = sm_config['fqdn'] +API_PORT = sm_config['api']['port'] +WEB_PORT = '443' +WEB_PATH = '/static/ui.html' +FRONTED_BY_NGINX = True +NGINX_PORT = '443' +BASE_URL = 'https://{}/'.format(FQDN) + +SECRET_KEY = '' + +MAIL_DEFAULT_SENDER = sm_config['email']['default-sender'] +SECURITY_REGISTERABLE = True +SECURITY_CONFIRMABLE = False +SECURITY_RECOVERABLE = False +SECURITY_PASSWORD_HASH = 'bcrypt' +SECURITY_PASSWORD_SALT = '' +SECURITY_TRACKABLE = True + +SECURITY_POST_LOGIN_VIEW = BASE_URL +SECURITY_POST_REGISTER_VIEW = BASE_URL +SECURITY_POST_CONFIRM_VIEW = BASE_URL +SECURITY_POST_RESET_VIEW = BASE_URL +SECURITY_POST_CHANGE_VIEW = BASE_URL + +# This address gets all change notifications (i.e. 'securityteam@example.com') +SECURITY_TEAM_EMAIL = sm_config['email']['security-team-email'] + +# These are only required if using SMTP instead of SES +EMAILS_USE_SMTP = sm_config['email']['smtp'] # Otherwise, Use SES +SES_REGION = sm_config['email']['ses-region'] +MAIL_SERVER = sm_config['email']['server'] +MAIL_PORT = 465 +MAIL_USE_SSL = True +MAIL_USERNAME = sm_config['email']['username'] +MAIL_PASSWORD = sm_config['email']['password'] + +WTF_CSRF_ENABLED = True +WTF_CSRF_SSL_STRICT = True # Checks Referer Header. Set to False for API access. +WTF_CSRF_METHODS = ['DELETE', 'POST', 'PUT', 'PATCH'] + +# "NONE", "SUMMARY", or "FULL" +SECURITYGROUP_INSTANCE_DETAIL = 'FULL' + +# Threads used by the scheduler. +# You will likely need at least one core thread for every account being monitored. +CORE_THREADS = 25 +MAX_THREADS = 30 + +# SSO SETTINGS: +ACTIVE_PROVIDERS = [] # "ping", "google" or "onelogin" + +PING_NAME = '' # Use to override the Ping name in the UI. +PING_REDIRECT_URI = "{BASE}api/1/auth/ping".format(BASE=BASE_URL) +PING_CLIENT_ID = '' # Provided by your administrator +PING_AUTH_ENDPOINT = '' # Often something ending in authorization.oauth2 +PING_ACCESS_TOKEN_URL = '' # Often something ending in token.oauth2 +PING_USER_API_URL = '' # Often something ending in idp/userinfo.openid +PING_JWKS_URL = '' # Often something ending in JWKS +PING_SECRET = '' # Provided by your administrator + +GOOGLE_CLIENT_ID = '' +GOOGLE_AUTH_ENDPOINT = '' +GOOGLE_SECRET = '' + +ONELOGIN_APP_ID = '' # OneLogin App ID provider by your administrator +ONELOGIN_EMAIL_FIELD = 'User.email' # SAML attribute used to provide email address +ONELOGIN_DEFAULT_ROLE = 'View' # Default RBAC when user doesn't already exist +ONELOGIN_HTTPS = True # If using HTTPS strict mode will check the requests are HTTPS +ONELOGIN_SETTINGS = { + # If strict is True, then the Python Toolkit will reject unsigned + # or unencrypted messages if it expects them to be signed or encrypted. + # Also it will reject the messages if the SAML standard is not strictly + # followed. Destination, NameId, Conditions ... are validated too. + "strict": True, + + # Enable debug mode (outputs errors). + "debug": True, + + # Service Provider Data that we are deploying. + "sp": { + # Identifier of the SP entity (must be a URI) + "entityId": "{BASE}metadata/".format(BASE=BASE_URL), + # Specifies info about where and how the message MUST be + # returned to the requester, in this case our SP. + "assertionConsumerService": { + # URL Location where the from the IdP will be returned + "url": "{BASE}api/1/auth/onelogin?acs".format(BASE=BASE_URL), + # SAML protocol binding to be used when returning the + # message. OneLogin Toolkit supports this endpoint for the + # HTTP-POST binding only. + "binding": "urn:oasis:names:tc:SAML:2.0:bindings:HTTP-POST" + }, + # If you need to specify requested attributes, set a + # attributeConsumingService. nameFormat, attributeValue and + # friendlyName can be omitted + #"attributeConsumingService": { + # "ServiceName": "SP test", + # "serviceDescription": "Test Service", + # "requestedAttributes": [ + # { + # "name": "", + # "isRequired": False, + # "nameFormat": "", + # "friendlyName": "", + # "attributeValue": "" + # } + # ] + #}, + # Specifies info about where and how the message MUST be + # returned to the requester, in this case our SP. + "singleLogoutService": { + # URL Location where the from the IdP will be returned + "url": "{BASE}api/1/auth/onelogin?sls".format(BASE=BASE_URL), + # SAML protocol binding to be used when returning the + # message. OneLogin Toolkit supports the HTTP-Redirect binding + # only for this endpoint. + "binding": "urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect" + }, + # Specifies the constraints on the name identifier to be used to + # represent the requested subject. + # Take a look on src/onelogin/saml2/constants.py to see the NameIdFormat that are supported. + "NameIDFormat": "urn:oasis:names:tc:SAML:1.1:nameid-format:unspecified", + # Usually x509cert and privateKey of the SP are provided by files placed at + # the certs folder. But we can also provide them with the following parameters + "x509cert": "", + "privateKey": "" + }, + + # Identity Provider Data that we want connected with our SP. + "idp": { + # Identifier of the IdP entity (must be a URI) + "entityId": "https://app.onelogin.com/saml/metadata/{APP_ID}".format(APP_ID=ONELOGIN_APP_ID), + # SSO endpoint info of the IdP. (Authentication Request protocol) + "singleSignOnService": { + # URL Target of the IdP where the Authentication Request Message + # will be sent. + "url": "https://app.onelogin.com/trust/saml2/http-post/sso/{APP_ID}".format(APP_ID=ONELOGIN_APP_ID), + # SAML protocol binding to be used when returning the + # message. OneLogin Toolkit supports the HTTP-Redirect binding + # only for this endpoint. + "binding": "urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect" + }, + # SLO endpoint info of the IdP. + "singleLogoutService": { + # URL Location of the IdP where SLO Request will be sent. + "url": "https://app.onelogin.com/trust/saml2/http-redirect/slo/{APP_ID}".format(APP_ID=ONELOGIN_APP_ID), + # SAML protocol binding to be used when returning the + # message. OneLogin Toolkit supports the HTTP-Redirect binding + # only for this endpoint. + "binding": "urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect" + }, + # Public x509 certificate of the IdP + "x509cert": "" + } +} + +from datetime import timedelta +PERMANENT_SESSION_LIFETIME=timedelta(minutes=60) +SESSION_REFRESH_EACH_REQUEST=True +SESSION_COOKIE_SECURE=True +SESSION_COOKIE_HTTPONLY=True +PREFERRED_URL_SCHEME='https' + +REMEMBER_COOKIE_DURATION=timedelta(minutes=60) # Can make longer if you want remember_me to be useful. +REMEMBER_COOKIE_SECURE=True +REMEMBER_COOKIE_HTTPONLY=True From 560665fdbca9fab903396646674878ba97546da6 Mon Sep 17 00:00:00 2001 From: jnbnyc Date: Mon, 7 Nov 2016 08:53:12 -0500 Subject: [PATCH 27/37] More log changes for Docker --- docker/nginx/Dockerfile | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docker/nginx/Dockerfile b/docker/nginx/Dockerfile index 4487ccc61..d30d51e92 100644 --- a/docker/nginx/Dockerfile +++ b/docker/nginx/Dockerfile @@ -38,10 +38,8 @@ RUN cd /usr/local/src/security_monkey/dart &&\ /bin/cp -R /usr/local/src/security_monkey/dart/build/web/* /usr/local/src/security_monkey/security_monkey/static/ RUN /bin/mkdir -p /var/log/security_monkey/ /etc/nginx/ssl/ &&\ - # /usr/bin/touch /var/log/security_monkey/security_monkey.access.log &&\ - # /usr/bin/touch /var/log/security_monkey/security_monkey.error.log - ln -s /dev/stdout /var/log/security_monkey/security_monkey.access.log &&\ - ln -s /dev/stderr /var/log/security_monkey/security_monkey.error.log + /usr/bin/touch /var/log/security_monkey/security_monkey.access.log &&\ + /usr/bin/touch /var/log/security_monkey/security_monkey.error.log WORKDIR /etc/nginx EXPOSE 443 From d84c3486c9d2405eea34dd5c83793371502c63b1 Mon Sep 17 00:00:00 2001 From: jnbnyc Date: Mon, 7 Nov 2016 08:53:35 -0500 Subject: [PATCH 28/37] Improvement for local docker-compose development --- docker-compose.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index b447709e4..c336fff89 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,7 +21,7 @@ services: api: container_name: secmonkey-api - build: . + image: secmonkey:latest volumes_from: - init depends_on: @@ -31,7 +31,7 @@ services: scheduler: container_name: secmonkey-scheduler - build: . + image: secmonkey:latest volumes_from: - init depends_on: @@ -44,6 +44,7 @@ services: build: context: ./ dockerfile: ./docker/nginx/Dockerfile + image: secmonkey-nginx:latest working_dir: /etc/nginx volumes: - ./docker/nginx/server.crt:/etc/nginx/ssl/server.crt @@ -65,6 +66,7 @@ services: init: container_name: init build: . + image: secmonkey:latest working_dir: /usr/local/src/security_monkey volumes: - ./data/aws_accounts.json:/usr/local/src/security_monkey/data/aws_accounts.json From 7b1a1db3e140322a4ddc686a3c1c7dbf40ae9093 Mon Sep 17 00:00:00 2001 From: jnbnyc Date: Mon, 7 Nov 2016 08:58:41 -0500 Subject: [PATCH 29/37] Update securitymonkey.conf to latest from http://securitymonkey.readthedocs.io/en/latest/quickstart.html#securitymonkey-conf --- docker/nginx/securitymonkey.conf | 52 ++++++++------------------------ 1 file changed, 12 insertions(+), 40 deletions(-) diff --git a/docker/nginx/securitymonkey.conf b/docker/nginx/securitymonkey.conf index 582d34eae..0d008a77a 100644 --- a/docker/nginx/securitymonkey.conf +++ b/docker/nginx/securitymonkey.conf @@ -1,47 +1,19 @@ +add_header X-Content-Type-Options "nosniff"; +add_header X-XSS-Protection "1; mode=block"; +add_header X-Frame-Options "SAMEORIGIN"; +add_header Strict-Transport-Security "max-age=631138519"; +add_header Content-Security-Policy "default-src 'self'; font-src 'self' https://fonts.gstatic.com; script-src 'self' https://ajax.googleapis.com; style-src 'self' https://fonts.googleapis.com;"; + server { - listen 0.0.0.0:80 default; - listen 0.0.0.0:443 ssl default; - ssl_certificate /etc/nginx/ssl/server.crt; - ssl_certificate_key /etc/nginx/ssl/server.key; + listen 0.0.0.0:443 ssl; + ssl_certificate /etc/ssl/certs/server.crt; + ssl_certificate_key /etc/ssl/private/server.key; access_log /var/log/security_monkey/security_monkey.access.log; error_log /var/log/security_monkey/security_monkey.error.log; - location /register { - proxy_read_timeout 120; - proxy_pass http://smapi:5000; - proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504; - proxy_redirect off; - proxy_buffering off; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } - - location /logout { - proxy_read_timeout 120; - proxy_pass http://smapi:5000; - proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504; - proxy_redirect off; - proxy_buffering off; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } - - location /login { - proxy_read_timeout 120; - proxy_pass http://smapi:5000; - proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504; - proxy_redirect off; - proxy_buffering off; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } - - location /api { + location ~* ^/(reset|confirm|healthcheck|register|login|logout|api) { proxy_read_timeout 120; - proxy_pass http://smapi:5000; + proxy_pass http://127.0.0.1:5000; proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504; proxy_redirect off; proxy_buffering off; @@ -61,4 +33,4 @@ server { index ui.html; } -} \ No newline at end of file +} From ca0d0031d31f3dc674d0760448794e42edc994b3 Mon Sep 17 00:00:00 2001 From: jnbnyc Date: Mon, 7 Nov 2016 09:05:33 -0500 Subject: [PATCH 30/37] Edit ssl crt/key location in securitymonkey.conf --- docker/nginx/securitymonkey.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/nginx/securitymonkey.conf b/docker/nginx/securitymonkey.conf index 0d008a77a..562cc90e9 100644 --- a/docker/nginx/securitymonkey.conf +++ b/docker/nginx/securitymonkey.conf @@ -6,8 +6,8 @@ add_header Content-Security-Policy "default-src 'self'; font-src 'self' https:// server { listen 0.0.0.0:443 ssl; - ssl_certificate /etc/ssl/certs/server.crt; - ssl_certificate_key /etc/ssl/private/server.key; + ssl_certificate /etc/nginx/ssl/server.crt; + ssl_certificate_key /etc/nginx/ssl/server.key; access_log /var/log/security_monkey/security_monkey.access.log; error_log /var/log/security_monkey/security_monkey.error.log; From 452690dddf49f477190a0ce5e7fd8cade4db8ba6 Mon Sep 17 00:00:00 2001 From: jnbnyc Date: Mon, 7 Nov 2016 09:07:10 -0500 Subject: [PATCH 31/37] Make NGINX output logs to stdout/err for SecurityMonkey Docker --- docker/nginx/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/nginx/Dockerfile b/docker/nginx/Dockerfile index d30d51e92..2c700f5c9 100644 --- a/docker/nginx/Dockerfile +++ b/docker/nginx/Dockerfile @@ -38,8 +38,8 @@ RUN cd /usr/local/src/security_monkey/dart &&\ /bin/cp -R /usr/local/src/security_monkey/dart/build/web/* /usr/local/src/security_monkey/security_monkey/static/ RUN /bin/mkdir -p /var/log/security_monkey/ /etc/nginx/ssl/ &&\ - /usr/bin/touch /var/log/security_monkey/security_monkey.access.log &&\ - /usr/bin/touch /var/log/security_monkey/security_monkey.error.log + ln -s /dev/stdout /var/log/security_monkey/security_monkey.access.log &&\ + ln -s /dev/stderr /var/log/security_monkey/security_monkey.error.log WORKDIR /etc/nginx EXPOSE 443 From 8fba6d127dbdead399cf85f414c1e228374f69f2 Mon Sep 17 00:00:00 2001 From: jnbnyc Date: Mon, 7 Nov 2016 09:37:19 -0500 Subject: [PATCH 32/37] Hardcode smapi NGINX endpoint instead of 127.0.0.1 for Docker --- docker/nginx/securitymonkey.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/nginx/securitymonkey.conf b/docker/nginx/securitymonkey.conf index 562cc90e9..83de9c368 100644 --- a/docker/nginx/securitymonkey.conf +++ b/docker/nginx/securitymonkey.conf @@ -13,7 +13,7 @@ server { location ~* ^/(reset|confirm|healthcheck|register|login|logout|api) { proxy_read_timeout 120; - proxy_pass http://127.0.0.1:5000; + proxy_pass http://smapi:5000; proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504; proxy_redirect off; proxy_buffering off; From 485a920bdea00880f49ab9c7fad99e2d09f3e41d Mon Sep 17 00:00:00 2001 From: jnbnyc Date: Mon, 7 Nov 2016 10:13:10 -0500 Subject: [PATCH 33/37] Updated docker documentation --- docs/docker.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/docker.rst b/docs/docker.rst index b0fb6f2ea..6e0844fd2 100644 --- a/docs/docker.rst +++ b/docs/docker.rst @@ -4,7 +4,7 @@ Docker Instructions The docker-compose.yml file describes the SecurityMonkey environment. This is intended for local development with the intention of deploying SecurityMonkey containers with a Docker Orchestration tool like Kubernetes. The Dockerfile builds SecurityMonkey into a container with several different entrypoints. These are for the different responsibilities SecurityMonkey has. -Also, the Dockerfile.nginx file is used to build an NGINX container that will front the API, serve the static assets, and provide TLS. +Also, the docker/nginx/Dockerfile file is used to build an NGINX container that will front the API, serve the static assets, and provide TLS. Quick Start: ------------ @@ -21,7 +21,7 @@ Quick Start: $ docker-compose up -d postgres ``this will start the database container`` - $ docker-compose up -d scratch + $ docker-compose up -d init ``this will start a container in which you canuse to setup the database, create users, and other manual configurations, see the below section for more info`` $ docker-compose up @@ -30,17 +30,17 @@ Quick Start: Commands: --------- - $ docker-compose build ``[api | scheduler | nginx | scratch]`` + $ docker-compose build ``[api | scheduler | nginx | init]`` - $ docker-compose up -d ``[postgres | api | scheduler | nginx | scratch]`` + $ docker-compose up -d ``[postgres | api | scheduler | nginx | init]`` More Info: ---------- :: - $ docker-compose up -d scratch + $ docker-compose up -d init -The scratch container is where the SecurityMonkey code is available for you to run manual configurations such as:: +The init container is where the SecurityMonkey code is available for you to run manual configurations such as:: $ python manage.py create_user admin@example.com Admin @@ -48,4 +48,4 @@ and/or:: $ python manage.py add_account --number $account --name $name -r SecurityMonkey -The scratch container provides a sandbox and is useful for local development. It is not required otherwise. +The init container provides a sandbox and is useful for local development. It is not required otherwise. From a198de61cb6aef5a052e6de733d632c8114c49a1 Mon Sep 17 00:00:00 2001 From: jnbnyc Date: Mon, 7 Nov 2016 21:49:07 -0500 Subject: [PATCH 34/37] Simplify config-docker.py in the use of environment variables as overrides --- env-config/config-docker.py | 130 +++++++++--------------------------- 1 file changed, 30 insertions(+), 100 deletions(-) diff --git a/env-config/config-docker.py b/env-config/config-docker.py index 1bc51e520..9743e3762 100644 --- a/env-config/config-docker.py +++ b/env-config/config-docker.py @@ -16,89 +16,19 @@ import os -# '': '', -# Setting default settings -sm_config = { - 'fqdn': 'ec2-XX-XXX-XXX-XXX.compute-1.amazonaws.com', - 'postgres': { - 'database': 'secmonkey', - 'host': 'localhost', - 'password': 'securitymonkeypassword', - 'port': '5432', - 'user': 'postgres' - }, - 'api': { - 'port': 5000 - }, - 'email': { - 'security-team-email': [], - 'smtp': False, - 'ses-region': 'us-east-1', - 'default-sender': 'securitymonkey@example.com', - 'server': 'smtp.example.com', - 'username': 'username', - 'password': 'password' - } -} - -logcfg = { - 'console': { - 'level': 'DEBUG' - }, - 'apscheduler': { - 'level': 'INFO' - } -} - -if 'SECURITY_MONKEY_POSTGRES_HOST' in os.environ: - sm_config['postgres']['host'] = os.environ.get('SECURITY_MONKEY_POSTGRES_HOST') - -if 'SECURITY_MONKEY_POSTGRES_USER' in os.environ: - sm_config['postgres']['user'] = os.environ.get('SECURITY_MONKEY_POSTGRES_USER') - -if 'SECURITY_MONKEY_POSTGRES_PASSWORD' in os.environ: - sm_config['postgres']['password'] = os.environ.get('SECURITY_MONKEY_POSTGRES_PASSWORD') - -if 'SECURITY_MONKEY_POSTGRES_DATABASE' in os.environ: - sm_config['postgres']['database'] = os.environ.get('SECURITY_MONKEY_POSTGRES_DATABASE') - -if 'SECURITY_MONKEY_POSTGRES_PORT' in os.environ: - sm_config['postgres']['port'] = os.environ.get('SECURITY_MONKEY_POSTGRES_PORT') - -if 'SECURITY_MONKEY_API_PORT' in os.environ: - sm_config['api']['port'] = os.environ.get('SECURITY_MONKEY_API_PORT') - -if 'SECURITY_MONKEY_FQDN' in os.environ: - sm_config['fqdn'] = os.environ.get('SECURITY_MONKEY_FQDN') - -if 'SECURITY_MONKEY_SECURITY_TEAM_EMAIL' in os.environ: - sm_config['email']['security-team-email'] = os.environ.get('SECURITY_MONKEY_SECURITY_TEAM_EMAIL') - -if 'SECURITY_MONKEY_SMTP' in os.environ: - # Must change String from environment variable into Boolean - if os.environ.get('SECURITY_MONKEY_SMTP') == 'True': - sm_config['email']['smtp'] = True - -if 'SECURITY_MONKEY_SES_REGION' in os.environ: - sm_config['email']['ses-region'] = os.environ.get('SECURITY_MONKEY_SES_REGION') - -if 'SECURITY_MONKEY_EMAIL_DEFAULT_SENDER' in os.environ: - sm_config['email']['default-sender'] = os.environ.get('SECURITY_MONKEY_EMAIL_DEFAULT_SENDER') - -if 'SECURITY_MONKEY_EMAIL_SERVER' in os.environ: - sm_config['email']['server'] = os.environ.get('SECURITY_MONKEY_EMAIL_SERVER') - -if 'SECURITY_MONKEY_EMAIL_USERNAME' in os.environ: - sm_config['email']['username'] = os.environ.get('SECURITY_MONKEY_EMAIL_USERNAME') - -if 'SECURITY_MONKEY_EMAIL_PASSWORD' in os.environ: - sm_config['email']['password'] = os.environ.get('SECURITY_MONKEY_EMAIL_PASSWORD') - -if 'SM_CONSOLE_LOG_LEVEL' in os.environ: - logcfg['console']['level'] = os.environ.get('SM_CONSOLE_LOG_LEVEL') -if 'SM_APSCHEDULER_LOG_LEVEL' in os.environ: - logcfg['apscheduler']['level'] = os.environ.get('SM_APPSCHEDULER_LOG_LEVEL') +def env_to_bool(input): + """ + Must change String from environment variable into Boolean + defaults to True + """ + if isinstance(input, str): + if input == 'False': + return False + else: + return True + else: + return input LOG_CFG = { @@ -130,32 +60,32 @@ 'loggers': { 'security_monkey': { 'handlers': ['console'], - 'level': logcfg['console']['level'] + 'level': os.getenv('SM_CONSOLE_LOG_LEVEL', 'DEBUG') }, 'apscheduler': { 'handlers': ['console'], - 'level': logcfg['apscheduler']['level'] + 'level': os.getenv('SM_APPSCHEDULER_LOG_LEVEL', 'INFO') } } } -SQLALCHEMY_DATABASE_URI = 'postgresql://%s:%s@%s:%s/%s' % ( - sm_config['postgres']['user'], - sm_config['postgres']['password'], - sm_config['postgres']['host'], - sm_config['postgres']['port'], - sm_config['postgres']['database'] +SQLALCHEMY_DATABASE_URI = 'postgresql://%s:%s@%s:%d/%s' % ( + os.getenv('SECURITY_MONKEY_POSTGRES_USER', 'postgres'), + os.getenv('SECURITY_MONKEY_POSTGRES_PASSWORD', 'securitymonkeypassword'), + os.getenv('SECURITY_MONKEY_POSTGRES_HOST', 'localhost'), + os.getenv('SECURITY_MONKEY_POSTGRES_PORT', 5432), + os.getenv('SECURITY_MONKEY_POSTGRES_DATABASE', 'secmonkey') ) -# print sm_config['postgres'] +# print postgres # print SQLALCHEMY_DATABASE_URI SQLALCHEMY_POOL_SIZE = 50 SQLALCHEMY_MAX_OVERFLOW = 15 ENVIRONMENT = 'ec2' USE_ROUTE53 = False -FQDN = sm_config['fqdn'] -API_PORT = sm_config['api']['port'] +FQDN = os.getenv('SECURITY_MONKEY_FQDN', 'ec2-XX-XXX-XXX-XXX.compute-1.amazonaws.com') +API_PORT = 5000 WEB_PORT = '443' WEB_PATH = '/static/ui.html' FRONTED_BY_NGINX = True @@ -164,7 +94,7 @@ SECRET_KEY = '' -MAIL_DEFAULT_SENDER = sm_config['email']['default-sender'] +MAIL_DEFAULT_SENDER = os.getenv('SECURITY_MONKEY_EMAIL_DEFAULT_SENDER', 'securitymonkey@example.com') SECURITY_REGISTERABLE = True SECURITY_CONFIRMABLE = False SECURITY_RECOVERABLE = False @@ -179,16 +109,16 @@ SECURITY_POST_CHANGE_VIEW = BASE_URL # This address gets all change notifications (i.e. 'securityteam@example.com') -SECURITY_TEAM_EMAIL = sm_config['email']['security-team-email'] +SECURITY_TEAM_EMAIL = os.getenv('SECURITY_MONKEY_SECURITY_TEAM_EMAIL', []) # These are only required if using SMTP instead of SES -EMAILS_USE_SMTP = sm_config['email']['smtp'] # Otherwise, Use SES -SES_REGION = sm_config['email']['ses-region'] -MAIL_SERVER = sm_config['email']['server'] +EMAILS_USE_SMTP = env_to_bool(os.getenv('SECURITY_MONKEY_SMTP', True)) # Otherwise, Use SES +SES_REGION = os.getenv('SECURITY_MONKEY_SES_REGION', 'us-east-1') +MAIL_SERVER = os.getenv('SECURITY_MONKEY_EMAIL_SERVER', 'smtp.example.com') MAIL_PORT = 465 MAIL_USE_SSL = True -MAIL_USERNAME = sm_config['email']['username'] -MAIL_PASSWORD = sm_config['email']['password'] +MAIL_USERNAME = os.getenv('SECURITY_MONKEY_EMAIL_USERNAME', 'username') +MAIL_PASSWORD = os.getenv('SECURITY_MONKEY_EMAIL_PASSWORD', 'password') WTF_CSRF_ENABLED = True WTF_CSRF_SSL_STRICT = True # Checks Referer Header. Set to False for API access. From 4cb743d852aa4cb80dbda4d623d02acb6911472d Mon Sep 17 00:00:00 2001 From: jnbnyc Date: Wed, 9 Nov 2016 16:32:12 -0500 Subject: [PATCH 35/37] Add support for local insecure development - Enables 80 in NGINX - Toggle to disable CSRF in settings - Toggle to disable ssl in NGINX if certs are not provided --- docker-compose.yml | 2 ++ docker/nginx/Dockerfile | 13 ++++++------- docker/nginx/securitymonkey.conf | 1 + docker/nginx/start-nginx.sh | 2 ++ env-config/config-docker.py | 5 +++-- 5 files changed, 14 insertions(+), 9 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index c336fff89..57d61f52a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -49,6 +49,8 @@ services: volumes: - ./docker/nginx/server.crt:/etc/nginx/ssl/server.crt - ./docker/nginx/server.key:/etc/nginx/ssl/server.key + - ./docker/nginx/securitymonkey.conf:/etc/nginx/conf.d/securitymonkey.conf + - ./docker/nginx/start-nginx.sh:/usr/local/src/security_monkey/docker/nginx/start-nginx.sh depends_on: - api ports: diff --git a/docker/nginx/Dockerfile b/docker/nginx/Dockerfile index 2c700f5c9..d888efc56 100644 --- a/docker/nginx/Dockerfile +++ b/docker/nginx/Dockerfile @@ -18,6 +18,10 @@ MAINTAINER Netflix Open Source Development ENV SECURITY_MONKEY_VERSION=v0.7.0 RUN apt-get update &&\ apt-get install -y curl git sudo apt-transport-https &&\ + curl https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - &&\ + curl https://storage.googleapis.com/download.dartlang.org/linux/debian/dart_stable.list > /etc/apt/sources.list.d/dart_stable.list && \ + apt-get update &&\ + apt-get install -y -q dart &&\ rm -rf /var/lib/apt/lists/* RUN cd /usr/local/src &&\ @@ -25,19 +29,14 @@ RUN cd /usr/local/src &&\ mkdir -p security_monkey ADD . /usr/local/src/security_monkey -RUN curl https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - &&\ - curl https://storage.googleapis.com/download.dartlang.org/linux/debian/dart_stable.list > /etc/apt/sources.list.d/dart_stable.list && \ - apt-get update &&\ - apt-get install -y -q dart &&\ - rm -rf /var/lib/apt/lists/* - RUN cd /usr/local/src/security_monkey/dart &&\ /usr/lib/dart/bin/pub get &&\ /usr/lib/dart/bin/pub build &&\ /bin/mkdir -p /usr/local/src/security_monkey/security_monkey/static/ &&\ /bin/cp -R /usr/local/src/security_monkey/dart/build/web/* /usr/local/src/security_monkey/security_monkey/static/ -RUN /bin/mkdir -p /var/log/security_monkey/ /etc/nginx/ssl/ &&\ +RUN /bin/rm /etc/nginx/conf.d/default.conf &&\ + /bin/mkdir -p /var/log/security_monkey/ /etc/nginx/ssl/ &&\ ln -s /dev/stdout /var/log/security_monkey/security_monkey.access.log &&\ ln -s /dev/stderr /var/log/security_monkey/security_monkey.error.log diff --git a/docker/nginx/securitymonkey.conf b/docker/nginx/securitymonkey.conf index 83de9c368..75489d142 100644 --- a/docker/nginx/securitymonkey.conf +++ b/docker/nginx/securitymonkey.conf @@ -5,6 +5,7 @@ add_header Strict-Transport-Security "max-age=631138519"; add_header Content-Security-Policy "default-src 'self'; font-src 'self' https://fonts.gstatic.com; script-src 'self' https://ajax.googleapis.com; style-src 'self' https://fonts.googleapis.com;"; server { + listen 0.0.0.0:80; listen 0.0.0.0:443 ssl; ssl_certificate /etc/nginx/ssl/server.crt; ssl_certificate_key /etc/nginx/ssl/server.key; diff --git a/docker/nginx/start-nginx.sh b/docker/nginx/start-nginx.sh index 3c2747336..efeed4d59 100755 --- a/docker/nginx/start-nginx.sh +++ b/docker/nginx/start-nginx.sh @@ -7,6 +7,8 @@ if [ ! -f "$SECURITY_MONKEY_SSL_CERT" ] || [ ! -f "$SECURITY_MONKEY_SSL_KEY" ]; # Fail if SSL is unavailable echo "$(date) Error: Missing files required for SSL" # exit 1 + sed -i.bak 's@.*ssl@# &@' /etc/nginx/conf.d/securitymonkey.conf &&\ + echo "$(date) Warn: Disabled ssl in securitymonkey.conf" fi exec nginx diff --git a/env-config/config-docker.py b/env-config/config-docker.py index 9743e3762..fc38b840d 100644 --- a/env-config/config-docker.py +++ b/env-config/config-docker.py @@ -120,8 +120,9 @@ def env_to_bool(input): MAIL_USERNAME = os.getenv('SECURITY_MONKEY_EMAIL_USERNAME', 'username') MAIL_PASSWORD = os.getenv('SECURITY_MONKEY_EMAIL_PASSWORD', 'password') -WTF_CSRF_ENABLED = True -WTF_CSRF_SSL_STRICT = True # Checks Referer Header. Set to False for API access. +WTF_CSRF_ENABLED = env_to_bool(os.getenv('SM_WTF_CSRF_ENABLED', True)) +# Checks Referer Header. Set to False for API access. +WTF_CSRF_SSL_STRICT = env_to_bool(os.getenv('SM_WTF_CSRF_SSL_STRICT', True)) WTF_CSRF_METHODS = ['DELETE', 'POST', 'PUT', 'PATCH'] # "NONE", "SUMMARY", or "FULL" From 00021e4fbe74caf31b4392811ed101a53cf4d7bd Mon Sep 17 00:00:00 2001 From: jnbnyc Date: Wed, 9 Nov 2016 17:37:29 -0500 Subject: [PATCH 36/37] Restore config-deploy.py from upstream --- env-config/config-deploy.py | 94 ++++++++++++++++++++++++++++++++++++- 1 file changed, 93 insertions(+), 1 deletion(-) diff --git a/env-config/config-deploy.py b/env-config/config-deploy.py index f5c5fb0ba..d72448114 100644 --- a/env-config/config-deploy.py +++ b/env-config/config-deploy.py @@ -107,7 +107,7 @@ MAX_THREADS = 30 # SSO SETTINGS: -ACTIVE_PROVIDERS = [] # "ping" or "google" +ACTIVE_PROVIDERS = [] # "ping", "google" or "onelogin" PING_NAME = '' # Use to override the Ping name in the UI. PING_REDIRECT_URI = "{BASE}api/1/auth/ping".format(BASE=BASE_URL) @@ -122,6 +122,98 @@ GOOGLE_AUTH_ENDPOINT = '' GOOGLE_SECRET = '' +ONELOGIN_APP_ID = '' # OneLogin App ID provider by your administrator +ONELOGIN_EMAIL_FIELD = 'User.email' # SAML attribute used to provide email address +ONELOGIN_DEFAULT_ROLE = 'View' # Default RBAC when user doesn't already exist +ONELOGIN_HTTPS = True # If using HTTPS strict mode will check the requests are HTTPS +ONELOGIN_SETTINGS = { + # If strict is True, then the Python Toolkit will reject unsigned + # or unencrypted messages if it expects them to be signed or encrypted. + # Also it will reject the messages if the SAML standard is not strictly + # followed. Destination, NameId, Conditions ... are validated too. + "strict": True, + + # Enable debug mode (outputs errors). + "debug": True, + + # Service Provider Data that we are deploying. + "sp": { + # Identifier of the SP entity (must be a URI) + "entityId": "{BASE}metadata/".format(BASE=BASE_URL), + # Specifies info about where and how the message MUST be + # returned to the requester, in this case our SP. + "assertionConsumerService": { + # URL Location where the from the IdP will be returned + "url": "{BASE}api/1/auth/onelogin?acs".format(BASE=BASE_URL), + # SAML protocol binding to be used when returning the + # message. OneLogin Toolkit supports this endpoint for the + # HTTP-POST binding only. + "binding": "urn:oasis:names:tc:SAML:2.0:bindings:HTTP-POST" + }, + # If you need to specify requested attributes, set a + # attributeConsumingService. nameFormat, attributeValue and + # friendlyName can be omitted + #"attributeConsumingService": { + # "ServiceName": "SP test", + # "serviceDescription": "Test Service", + # "requestedAttributes": [ + # { + # "name": "", + # "isRequired": False, + # "nameFormat": "", + # "friendlyName": "", + # "attributeValue": "" + # } + # ] + #}, + # Specifies info about where and how the message MUST be + # returned to the requester, in this case our SP. + "singleLogoutService": { + # URL Location where the from the IdP will be returned + "url": "{BASE}api/1/auth/onelogin?sls".format(BASE=BASE_URL), + # SAML protocol binding to be used when returning the + # message. OneLogin Toolkit supports the HTTP-Redirect binding + # only for this endpoint. + "binding": "urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect" + }, + # Specifies the constraints on the name identifier to be used to + # represent the requested subject. + # Take a look on src/onelogin/saml2/constants.py to see the NameIdFormat that are supported. + "NameIDFormat": "urn:oasis:names:tc:SAML:1.1:nameid-format:unspecified", + # Usually x509cert and privateKey of the SP are provided by files placed at + # the certs folder. But we can also provide them with the following parameters + "x509cert": "", + "privateKey": "" + }, + + # Identity Provider Data that we want connected with our SP. + "idp": { + # Identifier of the IdP entity (must be a URI) + "entityId": "https://app.onelogin.com/saml/metadata/{APP_ID}".format(APP_ID=ONELOGIN_APP_ID), + # SSO endpoint info of the IdP. (Authentication Request protocol) + "singleSignOnService": { + # URL Target of the IdP where the Authentication Request Message + # will be sent. + "url": "https://app.onelogin.com/trust/saml2/http-post/sso/{APP_ID}".format(APP_ID=ONELOGIN_APP_ID), + # SAML protocol binding to be used when returning the + # message. OneLogin Toolkit supports the HTTP-Redirect binding + # only for this endpoint. + "binding": "urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect" + }, + # SLO endpoint info of the IdP. + "singleLogoutService": { + # URL Location of the IdP where SLO Request will be sent. + "url": "https://app.onelogin.com/trust/saml2/http-redirect/slo/{APP_ID}".format(APP_ID=ONELOGIN_APP_ID), + # SAML protocol binding to be used when returning the + # message. OneLogin Toolkit supports the HTTP-Redirect binding + # only for this endpoint. + "binding": "urn:oasis:names:tc:SAML:2.0:bindings:HTTP-Redirect" + }, + # Public x509 certificate of the IdP + "x509cert": "" + } +} + from datetime import timedelta PERMANENT_SESSION_LIFETIME=timedelta(minutes=60) SESSION_REFRESH_EACH_REQUEST=True From 0ecc6be496dae72d56ef8d20f5a4bc2892e15b65 Mon Sep 17 00:00:00 2001 From: jnbnyc Date: Wed, 9 Nov 2016 17:41:16 -0500 Subject: [PATCH 37/37] Set API_PORT back to a String --- env-config/config-docker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/env-config/config-docker.py b/env-config/config-docker.py index fc38b840d..20127e133 100644 --- a/env-config/config-docker.py +++ b/env-config/config-docker.py @@ -85,7 +85,7 @@ def env_to_bool(input): ENVIRONMENT = 'ec2' USE_ROUTE53 = False FQDN = os.getenv('SECURITY_MONKEY_FQDN', 'ec2-XX-XXX-XXX-XXX.compute-1.amazonaws.com') -API_PORT = 5000 +API_PORT = '5000' WEB_PORT = '443' WEB_PATH = '/static/ui.html' FRONTED_BY_NGINX = True