From 26f4e92bccd78baf95e1ad8ea9fd43d9925bb460 Mon Sep 17 00:00:00 2001 From: tosih Date: Fri, 7 Jul 2017 10:04:21 -0700 Subject: [PATCH 1/7] Add mongo-seed-db k8 job. --- ansible/group_vars/alpha-mongo-seed-db.yml | 122 +++++++++++++++++++++ ansible/mongo-seed-db.yml | 6 + 2 files changed, 128 insertions(+) create mode 100644 ansible/group_vars/alpha-mongo-seed-db.yml create mode 100644 ansible/mongo-seed-db.yml diff --git a/ansible/group_vars/alpha-mongo-seed-db.yml b/ansible/group_vars/alpha-mongo-seed-db.yml new file mode 100644 index 00000000..191c6090 --- /dev/null +++ b/ansible/group_vars/alpha-mongo-seed-db.yml @@ -0,0 +1,122 @@ +name: mongo-seed-db + +container_image: quay.io/runnable/api + +container_run_args: > + npm run migrate-up + +container_envs: + - name: DOCKER_IMAGE_BUILDER_CACHE + value: "/home/anandkumarpatel/cache" + - name: VALID_REDIR_TLDS + value: "{{ domain }},runnablecloud.com" + - name: FULL_API_DOMAIN + value: "{{ api_url }}" + - name: FULL_FRONTEND_DOMAIN + value: "https://{{ domain }}" + - name: AWS_ACCESS_KEY_ID + value: "{{ api_aws_access_key_id }}" + - name: AWS_SECRET_ACCESS_KEY + value: "{{ api_aws_secret_access_key }}" + - name: BIG_POPPA_HOST + value: "http://{{ big_poppa_host_address }}:{{ big_poppa_port }}" + - name: CLIO_HTTP_URL + value: "http://{{ clio_host_address }}:{{ clio_port }}/" + - name: COOKIE_DOMAIN + value: "{{ domain }}" + - name: CREAM_HOST + value: "http://{{ cream_host_address }}:{{ cream_port }}" + - name: DATADOG_HOST + valueFrom: "spec.nodeName" + - name: DATADOG_PORT + value: "{{ datadog_port }}" + - name: DOCKER_BUILD_LINE_TIMEOUT_MS + value: "3600000" + - name: DOMAIN + value: "{{ domain }}" + - name: KEYMAKER_HTTP_URL + value: "http://{{ keymaker_host_address }}:{{ keymaker_port }}/" + - name: GITHUB_CALLBACK_URL + value: "{{ api_url }}/auth/github/callback" + - name: GITHUB_CLIENT_ID + value: "{{ api_github_client_id }}" + - name: GITHUB_CLIENT_SECRET + value: "{{ api_github_client_secret }}" + - name: GITHUB_DEPLOY_KEYS_BUCKET + value: "{{ api_github_deploy_keys_bucket }}" + - name: GITHUB_HOOK_URL + value: "{{ api_url }}/actions/github" + - name: GITHUB_UPGRADE_REDIRECT_URL + value: "https://app.{{ domain }}/githubAuthUpgrade/" + - name: GITHUB_WEBHOOK_URL + value: "https://{{ drake_hostname }}/github" + - name: GITHUB_VARNISH_HOST + value: "{{ github_varnish_host }}" + - name: GITHUB_VARNISH_PORT + value: "{{ github_varnish_port }}" + - name: GITHUB_PROTOCOL + value: "http" + - name: HELLO_RUNNABLE_GITHUB_ID + value: "{{ hello_runnable_github_id }}" + - name: HELLO_RUNNABLE_GITHUB_TOKEN + value: "{{ api_hello_runnable_github_token }}" + - name: KRAIN_PORT + value: "{{ krain_port }}" + - name: MIXPANEL_APP_ID + value: "{{ api_mixpanel_app_id | default('ansible_undefined') }}" + - name: MONGO_AUTH + value: "{{ api_mongo_auth }}" + - name: MONGO_DB + value: "{{ api_mongo_database }}" + - name: MONGO_HOSTS + value: "{{ mongo_hosts }}" + - name: MONGO_REPLSET_NAME + value: "{{ api_mongo_replset_name | default('ansible_undefined') }}" + - name: MONGO + value: "mongodb://{{ api_mongo_auth }}@{{ mongo_hosts }}/{{ api_mongo_database }}" + - name: NAVI_HOST + value: "http://{{ navi_host_address }}:{{ navi_http_port }}" + - name: NODE_ENV + value: "{{ node_env }}" + - name: NUM_WORKERS + value: "1" + - name: OPTIMUS_HOST + value: "http://{{ optimus_hostname }}" + - name: RABBITMQ_HOSTNAME + value: "{{ rabbit_host_address }}" + - name: RABBITMQ_PASSWORD + value: "{{ rabbit_password }}" + - name: RABBITMQ_PORT + value: "{{ rabbit_port }}" + - name: RABBITMQ_USERNAME + value: "{{ rabbit_username }}" + - name: REDIS_IPADDRESS + value: "{{ redis_host_address }}" + - name: REDIS_PORT + value: "{{ redis_port }}" + - name: S3_CONTEXT_RESOURCE_BUCKET + value: "{{ api_s3_context_bucket }}" + - name: S3_LOG_BUCKET + value: "{{ api_s3_log_bucket }}" + - name: SENDGRID_KEY + value: "{{ sendgrid_key }}" + - name: SWARM_HOST + value: "http://{{ swarm_host_address }}:{{ swarm_master_port }}" + - name: USER_CONTENT_DOMAIN + value: "{{ user_content_domain }}" + - name: AWS_ALIAS_HOST + value: "{{ api_aws_alias_host }}" + - name: NEW_RELIC_APP_NAME + value: "{{ api_new_relic_app_name | default('ansible_undefined') }}" + - name: NEW_RELIC_LICENSE_KEY + value: "{{ new_relic_license_key | default('ansible_undefined') }}" + - name: NEW_RELIC_LOG_LEVEL + value: "fatal" + - name: NEW_RELIC_NO_CONFIG_FILE + value: "true" + - name: INTERCOM_APP_ID + value: "{{ api_intercom_app_id | default('ansible_undefined') }}" + - name: INTERCOM_API_KEY + value: "{{ api_intercom_api_key | default('ansible_undefined') }}" + - name: USER_VAULT_ENDPOINT + value: "http://{{ user_vault_load_balancer }}:{{ user_vault_port }}" \ No newline at end of file diff --git a/ansible/mongo-seed-db.yml b/ansible/mongo-seed-db.yml new file mode 100644 index 00000000..426589c1 --- /dev/null +++ b/ansible/mongo-seed-db.yml @@ -0,0 +1,6 @@ +--- +- hosts: mongo + vars_files: + - group_vars/alpha-mongo-seed-db.yml + roles: + - role: k8-job From 0799267c04bd0a4528693aa8a7838cdd9ea172fb Mon Sep 17 00:00:00 2001 From: tosih Date: Fri, 7 Jul 2017 10:06:32 -0700 Subject: [PATCH 2/7] Clean up yml. --- ansible/group_vars/alpha-mongo-seed-db.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ansible/group_vars/alpha-mongo-seed-db.yml b/ansible/group_vars/alpha-mongo-seed-db.yml index 191c6090..16a8b229 100644 --- a/ansible/group_vars/alpha-mongo-seed-db.yml +++ b/ansible/group_vars/alpha-mongo-seed-db.yml @@ -2,8 +2,7 @@ name: mongo-seed-db container_image: quay.io/runnable/api -container_run_args: > - npm run migrate-up +container_run_args: npm run migrate-up container_envs: - name: DOCKER_IMAGE_BUILDER_CACHE From c2cb8e7472bce5b1f4b74341c87e06bc73211ac7 Mon Sep 17 00:00:00 2001 From: tosih Date: Fri, 7 Jul 2017 10:24:42 -0700 Subject: [PATCH 3/7] Fixes for Jorge. --- ansible/group_vars/alpha-mongo-seed-db.yml | 120 ++------------------- ansible/mongo-seed-db.yml | 6 -- 2 files changed, 6 insertions(+), 120 deletions(-) delete mode 100644 ansible/mongo-seed-db.yml diff --git a/ansible/group_vars/alpha-mongo-seed-db.yml b/ansible/group_vars/alpha-mongo-seed-db.yml index 16a8b229..767afef4 100644 --- a/ansible/group_vars/alpha-mongo-seed-db.yml +++ b/ansible/group_vars/alpha-mongo-seed-db.yml @@ -4,118 +4,10 @@ container_image: quay.io/runnable/api container_run_args: npm run migrate-up -container_envs: - - name: DOCKER_IMAGE_BUILDER_CACHE - value: "/home/anandkumarpatel/cache" - - name: VALID_REDIR_TLDS - value: "{{ domain }},runnablecloud.com" - - name: FULL_API_DOMAIN - value: "{{ api_url }}" - - name: FULL_FRONTEND_DOMAIN - value: "https://{{ domain }}" - - name: AWS_ACCESS_KEY_ID - value: "{{ api_aws_access_key_id }}" - - name: AWS_SECRET_ACCESS_KEY - value: "{{ api_aws_secret_access_key }}" - - name: BIG_POPPA_HOST - value: "http://{{ big_poppa_host_address }}:{{ big_poppa_port }}" - - name: CLIO_HTTP_URL - value: "http://{{ clio_host_address }}:{{ clio_port }}/" - - name: COOKIE_DOMAIN - value: "{{ domain }}" - - name: CREAM_HOST - value: "http://{{ cream_host_address }}:{{ cream_port }}" - - name: DATADOG_HOST - valueFrom: "spec.nodeName" - - name: DATADOG_PORT - value: "{{ datadog_port }}" - - name: DOCKER_BUILD_LINE_TIMEOUT_MS - value: "3600000" - - name: DOMAIN - value: "{{ domain }}" - - name: KEYMAKER_HTTP_URL - value: "http://{{ keymaker_host_address }}:{{ keymaker_port }}/" - - name: GITHUB_CALLBACK_URL - value: "{{ api_url }}/auth/github/callback" - - name: GITHUB_CLIENT_ID - value: "{{ api_github_client_id }}" - - name: GITHUB_CLIENT_SECRET - value: "{{ api_github_client_secret }}" - - name: GITHUB_DEPLOY_KEYS_BUCKET - value: "{{ api_github_deploy_keys_bucket }}" - - name: GITHUB_HOOK_URL - value: "{{ api_url }}/actions/github" - - name: GITHUB_UPGRADE_REDIRECT_URL - value: "https://app.{{ domain }}/githubAuthUpgrade/" - - name: GITHUB_WEBHOOK_URL - value: "https://{{ drake_hostname }}/github" - - name: GITHUB_VARNISH_HOST - value: "{{ github_varnish_host }}" - - name: GITHUB_VARNISH_PORT - value: "{{ github_varnish_port }}" - - name: GITHUB_PROTOCOL - value: "http" - - name: HELLO_RUNNABLE_GITHUB_ID - value: "{{ hello_runnable_github_id }}" - - name: HELLO_RUNNABLE_GITHUB_TOKEN - value: "{{ api_hello_runnable_github_token }}" - - name: KRAIN_PORT - value: "{{ krain_port }}" - - name: MIXPANEL_APP_ID - value: "{{ api_mixpanel_app_id | default('ansible_undefined') }}" - - name: MONGO_AUTH - value: "{{ api_mongo_auth }}" - - name: MONGO_DB - value: "{{ api_mongo_database }}" - - name: MONGO_HOSTS - value: "{{ mongo_hosts }}" - - name: MONGO_REPLSET_NAME - value: "{{ api_mongo_replset_name | default('ansible_undefined') }}" - - name: MONGO - value: "mongodb://{{ api_mongo_auth }}@{{ mongo_hosts }}/{{ api_mongo_database }}" - - name: NAVI_HOST - value: "http://{{ navi_host_address }}:{{ navi_http_port }}" - - name: NODE_ENV - value: "{{ node_env }}" - - name: NUM_WORKERS - value: "1" - - name: OPTIMUS_HOST - value: "http://{{ optimus_hostname }}" - - name: RABBITMQ_HOSTNAME - value: "{{ rabbit_host_address }}" - - name: RABBITMQ_PASSWORD - value: "{{ rabbit_password }}" - - name: RABBITMQ_PORT - value: "{{ rabbit_port }}" - - name: RABBITMQ_USERNAME - value: "{{ rabbit_username }}" - - name: REDIS_IPADDRESS - value: "{{ redis_host_address }}" - - name: REDIS_PORT - value: "{{ redis_port }}" - - name: S3_CONTEXT_RESOURCE_BUCKET - value: "{{ api_s3_context_bucket }}" - - name: S3_LOG_BUCKET - value: "{{ api_s3_log_bucket }}" - - name: SENDGRID_KEY - value: "{{ sendgrid_key }}" - - name: SWARM_HOST - value: "http://{{ swarm_host_address }}:{{ swarm_master_port }}" - - name: USER_CONTENT_DOMAIN - value: "{{ user_content_domain }}" - - name: AWS_ALIAS_HOST - value: "{{ api_aws_alias_host }}" - - name: NEW_RELIC_APP_NAME - value: "{{ api_new_relic_app_name | default('ansible_undefined') }}" - - name: NEW_RELIC_LICENSE_KEY - value: "{{ new_relic_license_key | default('ansible_undefined') }}" - - name: NEW_RELIC_LOG_LEVEL - value: "fatal" - - name: NEW_RELIC_NO_CONFIG_FILE +envs: + - name: IS_QUEUE_WORKER value: "true" - - name: INTERCOM_APP_ID - value: "{{ api_intercom_app_id | default('ansible_undefined') }}" - - name: INTERCOM_API_KEY - value: "{{ api_intercom_api_key | default('ansible_undefined') }}" - - name: USER_VAULT_ENDPOINT - value: "http://{{ user_vault_load_balancer }}:{{ user_vault_port }}" \ No newline at end of file + - name: ROLLBAR_KEY + value: "{{ api_workers_rollbar_key | default('ansible_undefined') }}" + +container_envs: "{{ api_base_container_envs + envs }}" \ No newline at end of file diff --git a/ansible/mongo-seed-db.yml b/ansible/mongo-seed-db.yml deleted file mode 100644 index 426589c1..00000000 --- a/ansible/mongo-seed-db.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -- hosts: mongo - vars_files: - - group_vars/alpha-mongo-seed-db.yml - roles: - - role: k8-job From 25025e2c896382ed41dfc42bbcb9a46387d897a0 Mon Sep 17 00:00:00 2001 From: tosih Date: Fri, 7 Jul 2017 10:28:45 -0700 Subject: [PATCH 4/7] Fixes for Jorge II --- ansible/mongo-seed-db.yml | 6 ++++++ ansible/single-host-part-2.yml | 1 + 2 files changed, 7 insertions(+) create mode 100644 ansible/mongo-seed-db.yml diff --git a/ansible/mongo-seed-db.yml b/ansible/mongo-seed-db.yml new file mode 100644 index 00000000..41f6cc82 --- /dev/null +++ b/ansible/mongo-seed-db.yml @@ -0,0 +1,6 @@ +--- +- hosts: api + vars_files: + - group_vars/alpha-mongo-seed-db.yml + roles: + - role: k8-job diff --git a/ansible/single-host-part-2.yml b/ansible/single-host-part-2.yml index b8dba6d0..543eff23 100644 --- a/ansible/single-host-part-2.yml +++ b/ansible/single-host-part-2.yml @@ -3,4 +3,5 @@ - include: rabbitmq.yml - include: redis.yml - include: mongo-create-users.yml +- include: mongo-seed-db.yml - include: rabbit-create-exchanges.yml From 04cdb756d407e97abcae91b496d1c49b7ea86653 Mon Sep 17 00:00:00 2001 From: tosih Date: Fri, 7 Jul 2017 13:34:31 -0700 Subject: [PATCH 5/7] Add api_base env vars. --- ansible/mongo-seed-db.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ansible/mongo-seed-db.yml b/ansible/mongo-seed-db.yml index 41f6cc82..ae15d73b 100644 --- a/ansible/mongo-seed-db.yml +++ b/ansible/mongo-seed-db.yml @@ -1,6 +1,7 @@ --- - hosts: api vars_files: + - group_vars/alpha-api-base.yml - group_vars/alpha-mongo-seed-db.yml roles: - role: k8-job From fd7ce20bba82e3178283e976c1ccd3c1a1ef745f Mon Sep 17 00:00:00 2001 From: tosih Date: Fri, 7 Jul 2017 14:30:54 -0700 Subject: [PATCH 6/7] Go back to old envs. --- ansible/group_vars/alpha-mongo-seed-db.yml | 120 +++++++++++++++++++-- ansible/mongo-seed-db.yml | 1 - 2 files changed, 114 insertions(+), 7 deletions(-) diff --git a/ansible/group_vars/alpha-mongo-seed-db.yml b/ansible/group_vars/alpha-mongo-seed-db.yml index 767afef4..f01da5b8 100644 --- a/ansible/group_vars/alpha-mongo-seed-db.yml +++ b/ansible/group_vars/alpha-mongo-seed-db.yml @@ -4,10 +4,118 @@ container_image: quay.io/runnable/api container_run_args: npm run migrate-up -envs: - - name: IS_QUEUE_WORKER +container_envs: + - name: DOCKER_IMAGE_BUILDER_CACHE + value: "/home/anandkumarpatel/cache" + - name: VALID_REDIR_TLDS + value: "{{ domain }},runnablecloud.com" + - name: FULL_API_DOMAIN + value: "{{ api_url }}" + - name: FULL_FRONTEND_DOMAIN + value: "https://{{ domain }}" + - name: AWS_ACCESS_KEY_ID + value: "{{ api_aws_access_key_id }}" + - name: AWS_SECRET_ACCESS_KEY + value: "{{ api_aws_secret_access_key }}" + - name: BIG_POPPA_HOST + value: "http://{{ big_poppa_host_address }}:{{ big_poppa_port }}" + - name: CLIO_HTTP_URL + value: "http://{{ clio_host_address }}:{{ clio_port }}/" + - name: COOKIE_DOMAIN + value: "{{ domain }}" + - name: CREAM_HOST + value: "http://{{ cream_host_address }}:{{ cream_port }}" + - name: DATADOG_HOST + valueFrom: "spec.nodeName" + - name: DATADOG_PORT + value: "{{ datadog_port }}" + - name: DOCKER_BUILD_LINE_TIMEOUT_MS + value: "3600000" + - name: DOMAIN + value: "{{ domain }}" + - name: KEYMAKER_HTTP_URL + value: "http://{{ keymaker_host_address }}:{{ keymaker_port }}/" + - name: GITHUB_CALLBACK_URL + value: "{{ api_url }}/auth/github/callback" + - name: GITHUB_CLIENT_ID + value: "{{ api_github_client_id }}" + - name: GITHUB_CLIENT_SECRET + value: "{{ api_github_client_secret }}" + - name: GITHUB_DEPLOY_KEYS_BUCKET + value: "{{ api_github_deploy_keys_bucket }}" + - name: GITHUB_HOOK_URL + value: "{{ api_url }}/actions/github" + - name: GITHUB_UPGRADE_REDIRECT_URL + value: "https://app.{{ domain }}/githubAuthUpgrade/" + - name: GITHUB_WEBHOOK_URL + value: "https://{{ drake_hostname }}/github" + - name: GITHUB_VARNISH_HOST + value: "{{ github_varnish_host }}" + - name: GITHUB_VARNISH_PORT + value: "{{ github_varnish_port }}" + - name: GITHUB_PROTOCOL + value: "http" + - name: HELLO_RUNNABLE_GITHUB_ID + value: "{{ hello_runnable_github_id }}" + - name: HELLO_RUNNABLE_GITHUB_TOKEN + value: "{{ api_hello_runnable_github_token }}" + - name: KRAIN_PORT + value: "{{ krain_port }}" + - name: MIXPANEL_APP_ID + value: "{{ api_mixpanel_app_id | default('ansible_undefined') }}" + - name: MONGO_AUTH + value: "{{ api_mongo_auth }}" + - name: MONGO_DB + value: "{{ api_mongo_database }}" + - name: MONGO_HOSTS + value: "{{ mongo_hosts }}" + - name: MONGO_REPLSET_NAME + value: "{{ api_mongo_replset_name | default('ansible_undefined') }}" + - name: MONGO + value: "mongodb://{{ api_mongo_auth }}@{{ mongo_hosts }}/{{ api_mongo_database }}" + - name: NAVI_HOST + value: "http://{{ navi_host_address }}:{{ navi_http_port }}" + - name: NODE_ENV + value: "{{ node_env }}" + - name: NUM_WORKERS + value: "1" + - name: OPTIMUS_HOST + value: "http://{{ optimus_hostname }}" + - name: RABBITMQ_HOSTNAME + value: "{{ rabbit_host_address }}" + - name: RABBITMQ_PASSWORD + value: "{{ rabbit_password }}" + - name: RABBITMQ_PORT + value: "{{ rabbit_port }}" + - name: RABBITMQ_USERNAME + value: "{{ rabbit_username }}" + - name: REDIS_IPADDRESS + value: "{{ redis_host_address }}" + - name: REDIS_PORT + value: "{{ redis_port }}" + - name: S3_CONTEXT_RESOURCE_BUCKET + value: "{{ api_s3_context_bucket }}" + - name: S3_LOG_BUCKET + value: "{{ api_s3_log_bucket }}" + - name: SENDGRID_KEY + value: "{{ sendgrid_key }}" + - name: SWARM_HOST + value: "http://{{ swarm_host_address }}:{{ swarm_master_port }}" + - name: USER_CONTENT_DOMAIN + value: "{{ user_content_domain }}" + - name: AWS_ALIAS_HOST + value: "{{ api_aws_alias_host }}" + - name: NEW_RELIC_APP_NAME + value: "{{ api_new_relic_app_name | default('ansible_undefined') }}" + - name: NEW_RELIC_LICENSE_KEY + value: "{{ new_relic_license_key | default('ansible_undefined') }}" + - name: NEW_RELIC_LOG_LEVEL + value: "fatal" + - name: NEW_RELIC_NO_CONFIG_FILE value: "true" - - name: ROLLBAR_KEY - value: "{{ api_workers_rollbar_key | default('ansible_undefined') }}" - -container_envs: "{{ api_base_container_envs + envs }}" \ No newline at end of file + - name: INTERCOM_APP_ID + value: "{{ api_intercom_app_id | default('ansible_undefined') }}" + - name: INTERCOM_API_KEY + value: "{{ api_intercom_api_key | default('ansible_undefined') }}" + - name: USER_VAULT_ENDPOINT + value: "http://{{ user_vault_load_balancer }}:{{ user_vault_port }}" diff --git a/ansible/mongo-seed-db.yml b/ansible/mongo-seed-db.yml index ae15d73b..41f6cc82 100644 --- a/ansible/mongo-seed-db.yml +++ b/ansible/mongo-seed-db.yml @@ -1,7 +1,6 @@ --- - hosts: api vars_files: - - group_vars/alpha-api-base.yml - group_vars/alpha-mongo-seed-db.yml roles: - role: k8-job From 1045234758dc5b2d2c52510107bf2db4e4fce109 Mon Sep 17 00:00:00 2001 From: tosih Date: Fri, 7 Jul 2017 15:12:07 -0700 Subject: [PATCH 7/7] Add higher log level. --- ansible/group_vars/alpha-mongo-seed-db.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ansible/group_vars/alpha-mongo-seed-db.yml b/ansible/group_vars/alpha-mongo-seed-db.yml index f01da5b8..523aee37 100644 --- a/ansible/group_vars/alpha-mongo-seed-db.yml +++ b/ansible/group_vars/alpha-mongo-seed-db.yml @@ -5,6 +5,8 @@ container_image: quay.io/runnable/api container_run_args: npm run migrate-up container_envs: + - name: LOG_LEVEL + value: trace - name: DOCKER_IMAGE_BUILDER_CACHE value: "/home/anandkumarpatel/cache" - name: VALID_REDIR_TLDS