ec2 failover -wip

lib/tasks/canary/build.js

@@ -2,12 +2,12 @@
 
 require('loadenv')()
 
-var api = require('../../models/api')
-var CanaryFailedError = require('../../errors/canary-failed-error')
-var Promise = require('bluebird')
-var request = Promise.promisifyAll(require('request'))
+const api = require('../../models/api')
+const CanaryFailedError = require('../../errors/canary-failed-error')
+const Promise = require('bluebird')
+const request = Promise.promisifyAll(require('request'))
 
-var CanaryBase = require('./canary-base')
+const CanaryBase = require('./canary-base')
 
 /**
  * Runs a canary test against the production API to ensure we can rebuild a

lib/tasks/canary/canary-base.js

@@ -1,11 +1,11 @@
 'use strict'
 
-var monitor = require('monitor-dog')
+const monitor = require('monitor-dog')
 const WorkerStopError = require('error-cat/errors/worker-stop-error')
-var CanaryFailedError = require('../../errors/canary-failed-error')
-var logger = require('../../logger')
-var exists = require('101/exists')
-var Promise = require('bluebird')
+const CanaryFailedError = require('../../errors/canary-failed-error')
+const logger = require('../../logger')
+const exists = require('101/exists')
+const Promise = require('bluebird')
 
 /**
  * Base class for all the canaries to work off of.

lib/tasks/canary/ec2-failover.js

@@ -0,0 +1,137 @@
+'use strict'
+
+require('loadenv')()
+const keypather = require('keypather')()
+const Promise = require('bluebird')
+const request = Promise.promisifyAll(require('request'))
+
+const api = require('../../models/api')
+const CanaryBase = require('./canary-base')
+const CanaryFailedError = require('../../errors/canary-failed-error')
+const rabbitmq = require('models/rabbitmq')
+
+const RUNNABLE_FAILOVER_TEST_GITHUB_ID = '19616978'
+
+/**
+ * Runs a canary test against the production API to ensure we can Fail Over a
+ * specific container. The results of the test are reported to datadog.
+ *
+ * For instructions on setting up this canary, refer to
+ * https://github.com/CodeNow/devops-scripts/wiki/Setup-FailOver-Canary
+ *
+ *
+ * @param {object} job The canary job to exectute.
+ * @return {Promise} Resolves when the rebuild has been successfully completed.
+ */
+module.exports = (job) => {
+  return new FailOverCanary(job).executeTest()
+}
+
+class FailOverCanary extends CanaryBase {
+  constructor (job) {
+    super(job)
+    this.queue = 'ec2-canary.failover.run'
+    this.name = 'FailOver Canary'
+    this.gauge = 'ec2-canary.failover'
+
+    this.log = this.log.child({
+      task: this.queue
+    })
+  }
+
+  setup () {
+    return api.connect(process.env.CANARY_API_FAILOVER_TOKEN)
+      .then((client) => {
+        this.client = client
+      })
+      .catch((err) => {
+        throw new CanaryFailedError('Error connecting to Runnable client', {err: err})
+      })
+  }
+
+  test () {
+    const dockerHosts = {}
+    return this.setup()
+      .then(() => {
+        this.log.debug('First, fetch the instances to get the docks')
+        return this.client.fetchInstancesAsync({
+          owner: {
+            github: RUNNABLE_FAILOVER_TEST_GITHUB_ID
+          }
+        })
+      })
+      .tap((instances) => {
+        this.log.debug('Found ' + instances.length + ' non-isolated instances')
+      })
+      .then((instances) => {
+        return Promise.filter(instances, (instance) => {
+          return instance.isIsolationGroupMaster
+        })
+          .map((instance) => {
+            const qs = {
+              isIsolationGroupMaster: false,
+              isolated: instance.isolated,
+              githubUsername: instance.owner.username
+            }
+            return this.client.fetchInstancesAsync(qs)
+              .map((isolatedInstances) => {
+                if (!isolatedInstances.isIsolationGroupMaster) {
+                  instances.push(isolatedInstances)
+                }
+              })
+          })
+          .return(instances)
+      })
+      .tap((instances) => {
+        this.log.debug('Found ' + instances.length + ' total instances')
+      })
+      .map((rawInstanceData) => {
+        this.log.debug('Validate that all of the instances are running')
+        const instance = this.client.newInstance(rawInstanceData)
+        const status = instance.status()
+        if (status !== 'running') {
+          throw new CanaryFailedError('Instance ' + rawInstanceData.name + ' not running', rawInstanceData)
+        }
+
+        const dockerHost = keypather.get(rawInstanceData, 'container.dockerHost')
+        if (!dockerHosts[dockerHost]) {
+          dockerHosts[dockerHost] = 0
+        }
+        dockerHosts[dockerHost]++
+        keypather.set(instance, 'opts.user.opts.userContentDomain', process.env.USER_CONTENT_DOMAIN)
+        return request.getAsync('http://' + instance.getContainerHostname())
+          .timeout(process.env.CANARY_FAILOVER_TEST_REQUEST_TIMEOUT)
+          .catch((e) => {
+            const errorData = {
+              error: e,
+              url: instance.getContainerHostname()
+            }
+            this.log.error(errorData, 'Instance ' + rawInstanceData.name + ' failed the get request')
+            throw new CanaryFailedError('Instance endpoint not accessible', errorData)
+          })
+      })
+      .then(() => {
+        const largestDock = Object
+          .keys(dockerHosts)
+          .sort((a, b) => {
+            // Sorting from large to small
+            if (dockerHosts[a] < dockerHosts[b]) {
+              return 1
+            } else if ((dockerHosts[a] > dockerHosts[b])) {
+              return -1
+            } else {
+              return 0
+            }
+          })[0] // Grab 0, since it should be the largest
+        this.log.debug('The Dock at ' + largestDock + ' contains ' + dockerHosts[largestDock] +
+          ' containers, and is about to be marked unhealthy')
+        return largestDock
+      })
+      .then((largestDock) => {
+        const ip = largestDock.split(':')[0]
+        const data = { ipAddress: ip }
+        this.log.debug('Killing EC2 instance ' + data)
+        rabbitmq.publishEvent('dock.purged', data)
+      })
+  }
+}

lib/tasks/canary/github-branch.js

@@ -2,13 +2,13 @@
 
 require('loadenv')()
 
-var api = require('../../models/api')
-var CanaryFailedError = require('../../errors/canary-failed-error')
-var github = require('../../models/github')
-var Promise = require('bluebird')
-var request = Promise.promisifyAll(require('request'))
+const api = require('../../models/api')
+const CanaryFailedError = require('../../errors/canary-failed-error')
+const github = require('../../models/github')
+const Promise = require('bluebird')
+const request = Promise.promisifyAll(require('request'))
 
-var CanaryBase = require('./canary-base')
+const CanaryBase = require('./canary-base')
 
 /**
  * Runs a canary test against the production API to ensure we can successfully

lib/tasks/canary/log.js

@@ -2,14 +2,14 @@
 
 require('loadenv')()
 
-var api = require('../../models/api')
-var CanaryFailedError = require('../../errors/canary-failed-error')
-var dockerStreamCleanser = require('docker-stream-cleanser')
-var Promise = require('bluebird')
-var PrimusClient = require('@runnable/api-client/lib/external/primus-client')
-var uuid = require('uuid')
+const api = require('../../models/api')
+const CanaryFailedError = require('../../errors/canary-failed-error')
+const dockerStreamCleanser = require('docker-stream-cleanser')
+const Promise = require('bluebird')
+const PrimusClient = require('@runnable/api-client/lib/external/primus-client')
+const uuid = require('uuid')
 
-var CanaryBase = require('./canary-base')
+const CanaryBase = require('./canary-base')
 
 /**
  * Runs a canary test against the production API to ensure we can get