feat(deploy): EC2 Deployment & Build with clever-deploy - WIP

lib/command.js

@@ -1,5 +1,7 @@
 var spawn = require('child_process').spawn
   , path  = require('path')
+  , os    = require('os')
+  , isWin = /^win32/.test(os.platform())
   , fs    = require('fs');
 
 /**
@@ -39,21 +41,24 @@ module.exports = function (thePath, program, cmd, args) {
   if (fs.existsSync(local)) {
     bin = local;
   } else {
-    // @TODO Support windows here
-    bin = process.env.PATH.split(':').reduce(function(binary, p) {
+    bin = process.env.PATH.split(isWin ? ';' : ':').reduce(function(binary, p) {
       p = path.resolve(p, bin);
       return fs.existsSync(p) && fs.statSync(p).isFile() ? p : binary;
     }, bin);
   }
 
-  // if the bin doesn't exist within the cleverstack binary
-  // try finding it within the project/module folder...
+  // if the bin doesn't exist within the cleverstack binary, try in the path
   if (!fs.existsSync(bin)) {
     bin = path.join(__dirname, cmd);
 
     // if bin/cmd doesn't exist, then try clever- prefix
     if (!fs.existsSync(bin)) {
       bin = path.join(__dirname, 'clever-' + cmd);
+
+      // try finding it within the project folder...
+      if (!fs.existsSync(bin)) {
+        bin = path.join(process.cwd(), 'bin', 'clever-' + cmd);
+      }
     }
   }
 
@@ -67,8 +72,12 @@ module.exports = function (thePath, program, cmd, args) {
   // Windows support
   args.unshift(bin);
 
+  var processEnv        = process.env;
+  processEnv.cliDir     = path.resolve(path.join(__dirname, '..'));
+  processEnv.libDir     = path.join(processEnv.cliDir, 'lib');
+
   // spawn
-  var proc = spawn('node', args, { stdio: 'inherit' });
+  var proc = spawn('node', args, { stdio: 'inherit', env: processEnv });
   proc.on('close', function(code) {
     process.exit(code);
   });

lib/util/deploy.js

@@ -0,0 +1,21 @@
+var fs = require('fs');
+
+exports.generateReleaseScript = function() {
+  return '#!/bin/bash\n\n' +
+    fs.readFileSync('./deploy/templates/lock/acquire.sh') + '\n\n' +
+    fs.readFileSync('./deploy/templates/release/s3/download/script.sh') + '\n\n' +
+    fs.readFileSync('./deploy/templates/release/crontab.sh') + '\n\n' +
+    fs.readFileSync('./deploy/templates/release/s3/download/release.sh') + '\n\n' +
+    fs.readFileSync('./deploy/templates/lock/release.sh');
+};
+
+exports.generateUserDataScript = function() {
+  return '#!/bin/bash\n\n' +
+    fs.readFileSync('./deploy/templates/userdata/config.sh') + '\n\n' +
+    fs.readFileSync('./deploy/templates/userdata/prepare.sh') + '\n\n' +
+    fs.readFileSync('./deploy/templates/userdata/ec2/hostname.sh') + '\n\n' +
+    fs.readFileSync('./deploy/templates/userdata/s3cfg.sh') + '\n\n' +
+    fs.readFileSync('./deploy/templates/userdata/s3/download.sh') + '\n\n' +
+    fs.readFileSync('./deploy/templates/userdata/nat/prerouting.sh') + '\n\n' +
+    fs.readFileSync('./deploy/templates/userdata/run.sh');
+};

lib/util/deploy/templates/lock/acquire.sh

@@ -0,0 +1,33 @@
+## ------ START:  Locking mechanism ------ ##
+SCRIPT_PID=$$
+BASENAME=${0##*/}
+PIDFILE_BASE=/tmp/$BASENAME
+PIDFILE=$PIDFILE_BASE.$$
+
+#Look for existing Lock files
+PIDFILES=`ls $PIDFILE_BASE*`
+
+if [ -n "$PIDFILES" ] ; then
+    echo "PID:$SCRIPT_PID - $(date) | Lock files are present."
+    for P in $PIDFILES ; do
+        # Get the PID
+        PID=`cat $P`
+        if [ -f /proc/$PID/cmdline ] ; then
+            CMD=`cat /proc/$PID/cmdline`
+            if [ "${CMD//$BASENAME}" != "$CMD" ] ; then
+                echo "PID:$SCRIPT_PID - $(date) | Lock acquisition failed, exited script."
+                exit 1
+            else
+                echo "PID:$SCRIPT_PID - $(date) | Lock found was bogus, deleting it."
+                rm -f $P
+            fi
+        else
+            echo "PID:$SCRIPT_PID - $(date) | Lock found is dead, deleting it."
+            rm -f $P
+        fi
+    done
+fi
+
+echo "PID:$SCRIPT_PID - $(date) | Lock acquired ($PIDFILE)"
+echo $$ > $PIDFILE
+## ------ END:    Locking mechanism ------ ##

lib/util/deploy/templates/lock/release.sh

@@ -0,0 +1,5 @@
+## ------ START: Release Locking mechanism ------ ##
+rm -f $PIDFILE
+echo "PID:$SCRIPT_PID - $(date) | Lock released ($PIDFILE)"
+exit 0
+## ------ END:  Release Locking mechanism ------ ##

lib/util/deploy/templates/release/crontab.sh

@@ -0,0 +1,9 @@
+echo "PID:$SCRIPT_PID - $(date) | Updating crontab for user ubuntu..."
+
+echo "* * * * * bash /opt/$REPO_NAME.sh 2>&1 >> /tmp/deploy.log
+* * * * * sleep 10; bash /opt/$REPO_NAME.sh 2>&1 >> /tmp/deploy.log
+* * * * * sleep 20; bash /opt/$REPO_NAME.sh 2>&1 >> /tmp/deploy.log
+* * * * * sleep 30; bash /opt/$REPO_NAME.sh 2>&1 >> /tmp/deploy.log
+* * * * * sleep 40; bash /opt/$REPO_NAME.sh 2>&1 >> /tmp/deploy.log
+* * * * * sleep 50; bash /opt/$REPO_NAME.sh 2>&1 >> /tmp/deploy.log" > ~/ubuntu.crontab
+crontab ~/ubuntu.crontab

lib/util/deploy/templates/release/s3/download/release.sh

@@ -0,0 +1,73 @@
+## ------ START:  Standard Node AMI Release Script ------ ##
+S3_PATH="s3://$S3_DEPLOYMENT_BUCKET/$NODE_ENV/$REPO_NAME.tgz"
+RUNNING_LIST=`forever list | grep -c '\n'`
+FOREVER_PID_FILE="/opt/$REPO_NAME.pid"
+FOREVER_LOG_FILE="/opt/$REPO_NAME.log"
+
+# Only do a release if we did not just get a new release script
+if [ $RELEASE_VERSION_DIFF -eq 0 ]; then
+
+  # MD5 Tracking hases for updating the applications code from S3
+  MD5OLD='/tmp/oldMd5.txt'
+  MD5NEW='/tmp/newMd5.txt'
+  touch $MD5OLD
+  s3cmd -f --config /home/ubuntu/.s3cfg ls $S3_PATH | md5sum | awk '{ print $1 }' > $MD5NEW
+  VERSION_DIFF=`diff $MD5NEW $MD5OLD | wc -l`
+
+  if [ $VERSION_DIFF -ne 0 ]; then
+    echo "PID:$SCRIPT_PID - $(date) | Release required (MD5 has changed), updating $REPO_NAME on $NODE_ENV..."
+
+    DEPLOYMENT_DIR="/opt/$REPO_NAME"
+    if [ ! -d $DEPLOYMENT_DIR ]; then
+      echo "PID:$SCRIPT_PID - $(date) | SETUP: the DEPLOYMENT_DIR and set owner as ubuntu..."
+        sudo mkdir $DEPLOYMENT_DIR
+        sudo chown ubuntu:ubuntu $DEPLOYMENT_DIR
+    fi
+
+    echo "PID:$SCRIPT_PID - $(date) | Changing to $DEPLOYMENT_DIR (DEPLOYMENT_DIR)..."
+    cd $DEPLOYMENT_DIR
+
+    echo "PID:$SCRIPT_PID - $(date) | Release starting download of new code release (S3)..."
+    s3cmd -f --config /home/ubuntu/.s3cfg get $S3_PATH /opt/$REPO_NAME.tgz
+    mkdir /opt/$REPO_NAME
+    sudo chown ubuntu:ubuntu /opt/$REPO_NAME
+    sudo chmod 775 /opt/$REPO_NAME
+    tar -xzvf /opt/$REPO_NAME.tgz
+
+    echo "PID:$SCRIPT_PID - $(date) | Release Running NPM install & init+update git submodules..."
+    sudo npm install
+    sudo git submodule init
+    sudo git submodule update
+
+    if [ $RUNNING_LIST -eq 3 ]; then # Node is running fine
+      echo "PID:$SCRIPT_PID - $(date) | Forever is RESTARTING the $NODE_ENV application..."
+      forever restart 0
+    elif [ $RUNNING_LIST -eq 1 ]; then # No node processes are running through forever
+      echo "PID:$SCRIPT_PID - $(date) | Forever is STARTING the $NODE_ENV application because it was NOT running..."
+      forever start --spinSleepTime 1000 --pidFile $FOREVER_PID_FILE -a -l $FOREVER_LOG_FILE app.js
+    else
+      echo "PID:$SCRIPT_PID - $(date) | Forever is running multiple processes, attempting to fix the problem..."
+      forever stopall
+      forever start --spinSleepTime 1000 --pidFile $FOREVER_PID_FILE -a -l $FOREVER_LOG_FILE app.js
+    fi
+
+    # Update the md5
+    echo "PID:$SCRIPT_PID - $(date) | Release updating current MD5 to $MD5NEW"
+    cat $MD5NEW > $MD5OLD
+
+    echo "PID:$SCRIPT_PID - $(date) | Release finished!"
+  else 
+
+    if [ $RUNNING_LIST -eq 1 ]; then # No node processes are running through forever
+      echo "PID:$SCRIPT_PID - $(date) | Forever is STARTING the $NODE_ENV application because it was NOT running..."
+      forever start --spinSleepTime 1000 --pidFile $FOREVER_PID_FILE -a -l $FOREVER_LOG_FILE /opt/$REPO_NAME/app.js
+    elif [ $RUNNING_LIST -ne 3 ]; then
+      echo "PID:$SCRIPT_PID - $(date) | Forever is running multiple processes, attempting to fix the problem..."
+      forever stopall
+      forever start --spinSleepTime 1000 --pidFile $FOREVER_PID_FILE -a -l $FOREVER_LOG_FILE /opt/$REPO_NAME/app.js
+    fi
+
+    echo "PID:$SCRIPT_PID - $(date) | Release not required, exiting"
+  fi
+fi
+## ------ END:  Standard Node AMI Release Script ------ ##

lib/util/deploy/templates/release/s3/download/script.sh

@@ -0,0 +1,23 @@
+## ------ START:  Release script self updater ------ ##
+RELEASE_VERSION_DIFF=0
+RELEASE_MD5_CURRENT='/tmp/releaseCurrentMd5.txt'
+RELEASE_MD5_NEW='/tmp/releaseNewMd5.txt'
+touch $RELEASE_MD5_CURRENT
+
+NEW_RELEASE_NOT_EMPTY=`s3cmd -f --config /home/ubuntu/.s3cfg ls s3://$S3_DEPLOYMENT_BUCKET/$NODE_ENV/$REPO_NAME.sh | wc -l`
+if [ $NEW_RELEASE_NOT_EMPTY -ne 0 ]; then
+
+  # Get the latest hash
+  s3cmd -f --config /home/ubuntu/.s3cfg ls s3://$S3_DEPLOYMENT_BUCKET/$NODE_ENV/$REPO_NAME.sh | md5sum | awk '{ print $1 }' > $RELEASE_MD5_NEW
+
+  RELEASE_VERSION_DIFF=`diff $RELEASE_MD5_NEW $RELEASE_MD5_CURRENT | wc -l`
+  if [ $RELEASE_VERSION_DIFF -ne 0 ]
+  then
+    echo "PID:$SCRIPT_PID - $(date) | Release SCRIPT MD5 has changed, fetching a new copy of this release script..."
+    s3cmd -f --config /home/ubuntu/.s3cfg get s3://$S3_DEPLOYMENT_BUCKET/$NODE_ENV/$REPO_NAME.sh /opt/$REPO_NAME.sh
+
+    echo "PID:$SCRIPT_PID - $(date) | Release SCRIPT was updated, updating RELEASE_MD5_CURRENT"
+    cat $RELEASE_MD5_NEW > $RELEASE_MD5_CURRENT
+  fi
+fi
+## ------ END:    Release script self updater ------ ##

lib/util/deploy/templates/userdata/config.sh

@@ -0,0 +1,15 @@
+## ------ START:  BASE:       Config ------ ##
+export PROJECT                  ='myProject'
+export NODE_ENV                 ='DEV'
+export S3_DEPLOYMENT_KEY        ='your_iam_key'
+export S3_DEPLOYMENT_SECRET     ='your_iam_secret'
+export S3_DEPLOYMENT_BUCKET     ='your_s3_bucket_name'
+export S3_DEPLOYMENT_PASSPHRASE ='your_s3_passphrase'
+## ------ END:    BASE:       Config ------ ##
+
+## ------ START:  ADVANCED:   Config ------ ##
+export IP                       =`/sbin/ifconfig|grep inet|head -n 1|cut -d: -f2|cut -d\  -f1`
+export DEPLOY_DIR               ='/opt'
+export SCRIPT_NAME              ='$PROJECT.sh'
+export S3_APP_SCRIPT_PATH       ="s3://$S3_DEPLOYMENT_BUCKET/$NODE_ENV/$SCRIPT_NAME"
+## ------ END:    ADVANCED:   Config ------ ##

lib/util/deploy/templates/userdata/ec2/hostname.sh

@@ -0,0 +1,17 @@
+## ------ START:  EC2:      Set the hostname of the server ------ ##
+HOSTNAME="$PROJECT-$NODE_ENV"
+
+echo $HOSTNAME > /etc/project
+echo "$HOSTNAME" > /etc/hostname
+echo "$IP $HOSTNAME" >> /etc/hosts
+
+cat <<'EOF' > /etc/rc.local
+if [ -f /etc/project ] ; then
+  echo "`cat /etc/project`-`/sbin/ifconfig|grep inet|head -n 1|cut -d: -f2|cut -d\  -f1|sed 's/\./\-/g'`" > /etc/hostname
+  hostname "`cat /etc/hostname`"
+fi
+exit 0
+EOF
+
+hostname "$HOSTNAME"
+## ------ END:    EC2:      Set the hostname of the server ------ ##

lib/util/deploy/templates/userdata/nat/prerouting.sh

@@ -0,0 +1,2 @@
+# Ubuntu cannot bind port 80, so we do this here in userdata as root so it redirects 80 to 8080 (where node is running)
+sudo iptables -t nat -A PREROUTING -p tcp --dport 80 -j REDIRECT --to-port 8080

lib/util/deploy/templates/userdata/nginx.sh

@@ -0,0 +1,112 @@
+sudo apt-get update
+sudo apt-get -y install nginx
+
+sudo cat << 'EOF' > /etc/nginx/nginx.conf
+
+user www-data;
+worker_processes 4;
+pid /var/run/nginx.pid;
+worker_rlimit_nofile 8192;
+
+events {
+    worker_connections 2048;
+    use epoll;
+    multi_accept on;
+}
+
+http {
+    include /etc/nginx/mime.types;
+    default_type application/octet-stream;
+
+    log_format main '$remote_addr - $remote_user [$time_local] $status '
+            '"$request" $body_bytes_sent "$http_referer" '
+            '"$http_user_agent" "$http_x_forwarded_for"';
+    log_format  byte '$bytes_sent';
+    access_log /var/log/nginx/access.log main;
+
+    client_header_timeout 3m;
+    client_body_timeout 3m;
+    send_timeout 3m;
+
+    server_tokens off;
+
+    client_header_buffer_size 1k;
+    large_client_header_buffers 4 4k;
+    server_names_hash_bucket_size 128;
+
+    gzip on;
+    gzip_min_length 1100;
+    gzip_buffers 4 8k;
+    gzip_types text/plain text/css application/x-javascript text/xml application/xml text/javascript;
+
+    output_buffers 1 32k;
+    postpone_output 1460;
+
+    sendfile on;
+    tcp_nopush on;
+    tcp_nodelay on;
+    directio 4m;
+
+    open_file_cache max=1000 inactive=20s;
+    open_file_cache_valid 30s;
+    open_file_cache_min_uses 2;
+    open_file_cache_errors on;
+
+    server {
+        listen 80;
+        access_log /var/log/nginx/dev.log main;
+
+        location /api/ {
+
+            rewrite /api/(.*) /$1  break;
+
+            proxy_pass         http://127.0.0.1:8080/;
+            proxy_redirect     off;
+            proxy_set_header   Host             $host;
+            proxy_set_header   X-Real-IP        $remote_addr;
+            proxy_set_header   X-Forwarded-For  $proxy_add_x_forwarded_for;
+            proxy_set_header   X-NginX-Proxy    true;
+            proxy_connect_timeout      90;
+            proxy_send_timeout         90;
+            proxy_read_timeout         90;
+            proxy_buffer_size          4k;
+            proxy_buffers              4 32k;
+            proxy_busy_buffers_size    64k;
+            proxy_temp_file_write_size 64k;
+
+            # websockets:
+            proxy_http_version 1.1;
+            proxy_set_header Upgrade $http_upgrade;
+            proxy_set_header Connection "upgrade";
+
+            client_max_body_size       30m;
+            client_body_buffer_size    128k;
+        }
+
+        location / {
+
+            proxy_pass         http://your-bucket.s3-website-us-east-1.amazonaws.com/;
+            proxy_redirect     off;
+            proxy_set_header   Host             "your-bucket.s3-website-us-east-1.amazonaws.com";
+            proxy_set_header   X-Real-IP        $remote_addr;
+            proxy_set_header   X-Forwarded-For  $proxy_add_x_forwarded_for;
+            proxy_set_header   X-NginX-Proxy    true;
+            proxy_connect_timeout      90;
+            proxy_send_timeout         90;
+            proxy_read_timeout         90;
+            proxy_buffer_size          4k;
+            proxy_buffers              4 32k;
+            proxy_busy_buffers_size    64k;
+            proxy_temp_file_write_size 64k;
+
+            client_max_body_size       30m;
+            client_body_buffer_size    128k;
+
+        }
+
+    }
+}
+
+EOF
+
+sudo /etc/init.d/nginx restart

lib/util/deploy/templates/userdata/prepare.sh

@@ -0,0 +1,13 @@
+## ------ START:  BASE:       Prepare Environment & Deployment Directories ------ ##
+cat <<'EOF' >> /home/ubuntu/.bashrc
+export NODE_ENV='$NODE_ENV'
+export SCRIPT_NAME='$SCRIPT_NAME'
+export S3_APP_SCRIPT_PATH='$S3_APP_SCRIPT_PATH'
+EOF
+
+if [ ! -d "$DEPLOY_DIR" ]; then
+  mkdir $DEPLOY_DIR
+  chmod 755 $DEPLOY_DIR
+  chown ubuntu:ubuntu $DEPLOY_DIR
+fi
+## ------ START:  BASE:       Prepare Environment & Deployment Directories ------ ##

lib/util/deploy/templates/userdata/run.sh

@@ -0,0 +1,2 @@
+sudo -u ubuntu bash /opt/$SCRIPT_NAME >> /tmp/deploy.log
+sudo chown ubuntu:ubuntu /tmp/deploy.log