Merge pull request #4128 from y-p/PR_travis_speedups

BLD: use binary packages to reduce travis-ci build times.

Author: y-p

.gitignore

@@ -35,3 +35,5 @@ pandas/io/*.json
 
 .idea/pandas.iml
 .build_cache_dir
+.vagrant
+*.whl

.travis.yml

@@ -10,13 +10,13 @@ env:
 matrix:
   include:
     - python: 2.7
-      env: NOSE_ARGS="slow and not network"  LOCALE_OVERRIDE="zh_CN.GB18030" FULL_DEPS=true UPLOAD=false
+      env: NOSE_ARGS="slow and not network"  LOCALE_OVERRIDE="zh_CN.GB18030" FULL_DEPS=true
     - python: 2.7
-      env: NOSE_ARGS="not slow" FULL_DEPS=true UPLOAD=true
+      env: NOSE_ARGS="not slow" FULL_DEPS=true
     - python: 3.2
-      env: NOSE_ARGS="not slow" FULL_DEPS=true UPLOAD=true
+      env: NOSE_ARGS="not slow" FULL_DEPS=true
     - python: 3.3
-      env: NOSE_ARGS="not slow" UPLOAD=true
+      env: NOSE_ARGS="not slow"
 
 # allow importing from site-packages,
 # so apt-get python-x works for system pythons

ci/install.sh

@@ -14,11 +14,18 @@
 #
 
 echo "inside $0"
+
 # Install Dependencie
-SITE_PKG_DIR=$VIRTUAL_ENV/lib/python$TRAVIS_PYTHON_VERSION/site-packages
-echo "Using SITE_PKG_DIR: $SITE_PKG_DIR"
+# as of pip 1.4rc2, wheel files are still being broken regularly, this is a known good
+# commit. should revert to pypi when a final release is out
+pip install -I git+https://github.com/pypa/pip@42102e9deaea99db08b681d06906c2945f6f95e2#egg=pip
+pip Install -I https://bitbucket.org/pypa/setuptools/downloads/setuptools-0.8b6.tar.gz
+pip install wheel
+
+# comment this line to disable the fetching of wheel files
+PIP_ARGS+=" -I --use-wheel --find-links=https://cache27-pypandas.rhcloud.com/"
 
-# workaround for travis ignoring system_site_packages in travis.yml
+# Force virtualenv to accpet system_site_packages
 rm -f $VIRTUAL_ENV/lib/python$TRAVIS_PYTHON_VERSION/no-global-site-packages.txt
 
 if [ x"$LOCALE_OVERRIDE" != x"" ]; then
@@ -30,75 +37,73 @@ fi;
 #scipy is not included in the cached venv
 if [ x"$FULL_DEPS" == x"true" ] ; then
    # for pytables gets the lib as well
-   sudo apt-get $APT_ARGS install libhdf5-serial-dev
+  time sudo apt-get $APT_ARGS install libhdf5-serial-dev
 
    if [ ${TRAVIS_PYTHON_VERSION:0:1} == "3" ]; then
-     sudo apt-get $APT_ARGS install python3-bs4
+     time sudo apt-get $APT_ARGS install python3-bs4
    elif [ ${TRAVIS_PYTHON_VERSION:0:1} == "2" ]; then
-     sudo apt-get $APT_ARGS install python-bs4
+     time sudo apt-get $APT_ARGS install python-bs4
    fi
 
    if [ ${TRAVIS_PYTHON_VERSION} == "3.2" ]; then
-       sudo apt-get $APT_ARGS install python3-scipy
+       time sudo apt-get $APT_ARGS install python3-scipy
    elif [ ${TRAVIS_PYTHON_VERSION} == "2.7" ]; then
-       sudo apt-get $APT_ARGS install python-scipy
+       time sudo apt-get $APT_ARGS install python-scipy
    fi
 fi
 
 # Hard Deps
-pip install $PIP_ARGS nose python-dateutil pytz
-pip install $PIP_ARGS cython
+time pip install $PIP_ARGS nose python-dateutil pytz>=2013a
+time pip install $PIP_ARGS cython==0.19.1
 
 if [ ${TRAVIS_PYTHON_VERSION} == "3.3" ]; then # should be >=3,3
-    pip install $PIP_ARGS numpy==1.7.0
+    time pip install $PIP_ARGS numpy==1.7.1
 elif [ ${TRAVIS_PYTHON_VERSION} == "3.2" ]; then
     # sudo apt-get $APT_ARGS install python3-numpy; # 1.6.2 or precise
-    pip install $PIP_ARGS numpy==1.6.1
+    time pip install $PIP_ARGS numpy==1.6.1
 else
-    pip install $PIP_ARGS numpy==1.6.1
+    time pip install $PIP_ARGS numpy==1.6.1
 fi
 
 # Optional Deps
 if [ x"$FULL_DEPS" == x"true" ]; then
     echo "Installing FULL_DEPS"
-    pip install $PIP_ARGS  cython
 
     if [ ${TRAVIS_PYTHON_VERSION:0:1} == "2" ]; then
-        pip install $PIP_ARGS xlwt
-        pip install $PIP_ARGS bottleneck
-        pip install $PIP_ARGS numexpr==2.0.1
-        pip install $PIP_ARGS tables==2.3.1
+        time pip install $PIP_ARGS xlwt
+        time pip install $PIP_ARGS bottleneck==0.6.0
+        time pip install $PIP_ARGS numexpr==2.1
+        time pip install $PIP_ARGS tables==2.3.1
     else
-        pip install $PIP_ARGS numexpr
-        pip install $PIP_ARGS tables
+        time pip install $PIP_ARGS numexpr==2.1
+        time pip install $PIP_ARGS tables==3.0.0
     fi
 
-    pip install $PIP_ARGS matplotlib
-    pip install $PIP_ARGS openpyxl
-    pip install $PIP_ARGS xlrd>=0.9.0
-    pip install $PIP_ARGS 'http://downloads.sourceforge.net/project/pytseries/scikits.timeseries/0.91.3/scikits.timeseries-0.91.3.tar.gz?r='
-    pip install $PIP_ARGS patsy
-    pip install $PIP_ARGS html5lib
+    time pip install $PIP_ARGS matplotlib==1.2.1
+    time pip install $PIP_ARGS openpyxl
+    time pip install $PIP_ARGS xlrd>=0.9.0
+    time pip install $PIP_ARGS 'http://downloads.sourceforge.net/project/pytseries/scikits.timeseries/0.91.3/scikits.timeseries-0.91.3.tar.gz?r='
+    time pip install $PIP_ARGS patsy
+    time pip install $PIP_ARGS html5lib
 
     if [ ${TRAVIS_PYTHON_VERSION:0:1} == "3" ]; then
-      sudo apt-get $APT_ARGS remove python3-lxml
+      time sudo apt-get $APT_ARGS remove python3-lxml
     elif [ ${TRAVIS_PYTHON_VERSION:0:1} == "2" ]; then
-      sudo apt-get $APT_ARGS remove python-lxml
+      time sudo apt-get $APT_ARGS remove python-lxml
     fi
 
-    pip install $PIP_ARGS lxml
     # fool statsmodels into thinking pandas was already installed
     # so it won't refuse to install itself.
 
     mkdir  $SITE_PKG_DIR/pandas
     touch $SITE_PKG_DIR/pandas/__init__.py
     echo "version='0.10.0-phony'" >  $SITE_PKG_DIR/pandas/version.py
-    pip install $PIP_ARGS git+git://github.com/statsmodels/statsmodels@c9062e43b8a5f7385537ca95#egg=statsmodels
+    time pip install $PIP_ARGS git+git://github.com/statsmodels/statsmodels@c9062e43b8a5f7385537ca95#egg=statsmodels
 
     rm -Rf $SITE_PKG_DIR/pandas # scrub phoney pandas
 fi
 
 # build pandas
-python setup.py build_ext install
+time python setup.py build_ext install
 
 true

ci/speedpack/Vagrantfile

@@ -0,0 +1,21 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+Vagrant.configure("2") do |config|
+  config.vm.box = "precise64"
+  config.vm.box_url = "http://files.vagrantup.com/precise64.box"
+
+#  config.vbguest.auto_update = true
+#  config.vbguest.no_remote = true
+
+  config.vm.synced_folder "wheelhouse", "/wheelhouse"
+
+  config.vm.provider :virtualbox do |vb|
+    vb.customize ["modifyvm", :id, "--cpus", "4"]
+    vb.customize ["modifyvm", :id, "--memory", "2048"]
+    vb.customize ["modifyvm", :id, "--natdnshostresolver1", "on"]
+    vb.customize ["modifyvm", :id, "--natdnsproxy1", "on"]
+  end
+
+  config.vm.provision :shell, :path => "build.sh"
+
+end

ci/speedpack/build.sh

@@ -0,0 +1,98 @@
+#!/bin/bash
+
+# This script is meant to run on a mint precise64 VM.
+# The generated wheel files should be compatible
+# with travis-ci as of 07/2013.
+#
+# Runtime can be up to an hour or more.
+
+echo "Running build.sh..."
+set -x
+
+WHEEL_DIR=/wheelhouse
+VERSIONS="2.6 2.7 3.2 3.3"
+SCRIPT_FILE="/tmp/run.sh"
+PARALLEL=false
+
+export PIP_ARGS=" --download-cache /tmp -w $WHEEL_DIR --use-wheel --find-links=$WHEEL_DIR"
+
+apt-get update
+apt-get install python-software-properties git -y
+apt-add-repository ppa:fkrull/deadsnakes -y
+apt-get update
+
+apt-get install python-pip libfreetype6-dev libpng12-dev -y
+pip install virtualenv
+apt-get install libhdf5-serial-dev g++ -y
+
+
+function generate_wheels {
+	VER=$1
+	set -x
+
+	if [ x"$VIRTUAL_ENV" != x"" ]; then
+		deactivate
+	fi
+
+	cd ~/
+	sudo rm -Rf venv-$VER
+	virtualenv -p python$VER venv-$VER
+	source venv-$VER/bin/activate
+
+	pip install -I --download-cache /tmp git+https://github.com/pypa/pip@42102e9d#egg=pip
+	pip install -I --download-cache  /tmp https://bitbucket.org/pypa/setuptools/downloads/setuptools-0.8b6.tar.gz
+	pip install -I --download-cache /tmp wheel
+
+	export INCLUDE_PATH=/usr/include/python$VER/
+	export C_INCLUDE_PATH=/usr/include/python$VER/
+	pip wheel $PIP_ARGS cython==0.19.1
+	pip install --use-wheel --find-links=$WHEEL_DIR cython==0.19.1
+
+	pip wheel $PIP_ARGS numpy==1.6.1
+	pip wheel $PIP_ARGS numpy==1.7.1
+	pip install --use-wheel --find-links=$WHEEL_DIR numpy==1.7.1
+	pip wheel $PIP_ARGS bottleneck==0.6.0
+
+	pip wheel $PIP_ARGS numexpr==1.4.2
+	pip install --use-wheel --find-links=$WHEEL_DIR numexpr==1.4.2
+	pip wheel $PIP_ARGS tables==2.3.1
+	pip wheel $PIP_ARGS tables==2.4.0
+
+	pip uninstall numexpr -y
+	pip wheel $PIP_ARGS numexpr==2.1
+	pip install --use-wheel --find-links=$WHEEL_DIR numexpr==2.1
+	pip wheel $PIP_ARGS tables==3.0.0
+	pip uninstall numexpr -y
+
+	pip wheel $PIP_ARGS matplotlib==1.2.1
+}
+
+
+for VER in $VERSIONS ; do
+	apt-get install python$VER python$VER-dev -y
+done
+
+if $PARALLEL; then
+	echo '#!/bin/bash' > $SCRIPT_FILE
+	echo "export WHEEL_DIR=$WHEEL_DIR" >> $SCRIPT_FILE
+	echo "export PIP_ARGS='$PIP_ARGS'">> $SCRIPT_FILE
+
+	declare -f generate_wheels >>  $SCRIPT_FILE
+	echo 'generate_wheels $1' >> $SCRIPT_FILE
+	chmod u+x $SCRIPT_FILE
+
+	pip install -I --download-cache /tmp git+https://github.com/pypa/pip@42102e9d#egg=pip
+	pip install --download-cache /tmp --no-install wheel
+	pip install --download-cache /tmp --no-install https://bitbucket.org/pypa/setuptools/downloads/setuptools-0.8b6.tar.gz
+
+	for VER in 2.6 2.7 3.2 3.3 ; do
+		$SCRIPT_FILE  $VER &
+	done
+
+	wait
+
+else
+	for VER in 2.6 2.7 3.2 3.3 ; do
+		generate_wheels $VER
+	done
+fi

ci/speedpack/nginx/nginx.conf.template

@@ -0,0 +1,48 @@
+#user  nobody;
+worker_processes  1;
+
+#error_log  logs/error.log;
+#error_log  logs/error.log  notice;
+#error_log  logs/error.log  info;
+
+#pid        logs/nginx.pid;
+
+
+events {
+    worker_connections  1024;
+}
+
+
+http {
+    include       mime.types;
+    default_type  application/octet-stream;
+
+    #log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+    #                  '$status $body_bytes_sent "$http_referer" '
+    #                  '"$http_user_agent" "$http_x_forwarded_for"';
+
+    #access_log  logs/access.log  on;
+
+    sendfile        on;
+    #tcp_nopush     on;
+
+    #keepalive_timeout  0;
+    keepalive_timeout  65;
+
+    #gzip  on;
+
+    server {
+        listen $OPENSHIFT_IP:$OPENSHIFT_PORT;
+
+        access_log  access.log ;
+        sendfile        on;
+
+        location / {
+                root ../../app-root/data/store/;
+                autoindex on;
+        }
+
+
+    }
+
+}