Skip to content

Commit 5f4be91

Browse files
cpcloudcpcloud
committed
BLD: add requirements files and update install.sh
This commit adds 5 requirements files: one for each Travis build. The corresponding sections in ci/install.sh are also removed in favor of these files. This allows pandas to have more fine-grained control over versions of hard and soft deps when using travis.
1 parent 99ae9ce commit 5f4be91

File tree

11 files changed

+122
-128
lines changed

11 files changed

+122
-128
lines changed

‎.gitignore‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,4 +36,5 @@ pandas/io/*.json
3636
.idea/pandas.iml
3737
.build_cache_dir
3838
.vagrant
39-
*.whl
39+
*.whl
40+
**/wheelhouse/*

‎.travis.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,13 +10,13 @@ env:
1010
matrix:
1111
include:
1212
- python: 2.7
13-
env: NOSE_ARGS="slow and not network" LOCALE_OVERRIDE="zh_CN.GB18030" FULL_DEPS=true
13+
env: NOSE_ARGS="slow and not network" LOCALE_OVERRIDE="zh_CN.GB18030" FULL_DEPS=true JOB_TAG=_LOCALE
1414
- python: 2.7
1515
env: NOSE_ARGS="not slow" FULL_DEPS=true
1616
- python: 3.2
1717
env: NOSE_ARGS="not slow" FULL_DEPS=true
1818
- python: 3.3
19-
env: NOSE_ARGS="not slow"
19+
env: NOSE_ARGS="not slow" FULL_DEPS=true
2020

2121
# allow importing from site-packages,
2222
# so apt-get python-x works for system pythons

‎ci/install.sh‎

Lines changed: 16 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -15,85 +15,42 @@
1515

1616
echo "inside $0"
1717

18-
# Install Dependencie
18+
# Install Dependencies
1919
# as of pip 1.4rc2, wheel files are still being broken regularly, this is a known good
2020
# commit. should revert to pypi when a final release is out
2121
pip install -I git+https://github.com/pypa/pip@42102e9deaea99db08b681d06906c2945f6f95e2#egg=pip
22-
pip Install -I https://bitbucket.org/pypa/setuptools/downloads/setuptools-0.8b6.tar.gz
22+
pv="${TRAVIS_PYTHON_VERSION:0:1}"
23+
[ "$pv" == "2" ] && pv=""
24+
[ "$pv" == "2" ] && DISTRIBUTE_VERSION="==0.6.35"
25+
26+
pip install -I distribute${DISTRIBUTE_VERSION}
2327
pip install wheel
2428

2529
# comment this line to disable the fetching of wheel files
26-
#PIP_ARGS+=" -I --use-wheel --find-links=https://cache27-pypandas.rhcloud.com/"
30+
PIP_ARGS+=" -I --use-wheel --find-links=http://cache27diy-cpycloud.rhcloud.com/${TRAVIS_PYTHON_VERSION}${JOB_TAG}/"
2731

2832
# Force virtualenv to accpet system_site_packages
2933
rm -f $VIRTUAL_ENV/lib/python$TRAVIS_PYTHON_VERSION/no-global-site-packages.txt
3034

31-
if [ x"$LOCALE_OVERRIDE" != x"" ]; then
35+
36+
if [ -n "$LOCALE_OVERRIDE" ]; then
3237
# make sure the locale is available
3338
# probably useless, since you would need to relogin
3439
sudo locale-gen "$LOCALE_OVERRIDE"
35-
fi;
36-
37-
#scipy is not included in the cached venv
38-
if [ x"$FULL_DEPS" == x"true" ] ; then
39-
# for pytables gets the lib as well
40-
time sudo apt-get $APT_ARGS install libhdf5-serial-dev
41-
42-
if [ ${TRAVIS_PYTHON_VERSION:0:1} == "3" ]; then
43-
time sudo apt-get $APT_ARGS install python3-bs4
44-
elif [ ${TRAVIS_PYTHON_VERSION:0:1} == "2" ]; then
45-
time sudo apt-get $APT_ARGS install python-bs4
46-
fi
47-
48-
if [ ${TRAVIS_PYTHON_VERSION} == "3.2" ]; then
49-
time sudo apt-get $APT_ARGS install python3-scipy
50-
elif [ ${TRAVIS_PYTHON_VERSION} == "2.7" ]; then
51-
time sudo apt-get $APT_ARGS install python-scipy
52-
fi
5340
fi
5441

55-
# Hard Deps
56-
for dep in nose 'python-dateutil' 'pytz>=2013a' 'cython==0.19.1'; do
57-
time pip install $PIP_ARGS $dep
58-
done
59-
60-
if [ ${TRAVIS_PYTHON_VERSION} == "3.3" ]; then # should be >=3,3
61-
time pip install $PIP_ARGS numpy==1.7.1
62-
elif [ ${TRAVIS_PYTHON_VERSION} == "3.2" ]; then
63-
# sudo apt-get $APT_ARGS install python3-numpy; # 1.6.2 or precise
64-
time pip install $PIP_ARGS numpy==1.6.1
65-
else
66-
time pip install $PIP_ARGS numpy==1.6.1
67-
fi
42+
time pip install $PIP_ARGS -r ci/requirements-${TRAVIS_PYTHON_VERSION}${JOB_TAG}.txt
6843

6944
# Optional Deps
7045
if [ x"$FULL_DEPS" == x"true" ]; then
7146
echo "Installing FULL_DEPS"
47+
# for pytables gets the lib as well
48+
time sudo apt-get $APT_ARGS install libhdf5-serial-dev
49+
time sudo apt-get $APT_ARGS install python${pv}-bs4
50+
time sudo apt-get $APT_ARGS install python${pv}-scipy
51+
52+
time sudo apt-get $APT_ARGS remove python${pv}-lxml
7253

73-
if [ ${TRAVIS_PYTHON_VERSION:0:1} == "2" ]; then
74-
time pip install $PIP_ARGS xlwt
75-
time pip install $PIP_ARGS bottleneck==0.6.0
76-
time pip install $PIP_ARGS numexpr==2.1
77-
time pip install $PIP_ARGS tables==2.3.1
78-
else
79-
time pip install $PIP_ARGS numexpr==2.1
80-
time pip install $PIP_ARGS tables==3.0.0
81-
fi
82-
83-
time pip install $PIP_ARGS matplotlib==1.2.1
84-
time pip install $PIP_ARGS openpyxl
85-
time pip install $PIP_ARGS xlrd>=0.9.0
86-
time pip install $PIP_ARGS 'http://downloads.sourceforge.net/project/pytseries/scikits.timeseries/0.91.3/scikits.timeseries-0.91.3.tar.gz?r='
87-
time pip install $PIP_ARGS patsy
88-
time pip install $PIP_ARGS html5lib
89-
90-
if [ ${TRAVIS_PYTHON_VERSION:0:1} == "3" ]; then
91-
time sudo apt-get $APT_ARGS remove python3-lxml
92-
elif [ ${TRAVIS_PYTHON_VERSION:0:1} == "2" ]; then
93-
time sudo apt-get $APT_ARGS remove python-lxml
94-
fi
95-
96-
pip install $PIP_ARGS lxml
9754
# fool statsmodels into thinking pandas was already installed
9855
# so it won't refuse to install itself.
9956

‎ci/requirements-2.6.txt‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
numpy==1.6.1
2+
cython==0.19.1
3+
python-dateutil==2.1
4+
pytz==2013b

‎ci/requirements-2.7.txt‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
python-dateutil==2.1
2+
pytz==2013b
3+
xlwt==0.7.5
4+
numpy==1.7.1
5+
cython==0.19.1
6+
bottleneck==0.6.0
7+
numexpr==2.1
8+
tables==2.3.1
9+
matplotlib==1.2.1
10+
openpyxl==1.6.2
11+
xlrd==0.9.2
12+
patsy==0.1.0
13+
html5lib==1.0b2
14+
lxml==3.2.1
15+
http://downloads.sourceforge.net/project/pytseries/scikits.timeseries/0.91.3/scikits.timeseries-0.91.3.tar.gz?r=

‎ci/requirements-2.7_LOCALE.txt‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
python-dateutil
2+
pytz==2013b
3+
xlwt==0.7.5
4+
openpyxl==1.6.2
5+
xlrd==0.9.2
6+
numpy==1.6.1
7+
cython==0.19.1
8+
bottleneck==0.6.0
9+
numexpr==2.1
10+
tables==2.3.1
11+
matplotlib==1.2.1
12+
patsy==0.1.0
13+
html5lib==1.0b2
14+
lxml==3.2.1

‎ci/requirements-3.2.txt‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
python-dateutil==2.1
2+
pytz==2013b
3+
openpyxl==1.6.2
4+
xlrd==0.9.2
5+
html5lib==1.0b2
6+
numpy==1.6.2
7+
cython==0.19.1
8+
numexpr==2.1
9+
tables==3.0.0
10+
matplotlib==1.2.1
11+
patsy==0.1.0
12+
lxml==3.2.1

‎ci/requirements-3.3.txt‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
python-dateutil==2.1
2+
pytz==2013b
3+
openpyxl==1.6.2
4+
xlrd==0.9.2
5+
html5lib==1.0b2
6+
numpy==1.7.1
7+
cython==0.19.1
8+
numexpr==2.1
9+
tables==3.0.0
10+
matplotlib==1.2.1
11+
patsy==0.1.0
12+
lxml==3.2.1

‎ci/speedpack/Vagrantfile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ Vagrant.configure("2") do |config|
77
# config.vbguest.auto_update = true
88
# config.vbguest.no_remote = true
99

10+
config.vm.synced_folder File.expand_path("..", Dir.pwd), "/reqf"
1011
config.vm.synced_folder "wheelhouse", "/wheelhouse"
1112

1213
config.vm.provider :virtualbox do |vb|

‎ci/speedpack/build.sh‎

Lines changed: 44 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -6,93 +6,71 @@
66
#
77
# Runtime can be up to an hour or more.
88

9-
echo "Running build.sh..."
10-
set -x
11-
12-
WHEEL_DIR=/wheelhouse
13-
VERSIONS="2.6 2.7 3.2 3.3"
14-
SCRIPT_FILE="/tmp/run.sh"
15-
PARALLEL=false
9+
echo "Building wheels..."
1610

17-
export PIP_ARGS=" --download-cache /tmp -w $WHEEL_DIR --use-wheel --find-links=$WHEEL_DIR"
11+
# print a trace for everything; RTFM
12+
set -x
1813

14+
# install and update some basics
1915
apt-get update
2016
apt-get install python-software-properties git -y
2117
apt-add-repository ppa:fkrull/deadsnakes -y
2218
apt-get update
2319

20+
# install some deps and virtualenv
2421
apt-get install python-pip libfreetype6-dev libpng12-dev -y
2522
pip install virtualenv
2623
apt-get install libhdf5-serial-dev g++ -y
24+
apt-get build-dep python-lxml -y
2725

26+
export PYTHONIOENCODING='utf-8'
2827

29-
function generate_wheels {
30-
VER=$1
31-
set -x
28+
function generate_wheels() {
29+
# get the requirements file
30+
local reqfile="$1"
3231

33-
if [ x"$VIRTUAL_ENV" != x"" ]; then
34-
deactivate
35-
fi
32+
# get the python version
33+
local TAG=$(echo $reqfile | grep -Po "(\d\.?[\d\-](_\w+)?)")
3634

37-
cd ~/
38-
sudo rm -Rf venv-$VER
39-
virtualenv -p python$VER venv-$VER
40-
source venv-$VER/bin/activate
35+
# base dir for wheel dirs
36+
local WHEELSTREET=/wheelhouse
37+
local WHEELHOUSE="$WHEELSTREET/$TAG"
4138

42-
pip install -I --download-cache /tmp git+https://github.com/pypa/pip@42102e9d#egg=pip
43-
pip install -I --download-cache /tmp https://bitbucket.org/pypa/setuptools/downloads/setuptools-0.8b6.tar.gz
44-
pip install -I --download-cache /tmp wheel
39+
local PY_VER="${TAG:0:3}"
40+
local PY_MAJOR="${PY_VER:0:1}"
41+
local PIP_ARGS="--use-wheel --find-links=$WHEELHOUSE --download-cache /tmp"
4542

46-
export INCLUDE_PATH=/usr/include/python$VER/
47-
export C_INCLUDE_PATH=/usr/include/python$VER/
48-
pip wheel $PIP_ARGS cython==0.19.1
49-
pip install --use-wheel --find-links=$WHEEL_DIR cython==0.19.1
43+
# install the python version if not installed
44+
apt-get install python$PY_VER python$PY_VER-dev -y
5045

51-
pip wheel $PIP_ARGS numpy==1.6.1
52-
pip wheel $PIP_ARGS numpy==1.7.1
53-
pip install --use-wheel --find-links=$WHEEL_DIR numpy==1.7.1
54-
pip wheel $PIP_ARGS bottleneck==0.6.0
46+
# create a new virtualenv
47+
rm -Rf /tmp/venv
48+
virtualenv -p python$PY_VER /tmp/venv
49+
source /tmp/venv/bin/activate
5550

56-
pip wheel $PIP_ARGS numexpr==1.4.2
57-
pip install --use-wheel --find-links=$WHEEL_DIR numexpr==1.4.2
58-
pip wheel $PIP_ARGS tables==2.3.1
59-
pip wheel $PIP_ARGS tables==2.4.0
51+
# install pip setuptools
52+
pip install -I --download-cache /tmp 'git+https://github.com/pypa/pip@42102e9d#egg=pip'
53+
DISTRIBUTE_VERSION=
54+
if [ "${PY_MAJOR}" == "2" ]; then
55+
DISTRIBUTE_VERSION="==0.6.35"
56+
fi
57+
pip install -I --download-cache /tmp distribute${DISTRIBUTE_VERSION}
58+
pip install -I --download-cache /tmp wheel
6059

61-
pip uninstall numexpr -y
62-
pip wheel $PIP_ARGS numexpr==2.1
63-
pip install --use-wheel --find-links=$WHEEL_DIR numexpr==2.1
64-
pip wheel $PIP_ARGS tables==3.0.0
65-
pip uninstall numexpr -y
60+
# make the dir if it doesn't exist
61+
mkdir -p $WHEELHOUSE
6662

67-
pip wheel $PIP_ARGS matplotlib==1.2.1
63+
# put the requirements file in the wheelhouse
64+
cp $reqfile $WHEELHOUSE
65+
66+
# install and build the wheels
67+
cat $reqfile | while read N; do
68+
pip wheel $PIP_ARGS --wheel-dir=$WHEELHOUSE $N
69+
pip install $PIP_ARGS --no-index $N
70+
done
6871
}
6972

7073

71-
for VER in $VERSIONS ; do
72-
apt-get install python$VER python$VER-dev -y
74+
for reqfile in $(ls -1 /reqf/requirements-*.*); do
75+
generate_wheels "$reqfile"
7376
done
74-
75-
if $PARALLEL; then
76-
echo '#!/bin/bash' > $SCRIPT_FILE
77-
echo "export WHEEL_DIR=$WHEEL_DIR" >> $SCRIPT_FILE
78-
echo "export PIP_ARGS='$PIP_ARGS'">> $SCRIPT_FILE
79-
80-
declare -f generate_wheels >> $SCRIPT_FILE
81-
echo 'generate_wheels $1' >> $SCRIPT_FILE
82-
chmod u+x $SCRIPT_FILE
83-
84-
pip install -I --download-cache /tmp git+https://github.com/pypa/pip@42102e9d#egg=pip
85-
pip install --download-cache /tmp --no-install wheel
86-
pip install --download-cache /tmp --no-install https://bitbucket.org/pypa/setuptools/downloads/setuptools-0.8b6.tar.gz
87-
88-
for VER in 2.6 2.7 3.2 3.3 ; do
89-
$SCRIPT_FILE $VER &
90-
done
91-
92-
wait
93-
94-
else
95-
for VER in 2.6 2.7 3.2 3.3 ; do
96-
generate_wheels $VER
97-
done
98-
fi

0 commit comments

Comments
 (0)