From 3de6a8a398cbf7ea0793cdebff520e19ca9fa18b Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Fri, 12 May 2017 15:48:21 +0200 Subject: [PATCH 1/5] ARROW-1016: Python: Include C++ headers (optionally) in wheels Change-Id: I0a50228ad8c91f89be27f6bb35f068e72bf43fc6 --- python/CMakeLists.txt | 1 + python/manylinux1/README.md | 2 +- python/setup.py | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index a9db4d877fb..d8287108d4c 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -246,6 +246,7 @@ endfunction(bundle_arrow_lib) if (PYARROW_BUNDLE_ARROW_CPP) # arrow + file(COPY ${ARROW_INCLUDE_DIR}/arrow DESTINATION ${BUILD_OUTPUT_ROOT_DIRECTORY}/include) bundle_arrow_lib(ARROW_SHARED_LIB) bundle_arrow_lib(ARROW_PYTHON_SHARED_LIB) endif() diff --git a/python/manylinux1/README.md b/python/manylinux1/README.md index 32af6f31da2..2e7f56bd620 100644 --- a/python/manylinux1/README.md +++ b/python/manylinux1/README.md @@ -32,7 +32,7 @@ git clone ../../ arrow # Build the native baseimage docker build -t arrow-base-x86_64 -f Dockerfile-x86_64 . # Build the python packages -docker run --rm -v $PWD:/io arrow-base-x86_64 /io/build_arrow.sh +docker run --rm -t -i -v $PWD:/io arrow-base-x86_64 /io/build_arrow.sh # Now the new packages are located in the dist/ folder ls -l dist/ ``` diff --git a/python/setup.py b/python/setup.py index 1c466170669..335aa02dce7 100644 --- a/python/setup.py +++ b/python/setup.py @@ -225,6 +225,8 @@ def move_lib(lib_name): os.symlink(lib_filename, link_name) if self.bundle_arrow_cpp: + print(pjoin(self.build_type, 'include'), pjoin(build_lib, 'pyarrow')) + shutil.move(pjoin(self.build_type, 'include'), pjoin(build_lib, 'pyarrow')) move_lib("arrow") move_lib("arrow_python") if self.with_jemalloc: From 590f629aa1e33eb60d27069b32d868746ecbe9d7 Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Fri, 12 May 2017 19:16:29 +0200 Subject: [PATCH 2/5] Add output to wrap cmake command Change-Id: Id8644ed9d2a5f961857fb381360842d007f9b033 --- python/setup.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/setup.py b/python/setup.py index 335aa02dce7..b5e4c4abac5 100644 --- a/python/setup.py +++ b/python/setup.py @@ -161,7 +161,9 @@ def _run_cmake(self): cmake_command = (['cmake', self.extra_cmake_args] + cmake_options + [source]) + print("-- Runnning cmake for pyarrow") self.spawn(cmake_command) + print("-- Finished cmake for pyarrow") args = ['make'] if os.environ.get('PYARROW_BUILD_VERBOSE', '0') == '1': args.append('VERBOSE=1') @@ -183,7 +185,9 @@ def _run_cmake(self): if "-G" in self.extra_cmake_args: cmake_command = cmake_command[:-2] + print("-- Runnning cmake for pyarrow") self.spawn(cmake_command) + print("-- Finished cmake for pyarrow") # Do the build self.spawn(['cmake', '--build', '.', '--config', self.build_type]) From 0b3209aa85bb5f12744e79bb78b1275ad1d613c6 Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Fri, 12 May 2017 19:22:59 +0200 Subject: [PATCH 3/5] More cmake debug output Change-Id: I69f8c26eb0f4d5b2f60dd675ee39ca49a304d59d --- python/setup.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/setup.py b/python/setup.py index b5e4c4abac5..87451014251 100644 --- a/python/setup.py +++ b/python/setup.py @@ -170,7 +170,9 @@ def _run_cmake(self): if 'PYARROW_PARALLEL' in os.environ: args.append('-j{0}'.format(os.environ['PYARROW_PARALLEL'])) + print("-- Running cmake --build for pyarrow") self.spawn(args) + print("-- Finished cmake --build for pyarrow") else: import shlex cmake_generator = 'Visual Studio 14 2015 Win64' @@ -189,7 +191,9 @@ def _run_cmake(self): self.spawn(cmake_command) print("-- Finished cmake for pyarrow") # Do the build + print("-- Running cmake --build for pyarrow") self.spawn(['cmake', '--build', '.', '--config', self.build_type]) + print("-- Finished cmake --build for pyarrow") if self.inplace: # a bit hacky From c982c97c121d09b8dbc0c114d81333b5fb7c2d16 Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Fri, 12 May 2017 19:33:20 +0200 Subject: [PATCH 4/5] Remove existing include folder Change-Id: I0f643791f72a122fbab4630d19de990e5bc443e2 --- python/setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/setup.py b/python/setup.py index 87451014251..148224afb05 100644 --- a/python/setup.py +++ b/python/setup.py @@ -234,6 +234,8 @@ def move_lib(lib_name): if self.bundle_arrow_cpp: print(pjoin(self.build_type, 'include'), pjoin(build_lib, 'pyarrow')) + if os.path.exists(pjoin(build_lib, 'pyarrow', 'include')): + shutil.rmtree(pjoin(build_lib, 'pyarrow', 'include')) shutil.move(pjoin(self.build_type, 'include'), pjoin(build_lib, 'pyarrow')) move_lib("arrow") move_lib("arrow_python") From bc9b5ee6c779b92661de826326aa5366611b1289 Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Fri, 12 May 2017 19:44:39 +0200 Subject: [PATCH 5/5] Build pyarrow using multiple cores Change-Id: I3b8d3adddc3b89740a5203f9192d51c874cebece --- ci/travis_script_manylinux.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/travis_script_manylinux.sh b/ci/travis_script_manylinux.sh index 69feb685b51..4e6be62bd3e 100755 --- a/ci/travis_script_manylinux.sh +++ b/ci/travis_script_manylinux.sh @@ -18,4 +18,4 @@ set -ex pushd python/manylinux1 git clone ../../ arrow docker build -t arrow-base-x86_64 -f Dockerfile-x86_64 . -docker run --rm -v $PWD:/io arrow-base-x86_64 /io/build_arrow.sh +docker run --rm -e PYARROW_PARALLEL=3 -v $PWD:/io arrow-base-x86_64 /io/build_arrow.sh