From 329f01790ad5a16965e744acf54440d7fb010c5e Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Wed, 20 Sep 2017 08:58:34 -0400 Subject: [PATCH 1/8] Run lint checks before compiling anything. Make cpplint warning Change-Id: Ib812f49e248540c7283a1e058f26925dbc36af00 --- ci/travis_before_script_cpp.sh | 12 ++++++++++++ ci/travis_script_cpp.sh | 2 -- ci/travis_script_python.sh | 8 +------- cpp/src/arrow/buffer.h | 2 +- 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh index d46fa2ff943..2c950cde105 100755 --- a/ci/travis_before_script_cpp.sh +++ b/ci/travis_before_script_cpp.sh @@ -99,6 +99,18 @@ else $ARROW_CPP_DIR fi +# Fail fast for code linting issues +$TRAVIS_MAKE lint +conda install -y -q flake8 + +# Fail fast on style checks +flake8 --count $ARROW_PYTHON_DIR/pyarrow + +# Check Cython files with some checks turned off +flake8 --count --config=$ARROW_PYTHON_DIR/pyarrow.flake8.cython \ + $ARROW_PYTHON_DIR/pyarrowpyarrow + +# Build and install libraries $TRAVIS_MAKE -j4 $TRAVIS_MAKE install diff --git a/ci/travis_script_cpp.sh b/ci/travis_script_cpp.sh index 4e3e7bbea1c..a2079036c45 100755 --- a/ci/travis_script_cpp.sh +++ b/ci/travis_script_cpp.sh @@ -27,8 +27,6 @@ git archive HEAD --prefix=apache-arrow/ --output=arrow-src.tar.gz pushd $CPP_BUILD_DIR -$TRAVIS_MAKE lint - # ARROW-209: checks depending on the LLVM toolchain are disabled temporarily # until we are able to install the full LLVM toolchain in Travis CI again diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh index e2cf2549348..f6ce18a2abe 100755 --- a/ci/travis_script_python.sh +++ b/ci/travis_script_python.sh @@ -38,7 +38,7 @@ which python conda install -y -q nomkl # Expensive dependencies install from Continuum package repo -conda install -y -q pip numpy pandas cython flake8 +conda install -y -q pip numpy pandas cython # Build C++ libraries pushd $ARROW_CPP_BUILD_DIR @@ -62,12 +62,6 @@ popd # Other stuff pip install pushd $ARROW_PYTHON_DIR -# Fail fast on style checks -flake8 --count pyarrow - -# Check Cython files with some checks turned off -flake8 --count --config=.flake8.cython pyarrow - pip install -r requirements.txt python setup.py build_ext --with-parquet --with-plasma \ install --single-version-externally-managed --record=record.text diff --git a/cpp/src/arrow/buffer.h b/cpp/src/arrow/buffer.h index d2152677860..4ed4d8d782d 100644 --- a/cpp/src/arrow/buffer.h +++ b/cpp/src/arrow/buffer.h @@ -173,7 +173,7 @@ class ARROW_EXPORT ResizableBuffer : public MutableBuffer { /// A Buffer whose lifetime is tied to a particular MemoryPool class ARROW_EXPORT PoolBuffer : public ResizableBuffer { public: - explicit PoolBuffer(MemoryPool* pool = nullptr); + PoolBuffer(MemoryPool* pool = nullptr); virtual ~PoolBuffer(); Status Resize(const int64_t new_size, bool shrink_to_fit = true) override; From 28fc3fb07589551959664db31997a9a0d8599b0c Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Wed, 20 Sep 2017 09:02:00 -0400 Subject: [PATCH 2/8] Typo Change-Id: Ifeae6a35fc35939bfdaf191b2639b3aee9f27274 --- ci/travis_before_script_cpp.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh index 2c950cde105..37f24fe1f3f 100755 --- a/ci/travis_before_script_cpp.sh +++ b/ci/travis_before_script_cpp.sh @@ -108,7 +108,7 @@ flake8 --count $ARROW_PYTHON_DIR/pyarrow # Check Cython files with some checks turned off flake8 --count --config=$ARROW_PYTHON_DIR/pyarrow.flake8.cython \ - $ARROW_PYTHON_DIR/pyarrowpyarrow + $ARROW_PYTHON_DIR/pyarrow # Build and install libraries $TRAVIS_MAKE -j4 From 7e50fad75bc352d7209c9bef088ef56afef4cb41 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Wed, 20 Sep 2017 09:31:07 -0400 Subject: [PATCH 3/8] Revert cpplint failure Change-Id: I249a02511b21b8dda43dc7e001cfcf4204fe9f21 --- cpp/src/arrow/buffer.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/arrow/buffer.h b/cpp/src/arrow/buffer.h index 4ed4d8d782d..d2152677860 100644 --- a/cpp/src/arrow/buffer.h +++ b/cpp/src/arrow/buffer.h @@ -173,7 +173,7 @@ class ARROW_EXPORT ResizableBuffer : public MutableBuffer { /// A Buffer whose lifetime is tied to a particular MemoryPool class ARROW_EXPORT PoolBuffer : public ResizableBuffer { public: - PoolBuffer(MemoryPool* pool = nullptr); + explicit PoolBuffer(MemoryPool* pool = nullptr); virtual ~PoolBuffer(); Status Resize(const int64_t new_size, bool shrink_to_fit = true) override; From b7db0830453f3b9591be4c7002d271b4b2abb3ea Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Wed, 20 Sep 2017 10:08:07 -0400 Subject: [PATCH 4/8] Only run lint checks when not running in --only-library mode Change-Id: I69076c41dad6fd8754fc154748ce6bd90bb8a7b2 --- ci/travis_before_script_cpp.sh | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh index 37f24fe1f3f..9b7410cc98e 100755 --- a/ci/travis_before_script_cpp.sh +++ b/ci/travis_before_script_cpp.sh @@ -99,16 +99,18 @@ else $ARROW_CPP_DIR fi -# Fail fast for code linting issues -$TRAVIS_MAKE lint -conda install -y -q flake8 +if [ $only_library_mode == "no" ]; then + # Fail fast for code linting issues + $TRAVIS_MAKE lint + conda install -y -q flake8 -# Fail fast on style checks -flake8 --count $ARROW_PYTHON_DIR/pyarrow + # Fail fast on style checks + flake8 --count $ARROW_PYTHON_DIR/pyarrow -# Check Cython files with some checks turned off -flake8 --count --config=$ARROW_PYTHON_DIR/pyarrow.flake8.cython \ - $ARROW_PYTHON_DIR/pyarrow + # Check Cython files with some checks turned off + flake8 --count --config=$ARROW_PYTHON_DIR/pyarrow.flake8.cython \ + $ARROW_PYTHON_DIR/pyarrow +fi # Build and install libraries $TRAVIS_MAKE -j4 From ed9e23a88d87c2d96e821ad0dd64c5569154b8f3 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Wed, 20 Sep 2017 10:37:14 -0400 Subject: [PATCH 5/8] Move linting to separate shell script Change-Id: I153daebbf521598fd88892b009e0b505fe60e350 --- .travis.yml | 5 +++++ ci/travis_before_script_cpp.sh | 13 ----------- ci/travis_lint.sh | 40 ++++++++++++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 13 deletions(-) create mode 100755 ci/travis_lint.sh diff --git a/.travis.yml b/.travis.yml index f6f64003d27..9a3ab9dcbfd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -58,6 +58,7 @@ matrix: - export ARROW_TRAVIS_USE_TOOLCHAIN=1 - export ARROW_TRAVIS_VALGRIND=1 - export ARROW_TRAVIS_PLASMA=1 + - $TRAVIS_BUILD_DIR/ci/travis_lint.sh - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh script: - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh @@ -72,6 +73,7 @@ matrix: before_script: - export ARROW_TRAVIS_USE_TOOLCHAIN=1 - export ARROW_TRAVIS_PLASMA=1 + - $TRAVIS_BUILD_DIR/ci/travis_lint.sh - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh script: - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh @@ -106,6 +108,7 @@ matrix: - source $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh - export CC="clang-4.0" - export CXX="clang++-4.0" + - $TRAVIS_BUILD_DIR/ci/travis_lint.sh - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh script: - $TRAVIS_BUILD_DIR/ci/travis_script_integration.sh @@ -123,6 +126,7 @@ matrix: before_script: - export CC="gcc-4.9" - export CXX="g++-4.9" + - $TRAVIS_BUILD_DIR/ci/travis_lint.sh - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library - $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh script: @@ -134,6 +138,7 @@ matrix: addons: rvm: 2.2 before_script: + - $TRAVIS_BUILD_DIR/ci/travis_lint.sh - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library - $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh script: diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh index 9b7410cc98e..a7f1d275634 100755 --- a/ci/travis_before_script_cpp.sh +++ b/ci/travis_before_script_cpp.sh @@ -99,19 +99,6 @@ else $ARROW_CPP_DIR fi -if [ $only_library_mode == "no" ]; then - # Fail fast for code linting issues - $TRAVIS_MAKE lint - conda install -y -q flake8 - - # Fail fast on style checks - flake8 --count $ARROW_PYTHON_DIR/pyarrow - - # Check Cython files with some checks turned off - flake8 --count --config=$ARROW_PYTHON_DIR/pyarrow.flake8.cython \ - $ARROW_PYTHON_DIR/pyarrow -fi - # Build and install libraries $TRAVIS_MAKE -j4 $TRAVIS_MAKE install diff --git a/ci/travis_lint.sh b/ci/travis_lint.sh new file mode 100755 index 00000000000..0b786b417ae --- /dev/null +++ b/ci/travis_lint.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -ex + +source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh + +# Fail fast for code linting issues +mkdir $ARROW_CPP_DIR/lint +pushd $ARROW_CPP_DIR/lint + +cmake .. +make lint + +popd + +# Fail fast on style checks +sudo pip install flake8 + +flake8 --count $ARROW_PYTHON_DIR/pyarrow + +# Check Cython files with some checks turned off +flake8 --count --config=$ARROW_PYTHON_DIR/pyarrow.flake8.cython \ + $ARROW_PYTHON_DIR/pyarrow From 910f684e202fabbc0b4559433065fa89924ef07b Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Wed, 20 Sep 2017 10:47:57 -0400 Subject: [PATCH 6/8] Fixes for linting. Do not cache .conda_packages Change-Id: I02dff04e3a9dd6eac7dec66e2809fe09a7b6c6fa --- .travis.yml | 2 -- ci/travis_lint.sh | 14 +++++++------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9a3ab9dcbfd..f469fe52e9d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -40,8 +40,6 @@ services: cache: ccache: true - directories: - - $HOME/.conda_packages matrix: fast_finish: true diff --git a/ci/travis_lint.sh b/ci/travis_lint.sh index 0b786b417ae..9b7b474524f 100755 --- a/ci/travis_lint.sh +++ b/ci/travis_lint.sh @@ -19,11 +19,9 @@ set -ex -source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh - # Fail fast for code linting issues -mkdir $ARROW_CPP_DIR/lint -pushd $ARROW_CPP_DIR/lint +mkdir $TRAVIS_BUILD_DIR/cpp/lint +pushd $TRAVIS_BUILD_DIR/cpp/lint cmake .. make lint @@ -33,8 +31,10 @@ popd # Fail fast on style checks sudo pip install flake8 -flake8 --count $ARROW_PYTHON_DIR/pyarrow +PYARROW_DIR=$TRAVIS_BUILD_DIR/python + +flake8 --count $PYTHON_DIR/pyarrow # Check Cython files with some checks turned off -flake8 --count --config=$ARROW_PYTHON_DIR/pyarrow.flake8.cython \ - $ARROW_PYTHON_DIR/pyarrow +flake8 --count --config=$PYTHON_DIR/.flake8.cython \ + $PYTHON_DIR/pyarrow From d1cf6792cb445c67a28fc016d37459989f4f77e6 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Wed, 20 Sep 2017 14:13:51 -0400 Subject: [PATCH 7/8] Set language: python when linting on macOS Change-Id: Ie388db2ce2e87905081457be4287927d7542fc40 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f469fe52e9d..b71885ef65d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -64,6 +64,7 @@ matrix: - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 2.7 - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6 - compiler: clang + language: python osx_image: xcode6.4 os: osx cache: @@ -136,7 +137,6 @@ matrix: addons: rvm: 2.2 before_script: - - $TRAVIS_BUILD_DIR/ci/travis_lint.sh - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh --only-library - $TRAVIS_BUILD_DIR/ci/travis_before_script_c_glib.sh script: From 0bb5202eb2dc03ade980fd5b829778d6fcb8633a Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Wed, 20 Sep 2017 20:40:28 -0400 Subject: [PATCH 8/8] System python not available on xcode 6.4 machines Change-Id: Icd46f484474699096ffacbe7b4ca9be1fa80f6c9 --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index b71885ef65d..4691eeb0656 100644 --- a/.travis.yml +++ b/.travis.yml @@ -64,7 +64,7 @@ matrix: - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 2.7 - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh 3.6 - compiler: clang - language: python + language: cpp osx_image: xcode6.4 os: osx cache: @@ -72,7 +72,6 @@ matrix: before_script: - export ARROW_TRAVIS_USE_TOOLCHAIN=1 - export ARROW_TRAVIS_PLASMA=1 - - $TRAVIS_BUILD_DIR/ci/travis_lint.sh - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh script: - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh