From 371a24a4ed4f0eb15017807074a9d1543868145e Mon Sep 17 00:00:00 2001 From: Michalis Papadimitriou Date: Wed, 30 Jun 2021 14:58:35 +0300 Subject: [PATCH 1/8] [Pre-commit] Add pre-commit hook configuration file --- .pre-commit-config.yaml | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000000..9c12a0cb227a --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,40 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.5.0 + hooks: + - id: check-added-large-files + - id: check-merge-conflict + - id: check-yaml + - id: end-of-file-fixer + - repo: local + hooks: + - id: run-file-types + name: Checking File Types... + entry: python3 tests/lint/check_file_type.py + language: system + always_run: true + pass_filenames: false + - id: run-headers-check + name: Checking ASF License Headers... + entry: tests/lint/check_asf_header.sh --local + language: system + always_run: true + pass_filenames: false + - id: run-black + name: Checking Black ... + entry: tests/lint/python_format.sh + language: system + always_run: true + pass_filenames: false + - id: run-python-linting + name: Checking Python Linting ... + entry: tests/lint/pylint.sh + language: system + always_run: true + pass_filenames: false + - id: run-mypy + name: Type Checking with MyPy ... + entry: tests/scripts/task_mypy.sh + language: system + always_run: true + pass_filenames: false From 114a9a351f15e29b4a133ec5a69da7cfea6c9738 Mon Sep 17 00:00:00 2001 From: Michalis Papadimitriou Date: Wed, 30 Jun 2021 15:00:56 +0300 Subject: [PATCH 2/8] [Pre-commit] Add header to configuratin file --- .pre-commit-config.yaml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9c12a0cb227a..55d80af21544 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v2.5.0 From 121b6cfba1479f6d434efd717e002fc4a21cf335 Mon Sep 17 00:00:00 2001 From: Michalis Papadimitriou Date: Wed, 30 Jun 2021 16:16:15 +0300 Subject: [PATCH 3/8] [Pre-commit] Add basic configuration instructions --- .pre-commit-config.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 55d80af21544..58afeeb97bac 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,6 +15,14 @@ # specific language governing permissions and limitations # under the License. +# https://pre-commit.com/ +# +# How to configure: +# $ pip install pre-commit +# $ pre-commit install +# How to prevent running it: +# $ git commit --no-verify + repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v2.5.0 @@ -23,6 +31,7 @@ repos: - id: check-merge-conflict - id: check-yaml - id: end-of-file-fixer + - id: trailing-whitespace - repo: local hooks: - id: run-file-types From 9246108a6ad7d74a89d090b0f749c906e1de31a9 Mon Sep 17 00:00:00 2001 From: Michalis Papadimitriou Date: Thu, 1 Jul 2021 14:15:36 +0300 Subject: [PATCH 4/8] [Pre-commit] Extend pre-commit pipelines with C++ linting --- .pre-commit-config.yaml | 64 ++++++++++++++++++++++++++--------------- 1 file changed, 41 insertions(+), 23 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 58afeeb97bac..648451df3b10 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,17 +15,28 @@ # specific language governing permissions and limitations # under the License. -# https://pre-commit.com/ +# Pre-commit hook +# See documentation at: https://pre-commit.com/ +# +# Pre-commit hook to run the sanity checks from Jenkins locally. +# +# Requirements: +# - How to configure: +# $ pip install pre-commit +# $ pre-commit install +# - How to prevent running it: +# $ git commit --no-verify +# +# Working versions required: +# $ pip3 install cpplint pylint==2.4.4 mypy==0.902 black==20.8b1 # -# How to configure: -# $ pip install pre-commit -# $ pre-commit install -# How to prevent running it: -# $ git commit --no-verify +default_language_version: + python: python3.8 +fail_fast: True repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.5.0 + rev: v2.3.0 hooks: - id: check-added-large-files - id: check-merge-conflict @@ -34,33 +45,40 @@ repos: - id: trailing-whitespace - repo: local hooks: + - id: run-unit-tests + name: Running Minimal Unit-Tests ... + entry: python -m pytest -v tests/python/all-platform-minimal-test + language: system + verbose: false + always_run: true + pass_filenames: false + - id: run-black + name: Running Black... + entry: tests/lint/python_format.sh + language: system + always_run: true - id: run-file-types name: Checking File Types... entry: python3 tests/lint/check_file_type.py - language: system + language: python always_run: true - pass_filenames: false - id: run-headers-check - name: Checking ASF License Headers... + name: Checking ASF License Headers ... entry: tests/lint/check_asf_header.sh --local language: system always_run: true - pass_filenames: false - - id: run-black - name: Checking Black ... - entry: tests/lint/python_format.sh + - id: run-headers-check + name: Linting the C++ code ... + entry: tests/lint/cpplint.sh language: system - always_run: true - pass_filenames: false - - id: run-python-linting - name: Checking Python Linting ... - entry: tests/lint/pylint.sh + always_run: False + - id: run-clang-format + name: Checking Clang format ... + entry: tests/lint/clang_format.sh language: system - always_run: true - pass_filenames: false + always_run: False - id: run-mypy - name: Type Checking with MyPy ... + name: Type Checking with MyPY ... entry: tests/scripts/task_mypy.sh language: system always_run: true - pass_filenames: false From a6845e81f53f586bc3cb8b833d5a60e5d863e784 Mon Sep 17 00:00:00 2001 From: Michalis Papadimitriou Date: Mon, 9 Aug 2021 17:28:43 +0300 Subject: [PATCH 5/8] [pre-commit] Add example usage comment for pre-commit hooks --- .pre-commit-config.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 648451df3b10..955dcdafc397 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -22,10 +22,12 @@ # # Requirements: # - How to configure: -# $ pip install pre-commit -# $ pre-commit install +# - $ pip install pre-commit +# - $ pre-commit install # - How to prevent running it: -# $ git commit --no-verify +# - git options: --no-verify or -n +# - $ git commit -n -m "YOUR COMMIT MESSAGE" +# # # Working versions required: # $ pip3 install cpplint pylint==2.4.4 mypy==0.902 black==20.8b1 From 0ac51e905a2c1348148775605bbd840c0d4a6194 Mon Sep 17 00:00:00 2001 From: Michalis Papadimitriou Date: Tue, 10 Aug 2021 12:25:04 +0300 Subject: [PATCH 6/8] [CI] Add in docker linting script mypy step --- docker/lint.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docker/lint.sh b/docker/lint.sh index d15ce71b7a98..e709bfb08445 100755 --- a/docker/lint.sh +++ b/docker/lint.sh @@ -20,7 +20,7 @@ source "$(dirname $0)/dev_common.sh" SCRIPT_NAME="$0" -DEFAULT_STEPS=( file_type asf cpplint clang_format pylint python_format jnilint cppdocs ) +DEFAULT_STEPS=( file_type asf cpplint clang_format pylint python_format jnilint cppdocs mypy ) inplace_fix=0 @@ -67,6 +67,9 @@ function run_lint_step() { cppdocs) cmd=( tests/lint/cppdocs.sh ) ;; + mypy) + cmd=( tests/scripts/task_mypy.sh ) + ;; *) echo "error: don't know how to run lint step: $1" >&2 echo "usage: ${SCRIPT_NAME} [-i] " >&2 From c6fcaf7153fa936238c2e9fa6aeaadcc43de79e8 Mon Sep 17 00:00:00 2001 From: Michalis Papadimitriou Date: Tue, 10 Aug 2021 13:05:22 +0300 Subject: [PATCH 7/8] [CI] Use lint docker image for pre-commit checks --- .pre-commit-config.yaml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 955dcdafc397..c3049e22331e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,10 +27,8 @@ # - How to prevent running it: # - git options: --no-verify or -n # - $ git commit -n -m "YOUR COMMIT MESSAGE" -# -# -# Working versions required: -# $ pip3 install cpplint pylint==2.4.4 mypy==0.902 black==20.8b1 +# - How to run it as standalone +# - $ pre-commit run # default_language_version: @@ -51,36 +49,38 @@ repos: name: Running Minimal Unit-Tests ... entry: python -m pytest -v tests/python/all-platform-minimal-test language: system - verbose: false + verbose: true always_run: true pass_filenames: false - id: run-black name: Running Black... - entry: tests/lint/python_format.sh + entry: docker/lint.sh python_format language: system always_run: true + verbose: true + pass_filenames: false - id: run-file-types name: Checking File Types... - entry: python3 tests/lint/check_file_type.py + entry: docker/lint.sh file_type language: python always_run: true - id: run-headers-check name: Checking ASF License Headers ... - entry: tests/lint/check_asf_header.sh --local + entry: docker/lint.sh asf language: system always_run: true - id: run-headers-check name: Linting the C++ code ... - entry: tests/lint/cpplint.sh + entry: docker/lint.sh cpplint language: system always_run: False - id: run-clang-format name: Checking Clang format ... - entry: tests/lint/clang_format.sh + entry: docker/lint.sh clang_format language: system always_run: False - id: run-mypy name: Type Checking with MyPY ... - entry: tests/scripts/task_mypy.sh + entry: docker/lint.sh mypy language: system always_run: true From b5872543dd90317799c7cb09cac5d78dcb460e26 Mon Sep 17 00:00:00 2001 From: Michalis Papadimitriou Date: Tue, 24 Aug 2021 17:44:30 +0300 Subject: [PATCH 8/8] [CI][pre-commit] Minor cleanups on docker runners of pre-commit lints --- .pre-commit-config.yaml | 16 ++++++++++------ docker/bash.sh | 20 ++++++++++++++++++-- docker/dev_common.sh | 3 +-- 3 files changed, 29 insertions(+), 10 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f2629f59703e..3a2c07de458a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -50,30 +50,34 @@ repos: entry: docker/lint.sh python_format language: system always_run: true - verbose: true pass_filenames: false - - id: run-file-types - name: Checking File Types... + - id: run-file-checks + name: Checking File Types.... entry: docker/lint.sh file_type - language: python + language: system always_run: true + pass_filenames: false - id: run-headers-check name: Checking ASF License Headers ... entry: docker/lint.sh asf language: system always_run: true + pass_filenames: false - id: run-headers-check name: Linting the C++ code ... entry: docker/lint.sh cpplint language: system - always_run: False + always_run: true + pass_filenames: false - id: run-clang-format name: Checking Clang format ... entry: docker/lint.sh clang_format language: system - always_run: False + always_run: true + pass_filenames: false - id: run-mypy name: Type Checking with MyPY ... entry: docker/lint.sh mypy language: system always_run: true + pass_filenames: false diff --git a/docker/bash.sh b/docker/bash.sh index 4667f27b0b9b..2a05abf4f2bc 100755 --- a/docker/bash.sh +++ b/docker/bash.sh @@ -20,7 +20,7 @@ # # Start a bash, mount REPO_MOUNT_POINT to be current directory. # -# Usage: docker/bash.sh [-i|--interactive] [--net=host] +# Usage: docker/bash.sh [-i|--interactive] [--net=host] [-t|--tty] # [--mount MOUNT_DIR] [--repo-mount-point REPO_MOUNT_POINT] # [--dry-run] # [--] [COMMAND] @@ -35,6 +35,7 @@ set -euo pipefail + function show_usage() { cat <