diff --git a/.github/workflows/iwyu.yml b/.github/workflows/iwyu.yml new file mode 100644 index 000000000000..f774890de440 --- /dev/null +++ b/.github/workflows/iwyu.yml @@ -0,0 +1,81 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Include What You Use Check + +on: + pull_request: + paths: + - '.github/workflows/iwyu.yml' + - 'cpp/**/*.cc' + - 'cpp/**/*.h' + +env: + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true + SETUP: 'bash .github/workflows/util/setup-helper.sh' + +concurrency: + group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + Run-Include-What-You-Use: + runs-on: ubuntu-22.04 + container: apache/gluten:vcpkg-centos-8 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Build Gluten native libraries + run: | + set -e + yum install tzdata -y + source /opt/rh/gcc-toolset-11/enable + pip3 install regex + cd $GITHUB_WORKSPACE/ + wget -q https://github.com/llvm/llvm-project/releases/download/llvmorg-18.1.8/clang+llvm-18.1.8-x86_64-linux-gnu-ubuntu-18.04.tar.xz + tar -xf clang+llvm-18.1.8-x86_64-linux-gnu-ubuntu-18.04.tar.xz + rm -rf clang+llvm-18.1.8-x86_64-linux-gnu-ubuntu-18.04.tar.xz + cd clang+llvm-18.1.8-x86_64-linux-gnu-ubuntu-18.04/bin/ && ls -1 | grep -v "^clang" | xargs rm -rf + ln -s /lib64/libtinfo.so.6.1 /lib64/libtinfo.so.5 + ln -s `pwd`/clang /usr/bin/clang + ln -s `pwd`/clang++ /usr/bin/clang++ + clang --version + ln -s /opt/rh/gcc-toolset-11/root/usr/lib/gcc/x86_64-redhat-linux/11/libstdc++.a /usr/lib64/libstdc++.a + export CXXFLAGS="-I/opt/rh/gcc-toolset-11/root/usr/include/c++/11 -I/opt/rh/gcc-toolset-11/root/usr/include/c++/11/x86_64-redhat-linux" + cd $GITHUB_WORKSPACE/ + export NUM_THREADS=$(nproc) + bash ./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_arrow=OFF --build_tests=OFF --build_benchmarks=OFF \ + --build_examples=OFF --enable_s3=ON --enable_gcs=ON --enable_hdfs=ON --enable_abfs=ON get_velox + bash ./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_arrow=OFF --build_tests=OFF --build_benchmarks=OFF \ + --build_examples=OFF --enable_s3=ON --enable_gcs=ON --enable_hdfs=ON --enable_abfs=ON build_velox + export CC="clang" + export CXX="clang++" + bash ./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_arrow=OFF --build_tests=OFF --build_benchmarks=OFF \ + --build_examples=OFF --enable_s3=ON --enable_gcs=ON --enable_hdfs=ON --enable_abfs=ON build_gluten_cpp + unset CC + unset CXX + rm -rf /usr/bin/clang /usr/bin/clang++ + - name: Check Include What You Use + run: | + cd $GITHUB_WORKSPACE/ + bash .github/workflows/util/setup-helper.sh install_iwyu + ls -lh /usr/bin/include-what-you-use + export CPLUS_INCLUDE_PATH=/opt/rh/gcc-toolset-11/root/usr/lib/gcc/x86_64-redhat-linux/11/include + export CXXFLAGS="--gcc-toolchain=/opt/rh/gcc-toolset-11/root/usr" + wget https://github.com/include-what-you-use/include-what-you-use/archive/refs/tags/0.25.tar.gz + tar -xzvf 0.25.tar.gz + cp include-what-you-use-0.25/iwyu_tool.py dev/iwyu_tool.py + python3 dev/check.py iwyu commit diff --git a/.github/workflows/util/setup-helper.sh b/.github/workflows/util/setup-helper.sh new file mode 100644 index 000000000000..0ddd52849822 --- /dev/null +++ b/.github/workflows/util/setup-helper.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +function install_maven { + ( + local maven_version="3.9.12" + local local_binary="apache-maven-${maven_version}-bin.tar.gz" + local mirror_host="https://www.apache.org/dyn/closer.lua" + local url="${mirror_host}/maven/maven-3/${maven_version}/binaries/${local_binary}?action=download" + cd /opt/ + wget -nv -O ${local_binary} ${url} + tar -xvf ${local_binary} && mv apache-maven-${maven_version} /usr/lib/maven + ) + export PATH=/usr/lib/maven/bin:$PATH + if [ -n "$GITHUB_ENV" ]; then + echo "PATH=/usr/lib/maven/bin:$PATH" >> $GITHUB_ENV + else + echo "Warning: GITHUB_ENV is not set. Skipping environment variable export." + fi +} + +function install_iwyu { + yum install -y llvm llvm-devel clang clang-devel llvm-toolset + CLANG_VERSION=`clang --version | awk '/clang version/ {print $3}' | cut -d. -f1` + echo $CLANG_VERSION + git clone https://github.com/include-what-you-use/include-what-you-use.git + cd include-what-you-use + git checkout clang_$CLANG_VERSION + mkdir build && cd build + cmake -G "Unix Makefiles" -DCMAKE_PREFIX_PATH=/usr/include/llvm ../ + make -j$(nproc) + ln -s `pwd`/bin/include-what-you-use /usr/bin/include-what-you-use +} + +for cmd in "$@" +do + echo "Running: $cmd" + "$cmd" +done diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 4fba7009e80d..a714d1e6d369 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -99,6 +99,7 @@ set(KNOWN_WARNINGS -Wno-error=unused-function \ -Wno-error=unused-variable \ -Wno-error=uninitialized \ + -Wno-pessimizing-move \ -Wno-error=maybe-uninitialized \ -Wno-unknown-warning-option \ -Wno-strict-aliasing \ diff --git a/cpp/core/memory/MemoryManager.cc b/cpp/core/memory/MemoryManager.cc index 8ccf8af2ccbf..10cb174d09a9 100644 --- a/cpp/core/memory/MemoryManager.cc +++ b/cpp/core/memory/MemoryManager.cc @@ -24,6 +24,7 @@ Registry& memoryManagerFactories() { static Registry registry; return registry; } + Registry& memoryManagerReleasers() { static Registry registry; return registry; diff --git a/dev/builddeps-veloxbe.sh b/dev/builddeps-veloxbe.sh index 0cd52f3ff97e..236bb732fb92 100755 --- a/dev/builddeps-veloxbe.sh +++ b/dev/builddeps-veloxbe.sh @@ -237,7 +237,6 @@ function build_gluten_cpp { rm -rf build mkdir build cd build - GLUTEN_CMAKE_OPTIONS="-DBUILD_VELOX_BACKEND=ON \ -DCMAKE_BUILD_TYPE=$BUILD_TYPE \ -DVELOX_HOME=$VELOX_HOME \ diff --git a/dev/check.py b/dev/check.py index f1ec051c90d5..3e65503bdbdc 100755 --- a/dev/check.py +++ b/dev/check.py @@ -186,6 +186,22 @@ def tidy_command(commit, files, fix): return status +def iwyu_command(commit, files, fix): + files = [file for file in files if regex.match(r".*\.cc$", file)] + + if not files: + return 0 + files_str = ",".join(files) + status, stdout, stderr = util.run( + f"{SCRIPTS}/iwyu_tool.py -p cpp/build {files_str}" + ) + + if stdout != "": + print(stdout) + + return status + + def get_commit(files): if files == "commit": return "HEAD^" @@ -254,10 +270,10 @@ def parse_args(): global parser parser = argparse.ArgumentParser( formatter_class=argparse.RawTextHelpFormatter, - description="""Check format/header/tidy + description="""Check format/header/tidy/iwyu - check.py {format,header,tidy} {commit,branch} [--fix] - check.py {format,header,tidy} {tree} [--fix] PATH + check.py {format,header,tidy,iwyu} {commit,branch} [--fix] + check.py {format,header,tidy,iwyu} {tree} [--fix] PATH """, ) command = parser.add_subparsers(dest="command") @@ -266,6 +282,7 @@ def parse_args(): format_command_parser = add_check_command(command, "format") header_command_parser = add_check_command(command, "header") tidy_command_parser = add_check_command(command, "tidy") + iwyu_command_parser = add_check_command(command, "iwyu") parser.set_defaults(path="") parser.set_defaults(command="help") @@ -292,6 +309,10 @@ def tidy(args): return run_command(args, tidy_command) +def iwyu(args): + return run_command(args, iwyu_command) + + def main(): args = parse_args() return globals()[args.command](args)