From f356fd1b7b299ebcb4d0745c0feb9a21fb175863 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Mon, 1 Nov 2021 16:37:21 +0100 Subject: [PATCH 1/3] ARROW-14554: [C++][CI] Fix OSS-Fuzz build failure ARROW-14393 downgraded the bundled GTest version to a version that doesn't build on the OSS-Fuzz setup (and there doesn't seem to be an obvious fix or workaround around those build errors). Re-bump the bundled GTest version, and try to fix the issue in the Windows verification script differently. Also, add a Crossbow task for the OSS-Fuzz build check, since the Github Actions cron job failure went unnoticed. --- ci/conda_env_cpp.txt | 2 +- ci/docker/conda-cpp.dockerfile | 1 + cpp/thirdparty/versions.txt | 4 +- dev/release/verify-release-candidate.bat | 7 -- .../tasks/fuzz-tests/github.oss-fuzz.yml | 75 +++++++------------ dev/tasks/tasks.yml | 10 +++ 6 files changed, 41 insertions(+), 58 deletions(-) rename .github/workflows/cpp_cron.yml => dev/tasks/fuzz-tests/github.oss-fuzz.yml (53%) diff --git a/ci/conda_env_cpp.txt b/ci/conda_env_cpp.txt index d2ccb66a26b..277bf6a425b 100644 --- a/ci/conda_env_cpp.txt +++ b/ci/conda_env_cpp.txt @@ -26,7 +26,7 @@ gflags glog gmock>=1.10.0 grpc-cpp>=1.27.3 -gtest=1.10.0 +gtest>=1.10.0 libprotobuf libutf8proc lz4-c diff --git a/ci/docker/conda-cpp.dockerfile b/ci/docker/conda-cpp.dockerfile index ff31930c06c..40a855b5dd2 100644 --- a/ci/docker/conda-cpp.dockerfile +++ b/ci/docker/conda-cpp.dockerfile @@ -48,6 +48,7 @@ ENV ARROW_BUILD_TESTS=ON \ ARROW_WITH_SNAPPY=ON \ ARROW_WITH_ZLIB=ON \ ARROW_WITH_ZSTD=ON \ + GTest_SOURCE=BUNDLED \ PARQUET_BUILD_EXAMPLES=ON \ PARQUET_BUILD_EXECUTABLES=ON \ PARQUET_HOME=$CONDA_PREFIX diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt index 12da4645849..3528d4eff9d 100644 --- a/cpp/thirdparty/versions.txt +++ b/cpp/thirdparty/versions.txt @@ -53,8 +53,8 @@ ARROW_GOOGLE_CLOUD_CPP_BUILD_VERSION=v1.31.1 ARROW_GOOGLE_CLOUD_CPP_BUILD_SHA256_CHECKSUM=dc7cbf95b506a84b48cf71e0462985d262183edeaabdacaaee2109852394a609 ARROW_GRPC_BUILD_VERSION=v1.35.0 ARROW_GRPC_BUILD_SHA256_CHECKSUM=27dd2fc5c9809ddcde8eb6fa1fa278a3486566dfc28335fca13eb8df8bd3b958 -ARROW_GTEST_BUILD_VERSION=1.10.0 -ARROW_GTEST_BUILD_SHA256_CHECKSUM=9dc9157a9a1551ec7a7e43daea9a694a0bb5fb8bec81235d8a1e6ef64c716dcb +ARROW_GTEST_BUILD_VERSION=1.11.0 +ARROW_GTEST_BUILD_SHA256_CHECKSUM=b4870bf121ff7795ba20d20bcdd8627b8e088f2d1dab299a031c1034eddc93d5 ARROW_JEMALLOC_BUILD_VERSION=5.2.1 ARROW_JEMALLOC_BUILD_SHA256_CHECKSUM=34330e5ce276099e2e8950d9335db5a875689a4c6a56751ef3b1d8c537f887f6 ARROW_LZ4_BUILD_VERSION=v1.9.3 diff --git a/dev/release/verify-release-candidate.bat b/dev/release/verify-release-candidate.bat index fee8c01bc63..cc5f88379eb 100644 --- a/dev/release/verify-release-candidate.bat +++ b/dev/release/verify-release-candidate.bat @@ -88,17 +88,10 @@ cmake -G "%GENERATOR%" ^ -DCMAKE_BUILD_TYPE=%CONFIGURATION% ^ -DCMAKE_INSTALL_PREFIX=%ARROW_HOME% ^ -DCMAKE_UNITY_BUILD=ON ^ - -DGTest_SOURCE=BUNDLED ^ .. || exit /B cmake --build . --target INSTALL --config Release || exit /B 1 -@rem NOTE(wesm): Building googletest is flaky for me with ninja. Building it -@rem first fixes the problem - -@rem ninja googletest_ep || exit /B 1 -@rem ninja install || exit /B 1 - @rem Get testing datasets for Parquet unit tests git clone https://github.com/apache/parquet-testing.git %_VERIFICATION_DIR%\parquet-testing set PARQUET_TEST_DATA=%_VERIFICATION_DIR%\parquet-testing\data diff --git a/.github/workflows/cpp_cron.yml b/dev/tasks/fuzz-tests/github.oss-fuzz.yml similarity index 53% rename from .github/workflows/cpp_cron.yml rename to dev/tasks/fuzz-tests/github.oss-fuzz.yml index c031e5961cb..43958b51ac1 100644 --- a/.github/workflows/cpp_cron.yml +++ b/dev/tasks/fuzz-tests/github.oss-fuzz.yml @@ -1,4 +1,3 @@ - # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -16,70 +15,50 @@ # specific language governing permissions and limitations # under the License. -name: C++ Cron - -on: - push: - paths: - - '.github/workflows/cpp_cron.yml' - pull_request: - paths: - - '.github/workflows/cpp_cron.yml' - schedule: - - cron: | - 0 */12 * * * +{% import 'macros.jinja' as macros with context %} -env: - ARROW_ENABLE_TIMING_TESTS: OFF - DOCKER_VOLUME_PREFIX: ".docker/" - ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} - ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} +{{ macros.github_header() }} jobs: - - oss-fuzz: - name: OSS-Fuzz build check + test: + name: OSS-Fuzz build test runs-on: ubuntu-latest - if: ${{ !contains(github.event.pull_request.title, 'WIP') && github.repository == 'apache/arrow' }} - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - ubuntu: [18.04] + {% if env is defined %} + env: + {% for key, value in env.items() %} + {{ key }}: "{{ value }}" + {% endfor %} + {% endif %} steps: - - name: Checkout Arrow - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Fetch Submodules and Tags - shell: bash - run: ci/scripts/util_checkout.sh + {{ macros.github_checkout_arrow()|indent }} + + - name: Checkout OSS-Fuzz + # (not using the dedicated Action for this as its checkout + # location choice is relatively obscure) + run: | + git clone --depth=50 https://github.com/google/oss-fuzz.git + - name: Free Up Disk Space shell: bash - run: ci/scripts/util_cleanup.sh - - name: Checkout OSS-Fuzz - uses: actions/checkout@v1 - with: - path: oss-fuzz - repository: google/oss-fuzz - ref: master + run: arrow/ci/scripts/util_cleanup.sh + - name: Install dependencies - working-directory: ../oss-fuzz + working-directory: oss-fuzz run: | python3 -m pip install setuptools python3 -m pip install -r infra/ci/requirements.txt + - name: Build image - shell: bash - working-directory: ../oss-fuzz + working-directory: oss-fuzz run: | python3 infra/helper.py build_image --pull arrow + - name: Build fuzzers - shell: bash - working-directory: ../oss-fuzz + working-directory: oss-fuzz run: | python3 infra/helper.py build_fuzzers arrow `pwd`/../arrow + - name: Check build - shell: bash - working-directory: ../oss-fuzz + working-directory: oss-fuzz run: | python3 infra/helper.py check_build arrow diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 944bc9838c8..70926ef8af5 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -104,6 +104,9 @@ groups: example-cpp: - example-*cpp* + fuzz: + - test-*fuzz* + verify-rc: - verify-rc-* @@ -1185,6 +1188,13 @@ tasks: params: run: ubuntu-docs + ############################## Fuzz tests ################################# + + test-build-cpp-fuzz: + ci: github + template: fuzz-tests/github.oss-fuzz.yml + + ############################## vcpkg tests ################################## test-build-vcpkg-win: From 0bc448eec1c9306d224ed6d5134793aa6b60ba8b Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Tue, 2 Nov 2021 17:14:32 +0100 Subject: [PATCH 2/3] Fix Valgrind failure --- cpp/src/arrow/testing/random_test.cc | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/cpp/src/arrow/testing/random_test.cc b/cpp/src/arrow/testing/random_test.cc index 002c6c9b7e8..8545d1b27e3 100644 --- a/cpp/src/arrow/testing/random_test.cc +++ b/cpp/src/arrow/testing/random_test.cc @@ -35,9 +35,15 @@ namespace random { // Use short arrays since especially in debug mode, generating list(list()) is slow constexpr int64_t kExpectedLength = 24; -class RandomArrayTest : public ::testing::TestWithParam> { +struct RandomTestParam { + RandomTestParam(std::shared_ptr field) : field(std::move(field)) {} + + std::shared_ptr field; +}; + +class RandomArrayTest : public ::testing::TestWithParam { protected: - std::shared_ptr GetField() { return GetParam(); } + std::shared_ptr GetField() { return GetParam().field; } }; TEST_P(RandomArrayTest, GenerateArray) { @@ -144,7 +150,7 @@ auto values = ::testing::Values( INSTANTIATE_TEST_SUITE_P( TestRandomArrayGeneration, RandomArrayTest, values, [](const ::testing::TestParamInfo& info) { - return std::to_string(info.index) + info.param->name(); + return std::to_string(info.index) + info.param.field->name(); }); template From e035ed5d6eaaf013caf1f3d83d7d77a0b08db452 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Tue, 2 Nov 2021 17:42:56 +0100 Subject: [PATCH 3/3] Fix lint --- cpp/src/arrow/testing/random_test.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cpp/src/arrow/testing/random_test.cc b/cpp/src/arrow/testing/random_test.cc index 8545d1b27e3..6b20d821d48 100644 --- a/cpp/src/arrow/testing/random_test.cc +++ b/cpp/src/arrow/testing/random_test.cc @@ -36,7 +36,8 @@ namespace random { constexpr int64_t kExpectedLength = 24; struct RandomTestParam { - RandomTestParam(std::shared_ptr field) : field(std::move(field)) {} + RandomTestParam(std::shared_ptr field) // NOLINT runtime/explicit + : field(std::move(field)) {} std::shared_ptr field; };