From 9d1817419552be2aaaabe49f31f1b1798aed6ce6 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Thu, 11 Jul 2019 10:51:42 +0000 Subject: [PATCH 01/13] coverage: use LLVM coverage Signed-off-by: Lizan Zhou --- ci/do_ci.sh | 19 +++---------- test/run_envoy_bazel_llvm_coverage.sh | 40 +++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 15 deletions(-) create mode 100755 test/run_envoy_bazel_llvm_coverage.sh diff --git a/ci/do_ci.sh b/ci/do_ci.sh index ec4d05496390f..877bc252188ec 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -241,24 +241,13 @@ elif [[ "$CI_TARGET" == "bazel.api" ]]; then @envoy_api//tools:tap2pcap_test exit 0 elif [[ "$CI_TARGET" == "bazel.coverage" ]]; then - setup_gcc_toolchain + setup_clang_toolchain echo "bazel coverage build with tests ${TEST_TARGETS}" - # gcovr is a pain to run with `bazel run`, so package it up into a - # relocatable and hermetic-ish .par file. - bazel build --python_version=PY2 @com_github_gcovr_gcovr//:gcovr.par - export GCOVR="/tmp/gcovr.par" - cp -f "${ENVOY_SRCDIR}/bazel-bin/external/com_github_gcovr_gcovr/gcovr.par" ${GCOVR} - - # Reduce the amount of memory and number of cores Bazel tries to use to - # prevent it from launching too many subprocesses. This should prevent the - # system from running out of memory and killing tasks. See discussion on - # https://github.com/envoyproxy/envoy/pull/5611. - # TODO(akonradi): use --local_cpu_resources flag once Bazel has a release - # after 0.21. - [ -z "$CIRCLECI" ] || export BAZEL_BUILD_OPTIONS="${BAZEL_BUILD_OPTIONS} --local_resources=12288,4,1" + # LLVM coverage is a memory hog too. + [ -z "$CIRCLECI" ] || export BAZEL_BUILD_OPTIONS="${BAZEL_BUILD_OPTIONS} --local_cpu_resources=6" - test/run_envoy_bazel_coverage.sh ${TEST_TARGETS} + test/run_envoy_bazel_llvm_coverage.sh ${TEST_TARGETS} collect_build_profile coverage exit 0 elif [[ "$CI_TARGET" == "bazel.clang_tidy" ]]; then diff --git a/test/run_envoy_bazel_llvm_coverage.sh b/test/run_envoy_bazel_llvm_coverage.sh new file mode 100755 index 0000000000000..a961385e02a51 --- /dev/null +++ b/test/run_envoy_bazel_llvm_coverage.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +set -e + +[[ -z "${SRCDIR}" ]] && SRCDIR="${PWD}" + +echo "Starting run_envoy_bazel_coverage.sh..." +echo " PWD=$(pwd)" +echo " SRCDIR=${SRCDIR}" +echo " VALIDATE_COVERAGE=${VALIDATE_COVERAGE}" + +# This is the target that will be run to generate coverage data. It can be overridden by consumer +# projects that want to run coverage on a different/combined target. +# Command-line arguments take precedence over ${COVERAGE_TARGET}. +if [[ $# -gt 0 ]]; then + COVERAGE_TARGETS=$* +elif [[ -n "${COVERAGE_TARGET}" ]]; then + COVERAGE_TARGETS=${COVERAGE_TARGET} +else + COVERAGE_TARGETS=//test/... +fi + +rm -rf $(find -L bazel-bin -name "test-*.profraw") + +BAZEL_USE_LLVM_NATIVE_COVERAGE=1 GCOV=llvm-profdata bazel coverage \ + ${BAZEL_BUILD_OPTIONS} -c fastbuild --copt=-DNDEBUG --instrumentation_filter=//source/...,//include/... \ + ${COVERAGE_TARGETS} + +COVERAGE_DIR="${SRCDIR}"/generated/coverage +mkdir -p "${COVERAGE_DIR}" + +echo "Merging profile data..." +llvm-profdata merge -sparse $(find -L bazel-out -name coverage.dat) -o ${COVERAGE_DIR}/coverage.profdata + +echo "Generating report..." +llvm-cov show bazel-bin/source/exe/envoy-static -instr-profile=${COVERAGE_DIR}/coverage.profdata \ + -ignore-filename-regex='(/external/|pb\.(validate\.)?(h|cc)|/chromium_url/)' -output-dir=${COVERAGE_DIR} -format=html +sed -i -e 's|>bazel-out/[^/]*/bin/\([^/]*\)/[^<]*/_virtual_includes/[^/]*|>\1|g' "${COVERAGE_DIR}/index.html" + +[[ -z "${ENVOY_COVERAGE_DIR}" ]] || rsync -av "${COVERAGE_DIR}"/ "${ENVOY_COVERAGE_DIR}" From aa03694c1655d4d3323c7a7a75602781ecc14286 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Fri, 12 Jul 2019 01:02:39 +0000 Subject: [PATCH 02/13] fix Signed-off-by: Lizan Zhou --- test/run_envoy_bazel_llvm_coverage.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/test/run_envoy_bazel_llvm_coverage.sh b/test/run_envoy_bazel_llvm_coverage.sh index a961385e02a51..4dfcc9b964c69 100755 --- a/test/run_envoy_bazel_llvm_coverage.sh +++ b/test/run_envoy_bazel_llvm_coverage.sh @@ -22,9 +22,10 @@ fi rm -rf $(find -L bazel-bin -name "test-*.profraw") -BAZEL_USE_LLVM_NATIVE_COVERAGE=1 GCOV=llvm-profdata bazel coverage \ - ${BAZEL_BUILD_OPTIONS} -c fastbuild --copt=-DNDEBUG --instrumentation_filter=//source/...,//include/... \ - ${COVERAGE_TARGETS} +BAZEL_USE_LLVM_NATIVE_COVERAGE=1 GCOV=llvm-profdata bazel coverage ${BAZEL_BUILD_OPTIONS} \ + -c fastbuild --copt=-DNDEBUG --instrumentation_filter=//source/...,//include/... --dynamic_mode=off \ + --strategy=TestRunner=local --test_sharding_strategy=disabled \ + --test_env=HEAPCHECK= --test_filter='-QuicPlatformTest.QuicStackTraceTest' ${COVERAGE_TARGETS} COVERAGE_DIR="${SRCDIR}"/generated/coverage mkdir -p "${COVERAGE_DIR}" @@ -34,7 +35,8 @@ llvm-profdata merge -sparse $(find -L bazel-out -name coverage.dat) -o ${COVERAG echo "Generating report..." llvm-cov show bazel-bin/source/exe/envoy-static -instr-profile=${COVERAGE_DIR}/coverage.profdata \ - -ignore-filename-regex='(/external/|pb\.(validate\.)?(h|cc)|/chromium_url/)' -output-dir=${COVERAGE_DIR} -format=html + -ignore-filename-regex='(/external/|pb\.(validate\.)?(h|cc)|/chromium_url/|/tmp)' -output-dir=${COVERAGE_DIR} -format=html +sed -i -e 's|>proc/self/cwd/|>|g' "${COVERAGE_DIR}/index.html" sed -i -e 's|>bazel-out/[^/]*/bin/\([^/]*\)/[^<]*/_virtual_includes/[^/]*|>\1|g' "${COVERAGE_DIR}/index.html" [[ -z "${ENVOY_COVERAGE_DIR}" ]] || rsync -av "${COVERAGE_DIR}"/ "${ENVOY_COVERAGE_DIR}" From 337ed68631bf185015614e27f702809fa8f76dec Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Fri, 12 Jul 2019 17:45:41 +0000 Subject: [PATCH 03/13] single binary Signed-off-by: Lizan Zhou --- test/run_envoy_bazel_llvm_coverage.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test/run_envoy_bazel_llvm_coverage.sh b/test/run_envoy_bazel_llvm_coverage.sh index 4dfcc9b964c69..c25706962abe7 100755 --- a/test/run_envoy_bazel_llvm_coverage.sh +++ b/test/run_envoy_bazel_llvm_coverage.sh @@ -22,10 +22,14 @@ fi rm -rf $(find -L bazel-bin -name "test-*.profraw") +# Make sure //test/coverage:coverage_tests is up-to-date. +SCRIPT_DIR="$(realpath "$(dirname "$0")")" +(BAZEL_BIN="${BAZEL_COVERAGE}" "${SCRIPT_DIR}"/coverage/gen_build.sh ${COVERAGE_TARGETS}) + BAZEL_USE_LLVM_NATIVE_COVERAGE=1 GCOV=llvm-profdata bazel coverage ${BAZEL_BUILD_OPTIONS} \ -c fastbuild --copt=-DNDEBUG --instrumentation_filter=//source/...,//include/... --dynamic_mode=off \ - --strategy=TestRunner=local --test_sharding_strategy=disabled \ - --test_env=HEAPCHECK= --test_filter='-QuicPlatformTest.QuicStackTraceTest' ${COVERAGE_TARGETS} + --strategy=TestRunner=local --test_sharding_strategy=disabled --test_arg="--log-path /dev/null" --test_arg="-l trace" \ + --test_env=HEAPCHECK= --test_filter='-QuicPlatformTest.QuicStackTraceTest' //test/coverage:coverage_tests COVERAGE_DIR="${SRCDIR}"/generated/coverage mkdir -p "${COVERAGE_DIR}" From 769f9d36f62c5c06596ecec5938c5ccf61e12905 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Fri, 12 Jul 2019 18:48:48 +0000 Subject: [PATCH 04/13] no gcov Signed-off-by: Lizan Zhou --- test/run_envoy_bazel_llvm_coverage.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/run_envoy_bazel_llvm_coverage.sh b/test/run_envoy_bazel_llvm_coverage.sh index c25706962abe7..8905e86ae0ad8 100755 --- a/test/run_envoy_bazel_llvm_coverage.sh +++ b/test/run_envoy_bazel_llvm_coverage.sh @@ -24,7 +24,7 @@ rm -rf $(find -L bazel-bin -name "test-*.profraw") # Make sure //test/coverage:coverage_tests is up-to-date. SCRIPT_DIR="$(realpath "$(dirname "$0")")" -(BAZEL_BIN="${BAZEL_COVERAGE}" "${SCRIPT_DIR}"/coverage/gen_build.sh ${COVERAGE_TARGETS}) +NO_GCOV=1 "${SCRIPT_DIR}"/coverage/gen_build.sh ${COVERAGE_TARGETS} BAZEL_USE_LLVM_NATIVE_COVERAGE=1 GCOV=llvm-profdata bazel coverage ${BAZEL_BUILD_OPTIONS} \ -c fastbuild --copt=-DNDEBUG --instrumentation_filter=//source/...,//include/... --dynamic_mode=off \ @@ -38,7 +38,7 @@ echo "Merging profile data..." llvm-profdata merge -sparse $(find -L bazel-out -name coverage.dat) -o ${COVERAGE_DIR}/coverage.profdata echo "Generating report..." -llvm-cov show bazel-bin/source/exe/envoy-static -instr-profile=${COVERAGE_DIR}/coverage.profdata \ +llvm-cov show bazel-bin/test/coverage/coverage_tests -instr-profile=${COVERAGE_DIR}/coverage.profdata \ -ignore-filename-regex='(/external/|pb\.(validate\.)?(h|cc)|/chromium_url/|/tmp)' -output-dir=${COVERAGE_DIR} -format=html sed -i -e 's|>proc/self/cwd/|>|g' "${COVERAGE_DIR}/index.html" sed -i -e 's|>bazel-out/[^/]*/bin/\([^/]*\)/[^<]*/_virtual_includes/[^/]*|>\1|g' "${COVERAGE_DIR}/index.html" From d00d550ad2026f270a44b4e0e2ace75ac2dcb0de Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Fri, 12 Jul 2019 21:41:24 +0000 Subject: [PATCH 05/13] timeout Signed-off-by: Lizan Zhou --- test/run_envoy_bazel_llvm_coverage.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/run_envoy_bazel_llvm_coverage.sh b/test/run_envoy_bazel_llvm_coverage.sh index 8905e86ae0ad8..04c1494df7106 100755 --- a/test/run_envoy_bazel_llvm_coverage.sh +++ b/test/run_envoy_bazel_llvm_coverage.sh @@ -27,7 +27,7 @@ SCRIPT_DIR="$(realpath "$(dirname "$0")")" NO_GCOV=1 "${SCRIPT_DIR}"/coverage/gen_build.sh ${COVERAGE_TARGETS} BAZEL_USE_LLVM_NATIVE_COVERAGE=1 GCOV=llvm-profdata bazel coverage ${BAZEL_BUILD_OPTIONS} \ - -c fastbuild --copt=-DNDEBUG --instrumentation_filter=//source/...,//include/... --dynamic_mode=off \ + -c fastbuild --copt=-DNDEBUG --instrumentation_filter=//source/...,//include/... --dynamic_mode=off --test_timeout=4000 \ --strategy=TestRunner=local --test_sharding_strategy=disabled --test_arg="--log-path /dev/null" --test_arg="-l trace" \ --test_env=HEAPCHECK= --test_filter='-QuicPlatformTest.QuicStackTraceTest' //test/coverage:coverage_tests From e6490ff2daac32f9f1808316ffe286d406ea7bbd Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Fri, 12 Jul 2019 22:32:21 +0000 Subject: [PATCH 06/13] filter test Signed-off-by: Lizan Zhou --- test/run_envoy_bazel_llvm_coverage.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/run_envoy_bazel_llvm_coverage.sh b/test/run_envoy_bazel_llvm_coverage.sh index 04c1494df7106..8a7fca9d7b52b 100755 --- a/test/run_envoy_bazel_llvm_coverage.sh +++ b/test/run_envoy_bazel_llvm_coverage.sh @@ -39,7 +39,7 @@ llvm-profdata merge -sparse $(find -L bazel-out -name coverage.dat) -o ${COVERAG echo "Generating report..." llvm-cov show bazel-bin/test/coverage/coverage_tests -instr-profile=${COVERAGE_DIR}/coverage.profdata \ - -ignore-filename-regex='(/external/|pb\.(validate\.)?(h|cc)|/chromium_url/|/tmp)' -output-dir=${COVERAGE_DIR} -format=html + -ignore-filename-regex='(/external/|pb\.(validate\.)?(h|cc)|/chromium_url/|/test/|/tmp)' -output-dir=${COVERAGE_DIR} -format=html sed -i -e 's|>proc/self/cwd/|>|g' "${COVERAGE_DIR}/index.html" sed -i -e 's|>bazel-out/[^/]*/bin/\([^/]*\)/[^<]*/_virtual_includes/[^/]*|>\1|g' "${COVERAGE_DIR}/index.html" From 8e5906dd8222abab5c0f6fd98e461aa295fbe2c1 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Sat, 13 Jul 2019 00:07:38 +0000 Subject: [PATCH 07/13] validate coverage Signed-off-by: Lizan Zhou --- ci/do_ci.sh | 2 +- test/run_envoy_bazel_coverage.sh | 99 +++++---------------------- test/run_envoy_bazel_llvm_coverage.sh | 46 ------------- 3 files changed, 18 insertions(+), 129 deletions(-) delete mode 100755 test/run_envoy_bazel_llvm_coverage.sh diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 877bc252188ec..d9a16221574a7 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -247,7 +247,7 @@ elif [[ "$CI_TARGET" == "bazel.coverage" ]]; then # LLVM coverage is a memory hog too. [ -z "$CIRCLECI" ] || export BAZEL_BUILD_OPTIONS="${BAZEL_BUILD_OPTIONS} --local_cpu_resources=6" - test/run_envoy_bazel_llvm_coverage.sh ${TEST_TARGETS} + test/run_envoy_bazel_coverage.sh ${TEST_TARGETS} collect_build_profile coverage exit 0 elif [[ "$CI_TARGET" == "bazel.clang_tidy" ]]; then diff --git a/test/run_envoy_bazel_coverage.sh b/test/run_envoy_bazel_coverage.sh index 6c1d22de56b60..ea93c116150f4 100755 --- a/test/run_envoy_bazel_coverage.sh +++ b/test/run_envoy_bazel_coverage.sh @@ -3,21 +3,11 @@ set -e [[ -z "${SRCDIR}" ]] && SRCDIR="${PWD}" -[[ -z "${GCOVR_DIR}" ]] && GCOVR_DIR="${SRCDIR}/bazel-$(basename "${SRCDIR}")" -[[ -z "${TESTLOGS_DIR}" ]] && TESTLOGS_DIR="${SRCDIR}/bazel-testlogs" -[[ -z "${BAZEL_COVERAGE}" ]] && BAZEL_COVERAGE=bazel -[[ -z "${GCOVR}" ]] && GCOVR=gcovr -[[ -z "${WORKSPACE}" ]] && WORKSPACE=envoy [[ -z "${VALIDATE_COVERAGE}" ]] && VALIDATE_COVERAGE=true echo "Starting run_envoy_bazel_coverage.sh..." echo " PWD=$(pwd)" echo " SRCDIR=${SRCDIR}" -echo " GCOVR_DIR=${GCOVR_DIR}" -echo " TESTLOGS_DIR=${TESTLOGS_DIR}" -echo " BAZEL_COVERAGE=${BAZEL_COVERAGE}" -echo " GCOVR=${GCOVR}" -echo " WORKSPACE=${WORKSPACE}" echo " VALIDATE_COVERAGE=${VALIDATE_COVERAGE}" # This is the target that will be run to generate coverage data. It can be overridden by consumer @@ -31,90 +21,35 @@ else COVERAGE_TARGETS=//test/... fi -# This is where we are going to copy the .gcno files into. -GCNO_ROOT=bazel-out/k8-dbg/bin/test/coverage/coverage_tests.runfiles/"${WORKSPACE}" -echo " GCNO_ROOT=${GCNO_ROOT}" - -echo "Cleaning .gcno from previous coverage runs..." -NUM_PREVIOUS_GCNO_FILES=0 -for f in $(find -L "${GCNO_ROOT}" -name "*.gcno") -do - rm -f "${f}" - let NUM_PREVIOUS_GCNO_FILES=NUM_PREVIOUS_GCNO_FILES+1 -done -echo "Cleanup completed. ${NUM_PREVIOUS_GCNO_FILES} files deleted." - # Make sure //test/coverage:coverage_tests is up-to-date. SCRIPT_DIR="$(realpath "$(dirname "$0")")" -(BAZEL_BIN="${BAZEL_COVERAGE}" "${SCRIPT_DIR}"/coverage/gen_build.sh ${COVERAGE_TARGETS}) - -echo "Cleaning .gcda/.gcov from previous coverage runs..." -NUM_PREVIOUS_GCOV_FILES=0 -for f in $(find -L "${GCOVR_DIR}" -name "*.gcda" -o -name "*.gcov") -do - rm -f "${f}" - let NUM_PREVIOUS_GCOV_FILES=NUM_PREVIOUS_GCOV_FILES+1 -done -echo "Cleanup completed. ${NUM_PREVIOUS_GCOV_FILES} files deleted." +NO_GCOV=1 "${SCRIPT_DIR}"/coverage/gen_build.sh ${COVERAGE_TARGETS} -# Force dbg for path consistency later, don't include debug code in coverage. -BAZEL_BUILD_OPTIONS="${BAZEL_BUILD_OPTIONS} -c dbg --copt=-DNDEBUG" - -# Run all tests under "bazel test", no sandbox. We're going to generate the -# .gcda inplace in the bazel-out/ directory. This is in contrast to the "bazel -# coverage" method, which is currently broken for C++ (see -# https://github.com/bazelbuild/bazel/issues/1118). This works today as we have -# a single coverage test binary and do not require the "bazel coverage" support -# for collecting multiple traces and glueing them together. -"${BAZEL_COVERAGE}" test //test/coverage:coverage_tests ${BAZEL_BUILD_OPTIONS} \ - --cache_test_results=no --cxxopt="--coverage" --cxxopt="-DENVOY_CONFIG_COVERAGE=1" \ - --linkopt="--coverage" --define ENVOY_CONFIG_COVERAGE=1 --test_output=streamed \ - --strategy=Genrule=standalone --spawn_strategy=standalone --test_timeout=4000 \ - --test_arg="--log-path /dev/null" --test_arg="-l trace" - -# The Bazel build has a lot of whack in it, in particular generated files, headers from external -# deps, etc. So, we exclude this from gcov to avoid false reporting of these files in the html and -# stats. The #foo# pattern is because gcov produces files such as -# bazel-out#local-fastbuild#bin#external#spdlog_git#_virtual_includes#spdlog#spdlog#details#pattern_formatter_impl.h.gcov. -# To find these while modifying this regex, perform a gcov run with -k set. -[[ -z "${GCOVR_EXCLUDE_REGEX}" ]] && GCOVR_EXCLUDE_REGEX=".*pb\\..*|.*#test#.*|external#.*|.*#external#.*|.*#prebuilt#.*|.*#config_validation#.*|.*#chromium_url#.*" -[[ -z "${GCOVR_EXCLUDE_DIR}" ]] && GCOVR_EXCLUDE_DIR=".*/external/.*" +BAZEL_USE_LLVM_NATIVE_COVERAGE=1 GCOV=llvm-profdata bazel coverage ${BAZEL_BUILD_OPTIONS} \ + -c fastbuild --copt=-DNDEBUG --instrumentation_filter=//source/...,//include/... --dynamic_mode=off --test_timeout=4000 \ + --strategy=TestRunner=local --test_sharding_strategy=disabled --test_arg="--log-path /dev/null" --test_arg="-l trace" \ + --test_env=HEAPCHECK= --test_filter='-QuicPlatformTest.QuicStackTraceTest' //test/coverage:coverage_tests COVERAGE_DIR="${SRCDIR}"/generated/coverage mkdir -p "${COVERAGE_DIR}" -COVERAGE_SUMMARY="${COVERAGE_DIR}/coverage_summary.txt" - -# Copy .gcno objects into the same location that we find the .gcda. -# TODO(htuch): Should use rsync, but there are some symlink loops to fight. -echo "Finding and copying .gcno files in GCOVR_DIR: ${GCOVR_DIR}" -mkdir -p ${GCNO_ROOT} -NUM_GCNO_FILES=0 -for f in $(find -L bazel-out/ -name "*.gcno") -do - cp --parents "$f" ${GCNO_ROOT}/ - let NUM_GCNO_FILES=NUM_GCNO_FILES+1 -done -echo "OK: copied ${NUM_GCNO_FILES} .gcno files" -# gcovr is extremely picky about where it is run and where the paths of the -# original source are relative to its execution location. -cd -P "${GCOVR_DIR}" -echo "Running gcovr in $(pwd)..." -time "${GCOVR}" -v --gcov-exclude="${GCOVR_EXCLUDE_REGEX}" \ - --exclude-directories="${GCOVR_EXCLUDE_DIR}" -r . \ - --html --html-details --exclude-unreachable-branches --print-summary \ - -o "${COVERAGE_DIR}"/coverage.html > "${COVERAGE_SUMMARY}" +COVERAGE_IGNORE_REGEX="(/external/|pb\.(validate\.)?(h|cc)|/chromium_url/|/test/|/tmp)" +COVERAGE_BINARY="bazel-bin/test/coverage/coverage_tests" +COVERAGE_DATA="bazel-out/k8-fastbuild/testlogs/test/coverage/coverage_tests/coverage.dat" -# Clean up the generated test/coverage/BUILD file: subsequent bazel invocations -# can choke on it if it references things that changed since the last coverage -# run. -rm "${SRCDIR}"/test/coverage/BUILD +echo "Generating report..." +llvm-cov show "${COVERAGE_BINARY}" -instr-profile="${COVERAGE_DATA}" \ + -ignore-filename-regex="${COVERAGE_IGNORE_REGEX}" -output-dir=${COVERAGE_DIR} -format=html +sed -i -e 's|>proc/self/cwd/|>|g' "${COVERAGE_DIR}/index.html" +sed -i -e 's|>bazel-out/[^/]*/bin/\([^/]*\)/[^<]*/_virtual_includes/[^/]*|>\1|g' "${COVERAGE_DIR}/index.html" [[ -z "${ENVOY_COVERAGE_DIR}" ]] || rsync -av "${COVERAGE_DIR}"/ "${ENVOY_COVERAGE_DIR}" if [ "$VALIDATE_COVERAGE" == "true" ] then - COVERAGE_VALUE=$(grep -Po 'lines: \K(\d|\.)*' "${COVERAGE_SUMMARY}") + COVERAGE_VALUE=$(llvm-cov export "${COVERAGE_BINARY}" -instr-profile="${COVERAGE_DATA}" \ + -ignore-filename-regex="${COVERAGE_IGNORE_REGEX}" -summary-only | \ + python3 -c "import sys, json; print(json.load(sys.stdin)['data'][0]['totals']['lines']['percent'])") COVERAGE_THRESHOLD=97.5 COVERAGE_FAILED=$(echo "${COVERAGE_VALUE}<${COVERAGE_THRESHOLD}" | bc) if test ${COVERAGE_FAILED} -eq 1; then @@ -123,5 +58,5 @@ then else echo Code coverage ${COVERAGE_VALUE} is good and higher than limit of ${COVERAGE_THRESHOLD} fi - echo "HTML coverage report is in ${COVERAGE_DIR}/coverage.html" fi +echo "HTML coverage report is in ${COVERAGE_DIR}/index.html" diff --git a/test/run_envoy_bazel_llvm_coverage.sh b/test/run_envoy_bazel_llvm_coverage.sh deleted file mode 100755 index 8a7fca9d7b52b..0000000000000 --- a/test/run_envoy_bazel_llvm_coverage.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash - -set -e - -[[ -z "${SRCDIR}" ]] && SRCDIR="${PWD}" - -echo "Starting run_envoy_bazel_coverage.sh..." -echo " PWD=$(pwd)" -echo " SRCDIR=${SRCDIR}" -echo " VALIDATE_COVERAGE=${VALIDATE_COVERAGE}" - -# This is the target that will be run to generate coverage data. It can be overridden by consumer -# projects that want to run coverage on a different/combined target. -# Command-line arguments take precedence over ${COVERAGE_TARGET}. -if [[ $# -gt 0 ]]; then - COVERAGE_TARGETS=$* -elif [[ -n "${COVERAGE_TARGET}" ]]; then - COVERAGE_TARGETS=${COVERAGE_TARGET} -else - COVERAGE_TARGETS=//test/... -fi - -rm -rf $(find -L bazel-bin -name "test-*.profraw") - -# Make sure //test/coverage:coverage_tests is up-to-date. -SCRIPT_DIR="$(realpath "$(dirname "$0")")" -NO_GCOV=1 "${SCRIPT_DIR}"/coverage/gen_build.sh ${COVERAGE_TARGETS} - -BAZEL_USE_LLVM_NATIVE_COVERAGE=1 GCOV=llvm-profdata bazel coverage ${BAZEL_BUILD_OPTIONS} \ - -c fastbuild --copt=-DNDEBUG --instrumentation_filter=//source/...,//include/... --dynamic_mode=off --test_timeout=4000 \ - --strategy=TestRunner=local --test_sharding_strategy=disabled --test_arg="--log-path /dev/null" --test_arg="-l trace" \ - --test_env=HEAPCHECK= --test_filter='-QuicPlatformTest.QuicStackTraceTest' //test/coverage:coverage_tests - -COVERAGE_DIR="${SRCDIR}"/generated/coverage -mkdir -p "${COVERAGE_DIR}" - -echo "Merging profile data..." -llvm-profdata merge -sparse $(find -L bazel-out -name coverage.dat) -o ${COVERAGE_DIR}/coverage.profdata - -echo "Generating report..." -llvm-cov show bazel-bin/test/coverage/coverage_tests -instr-profile=${COVERAGE_DIR}/coverage.profdata \ - -ignore-filename-regex='(/external/|pb\.(validate\.)?(h|cc)|/chromium_url/|/test/|/tmp)' -output-dir=${COVERAGE_DIR} -format=html -sed -i -e 's|>proc/self/cwd/|>|g' "${COVERAGE_DIR}/index.html" -sed -i -e 's|>bazel-out/[^/]*/bin/\([^/]*\)/[^<]*/_virtual_includes/[^/]*|>\1|g' "${COVERAGE_DIR}/index.html" - -[[ -z "${ENVOY_COVERAGE_DIR}" ]] || rsync -av "${COVERAGE_DIR}"/ "${ENVOY_COVERAGE_DIR}" From c68d74c9ff90579d00802ec03eb15e4b913aa995 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Sat, 13 Jul 2019 01:05:58 +0000 Subject: [PATCH 08/13] demangler Signed-off-by: Lizan Zhou --- test/run_envoy_bazel_coverage.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/run_envoy_bazel_coverage.sh b/test/run_envoy_bazel_coverage.sh index ea93c116150f4..0ff2d955018df 100755 --- a/test/run_envoy_bazel_coverage.sh +++ b/test/run_envoy_bazel_coverage.sh @@ -38,7 +38,7 @@ COVERAGE_BINARY="bazel-bin/test/coverage/coverage_tests" COVERAGE_DATA="bazel-out/k8-fastbuild/testlogs/test/coverage/coverage_tests/coverage.dat" echo "Generating report..." -llvm-cov show "${COVERAGE_BINARY}" -instr-profile="${COVERAGE_DATA}" \ +llvm-cov show "${COVERAGE_BINARY}" -instr-profile="${COVERAGE_DATA}" -Xdemangler=c++filt \ -ignore-filename-regex="${COVERAGE_IGNORE_REGEX}" -output-dir=${COVERAGE_DIR} -format=html sed -i -e 's|>proc/self/cwd/|>|g' "${COVERAGE_DIR}/index.html" sed -i -e 's|>bazel-out/[^/]*/bin/\([^/]*\)/[^<]*/_virtual_includes/[^/]*|>\1|g' "${COVERAGE_DIR}/index.html" From b697f0394d6261a621f811aebec51615f8217cf2 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Sat, 13 Jul 2019 07:17:51 +0000 Subject: [PATCH 09/13] tweak Signed-off-by: Lizan Zhou --- test/run_envoy_bazel_coverage.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/run_envoy_bazel_coverage.sh b/test/run_envoy_bazel_coverage.sh index 0ff2d955018df..b08510acc36b4 100755 --- a/test/run_envoy_bazel_coverage.sh +++ b/test/run_envoy_bazel_coverage.sh @@ -26,9 +26,10 @@ SCRIPT_DIR="$(realpath "$(dirname "$0")")" NO_GCOV=1 "${SCRIPT_DIR}"/coverage/gen_build.sh ${COVERAGE_TARGETS} BAZEL_USE_LLVM_NATIVE_COVERAGE=1 GCOV=llvm-profdata bazel coverage ${BAZEL_BUILD_OPTIONS} \ - -c fastbuild --copt=-DNDEBUG --instrumentation_filter=//source/...,//include/... --dynamic_mode=off --test_timeout=4000 \ - --strategy=TestRunner=local --test_sharding_strategy=disabled --test_arg="--log-path /dev/null" --test_arg="-l trace" \ - --test_env=HEAPCHECK= --test_filter='-QuicPlatformTest.QuicStackTraceTest' //test/coverage:coverage_tests + -c fastbuild --copt=-DNDEBUG --instrumentation_filter=//source/...,//include/... \ + --test_timeout=2000 --cxxopt="-DENVOY_CONFIG_COVERAGE=1" --test_output=streamed \ + --test_arg="--log-path /dev/null" --test_arg="-l trace" --test_env=HEAPCHECK= \ + --test_filter='-QuicPlatformTest.QuicStackTraceTest' //test/coverage:coverage_tests COVERAGE_DIR="${SRCDIR}"/generated/coverage mkdir -p "${COVERAGE_DIR}" From 03436270f26540e1a92b5205b1e41e5d9ff7a804 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Sat, 13 Jul 2019 07:21:20 +0000 Subject: [PATCH 10/13] threshold Signed-off-by: Lizan Zhou --- test/run_envoy_bazel_coverage.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/run_envoy_bazel_coverage.sh b/test/run_envoy_bazel_coverage.sh index b08510acc36b4..1f3c74720090f 100755 --- a/test/run_envoy_bazel_coverage.sh +++ b/test/run_envoy_bazel_coverage.sh @@ -51,7 +51,7 @@ then COVERAGE_VALUE=$(llvm-cov export "${COVERAGE_BINARY}" -instr-profile="${COVERAGE_DATA}" \ -ignore-filename-regex="${COVERAGE_IGNORE_REGEX}" -summary-only | \ python3 -c "import sys, json; print(json.load(sys.stdin)['data'][0]['totals']['lines']['percent'])") - COVERAGE_THRESHOLD=97.5 + COVERAGE_THRESHOLD=97.0 COVERAGE_FAILED=$(echo "${COVERAGE_VALUE}<${COVERAGE_THRESHOLD}" | bc) if test ${COVERAGE_FAILED} -eq 1; then echo Code coverage ${COVERAGE_VALUE} is lower than limit of ${COVERAGE_THRESHOLD} From dd8a863ed39ed90845a9a0acb9786cde5f3e030d Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Sun, 14 Jul 2019 22:47:50 +0000 Subject: [PATCH 11/13] remove gcovr deps Signed-off-by: Lizan Zhou --- DEVELOPER.md | 4 +++- bazel/README.md | 6 ++---- bazel/external/gcovr.BUILD | 24 ------------------------ bazel/repositories.bzl | 14 -------------- bazel/repository_locations.bzl | 12 ------------ ci/build_setup.sh | 15 --------------- 6 files changed, 5 insertions(+), 70 deletions(-) delete mode 100644 bazel/external/gcovr.BUILD diff --git a/DEVELOPER.md b/DEVELOPER.md index ee76c57463108..465644c0e02ce 100644 --- a/DEVELOPER.md +++ b/DEVELOPER.md @@ -2,7 +2,9 @@ Envoy is built using the Bazel build system. CircleCI builds, tests, and runs coverage against all pull requests and the master branch. -To get started building Envoy locally, see the [Bazel quick start](https://github.com/envoyproxy/envoy/blob/master/bazel/README.md#quick-start-bazel-build-for-developers). To run tests, there are Bazel [targets](https://github.com/envoyproxy/envoy/blob/master/bazel/README.md#testing-envoy-with-bazel) for Google Test. To generate a coverage report, use the tooling for [gcovr](https://github.com/envoyproxy/envoy/blob/master/bazel/README.md#coverage-builds). +To get started building Envoy locally, see the [Bazel quick start](https://github.com/envoyproxy/envoy/blob/master/bazel/README.md#quick-start-bazel-build-for-developers). +To run tests, there are Bazel [targets](https://github.com/envoyproxy/envoy/blob/master/bazel/README.md#testing-envoy-with-bazel) for Google Test. +To generate a coverage report, there is a [coverage build script](https://github.com/envoyproxy/envoy/blob/master/bazel/README.md#coverage-builds). If you plan to contribute to Envoy, you may find it useful to install the Envoy [development support toolchain](https://github.com/envoyproxy/envoy/blob/master/support/README.md), which helps automate parts of the development process, particularly those involving code review. diff --git a/bazel/README.md b/bazel/README.md index 25743ffdc0974..657177dbccfd6 100644 --- a/bazel/README.md +++ b/bazel/README.md @@ -435,10 +435,8 @@ https://github.com/bazelbuild/bazel/issues/2805. # Coverage builds -To generate coverage results, make sure you have -[`gcovr`](https://github.com/gcovr/gcovr) 3.3 in your `PATH` (or set `GCOVR` to -point at it) and are using a GCC toolchain (clang does not work currently, see -https://github.com/envoyproxy/envoy/issues/1000). Then run: +To generate coverage results, make sure you are using a clang toolchain and have `llvm-cov` and +`llvm-profdata` in your `PATH`. Then run: ``` test/run_envoy_bazel_coverage.sh diff --git a/bazel/external/gcovr.BUILD b/bazel/external/gcovr.BUILD deleted file mode 100644 index 0215a25da848f..0000000000000 --- a/bazel/external/gcovr.BUILD +++ /dev/null @@ -1,24 +0,0 @@ -licenses(["notice"]) # Apache 2 - -load("@subpar//:subpar.bzl", "par_binary") - -# gcovr is difficult to run from a CI environment because it has hard -# assumptions about its local working directory, which interact poorly -# with `bazel run`. To make gcovr more mobile, we package it into a -# .par file (a mostly-hermetic "Python binary"). -par_binary( - name = "gcovr", - srcs = [":renamed_gcovr.py"], - main = ":renamed_gcovr.py", - python_version = "PY2", - visibility = ["//visibility:public"], -) - -# par_binary expects its `srcs` to contain only *.py files, but gcovr is -# distributed as a script with no filename extension. Rename it here. -genrule( - name = "gcovr_to_exec_py", - srcs = ["scripts/gcovr"], - outs = ["renamed_gcovr.py"], - cmd = "cat $(location scripts/gcovr) > $(location renamed_gcovr.py)", -) diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index ed14486d3a5e9..027d69baa27fc 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -140,7 +140,6 @@ def envoy_dependencies(skip_targets = []): _com_github_envoyproxy_sqlparser() _com_github_fmtlib_fmt() _com_github_gabime_spdlog() - _com_github_gcovr_gcovr() _com_github_google_benchmark() _com_github_google_jwt_verify() _com_github_google_libprotobuf_mutator() @@ -164,9 +163,6 @@ def envoy_dependencies(skip_targets = []): _io_opentracing_cpp() _net_zlib() - # Used for bundling gcovr into a relocatable .par file. - _repository_impl("subpar") - _python_deps() _cc_deps() _go_deps(skip_targets) @@ -257,16 +253,6 @@ def _com_github_gabime_spdlog(): actual = "@com_github_gabime_spdlog//:spdlog", ) -def _com_github_gcovr_gcovr(): - _repository_impl( - name = "com_github_gcovr_gcovr", - build_file = "@envoy//bazel/external:gcovr.BUILD", - ) - native.bind( - name = "gcovr", - actual = "@com_github_gcovr_gcovr//:gcovr", - ) - def _com_github_google_benchmark(): location = REPOSITORY_LOCATIONS["com_github_google_benchmark"] http_archive( diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 383f40e90c600..d809a6f871924 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -66,11 +66,6 @@ REPOSITORY_LOCATIONS = dict( strip_prefix = "spdlog-1.3.1", urls = ["https://github.com/gabime/spdlog/archive/v1.3.1.tar.gz"], ), - com_github_gcovr_gcovr = dict( - sha256 = "1c52a71f245adfe1b45e30fbe5015337fe66546f17f40038b3969b7b42acceed", - strip_prefix = "gcovr-3.4", - urls = ["https://github.com/gcovr/gcovr/archive/3.4.tar.gz"], - ), com_github_google_libprotobuf_mutator = dict( sha256 = "97b3639630040f41c45f45838ab00b78909e6b4cb69c8028e01302bea5b79495", strip_prefix = "libprotobuf-mutator-c3d2faf04a1070b0b852b0efdef81e1a81ba925e", @@ -229,13 +224,6 @@ REPOSITORY_LOCATIONS = dict( sha256 = "105f8d68616f8248e24bf0e9372ef04d3cc10104f1980f54d57b2ce73a5ad56a", urls = ["https://pypi.python.org/packages/source/s/six/six-1.10.0.tar.gz#md5=34eed507548117b2ab523ab14b2f8b55"], ), - # I'd love to name this `com_github_google_subpar`, but something in the Subpar - # code assumes its repository name is just `subpar`. - subpar = dict( - sha256 = "b80297a1b8d38027a86836dbadc22f55dc3ecad56728175381aa6330705ac10f", - strip_prefix = "subpar-2.0.0", - urls = ["https://github.com/google/subpar/archive/2.0.0.tar.gz"], - ), io_opencensus_cpp = dict( sha256 = "d6d68704c419a9e892bd1f942e09509ebc5a318499a1abcf2c09734e5dc56e19", strip_prefix = "opencensus-cpp-1145dd77ffb7a2845c71c8e6ca188ef55e4ff607", diff --git a/ci/build_setup.sh b/ci/build_setup.sh index a8069429be0d1..8323eb7440110 100755 --- a/ci/build_setup.sh +++ b/ci/build_setup.sh @@ -40,21 +40,6 @@ then fi export ENVOY_FILTER_EXAMPLE_SRCDIR="${BUILD_DIR}/envoy-filter-example" -# Make sure that /source doesn't contain /build on the underlying host -# filesystem, including via hard links or symlinks. We can get into weird -# loops with Bazel symlinking and gcovr's path traversal if this is true, so -# best to keep /source and /build in distinct directories on the host -# filesystem. -SENTINEL="${BUILD_DIR}"/bazel.sentinel -touch "${SENTINEL}" -if [[ -n "$(find -L "${ENVOY_SRCDIR}" -name "$(basename "${SENTINEL}")")" ]] -then - rm -f "${SENTINEL}" - echo "/source mount must not contain /build mount" - exit 1 -fi -rm -f "${SENTINEL}" - # Environment setup. export USER=bazel export TEST_TMPDIR=${BUILD_DIR}/tmp From e15fbf27739be7a345a8a2d98d2f8b7846e4bb16 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Tue, 16 Jul 2019 02:06:03 +0000 Subject: [PATCH 12/13] remove gcc_only_test, test_filter Signed-off-by: Lizan Zhou --- test/coverage/gcc_only_test/BUILD | 16 ---------------- test/coverage/gcc_only_test/gcc_only_test.cc | 12 ------------ test/coverage/gen_build.sh | 7 ------- test/run_envoy_bazel_coverage.sh | 4 ++-- 4 files changed, 2 insertions(+), 37 deletions(-) delete mode 100644 test/coverage/gcc_only_test/BUILD delete mode 100644 test/coverage/gcc_only_test/gcc_only_test.cc diff --git a/test/coverage/gcc_only_test/BUILD b/test/coverage/gcc_only_test/BUILD deleted file mode 100644 index 1482a741b3e3c..0000000000000 --- a/test/coverage/gcc_only_test/BUILD +++ /dev/null @@ -1,16 +0,0 @@ -licenses(["notice"]) # Apache 2 - -load( - "//bazel:envoy_build_system.bzl", - "envoy_cc_test", - "envoy_package", -) - -envoy_package() - -envoy_cc_test( - name = "gcc_only_test", - srcs = ["gcc_only_test.cc"], - coverage = False, - tags = ["manual"], -) diff --git a/test/coverage/gcc_only_test/gcc_only_test.cc b/test/coverage/gcc_only_test/gcc_only_test.cc deleted file mode 100644 index 31346323d20e4..0000000000000 --- a/test/coverage/gcc_only_test/gcc_only_test.cc +++ /dev/null @@ -1,12 +0,0 @@ -#include "gtest/gtest.h" - -namespace Envoy { - -TEST(GccOnly, CompilerCheck) { -#if defined(__clang__) or not defined(__GNUC__) - // clang is incompatible with gcov. - FAIL() << "GCC is required for coverage runs"; -#endif -} - -} // namespace Envoy diff --git a/test/coverage/gen_build.sh b/test/coverage/gen_build.sh index 5532e78a5e95a..5998e3eefd497 100755 --- a/test/coverage/gen_build.sh +++ b/test/coverage/gen_build.sh @@ -44,13 +44,6 @@ if [ -n "${EXTRA_QUERY_PATHS}" ]; then TARGETS="$TARGETS $("${BAZEL_BIN}" query ${BAZEL_QUERY_OPTIONS} "attr('tags', 'coverage_test_lib', ${EXTRA_QUERY_PATHS})" | grep "^//")" fi -# gcov requires gcc -if [ "${NO_GCOV}" != 1 ] -then - # Here we use the synthetic library target created by envoy_build_system.bzl - TARGETS="${TARGETS} ${REPOSITORY}//test/coverage/gcc_only_test:gcc_only_test_lib_internal_only" -fi - ( cat << EOF # This file is generated by test/coverage/gen_build.sh automatically prior to diff --git a/test/run_envoy_bazel_coverage.sh b/test/run_envoy_bazel_coverage.sh index 1f3c74720090f..a16062927ad2e 100755 --- a/test/run_envoy_bazel_coverage.sh +++ b/test/run_envoy_bazel_coverage.sh @@ -23,13 +23,13 @@ fi # Make sure //test/coverage:coverage_tests is up-to-date. SCRIPT_DIR="$(realpath "$(dirname "$0")")" -NO_GCOV=1 "${SCRIPT_DIR}"/coverage/gen_build.sh ${COVERAGE_TARGETS} +"${SCRIPT_DIR}"/coverage/gen_build.sh ${COVERAGE_TARGETS} BAZEL_USE_LLVM_NATIVE_COVERAGE=1 GCOV=llvm-profdata bazel coverage ${BAZEL_BUILD_OPTIONS} \ -c fastbuild --copt=-DNDEBUG --instrumentation_filter=//source/...,//include/... \ --test_timeout=2000 --cxxopt="-DENVOY_CONFIG_COVERAGE=1" --test_output=streamed \ --test_arg="--log-path /dev/null" --test_arg="-l trace" --test_env=HEAPCHECK= \ - --test_filter='-QuicPlatformTest.QuicStackTraceTest' //test/coverage:coverage_tests + //test/coverage:coverage_tests COVERAGE_DIR="${SRCDIR}"/generated/coverage mkdir -p "${COVERAGE_DIR}" From d51fbba0a8e45b5a1f1548a33d8b135c3d0ffd97 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Tue, 16 Jul 2019 07:48:24 +0000 Subject: [PATCH 13/13] threshold Signed-off-by: Lizan Zhou --- test/run_envoy_bazel_coverage.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/run_envoy_bazel_coverage.sh b/test/run_envoy_bazel_coverage.sh index a16062927ad2e..a3542492ef2c1 100755 --- a/test/run_envoy_bazel_coverage.sh +++ b/test/run_envoy_bazel_coverage.sh @@ -51,7 +51,7 @@ then COVERAGE_VALUE=$(llvm-cov export "${COVERAGE_BINARY}" -instr-profile="${COVERAGE_DATA}" \ -ignore-filename-regex="${COVERAGE_IGNORE_REGEX}" -summary-only | \ python3 -c "import sys, json; print(json.load(sys.stdin)['data'][0]['totals']['lines']['percent'])") - COVERAGE_THRESHOLD=97.0 + COVERAGE_THRESHOLD=96.9 COVERAGE_FAILED=$(echo "${COVERAGE_VALUE}<${COVERAGE_THRESHOLD}" | bc) if test ${COVERAGE_FAILED} -eq 1; then echo Code coverage ${COVERAGE_VALUE} is lower than limit of ${COVERAGE_THRESHOLD}