From e278d85a065ea8b83fd55aecefa3e82e3909ea17 Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Mon, 24 Aug 2020 13:12:18 +0200 Subject: [PATCH 1/2] Stop executing integration tests in CI coverage task Signed-off-by: Otto van der Schaaf --- ci/do_ci.sh | 2 +- test/run_nighthawk_bazel_coverage.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/do_ci.sh b/ci/do_ci.sh index 679ae7460..4ae0885a9 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -33,7 +33,7 @@ function do_clang_tidy() { } function do_coverage() { - export TEST_TARGETS="//test/..." + export TEST_TARGETS="//test/... -//test:python_test" echo "bazel coverage build with tests ${TEST_TARGETS}" test/run_nighthawk_bazel_coverage.sh ${TEST_TARGETS} exit 0 diff --git a/test/run_nighthawk_bazel_coverage.sh b/test/run_nighthawk_bazel_coverage.sh index 0512e3b23..fcbfcce58 100755 --- a/test/run_nighthawk_bazel_coverage.sh +++ b/test/run_nighthawk_bazel_coverage.sh @@ -31,7 +31,7 @@ else fi BAZEL_BUILD_OPTIONS+=" --config=test-coverage --test_tag_filters=-nocoverage --test_env=ENVOY_IP_TEST_VERSIONS=v4only" -bazel coverage ${BAZEL_BUILD_OPTIONS} --cache_test_results=no --test_output=all ${COVERAGE_TARGETS} +bazel coverage ${BAZEL_BUILD_OPTIONS} --cache_test_results=no --test_output=all -- ${COVERAGE_TARGETS} COVERAGE_DATA="${COVERAGE_DIR}/coverage.dat" cp bazel-out/_coverage/_coverage_report.dat "${COVERAGE_DATA}" @@ -43,7 +43,7 @@ COVERAGE_VALUE=${COVERAGE_VALUE%?} if [ "$VALIDATE_COVERAGE" == "true" ] then - COVERAGE_THRESHOLD=98.4 + COVERAGE_THRESHOLD=92.1 COVERAGE_FAILED=$(echo "${COVERAGE_VALUE}<${COVERAGE_THRESHOLD}" | bc) if test ${COVERAGE_FAILED} -eq 1; then echo Code coverage ${COVERAGE_VALUE} is lower than limit of ${COVERAGE_THRESHOLD} From d559ccacc26b49031d2db1cf38209af4889d7d25 Mon Sep 17 00:00:00 2001 From: Otto van der Schaaf Date: Thu, 27 Aug 2020 10:04:44 +0200 Subject: [PATCH 2/2] Add separate coverage CI task for the integration tests Signed-off-by: Otto van der Schaaf --- .circleci/config.yml | 10 ++++++++++ ci/do_ci.sh | 20 +++++++++++++++++--- test/run_nighthawk_bazel_coverage.sh | 7 +++---- 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f7ad3fa43..f542d172d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -41,6 +41,16 @@ jobs: - store_artifacts: path: /root/project/generated destination: / + integration_test_coverage: + docker: + - image: *envoy-build-image + resource_class: xlarge + steps: + - checkout + - run: ci/do_ci.sh coverage_integration + - store_artifacts: + path: /root/project/generated + destination: / benchmark: docker: - image: *envoy-build-image diff --git a/ci/do_ci.sh b/ci/do_ci.sh index ea6e55589..5b0ebd239 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -32,8 +32,17 @@ function do_clang_tidy() { ci/run_clang_tidy.sh } -function do_coverage() { +function do_unit_test_coverage() { export TEST_TARGETS="//test/... -//test:python_test" + export COVERAGE_THRESHOLD=92.3 + echo "bazel coverage build with tests ${TEST_TARGETS}" + test/run_nighthawk_bazel_coverage.sh ${TEST_TARGETS} + exit 0 +} + +function do_integration_test_coverage() { + export TEST_TARGETS="//test:python_test" + export COVERAGE_THRESHOLD=78.2 echo "bazel coverage build with tests ${TEST_TARGETS}" test/run_nighthawk_bazel_coverage.sh ${TEST_TARGETS} exit 0 @@ -208,7 +217,12 @@ case "$1" in ;; coverage) setup_clang_toolchain - do_coverage + do_unit_test_coverage + exit 0 + ;; + coverage_integration) + setup_clang_toolchain + do_integration_test_coverage exit 0 ;; asan) @@ -242,7 +256,7 @@ case "$1" in exit 0 ;; *) - echo "must be one of [build,test,clang_tidy,coverage,asan,tsan,benchmark_with_own_binaries,docker,check_format,fix_format,test_gcc]" + echo "must be one of [build,test,clang_tidy,coverage,coverage_integration,asan,tsan,benchmark_with_own_binaries,docker,check_format,fix_format,test_gcc]" exit 1 ;; esac diff --git a/test/run_nighthawk_bazel_coverage.sh b/test/run_nighthawk_bazel_coverage.sh index fcbfcce58..ae1e81328 100755 --- a/test/run_nighthawk_bazel_coverage.sh +++ b/test/run_nighthawk_bazel_coverage.sh @@ -7,13 +7,13 @@ set +x set -u SRCDIR="${SRCDIR:=${PWD}}" -VALIDATE_COVERAGE="${VALIDATE_COVERAGE:=true}" ENVOY_COVERAGE_DIR="${ENVOY_COVERAGE_DIR:=}" +COVERAGE_THRESHOLD=${COVERAGE_THRESHOLD:=0} echo "Starting run_nighthawk_bazel_coverage.sh..." echo " PWD=$(pwd)" echo " SRCDIR=${SRCDIR}" -echo " VALIDATE_COVERAGE=${VALIDATE_COVERAGE}" +echo " COVERAGE_THRESHOLD=${COVERAGE_THRESHOLD}" COVERAGE_DIR="${SRCDIR}"/generated/coverage rm -rf "${COVERAGE_DIR}" @@ -41,9 +41,8 @@ COVERAGE_VALUE=${COVERAGE_VALUE%?} [[ -z "${ENVOY_COVERAGE_DIR}" ]] || rsync -av "${COVERAGE_DIR}"/ "${ENVOY_COVERAGE_DIR}" -if [ "$VALIDATE_COVERAGE" == "true" ] +if [ "$COVERAGE_THRESHOLD" != "0" ] then - COVERAGE_THRESHOLD=92.1 COVERAGE_FAILED=$(echo "${COVERAGE_VALUE}<${COVERAGE_THRESHOLD}" | bc) if test ${COVERAGE_FAILED} -eq 1; then echo Code coverage ${COVERAGE_VALUE} is lower than limit of ${COVERAGE_THRESHOLD}