CI: Track integration test coverage separately from unit tests

.circleci/config.yml

@@ -41,6 +41,16 @@ jobs:
       - store_artifacts:
           path: /root/project/generated
           destination: /
+  integration_test_coverage:
+    docker:
+      - image: *envoy-build-image
+    resource_class: xlarge
+    steps:
+      - checkout
+      - run: ci/do_ci.sh coverage_integration
+      - store_artifacts:
+          path: /root/project/generated
+          destination: /
   benchmark:
     docker:
       - image: *envoy-build-image

ci/do_ci.sh

@@ -32,8 +32,17 @@ function do_clang_tidy() {
     ci/run_clang_tidy.sh
 }
 
-function do_coverage() {
-    export TEST_TARGETS="//test/..."
+function do_unit_test_coverage() {
+    export TEST_TARGETS="//test/... -//test:python_test"
+    export COVERAGE_THRESHOLD=92.3
+    echo "bazel coverage build with tests ${TEST_TARGETS}"
+    test/run_nighthawk_bazel_coverage.sh ${TEST_TARGETS}
+    exit 0
+}
+
+function do_integration_test_coverage() {
+    export TEST_TARGETS="//test:python_test"
+    export COVERAGE_THRESHOLD=78.2
     echo "bazel coverage build with tests ${TEST_TARGETS}"
     test/run_nighthawk_bazel_coverage.sh ${TEST_TARGETS}
     exit 0
@@ -208,7 +217,12 @@ case "$1" in
     ;;
     coverage)
         setup_clang_toolchain
-        do_coverage
+        do_unit_test_coverage
+        exit 0
+    ;;
+    coverage_integration)
+        setup_clang_toolchain
+        do_integration_test_coverage
         exit 0
     ;;
     asan)
@@ -242,7 +256,7 @@ case "$1" in
         exit 0
     ;;
     *)
-        echo "must be one of [build,test,clang_tidy,coverage,asan,tsan,benchmark_with_own_binaries,docker,check_format,fix_format,test_gcc]"
+        echo "must be one of [build,test,clang_tidy,coverage,coverage_integration,asan,tsan,benchmark_with_own_binaries,docker,check_format,fix_format,test_gcc]"
         exit 1
     ;;
 esac

test/run_nighthawk_bazel_coverage.sh

@@ -7,13 +7,13 @@ set +x
 set -u
 
 SRCDIR="${SRCDIR:=${PWD}}"
-VALIDATE_COVERAGE="${VALIDATE_COVERAGE:=true}"
 ENVOY_COVERAGE_DIR="${ENVOY_COVERAGE_DIR:=}"
+COVERAGE_THRESHOLD=${COVERAGE_THRESHOLD:=0}
 
 echo "Starting run_nighthawk_bazel_coverage.sh..."
 echo "    PWD=$(pwd)"
 echo "    SRCDIR=${SRCDIR}"
-echo "    VALIDATE_COVERAGE=${VALIDATE_COVERAGE}"
+echo "    COVERAGE_THRESHOLD=${COVERAGE_THRESHOLD}"
 
 COVERAGE_DIR="${SRCDIR}"/generated/coverage
 rm -rf "${COVERAGE_DIR}"
@@ -31,7 +31,7 @@ else
 fi
 
 BAZEL_BUILD_OPTIONS+=" --config=test-coverage --test_tag_filters=-nocoverage --test_env=ENVOY_IP_TEST_VERSIONS=v4only"
-bazel coverage ${BAZEL_BUILD_OPTIONS} --cache_test_results=no --test_output=all ${COVERAGE_TARGETS}
+bazel coverage ${BAZEL_BUILD_OPTIONS} --cache_test_results=no --test_output=all -- ${COVERAGE_TARGETS}
 COVERAGE_DATA="${COVERAGE_DIR}/coverage.dat"
 
 cp bazel-out/_coverage/_coverage_report.dat "${COVERAGE_DATA}"
@@ -41,9 +41,8 @@ COVERAGE_VALUE=${COVERAGE_VALUE%?}
 
 [[ -z "${ENVOY_COVERAGE_DIR}" ]] || rsync -av "${COVERAGE_DIR}"/ "${ENVOY_COVERAGE_DIR}"
 
-if [ "$VALIDATE_COVERAGE" == "true" ]
+if [ "$COVERAGE_THRESHOLD" != "0" ]
 then
-  COVERAGE_THRESHOLD=98.4
   COVERAGE_FAILED=$(echo "${COVERAGE_VALUE}<${COVERAGE_THRESHOLD}" | bc)
   if test ${COVERAGE_FAILED} -eq 1; then
       echo Code coverage ${COVERAGE_VALUE} is lower than limit of ${COVERAGE_THRESHOLD}