diff --git a/.bazelrc b/.bazelrc index dbdd016de..54769468c 100644 --- a/.bazelrc +++ b/.bazelrc @@ -1,6 +1,9 @@ # The following .bazelrc content is forked from the main Envoy repository. This is necessary since # this needs to be available before we can access the Envoy repository contents via Bazel. +build:clang-asan --test_timeout=900 +build:clang-tsan --test_timeout=900 + # Envoy specific Bazel build/test options. # Bazel doesn't need more than 200MB of memory for local build based on memory profiling: diff --git a/ci/envoy_build_sha.sh b/ci/envoy_build_sha.sh index 709617afe..d2a984438 100644 --- a/ci/envoy_build_sha.sh +++ b/ci/envoy_build_sha.sh @@ -1,2 +1,2 @@ -ENVOY_BUILD_SHA=$(grep envoyproxy/envoy-build .circleci/config.yml | sed -e 's#.*envoyproxy/envoy-build:\(.*\)#\1#' | uniq) -[[ $(wc -l <<< "${ENVOY_BUILD_SHA}" | awk '{$1=$1};1') == 1 ]] || (echo ".circleci/config.yml hashes are inconsistent!" && exit 1) +ENVOY_BUILD_SHA=$(grep envoyproxy/envoy-build-ubuntu $(dirname $0)/../.bazelrc | sed -e 's#.*envoyproxy/envoy-build-ubuntu:\(.*\)#\1#' | uniq) +[[ $(wc -l <<< "${ENVOY_BUILD_SHA}" | awk '{$1=$1};1') == 1 ]] || (echo ".bazelrc envoyproxy/envoy-build-ubuntu hashes are inconsistent!" && exit 1) \ No newline at end of file diff --git a/test/BUILD b/test/BUILD index 91dfad7c2..868394027 100644 --- a/test/BUILD +++ b/test/BUILD @@ -266,6 +266,7 @@ envoy_cc_test( deps = [ "//source/client:nighthawk_client_lib", "//source/client:nighthawk_service_lib", + "//source/client:output_transform_main_lib", "//source/common:nighthawk_common_lib", "//source/server:http_test_server_filter_lib", "//test/test_common:environment_lib", diff --git a/test/integration/BUILD b/test/integration/BUILD index 43cd271bb..720e83f00 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -23,6 +23,7 @@ py_library( "configurations/nighthawk_https_origin.yaml", "configurations/sni_origin.yaml", "//:nighthawk_client", + "//:nighthawk_output_transform", "//:nighthawk_service", "//:nighthawk_test_server", "@envoy//test/config/integration/certs", @@ -83,6 +84,12 @@ py_library( deps = [":integration_test_base"], ) +py_library( + name = "test_output_transform_lib", + srcs = ["test_output_transform.py"], + deps = [":integration_test_base"], +) + py_binary( name = "integration_test", srcs = ["integration_test.py"], @@ -99,6 +106,7 @@ py_binary( ":test_grpc_service_lib", ":test_integration_basics_lib", ":test_integration_zipkin_lib", + ":test_output_transform_lib", ":test_remote_execution_lib", ], ) diff --git a/test/integration/test_grpc_service.py b/test/integration/test_grpc_service.py index 418b424b2..f0624caad 100644 --- a/test/integration/test_grpc_service.py +++ b/test/integration/test_grpc_service.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 import pytest -from integration_test_fixtures import (http_test_server_fixture) -from utility import * +from test.integration.integration_test_fixtures import http_test_server_fixture +from test.integration import utility def test_grpc_service_happy_flow(http_test_server_fixture): @@ -14,8 +14,8 @@ def test_grpc_service_happy_flow(http_test_server_fixture): http_test_server_fixture.getTestServerRootUri() ]) counters = http_test_server_fixture.getNighthawkCounterMapFromJson(parsed_json) - assertGreaterEqual(counters["benchmark.http_2xx"], 5) - assertEqual(counters["requestsource.internal.upstream_rq_200"], 1) + utility.assertGreaterEqual(counters["benchmark.http_2xx"], 5) + utility.assertEqual(counters["requestsource.internal.upstream_rq_200"], 1) def test_grpc_service_down(http_test_server_fixture): @@ -26,10 +26,10 @@ def test_grpc_service_down(http_test_server_fixture): ], expect_failure=True) counters = http_test_server_fixture.getNighthawkCounterMapFromJson(parsed_json) - assertEqual(counters["requestsource.upstream_rq_pending_failure_eject"], 1) + utility.assertEqual(counters["requestsource.upstream_rq_pending_failure_eject"], 1) -@pytest.mark.skipif(isSanitizerRun(), reason="Slow in sanitizer runs") +@pytest.mark.skipif(utility.isSanitizerRun(), reason="Slow in sanitizer runs") def test_grpc_service_stress(http_test_server_fixture): http_test_server_fixture.startNighthawkGrpcService("dummy-request-source") parsed_json, _ = http_test_server_fixture.runNighthawkClient([ @@ -40,5 +40,27 @@ def test_grpc_service_stress(http_test_server_fixture): http_test_server_fixture.getTestServerRootUri() ]) counters = http_test_server_fixture.getNighthawkCounterMapFromJson(parsed_json) - assertGreaterEqual(counters["benchmark.http_2xx"], 5000) - assertEqual(counters["requestsource.internal.upstream_rq_200"], 4) + utility.assertGreaterEqual(counters["benchmark.http_2xx"], 5000) + utility.assertEqual(counters["requestsource.internal.upstream_rq_200"], 4) + + +def _run_service_with_args(args): + return utility.run_binary_with_args("nighthawk_service", args) + + +def test_grpc_service_help(): + (exit_code, output) = _run_service_with_args("--help") + utility.assertEqual(exit_code, 0) + utility.assertIn("USAGE", output) + + +def test_grpc_service_bad_arguments(): + (exit_code, output) = _run_service_with_args("--foo") + utility.assertEqual(exit_code, 1) + utility.assertIn("PARSE ERROR: Argument: --foo", output) + + +def test_grpc_service_nonexisting_listener_address(): + (exit_code, output) = _run_service_with_args("--listen 1.1.1.1:1") + utility.assertEqual(exit_code, 1) + utility.assertIn("Failure: Could not start the grpc service", output) diff --git a/test/integration/test_integration_basics.py b/test/integration/test_integration_basics.py index bb0315588..df02ef193 100644 --- a/test/integration/test_integration_basics.py +++ b/test/integration/test_integration_basics.py @@ -652,3 +652,19 @@ def test_http_request_release_timing(http_test_server_fixture, qps_parameterizat int(global_histograms["benchmark_http_client.queue_to_connect"]["count"]), total_requests) assertCounterEqual(counters, "benchmark.http_2xx", (total_requests)) + + +def _run_client_with_args(args): + return run_binary_with_args("nighthawk_client", args) + + +def test_client_help(): + (exit_code, output) = _run_client_with_args("--help") + assertEqual(exit_code, 0) + assertIn("USAGE", output) + + +def test_client_bad_arg(): + (exit_code, output) = _run_client_with_args("127.0.0.1 --foo") + assertEqual(exit_code, 1) + assertIn("PARSE ERROR: Argument: --foo", output) diff --git a/test/integration/test_output_transform.py b/test/integration/test_output_transform.py new file mode 100644 index 000000000..3889fcdb0 --- /dev/null +++ b/test/integration/test_output_transform.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +import pytest + +from test.integration import utility +import os +import subprocess + + +def _run_output_transform_with_args(args): + return utility.run_binary_with_args("nighthawk_output_transform", args) + + +def test_output_transform_help(): + (exit_code, output) = _run_output_transform_with_args("--help") + utility.assertEqual(exit_code, 0) + utility.assertIn("USAGE", output) + + +def test_output_transform_bad_arguments(): + (exit_code, output) = _run_output_transform_with_args("--foo") + utility.assertEqual(exit_code, 1) + utility.assertIn("PARSE ERROR: Argument: --foo", output) + + +def test_output_transform_101(): + """ + Runs an arbitrary load test, which outputs to json. + This json output is then transformed to human readable output. + """ + + test_rundir = os.path.join(os.environ["TEST_SRCDIR"], os.environ["TEST_WORKSPACE"]) + process = subprocess.run([ + os.path.join(test_rundir, "nighthawk_client"), "--duration", "1", "--rps", "1", "127.0.0.1", + "--output-format", "json" + ], + stdout=subprocess.PIPE) + output = process.stdout + process = subprocess.run( + [os.path.join(test_rundir, "nighthawk_output_transform"), "--output-format", "human"], + stdout=subprocess.PIPE, + input=output) + utility.assertEqual(process.returncode, 0) + utility.assertIn("Nighthawk - A layer 7 protocol benchmarking tool", + process.stdout.decode("utf-8")) diff --git a/test/integration/utility.py b/test/integration/utility.py index 28e9a2d37..907ffab6f 100644 --- a/test/integration/utility.py +++ b/test/integration/utility.py @@ -1,4 +1,5 @@ import os +import subprocess def assertEqual(a, b): @@ -57,3 +58,19 @@ def assertCounterBetweenInclusive(counters, name, min_value, max_value): def isSanitizerRun(): return True if os.environ.get("NH_INTEGRATION_TEST_SANITIZER_RUN", 0) == "1" else False + + +def run_binary_with_args(binary, args): + """Executes a Nighthawk binary with the provided arguments. + + Args: + binary: A string, the name of the to-be-called binary, e.g. "nighthawk_client". + args: A string, the command line arguments to the binary, e.g. "--foo --bar". + + Returns: + A tuple in the form (exit_code, output), where exit_code is the code the Nighthawk + service terminated with and the output is its standard output. + """ + test_rundir = os.path.join(os.environ["TEST_SRCDIR"], os.environ["TEST_WORKSPACE"]) + args = "%s %s" % (os.path.join(test_rundir, binary), args) + return subprocess.getstatusoutput(args) diff --git a/test/rate_limiter_test.cc b/test/rate_limiter_test.cc index 8dee7ffb8..76d9b3e5d 100644 --- a/test/rate_limiter_test.cc +++ b/test/rate_limiter_test.cc @@ -166,8 +166,11 @@ TEST_F(RateLimiterTest, DistributionSamplingRateLimiterImplTest) { EXPECT_CALL(unsafe_mock_rate_limiter, timeSource) .Times(AtLeast(1)) .WillRepeatedly(ReturnRef(time_system)); + auto sampler = std::make_unique(1); + EXPECT_EQ(sampler->min(), 0); + EXPECT_EQ(sampler->max(), 1); RateLimiterPtr rate_limiter = std::make_unique( - std::make_unique(1), std::move(mock_rate_limiter)); + std::move(sampler), std::move(mock_rate_limiter)); EXPECT_CALL(unsafe_mock_rate_limiter, tryAcquireOne).Times(tries).WillRepeatedly(Return(true)); EXPECT_CALL(unsafe_mock_rate_limiter, releaseOne).Times(tries); @@ -383,6 +386,10 @@ class GraduallyOpeningRateLimiterFilterTest : public Test { } // Verify we acquired everything. EXPECT_FALSE(rate_limiter->tryAcquireOne()); + // Verify releaseOne works. + rate_limiter->releaseOne(); + EXPECT_TRUE(rate_limiter->tryAcquireOne()); + EXPECT_FALSE(rate_limiter->tryAcquireOne()); return acquisition_timings; } }; diff --git a/test/run_nighthawk_bazel_coverage.sh b/test/run_nighthawk_bazel_coverage.sh index 6e679e4a8..6ff69c567 100755 --- a/test/run_nighthawk_bazel_coverage.sh +++ b/test/run_nighthawk_bazel_coverage.sh @@ -2,17 +2,23 @@ # derived from test/run_envoy_bazel_coverage.sh over the Envoy repo. -set -e -set -x +set -eo pipefail +set +x +set -u -[[ -z "${SRCDIR}" ]] && SRCDIR="${PWD}" -[[ -z "${VALIDATE_COVERAGE}" ]] && VALIDATE_COVERAGE=true +SRCDIR="${SRCDIR:=${PWD}}" +VALIDATE_COVERAGE="${VALIDATE_COVERAGE:=true}" +ENVOY_COVERAGE_DIR="${ENVOY_COVERAGE_DIR:=}" echo "Starting run_nighthawk_bazel_coverage.sh..." echo " PWD=$(pwd)" echo " SRCDIR=${SRCDIR}" echo " VALIDATE_COVERAGE=${VALIDATE_COVERAGE}" +COVERAGE_DIR="${SRCDIR}"/generated/coverage +rm -rf "${COVERAGE_DIR}" +mkdir -p "${COVERAGE_DIR}" + # This is the target that will be run to generate coverage data. It can be overridden by consumer # projects that want to run coverage on a different/combined target. # Command-line arguments take precedence over ${COVERAGE_TARGET}. @@ -25,16 +31,12 @@ else fi BAZEL_BUILD_OPTIONS+=" --config=test-coverage --test_tag_filters=-nocoverage --test_env=ENVOY_IP_TEST_VERSIONS=v4only" -bazel coverage ${BAZEL_BUILD_OPTIONS} --test_output=all ${COVERAGE_TARGETS} - -COVERAGE_DIR="${SRCDIR}"/generated/coverage -mkdir -p "${COVERAGE_DIR}" - +bazel coverage ${BAZEL_BUILD_OPTIONS} --cache_test_results=no --test_output=all ${COVERAGE_TARGETS} COVERAGE_DATA="${COVERAGE_DIR}/coverage.dat" cp bazel-out/_coverage/_coverage_report.dat "${COVERAGE_DATA}" -COVERAGE_VALUE=$(genhtml --prefix ${PWD} --output "${COVERAGE_DIR}" "${COVERAGE_DATA}" | tee /dev/stderr | grep lines... | cut -d ' ' -f 4) +COVERAGE_VALUE=$(genhtml --prefix ${PWD} --output "${COVERAGE_DIR}" "${COVERAGE_DATA}" | grep lines... | cut -d ' ' -f 4) COVERAGE_VALUE=${COVERAGE_VALUE%?} [[ -z "${ENVOY_COVERAGE_DIR}" ]] || rsync -av "${COVERAGE_DIR}"/ "${ENVOY_COVERAGE_DIR}" @@ -42,7 +44,7 @@ COVERAGE_VALUE=${COVERAGE_VALUE%?} if [ "$VALIDATE_COVERAGE" == "true" ] then # TODO(#370): restore the coverage threshold. - COVERAGE_THRESHOLD=98.1 + COVERAGE_THRESHOLD=98.5 COVERAGE_FAILED=$(echo "${COVERAGE_VALUE}<${COVERAGE_THRESHOLD}" | bc) if test ${COVERAGE_FAILED} -eq 1; then echo Code coverage ${COVERAGE_VALUE} is lower than limit of ${COVERAGE_THRESHOLD} diff --git a/test/statistic_test.cc b/test/statistic_test.cc index c36faeb47..c73e1d6b8 100644 --- a/test/statistic_test.cc +++ b/test/statistic_test.cc @@ -320,7 +320,12 @@ TEST(StatisticTest, NullStatistic) { NullStatistic stat; EXPECT_EQ(0, stat.count()); stat.addValue(1); - EXPECT_EQ(0, stat.count()); + EXPECT_EQ(0, stat.mean()); + EXPECT_EQ(0, stat.pvariance()); + EXPECT_EQ(0, stat.pstdev()); + EXPECT_NE(nullptr, stat.combine(stat)); + EXPECT_EQ(0, stat.significantDigits()); + EXPECT_NE(nullptr, stat.createNewInstanceOfSameType()); } } // namespace Nighthawk