diff --git a/WORKSPACE b/WORKSPACE index 75b45cee1..9e2ecf427 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -26,7 +26,7 @@ # # A Bazel (http://bazel.io) workspace for the Google Cloud Endpoints runtime. -ISTIO_PROXY = "cfdb30323aa72b680bab1664035a1bbc200379c0" +ISTIO_PROXY = "9442a836988332f397f874e0b8ae9edafcd53425" git_repository( name = "nginx", @@ -126,7 +126,7 @@ bind( git_repository( name = "tools", - commit = "1bcac83ed2dc9c5e0be156a4c1801d435667f642", + commit = "e5cfdcd2edfe36b46c620fe2236f079edd5fe80b", remote = "https://github.com/cloudendpoints/endpoints-tools", ) @@ -158,7 +158,7 @@ git_repository( # git_repository( name = "io_bazel_rules_go", - commit = "76c63b5cd0d47c1f2b47ab4953db96c574af1c1d", + commit = "2d9f328a9723baf2d037ba9db28d9d0e30683938", # Apr 6, 2017 (buildifier fix) remote = "https://github.com/bazelbuild/rules_go.git", ) diff --git a/docker/ingress/build.sh b/docker/ingress/build.sh index a3dd507b5..2771e40b1 100755 --- a/docker/ingress/build.sh +++ b/docker/ingress/build.sh @@ -43,4 +43,4 @@ cp $DEB $ROOT/docker/ingress cp $ROOT/bazel-bin/test/src/ingress $ROOT/docker/ingress cp $ROOT/test/src/controller/nginx.tmpl $ROOT/docker/ingress docker build -t ${IMAGE} $ROOT/docker/ingress || error_exit "Failed to build" -gcloud docker push ${IMAGE} || error_exit "Failed to upload docker image" +gcloud docker -- push ${IMAGE} || error_exit "Failed to upload docker image" diff --git a/script/jenkins-build-docker-slave b/script/jenkins-build-docker-slave index d0fe07ba3..4a3dd8f31 100755 --- a/script/jenkins-build-docker-slave +++ b/script/jenkins-build-docker-slave @@ -43,7 +43,7 @@ TOOLS_BUCKET='' function update_image() { local image="${1}" [[ ${FORCE} == true ]] && return 0 - ${GCLOUD} docker pull "${image}" + ${GCLOUD} docker -- pull "${image}" local image_exists=${?} [[ ${image_exists} -eq 0 ]] && return 1 return 0 @@ -75,7 +75,7 @@ if [[ "${BUILD}" == true ]]; then run retry -n 10 docker build --build-arg TOOLS_BUCKET="${TOOLS_BUCKET}" --no-cache -t ${IMAGE} \ -f "${DOCKER_FILE}" . || error_exit "Could not build ${SLAVE}" echo "Pushing Docker image: ${IMAGE}" - retry -n 3 ${GCLOUD} docker push "${IMAGE}" \ + retry -n 3 ${GCLOUD} docker -- push "${IMAGE}" \ || error_exit "Failed to push tag ${IMAGE}." fi fi @@ -86,7 +86,7 @@ if [[ -n "${TAG}" ]]; then retry -n 3 docker tag "${IMAGE}" ${TAG} echo "Pushing Docker image: ${TAG}" - retry -n 3 ${GCLOUD} docker push "${TAG}" \ + retry -n 3 ${GCLOUD} docker -- push "${TAG}" \ || error_exit "Failed to push tag ${TAG}." fi diff --git a/script/linux-build-docker b/script/linux-build-docker index fa38bcae5..12389783e 100755 --- a/script/linux-build-docker +++ b/script/linux-build-docker @@ -48,7 +48,7 @@ done [[ -f "${DEB}" ]] || error_exit "Cannot find Debian package ${DEB}" echo "Checking if docker image ${IMAGE} exists.." -gcloud docker pull "${IMAGE}" \ +gcloud docker -- pull "${IMAGE}" \ && { echo "Image ${IMAGE} already exists; skipping"; exit 0; } echo "Building Endpoints Runtime docker image." @@ -86,7 +86,7 @@ if [[ -n "${CONFIG}" ]]; then echo "Pushing Custom Docker image: ${IMAGE}" retry -n 10 -s 10 \ - gcloud docker push "${IMAGE}" \ + gcloud docker -- push "${IMAGE}" \ || error_exit "Failed to upload custom Docker image to gcr." else @@ -94,6 +94,6 @@ else # Try 10 times, shortest wait is 10 seconds, exponential back-off. retry -n 10 -s 10 \ - gcloud docker push "${IMAGE}" \ + gcloud docker -- push "${IMAGE}" \ || error_exit "Failed to upload Docker image to gcr." fi diff --git a/script/linux-gae-instance b/script/linux-gae-instance index 343c9cc0c..62c2fb5b3 100755 --- a/script/linux-gae-instance +++ b/script/linux-gae-instance @@ -177,7 +177,7 @@ function download_nginx_logs() { retry -n 2 -s 10 \ ${GCLOUD} compute ssh --project=${PROJECT} --zone=${vm_zone} ${vm_name} \ - "sudo tar czf \"${remote_archive}\" /var/log/ ; sudo chmod 777 \"${remote_archive}\"" \ + --command="sudo tar czf ${remote_archive} /var/log/; sudo chmod 777 ${remote_archive}" \ || error_exit "Cannot tar backend VM logs." retry -n 2 -s 10 \ diff --git a/script/release-stable b/script/release-stable index f07713500..512dadfca 100755 --- a/script/release-stable +++ b/script/release-stable @@ -51,11 +51,13 @@ END_USAGE GCLOUD=$(which gcloud) || GCLOUD=~/google-cloud-sdk/bin/gcloud DEBIAN_PACKAGE_REPOSITORY='' +VERSION="$(command cat ${ROOT}/src/nginx/version)" -while getopts :g:r: arg; do +while getopts :g:r:v: arg; do case ${arg} in g) GCLOUD="${OPTARG}";; r) DEBIAN_PACKAGE_REPOSITORY="${OPTARG}";; + v) VERSION="${OPTARG}";; *) usage "Invalid option: -${OPTARG}";; esac done @@ -70,7 +72,6 @@ which rapture > /dev/null 2>&1 \ set -x -VERSION="$(command cat ${ROOT}/src/nginx/version)" # Minor base is 1.0 if version is 1.0.0 MINOR_BASE_VERSION=${VERSION%.*} # Major base is 1 if version is 1.0.0 @@ -82,7 +83,7 @@ function tag_stable_image() { docker_tag_f "${image}" "${stable}" - retry "${GCLOUD}" docker push "${stable}" \ + retry "${GCLOUD}" docker -- push "${stable}" \ || error_exit "ERROR: failed to push '${stable}'" } diff --git a/src/grpc/BUILD b/src/grpc/BUILD index c51aa23c6..f6a850b10 100644 --- a/src/grpc/BUILD +++ b/src/grpc/BUILD @@ -58,6 +58,7 @@ cc_library( ], visibility = ["//visibility:public"], deps = [ + "@istio_proxy_git//contrib/endpoints/src/grpc/transcoding:transcoder_input_stream", "//external:grpc", "//external:grpc++", "//external:protobuf", diff --git a/src/grpc/zero_copy_stream.cc b/src/grpc/zero_copy_stream.cc index dd76a3d23..3a037ea4c 100644 --- a/src/grpc/zero_copy_stream.cc +++ b/src/grpc/zero_copy_stream.cc @@ -68,7 +68,7 @@ void GrpcZeroCopyInputStream::BackUp(int count) { } } -::google::protobuf::int64 GrpcZeroCopyInputStream::ByteCount() const { +int64_t GrpcZeroCopyInputStream::BytesAvailable() const { return (current_buffer_size_ - position_) + serializer_.ByteCount(); } diff --git a/src/grpc/zero_copy_stream.h b/src/grpc/zero_copy_stream.h index b6a215bfe..c0de9e278 100644 --- a/src/grpc/zero_copy_stream.h +++ b/src/grpc/zero_copy_stream.h @@ -28,6 +28,7 @@ #include +#include "contrib/endpoints/src/grpc/transcoding/transcoder_input_stream.h" #include "google/protobuf/io/zero_copy_stream.h" #include "grpc++/support/byte_buffer.h" #include "src/grpc/message_serializer.h" @@ -38,7 +39,7 @@ namespace grpc { // ZeroCopyInputStream implementation over a stream of gRPC messages. class GrpcZeroCopyInputStream - : public ::google::protobuf::io::ZeroCopyInputStream { + : public ::google::api_manager::transcoding::TranscoderInputStream { public: GrpcZeroCopyInputStream(); @@ -53,8 +54,9 @@ class GrpcZeroCopyInputStream bool Next(const void** data, int* size); void BackUp(int count); - bool Skip(int count) { return false; } // not supported - ::google::protobuf::int64 ByteCount() const; + bool Skip(int count) { return false; } // not supported + ::google::protobuf::int64 ByteCount() const { return 0; } // Not implemented + int64_t BytesAvailable() const; private: GrpcMessageSerializer serializer_; diff --git a/src/grpc/zero_copy_stream_test.cc b/src/grpc/zero_copy_stream_test.cc index f21d42296..6b12cde25 100644 --- a/src/grpc/zero_copy_stream_test.cc +++ b/src/grpc/zero_copy_stream_test.cc @@ -86,10 +86,10 @@ TEST_F(GrpcZeroCopyInputStreamTest, SimpleRead) { stream.AddMessage(CreateByteBuffer(SliceData{slice21, slice22}), true); stream.Finish(); - // Test ByteCount() + // Test BytesAvailable() EXPECT_EQ(slice11.size() + slice12.size() + slice21.size() + slice22.size() + 10, // +10 bytes for two delimiters - stream.ByteCount()); + stream.BytesAvailable()); // Test the message1 delimiter ASSERT_TRUE(stream.Next(&data, &size)); @@ -97,28 +97,28 @@ TEST_F(GrpcZeroCopyInputStreamTest, SimpleRead) { EXPECT_EQ(slice11.size() + slice12.size(), DelimiterToSize(reinterpret_cast(data))); - // Test ByteCount() + // Test BytesAvailable() EXPECT_EQ(slice11.size() + slice12.size() + slice21.size() + slice22.size() + 5, // +5 bytes for one delimiter - stream.ByteCount()); + stream.BytesAvailable()); // Test the slices ASSERT_TRUE(stream.Next(&data, &size)); ASSERT_EQ(slice11.size(), size); EXPECT_EQ(slice11, std::string(reinterpret_cast(data), size)); - // Test ByteCount() + // Test BytesAvailable() EXPECT_EQ(slice12.size() + slice21.size() + slice22.size() + 5, // +5 bytes for one delimiter - stream.ByteCount()); + stream.BytesAvailable()); ASSERT_TRUE(stream.Next(&data, &size)); ASSERT_EQ(slice12.size(), size); EXPECT_EQ(slice12, std::string(reinterpret_cast(data), size)); - // Test ByteCount() + // Test BytesAvailable() EXPECT_EQ(slice21.size() + slice22.size() + 5, // +5 bytes for one delimiter - stream.ByteCount()); + stream.BytesAvailable()); // Test the message2 delimiter ASSERT_TRUE(stream.Next(&data, &size)); @@ -126,23 +126,23 @@ TEST_F(GrpcZeroCopyInputStreamTest, SimpleRead) { EXPECT_EQ(slice21.size() + slice22.size(), DelimiterToSize(reinterpret_cast(data))); - // Test ByteCount() - EXPECT_EQ(slice21.size() + slice22.size(), stream.ByteCount()); + // Test BytesAvailable() + EXPECT_EQ(slice21.size() + slice22.size(), stream.BytesAvailable()); // Test the slices ASSERT_TRUE(stream.Next(&data, &size)); ASSERT_EQ(slice21.size(), size); EXPECT_EQ(slice21, std::string(reinterpret_cast(data), size)); - // Test ByteCount() - EXPECT_EQ(slice22.size(), stream.ByteCount()); + // Test BytesAvailable() + EXPECT_EQ(slice22.size(), stream.BytesAvailable()); ASSERT_TRUE(stream.Next(&data, &size)); ASSERT_EQ(slice22.size(), size); EXPECT_EQ(slice22, std::string(reinterpret_cast(data), size)); // Test the end of the stream - EXPECT_EQ(0, stream.ByteCount()); + EXPECT_EQ(0, stream.BytesAvailable()); EXPECT_FALSE(stream.Next(&data, &size)); } @@ -167,9 +167,9 @@ TEST_F(GrpcZeroCopyInputStreamTest, Backups) { // Back up stream.BackUp(5); - // Test the ByteCount() + // Test the BytesAvailable() EXPECT_EQ(slice1.size() + slice2.size() + 5, // +5 bytes for the delimiter - stream.ByteCount()); + stream.BytesAvailable()); // Test the slice again ASSERT_TRUE(stream.Next(&data, &size)); @@ -184,14 +184,14 @@ TEST_F(GrpcZeroCopyInputStreamTest, Backups) { // Back up & test again stream.BackUp(size); - EXPECT_EQ(slice1.size() + slice2.size(), stream.ByteCount()); + EXPECT_EQ(slice1.size() + slice2.size(), stream.BytesAvailable()); ASSERT_TRUE(stream.Next(&data, &size)); ASSERT_EQ(slice1.size(), size); EXPECT_EQ(slice1, std::string(reinterpret_cast(data), size)); // Now Back up 10 bytes & test again stream.BackUp(10); - EXPECT_EQ(10 + slice2.size(), stream.ByteCount()); + EXPECT_EQ(10 + slice2.size(), stream.BytesAvailable()); ASSERT_TRUE(stream.Next(&data, &size)); ASSERT_EQ(10, size); EXPECT_EQ(slice1.substr(slice1.size() - 10), @@ -204,14 +204,14 @@ TEST_F(GrpcZeroCopyInputStreamTest, Backups) { // Back up and test again stream.BackUp(size); - EXPECT_EQ(slice2.size(), stream.ByteCount()); + EXPECT_EQ(slice2.size(), stream.BytesAvailable()); ASSERT_TRUE(stream.Next(&data, &size)); ASSERT_EQ(slice2.size(), size); EXPECT_EQ(slice2, std::string(reinterpret_cast(data), size)); // Now Back up size - 1 bytes (all but 1) and check again stream.BackUp(size - 1); - EXPECT_EQ(slice2.size() - 1, stream.ByteCount()); + EXPECT_EQ(slice2.size() - 1, stream.BytesAvailable()); ASSERT_TRUE(stream.Next(&data, &size)); ASSERT_EQ(slice2.size() - 1, size); EXPECT_EQ(slice2.substr(1), diff --git a/src/nginx/grpc.cc b/src/nginx/grpc.cc index 113d55fee..b6f54d3a3 100644 --- a/src/nginx/grpc.cc +++ b/src/nginx/grpc.cc @@ -93,8 +93,14 @@ std::pair> GrpcGetStub( return std::make_pair(Status::OK, it->second); } - auto result = std::make_shared<::grpc::GenericStub>( - ::grpc::CreateChannel(address, ::grpc::InsecureChannelCredentials())); + ::grpc::ChannelArguments channel_arguments; + + channel_arguments.SetMaxReceiveMessageSize(INT_MAX); + channel_arguments.SetMaxSendMessageSize(INT_MAX); + + auto result = + std::make_shared<::grpc::GenericStub>(::grpc::CreateCustomChannel( + address, ::grpc::InsecureChannelCredentials(), channel_arguments)); if (result) { espcf->grpc_stubs.emplace(address, result); diff --git a/src/nginx/t/BUILD b/src/nginx/t/BUILD index b0effdcd6..8295b47b0 100644 --- a/src/nginx/t/BUILD +++ b/src/nginx/t/BUILD @@ -110,6 +110,7 @@ nginx_suite( "grpc_grpc_fallback.t", "grpc_grpc_override.t", "grpc_large_request.t", + "grpc_large_response.t", "grpc_large_streaming.t", "grpc_long_streaming.t", "grpc_metadata.t", @@ -171,6 +172,7 @@ nginx_suite( "check_custom_api_key.t", "check_default.t", "check_invalid_api_key.t", + "check_key_restriction.t", "check_no_consumer.t", "check_referer.t", "check_report_body.t", @@ -188,6 +190,9 @@ nginx_suite( "multiple_apis.t", "no_backend.t", "no_service_control.t", + "quota.t", + "quota_api_not_available.t", + "quota_exhausted.t", "reject_unrecognized.t", "report_3xx.t", "report_4xx.t", diff --git a/src/nginx/t/check_key_restriction.t b/src/nginx/t/check_key_restriction.t new file mode 100644 index 000000000..961a5d263 --- /dev/null +++ b/src/nginx/t/check_key_restriction.t @@ -0,0 +1,247 @@ +# Copyright (C) Extensible Service Proxy Authors +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# +################################################################################ +# +use strict; +use warnings; +use JSON::PP; +use Data::Dumper; + +################################################################################ + +use src::nginx::t::ApiManager; # Must be first (sets up import path to + # the Nginx test module) +use src::nginx::t::HttpServer; +use src::nginx::t::ServiceControl; +use Test::Nginx; # Imports Nginx's test module +use Test::More; # And the test framework + +################################################################################ + +# Port assignments +my $NginxPort = ApiManager::pick_port(); +my $BackendPort = ApiManager::pick_port(); +my $ServiceControlPort = ApiManager::pick_port(); + +my $t = Test::Nginx->new()->has(qw/http proxy/)->plan(22); + +# Save servce configuration that disables the report cache. +# Report request will be sent for each client request +$t->write_file('server.pb.txt', <<"EOF"); +service_control_config { + report_aggregator_config { + cache_entries: 0 + flush_interval_ms: 1000 + } +} +EOF + +# Save service name in the service configuration protocol buffer file. +$t->write_file( 'service.pb.txt', + ApiManager::get_bookstore_service_config . <<"EOF"); +control { + environment: "http://127.0.0.1:${ServiceControlPort}" +} +EOF + +ApiManager::write_file_expand( $t, 'nginx.conf', <<"EOF"); +%%TEST_GLOBALS%% +daemon off; +events { + worker_connections 32; +} +http { + %%TEST_GLOBALS_HTTP%% + server_tokens off; + set_real_ip_from 0.0.0.0/1; + set_real_ip_from 0::/1; + real_ip_header X-Forwarded-For; + real_ip_recursive on; + server { + listen 127.0.0.1:${NginxPort}; + server_name localhost; + location / { + endpoints { + api service.pb.txt; + server_config server.pb.txt; + %%TEST_CONFIG%% + on; + } + proxy_pass http://127.0.0.1:${BackendPort}; + } + } +} +EOF + +my $report_done = 'report_done'; + +$t->run_daemon( \&bookstore, $t, $BackendPort, 'bookstore.log' ); +$t->run_daemon( \&servicecontrol, $t, $ServiceControlPort, + 'servicecontrol.log', $report_done); +is( $t->waitforsocket("127.0.0.1:${BackendPort}"), 1, 'Bookstore socket ready.' ); +is( $t->waitforsocket("127.0.0.1:${ServiceControlPort}"), 1, + 'Service control socket ready.' ); +$t->run(); + +################################################################################ + +my $response = ApiManager::http($NginxPort,<<'EOF'); +GET /shelves?key=this-is-an-api-key HTTP/1.0 +Referer: http://google.com/bookstore/root +Host: localhost +X-Forwarded-For: 10.20.30.40 +X-Android-Package: com.goolge.cloud.esp +X-Android-Cert: AIzaSyB4Gz8nyaSaWo63IPUcy5d_L8dpKtOTSD0 +X-Ios-Bundle-Identifier: 5b40ad6af9a806305a0a56d7cb91b82a27c26909 + +EOF + +is($t->waitforfile("$t->{_testdir}/${report_done}"), 1, + 'Report body file ready.'); +$t->stop_daemons(); + +my ( $response_headers, $response_body ) = split /\r\n\r\n/, $response, 2; + +like( $response_headers, qr/HTTP\/1\.1 200 OK/, 'Returned HTTP 200.' ); +is( $response_body, <<'EOF', 'Shelves returned in the response body.' ); +{ "shelves": [ + { "name": "shelves/1", "theme": "Fiction" }, + { "name": "shelves/2", "theme": "Fantasy" } + ] +} +EOF + +my @requests = ApiManager::read_http_stream( $t, 'bookstore.log' ); +is( scalar @requests, 1, 'Backend received one request' ); + +my $r = shift @requests; + +is( $r->{verb}, 'GET', 'Backend request was a get' ); +is( $r->{uri}, '/shelves?key=this-is-an-api-key', 'Backend uri was /shelves' ); +is( $r->{headers}->{host}, "127.0.0.1:${BackendPort}", 'Host header was set' ); + +@requests = ApiManager::read_http_stream( $t, 'servicecontrol.log' ); +is( scalar @requests, 2, 'Service control received two requests' ); + +# check +$r = shift @requests; +is( $r->{verb}, 'POST', ':check verb was post' ); +is( $r->{uri}, '/v1/services/endpoints-test.cloudendpointsapis.com:check', + ':check was called'); +is( $r->{headers}->{host}, "127.0.0.1:${ServiceControlPort}", + 'Host header was set'); +is( $r->{headers}->{'content-type'}, 'application/x-protobuf', + ':check Content-Type was protocol buffer'); + +my $check_request = decode_json(ServiceControl::convert_proto( + $r->{body}, 'check_request', 'json' ) ); + +is( $check_request->{operation}->{labels}-> + {'servicecontrol.googleapis.com/caller_ip'}, "10.20.30.40", + "servicecontrol.googleapis.com/caller_ip was overrode by ". + "X-Forwarded-For header" ); +is( $check_request->{operation}->{labels}-> + {'servicecontrol.googleapis.com/android_package_name'}, + "com.goolge.cloud.esp", + "servicecontrol.googleapis.com/android_package_name ". + "is 'com.goolge.cloud.esp'" ); +is( $check_request->{operation}->{labels}-> + {'servicecontrol.googleapis.com/android_cert_fingerprint'}, + "AIzaSyB4Gz8nyaSaWo63IPUcy5d_L8dpKtOTSD0", + "servicecontrol.googleapis.com/android_cert_fingerprint ". + "is 'AIzaSyB4Gz8nyaSaWo63IPUcy5d_L8dpKtOTSD0'" ); +is( $check_request->{operation}->{labels}-> + {'servicecontrol.googleapis.com/ios_bundle_id'}, + "5b40ad6af9a806305a0a56d7cb91b82a27c26909", + "servicecontrol.googleapis.com/ios_bundle_id ". + "is '5b40ad6af9a806305a0a56d7cb91b82a27c26909'" ); + +# report +$r = shift @requests; + +is( $r->{verb}, 'POST', ':report verb was post' ); +is( $r->{uri}, '/v1/services/endpoints-test.cloudendpointsapis.com:report', + ':report was called'); +is( $r->{headers}->{host}, "127.0.0.1:${ServiceControlPort}", + 'Host header was set'); +is( $r->{headers}->{'content-type'}, 'application/x-protobuf', + ':check Content-Type was protocol buffer' ); + +my $report_request = decode_json(ServiceControl::convert_proto( + $r->{body}, 'report_request', 'json' ) ); + +################################################################################ + +sub bookstore { + my ( $t, $port, $file ) = @_; + my $server = HttpServer->new( $port, $t->testdir() . '/' . $file ) + or die "Can't create test server socket: $!\n"; + local $SIG{PIPE} = 'IGNORE'; + + $server->on( 'GET', '/shelves?key=this-is-an-api-key', <<'EOF'); +HTTP/1.1 200 OK +Connection: close + +{ "shelves": [ + { "name": "shelves/1", "theme": "Fiction" }, + { "name": "shelves/2", "theme": "Fantasy" } + ] +} +EOF + $server->run(); +} + +sub servicecontrol { + my ( $t, $port, $file, $done) = @_; + my $server = HttpServer->new( $port, $t->testdir() . '/' . $file ) + or die "Can't create test server socket: $!\n"; + local $SIG{PIPE} = 'IGNORE'; + + $server->on_sub('POST', + '/v1/services/endpoints-test.cloudendpointsapis.com:check', sub { + my ($headers, $body, $client) = @_; + print $client <<'EOF'; +HTTP/1.1 200 OK +Connection: close + +EOF + }); + + $server->on_sub('POST', + '/v1/services/endpoints-test.cloudendpointsapis.com:report', sub { + my ($headers, $body, $client) = @_; + print $client <<'EOF'; +HTTP/1.1 200 OK +Connection: close + +EOF + + $t->write_file($done, ':report done'); + }); + + $server->run(); +} + +################################################################################ diff --git a/src/nginx/t/cloud_trace.t b/src/nginx/t/cloud_trace.t index de001b46c..01406ff6e 100644 --- a/src/nginx/t/cloud_trace.t +++ b/src/nginx/t/cloud_trace.t @@ -44,7 +44,7 @@ my $BackendPort = ApiManager::pick_port(); my $ServiceControlPort = ApiManager::pick_port(); my $CloudTracePort = ApiManager::pick_port(); -my $t = Test::Nginx->new()->has(qw/http proxy/)->plan(27); +my $t = Test::Nginx->new()->has(qw/http proxy/)->plan(29); my $config = ApiManager::get_bookstore_service_config_allow_unregistered . ApiManager::read_test_file('testdata/logs_metrics.pb.txt') . <<"EOF"; @@ -151,11 +151,15 @@ is($json_obj->{traces}->[0]->{spans}->[3]->{name}, 'Call ServiceControl server', 'Next trace span is Call ServiceControl server'); is($json_obj->{traces}->[0]->{spans}->[3]->{parentSpanId}, $check_service_control_cache_id, 'Parent of Call ServiceControl sever span is CheckServiceControlCache'); -is($json_obj->{traces}->[0]->{spans}->[4]->{name}, 'Backend', +is($json_obj->{traces}->[0]->{spans}->[4]->{name}, 'QuotaControl', 'Next trace span is Backend'); is($json_obj->{traces}->[0]->{spans}->[4]->{parentSpanId}, $rootid, 'Parent of Beckend span is root'); -my $backend_span_id = $json_obj->{traces}->[0]->{spans}->[4]->{spanId}; +is($json_obj->{traces}->[0]->{spans}->[5]->{name}, 'Backend', + 'Next trace span is Backend'); +is($json_obj->{traces}->[0]->{spans}->[5]->{parentSpanId}, $rootid, + 'Parent of Beckend span is root'); +my $backend_span_id = $json_obj->{traces}->[0]->{spans}->[5]->{spanId}; my @bookstore_requests = ApiManager::read_http_stream($t, 'bookstore.log'); is(scalar @bookstore_requests, 1, 'Bookstore received 1 request.'); diff --git a/src/nginx/t/grpc_large_response.t b/src/nginx/t/grpc_large_response.t new file mode 100644 index 000000000..31b512a7d --- /dev/null +++ b/src/nginx/t/grpc_large_response.t @@ -0,0 +1,120 @@ +# Copyright (C) Extensible Service Proxy Authors +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# +################################################################################ +# +use strict; +use warnings; + +################################################################################ + +use src::nginx::t::ApiManager; # Must be first (sets up import path to the Nginx test module) +use src::nginx::t::HttpServer; +use Test::Nginx; # Imports Nginx's test module +use Test::More; # And the test framework + +################################################################################ + +# Port assignments +my $Http2NginxPort = ApiManager::pick_port(); +my $ServiceControlPort = ApiManager::pick_port(); +my $GrpcBackendPort = ApiManager::pick_port(); + +my $t = Test::Nginx->new()->has(qw/http proxy/)->plan(4); + +$t->write_file('service.pb.txt', ApiManager::get_grpc_test_service_config($GrpcBackendPort) . <<"EOF"); +control { + environment: "http://127.0.0.1:${ServiceControlPort}" +} +EOF + +$t->write_file_expand('nginx.conf', <<"EOF"); +%%TEST_GLOBALS%% +daemon off; +events { + worker_connections 32; +} +http { + %%TEST_GLOBALS_HTTP%% + server { + listen 127.0.0.1:${Http2NginxPort} http2; + server_name localhost; + client_max_body_size 32m; + location / { + endpoints { + api service.pb.txt; + on; + } + grpc_pass 127.0.0.2:${GrpcBackendPort}; + } + } +} +EOF + +$t->run_daemon(\&service_control, $t, $ServiceControlPort, 'requests.log'); +$t->run_daemon(\&ApiManager::grpc_test_server, $t, "127.0.0.1:${GrpcBackendPort}"); +is($t->waitforsocket("127.0.0.1:${ServiceControlPort}"), 1, 'Service control socket ready.'); +is($t->waitforsocket("127.0.0.1:${GrpcBackendPort}"), 1, 'GRPC test server socket ready.'); +$t->run(); +is($t->waitforsocket("127.0.0.1:${Http2NginxPort}"), 1, 'Nginx socket ready.'); + +################################################################################ +my $test_results = &ApiManager::run_grpc_test($t, <<"EOF"); +server_addr: "127.0.0.1:${Http2NginxPort}" +plans { + echo { + call_config { + api_key: "this-is-an-api-key" + } + request { + space_payload_size: 30000000 + } + } +} +EOF + +$t->stop_daemons(); + +like($test_results, qr/echo \{/, "Test returned error"); + +################################################################################ + +sub service_control { + my ($t, $port, $file) = @_; + my $server = HttpServer->new($port, $t->testdir() . '/' . $file) + or die "Can't create test server socket: $!\n"; + + $server->on_sub('POST', '/v1/services/endpoints-grpc-test.cloudendpointsapis.com:check', sub { + my ($headers, $body, $client) = @_; + print $client <<'EOF'; +HTTP/1.1 200 OK +Connection: close + +EOF + }); + + $server->run(); +} + +################################################################################ diff --git a/src/nginx/t/quota.t b/src/nginx/t/quota.t new file mode 100644 index 000000000..a4cc1998d --- /dev/null +++ b/src/nginx/t/quota.t @@ -0,0 +1,253 @@ +# Copyright (C) Extensible Service Proxy Authors +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# +################################################################################ +# +use strict; +use warnings; +use JSON::PP; + +################################################################################ + +use src::nginx::t::ApiManager; # Must be first (sets up import path to + # the Nginx test module) +use src::nginx::t::HttpServer; +use src::nginx::t::ServiceControl; +use Test::Nginx; # Imports Nginx's test module +use Test::More; # And the test framework + +################################################################################ + +# Port assignments +my $NginxPort = ApiManager::pick_port(); +my $BackendPort = ApiManager::pick_port(); +my $ServiceControlPort = ApiManager::pick_port(); + +my $t = Test::Nginx->new()->has(qw/http proxy/)->plan(26); + +# Save servce configuration that disables the report cache. +# Report request will be sent for each client request +$t->write_file('server.pb.txt', <<"EOF"); +service_control_config { + report_aggregator_config { + cache_entries: 0 + flush_interval_ms: 1000 + } +} +EOF + +# Save service name in the service configuration protocol buffer file. +$t->write_file( 'service.pb.txt', + ApiManager::get_bookstore_service_config . <<"EOF"); +control { + environment: "http://127.0.0.1:${ServiceControlPort}" +} +quota { + metric_rules [ + { + selector: "ListShelves" + metric_costs: [ + { + key: "metrics_first" + value: 2 + }, + { + key: "metrics_second" + value: 1 + } + ] + } + ] +} +EOF + +ApiManager::write_file_expand( $t, 'nginx.conf', <<"EOF"); +%%TEST_GLOBALS%% +daemon off; +events { + worker_connections 32; +} +http { + %%TEST_GLOBALS_HTTP%% + server_tokens off; + server { + listen 127.0.0.1:${NginxPort}; + server_name localhost; + location / { + endpoints { + api service.pb.txt; + server_config server.pb.txt; + %%TEST_CONFIG%% + on; + } + proxy_pass http://127.0.0.1:${BackendPort}; + } + } +} +EOF + +$t->run_daemon( \&bookstore, $t, $BackendPort, 'bookstore.log' ); +$t->run_daemon( \&servicecontrol, $t, $ServiceControlPort, 'servicecontrol.log' ); +is( $t->waitforsocket("127.0.0.1:${BackendPort}"), 1, 'Bookstore socket ready.' ); +is( $t->waitforsocket("127.0.0.1:${ServiceControlPort}"), 1, + 'Service control socket ready.' ); +$t->run(); + +################################################################################ + +my $response = ApiManager::http_get( + $NginxPort, '/shelves?key=this-is-an-api-key' ); + +$t->stop_daemons(); + +my ( $response_headers, $response_body ) = split /\r\n\r\n/, $response, 2; + +like( $response_headers, qr/HTTP\/1\.1 200 OK/, 'Returned HTTP 200.' ); +is( $response_body, <<'EOF', 'Shelves returned in the response body.' ); +{ "shelves": [ + { "name": "shelves/1", "theme": "Fiction" }, + { "name": "shelves/2", "theme": "Fantasy" } + ] +} +EOF + +my @requests = ApiManager::read_http_stream( $t, 'bookstore.log' ); +is( scalar @requests, 1, 'Backend received one request' ); + +my $r = shift @requests; + +is( $r->{verb}, 'GET', 'Backend request was a get' ); +is( $r->{uri}, '/shelves?key=this-is-an-api-key', 'Backend uri was /shelves' ); +is( $r->{headers}->{host}, "127.0.0.1:${BackendPort}", 'Host header was set' ); + +@requests = ApiManager::read_http_stream( $t, 'servicecontrol.log' ); +is( scalar @requests, 3, 'Service control received three requests' ); + +# check +$r = shift @requests; +is( $r->{verb}, 'POST', ':check verb was post' ); +is( $r->{uri}, '/v1/services/endpoints-test.cloudendpointsapis.com:check', + ':check was called'); +is( $r->{headers}->{host}, "127.0.0.1:${ServiceControlPort}", + 'Host header was set'); +is( $r->{headers}->{'content-type'}, 'application/x-protobuf', + ':check Content-Type was protocol buffer'); + +# test allocateQuota request was requested +$r = shift @requests; +is( $r->{verb}, 'POST', ':allocateQuota verb was post' ); +is( $r->{uri}, + '/v1/services/endpoints-test.cloudendpointsapis.com:allocateQuota', + ':allocateQuota was called'); +is( $r->{headers}->{host}, "127.0.0.1:${ServiceControlPort}", + 'Host header was set'); +is( $r->{headers}->{'content-type'}, 'application/x-protobuf', + ':check Content-Type was protocol buffer' ); + +my $allocate_quota_request = decode_json(ServiceControl::convert_proto( + $r->{body}, 'quota_request', 'json' ) ); + +my @quotaMetrics = + @{ $allocate_quota_request->{allocateOperation}->{quotaMetrics} }; +is( @quotaMetrics, 2, "Quota metrics should have two elements" ); + +my @sorted_quotaMetrics = + sort { $a->{metricName} cmp $b->{metricName} } @quotaMetrics; + +is( $sorted_quotaMetrics[0]->{metricName}, "metrics_first", + "Quota metric name is 'metrics_first'" ); +is( $sorted_quotaMetrics[0]->{metricValues}[0]->{int64Value}, 2, + "Quota metric value is 2" ); +is( $sorted_quotaMetrics[1]->{metricName}, "metrics_second", + "Quota metric name is 'metrics_second'" ); +is( $sorted_quotaMetrics[1]->{metricValues}[0]->{int64Value}, 1, + "Quota metric value is 1" ); + +# check report +$r = shift @requests; + +is( $r->{verb}, 'POST', ':report verb was post' ); +is( $r->{uri}, '/v1/services/endpoints-test.cloudendpointsapis.com:report', + ':report was called'); +is( $r->{headers}->{host}, "127.0.0.1:${ServiceControlPort}", + 'Host header was set'); +is( $r->{headers}->{'content-type'}, 'application/x-protobuf', + ':check Content-Type was protocol buffer' ); + +################################################################################ + +sub bookstore { + my ( $t, $port, $file ) = @_; + my $server = HttpServer->new( $port, $t->testdir() . '/' . $file ) + or die "Can't create test server socket: $!\n"; + local $SIG{PIPE} = 'IGNORE'; + + $server->on( 'GET', '/shelves?key=this-is-an-api-key', <<'EOF'); +HTTP/1.1 200 OK +Connection: close + +{ "shelves": [ + { "name": "shelves/1", "theme": "Fiction" }, + { "name": "shelves/2", "theme": "Fantasy" } + ] +} +EOF + $server->run(); +} + +my @quota_responses = (); +my $quota_response_index = 0; + +sub servicecontrol { + my ( $t, $port, $file ) = @_; + my $server = HttpServer->new( $port, $t->testdir() . '/' . $file ) + or die "Can't create test server socket: $!\n"; + local $SIG{PIPE} = 'IGNORE'; + + $server->on( 'POST', + '/v1/services/endpoints-test.cloudendpointsapis.com:check', <<'EOF'); +HTTP/1.1 200 OK +Connection: close + +EOF + + $server->on('POST', + '/v1/services/endpoints-test.cloudendpointsapis.com:allocateQuota', + <<'EOF'); +HTTP/1.1 200 OK +Connection: close + +EOF + + $server->on( 'POST', + '/v1/services/endpoints-test.cloudendpointsapis.com:report', <<'EOF'); +HTTP/1.1 200 OK +Connection: close + +EOF + + $server->run(); +} + +################################################################################ diff --git a/src/nginx/t/quota_api_not_available.t b/src/nginx/t/quota_api_not_available.t new file mode 100644 index 000000000..277b1c3f7 --- /dev/null +++ b/src/nginx/t/quota_api_not_available.t @@ -0,0 +1,200 @@ +# Copyright (C) Extensible Service Proxy Authors +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# +################################################################################ +# +use strict; +use warnings; + +use JSON::PP; +use Data::Dumper; + +################################################################################ + +use src::nginx::t::ApiManager; # Must be first (sets up import path + # to the Nginx test module) +use src::nginx::t::HttpServer; +use src::nginx::t::ServiceControl; +use Test::Nginx; # Imports Nginx's test module +use Test::More; # And the test framework + +################################################################################ + +# Port assignments +my $NginxPort = ApiManager::pick_port(); +my $BackendPort = ApiManager::pick_port(); +my $ServiceControlPort = ApiManager::pick_port(); + +my $t = Test::Nginx->new()->has(qw/http proxy/)->plan(9); + +# Save servce configuration that disables the report cache. +# Report request will be sent for each client request +$t->write_file('server.pb.txt', <<"EOF"); +service_control_config { + report_aggregator_config { + cache_entries: 0 + flush_interval_ms: 1000 + } +} +EOF + +# Save service name in the service configuration protocol buffer file. +$t->write_file( 'service.pb.txt', + ApiManager::get_bookstore_service_config . <<"EOF"); +control { + environment: "http://127.0.0.1:${ServiceControlPort}" +} +quota { + metric_rules [ + { + selector: "ListShelves" + metric_costs: [ + { + key: "metrics_first" + value: 2 + }, + { + key: "metrics_second" + value: 1 + } + ] + } + ] +} +EOF + +ApiManager::write_file_expand( $t, 'nginx.conf', <<"EOF"); +%%TEST_GLOBALS%% +daemon off; +events { + worker_connections 32; +} +http { + %%TEST_GLOBALS_HTTP%% + server_tokens off; + server { + listen 127.0.0.1:${NginxPort}; + server_name localhost; + location / { + endpoints { + api service.pb.txt; + server_config server.pb.txt; + %%TEST_CONFIG%% + on; + } + proxy_pass http://127.0.0.1:${BackendPort}; + } + } +} +EOF + +$t->run_daemon( \&bookstore, $t, $BackendPort, 'bookstore.log' ); +$t->run_daemon( \&servicecontrol, $t, $ServiceControlPort, + 'servicecontrol.log' ); +is( $t->waitforsocket("127.0.0.1:${BackendPort}"), 1, + 'Bookstore socket ready.' ); +is( $t->waitforsocket("127.0.0.1:${ServiceControlPort}"), 1, + 'Service control socket ready.' ); +$t->run(); + +################################################################################ + +my $response = ApiManager::http_get( $NginxPort, + '/shelves?key=this-is-an-api-key' ); + +$t->stop_daemons(); + +my ( $response_headers, $response_body ) = split /\r\n\r\n/, $response, 2; + +like( $response_headers, qr/HTTP\/1\.1 200 OK/, 'Returned HTTP 200.' ); +is( $response_body, <<'EOF', 'Shelves returned in the response body.' ); +{ "shelves": [ + { "name": "shelves/1", "theme": "Fiction" }, + { "name": "shelves/2", "theme": "Fantasy" } + ] +} +EOF + +my @requests = ApiManager::read_http_stream( $t, 'bookstore.log' ); +is( scalar @requests, 1, 'Backend received empty request' ); + +my $r = shift @requests; + +is( $r->{verb}, 'GET', 'Backend request was a get' ); +is( $r->{uri}, '/shelves?key=this-is-an-api-key', 'Backend uri was /shelves' ); +is( $r->{headers}->{host}, "127.0.0.1:${BackendPort}", 'Host header was set' ); + +@requests = ApiManager::read_http_stream( $t, 'servicecontrol.log' ); +is( scalar @requests, 3, 'Service control received three requests' ); + +################################################################################ + +sub bookstore { + my ( $t, $port, $file ) = @_; + my $server = HttpServer->new( $port, $t->testdir() . '/' . $file ) + or die "Can't create test server socket: $!\n"; + local $SIG{PIPE} = 'IGNORE'; + + $server->on( 'GET', '/shelves?key=this-is-an-api-key', <<'EOF'); +HTTP/1.1 200 OK +Connection: close + +{ "shelves": [ + { "name": "shelves/1", "theme": "Fiction" }, + { "name": "shelves/2", "theme": "Fantasy" } + ] +} +EOF + $server->run(); +} + +my @quota_responses = (); +my $quota_response_index = 0; + +sub servicecontrol { + my ( $t, $port, $file ) = @_; + my $server = HttpServer->new( $port, $t->testdir() . '/' . $file ) + or die "Can't create test server socket: $!\n"; + local $SIG{PIPE} = 'IGNORE'; + + $server->on( 'POST', + '/v1/services/endpoints-test.cloudendpointsapis.com:check', <<'EOF'); +HTTP/1.1 200 OK +Connection: close + +EOF + # The allocateQuota request receives HTTP 404 Not Found error code. + # This simulates QuotaController is not available. + + $server->on( 'POST', + '/v1/services/endpoints-test.cloudendpointsapis.com:report', <<'EOF'); +HTTP/1.1 200 OK +Connection: close + +EOF + + $server->run(); +} + +################################################################################ diff --git a/src/nginx/t/quota_exhausted.t b/src/nginx/t/quota_exhausted.t new file mode 100644 index 000000000..0fdd8f404 --- /dev/null +++ b/src/nginx/t/quota_exhausted.t @@ -0,0 +1,218 @@ +# Copyright (C) Extensible Service Proxy Authors +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# +################################################################################ +# +use strict; +use warnings; + +use JSON::PP; +use Data::Dumper; + +################################################################################ + +use src::nginx::t::ApiManager; # Must be first (sets up import path to + # the Nginx test module) +use src::nginx::t::HttpServer; +use src::nginx::t::ServiceControl; +use Test::Nginx; # Imports Nginx's test module +use Test::More; # And the test framework + +################################################################################ + +# Port assignments +my $NginxPort = ApiManager::pick_port(); +my $BackendPort = ApiManager::pick_port(); +my $ServiceControlPort = ApiManager::pick_port(); + +my $t = Test::Nginx->new()->has(qw/http proxy/)->plan(6); + +# Save servce configuration that disables the report cache. +# Report request will be sent for each client request +$t->write_file('server.pb.txt', <<"EOF"); +service_control_config { + report_aggregator_config { + cache_entries: 0 + flush_interval_ms: 1000 + } +} +EOF + +# Save service name in the service configuration protocol buffer file. +$t->write_file( 'service.pb.txt', + ApiManager::get_bookstore_service_config . <<"EOF"); +control { + environment: "http://127.0.0.1:${ServiceControlPort}" +} +quota { + metric_rules [ + { + selector: "ListShelves" + metric_costs: [ + { + key: "metrics_first" + value: 2 + }, + { + key: "metrics_second" + value: 1 + } + ] + } + ] +} +EOF + +ApiManager::write_file_expand( $t, 'nginx.conf', <<"EOF"); +%%TEST_GLOBALS%% +daemon off; +events { + worker_connections 32; +} +http { + %%TEST_GLOBALS_HTTP%% + server_tokens off; + server { + listen 127.0.0.1:${NginxPort}; + server_name localhost; + location / { + endpoints { + api service.pb.txt; + server_config server.pb.txt; + %%TEST_CONFIG%% + on; + } + proxy_pass http://127.0.0.1:${BackendPort}; + } + } +} +EOF + +$t->run_daemon( \&bookstore, $t, $BackendPort, 'bookstore.log' ); +$t->run_daemon( \&servicecontrol, $t, $ServiceControlPort, + 'servicecontrol.log' ); +is( $t->waitforsocket("127.0.0.1:${BackendPort}"), + 1, 'Bookstore socket ready.' ); +is( $t->waitforsocket("127.0.0.1:${ServiceControlPort}"), + 1, 'Service control socket ready.' ); +$t->run(); + +################################################################################ + +my $response = + ApiManager::http_get( $NginxPort, '/shelves?key=this-is-an-api-key' ); + +$t->stop_daemons(); + +my ( $response_headers, $response_body ) = split /\r\n\r\n/, $response, 2; + +like( $response_headers, qr/HTTP\/1\.1 429/, 'Returned HTTP 429.' ); +is( $response_body, <<'EOF', 'Shelves returned in the response body.' ); +{ + "code": 8, + "message": "Quota allocation failed.", + "details": [ + { + "@type": "type.googleapis.com/google.rpc.DebugInfo", + "stackEntries": [], + "detail": "internal" + } + ] +} +EOF + +my @requests = ApiManager::read_http_stream( $t, 'bookstore.log' ); +is( scalar @requests, 0, 'Backend received empty request' ); + + +@requests = ApiManager::read_http_stream( $t, 'servicecontrol.log' ); +is( scalar @requests, 3, 'Service control received three requests' ); + + +################################################################################ + +sub bookstore { + my ( $t, $port, $file ) = @_; + my $server = HttpServer->new( $port, $t->testdir() . '/' . $file ) + or die "Can't create test server socket: $!\n"; + local $SIG{PIPE} = 'IGNORE'; + + $server->on( 'GET', '/shelves?key=this-is-an-api-key', <<'EOF'); +HTTP/1.1 200 OK +Connection: close + +{ "shelves": [ + { "name": "shelves/1", "theme": "Fiction" }, + { "name": "shelves/2", "theme": "Fantasy" } + ] +} +EOF + $server->run(); +} + +my @quota_responses = (); +my $quota_response_index = 0; + +sub servicecontrol { + my ( $t, $port, $file ) = @_; + my $server = HttpServer->new( $port, $t->testdir() . '/' . $file ) + or die "Can't create test server socket: $!\n"; + local $SIG{PIPE} = 'IGNORE'; + + my $quota_response_exhausted = + ServiceControl::convert_proto( <<'EOF', 'quota_response', 'binary' ); +operation_id: "006eaa26-5c2f-41bc-b6d8-0972eff8bdf6" +allocate_errors { + code: RESOURCE_EXHAUSTED + description: "Insufficient tokens for quota group and limit \'apiWriteQpsPerProject_LOW\' of service \'jaebonginternal.sandbox.google.com\', using the limit by ID \'container:1002409420961\'." +} +service_config_id: "2017-02-08r9" +EOF + + $server->on( 'POST', + '/v1/services/endpoints-test.cloudendpointsapis.com:check', <<'EOF'); +HTTP/1.1 200 OK +Connection: close + +EOF + + $server->on('POST', + '/v1/services/endpoints-test.cloudendpointsapis.com:allocateQuota', + <<'EOF' . $quota_response_exhausted); +HTTP/1.1 200 OK +Connection: close + +EOF + + $server->on( 'POST', + '/v1/services/endpoints-test.cloudendpointsapis.com:report', <<'EOF'); +HTTP/1.1 200 OK +Connection: close + +EOF + + $server->run(); +} + +################################################################################ diff --git a/src/tools/proto_pass_perf.cc b/src/tools/proto_pass_perf.cc index a68548800..b890fc4c1 100644 --- a/src/tools/proto_pass_perf.cc +++ b/src/tools/proto_pass_perf.cc @@ -43,12 +43,15 @@ using google::api_manager::service_control::Proto; using ::google::api::servicecontrol::v1::CheckRequest; using ::google::api::servicecontrol::v1::CheckResponse; +using ::google::api::servicecontrol::v1::AllocateQuotaRequest; +using ::google::api::servicecontrol::v1::AllocateQuotaResponse; using ::google::api::servicecontrol::v1::ReportRequest; using ::google::api::servicecontrol::v1::ReportResponse; using ::google::protobuf::Arena; using ::google::protobuf::util::Status; using ::google::service_control_client::CheckAggregationOptions; +using ::google::service_control_client::QuotaAggregationOptions; using ::google::service_control_client::ReportAggregationOptions; using ::google::service_control_client::ServiceControlClient; using ::google::service_control_client::ServiceControlClientOptions; @@ -87,6 +90,7 @@ void FillReportRequestInfo(ReportRequestInfo* request) { int total_called_checks = 0; int total_called_reports = 0; +int total_called_quotas = 0; std::string request_text; // Compare the performance for passing Service Control Report protobuf to its @@ -103,10 +107,15 @@ int main() { CheckAggregationOptions(1000000 /*entries*/, 1000000 /* refresh_interval_ms */, 1000000 /*flush_interval_ms*/), + QuotaAggregationOptions(1000000 /*entries*/, + 1000000 /* refresh_interval_ms */), ReportAggregationOptions(1000000 /*entries*/, 1000000 /* refresh_interval_ms */)); options.check_transport = [](const CheckRequest&, CheckResponse*, TransportDoneFunc) { ++total_called_checks; }; + options.quota_transport = [](const AllocateQuotaRequest&, + AllocateQuotaResponse*, + TransportDoneFunc) { ++total_called_quotas; }; options.report_transport = [](const ReportRequest&, ReportResponse*, TransportDoneFunc) { ++total_called_reports; }; client = CreateServiceControlClient(kServiceName, kServiceConfigId, options); diff --git a/src/tools/service_control_json_gen.cc b/src/tools/service_control_json_gen.cc index de7f2594f..f63e557ec 100644 --- a/src/tools/service_control_json_gen.cc +++ b/src/tools/service_control_json_gen.cc @@ -46,6 +46,8 @@ enum OutputType { BINARY, TEXT, JSON }; enum ProtoMessageType { CHECK_REQUEST, CHECK_RESPONSE, + QUOTA_REQUEST, + QUOTA_RESPONSE, REPORT_REQUEST, REPORT_RESPONSE }; @@ -69,12 +71,16 @@ const int kTextOutput = 4; const int kBinaryOutput = 5; const int kCheckRequstProto = 6; const int kCheckResponseProto = 7; -const int kReportRequstProto = 8; -const int kReportResponseProto = 9; -const int kReportRequestSize = 10; +const int kAllocateQuotaRequstProto = 8; +const int kAllocateQuotaResponseProto = 9; +const int kReportRequstProto = 10; +const int kReportResponseProto = 11; +const int kReportRequestSize = 12; ::google::api::servicecontrol::v1::CheckRequest check_request; ::google::api::servicecontrol::v1::CheckResponse check_response; +::google::api::servicecontrol::v1::AllocateQuotaRequest quota_request; +::google::api::servicecontrol::v1::AllocateQuotaResponse quota_response; ::google::api::servicecontrol::v1::ReportRequest report_request; ::google::api::servicecontrol::v1::ReportResponse report_response; @@ -94,6 +100,8 @@ void ProcessCmdLine(int argc, char** argv) { {"binary", no_argument, nullptr, kBinaryOutput}, {"check_request", no_argument, nullptr, kCheckRequstProto}, {"check_response", no_argument, nullptr, kCheckResponseProto}, + {"quota_request", no_argument, nullptr, kAllocateQuotaRequstProto}, + {"quota_response", no_argument, nullptr, kAllocateQuotaResponseProto}, {"report_request", no_argument, nullptr, kReportRequstProto}, {"report_response", no_argument, nullptr, kReportResponseProto}, {"report_request_size", required_argument, nullptr, kReportRequestSize}, @@ -133,6 +141,12 @@ void ProcessCmdLine(int argc, char** argv) { case kCheckResponseProto: proto_type = CHECK_RESPONSE; break; + case kAllocateQuotaRequstProto: + proto_type = QUOTA_REQUEST; + break; + case kAllocateQuotaResponseProto: + proto_type = QUOTA_RESPONSE; + break; case kReportRequstProto: proto_type = REPORT_REQUEST; break; @@ -163,6 +177,8 @@ void ProcessCmdLine(int argc, char** argv) { "Protobuf message type:\n" " --check_request: CheckRequest protobuf message.\n" " --check_response: CheckResponse protobuf message.\n" + " --quota_request: AllocateQuotaRequest protobuf message.\n" + " --quota_response: AllocateQuotaResponse protobuf message.\n" " --report_request: ReportRequest protobuf message.\n" " --report_response: ReportResponse protobuf message.\n" "Input:\n" @@ -316,6 +332,12 @@ int main(int argc, char** argv) { case CHECK_RESPONSE: request = &check_response; break; + case QUOTA_REQUEST: + request = "a_request; + break; + case QUOTA_RESPONSE: + request = "a_response; + break; case REPORT_REQUEST: request = &report_request; break; diff --git a/test/bookstore/linux-build-bookstore-docker b/test/bookstore/linux-build-bookstore-docker index 3b5e2cf5b..0ad905962 100755 --- a/test/bookstore/linux-build-bookstore-docker +++ b/test/bookstore/linux-build-bookstore-docker @@ -42,7 +42,7 @@ done [[ -n "${IMAGE}" ]] || error_exit "Specify required image argument via '-i'" echo "Checking if docker image ${IMAGE} exists.." -gcloud docker pull "${IMAGE}" \ +gcloud docker -- pull "${IMAGE}" \ && echo "Image ${IMAGE} already exists; skipping" \ && exit 0 @@ -54,5 +54,5 @@ retry -n 3 docker build --no-cache -t ${IMAGE} \ echo "Pushing Docker image: ${IMAGE}" # Try 10 times, shortest wait is 10 seconds, exponential back-off. retry -n 10 -s 10 \ - gcloud docker push "${IMAGE}" \ + gcloud docker -- push "${IMAGE}" \ || error_exit "Failed to upload Docker image to gcr." diff --git a/test/echo/docker-remote/build-docker.sh b/test/echo/docker-remote/build-docker.sh index 23fe5599b..ec5d35abf 100755 --- a/test/echo/docker-remote/build-docker.sh +++ b/test/echo/docker-remote/build-docker.sh @@ -44,7 +44,7 @@ sed "s/\${PROJECT}/${PROJECT}/" ../service.json.temp > ./service.json ${GCLOUD} docker -- build --no-cache -t ${TAG} . docker tag -f ${TAG} gcr.io/${PROJECT}/${TAG} -${GCLOUD} docker push gcr.io/${PROJECT}/${TAG} +${GCLOUD} docker -- push gcr.io/${PROJECT}/${TAG} rm service.json diff --git a/test/echo/docker/build-docker.sh b/test/echo/docker/build-docker.sh index ad84494a9..9504f48a5 100755 --- a/test/echo/docker/build-docker.sh +++ b/test/echo/docker/build-docker.sh @@ -44,7 +44,7 @@ sed "s/\${PROJECT}/${PROJECT}/" ../service.json.temp > ./service.json docker build --no-cache -t ${TAG} . docker tag -f ${TAG} gcr.io/${PROJECT}/${TAG} -${GCLOUD} docker push gcr.io/${PROJECT}/${TAG} +${GCLOUD} docker -- push gcr.io/${PROJECT}/${TAG} rm service.json diff --git a/test/grpc/Dockerfile.temp b/test/grpc/Dockerfile.temp index 575c2dab9..47110258c 100644 --- a/test/grpc/Dockerfile.temp +++ b/test/grpc/Dockerfile.temp @@ -7,7 +7,7 @@ # 2) cp bazel-bin/test/grpc/grpc-test-server test/grpc # 3) IMAGE=gcr.io/endpointsv2/grpc-test-server:latest # 4) docker build --no-cache -t "${IMAGE}" test/grpc -# 5) gcloud docker push "${IMAGE}" +# 5) gcloud docker -- push "${IMAGE}" FROM debian:jessie diff --git a/test/grpc/client-test-lib.cc b/test/grpc/client-test-lib.cc index eb85ee204..1fbf87a6d 100644 --- a/test/grpc/client-test-lib.cc +++ b/test/grpc/client-test-lib.cc @@ -46,13 +46,14 @@ using ::google::api::servicecontrol::v1::ReportRequest; using ::google::protobuf::util::MessageDifferencer; using ::grpc::Alarm; using ::grpc::Channel; +using ::grpc::ChannelArguments; using ::grpc::ChannelCredentials; using ::grpc::ClientContext; using ::grpc::ClientAsyncReaderWriterInterface; using ::grpc::ClientAsyncResponseReaderInterface; using ::grpc::ClientReaderWriter; using ::grpc::CompletionQueue; -using ::grpc::CreateChannel; +using ::grpc::CreateCustomChannel; using ::grpc::InsecureChannelCredentials; using ::grpc::SslCredentials; using ::grpc::SslCredentialsOptions; @@ -92,7 +93,11 @@ std::shared_ptr GetCreds(const EchoStreamTest &desc) { template static std::unique_ptr GetStub(const std::string &addr, const T &desc) { - std::shared_ptr channel(CreateChannel(addr, GetCreds(desc))); + ChannelArguments args; + args.SetMaxReceiveMessageSize(INT_MAX); + args.SetMaxSendMessageSize(INT_MAX); + std::shared_ptr channel( + CreateCustomChannel(addr, GetCreds(desc), args)); return std::unique_ptr(Test::NewStub(channel)); } @@ -220,6 +225,10 @@ class Echo { desc_.mutable_request()->set_text( RandomString(desc_.request().random_payload_max_size())); } + if (desc_.request().space_payload_size() > 0) { + desc_.mutable_request()->set_text( + std::string(desc_.request().space_payload_size(), ' ')); + } } EchoTest desc_; diff --git a/test/grpc/grpc-test-server.cc b/test/grpc/grpc-test-server.cc index de96e198f..f020004d6 100644 --- a/test/grpc/grpc-test-server.cc +++ b/test/grpc/grpc-test-server.cc @@ -360,6 +360,8 @@ void TestServer::Run(const char *addr) { ServerBuilder builder; builder.AddListeningPort(addr, InsecureServerCredentials()); builder.RegisterService(this); + builder.SetMaxReceiveMessageSize(INT_MAX); + builder.SetMaxSendMessageSize(INT_MAX); cq_ = builder.AddCompletionQueue(); std::unique_ptr server(builder.BuildAndStart()); diff --git a/test/grpc/grpc-test.proto b/test/grpc/grpc-test.proto index 3db36e026..fd3d66acb 100644 --- a/test/grpc/grpc-test.proto +++ b/test/grpc/grpc-test.proto @@ -44,6 +44,10 @@ message EchoRequest { // The payload size is random between 0 and value of this field. int32 random_payload_max_size = 3; + // If non 0, the text payload is generated with space characters + // The payload size is specified by the value of this field. + int32 space_payload_size = 6; + // The metadata that server should return map return_initial_metadata = 4; map return_trailing_metadata = 5; diff --git a/test/grpc/linux-build-grpc-docker b/test/grpc/linux-build-grpc-docker index 25e462bfd..a5756939d 100755 --- a/test/grpc/linux-build-grpc-docker +++ b/test/grpc/linux-build-grpc-docker @@ -57,7 +57,7 @@ done [[ -n "${IMAGE}" ]] || error_exit "Specify required image argument via '-i'" echo "Checking if docker image ${IMAGE} exists.." -gcloud docker pull "${IMAGE}" \ +gcloud docker -- pull "${IMAGE}" \ && { echo "Image ${IMAGE} already exists; skipping"; exit 0; } BAZEL_TARGET="${ROOT}/bazel-bin/${TEST_SERVER_BIN}" @@ -82,5 +82,5 @@ retry -n 3 docker build --no-cache -t "${IMAGE}" \ echo "Pushing Docker image: ${IMAGE}" # Try 10 times, shortest wait is 10 seconds, exponential back-off. retry -n 10 -s 10 \ - gcloud docker push "${IMAGE}" \ + gcloud docker -- push "${IMAGE}" \ || error_exit "Failed to upload Docker image to gcr." diff --git a/test/transcoding/Dockerfile b/test/transcoding/Dockerfile index 20c14ceaa..a01f41971 100644 --- a/test/transcoding/Dockerfile +++ b/test/transcoding/Dockerfile @@ -8,7 +8,7 @@ # 2) cp bazel-bin/test/transcoding/bookstore-server test/transcoding # 3) IMAGE=gcr.io/endpointsv2/bookstore-grpc:latest # 4) docker build --no-cache -t "${IMAGE}" test/transcoding -# 5) gcloud docker push "${IMAGE}" +# 5) gcloud docker -- push "${IMAGE}" FROM debian:jessie diff --git a/third_party/BUILD.golang_protobuf b/third_party/BUILD.golang_protobuf index e9b0c65f5..24f0b8930 100644 --- a/third_party/BUILD.golang_protobuf +++ b/third_party/BUILD.golang_protobuf @@ -15,6 +15,7 @@ go_library( "proto/lib.go", "proto/message_set.go", "proto/pointer_reflect.go", + "proto/pointer_unsafe.go", "proto/properties.go", "proto/text.go", "proto/text_parser.go",