Firebase merge from master

WORKSPACE

@@ -26,7 +26,7 @@
 #
 # A Bazel (http://bazel.io) workspace for the Google Cloud Endpoints runtime.
 
-ISTIO_PROXY = "cfdb30323aa72b680bab1664035a1bbc200379c0"
+ISTIO_PROXY = "9442a836988332f397f874e0b8ae9edafcd53425"
 
 git_repository(
     name = "nginx",
@@ -126,7 +126,7 @@ bind(
 
 git_repository(
     name = "tools",
-    commit = "1bcac83ed2dc9c5e0be156a4c1801d435667f642",
+    commit = "e5cfdcd2edfe36b46c620fe2236f079edd5fe80b",
     remote = "https://github.com/cloudendpoints/endpoints-tools",
 )
 
@@ -158,7 +158,7 @@ git_repository(
 #
 git_repository(
     name = "io_bazel_rules_go",
-    commit = "76c63b5cd0d47c1f2b47ab4953db96c574af1c1d",
+    commit = "2d9f328a9723baf2d037ba9db28d9d0e30683938", # Apr 6, 2017 (buildifier fix)
     remote = "https://github.com/bazelbuild/rules_go.git",
 )
 

docker/ingress/build.sh

@@ -43,4 +43,4 @@ cp $DEB                                 $ROOT/docker/ingress
 cp $ROOT/bazel-bin/test/src/ingress     $ROOT/docker/ingress
 cp $ROOT/test/src/controller/nginx.tmpl $ROOT/docker/ingress
 docker build -t ${IMAGE} $ROOT/docker/ingress || error_exit "Failed to build"
-gcloud docker push ${IMAGE}                   || error_exit "Failed to upload docker image"
+gcloud docker -- push ${IMAGE}                   || error_exit "Failed to upload docker image"

script/jenkins-build-docker-slave

@@ -43,7 +43,7 @@ TOOLS_BUCKET=''
 function update_image() {
   local image="${1}"
   [[ ${FORCE} == true ]] && return 0
-  ${GCLOUD} docker pull "${image}"
+  ${GCLOUD} docker -- pull "${image}"
   local image_exists=${?}
   [[ ${image_exists} -eq 0 ]] && return 1
   return 0
@@ -75,7 +75,7 @@ if [[ "${BUILD}" == true ]]; then
     run retry -n 10 docker build --build-arg TOOLS_BUCKET="${TOOLS_BUCKET}" --no-cache -t ${IMAGE} \
       -f "${DOCKER_FILE}" . || error_exit "Could not build ${SLAVE}"
     echo "Pushing Docker image: ${IMAGE}"
-    retry -n 3 ${GCLOUD} docker push "${IMAGE}" \
+    retry -n 3 ${GCLOUD} docker -- push "${IMAGE}" \
       || error_exit "Failed to push tag ${IMAGE}."
   fi
 fi
@@ -86,7 +86,7 @@ if [[ -n "${TAG}" ]]; then
   retry -n 3 docker tag "${IMAGE}" ${TAG}
 
   echo "Pushing Docker image: ${TAG}"
-  retry -n 3 ${GCLOUD} docker push "${TAG}" \
+  retry -n 3 ${GCLOUD} docker -- push "${TAG}" \
     || error_exit "Failed to push tag ${TAG}."
 fi
 

script/linux-build-docker

@@ -48,7 +48,7 @@ done
 [[ -f "${DEB}" ]] || error_exit "Cannot find Debian package ${DEB}"
 
 echo "Checking if docker image ${IMAGE} exists.."
-gcloud docker pull "${IMAGE}" \
+gcloud docker -- pull "${IMAGE}" \
   && { echo "Image ${IMAGE} already exists; skipping"; exit 0; }
 
 echo "Building Endpoints Runtime docker image."
@@ -86,14 +86,14 @@ if  [[ -n "${CONFIG}" ]]; then
 
   echo "Pushing Custom Docker image: ${IMAGE}"
   retry -n 10 -s 10 \
-      gcloud docker push "${IMAGE}" \
+      gcloud docker -- push "${IMAGE}" \
     || error_exit "Failed to upload custom Docker image to gcr."
 else
 
   echo "Pushing Docker image: ${IMAGE}"
 
   # Try 10 times, shortest wait is 10 seconds, exponential back-off.
   retry -n 10 -s 10 \
-      gcloud docker push "${IMAGE}" \
+      gcloud docker -- push "${IMAGE}" \
     || error_exit "Failed to upload Docker image to gcr."
 fi

script/linux-gae-instance

@@ -177,7 +177,7 @@ function download_nginx_logs() {
 
   retry -n 2 -s 10 \
     ${GCLOUD} compute ssh --project=${PROJECT} --zone=${vm_zone} ${vm_name} \
-        "sudo tar czf \"${remote_archive}\" /var/log/ ; sudo chmod 777 \"${remote_archive}\"" \
+        --command="sudo tar czf ${remote_archive} /var/log/; sudo chmod 777 ${remote_archive}" \
     || error_exit "Cannot tar backend VM logs."
 
   retry -n 2 -s 10 \

script/release-stable

@@ -51,11 +51,13 @@ END_USAGE
 
 GCLOUD=$(which gcloud) || GCLOUD=~/google-cloud-sdk/bin/gcloud
 DEBIAN_PACKAGE_REPOSITORY=''
+VERSION="$(command cat ${ROOT}/src/nginx/version)"
 
-while getopts :g:r: arg; do
+while getopts :g:r:v: arg; do
   case ${arg} in
     g) GCLOUD="${OPTARG}";;
     r) DEBIAN_PACKAGE_REPOSITORY="${OPTARG}";;
+    v) VERSION="${OPTARG}";;
     *) usage "Invalid option: -${OPTARG}";;
   esac
 done
@@ -70,7 +72,6 @@ which rapture > /dev/null 2>&1 \
 
 set -x
 
-VERSION="$(command cat ${ROOT}/src/nginx/version)"
 # Minor base is 1.0  if version is 1.0.0
 MINOR_BASE_VERSION=${VERSION%.*}
 # Major base is 1  if version is 1.0.0
@@ -82,7 +83,7 @@ function tag_stable_image() {
 
   docker_tag_f "${image}" "${stable}"
 
-  retry "${GCLOUD}" docker push "${stable}" \
+  retry "${GCLOUD}" docker -- push "${stable}" \
     || error_exit "ERROR: failed to push '${stable}'"
 }
 

src/grpc/BUILD

@@ -58,6 +58,7 @@ cc_library(
     ],
     visibility = ["//visibility:public"],
     deps = [
+        "@istio_proxy_git//contrib/endpoints/src/grpc/transcoding:transcoder_input_stream",
         "//external:grpc",
         "//external:grpc++",
         "//external:protobuf",

src/grpc/zero_copy_stream.cc

@@ -68,7 +68,7 @@ void GrpcZeroCopyInputStream::BackUp(int count) {
   }
 }
 
-::google::protobuf::int64 GrpcZeroCopyInputStream::ByteCount() const {
+int64_t GrpcZeroCopyInputStream::BytesAvailable() const {
   return (current_buffer_size_ - position_) + serializer_.ByteCount();
 }
 

src/grpc/zero_copy_stream.h

@@ -28,6 +28,7 @@
 
 #include <deque>
 
+#include "contrib/endpoints/src/grpc/transcoding/transcoder_input_stream.h"
 #include "google/protobuf/io/zero_copy_stream.h"
 #include "grpc++/support/byte_buffer.h"
 #include "src/grpc/message_serializer.h"
@@ -38,7 +39,7 @@ namespace grpc {
 
 // ZeroCopyInputStream implementation over a stream of gRPC messages.
 class GrpcZeroCopyInputStream
-    : public ::google::protobuf::io::ZeroCopyInputStream {
+    : public ::google::api_manager::transcoding::TranscoderInputStream {
  public:
   GrpcZeroCopyInputStream();
 
@@ -53,8 +54,9 @@ class GrpcZeroCopyInputStream
 
   bool Next(const void** data, int* size);
   void BackUp(int count);
-  bool Skip(int count) { return false; }  // not supported
-  ::google::protobuf::int64 ByteCount() const;
+  bool Skip(int count) { return false; }                     // not supported
+  ::google::protobuf::int64 ByteCount() const { return 0; }  // Not implemented
+  int64_t BytesAvailable() const;
 
  private:
   GrpcMessageSerializer serializer_;

src/grpc/zero_copy_stream_test.cc

@@ -86,63 +86,63 @@ TEST_F(GrpcZeroCopyInputStreamTest, SimpleRead) {
   stream.AddMessage(CreateByteBuffer(SliceData{slice21, slice22}), true);
   stream.Finish();
 
-  // Test ByteCount()
+  // Test BytesAvailable()
   EXPECT_EQ(slice11.size() + slice12.size() + slice21.size() + slice22.size() +
                 10,  // +10 bytes for two delimiters
-            stream.ByteCount());
+            stream.BytesAvailable());
 
   // Test the message1 delimiter
   ASSERT_TRUE(stream.Next(&data, &size));
   ASSERT_EQ(5, size);
   EXPECT_EQ(slice11.size() + slice12.size(),
             DelimiterToSize(reinterpret_cast<const unsigned char *>(data)));
 
-  // Test ByteCount()
+  // Test BytesAvailable()
   EXPECT_EQ(slice11.size() + slice12.size() + slice21.size() + slice22.size() +
                 5,  // +5 bytes for one delimiter
-            stream.ByteCount());
+            stream.BytesAvailable());
 
   // Test the slices
   ASSERT_TRUE(stream.Next(&data, &size));
   ASSERT_EQ(slice11.size(), size);
   EXPECT_EQ(slice11, std::string(reinterpret_cast<const char *>(data), size));
 
-  // Test ByteCount()
+  // Test BytesAvailable()
   EXPECT_EQ(slice12.size() + slice21.size() + slice22.size() +
                 5,  // +5 bytes for one delimiter
-            stream.ByteCount());
+            stream.BytesAvailable());
 
   ASSERT_TRUE(stream.Next(&data, &size));
   ASSERT_EQ(slice12.size(), size);
   EXPECT_EQ(slice12, std::string(reinterpret_cast<const char *>(data), size));
 
-  // Test ByteCount()
+  // Test BytesAvailable()
   EXPECT_EQ(slice21.size() + slice22.size() + 5,  // +5 bytes for one delimiter
-            stream.ByteCount());
+            stream.BytesAvailable());
 
   // Test the message2 delimiter
   ASSERT_TRUE(stream.Next(&data, &size));
   ASSERT_EQ(5, size);
   EXPECT_EQ(slice21.size() + slice22.size(),
             DelimiterToSize(reinterpret_cast<const unsigned char *>(data)));
 
-  // Test ByteCount()
-  EXPECT_EQ(slice21.size() + slice22.size(), stream.ByteCount());
+  // Test BytesAvailable()
+  EXPECT_EQ(slice21.size() + slice22.size(), stream.BytesAvailable());
 
   // Test the slices
   ASSERT_TRUE(stream.Next(&data, &size));
   ASSERT_EQ(slice21.size(), size);
   EXPECT_EQ(slice21, std::string(reinterpret_cast<const char *>(data), size));
 
-  // Test ByteCount()
-  EXPECT_EQ(slice22.size(), stream.ByteCount());
+  // Test BytesAvailable()
+  EXPECT_EQ(slice22.size(), stream.BytesAvailable());
 
   ASSERT_TRUE(stream.Next(&data, &size));
   ASSERT_EQ(slice22.size(), size);
   EXPECT_EQ(slice22, std::string(reinterpret_cast<const char *>(data), size));
 
   // Test the end of the stream
-  EXPECT_EQ(0, stream.ByteCount());
+  EXPECT_EQ(0, stream.BytesAvailable());
   EXPECT_FALSE(stream.Next(&data, &size));
 }
 
@@ -167,9 +167,9 @@ TEST_F(GrpcZeroCopyInputStreamTest, Backups) {
   // Back up
   stream.BackUp(5);
 
-  // Test the ByteCount()
+  // Test the BytesAvailable()
   EXPECT_EQ(slice1.size() + slice2.size() + 5,  // +5 bytes for the delimiter
-            stream.ByteCount());
+            stream.BytesAvailable());
 
   // Test the slice again
   ASSERT_TRUE(stream.Next(&data, &size));
@@ -184,14 +184,14 @@ TEST_F(GrpcZeroCopyInputStreamTest, Backups) {
 
   // Back up & test again
   stream.BackUp(size);
-  EXPECT_EQ(slice1.size() + slice2.size(), stream.ByteCount());
+  EXPECT_EQ(slice1.size() + slice2.size(), stream.BytesAvailable());
   ASSERT_TRUE(stream.Next(&data, &size));
   ASSERT_EQ(slice1.size(), size);
   EXPECT_EQ(slice1, std::string(reinterpret_cast<const char *>(data), size));
 
   // Now Back up 10 bytes & test again
   stream.BackUp(10);
-  EXPECT_EQ(10 + slice2.size(), stream.ByteCount());
+  EXPECT_EQ(10 + slice2.size(), stream.BytesAvailable());
   ASSERT_TRUE(stream.Next(&data, &size));
   ASSERT_EQ(10, size);
   EXPECT_EQ(slice1.substr(slice1.size() - 10),
@@ -204,14 +204,14 @@ TEST_F(GrpcZeroCopyInputStreamTest, Backups) {
 
   // Back up and test again
   stream.BackUp(size);
-  EXPECT_EQ(slice2.size(), stream.ByteCount());
+  EXPECT_EQ(slice2.size(), stream.BytesAvailable());
   ASSERT_TRUE(stream.Next(&data, &size));
   ASSERT_EQ(slice2.size(), size);
   EXPECT_EQ(slice2, std::string(reinterpret_cast<const char *>(data), size));
 
   // Now Back up size - 1 bytes (all but 1) and check again
   stream.BackUp(size - 1);
-  EXPECT_EQ(slice2.size() - 1, stream.ByteCount());
+  EXPECT_EQ(slice2.size() - 1, stream.BytesAvailable());
   ASSERT_TRUE(stream.Next(&data, &size));
   ASSERT_EQ(slice2.size() - 1, size);
   EXPECT_EQ(slice2.substr(1),

src/nginx/grpc.cc

@@ -93,8 +93,14 @@ std::pair<Status, std::shared_ptr<::grpc::GenericStub>> GrpcGetStub(
     return std::make_pair(Status::OK, it->second);
   }
 
-  auto result = std::make_shared<::grpc::GenericStub>(
-      ::grpc::CreateChannel(address, ::grpc::InsecureChannelCredentials()));
+  ::grpc::ChannelArguments channel_arguments;
+
+  channel_arguments.SetMaxReceiveMessageSize(INT_MAX);
+  channel_arguments.SetMaxSendMessageSize(INT_MAX);
+
+  auto result =
+      std::make_shared<::grpc::GenericStub>(::grpc::CreateCustomChannel(
+          address, ::grpc::InsecureChannelCredentials(), channel_arguments));
 
   if (result) {
     espcf->grpc_stubs.emplace(address, result);

src/nginx/t/BUILD

@@ -110,6 +110,7 @@ nginx_suite(
         "grpc_grpc_fallback.t",
         "grpc_grpc_override.t",
         "grpc_large_request.t",
+        "grpc_large_response.t",
         "grpc_large_streaming.t",
         "grpc_long_streaming.t",
         "grpc_metadata.t",
@@ -171,6 +172,7 @@ nginx_suite(
         "check_custom_api_key.t",
         "check_default.t",
         "check_invalid_api_key.t",
+        "check_key_restriction.t",
         "check_no_consumer.t",
         "check_referer.t",
         "check_report_body.t",
@@ -188,6 +190,9 @@ nginx_suite(
         "multiple_apis.t",
         "no_backend.t",
         "no_service_control.t",
+        "quota.t",
+        "quota_api_not_available.t",
+        "quota_exhausted.t",
         "reject_unrecognized.t",
         "report_3xx.t",
         "report_4xx.t",