From 78b931b935d69422a0c6a0c9424ac50c054b0677 Mon Sep 17 00:00:00 2001 From: enricoschiattarella Date: Tue, 10 Jan 2017 13:26:08 -0800 Subject: [PATCH 01/83] Simple TCP server to show how to retrieve original dest IP:port after an iptables redirect (#38) * Simple TCP server to show how to retrieve original dest IP:port after an iptables redirect * Fixed style. --- contrib/tools/server/Makefile | 6 ++ contrib/tools/server/README.md | 25 +++++ contrib/tools/server/server.c | 169 +++++++++++++++++++++++++++++++++ 3 files changed, 200 insertions(+) create mode 100644 contrib/tools/server/Makefile create mode 100644 contrib/tools/server/README.md create mode 100644 contrib/tools/server/server.c diff --git a/contrib/tools/server/Makefile b/contrib/tools/server/Makefile new file mode 100644 index 00000000000..d7c7d758686 --- /dev/null +++ b/contrib/tools/server/Makefile @@ -0,0 +1,6 @@ +server: + $(CC) -g -o server server.c + +clean: + rm server + diff --git a/contrib/tools/server/README.md b/contrib/tools/server/README.md new file mode 100644 index 00000000000..f962c2c640b --- /dev/null +++ b/contrib/tools/server/README.md @@ -0,0 +1,25 @@ +# Sample TCP Server + +This is a simple TCP server that listens on the specified port (3490 by default) and replies to incoming connections by providing info about src/destination ip/port. + +It demonstrates the use of the getsockopt() system call with the ORIGINAL_DST option to retrieve the original destination ip/port after an iptables redirect. + +So, for example, if you have the server listening on port 3490 on the local machine and an iptables rule like: + +``` +iptables -t nat -I OUTPUT 1 -p tcp --dport 4000:5000 -j REDIRECT --to-port 3490 +``` +your will see: + +``` +$ telnet localhost 3490 +FROM 127.0.0.1:44978, TO 127.0.0.1:3490, ORIG DEST 127.0.0.1:3490 + +$ telnet localhost 4100 +FROM 127.0.0.1:35476, TO 127.0.0.1:3490, ORIG DEST 127.0.0.1:4100 + +$ telnet 1.1.1.1 5000 +FROM 100.100.100.100:60275, TO 127.0.0.1:3490, ORIG DEST 1.1.1.1:5000 +``` + + diff --git a/contrib/tools/server/server.c b/contrib/tools/server/server.c new file mode 100644 index 00000000000..77a2d97e004 --- /dev/null +++ b/contrib/tools/server/server.c @@ -0,0 +1,169 @@ +/* +** server.c + * Demo server to check that we can extract all parameters from an incoming +*request + * and respond to it +*/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define DEFAULT_PORT \ + "3490" // the port users will be connecting to, if not specified on command + // line + +#define QUEUE_SIZE 10 // pending connections queue size + +void sigchld_handler(int s) { + int saved_errno = errno; + + while (waitpid(-1, NULL, WNOHANG) > 0) + ; + + errno = saved_errno; +} + +int main(int argc, char *argv[]) { + struct sigaction sa; + const int yes = 1; + char *port = DEFAULT_PORT; + int rv; + + if (argc > 1) { + port = argv[1]; + } + + struct addrinfo hints; + memset(&hints, 0, sizeof hints); + hints.ai_family = AF_INET; + hints.ai_socktype = SOCK_STREAM; + hints.ai_flags = AI_PASSIVE; // use my IP + + struct addrinfo *server_info = NULL; + if ((rv = getaddrinfo(NULL, port, &hints, &server_info)) != 0) { + fprintf(stderr, "getaddrinfo: %s\n", gai_strerror(rv)); + return 1; + } + + struct addrinfo *p; + int sockfd; // listen on sock_fd + for (p = server_info; p != NULL; p = p->ai_next) { + if ((sockfd = socket(p->ai_family, p->ai_socktype, p->ai_protocol)) == -1) { + perror("server: socket"); + continue; + } + + if (setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(int)) == -1) { + perror("setsockopt"); + exit(1); + } + + if (bind(sockfd, p->ai_addr, p->ai_addrlen) == -1) { + close(sockfd); + perror("server: bind"); + continue; + } + + break; + } + + freeaddrinfo(server_info); + + if (p == NULL) { + fprintf(stderr, "server: failed to bind\n"); + exit(1); + } + if (listen(sockfd, QUEUE_SIZE) == -1) { + perror("listen"); + exit(1); + } + + sa.sa_handler = sigchld_handler; + sigemptyset(&sa.sa_mask); + sa.sa_flags = SA_RESTART; + if (sigaction(SIGCHLD, &sa, NULL) == -1) { + perror("sigaction"); + exit(1); + } + + char bind_addr_str[INET6_ADDRSTRLEN] = {0}; + struct sockaddr_in *bind_sock_addr = (struct sockaddr_in *)p->ai_addr; + inet_ntop(p->ai_family, &bind_sock_addr->sin_addr, bind_addr_str, + p->ai_addrlen); + printf("server %s: waiting for connections on port %s:%u...\n", port, + bind_addr_str, ntohs(bind_sock_addr->sin_port)); + + while (1) { + socklen_t addr_len = sizeof(struct sockaddr_storage); + int addr_str_len = INET6_ADDRSTRLEN; + + struct sockaddr_storage their_addr = { + 0}; // connector's address information + int accepted_fd = accept(sockfd, (struct sockaddr *)&their_addr, &addr_len); + if (accepted_fd == -1) { + perror("accept"); + continue; + } + + struct sockaddr_in *src_sock_addr = (struct sockaddr_in *)&their_addr; + char their_addr_str[INET6_ADDRSTRLEN] = {0}; + inet_ntop(their_addr.ss_family, &src_sock_addr->sin_addr, their_addr_str, + addr_str_len); + printf("server %s: got connection FROM %s:%u\n", port, their_addr_str, + ntohs(src_sock_addr->sin_port)); + + struct sockaddr_storage my_addr = {0}; // my address information + struct sockaddr_in *dst_sock_addr = (struct sockaddr_in *)&my_addr; + char my_addr_str[INET6_ADDRSTRLEN] = {0}; + getsockname(accepted_fd, (struct sockaddr *)dst_sock_addr, &addr_len); + inet_ntop(my_addr.ss_family, &dst_sock_addr->sin_addr, my_addr_str, + addr_str_len); + printf("server %s: got connection TO %s:%u\n", port, my_addr_str, + ntohs(dst_sock_addr->sin_port)); + + struct sockaddr_storage orig_addr = {0}; // orig address information + struct sockaddr_in *orig_sock_addr = (struct sockaddr_in *)&orig_addr; + char orig_addr_str[INET6_ADDRSTRLEN] = {0}; + int status = getsockopt(accepted_fd, SOL_IP, SO_ORIGINAL_DST, + orig_sock_addr, &addr_len); + + if (status == 0) { + inet_ntop(orig_addr.ss_family, &orig_sock_addr->sin_addr, orig_addr_str, + addr_str_len); + printf("server %s: ORIG DEST %s:%u\n", port, orig_addr_str, + ntohs(orig_sock_addr->sin_port)); + } else { + printf("Could not get orig destination from accepted socket.\n"); + } + + if (!fork()) { // this is the child process + + close(sockfd); + char msg[256] = {0}; + snprintf(msg, 256, "FROM %s:%u, TO %s:%u, ORIG DEST %s:%u\n", + their_addr_str, ntohs(src_sock_addr->sin_port), my_addr_str, + ntohs(dst_sock_addr->sin_port), orig_addr_str, + ntohs(orig_sock_addr->sin_port)); + + if (send(accepted_fd, msg, strlen(msg), 0) == -1) { + perror("send"); + } + close(accepted_fd); + exit(0); + } + close(accepted_fd); + } + + return 0; +} From 8b3a4429965ed1e792704e795e5d22782eca505b Mon Sep 17 00:00:00 2001 From: Kuat Date: Thu, 12 Jan 2017 11:23:47 -0800 Subject: [PATCH 02/83] Rebase Envoy (#41) --- src/envoy/repositories.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/envoy/repositories.bzl b/src/envoy/repositories.bzl index 56fabce07ac..4460b4c1c9d 100644 --- a/src/envoy/repositories.bzl +++ b/src/envoy/repositories.bzl @@ -629,6 +629,6 @@ cc_test( native.new_git_repository( name = "envoy_git", remote = "https://github.com/lyft/envoy.git", - commit = "6b1336a786ebe56c45a1a349ddf706e0526c1ec1", # 2017-01-03 + commit = "38990918a8672a0332b1ca2e048db90a5166d678", build_file_content = BUILD, ) From c3abd2aeb2b43e2c68b71b48449a5fa0de0f2fb8 Mon Sep 17 00:00:00 2001 From: Kuat Date: Thu, 12 Jan 2017 14:52:56 -0800 Subject: [PATCH 03/83] Update prototype to use iptables (#42) --- src/envoy/prototype/README.md | 13 +++++++++++++ src/envoy/prototype/envoy-esp.conf | 11 +++++++++-- test/backend/echo/echo.go | 21 +++++++-------------- 3 files changed, 29 insertions(+), 16 deletions(-) diff --git a/src/envoy/prototype/README.md b/src/envoy/prototype/README.md index 2ddbe98459c..00e73a169d6 100644 --- a/src/envoy/prototype/README.md +++ b/src/envoy/prototype/README.md @@ -36,6 +36,19 @@ This Proxy will use Envoy and talk to Mixer server. go run echo.go ``` +* Modify your iptables: + +``` + sudo iptables -t nat -A OUTPUT -p tcp --dport 9090 -j REDIRECT --to-port 9092 +``` + +Once you are done, you should remove this rule: + +``` + sudo iptables -t nat -D OUTPUT -p tcp --dport 9090 -j REDIRECT --to-port 9092 +``` + + * Start Envoy proxy, run ``` diff --git a/src/envoy/prototype/envoy-esp.conf b/src/envoy/prototype/envoy-esp.conf index dff3bd67b1a..11690bed565 100644 --- a/src/envoy/prototype/envoy-esp.conf +++ b/src/envoy/prototype/envoy-esp.conf @@ -1,7 +1,14 @@ { "listeners": [ + { + "port": 9092, + "bind_to_port": true, + "use_original_dst": true, + "filters": [] + }, { "port": 9090, + "bind_to_port": false, "filters": [ { "type": "read", @@ -26,7 +33,7 @@ }, "access_log": [ { - "path": "/tmp/access.envoy" + "path": "/dev/stdout" } ], "filters": [ @@ -50,7 +57,7 @@ } ], "admin": { - "access_log_path": "/tmp/access.envoy", + "access_log_path": "/dev/stdout", "port": 9001 }, "cluster_manager": { diff --git a/test/backend/echo/echo.go b/test/backend/echo/echo.go index 51aaa07e353..ef13f515ff1 100644 --- a/test/backend/echo/echo.go +++ b/test/backend/echo/echo.go @@ -23,19 +23,22 @@ import ( "io/ioutil" "net/http" "strconv" - "sync" - "time" ) var ( port = flag.Int("port", 8080, "default http port") - mu sync.Mutex requests = 0 data = 0 ) func handler(w http.ResponseWriter, r *http.Request) { + fmt.Printf("%v %v %v %v\n", r.Method, r.URL, r.Proto, r.RemoteAddr) + for name, headers := range r.Header { + for _, h := range headers { + fmt.Printf("%v: %v\n", name, h) + } + } body, err := ioutil.ReadAll(r.Body) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) @@ -51,24 +54,14 @@ func handler(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) w.Write(body) - mu.Lock() requests++ data += len(body) - defer mu.Unlock() + fmt.Printf("Requests Requests: %v Data: %v\n", requests, data) } func main() { flag.Parse() - go func() { - for { - mu.Lock() - fmt.Printf("Requests Requests: %v Data: %v\n", requests, data) - mu.Unlock() - time.Sleep(time.Second) - } - }() - fmt.Printf("Listening on port %v\n", *port) http.HandleFunc("/", handler) From add363a037389ebf985defa8898465b3269e380f Mon Sep 17 00:00:00 2001 From: Kuat Date: Thu, 12 Jan 2017 20:18:53 -0800 Subject: [PATCH 04/83] Rebase to fixed Envoy (#43) --- src/envoy/repositories.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/envoy/repositories.bzl b/src/envoy/repositories.bzl index 4460b4c1c9d..dd36a23d456 100644 --- a/src/envoy/repositories.bzl +++ b/src/envoy/repositories.bzl @@ -629,6 +629,6 @@ cc_test( native.new_git_repository( name = "envoy_git", remote = "https://github.com/lyft/envoy.git", - commit = "38990918a8672a0332b1ca2e048db90a5166d678", + commit = "c36bb76e304ab4c5584a07303e2953a170888319", build_file_content = BUILD, ) From ee3cdeeb33b1de0b87c80809802de5246cbb3e25 Mon Sep 17 00:00:00 2001 From: Qian Sun Date: Fri, 13 Jan 2017 08:49:25 -0800 Subject: [PATCH 05/83] Handle HEAD request. (#34) * Handle HEAD request. * Try with GET if HEAD fails. * Address comments. * Format file. --- .../src/api_manager/context/service_context.cc | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/contrib/endpoints/src/api_manager/context/service_context.cc b/contrib/endpoints/src/api_manager/context/service_context.cc index b9f753f0625..d8fc9dfc409 100644 --- a/contrib/endpoints/src/api_manager/context/service_context.cc +++ b/contrib/endpoints/src/api_manager/context/service_context.cc @@ -41,6 +41,9 @@ const double kDefaultTraceSampleQps = 0.1; // The time window to send intermediate report for Grpc streaming (second). // Default to 10s. const int kIntermediateReportInterval = 10; + +const char kHTTPHeadMethod[] = "HEAD"; +const char kHTTPGetMethod[] = "GET"; } ServiceContext::ServiceContext(std::unique_ptr env, @@ -74,7 +77,15 @@ MethodCallInfo ServiceContext::GetMethodCallInfo( if (config_ == nullptr) { return MethodCallInfo(); } - return config_->GetMethodCallInfo(http_method, url, query_params); + MethodCallInfo method_call_info = + config_->GetMethodCallInfo(http_method, url, query_params); + // HEAD should be treated as GET unless it is specified from service_config. + if (method_call_info.method_info == nullptr && + http_method == kHTTPHeadMethod) { + method_call_info = + config_->GetMethodCallInfo(kHTTPGetMethod, url, query_params); + } + return method_call_info; } const std::string& ServiceContext::project_id() const { From efc32a2a83434e7169f00c56c08c4b7361febf3b Mon Sep 17 00:00:00 2001 From: Kuat Date: Fri, 20 Jan 2017 16:39:05 -0800 Subject: [PATCH 06/83] Expose bazel target (#48) --- src/envoy/prototype/BUILD | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/envoy/prototype/BUILD b/src/envoy/prototype/BUILD index e5cbaeba10c..5c8ab20e27a 100644 --- a/src/envoy/prototype/BUILD +++ b/src/envoy/prototype/BUILD @@ -15,6 +15,8 @@ ################################################################################ # +package(default_visibility = [":__subpackages__"]) + cc_binary( name = "envoy_esp", srcs = [ From 92541b79f45fb3ada59e36d3abd87bd7724b27f9 Mon Sep 17 00:00:00 2001 From: Kuat Date: Fri, 20 Jan 2017 16:51:15 -0800 Subject: [PATCH 07/83] Try again (#49) --- src/envoy/prototype/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/envoy/prototype/BUILD b/src/envoy/prototype/BUILD index 5c8ab20e27a..258f2046cd7 100644 --- a/src/envoy/prototype/BUILD +++ b/src/envoy/prototype/BUILD @@ -15,7 +15,7 @@ ################################################################################ # -package(default_visibility = [":__subpackages__"]) +package(default_visibility = ["//visibility:public"]) cc_binary( name = "envoy_esp", From cea882799327dbea20ce849be36088a349a35f18 Mon Sep 17 00:00:00 2001 From: Wayne Zhang Date: Thu, 26 Jan 2017 22:02:18 -0800 Subject: [PATCH 08/83] Integrate with mixer client. (#55) * Integrate with mixer client. * Restore repositories.bzl back. --- WORKSPACE | 22 +- contrib/endpoints/repositories.bzl | 52 +--- contrib/endpoints/src/api_manager/mixer/BUILD | 6 +- .../endpoints/src/api_manager/mixer/mixer.cc | 255 ++++++------------ .../endpoints/src/api_manager/mixer/mixer.h | 6 +- src/envoy/prototype/api_manager_filter.cc | 9 +- src/envoy/prototype/envoy-esp.conf | 12 - src/envoy/prototype/server_config.pb.txt | 4 +- 8 files changed, 129 insertions(+), 237 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index bbafd205918..95dcdde5cdc 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -31,16 +31,34 @@ googletest_repositories() load( "//contrib/endpoints:repositories.bzl", "grpc_repositories", - "mixerapi_repositories", + "mixer_client_repositories", "servicecontrol_client_repositories", ) grpc_repositories() -mixerapi_repositories() +mixer_client_repositories() servicecontrol_client_repositories() +# Workaround for Bazel > 0.4.0 since it needs newer protobuf.bzl from: +# https://github.com/google/protobuf/pull/2246 +# Do not use this git_repository for anything else than protobuf.bzl +new_git_repository( + name = "protobuf_bzl", + # Injecting an empty BUILD file to prevent using any build target + build_file_content = "", + commit = "05090726144b6e632c50f47720ff51049bfcbef6", + remote = "https://github.com/google/protobuf.git", +) + +load( + "@mixerclient_git//:repositories.bzl", + "mixerapi_repositories", +) + +mixerapi_repositories(protobuf_repo="@protobuf_bzl//") + load( "//src/envoy:repositories.bzl", "envoy_repositories", diff --git a/contrib/endpoints/repositories.bzl b/contrib/endpoints/repositories.bzl index f3df61722a2..7b77aa91c5a 100644 --- a/contrib/endpoints/repositories.bzl +++ b/contrib/endpoints/repositories.bzl @@ -334,51 +334,15 @@ def servicecontrol_client_repositories(bind=True): actual = "@servicecontrol_client_git//:service_control_client_lib", ) -def mixerapi_repositories(protobuf_repo="@protobuf_git//", bind=True): - BUILD = """ -# Copyright 2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -################################################################################ -# -licenses(["notice"]) - -load("{}:protobuf.bzl", "cc_proto_library") - -cc_proto_library( - name = "mixer_api_cc_proto", - srcs = glob( - ["mixer/api/v1/*.proto"], - ), - default_runtime = "//external:protobuf", - protoc = "//external:protoc", - visibility = ["//visibility:public"], - deps = [ - "//external:cc_wkt_protos", - "//external:servicecontrol", - ], -) -""".format(protobuf_repo) - - native.new_git_repository( - name = "mixerapi_git", - commit = "fc5a396185edc72d06d1937f30a8148a37d4fc1b", - remote = "https://github.com/istio/api.git", - build_file_content = BUILD, +def mixer_client_repositories(bind=True): + native.git_repository( + name = "mixerclient_git", + commit = "1569430f1e27b31e23c029c6bec0d8d5062d9e55", + remote = "https://github.com/istio/mixerclient.git", ) + if bind: native.bind( - name = "mixer_api_cc_proto", - actual = "@mixerapi_git//:mixer_api_cc_proto", + name = "mixer_client_lib", + actual = "@mixerclient_git//:mixer_client_lib", ) diff --git a/contrib/endpoints/src/api_manager/mixer/BUILD b/contrib/endpoints/src/api_manager/mixer/BUILD index 8082c61e201..affe09c7ddd 100644 --- a/contrib/endpoints/src/api_manager/mixer/BUILD +++ b/contrib/endpoints/src/api_manager/mixer/BUILD @@ -32,11 +32,11 @@ cc_library( ], }), deps = [ - "//external:grpc++", - "//external:mixer_api_cc_proto", - "//external:protobuf", "//contrib/endpoints/src/api_manager:impl_headers", "//contrib/endpoints/src/api_manager/service_control", "//contrib/endpoints/src/api_manager/utils", + "//external:grpc++", + "//external:mixer_client_lib", + "//external:protobuf", ], ) diff --git a/contrib/endpoints/src/api_manager/mixer/mixer.cc b/contrib/endpoints/src/api_manager/mixer/mixer.cc index 0c0bb5d793e..47f6c4a8a2f 100644 --- a/contrib/endpoints/src/api_manager/mixer/mixer.cc +++ b/contrib/endpoints/src/api_manager/mixer/mixer.cc @@ -14,169 +14,103 @@ */ #include "contrib/endpoints/src/api_manager/mixer/mixer.h" -#include -#include "mixer/api/v1/service.pb.h" - using ::google::api_manager::utils::Status; -using ::google::protobuf::util::error::Code; -using ::google::protobuf::Map; +using ::istio::mixer_client::Attributes; namespace google { namespace api_manager { namespace mixer { namespace { -const char kMixerServiceName[] = "istio.mixer.v1.Mixer"; - -enum AttributeIndex { - ATTR_SERVICE_NAME = 0, - ATTR_PEER_ID, - ATTR_OPERATION_NAME, - ATTR_API_KEY, - ATTR_RESPONSE_CODE, - ATTR_URL, - ATTR_LOCATION, - ATTR_API_NAME, - ATTR_API_VERSION, - ATTR_API_METHOD, - ATTR_REQUEST_SIZE, - ATTR_RESPONSE_SIZE, - ATTR_LOG_MESSAGE, -}; - -struct AttributeDict { - int index; - std::string name; -} kAttributeNames[] = { - { - ATTR_SERVICE_NAME, "serviceName", - }, - { - ATTR_PEER_ID, "peerId", - }, - { - ATTR_OPERATION_NAME, "operationName", - }, - { - ATTR_API_KEY, "apiKey", - }, - { - ATTR_RESPONSE_CODE, "responseCode", - }, - { - ATTR_URL, "URL", - }, - { - ATTR_LOCATION, "location", - }, - { - ATTR_API_NAME, "apiName", - }, - { - ATTR_API_VERSION, "apiVersion", - }, - { - ATTR_API_METHOD, "apiMethod", - }, - { - ATTR_REQUEST_SIZE, "requestSize", - }, - { - ATTR_RESPONSE_SIZE, "responesSize", - }, - { - ATTR_LOG_MESSAGE, "logMessage", - }, -}; - -void SetAttributeDict(Map* dict) { - for (auto attr : kAttributeNames) { - (*dict)[attr.index] = attr.name; - } +const std::string kAttrNameServiceName = "serviceName"; +const std::string kAttrNamePeerId = "peerId"; +const std::string kAttrNameOperationName = "operationName"; +const std::string kAttrNameApiKey = "apiKey"; +const std::string kAttrNameResponseCode = "responseCode"; +const std::string kAttrNameURL = "url"; +const std::string kAttrNameLocation = "location"; +const std::string kAttrNameApiName = "apiName"; +const std::string kAttrNameApiVersion = "apiVersion"; +const std::string kAttrNameApiMethod = "apiMethod"; +const std::string kAttrNameRequestSize = "requestSize"; +const std::string kAttrNameResponseSize = "responseSize"; +const std::string kAttrNameLogMessage = "logMessage"; + +Attributes::Value StringValue(const std::string& str) { + Attributes::Value v; + v.type = Attributes::Value::STRING; + v.str_v = str; + return v; } -void CovertToPb(const service_control::CheckRequestInfo& info, - const std::string& service_name, - ::istio::mixer::v1::Attributes* attr) { - SetAttributeDict(attr->mutable_dictionary()); - - auto* str_attrs = attr->mutable_string_attributes(); - (*str_attrs)[ATTR_SERVICE_NAME] = service_name; - (*str_attrs)[ATTR_PEER_ID] = "Proxy"; - (*str_attrs)[ATTR_OPERATION_NAME] = info.operation_name; - (*str_attrs)[ATTR_API_KEY] = info.api_key; +Attributes::Value Int64Value(int64_t value) { + Attributes::Value v; + v.type = Attributes::Value::INT64; + v.value.int64_v = value; + return v; } -void CovertToPb(const service_control::ReportRequestInfo& info, - const std::string& service_name, - ::istio::mixer::v1::Attributes* attr) { - SetAttributeDict(attr->mutable_dictionary()); +void FillCheckAttributes(const service_control::CheckRequestInfo& info, + const std::string& service_name, + ::istio::mixer_client::Attributes* attr) { + attr->attributes[kAttrNameServiceName] = StringValue(service_name); + attr->attributes[kAttrNamePeerId] = StringValue("Proxy"); + attr->attributes[kAttrNameOperationName] = StringValue(info.operation_name); + attr->attributes[kAttrNameApiKey] = StringValue(info.api_key); +} - auto* str_attrs = attr->mutable_string_attributes(); - (*str_attrs)[ATTR_SERVICE_NAME] = service_name; - (*str_attrs)[ATTR_PEER_ID] = "Proxy"; - (*str_attrs)[ATTR_OPERATION_NAME] = info.operation_name; - (*str_attrs)[ATTR_API_KEY] = info.api_key; +void FillReportAttributes(const service_control::ReportRequestInfo& info, + const std::string& service_name, + ::istio::mixer_client::Attributes* attr) { + attr->attributes[kAttrNameServiceName] = StringValue(service_name); + attr->attributes[kAttrNamePeerId] = StringValue("Proxy"); + attr->attributes[kAttrNameOperationName] = StringValue(info.operation_name); + attr->attributes[kAttrNameApiKey] = StringValue(info.api_key); - (*str_attrs)[ATTR_URL] = info.url; - (*str_attrs)[ATTR_LOCATION] = info.location; + attr->attributes[kAttrNameURL] = StringValue(info.url); + attr->attributes[kAttrNameLocation] = StringValue(info.location); - (*str_attrs)[ATTR_API_NAME] = info.api_name; - (*str_attrs)[ATTR_API_VERSION] = info.api_version; - (*str_attrs)[ATTR_API_METHOD] = info.api_method; + attr->attributes[kAttrNameApiName] = StringValue(info.api_name); + attr->attributes[kAttrNameApiVersion] = StringValue(info.api_version); + attr->attributes[kAttrNameApiMethod] = StringValue(info.api_method); - (*str_attrs)[ATTR_LOG_MESSAGE] = info.log_message; + attr->attributes[kAttrNameLogMessage] = StringValue(info.log_message); - auto* int_attrs = attr->mutable_int64_attributes(); - (*int_attrs)[ATTR_RESPONSE_CODE] = info.response_code; - (*int_attrs)[ATTR_REQUEST_SIZE] = info.request_size; - (*int_attrs)[ATTR_RESPONSE_SIZE] = info.response_size; + attr->attributes[kAttrNameResponseCode] = Int64Value(info.response_code); + attr->attributes[kAttrNameRequestSize] = Int64Value(info.request_size); + attr->attributes[kAttrNameResponseSize] = Int64Value(info.response_size); } } // namespace Mixer::Mixer(ApiManagerEnvInterface* env, const Config* config) - : env_(env), request_index_(0), config_(config) {} + : env_(env), config_(config) {} Mixer::~Mixer() {} -Status Mixer::Init() { return Status::OK; } +Status Mixer::Init() { + ::istio::mixer_client::MixerClientOptions options; + options.mixer_server = + config_->server_config()->mixer_options().mixer_server(); + mixer_client_ = ::istio::mixer_client::CreateMixerClient(options); + return Status::OK; +} Status Mixer::Close() { return Status::OK; } Status Mixer::Report(const service_control::ReportRequestInfo& info) { - std::unique_ptr grpc_request(new GRPCRequest([this]( - Status status, std::string&& body) { - if (status.ok()) { - // Handle 200 response - ::istio::mixer::v1::ReportResponse response; - if (!response.ParseFromString(body)) { - status = - Status(Code::INVALID_ARGUMENT, std::string("Invalid response")); - env_->LogError(std::string("Failed parse report response: ") + body); - } - env_->LogInfo(std::string("Report response: ") + response.DebugString()); - } else { - env_->LogError(std::string("Failed to call Mixer::report, Error: ") + - status.ToString()); - } - })); - - ::istio::mixer::v1::ReportRequest request; - request.set_request_index(++request_index_); - CovertToPb(info, config_->service_name(), request.mutable_attribute_update()); - env_->LogInfo(std::string("Send Report: ") + request.DebugString()); - - std::string request_body; - request.SerializeToString(&request_body); - - grpc_request - ->set_server(config_->server_config()->mixer_options().mixer_server()) - .set_service(kMixerServiceName) - .set_method("Report") - .set_body(request_body); - - env_->RunGRPCRequest(std::move(grpc_request)); + ::istio::mixer_client::Attributes attributes; + FillReportAttributes(info, config_->service_name(), &attributes); + env_->LogInfo("Send Report: "); + mixer_client_->Report( + attributes, [this](const ::google::protobuf::util::Status& status) { + if (status.ok()) { + env_->LogInfo("Report response: OK"); + } else { + env_->LogError(std::string("Failed to call Mixer::report, Error: ") + + status.ToString()); + } + }); return Status::OK; } @@ -185,40 +119,23 @@ void Mixer::Check( cloud_trace::CloudTraceSpan* parent_span, std::function on_done) { - std::unique_ptr grpc_request(new GRPCRequest([this, on_done]( - Status status, std::string&& body) { - if (status.ok()) { - // Handle 200 response - ::istio::mixer::v1::CheckResponse response; - if (!response.ParseFromString(body)) { - status = - Status(Code::INVALID_ARGUMENT, std::string("Invalid response")); - env_->LogError(std::string("Failed parse check response: ") + body); - } - env_->LogInfo(std::string("Check response: ") + response.DebugString()); - } else { - env_->LogError(std::string("Failed to call Mixer::check, Error: ") + - status.ToString()); - } - service_control::CheckResponseInfo info; - on_done(status, info); - })); - - ::istio::mixer::v1::CheckRequest request; - request.set_request_index(++request_index_); - CovertToPb(info, config_->service_name(), request.mutable_attribute_update()); - env_->LogInfo(std::string("Send Check: ") + request.DebugString()); - - std::string request_body; - request.SerializeToString(&request_body); - - grpc_request - ->set_server(config_->server_config()->mixer_options().mixer_server()) - .set_service(kMixerServiceName) - .set_method("Check") - .set_body(request_body); - - env_->RunGRPCRequest(std::move(grpc_request)); + ::istio::mixer_client::Attributes attributes; + FillCheckAttributes(info, config_->service_name(), &attributes); + env_->LogInfo("Send Check: "); + mixer_client_->Check( + attributes, + [this, on_done](const ::google::protobuf::util::Status& status) { + if (status.ok()) { + env_->LogInfo("Check response: OK"); + } else { + env_->LogError(std::string("Failed to call Mixer::check, Error: ") + + status.ToString()); + } + service_control::CheckResponseInfo info; + on_done(Status(status.error_code(), status.error_message(), + Status::SERVICE_CONTROL), + info); + }); } Status Mixer::GetStatistics(service_control::Statistics* esp_stat) const { diff --git a/contrib/endpoints/src/api_manager/mixer/mixer.h b/contrib/endpoints/src/api_manager/mixer/mixer.h index 0318a5f06f1..4cf2d1f54a7 100644 --- a/contrib/endpoints/src/api_manager/mixer/mixer.h +++ b/contrib/endpoints/src/api_manager/mixer/mixer.h @@ -18,6 +18,7 @@ #include "contrib/endpoints/include/api_manager/env_interface.h" #include "contrib/endpoints/src/api_manager/config.h" #include "contrib/endpoints/src/api_manager/service_control/interface.h" +#include "include/client.h" namespace google { namespace api_manager { @@ -51,9 +52,10 @@ class Mixer : public service_control::Interface { // The Api Manager environment interface. ApiManagerEnvInterface* env_; - int64_t request_index_; - + // The config. const Config* config_; + // The mixer client + std::unique_ptr<::istio::mixer_client::MixerClient> mixer_client_; }; } // namespace mixer diff --git a/src/envoy/prototype/api_manager_filter.cc b/src/envoy/prototype/api_manager_filter.cc index 2fca48de23c..4ccd9e6d120 100644 --- a/src/envoy/prototype/api_manager_filter.cc +++ b/src/envoy/prototype/api_manager_filter.cc @@ -214,13 +214,16 @@ class Instance : public Http::StreamFilter, status.ToJson()); if (!status.ok() && state_ != Responded) { state_ = Responded; - Utility::sendLocalReply(*decoder_callbacks_, Code(status.HttpCode()), - status.ToJson()); + decoder_callbacks_->dispatcher().post([this, status]() { + Utility::sendLocalReply(*decoder_callbacks_, Code(status.HttpCode()), + status.ToJson()); + }); return; } state_ = Complete; if (!initiating_call_) { - decoder_callbacks_->continueDecoding(); + decoder_callbacks_->dispatcher().post( + [this]() { decoder_callbacks_->continueDecoding(); }); } } diff --git a/src/envoy/prototype/envoy-esp.conf b/src/envoy/prototype/envoy-esp.conf index 11690bed565..d3ec54f405d 100644 --- a/src/envoy/prototype/envoy-esp.conf +++ b/src/envoy/prototype/envoy-esp.conf @@ -73,18 +73,6 @@ } ] }, - { - "name": "mixer_server", - "connect_timeout_ms": 5000, - "type": "strict_dns", - "lb_type": "round_robin", - "features": "http2", - "hosts": [ - { - "url": "tcp://localhost:9091" - } - ] - }, { "name": "api_manager", "connect_timeout_ms": 5000, diff --git a/src/envoy/prototype/server_config.pb.txt b/src/envoy/prototype/server_config.pb.txt index 9703886cc9e..19cb6cbb2c0 100644 --- a/src/envoy/prototype/server_config.pb.txt +++ b/src/envoy/prototype/server_config.pb.txt @@ -2,5 +2,5 @@ cloud_tracing_config { force_disable: true } mixer_options { - mixer_server: "mixer_server" -} \ No newline at end of file + mixer_server: "localhost:9091" +} From 6d7f0eebec4d00856987f9e8f55e4b9b4ddd8455 Mon Sep 17 00:00:00 2001 From: Wayne Zhang Date: Thu, 26 Jan 2017 23:38:51 -0800 Subject: [PATCH 09/83] Add originIp and originHost attributes. (#56) --- .../endpoints/include/api_manager/request.h | 2 + .../api_manager/context/request_context.cc | 3 +- .../endpoints/src/api_manager/mixer/mixer.cc | 107 ++++++++++++------ .../endpoints/src/api_manager/mixer/mixer.h | 10 ++ .../src/api_manager/service_control/info.h | 8 +- src/envoy/prototype/api_manager_filter.cc | 1 + 6 files changed, 95 insertions(+), 36 deletions(-) diff --git a/contrib/endpoints/include/api_manager/request.h b/contrib/endpoints/include/api_manager/request.h index ad604b5a41b..8d96ff356d6 100644 --- a/contrib/endpoints/include/api_manager/request.h +++ b/contrib/endpoints/include/api_manager/request.h @@ -43,6 +43,8 @@ class Request { // Gets Client IP // This will be used by service control Check() call. virtual std::string GetClientIP() = 0; + // Gets Client Host. + virtual std::string GetClientHost() { return ""; } // Get GRPC stats. virtual int64_t GetGrpcRequestBytes() = 0; diff --git a/contrib/endpoints/src/api_manager/context/request_context.cc b/contrib/endpoints/src/api_manager/context/request_context.cc index cbf4e925e37..59cc7c3d53f 100644 --- a/contrib/endpoints/src/api_manager/context/request_context.cc +++ b/contrib/endpoints/src/api_manager/context/request_context.cc @@ -171,6 +171,8 @@ void RequestContext::FillOperationInfo(service_control::OperationInfo *info) { info->producer_project_id = service_context()->project_id(); info->referer = http_referer_; info->request_start_time = start_time_; + info->client_ip = request_->GetClientIP(); + info->client_host = request_->GetClientHost(); } void RequestContext::FillLocation(service_control::ReportRequestInfo *info) { @@ -221,7 +223,6 @@ void RequestContext::FillLogMessage(service_control::ReportRequestInfo *info) { void RequestContext::FillCheckRequestInfo( service_control::CheckRequestInfo *info) { FillOperationInfo(info); - info->client_ip = request_->GetClientIP(); info->allow_unregistered_calls = method()->allow_unregistered_calls(); } diff --git a/contrib/endpoints/src/api_manager/mixer/mixer.cc b/contrib/endpoints/src/api_manager/mixer/mixer.cc index 47f6c4a8a2f..e9ea9a30ea5 100644 --- a/contrib/endpoints/src/api_manager/mixer/mixer.cc +++ b/contrib/endpoints/src/api_manager/mixer/mixer.cc @@ -22,8 +22,11 @@ namespace api_manager { namespace mixer { namespace { +const std::string kProxyPeerID = "Istio/Proxy"; + const std::string kAttrNameServiceName = "serviceName"; const std::string kAttrNamePeerId = "peerId"; +const std::string kAttrNameOperationId = "operationId"; const std::string kAttrNameOperationName = "operationName"; const std::string kAttrNameApiKey = "apiKey"; const std::string kAttrNameResponseCode = "responseCode"; @@ -35,6 +38,9 @@ const std::string kAttrNameApiMethod = "apiMethod"; const std::string kAttrNameRequestSize = "requestSize"; const std::string kAttrNameResponseSize = "responseSize"; const std::string kAttrNameLogMessage = "logMessage"; +const std::string kAttrNameResponseTime = "responseTime"; +const std::string kAttrNameOriginIp = "originIp"; +const std::string kAttrNameOriginHost = "originHost"; Attributes::Value StringValue(const std::string& str) { Attributes::Value v; @@ -50,37 +56,6 @@ Attributes::Value Int64Value(int64_t value) { return v; } -void FillCheckAttributes(const service_control::CheckRequestInfo& info, - const std::string& service_name, - ::istio::mixer_client::Attributes* attr) { - attr->attributes[kAttrNameServiceName] = StringValue(service_name); - attr->attributes[kAttrNamePeerId] = StringValue("Proxy"); - attr->attributes[kAttrNameOperationName] = StringValue(info.operation_name); - attr->attributes[kAttrNameApiKey] = StringValue(info.api_key); -} - -void FillReportAttributes(const service_control::ReportRequestInfo& info, - const std::string& service_name, - ::istio::mixer_client::Attributes* attr) { - attr->attributes[kAttrNameServiceName] = StringValue(service_name); - attr->attributes[kAttrNamePeerId] = StringValue("Proxy"); - attr->attributes[kAttrNameOperationName] = StringValue(info.operation_name); - attr->attributes[kAttrNameApiKey] = StringValue(info.api_key); - - attr->attributes[kAttrNameURL] = StringValue(info.url); - attr->attributes[kAttrNameLocation] = StringValue(info.location); - - attr->attributes[kAttrNameApiName] = StringValue(info.api_name); - attr->attributes[kAttrNameApiVersion] = StringValue(info.api_version); - attr->attributes[kAttrNameApiMethod] = StringValue(info.api_method); - - attr->attributes[kAttrNameLogMessage] = StringValue(info.log_message); - - attr->attributes[kAttrNameResponseCode] = Int64Value(info.response_code); - attr->attributes[kAttrNameRequestSize] = Int64Value(info.request_size); - attr->attributes[kAttrNameResponseSize] = Int64Value(info.response_size); -} - } // namespace Mixer::Mixer(ApiManagerEnvInterface* env, const Config* config) @@ -98,9 +73,75 @@ Status Mixer::Init() { Status Mixer::Close() { return Status::OK; } +void Mixer::FillCommonAttributes(const service_control::OperationInfo& info, + ::istio::mixer_client::Attributes* attr) { + attr->attributes[kAttrNameServiceName] = StringValue(config_->service_name()); + attr->attributes[kAttrNamePeerId] = StringValue(kProxyPeerID); + + if (!info.operation_id.empty()) { + attr->attributes[kAttrNameOperationId] = StringValue(info.operation_id); + } + if (!info.operation_name.empty()) { + attr->attributes[kAttrNameOperationName] = StringValue(info.operation_name); + } + if (!info.api_key.empty()) { + attr->attributes[kAttrNameApiKey] = StringValue(info.api_key); + } + if (!info.client_ip.empty()) { + attr->attributes[kAttrNameOriginIp] = StringValue(info.client_ip); + } + if (!info.client_host.empty()) { + attr->attributes[kAttrNameOriginHost] = StringValue(info.client_host); + } +} + +void Mixer::FillCheckAttributes(const service_control::CheckRequestInfo& info, + ::istio::mixer_client::Attributes* attr) { + FillCommonAttributes(info, attr); +} + +void Mixer::FillReportAttributes(const service_control::ReportRequestInfo& info, + ::istio::mixer_client::Attributes* attr) { + FillCommonAttributes(info, attr); + + if (!info.url.empty()) { + attr->attributes[kAttrNameURL] = StringValue(info.url); + } + if (!info.location.empty()) { + attr->attributes[kAttrNameLocation] = StringValue(info.location); + } + + if (!info.api_name.empty()) { + attr->attributes[kAttrNameApiName] = StringValue(info.api_name); + } + if (!info.api_version.empty()) { + attr->attributes[kAttrNameApiVersion] = StringValue(info.api_version); + } + if (!info.api_method.empty()) { + attr->attributes[kAttrNameApiMethod] = StringValue(info.api_method); + } + + if (!info.log_message.empty()) { + attr->attributes[kAttrNameLogMessage] = StringValue(info.log_message); + } + + attr->attributes[kAttrNameResponseCode] = Int64Value(info.response_code); + if (info.request_size >= 0) { + attr->attributes[kAttrNameRequestSize] = Int64Value(info.request_size); + } + if (info.response_size >= 0) { + attr->attributes[kAttrNameResponseSize] = Int64Value(info.response_size); + } + + if (info.latency.request_time_ms >= 0) { + attr->attributes[kAttrNameResponseTime] = + Int64Value(info.latency.request_time_ms); + } +} + Status Mixer::Report(const service_control::ReportRequestInfo& info) { ::istio::mixer_client::Attributes attributes; - FillReportAttributes(info, config_->service_name(), &attributes); + FillReportAttributes(info, &attributes); env_->LogInfo("Send Report: "); mixer_client_->Report( attributes, [this](const ::google::protobuf::util::Status& status) { @@ -120,7 +161,7 @@ void Mixer::Check( std::function on_done) { ::istio::mixer_client::Attributes attributes; - FillCheckAttributes(info, config_->service_name(), &attributes); + FillCheckAttributes(info, &attributes); env_->LogInfo("Send Check: "); mixer_client_->Check( attributes, diff --git a/contrib/endpoints/src/api_manager/mixer/mixer.h b/contrib/endpoints/src/api_manager/mixer/mixer.h index 4cf2d1f54a7..b020d2636f9 100644 --- a/contrib/endpoints/src/api_manager/mixer/mixer.h +++ b/contrib/endpoints/src/api_manager/mixer/mixer.h @@ -50,6 +50,16 @@ class Mixer : public service_control::Interface { // The constructor. Mixer(ApiManagerEnvInterface* env, const Config* config); + // Fill common attributes for both check and report. + void FillCommonAttributes(const service_control::OperationInfo& info, + ::istio::mixer_client::Attributes* attr); + // Fill attributes for check. + void FillCheckAttributes(const service_control::CheckRequestInfo& info, + ::istio::mixer_client::Attributes* attr); + // Fill attributes for report. + void FillReportAttributes(const service_control::ReportRequestInfo& info, + ::istio::mixer_client::Attributes* attr); + // The Api Manager environment interface. ApiManagerEnvInterface* env_; // The config. diff --git a/contrib/endpoints/src/api_manager/service_control/info.h b/contrib/endpoints/src/api_manager/service_control/info.h index 15364ad58c6..f203057cc9e 100644 --- a/contrib/endpoints/src/api_manager/service_control/info.h +++ b/contrib/endpoints/src/api_manager/service_control/info.h @@ -60,13 +60,17 @@ struct OperationInfo { // and Report. std::chrono::system_clock::time_point request_start_time; + // The client IP address. + std::string client_ip; + + // The client host name. + std::string client_host; + OperationInfo() {} }; // Information to fill Check request protobuf. struct CheckRequestInfo : public OperationInfo { - // The client IP address. - std::string client_ip; // Whether the method allow unregistered calls. bool allow_unregistered_calls; diff --git a/src/envoy/prototype/api_manager_filter.cc b/src/envoy/prototype/api_manager_filter.cc index 4ccd9e6d120..869d8e6c500 100644 --- a/src/envoy/prototype/api_manager_filter.cc +++ b/src/envoy/prototype/api_manager_filter.cc @@ -81,6 +81,7 @@ class Request : public google::api_manager::Request { return header_map_.Path()->value().c_str(); } virtual std::string GetClientIP() override { return ""; } + virtual std::string GetClientHost() override { return ""; } virtual bool FindQuery(const std::string& name, std::string* query) override { if (!query_parsed_) { auto header = header_map_.Path(); From 4cd5c9f1d7c8367239021c35b9d72229a82353de Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Fri, 27 Jan 2017 17:54:34 -0800 Subject: [PATCH 10/83] Add uuid-dev dependency in README.md (#45) --- src/envoy/prototype/README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/envoy/prototype/README.md b/src/envoy/prototype/README.md index 00e73a169d6..dc73f63f6c9 100644 --- a/src/envoy/prototype/README.md +++ b/src/envoy/prototype/README.md @@ -1,6 +1,11 @@ This Proxy will use Envoy and talk to Mixer server. +## Install dependencies + +``` + apt-get install uuid-dev +``` ## Build Mixer server From bcdb75d2703050cb3d7b9b11b242a2304068cf90 Mon Sep 17 00:00:00 2001 From: Wayne Zhang Date: Sat, 28 Jan 2017 14:15:11 -0800 Subject: [PATCH 11/83] Extract originIp and OriginHost. (#57) * Extract originIp and OriginHost. * Make header x-forwarded-host const. --- src/envoy/prototype/api_manager_filter.cc | 56 +++++++++++++++++++++-- 1 file changed, 51 insertions(+), 5 deletions(-) diff --git a/src/envoy/prototype/api_manager_filter.cc b/src/envoy/prototype/api_manager_filter.cc index 869d8e6c500..d616ddfdf0f 100644 --- a/src/envoy/prototype/api_manager_filter.cc +++ b/src/envoy/prototype/api_manager_filter.cc @@ -13,6 +13,12 @@ namespace Http { namespace ApiManager { +namespace { + +// Define lower case string for X-Forwarded-Host. +const LowerCaseString kHeaderNameXFH("x-forwarded-host", false); + +} // namespace std::string ReadFile(const std::string& file_name) { std::ifstream t(file_name); @@ -65,12 +71,18 @@ typedef std::shared_ptr ConfigPtr; class Request : public google::api_manager::Request { private: HeaderMap& header_map_; + std::string downstream_address_; + std::string virtual_host_; bool query_parsed_; std::map query_params_; public: - Request(HeaderMap& header_map) - : header_map_(header_map), query_parsed_(false) {} + Request(HeaderMap& header_map, const std::string& downstream_address, + const std::string& virtual_host) + : header_map_(header_map), + downstream_address_(downstream_address), + virtual_host_(virtual_host), + query_parsed_(false) {} virtual std::string GetRequestHTTPMethod() override { return header_map_.Method()->value().c_str(); } @@ -80,8 +92,31 @@ class Request : public google::api_manager::Request { virtual std::string GetUnparsedRequestPath() override { return header_map_.Path()->value().c_str(); } - virtual std::string GetClientIP() override { return ""; } - virtual std::string GetClientHost() override { return ""; } + + virtual std::string GetClientIP() override { + if (!header_map_.ForwardedFor()) { + return downstream_address_; + } + std::vector xff_address_list = + StringUtil::split(header_map_.ForwardedFor()->value().c_str(), ','); + if (xff_address_list.empty()) { + return downstream_address_; + } + return xff_address_list.front(); + } + + virtual std::string GetClientHost() override { + const HeaderEntry* entry = header_map_.get(kHeaderNameXFH); + if (entry == nullptr) { + return virtual_host_; + } + auto xff_list = StringUtil::split(entry->value().c_str(), ','); + if (xff_list.empty()) { + return virtual_host_; + } + return xff_list.back(); + } + virtual bool FindQuery(const std::string& name, std::string* query) override { if (!query_parsed_) { auto header = header_map_.Path(); @@ -159,6 +194,15 @@ class Instance : public Http::StreamFilter, bool initiating_call_; + std::string getRouteVirtualHost(HeaderMap& headers) const { + const Router::Route* route = + decoder_callbacks_->routeTable().route(headers); + if (route && route->routeEntry()) { + return route->routeEntry()->virtualHost().name(); + } + return ""; + } + public: Instance(ConfigPtr config) : api_manager_(config->api_manager()), @@ -170,7 +214,9 @@ class Instance : public Http::StreamFilter, FilterHeadersStatus decodeHeaders(HeaderMap& headers, bool end_stream) override { log().debug("Called ApiManager::Instance : {}", __func__); - std::unique_ptr request(new Request(headers)); + std::unique_ptr request( + new Request(headers, decoder_callbacks_->downstreamAddress(), + getRouteVirtualHost(headers))); request_handler_ = api_manager_->CreateRequestHandler(std::move(request)); state_ = Calling; initiating_call_ = true; From 2c9c1af3d6ac5ab4d295444c30acd42599edfcb3 Mon Sep 17 00:00:00 2001 From: Qian Sun Date: Mon, 30 Jan 2017 15:03:42 -0800 Subject: [PATCH 12/83] Update buckets for UI. (#58) * Update buckets for UI. * Only update time_distribution. --- .../src/api_manager/service_control/proto.cc | 2 +- .../testdata/final_report_request.golden | 150 ++++++++++++++++-- .../testdata/report_request.golden | 150 ++++++++++++++++-- .../testdata/report_request_failed.golden | 75 ++++++++- 4 files changed, 346 insertions(+), 31 deletions(-) diff --git a/contrib/endpoints/src/api_manager/service_control/proto.cc b/contrib/endpoints/src/api_manager/service_control/proto.cc index 17b921db076..3e7eba64d21 100644 --- a/contrib/endpoints/src/api_manager/service_control/proto.cc +++ b/contrib/endpoints/src/api_manager/service_control/proto.cc @@ -96,7 +96,7 @@ struct DistributionHelperOptions { double scale; }; -const DistributionHelperOptions time_distribution = {8, 10.0, 1e-6}; +const DistributionHelperOptions time_distribution = {29, 2.0, 1e-6}; const DistributionHelperOptions size_distribution = {8, 10.0, 1}; const double kMsToSecs = 1e-3; diff --git a/contrib/endpoints/src/api_manager/service_control/testdata/final_report_request.golden b/contrib/endpoints/src/api_manager/service_control/testdata/final_report_request.golden index b9eff9f28eb..d60b039987d 100644 --- a/contrib/endpoints/src/api_manager/service_control/testdata/final_report_request.golden +++ b/contrib/endpoints/src/api_manager/service_control/testdata/final_report_request.golden @@ -201,13 +201,34 @@ operations { bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 bucket_counts: 1 bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 exponential_buckets { - num_finite_buckets: 8 - growth_factor: 10 + num_finite_buckets: 29 + growth_factor: 2 scale: 1e-06 } } @@ -227,13 +248,34 @@ operations { bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 bucket_counts: 1 bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 exponential_buckets { - num_finite_buckets: 8 - growth_factor: 10 + num_finite_buckets: 29 + growth_factor: 2 scale: 1e-06 } } @@ -253,13 +295,34 @@ operations { bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 bucket_counts: 1 bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 exponential_buckets { - num_finite_buckets: 8 - growth_factor: 10 + num_finite_buckets: 29 + growth_factor: 2 scale: 1e-06 } } @@ -279,13 +342,34 @@ operations { bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 bucket_counts: 1 bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 exponential_buckets { - num_finite_buckets: 8 - growth_factor: 10 + num_finite_buckets: 29 + growth_factor: 2 scale: 1e-06 } } @@ -304,14 +388,35 @@ operations { bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 bucket_counts: 1 bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 exponential_buckets { - num_finite_buckets: 8 - growth_factor: 10 + num_finite_buckets: 29 + growth_factor: 2 scale: 1e-06 } } @@ -330,14 +435,35 @@ operations { bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 bucket_counts: 1 bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 exponential_buckets { - num_finite_buckets: 8 - growth_factor: 10 + num_finite_buckets: 29 + growth_factor: 2 scale: 1e-06 } } diff --git a/contrib/endpoints/src/api_manager/service_control/testdata/report_request.golden b/contrib/endpoints/src/api_manager/service_control/testdata/report_request.golden index 5d8a266ef88..faf96383418 100644 --- a/contrib/endpoints/src/api_manager/service_control/testdata/report_request.golden +++ b/contrib/endpoints/src/api_manager/service_control/testdata/report_request.golden @@ -213,13 +213,34 @@ operations { bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 bucket_counts: 1 bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 exponential_buckets { - num_finite_buckets: 8 - growth_factor: 10 + num_finite_buckets: 29 + growth_factor: 2 scale: 1e-06 } } @@ -239,13 +260,34 @@ operations { bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 bucket_counts: 1 bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 exponential_buckets { - num_finite_buckets: 8 - growth_factor: 10 + num_finite_buckets: 29 + growth_factor: 2 scale: 1e-06 } } @@ -265,13 +307,34 @@ operations { bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 bucket_counts: 1 bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 exponential_buckets { - num_finite_buckets: 8 - growth_factor: 10 + num_finite_buckets: 29 + growth_factor: 2 scale: 1e-06 } } @@ -291,13 +354,34 @@ operations { bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 bucket_counts: 1 bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 exponential_buckets { - num_finite_buckets: 8 - growth_factor: 10 + num_finite_buckets: 29 + growth_factor: 2 scale: 1e-06 } } @@ -316,14 +400,35 @@ operations { bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 bucket_counts: 1 bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 exponential_buckets { - num_finite_buckets: 8 - growth_factor: 10 + num_finite_buckets: 29 + growth_factor: 2 scale: 1e-06 } } @@ -342,14 +447,35 @@ operations { bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 bucket_counts: 1 bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 exponential_buckets { - num_finite_buckets: 8 - growth_factor: 10 + num_finite_buckets: 29 + growth_factor: 2 scale: 1e-06 } } diff --git a/contrib/endpoints/src/api_manager/service_control/testdata/report_request_failed.golden b/contrib/endpoints/src/api_manager/service_control/testdata/report_request_failed.golden index a3ffd58c30d..710772cd215 100644 --- a/contrib/endpoints/src/api_manager/service_control/testdata/report_request_failed.golden +++ b/contrib/endpoints/src/api_manager/service_control/testdata/report_request_failed.golden @@ -152,13 +152,34 @@ operations { bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 bucket_counts: 1 bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 exponential_buckets { - num_finite_buckets: 8 - growth_factor: 10 + num_finite_buckets: 29 + growth_factor: 2 scale: 1e-06 } } @@ -178,13 +199,34 @@ operations { bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 bucket_counts: 1 bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 exponential_buckets { - num_finite_buckets: 8 - growth_factor: 10 + num_finite_buckets: 29 + growth_factor: 2 scale: 1e-06 } } @@ -203,14 +245,35 @@ operations { bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 bucket_counts: 1 bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 + bucket_counts: 0 exponential_buckets { - num_finite_buckets: 8 - growth_factor: 10 + num_finite_buckets: 29 + growth_factor: 2 scale: 1e-06 } } From 27962094adf801685a8547284fb60ca7c75ad1f8 Mon Sep 17 00:00:00 2001 From: Wayne Zhang Date: Tue, 31 Jan 2017 16:12:56 -0800 Subject: [PATCH 13/83] Add targetService attribute. (#59) --- contrib/endpoints/repositories.bzl | 2 +- contrib/endpoints/src/api_manager/mixer/mixer.cc | 11 +++++++++++ contrib/endpoints/src/api_manager/mixer/mixer.h | 2 ++ src/envoy/prototype/api_manager_env.cc | 2 +- src/envoy/prototype/api_manager_filter.cc | 3 +-- src/envoy/repositories.bzl | 2 +- 6 files changed, 17 insertions(+), 5 deletions(-) diff --git a/contrib/endpoints/repositories.bzl b/contrib/endpoints/repositories.bzl index 7b77aa91c5a..b1f14aae8b5 100644 --- a/contrib/endpoints/repositories.bzl +++ b/contrib/endpoints/repositories.bzl @@ -337,7 +337,7 @@ def servicecontrol_client_repositories(bind=True): def mixer_client_repositories(bind=True): native.git_repository( name = "mixerclient_git", - commit = "1569430f1e27b31e23c029c6bec0d8d5062d9e55", + commit = "80e450a5126960e8e6337c3631cf2ef984038eab", remote = "https://github.com/istio/mixerclient.git", ) diff --git a/contrib/endpoints/src/api_manager/mixer/mixer.cc b/contrib/endpoints/src/api_manager/mixer/mixer.cc index e9ea9a30ea5..53837edaf67 100644 --- a/contrib/endpoints/src/api_manager/mixer/mixer.cc +++ b/contrib/endpoints/src/api_manager/mixer/mixer.cc @@ -23,6 +23,7 @@ namespace mixer { namespace { const std::string kProxyPeerID = "Istio/Proxy"; +const std::string kEnvNameTargetService = "TARGET_SERVICE"; const std::string kAttrNameServiceName = "serviceName"; const std::string kAttrNamePeerId = "peerId"; @@ -41,6 +42,7 @@ const std::string kAttrNameLogMessage = "logMessage"; const std::string kAttrNameResponseTime = "responseTime"; const std::string kAttrNameOriginIp = "originIp"; const std::string kAttrNameOriginHost = "originHost"; +const std::string kAttrNameTargetService = "targetService"; Attributes::Value StringValue(const std::string& str) { Attributes::Value v; @@ -68,6 +70,10 @@ Status Mixer::Init() { options.mixer_server = config_->server_config()->mixer_options().mixer_server(); mixer_client_ = ::istio::mixer_client::CreateMixerClient(options); + auto target_service = getenv(kEnvNameTargetService.c_str()); + if (target_service) { + target_service_ = target_service; + } return Status::OK; } @@ -93,6 +99,9 @@ void Mixer::FillCommonAttributes(const service_control::OperationInfo& info, if (!info.client_host.empty()) { attr->attributes[kAttrNameOriginHost] = StringValue(info.client_host); } + if (!target_service_.empty()) { + attr->attributes[kAttrNameTargetService] = StringValue(target_service_); + } } void Mixer::FillCheckAttributes(const service_control::CheckRequestInfo& info, @@ -143,6 +152,7 @@ Status Mixer::Report(const service_control::ReportRequestInfo& info) { ::istio::mixer_client::Attributes attributes; FillReportAttributes(info, &attributes); env_->LogInfo("Send Report: "); + env_->LogInfo(attributes.DebugString()); mixer_client_->Report( attributes, [this](const ::google::protobuf::util::Status& status) { if (status.ok()) { @@ -163,6 +173,7 @@ void Mixer::Check( ::istio::mixer_client::Attributes attributes; FillCheckAttributes(info, &attributes); env_->LogInfo("Send Check: "); + env_->LogInfo(attributes.DebugString()); mixer_client_->Check( attributes, [this, on_done](const ::google::protobuf::util::Status& status) { diff --git a/contrib/endpoints/src/api_manager/mixer/mixer.h b/contrib/endpoints/src/api_manager/mixer/mixer.h index b020d2636f9..81276e1227d 100644 --- a/contrib/endpoints/src/api_manager/mixer/mixer.h +++ b/contrib/endpoints/src/api_manager/mixer/mixer.h @@ -66,6 +66,8 @@ class Mixer : public service_control::Interface { const Config* config_; // The mixer client std::unique_ptr<::istio::mixer_client::MixerClient> mixer_client_; + // Target service + std::string target_service_; }; } // namespace mixer diff --git a/src/envoy/prototype/api_manager_env.cc b/src/envoy/prototype/api_manager_env.cc index 6b1619a6faa..f8536bcc10a 100644 --- a/src/envoy/prototype/api_manager_env.cc +++ b/src/envoy/prototype/api_manager_env.cc @@ -103,7 +103,7 @@ class HTTPRequest : public Http::Message { virtual void body(Buffer::InstancePtr &&body) override {} virtual HeaderMap *trailers() override { return nullptr; } virtual void trailers(HeaderMapPtr &&trailers) override {} - virtual std::string bodyAsString() override { return ""; } + virtual std::string bodyAsString() const override { return ""; } }; class HTTPRequestCallbacks : public AsyncClient::Callbacks { diff --git a/src/envoy/prototype/api_manager_filter.cc b/src/envoy/prototype/api_manager_filter.cc index d616ddfdf0f..87a3beeb48a 100644 --- a/src/envoy/prototype/api_manager_filter.cc +++ b/src/envoy/prototype/api_manager_filter.cc @@ -195,8 +195,7 @@ class Instance : public Http::StreamFilter, bool initiating_call_; std::string getRouteVirtualHost(HeaderMap& headers) const { - const Router::Route* route = - decoder_callbacks_->routeTable().route(headers); + const Router::Route* route = decoder_callbacks_->route(); if (route && route->routeEntry()) { return route->routeEntry()->virtualHost().name(); } diff --git a/src/envoy/repositories.bzl b/src/envoy/repositories.bzl index dd36a23d456..e19efc6dce1 100644 --- a/src/envoy/repositories.bzl +++ b/src/envoy/repositories.bzl @@ -629,6 +629,6 @@ cc_test( native.new_git_repository( name = "envoy_git", remote = "https://github.com/lyft/envoy.git", - commit = "c36bb76e304ab4c5584a07303e2953a170888319", + commit = "39f42378fa41c10996d4c3ffba534951de30ceb8", build_file_content = BUILD, ) From 8218f92089a6ac36f851f3cacbf761e443e5f8a1 Mon Sep 17 00:00:00 2001 From: Wayne Zhang Date: Wed, 1 Feb 2017 12:08:31 -0800 Subject: [PATCH 14/83] Use envoy new access_log handler for sending Report. (#60) * use access_log handler. * Not to use Loggable base class. --- src/envoy/prototype/api_manager_filter.cc | 61 ++++++++++++++--------- src/envoy/prototype/envoy-esp.conf | 3 +- src/envoy/repositories.bzl | 2 +- 3 files changed, 40 insertions(+), 26 deletions(-) diff --git a/src/envoy/prototype/api_manager_filter.cc b/src/envoy/prototype/api_manager_filter.cc index 87a3beeb48a..3983a4d8b41 100644 --- a/src/envoy/prototype/api_manager_filter.cc +++ b/src/envoy/prototype/api_manager_filter.cc @@ -163,24 +163,30 @@ class Request : public google::api_manager::Request { }; class Response : public google::api_manager::Response { + const AccessLog::RequestInfo& request_info_; + + public: + Response(const AccessLog::RequestInfo& request_info) + : request_info_(request_info) {} + google::api_manager::utils::Status GetResponseStatus() { return google::api_manager::utils::Status::OK; } - std::size_t GetRequestSize() { return 0; } + std::size_t GetRequestSize() { return request_info_.bytesReceived(); } - std::size_t GetResponseSize() { return 0; } + std::size_t GetResponseSize() { return request_info_.bytesSent(); } google::api_manager::utils::Status GetLatencyInfo( google::api_manager::service_control::LatencyInfo* info) { + info->request_time_ms = request_info_.duration().count(); return google::api_manager::utils::Status::OK; } }; const Http::HeaderMapImpl BadRequest{{Http::Headers::get().Status, "400"}}; -class Instance : public Http::StreamFilter, - public Logger::Loggable { +class Instance : public Http::StreamFilter, public Http::AccessLog::Instance { private: std::shared_ptr api_manager_; std::unique_ptr @@ -207,12 +213,12 @@ class Instance : public Http::StreamFilter, : api_manager_(config->api_manager()), state_(NotStarted), initiating_call_(false) { - log().debug("Called ApiManager::Instance : {}", __func__); + Log().debug("Called ApiManager::Instance : {}", __func__); } FilterHeadersStatus decodeHeaders(HeaderMap& headers, bool end_stream) override { - log().debug("Called ApiManager::Instance : {}", __func__); + Log().debug("Called ApiManager::Instance : {}", __func__); std::unique_ptr request( new Request(headers, decoder_callbacks_->downstreamAddress(), getRouteVirtualHost(headers))); @@ -227,13 +233,13 @@ class Instance : public Http::StreamFilter, if (state_ == Complete) { return FilterHeadersStatus::Continue; } - log().debug("Called ApiManager::Instance : {} Stop", __func__); + Log().debug("Called ApiManager::Instance : {} Stop", __func__); return FilterHeadersStatus::StopIteration; } FilterDataStatus decodeData(Buffer::Instance& data, bool end_stream) override { - log().debug("Called ApiManager::Instance : {} ({}, {})", __func__, + Log().debug("Called ApiManager::Instance : {} ({}, {})", __func__, data.length(), end_stream); if (state_ == Calling) { return FilterDataStatus::StopIterationAndBuffer; @@ -242,7 +248,7 @@ class Instance : public Http::StreamFilter, } FilterTrailersStatus decodeTrailers(HeaderMap& trailers) override { - log().debug("Called ApiManager::Instance : {}", __func__); + Log().debug("Called ApiManager::Instance : {}", __func__); if (state_ == Calling) { return FilterTrailersStatus::StopIteration; } @@ -250,13 +256,13 @@ class Instance : public Http::StreamFilter, } void setDecoderFilterCallbacks( StreamDecoderFilterCallbacks& callbacks) override { - log().debug("Called ApiManager::Instance : {}", __func__); + Log().debug("Called ApiManager::Instance : {}", __func__); decoder_callbacks_ = &callbacks; decoder_callbacks_->addResetStreamCallback( [this]() { state_ = Responded; }); } void completeCheck(const google::api_manager::utils::Status& status) { - log().debug("Called ApiManager::Instance : check complete {}", + Log().debug("Called ApiManager::Instance : check complete {}", status.ToJson()); if (!status.ok() && state_ != Responded) { state_ = Responded; @@ -275,30 +281,37 @@ class Instance : public Http::StreamFilter, virtual FilterHeadersStatus encodeHeaders(HeaderMap& headers, bool end_stream) override { - log().debug("Called ApiManager::Instance : {}", __func__); + Log().debug("Called ApiManager::Instance : {}", __func__); return FilterHeadersStatus::Continue; } virtual FilterDataStatus encodeData(Buffer::Instance& data, bool end_stream) override { - log().debug("Called ApiManager::Instance : {}", __func__); + Log().debug("Called ApiManager::Instance : {}", __func__); return FilterDataStatus::Continue; } virtual FilterTrailersStatus encodeTrailers(HeaderMap& trailers) override { - log().debug("Called ApiManager::Instance : {}", __func__); + Log().debug("Called ApiManager::Instance : {}", __func__); return FilterTrailersStatus::Continue; } virtual void setEncoderFilterCallbacks( StreamEncoderFilterCallbacks& callbacks) override { - log().debug("Called ApiManager::Instance : {}", __func__); + Log().debug("Called ApiManager::Instance : {}", __func__); encoder_callbacks_ = &callbacks; } - // note: cannot extend ~ActiveStream for access log, placing it here - ~Instance() { - log().debug("Called ApiManager::Instance : {}", __func__); - std::unique_ptr response(new Response()); - request_handler_->Report(std::move(response), - [this]() { log().debug("Report returns"); }); + virtual void log(const HeaderMap* request_headers, + const HeaderMap* response_headers, + const AccessLog::RequestInfo& request_info) override { + Log().debug("Called ApiManager::Instance : {}", __func__); + std::unique_ptr response( + new Response(request_info)); + request_handler_->Report(std::move(response), []() {}); + } + + spdlog::logger& Log() { + static spdlog::logger& instance = + Logger::Registry::getLog(Logger::Id::http); + return instance; } }; } @@ -320,8 +333,10 @@ class ApiManagerConfig : public HttpFilterConfigFactory { new Http::ApiManager::Config(config, server)); return [api_manager_config]( Http::FilterChainFactoryCallbacks& callbacks) -> void { - auto instance = new Http::ApiManager::Instance(api_manager_config); - callbacks.addStreamFilter(Http::StreamFilterPtr{instance}); + std::shared_ptr instance( + new Http::ApiManager::Instance(api_manager_config)); + callbacks.addStreamFilter(Http::StreamFilterPtr(instance)); + callbacks.addAccessLogHandler(Http::AccessLog::InstancePtr(instance)); }; } }; diff --git a/src/envoy/prototype/envoy-esp.conf b/src/envoy/prototype/envoy-esp.conf index d3ec54f405d..27ea9f1b397 100644 --- a/src/envoy/prototype/envoy-esp.conf +++ b/src/envoy/prototype/envoy-esp.conf @@ -85,6 +85,5 @@ ] } ] - }, - "tracing_enabled": "true" + } } diff --git a/src/envoy/repositories.bzl b/src/envoy/repositories.bzl index e19efc6dce1..d14053308e1 100644 --- a/src/envoy/repositories.bzl +++ b/src/envoy/repositories.bzl @@ -629,6 +629,6 @@ cc_test( native.new_git_repository( name = "envoy_git", remote = "https://github.com/lyft/envoy.git", - commit = "39f42378fa41c10996d4c3ffba534951de30ceb8", + commit = "0bac7508c6803ec315c2228672728281b99149bd", build_file_content = BUILD, ) From 1d2d6a84989321ed20c3504704ce6c249188c166 Mon Sep 17 00:00:00 2001 From: Wayne Zhang Date: Wed, 1 Feb 2017 17:57:55 -0800 Subject: [PATCH 15/83] Update to the latest envoy with #396. (#61) --- src/envoy/repositories.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/envoy/repositories.bzl b/src/envoy/repositories.bzl index d14053308e1..7cc4cb2db48 100644 --- a/src/envoy/repositories.bzl +++ b/src/envoy/repositories.bzl @@ -629,6 +629,6 @@ cc_test( native.new_git_repository( name = "envoy_git", remote = "https://github.com/lyft/envoy.git", - commit = "0bac7508c6803ec315c2228672728281b99149bd", + commit = "02c6fc97b4c21d25ab596a25208fbe283e927f6a", build_file_content = BUILD, ) From 53fd02610571f8c05af2e50fea911f89f935e678 Mon Sep 17 00:00:00 2001 From: Kuat Date: Thu, 2 Feb 2017 11:52:57 -0800 Subject: [PATCH 16/83] Fix tclap dependency fetching error (#62) --- src/envoy/repositories.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/envoy/repositories.bzl b/src/envoy/repositories.bzl index 7cc4cb2db48..94f371ebc7d 100644 --- a/src/envoy/repositories.bzl +++ b/src/envoy/repositories.bzl @@ -221,7 +221,7 @@ cc_library( native.new_http_archive( name = "tclap_tar", - url = "https://sourceforge.net/projects/tclap/files/tclap-1.2.1.tar.gz/download", + url = "https://storage.googleapis.com/istio-build-deps/tclap-1.2.1.tar.gz", type = "tar.gz", strip_prefix = "tclap-1.2.1", build_file_content = BUILD, From 33f54ed2b98c9e57a74e78bbb34756764c206f7a Mon Sep 17 00:00:00 2001 From: Wayne Zhang Date: Mon, 6 Feb 2017 10:08:35 -0800 Subject: [PATCH 17/83] Integrate mixer client directly with envoy. (#66) * Integrate mixer client directly with envoy. * Send response header in Report. * rename filter name from esp to mixer. * add README. --- src/envoy/mixer/BUILD | 41 +++++ src/envoy/mixer/README.md | 49 ++++++ src/envoy/mixer/envoy-mixer.conf | 71 ++++++++ src/envoy/mixer/http_control.cc | 184 +++++++++++++++++++++ src/envoy/mixer/http_control.h | 63 ++++++++ src/envoy/mixer/http_filter.cc | 267 +++++++++++++++++++++++++++++++ 6 files changed, 675 insertions(+) create mode 100644 src/envoy/mixer/BUILD create mode 100644 src/envoy/mixer/README.md create mode 100644 src/envoy/mixer/envoy-mixer.conf create mode 100644 src/envoy/mixer/http_control.cc create mode 100644 src/envoy/mixer/http_control.h create mode 100644 src/envoy/mixer/http_filter.cc diff --git a/src/envoy/mixer/BUILD b/src/envoy/mixer/BUILD new file mode 100644 index 00000000000..428c87101b9 --- /dev/null +++ b/src/envoy/mixer/BUILD @@ -0,0 +1,41 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ +# + +package(default_visibility = ["//visibility:public"]) + +cc_library( + name = "filter_lib", + srcs = [ + "http_control.cc", + "http_control.h", + "http_filter.cc", + ], + deps = [ + "//external:mixer_client_lib", + "@envoy_git//:envoy-common", + ], + alwayslink = 1, +) + +cc_binary( + name = "envoy_esp", + linkstatic = 1, + deps = [ + ":filter_lib", + "@envoy_git//:envoy-main", + ], +) diff --git a/src/envoy/mixer/README.md b/src/envoy/mixer/README.md new file mode 100644 index 00000000000..94276e37826 --- /dev/null +++ b/src/envoy/mixer/README.md @@ -0,0 +1,49 @@ + +This Proxy will use Envoy and talk to Mixer server. + +## Build Mixer server + +* Follow https://github.com/istio/mixer/blob/master/doc/devel/development.md to set up environment, and build via: + +``` + cd $(ISTIO)/mixer + bazel build ...:all +``` + +## Build Envoy proxy + +* Build target envoy_esp: + +``` + bazel build //src/envoy/mixer:envoy_esp +``` + +## How to run it + +* Start mixer server. In mixer folder run: + +``` + bazel-bin/cmd/server/mixs server +``` + + The server will run at port 9091 + +* Start backend Echo server. + +``` + cd test/backend/echo + go run echo.go +``` + +* Start Envoy proxy, run + +``` + bazel-bin/src/envoy/mixer/envoy_esp -c src/envoy/prototype/envoy-mixer.conf +``` + +* Then issue HTTP request to proxy. + +``` + curl http://localhost:9090/echo -d "hello world" +``` + diff --git a/src/envoy/mixer/envoy-mixer.conf b/src/envoy/mixer/envoy-mixer.conf new file mode 100644 index 00000000000..3b494c96b59 --- /dev/null +++ b/src/envoy/mixer/envoy-mixer.conf @@ -0,0 +1,71 @@ +{ + "listeners": [ + { + "port": 9090, + "bind_to_port": true, + "filters": [ + { + "type": "read", + "name": "http_connection_manager", + "config": { + "codec_type": "auto", + "stat_prefix": "ingress_http", + "route_config": { + "virtual_hosts": [ + { + "name": "backend", + "domains": ["*"], + "routes": [ + { + "timeout_ms": 0, + "prefix": "/", + "cluster": "service1" + } + ] + } + ] + }, + "access_log": [ + { + "path": "/dev/stdout" + } + ], + "filters": [ + { + "type": "both", + "name": "mixer", + "config": { + "mixer_server": "localhost:9091" + } + }, + { + "type": "decoder", + "name": "router", + "config": {} + } + ] + } + } + ] + } + ], + "admin": { + "access_log_path": "/dev/stdout", + "port": 9001 + }, + "cluster_manager": { + "clusters": [ + { + "name": "service1", + "connect_timeout_ms": 5000, + "type": "strict_dns", + "lb_type": "round_robin", + "hosts": [ + { + "url": "tcp://localhost:8080" + } + ] + } + ] + } +} diff --git a/src/envoy/mixer/http_control.cc b/src/envoy/mixer/http_control.cc new file mode 100644 index 00000000000..39e96ecbd3c --- /dev/null +++ b/src/envoy/mixer/http_control.cc @@ -0,0 +1,184 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/envoy/mixer/http_control.h" +#include "common/common/utility.h" +#include "common/http/utility.h" + +using ::google::protobuf::util::Status; +using ::istio::mixer_client::Attributes; +using ::istio::mixer_client::DoneFunc; + +namespace Http { +namespace Mixer { +namespace { + +const std::string kProxyPeerID = "Istio/Proxy"; +const std::string kEnvNameTargetService = "TARGET_SERVICE"; + +// Define lower case string for X-Forwarded-Host. +const LowerCaseString kHeaderNameXFH("x-forwarded-host", false); + +const std::string kRequestHeaderPrefix = "requestHeader:"; +const std::string kRequestParamPrefix = "requestParameter:"; +const std::string kResponseHeaderPrefix = "responseHeader:"; + +// Define attribute names +const std::string kAttrNamePeerId = "peerId"; +const std::string kAttrNameURL = "url"; +const std::string kAttrNameHttpMethod = "httpMethod"; +const std::string kAttrNameRequestSize = "requestSize"; +const std::string kAttrNameResponseSize = "responseSize"; +const std::string kAttrNameLogMessage = "logMessage"; +const std::string kAttrNameResponseTime = "responseTime"; +const std::string kAttrNameOriginIp = "originIp"; +const std::string kAttrNameOriginHost = "originHost"; +const std::string kAttrNameTargetService = "targetService"; + +Attributes::Value StringValue(const std::string& str) { + Attributes::Value v; + v.type = Attributes::Value::STRING; + v.str_v = str; + return v; +} + +Attributes::Value Int64Value(int64_t value) { + Attributes::Value v; + v.type = Attributes::Value::INT64; + v.value.int64_v = value; + return v; +} + +void SetStringAttribute(const std::string& name, const std::string& value, + Attributes* attr) { + if (!value.empty()) { + attr->attributes[name] = StringValue(value); + } +} + +std::string GetFirstForwardedFor(const HeaderMap& header_map) { + if (!header_map.ForwardedFor()) { + return ""; + } + std::vector xff_address_list = + StringUtil::split(header_map.ForwardedFor()->value().c_str(), ','); + if (xff_address_list.empty()) { + return ""; + } + return xff_address_list.front(); +} + +std::string GetLastForwardedHost(const HeaderMap& header_map) { + const HeaderEntry* entry = header_map.get(kHeaderNameXFH); + if (entry == nullptr) { + return ""; + } + auto xff_list = StringUtil::split(entry->value().c_str(), ','); + if (xff_list.empty()) { + return ""; + } + return xff_list.back(); +} + +void FillRequestHeaderAttributes(const HeaderMap& header_map, + Attributes* attr) { + // Pass in all headers + header_map.iterate( + [](const HeaderEntry& header, void* context) { + auto attr = static_cast(context); + attr->attributes[kRequestHeaderPrefix + header.key().c_str()] = + StringValue(header.value().c_str()); + }, + attr); + + // Pass in all Query parameters. + auto path = header_map.Path(); + if (path != nullptr) { + for (const auto& it : Utility::parseQueryString(path->value().c_str())) { + attr->attributes[kRequestParamPrefix + it.first] = StringValue(it.second); + } + } + + SetStringAttribute(kAttrNameOriginIp, GetFirstForwardedFor(header_map), attr); + SetStringAttribute(kAttrNameOriginHost, GetLastForwardedHost(header_map), + attr); +} + +void FillResponseHeaderAttributes(const HeaderMap& header_map, + Attributes* attr) { + header_map.iterate( + [](const HeaderEntry& header, void* context) { + auto attr = static_cast(context); + attr->attributes[kResponseHeaderPrefix + header.key().c_str()] = + StringValue(header.value().c_str()); + }, + attr); +} + +void FillRequestInfoAttributes(const AccessLog::RequestInfo& info, + Attributes* attr) { + if (info.bytesReceived() >= 0) { + attr->attributes[kAttrNameRequestSize] = Int64Value(info.bytesReceived()); + } + if (info.bytesSent() >= 0) { + attr->attributes[kAttrNameResponseSize] = Int64Value(info.bytesSent()); + } + if (info.duration().count() >= 0) { + attr->attributes[kAttrNameResponseTime] = + Int64Value(info.duration().count()); + } +} + +} // namespace + +HttpControl::HttpControl(const std::string& mixer_server) { + ::istio::mixer_client::MixerClientOptions options; + options.mixer_server = mixer_server; + mixer_client_ = ::istio::mixer_client::CreateMixerClient(options); + + auto target_service = getenv(kEnvNameTargetService.c_str()); + if (target_service) { + target_service_ = target_service; + } +} + +void HttpControl::FillCheckAttributes(const HeaderMap& header_map, + Attributes* attr) { + FillRequestHeaderAttributes(header_map, attr); + + SetStringAttribute(kAttrNameTargetService, target_service_, attr); + attr->attributes[kAttrNamePeerId] = StringValue(kProxyPeerID); +} + +void HttpControl::Check(HttpRequestDataPtr request_data, HeaderMap& headers, + DoneFunc on_done) { + FillCheckAttributes(headers, &request_data->attributes); + log().debug("Send Check: {}", request_data->attributes.DebugString()); + mixer_client_->Check(request_data->attributes, on_done); +} + +void HttpControl::Report(HttpRequestDataPtr request_data, + const HeaderMap* response_headers, + const AccessLog::RequestInfo& request_info, + DoneFunc on_done) { + // Use all Check attributes for Report. + // Add additional Report attributes. + FillResponseHeaderAttributes(*response_headers, &request_data->attributes); + FillRequestInfoAttributes(request_info, &request_data->attributes); + log().debug("Send Report: {}", request_data->attributes.DebugString()); + mixer_client_->Report(request_data->attributes, on_done); +} + +} // namespace Mixer +} // namespace Http diff --git a/src/envoy/mixer/http_control.h b/src/envoy/mixer/http_control.h new file mode 100644 index 00000000000..71c7e5620d6 --- /dev/null +++ b/src/envoy/mixer/http_control.h @@ -0,0 +1,63 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "precompiled/precompiled.h" + +#include "common/common/logger.h" +#include "common/http/headers.h" +#include "envoy/http/access_log.h" +#include "include/client.h" + +namespace Http { +namespace Mixer { + +// Store data from Check to report +struct HttpRequestData { + ::istio::mixer_client::Attributes attributes; +}; +typedef std::shared_ptr HttpRequestDataPtr; + +// The mixer client class to control HTTP requests. +// It has Check() to validate if a request can be processed. +// At the end of request, call Report(). +class HttpControl final : public Logger::Loggable { + public: + // The constructor. + HttpControl(const std::string& mixer_server); + + // Make mixer check call. + void Check(HttpRequestDataPtr request_data, HeaderMap& headers, + ::istio::mixer_client::DoneFunc on_done); + + // Make mixer report call. + void Report(HttpRequestDataPtr request_data, + const HeaderMap* response_headers, + const AccessLog::RequestInfo& request_info, + ::istio::mixer_client::DoneFunc on_done); + + private: + void FillCheckAttributes(const HeaderMap& header_map, + ::istio::mixer_client::Attributes* attr); + + // The mixer client + std::unique_ptr<::istio::mixer_client::MixerClient> mixer_client_; + // Target service + std::string target_service_; +}; + +} // namespace Mixer +} // namespace Http diff --git a/src/envoy/mixer/http_filter.cc b/src/envoy/mixer/http_filter.cc new file mode 100644 index 00000000000..ec209b1ff98 --- /dev/null +++ b/src/envoy/mixer/http_filter.cc @@ -0,0 +1,267 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "precompiled/precompiled.h" + +#include "common/common/logger.h" +#include "common/http/headers.h" +#include "common/http/utility.h" +#include "envoy/server/instance.h" +#include "server/config/network/http_connection_manager.h" +#include "src/envoy/mixer/http_control.h" + +using ::google::protobuf::util::Status; +using StatusCode = ::google::protobuf::util::error::Code; +using ::istio::mixer_client::DoneFunc; + +namespace Http { +namespace Mixer { +namespace { + +// Define lower case string for X-Forwarded-Host. +const LowerCaseString kHeaderNameXFH("x-forwarded-host", false); + +// Convert Status::code to HTTP code +int HttpCode(int code) { + // Map Canonical codes to HTTP status codes. This is based on the mapping + // defined by the protobuf http error space. + switch (code) { + case StatusCode::OK: + return 200; + case StatusCode::CANCELLED: + return 499; + case StatusCode::UNKNOWN: + return 500; + case StatusCode::INVALID_ARGUMENT: + return 400; + case StatusCode::DEADLINE_EXCEEDED: + return 504; + case StatusCode::NOT_FOUND: + return 404; + case StatusCode::ALREADY_EXISTS: + return 409; + case StatusCode::PERMISSION_DENIED: + return 403; + case StatusCode::RESOURCE_EXHAUSTED: + return 429; + case StatusCode::FAILED_PRECONDITION: + return 400; + case StatusCode::ABORTED: + return 409; + case StatusCode::OUT_OF_RANGE: + return 400; + case StatusCode::UNIMPLEMENTED: + return 501; + case StatusCode::INTERNAL: + return 500; + case StatusCode::UNAVAILABLE: + return 503; + case StatusCode::DATA_LOSS: + return 500; + case StatusCode::UNAUTHENTICATED: + return 401; + default: + return 500; + } +} + +} // namespace + +class Config : public Logger::Loggable { + private: + std::shared_ptr http_control_; + Upstream::ClusterManager& cm_; + + public: + Config(const Json::Object& config, Server::Instance& server) + : cm_(server.clusterManager()) { + std::string mixer_server; + if (config.hasObject("mixer_server")) { + mixer_server = config.getString("mixer_server"); + } else { + log().error( + "mixer_server is required but not specified in the config: {}", + __func__); + } + + http_control_ = std::make_shared(mixer_server); + log().debug("Called Mixer::Config contructor with mixer_server: ", + mixer_server); + } + + std::shared_ptr& http_control() { return http_control_; } +}; + +typedef std::shared_ptr ConfigPtr; + +class Instance : public Http::StreamFilter, public Http::AccessLog::Instance { + private: + std::shared_ptr http_control_; + std::shared_ptr request_data_; + + enum State { NotStarted, Calling, Complete, Responded }; + State state_; + + StreamDecoderFilterCallbacks* decoder_callbacks_; + StreamEncoderFilterCallbacks* encoder_callbacks_; + + bool initiating_call_; + + public: + Instance(ConfigPtr config) + : http_control_(config->http_control()), + state_(NotStarted), + initiating_call_(false) { + Log().debug("Called Mixer::Instance : {}", __func__); + } + + // Jump thread; on_done will be called at the dispatcher thread. + DoneFunc wrapper(DoneFunc on_done) { + auto& dispatcher = decoder_callbacks_->dispatcher(); + return [&dispatcher, on_done](const Status& status) { + dispatcher.post([status, on_done]() { on_done(status); }); + }; + } + + FilterHeadersStatus decodeHeaders(HeaderMap& headers, + bool end_stream) override { + Log().debug("Called Mixer::Instance : {}", __func__); + state_ = Calling; + initiating_call_ = true; + request_data_ = std::make_shared(); + http_control_->Check( + request_data_, headers, + wrapper([this](const Status& status) { completeCheck(status); })); + initiating_call_ = false; + + if (state_ == Complete) { + return FilterHeadersStatus::Continue; + } + Log().debug("Called Mixer::Instance : {} Stop", __func__); + return FilterHeadersStatus::StopIteration; + } + + FilterDataStatus decodeData(Buffer::Instance& data, + bool end_stream) override { + Log().debug("Called Mixer::Instance : {} ({}, {})", __func__, data.length(), + end_stream); + if (state_ == Calling) { + return FilterDataStatus::StopIterationAndBuffer; + } + return FilterDataStatus::Continue; + } + + FilterTrailersStatus decodeTrailers(HeaderMap& trailers) override { + Log().debug("Called Mixer::Instance : {}", __func__); + if (state_ == Calling) { + return FilterTrailersStatus::StopIteration; + } + return FilterTrailersStatus::Continue; + } + void setDecoderFilterCallbacks( + StreamDecoderFilterCallbacks& callbacks) override { + Log().debug("Called Mixer::Instance : {}", __func__); + decoder_callbacks_ = &callbacks; + decoder_callbacks_->addResetStreamCallback( + [this]() { state_ = Responded; }); + } + void completeCheck(const Status& status) { + Log().debug("Called Mixer::Instance : check complete {}", + status.ToString()); + if (!status.ok() && state_ != Responded) { + state_ = Responded; + Utility::sendLocalReply(*decoder_callbacks_, + Code(HttpCode(status.error_code())), + status.ToString()); + return; + } + state_ = Complete; + if (!initiating_call_) { + decoder_callbacks_->continueDecoding(); + } + } + + virtual FilterHeadersStatus encodeHeaders(HeaderMap& headers, + bool end_stream) override { + Log().debug("Called Mixer::Instance : {}", __func__); + return FilterHeadersStatus::Continue; + } + virtual FilterDataStatus encodeData(Buffer::Instance& data, + bool end_stream) override { + Log().debug("Called Mixer::Instance : {}", __func__); + return FilterDataStatus::Continue; + } + virtual FilterTrailersStatus encodeTrailers(HeaderMap& trailers) override { + Log().debug("Called Mixer::Instance : {}", __func__); + return FilterTrailersStatus::Continue; + } + virtual void setEncoderFilterCallbacks( + StreamEncoderFilterCallbacks& callbacks) override { + Log().debug("Called Mixer::Instance : {}", __func__); + encoder_callbacks_ = &callbacks; + } + + virtual void log(const HeaderMap* request_headers, + const HeaderMap* response_headers, + const AccessLog::RequestInfo& request_info) override { + Log().debug("Called Mixer::Instance : {}", __func__); + // Make sure not to use any class members at the callback. + // The class may be gone when it is called. + // Log() is a static function so it is OK. + http_control_->Report(request_data_, response_headers, request_info, + [](const Status& status) { + Log().debug("Report returns status: {}", + status.ToString()); + }); + } + + static spdlog::logger& Log() { + static spdlog::logger& instance = + Logger::Registry::getLog(Logger::Id::http); + return instance; + } +}; + +} // namespace Mixer +} // namespace Http + +namespace Server { +namespace Configuration { + +class MixerConfig : public HttpFilterConfigFactory { + public: + HttpFilterFactoryCb tryCreateFilterFactory( + HttpFilterType type, const std::string& name, const Json::Object& config, + const std::string&, Server::Instance& server) override { + if (type != HttpFilterType::Both || name != "mixer") { + return nullptr; + } + + Http::Mixer::ConfigPtr mixer_config( + new Http::Mixer::Config(config, server)); + return + [mixer_config](Http::FilterChainFactoryCallbacks& callbacks) -> void { + std::shared_ptr instance( + new Http::Mixer::Instance(mixer_config)); + callbacks.addStreamFilter(Http::StreamFilterPtr(instance)); + callbacks.addAccessLogHandler(Http::AccessLog::InstancePtr(instance)); + }; + } +}; + +static RegisterHttpFilterConfigFactory register_; + +} // namespace Configuration +} // namespace server From eb8bfccc391e528fba316c4e081515eb18583e5c Mon Sep 17 00:00:00 2001 From: Wayne Zhang Date: Mon, 6 Feb 2017 12:18:42 -0800 Subject: [PATCH 18/83] Add release binary script. (#68) --- script/release-binary | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100755 script/release-binary diff --git a/script/release-binary b/script/release-binary new file mode 100755 index 00000000000..9bd7b51bf36 --- /dev/null +++ b/script/release-binary @@ -0,0 +1,37 @@ +#!/bin/bash +# +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ +# + +# The bucket name to store proxy binary +BUCKET_NAME="istio-build/proxy" + +# The proxy binary name. +BINARY_FORMAT='proxy-alpha-%H' +BINARY_NAME="$(git show -q HEAD --pretty=format:"${BINARY_FORMAT}")" + +# Build the binary +bazel build --config=release //src/envoy/mixer:envoy_esp + +SRC="bazel-bin/src/envoy/mixer/envoy_esp" +DST="gs://${BUCKET_NAME}/${BINARY_NAME}" + +# Copy it to the bucket. +echo "Copying ${SRC} to ${DST}" +gsutil cp ${SRC} ${DST} + + From 89f0fbfe5c2105793583cdd4ac2286c87961b98f Mon Sep 17 00:00:00 2001 From: Kuat Date: Mon, 6 Feb 2017 15:47:29 -0800 Subject: [PATCH 19/83] Push tar.gz to GCS (#69) * Push tar.gz to GCS * Rename envoy_esp --- script/release-binary | 10 ++++++---- src/envoy/mixer/BUILD | 12 ++++++++++-- src/envoy/mixer/README.md | 6 +++--- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/script/release-binary b/script/release-binary index 9bd7b51bf36..af90b1aa0da 100755 --- a/script/release-binary +++ b/script/release-binary @@ -16,18 +16,20 @@ # ################################################################################ # +set -ex + +# Make sure to this script on x86_64 Ubuntu Xenial # The bucket name to store proxy binary BUCKET_NAME="istio-build/proxy" # The proxy binary name. -BINARY_FORMAT='proxy-alpha-%H' +BINARY_FORMAT='envoy-alpha-%H.tar.gz' BINARY_NAME="$(git show -q HEAD --pretty=format:"${BINARY_FORMAT}")" # Build the binary -bazel build --config=release //src/envoy/mixer:envoy_esp - -SRC="bazel-bin/src/envoy/mixer/envoy_esp" +bazel build --config=release //src/envoy/mixer:envoy_tar +SRC="bazel-bin/src/envoy/mixer/envoy_tar.tar.gz" DST="gs://${BUCKET_NAME}/${BINARY_NAME}" # Copy it to the bucket. diff --git a/src/envoy/mixer/BUILD b/src/envoy/mixer/BUILD index 428c87101b9..45c96e25f96 100644 --- a/src/envoy/mixer/BUILD +++ b/src/envoy/mixer/BUILD @@ -15,7 +15,7 @@ ################################################################################ # -package(default_visibility = ["//visibility:public"]) +load("@bazel_tools//tools/build_defs/pkg:pkg.bzl", "pkg_tar") cc_library( name = "filter_lib", @@ -32,10 +32,18 @@ cc_library( ) cc_binary( - name = "envoy_esp", + name = "envoy", linkstatic = 1, deps = [ ":filter_lib", "@envoy_git//:envoy-main", ], ) + +pkg_tar( + name = "envoy_tar", + extension = "tar.gz", + files = [":envoy"], + mode = "0755", + package_dir = "/usr/local/bin/", +) diff --git a/src/envoy/mixer/README.md b/src/envoy/mixer/README.md index 94276e37826..9094d748442 100644 --- a/src/envoy/mixer/README.md +++ b/src/envoy/mixer/README.md @@ -12,10 +12,10 @@ This Proxy will use Envoy and talk to Mixer server. ## Build Envoy proxy -* Build target envoy_esp: +* Build target envoy: ``` - bazel build //src/envoy/mixer:envoy_esp + bazel build //src/envoy/mixer:envoy ``` ## How to run it @@ -38,7 +38,7 @@ This Proxy will use Envoy and talk to Mixer server. * Start Envoy proxy, run ``` - bazel-bin/src/envoy/mixer/envoy_esp -c src/envoy/prototype/envoy-mixer.conf + bazel-bin/src/envoy/mixer/envoy -c src/envoy/prototype/envoy-mixer.conf ``` * Then issue HTTP request to proxy. From 76355366e4adc6d4002d69d5aafa57e1606338ee Mon Sep 17 00:00:00 2001 From: Wayne Zhang Date: Wed, 8 Feb 2017 09:34:56 -0800 Subject: [PATCH 20/83] Remove mixer_client from api_manager. (#72) --- WORKSPACE | 10 +- contrib/endpoints/repositories.bzl | 12 - .../endpoints/src/api_manager/context/BUILD | 1 - .../api_manager/context/service_context.cc | 15 +- contrib/endpoints/src/api_manager/mixer/BUILD | 42 - .../endpoints/src/api_manager/mixer/mixer.cc | 204 ----- .../endpoints/src/api_manager/mixer/mixer.h | 77 -- .../src/api_manager/proto/server_config.proto | 9 - .../api_manager/server_config_proto_test.cc | 6 - .../BUILD => mixer/repositories.bzl} | 29 +- src/envoy/prototype/README.md | 73 -- src/envoy/prototype/api_manager_env.cc | 219 ------ src/envoy/prototype/api_manager_env.h | 33 - src/envoy/prototype/api_manager_filter.cc | 346 -------- .../prototype/dummy_api_manager_cluster.py | 91 --- src/envoy/prototype/envoy-esp.conf | 89 --- .../prototype/generic_service_config.json | 736 ------------------ src/envoy/prototype/server_config.pb.txt | 6 - 18 files changed, 22 insertions(+), 1976 deletions(-) delete mode 100644 contrib/endpoints/src/api_manager/mixer/BUILD delete mode 100644 contrib/endpoints/src/api_manager/mixer/mixer.cc delete mode 100644 contrib/endpoints/src/api_manager/mixer/mixer.h rename src/envoy/{prototype/BUILD => mixer/repositories.bzl} (57%) delete mode 100644 src/envoy/prototype/README.md delete mode 100644 src/envoy/prototype/api_manager_env.cc delete mode 100644 src/envoy/prototype/api_manager_env.h delete mode 100644 src/envoy/prototype/api_manager_filter.cc delete mode 100644 src/envoy/prototype/dummy_api_manager_cluster.py delete mode 100644 src/envoy/prototype/envoy-esp.conf delete mode 100644 src/envoy/prototype/generic_service_config.json delete mode 100644 src/envoy/prototype/server_config.pb.txt diff --git a/WORKSPACE b/WORKSPACE index 95dcdde5cdc..e152300d73d 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -31,16 +31,20 @@ googletest_repositories() load( "//contrib/endpoints:repositories.bzl", "grpc_repositories", - "mixer_client_repositories", "servicecontrol_client_repositories", ) grpc_repositories() -mixer_client_repositories() - servicecontrol_client_repositories() +load( + "//src/envoy/mixer:repositories.bzl", + "mixer_client_repositories", +) + +mixer_client_repositories() + # Workaround for Bazel > 0.4.0 since it needs newer protobuf.bzl from: # https://github.com/google/protobuf/pull/2246 # Do not use this git_repository for anything else than protobuf.bzl diff --git a/contrib/endpoints/repositories.bzl b/contrib/endpoints/repositories.bzl index b1f14aae8b5..e3363cc86d2 100644 --- a/contrib/endpoints/repositories.bzl +++ b/contrib/endpoints/repositories.bzl @@ -334,15 +334,3 @@ def servicecontrol_client_repositories(bind=True): actual = "@servicecontrol_client_git//:service_control_client_lib", ) -def mixer_client_repositories(bind=True): - native.git_repository( - name = "mixerclient_git", - commit = "80e450a5126960e8e6337c3631cf2ef984038eab", - remote = "https://github.com/istio/mixerclient.git", - ) - - if bind: - native.bind( - name = "mixer_client_lib", - actual = "@mixerclient_git//:mixer_client_lib", - ) diff --git a/contrib/endpoints/src/api_manager/context/BUILD b/contrib/endpoints/src/api_manager/context/BUILD index e158c13fc92..59365102318 100644 --- a/contrib/endpoints/src/api_manager/context/BUILD +++ b/contrib/endpoints/src/api_manager/context/BUILD @@ -39,7 +39,6 @@ cc_library( "//contrib/endpoints/src/api_manager/auth", "//contrib/endpoints/src/api_manager/auth:service_account_token", "//contrib/endpoints/src/api_manager/cloud_trace", - "//contrib/endpoints/src/api_manager/mixer", "//contrib/endpoints/src/api_manager/service_control", "//contrib/endpoints/src/api_manager/utils", "//external:cc_wkt_protos", diff --git a/contrib/endpoints/src/api_manager/context/service_context.cc b/contrib/endpoints/src/api_manager/context/service_context.cc index d8fc9dfc409..e9598a05814 100644 --- a/contrib/endpoints/src/api_manager/context/service_context.cc +++ b/contrib/endpoints/src/api_manager/context/service_context.cc @@ -15,7 +15,6 @@ //////////////////////////////////////////////////////////////////////////////// // #include "contrib/endpoints/src/api_manager/context/service_context.h" -#include "contrib/endpoints/src/api_manager/mixer/mixer.h" #include "contrib/endpoints/src/api_manager/service_control/aggregated.h" @@ -97,16 +96,10 @@ const std::string& ServiceContext::project_id() const { } std::unique_ptr ServiceContext::CreateInterface() { - if (config_->server_config() && - config_->server_config()->has_mixer_options()) { - return std::unique_ptr( - mixer::Mixer::Create(env_.get(), config_.get())); - } else { - return std::unique_ptr( - service_control::Aggregated::Create( - config_->service(), config_->server_config(), env_.get(), - &service_account_token_)); - } + return std::unique_ptr( + service_control::Aggregated::Create(config_->service(), + config_->server_config(), env_.get(), + &service_account_token_)); } std::unique_ptr diff --git a/contrib/endpoints/src/api_manager/mixer/BUILD b/contrib/endpoints/src/api_manager/mixer/BUILD deleted file mode 100644 index affe09c7ddd..00000000000 --- a/contrib/endpoints/src/api_manager/mixer/BUILD +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright 2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -################################################################################ -# -package(default_visibility = ["//contrib/endpoints/src/api_manager:__subpackages__"]) - -cc_library( - name = "mixer", - srcs = [ - "mixer.cc", - ], - hdrs = [ - "mixer.h", - ], - linkopts = select({ - "//:darwin": [], - "//conditions:default": [ - "-lm", - "-luuid", - ], - }), - deps = [ - "//contrib/endpoints/src/api_manager:impl_headers", - "//contrib/endpoints/src/api_manager/service_control", - "//contrib/endpoints/src/api_manager/utils", - "//external:grpc++", - "//external:mixer_client_lib", - "//external:protobuf", - ], -) diff --git a/contrib/endpoints/src/api_manager/mixer/mixer.cc b/contrib/endpoints/src/api_manager/mixer/mixer.cc deleted file mode 100644 index 53837edaf67..00000000000 --- a/contrib/endpoints/src/api_manager/mixer/mixer.cc +++ /dev/null @@ -1,204 +0,0 @@ -/* Copyright 2016 Google Inc. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "contrib/endpoints/src/api_manager/mixer/mixer.h" - -using ::google::api_manager::utils::Status; -using ::istio::mixer_client::Attributes; - -namespace google { -namespace api_manager { -namespace mixer { -namespace { - -const std::string kProxyPeerID = "Istio/Proxy"; -const std::string kEnvNameTargetService = "TARGET_SERVICE"; - -const std::string kAttrNameServiceName = "serviceName"; -const std::string kAttrNamePeerId = "peerId"; -const std::string kAttrNameOperationId = "operationId"; -const std::string kAttrNameOperationName = "operationName"; -const std::string kAttrNameApiKey = "apiKey"; -const std::string kAttrNameResponseCode = "responseCode"; -const std::string kAttrNameURL = "url"; -const std::string kAttrNameLocation = "location"; -const std::string kAttrNameApiName = "apiName"; -const std::string kAttrNameApiVersion = "apiVersion"; -const std::string kAttrNameApiMethod = "apiMethod"; -const std::string kAttrNameRequestSize = "requestSize"; -const std::string kAttrNameResponseSize = "responseSize"; -const std::string kAttrNameLogMessage = "logMessage"; -const std::string kAttrNameResponseTime = "responseTime"; -const std::string kAttrNameOriginIp = "originIp"; -const std::string kAttrNameOriginHost = "originHost"; -const std::string kAttrNameTargetService = "targetService"; - -Attributes::Value StringValue(const std::string& str) { - Attributes::Value v; - v.type = Attributes::Value::STRING; - v.str_v = str; - return v; -} - -Attributes::Value Int64Value(int64_t value) { - Attributes::Value v; - v.type = Attributes::Value::INT64; - v.value.int64_v = value; - return v; -} - -} // namespace - -Mixer::Mixer(ApiManagerEnvInterface* env, const Config* config) - : env_(env), config_(config) {} - -Mixer::~Mixer() {} - -Status Mixer::Init() { - ::istio::mixer_client::MixerClientOptions options; - options.mixer_server = - config_->server_config()->mixer_options().mixer_server(); - mixer_client_ = ::istio::mixer_client::CreateMixerClient(options); - auto target_service = getenv(kEnvNameTargetService.c_str()); - if (target_service) { - target_service_ = target_service; - } - return Status::OK; -} - -Status Mixer::Close() { return Status::OK; } - -void Mixer::FillCommonAttributes(const service_control::OperationInfo& info, - ::istio::mixer_client::Attributes* attr) { - attr->attributes[kAttrNameServiceName] = StringValue(config_->service_name()); - attr->attributes[kAttrNamePeerId] = StringValue(kProxyPeerID); - - if (!info.operation_id.empty()) { - attr->attributes[kAttrNameOperationId] = StringValue(info.operation_id); - } - if (!info.operation_name.empty()) { - attr->attributes[kAttrNameOperationName] = StringValue(info.operation_name); - } - if (!info.api_key.empty()) { - attr->attributes[kAttrNameApiKey] = StringValue(info.api_key); - } - if (!info.client_ip.empty()) { - attr->attributes[kAttrNameOriginIp] = StringValue(info.client_ip); - } - if (!info.client_host.empty()) { - attr->attributes[kAttrNameOriginHost] = StringValue(info.client_host); - } - if (!target_service_.empty()) { - attr->attributes[kAttrNameTargetService] = StringValue(target_service_); - } -} - -void Mixer::FillCheckAttributes(const service_control::CheckRequestInfo& info, - ::istio::mixer_client::Attributes* attr) { - FillCommonAttributes(info, attr); -} - -void Mixer::FillReportAttributes(const service_control::ReportRequestInfo& info, - ::istio::mixer_client::Attributes* attr) { - FillCommonAttributes(info, attr); - - if (!info.url.empty()) { - attr->attributes[kAttrNameURL] = StringValue(info.url); - } - if (!info.location.empty()) { - attr->attributes[kAttrNameLocation] = StringValue(info.location); - } - - if (!info.api_name.empty()) { - attr->attributes[kAttrNameApiName] = StringValue(info.api_name); - } - if (!info.api_version.empty()) { - attr->attributes[kAttrNameApiVersion] = StringValue(info.api_version); - } - if (!info.api_method.empty()) { - attr->attributes[kAttrNameApiMethod] = StringValue(info.api_method); - } - - if (!info.log_message.empty()) { - attr->attributes[kAttrNameLogMessage] = StringValue(info.log_message); - } - - attr->attributes[kAttrNameResponseCode] = Int64Value(info.response_code); - if (info.request_size >= 0) { - attr->attributes[kAttrNameRequestSize] = Int64Value(info.request_size); - } - if (info.response_size >= 0) { - attr->attributes[kAttrNameResponseSize] = Int64Value(info.response_size); - } - - if (info.latency.request_time_ms >= 0) { - attr->attributes[kAttrNameResponseTime] = - Int64Value(info.latency.request_time_ms); - } -} - -Status Mixer::Report(const service_control::ReportRequestInfo& info) { - ::istio::mixer_client::Attributes attributes; - FillReportAttributes(info, &attributes); - env_->LogInfo("Send Report: "); - env_->LogInfo(attributes.DebugString()); - mixer_client_->Report( - attributes, [this](const ::google::protobuf::util::Status& status) { - if (status.ok()) { - env_->LogInfo("Report response: OK"); - } else { - env_->LogError(std::string("Failed to call Mixer::report, Error: ") + - status.ToString()); - } - }); - return Status::OK; -} - -void Mixer::Check( - const service_control::CheckRequestInfo& info, - cloud_trace::CloudTraceSpan* parent_span, - std::function - on_done) { - ::istio::mixer_client::Attributes attributes; - FillCheckAttributes(info, &attributes); - env_->LogInfo("Send Check: "); - env_->LogInfo(attributes.DebugString()); - mixer_client_->Check( - attributes, - [this, on_done](const ::google::protobuf::util::Status& status) { - if (status.ok()) { - env_->LogInfo("Check response: OK"); - } else { - env_->LogError(std::string("Failed to call Mixer::check, Error: ") + - status.ToString()); - } - service_control::CheckResponseInfo info; - on_done(Status(status.error_code(), status.error_message(), - Status::SERVICE_CONTROL), - info); - }); -} - -Status Mixer::GetStatistics(service_control::Statistics* esp_stat) const { - return Status::OK; -} - -service_control::Interface* Mixer::Create(ApiManagerEnvInterface* env, - const Config* config) { - return new Mixer(env, config); -} - -} // namespace mixer -} // namespace api_manager -} // namespace google diff --git a/contrib/endpoints/src/api_manager/mixer/mixer.h b/contrib/endpoints/src/api_manager/mixer/mixer.h deleted file mode 100644 index 81276e1227d..00000000000 --- a/contrib/endpoints/src/api_manager/mixer/mixer.h +++ /dev/null @@ -1,77 +0,0 @@ -/* Copyright 2016 Google Inc. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef API_MANAGER_MIXER_MIXER_H_ -#define API_MANAGER_MIXER_MIXER_H_ - -#include "contrib/endpoints/include/api_manager/env_interface.h" -#include "contrib/endpoints/src/api_manager/config.h" -#include "contrib/endpoints/src/api_manager/service_control/interface.h" -#include "include/client.h" - -namespace google { -namespace api_manager { -namespace mixer { - -// This implementation uses service-control-client-cxx module. -class Mixer : public service_control::Interface { - public: - static service_control::Interface* Create(ApiManagerEnvInterface* env, - const Config* config); - - virtual ~Mixer(); - - virtual utils::Status Report(const service_control::ReportRequestInfo& info); - - virtual void Check( - const service_control::CheckRequestInfo& info, - cloud_trace::CloudTraceSpan* parent_span, - std::function - on_done); - - virtual utils::Status Init(); - virtual utils::Status Close(); - - virtual utils::Status GetStatistics(service_control::Statistics* stat) const; - - private: - // The constructor. - Mixer(ApiManagerEnvInterface* env, const Config* config); - - // Fill common attributes for both check and report. - void FillCommonAttributes(const service_control::OperationInfo& info, - ::istio::mixer_client::Attributes* attr); - // Fill attributes for check. - void FillCheckAttributes(const service_control::CheckRequestInfo& info, - ::istio::mixer_client::Attributes* attr); - // Fill attributes for report. - void FillReportAttributes(const service_control::ReportRequestInfo& info, - ::istio::mixer_client::Attributes* attr); - - // The Api Manager environment interface. - ApiManagerEnvInterface* env_; - // The config. - const Config* config_; - // The mixer client - std::unique_ptr<::istio::mixer_client::MixerClient> mixer_client_; - // Target service - std::string target_service_; -}; - -} // namespace mixer -} // namespace api_manager -} // namespace google - -#endif // API_MANAGER_MIXER_MIXER_H_ diff --git a/contrib/endpoints/src/api_manager/proto/server_config.proto b/contrib/endpoints/src/api_manager/proto/server_config.proto index 343414aceae..1fe04af6d07 100644 --- a/contrib/endpoints/src/api_manager/proto/server_config.proto +++ b/contrib/endpoints/src/api_manager/proto/server_config.proto @@ -35,10 +35,6 @@ message ServerConfig { // Server config used for API authentication ApiAuthenticationConfig api_authentication_config = 5; - // Mixer option flag. If not present, default to use service_control. When - // Envoy/esp talks to Mixer, has to specify this field. - MixerOptions mixer_options = 6; - // Experimental flags Experimental experimental = 999; } @@ -143,11 +139,6 @@ message ApiAuthenticationConfig { bool force_disable = 1; } -message MixerOptions { - // For envoy, it is the cluster name for mixer server. - string mixer_server = 1; -} - message Experimental { // Disable timed printouts of ESP status to the error log. bool disable_log_status = 1; diff --git a/contrib/endpoints/src/api_manager/server_config_proto_test.cc b/contrib/endpoints/src/api_manager/server_config_proto_test.cc index b39309727f1..80c62e05471 100644 --- a/contrib/endpoints/src/api_manager/server_config_proto_test.cc +++ b/contrib/endpoints/src/api_manager/server_config_proto_test.cc @@ -76,9 +76,6 @@ experimental { disable_log_status: false } -mixer_options { - mixer_server: "mixer_server" -} )"; TEST(ServerConfigProto, ServerConfigFromString) { @@ -125,9 +122,6 @@ TEST(ServerConfigProto, ServerConfigFromString) { // Check disable_log_status EXPECT_EQ(false, server_config.experimental().disable_log_status()); - - // Check mixer options - EXPECT_EQ("mixer_server", server_config.mixer_options().mixer_server()); } TEST(ServerConfigProto, ValidateSampleServerConfig) { diff --git a/src/envoy/prototype/BUILD b/src/envoy/mixer/repositories.bzl similarity index 57% rename from src/envoy/prototype/BUILD rename to src/envoy/mixer/repositories.bzl index 258f2046cd7..d9888fb0daf 100644 --- a/src/envoy/prototype/BUILD +++ b/src/envoy/mixer/repositories.bzl @@ -15,22 +15,15 @@ ################################################################################ # -package(default_visibility = ["//visibility:public"]) +def mixer_client_repositories(bind=True): + native.git_repository( + name = "mixerclient_git", + commit = "80e450a5126960e8e6337c3631cf2ef984038eab", + remote = "https://github.com/istio/mixerclient.git", + ) -cc_binary( - name = "envoy_esp", - srcs = [ - "api_manager_filter.cc", - "api_manager_env.cc", - "api_manager_env.h", - ], - deps = [ - "//external:protobuf", - "//contrib/endpoints/include:api_manager", - "//contrib/endpoints/src/grpc/transcoding:transcoding", - "//external:servicecontrol", - "@envoy_git//:envoy-common", - "@envoy_git//:envoy-main" - ], - linkstatic=1, -) + if bind: + native.bind( + name = "mixer_client_lib", + actual = "@mixerclient_git//:mixer_client_lib", + ) diff --git a/src/envoy/prototype/README.md b/src/envoy/prototype/README.md deleted file mode 100644 index dc73f63f6c9..00000000000 --- a/src/envoy/prototype/README.md +++ /dev/null @@ -1,73 +0,0 @@ - -This Proxy will use Envoy and talk to Mixer server. - -## Install dependencies - -``` - apt-get install uuid-dev -``` - -## Build Mixer server - -* Follow https://github.com/istio/mixer/blob/master/doc/devel/development.md to set up environment, and build via: - -``` - cd $(ISTIO)/mixer - bazel build ...:all -``` - -## Build Envoy proxy - -* Build target envoy_esp: - -``` - bazel build //src/envoy/prototype:envoy_esp -``` - -## How to run it - -* Start mixer server. In mixer folder run: - -``` - bazel-bin/cmd/server/mixs server -``` - - The server will run at port 9091 - -* Start backend Echo server. - -``` - cd test/backend/echo - go run echo.go -``` - -* Modify your iptables: - -``` - sudo iptables -t nat -A OUTPUT -p tcp --dport 9090 -j REDIRECT --to-port 9092 -``` - -Once you are done, you should remove this rule: - -``` - sudo iptables -t nat -D OUTPUT -p tcp --dport 9090 -j REDIRECT --to-port 9092 -``` - - -* Start Envoy proxy, run - -``` - bazel-bin/src/envoy/prototype/envoy_esp -c src/envoy/prototype/envoy-esp.conf -``` - -* Then issue HTTP request to proxy. - -``` - curl http://localhost:9090/echo?key=API-KEY -d "hello world" -``` - -## How to add attributes or facts - -Now only some of attributes are passed to mixer. If you want to add more attributes, you can -modify this -[file](https://github.com/istio/proxy/blob/master/contrib/endpoints/src/api_manager/mixer/mixer.cc). diff --git a/src/envoy/prototype/api_manager_env.cc b/src/envoy/prototype/api_manager_env.cc deleted file mode 100644 index f8536bcc10a..00000000000 --- a/src/envoy/prototype/api_manager_env.cc +++ /dev/null @@ -1,219 +0,0 @@ -#include "api_manager_env.h" - -#include "common/http/headers.h" -#include "common/http/message_impl.h" -#include "envoy/event/timer.h" -#include "google/protobuf/stubs/status.h" -#include "source/common/grpc/common.h" - -using ::google::api_manager::utils::Status; -using ::google::protobuf::util::error::Code; - -namespace Http { -namespace ApiManager { - -void Http::ApiManager::Env::Log(LogLevel level, const char *message) { - switch (level) { - case LogLevel::DEBUG: - log().debug("{}", message); - break; - case LogLevel::INFO: - log().info("{}", message); - break; - case LogLevel::WARNING: - log().warn("{}", message); - break; - case LogLevel::ERROR: - log().error("{}", message); - break; - } -} - -class PeriodicTimer : public google::api_manager::PeriodicTimer, - public Logger::Loggable { - private: - Server::Instance &server_; - Event::TimerPtr timer_; - - public: - PeriodicTimer(Server::Instance &server) : server_(server) {} - ~PeriodicTimer() { Stop(); } - void Stop() { - if (timer_) { - timer_->disableTimer(); - timer_ = nullptr; - } - } - void Schedule(std::chrono::milliseconds interval, - std::function continuation) { - Stop(); - timer_ = server_.dispatcher().createTimer([this, continuation, interval]() { - continuation(); - Schedule(interval, continuation); - }); - timer_->enableTimer(interval); - } -}; - -std::unique_ptr Env::StartPeriodicTimer( - std::chrono::milliseconds interval, std::function continuation) { - log().debug("start periodic timer"); - auto single = new PeriodicTimer(server); - single->Schedule(interval, continuation); - std::unique_ptr timer(single); - return timer; -} - -static const LowerCaseString kApiManagerUrl("x-api-manager-url"); -static const LowerCaseString kGrpcTEKey("te"); -static const std::string kGrpcTEValue("trailers"); - -class HTTPRequest : public Http::Message { - private: - HeaderMapImpl header_map_; - Buffer::OwnedImpl body_; - - public: - HTTPRequest(google::api_manager::HTTPRequest *request) - : body_(request->body()) { - header_map_.addStaticKey(Headers::get().Method, request->method()); - - size_t path_pos = request->url().find('/', 8); - if (path_pos == std::string::npos) { - header_map_.addStaticKey(Headers::get().Path, "/"); - } else { - header_map_.addStaticKey(Headers::get().Path, - request->url().substr(path_pos)); - } - - header_map_.addStaticKey(Headers::get().Scheme, "http"); - header_map_.addStaticKey(Headers::get().Host, "localhost"); - header_map_.addStaticKey(Headers::get().ContentLength, body_.length()); - header_map_.addStaticKey(kApiManagerUrl, request->url()); - for (const auto header : request->request_headers()) { - LowerCaseString lower_key(header.first); - HeaderString key, value; - key.setCopy(lower_key.get().data(), lower_key.get().size()); - value.setCopy(header.second.data(), header.second.size()); - header_map_.addViaMove(std::move(key), std::move(value)); - } - } - virtual HeaderMap &headers() override { return header_map_; } - virtual Buffer::Instance *body() override { return &body_; } - virtual void body(Buffer::InstancePtr &&body) override {} - virtual HeaderMap *trailers() override { return nullptr; } - virtual void trailers(HeaderMapPtr &&trailers) override {} - virtual std::string bodyAsString() const override { return ""; } -}; - -class HTTPRequestCallbacks : public AsyncClient::Callbacks { - private: - std::unique_ptr request_; - std::unique_ptr sent_request_; - - public: - HTTPRequestCallbacks( - std::unique_ptr &&request) - : request_(std::move(request)) {} - virtual void onSuccess(MessagePtr &&response) override { - google::api_manager::utils::Status status( - std::stoi(response->headers().Status()->value().c_str()), ""); - std::map headers; - response->headers().iterate( - [&](const HeaderEntry &header, void *) -> void { - // TODO: fix it - // headers.emplace(header.key().c_str(), header.value().c_str()); - }, - nullptr); - request_->OnComplete(status, std::move(headers), response->bodyAsString()); - delete this; - } - virtual void onFailure(AsyncClient::FailureReason reason) override { - google::api_manager::utils::Status status(-1, - "Cannot connect to HTTP server."); - std::map headers; - request_->OnComplete(status, std::move(headers), ""); - delete this; - } -}; - -namespace { -// Copy the code here from envoy/grpc/common.cc -Buffer::InstancePtr SerializeGrpcBody(const std::string &body_str) { - // http://www.grpc.io/docs/guides/wire.html - Buffer::InstancePtr body(new Buffer::OwnedImpl()); - uint8_t compressed = 0; - body->add(&compressed, sizeof(compressed)); - uint32_t size = htonl(body_str.size()); - body->add(&size, sizeof(size)); - body->add(body_str); - return body; -} -Http::MessagePtr PrepareGrpcHeaders(const std::string &upstream_cluster, - const std::string &service_full_name, - const std::string &method_name) { - Http::MessagePtr message(new Http::RequestMessageImpl()); - message->headers().insertMethod().value( - Http::Headers::get().MethodValues.Post); - message->headers().insertPath().value( - fmt::format("/{}/{}", service_full_name, method_name)); - message->headers().insertHost().value(upstream_cluster); - message->headers().insertContentType().value(Grpc::Common::GRPC_CONTENT_TYPE); - message->headers().addStatic(kGrpcTEKey, kGrpcTEValue); - return message; -} -} // annoymous namespace - -class GrpcRequestCallbacks : public AsyncClient::Callbacks { - private: - Env *env_; - std::unique_ptr request_; - - public: - GrpcRequestCallbacks( - Env *env, std::unique_ptr &&request) - : env_(env), request_(std::move(request)) {} - virtual void onSuccess(MessagePtr &&response) override { - google::api_manager::utils::Status status( - std::stoi(response->headers().Status()->value().c_str()), ""); - Grpc::Common::validateResponse(*response); - env_->LogInfo("pass validate"); - // remove 5 bytes of grpc header - response->body()->drain(5); - request_->OnComplete(status, response->bodyAsString()); - delete this; - } - virtual void onFailure(AsyncClient::FailureReason reason) override { - google::api_manager::utils::Status status(-1, - "Cannot connect to gRPC server."); - request_->OnComplete(status, ""); - delete this; - } -}; - -void Env::RunHTTPRequest( - std::unique_ptr request) { - auto &client = cm_.httpAsyncClientForCluster("api_manager"); - - MessagePtr message{new HTTPRequest(request.get())}; - HTTPRequestCallbacks *callbacks = - new HTTPRequestCallbacks(std::move(request)); - client.send( - std::move(message), *callbacks, - Optional(std::chrono::milliseconds(10000))); -} - -void Env::RunGRPCRequest( - std::unique_ptr request) { - auto &client = cm_.httpAsyncClientForCluster(request->server()); - - Http::MessagePtr message = - PrepareGrpcHeaders("localhost", request->service(), request->method()); - message->body(SerializeGrpcBody(request->body())); - auto callbacks = new GrpcRequestCallbacks(this, std::move(request)); - client.send( - std::move(message), *callbacks, - Optional(std::chrono::milliseconds(10000))); -} -} -} diff --git a/src/envoy/prototype/api_manager_env.h b/src/envoy/prototype/api_manager_env.h deleted file mode 100644 index 0ae8136f5be..00000000000 --- a/src/envoy/prototype/api_manager_env.h +++ /dev/null @@ -1,33 +0,0 @@ -#pragma once - -#include "precompiled/precompiled.h" - -#include "common/common/logger.h" -#include "contrib/endpoints/include/api_manager/env_interface.h" -#include "envoy/upstream/cluster_manager.h" -#include "server/server.h" - -namespace Http { -namespace ApiManager { - -class Env : public google::api_manager::ApiManagerEnvInterface, - public Logger::Loggable { - private: - Server::Instance& server; - Upstream::ClusterManager& cm_; - - public: - Env(Server::Instance& server) - : server(server), cm_(server.clusterManager()){}; - - virtual void Log(LogLevel level, const char* message) override; - virtual std::unique_ptr - StartPeriodicTimer(std::chrono::milliseconds interval, - std::function continuation) override; - virtual void RunHTTPRequest( - std::unique_ptr request) override; - virtual void RunGRPCRequest( - std::unique_ptr request) override; -}; -} -} diff --git a/src/envoy/prototype/api_manager_filter.cc b/src/envoy/prototype/api_manager_filter.cc deleted file mode 100644 index 3983a4d8b41..00000000000 --- a/src/envoy/prototype/api_manager_filter.cc +++ /dev/null @@ -1,346 +0,0 @@ -#include "precompiled/precompiled.h" - -#include "api_manager_env.h" -#include "common/common/logger.h" -#include "common/grpc/common.h" -#include "common/http/filter/ratelimit.h" -#include "common/http/headers.h" -#include "common/http/utility.h" -#include "contrib/endpoints/include/api_manager/api_manager.h" -#include "contrib/endpoints/src/grpc/transcoding/transcoder.h" -#include "envoy/server/instance.h" -#include "server/config/network/http_connection_manager.h" - -namespace Http { -namespace ApiManager { -namespace { - -// Define lower case string for X-Forwarded-Host. -const LowerCaseString kHeaderNameXFH("x-forwarded-host", false); - -} // namespace - -std::string ReadFile(const std::string& file_name) { - std::ifstream t(file_name); - std::string content((std::istreambuf_iterator(t)), - std::istreambuf_iterator()); - return content; -} - -class Config : public Logger::Loggable { - private: - google::api_manager::ApiManagerFactory api_manager_factory_; - std::shared_ptr api_manager_; - Upstream::ClusterManager& cm_; - - public: - Config(const Json::Object& config, Server::Instance& server) - : cm_(server.clusterManager()) { - std::string service_config_content; - if (config.hasObject("service_config")) { - const std::string service_config = config.getString("service_config"); - service_config_content = ReadFile(service_config); - } else { - log().error( - "Service_config is required but not specified in the config: {}", - __func__); - } - - std::string server_config_content; - if (config.hasObject("server_config")) { - const std::string server_config = config.getString("server_config"); - server_config_content = ReadFile(server_config); - } - std::unique_ptr env( - new Env(server)); - - api_manager_ = api_manager_factory_.GetOrCreateApiManager( - std::move(env), service_config_content, server_config_content); - - api_manager_->Init(); - log().debug("Called ApiManager::Config constructor: {}", __func__); - } - - std::shared_ptr& api_manager() { - return api_manager_; - } -}; - -typedef std::shared_ptr ConfigPtr; - -class Request : public google::api_manager::Request { - private: - HeaderMap& header_map_; - std::string downstream_address_; - std::string virtual_host_; - bool query_parsed_; - std::map query_params_; - - public: - Request(HeaderMap& header_map, const std::string& downstream_address, - const std::string& virtual_host) - : header_map_(header_map), - downstream_address_(downstream_address), - virtual_host_(virtual_host), - query_parsed_(false) {} - virtual std::string GetRequestHTTPMethod() override { - return header_map_.Method()->value().c_str(); - } - virtual std::string GetRequestPath() override { - return header_map_.Path()->value().c_str(); - } - virtual std::string GetUnparsedRequestPath() override { - return header_map_.Path()->value().c_str(); - } - - virtual std::string GetClientIP() override { - if (!header_map_.ForwardedFor()) { - return downstream_address_; - } - std::vector xff_address_list = - StringUtil::split(header_map_.ForwardedFor()->value().c_str(), ','); - if (xff_address_list.empty()) { - return downstream_address_; - } - return xff_address_list.front(); - } - - virtual std::string GetClientHost() override { - const HeaderEntry* entry = header_map_.get(kHeaderNameXFH); - if (entry == nullptr) { - return virtual_host_; - } - auto xff_list = StringUtil::split(entry->value().c_str(), ','); - if (xff_list.empty()) { - return virtual_host_; - } - return xff_list.back(); - } - - virtual bool FindQuery(const std::string& name, std::string* query) override { - if (!query_parsed_) { - auto header = header_map_.Path(); - if (header != nullptr) { - std::string path = header->value().c_str(); - Utility::parseQueryString(path).swap(query_params_); - } - query_parsed_ = true; - } - auto entry = query_params_.find(name); - if (entry == query_params_.end()) { - return false; - } - *query = entry->second; - return true; - } - - virtual bool FindHeader(const std::string& name, - std::string* header) override { - LowerCaseString key(name); - const HeaderEntry* entry = header_map_.get(key); - if (entry == nullptr) { - return false; - } - *header = entry->value().c_str(); - return true; - } - - virtual google::api_manager::protocol::Protocol GetRequestProtocol() - override { - return google::api_manager::protocol::Protocol::HTTP; - } - virtual google::api_manager::utils::Status AddHeaderToBackend( - const std::string& key, const std::string& value) override { - return google::api_manager::utils::Status::OK; - } - virtual void SetAuthToken(const std::string& auth_token) override {} - - virtual int64_t GetGrpcRequestBytes() { return 0; } - virtual int64_t GetGrpcResponseBytes() { return 0; } - virtual int64_t GetGrpcRequestMessageCounts() { return 0; } - virtual int64_t GetGrpcResponseMessageCounts() { return 0; } - virtual std::string GetQueryParameters() { return ""; } -}; - -class Response : public google::api_manager::Response { - const AccessLog::RequestInfo& request_info_; - - public: - Response(const AccessLog::RequestInfo& request_info) - : request_info_(request_info) {} - - google::api_manager::utils::Status GetResponseStatus() { - return google::api_manager::utils::Status::OK; - } - - std::size_t GetRequestSize() { return request_info_.bytesReceived(); } - - std::size_t GetResponseSize() { return request_info_.bytesSent(); } - - google::api_manager::utils::Status GetLatencyInfo( - google::api_manager::service_control::LatencyInfo* info) { - info->request_time_ms = request_info_.duration().count(); - return google::api_manager::utils::Status::OK; - } -}; - -const Http::HeaderMapImpl BadRequest{{Http::Headers::get().Status, "400"}}; - -class Instance : public Http::StreamFilter, public Http::AccessLog::Instance { - private: - std::shared_ptr api_manager_; - std::unique_ptr - request_handler_; - - enum State { NotStarted, Calling, Complete, Responded }; - State state_; - - StreamDecoderFilterCallbacks* decoder_callbacks_; - StreamEncoderFilterCallbacks* encoder_callbacks_; - - bool initiating_call_; - - std::string getRouteVirtualHost(HeaderMap& headers) const { - const Router::Route* route = decoder_callbacks_->route(); - if (route && route->routeEntry()) { - return route->routeEntry()->virtualHost().name(); - } - return ""; - } - - public: - Instance(ConfigPtr config) - : api_manager_(config->api_manager()), - state_(NotStarted), - initiating_call_(false) { - Log().debug("Called ApiManager::Instance : {}", __func__); - } - - FilterHeadersStatus decodeHeaders(HeaderMap& headers, - bool end_stream) override { - Log().debug("Called ApiManager::Instance : {}", __func__); - std::unique_ptr request( - new Request(headers, decoder_callbacks_->downstreamAddress(), - getRouteVirtualHost(headers))); - request_handler_ = api_manager_->CreateRequestHandler(std::move(request)); - state_ = Calling; - initiating_call_ = true; - request_handler_->Check([this](google::api_manager::utils::Status status) { - completeCheck(status); - }); - initiating_call_ = false; - - if (state_ == Complete) { - return FilterHeadersStatus::Continue; - } - Log().debug("Called ApiManager::Instance : {} Stop", __func__); - return FilterHeadersStatus::StopIteration; - } - - FilterDataStatus decodeData(Buffer::Instance& data, - bool end_stream) override { - Log().debug("Called ApiManager::Instance : {} ({}, {})", __func__, - data.length(), end_stream); - if (state_ == Calling) { - return FilterDataStatus::StopIterationAndBuffer; - } - return FilterDataStatus::Continue; - } - - FilterTrailersStatus decodeTrailers(HeaderMap& trailers) override { - Log().debug("Called ApiManager::Instance : {}", __func__); - if (state_ == Calling) { - return FilterTrailersStatus::StopIteration; - } - return FilterTrailersStatus::Continue; - } - void setDecoderFilterCallbacks( - StreamDecoderFilterCallbacks& callbacks) override { - Log().debug("Called ApiManager::Instance : {}", __func__); - decoder_callbacks_ = &callbacks; - decoder_callbacks_->addResetStreamCallback( - [this]() { state_ = Responded; }); - } - void completeCheck(const google::api_manager::utils::Status& status) { - Log().debug("Called ApiManager::Instance : check complete {}", - status.ToJson()); - if (!status.ok() && state_ != Responded) { - state_ = Responded; - decoder_callbacks_->dispatcher().post([this, status]() { - Utility::sendLocalReply(*decoder_callbacks_, Code(status.HttpCode()), - status.ToJson()); - }); - return; - } - state_ = Complete; - if (!initiating_call_) { - decoder_callbacks_->dispatcher().post( - [this]() { decoder_callbacks_->continueDecoding(); }); - } - } - - virtual FilterHeadersStatus encodeHeaders(HeaderMap& headers, - bool end_stream) override { - Log().debug("Called ApiManager::Instance : {}", __func__); - return FilterHeadersStatus::Continue; - } - virtual FilterDataStatus encodeData(Buffer::Instance& data, - bool end_stream) override { - Log().debug("Called ApiManager::Instance : {}", __func__); - return FilterDataStatus::Continue; - } - virtual FilterTrailersStatus encodeTrailers(HeaderMap& trailers) override { - Log().debug("Called ApiManager::Instance : {}", __func__); - return FilterTrailersStatus::Continue; - } - virtual void setEncoderFilterCallbacks( - StreamEncoderFilterCallbacks& callbacks) override { - Log().debug("Called ApiManager::Instance : {}", __func__); - encoder_callbacks_ = &callbacks; - } - - virtual void log(const HeaderMap* request_headers, - const HeaderMap* response_headers, - const AccessLog::RequestInfo& request_info) override { - Log().debug("Called ApiManager::Instance : {}", __func__); - std::unique_ptr response( - new Response(request_info)); - request_handler_->Report(std::move(response), []() {}); - } - - spdlog::logger& Log() { - static spdlog::logger& instance = - Logger::Registry::getLog(Logger::Id::http); - return instance; - } -}; -} -} - -namespace Server { -namespace Configuration { - -class ApiManagerConfig : public HttpFilterConfigFactory { - public: - HttpFilterFactoryCb tryCreateFilterFactory( - HttpFilterType type, const std::string& name, const Json::Object& config, - const std::string&, Server::Instance& server) override { - if (type != HttpFilterType::Both || name != "esp") { - return nullptr; - } - - Http::ApiManager::ConfigPtr api_manager_config( - new Http::ApiManager::Config(config, server)); - return [api_manager_config]( - Http::FilterChainFactoryCallbacks& callbacks) -> void { - std::shared_ptr instance( - new Http::ApiManager::Instance(api_manager_config)); - callbacks.addStreamFilter(Http::StreamFilterPtr(instance)); - callbacks.addAccessLogHandler(Http::AccessLog::InstancePtr(instance)); - }; - } -}; - -static RegisterHttpFilterConfigFactory register_; -} -} diff --git a/src/envoy/prototype/dummy_api_manager_cluster.py b/src/envoy/prototype/dummy_api_manager_cluster.py deleted file mode 100644 index 3f83476aae8..00000000000 --- a/src/envoy/prototype/dummy_api_manager_cluster.py +++ /dev/null @@ -1,91 +0,0 @@ -from BaseHTTPServer import BaseHTTPRequestHandler -from BaseHTTPServer import HTTPServer -from subprocess import Popen, PIPE, STDOUT - -PUBLIC_KEY = '''{ - "keys": [ - { - "kty": "RSA", - "alg": "RS256", - "use": "sig", - "kid": "686f442dead739e241342519849bd0d140707a86", - "n": "yVGRzBnVmCHC9uMH_mCna7FiSKOjMEsDdgf8hds41KYfUHH9cp4P41iQEBnqPYvQiMYaZDuatkMDh25ukmkSyfLwJIVQpnHgMDwoZ1cx83Z0PwC8wAAJYqKyItKNfFN4UJ6DZIOVU-Iqgxi8VOtGwMNx2LiD1NoFVfXyz52UJ_QLiUGzErVwTGv4UD6NtaWKFkctTnEG-9rZvDF8QknnzxAomVa2OcV8OHeszx6N8UE1tm9Kq4xj2Uu8D3dDrfu2jr45Pi6RHIZOTAnu8G7wTDNaiGbENrbHSk6WAjLZBOcWZj9SDlDlwH2xFoKdpNKRmLKpPQblHem_y1KRYwvJTw", - "e": "AQAB" - }, - { - "kty": "RSA", - "alg": "RS256", - "use": "sig", - "kid": "d57d66bfbff089769e3545589572d3b53955a8cb", - "n": "zX99YCqicbn_j-5YTJ-2FsgONUg7cmqJiwvHXrUopRvn2Tukrd0B5Sg-Rq1hdZTYgIym7lSMw_zLxIQCH54sUfydX5MMWr6FOxVUbYl-E0Oko85Yer9dFv61rN0USj_A12QRMmjCZkcqH_6MWWuA1QWaejyStopjpLEYnUD3bP6oS604eZWkkOp8Nu-Vg4NqqX7ZClIcQqe03xv3sFHiPuhB-qaifhpIPpKCiYSKxAY4_GxwCJ_ml_uJ5k1tIrDykAlie6aWxv8hogOXrQmNRCO2Qcumwb7d9cci1UxsEYOtpZxhTiZqWsbrBxwfvLqU_rvsCT8vOjrBPOkTtYl22w", - "e": "AQAB" - }, - { - "kty": "RSA", - "alg": "RS256", - "use": "sig", - "kid": "516ca9a83b130d4b605684fe7df83b43f33fba43", - "n": "rSLxCvtRLoJUt2xPTSjDgN8_7kuBLsZz0JD4HRiwvId_IYY-T5MsUirBx01AtI3KRlmeQ-1QIvET4cNN99QL8U6e7Hh75VWJvQhh0_TFdKbv_75b81-rzx-EvIoVe0czeY3QeRp3DDORRk9o7xgri6U3VfT2zcQIDAPY14h3wLHE5DQuH9EpbxjS_wNrQxRYZH_mWgmWU7h001WHWRDKdEIgtzQkinKOTig90Wy4pt_vgMGcv0mh_wYvIcmB62Qj1sHUUOa5NJWUaZLC4Stj7CH0dLiBrVqg3JFqvO60Oo3wrcdmNQl0RaRHFjBYLameUPSP6M2BevfCRx8-Ix26rw", - "e": "AQAB" - }, - { - "kty": "RSA", - "alg": "RS256", - "use": "sig", - "kid": "33a681b36b8983913b05f8d5eab165c60694b6fc", - "n": "pRbWhLXIOvF1Naxh5n2evZqpJ3kPEFL4b-jHRHOhMnzqsi_aeQtIwYLVM8zdYYfgCxRq8umG1YwMenBKKPzRWr4MkFAB64O3UPgdyg-3je-fgCMziqS_3KH7AXHekxG_ZHpwbkgilRMtJNiDnSZWGad8XAfW3VFct4RqRAaf7h-6Za0IM7R3u4VYkUfosNqKtoDJDQrng9Nbv9ryUk8u1WikKF0M1r-wrZoDA7QFRAFkbdfYyvHL2daUflNDIXmFkUeHGGApMlJQ3wJk7Tln4txGMSUdJoD3JWEyKa2W1WshtqBHnQb8VlL77H-ch9zbn5pGZCoJ0MsgJr0vjoea8w", - "e": "AQAB" - }, - { - "kty": "RSA", - "alg": "RS256", - "use": "sig", - "kid": "986f2c30b9843defdc8f8debe138d640d1d67d9d", - "n": "qfE-u8I60zP_Vg3UsP_X9Qv1idkrnsXKkcQcsGWUdqs4jM2VixGBJDMJWbP55aJA93Sl49_dzAv_lr5-ctc1L04ke6nHX19EEBDtFTjGKyEdI-X1C-6HXCQmpI1XvUA8DzTOIZd5KEXJgBA9tpn6qkMAHzXyMHOfg8nhW36She4QggFjfF_RDKOA2-jRXDIjinOmIaLhVq4hsC9hCshhsfreLPw3HH8UhRONbkoFU_8ZBgAQLdVB1TQwp-ZDiVyh9od9a-RmfIiUl27AK5LDrtpVRCtXj6bv9OMx4QWVX8G-NfQexDwAp1pICaE5qYyJbiIK25E07vPTJ1ZtMpodFQ", - "e": "AQAB" - } - ] -} -''' - -def print_body(binary, expected): - p = Popen(['bazel-bin/src/tools/service_control_json_gen', - '--text', - '--'+expected, - '--stdin'], stdin=PIPE) - p.communicate(input=binary) - -class Handler(BaseHTTPRequestHandler): - def do_GET(self): - self.handle_request("GET") - - def do_POST(self): - self.handle_request("POST") - - - - def handle_request(self, method): - print(method, self.path, self.headers.items()) - url = self.headers.get('x-api-manager-url', '') - if url == 'https://www.googleapis.com/service_accounts/v1/jwk/loadtest@esp-test-client.iam.gserviceaccount.com': - self.send_response(200) - self.send_header('Content-Length', str(len(PUBLIC_KEY))) - self.end_headers() - self.wfile.write(PUBLIC_KEY) - else: - #content_len = self.headers.getheader('content-length', 0) - #post_body = self.rfile.read(int(content_len)) - if url.endswith(":report"): - print_body(post_body, "report_request") - elif url.endswith(":check"): - print_body(post_body, "check_request") - - self.send_response(200) - self.send_header('Content-Length', "0") - self.end_headers() - - -if __name__ == '__main__': - server = HTTPServer(('localhost', 8081), Handler) - print 'Starting server, use to stop' - server.serve_forever() diff --git a/src/envoy/prototype/envoy-esp.conf b/src/envoy/prototype/envoy-esp.conf deleted file mode 100644 index 27ea9f1b397..00000000000 --- a/src/envoy/prototype/envoy-esp.conf +++ /dev/null @@ -1,89 +0,0 @@ -{ - "listeners": [ - { - "port": 9092, - "bind_to_port": true, - "use_original_dst": true, - "filters": [] - }, - { - "port": 9090, - "bind_to_port": false, - "filters": [ - { - "type": "read", - "name": "http_connection_manager", - "config": { - "codec_type": "auto", - "stat_prefix": "ingress_http", - "route_config": { - "virtual_hosts": [ - { - "name": "backend", - "domains": ["*"], - "routes": [ - { - "timeout_ms": 0, - "prefix": "/", - "cluster": "service1" - } - ] - } - ] - }, - "access_log": [ - { - "path": "/dev/stdout" - } - ], - "filters": [ - { - "type": "both", - "name": "esp", - "config": { - "service_config": "src/envoy/prototype/generic_service_config.json", - "server_config": "src/envoy/prototype/server_config.pb.txt" - } - }, - { - "type": "decoder", - "name": "router", - "config": {} - } - ] - } - } - ] - } - ], - "admin": { - "access_log_path": "/dev/stdout", - "port": 9001 - }, - "cluster_manager": { - "clusters": [ - { - "name": "service1", - "connect_timeout_ms": 5000, - "type": "strict_dns", - "lb_type": "round_robin", - "hosts": [ - { - "url": "tcp://localhost:8080" - } - ] - }, - { - "name": "api_manager", - "connect_timeout_ms": 5000, - "type": "strict_dns", - "lb_type": "round_robin", - "hosts": [ - { - "url": "tcp://localhost:8081" - } - ] - } - ] - } -} diff --git a/src/envoy/prototype/generic_service_config.json b/src/envoy/prototype/generic_service_config.json deleted file mode 100644 index ac926e5e4c2..00000000000 --- a/src/envoy/prototype/generic_service_config.json +++ /dev/null @@ -1,736 +0,0 @@ -{ - "apis": [ - { - "methods": [ - { - "name": "Get", - "requestTypeUrl": "type.googleapis.com/google.protobuf.Empty", - "responseTypeUrl": "type.googleapis.com/google.protobuf.Value" - }, - { - "name": "Delete", - "requestTypeUrl": "type.googleapis.com/google.protobuf.Empty", - "responseTypeUrl": "type.googleapis.com/google.protobuf.Value" - }, - { - "name": "Patch", - "requestTypeUrl": "type.googleapis.com/google.protobuf.Empty", - "responseTypeUrl": "type.googleapis.com/google.protobuf.Value" - }, - { - "name": "Post", - "requestTypeUrl": "type.googleapis.com/google.protobuf.Empty", - "responseTypeUrl": "type.googleapis.com/google.protobuf.Value" - }, - { - "name": "Put", - "requestTypeUrl": "type.googleapis.com/google.protobuf.Empty", - "responseTypeUrl": "type.googleapis.com/google.protobuf.Value" - } - ], - "name": "generic_qiwzhang_app_1_appspot_com_1_0_0", - "sourceContext": { - "fileName": "sss.yaml" - }, - "version": "1.0.0" - } - ], - "authentication": {}, - "backend": { - "rules": [ - { - "selector": "generic_qiwzhang_app_1_appspot_com_1_0_0.Get" - }, - { - "selector": "generic_qiwzhang_app_1_appspot_com_1_0_0.Delete" - }, - { - "selector": "generic_qiwzhang_app_1_appspot_com_1_0_0.Patch" - }, - { - "selector": "generic_qiwzhang_app_1_appspot_com_1_0_0.Post" - }, - { - "selector": "generic_qiwzhang_app_1_appspot_com_1_0_0.Put" - } - ] - }, - "configVersion": 3, - "context": {}, - "control": { - "environment": "servicecontrol.googleapis.com" - }, - "customError": {}, - "documentation": {}, - "enums": [ - { - "enumvalue": [ - { - "name": "NULL_VALUE" - } - ], - "name": "google.protobuf.NullValue", - "sourceContext": { - "fileName": "struct.proto" - } - } - ], - "http": { - "rules": [ - { - "get": "/**", - "selector": "generic_qiwzhang_app_1_appspot_com_1_0_0.Get" - }, - { - "delete": "/**", - "selector": "generic_qiwzhang_app_1_appspot_com_1_0_0.Delete" - }, - { - "patch": "/**", - "selector": "generic_qiwzhang_app_1_appspot_com_1_0_0.Patch" - }, - { - "post": "/**", - "selector": "generic_qiwzhang_app_1_appspot_com_1_0_0.Post" - }, - { - "put": "/**", - "selector": "generic_qiwzhang_app_1_appspot_com_1_0_0.Put" - } - ] - }, - "id": "2017-01-06r0", - "logging": { - "producerDestinations": [ - { - "logs": [ - "endpoints_log" - ], - "monitoredResource": "api" - } - ] - }, - "logs": [ - { - "name": "endpoints_log" - } - ], - "metrics": [ - { - "labels": [ - { - "key": "/credential_id" - }, - { - "key": "/protocol" - }, - { - "key": "/response_code" - }, - { - "key": "/response_code_class" - }, - { - "key": "/status_code" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/request_count", - "valueType": "INT64" - }, - { - "labels": [ - { - "key": "/credential_id" - }, - { - "key": "/error_type" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/error_count", - "valueType": "INT64" - }, - { - "labels": [ - { - "key": "/credential_id" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/total_latencies", - "valueType": "DISTRIBUTION" - }, - { - "labels": [ - { - "key": "/protocol" - }, - { - "key": "/response_code" - }, - { - "key": "/response_code_class" - }, - { - "key": "/status_code" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/request_count", - "valueType": "INT64" - }, - { - "labels": [ - { - "key": "/error_type" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/error_count", - "valueType": "INT64" - }, - { - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/total_latencies", - "valueType": "DISTRIBUTION" - }, - { - "labels": [ - { - "key": "/credential_id" - }, - { - "key": "/end_user" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/top_request_count_by_end_user", - "valueType": "INT64" - }, - { - "labels": [ - { - "key": "/credential_id" - }, - { - "key": "/end_user_country" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/top_request_count_by_end_user_country", - "valueType": "INT64" - }, - { - "labels": [ - { - "key": "/credential_id" - }, - { - "key": "/referer" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/top_request_count_by_referer", - "valueType": "INT64" - }, - { - "labels": [ - { - "key": "/protocol" - }, - { - "key": "/response_code" - }, - { - "key": "/consumer_id" - }, - { - "key": "/status_code" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/top_request_count_by_consumer", - "valueType": "INT64" - }, - { - "labels": [ - { - "key": "/credential_id" - }, - { - "key": "/quota_group_name" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/quota_used_count", - "valueType": "INT64" - }, - { - "labels": [ - { - "key": "/quota_group_name" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/quota_billable_count", - "valueType": "INT64" - }, - { - "labels": [ - { - "key": "/credential_id" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/request_overhead_latencies", - "valueType": "DISTRIBUTION" - }, - { - "labels": [ - { - "key": "/credential_id" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/backend_latencies", - "valueType": "DISTRIBUTION" - }, - { - "labels": [ - { - "key": "/credential_id" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/request_sizes", - "valueType": "DISTRIBUTION" - }, - { - "labels": [ - { - "key": "/credential_id" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/response_sizes", - "valueType": "DISTRIBUTION" - }, - { - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/request_overhead_latencies", - "valueType": "DISTRIBUTION" - }, - { - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/backend_latencies", - "valueType": "DISTRIBUTION" - }, - { - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/request_sizes", - "valueType": "DISTRIBUTION" - }, - { - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/response_sizes", - "valueType": "DISTRIBUTION" - }, - { - "labels": [ - { - "key": "/consumer_id" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/top_request_sizes_by_consumer", - "valueType": "DISTRIBUTION" - }, - { - "labels": [ - { - "key": "/consumer_id" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/top_response_sizes_by_consumer", - "valueType": "DISTRIBUTION" - }, - { - "labels": [ - { - "key": "/credential_id" - }, - { - "key": "/protocol" - }, - { - "key": "/response_code" - }, - { - "key": "/response_code_class" - }, - { - "key": "/status_code" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/by_consumer/request_count", - "valueType": "INT64" - }, - { - "labels": [ - { - "key": "/credential_id" - }, - { - "key": "/error_type" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/by_consumer/error_count", - "valueType": "INT64" - }, - { - "labels": [ - { - "key": "/credential_id" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/by_consumer/total_latencies", - "valueType": "DISTRIBUTION" - }, - { - "labels": [ - { - "key": "/credential_id" - }, - { - "key": "/quota_group_name" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/by_consumer/quota_used_count", - "valueType": "INT64" - }, - { - "labels": [ - { - "key": "/credential_id" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/by_consumer/request_overhead_latencies", - "valueType": "DISTRIBUTION" - }, - { - "labels": [ - { - "key": "/credential_id" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/by_consumer/backend_latencies", - "valueType": "DISTRIBUTION" - }, - { - "labels": [ - { - "key": "/credential_id" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/by_consumer/request_sizes", - "valueType": "DISTRIBUTION" - }, - { - "labels": [ - { - "key": "/credential_id" - } - ], - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/by_consumer/response_sizes", - "valueType": "DISTRIBUTION" - }, - { - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/streaming_request_message_counts", - "valueType": "DISTRIBUTION" - }, - { - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/streaming_response_message_counts", - "valueType": "DISTRIBUTION" - }, - { - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/streaming_request_message_counts", - "valueType": "DISTRIBUTION" - }, - { - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/streaming_response_message_counts", - "valueType": "DISTRIBUTION" - }, - { - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/streaming_durations", - "valueType": "DISTRIBUTION" - }, - { - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/streaming_durations", - "valueType": "DISTRIBUTION" - }, - { - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/request_bytes", - "valueType": "INT64" - }, - { - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/producer/response_bytes", - "valueType": "INT64" - }, - { - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/request_bytes", - "valueType": "INT64" - }, - { - "metricKind": "DELTA", - "name": "serviceruntime.googleapis.com/api/consumer/response_bytes", - "valueType": "INT64" - } - ], - "monitoredResources": [ - { - "labels": [ - { - "key": "cloud.googleapis.com/location" - }, - { - "key": "cloud.googleapis.com/uid" - }, - { - "key": "serviceruntime.googleapis.com/api_version" - }, - { - "key": "serviceruntime.googleapis.com/api_method" - }, - { - "key": "serviceruntime.googleapis.com/consumer_project" - }, - { - "key": "cloud.googleapis.com/project" - }, - { - "key": "cloud.googleapis.com/service" - } - ], - "type": "api" - } - ], - "monitoring": { - "consumerDestinations": [ - { - "metrics": [ - "serviceruntime.googleapis.com/api/consumer/request_count", - "serviceruntime.googleapis.com/api/consumer/error_count", - "serviceruntime.googleapis.com/api/consumer/quota_used_count", - "serviceruntime.googleapis.com/api/consumer/total_latencies", - "serviceruntime.googleapis.com/api/consumer/request_overhead_latencies", - "serviceruntime.googleapis.com/api/consumer/backend_latencies", - "serviceruntime.googleapis.com/api/consumer/request_sizes", - "serviceruntime.googleapis.com/api/consumer/response_sizes", - "serviceruntime.googleapis.com/api/consumer/top_request_count_by_end_user", - "serviceruntime.googleapis.com/api/consumer/top_request_count_by_end_user_country", - "serviceruntime.googleapis.com/api/consumer/top_request_count_by_referer", - "serviceruntime.googleapis.com/api/consumer/streaming_request_message_counts", - "serviceruntime.googleapis.com/api/consumer/streaming_response_message_counts", - "serviceruntime.googleapis.com/api/consumer/streaming_durations", - "serviceruntime.googleapis.com/api/consumer/request_bytes", - "serviceruntime.googleapis.com/api/consumer/response_bytes" - ], - "monitoredResource": "api" - } - ], - "producerDestinations": [ - { - "metrics": [ - "serviceruntime.googleapis.com/api/producer/request_count", - "serviceruntime.googleapis.com/api/producer/error_count", - "serviceruntime.googleapis.com/api/producer/total_latencies", - "serviceruntime.googleapis.com/api/producer/request_overhead_latencies", - "serviceruntime.googleapis.com/api/producer/backend_latencies", - "serviceruntime.googleapis.com/api/producer/request_sizes", - "serviceruntime.googleapis.com/api/producer/response_sizes", - "serviceruntime.googleapis.com/api/producer/top_request_count_by_consumer", - "serviceruntime.googleapis.com/api/producer/top_request_sizes_by_consumer", - "serviceruntime.googleapis.com/api/producer/top_response_sizes_by_consumer", - "serviceruntime.googleapis.com/api/producer/streaming_request_message_counts", - "serviceruntime.googleapis.com/api/producer/streaming_response_message_counts", - "serviceruntime.googleapis.com/api/producer/streaming_durations", - "serviceruntime.googleapis.com/api/producer/request_bytes", - "serviceruntime.googleapis.com/api/producer/response_bytes", - "serviceruntime.googleapis.com/api/producer/by_consumer/request_count", - "serviceruntime.googleapis.com/api/producer/by_consumer/error_count", - "serviceruntime.googleapis.com/api/producer/by_consumer/total_latencies", - "serviceruntime.googleapis.com/api/producer/by_consumer/quota_used_count", - "serviceruntime.googleapis.com/api/producer/by_consumer/request_overhead_latencies", - "serviceruntime.googleapis.com/api/producer/by_consumer/backend_latencies", - "serviceruntime.googleapis.com/api/producer/by_consumer/request_sizes", - "serviceruntime.googleapis.com/api/producer/by_consumer/response_sizes" - ], - "monitoredResource": "api" - } - ] - }, - "name": "generic.qiwzhang-app-1.appspot.com", - "producerProjectId": "qiwzhang-app-1", - "systemParameters": {}, - "title": "Basic HTTP service configuration", - "types": [ - { - "fields": [ - { - "cardinality": "CARDINALITY_REPEATED", - "jsonName": "values", - "kind": "TYPE_MESSAGE", - "name": "values", - "number": 1, - "typeUrl": "type.googleapis.com/google.protobuf.Value" - } - ], - "name": "google.protobuf.ListValue", - "sourceContext": { - "fileName": "struct.proto" - } - }, - { - "fields": [ - { - "cardinality": "CARDINALITY_REPEATED", - "jsonName": "fields", - "kind": "TYPE_MESSAGE", - "name": "fields", - "number": 1, - "typeUrl": "type.googleapis.com/google.protobuf.Struct.FieldsEntry" - } - ], - "name": "google.protobuf.Struct", - "sourceContext": { - "fileName": "struct.proto" - } - }, - { - "fields": [ - { - "cardinality": "CARDINALITY_OPTIONAL", - "jsonName": "key", - "kind": "TYPE_STRING", - "name": "key", - "number": 1 - }, - { - "cardinality": "CARDINALITY_OPTIONAL", - "jsonName": "value", - "kind": "TYPE_MESSAGE", - "name": "value", - "number": 2, - "typeUrl": "type.googleapis.com/google.protobuf.Value" - } - ], - "name": "google.protobuf.Struct.FieldsEntry", - "sourceContext": { - "fileName": "struct.proto" - } - }, - { - "name": "google.protobuf.Empty", - "sourceContext": { - "fileName": "struct.proto" - } - }, - { - "fields": [ - { - "cardinality": "CARDINALITY_OPTIONAL", - "jsonName": "nullValue", - "kind": "TYPE_ENUM", - "name": "null_value", - "number": 1, - "typeUrl": "type.googleapis.com/google.protobuf.NullValue" - }, - { - "cardinality": "CARDINALITY_OPTIONAL", - "jsonName": "numberValue", - "kind": "TYPE_DOUBLE", - "name": "number_value", - "number": 2 - }, - { - "cardinality": "CARDINALITY_OPTIONAL", - "jsonName": "stringValue", - "kind": "TYPE_STRING", - "name": "string_value", - "number": 3 - }, - { - "cardinality": "CARDINALITY_OPTIONAL", - "jsonName": "boolValue", - "kind": "TYPE_BOOL", - "name": "bool_value", - "number": 4 - }, - { - "cardinality": "CARDINALITY_OPTIONAL", - "jsonName": "structValue", - "kind": "TYPE_MESSAGE", - "name": "struct_value", - "number": 5, - "typeUrl": "type.googleapis.com/google.protobuf.Struct" - }, - { - "cardinality": "CARDINALITY_OPTIONAL", - "jsonName": "listValue", - "kind": "TYPE_MESSAGE", - "name": "list_value", - "number": 6, - "typeUrl": "type.googleapis.com/google.protobuf.ListValue" - } - ], - "name": "google.protobuf.Value", - "sourceContext": { - "fileName": "struct.proto" - } - } - ], - "usage": { - "rules": [ - { - "allowUnregisteredCalls": true, - "selector": "generic_qiwzhang_app_1_appspot_com_1_0_0.Get" - }, - { - "allowUnregisteredCalls": true, - "selector": "generic_qiwzhang_app_1_appspot_com_1_0_0.Delete" - }, - { - "allowUnregisteredCalls": true, - "selector": "generic_qiwzhang_app_1_appspot_com_1_0_0.Patch" - }, - { - "allowUnregisteredCalls": true, - "selector": "generic_qiwzhang_app_1_appspot_com_1_0_0.Post" - }, - { - "allowUnregisteredCalls": true, - "selector": "generic_qiwzhang_app_1_appspot_com_1_0_0.Put" - } - ] - }, - "visibility": {} -} diff --git a/src/envoy/prototype/server_config.pb.txt b/src/envoy/prototype/server_config.pb.txt deleted file mode 100644 index 19cb6cbb2c0..00000000000 --- a/src/envoy/prototype/server_config.pb.txt +++ /dev/null @@ -1,6 +0,0 @@ -cloud_tracing_config { - force_disable: true -} -mixer_options { - mixer_server: "localhost:9091" -} From a6a33fdd6f2cd6b563018ed26f6f9f82bb0f06c8 Mon Sep 17 00:00:00 2001 From: Qian Sun Date: Wed, 8 Feb 2017 12:42:11 -0800 Subject: [PATCH 21/83] Update mixer client SHA. (#74) --- src/envoy/mixer/repositories.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/envoy/mixer/repositories.bzl b/src/envoy/mixer/repositories.bzl index d9888fb0daf..d5dfe4ac19c 100644 --- a/src/envoy/mixer/repositories.bzl +++ b/src/envoy/mixer/repositories.bzl @@ -18,7 +18,7 @@ def mixer_client_repositories(bind=True): native.git_repository( name = "mixerclient_git", - commit = "80e450a5126960e8e6337c3631cf2ef984038eab", + commit = "7b8544d765f9d7d86d28770c8d27d69cbf9509ac", remote = "https://github.com/istio/mixerclient.git", ) From d29a195a7478cd5bf56706aed50c874d813cb5e6 Mon Sep 17 00:00:00 2001 From: Qian Sun Date: Wed, 8 Feb 2017 12:49:30 -0800 Subject: [PATCH 22/83] Update readme. (#73) --- src/envoy/mixer/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/envoy/mixer/README.md b/src/envoy/mixer/README.md index 9094d748442..43304785198 100644 --- a/src/envoy/mixer/README.md +++ b/src/envoy/mixer/README.md @@ -38,7 +38,7 @@ This Proxy will use Envoy and talk to Mixer server. * Start Envoy proxy, run ``` - bazel-bin/src/envoy/mixer/envoy -c src/envoy/prototype/envoy-mixer.conf + bazel-bin/src/envoy/mixer/envoy -c src/envoy/mixer/envoy-mixer.conf ``` * Then issue HTTP request to proxy. From fdac61b43eaf7963a9f79073a631d790f67535c3 Mon Sep 17 00:00:00 2001 From: Sebastien Vas Date: Wed, 8 Feb 2017 16:58:32 -0800 Subject: [PATCH 23/83] Adds Jenkinsfile and updates release-binary to create a SHA. (#71) * Adds Jenkinsfile and update release-binary * Update Jenkinsfile and gitignore * Fixes typo and use normal build Node * Uses default bazel config * Using batch mode * Update bazel memory settings * Do not use Jenkins bazel env * Set .bazelrc for postsubmit --- .bazelrc.jenkins | 8 ++++++ .gitignore | 2 ++ Jenkinsfile | 61 +++++++++++++++++++++++++++++++++++++++++++ script/release-binary | 15 ++++++----- 4 files changed, 80 insertions(+), 6 deletions(-) create mode 100644 .bazelrc.jenkins create mode 100644 Jenkinsfile diff --git a/.bazelrc.jenkins b/.bazelrc.jenkins new file mode 100644 index 00000000000..664f7b13380 --- /dev/null +++ b/.bazelrc.jenkins @@ -0,0 +1,8 @@ +# This is from Bazel's former travis setup, to avoid blowing up the RAM usage. +startup --host_jvm_args=-Xmx8192m +startup --host_jvm_args=-Xms8192m +startup --batch + +# This is so we understand failures better +build --verbose_failures + diff --git a/.gitignore b/.gitignore index a6ef824c1f8..469486c8f36 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ /bazel-* +.idea/* +*.iml diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 00000000000..db6abcaa5f4 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,61 @@ +#!groovy + +@Library('testutils') + +import org.istio.testutils.Utilities +import org.istio.testutils.GitUtilities +import org.istio.testutils.Bazel + +// Utilities shared amongst modules +def gitUtils = new GitUtilities() +def utils = new Utilities() +def bazel = new Bazel() + +node { + gitUtils.initialize() + // Proxy does build work correctly with Hazelcast. + // Must use .bazelrc.jenkins + bazel.setVars('', '') +} + +mainFlow(utils) { + if (utils.runStage('PRESUBMIT')) { + def success = true + utils.updatePullRequest('run') + try { + presubmit(gitUtils, bazel) + } catch (Exception e) { + success = false + throw e + } finally { + utils.updatePullRequest('verify', success) + } + } + if (utils.runStage('POSTSUBMIT')) { + buildNode(gitUtils) { + bazel.updateBazelRc() + sh 'script/release-binary' + } + } +} + +def presubmit(gitUtils, bazel) { + buildNode(gitUtils) { + stage('Code Check') { + sh('script/check-style') + } + bazel.updateBazelRc() + stage('Bazel Fetch') { + bazel.fetch('-k //...') + } + stage('Bazel Build') { + bazel.build('//...') + } + stage('Bazel Tests') { + bazel.test('//...') + } + stage('Push Test Binary') { + sh 'script/release-binary' + } + } +} diff --git a/script/release-binary b/script/release-binary index af90b1aa0da..bf83a7d3115 100755 --- a/script/release-binary +++ b/script/release-binary @@ -24,16 +24,19 @@ set -ex BUCKET_NAME="istio-build/proxy" # The proxy binary name. -BINARY_FORMAT='envoy-alpha-%H.tar.gz' -BINARY_NAME="$(git show -q HEAD --pretty=format:"${BINARY_FORMAT}")" +SHA="$(git rev-parse --verify HEAD)" +BINARY_NAME="envoy-alpha-${SHA}.tar.gz" +SHA256_NAME="envoy-alpha-${SHA}.sha256" # Build the binary bazel build --config=release //src/envoy/mixer:envoy_tar -SRC="bazel-bin/src/envoy/mixer/envoy_tar.tar.gz" -DST="gs://${BUCKET_NAME}/${BINARY_NAME}" +BAZEL_TARGET="bazel-bin/src/envoy/mixer/envoy_tar.tar.gz" +ln "${BAZEL_TARGET}" "${BINARY_NAME}" +sha256sum "${BINARY_NAME}" > "${SHA256_NAME}" +DST="gs://${BUCKET_NAME}/" # Copy it to the bucket. -echo "Copying ${SRC} to ${DST}" -gsutil cp ${SRC} ${DST} +echo "Copying ${BINARY_NAME} ${SHA256_NAME} to ${DST}" +gsutil cp "${BINARY_NAME}" "${SHA256_NAME}" "${DST}" From 6e372fc510de08da504c2dd21060c09255c1aee7 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Wed, 8 Feb 2017 19:18:26 -0800 Subject: [PATCH 24/83] Update grpc and protobuf (#70) * protobuf v3.2.0 * grpc v1.1.1 * Align auth lib with grpc 1.1.1 --- contrib/endpoints/repositories.bzl | 5 +- .../auth/lib/auth_jwt_validator.cc | 47 +++-- .../src/api_manager/auth/lib/auth_token.cc | 3 +- .../src/api_manager/auth/lib/grpc_internals.h | 176 +----------------- repositories.bzl | 4 +- 5 files changed, 40 insertions(+), 195 deletions(-) diff --git a/contrib/endpoints/repositories.bzl b/contrib/endpoints/repositories.bzl index e3363cc86d2..bae14e92d61 100644 --- a/contrib/endpoints/repositories.bzl +++ b/contrib/endpoints/repositories.bzl @@ -163,7 +163,7 @@ def grpc_repositories(bind=True): native.git_repository( name = "grpc_git", - commit = "d28417c856366df704200f544e72d31056931bce", + commit = "bb3edafea245a9780cc4c10f0b58da21e8193f38", # v1.1.1 remote = "https://github.com/grpc/grpc.git", ) @@ -190,7 +190,7 @@ def grpc_repositories(bind=True): native.bind( name = "grpc_lib", - actual = "@grpc_git//:grpc++_reflection", + actual = "@grpc_git//:grpc++_codegen_proto", ) def googleapis_repositories(protobuf_repo="@protobuf_git//", bind=True): @@ -333,4 +333,3 @@ def servicecontrol_client_repositories(bind=True): name = "servicecontrol_client", actual = "@servicecontrol_client_git//:service_control_client_lib", ) - diff --git a/contrib/endpoints/src/api_manager/auth/lib/auth_jwt_validator.cc b/contrib/endpoints/src/api_manager/auth/lib/auth_jwt_validator.cc index 8da6ef2166c..1c84bdb837d 100644 --- a/contrib/endpoints/src/api_manager/auth/lib/auth_jwt_validator.cc +++ b/contrib/endpoints/src/api_manager/auth/lib/auth_jwt_validator.cc @@ -150,6 +150,8 @@ class JwtValidatorImpl : public JwtValidator { RSA *rsa_; EVP_PKEY *pkey_; EVP_MD_CTX *md_ctx_; + + grpc_exec_ctx exec_ctx_; }; // Gets EVP_MD mapped from an alg (algorithm string). @@ -159,12 +161,12 @@ const EVP_MD *EvpMdFromAlg(const char *alg); size_t HashSizeFromAlg(const char *alg); // Parses str into grpc_json object. Does not own buffer. -grpc_json *DecodeBase64AndParseJson(const char *str, size_t len, - gpr_slice *buffer); +grpc_json *DecodeBase64AndParseJson(grpc_exec_ctx *exec_ctx, const char *str, + size_t len, gpr_slice *buffer); // Gets BIGNUM from b64 string, used for extracting pkey from jwk. // Result owned by rsa_. -BIGNUM *BigNumFromBase64String(const char *b64); +BIGNUM *BigNumFromBase64String(grpc_exec_ctx *exec_ctx, const char *b64); } // namespace @@ -185,7 +187,8 @@ JwtValidatorImpl::JwtValidatorImpl(const char *jwt, size_t jwt_len) x509_(nullptr), rsa_(nullptr), pkey_(nullptr), - md_ctx_(nullptr) { + md_ctx_(nullptr), + exec_ctx_(GRPC_EXEC_CTX_INIT) { header_buffer_ = gpr_empty_slice(); signed_buffer_ = gpr_empty_slice(); sig_buffer_ = gpr_empty_slice(); @@ -204,7 +207,7 @@ JwtValidatorImpl::~JwtValidatorImpl() { grpc_json_destroy(pkey_json_); } if (claims_ != nullptr) { - grpc_jwt_claims_destroy(claims_); + grpc_jwt_claims_destroy(&exec_ctx_, claims_); } if (!GPR_SLICE_IS_EMPTY(header_buffer_)) { gpr_slice_unref(header_buffer_); @@ -304,7 +307,8 @@ grpc_jwt_verifier_status JwtValidatorImpl::ParseImpl() { if (dot == nullptr) { return GRPC_JWT_VERIFIER_BAD_FORMAT; } - header_json_ = DecodeBase64AndParseJson(cur, dot - cur, &header_buffer_); + header_json_ = + DecodeBase64AndParseJson(&exec_ctx_, cur, dot - cur, &header_buffer_); CreateJoseHeader(); if (header_ == nullptr) { return GRPC_JWT_VERIFIER_BAD_FORMAT; @@ -323,7 +327,7 @@ grpc_jwt_verifier_status JwtValidatorImpl::ParseImpl() { // case, and it is owned by claims_ for successful case. gpr_slice claims_buffer = gpr_empty_slice(); grpc_json *claims_json = - DecodeBase64AndParseJson(cur, dot - cur, &claims_buffer); + DecodeBase64AndParseJson(&exec_ctx_, cur, dot - cur, &claims_buffer); if (claims_json == nullptr) { if (!GPR_SLICE_IS_EMPTY(claims_buffer)) { gpr_slice_unref(claims_buffer); @@ -332,10 +336,13 @@ grpc_jwt_verifier_status JwtValidatorImpl::ParseImpl() { } UpdateAudience(claims_json); // Takes ownershp of claims_json and claims_buffer. - claims_ = grpc_jwt_claims_from_json(claims_json, claims_buffer); - if (claims_ == nullptr) { + claims_ = grpc_jwt_claims_from_json(&exec_ctx_, claims_json, claims_buffer); + + // issuer is mandatory. grpc_jwt_claims_issuer checks if claims_ is nullptr. + if (grpc_jwt_claims_issuer(claims_) == nullptr) { return GRPC_JWT_VERIFIER_BAD_FORMAT; } + // Check timestamp. // Passing in its own audience to skip audience check. // Audience check should be done by the caller. @@ -354,8 +361,8 @@ grpc_jwt_verifier_status JwtValidatorImpl::ParseImpl() { return GRPC_JWT_VERIFIER_BAD_FORMAT; } cur = dot + 1; - sig_buffer_ = - grpc_base64_decode_with_len(cur, jwt_len - signed_jwt_len - 1, 1); + sig_buffer_ = grpc_base64_decode_with_len(&exec_ctx_, cur, + jwt_len - signed_jwt_len - 1, 1); if (GPR_SLICE_IS_EMPTY(sig_buffer_)) { return GRPC_JWT_VERIFIER_BAD_FORMAT; } @@ -576,9 +583,11 @@ bool JwtValidatorImpl::ExtractPubkeyFromJwk(const grpc_json *jkey) { } const char *rsa_n = GetStringValue(jkey, "n"); - rsa_->n = rsa_n == nullptr ? nullptr : BigNumFromBase64String(rsa_n); + rsa_->n = + rsa_n == nullptr ? nullptr : BigNumFromBase64String(&exec_ctx_, rsa_n); const char *rsa_e = GetStringValue(jkey, "e"); - rsa_->e = rsa_e == nullptr ? nullptr : BigNumFromBase64String(rsa_e); + rsa_->e = + rsa_e == nullptr ? nullptr : BigNumFromBase64String(&exec_ctx_, rsa_e); if (rsa_->e == nullptr || rsa_->n == nullptr) { gpr_log(GPR_ERROR, "Missing RSA public key field."); @@ -651,7 +660,7 @@ grpc_jwt_verifier_status JwtValidatorImpl::VerifyHsSignature(const char *pkey, const EVP_MD *md = EvpMdFromAlg(header_->alg); GPR_ASSERT(md != nullptr); // Checked before. - pkey_buffer_ = grpc_base64_decode_with_len(pkey, pkey_len, 1); + pkey_buffer_ = grpc_base64_decode_with_len(&exec_ctx_, pkey, pkey_len, 1); if (GPR_SLICE_IS_EMPTY(pkey_buffer_)) { gpr_log(GPR_ERROR, "Unable to decode base64 of secret"); return GRPC_JWT_VERIFIER_KEY_RETRIEVAL_ERROR; @@ -735,11 +744,11 @@ size_t HashSizeFromAlg(const char *alg) { } } -grpc_json *DecodeBase64AndParseJson(const char *str, size_t len, - gpr_slice *buffer) { +grpc_json *DecodeBase64AndParseJson(grpc_exec_ctx *exec_ctx, const char *str, + size_t len, gpr_slice *buffer) { grpc_json *json; - *buffer = grpc_base64_decode_with_len(str, len, 1); + *buffer = grpc_base64_decode_with_len(exec_ctx, str, len, 1); if (GPR_SLICE_IS_EMPTY(*buffer)) { gpr_log(GPR_ERROR, "Invalid base64."); return nullptr; @@ -753,12 +762,12 @@ grpc_json *DecodeBase64AndParseJson(const char *str, size_t len, return json; } -BIGNUM *BigNumFromBase64String(const char *b64) { +BIGNUM *BigNumFromBase64String(grpc_exec_ctx *exec_ctx, const char *b64) { BIGNUM *result = nullptr; gpr_slice bin; if (b64 == nullptr) return nullptr; - bin = grpc_base64_decode(b64, 1); + bin = grpc_base64_decode(exec_ctx, b64, 1); if (GPR_SLICE_IS_EMPTY(bin)) { gpr_log(GPR_ERROR, "Invalid base64 for big num."); return nullptr; diff --git a/contrib/endpoints/src/api_manager/auth/lib/auth_token.cc b/contrib/endpoints/src/api_manager/auth/lib/auth_token.cc index 72b53790158..85f06ca0417 100644 --- a/contrib/endpoints/src/api_manager/auth/lib/auth_token.cc +++ b/contrib/endpoints/src/api_manager/auth/lib/auth_token.cc @@ -200,7 +200,8 @@ char *GenerateJwtClaim(const char *issuer, const char *subject, } char *GenerateSignatueHs256(const char *data, const char *key) { - gpr_slice key_buffer = grpc_base64_decode(key, 1); + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + gpr_slice key_buffer = grpc_base64_decode(&exec_ctx, key, 1); if (GPR_SLICE_IS_EMPTY(key_buffer)) { gpr_log(GPR_ERROR, "Unable to decode base64 of secret"); return nullptr; diff --git a/contrib/endpoints/src/api_manager/auth/lib/grpc_internals.h b/contrib/endpoints/src/api_manager/auth/lib/grpc_internals.h index 66074a12776..cdf58840654 100644 --- a/contrib/endpoints/src/api_manager/auth/lib/grpc_internals.h +++ b/contrib/endpoints/src/api_manager/auth/lib/grpc_internals.h @@ -15,11 +15,6 @@ #ifndef API_MANAGER_AUTH_LIB_GRPC_INTERNALS_H_ #define API_MANAGER_AUTH_LIB_GRPC_INTERNALS_H_ -// This header file contains definitions for all grpc internal -// dependencies. The code that depends on grpc internals should -// include this file instead of including the original headers -// in grpc. - // This header file is for internal use only since it declares grpc // internals that auth depends on. A public header file should not // include any internal grpc header files. @@ -30,171 +25,12 @@ extern "C" { -#include -#include - -////////////////////////////////////////////////////// -// definitions from grpc/src/core/json/json_common.h -////////////////////////////////////////////////////// - -/* The various json types. */ -typedef enum { - GRPC_JSON_OBJECT, - GRPC_JSON_ARRAY, - GRPC_JSON_STRING, - GRPC_JSON_NUMBER, - GRPC_JSON_TRUE, - GRPC_JSON_FALSE, - GRPC_JSON_NULL, - GRPC_JSON_TOP_LEVEL -} grpc_json_type; - -////////////////////////////////////////////////////// -// definitions from grpc/src/core/json/json.h -////////////////////////////////////////////////////// - -/* A tree-like structure to hold json values. The key and value pointers - * are not owned by it. - */ -typedef struct grpc_json { - struct grpc_json *next; - struct grpc_json *prev; - struct grpc_json *child; - struct grpc_json *parent; - - grpc_json_type type; - const char *key; - const char *value; -} grpc_json; - -/* The next two functions are going to parse the input string, and - * modify it in the process, in order to use its space to store - * all of the keys and values for the returned object tree. - * - * They assume UTF-8 input stream, and will output UTF-8 encoded - * strings in the tree. The input stream's UTF-8 isn't validated, - * as in, what you input is what you get as an output. - * - * All the keys and values in the grpc_json objects will be strings - * pointing at your input buffer. - * - * Delete the allocated tree afterward using grpc_json_destroy(). - */ -grpc_json *grpc_json_parse_string_with_len(char *input, size_t size); -grpc_json *grpc_json_parse_string(char *input); - -/* Use these to create or delete a grpc_json object. - * Deletion is recursive. We will not attempt to free any of the strings - * in any of the objects of that tree. - */ -grpc_json *grpc_json_create(grpc_json_type type); -void grpc_json_destroy(grpc_json *json); - -/* This function will create a new string using gpr_realloc, and will - * deserialize the grpc_json tree into it. It'll be zero-terminated, - * but will be allocated in chunks of 256 bytes. - * - * The indent parameter controls the way the output is formatted. - * If indent is 0, then newlines will be suppressed as well, and the - * output will be condensed at its maximum. - */ -char *grpc_json_dump_to_string(grpc_json *json, int indent); - -////////////////////////////////////////////////////// -// definitions from grpc/src/core/security/base64 -////////////////////////////////////////////////////// - -/* Encodes data using base64. It is the caller's responsability to free - the returned char * using gpr_free. Returns nullptr on nullptr input. */ -char *grpc_base64_encode(const void *data, size_t data_size, int url_safe, - int multiline); - -/* Decodes data according to the base64 specification. Returns an empty - slice in case of failure. */ -gpr_slice grpc_base64_decode(const char *b64, int url_safe); - -/* Same as above except that the length is provided by the caller. */ -gpr_slice grpc_base64_decode_with_len(const char *b64, size_t b64_len, - int url_safe); - -////////////////////////////////////////////////////// -// definitions from grpc/src/core/auth/security/json_key.h -////////////////////////////////////////////////////// - -/* --- auth_json_key parsing. --- */ - -typedef struct { - const char *type; - char *private_key_id; - char *client_id; - char *client_email; - RSA *private_key; -} grpc_auth_json_key; - -/* Creates a json_key object from string. Returns an invalid object if a parsing - error has been encountered. */ -grpc_auth_json_key grpc_auth_json_key_create_from_string( - const char *json_string); - -/* Destructs the object. */ -void grpc_auth_json_key_destruct(grpc_auth_json_key *json_key); - -/* Caller is responsible for calling gpr_free on the returned value. May return - nullptr on invalid input. The scope parameter may be nullptr. */ -char *grpc_jwt_encode_and_sign(const grpc_auth_json_key *json_key, - const char *audience, - gpr_timespec token_lifetime, const char *scope); - -////////////////////////////////////////////////////// -// definitions from grpc/src/core/support/string.h -////////////////////////////////////////////////////// - -/* Minimum buffer size for calling ltoa */ -#define GPR_LTOA_MIN_BUFSIZE (3 * sizeof(long)) - -/* Convert a long to a string in base 10; returns the length of the - output string (or 0 on failure). - output must be at least GPR_LTOA_MIN_BUFSIZE bytes long. */ -int gpr_ltoa(long value, char *output); - -////////////////////////////////////////////////////// -// definitions from grpc/src/core/security/jwt_verifier.h -////////////////////////////////////////////////////// - -/* --- grpc_jwt_verifier_status. --- */ - -typedef enum { - GRPC_JWT_VERIFIER_OK = 0, - GRPC_JWT_VERIFIER_BAD_SIGNATURE, - GRPC_JWT_VERIFIER_BAD_FORMAT, - GRPC_JWT_VERIFIER_BAD_AUDIENCE, - GRPC_JWT_VERIFIER_KEY_RETRIEVAL_ERROR, - GRPC_JWT_VERIFIER_TIME_CONSTRAINT_FAILURE, - GRPC_JWT_VERIFIER_GENERIC_ERROR -} grpc_jwt_verifier_status; - -const char *grpc_jwt_verifier_status_to_string(grpc_jwt_verifier_status status); - -/* --- grpc_jwt_claims. --- */ - -typedef struct grpc_jwt_claims grpc_jwt_claims; - -void grpc_jwt_claims_destroy(grpc_jwt_claims *claims); - -/* Returns the whole JSON tree of the claims. */ -const grpc_json *grpc_jwt_claims_json(const grpc_jwt_claims *claims); - -/* Access to registered claims in https://tools.ietf.org/html/rfc7519#page-9 */ -const char *grpc_jwt_claims_subject(const grpc_jwt_claims *claims); -const char *grpc_jwt_claims_issuer(const grpc_jwt_claims *claims); -const char *grpc_jwt_claims_audience(const grpc_jwt_claims *claims); -gpr_timespec grpc_jwt_claims_expires_at(const grpc_jwt_claims *claims); - -/* --- TESTING ONLY exposed functions. --- */ - -grpc_jwt_claims *grpc_jwt_claims_from_json(grpc_json *json, gpr_slice buffer); -grpc_jwt_verifier_status grpc_jwt_claims_check(const grpc_jwt_claims *claims, - const char *audience); +#include "src/core/lib/json/json.h" +#include "src/core/lib/json/json_common.h" +#include "src/core/lib/security/credentials/jwt/json_token.h" +#include "src/core/lib/security/credentials/jwt/jwt_verifier.h" +#include "src/core/lib/security/util/b64.h" +#include "src/core/lib/support/string.h" } #endif // API_MANAGER_AUTH_LIB_GRPC_INTERNALS_H_ diff --git a/repositories.bzl b/repositories.bzl index 8764aa24d32..31fe15ffb63 100644 --- a/repositories.bzl +++ b/repositories.bzl @@ -36,7 +36,7 @@ def boringssl_repositories(bind=True): def protobuf_repositories(bind=True): native.git_repository( name = "protobuf_git", - commit = "a428e42072765993ff674fda72863c9f1aa2d268", # v3.1.0 + commit = "593e917c176b5bc5aafa57bf9f6030d749d91cd5", # v3.2.0 remote = "https://github.com/google/protobuf.git", ) @@ -68,7 +68,7 @@ def protobuf_repositories(bind=True): native.bind( name = "protobuf_clib", - actual = "@protobuf_git//:protobuf_lite", + actual = "@protobuf_git//:protoc_lib", ) From 28105ca86ab5487990eb7e13dd8a2ced1fb708be Mon Sep 17 00:00:00 2001 From: Wayne Zhang Date: Thu, 9 Feb 2017 13:10:47 -0800 Subject: [PATCH 25/83] Add sourceService. (#78) --- src/envoy/mixer/http_control.cc | 11 ++++++++++- src/envoy/mixer/http_control.h | 2 ++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/envoy/mixer/http_control.cc b/src/envoy/mixer/http_control.cc index 39e96ecbd3c..b1b5ba816e5 100644 --- a/src/envoy/mixer/http_control.cc +++ b/src/envoy/mixer/http_control.cc @@ -25,7 +25,6 @@ namespace Mixer { namespace { const std::string kProxyPeerID = "Istio/Proxy"; -const std::string kEnvNameTargetService = "TARGET_SERVICE"; // Define lower case string for X-Forwarded-Host. const LowerCaseString kHeaderNameXFH("x-forwarded-host", false); @@ -44,6 +43,11 @@ const std::string kAttrNameLogMessage = "logMessage"; const std::string kAttrNameResponseTime = "responseTime"; const std::string kAttrNameOriginIp = "originIp"; const std::string kAttrNameOriginHost = "originHost"; + +const std::string kEnvNameSourceService = "SOURCE_SERVICE"; +const std::string kEnvNameTargetService = "TARGET_SERVICE"; + +const std::string kAttrNameSourceService = "sourceService"; const std::string kAttrNameTargetService = "targetService"; Attributes::Value StringValue(const std::string& str) { @@ -147,6 +151,10 @@ HttpControl::HttpControl(const std::string& mixer_server) { options.mixer_server = mixer_server; mixer_client_ = ::istio::mixer_client::CreateMixerClient(options); + auto source_service = getenv(kEnvNameSourceService.c_str()); + if (source_service) { + source_service_ = source_service; + } auto target_service = getenv(kEnvNameTargetService.c_str()); if (target_service) { target_service_ = target_service; @@ -157,6 +165,7 @@ void HttpControl::FillCheckAttributes(const HeaderMap& header_map, Attributes* attr) { FillRequestHeaderAttributes(header_map, attr); + SetStringAttribute(kAttrNameSourceService, source_service_, attr); SetStringAttribute(kAttrNameTargetService, target_service_, attr); attr->attributes[kAttrNamePeerId] = StringValue(kProxyPeerID); } diff --git a/src/envoy/mixer/http_control.h b/src/envoy/mixer/http_control.h index 71c7e5620d6..312016e419c 100644 --- a/src/envoy/mixer/http_control.h +++ b/src/envoy/mixer/http_control.h @@ -55,6 +55,8 @@ class HttpControl final : public Logger::Loggable { // The mixer client std::unique_ptr<::istio::mixer_client::MixerClient> mixer_client_; + // Source service + std::string source_service_; // Target service std::string target_service_; }; From 21f1cb3a9333e2cda7658e09aace7fdb96c6b917 Mon Sep 17 00:00:00 2001 From: Wayne Zhang Date: Thu, 9 Feb 2017 15:22:10 -0800 Subject: [PATCH 26/83] Add script to build docker image. (#77) * Add script to build docker image. * Add start_envoy for docker image. --- WORKSPACE | 16 +++++ script/release-docker | 46 +++++++++++++ src/envoy/mixer/BUILD | 40 +++++++++++ src/envoy/mixer/README.md | 2 +- .../{envoy-mixer.conf => envoy.conf.template} | 6 +- src/envoy/mixer/start_envoy | 67 +++++++++++++++++++ 6 files changed, 173 insertions(+), 4 deletions(-) create mode 100755 script/release-docker rename src/envoy/mixer/{envoy-mixer.conf => envoy.conf.template} (92%) create mode 100755 src/envoy/mixer/start_envoy diff --git a/WORKSPACE b/WORKSPACE index e152300d73d..3b255e3fa7a 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -69,3 +69,19 @@ load( ) envoy_repositories() + +new_http_archive( + name = "docker_ubuntu", + build_file_content = """ +load("@bazel_tools//tools/build_defs/docker:docker.bzl", "docker_build") +docker_build( + name = "xenial", + tars = ["xenial/ubuntu-xenial-core-cloudimg-amd64-root.tar.gz"], + visibility = ["//visibility:public"], +) +""", + sha256 = "de31e6fcb843068965de5945c11a6f86399be5e4208c7299fb7311634fb41943", + strip_prefix = "docker-brew-ubuntu-core-e406914e5f648003dfe8329b512c30c9ad0d2f9c", + type = "zip", + url = "https://codeload.github.com/tianon/docker-brew-ubuntu-core/zip/e406914e5f648003dfe8329b512c30c9ad0d2f9c", +) diff --git a/script/release-docker b/script/release-docker new file mode 100755 index 00000000000..d21caffee55 --- /dev/null +++ b/script/release-docker @@ -0,0 +1,46 @@ +#!/bin/bash +# +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ +# +set -ex + +# This docker image can be used as: +# +# docker run IMAGE -a backend_address -p PORT -m MIXER_SERVER +# + +PROJECT=istio-testing +IMAGE_PREFIX="gcr.io/${PROJECT}/proxy" + +DATE_PART=$(date +"%Y%m%d") +SHA_PART=$(git show -q HEAD --pretty=format:%h) +DOCKER_TAG="${DATE_PART}${SHA_PART}" + +IMAGE_NAME="${IMAGE_PREFIX}:${DOCKER_TAG}" + +gcloud docker --authorize-only + +bazel run --config=release //src/envoy/mixer:proxy "${IMAGE_NAME}" + +gcloud docker -- push "${IMAGE_NAME}" + +IMAGE_LATEST="${IMAGE_PREFIX}:latest" + +docker tag -f "${IMAGE_NAME}" "${IMAGE_LATEST}" + +gcloud docker -- push "${IMAGE_LATEST}" + diff --git a/src/envoy/mixer/BUILD b/src/envoy/mixer/BUILD index 45c96e25f96..917e627e073 100644 --- a/src/envoy/mixer/BUILD +++ b/src/envoy/mixer/BUILD @@ -16,6 +16,7 @@ # load("@bazel_tools//tools/build_defs/pkg:pkg.bzl", "pkg_tar") +load("@bazel_tools//tools/build_defs/docker:docker.bzl", "docker_build") cc_library( name = "filter_lib", @@ -47,3 +48,42 @@ pkg_tar( mode = "0755", package_dir = "/usr/local/bin/", ) + +pkg_tar( + name = "start_envoy_tar", + extension = "tar.gz", + files = ["start_envoy"], + mode = "0755", + package_dir = "/usr/local/bin/", +) + +pkg_tar( + name = "config_tar", + extension = "tar.gz", + files = ["envoy.conf.template"], + mode = "0666", + package_dir = "/etc/opt/proxy", + tags = ["manual"], +) + +docker_build( + name = "proxy", + base = "@docker_ubuntu//:xenial", + entrypoint = [ + "/usr/local/bin/start_envoy", + "-e", + "/usr/local/bin/envoy", + "-c", + "/etc/opt/proxy/envoy.conf", + "-t", + "/etc/opt/proxy/envoy.conf.template", + ], + ports = ["9090"], + repository = "istio", + tags = ["manual"], + tars = [ + ":config_tar", + ":envoy_tar", + ":start_envoy_tar", + ], +) diff --git a/src/envoy/mixer/README.md b/src/envoy/mixer/README.md index 43304785198..21f7403f77c 100644 --- a/src/envoy/mixer/README.md +++ b/src/envoy/mixer/README.md @@ -38,7 +38,7 @@ This Proxy will use Envoy and talk to Mixer server. * Start Envoy proxy, run ``` - bazel-bin/src/envoy/mixer/envoy -c src/envoy/mixer/envoy-mixer.conf + bazel-bin/src/envoy/mixer/start_envoy ``` * Then issue HTTP request to proxy. diff --git a/src/envoy/mixer/envoy-mixer.conf b/src/envoy/mixer/envoy.conf.template similarity index 92% rename from src/envoy/mixer/envoy-mixer.conf rename to src/envoy/mixer/envoy.conf.template index 3b494c96b59..f1a00d1fb3e 100644 --- a/src/envoy/mixer/envoy-mixer.conf +++ b/src/envoy/mixer/envoy.conf.template @@ -1,7 +1,7 @@ { "listeners": [ { - "port": 9090, + "port": ${PORT}, "bind_to_port": true, "filters": [ { @@ -35,7 +35,7 @@ "type": "both", "name": "mixer", "config": { - "mixer_server": "localhost:9091" + "mixer_server": "${MIXER_SERVER}" } }, { @@ -62,7 +62,7 @@ "lb_type": "round_robin", "hosts": [ { - "url": "tcp://localhost:8080" + "url": "tcp://${BACKEND}" } ] } diff --git a/src/envoy/mixer/start_envoy b/src/envoy/mixer/start_envoy new file mode 100755 index 00000000000..0a24b959ae2 --- /dev/null +++ b/src/envoy/mixer/start_envoy @@ -0,0 +1,67 @@ +#!/bin/bash +# +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ +# + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" + +function usage() { + [[ -n "${1}" ]] && echo "${1}" + + cat < "${CONFIG}" + +"${ENVOY}" -c "${CONFIG}" "${DEBUG}" + From 673b902a4da5b03fdad670af363ca265864505e9 Mon Sep 17 00:00:00 2001 From: Wayne Zhang Date: Fri, 10 Feb 2017 11:11:57 -0800 Subject: [PATCH 27/83] Use official attribute names (#80) * Use official attribute names * fix format --- src/envoy/mixer/http_control.cc | 37 +++++++++++---------------------- 1 file changed, 12 insertions(+), 25 deletions(-) diff --git a/src/envoy/mixer/http_control.cc b/src/envoy/mixer/http_control.cc index b1b5ba816e5..6ae266ef088 100644 --- a/src/envoy/mixer/http_control.cc +++ b/src/envoy/mixer/http_control.cc @@ -24,31 +24,25 @@ namespace Http { namespace Mixer { namespace { -const std::string kProxyPeerID = "Istio/Proxy"; - // Define lower case string for X-Forwarded-Host. const LowerCaseString kHeaderNameXFH("x-forwarded-host", false); -const std::string kRequestHeaderPrefix = "requestHeader:"; -const std::string kRequestParamPrefix = "requestParameter:"; -const std::string kResponseHeaderPrefix = "responseHeader:"; +const std::string kRequestHeaderPrefix = "request.headers."; +const std::string kResponseHeaderPrefix = "response.headers."; // Define attribute names -const std::string kAttrNamePeerId = "peerId"; -const std::string kAttrNameURL = "url"; -const std::string kAttrNameHttpMethod = "httpMethod"; -const std::string kAttrNameRequestSize = "requestSize"; -const std::string kAttrNameResponseSize = "responseSize"; -const std::string kAttrNameLogMessage = "logMessage"; -const std::string kAttrNameResponseTime = "responseTime"; -const std::string kAttrNameOriginIp = "originIp"; -const std::string kAttrNameOriginHost = "originHost"; +const std::string kAttrNameRequestPath = "request.path"; +const std::string kAttrNameRequestSize = "request.size"; +const std::string kAttrNameResponseSize = "response.size"; +const std::string kAttrNameResponseTime = "response.time"; +const std::string kAttrNameOriginIp = "origin.ip"; +const std::string kAttrNameOriginHost = "origin.host"; const std::string kEnvNameSourceService = "SOURCE_SERVICE"; const std::string kEnvNameTargetService = "TARGET_SERVICE"; -const std::string kAttrNameSourceService = "sourceService"; -const std::string kAttrNameTargetService = "targetService"; +const std::string kAttrNameSourceService = "source.service"; +const std::string kAttrNameTargetService = "target.service"; Attributes::Value StringValue(const std::string& str) { Attributes::Value v; @@ -106,14 +100,8 @@ void FillRequestHeaderAttributes(const HeaderMap& header_map, }, attr); - // Pass in all Query parameters. - auto path = header_map.Path(); - if (path != nullptr) { - for (const auto& it : Utility::parseQueryString(path->value().c_str())) { - attr->attributes[kRequestParamPrefix + it.first] = StringValue(it.second); - } - } - + SetStringAttribute(kAttrNameRequestPath, header_map.Path()->value().c_str(), + attr); SetStringAttribute(kAttrNameOriginIp, GetFirstForwardedFor(header_map), attr); SetStringAttribute(kAttrNameOriginHost, GetLastForwardedHost(header_map), attr); @@ -167,7 +155,6 @@ void HttpControl::FillCheckAttributes(const HeaderMap& header_map, SetStringAttribute(kAttrNameSourceService, source_service_, attr); SetStringAttribute(kAttrNameTargetService, target_service_, attr); - attr->attributes[kAttrNamePeerId] = StringValue(kProxyPeerID); } void HttpControl::Check(HttpRequestDataPtr request_data, HeaderMap& headers, From 366f5ae62eaf24888240c8b998cc43f3a0c975ca Mon Sep 17 00:00:00 2001 From: Sebastien Vas Date: Fri, 10 Feb 2017 11:53:45 -0800 Subject: [PATCH 28/83] Creates a KEY for mixer client dep. Updates release-binary (#79) * Updated mixer repo to use a key for commit * release-binary skip build if file exists. --- script/release-binary | 15 ++++++++++----- src/envoy/mixer/repositories.bzl | 4 +++- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/script/release-binary b/script/release-binary index bf83a7d3115..61d446533b8 100755 --- a/script/release-binary +++ b/script/release-binary @@ -19,24 +19,29 @@ set -ex # Make sure to this script on x86_64 Ubuntu Xenial +UBUNTU_RELEASE="$(lsb_release -c -s)" +[[ "${UBUNTU_RELEASE}" == 'xenial' ]] || { echo 'must run on Ubuntu Xenial'; exit 1; } # The bucket name to store proxy binary -BUCKET_NAME="istio-build/proxy" +DST="gs://istio-build/proxy" # The proxy binary name. SHA="$(git rev-parse --verify HEAD)" BINARY_NAME="envoy-alpha-${SHA}.tar.gz" SHA256_NAME="envoy-alpha-${SHA}.sha256" +# If binary already exists skip. +gsutil stat "${DST}/${BINARY_NAME}" \ + && { echo 'Binary already exists'; exit 0; } \ + || echo 'Building a new binary.' + # Build the binary bazel build --config=release //src/envoy/mixer:envoy_tar BAZEL_TARGET="bazel-bin/src/envoy/mixer/envoy_tar.tar.gz" ln "${BAZEL_TARGET}" "${BINARY_NAME}" sha256sum "${BINARY_NAME}" > "${SHA256_NAME}" -DST="gs://${BUCKET_NAME}/" # Copy it to the bucket. -echo "Copying ${BINARY_NAME} ${SHA256_NAME} to ${DST}" -gsutil cp "${BINARY_NAME}" "${SHA256_NAME}" "${DST}" - +echo "Copying ${BINARY_NAME} ${SHA256_NAME} to ${DST}/" +gsutil cp "${BINARY_NAME}" "${SHA256_NAME}" "${DST}/" diff --git a/src/envoy/mixer/repositories.bzl b/src/envoy/mixer/repositories.bzl index d5dfe4ac19c..1ec26f2a47c 100644 --- a/src/envoy/mixer/repositories.bzl +++ b/src/envoy/mixer/repositories.bzl @@ -15,10 +15,12 @@ ################################################################################ # +MIXER_CLIENT = "7b8544d765f9d7d86d28770c8d27d69cbf9509ac" + def mixer_client_repositories(bind=True): native.git_repository( name = "mixerclient_git", - commit = "7b8544d765f9d7d86d28770c8d27d69cbf9509ac", + commit = MIXER_CLIENT, remote = "https://github.com/istio/mixerclient.git", ) From a39d55542630f92885911bd9e86b468444baff39 Mon Sep 17 00:00:00 2001 From: Wayne Zhang Date: Fri, 10 Feb 2017 15:36:17 -0800 Subject: [PATCH 29/83] Update src/envoy/mixer/README. (#82) --- src/envoy/mixer/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/envoy/mixer/README.md b/src/envoy/mixer/README.md index 21f7403f77c..20400c64045 100644 --- a/src/envoy/mixer/README.md +++ b/src/envoy/mixer/README.md @@ -38,7 +38,7 @@ This Proxy will use Envoy and talk to Mixer server. * Start Envoy proxy, run ``` - bazel-bin/src/envoy/mixer/start_envoy + src/envoy/mixer/start_envoy ``` * Then issue HTTP request to proxy. From 88cfb2474f4cf6c032ead911c17dc9debd779841 Mon Sep 17 00:00:00 2001 From: wattli Date: Fri, 10 Feb 2017 15:49:13 -0800 Subject: [PATCH 30/83] Fix src/envoy/mixer/README.md (#85) --- src/envoy/mixer/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/envoy/mixer/README.md b/src/envoy/mixer/README.md index 20400c64045..5bf2198b30d 100644 --- a/src/envoy/mixer/README.md +++ b/src/envoy/mixer/README.md @@ -24,6 +24,8 @@ This Proxy will use Envoy and talk to Mixer server. ``` bazel-bin/cmd/server/mixs server + --globalConfigFile testdata/globalconfig.yml + --serviceConfigFile testdata/serviceconfig.yml --logtostderr ``` The server will run at port 9091 From d5a878a10b899332639a46d83ce1e56356575baa Mon Sep 17 00:00:00 2001 From: Qian Sun Date: Mon, 13 Feb 2017 16:27:14 -0800 Subject: [PATCH 31/83] Get attributes from envoy config. (#87) * Send all attributes. * Remove unused const strings. * Address comment. --- src/envoy/mixer/http_control.cc | 24 ++++++------------------ src/envoy/mixer/http_control.h | 9 ++++----- src/envoy/mixer/http_filter.cc | 10 +++++++++- 3 files changed, 19 insertions(+), 24 deletions(-) diff --git a/src/envoy/mixer/http_control.cc b/src/envoy/mixer/http_control.cc index 6ae266ef088..870d70956fb 100644 --- a/src/envoy/mixer/http_control.cc +++ b/src/envoy/mixer/http_control.cc @@ -38,12 +38,6 @@ const std::string kAttrNameResponseTime = "response.time"; const std::string kAttrNameOriginIp = "origin.ip"; const std::string kAttrNameOriginHost = "origin.host"; -const std::string kEnvNameSourceService = "SOURCE_SERVICE"; -const std::string kEnvNameTargetService = "TARGET_SERVICE"; - -const std::string kAttrNameSourceService = "source.service"; -const std::string kAttrNameTargetService = "target.service"; - Attributes::Value StringValue(const std::string& str) { Attributes::Value v; v.type = Attributes::Value::STRING; @@ -134,27 +128,21 @@ void FillRequestInfoAttributes(const AccessLog::RequestInfo& info, } // namespace -HttpControl::HttpControl(const std::string& mixer_server) { +HttpControl::HttpControl(const std::string& mixer_server, + std::map&& attributes) + : config_attributes_(std::move(attributes)) { ::istio::mixer_client::MixerClientOptions options; options.mixer_server = mixer_server; mixer_client_ = ::istio::mixer_client::CreateMixerClient(options); - - auto source_service = getenv(kEnvNameSourceService.c_str()); - if (source_service) { - source_service_ = source_service; - } - auto target_service = getenv(kEnvNameTargetService.c_str()); - if (target_service) { - target_service_ = target_service; - } } void HttpControl::FillCheckAttributes(const HeaderMap& header_map, Attributes* attr) { FillRequestHeaderAttributes(header_map, attr); - SetStringAttribute(kAttrNameSourceService, source_service_, attr); - SetStringAttribute(kAttrNameTargetService, target_service_, attr); + for (const auto& attribute : config_attributes_) { + SetStringAttribute(attribute.first, attribute.second, attr); + } } void HttpControl::Check(HttpRequestDataPtr request_data, HeaderMap& headers, diff --git a/src/envoy/mixer/http_control.h b/src/envoy/mixer/http_control.h index 312016e419c..bf52e49a81a 100644 --- a/src/envoy/mixer/http_control.h +++ b/src/envoy/mixer/http_control.h @@ -37,7 +37,8 @@ typedef std::shared_ptr HttpRequestDataPtr; class HttpControl final : public Logger::Loggable { public: // The constructor. - HttpControl(const std::string& mixer_server); + HttpControl(const std::string& mixer_server, + std::map&& attributes); // Make mixer check call. void Check(HttpRequestDataPtr request_data, HeaderMap& headers, @@ -55,10 +56,8 @@ class HttpControl final : public Logger::Loggable { // The mixer client std::unique_ptr<::istio::mixer_client::MixerClient> mixer_client_; - // Source service - std::string source_service_; - // Target service - std::string target_service_; + // The attributes read from the config file. + std::map config_attributes_; }; } // namespace Mixer diff --git a/src/envoy/mixer/http_filter.cc b/src/envoy/mixer/http_filter.cc index ec209b1ff98..a37ac032ee1 100644 --- a/src/envoy/mixer/http_filter.cc +++ b/src/envoy/mixer/http_filter.cc @@ -96,7 +96,15 @@ class Config : public Logger::Loggable { __func__); } - http_control_ = std::make_shared(mixer_server); + std::map attributes; + if (config.hasObject("attributes")) { + for (const std::string& attr : config.getStringArray("attributes")) { + attributes[attr] = config.getString(attr); + } + } + + http_control_ = + std::make_shared(mixer_server, std::move(attributes)); log().debug("Called Mixer::Config contructor with mixer_server: ", mixer_server); } From 25adae9db6ab1b499815ba7a8fd87d4309cdff69 Mon Sep 17 00:00:00 2001 From: GregHanson Date: Tue, 14 Feb 2017 15:51:40 -0600 Subject: [PATCH 32/83] updated SHA to point to newer envoy with RDS API feature (#94) --- src/envoy/repositories.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/envoy/repositories.bzl b/src/envoy/repositories.bzl index 94f371ebc7d..ec140710592 100644 --- a/src/envoy/repositories.bzl +++ b/src/envoy/repositories.bzl @@ -629,6 +629,6 @@ cc_test( native.new_git_repository( name = "envoy_git", remote = "https://github.com/lyft/envoy.git", - commit = "02c6fc97b4c21d25ab596a25208fbe283e927f6a", + commit = "fa1d9680d809668fef2ec9386769c79486029f04", build_file_content = BUILD, ) From 1cdc2ac135ba642d79fb584104232b2964fd5c6e Mon Sep 17 00:00:00 2001 From: Sebastien Vas Date: Tue, 14 Feb 2017 16:35:15 -0800 Subject: [PATCH 33/83] Disable travis on stable branches (#96) --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index d8559616e6d..88be42c285b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,10 @@ sudo: required dist: xenial +branches: + except: + - stable + lang: go go: From 2f69cc985fc28e8131eba36ee24c21cfa535384e Mon Sep 17 00:00:00 2001 From: Kuat Date: Wed, 15 Feb 2017 14:36:06 -0800 Subject: [PATCH 34/83] Publish debug binaries (no release yet) (#98) --- script/release-binary | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/script/release-binary b/script/release-binary index 61d446533b8..13c4aea2bc7 100755 --- a/script/release-binary +++ b/script/release-binary @@ -36,7 +36,7 @@ gsutil stat "${DST}/${BINARY_NAME}" \ || echo 'Building a new binary.' # Build the binary -bazel build --config=release //src/envoy/mixer:envoy_tar +bazel build -c dbg //src/envoy/mixer:envoy_tar BAZEL_TARGET="bazel-bin/src/envoy/mixer/envoy_tar.tar.gz" ln "${BAZEL_TARGET}" "${BINARY_NAME}" sha256sum "${BINARY_NAME}" > "${SHA256_NAME}" @@ -44,4 +44,3 @@ sha256sum "${BINARY_NAME}" > "${SHA256_NAME}" # Copy it to the bucket. echo "Copying ${BINARY_NAME} ${SHA256_NAME} to ${DST}/" gsutil cp "${BINARY_NAME}" "${SHA256_NAME}" "${DST}/" - From 84881332f5e3f31e64b06e46d61964b5ed390bac Mon Sep 17 00:00:00 2001 From: Sebastien Vas Date: Thu, 16 Feb 2017 13:38:01 -0800 Subject: [PATCH 35/83] Copies the binary instead of linking for release (#102) --- script/release-binary | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/release-binary b/script/release-binary index 13c4aea2bc7..98bbc67a173 100755 --- a/script/release-binary +++ b/script/release-binary @@ -38,7 +38,7 @@ gsutil stat "${DST}/${BINARY_NAME}" \ # Build the binary bazel build -c dbg //src/envoy/mixer:envoy_tar BAZEL_TARGET="bazel-bin/src/envoy/mixer/envoy_tar.tar.gz" -ln "${BAZEL_TARGET}" "${BINARY_NAME}" +cp -f "${BAZEL_TARGET}" "${BINARY_NAME}" sha256sum "${BINARY_NAME}" > "${SHA256_NAME}" # Copy it to the bucket. From 0804a07680b2bc6a6ab93ffccead554b9680b6b5 Mon Sep 17 00:00:00 2001 From: Wayne Zhang Date: Tue, 21 Feb 2017 12:46:14 -0800 Subject: [PATCH 36/83] Not to use api_key if its service is not actived. (#109) --- .../src/api_manager/context/request_context.cc | 3 ++- .../src/api_manager/service_control/proto.cc | 15 +++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/contrib/endpoints/src/api_manager/context/request_context.cc b/contrib/endpoints/src/api_manager/context/request_context.cc index 59cc7c3d53f..75bcb177f9d 100644 --- a/contrib/endpoints/src/api_manager/context/request_context.cc +++ b/contrib/endpoints/src/api_manager/context/request_context.cc @@ -165,7 +165,8 @@ void RequestContext::FillOperationInfo(service_control::OperationInfo *info) { info->operation_name = kUnrecognizedOperation; } info->operation_id = operation_id_; - if (check_response_info_.is_api_key_valid) { + if (check_response_info_.is_api_key_valid && + check_response_info_.service_is_activated) { info->api_key = api_key_; } info->producer_project_id = service_context()->project_id(); diff --git a/contrib/endpoints/src/api_manager/service_control/proto.cc b/contrib/endpoints/src/api_manager/service_control/proto.cc index 3e7eba64d21..694f299d22c 100644 --- a/contrib/endpoints/src/api_manager/service_control/proto.cc +++ b/contrib/endpoints/src/api_manager/service_control/proto.cc @@ -956,14 +956,13 @@ Status Proto::FillReportRequest(const ReportRequestInfo& info, } } - // Not to send consumer metrics for following cases: - // 1) api_key is not provided, or - // 2) the service is not activated for the consumer project, - bool send_consumer_metric = true; - if (info.api_key.empty() || - !info.check_response_info.service_is_activated) { - send_consumer_metric = false; - } + // Not to send consumer metrics if api_key is empty. + // api_key is empty in one of following cases: + // 1) api_key is not provided, + // 2) api_key is invalid determined by the server from the Check call. + // 3) the service is not activated for the consumer project. + bool send_consumer_metric = !info.api_key.empty(); + // Populate all metrics. for (auto it = metrics_.begin(), end = metrics_.end(); it != end; it++) { const SupportedMetric* m = *it; From 7444da1c3c22abaab03e9fbe67ba3e77a1cea590 Mon Sep 17 00:00:00 2001 From: Kuat Date: Tue, 21 Feb 2017 17:36:10 -0800 Subject: [PATCH 37/83] Update envoy and add c-ares (#107) * Update envoy and add c-ares depedencies * Update release script with debug and normal binary * remove debug ls * formatting --- script/release-binary | 14 ++++- src/envoy/repositories.bzl | 125 ++++++++++++++++++++++++++++++++++++- 2 files changed, 137 insertions(+), 2 deletions(-) diff --git a/script/release-binary b/script/release-binary index 98bbc67a173..0ff4dd63f91 100755 --- a/script/release-binary +++ b/script/release-binary @@ -35,7 +35,19 @@ gsutil stat "${DST}/${BINARY_NAME}" \ && { echo 'Binary already exists'; exit 0; } \ || echo 'Building a new binary.' -# Build the binary +# Build the release binary +bazel build --config=release //src/envoy/mixer:envoy_tar +BAZEL_TARGET="bazel-bin/src/envoy/mixer/envoy_tar.tar.gz" +cp -f "${BAZEL_TARGET}" "${BINARY_NAME}" +sha256sum "${BINARY_NAME}" > "${SHA256_NAME}" + +# Copy it to the bucket. +echo "Copying ${BINARY_NAME} ${SHA256_NAME} to ${DST}/" +gsutil cp "${BINARY_NAME}" "${SHA256_NAME}" "${DST}/" + +# Build the debug binary +BINARY_NAME="envoy-debug-${SHA}.tar.gz" +SHA256_NAME="envoy-debug-${SHA}.sha256" bazel build -c dbg //src/envoy/mixer:envoy_tar BAZEL_TARGET="bazel-bin/src/envoy/mixer/envoy_tar.tar.gz" cp -f "${BAZEL_TARGET}" "${BINARY_NAME}" diff --git a/src/envoy/repositories.bzl b/src/envoy/repositories.bzl index ec140710592..150f445015e 100644 --- a/src/envoy/repositories.bzl +++ b/src/envoy/repositories.bzl @@ -442,6 +442,127 @@ cc_library( actual = "@nghttp2_tar//:nghttp2", ) +def ares_repositories(bind=True): + BUILD = """ +cc_library( + name = "ares", + srcs = [ + "ares__close_sockets.c", + "ares__get_hostent.c", + "ares__read_line.c", + "ares__timeval.c", + "ares_cancel.c", + "ares_create_query.c", + "ares_data.c", + "ares_destroy.c", + "ares_expand_name.c", + "ares_expand_string.c", + "ares_fds.c", + "ares_free_hostent.c", + "ares_free_string.c", + "ares_getenv.c", + "ares_gethostbyaddr.c", + "ares_gethostbyname.c", + "ares_getnameinfo.c", + "ares_getopt.c", + "ares_getsock.c", + "ares_init.c", + "ares_library_init.c", + "ares_llist.c", + "ares_mkquery.c", + "ares_nowarn.c", + "ares_options.c", + "ares_parse_a_reply.c", + "ares_parse_aaaa_reply.c", + "ares_parse_mx_reply.c", + "ares_parse_naptr_reply.c", + "ares_parse_ns_reply.c", + "ares_parse_ptr_reply.c", + "ares_parse_soa_reply.c", + "ares_parse_srv_reply.c", + "ares_parse_txt_reply.c", + "ares_platform.c", + "ares_process.c", + "ares_query.c", + "ares_search.c", + "ares_send.c", + "ares_strcasecmp.c", + "ares_strdup.c", + "ares_strerror.c", + "ares_timeout.c", + "ares_version.c", + "ares_writev.c", + "bitncmp.c", + "inet_net_pton.c", + "inet_ntop.c", + "windows_port.c", + ], + hdrs = [ + "ares_config.h", + "ares.h", + "ares_build.h", + "ares_data.h", + "ares_dns.h", + "ares_getenv.h", + "ares_getopt.h", + "ares_inet_net_pton.h", + "ares_iphlpapi.h", + "ares_ipv6.h", + "ares_library_init.h", + "ares_llist.h", + "ares_nowarn.h", + "ares_platform.h", + "ares_private.h", + "ares_rules.h", + "ares_setup.h", + "ares_strcasecmp.h", + "ares_strdup.h", + "ares_version.h", + "ares_writev.h", + "bitncmp.h", + "nameser.h", + "setup_once.h", + ], + copts = [ + "-DHAVE_CONFIG_H", + ], + includes = ["."], + visibility = ["//visibility:public"], +) + +genrule( + name = "config", + srcs = glob(["**/*"]), + outs = ["ares_config.h"], + cmd = "pushd external/cares_git ; ./buildconf ; ./configure ; cp ares_config.h ../../$@", + visibility = ["//visibility:public"], +) + +genrule( + name = "ares_build", + srcs = [ + "ares_build.h.dist", + ], + outs = [ + "ares_build.h", + ], + cmd = "cp $(SRCS) $@", + visibility = ["//visibility:public"], +) +""" + + native.new_git_repository( + name = "cares_git", + remote = "https://github.com/c-ares/c-ares.git", + commit = "7691f773af79bf75a62d1863fd0f13ebf9dc51b1", # v1.12.0 + build_file_content = BUILD, + ) + + if bind: + native.bind( + name = "ares", + actual = "@cares_git//:ares", + ) def envoy_repositories(bind=True): libevent_repositories(bind) @@ -451,6 +572,7 @@ def envoy_repositories(bind=True): http_parser_repositories(bind) rapidjson_repositories(bind) nghttp2_repositories(bind) + ares_repositories(bind) BUILD = """ load("@protobuf_git//:protobuf.bzl", "cc_proto_library") @@ -541,6 +663,7 @@ cc_library( alwayslink=1, deps = [ ":envoy-ratelimit-pb", + "//external:ares", "//external:libssl", "//external:nghttp2", "//external:spdlog", @@ -629,6 +752,6 @@ cc_test( native.new_git_repository( name = "envoy_git", remote = "https://github.com/lyft/envoy.git", - commit = "fa1d9680d809668fef2ec9386769c79486029f04", + commit = "abef64f738d269f6c7f8265233d3ea377eef9596", # 2/16/2017 build_file_content = BUILD, ) From d09325482e840f9ea73b96791535e88b04387a55 Mon Sep 17 00:00:00 2001 From: Qian Sun Date: Tue, 21 Feb 2017 18:29:40 -0800 Subject: [PATCH 38/83] Send StatusCode Attributes to Mixer. (#110) --- src/envoy/mixer/http_control.cc | 15 ++++++++++++--- src/envoy/mixer/http_control.h | 2 +- src/envoy/mixer/http_filter.cc | 12 +++++++----- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/envoy/mixer/http_control.cc b/src/envoy/mixer/http_control.cc index 870d70956fb..bbff6b94992 100644 --- a/src/envoy/mixer/http_control.cc +++ b/src/envoy/mixer/http_control.cc @@ -37,6 +37,7 @@ const std::string kAttrNameResponseSize = "response.size"; const std::string kAttrNameResponseTime = "response.time"; const std::string kAttrNameOriginIp = "origin.ip"; const std::string kAttrNameOriginHost = "origin.host"; +const std::string kResponseStatusCode = "response.status.code"; Attributes::Value StringValue(const std::string& str) { Attributes::Value v; @@ -113,7 +114,7 @@ void FillResponseHeaderAttributes(const HeaderMap& header_map, } void FillRequestInfoAttributes(const AccessLog::RequestInfo& info, - Attributes* attr) { + int check_status_code, Attributes* attr) { if (info.bytesReceived() >= 0) { attr->attributes[kAttrNameRequestSize] = Int64Value(info.bytesReceived()); } @@ -124,6 +125,13 @@ void FillRequestInfoAttributes(const AccessLog::RequestInfo& info, attr->attributes[kAttrNameResponseTime] = Int64Value(info.duration().count()); } + if (info.responseCode().valid()) { + attr->attributes[kResponseStatusCode] = + StringValue(std::to_string(info.responseCode().value())); + } else { + attr->attributes[kResponseStatusCode] = + StringValue(std::to_string(check_status_code)); + } } } // namespace @@ -155,11 +163,12 @@ void HttpControl::Check(HttpRequestDataPtr request_data, HeaderMap& headers, void HttpControl::Report(HttpRequestDataPtr request_data, const HeaderMap* response_headers, const AccessLog::RequestInfo& request_info, - DoneFunc on_done) { + int check_status, DoneFunc on_done) { // Use all Check attributes for Report. // Add additional Report attributes. FillResponseHeaderAttributes(*response_headers, &request_data->attributes); - FillRequestInfoAttributes(request_info, &request_data->attributes); + FillRequestInfoAttributes(request_info, check_status, + &request_data->attributes); log().debug("Send Report: {}", request_data->attributes.DebugString()); mixer_client_->Report(request_data->attributes, on_done); } diff --git a/src/envoy/mixer/http_control.h b/src/envoy/mixer/http_control.h index bf52e49a81a..65863d950fa 100644 --- a/src/envoy/mixer/http_control.h +++ b/src/envoy/mixer/http_control.h @@ -47,7 +47,7 @@ class HttpControl final : public Logger::Loggable { // Make mixer report call. void Report(HttpRequestDataPtr request_data, const HeaderMap* response_headers, - const AccessLog::RequestInfo& request_info, + const AccessLog::RequestInfo& request_info, int check_status_code, ::istio::mixer_client::DoneFunc on_done); private: diff --git a/src/envoy/mixer/http_filter.cc b/src/envoy/mixer/http_filter.cc index a37ac032ee1..d0796b506dd 100644 --- a/src/envoy/mixer/http_filter.cc +++ b/src/envoy/mixer/http_filter.cc @@ -105,7 +105,7 @@ class Config : public Logger::Loggable { http_control_ = std::make_shared(mixer_server, std::move(attributes)); - log().debug("Called Mixer::Config contructor with mixer_server: ", + log().debug("Called Mixer::Config constructor with mixer_server: ", mixer_server); } @@ -126,12 +126,14 @@ class Instance : public Http::StreamFilter, public Http::AccessLog::Instance { StreamEncoderFilterCallbacks* encoder_callbacks_; bool initiating_call_; + int check_status_code_; public: Instance(ConfigPtr config) : http_control_(config->http_control()), state_(NotStarted), - initiating_call_(false) { + initiating_call_(false), + check_status_code_(HttpCode(StatusCode::UNKNOWN)) { Log().debug("Called Mixer::Instance : {}", __func__); } @@ -190,8 +192,8 @@ class Instance : public Http::StreamFilter, public Http::AccessLog::Instance { status.ToString()); if (!status.ok() && state_ != Responded) { state_ = Responded; - Utility::sendLocalReply(*decoder_callbacks_, - Code(HttpCode(status.error_code())), + check_status_code_ = HttpCode(status.error_code()); + Utility::sendLocalReply(*decoder_callbacks_, Code(check_status_code_), status.ToString()); return; } @@ -229,7 +231,7 @@ class Instance : public Http::StreamFilter, public Http::AccessLog::Instance { // The class may be gone when it is called. // Log() is a static function so it is OK. http_control_->Report(request_data_, response_headers, request_info, - [](const Status& status) { + check_status_code_, [](const Status& status) { Log().debug("Report returns status: {}", status.ToString()); }); From ef1aecf0f18cfbdb44a0fa247eba510b4785595c Mon Sep 17 00:00:00 2001 From: Wayne Zhang Date: Wed, 22 Feb 2017 10:04:36 -0800 Subject: [PATCH 39/83] Add send_attribute filter. (#115) * Add send_attribute filter. * Fix format * rename variable serialized_attributes_ * Address the comments. --- src/envoy/mixer/BUILD | 13 +++ src/envoy/mixer/README.md | 48 +++++++++ src/envoy/mixer/envoy.conf.template | 68 +++++++++++- src/envoy/mixer/forward_attribute_filter.cc | 114 ++++++++++++++++++++ src/envoy/mixer/http_control.cc | 30 ++++-- src/envoy/mixer/http_control.h | 4 +- src/envoy/mixer/http_filter.cc | 22 ++-- src/envoy/mixer/string_map.proto | 23 ++++ src/envoy/mixer/utils.cc | 50 +++++++++ src/envoy/mixer/utils.h | 39 +++++++ 10 files changed, 391 insertions(+), 20 deletions(-) create mode 100644 src/envoy/mixer/forward_attribute_filter.cc create mode 100644 src/envoy/mixer/string_map.proto create mode 100644 src/envoy/mixer/utils.cc create mode 100644 src/envoy/mixer/utils.h diff --git a/src/envoy/mixer/BUILD b/src/envoy/mixer/BUILD index 917e627e073..d04b5105029 100644 --- a/src/envoy/mixer/BUILD +++ b/src/envoy/mixer/BUILD @@ -17,15 +17,28 @@ load("@bazel_tools//tools/build_defs/pkg:pkg.bzl", "pkg_tar") load("@bazel_tools//tools/build_defs/docker:docker.bzl", "docker_build") +load("@protobuf_git//:protobuf.bzl", "cc_proto_library") + +cc_proto_library( + name = "string_map_proto", + srcs = ["string_map.proto"], + default_runtime = "//external:protobuf", + protoc = "//external:protoc", + visibility = ["//visibility:public"], +) cc_library( name = "filter_lib", srcs = [ + "forward_attribute_filter.cc", "http_control.cc", "http_control.h", "http_filter.cc", + "utils.cc", + "utils.h", ], deps = [ + ":string_map_proto", "//external:mixer_client_lib", "@envoy_git//:envoy-common", ], diff --git a/src/envoy/mixer/README.md b/src/envoy/mixer/README.md index 5bf2198b30d..d7d39f30e03 100644 --- a/src/envoy/mixer/README.md +++ b/src/envoy/mixer/README.md @@ -49,3 +49,51 @@ This Proxy will use Envoy and talk to Mixer server. curl http://localhost:9090/echo -d "hello world" ``` +## How to configurate HTTP filters + +This module has two HTTP filters: +1. mixer filter: intercept all HTTP requests, call the mixer. +2. forward_attribute filter: Forward attributes to the upstream istio/proxy. + +### *mixer* filter: + +This filter will intercept all HTTP requests and call Mixer. Here is its config: + +``` + "filters": [ + "type": "both", + "name": "mixer", + "config": { + "mixer_server": "${MIXER_SERVER}", + "attributes" : { + "attribute_name1": "attribute_value1", + "attribute_name2": "attribute_value2" + } + } +``` + +Notes: +* mixer_server is required +* attributes: these attributes will be send to the mixer + +### *forward_attribute* HTTP filter: + +This filer will forward attributes to the upstream istio/proxy. + +``` + "filters": [ + "type": "decoder", + "name": "forward_attribute", + "config": { + "attributes": { + "attribute_name1": "attribute_value1", + "attribute_name2": "attribute_value2" + } + } +``` + +Notes: +* attributes: these attributes will be forwarded to the upstream istio/proxy. + + + diff --git a/src/envoy/mixer/envoy.conf.template b/src/envoy/mixer/envoy.conf.template index f1a00d1fb3e..c5377632388 100644 --- a/src/envoy/mixer/envoy.conf.template +++ b/src/envoy/mixer/envoy.conf.template @@ -35,7 +35,62 @@ "type": "both", "name": "mixer", "config": { - "mixer_server": "${MIXER_SERVER}" + "mixer_server": "${MIXER_SERVER}", + "attributes": { + "target.uid": "POD222", + "target.namespace": "XYZ222" + } + } + }, + { + "type": "decoder", + "name": "router", + "config": {} + } + ] + } + } + ] + }, + { + "port": 7070, + "bind_to_port": true, + "filters": [ + { + "type": "read", + "name": "http_connection_manager", + "config": { + "codec_type": "auto", + "stat_prefix": "ingress_http", + "route_config": { + "virtual_hosts": [ + { + "name": "backend", + "domains": ["*"], + "routes": [ + { + "timeout_ms": 0, + "prefix": "/", + "cluster": "service2" + } + ] + } + ] + }, + "access_log": [ + { + "path": "/dev/stdout" + } + ], + "filters": [ + { + "type": "decoder", + "name": "forward_attribute", + "config": { + "attributes": { + "source.uid": "POD11", + "source.namespace": "XYZ11" + } } }, { @@ -65,6 +120,17 @@ "url": "tcp://${BACKEND}" } ] + }, + { + "name": "service2", + "connect_timeout_ms": 5000, + "type": "strict_dns", + "lb_type": "round_robin", + "hosts": [ + { + "url": "tcp://localhost:9090" + } + ] } ] } diff --git a/src/envoy/mixer/forward_attribute_filter.cc b/src/envoy/mixer/forward_attribute_filter.cc new file mode 100644 index 00000000000..08d58916f89 --- /dev/null +++ b/src/envoy/mixer/forward_attribute_filter.cc @@ -0,0 +1,114 @@ +/* Copyright 2017 Istio Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "precompiled/precompiled.h" + +#include "common/common/base64.h" +#include "common/common/logger.h" +#include "common/http/headers.h" +#include "common/http/utility.h" +#include "envoy/server/instance.h" +#include "server/config/network/http_connection_manager.h" +#include "src/envoy/mixer/utils.h" + +namespace Http { +namespace ForwardAttribute { +namespace { + +// The Json object name to specify attributes which will be forwarded +// to the upstream istio proxy. +const std::string kJsonNameAttributes("attributes"); + +} // namespace + +class Config : public Logger::Loggable { + private: + std::string attributes_; + + public: + Config(const Json::Object& config) { + Utils::StringMap attributes = + Utils::ExtractStringMap(config, kJsonNameAttributes); + if (!attributes.empty()) { + std::string serialized_str = Utils::SerializeStringMap(attributes); + attributes_ = + Base64::encode(serialized_str.c_str(), serialized_str.size()); + } + } + + const std::string& attributes() const { return attributes_; } +}; + +typedef std::shared_ptr ConfigPtr; + +class ForwardAttributeFilter : public Http::StreamDecoderFilter { + private: + ConfigPtr config_; + + public: + ForwardAttributeFilter(ConfigPtr config) : config_(config) {} + + FilterHeadersStatus decodeHeaders(HeaderMap& headers, + bool end_stream) override { + if (!config_->attributes().empty()) { + headers.addStatic(Utils::kIstioAttributeHeader, config_->attributes()); + } + return FilterHeadersStatus::Continue; + } + + FilterDataStatus decodeData(Buffer::Instance& data, + bool end_stream) override { + return FilterDataStatus::Continue; + } + + FilterTrailersStatus decodeTrailers(HeaderMap& trailers) override { + return FilterTrailersStatus::Continue; + } + + void setDecoderFilterCallbacks( + StreamDecoderFilterCallbacks& callbacks) override {} +}; + +} // namespace ForwardAttribute +} // namespace Http + +namespace Server { +namespace Configuration { + +class ForwardAttributeConfig : public HttpFilterConfigFactory { + public: + HttpFilterFactoryCb tryCreateFilterFactory( + HttpFilterType type, const std::string& name, const Json::Object& config, + const std::string&, Server::Instance& server) override { + if (type != HttpFilterType::Decoder || name != "forward_attribute") { + return nullptr; + } + + Http::ForwardAttribute::ConfigPtr add_header_config( + new Http::ForwardAttribute::Config(config)); + return [add_header_config]( + Http::FilterChainFactoryCallbacks& callbacks) -> void { + std::shared_ptr instance( + new Http::ForwardAttribute::ForwardAttributeFilter( + add_header_config)); + callbacks.addStreamDecoderFilter(Http::StreamDecoderFilterPtr(instance)); + }; + } +}; + +static RegisterHttpFilterConfigFactory register_; + +} // namespace Configuration +} // namespace server diff --git a/src/envoy/mixer/http_control.cc b/src/envoy/mixer/http_control.cc index bbff6b94992..7e1f7deffcd 100644 --- a/src/envoy/mixer/http_control.cc +++ b/src/envoy/mixer/http_control.cc @@ -1,4 +1,5 @@ -/* +/* Copyright 2017 Istio Authors. All Rights Reserved. + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -13,9 +14,14 @@ */ #include "src/envoy/mixer/http_control.h" + +#include "common/common/base64.h" #include "common/common/utility.h" #include "common/http/utility.h" +#include "src/envoy/mixer/string_map.pb.h" +#include "src/envoy/mixer/utils.h" + using ::google::protobuf::util::Status; using ::istio::mixer_client::Attributes; using ::istio::mixer_client::DoneFunc; @@ -77,7 +83,8 @@ std::string GetLastForwardedHost(const HeaderMap& header_map) { if (entry == nullptr) { return ""; } - auto xff_list = StringUtil::split(entry->value().c_str(), ','); + std::vector xff_list = + StringUtil::split(entry->value().c_str(), ','); if (xff_list.empty()) { return ""; } @@ -89,7 +96,7 @@ void FillRequestHeaderAttributes(const HeaderMap& header_map, // Pass in all headers header_map.iterate( [](const HeaderEntry& header, void* context) { - auto attr = static_cast(context); + Attributes* attr = static_cast(context); attr->attributes[kRequestHeaderPrefix + header.key().c_str()] = StringValue(header.value().c_str()); }, @@ -106,7 +113,7 @@ void FillResponseHeaderAttributes(const HeaderMap& header_map, Attributes* attr) { header_map.iterate( [](const HeaderEntry& header, void* context) { - auto attr = static_cast(context); + Attributes* attr = static_cast(context); attr->attributes[kResponseHeaderPrefix + header.key().c_str()] = StringValue(header.value().c_str()); }, @@ -144,8 +151,19 @@ HttpControl::HttpControl(const std::string& mixer_server, mixer_client_ = ::istio::mixer_client::CreateMixerClient(options); } -void HttpControl::FillCheckAttributes(const HeaderMap& header_map, - Attributes* attr) { +void HttpControl::FillCheckAttributes(HeaderMap& header_map, Attributes* attr) { + // Extract attributes from x-istio-attributes header + const HeaderEntry* entry = header_map.get(Utils::kIstioAttributeHeader); + if (entry) { + ::istio::proxy::mixer::StringMap pb; + std::string str(entry->value().c_str(), entry->value().size()); + pb.ParseFromString(Base64::decode(str)); + for (const auto& it : pb.map()) { + SetStringAttribute(it.first, it.second, attr); + } + header_map.remove(Utils::kIstioAttributeHeader); + } + FillRequestHeaderAttributes(header_map, attr); for (const auto& attribute : config_attributes_) { diff --git a/src/envoy/mixer/http_control.h b/src/envoy/mixer/http_control.h index 65863d950fa..c5938cf369d 100644 --- a/src/envoy/mixer/http_control.h +++ b/src/envoy/mixer/http_control.h @@ -1,4 +1,4 @@ -/* Copyright 2016 Google Inc. All Rights Reserved. +/* Copyright 2017 Istio Authors. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -51,7 +51,7 @@ class HttpControl final : public Logger::Loggable { ::istio::mixer_client::DoneFunc on_done); private: - void FillCheckAttributes(const HeaderMap& header_map, + void FillCheckAttributes(HeaderMap& header_map, ::istio::mixer_client::Attributes* attr); // The mixer client diff --git a/src/envoy/mixer/http_filter.cc b/src/envoy/mixer/http_filter.cc index d0796b506dd..a990b0fd7e3 100644 --- a/src/envoy/mixer/http_filter.cc +++ b/src/envoy/mixer/http_filter.cc @@ -1,4 +1,4 @@ -/* Copyright 2016 Google Inc. All Rights Reserved. +/* Copyright 2017 Istio Authors. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,6 +21,7 @@ #include "envoy/server/instance.h" #include "server/config/network/http_connection_manager.h" #include "src/envoy/mixer/http_control.h" +#include "src/envoy/mixer/utils.h" using ::google::protobuf::util::Status; using StatusCode = ::google::protobuf::util::error::Code; @@ -30,8 +31,11 @@ namespace Http { namespace Mixer { namespace { -// Define lower case string for X-Forwarded-Host. -const LowerCaseString kHeaderNameXFH("x-forwarded-host", false); +// The Json object name for mixer-server. +const std::string kJsonNameMixerServer("mixer_server"); + +// The Json object name for static attributes. +const std::string kJsonNameMixerAttributes("attributes"); // Convert Status::code to HTTP code int HttpCode(int code) { @@ -88,20 +92,16 @@ class Config : public Logger::Loggable { Config(const Json::Object& config, Server::Instance& server) : cm_(server.clusterManager()) { std::string mixer_server; - if (config.hasObject("mixer_server")) { - mixer_server = config.getString("mixer_server"); + if (config.hasObject(kJsonNameMixerServer)) { + mixer_server = config.getString(kJsonNameMixerServer); } else { log().error( "mixer_server is required but not specified in the config: {}", __func__); } - std::map attributes; - if (config.hasObject("attributes")) { - for (const std::string& attr : config.getStringArray("attributes")) { - attributes[attr] = config.getString(attr); - } - } + std::map attributes = + Utils::ExtractStringMap(config, kJsonNameMixerAttributes); http_control_ = std::make_shared(mixer_server, std::move(attributes)); diff --git a/src/envoy/mixer/string_map.proto b/src/envoy/mixer/string_map.proto new file mode 100644 index 00000000000..bd9140c37d7 --- /dev/null +++ b/src/envoy/mixer/string_map.proto @@ -0,0 +1,23 @@ +// Copyright 2017 Istio Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package istio.proxy.mixer; + +// A message with a map of string to string. It is used to serialize +// a string map. +message StringMap { + map map = 1; +} diff --git a/src/envoy/mixer/utils.cc b/src/envoy/mixer/utils.cc new file mode 100644 index 00000000000..6dc3a78356d --- /dev/null +++ b/src/envoy/mixer/utils.cc @@ -0,0 +1,50 @@ +/* Copyright 2017 Istio Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/envoy/mixer/utils.h" +#include "src/envoy/mixer/string_map.pb.h" + +namespace Http { +namespace Utils { + +const LowerCaseString kIstioAttributeHeader("x-istio-attributes"); + +StringMap ExtractStringMap(const Json::Object& json, const std::string& name) { + StringMap map; + if (json.hasObject(name)) { + Json::ObjectPtr json_obj = json.getObject(name); + Json::Object* raw_obj = json_obj.get(); + json_obj->iterate( + [&map, raw_obj](const std::string& key, const Json::Object&) -> bool { + map[key] = raw_obj->getString(key); + return true; + }); + } + return map; +} + +std::string SerializeStringMap(const StringMap& string_map) { + ::istio::proxy::mixer::StringMap pb; + ::google::protobuf::Map* map_pb = pb.mutable_map(); + for (const auto& it : string_map) { + (*map_pb)[it.first] = it.second; + } + std::string str; + pb.SerializeToString(&str); + return str; +} + +} // namespace Utils +} // namespace Http diff --git a/src/envoy/mixer/utils.h b/src/envoy/mixer/utils.h new file mode 100644 index 00000000000..9273a9a2d8b --- /dev/null +++ b/src/envoy/mixer/utils.h @@ -0,0 +1,39 @@ +/* Copyright 2017 Istio Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "precompiled/precompiled.h" + +#include "common/http/headers.h" +#include "envoy/json/json_object.h" + +namespace Http { +namespace Utils { + +// The internal header to pass istio attributes. +extern const LowerCaseString kIstioAttributeHeader; + +// The string map. +typedef std::map StringMap; + +// Extracts name/value attributes from a json object. +StringMap ExtractStringMap(const Json::Object& json, const std::string& name); + +// Serialize a string map to string. +std::string SerializeStringMap(const StringMap& map); + +} // namespace Utils +} // namespace Http From 20e19d217be714ce1440b6dcdeab21cf1c14c2de Mon Sep 17 00:00:00 2001 From: Qian Sun Date: Wed, 22 Feb 2017 12:50:08 -0800 Subject: [PATCH 40/83] Fail request if api_key is not valid. (#116) * Fail request if api_key is not valid. * Format code. * Update comments. * Address comment. --- .../src/api_manager/service_control/aggregated.cc | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/contrib/endpoints/src/api_manager/service_control/aggregated.cc b/contrib/endpoints/src/api_manager/service_control/aggregated.cc index d2754d69dbd..e59a69dc8cc 100644 --- a/contrib/endpoints/src/api_manager/service_control/aggregated.cc +++ b/contrib/endpoints/src/api_manager/service_control/aggregated.cc @@ -287,8 +287,14 @@ void Aggregated::Check( if (status.ok()) { Status status = Proto::ConvertCheckResponse( *response, service_control_proto_.service_name(), &response_info); - // If allow_unregistered_calls is true, it is always OK to proceed. - if (allow_unregistered_calls) { + // If server replied with either invalid api_key or not activated service, + // the request is rejected even allow_unregistered_calls is true. Most + // likely, users provide a wrong api key. By failing the request, the + // users will be notified with the error and have chance to correct it. + // Otherwise, the Report call will fail. It is very hard to notice and + // debug the Report failure. + if (allow_unregistered_calls && response_info.is_api_key_valid && + response_info.service_is_activated) { on_done(Status::OK, response_info); } else { on_done(status, response_info); From 7fe10df1ef85fd5b3417c58173387bd077360027 Mon Sep 17 00:00:00 2001 From: Wayne Zhang Date: Thu, 23 Feb 2017 16:04:30 -0800 Subject: [PATCH 41/83] Rename response.http.code (#125) --- src/envoy/mixer/http_control.cc | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/envoy/mixer/http_control.cc b/src/envoy/mixer/http_control.cc index 7e1f7deffcd..84e31de5528 100644 --- a/src/envoy/mixer/http_control.cc +++ b/src/envoy/mixer/http_control.cc @@ -43,7 +43,7 @@ const std::string kAttrNameResponseSize = "response.size"; const std::string kAttrNameResponseTime = "response.time"; const std::string kAttrNameOriginIp = "origin.ip"; const std::string kAttrNameOriginHost = "origin.host"; -const std::string kResponseStatusCode = "response.status.code"; +const std::string kResponseHttpCode = "response.http.code"; Attributes::Value StringValue(const std::string& str) { Attributes::Value v; @@ -133,11 +133,10 @@ void FillRequestInfoAttributes(const AccessLog::RequestInfo& info, Int64Value(info.duration().count()); } if (info.responseCode().valid()) { - attr->attributes[kResponseStatusCode] = - StringValue(std::to_string(info.responseCode().value())); + attr->attributes[kResponseHttpCode] = + Int64Value(info.responseCode().value()); } else { - attr->attributes[kResponseStatusCode] = - StringValue(std::to_string(check_status_code)); + attr->attributes[kResponseHttpCode] = Int64Value(check_status_code); } } From fdfb73f4ea245dc158d8cce5f4fd66f551099465 Mon Sep 17 00:00:00 2001 From: Wayne Zhang Date: Fri, 24 Feb 2017 17:21:27 -0800 Subject: [PATCH 42/83] Send headers as string map. (#129) * Send headers as string map. * Remove origin.ip and origin.host. * Fix format --- src/envoy/mixer/http_control.cc | 120 +++++++++++++++---------------- src/envoy/mixer/repositories.bzl | 2 +- 2 files changed, 60 insertions(+), 62 deletions(-) diff --git a/src/envoy/mixer/http_control.cc b/src/envoy/mixer/http_control.cc index 84e31de5528..677a8a9a3f3 100644 --- a/src/envoy/mixer/http_control.cc +++ b/src/envoy/mixer/http_control.cc @@ -30,19 +30,17 @@ namespace Http { namespace Mixer { namespace { -// Define lower case string for X-Forwarded-Host. -const LowerCaseString kHeaderNameXFH("x-forwarded-host", false); - -const std::string kRequestHeaderPrefix = "request.headers."; -const std::string kResponseHeaderPrefix = "response.headers."; - // Define attribute names -const std::string kAttrNameRequestPath = "request.path"; -const std::string kAttrNameRequestSize = "request.size"; -const std::string kAttrNameResponseSize = "response.size"; -const std::string kAttrNameResponseTime = "response.time"; -const std::string kAttrNameOriginIp = "origin.ip"; -const std::string kAttrNameOriginHost = "origin.host"; +const std::string kRequestPath = "request.path"; +const std::string kRequestHost = "request.host"; +const std::string kRequestSize = "request.size"; +const std::string kRequestTime = "request.time"; +const std::string kRequestHeaders = "request.headers"; + +const std::string kResponseHeaders = "response.headers"; +const std::string kResponseSize = "response.size"; +const std::string kResponseTime = "response.time"; +const std::string kResponseLatency = "response.latency"; const std::string kResponseHttpCode = "response.http.code"; Attributes::Value StringValue(const std::string& str) { @@ -52,6 +50,14 @@ Attributes::Value StringValue(const std::string& str) { return v; } +Attributes::Value StringMapValue( + std::map&& string_map) { + Attributes::Value v; + v.type = Attributes::Value::STRING_MAP; + v.string_map_v.swap(string_map); + return v; +} + Attributes::Value Int64Value(int64_t value) { Attributes::Value v; v.type = Attributes::Value::INT64; @@ -59,79 +65,70 @@ Attributes::Value Int64Value(int64_t value) { return v; } -void SetStringAttribute(const std::string& name, const std::string& value, - Attributes* attr) { - if (!value.empty()) { - attr->attributes[name] = StringValue(value); - } +Attributes::Value TimeValue( + std::chrono::time_point value) { + Attributes::Value v; + v.type = Attributes::Value::TIME; + v.time_v = value; + return v; } -std::string GetFirstForwardedFor(const HeaderMap& header_map) { - if (!header_map.ForwardedFor()) { - return ""; - } - std::vector xff_address_list = - StringUtil::split(header_map.ForwardedFor()->value().c_str(), ','); - if (xff_address_list.empty()) { - return ""; - } - return xff_address_list.front(); +Attributes::Value DurationValue(std::chrono::nanoseconds value) { + Attributes::Value v; + v.type = Attributes::Value::DURATION; + v.duration_nanos_v = value; + return v; } -std::string GetLastForwardedHost(const HeaderMap& header_map) { - const HeaderEntry* entry = header_map.get(kHeaderNameXFH); - if (entry == nullptr) { - return ""; - } - std::vector xff_list = - StringUtil::split(entry->value().c_str(), ','); - if (xff_list.empty()) { - return ""; +void SetStringAttribute(const std::string& name, const std::string& value, + Attributes* attr) { + if (!value.empty()) { + attr->attributes[name] = StringValue(value); } - return xff_list.back(); } -void FillRequestHeaderAttributes(const HeaderMap& header_map, - Attributes* attr) { - // Pass in all headers +std::map ExtractHeaders(const HeaderMap& header_map) { + std::map headers; header_map.iterate( [](const HeaderEntry& header, void* context) { - Attributes* attr = static_cast(context); - attr->attributes[kRequestHeaderPrefix + header.key().c_str()] = - StringValue(header.value().c_str()); + std::map* header_map = + static_cast*>(context); + (*header_map)[header.key().c_str()] = header.value().c_str(); }, - attr); + &headers); + return headers; +} - SetStringAttribute(kAttrNameRequestPath, header_map.Path()->value().c_str(), - attr); - SetStringAttribute(kAttrNameOriginIp, GetFirstForwardedFor(header_map), attr); - SetStringAttribute(kAttrNameOriginHost, GetLastForwardedHost(header_map), - attr); +void FillRequestHeaderAttributes(const HeaderMap& header_map, + Attributes* attr) { + SetStringAttribute(kRequestPath, header_map.Path()->value().c_str(), attr); + SetStringAttribute(kRequestHost, header_map.Host()->value().c_str(), attr); + attr->attributes[kRequestTime] = TimeValue(std::chrono::system_clock::now()); + attr->attributes[kRequestHeaders] = + StringMapValue(ExtractHeaders(header_map)); } void FillResponseHeaderAttributes(const HeaderMap& header_map, Attributes* attr) { - header_map.iterate( - [](const HeaderEntry& header, void* context) { - Attributes* attr = static_cast(context); - attr->attributes[kResponseHeaderPrefix + header.key().c_str()] = - StringValue(header.value().c_str()); - }, - attr); + attr->attributes[kResponseHeaders] = + StringMapValue(ExtractHeaders(header_map)); + attr->attributes[kResponseTime] = TimeValue(std::chrono::system_clock::now()); } void FillRequestInfoAttributes(const AccessLog::RequestInfo& info, int check_status_code, Attributes* attr) { if (info.bytesReceived() >= 0) { - attr->attributes[kAttrNameRequestSize] = Int64Value(info.bytesReceived()); + attr->attributes[kRequestSize] = Int64Value(info.bytesReceived()); } if (info.bytesSent() >= 0) { - attr->attributes[kAttrNameResponseSize] = Int64Value(info.bytesSent()); + attr->attributes[kResponseSize] = Int64Value(info.bytesSent()); } - if (info.duration().count() >= 0) { - attr->attributes[kAttrNameResponseTime] = - Int64Value(info.duration().count()); + + if (info.duration().count() > 0) { + attr->attributes[kResponseLatency] = DurationValue( + std::chrono::duration_cast(info.duration())); } + if (info.responseCode().valid()) { attr->attributes[kResponseHttpCode] = Int64Value(info.responseCode().value()); @@ -184,6 +181,7 @@ void HttpControl::Report(HttpRequestDataPtr request_data, // Use all Check attributes for Report. // Add additional Report attributes. FillResponseHeaderAttributes(*response_headers, &request_data->attributes); + FillRequestInfoAttributes(request_info, check_status, &request_data->attributes); log().debug("Send Report: {}", request_data->attributes.DebugString()); diff --git a/src/envoy/mixer/repositories.bzl b/src/envoy/mixer/repositories.bzl index 1ec26f2a47c..76a4aa9a113 100644 --- a/src/envoy/mixer/repositories.bzl +++ b/src/envoy/mixer/repositories.bzl @@ -15,7 +15,7 @@ ################################################################################ # -MIXER_CLIENT = "7b8544d765f9d7d86d28770c8d27d69cbf9509ac" +MIXER_CLIENT = "d0c57ab0d74887feb11f8644fa98538208f14dbc" def mixer_client_repositories(bind=True): native.git_repository( From 3d402e6defd367ea3e3047a86451110951a4ed22 Mon Sep 17 00:00:00 2001 From: Jason Young Date: Fri, 24 Feb 2017 17:50:23 -0800 Subject: [PATCH 43/83] unify bazel's docker build targets with other istio repos (#127) --- WORKSPACE | 8 ++++++++ docker/BUILD | 11 +++++++++++ src/envoy/mixer/BUILD | 12 ++++++++---- src/envoy/mixer/proxy_docker.bzl | 9 +++++++++ 4 files changed, 36 insertions(+), 4 deletions(-) create mode 100644 docker/BUILD create mode 100644 src/envoy/mixer/proxy_docker.bzl diff --git a/WORKSPACE b/WORKSPACE index 3b255e3fa7a..69546d28831 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -85,3 +85,11 @@ docker_build( type = "zip", url = "https://codeload.github.com/tianon/docker-brew-ubuntu-core/zip/e406914e5f648003dfe8329b512c30c9ad0d2f9c", ) + +# See github.com/istio/manager/blob/master/docker/debug/build-and-publish-debug-image.sh +# for instructions on how to re-build and publish this base image layer. +http_file( + name = "ubuntu_xenial_debug", + url = "https://storage.googleapis.com/istio-build/manager/ubuntu_xenial_debug.tar.gz", + sha256 = "02f0ea4b04012562dec4b75ee4337ac77a0003418d02a91bde1b4b4d162a41e4", +) diff --git a/docker/BUILD b/docker/BUILD new file mode 100644 index 00000000000..6e017ee6064 --- /dev/null +++ b/docker/BUILD @@ -0,0 +1,11 @@ +# Add aliases for envoy specific targets so that the docker build +# targets are consistent for all istio repos, e.g. //docker: +alias( + name = "proxy", + actual = "//src/envoy/mixer:proxy", +) + +alias( + name = "proxy_debug", + actual = "//src/envoy/mixer:proxy_debug", +) diff --git a/src/envoy/mixer/BUILD b/src/envoy/mixer/BUILD index d04b5105029..132bc364a1d 100644 --- a/src/envoy/mixer/BUILD +++ b/src/envoy/mixer/BUILD @@ -15,8 +15,9 @@ ################################################################################ # + load("@bazel_tools//tools/build_defs/pkg:pkg.bzl", "pkg_tar") -load("@bazel_tools//tools/build_defs/docker:docker.bzl", "docker_build") +load("//src/envoy/mixer:proxy_docker.bzl", "proxy_docker_build") load("@protobuf_git//:protobuf.bzl", "cc_proto_library") cc_proto_library( @@ -79,9 +80,11 @@ pkg_tar( tags = ["manual"], ) -docker_build( - name = "proxy", - base = "@docker_ubuntu//:xenial", +proxy_docker_build( + images = [ + {"name": "proxy", "base": "@docker_ubuntu//:xenial"}, + {"name": "proxy_debug", "base": "@ubuntu_xenial_debug//file"}, + ], entrypoint = [ "/usr/local/bin/start_envoy", "-e", @@ -99,4 +102,5 @@ docker_build( ":envoy_tar", ":start_envoy_tar", ], + visibility = ["//visibility:public"], ) diff --git a/src/envoy/mixer/proxy_docker.bzl b/src/envoy/mixer/proxy_docker.bzl new file mode 100644 index 00000000000..8522729628f --- /dev/null +++ b/src/envoy/mixer/proxy_docker.bzl @@ -0,0 +1,9 @@ +load("@bazel_tools//tools/build_defs/docker:docker.bzl", "docker_build") + +def proxy_docker_build(images, **kwargs): + for image in images: + docker_build( + name = image['name'], + base = image['base'], + **kwargs + ) From d71d5f3d2b4fa47d18e060b81a5b9fb3f2032f33 Mon Sep 17 00:00:00 2001 From: Jason Young Date: Tue, 28 Feb 2017 14:21:18 -0800 Subject: [PATCH 44/83] update base debug docker image reference (#133) --- WORKSPACE | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 69546d28831..ddfa05d578b 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -86,10 +86,13 @@ docker_build( url = "https://codeload.github.com/tianon/docker-brew-ubuntu-core/zip/e406914e5f648003dfe8329b512c30c9ad0d2f9c", ) + +DEBUG_BASE_IMAGE_SHA="3f57ae2aceef79e4000fb07ec850bbf4bce811e6f81dc8cfd970e16cdf33e622" + # See github.com/istio/manager/blob/master/docker/debug/build-and-publish-debug-image.sh # for instructions on how to re-build and publish this base image layer. http_file( name = "ubuntu_xenial_debug", - url = "https://storage.googleapis.com/istio-build/manager/ubuntu_xenial_debug.tar.gz", - sha256 = "02f0ea4b04012562dec4b75ee4337ac77a0003418d02a91bde1b4b4d162a41e4", + url = "https://storage.googleapis.com/istio-build/manager/ubuntu_xenial_debug-" + DEBUG_BASE_IMAGE_SHA + ".tar.gz", + sha256 = DEBUG_BASE_IMAGE_SHA, ) From f5281adb8e1a84e93815e06c67864a55e2f0d8e5 Mon Sep 17 00:00:00 2001 From: Sebastien Vas Date: Tue, 28 Feb 2017 15:12:53 -0800 Subject: [PATCH 45/83] Update postsubmit to create docker images (#132) --- Jenkinsfile | 46 +++++++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index db6abcaa5f4..a8ded63359b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -11,30 +11,20 @@ def gitUtils = new GitUtilities() def utils = new Utilities() def bazel = new Bazel() -node { - gitUtils.initialize() - // Proxy does build work correctly with Hazelcast. - // Must use .bazelrc.jenkins - bazel.setVars('', '') -} - mainFlow(utils) { - if (utils.runStage('PRESUBMIT')) { - def success = true - utils.updatePullRequest('run') - try { + pullRequest(utils) { + node { + gitUtils.initialize() + // Proxy does build work correctly with Hazelcast. + // Must use .bazelrc.jenkins + bazel.setVars('', '') + } + + if (utils.runStage('PRESUBMIT')) { presubmit(gitUtils, bazel) - } catch (Exception e) { - success = false - throw e - } finally { - utils.updatePullRequest('verify', success) } - } - if (utils.runStage('POSTSUBMIT')) { - buildNode(gitUtils) { - bazel.updateBazelRc() - sh 'script/release-binary' + if (utils.runStage('POSTSUBMIT')) { + postsubmit(gitUtils, bazel, utils) } } } @@ -59,3 +49,17 @@ def presubmit(gitUtils, bazel) { } } } + +def postsubmit(gitUtils, bazel, utils) { + buildNode(gitUtils) { + bazel.updateBazelRc() + stage('Push Binary') { + sh 'script/release-binary' + } + stage('Docker Push') { + def images = 'proxy,proxy_debug' + def tags = "${gitUtils.GIT_SHORT_SHA},\$(date +%Y-%m-%d-%H.%M.%S),latest" + utils.publishDockerImages(images, tags) + } + } +} \ No newline at end of file From 0c7c549dd722e01fb7690f9d588ae645df9dde6a Mon Sep 17 00:00:00 2001 From: Sebastien Vas Date: Tue, 28 Feb 2017 16:54:00 -0800 Subject: [PATCH 46/83] Adding config release for bazel build (#135) --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index a8ded63359b..9c59a146e50 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -59,7 +59,7 @@ def postsubmit(gitUtils, bazel, utils) { stage('Docker Push') { def images = 'proxy,proxy_debug' def tags = "${gitUtils.GIT_SHORT_SHA},\$(date +%Y-%m-%d-%H.%M.%S),latest" - utils.publishDockerImages(images, tags) + utils.publishDockerImages(images, tags, 'release') } } } \ No newline at end of file From fc9497a8d99820c9084dec8a23b5b723949bfbdf Mon Sep 17 00:00:00 2001 From: Wayne Zhang Date: Tue, 28 Feb 2017 17:11:01 -0800 Subject: [PATCH 47/83] Fix mixer client crash. (#136) --- src/envoy/mixer/http_control.cc | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/envoy/mixer/http_control.cc b/src/envoy/mixer/http_control.cc index 677a8a9a3f3..6e64659c2bb 100644 --- a/src/envoy/mixer/http_control.cc +++ b/src/envoy/mixer/http_control.cc @@ -108,10 +108,12 @@ void FillRequestHeaderAttributes(const HeaderMap& header_map, StringMapValue(ExtractHeaders(header_map)); } -void FillResponseHeaderAttributes(const HeaderMap& header_map, +void FillResponseHeaderAttributes(const HeaderMap* header_map, Attributes* attr) { - attr->attributes[kResponseHeaders] = - StringMapValue(ExtractHeaders(header_map)); + if (header_map) { + attr->attributes[kResponseHeaders] = + StringMapValue(ExtractHeaders(*header_map)); + } attr->attributes[kResponseTime] = TimeValue(std::chrono::system_clock::now()); } @@ -180,7 +182,7 @@ void HttpControl::Report(HttpRequestDataPtr request_data, int check_status, DoneFunc on_done) { // Use all Check attributes for Report. // Add additional Report attributes. - FillResponseHeaderAttributes(*response_headers, &request_data->attributes); + FillResponseHeaderAttributes(response_headers, &request_data->attributes); FillRequestInfoAttributes(request_info, check_status, &request_data->attributes); From 83e1d58c8e841c47fb9c4b14352249f9e96f927e Mon Sep 17 00:00:00 2001 From: Wayne Zhang Date: Wed, 1 Mar 2017 16:30:27 -0800 Subject: [PATCH 48/83] Get mixerclient with response parsing. (#138) --- src/envoy/mixer/repositories.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/envoy/mixer/repositories.bzl b/src/envoy/mixer/repositories.bzl index 76a4aa9a113..2f665d71c4b 100644 --- a/src/envoy/mixer/repositories.bzl +++ b/src/envoy/mixer/repositories.bzl @@ -15,7 +15,7 @@ ################################################################################ # -MIXER_CLIENT = "d0c57ab0d74887feb11f8644fa98538208f14dbc" +MIXER_CLIENT = "5b5745f29ac5a8babe79ada573defaa83f3bb9e7" def mixer_client_repositories(bind=True): native.git_repository( From a118aea2493fabbd2bfadb4dcf5f873f81980402 Mon Sep 17 00:00:00 2001 From: wattli Date: Wed, 1 Mar 2017 19:00:35 -0800 Subject: [PATCH 49/83] Update nghttp2 to sync with envoy (#140) * Fix src/envoy/mixer/README.md * Update nghttp2 to sync with envoy * update * fix typo --- src/envoy/repositories.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/envoy/repositories.bzl b/src/envoy/repositories.bzl index 150f445015e..ca7e9468ded 100644 --- a/src/envoy/repositories.bzl +++ b/src/envoy/repositories.bzl @@ -431,8 +431,8 @@ cc_library( native.new_http_archive( name = "nghttp2_tar", - url = "https://github.com/nghttp2/nghttp2/releases/download/v1.14.1/nghttp2-1.14.1.tar.gz", - strip_prefix = "nghttp2-1.14.1", + url = "https://github.com/nghttp2/nghttp2/releases/download/v1.20.0/nghttp2-1.20.0.tar.gz", + strip_prefix = "nghttp2-1.20.0", build_file_content = BUILD, ) @@ -752,6 +752,6 @@ cc_test( native.new_git_repository( name = "envoy_git", remote = "https://github.com/lyft/envoy.git", - commit = "abef64f738d269f6c7f8265233d3ea377eef9596", # 2/16/2017 + commit = "70e5d651b55d356770529e5bee9c6b2707d9cf21", # 3/1/2017 build_file_content = BUILD, ) From 9d10adc1099769ed2097d91314fa8089393f9210 Mon Sep 17 00:00:00 2001 From: wattli Date: Thu, 2 Mar 2017 15:00:52 -0800 Subject: [PATCH 50/83] Populate origin.user attribute from the SAN field of client cert (#142) * Test * test * test * revert file * address comments * test * fix typo * fix format * fix format --- src/envoy/mixer/http_control.cc | 13 ++++++++----- src/envoy/mixer/http_control.h | 2 +- src/envoy/mixer/http_filter.cc | 17 ++++++++++++++++- 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/src/envoy/mixer/http_control.cc b/src/envoy/mixer/http_control.cc index 6e64659c2bb..c78f4bd9049 100644 --- a/src/envoy/mixer/http_control.cc +++ b/src/envoy/mixer/http_control.cc @@ -31,17 +31,19 @@ namespace Mixer { namespace { // Define attribute names -const std::string kRequestPath = "request.path"; +const std::string kOriginUser = "origin.user"; + +const std::string kRequestHeaders = "request.headers"; const std::string kRequestHost = "request.host"; +const std::string kRequestPath = "request.path"; const std::string kRequestSize = "request.size"; const std::string kRequestTime = "request.time"; -const std::string kRequestHeaders = "request.headers"; const std::string kResponseHeaders = "response.headers"; +const std::string kResponseHttpCode = "response.http.code"; +const std::string kResponseLatency = "response.latency"; const std::string kResponseSize = "response.size"; const std::string kResponseTime = "response.time"; -const std::string kResponseLatency = "response.latency"; -const std::string kResponseHttpCode = "response.http.code"; Attributes::Value StringValue(const std::string& str) { Attributes::Value v; @@ -170,8 +172,9 @@ void HttpControl::FillCheckAttributes(HeaderMap& header_map, Attributes* attr) { } void HttpControl::Check(HttpRequestDataPtr request_data, HeaderMap& headers, - DoneFunc on_done) { + std::string origin_user, DoneFunc on_done) { FillCheckAttributes(headers, &request_data->attributes); + SetStringAttribute(kOriginUser, origin_user, &request_data->attributes); log().debug("Send Check: {}", request_data->attributes.DebugString()); mixer_client_->Check(request_data->attributes, on_done); } diff --git a/src/envoy/mixer/http_control.h b/src/envoy/mixer/http_control.h index c5938cf369d..e9ddc734f45 100644 --- a/src/envoy/mixer/http_control.h +++ b/src/envoy/mixer/http_control.h @@ -42,7 +42,7 @@ class HttpControl final : public Logger::Loggable { // Make mixer check call. void Check(HttpRequestDataPtr request_data, HeaderMap& headers, - ::istio::mixer_client::DoneFunc on_done); + std::string origin_user, ::istio::mixer_client::DoneFunc on_done); // Make mixer report call. void Report(HttpRequestDataPtr request_data, diff --git a/src/envoy/mixer/http_filter.cc b/src/envoy/mixer/http_filter.cc index a990b0fd7e3..c59f5f43eea 100644 --- a/src/envoy/mixer/http_filter.cc +++ b/src/envoy/mixer/http_filter.cc @@ -19,6 +19,7 @@ #include "common/http/headers.h" #include "common/http/utility.h" #include "envoy/server/instance.h" +#include "envoy/ssl/connection.h" #include "server/config/network/http_connection_manager.h" #include "src/envoy/mixer/http_control.h" #include "src/envoy/mixer/utils.h" @@ -151,8 +152,16 @@ class Instance : public Http::StreamFilter, public Http::AccessLog::Instance { state_ = Calling; initiating_call_ = true; request_data_ = std::make_shared(); + + std::string origin_user; + Ssl::Connection* ssl = + const_cast(decoder_callbacks_->ssl()); + if (ssl != nullptr) { + origin_user = ssl->uriSanPeerCertificate(); + } + http_control_->Check( - request_data_, headers, + request_data_, headers, origin_user, wrapper([this](const Status& status) { completeCheck(status); })); initiating_call_ = false; @@ -180,6 +189,7 @@ class Instance : public Http::StreamFilter, public Http::AccessLog::Instance { } return FilterTrailersStatus::Continue; } + void setDecoderFilterCallbacks( StreamDecoderFilterCallbacks& callbacks) override { Log().debug("Called Mixer::Instance : {}", __func__); @@ -187,6 +197,7 @@ class Instance : public Http::StreamFilter, public Http::AccessLog::Instance { decoder_callbacks_->addResetStreamCallback( [this]() { state_ = Responded; }); } + void completeCheck(const Status& status) { Log().debug("Called Mixer::Instance : check complete {}", status.ToString()); @@ -197,6 +208,7 @@ class Instance : public Http::StreamFilter, public Http::AccessLog::Instance { status.ToString()); return; } + state_ = Complete; if (!initiating_call_) { decoder_callbacks_->continueDecoding(); @@ -208,15 +220,18 @@ class Instance : public Http::StreamFilter, public Http::AccessLog::Instance { Log().debug("Called Mixer::Instance : {}", __func__); return FilterHeadersStatus::Continue; } + virtual FilterDataStatus encodeData(Buffer::Instance& data, bool end_stream) override { Log().debug("Called Mixer::Instance : {}", __func__); return FilterDataStatus::Continue; } + virtual FilterTrailersStatus encodeTrailers(HeaderMap& trailers) override { Log().debug("Called Mixer::Instance : {}", __func__); return FilterTrailersStatus::Continue; } + virtual void setEncoderFilterCallbacks( StreamEncoderFilterCallbacks& callbacks) override { Log().debug("Called Mixer::Instance : {}", __func__); From 6e29abce8a4d06037b854c96b8434aecac8079e6 Mon Sep 17 00:00:00 2001 From: Wayne Zhang Date: Fri, 3 Mar 2017 15:28:26 -0800 Subject: [PATCH 51/83] Update to latest mixer_client. (#145) * Update to latest mixer_client. * Updated the sha. --- src/envoy/mixer/http_control.cc | 61 ++++++++------------------------ src/envoy/mixer/repositories.bzl | 2 +- 2 files changed, 15 insertions(+), 48 deletions(-) diff --git a/src/envoy/mixer/http_control.cc b/src/envoy/mixer/http_control.cc index c78f4bd9049..3b97d20a35f 100644 --- a/src/envoy/mixer/http_control.cc +++ b/src/envoy/mixer/http_control.cc @@ -45,47 +45,10 @@ const std::string kResponseLatency = "response.latency"; const std::string kResponseSize = "response.size"; const std::string kResponseTime = "response.time"; -Attributes::Value StringValue(const std::string& str) { - Attributes::Value v; - v.type = Attributes::Value::STRING; - v.str_v = str; - return v; -} - -Attributes::Value StringMapValue( - std::map&& string_map) { - Attributes::Value v; - v.type = Attributes::Value::STRING_MAP; - v.string_map_v.swap(string_map); - return v; -} - -Attributes::Value Int64Value(int64_t value) { - Attributes::Value v; - v.type = Attributes::Value::INT64; - v.value.int64_v = value; - return v; -} - -Attributes::Value TimeValue( - std::chrono::time_point value) { - Attributes::Value v; - v.type = Attributes::Value::TIME; - v.time_v = value; - return v; -} - -Attributes::Value DurationValue(std::chrono::nanoseconds value) { - Attributes::Value v; - v.type = Attributes::Value::DURATION; - v.duration_nanos_v = value; - return v; -} - void SetStringAttribute(const std::string& name, const std::string& value, Attributes* attr) { if (!value.empty()) { - attr->attributes[name] = StringValue(value); + attr->attributes[name] = Attributes::StringValue(value); } } @@ -105,39 +68,43 @@ void FillRequestHeaderAttributes(const HeaderMap& header_map, Attributes* attr) { SetStringAttribute(kRequestPath, header_map.Path()->value().c_str(), attr); SetStringAttribute(kRequestHost, header_map.Host()->value().c_str(), attr); - attr->attributes[kRequestTime] = TimeValue(std::chrono::system_clock::now()); + attr->attributes[kRequestTime] = + Attributes::TimeValue(std::chrono::system_clock::now()); attr->attributes[kRequestHeaders] = - StringMapValue(ExtractHeaders(header_map)); + Attributes::StringMapValue(ExtractHeaders(header_map)); } void FillResponseHeaderAttributes(const HeaderMap* header_map, Attributes* attr) { if (header_map) { attr->attributes[kResponseHeaders] = - StringMapValue(ExtractHeaders(*header_map)); + Attributes::StringMapValue(ExtractHeaders(*header_map)); } - attr->attributes[kResponseTime] = TimeValue(std::chrono::system_clock::now()); + attr->attributes[kResponseTime] = + Attributes::TimeValue(std::chrono::system_clock::now()); } void FillRequestInfoAttributes(const AccessLog::RequestInfo& info, int check_status_code, Attributes* attr) { if (info.bytesReceived() >= 0) { - attr->attributes[kRequestSize] = Int64Value(info.bytesReceived()); + attr->attributes[kRequestSize] = + Attributes::Int64Value(info.bytesReceived()); } if (info.bytesSent() >= 0) { - attr->attributes[kResponseSize] = Int64Value(info.bytesSent()); + attr->attributes[kResponseSize] = Attributes::Int64Value(info.bytesSent()); } if (info.duration().count() > 0) { - attr->attributes[kResponseLatency] = DurationValue( + attr->attributes[kResponseLatency] = Attributes::DurationValue( std::chrono::duration_cast(info.duration())); } if (info.responseCode().valid()) { attr->attributes[kResponseHttpCode] = - Int64Value(info.responseCode().value()); + Attributes::Int64Value(info.responseCode().value()); } else { - attr->attributes[kResponseHttpCode] = Int64Value(check_status_code); + attr->attributes[kResponseHttpCode] = + Attributes::Int64Value(check_status_code); } } diff --git a/src/envoy/mixer/repositories.bzl b/src/envoy/mixer/repositories.bzl index 2f665d71c4b..868f52aabd0 100644 --- a/src/envoy/mixer/repositories.bzl +++ b/src/envoy/mixer/repositories.bzl @@ -15,7 +15,7 @@ ################################################################################ # -MIXER_CLIENT = "5b5745f29ac5a8babe79ada573defaa83f3bb9e7" +MIXER_CLIENT = "456d37e71a7608636ddaf5f3d1acbce015870ebf" def mixer_client_repositories(bind=True): native.git_repository( From a5567b9c2a9f1610ef067b98bfa068cb23c59b7c Mon Sep 17 00:00:00 2001 From: Wayne Zhang Date: Tue, 7 Mar 2017 14:42:58 -0800 Subject: [PATCH 52/83] Not call report if decodeHeaders is not called. (#150) --- src/envoy/mixer/http_filter.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/envoy/mixer/http_filter.cc b/src/envoy/mixer/http_filter.cc index c59f5f43eea..6a8da6a6f45 100644 --- a/src/envoy/mixer/http_filter.cc +++ b/src/envoy/mixer/http_filter.cc @@ -242,6 +242,8 @@ class Instance : public Http::StreamFilter, public Http::AccessLog::Instance { const HeaderMap* response_headers, const AccessLog::RequestInfo& request_info) override { Log().debug("Called Mixer::Instance : {}", __func__); + // If decodeHaeders() is not called, not to call Mixer report. + if (!request_data_) return; // Make sure not to use any class members at the callback. // The class may be gone when it is called. // Log() is a static function so it is OK. From c0c317dc0b2c8d433b5f9e3f349281f2f4f343b2 Mon Sep 17 00:00:00 2001 From: Wayne Zhang Date: Thu, 9 Mar 2017 15:16:52 -0800 Subject: [PATCH 53/83] Update mixerclient with sync-ed grpc write and fail-fast. (#155) * Update mixerclient with sync-ed write and fail-fast. * Update to latest test. * Update again --- src/envoy/mixer/repositories.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/envoy/mixer/repositories.bzl b/src/envoy/mixer/repositories.bzl index 868f52aabd0..473e2650882 100644 --- a/src/envoy/mixer/repositories.bzl +++ b/src/envoy/mixer/repositories.bzl @@ -15,7 +15,7 @@ ################################################################################ # -MIXER_CLIENT = "456d37e71a7608636ddaf5f3d1acbce015870ebf" +MIXER_CLIENT = "92a305961bcea90ed349ffdbb0ea299c6f6bacad" def mixer_client_repositories(bind=True): native.git_repository( From 25f43e36e7365258eb4498083c8a52d03eeee58d Mon Sep 17 00:00:00 2001 From: Wayne Zhang Date: Thu, 9 Mar 2017 17:54:38 -0800 Subject: [PATCH 54/83] Update envoy to PR553 (#156) * Update envoy to PR553 * Update libevent to 2.1.8 --- src/envoy/repositories.bzl | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/envoy/repositories.bzl b/src/envoy/repositories.bzl index ca7e9468ded..e7c5f5584da 100644 --- a/src/envoy/repositories.bzl +++ b/src/envoy/repositories.bzl @@ -62,6 +62,7 @@ event_srcs = [ "evthread.c", "evutil.c", "evutil_rand.c", + "evutil_time.c", "http.c", "listener.c", "log.c", @@ -86,9 +87,7 @@ cc_library( "defer-internal.h", "evbuffer-internal.h", "event-internal.h", - "event.h", "evthread-internal.h", - "evutil.h", "http-internal.h", "iocp-internal.h", "ipv6-internal.h", @@ -134,8 +133,8 @@ cc_library( native.new_http_archive( name = "libevent_git", - url = "https://github.com/libevent/libevent/releases/download/release-2.0.22-stable/libevent-2.0.22-stable.tar.gz", - strip_prefix = "libevent-2.0.22-stable", + url = "https://github.com/libevent/libevent/releases/download/release-2.1.8-stable/libevent-2.1.8-stable.tar.gz", + strip_prefix = "libevent-2.1.8-stable", build_file_content = BUILD, ) @@ -752,6 +751,6 @@ cc_test( native.new_git_repository( name = "envoy_git", remote = "https://github.com/lyft/envoy.git", - commit = "70e5d651b55d356770529e5bee9c6b2707d9cf21", # 3/1/2017 + commit = "9679c08a21988a8e95a4e9b2ba65712ec25eadc1", # https://github.com/lyft/envoy/pull/553 build_file_content = BUILD, ) From 5bf8276f8605cecd2ef8c5f563ce9b0f8497bf51 Mon Sep 17 00:00:00 2001 From: Sebastien Vas Date: Mon, 13 Mar 2017 09:24:07 -0700 Subject: [PATCH 55/83] Uses a specific version of the Shared Pipeline lib (#158) --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 9c59a146e50..35b530333b1 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,6 +1,6 @@ #!groovy -@Library('testutils') +@Library('testutils@stable-838b134') import org.istio.testutils.Utilities import org.istio.testutils.GitUtilities From f0e541fdc224f936e062ad26596688144ba67c7f Mon Sep 17 00:00:00 2001 From: Sarvani Vakkalanka Date: Mon, 13 Mar 2017 12:59:15 -0700 Subject: [PATCH 56/83] Update lyft/envoy commit Id to latest. (#161) * Update lyft/envoy commit Id to latest. * Remove the comment about pull request * Add new line - will delete in next commit. --- src/envoy/repositories.bzl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/envoy/repositories.bzl b/src/envoy/repositories.bzl index e7c5f5584da..a255ed1c39b 100644 --- a/src/envoy/repositories.bzl +++ b/src/envoy/repositories.bzl @@ -751,6 +751,7 @@ cc_test( native.new_git_repository( name = "envoy_git", remote = "https://github.com/lyft/envoy.git", - commit = "9679c08a21988a8e95a4e9b2ba65712ec25eadc1", # https://github.com/lyft/envoy/pull/553 + commit = "9dcac8ca111ecc8da059d1f8d42eb766b44bacd6", build_file_content = BUILD, ) + From 76d96cd4d254570c66d8b5604e47d301f7eddc7d Mon Sep 17 00:00:00 2001 From: Kuat Date: Tue, 14 Mar 2017 12:21:24 -0700 Subject: [PATCH 57/83] Update repositories.bzl (#169) --- src/envoy/repositories.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/envoy/repositories.bzl b/src/envoy/repositories.bzl index a255ed1c39b..5a74586ddc8 100644 --- a/src/envoy/repositories.bzl +++ b/src/envoy/repositories.bzl @@ -751,7 +751,7 @@ cc_test( native.new_git_repository( name = "envoy_git", remote = "https://github.com/lyft/envoy.git", - commit = "9dcac8ca111ecc8da059d1f8d42eb766b44bacd6", + commit = "b72309da41fba0c1222a72262b83bedc7294df65", # Mar 13 2017 build_file_content = BUILD, ) From 4d1dbe012056041cdbd047326d765943be0c6258 Mon Sep 17 00:00:00 2001 From: Wayne Zhang Date: Wed, 15 Mar 2017 12:10:22 -0700 Subject: [PATCH 58/83] Always set response latency (#172) --- src/envoy/mixer/http_control.cc | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/envoy/mixer/http_control.cc b/src/envoy/mixer/http_control.cc index 3b97d20a35f..50a637ca356 100644 --- a/src/envoy/mixer/http_control.cc +++ b/src/envoy/mixer/http_control.cc @@ -94,10 +94,8 @@ void FillRequestInfoAttributes(const AccessLog::RequestInfo& info, attr->attributes[kResponseSize] = Attributes::Int64Value(info.bytesSent()); } - if (info.duration().count() > 0) { - attr->attributes[kResponseLatency] = Attributes::DurationValue( - std::chrono::duration_cast(info.duration())); - } + attr->attributes[kResponseLatency] = Attributes::DurationValue( + std::chrono::duration_cast(info.duration())); if (info.responseCode().valid()) { attr->attributes[kResponseHttpCode] = From 52ef466a3fb296a4c1e11336fb92ab3f7064f102 Mon Sep 17 00:00:00 2001 From: Wayne Zhang Date: Thu, 16 Mar 2017 10:15:18 -0700 Subject: [PATCH 59/83] Update mixerclient to sync_transport change. (#178) --- src/envoy/mixer/repositories.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/envoy/mixer/repositories.bzl b/src/envoy/mixer/repositories.bzl index 473e2650882..910055aff48 100644 --- a/src/envoy/mixer/repositories.bzl +++ b/src/envoy/mixer/repositories.bzl @@ -15,7 +15,7 @@ ################################################################################ # -MIXER_CLIENT = "92a305961bcea90ed349ffdbb0ea299c6f6bacad" +MIXER_CLIENT = "e292a59fbbcbeb121d532156c0ac5aad5ab5c9aa" def mixer_client_repositories(bind=True): native.git_repository( From 531cfa007231e3af08b989b0d09ce45f6b375fd9 Mon Sep 17 00:00:00 2001 From: Kuat Date: Thu, 16 Mar 2017 17:03:05 -0700 Subject: [PATCH 60/83] Use opaque config to turn on/off forward attribute and mixer filter (#179) * Modify mixer filter * Swap defaults * Make the filter decoder only * cache mixer disabled decision --- src/envoy/mixer/BUILD | 1 - src/envoy/mixer/forward_attribute_filter.cc | 114 ------------------ src/envoy/mixer/http_filter.cc | 126 ++++++++++++++------ 3 files changed, 87 insertions(+), 154 deletions(-) delete mode 100644 src/envoy/mixer/forward_attribute_filter.cc diff --git a/src/envoy/mixer/BUILD b/src/envoy/mixer/BUILD index 132bc364a1d..6bf9ea64d72 100644 --- a/src/envoy/mixer/BUILD +++ b/src/envoy/mixer/BUILD @@ -31,7 +31,6 @@ cc_proto_library( cc_library( name = "filter_lib", srcs = [ - "forward_attribute_filter.cc", "http_control.cc", "http_control.h", "http_filter.cc", diff --git a/src/envoy/mixer/forward_attribute_filter.cc b/src/envoy/mixer/forward_attribute_filter.cc deleted file mode 100644 index 08d58916f89..00000000000 --- a/src/envoy/mixer/forward_attribute_filter.cc +++ /dev/null @@ -1,114 +0,0 @@ -/* Copyright 2017 Istio Authors. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "precompiled/precompiled.h" - -#include "common/common/base64.h" -#include "common/common/logger.h" -#include "common/http/headers.h" -#include "common/http/utility.h" -#include "envoy/server/instance.h" -#include "server/config/network/http_connection_manager.h" -#include "src/envoy/mixer/utils.h" - -namespace Http { -namespace ForwardAttribute { -namespace { - -// The Json object name to specify attributes which will be forwarded -// to the upstream istio proxy. -const std::string kJsonNameAttributes("attributes"); - -} // namespace - -class Config : public Logger::Loggable { - private: - std::string attributes_; - - public: - Config(const Json::Object& config) { - Utils::StringMap attributes = - Utils::ExtractStringMap(config, kJsonNameAttributes); - if (!attributes.empty()) { - std::string serialized_str = Utils::SerializeStringMap(attributes); - attributes_ = - Base64::encode(serialized_str.c_str(), serialized_str.size()); - } - } - - const std::string& attributes() const { return attributes_; } -}; - -typedef std::shared_ptr ConfigPtr; - -class ForwardAttributeFilter : public Http::StreamDecoderFilter { - private: - ConfigPtr config_; - - public: - ForwardAttributeFilter(ConfigPtr config) : config_(config) {} - - FilterHeadersStatus decodeHeaders(HeaderMap& headers, - bool end_stream) override { - if (!config_->attributes().empty()) { - headers.addStatic(Utils::kIstioAttributeHeader, config_->attributes()); - } - return FilterHeadersStatus::Continue; - } - - FilterDataStatus decodeData(Buffer::Instance& data, - bool end_stream) override { - return FilterDataStatus::Continue; - } - - FilterTrailersStatus decodeTrailers(HeaderMap& trailers) override { - return FilterTrailersStatus::Continue; - } - - void setDecoderFilterCallbacks( - StreamDecoderFilterCallbacks& callbacks) override {} -}; - -} // namespace ForwardAttribute -} // namespace Http - -namespace Server { -namespace Configuration { - -class ForwardAttributeConfig : public HttpFilterConfigFactory { - public: - HttpFilterFactoryCb tryCreateFilterFactory( - HttpFilterType type, const std::string& name, const Json::Object& config, - const std::string&, Server::Instance& server) override { - if (type != HttpFilterType::Decoder || name != "forward_attribute") { - return nullptr; - } - - Http::ForwardAttribute::ConfigPtr add_header_config( - new Http::ForwardAttribute::Config(config)); - return [add_header_config]( - Http::FilterChainFactoryCallbacks& callbacks) -> void { - std::shared_ptr instance( - new Http::ForwardAttribute::ForwardAttributeFilter( - add_header_config)); - callbacks.addStreamDecoderFilter(Http::StreamDecoderFilterPtr(instance)); - }; - } -}; - -static RegisterHttpFilterConfigFactory register_; - -} // namespace Configuration -} // namespace server diff --git a/src/envoy/mixer/http_filter.cc b/src/envoy/mixer/http_filter.cc index 6a8da6a6f45..59a459e166c 100644 --- a/src/envoy/mixer/http_filter.cc +++ b/src/envoy/mixer/http_filter.cc @@ -15,6 +15,7 @@ #include "precompiled/precompiled.h" +#include "common/common/base64.h" #include "common/common/logger.h" #include "common/http/headers.h" #include "common/http/utility.h" @@ -36,7 +37,17 @@ namespace { const std::string kJsonNameMixerServer("mixer_server"); // The Json object name for static attributes. -const std::string kJsonNameMixerAttributes("attributes"); +const std::string kJsonNameMixerAttributes("mixer_attributes"); + +// The Json object name to specify attributes which will be forwarded +// to the upstream istio proxy. +const std::string kJsonNameForwardAttributes("forward_attributes"); + +// Switch to turn off attribute forwarding +const std::string kJsonNameForwardSwitch("mixer_forward"); + +// Switch to turn off mixer check/report/quota +const std::string kJsonNameMixerSwitch("mixer_control"); // Convert Status::code to HTTP code int HttpCode(int code) { @@ -88,6 +99,7 @@ class Config : public Logger::Loggable { private: std::shared_ptr http_control_; Upstream::ClusterManager& cm_; + std::string forward_attributes_; public: Config(const Json::Object& config, Server::Instance& server) @@ -101,37 +113,75 @@ class Config : public Logger::Loggable { __func__); } - std::map attributes = + Utils::StringMap attributes = + Utils::ExtractStringMap(config, kJsonNameForwardAttributes); + if (!attributes.empty()) { + std::string serialized_str = Utils::SerializeStringMap(attributes); + forward_attributes_ = + Base64::encode(serialized_str.c_str(), serialized_str.size()); + log().debug("Mixer forward attributes set: ", serialized_str); + } + + std::map mixer_attributes = Utils::ExtractStringMap(config, kJsonNameMixerAttributes); - http_control_ = - std::make_shared(mixer_server, std::move(attributes)); + http_control_ = std::make_shared(mixer_server, + std::move(mixer_attributes)); log().debug("Called Mixer::Config constructor with mixer_server: ", mixer_server); } std::shared_ptr& http_control() { return http_control_; } + const std::string& forward_attributes() const { return forward_attributes_; } }; typedef std::shared_ptr ConfigPtr; -class Instance : public Http::StreamFilter, public Http::AccessLog::Instance { +class Instance : public Http::StreamDecoderFilter, + public Http::AccessLog::Instance { private: std::shared_ptr http_control_; + ConfigPtr config_; std::shared_ptr request_data_; enum State { NotStarted, Calling, Complete, Responded }; State state_; StreamDecoderFilterCallbacks* decoder_callbacks_; - StreamEncoderFilterCallbacks* encoder_callbacks_; bool initiating_call_; int check_status_code_; + bool mixer_disabled_; + + // mixer control switch (off by default) + bool mixer_disabled() { + auto route = decoder_callbacks_->route()->routeEntry(); + if (route != nullptr) { + auto key = route->opaqueConfig().find(kJsonNameMixerSwitch); + if (key != route->opaqueConfig().end() && key->second == "on") { + return false; + } + } + return true; + } + + // attribute forward switch (on by default) + bool forward_disabled() { + auto route = decoder_callbacks_->route()->routeEntry(); + if (route != nullptr) { + auto key = route->opaqueConfig().find(kJsonNameForwardSwitch); + if (key != route->opaqueConfig().end() && key->second == "off") { + return true; + } + } + return false; + } + public: Instance(ConfigPtr config) : http_control_(config->http_control()), + config_(config), state_(NotStarted), initiating_call_(false), check_status_code_(HttpCode(StatusCode::UNKNOWN)) { @@ -149,6 +199,17 @@ class Instance : public Http::StreamFilter, public Http::AccessLog::Instance { FilterHeadersStatus decodeHeaders(HeaderMap& headers, bool end_stream) override { Log().debug("Called Mixer::Instance : {}", __func__); + + if (!config_->forward_attributes().empty() && !forward_disabled()) { + headers.addStatic(Utils::kIstioAttributeHeader, + config_->forward_attributes()); + } + + mixer_disabled_ = mixer_disabled(); + if (mixer_disabled_) { + return FilterHeadersStatus::Continue; + } + state_ = Calling; initiating_call_ = true; request_data_ = std::make_shared(); @@ -174,6 +235,10 @@ class Instance : public Http::StreamFilter, public Http::AccessLog::Instance { FilterDataStatus decodeData(Buffer::Instance& data, bool end_stream) override { + if (mixer_disabled_) { + return FilterDataStatus::Continue; + } + Log().debug("Called Mixer::Instance : {} ({}, {})", __func__, data.length(), end_stream); if (state_ == Calling) { @@ -183,6 +248,10 @@ class Instance : public Http::StreamFilter, public Http::AccessLog::Instance { } FilterTrailersStatus decodeTrailers(HeaderMap& trailers) override { + if (mixer_disabled_) { + return FilterTrailersStatus::Continue; + } + Log().debug("Called Mixer::Instance : {}", __func__); if (state_ == Calling) { return FilterTrailersStatus::StopIteration; @@ -194,8 +263,10 @@ class Instance : public Http::StreamFilter, public Http::AccessLog::Instance { StreamDecoderFilterCallbacks& callbacks) override { Log().debug("Called Mixer::Instance : {}", __func__); decoder_callbacks_ = &callbacks; - decoder_callbacks_->addResetStreamCallback( - [this]() { state_ = Responded; }); + if (!mixer_disabled()) { + decoder_callbacks_->addResetStreamCallback( + [this]() { state_ = Responded; }); + } } void completeCheck(const Status& status) { @@ -215,29 +286,6 @@ class Instance : public Http::StreamFilter, public Http::AccessLog::Instance { } } - virtual FilterHeadersStatus encodeHeaders(HeaderMap& headers, - bool end_stream) override { - Log().debug("Called Mixer::Instance : {}", __func__); - return FilterHeadersStatus::Continue; - } - - virtual FilterDataStatus encodeData(Buffer::Instance& data, - bool end_stream) override { - Log().debug("Called Mixer::Instance : {}", __func__); - return FilterDataStatus::Continue; - } - - virtual FilterTrailersStatus encodeTrailers(HeaderMap& trailers) override { - Log().debug("Called Mixer::Instance : {}", __func__); - return FilterTrailersStatus::Continue; - } - - virtual void setEncoderFilterCallbacks( - StreamEncoderFilterCallbacks& callbacks) override { - Log().debug("Called Mixer::Instance : {}", __func__); - encoder_callbacks_ = &callbacks; - } - virtual void log(const HeaderMap* request_headers, const HeaderMap* response_headers, const AccessLog::RequestInfo& request_info) override { @@ -272,19 +320,19 @@ class MixerConfig : public HttpFilterConfigFactory { HttpFilterFactoryCb tryCreateFilterFactory( HttpFilterType type, const std::string& name, const Json::Object& config, const std::string&, Server::Instance& server) override { - if (type != HttpFilterType::Both || name != "mixer") { + if (type != HttpFilterType::Decoder || name != "mixer") { return nullptr; } Http::Mixer::ConfigPtr mixer_config( new Http::Mixer::Config(config, server)); - return - [mixer_config](Http::FilterChainFactoryCallbacks& callbacks) -> void { - std::shared_ptr instance( - new Http::Mixer::Instance(mixer_config)); - callbacks.addStreamFilter(Http::StreamFilterPtr(instance)); - callbacks.addAccessLogHandler(Http::AccessLog::InstancePtr(instance)); - }; + return [mixer_config]( + Http::FilterChainFactoryCallbacks& callbacks) -> void { + std::shared_ptr instance( + new Http::Mixer::Instance(mixer_config)); + callbacks.addStreamDecoderFilter(Http::StreamDecoderFilterPtr(instance)); + callbacks.addAccessLogHandler(Http::AccessLog::InstancePtr(instance)); + }; } }; From 95535f5fb6631adcb453b8f84a2e76d13ea23886 Mon Sep 17 00:00:00 2001 From: Kuat Date: Thu, 16 Mar 2017 18:29:47 -0700 Subject: [PATCH 61/83] Fix a bug in opaque config change and test it out (#182) * Fix a bug and test it out * Update filter type * Update README.md --- src/envoy/mixer/README.md | 39 ++++++++++++----------------- src/envoy/mixer/envoy.conf.template | 15 +++++++---- src/envoy/mixer/http_filter.cc | 6 ++--- 3 files changed, 28 insertions(+), 32 deletions(-) diff --git a/src/envoy/mixer/README.md b/src/envoy/mixer/README.md index d7d39f30e03..4e30eb08f56 100644 --- a/src/envoy/mixer/README.md +++ b/src/envoy/mixer/README.md @@ -46,26 +46,29 @@ This Proxy will use Envoy and talk to Mixer server. * Then issue HTTP request to proxy. ``` + # request to server-side proxy curl http://localhost:9090/echo -d "hello world" + # request to client-side proxy that gets sent to server-side proxy + curl http://localhost:7070/echo -d "hello world" ``` ## How to configurate HTTP filters -This module has two HTTP filters: -1. mixer filter: intercept all HTTP requests, call the mixer. -2. forward_attribute filter: Forward attributes to the upstream istio/proxy. - ### *mixer* filter: This filter will intercept all HTTP requests and call Mixer. Here is its config: ``` "filters": [ - "type": "both", + "type": "decoder", "name": "mixer", "config": { "mixer_server": "${MIXER_SERVER}", - "attributes" : { + "mixer_attributes" : { + "attribute_name1": "attribute_value1", + "attribute_name2": "attribute_value2" + }, + "forward_attributes" : { "attribute_name1": "attribute_value1", "attribute_name2": "attribute_value2" } @@ -74,26 +77,16 @@ This filter will intercept all HTTP requests and call Mixer. Here is its config: Notes: * mixer_server is required -* attributes: these attributes will be send to the mixer +* mixer_attributes: these attributes will be send to the mixer +* forward_attributes: these attributes will be forwarded to the upstream istio/proxy. -### *forward_attribute* HTTP filter: - -This filer will forward attributes to the upstream istio/proxy. +By default, mixer filter forwards attributes and does not invoke mixer server. You can customize this behavior per HTTP route by supplying an opaque config: ``` - "filters": [ - "type": "decoder", - "name": "forward_attribute", - "config": { - "attributes": { - "attribute_name1": "attribute_value1", - "attribute_name2": "attribute_value2" - } + "opaque_config": { + "mixer_control": "on", + "mixer_forward": "off" } ``` -Notes: -* attributes: these attributes will be forwarded to the upstream istio/proxy. - - - +This config reverts the behavior by sending requests to mixer server but not forwarding any attributes. diff --git a/src/envoy/mixer/envoy.conf.template b/src/envoy/mixer/envoy.conf.template index c5377632388..86aa0e1dc33 100644 --- a/src/envoy/mixer/envoy.conf.template +++ b/src/envoy/mixer/envoy.conf.template @@ -19,7 +19,11 @@ { "timeout_ms": 0, "prefix": "/", - "cluster": "service1" + "cluster": "service1", + "opaque_config": { + "mixer_control": "on", + "mixer_forward": "off" + } } ] } @@ -32,11 +36,11 @@ ], "filters": [ { - "type": "both", + "type": "decoder", "name": "mixer", "config": { "mixer_server": "${MIXER_SERVER}", - "attributes": { + "mixer_attributes": { "target.uid": "POD222", "target.namespace": "XYZ222" } @@ -85,9 +89,10 @@ "filters": [ { "type": "decoder", - "name": "forward_attribute", + "name": "mixer", "config": { - "attributes": { + "mixer_server": "${MIXER_SERVER}", + "forward_attributes": { "source.uid": "POD11", "source.namespace": "XYZ11" } diff --git a/src/envoy/mixer/http_filter.cc b/src/envoy/mixer/http_filter.cc index 59a459e166c..d6ca93e2bb5 100644 --- a/src/envoy/mixer/http_filter.cc +++ b/src/envoy/mixer/http_filter.cc @@ -263,10 +263,8 @@ class Instance : public Http::StreamDecoderFilter, StreamDecoderFilterCallbacks& callbacks) override { Log().debug("Called Mixer::Instance : {}", __func__); decoder_callbacks_ = &callbacks; - if (!mixer_disabled()) { - decoder_callbacks_->addResetStreamCallback( - [this]() { state_ = Responded; }); - } + decoder_callbacks_->addResetStreamCallback( + [this]() { state_ = Responded; }); } void completeCheck(const Status& status) { From 93ea507f3a141bed4bb1240101f77ce6c4844e6e Mon Sep 17 00:00:00 2001 From: Wayne Zhang Date: Fri, 17 Mar 2017 12:38:39 -0700 Subject: [PATCH 62/83] Update mixer client to mixer api with gogoproto. (#184) --- src/envoy/mixer/repositories.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/envoy/mixer/repositories.bzl b/src/envoy/mixer/repositories.bzl index 910055aff48..9d06b24a14b 100644 --- a/src/envoy/mixer/repositories.bzl +++ b/src/envoy/mixer/repositories.bzl @@ -15,7 +15,7 @@ ################################################################################ # -MIXER_CLIENT = "e292a59fbbcbeb121d532156c0ac5aad5ab5c9aa" +MIXER_CLIENT = "1d6b587755846fe1b3a44fb53e3ab9ce0534af2c" def mixer_client_repositories(bind=True): native.git_repository( From 20c5bab707136751fd7279da743826a75b950431 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Mon, 20 Mar 2017 11:04:55 -0700 Subject: [PATCH 63/83] Move .bazelrc to tools/bazel.rc (#186) * Move .bazelrc to tools/bazel.rc * Update Jenkinsfile with latest version of pipeline --- .travis.yml | 3 +-- Jenkinsfile | 29 +++++++++++++---------------- .bazelrc => tools/bazel.rc | 0 3 files changed, 14 insertions(+), 18 deletions(-) rename .bazelrc => tools/bazel.rc (100%) diff --git a/.travis.yml b/.travis.yml index 88be42c285b..418e24d2dee 100644 --- a/.travis.yml +++ b/.travis.yml @@ -35,8 +35,7 @@ before_install: - sudo dpkg -i bazel_${BAZEL_VERSION}-linux-x86_64.deb - sudo apt-get -f install -qqy uuid-dev - cd ${TRAVIS_BUILD_DIR} - - mv .bazelrc .bazelrc.orig - - cat .bazelrc.travis .bazelrc.orig > .bazelrc + - mv .bazelrc.travis .bazelrc script: - script/check-style diff --git a/Jenkinsfile b/Jenkinsfile index 35b530333b1..6575dc498f5 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,6 +1,6 @@ #!groovy -@Library('testutils@stable-838b134') +@Library('testutils@stable-63c264e') import org.istio.testutils.Utilities import org.istio.testutils.GitUtilities @@ -12,20 +12,17 @@ def utils = new Utilities() def bazel = new Bazel() mainFlow(utils) { - pullRequest(utils) { - node { - gitUtils.initialize() - // Proxy does build work correctly with Hazelcast. - // Must use .bazelrc.jenkins - bazel.setVars('', '') - } - - if (utils.runStage('PRESUBMIT')) { - presubmit(gitUtils, bazel) - } - if (utils.runStage('POSTSUBMIT')) { - postsubmit(gitUtils, bazel, utils) - } + node { + gitUtils.initialize() + // Proxy does build work correctly with Hazelcast. + // Must use .bazelrc.jenkins + bazel.setVars('', '') + } + if (utils.runStage('PRESUBMIT')) { + presubmit(gitUtils, bazel) + } + if (utils.runStage('POSTSUBMIT')) { + postsubmit(gitUtils, bazel, utils) } } @@ -62,4 +59,4 @@ def postsubmit(gitUtils, bazel, utils) { utils.publishDockerImages(images, tags, 'release') } } -} \ No newline at end of file +} diff --git a/.bazelrc b/tools/bazel.rc similarity index 100% rename from .bazelrc rename to tools/bazel.rc From 557d0c6a7f1136e98ca1a05494df0b51c5bad7d2 Mon Sep 17 00:00:00 2001 From: Jae Kim Date: Mon, 20 Mar 2017 13:42:37 -0700 Subject: [PATCH 64/83] Support apikey based traffic restriction (#189) * b/36368559 support apikey based traffic restriction * Fixed code formatting --- .../api_manager/context/request_context.cc | 14 ++++++ .../src/api_manager/service_control/info.h | 5 +++ .../src/api_manager/service_control/proto.cc | 18 ++++++++ .../api_manager/service_control/proto_test.cc | 18 ++++++++ .../testdata/check_request_android_ios.golden | 43 +++++++++++++++++++ 5 files changed, 98 insertions(+) create mode 100644 contrib/endpoints/src/api_manager/service_control/testdata/check_request_android_ios.golden diff --git a/contrib/endpoints/src/api_manager/context/request_context.cc b/contrib/endpoints/src/api_manager/context/request_context.cc index 75bcb177f9d..24508a4a981 100644 --- a/contrib/endpoints/src/api_manager/context/request_context.cc +++ b/contrib/endpoints/src/api_manager/context/request_context.cc @@ -51,6 +51,16 @@ const char kDefaultApiKeyQueryName1[] = "key"; const char kDefaultApiKeyQueryName2[] = "api_key"; const char kDefaultApiKeyHeaderName[] = "x-api-key"; +// Header for android package name, used for api key restriction check. +const char kXAndroidPackage[] = "x-android-package"; + +// Header for android certificate fingerprint, used for api key restriction +// check. +const char kXAndroidCert[] = "x-android-cert"; + +// Header for IOS bundle identifier, used for api key restriction check. +const char kXIosBundleId[] = "x-ios-bundle-identifier"; + // Default location const char kDefaultLocation[] = "us-central1"; @@ -225,6 +235,10 @@ void RequestContext::FillCheckRequestInfo( service_control::CheckRequestInfo *info) { FillOperationInfo(info); info->allow_unregistered_calls = method()->allow_unregistered_calls(); + + request_->FindHeader(kXAndroidPackage, &info->android_package_name); + request_->FindHeader(kXAndroidCert, &info->android_cert_fingerprint); + request_->FindHeader(kXIosBundleId, &info->ios_bundle_id); } void RequestContext::FillReportRequestInfo( diff --git a/contrib/endpoints/src/api_manager/service_control/info.h b/contrib/endpoints/src/api_manager/service_control/info.h index f203057cc9e..1a62fdfed13 100644 --- a/contrib/endpoints/src/api_manager/service_control/info.h +++ b/contrib/endpoints/src/api_manager/service_control/info.h @@ -74,6 +74,11 @@ struct CheckRequestInfo : public OperationInfo { // Whether the method allow unregistered calls. bool allow_unregistered_calls; + // used for api key restriction check + std::string android_package_name; + std::string android_cert_fingerprint; + std::string ios_bundle_id; + CheckRequestInfo() : allow_unregistered_calls(false) {} }; diff --git a/contrib/endpoints/src/api_manager/service_control/proto.cc b/contrib/endpoints/src/api_manager/service_control/proto.cc index 694f299d22c..cfaa9a5bf53 100644 --- a/contrib/endpoints/src/api_manager/service_control/proto.cc +++ b/contrib/endpoints/src/api_manager/service_control/proto.cc @@ -420,6 +420,12 @@ const char kServiceControlServiceAgent[] = const char kServiceControlUserAgent[] = "servicecontrol.googleapis.com/user_agent"; const char kServiceControlPlatform[] = "servicecontrol.googleapis.com/platform"; +const char kServiceControlAndroidPackageName[] = + "servicecontrol.googleapis.com/android_package_name"; +const char kServiceControlAndroidCertFingerprint[] = + "servicecontrol.googleapis.com/android_cert_fingerprint"; +const char kServiceControlIosBundleId[] = + "servicecontrol.googleapis.com/ios_bundle_id"; // User agent label value // The value for kUserAgent should be configured at service control server. @@ -928,6 +934,18 @@ Status Proto::FillCheckRequest(const CheckRequestInfo& info, (*labels)[kServiceControlUserAgent] = kUserAgent; (*labels)[kServiceControlServiceAgent] = kServiceAgentPrefix + utils::Version::instance().get(); + + if (!info.android_package_name.empty()) { + (*labels)[kServiceControlAndroidPackageName] = info.android_package_name; + } + if (!info.android_cert_fingerprint.empty()) { + (*labels)[kServiceControlAndroidCertFingerprint] = + info.android_cert_fingerprint; + } + if (!info.ios_bundle_id.empty()) { + (*labels)[kServiceControlIosBundleId] = info.ios_bundle_id; + } + return Status::OK; } diff --git a/contrib/endpoints/src/api_manager/service_control/proto_test.cc b/contrib/endpoints/src/api_manager/service_control/proto_test.cc index 609cb54abe7..0a3f05c3668 100644 --- a/contrib/endpoints/src/api_manager/service_control/proto_test.cc +++ b/contrib/endpoints/src/api_manager/service_control/proto_test.cc @@ -161,6 +161,24 @@ TEST_F(ProtoTest, FillGoodCheckRequestTest) { ASSERT_EQ(expected_text, text); } +TEST_F(ProtoTest, FillGoodCheckRequestAndroidIosTest) { + CheckRequestInfo info; + FillOperationInfo(&info); + FillCheckRequestInfo(&info); + + info.android_package_name = "com.google.cloud"; + info.android_cert_fingerprint = "AIzaSyB4Gz8nyaSaWo63IPUcy5d_L8dpKtOTSD0"; + info.ios_bundle_id = "5b40ad6af9a806305a0a56d7cb91b82a27c26909"; + + gasv1::CheckRequest request; + ASSERT_TRUE(scp_.FillCheckRequest(info, &request).ok()); + + std::string text = CheckRequestToString(&request); + std::string expected_text = + ReadTestBaseline("check_request_android_ios.golden"); + ASSERT_EQ(expected_text, text); +} + TEST_F(ProtoTest, FillNoApiKeyCheckRequestTest) { CheckRequestInfo info; info.operation_id = "operation_id"; diff --git a/contrib/endpoints/src/api_manager/service_control/testdata/check_request_android_ios.golden b/contrib/endpoints/src/api_manager/service_control/testdata/check_request_android_ios.golden new file mode 100644 index 00000000000..c847061da4f --- /dev/null +++ b/contrib/endpoints/src/api_manager/service_control/testdata/check_request_android_ios.golden @@ -0,0 +1,43 @@ +service_name: "test_service" +operation { + operation_id: "operation_id" + operation_name: "operation_name" + consumer_id: "api_key:api_key_x" + start_time { + seconds: 100000 + nanos: 100000 + } + end_time { + seconds: 100000 + nanos: 100000 + } + labels { + key: "servicecontrol.googleapis.com/android_cert_fingerprint" + value: "AIzaSyB4Gz8nyaSaWo63IPUcy5d_L8dpKtOTSD0" + } + labels { + key: "servicecontrol.googleapis.com/android_package_name" + value: "com.google.cloud" + } + labels { + key: "servicecontrol.googleapis.com/caller_ip" + value: "1.2.3.4" + } + labels { + key: "servicecontrol.googleapis.com/ios_bundle_id" + value: "5b40ad6af9a806305a0a56d7cb91b82a27c26909" + } + labels { + key: "servicecontrol.googleapis.com/referer" + value: "referer" + } + labels { + key: "servicecontrol.googleapis.com/service_agent" + value: "ESP/{{service_agent_version}}" + } + labels { + key: "servicecontrol.googleapis.com/user_agent" + value: "ESP" + } +} +service_config_id: "2016-09-19r0" From 978b9678d57d1123d7dfec135f5aa6824d26f0d5 Mon Sep 17 00:00:00 2001 From: Kuat Date: Mon, 20 Mar 2017 14:04:37 -0700 Subject: [PATCH 65/83] Fix crash in unreachable/overloaded RDS (#190) --- src/envoy/mixer/http_filter.cc | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/envoy/mixer/http_filter.cc b/src/envoy/mixer/http_filter.cc index d6ca93e2bb5..8708333de40 100644 --- a/src/envoy/mixer/http_filter.cc +++ b/src/envoy/mixer/http_filter.cc @@ -156,11 +156,14 @@ class Instance : public Http::StreamDecoderFilter, // mixer control switch (off by default) bool mixer_disabled() { - auto route = decoder_callbacks_->route()->routeEntry(); + auto route = decoder_callbacks_->route(); if (route != nullptr) { - auto key = route->opaqueConfig().find(kJsonNameMixerSwitch); - if (key != route->opaqueConfig().end() && key->second == "on") { - return false; + auto entry = route->routeEntry(); + if (entry != nullptr) { + auto key = entry->opaqueConfig().find(kJsonNameMixerSwitch); + if (key != entry->opaqueConfig().end() && key->second == "on") { + return false; + } } } return true; @@ -168,11 +171,14 @@ class Instance : public Http::StreamDecoderFilter, // attribute forward switch (on by default) bool forward_disabled() { - auto route = decoder_callbacks_->route()->routeEntry(); + auto route = decoder_callbacks_->route(); if (route != nullptr) { - auto key = route->opaqueConfig().find(kJsonNameForwardSwitch); - if (key != route->opaqueConfig().end() && key->second == "off") { - return true; + auto entry = route->routeEntry(); + if (entry != nullptr) { + auto key = entry->opaqueConfig().find(kJsonNameForwardSwitch); + if (key != entry->opaqueConfig().end() && key->second == "off") { + return true; + } } } return false; From 955d54a055c08474bf64a063e13eb81b75253b7f Mon Sep 17 00:00:00 2001 From: Wayne Zhang Date: Mon, 20 Mar 2017 14:12:20 -0700 Subject: [PATCH 66/83] Add mixer client end to end integration test. (#177) * Add mixer client end to end integration test. * Split some repositories into a separate file. * use real mixer for fake mixer_server. * Test repository * use mixer bzl file. * Use mixer repositories * Not to use mixer repository. * Add return line at the end of WORKSPACE. --- BUILD | 4 + WORKSPACE | 16 + src/envoy/mixer/BUILD | 16 +- src/envoy/mixer/integration_test/BUILD | 58 +++ .../mixer/integration_test/attributes.go | 114 ++++++ src/envoy/mixer/integration_test/envoy.conf | 142 ++++++++ src/envoy/mixer/integration_test/envoy.go | 103 ++++++ .../mixer/integration_test/http_client.go | 80 +++++ .../mixer/integration_test/http_server.go | 111 ++++++ .../mixer/integration_test/mixer_server.go | 122 +++++++ .../mixer/integration_test/mixer_test.go | 333 ++++++++++++++++++ .../mixer/integration_test/repositories.bzl | 272 ++++++++++++++ src/envoy/mixer/integration_test/setup.go | 64 ++++ 13 files changed, 1430 insertions(+), 5 deletions(-) create mode 100644 src/envoy/mixer/integration_test/BUILD create mode 100644 src/envoy/mixer/integration_test/attributes.go create mode 100644 src/envoy/mixer/integration_test/envoy.conf create mode 100644 src/envoy/mixer/integration_test/envoy.go create mode 100644 src/envoy/mixer/integration_test/http_client.go create mode 100644 src/envoy/mixer/integration_test/http_server.go create mode 100644 src/envoy/mixer/integration_test/mixer_server.go create mode 100644 src/envoy/mixer/integration_test/mixer_test.go create mode 100644 src/envoy/mixer/integration_test/repositories.bzl create mode 100644 src/envoy/mixer/integration_test/setup.go diff --git a/BUILD b/BUILD index ac06d245289..510620e8246 100644 --- a/BUILD +++ b/BUILD @@ -24,3 +24,7 @@ config_setting( }, visibility = ["//visibility:public"], ) + +load("@io_bazel_rules_go//go:def.bzl", "go_prefix") + +go_prefix("istio.io/proxy") diff --git a/WORKSPACE b/WORKSPACE index ddfa05d578b..4bc7137304c 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -96,3 +96,19 @@ http_file( url = "https://storage.googleapis.com/istio-build/manager/ubuntu_xenial_debug-" + DEBUG_BASE_IMAGE_SHA + ".tar.gz", sha256 = DEBUG_BASE_IMAGE_SHA, ) + +# Following go repositories are for building go integration test for mixer filter. +git_repository( + name = "io_bazel_rules_go", + commit = "9496d79880a7d55b8e4a96f04688d70a374eaaf4", # Mar 3, 2017 (v0.4.1) + remote = "https://github.com/bazelbuild/rules_go.git", +) + +git_repository( + name = "org_pubref_rules_protobuf", + commit = "d42e895387c658eda90276aea018056fcdcb30e4", # Mar 07 2017 (gogo* support) + remote = "https://github.com/pubref/rules_protobuf", +) + +load("//src/envoy/mixer/integration_test:repositories.bzl", "go_mixer_repositories") +go_mixer_repositories() diff --git a/src/envoy/mixer/BUILD b/src/envoy/mixer/BUILD index 6bf9ea64d72..4d57f801626 100644 --- a/src/envoy/mixer/BUILD +++ b/src/envoy/mixer/BUILD @@ -15,7 +15,6 @@ ################################################################################ # - load("@bazel_tools//tools/build_defs/pkg:pkg.bzl", "pkg_tar") load("//src/envoy/mixer:proxy_docker.bzl", "proxy_docker_build") load("@protobuf_git//:protobuf.bzl", "cc_proto_library") @@ -48,6 +47,7 @@ cc_library( cc_binary( name = "envoy", linkstatic = 1, + visibility = [":__subpackages__"], deps = [ ":filter_lib", "@envoy_git//:envoy-main", @@ -80,10 +80,6 @@ pkg_tar( ) proxy_docker_build( - images = [ - {"name": "proxy", "base": "@docker_ubuntu//:xenial"}, - {"name": "proxy_debug", "base": "@ubuntu_xenial_debug//file"}, - ], entrypoint = [ "/usr/local/bin/start_envoy", "-e", @@ -93,6 +89,16 @@ proxy_docker_build( "-t", "/etc/opt/proxy/envoy.conf.template", ], + images = [ + { + "name": "proxy", + "base": "@docker_ubuntu//:xenial", + }, + { + "name": "proxy_debug", + "base": "@ubuntu_xenial_debug//file", + }, + ], ports = ["9090"], repository = "istio", tags = ["manual"], diff --git a/src/envoy/mixer/integration_test/BUILD b/src/envoy/mixer/integration_test/BUILD new file mode 100644 index 00000000000..c45865457ce --- /dev/null +++ b/src/envoy/mixer/integration_test/BUILD @@ -0,0 +1,58 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ +# + +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = [ + "attributes.go", + "envoy.go", + "http_client.go", + "http_server.go", + "mixer_server.go", + "setup.go", + ], + deps = [ + "@com_github_gogo_protobuf//types:go_default_library", + "@com_github_golang_glog//:go_default_library", + "@com_github_golang_protobuf//proto:go_default_library", + "@com_github_googleapis_googleapis//:google/rpc", + "@com_github_istio_api//:mixer/v1", + "@com_github_istio_mixer//pkg/api:go_default_library", + "@com_github_istio_mixer//pkg/attribute:go_default_library", + "@com_github_istio_mixer//pkg/pool:go_default_library", + "@com_github_istio_mixer//pkg/tracing:go_default_library", + "@org_golang_google_grpc//:go_default_library", + ], +) + +go_test( + name = "mixer_test", + size = "small", + srcs = [ + "mixer_test.go", + ], + data = [ + "envoy.conf", + "//src/envoy/mixer:envoy", + ], + library = ":go_default_library", + # shared memory path /envoy_shared_memory_0 used by Envoy + # hot start is not working in sandbox mode. + local = True, +) diff --git a/src/envoy/mixer/integration_test/attributes.go b/src/envoy/mixer/integration_test/attributes.go new file mode 100644 index 00000000000..4c97208fd4b --- /dev/null +++ b/src/envoy/mixer/integration_test/attributes.go @@ -0,0 +1,114 @@ +// Copyright 2017 Istio Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "encoding/json" + "fmt" + "reflect" + + "istio.io/mixer/pkg/attribute" +) + +func verifyStringMap(actual map[string]string, expected map[string]interface{}) error { + for k, v := range expected { + vstring := v.(string) + // "-" make sure the key does not exist. + if vstring == "-" { + if _, ok := actual[k]; ok { + return fmt.Errorf("key %+v is NOT expected", k) + } + } else { + if val, ok := actual[k]; ok { + // "*" only check key exist + if val != vstring && vstring != "*" { + return fmt.Errorf("key %+v value doesn't match. Actual %+v, expected %+v", + k, val, vstring) + } + } else { + return fmt.Errorf("key %+v is expected", k) + } + } + } + return nil +} + +// Please see the comment at top of mixer_test.go for verification rules +func Verify(b *attribute.MutableBag, json_results string) error { + var r map[string]interface{} + if err := json.Unmarshal([]byte(json_results), &r); err != nil { + return fmt.Errorf("unable to decode json %v", err) + } + + all_keys := make(map[string]bool) + for _, k := range b.Names() { + all_keys[k] = true + } + + for k, v := range r { + switch vv := v.(type) { + case string: + // "*" means only checking key. + if vv == "*" { + if _, ok := b.Get(k); !ok { + return fmt.Errorf("attribute %+v is expected", k) + } + } else { + if val, ok := b.Get(k); ok { + if val.(string) != v.(string) { + return fmt.Errorf("attribute %+v value doesn't match. Actual %+v, expected %+v", + k, val.(string), v.(string)) + } + } else { + return fmt.Errorf("attribute %+v is expected", k) + } + } + case float64: + // Json converts all integers to float64, + // Our tests only verify size related attributes which are int64 type + if val, ok := b.Get(k); ok { + vint64 := int64(vv) + if val.(int64) != vint64 { + return fmt.Errorf("attribute %+v value doesn't match. Actual %+v, expected %+v", + k, val.(int64), vint64) + } + } else { + return fmt.Errorf("attribute %+v is expected", k) + } + case map[string]interface{}: + if val, ok := b.Get(k); ok { + if err := verifyStringMap(val.(map[string]string), v.(map[string]interface{})); err != nil { + return fmt.Errorf("attribute %+v StringMap doesn't match: %+v", k, err) + } + } else { + return fmt.Errorf("attribute %+v is expected", k) + } + default: + return fmt.Errorf("attribute %+v is of a type %+v that I don't know how to handle ", + k, reflect.TypeOf(v)) + } + delete(all_keys, k) + + } + + if len(all_keys) > 0 { + var s string + for k, _ := range all_keys { + s += k + ", " + } + return fmt.Errorf("Following attributes are not expected: %s", s) + } + return nil +} diff --git a/src/envoy/mixer/integration_test/envoy.conf b/src/envoy/mixer/integration_test/envoy.conf new file mode 100644 index 00000000000..2083d7f4e5a --- /dev/null +++ b/src/envoy/mixer/integration_test/envoy.conf @@ -0,0 +1,142 @@ +{ + "listeners": [ + { + "port": 29090, + "bind_to_port": true, + "filters": [ + { + "type": "read", + "name": "http_connection_manager", + "config": { + "codec_type": "auto", + "stat_prefix": "ingress_http", + "route_config": { + "virtual_hosts": [ + { + "name": "backend", + "domains": ["*"], + "routes": [ + { + "timeout_ms": 0, + "prefix": "/", + "cluster": "service1", + "opaque_config": { + "mixer_control": "on", + "mixer_forward": "off" + } + } + ] + } + ] + }, + "access_log": [ + { + "path": "/dev/stdout" + } + ], + "filters": [ + { + "type": "decoder", + "name": "mixer", + "config": { + "mixer_server": "localhost:29091", + "mixer_attributes": { + "target.uid": "POD222", + "target.namespace": "XYZ222" + } + } + }, + { + "type": "decoder", + "name": "router", + "config": {} + } + ] + } + } + ] + }, + { + "port": 27070, + "bind_to_port": true, + "filters": [ + { + "type": "read", + "name": "http_connection_manager", + "config": { + "codec_type": "auto", + "stat_prefix": "ingress_http", + "route_config": { + "virtual_hosts": [ + { + "name": "backend", + "domains": ["*"], + "routes": [ + { + "timeout_ms": 0, + "prefix": "/", + "cluster": "service2" + } + ] + } + ] + }, + "access_log": [ + { + "path": "/dev/stdout" + } + ], + "filters": [ + { + "type": "decoder", + "name": "mixer", + "config": { + "mixer_server": "localhost:29091", + "forward_attributes": { + "source.uid": "POD11", + "source.namespace": "XYZ11" + } + } + }, + { + "type": "decoder", + "name": "router", + "config": {} + } + ] + } + } + ] + } + ], + "admin": { + "access_log_path": "/dev/stdout", + "port": 29001 + }, + "cluster_manager": { + "clusters": [ + { + "name": "service1", + "connect_timeout_ms": 5000, + "type": "strict_dns", + "lb_type": "round_robin", + "hosts": [ + { + "url": "tcp://localhost:28080" + } + ] + }, + { + "name": "service2", + "connect_timeout_ms": 5000, + "type": "strict_dns", + "lb_type": "round_robin", + "hosts": [ + { + "url": "tcp://localhost:29090" + } + ] + } + ] + } +} diff --git a/src/envoy/mixer/integration_test/envoy.go b/src/envoy/mixer/integration_test/envoy.go new file mode 100644 index 00000000000..0f9a1fc01c7 --- /dev/null +++ b/src/envoy/mixer/integration_test/envoy.go @@ -0,0 +1,103 @@ +// Copyright 2017 Istio Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "log" + "os" + "os/exec" + "strings" +) + +func getTestBinRootPath() string { + switch { + // custom path + case os.Getenv("TEST_BIN_ROOT") != "": + return os.Getenv("TEST_BIN_ROOT") + // running under bazel + case os.Getenv("TEST_SRCDIR") != "": + return os.Getenv("TEST_SRCDIR") + "/__main__" + // running with native go + case os.Getenv("GOPATH") != "": + list := strings.Split(os.Getenv("GOPATH"), + string(os.PathListSeparator)) + return list[0] + "/bazel-bin" + default: + return "bazel-bin" + } +} + +func getTestDataRootPath() string { + switch { + // custom path + case os.Getenv("TEST_DATA_ROOT") != "": + return os.Getenv("TEST_DATA_ROOT") + // running under bazel + case os.Getenv("TEST_SRCDIR") != "": + return os.Getenv("TEST_SRCDIR") + "/__main__" + // running with native go + case os.Getenv("GOPATH") != "": + list := strings.Split(os.Getenv("GOPATH"), + string(os.PathListSeparator)) + return list[0] + default: + return "" + } +} + +type Envoy struct { + cmd *exec.Cmd +} + +// Run command and return the merged output from stderr and stdout, error code +func Run(name string, args ...string) (s string, err error) { + log.Println(">", name, strings.Join(args, " ")) + c := exec.Command(name, args...) + bytes, err := c.CombinedOutput() + s = string(bytes) + for _, line := range strings.Split(s, "\n") { + log.Println(line) + } + if err != nil { + log.Println(err) + } + return +} + +func NewEnvoy() (*Envoy, error) { + path := getTestBinRootPath() + "/src/envoy/mixer/envoy" + conf := getTestDataRootPath() + + "/src/envoy/mixer/integration_test/envoy.conf" + log.Printf("Envoy binary: %v\n", path) + log.Printf("Envoy config: %v\n", conf) + + cmd := exec.Command(path, "-c", conf, "-l", "debug") + cmd.Stderr = os.Stderr + cmd.Stdout = os.Stdout + return &Envoy{ + cmd: cmd, + }, nil +} + +func (s *Envoy) Start() error { + return s.cmd.Start() +} + +func (s *Envoy) Stop() error { + log.Printf("Kill Envoy ...\n") + err := s.cmd.Process.Kill() + log.Printf("Kill Envoy ... Done\n") + return err +} diff --git a/src/envoy/mixer/integration_test/http_client.go b/src/envoy/mixer/integration_test/http_client.go new file mode 100644 index 00000000000..0212ddf775e --- /dev/null +++ b/src/envoy/mixer/integration_test/http_client.go @@ -0,0 +1,80 @@ +// Copyright 2017 Istio Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "io/ioutil" + "log" + "net/http" + "net/url" + "strings" +) + +func HTTPGet(url string) (code int, resp_body string, err error) { + log.Println("HTTP GET", url) + client := &http.Client{} + resp, err := client.Get(url) + if err != nil { + log.Println(err) + return 0, "", err + } + defer resp.Body.Close() + body, _ := ioutil.ReadAll(resp.Body) + resp_body = string(body) + code = resp.StatusCode + log.Println(resp_body) + return code, resp_body, nil +} + +func HTTPPost(url string, content_type string, req_body string) (code int, resp_body string, err error) { + log.Println("HTTP POST", url) + client := &http.Client{} + resp, err := client.Post(url, content_type, strings.NewReader(req_body)) + if err != nil { + log.Println(err) + return 0, "", err + } + defer resp.Body.Close() + body, _ := ioutil.ReadAll(resp.Body) + resp_body = string(body) + code = resp.StatusCode + log.Println(resp_body) + return code, resp_body, nil +} + +func HTTPGetWithHeaders(l string, headers map[string]string) (code int, resp_body string, err error) { + log.Println("HTTP GET with headers: ", l) + client := &http.Client{} + req := http.Request{} + + req.Header = map[string][]string{} + for k, v := range headers { + req.Header[k] = []string{v} + } + req.Method = http.MethodGet + req.URL, _ = url.Parse(l) + + resp, err := client.Do(&req) + if err != nil { + log.Println(err) + return 0, "", err + } + defer resp.Body.Close() + body, _ := ioutil.ReadAll(resp.Body) + resp_body = string(body) + code = resp.StatusCode + log.Println(resp_body) + return code, resp_body, nil +} diff --git a/src/envoy/mixer/integration_test/http_server.go b/src/envoy/mixer/integration_test/http_server.go new file mode 100644 index 00000000000..77677286b58 --- /dev/null +++ b/src/envoy/mixer/integration_test/http_server.go @@ -0,0 +1,111 @@ +// Copyright 2017 Istio Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "fmt" + "io/ioutil" + "log" + "net" + "net/http" + "strings" + "time" +) + +const ( + FailHeader = "x-istio-backend-fail" + FailBody = "Bad request from backend." +) + +type HttpServer struct { + port uint16 + lis net.Listener +} + +func handler(w http.ResponseWriter, r *http.Request) { + log.Printf("%v %v %v %v\n", r.Method, r.URL, r.Proto, r.RemoteAddr) + for name, headers := range r.Header { + for _, h := range headers { + log.Printf("%v: %v\n", name, h) + } + } + body, err := ioutil.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // Fail if there is such header. + if r.Header.Get(FailHeader) != "" { + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte(FailBody)) + return + } + + // echo back the Content-Type and Content-Length in the response + for _, k := range []string{"Content-Type", "Content-Length"} { + if v := r.Header.Get(k); v != "" { + w.Header().Set(k, v) + } + } + w.WriteHeader(http.StatusOK) + w.Write(body) +} + +func NewHttpServer(port uint16) (*HttpServer, error) { + log.Printf("Http server listening on port %v\n", port) + lis, err := net.Listen("tcp", fmt.Sprintf(":%d", port)) + if err != nil { + log.Fatal(err) + return nil, err + } + return &HttpServer{ + port: port, + lis: lis, + }, nil +} + +func (s *HttpServer) Start() { + go func() { + http.HandleFunc("/", handler) + http.Serve(s.lis, nil) + }() + + addr := fmt.Sprintf("http://localhost:%s", s.port) + + const maxAttempts = 10 + for i := 0; i < maxAttempts; i++ { + time.Sleep(time.Second) + client := http.Client{} + log.Println("Pinging the server...") + rsp, err := client.Post( + addr+"/echo", "text/plain", strings.NewReader("PING")) + if err == nil && rsp.StatusCode == http.StatusOK { + log.Println("Got a response...") + png, err := ioutil.ReadAll(rsp.Body) + if err == nil && string(png) == "PING" { + log.Println("Server is up and running...") + return + } + } + log.Println("Will wait a second and try again.") + } +} + +func (s *HttpServer) Stop() { + log.Printf("Close HTTP server\n") + s.lis.Close() + log.Printf("Close HTTP server -- Done\n") +} diff --git a/src/envoy/mixer/integration_test/mixer_server.go b/src/envoy/mixer/integration_test/mixer_server.go new file mode 100644 index 00000000000..e36f04b484a --- /dev/null +++ b/src/envoy/mixer/integration_test/mixer_server.go @@ -0,0 +1,122 @@ +// Copyright 2017 Istio Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "context" + "fmt" + "log" + "net" + + rpc "github.com/googleapis/googleapis/google/rpc" + "google.golang.org/grpc" + + mixerpb "istio.io/api/mixer/v1" + "istio.io/mixer/pkg/api" + "istio.io/mixer/pkg/attribute" + "istio.io/mixer/pkg/pool" + "istio.io/mixer/pkg/tracing" +) + +type Handler struct { + bag *attribute.MutableBag + ch chan int + r_status rpc.Status +} + +func newHandler() *Handler { + return &Handler{ + bag: nil, + ch: make(chan int, 1), + r_status: rpc.Status{}, + } +} + +func (h *Handler) run(bag *attribute.MutableBag) rpc.Status { + h.bag = attribute.CopyBag(bag) + h.ch <- 1 + return h.r_status +} + +type MixerServer struct { + lis net.Listener + gs *grpc.Server + gp *pool.GoroutinePool + s mixerpb.MixerServer + + check *Handler + report *Handler + quota *Handler +} + +func (ts *MixerServer) Check(ctx context.Context, bag *attribute.MutableBag, + request *mixerpb.CheckRequest, response *mixerpb.CheckResponse) { + response.RequestIndex = request.RequestIndex + response.Result = ts.check.run(bag) +} + +func (ts *MixerServer) Report(ctx context.Context, bag *attribute.MutableBag, + request *mixerpb.ReportRequest, response *mixerpb.ReportResponse) { + response.RequestIndex = request.RequestIndex + response.Result = ts.report.run(bag) +} + +func (ts *MixerServer) Quota(ctx context.Context, bag *attribute.MutableBag, + request *mixerpb.QuotaRequest, response *mixerpb.QuotaResponse) { + response.RequestIndex = request.RequestIndex + response.Result = ts.quota.run(bag) + response.Amount = 0 +} + +func NewMixerServer(port uint16) (*MixerServer, error) { + log.Printf("Mixer server listening on port %v\n", port) + s := &MixerServer{ + check: newHandler(), + report: newHandler(), + quota: newHandler(), + } + + var err error + s.lis, err = net.Listen("tcp", fmt.Sprintf(":%d", port)) + if err != nil { + log.Fatalf("failed to listen: %v", err) + return nil, err + } + + var opts []grpc.ServerOption + opts = append(opts, grpc.MaxConcurrentStreams(32)) + opts = append(opts, grpc.MaxMsgSize(1024*1024)) + s.gs = grpc.NewServer(opts...) + + s.gp = pool.NewGoroutinePool(128, false) + s.gp.AddWorkers(32) + + s.s = api.NewGRPCServer(s, tracing.DisabledTracer(), s.gp) + mixerpb.RegisterMixerServer(s.gs, s.s) + return s, nil +} + +func (s *MixerServer) Start() { + go func() { + _ = s.gs.Serve(s.lis) + log.Printf("Mixer server exited\n") + }() +} + +func (s *MixerServer) Stop() { + log.Printf("Stop Mixer server\n") + s.gs.Stop() + log.Printf("Stop Mixer server -- Done\n") +} diff --git a/src/envoy/mixer/integration_test/mixer_test.go b/src/envoy/mixer/integration_test/mixer_test.go new file mode 100644 index 00000000000..b8f382bd58c --- /dev/null +++ b/src/envoy/mixer/integration_test/mixer_test.go @@ -0,0 +1,333 @@ +// Copyright 2017 Istio Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "fmt" + "testing" + + rpc "github.com/googleapis/googleapis/google/rpc" +) + +const ( + mixerFailMessage = "Unauthenticated by mixer." +) + +// Attributes verification rules +// 1) If value is *, key must exist, but value is not checked. +// 1) If value is -, key must NOT exist. +// 3) At top level attributes, not inside StringMap, all keys must +// be listed. Extra keys are NOT allowed +// 3) Inside StringMap, not need to list all keys. Extra keys are allowed +// +// Attributes provided from envoy config +// * source.id and source.namespace are forwarded from client proxy +// * target.id and target.namespace are from server proxy +// +// HTTP header "x-istio-attributes" is used to forward attributes between +// proxy. It should be removed before calling mixer and backend. +// +// Check attributes from a good GET request +const checkAttributesOkGet = ` +{ + "request.host": "localhost:27070", + "request.path": "/echo", + "request.time": "*", + "source.uid": "POD11", + "source.namespace": "XYZ11", + "target.uid": "POD222", + "target.namespace": "XYZ222", + "request.headers": { + ":method": "GET", + ":path": "/echo", + ":authority": "localhost:27070", + "x-forwarded-proto": "http", + "x-istio-attributes": "-", + "x-request-id": "*" + } +} +` + +// Report attributes from a good GET request +const reportAttributesOkGet = ` +{ + "request.host": "localhost:27070", + "request.path": "/echo", + "request.time": "*", + "source.uid": "POD11", + "source.namespace": "XYZ11", + "target.uid": "POD222", + "target.namespace": "XYZ222", + "request.headers": { + ":method": "GET", + ":path": "/echo", + ":authority": "localhost:27070", + "x-forwarded-proto": "http", + "x-istio-attributes": "-", + "x-request-id": "*" + }, + "request.size": 0, + "response.time": "*", + "response.size": 0, + "response.latency": "*", + "response.http.code": 200, + "response.headers": { + "date": "*", + "content-type": "text/plain; charset=utf-8", + "content-length": "0", + ":status": "200", + "server": "envoy" + } +} +` + +// Check attributes from a good POST request +const checkAttributesOkPost = ` +{ + "request.host": "localhost:27070", + "request.path": "/echo", + "request.time": "*", + "source.uid": "POD11", + "source.namespace": "XYZ11", + "target.uid": "POD222", + "target.namespace": "XYZ222", + "request.headers": { + ":method": "POST", + ":path": "/echo", + ":authority": "localhost:27070", + "x-forwarded-proto": "http", + "x-istio-attributes": "-", + "x-request-id": "*" + } +} +` + +// Report attributes from a good POST request +const reportAttributesOkPost = ` +{ + "request.host": "localhost:27070", + "request.path": "/echo", + "request.time": "*", + "source.uid": "POD11", + "source.namespace": "XYZ11", + "target.uid": "POD222", + "target.namespace": "XYZ222", + "request.headers": { + ":method": "POST", + ":path": "/echo", + ":authority": "localhost:27070", + "x-forwarded-proto": "http", + "x-istio-attributes": "-", + "x-request-id": "*" + }, + "request.size": 12, + "response.time": "*", + "response.size": 12, + "response.latency": "*", + "response.http.code": 200, + "response.headers": { + "date": "*", + "content-type": "text/plain", + "content-length": "12", + ":status": "200", + "server": "envoy" + } +} +` + +// Check attributes from a fail GET request from mixer +const checkAttributesMixerFail = ` +{ + "request.host": "localhost:27070", + "request.path": "/echo", + "request.time": "*", + "source.uid": "POD11", + "source.namespace": "XYZ11", + "target.uid": "POD222", + "target.namespace": "XYZ222", + "request.headers": { + ":method": "GET", + ":path": "/echo", + ":authority": "localhost:27070", + "x-forwarded-proto": "http", + "x-istio-attributes": "-", + "x-request-id": "*" + } +} +` + +// Report attributes from a fail GET request from mixer +const reportAttributesMixerFail = ` +{ + "request.host": "localhost:27070", + "request.path": "/echo", + "request.time": "*", + "source.uid": "POD11", + "source.namespace": "XYZ11", + "target.uid": "POD222", + "target.namespace": "XYZ222", + "request.headers": { + ":method": "GET", + ":path": "/echo", + ":authority": "localhost:27070", + "x-forwarded-proto": "http", + "x-istio-attributes": "-", + "x-request-id": "*" + }, + "request.size": 0, + "response.time": "*", + "response.size": 41, + "response.latency": "*", + "response.http.code": 401, + "response.headers": { + "date": "*", + "content-type": "text/plain", + "content-length": "41", + ":status": "401", + "server": "envoy" + } +} +` + +// Check attributes from a fail GET request from backend +const checkAttributesBackendFail = ` +{ + "request.host": "localhost:27070", + "request.path": "/echo", + "request.time": "*", + "source.uid": "POD11", + "source.namespace": "XYZ11", + "target.uid": "POD222", + "target.namespace": "XYZ222", + "request.headers": { + ":method": "GET", + ":path": "/echo", + ":authority": "localhost:27070", + "x-forwarded-proto": "http", + "x-istio-attributes": "-", + "x-request-id": "*" + } +} +` + +// Report attributes from a fail GET request from backend +const reportAttributesBackendFail = ` +{ + "request.host": "localhost:27070", + "request.path": "/echo", + "request.time": "*", + "source.uid": "POD11", + "source.namespace": "XYZ11", + "target.uid": "POD222", + "target.namespace": "XYZ222", + "request.headers": { + ":method": "GET", + ":path": "/echo", + ":authority": "localhost:27070", + "x-forwarded-proto": "http", + "x-istio-attributes": "-", + "x-request-id": "*" + }, + "request.size": 0, + "response.time": "*", + "response.size": 25, + "response.latency": "*", + "response.http.code": 400, + "response.headers": { + "date": "*", + "content-type": "text/plain; charset=utf-8", + "content-length": "25", + ":status": "400", + "server": "envoy" + } +} +` + +func verifyAttributes( + s *TestSetup, tag string, check string, report string, t *testing.T) { + _ = <-s.mixer.check.ch + if err := Verify(s.mixer.check.bag, check); err != nil { + t.Fatalf("Failed to verify %s check: %v\n, Attributes: %+v", + tag, err, s.mixer.check.bag) + } + _ = <-s.mixer.report.ch + if err := Verify(s.mixer.report.bag, report); err != nil { + t.Fatalf("Failed to verify %s report: %v\n, Attributes: %+v", + tag, err, s.mixer.report.bag) + } +} + +func TestMixer(t *testing.T) { + s, err := SetUp() + if err != nil { + t.Fatalf("Failed to setup test: %v", err) + } + defer s.TearDown() + + // There is a client proxy with filter "forward_attribute" + // and a server proxy with filter "mixer" calling mixer. + // This request will connect to client proxy, to server proxy + // and to the backend. + url := fmt.Sprintf("http://localhost:%d/echo", ClientProxyPort) + + // Issues a GET echo request with 0 size body + if _, _, err := HTTPGet(url); err != nil { + t.Errorf("Failed in GET request: %v", err) + } + verifyAttributes(&s, "OkGet", + checkAttributesOkGet, reportAttributesOkGet, t) + + // Issues a POST echo request with + if _, _, err := HTTPPost(url, "text/plain", "Hello World!"); err != nil { + t.Errorf("Failed in POST request: %v", err) + } + verifyAttributes(&s, "OkPost", + checkAttributesOkPost, reportAttributesOkPost, t) + + // Issues a failed request caused by mixer + s.mixer.check.r_status = rpc.Status{ + Code: int32(rpc.UNAUTHENTICATED), + Message: mixerFailMessage, + } + code, resp_body, err := HTTPGet(url) + s.mixer.check.r_status = rpc.Status{} + if err != nil { + t.Errorf("Failed in GET request: error: %v", err) + } + if code != 401 { + t.Errorf("Status code 401 is expected.") + } + if resp_body != "UNAUTHENTICATED:"+mixerFailMessage { + t.Errorf("Error response body is not expected.") + } + verifyAttributes(&s, "MixerFail", + checkAttributesMixerFail, reportAttributesMixerFail, t) + + // Issues a failed request caused by backend + headers := map[string]string{} + headers[FailHeader] = "Yes" + code, resp_body, err = HTTPGetWithHeaders(url, headers) + if err != nil { + t.Errorf("Failed in GET request: error: %v", err) + } + if code != 400 { + t.Errorf("Status code 400 is expected.") + } + if resp_body != FailBody { + t.Errorf("Error response body is not expected.") + } + verifyAttributes(&s, "BackendFail", + checkAttributesBackendFail, reportAttributesBackendFail, t) +} diff --git a/src/envoy/mixer/integration_test/repositories.bzl b/src/envoy/mixer/integration_test/repositories.bzl new file mode 100644 index 00000000000..7eec527fa39 --- /dev/null +++ b/src/envoy/mixer/integration_test/repositories.bzl @@ -0,0 +1,272 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_repositories", "new_go_repository", "go_repository") +load("@org_pubref_rules_protobuf//protobuf:rules.bzl", "proto_repositories") + +load("@org_pubref_rules_protobuf//gogo:rules.bzl", "gogo_proto_repositories") +load("@org_pubref_rules_protobuf//cpp:rules.bzl", "cpp_proto_repositories") + +def go_istio_api_repositories(use_local=False): + ISTIO_API_BUILD_FILE = """ +# build protos from istio.io/api repo + +package(default_visibility = ["//visibility:public"]) + +load("@io_bazel_rules_go//go:def.bzl", "go_prefix") + +go_prefix("istio.io/api") + +load("@org_pubref_rules_protobuf//gogo:rules.bzl", "gogoslick_proto_library") + +gogoslick_proto_library( + name = "mixer/v1", + importmap = { + "gogoproto/gogo.proto": "github.com/gogo/protobuf/gogoproto", + "google/rpc/status.proto": "github.com/googleapis/googleapis/google/rpc", + "google/protobuf/timestamp.proto": "github.com/gogo/protobuf/types", + "google/protobuf/duration.proto": "github.com/gogo/protobuf/types", + }, + imports = [ + "../../external/com_github_gogo_protobuf", + "../../external/com_github_google_protobuf/src", + "../../external/com_github_googleapis_googleapis", + ], + inputs = [ + "@com_github_google_protobuf//:well_known_protos", + "@com_github_googleapis_googleapis//:status_proto", + "@com_github_gogo_protobuf//gogoproto:go_default_library_protos", + ], + protos = [ + "mixer/v1/attributes.proto", + "mixer/v1/check.proto", + "mixer/v1/quota.proto", + "mixer/v1/report.proto", + "mixer/v1/service.proto", + ], + verbose = 0, + visibility = ["//visibility:public"], + with_grpc = True, + deps = [ + "@com_github_gogo_protobuf//gogoproto:go_default_library", + "@com_github_gogo_protobuf//sortkeys:go_default_library", + "@com_github_gogo_protobuf//types:go_default_library", + "@com_github_googleapis_googleapis//:google/rpc", + ], +) + +DESCRIPTOR_FILE_GROUP = [ + "mixer/v1/config/descriptor/attribute_descriptor.proto", + "mixer/v1/config/descriptor/label_descriptor.proto", + "mixer/v1/config/descriptor/log_entry_descriptor.proto", + "mixer/v1/config/descriptor/metric_descriptor.proto", + "mixer/v1/config/descriptor/monitored_resource_descriptor.proto", + "mixer/v1/config/descriptor/principal_descriptor.proto", + "mixer/v1/config/descriptor/quota_descriptor.proto", + "mixer/v1/config/descriptor/value_type.proto", +] + +gogoslick_proto_library( + name = "mixer/v1/config", + importmap = { + "google/protobuf/struct.proto": "github.com/gogo/protobuf/types", + "mixer/v1/config/descriptor/log_entry_descriptor.proto": "istio.io/api/mixer/v1/config/descriptor", + "mixer/v1/config/descriptor/metric_descriptor.proto": "istio.io/api/mixer/v1/config/descriptor", + "mixer/v1/config/descriptor/monitored_resource_descriptor.proto": "istio.io/api/mixer/v1/config/descriptor", + "mixer/v1/config/descriptor/principal_descriptor.proto": "istio.io/api/mixer/v1/config/descriptor", + "mixer/v1/config/descriptor/quota_descriptor.proto": "istio.io/api/mixer/v1/config/descriptor", + }, + imports = [ + "../../external/com_github_google_protobuf/src", + ], + inputs = DESCRIPTOR_FILE_GROUP + [ + "@com_github_google_protobuf//:well_known_protos", + ], + protos = [ + "mixer/v1/config/cfg.proto", + ], + verbose = 0, + visibility = ["//visibility:public"], + with_grpc = False, + deps = [ + ":mixer/v1/config/descriptor", + "@com_github_gogo_protobuf//sortkeys:go_default_library", + "@com_github_gogo_protobuf//types:go_default_library", + "@com_github_googleapis_googleapis//:google/rpc", + ], +) + +gogoslick_proto_library( + name = "mixer/v1/config/descriptor", + importmap = { + "google/protobuf/duration.proto": "github.com/gogo/protobuf/types", + }, + imports = [ + "../../external/com_github_google_protobuf/src", + ], + inputs = [ + "@com_github_google_protobuf//:well_known_protos", + ], + protos = DESCRIPTOR_FILE_GROUP, + verbose = 0, + visibility = ["//visibility:public"], + with_grpc = False, + deps = [ + "@com_github_gogo_protobuf//types:go_default_library", + ], +) +""" + if use_local: + native.new_local_repository( + name = "com_github_istio_api", + build_file_content = ISTIO_API_BUILD_FILE, + path = "../api", + ) + else: + native.new_git_repository( + name = "com_github_istio_api", + build_file_content = ISTIO_API_BUILD_FILE, + commit = "2cb09827d7f09a6e88eac2c2249dcb45c5419f09", # Mar. 14, 2017 (no releases) + remote = "https://github.com/istio/api.git", + ) + +def go_googleapis_repositories(): + GOOGLEAPIS_BUILD_FILE = """ +package(default_visibility = ["//visibility:public"]) + +load("@io_bazel_rules_go//go:def.bzl", "go_prefix") +go_prefix("github.com/googleapis/googleapis") + +load("@org_pubref_rules_protobuf//gogo:rules.bzl", "gogoslick_proto_library") + +gogoslick_proto_library( + name = "google/rpc", + protos = [ + "google/rpc/code.proto", + "google/rpc/error_details.proto", + "google/rpc/status.proto", + ], + importmap = { + "google/protobuf/any.proto": "github.com/gogo/protobuf/types", + "google/protobuf/duration.proto": "github.com/gogo/protobuf/types", + }, + imports = [ + "../../external/com_github_google_protobuf/src", + ], + inputs = [ + "@com_github_google_protobuf//:well_known_protos", + ], + deps = [ + "@com_github_gogo_protobuf//types:go_default_library", + ], + verbose = 0, +) + +load("@org_pubref_rules_protobuf//cpp:rules.bzl", "cc_proto_library") + +cc_proto_library( + name = "cc_status_proto", + protos = [ + "google/rpc/status.proto", + ], + imports = [ + "../../external/com_github_google_protobuf/src", + ], + verbose = 0, +) + +filegroup( + name = "status_proto", + srcs = [ "google/rpc/status.proto" ], +) + +filegroup( + name = "code_proto", + srcs = [ "google/rpc/code.proto" ], +) +""" + native.new_git_repository( + name = "com_github_googleapis_googleapis", + build_file_content = GOOGLEAPIS_BUILD_FILE, + commit = "13ac2436c5e3d568bd0e938f6ed58b77a48aba15", # Oct 21, 2016 (only release pre-dates sha) + remote = "https://github.com/googleapis/googleapis.git", + ) + +def go_mixer_repositories(use_local_api=False): + go_istio_api_repositories(use_local_api) + go_googleapis_repositories() + + go_repositories() + proto_repositories() + + gogo_proto_repositories() + + new_go_repository( + name = "com_github_golang_glog", + commit = "23def4e6c14b4da8ac2ed8007337bc5eb5007998", # Jan 26, 2016 (no releases) + importpath = "github.com/golang/glog", + ) + + new_go_repository( + name = "com_github_ghodss_yaml", + commit = "04f313413ffd65ce25f2541bfd2b2ceec5c0908c", # Dec 6, 2016 (no releases) + importpath = "github.com/ghodss/yaml", + ) + + new_go_repository( + name = "in_gopkg_yaml_v2", + commit = "14227de293ca979cf205cd88769fe71ed96a97e2", # Jan 24, 2017 (no releases) + importpath = "gopkg.in/yaml.v2", + ) + + new_go_repository( + name = "com_github_golang_protobuf", + commit = "8ee79997227bf9b34611aee7946ae64735e6fd93", # Nov 16, 2016 (no releases) + importpath = "github.com/golang/protobuf", + ) + + new_go_repository( + name = "org_golang_google_grpc", + commit = "708a7f9f3283aa2d4f6132d287d78683babe55c8", # Dec 5, 2016 (v1.0.5) + importpath = "google.golang.org/grpc", + ) + + new_go_repository( + name = "com_github_spf13_cobra", + commit = "35136c09d8da66b901337c6e86fd8e88a1a255bd", # Jan 30, 2017 (no releases) + importpath = "github.com/spf13/cobra", + ) + + new_go_repository( + name = "com_github_spf13_pflag", + commit = "9ff6c6923cfffbcd502984b8e0c80539a94968b7", # Jan 30, 2017 (no releases) + importpath = "github.com/spf13/pflag", + ) + + new_go_repository( + name = "com_github_hashicorp_go_multierror", + commit = "ed905158d87462226a13fe39ddf685ea65f1c11f", # Dec 16, 2016 (no releases) + importpath = "github.com/hashicorp/go-multierror", + ) + + new_go_repository( + name = "com_github_hashicorp_errwrap", + commit = "7554cd9344cec97297fa6649b055a8c98c2a1e55", # Oct 27, 2014 (no releases) + importpath = "github.com/hashicorp/errwrap", + ) + + new_go_repository( + name = "com_github_opentracing_opentracing_go", + commit = "0c3154a3c2ce79d3271985848659870599dfb77c", # Sep 26, 2016 (v1.0.0) + importpath = "github.com/opentracing/opentracing-go", + ) + + new_go_repository( + name = "com_github_opentracing_basictracer", + commit = "1b32af207119a14b1b231d451df3ed04a72efebf", # Sep 29, 2016 (no releases) + importpath = "github.com/opentracing/basictracer-go", + ) + + go_repository( + name = "com_github_istio_mixer", + commit = "064001053b51f73adc3a80ff87ef41a15316c300", + importpath = "github.com/istio/mixer", + ) + diff --git a/src/envoy/mixer/integration_test/setup.go b/src/envoy/mixer/integration_test/setup.go new file mode 100644 index 00000000000..7dc093d2449 --- /dev/null +++ b/src/envoy/mixer/integration_test/setup.go @@ -0,0 +1,64 @@ +// Copyright 2017 Istio Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "log" +) + +const ( + // These ports should match with used envoy.conf + // Default is using one in this folder. + ServerProxyPort = 29090 + ClientProxyPort = 27070 + MixerPort = 29091 + BackendPort = 28080 +) + +type TestSetup struct { + envoy *Envoy + mixer *MixerServer + backend *HttpServer +} + +func SetUp() (ts TestSetup, err error) { + ts.envoy, err = NewEnvoy() + if err != nil { + log.Printf("unable to create Envoy %v", err) + } else { + ts.envoy.Start() + } + + ts.mixer, err = NewMixerServer(MixerPort) + if err != nil { + log.Printf("unable to create mixer server %v", err) + } else { + ts.mixer.Start() + } + + ts.backend, err = NewHttpServer(BackendPort) + if err != nil { + log.Printf("unable to create HTTP server %v", err) + } else { + ts.backend.Start() + } + return ts, err +} + +func (ts *TestSetup) TearDown() { + ts.envoy.Stop() + ts.mixer.Stop() + ts.backend.Stop() +} From 57419f1a2a4b3d3b00bec59a69458f8911ffe8f7 Mon Sep 17 00:00:00 2001 From: wattli Date: Tue, 21 Mar 2017 13:48:11 -0700 Subject: [PATCH 67/83] Fix broken link (#193) --- src/envoy/mixer/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/envoy/mixer/README.md b/src/envoy/mixer/README.md index 4e30eb08f56..018de6ab206 100644 --- a/src/envoy/mixer/README.md +++ b/src/envoy/mixer/README.md @@ -3,7 +3,7 @@ This Proxy will use Envoy and talk to Mixer server. ## Build Mixer server -* Follow https://github.com/istio/mixer/blob/master/doc/devel/development.md to set up environment, and build via: +* Follow https://github.com/istio/mixer/blob/master/doc/dev/development.md to set up environment, and build via: ``` cd $(ISTIO)/mixer From d2bc18a5b81af863c2313c325aa4e9d9acd9455f Mon Sep 17 00:00:00 2001 From: Wayne Zhang Date: Tue, 21 Mar 2017 19:14:25 -0700 Subject: [PATCH 68/83] Make quota call (#192) * hookup quota call * Make quota call. * Update indent. --- src/envoy/mixer/README.md | 4 +- src/envoy/mixer/envoy.conf.template | 3 +- src/envoy/mixer/http_control.cc | 30 ++++++++++- src/envoy/mixer/http_control.h | 2 + src/envoy/mixer/integration_test/envoy.conf | 4 +- .../mixer/integration_test/mixer_server.go | 8 +-- .../mixer/integration_test/mixer_test.go | 52 +++++++++++++++---- src/envoy/mixer/repositories.bzl | 2 +- 8 files changed, 87 insertions(+), 18 deletions(-) diff --git a/src/envoy/mixer/README.md b/src/envoy/mixer/README.md index 018de6ab206..5cfb5f57fcb 100644 --- a/src/envoy/mixer/README.md +++ b/src/envoy/mixer/README.md @@ -66,7 +66,8 @@ This filter will intercept all HTTP requests and call Mixer. Here is its config: "mixer_server": "${MIXER_SERVER}", "mixer_attributes" : { "attribute_name1": "attribute_value1", - "attribute_name2": "attribute_value2" + "attribute_name2": "attribute_value2", + "quota.name": "RequestCount" }, "forward_attributes" : { "attribute_name1": "attribute_value1", @@ -79,6 +80,7 @@ Notes: * mixer_server is required * mixer_attributes: these attributes will be send to the mixer * forward_attributes: these attributes will be forwarded to the upstream istio/proxy. +* "quota.name" and "quota.amount" are used for quota call. "quota.amount" is default to 1 if missing. By default, mixer filter forwards attributes and does not invoke mixer server. You can customize this behavior per HTTP route by supplying an opaque config: diff --git a/src/envoy/mixer/envoy.conf.template b/src/envoy/mixer/envoy.conf.template index 86aa0e1dc33..1ca8ffc62b3 100644 --- a/src/envoy/mixer/envoy.conf.template +++ b/src/envoy/mixer/envoy.conf.template @@ -42,7 +42,8 @@ "mixer_server": "${MIXER_SERVER}", "mixer_attributes": { "target.uid": "POD222", - "target.namespace": "XYZ222" + "target.namespace": "XYZ222", + "quota.name": "RequestCount" } } }, diff --git a/src/envoy/mixer/http_control.cc b/src/envoy/mixer/http_control.cc index 50a637ca356..0e015b58ba6 100644 --- a/src/envoy/mixer/http_control.cc +++ b/src/envoy/mixer/http_control.cc @@ -114,6 +114,23 @@ HttpControl::HttpControl(const std::string& mixer_server, ::istio::mixer_client::MixerClientOptions options; options.mixer_server = mixer_server; mixer_client_ = ::istio::mixer_client::CreateMixerClient(options); + + // Extract quota attributes + auto it = config_attributes_.find(::istio::mixer_client::kQuotaName); + if (it != config_attributes_.end()) { + quota_attributes_.attributes[ ::istio::mixer_client::kQuotaName] = + Attributes::StringValue(it->second); + config_attributes_.erase(it); + + int64_t amount = 1; // default amount to 1. + it = config_attributes_.find(::istio::mixer_client::kQuotaAmount); + if (it != config_attributes_.end()) { + amount = std::stoi(it->second); + config_attributes_.erase(it); + } + quota_attributes_.attributes[ ::istio::mixer_client::kQuotaAmount] = + Attributes::Int64Value(amount); + } } void HttpControl::FillCheckAttributes(HeaderMap& header_map, Attributes* attr) { @@ -141,7 +158,18 @@ void HttpControl::Check(HttpRequestDataPtr request_data, HeaderMap& headers, FillCheckAttributes(headers, &request_data->attributes); SetStringAttribute(kOriginUser, origin_user, &request_data->attributes); log().debug("Send Check: {}", request_data->attributes.DebugString()); - mixer_client_->Check(request_data->attributes, on_done); + + auto check_on_done = [this, on_done](const Status& status) { + if (status.ok()) { + if (!quota_attributes_.attributes.empty()) { + log().debug("Send Quota: {}", quota_attributes_.DebugString()); + mixer_client_->Quota(quota_attributes_, on_done); + return; // Not to call on_done again. + } + } + on_done(status); + }; + mixer_client_->Check(request_data->attributes, check_on_done); } void HttpControl::Report(HttpRequestDataPtr request_data, diff --git a/src/envoy/mixer/http_control.h b/src/envoy/mixer/http_control.h index e9ddc734f45..9405945d714 100644 --- a/src/envoy/mixer/http_control.h +++ b/src/envoy/mixer/http_control.h @@ -58,6 +58,8 @@ class HttpControl final : public Logger::Loggable { std::unique_ptr<::istio::mixer_client::MixerClient> mixer_client_; // The attributes read from the config file. std::map config_attributes_; + // Quota attributes; extracted from envoy filter config. + ::istio::mixer_client::Attributes quota_attributes_; }; } // namespace Mixer diff --git a/src/envoy/mixer/integration_test/envoy.conf b/src/envoy/mixer/integration_test/envoy.conf index 2083d7f4e5a..4d993753f8b 100644 --- a/src/envoy/mixer/integration_test/envoy.conf +++ b/src/envoy/mixer/integration_test/envoy.conf @@ -42,7 +42,9 @@ "mixer_server": "localhost:29091", "mixer_attributes": { "target.uid": "POD222", - "target.namespace": "XYZ222" + "target.namespace": "XYZ222", + "quota.name": "RequestCount", + "quota.amount": "5" } } }, diff --git a/src/envoy/mixer/integration_test/mixer_server.go b/src/envoy/mixer/integration_test/mixer_server.go index e36f04b484a..fbe5a5bf9c5 100644 --- a/src/envoy/mixer/integration_test/mixer_server.go +++ b/src/envoy/mixer/integration_test/mixer_server.go @@ -56,9 +56,10 @@ type MixerServer struct { gp *pool.GoroutinePool s mixerpb.MixerServer - check *Handler - report *Handler - quota *Handler + check *Handler + report *Handler + quota *Handler + quota_request *mixerpb.QuotaRequest } func (ts *MixerServer) Check(ctx context.Context, bag *attribute.MutableBag, @@ -76,6 +77,7 @@ func (ts *MixerServer) Report(ctx context.Context, bag *attribute.MutableBag, func (ts *MixerServer) Quota(ctx context.Context, bag *attribute.MutableBag, request *mixerpb.QuotaRequest, response *mixerpb.QuotaResponse) { response.RequestIndex = request.RequestIndex + ts.quota_request = request response.Result = ts.quota.run(bag) response.Amount = 0 } diff --git a/src/envoy/mixer/integration_test/mixer_test.go b/src/envoy/mixer/integration_test/mixer_test.go index b8f382bd58c..862391f873a 100644 --- a/src/envoy/mixer/integration_test/mixer_test.go +++ b/src/envoy/mixer/integration_test/mixer_test.go @@ -22,7 +22,8 @@ import ( ) const ( - mixerFailMessage = "Unauthenticated by mixer." + mixerAuthFailMessage = "Unauthenticated by mixer." + mixerQuotaFailMessage = "Not enough quota by mixer." ) // Attributes verification rules @@ -134,14 +135,14 @@ const reportAttributesOkPost = ` }, "request.size": 12, "response.time": "*", - "response.size": 12, + "response.size": 45, "response.latency": "*", - "response.http.code": 200, + "response.http.code": 429, "response.headers": { "date": "*", "content-type": "text/plain", - "content-length": "12", - ":status": "200", + "content-length": "45", + ":status": "429", "server": "envoy" } } @@ -262,6 +263,7 @@ func verifyAttributes( t.Fatalf("Failed to verify %s check: %v\n, Attributes: %+v", tag, err, s.mixer.check.bag) } + _ = <-s.mixer.report.ch if err := Verify(s.mixer.report.bag, report); err != nil { t.Fatalf("Failed to verify %s report: %v\n, Attributes: %+v", @@ -269,6 +271,18 @@ func verifyAttributes( } } +func verifyQuota(s *TestSetup, tag string, t *testing.T) { + _ = <-s.mixer.quota.ch + if s.mixer.quota_request.Quota != "RequestCount" { + t.Fatalf("Failed to verify %s quota name (=RequestCount): %v\n", + tag, s.mixer.quota_request.Quota) + } + if s.mixer.quota_request.Amount != 5 { + t.Fatalf("Failed to verify %s quota amount (=5): %v\n", + tag, s.mixer.quota_request.Amount) + } +} + func TestMixer(t *testing.T) { s, err := SetUp() if err != nil { @@ -288,20 +302,36 @@ func TestMixer(t *testing.T) { } verifyAttributes(&s, "OkGet", checkAttributesOkGet, reportAttributesOkGet, t) + verifyQuota(&s, "OkGet", t) - // Issues a POST echo request with - if _, _, err := HTTPPost(url, "text/plain", "Hello World!"); err != nil { + // Issues a failed POST request caused by Mixer Quota + s.mixer.quota.r_status = rpc.Status{ + Code: int32(rpc.RESOURCE_EXHAUSTED), + Message: mixerQuotaFailMessage, + } + code, resp_body, err := HTTPPost(url, "text/plain", "Hello World!") + // Make sure to restore r_status for next request. + s.mixer.quota.r_status = rpc.Status{} + if err != nil { t.Errorf("Failed in POST request: %v", err) } + if code != 429 { + t.Errorf("Status code 429 is expected.") + } + if resp_body != "RESOURCE_EXHAUSTED:"+mixerQuotaFailMessage { + t.Errorf("Error response body is not expected.") + } verifyAttributes(&s, "OkPost", checkAttributesOkPost, reportAttributesOkPost, t) + verifyQuota(&s, "OkPost", t) // Issues a failed request caused by mixer s.mixer.check.r_status = rpc.Status{ Code: int32(rpc.UNAUTHENTICATED), - Message: mixerFailMessage, + Message: mixerAuthFailMessage, } - code, resp_body, err := HTTPGet(url) + code, resp_body, err = HTTPGet(url) + // Make sure to restore r_status for next request. s.mixer.check.r_status = rpc.Status{} if err != nil { t.Errorf("Failed in GET request: error: %v", err) @@ -309,11 +339,12 @@ func TestMixer(t *testing.T) { if code != 401 { t.Errorf("Status code 401 is expected.") } - if resp_body != "UNAUTHENTICATED:"+mixerFailMessage { + if resp_body != "UNAUTHENTICATED:"+mixerAuthFailMessage { t.Errorf("Error response body is not expected.") } verifyAttributes(&s, "MixerFail", checkAttributesMixerFail, reportAttributesMixerFail, t) + // Not quota call due to Mixer failure. // Issues a failed request caused by backend headers := map[string]string{} @@ -330,4 +361,5 @@ func TestMixer(t *testing.T) { } verifyAttributes(&s, "BackendFail", checkAttributesBackendFail, reportAttributesBackendFail, t) + verifyQuota(&s, "BackendFail", t) } diff --git a/src/envoy/mixer/repositories.bzl b/src/envoy/mixer/repositories.bzl index 9d06b24a14b..7edd7e89834 100644 --- a/src/envoy/mixer/repositories.bzl +++ b/src/envoy/mixer/repositories.bzl @@ -15,7 +15,7 @@ ################################################################################ # -MIXER_CLIENT = "1d6b587755846fe1b3a44fb53e3ab9ce0534af2c" +MIXER_CLIENT = "b76b5131c4650cefff4af7e4267883a33d66bca1" def mixer_client_repositories(bind=True): native.git_repository( From 61a53a46cff95afb77fda6b58e128ede73f54e06 Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Wed, 22 Mar 2017 16:41:28 -0700 Subject: [PATCH 69/83] Update envoy and update configs (#195) * Update envoy and update configs * Use gcc-4.9 for travis * Use bazel 0.4.5 * Fix SHA of lightstep-tracer-common --- .travis.yml | 26 ++++++----- src/envoy/mixer/envoy.conf.template | 6 +-- src/envoy/mixer/integration_test/envoy.conf | 6 +-- src/envoy/repositories.bzl | 52 ++++++++------------- 4 files changed, 39 insertions(+), 51 deletions(-) diff --git a/.travis.yml b/.travis.yml index 418e24d2dee..b2c2e63a881 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,25 +1,27 @@ sudo: required -dist: xenial + +dist: trusty + +addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - gcc-4.9 + - g++-4.9 + - wget branches: except: - stable -lang: go - -go: - - 1.7.x +language: cpp jdk: - oraclejdk8 env: - - BAZEL_VERSION=0.4.3 - -addons: - apt: - packages: - - wget + - BAZEL_VERSION=0.4.5 cache: directories: @@ -39,7 +41,7 @@ before_install: script: - script/check-style - - bazel --output_base=${HOME}/bazel/outbase test //... + - CC=/usr/bin/gcc-4.9 CXX=/usr/bin/g++-4.9 bazel --output_base=${HOME}/bazel/outbase test //... notifications: slack: istio-dev:wEEEbaabdP5ieCgDOFetA9nX diff --git a/src/envoy/mixer/envoy.conf.template b/src/envoy/mixer/envoy.conf.template index 1ca8ffc62b3..6e20da737ba 100644 --- a/src/envoy/mixer/envoy.conf.template +++ b/src/envoy/mixer/envoy.conf.template @@ -1,7 +1,7 @@ { "listeners": [ { - "port": ${PORT}, + "address": "tcp://0.0.0.0:${PORT}", "bind_to_port": true, "filters": [ { @@ -58,7 +58,7 @@ ] }, { - "port": 7070, + "address": "tcp://0.0.0.0:7070", "bind_to_port": true, "filters": [ { @@ -112,7 +112,7 @@ ], "admin": { "access_log_path": "/dev/stdout", - "port": 9001 + "address": "tcp://0.0.0.0:9001" }, "cluster_manager": { "clusters": [ diff --git a/src/envoy/mixer/integration_test/envoy.conf b/src/envoy/mixer/integration_test/envoy.conf index 4d993753f8b..4374ec1b4ea 100644 --- a/src/envoy/mixer/integration_test/envoy.conf +++ b/src/envoy/mixer/integration_test/envoy.conf @@ -1,7 +1,7 @@ { "listeners": [ { - "port": 29090, + "address": "tcp://0.0.0.0:29090", "bind_to_port": true, "filters": [ { @@ -59,7 +59,7 @@ ] }, { - "port": 27070, + "address": "tcp://0.0.0.0:27070", "bind_to_port": true, "filters": [ { @@ -113,7 +113,7 @@ ], "admin": { "access_log_path": "/dev/stdout", - "port": 29001 + "address": "tcp://0.0.0.0:29001" }, "cluster_manager": { "clusters": [ diff --git a/src/envoy/repositories.bzl b/src/envoy/repositories.bzl index 5a74586ddc8..bb150928ba0 100644 --- a/src/envoy/repositories.bzl +++ b/src/envoy/repositories.bzl @@ -235,25 +235,6 @@ def lightstep_repositories(bind=True): BUILD = """ load("@protobuf_git//:protobuf.bzl", "cc_proto_library") -genrule( - name = "envoy_carrier_pb", - srcs = ["src/c++11/envoy/envoy_carrier.proto"], - outs = ["lightstep/envoy_carrier.proto"], - cmd = "cp $(SRCS) $@", -) - -cc_proto_library( - name = "envoy_carrier_proto", - srcs = ["lightstep/envoy_carrier.proto"], - include = ".", - deps = [ - "//external:cc_wkt_protos", - ], - protoc = "//external:protoc", - default_runtime = "//external:protobuf", - visibility = ["//visibility:public"], -) - cc_library( name = "lightstep_core", srcs = [ @@ -266,7 +247,7 @@ cc_library( "src/c++11/lightstep/impl.h", "src/c++11/lightstep/options.h", "src/c++11/lightstep/propagation.h", - "src/c++11/lightstep/envoy.h", + "src/c++11/lightstep/carrier.h", "src/c++11/lightstep/span.h", "src/c++11/lightstep/tracer.h", "src/c++11/lightstep/util.h", @@ -275,7 +256,7 @@ cc_library( "src/c++11/mapbox_variant/variant.hpp", ], copts = [ - "-DPACKAGE_VERSION='\\"0.19\\"'", + "-DPACKAGE_VERSION='\\"0.36\\"'", "-Iexternal/lightstep_git/src/c++11/lightstep", "-Iexternal/lightstep_git/src/c++11/mapbox_variant", ], @@ -285,7 +266,7 @@ cc_library( visibility = ["//visibility:public"], deps = [ "@lightstep_common_git//:collector_proto", - ":envoy_carrier_proto", + "@lightstep_common_git//:lightstep_carrier_proto", "//external:protobuf", ], )""" @@ -293,16 +274,21 @@ cc_library( COMMON_BUILD = """ load("@protobuf_git//:protobuf.bzl", "cc_proto_library") -genrule( - name = "collector_pb", +cc_proto_library( + name = "collector_proto", srcs = ["collector.proto"], - outs = ["lightstep/collector.proto"], - cmd = "cp $(SRCS) $@", + include = ".", + deps = [ + "//external:cc_wkt_protos", + ], + protoc = "//external:protoc", + default_runtime = "//external:protobuf", + visibility = ["//visibility:public"], ) cc_proto_library( - name = "collector_proto", - srcs = ["lightstep/collector.proto"], + name = "lightstep_carrier_proto", + srcs = ["lightstep_carrier.proto"], include = ".", deps = [ "//external:cc_wkt_protos", @@ -310,19 +296,20 @@ cc_proto_library( protoc = "//external:protoc", default_runtime = "//external:protobuf", visibility = ["//visibility:public"], -)""" +) +""" native.new_git_repository( name = "lightstep_common_git", remote = "https://github.com/lightstep/lightstep-tracer-common.git", - commit = "8d932f7f76cd286691e6179621d0012b0ff1e6aa", + commit = "cbbecd671c1ae1f20ae873c5da688c8c14d04ec3", build_file_content = COMMON_BUILD, ) native.new_git_repository( name = "lightstep_git", remote = "https://github.com/lightstep/lightstep-tracer-cpp.git", - commit = "5a71d623cac17a059041b04fabca4ed86ffff7cc", + commit = "f1dc8f3dfd529350e053fd21273e627f409ae428", # 0.36 build_file_content = BUILD, ) @@ -751,7 +738,6 @@ cc_test( native.new_git_repository( name = "envoy_git", remote = "https://github.com/lyft/envoy.git", - commit = "b72309da41fba0c1222a72262b83bedc7294df65", # Mar 13 2017 + commit = "09f078b016da908ba8b861857f2a12a43933ba40", # Mar 21 2017 build_file_content = BUILD, ) - From 5e601ca3646135d3027568a2c8927e8f291628fc Mon Sep 17 00:00:00 2001 From: Wayne Zhang Date: Thu, 23 Mar 2017 12:48:29 -0700 Subject: [PATCH 70/83] Enable check cache and refactory mixer config loading (#197) * Refactory the mixer config loading. * fix format * Add integration test. * updated README.md * s/send/sent/ --- src/envoy/mixer/BUILD | 2 + src/envoy/mixer/README.md | 37 +++++-- src/envoy/mixer/config.cc | 96 +++++++++++++++++ src/envoy/mixer/config.h | 53 +++++++++ src/envoy/mixer/envoy.conf.template | 12 ++- src/envoy/mixer/http_control.cc | 29 ++--- src/envoy/mixer/http_control.h | 8 +- src/envoy/mixer/http_filter.cc | 36 ++----- src/envoy/mixer/integration_test/envoy.conf | 12 ++- .../mixer/integration_test/mixer_server.go | 3 + .../mixer/integration_test/mixer_test.go | 101 +++++++----------- src/envoy/mixer/repositories.bzl | 2 +- src/envoy/mixer/utils.cc | 14 --- src/envoy/mixer/utils.h | 3 - 14 files changed, 261 insertions(+), 147 deletions(-) create mode 100644 src/envoy/mixer/config.cc create mode 100644 src/envoy/mixer/config.h diff --git a/src/envoy/mixer/BUILD b/src/envoy/mixer/BUILD index 4d57f801626..aa7c455915d 100644 --- a/src/envoy/mixer/BUILD +++ b/src/envoy/mixer/BUILD @@ -30,6 +30,8 @@ cc_proto_library( cc_library( name = "filter_lib", srcs = [ + "config.cc", + "config.h", "http_control.cc", "http_control.h", "http_filter.cc", diff --git a/src/envoy/mixer/README.md b/src/envoy/mixer/README.md index 5cfb5f57fcb..493b2d185cf 100644 --- a/src/envoy/mixer/README.md +++ b/src/envoy/mixer/README.md @@ -52,9 +52,7 @@ This Proxy will use Envoy and talk to Mixer server. curl http://localhost:7070/echo -d "hello world" ``` -## How to configurate HTTP filters - -### *mixer* filter: +## How to configurate HTTP Mixer filters This filter will intercept all HTTP requests and call Mixer. Here is its config: @@ -72,17 +70,25 @@ This filter will intercept all HTTP requests and call Mixer. Here is its config: "forward_attributes" : { "attribute_name1": "attribute_value1", "attribute_name2": "attribute_value2" - } + }, + "quota_name": "RequestCount", + "quota_amount": "1", + "check_cache_keys": [ + "request.host", + "request.path", + "origin.user" + ] } ``` Notes: * mixer_server is required -* mixer_attributes: these attributes will be send to the mixer -* forward_attributes: these attributes will be forwarded to the upstream istio/proxy. -* "quota.name" and "quota.amount" are used for quota call. "quota.amount" is default to 1 if missing. +* mixer_attributes: these attributes will be sent to the mixer in both Check and Report calls. +* forward_attributes: these attributes will be forwarded to the upstream istio/proxy. It will send them to mixer in Check and Report calls. +* quota_name, quota_amount are used for making quota call. quota_amount is default to 1 if missing. +* check_cache_keys is to cache check calls. If missing or empty, check calls are not cached. -By default, mixer filter forwards attributes and does not invoke mixer server. You can customize this behavior per HTTP route by supplying an opaque config: +By default, mixer filter forwards attributes and does not invoke mixer server. You can customize this behavior per HTTP route by supplying an opaque config in the route config: ``` "opaque_config": { @@ -91,4 +97,17 @@ By default, mixer filter forwards attributes and does not invoke mixer server. Y } ``` -This config reverts the behavior by sending requests to mixer server but not forwarding any attributes. +This route opaque config reverts the behavior by sending requests to mixer server but not forwarding any attributes. + + +## How to enable quota (rate limiting) + +Quota (rate limiting) is enforced by the mixer. Mixer needs to be configured with Quota in its global config and service config. Its quota config will have +"quota name", its limit within a window. If "Quota" is added but param is missing, the default config is: quota name is "RequestCount", the limit is 10 with 1 second window. Essentially, it is imposing 10 qps rate limiting. + +Mixer client can be configured to make Quota call for all requests. If "quota_name" is specified in the mixer filter config, mixer client will call Quota with the specified quota name. If "quota_amount" is specified, it will call with that amount, otherwise the used amount is 1. + + +## How to pass some attributes from client proxy to mixer. + +Usually client proxy is not configured to call mixer (it can be enabled in the route opaque_config). Client proxy can pass some attributes to mixer by using "forward_attributes" field. Its attributes will be sent to the upstream proxy (the server proxy). If the server proxy is calling mixer, these attributes will be sent to the mixer. diff --git a/src/envoy/mixer/config.cc b/src/envoy/mixer/config.cc new file mode 100644 index 00000000000..5675a5a311f --- /dev/null +++ b/src/envoy/mixer/config.cc @@ -0,0 +1,96 @@ +/* Copyright 2017 Istio Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/envoy/mixer/config.h" + +using ::istio::mixer_client::Attributes; + +namespace Http { +namespace Mixer { +namespace { + +// The Json object name for mixer-server. +const std::string kJsonNameMixerServer("mixer_server"); + +// The Json object name for static attributes. +const std::string kJsonNameMixerAttributes("mixer_attributes"); + +// The Json object name to specify attributes which will be forwarded +// to the upstream istio proxy. +const std::string kJsonNameForwardAttributes("forward_attributes"); + +// The Json object name for quota name and amount. +const std::string kJsonNameQuotaName("quota_name"); +const std::string kJsonNameQuotaAmount("quota_amount"); + +// The Json object name for check cache keys. +const std::string kJsonNameCheckCacheKeys("check_cache_keys"); + +void ReadString(const Json::Object& json, const std::string& name, + std::string* value) { + if (json.hasObject(name)) { + *value = json.getString(name); + } +} + +void ReadStringMap(const Json::Object& json, const std::string& name, + std::map* map) { + if (json.hasObject(name)) { + json.getObject(name)->iterate( + [map](const std::string& key, const Json::Object& obj) -> bool { + (*map)[key] = obj.asString(); + return true; + }); + } +} + +void ReadStringVector(const Json::Object& json, const std::string& name, + std::vector* value) { + if (json.hasObject(name)) { + auto v = json.getStringArray(name); + value->swap(v); + } +} + +} // namespace + +void MixerConfig::Load(const Json::Object& json) { + ReadString(json, kJsonNameMixerServer, &mixer_server); + + ReadStringMap(json, kJsonNameMixerAttributes, &mixer_attributes); + ReadStringMap(json, kJsonNameForwardAttributes, &forward_attributes); + + ReadString(json, kJsonNameQuotaName, "a_name); + ReadString(json, kJsonNameQuotaAmount, "a_amount); + + ReadStringVector(json, kJsonNameCheckCacheKeys, &check_cache_keys); +} + +void MixerConfig::ExtractQuotaAttributes(Attributes* attr) const { + if (!quota_name.empty()) { + attr->attributes[ ::istio::mixer_client::kQuotaName] = + Attributes::StringValue(quota_name); + + int64_t amount = 1; // default amount to 1. + if (!quota_amount.empty()) { + amount = std::stoi(quota_amount); + } + attr->attributes[ ::istio::mixer_client::kQuotaAmount] = + Attributes::Int64Value(amount); + } +} + +} // namespace Mixer +} // namespace Http diff --git a/src/envoy/mixer/config.h b/src/envoy/mixer/config.h new file mode 100644 index 00000000000..ee301aba2b5 --- /dev/null +++ b/src/envoy/mixer/config.h @@ -0,0 +1,53 @@ +/* Copyright 2017 Istio Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "precompiled/precompiled.h" + +#include "envoy/json/json_object.h" +#include "include/attribute.h" + +namespace Http { +namespace Mixer { + +// A config for mixer filter +struct MixerConfig { + // the mixer server address + std::string mixer_server; + + // These static attributes will be send to mixer in both + // Check and Report. + std::map mixer_attributes; + + // These attributes will be forwarded to upstream. + std::map forward_attributes; + + // Quota attributes. + std::string quota_name; + std::string quota_amount; + + // The attribute names for check cache. + std::vector check_cache_keys; + + // Load the config from envoy config. + void Load(const Json::Object& json); + + // Extract quota attributes. + void ExtractQuotaAttributes(::istio::mixer_client::Attributes* attr) const; +}; + +} // namespace Mixer +} // namespace Http diff --git a/src/envoy/mixer/envoy.conf.template b/src/envoy/mixer/envoy.conf.template index 6e20da737ba..c7574734ff3 100644 --- a/src/envoy/mixer/envoy.conf.template +++ b/src/envoy/mixer/envoy.conf.template @@ -42,9 +42,15 @@ "mixer_server": "${MIXER_SERVER}", "mixer_attributes": { "target.uid": "POD222", - "target.namespace": "XYZ222", - "quota.name": "RequestCount" - } + "target.namespace": "XYZ222" + }, + "quota_name": "RequestCount", + "quota_amount": "1", + "check_cache_keys": [ + "request.host", + "request.path", + "origin.user" + ] } }, { diff --git a/src/envoy/mixer/http_control.cc b/src/envoy/mixer/http_control.cc index 0e015b58ba6..b44cf707172 100644 --- a/src/envoy/mixer/http_control.cc +++ b/src/envoy/mixer/http_control.cc @@ -108,29 +108,16 @@ void FillRequestInfoAttributes(const AccessLog::RequestInfo& info, } // namespace -HttpControl::HttpControl(const std::string& mixer_server, - std::map&& attributes) - : config_attributes_(std::move(attributes)) { +HttpControl::HttpControl(const MixerConfig& mixer_config) + : mixer_config_(mixer_config) { ::istio::mixer_client::MixerClientOptions options; - options.mixer_server = mixer_server; + options.mixer_server = mixer_config_.mixer_server; + options.check_options.cache_keys.insert( + mixer_config_.check_cache_keys.begin(), + mixer_config_.check_cache_keys.end()); mixer_client_ = ::istio::mixer_client::CreateMixerClient(options); - // Extract quota attributes - auto it = config_attributes_.find(::istio::mixer_client::kQuotaName); - if (it != config_attributes_.end()) { - quota_attributes_.attributes[ ::istio::mixer_client::kQuotaName] = - Attributes::StringValue(it->second); - config_attributes_.erase(it); - - int64_t amount = 1; // default amount to 1. - it = config_attributes_.find(::istio::mixer_client::kQuotaAmount); - if (it != config_attributes_.end()) { - amount = std::stoi(it->second); - config_attributes_.erase(it); - } - quota_attributes_.attributes[ ::istio::mixer_client::kQuotaAmount] = - Attributes::Int64Value(amount); - } + mixer_config_.ExtractQuotaAttributes("a_attributes_); } void HttpControl::FillCheckAttributes(HeaderMap& header_map, Attributes* attr) { @@ -148,7 +135,7 @@ void HttpControl::FillCheckAttributes(HeaderMap& header_map, Attributes* attr) { FillRequestHeaderAttributes(header_map, attr); - for (const auto& attribute : config_attributes_) { + for (const auto& attribute : mixer_config_.mixer_attributes) { SetStringAttribute(attribute.first, attribute.second, attr); } } diff --git a/src/envoy/mixer/http_control.h b/src/envoy/mixer/http_control.h index 9405945d714..c473facef3c 100644 --- a/src/envoy/mixer/http_control.h +++ b/src/envoy/mixer/http_control.h @@ -21,6 +21,7 @@ #include "common/http/headers.h" #include "envoy/http/access_log.h" #include "include/client.h" +#include "src/envoy/mixer/config.h" namespace Http { namespace Mixer { @@ -37,8 +38,7 @@ typedef std::shared_ptr HttpRequestDataPtr; class HttpControl final : public Logger::Loggable { public: // The constructor. - HttpControl(const std::string& mixer_server, - std::map&& attributes); + HttpControl(const MixerConfig& mixer_config); // Make mixer check call. void Check(HttpRequestDataPtr request_data, HeaderMap& headers, @@ -56,8 +56,8 @@ class HttpControl final : public Logger::Loggable { // The mixer client std::unique_ptr<::istio::mixer_client::MixerClient> mixer_client_; - // The attributes read from the config file. - std::map config_attributes_; + // The mixer config + const MixerConfig& mixer_config_; // Quota attributes; extracted from envoy filter config. ::istio::mixer_client::Attributes quota_attributes_; }; diff --git a/src/envoy/mixer/http_filter.cc b/src/envoy/mixer/http_filter.cc index 8708333de40..c46df8adf4b 100644 --- a/src/envoy/mixer/http_filter.cc +++ b/src/envoy/mixer/http_filter.cc @@ -22,6 +22,7 @@ #include "envoy/server/instance.h" #include "envoy/ssl/connection.h" #include "server/config/network/http_connection_manager.h" +#include "src/envoy/mixer/config.h" #include "src/envoy/mixer/http_control.h" #include "src/envoy/mixer/utils.h" @@ -33,16 +34,6 @@ namespace Http { namespace Mixer { namespace { -// The Json object name for mixer-server. -const std::string kJsonNameMixerServer("mixer_server"); - -// The Json object name for static attributes. -const std::string kJsonNameMixerAttributes("mixer_attributes"); - -// The Json object name to specify attributes which will be forwarded -// to the upstream istio proxy. -const std::string kJsonNameForwardAttributes("forward_attributes"); - // Switch to turn off attribute forwarding const std::string kJsonNameForwardSwitch("mixer_forward"); @@ -100,35 +91,30 @@ class Config : public Logger::Loggable { std::shared_ptr http_control_; Upstream::ClusterManager& cm_; std::string forward_attributes_; + MixerConfig mixer_config_; public: Config(const Json::Object& config, Server::Instance& server) : cm_(server.clusterManager()) { - std::string mixer_server; - if (config.hasObject(kJsonNameMixerServer)) { - mixer_server = config.getString(kJsonNameMixerServer); - } else { + mixer_config_.Load(config); + if (mixer_config_.mixer_server.empty()) { log().error( "mixer_server is required but not specified in the config: {}", __func__); + } else { + log().debug("Called Mixer::Config constructor with mixer_server: ", + mixer_config_.mixer_server); } - Utils::StringMap attributes = - Utils::ExtractStringMap(config, kJsonNameForwardAttributes); - if (!attributes.empty()) { - std::string serialized_str = Utils::SerializeStringMap(attributes); + if (!mixer_config_.forward_attributes.empty()) { + std::string serialized_str = + Utils::SerializeStringMap(mixer_config_.forward_attributes); forward_attributes_ = Base64::encode(serialized_str.c_str(), serialized_str.size()); log().debug("Mixer forward attributes set: ", serialized_str); } - std::map mixer_attributes = - Utils::ExtractStringMap(config, kJsonNameMixerAttributes); - - http_control_ = std::make_shared(mixer_server, - std::move(mixer_attributes)); - log().debug("Called Mixer::Config constructor with mixer_server: ", - mixer_server); + http_control_ = std::make_shared(mixer_config_); } std::shared_ptr& http_control() { return http_control_; } diff --git a/src/envoy/mixer/integration_test/envoy.conf b/src/envoy/mixer/integration_test/envoy.conf index 4374ec1b4ea..1e9bd153893 100644 --- a/src/envoy/mixer/integration_test/envoy.conf +++ b/src/envoy/mixer/integration_test/envoy.conf @@ -42,10 +42,14 @@ "mixer_server": "localhost:29091", "mixer_attributes": { "target.uid": "POD222", - "target.namespace": "XYZ222", - "quota.name": "RequestCount", - "quota.amount": "5" - } + "target.namespace": "XYZ222" + }, + "quota_name": "RequestCount", + "quota_amount": "5", + "check_cache_keys": [ + "request.host", + "request.path" + ] } }, { diff --git a/src/envoy/mixer/integration_test/mixer_server.go b/src/envoy/mixer/integration_test/mixer_server.go index fbe5a5bf9c5..3e851274819 100644 --- a/src/envoy/mixer/integration_test/mixer_server.go +++ b/src/envoy/mixer/integration_test/mixer_server.go @@ -33,6 +33,7 @@ import ( type Handler struct { bag *attribute.MutableBag ch chan int + count int r_status rpc.Status } @@ -40,6 +41,7 @@ func newHandler() *Handler { return &Handler{ bag: nil, ch: make(chan int, 1), + count: 0, r_status: rpc.Status{}, } } @@ -47,6 +49,7 @@ func newHandler() *Handler { func (h *Handler) run(bag *attribute.MutableBag) rpc.Status { h.bag = attribute.CopyBag(bag) h.ch <- 1 + h.count++ return h.r_status } diff --git a/src/envoy/mixer/integration_test/mixer_test.go b/src/envoy/mixer/integration_test/mixer_test.go index 862391f873a..99cef6501b3 100644 --- a/src/envoy/mixer/integration_test/mixer_test.go +++ b/src/envoy/mixer/integration_test/mixer_test.go @@ -94,27 +94,6 @@ const reportAttributesOkGet = ` } ` -// Check attributes from a good POST request -const checkAttributesOkPost = ` -{ - "request.host": "localhost:27070", - "request.path": "/echo", - "request.time": "*", - "source.uid": "POD11", - "source.namespace": "XYZ11", - "target.uid": "POD222", - "target.namespace": "XYZ222", - "request.headers": { - ":method": "POST", - ":path": "/echo", - ":authority": "localhost:27070", - "x-forwarded-proto": "http", - "x-istio-attributes": "-", - "x-request-id": "*" - } -} -` - // Report attributes from a good POST request const reportAttributesOkPost = ` { @@ -152,7 +131,7 @@ const reportAttributesOkPost = ` const checkAttributesMixerFail = ` { "request.host": "localhost:27070", - "request.path": "/echo", + "request.path": "/echo111", "request.time": "*", "source.uid": "POD11", "source.namespace": "XYZ11", @@ -160,7 +139,7 @@ const checkAttributesMixerFail = ` "target.namespace": "XYZ222", "request.headers": { ":method": "GET", - ":path": "/echo", + ":path": "/echo111", ":authority": "localhost:27070", "x-forwarded-proto": "http", "x-istio-attributes": "-", @@ -173,7 +152,7 @@ const checkAttributesMixerFail = ` const reportAttributesMixerFail = ` { "request.host": "localhost:27070", - "request.path": "/echo", + "request.path": "/echo111", "request.time": "*", "source.uid": "POD11", "source.namespace": "XYZ11", @@ -181,7 +160,7 @@ const reportAttributesMixerFail = ` "target.namespace": "XYZ222", "request.headers": { ":method": "GET", - ":path": "/echo", + ":path": "/echo111", ":authority": "localhost:27070", "x-forwarded-proto": "http", "x-istio-attributes": "-", @@ -202,27 +181,6 @@ const reportAttributesMixerFail = ` } ` -// Check attributes from a fail GET request from backend -const checkAttributesBackendFail = ` -{ - "request.host": "localhost:27070", - "request.path": "/echo", - "request.time": "*", - "source.uid": "POD11", - "source.namespace": "XYZ11", - "target.uid": "POD222", - "target.namespace": "XYZ222", - "request.headers": { - ":method": "GET", - ":path": "/echo", - ":authority": "localhost:27070", - "x-forwarded-proto": "http", - "x-istio-attributes": "-", - "x-request-id": "*" - } -} -` - // Report attributes from a fail GET request from backend const reportAttributesBackendFail = ` { @@ -256,14 +214,22 @@ const reportAttributesBackendFail = ` } ` -func verifyAttributes( - s *TestSetup, tag string, check string, report string, t *testing.T) { +func verifyCheckCount(s *TestSetup, tag string, count int, t *testing.T) { + if s.mixer.check.count != count { + t.Fatalf("%s check count doesn't match: %v\n, expected: %+v", + tag, s.mixer.check.count, count) + } +} + +func verifyCheckAttributes(s *TestSetup, tag string, check string, t *testing.T) { _ = <-s.mixer.check.ch if err := Verify(s.mixer.check.bag, check); err != nil { t.Fatalf("Failed to verify %s check: %v\n, Attributes: %+v", tag, err, s.mixer.check.bag) } +} +func verifyReportAttributes(s *TestSetup, tag string, report string, t *testing.T) { _ = <-s.mixer.report.ch if err := Verify(s.mixer.report.bag, report); err != nil { t.Fatalf("Failed to verify %s report: %v\n, Attributes: %+v", @@ -297,14 +263,16 @@ func TestMixer(t *testing.T) { url := fmt.Sprintf("http://localhost:%d/echo", ClientProxyPort) // Issues a GET echo request with 0 size body + tag := "OKGet" if _, _, err := HTTPGet(url); err != nil { - t.Errorf("Failed in GET request: %v", err) + t.Errorf("Failed in request %s: %v", tag, err) } - verifyAttributes(&s, "OkGet", - checkAttributesOkGet, reportAttributesOkGet, t) - verifyQuota(&s, "OkGet", t) + verifyCheckAttributes(&s, tag, checkAttributesOkGet, t) + verifyReportAttributes(&s, tag, reportAttributesOkGet, t) + verifyQuota(&s, tag, t) // Issues a failed POST request caused by Mixer Quota + tag = "QuotaFail" s.mixer.quota.r_status = rpc.Status{ Code: int32(rpc.RESOURCE_EXHAUSTED), Message: mixerQuotaFailMessage, @@ -313,7 +281,7 @@ func TestMixer(t *testing.T) { // Make sure to restore r_status for next request. s.mixer.quota.r_status = rpc.Status{} if err != nil { - t.Errorf("Failed in POST request: %v", err) + t.Errorf("Failed in request %s: %v", tag, err) } if code != 429 { t.Errorf("Status code 429 is expected.") @@ -321,11 +289,15 @@ func TestMixer(t *testing.T) { if resp_body != "RESOURCE_EXHAUSTED:"+mixerQuotaFailMessage { t.Errorf("Error response body is not expected.") } - verifyAttributes(&s, "OkPost", - checkAttributesOkPost, reportAttributesOkPost, t) - verifyQuota(&s, "OkPost", t) + // Use cached check. so server check count should remain 1. + verifyCheckCount(&s, tag, 1, t) + verifyReportAttributes(&s, tag, reportAttributesOkPost, t) + verifyQuota(&s, tag, t) // Issues a failed request caused by mixer + // Use a different path to avoid check cache + url = fmt.Sprintf("http://localhost:%d/echo111", ClientProxyPort) + tag = "MixerFail" s.mixer.check.r_status = rpc.Status{ Code: int32(rpc.UNAUTHENTICATED), Message: mixerAuthFailMessage, @@ -334,7 +306,7 @@ func TestMixer(t *testing.T) { // Make sure to restore r_status for next request. s.mixer.check.r_status = rpc.Status{} if err != nil { - t.Errorf("Failed in GET request: error: %v", err) + t.Errorf("Failed in request %s: %v", tag, err) } if code != 401 { t.Errorf("Status code 401 is expected.") @@ -342,16 +314,19 @@ func TestMixer(t *testing.T) { if resp_body != "UNAUTHENTICATED:"+mixerAuthFailMessage { t.Errorf("Error response body is not expected.") } - verifyAttributes(&s, "MixerFail", - checkAttributesMixerFail, reportAttributesMixerFail, t) + verifyCheckAttributes(&s, tag, checkAttributesMixerFail, t) + verifyReportAttributes(&s, tag, reportAttributesMixerFail, t) // Not quota call due to Mixer failure. // Issues a failed request caused by backend + // Use the first path to use check cache + url = fmt.Sprintf("http://localhost:%d/echo", ClientProxyPort) + tag = "BackendFail" headers := map[string]string{} headers[FailHeader] = "Yes" code, resp_body, err = HTTPGetWithHeaders(url, headers) if err != nil { - t.Errorf("Failed in GET request: error: %v", err) + t.Errorf("Failed in request %s: %v", tag, err) } if code != 400 { t.Errorf("Status code 400 is expected.") @@ -359,7 +334,7 @@ func TestMixer(t *testing.T) { if resp_body != FailBody { t.Errorf("Error response body is not expected.") } - verifyAttributes(&s, "BackendFail", - checkAttributesBackendFail, reportAttributesBackendFail, t) - verifyQuota(&s, "BackendFail", t) + verifyCheckCount(&s, tag, 2, t) + verifyReportAttributes(&s, tag, reportAttributesBackendFail, t) + verifyQuota(&s, tag, t) } diff --git a/src/envoy/mixer/repositories.bzl b/src/envoy/mixer/repositories.bzl index 7edd7e89834..ffa2a20409f 100644 --- a/src/envoy/mixer/repositories.bzl +++ b/src/envoy/mixer/repositories.bzl @@ -15,7 +15,7 @@ ################################################################################ # -MIXER_CLIENT = "b76b5131c4650cefff4af7e4267883a33d66bca1" +MIXER_CLIENT = "0e6f858bc7b52dc8f143f46ac32afc79d504e8a4" def mixer_client_repositories(bind=True): native.git_repository( diff --git a/src/envoy/mixer/utils.cc b/src/envoy/mixer/utils.cc index 6dc3a78356d..32b09667208 100644 --- a/src/envoy/mixer/utils.cc +++ b/src/envoy/mixer/utils.cc @@ -21,20 +21,6 @@ namespace Utils { const LowerCaseString kIstioAttributeHeader("x-istio-attributes"); -StringMap ExtractStringMap(const Json::Object& json, const std::string& name) { - StringMap map; - if (json.hasObject(name)) { - Json::ObjectPtr json_obj = json.getObject(name); - Json::Object* raw_obj = json_obj.get(); - json_obj->iterate( - [&map, raw_obj](const std::string& key, const Json::Object&) -> bool { - map[key] = raw_obj->getString(key); - return true; - }); - } - return map; -} - std::string SerializeStringMap(const StringMap& string_map) { ::istio::proxy::mixer::StringMap pb; ::google::protobuf::Map* map_pb = pb.mutable_map(); diff --git a/src/envoy/mixer/utils.h b/src/envoy/mixer/utils.h index 9273a9a2d8b..e94ed91706c 100644 --- a/src/envoy/mixer/utils.h +++ b/src/envoy/mixer/utils.h @@ -29,9 +29,6 @@ extern const LowerCaseString kIstioAttributeHeader; // The string map. typedef std::map StringMap; -// Extracts name/value attributes from a json object. -StringMap ExtractStringMap(const Json::Object& json, const std::string& name); - // Serialize a string map to string. std::string SerializeStringMap(const StringMap& map); From 3bcb6c4096ce672798f4d544568547b95fcc4bb2 Mon Sep 17 00:00:00 2001 From: Wayne Zhang Date: Fri, 24 Mar 2017 12:25:48 -0700 Subject: [PATCH 71/83] Split into separate tests. (#201) --- src/envoy/mixer/integration_test/BUILD | 30 +- .../mixer/integration_test/attributes.go | 16 +- .../integration_test/check_cache_test.go | 40 +++ .../integration_test/check_report_test.go | 154 ++++++++ src/envoy/mixer/integration_test/envoy.go | 35 +- .../{envoy.conf => envoy_conf.go} | 138 +++++-- .../integration_test/failed_request_test.go | 161 +++++++++ .../mixer/integration_test/mixer_server.go | 2 +- .../mixer/integration_test/mixer_test.go | 340 ------------------ .../mixer/integration_test/quota_test.go | 64 ++++ src/envoy/mixer/integration_test/setup.go | 71 ++-- .../mixer/integration_test/test_suite.bzl | 29 ++ 12 files changed, 659 insertions(+), 421 deletions(-) create mode 100644 src/envoy/mixer/integration_test/check_cache_test.go create mode 100644 src/envoy/mixer/integration_test/check_report_test.go rename src/envoy/mixer/integration_test/{envoy.conf => envoy_conf.go} (56%) create mode 100644 src/envoy/mixer/integration_test/failed_request_test.go delete mode 100644 src/envoy/mixer/integration_test/mixer_test.go create mode 100644 src/envoy/mixer/integration_test/quota_test.go create mode 100644 src/envoy/mixer/integration_test/test_suite.bzl diff --git a/src/envoy/mixer/integration_test/BUILD b/src/envoy/mixer/integration_test/BUILD index c45865457ce..a6ff4916db5 100644 --- a/src/envoy/mixer/integration_test/BUILD +++ b/src/envoy/mixer/integration_test/BUILD @@ -1,4 +1,4 @@ -# Copyright 2016 Google Inc. All Rights Reserved. +# Copyright 2017 Istio Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,13 +15,15 @@ ################################################################################ # -load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") +load("@io_bazel_rules_go//go:def.bzl", "go_library") +load(":test_suite.bzl", "go_test_suite") go_library( name = "go_default_library", srcs = [ "attributes.go", "envoy.go", + "envoy_conf.go", "http_client.go", "http_server.go", "mixer_server.go", @@ -41,18 +43,22 @@ go_library( ], ) -go_test( - name = "mixer_test", - size = "small", - srcs = [ - "mixer_test.go", - ], +go_test_suite( data = [ - "envoy.conf", "//src/envoy/mixer:envoy", ], library = ":go_default_library", - # shared memory path /envoy_shared_memory_0 used by Envoy - # hot start is not working in sandbox mode. - local = True, + tags = [ + # Use fixed ports, not in sanbbox, have to be run exclusively. + "exclusive", + # shared memory path /envoy_shared_memory_0 used by Envoy + # hot start is not working in sandbox mode. + "local", + ], + tests = [ + "check_cache_test.go", + "check_report_test.go", + "failed_request_test.go", + "quota_test.go", + ], ) diff --git a/src/envoy/mixer/integration_test/attributes.go b/src/envoy/mixer/integration_test/attributes.go index 4c97208fd4b..6aecd20d8bb 100644 --- a/src/envoy/mixer/integration_test/attributes.go +++ b/src/envoy/mixer/integration_test/attributes.go @@ -45,7 +45,21 @@ func verifyStringMap(actual map[string]string, expected map[string]interface{}) return nil } -// Please see the comment at top of mixer_test.go for verification rules +// Attributes verification rules: +// +// 1) If value is *, key must exist, but value is not checked. +// 1) If value is -, key must NOT exist. +// 3) At top level attributes, not inside StringMap, all keys must +// be listed. Extra keys are NOT allowed +// 3) Inside StringMap, not need to list all keys. Extra keys are allowed +// +// Attributes provided from envoy config +// * source.id and source.namespace are forwarded from client proxy +// * target.id and target.namespace are from server proxy +// +// HTTP header "x-istio-attributes" is used to forward attributes between +// proxy. It should be removed before calling mixer and backend. +// func Verify(b *attribute.MutableBag, json_results string) error { var r map[string]interface{} if err := json.Unmarshal([]byte(json_results), &r); err != nil { diff --git a/src/envoy/mixer/integration_test/check_cache_test.go b/src/envoy/mixer/integration_test/check_cache_test.go new file mode 100644 index 00000000000..ab79320ae82 --- /dev/null +++ b/src/envoy/mixer/integration_test/check_cache_test.go @@ -0,0 +1,40 @@ +// Copyright 2017 Istio Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "fmt" + "testing" +) + +func TestCheckCache(t *testing.T) { + s, err := SetUp(t, basicConfig+","+checkCacheConfig) + if err != nil { + t.Fatalf("Failed to setup test: %v", err) + } + defer s.TearDown() + + url := fmt.Sprintf("http://localhost:%d/echo", ClientProxyPort) + + // Issues a GET echo request with 0 size body + tag := "OKGet" + for i := 0; i < 10; i++ { + if _, _, err := HTTPGet(url); err != nil { + t.Errorf("Failed in request %s: %v", tag, err) + } + // Only the first check is called. + s.VerifyCheckCount(tag, 1) + } +} diff --git a/src/envoy/mixer/integration_test/check_report_test.go b/src/envoy/mixer/integration_test/check_report_test.go new file mode 100644 index 00000000000..71d8a3f6bb9 --- /dev/null +++ b/src/envoy/mixer/integration_test/check_report_test.go @@ -0,0 +1,154 @@ +// Copyright 2017 Istio Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "fmt" + "testing" +) + +// Check attributes from a good GET request +const checkAttributesOkGet = ` +{ + "request.host": "localhost:27070", + "request.path": "/echo", + "request.time": "*", + "source.uid": "POD11", + "source.namespace": "XYZ11", + "target.uid": "POD222", + "target.namespace": "XYZ222", + "request.headers": { + ":method": "GET", + ":path": "/echo", + ":authority": "localhost:27070", + "x-forwarded-proto": "http", + "x-istio-attributes": "-", + "x-request-id": "*" + } +} +` + +// Report attributes from a good GET request +const reportAttributesOkGet = ` +{ + "request.host": "localhost:27070", + "request.path": "/echo", + "request.time": "*", + "source.uid": "POD11", + "source.namespace": "XYZ11", + "target.uid": "POD222", + "target.namespace": "XYZ222", + "request.headers": { + ":method": "GET", + ":path": "/echo", + ":authority": "localhost:27070", + "x-forwarded-proto": "http", + "x-istio-attributes": "-", + "x-request-id": "*" + }, + "request.size": 0, + "response.time": "*", + "response.size": 0, + "response.latency": "*", + "response.http.code": 200, + "response.headers": { + "date": "*", + "content-type": "text/plain; charset=utf-8", + "content-length": "0", + ":status": "200", + "server": "envoy" + } +} +` + +// Check attributes from a good POST request +const checkAttributesOkPost = ` +{ + "request.host": "localhost:27070", + "request.path": "/echo", + "request.time": "*", + "source.uid": "POD11", + "source.namespace": "XYZ11", + "target.uid": "POD222", + "target.namespace": "XYZ222", + "request.headers": { + ":method": "POST", + ":path": "/echo", + ":authority": "localhost:27070", + "x-forwarded-proto": "http", + "x-istio-attributes": "-", + "x-request-id": "*" + } +} +` + +// Report attributes from a good POST request +const reportAttributesOkPost = ` +{ + "request.host": "localhost:27070", + "request.path": "/echo", + "request.time": "*", + "source.uid": "POD11", + "source.namespace": "XYZ11", + "target.uid": "POD222", + "target.namespace": "XYZ222", + "request.headers": { + ":method": "POST", + ":path": "/echo", + ":authority": "localhost:27070", + "x-forwarded-proto": "http", + "x-istio-attributes": "-", + "x-request-id": "*" + }, + "request.size": 12, + "response.time": "*", + "response.size": 12, + "response.latency": "*", + "response.http.code": 200, + "response.headers": { + "date": "*", + "content-type": "text/plain", + "content-length": "12", + ":status": "200", + "server": "envoy" + } +} +` + +func TestCheckReportAttributes(t *testing.T) { + s, err := SetUp(t, basicConfig) + if err != nil { + t.Fatalf("Failed to setup test: %v", err) + } + defer s.TearDown() + + url := fmt.Sprintf("http://localhost:%d/echo", ClientProxyPort) + + // Issues a GET echo request with 0 size body + tag := "OKGet" + if _, _, err := HTTPGet(url); err != nil { + t.Errorf("Failed in request %s: %v", tag, err) + } + s.VerifyCheck(tag, checkAttributesOkGet) + s.VerifyReport(tag, reportAttributesOkGet) + + // Issues a POST request. + tag = "OKPost" + if _, _, err := HTTPPost(url, "text/plain", "Hello World!"); err != nil { + t.Errorf("Failed in request %s: %v", tag, err) + } + s.VerifyCheck(tag, checkAttributesOkPost) + s.VerifyReport(tag, reportAttributesOkPost) +} diff --git a/src/envoy/mixer/integration_test/envoy.go b/src/envoy/mixer/integration_test/envoy.go index 0f9a1fc01c7..b34fb5c240a 100644 --- a/src/envoy/mixer/integration_test/envoy.go +++ b/src/envoy/mixer/integration_test/envoy.go @@ -39,24 +39,6 @@ func getTestBinRootPath() string { } } -func getTestDataRootPath() string { - switch { - // custom path - case os.Getenv("TEST_DATA_ROOT") != "": - return os.Getenv("TEST_DATA_ROOT") - // running under bazel - case os.Getenv("TEST_SRCDIR") != "": - return os.Getenv("TEST_SRCDIR") + "/__main__" - // running with native go - case os.Getenv("GOPATH") != "": - list := strings.Split(os.Getenv("GOPATH"), - string(os.PathListSeparator)) - return list[0] - default: - return "" - } -} - type Envoy struct { cmd *exec.Cmd } @@ -76,14 +58,17 @@ func Run(name string, args ...string) (s string, err error) { return } -func NewEnvoy() (*Envoy, error) { - path := getTestBinRootPath() + "/src/envoy/mixer/envoy" - conf := getTestDataRootPath() + - "/src/envoy/mixer/integration_test/envoy.conf" - log.Printf("Envoy binary: %v\n", path) - log.Printf("Envoy config: %v\n", conf) +func NewEnvoy(conf string) (*Envoy, error) { + bin_path := getTestBinRootPath() + "/src/envoy/mixer/envoy" + log.Printf("Envoy binary: %v\n", bin_path) + + conf_path := "/tmp/envoy.conf" + log.Printf("Envoy config: in %v\n%v\n", conf_path, conf) + if err := CreateEnvoyConf(conf_path, conf); err != nil { + return nil, err + } - cmd := exec.Command(path, "-c", conf, "-l", "debug") + cmd := exec.Command(bin_path, "-c", conf_path, "-l", "debug") cmd.Stderr = os.Stderr cmd.Stdout = os.Stdout return &Envoy{ diff --git a/src/envoy/mixer/integration_test/envoy.conf b/src/envoy/mixer/integration_test/envoy_conf.go similarity index 56% rename from src/envoy/mixer/integration_test/envoy.conf rename to src/envoy/mixer/integration_test/envoy_conf.go index 1e9bd153893..166b11543aa 100644 --- a/src/envoy/mixer/integration_test/envoy.conf +++ b/src/envoy/mixer/integration_test/envoy_conf.go @@ -1,7 +1,83 @@ +// Copyright 2017 Istio Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package test + +import ( + "fmt" + "os" + "text/template" +) + +const ( + // These ports should match with used envoy.conf + // Default is using one in this folder. + ServerProxyPort = 29090 + ClientProxyPort = 27070 + MixerPort = 29091 + BackendPort = 28080 + AdminPort = 29001 +) + +type ConfParam struct { + ClientPort int + ServerPort int + AdminPort int + MixerServer string + Backend string + ClientConfig string + ServerConfig string +} + +// A basic config +const basicConfig = ` + "mixer_attributes": { + "target.uid": "POD222", + "target.namespace": "XYZ222" + } +` + +// A config with quota +const quotaConfig = ` + "quota_name": "RequestCount", + "quota_amount": "5" +` + +// A config with check cache keys +const checkCacheConfig = ` + "check_cache_keys": [ + "request.host", + "request.path", + "origin.user" + ] +` + +// The default client proxy mixer config +const defaultClientMixerConfig = ` + "forward_attributes": { + "source.uid": "POD11", + "source.namespace": "XYZ11" + } +` + +// The envoy config template +const envoyConfTempl = ` { "listeners": [ { - "address": "tcp://0.0.0.0:29090", + "address": "tcp://0.0.0.0:{{.ServerPort}}", "bind_to_port": true, "filters": [ { @@ -39,17 +115,8 @@ "type": "decoder", "name": "mixer", "config": { - "mixer_server": "localhost:29091", - "mixer_attributes": { - "target.uid": "POD222", - "target.namespace": "XYZ222" - }, - "quota_name": "RequestCount", - "quota_amount": "5", - "check_cache_keys": [ - "request.host", - "request.path" - ] + "mixer_server": "{{.MixerServer}}", +{{.ServerConfig}} } }, { @@ -63,7 +130,7 @@ ] }, { - "address": "tcp://0.0.0.0:27070", + "address": "tcp://0.0.0.0:{{.ClientPort}}", "bind_to_port": true, "filters": [ { @@ -97,11 +164,8 @@ "type": "decoder", "name": "mixer", "config": { - "mixer_server": "localhost:29091", - "forward_attributes": { - "source.uid": "POD11", - "source.namespace": "XYZ11" - } + "mixer_server": "{{.MixerServer}}", +{{.ClientConfig}} } }, { @@ -117,7 +181,7 @@ ], "admin": { "access_log_path": "/dev/stdout", - "address": "tcp://0.0.0.0:29001" + "address": "tcp://0.0.0.0:{{.AdminPort}}" }, "cluster_manager": { "clusters": [ @@ -128,7 +192,7 @@ "lb_type": "round_robin", "hosts": [ { - "url": "tcp://localhost:28080" + "url": "tcp://{{.Backend}}" } ] }, @@ -139,10 +203,42 @@ "lb_type": "round_robin", "hosts": [ { - "url": "tcp://localhost:29090" + "url": "tcp://localhost:{{.ServerPort}}" } ] } ] } } +` + +func (c *ConfParam) write(path string) error { + tmpl, err := template.New("test").Parse(envoyConfTempl) + if err != nil { + return fmt.Errorf("Failed to parse config template: %v", err) + } + + f, err := os.Create(path) + if err != nil { + return fmt.Errorf("Failed to create file %v: %v", path, err) + } + defer f.Close() + return tmpl.Execute(f, *c) +} + +func getConf() ConfParam { + return ConfParam{ + ClientPort: ClientProxyPort, + ServerPort: ServerProxyPort, + AdminPort: AdminPort, + MixerServer: fmt.Sprintf("localhost:%d", MixerPort), + Backend: fmt.Sprintf("localhost:%d", BackendPort), + ClientConfig: defaultClientMixerConfig, + } +} + +func CreateEnvoyConf(path string, conf string) error { + c := getConf() + c.ServerConfig = conf + return c.write(path) +} diff --git a/src/envoy/mixer/integration_test/failed_request_test.go b/src/envoy/mixer/integration_test/failed_request_test.go new file mode 100644 index 00000000000..9ae99bcf81d --- /dev/null +++ b/src/envoy/mixer/integration_test/failed_request_test.go @@ -0,0 +1,161 @@ +// Copyright 2017 Istio Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "fmt" + "testing" + + rpc "github.com/googleapis/googleapis/google/rpc" +) + +const ( + mixerAuthFailMessage = "Unauthenticated by mixer." +) + +// Check attributes from a fail GET request from mixer +const checkAttributesMixerFail = ` +{ + "request.host": "localhost:27070", + "request.path": "/echo", + "request.time": "*", + "source.uid": "POD11", + "source.namespace": "XYZ11", + "target.uid": "POD222", + "target.namespace": "XYZ222", + "request.headers": { + ":method": "GET", + ":path": "/echo", + ":authority": "localhost:27070", + "x-forwarded-proto": "http", + "x-istio-attributes": "-", + "x-request-id": "*" + } +} +` + +// Report attributes from a fail GET request from mixer +const reportAttributesMixerFail = ` +{ + "request.host": "localhost:27070", + "request.path": "/echo", + "request.time": "*", + "source.uid": "POD11", + "source.namespace": "XYZ11", + "target.uid": "POD222", + "target.namespace": "XYZ222", + "request.headers": { + ":method": "GET", + ":path": "/echo", + ":authority": "localhost:27070", + "x-forwarded-proto": "http", + "x-istio-attributes": "-", + "x-request-id": "*" + }, + "request.size": 0, + "response.time": "*", + "response.size": 41, + "response.latency": "*", + "response.http.code": 401, + "response.headers": { + "date": "*", + "content-type": "text/plain", + "content-length": "41", + ":status": "401", + "server": "envoy" + } +} +` + +// Report attributes from a fail GET request from backend +const reportAttributesBackendFail = ` +{ + "request.host": "localhost:27070", + "request.path": "/echo", + "request.time": "*", + "source.uid": "POD11", + "source.namespace": "XYZ11", + "target.uid": "POD222", + "target.namespace": "XYZ222", + "request.headers": { + ":method": "GET", + ":path": "/echo", + ":authority": "localhost:27070", + "x-forwarded-proto": "http", + "x-istio-attributes": "-", + "x-request-id": "*" + }, + "request.size": 0, + "response.time": "*", + "response.size": 25, + "response.latency": "*", + "response.http.code": 400, + "response.headers": { + "date": "*", + "content-type": "text/plain; charset=utf-8", + "content-length": "25", + ":status": "400", + "server": "envoy" + } +} +` + +func TestFailedRequest(t *testing.T) { + s, err := SetUp(t, basicConfig) + if err != nil { + t.Fatalf("Failed to setup test: %v", err) + } + defer s.TearDown() + + url := fmt.Sprintf("http://localhost:%d/echo", ClientProxyPort) + + tag := "MixerFail" + s.mixer.check.r_status = rpc.Status{ + Code: int32(rpc.UNAUTHENTICATED), + Message: mixerAuthFailMessage, + } + code, resp_body, err := HTTPGet(url) + // Make sure to restore r_status for next request. + s.mixer.check.r_status = rpc.Status{} + if err != nil { + t.Errorf("Failed in request %s: %v", tag, err) + } + if code != 401 { + t.Errorf("Status code 401 is expected.") + } + if resp_body != "UNAUTHENTICATED:"+mixerAuthFailMessage { + t.Errorf("Error response body is not expected.") + } + s.VerifyCheck(tag, checkAttributesMixerFail) + s.VerifyReport(tag, reportAttributesMixerFail) + + // Issues a failed request caused by backend + tag = "BackendFail" + headers := map[string]string{} + headers[FailHeader] = "Yes" + code, resp_body, err = HTTPGetWithHeaders(url, headers) + if err != nil { + t.Errorf("Failed in request %s: %v", tag, err) + } + if code != 400 { + t.Errorf("Status code 400 is expected.") + } + if resp_body != FailBody { + t.Errorf("Error response body is not expected.") + } + // Same Check attributes as the first one. + s.VerifyCheck(tag, checkAttributesMixerFail) + s.VerifyReport(tag, reportAttributesBackendFail) +} diff --git a/src/envoy/mixer/integration_test/mixer_server.go b/src/envoy/mixer/integration_test/mixer_server.go index 3e851274819..6e3db085e5d 100644 --- a/src/envoy/mixer/integration_test/mixer_server.go +++ b/src/envoy/mixer/integration_test/mixer_server.go @@ -40,7 +40,7 @@ type Handler struct { func newHandler() *Handler { return &Handler{ bag: nil, - ch: make(chan int, 1), + ch: make(chan int, 10), // Allow maximum 10 requests count: 0, r_status: rpc.Status{}, } diff --git a/src/envoy/mixer/integration_test/mixer_test.go b/src/envoy/mixer/integration_test/mixer_test.go deleted file mode 100644 index 99cef6501b3..00000000000 --- a/src/envoy/mixer/integration_test/mixer_test.go +++ /dev/null @@ -1,340 +0,0 @@ -// Copyright 2017 Istio Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package test - -import ( - "fmt" - "testing" - - rpc "github.com/googleapis/googleapis/google/rpc" -) - -const ( - mixerAuthFailMessage = "Unauthenticated by mixer." - mixerQuotaFailMessage = "Not enough quota by mixer." -) - -// Attributes verification rules -// 1) If value is *, key must exist, but value is not checked. -// 1) If value is -, key must NOT exist. -// 3) At top level attributes, not inside StringMap, all keys must -// be listed. Extra keys are NOT allowed -// 3) Inside StringMap, not need to list all keys. Extra keys are allowed -// -// Attributes provided from envoy config -// * source.id and source.namespace are forwarded from client proxy -// * target.id and target.namespace are from server proxy -// -// HTTP header "x-istio-attributes" is used to forward attributes between -// proxy. It should be removed before calling mixer and backend. -// -// Check attributes from a good GET request -const checkAttributesOkGet = ` -{ - "request.host": "localhost:27070", - "request.path": "/echo", - "request.time": "*", - "source.uid": "POD11", - "source.namespace": "XYZ11", - "target.uid": "POD222", - "target.namespace": "XYZ222", - "request.headers": { - ":method": "GET", - ":path": "/echo", - ":authority": "localhost:27070", - "x-forwarded-proto": "http", - "x-istio-attributes": "-", - "x-request-id": "*" - } -} -` - -// Report attributes from a good GET request -const reportAttributesOkGet = ` -{ - "request.host": "localhost:27070", - "request.path": "/echo", - "request.time": "*", - "source.uid": "POD11", - "source.namespace": "XYZ11", - "target.uid": "POD222", - "target.namespace": "XYZ222", - "request.headers": { - ":method": "GET", - ":path": "/echo", - ":authority": "localhost:27070", - "x-forwarded-proto": "http", - "x-istio-attributes": "-", - "x-request-id": "*" - }, - "request.size": 0, - "response.time": "*", - "response.size": 0, - "response.latency": "*", - "response.http.code": 200, - "response.headers": { - "date": "*", - "content-type": "text/plain; charset=utf-8", - "content-length": "0", - ":status": "200", - "server": "envoy" - } -} -` - -// Report attributes from a good POST request -const reportAttributesOkPost = ` -{ - "request.host": "localhost:27070", - "request.path": "/echo", - "request.time": "*", - "source.uid": "POD11", - "source.namespace": "XYZ11", - "target.uid": "POD222", - "target.namespace": "XYZ222", - "request.headers": { - ":method": "POST", - ":path": "/echo", - ":authority": "localhost:27070", - "x-forwarded-proto": "http", - "x-istio-attributes": "-", - "x-request-id": "*" - }, - "request.size": 12, - "response.time": "*", - "response.size": 45, - "response.latency": "*", - "response.http.code": 429, - "response.headers": { - "date": "*", - "content-type": "text/plain", - "content-length": "45", - ":status": "429", - "server": "envoy" - } -} -` - -// Check attributes from a fail GET request from mixer -const checkAttributesMixerFail = ` -{ - "request.host": "localhost:27070", - "request.path": "/echo111", - "request.time": "*", - "source.uid": "POD11", - "source.namespace": "XYZ11", - "target.uid": "POD222", - "target.namespace": "XYZ222", - "request.headers": { - ":method": "GET", - ":path": "/echo111", - ":authority": "localhost:27070", - "x-forwarded-proto": "http", - "x-istio-attributes": "-", - "x-request-id": "*" - } -} -` - -// Report attributes from a fail GET request from mixer -const reportAttributesMixerFail = ` -{ - "request.host": "localhost:27070", - "request.path": "/echo111", - "request.time": "*", - "source.uid": "POD11", - "source.namespace": "XYZ11", - "target.uid": "POD222", - "target.namespace": "XYZ222", - "request.headers": { - ":method": "GET", - ":path": "/echo111", - ":authority": "localhost:27070", - "x-forwarded-proto": "http", - "x-istio-attributes": "-", - "x-request-id": "*" - }, - "request.size": 0, - "response.time": "*", - "response.size": 41, - "response.latency": "*", - "response.http.code": 401, - "response.headers": { - "date": "*", - "content-type": "text/plain", - "content-length": "41", - ":status": "401", - "server": "envoy" - } -} -` - -// Report attributes from a fail GET request from backend -const reportAttributesBackendFail = ` -{ - "request.host": "localhost:27070", - "request.path": "/echo", - "request.time": "*", - "source.uid": "POD11", - "source.namespace": "XYZ11", - "target.uid": "POD222", - "target.namespace": "XYZ222", - "request.headers": { - ":method": "GET", - ":path": "/echo", - ":authority": "localhost:27070", - "x-forwarded-proto": "http", - "x-istio-attributes": "-", - "x-request-id": "*" - }, - "request.size": 0, - "response.time": "*", - "response.size": 25, - "response.latency": "*", - "response.http.code": 400, - "response.headers": { - "date": "*", - "content-type": "text/plain; charset=utf-8", - "content-length": "25", - ":status": "400", - "server": "envoy" - } -} -` - -func verifyCheckCount(s *TestSetup, tag string, count int, t *testing.T) { - if s.mixer.check.count != count { - t.Fatalf("%s check count doesn't match: %v\n, expected: %+v", - tag, s.mixer.check.count, count) - } -} - -func verifyCheckAttributes(s *TestSetup, tag string, check string, t *testing.T) { - _ = <-s.mixer.check.ch - if err := Verify(s.mixer.check.bag, check); err != nil { - t.Fatalf("Failed to verify %s check: %v\n, Attributes: %+v", - tag, err, s.mixer.check.bag) - } -} - -func verifyReportAttributes(s *TestSetup, tag string, report string, t *testing.T) { - _ = <-s.mixer.report.ch - if err := Verify(s.mixer.report.bag, report); err != nil { - t.Fatalf("Failed to verify %s report: %v\n, Attributes: %+v", - tag, err, s.mixer.report.bag) - } -} - -func verifyQuota(s *TestSetup, tag string, t *testing.T) { - _ = <-s.mixer.quota.ch - if s.mixer.quota_request.Quota != "RequestCount" { - t.Fatalf("Failed to verify %s quota name (=RequestCount): %v\n", - tag, s.mixer.quota_request.Quota) - } - if s.mixer.quota_request.Amount != 5 { - t.Fatalf("Failed to verify %s quota amount (=5): %v\n", - tag, s.mixer.quota_request.Amount) - } -} - -func TestMixer(t *testing.T) { - s, err := SetUp() - if err != nil { - t.Fatalf("Failed to setup test: %v", err) - } - defer s.TearDown() - - // There is a client proxy with filter "forward_attribute" - // and a server proxy with filter "mixer" calling mixer. - // This request will connect to client proxy, to server proxy - // and to the backend. - url := fmt.Sprintf("http://localhost:%d/echo", ClientProxyPort) - - // Issues a GET echo request with 0 size body - tag := "OKGet" - if _, _, err := HTTPGet(url); err != nil { - t.Errorf("Failed in request %s: %v", tag, err) - } - verifyCheckAttributes(&s, tag, checkAttributesOkGet, t) - verifyReportAttributes(&s, tag, reportAttributesOkGet, t) - verifyQuota(&s, tag, t) - - // Issues a failed POST request caused by Mixer Quota - tag = "QuotaFail" - s.mixer.quota.r_status = rpc.Status{ - Code: int32(rpc.RESOURCE_EXHAUSTED), - Message: mixerQuotaFailMessage, - } - code, resp_body, err := HTTPPost(url, "text/plain", "Hello World!") - // Make sure to restore r_status for next request. - s.mixer.quota.r_status = rpc.Status{} - if err != nil { - t.Errorf("Failed in request %s: %v", tag, err) - } - if code != 429 { - t.Errorf("Status code 429 is expected.") - } - if resp_body != "RESOURCE_EXHAUSTED:"+mixerQuotaFailMessage { - t.Errorf("Error response body is not expected.") - } - // Use cached check. so server check count should remain 1. - verifyCheckCount(&s, tag, 1, t) - verifyReportAttributes(&s, tag, reportAttributesOkPost, t) - verifyQuota(&s, tag, t) - - // Issues a failed request caused by mixer - // Use a different path to avoid check cache - url = fmt.Sprintf("http://localhost:%d/echo111", ClientProxyPort) - tag = "MixerFail" - s.mixer.check.r_status = rpc.Status{ - Code: int32(rpc.UNAUTHENTICATED), - Message: mixerAuthFailMessage, - } - code, resp_body, err = HTTPGet(url) - // Make sure to restore r_status for next request. - s.mixer.check.r_status = rpc.Status{} - if err != nil { - t.Errorf("Failed in request %s: %v", tag, err) - } - if code != 401 { - t.Errorf("Status code 401 is expected.") - } - if resp_body != "UNAUTHENTICATED:"+mixerAuthFailMessage { - t.Errorf("Error response body is not expected.") - } - verifyCheckAttributes(&s, tag, checkAttributesMixerFail, t) - verifyReportAttributes(&s, tag, reportAttributesMixerFail, t) - // Not quota call due to Mixer failure. - - // Issues a failed request caused by backend - // Use the first path to use check cache - url = fmt.Sprintf("http://localhost:%d/echo", ClientProxyPort) - tag = "BackendFail" - headers := map[string]string{} - headers[FailHeader] = "Yes" - code, resp_body, err = HTTPGetWithHeaders(url, headers) - if err != nil { - t.Errorf("Failed in request %s: %v", tag, err) - } - if code != 400 { - t.Errorf("Status code 400 is expected.") - } - if resp_body != FailBody { - t.Errorf("Error response body is not expected.") - } - verifyCheckCount(&s, tag, 2, t) - verifyReportAttributes(&s, tag, reportAttributesBackendFail, t) - verifyQuota(&s, tag, t) -} diff --git a/src/envoy/mixer/integration_test/quota_test.go b/src/envoy/mixer/integration_test/quota_test.go new file mode 100644 index 00000000000..e99afb8e426 --- /dev/null +++ b/src/envoy/mixer/integration_test/quota_test.go @@ -0,0 +1,64 @@ +// Copyright 2017 Istio Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "fmt" + "testing" + + rpc "github.com/googleapis/googleapis/google/rpc" +) + +const ( + mixerQuotaFailMessage = "Not enough quota by mixer." +) + +func TestQuotaCall(t *testing.T) { + s, err := SetUp(t, basicConfig+","+quotaConfig) + if err != nil { + t.Fatalf("Failed to setup test: %v", err) + } + defer s.TearDown() + + url := fmt.Sprintf("http://localhost:%d/echo", ClientProxyPort) + + // Issues a GET echo request with 0 size body + tag := "OKGet" + if _, _, err := HTTPGet(url); err != nil { + t.Errorf("Failed in request %s: %v", tag, err) + } + s.VerifyQuota(tag, "RequestCount", 5) + + // Issues a failed POST request caused by Mixer Quota + tag = "QuotaFail" + s.mixer.quota_request = nil + s.mixer.quota.r_status = rpc.Status{ + Code: int32(rpc.RESOURCE_EXHAUSTED), + Message: mixerQuotaFailMessage, + } + code, resp_body, err := HTTPPost(url, "text/plain", "Hello World!") + // Make sure to restore r_status for next request. + s.mixer.quota.r_status = rpc.Status{} + if err != nil { + t.Errorf("Failed in request %s: %v", tag, err) + } + if code != 429 { + t.Errorf("Status code 429 is expected.") + } + if resp_body != "RESOURCE_EXHAUSTED:"+mixerQuotaFailMessage { + t.Errorf("Error response body is not expected.") + } + s.VerifyQuota(tag, "RequestCount", 5) +} diff --git a/src/envoy/mixer/integration_test/setup.go b/src/envoy/mixer/integration_test/setup.go index 7dc093d2449..52bfdd2f711 100644 --- a/src/envoy/mixer/integration_test/setup.go +++ b/src/envoy/mixer/integration_test/setup.go @@ -16,49 +16,78 @@ package test import ( "log" -) - -const ( - // These ports should match with used envoy.conf - // Default is using one in this folder. - ServerProxyPort = 29090 - ClientProxyPort = 27070 - MixerPort = 29091 - BackendPort = 28080 + "testing" ) type TestSetup struct { envoy *Envoy mixer *MixerServer backend *HttpServer + t *testing.T } -func SetUp() (ts TestSetup, err error) { - ts.envoy, err = NewEnvoy() +func SetUp(t *testing.T, conf string) (s TestSetup, err error) { + s.t = t + s.envoy, err = NewEnvoy(conf) if err != nil { log.Printf("unable to create Envoy %v", err) } else { - ts.envoy.Start() + s.envoy.Start() } - ts.mixer, err = NewMixerServer(MixerPort) + s.mixer, err = NewMixerServer(MixerPort) if err != nil { log.Printf("unable to create mixer server %v", err) } else { - ts.mixer.Start() + s.mixer.Start() } - ts.backend, err = NewHttpServer(BackendPort) + s.backend, err = NewHttpServer(BackendPort) if err != nil { log.Printf("unable to create HTTP server %v", err) } else { - ts.backend.Start() + s.backend.Start() + } + return s, err +} + +func (s *TestSetup) TearDown() { + s.envoy.Stop() + s.mixer.Stop() + s.backend.Stop() +} + +func (s *TestSetup) VerifyCheckCount(tag string, expected int) { + if s.mixer.check.count != expected { + s.t.Fatalf("%s check count doesn't match: %v\n, expected: %+v", + tag, s.mixer.check.count, expected) } - return ts, err } -func (ts *TestSetup) TearDown() { - ts.envoy.Stop() - ts.mixer.Stop() - ts.backend.Stop() +func (s *TestSetup) VerifyCheck(tag string, result string) { + _ = <-s.mixer.check.ch + if err := Verify(s.mixer.check.bag, result); err != nil { + s.t.Fatalf("Failed to verify %s check: %v\n, Attributes: %+v", + tag, err, s.mixer.check.bag) + } +} + +func (s *TestSetup) VerifyReport(tag string, result string) { + _ = <-s.mixer.report.ch + if err := Verify(s.mixer.report.bag, result); err != nil { + s.t.Fatalf("Failed to verify %s report: %v\n, Attributes: %+v", + tag, err, s.mixer.report.bag) + } +} + +func (s *TestSetup) VerifyQuota(tag string, name string, amount int64) { + _ = <-s.mixer.quota.ch + if s.mixer.quota_request.Quota != name { + s.t.Fatalf("Failed to verify %s quota name: %v, expected: %v\n", + tag, s.mixer.quota_request.Quota, name) + } + if s.mixer.quota_request.Amount != amount { + s.t.Fatalf("Failed to verify %s quota amount: %v, expected: %v\n", + tag, s.mixer.quota_request.Amount, amount) + } } diff --git a/src/envoy/mixer/integration_test/test_suite.bzl b/src/envoy/mixer/integration_test/test_suite.bzl new file mode 100644 index 00000000000..afa984f4773 --- /dev/null +++ b/src/envoy/mixer/integration_test/test_suite.bzl @@ -0,0 +1,29 @@ +# Copyright 2017 Istio Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ +# + +load("@io_bazel_rules_go//go:def.bzl", "go_test") + +def go_test_suite(tests, library, data, tags, size="small"): + for test in tests: + go_test( + name = test.split(".")[0], + size = size, + srcs = [test], + data = data, + library = library, + tags = tags, + ) From c93dcc5d7605fe1cc375ac5462499f2316b31870 Mon Sep 17 00:00:00 2001 From: Wayne Zhang Date: Sat, 25 Mar 2017 19:09:45 -0700 Subject: [PATCH 72/83] Update README on how to enable check cache. (#204) * Update README on how to enable check cache. * Update the comment. --- src/envoy/mixer/README.md | 38 ++++++++++++++++++++++++++++---- src/envoy/mixer/config.cc | 31 +++++++++++++------------- src/envoy/mixer/config.h | 1 + src/envoy/mixer/http_control.cc | 30 +++++++++++++++++++++---- src/envoy/mixer/repositories.bzl | 2 +- 5 files changed, 78 insertions(+), 24 deletions(-) diff --git a/src/envoy/mixer/README.md b/src/envoy/mixer/README.md index 493b2d185cf..48229a3631e 100644 --- a/src/envoy/mixer/README.md +++ b/src/envoy/mixer/README.md @@ -73,6 +73,7 @@ This filter will intercept all HTTP requests and call Mixer. Here is its config: }, "quota_name": "RequestCount", "quota_amount": "1", + "check_cache_expiration_in_seconds": "600", "check_cache_keys": [ "request.host", "request.path", @@ -85,16 +86,23 @@ Notes: * mixer_server is required * mixer_attributes: these attributes will be sent to the mixer in both Check and Report calls. * forward_attributes: these attributes will be forwarded to the upstream istio/proxy. It will send them to mixer in Check and Report calls. -* quota_name, quota_amount are used for making quota call. quota_amount is default to 1 if missing. +* quota_name, quota_amount are used for making quota call. quota_amount defaults to 1. * check_cache_keys is to cache check calls. If missing or empty, check calls are not cached. -By default, mixer filter forwards attributes and does not invoke mixer server. You can customize this behavior per HTTP route by supplying an opaque config in the route config: +## HTTP Route opaque config +By default, the mixer filter only forwards attributes and does not call mixer server. This behavior can be changed per HTTP route by supplying an opaque config: ``` - "opaque_config": { + "routes": [ + { + "timeout_ms": 0, + "prefix": "/", + "cluster": "service1", + "opaque_config": { "mixer_control": "on", "mixer_forward": "off" - } + } + } ``` This route opaque config reverts the behavior by sending requests to mixer server but not forwarding any attributes. @@ -111,3 +119,25 @@ Mixer client can be configured to make Quota call for all requests. If "quota_n ## How to pass some attributes from client proxy to mixer. Usually client proxy is not configured to call mixer (it can be enabled in the route opaque_config). Client proxy can pass some attributes to mixer by using "forward_attributes" field. Its attributes will be sent to the upstream proxy (the server proxy). If the server proxy is calling mixer, these attributes will be sent to the mixer. + + +## How to enable cache for Check calls + +Check calls can be cached. By default, it is not enabled. It can be enabled by supplying non-empty "check_cache_keys" string list in the mixer filter config. Only these attributes in the Check request, their keys and values, are used to calculate the key for the cache lookup. If it is a cache hit, the cached response will be used. +The cached response will be expired in 5 minutes by default. It can be overrided by supplying "check_cache_expiration_in_seconds" in the mixer filter config. The Check response from the mixer has an expiration field. If it is filled, it will be used. By design, the mixer will control the cache expiration time. + +Following is a sample mixer filter config to enable the Check call cache: +``` + "check_cache_expiration_in_seconds": "600", + "check_cache_keys": [ + "request.host", + "request.path", + "source.labels", + "request.headers/:method", + "origin.user" + ] +``` + +For the string map attributes in the above example: +1) "request.headers" attribute is a string map, "request.headers/:method" cache key means only its ":method" key and value are used for cache key. +2) "source.labels" attribute is a string map, "source.labels" cache key means all key value pairs for the string map will be used. diff --git a/src/envoy/mixer/config.cc b/src/envoy/mixer/config.cc index 5675a5a311f..8e06b1e9c2b 100644 --- a/src/envoy/mixer/config.cc +++ b/src/envoy/mixer/config.cc @@ -22,21 +22,22 @@ namespace Mixer { namespace { // The Json object name for mixer-server. -const std::string kJsonNameMixerServer("mixer_server"); +const std::string kMixerServer("mixer_server"); // The Json object name for static attributes. -const std::string kJsonNameMixerAttributes("mixer_attributes"); +const std::string kMixerAttributes("mixer_attributes"); // The Json object name to specify attributes which will be forwarded // to the upstream istio proxy. -const std::string kJsonNameForwardAttributes("forward_attributes"); +const std::string kForwardAttributes("forward_attributes"); // The Json object name for quota name and amount. -const std::string kJsonNameQuotaName("quota_name"); -const std::string kJsonNameQuotaAmount("quota_amount"); +const std::string kQuotaName("quota_name"); +const std::string kQuotaAmount("quota_amount"); // The Json object name for check cache keys. -const std::string kJsonNameCheckCacheKeys("check_cache_keys"); +const std::string kCheckCacheKeys("check_cache_keys"); +const std::string kCheckCacheExpiration("check_cache_expiration_in_seconds"); void ReadString(const Json::Object& json, const std::string& name, std::string* value) { @@ -67,28 +68,28 @@ void ReadStringVector(const Json::Object& json, const std::string& name, } // namespace void MixerConfig::Load(const Json::Object& json) { - ReadString(json, kJsonNameMixerServer, &mixer_server); + ReadString(json, kMixerServer, &mixer_server); - ReadStringMap(json, kJsonNameMixerAttributes, &mixer_attributes); - ReadStringMap(json, kJsonNameForwardAttributes, &forward_attributes); + ReadStringMap(json, kMixerAttributes, &mixer_attributes); + ReadStringMap(json, kForwardAttributes, &forward_attributes); - ReadString(json, kJsonNameQuotaName, "a_name); - ReadString(json, kJsonNameQuotaAmount, "a_amount); + ReadString(json, kQuotaName, "a_name); + ReadString(json, kQuotaAmount, "a_amount); - ReadStringVector(json, kJsonNameCheckCacheKeys, &check_cache_keys); + ReadStringVector(json, kCheckCacheKeys, &check_cache_keys); + ReadString(json, kCheckCacheExpiration, &check_cache_expiration); } void MixerConfig::ExtractQuotaAttributes(Attributes* attr) const { if (!quota_name.empty()) { - attr->attributes[ ::istio::mixer_client::kQuotaName] = + attr->attributes[Attributes::kQuotaName] = Attributes::StringValue(quota_name); int64_t amount = 1; // default amount to 1. if (!quota_amount.empty()) { amount = std::stoi(quota_amount); } - attr->attributes[ ::istio::mixer_client::kQuotaAmount] = - Attributes::Int64Value(amount); + attr->attributes[Attributes::kQuotaAmount] = Attributes::Int64Value(amount); } } diff --git a/src/envoy/mixer/config.h b/src/envoy/mixer/config.h index ee301aba2b5..fc08f33c439 100644 --- a/src/envoy/mixer/config.h +++ b/src/envoy/mixer/config.h @@ -41,6 +41,7 @@ struct MixerConfig { // The attribute names for check cache. std::vector check_cache_keys; + std::string check_cache_expiration; // Load the config from envoy config. void Load(const Json::Object& json); diff --git a/src/envoy/mixer/http_control.cc b/src/envoy/mixer/http_control.cc index b44cf707172..8283482dff3 100644 --- a/src/envoy/mixer/http_control.cc +++ b/src/envoy/mixer/http_control.cc @@ -23,8 +23,12 @@ #include "src/envoy/mixer/utils.h" using ::google::protobuf::util::Status; +using ::istio::mixer_client::CheckOptions; using ::istio::mixer_client::Attributes; using ::istio::mixer_client::DoneFunc; +using ::istio::mixer_client::MixerClientOptions; +using ::istio::mixer_client::ReportOptions; +using ::istio::mixer_client::QuotaOptions; namespace Http { namespace Mixer { @@ -45,6 +49,26 @@ const std::string kResponseLatency = "response.latency"; const std::string kResponseSize = "response.size"; const std::string kResponseTime = "response.time"; +// Check cache size: 10000 cache entries. +const int kCheckCacheEntries = 10000; +// Default check cache expired in 5 minutes. +const int kCheckCacheExpirationInSeconds = 300; + +CheckOptions GetCheckOptions(const MixerConfig& config) { + int expiration = kCheckCacheExpirationInSeconds; + if (!config.check_cache_expiration.empty()) { + expiration = std::stoi(config.check_cache_expiration); + } + + // Remove expired items from cache 1 second later. + CheckOptions options(kCheckCacheEntries, expiration * 1000, + (expiration + 1) * 1000); + + options.cache_keys = config.check_cache_keys; + + return options; +} + void SetStringAttribute(const std::string& name, const std::string& value, Attributes* attr) { if (!value.empty()) { @@ -110,11 +134,9 @@ void FillRequestInfoAttributes(const AccessLog::RequestInfo& info, HttpControl::HttpControl(const MixerConfig& mixer_config) : mixer_config_(mixer_config) { - ::istio::mixer_client::MixerClientOptions options; + MixerClientOptions options(GetCheckOptions(mixer_config), ReportOptions(), + QuotaOptions()); options.mixer_server = mixer_config_.mixer_server; - options.check_options.cache_keys.insert( - mixer_config_.check_cache_keys.begin(), - mixer_config_.check_cache_keys.end()); mixer_client_ = ::istio::mixer_client::CreateMixerClient(options); mixer_config_.ExtractQuotaAttributes("a_attributes_); diff --git a/src/envoy/mixer/repositories.bzl b/src/envoy/mixer/repositories.bzl index ffa2a20409f..4382e71f921 100644 --- a/src/envoy/mixer/repositories.bzl +++ b/src/envoy/mixer/repositories.bzl @@ -15,7 +15,7 @@ ################################################################################ # -MIXER_CLIENT = "0e6f858bc7b52dc8f143f46ac32afc79d504e8a4" +MIXER_CLIENT = "c5d857e28bfcc53f20f59466b464f99526737545" def mixer_client_repositories(bind=True): native.git_repository( From 8c2bbceb284dd4347806e91dd09a0e78ba78d80b Mon Sep 17 00:00:00 2001 From: htuch Date: Sat, 1 Apr 2017 14:48:36 -0400 Subject: [PATCH 73/83] build: support Envoy native Bazel build. (#210) * build: support Envoy native Bazel build. This patch switches the Envoy build from src/envoy/repositories.bzl to using the upstream native build. See https://github.com/lyft/envoy/pull/663 for the corresponding changes on the Envoy side. * Use Envoy master with BUILD.wip rename merged. * Fix clang-format issues. --- src/envoy/BUILD | 34 ------ src/envoy/mixer/BUILD | 5 +- src/envoy/mixer/http_filter.cc | 18 ++-- src/envoy/repositories.bzl | 189 ++------------------------------- 4 files changed, 22 insertions(+), 224 deletions(-) diff --git a/src/envoy/BUILD b/src/envoy/BUILD index 1c75af50a75..e69de29bb2d 100644 --- a/src/envoy/BUILD +++ b/src/envoy/BUILD @@ -1,34 +0,0 @@ -# Copyright 2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -################################################################################ -# -cc_test( - name = "envoy_test", - data = [ - "@envoy_git//:envoy-testdata", - ], - copts = [ - "-include ./external/envoy_git/test/precompiled/precompiled_test.h", - ], - deps = [ - "@envoy_git//:envoy-test-lib", - "//external:googletest_main", - ], - args = [ - #TODO: Make all test pass - "--gtest_filter=RouterTest.*", - ], - linkstatic=1, -) diff --git a/src/envoy/mixer/BUILD b/src/envoy/mixer/BUILD index aa7c455915d..45c69b7f2d5 100644 --- a/src/envoy/mixer/BUILD +++ b/src/envoy/mixer/BUILD @@ -41,7 +41,7 @@ cc_library( deps = [ ":string_map_proto", "//external:mixer_client_lib", - "@envoy_git//:envoy-common", + "@envoy//source/exe:envoy_common_lib", ], alwayslink = 1, ) @@ -49,10 +49,11 @@ cc_library( cc_binary( name = "envoy", linkstatic = 1, + linkopts = ["-lrt"], visibility = [":__subpackages__"], deps = [ ":filter_lib", - "@envoy_git//:envoy-main", + "@envoy//source/exe:envoy_main_lib", ], ) diff --git a/src/envoy/mixer/http_filter.cc b/src/envoy/mixer/http_filter.cc index c46df8adf4b..9f8e4ffb75c 100644 --- a/src/envoy/mixer/http_filter.cc +++ b/src/envoy/mixer/http_filter.cc @@ -316,17 +316,19 @@ class MixerConfig : public HttpFilterConfigFactory { Http::Mixer::ConfigPtr mixer_config( new Http::Mixer::Config(config, server)); - return [mixer_config]( - Http::FilterChainFactoryCallbacks& callbacks) -> void { - std::shared_ptr instance( - new Http::Mixer::Instance(mixer_config)); - callbacks.addStreamDecoderFilter(Http::StreamDecoderFilterPtr(instance)); - callbacks.addAccessLogHandler(Http::AccessLog::InstancePtr(instance)); - }; + return + [mixer_config](Http::FilterChainFactoryCallbacks& callbacks) -> void { + std::shared_ptr instance( + new Http::Mixer::Instance(mixer_config)); + callbacks.addStreamDecoderFilter( + Http::StreamDecoderFilterSharedPtr(instance)); + callbacks.addAccessLogHandler( + Http::AccessLog::InstanceSharedPtr(instance)); + }; } }; static RegisterHttpFilterConfigFactory register_; } // namespace Configuration -} // namespace server +} // namespace Server diff --git a/src/envoy/repositories.bzl b/src/envoy/repositories.bzl index bb150928ba0..bad332e188d 100644 --- a/src/envoy/repositories.bzl +++ b/src/envoy/repositories.bzl @@ -559,185 +559,14 @@ def envoy_repositories(bind=True): rapidjson_repositories(bind) nghttp2_repositories(bind) ares_repositories(bind) - - BUILD = """ -load("@protobuf_git//:protobuf.bzl", "cc_proto_library") - -exports_files(["source/precompiled/precompiled.h"]) - -package(default_visibility = ["//visibility:public"]) - -genrule( - name = "envoy-ratelimit-proto", - srcs = [ - "source/common/ratelimit/ratelimit.proto", - ], - outs = [ - "source/common/generated/ratelimit.proto", - ], - cmd = "cp $(SRCS) $@", -) - -cc_proto_library( - name = "envoy-ratelimit-pb", - srcs = [ - "source/common/generated/ratelimit.proto", - ], - default_runtime = "//external:protobuf", - protoc = "//external:protoc", - include = "source", -) - -genrule( - name = "envoy-test-proto", - srcs = [ - "test/proto/helloworld.proto", - ], - outs = [ - "test/generated/helloworld.proto", - ], - cmd = "cp $(SRCS) $@", -) - -cc_proto_library( - name = "envoy-test-pb", - srcs = [ - "test/generated/helloworld.proto", - ], - default_runtime = "//external:protobuf", - protoc = "//external:protoc", - include = "test", -) - -genrule( - name = "envoy-version", - srcs = glob([ - ".git/**", - ]), - tools = [ - "tools/gen_git_sha.sh", - ], - outs = [ - "source/common/version_generated.cc", - ], - cmd = "touch $@ && $(location tools/gen_git_sha.sh) $$(dirname $(location tools/gen_git_sha.sh)) $@", - local = 1, -) - -cc_library( - name = "envoy-common", - srcs = glob([ - "source/**/*.cc", - "source/**/*.h", - "include/**/*.h", - ], exclude=["source/exe/main.cc"]) + [ - "source/common/version_generated.cc", - ], - copts = [ - "-I./external/envoy_git/source", - "-include ./external/envoy_git/source/precompiled/precompiled.h", - ], - includes = [ - "include", - ], - linkopts = [ - "-lpthread", - "-lanl", - "-lrt", - ], - linkstatic=1, - alwayslink=1, - deps = [ - ":envoy-ratelimit-pb", - "//external:ares", - "//external:libssl", - "//external:nghttp2", - "//external:spdlog", - "//external:tclap", - "//external:lightstep", - "//external:event", - "//external:protobuf", - "//external:http_parser", - "//external:rapidjson", - "//external:event_pthreads", - ], -) - -cc_library( - name = "envoy-main", - srcs = [ - "source/exe/main.cc", - ], - copts = [ - "-I./external/envoy_git/source", - "-include ./external/envoy_git/source/precompiled/precompiled.h", - ], - deps = [ - ":envoy-common", - ], - linkstatic=1, -) - -cc_binary( - name = "envoy", - srcs = [ - "source/exe/main.cc", - ], - copts = [ - "-I./external/envoy_git/source", - "-include ./external/envoy_git/source/precompiled/precompiled.h", - ], - deps = [ - ":envoy-common", - ], - linkstatic=1, -) - -cc_library( - name = "envoy-test-lib", - srcs = glob([ - "test/**/*.cc", - "test/**/*.h", - ]), - copts = [ - "-I./external/envoy_git/source", - "-include ./external/envoy_git/test/precompiled/precompiled_test.h", - ], - includes = [ - "include", - ], - deps = [ - ":envoy-common", - ":envoy-test-pb", - "//external:googletest", - ], - alwayslink=1, -) - -filegroup( - name = "envoy-testdata", - srcs = glob([ - "generated/**/*", - "test/**/*", - ]), -) - -cc_test( - name = "envoy-test", - data = [ - ":envoy-testdata", - ], - deps = [ - ":envoy-test-lib", - ":envoy-test-pb", - "//external:googletest", - ], - linkstatic=1, -)""" - - native.new_git_repository( - name = "envoy_git", + # @boringssl is defined in //:repositories.bzl, but bound to libssl for + # grpc. Rebind to what envoy expects here. + native.bind( + name = "ssl", + actual = "@boringssl//:ssl", + ) + native.git_repository( + name = "envoy", remote = "https://github.com/lyft/envoy.git", - commit = "09f078b016da908ba8b861857f2a12a43933ba40", # Mar 21 2017 - build_file_content = BUILD, + commit = "bf3f23ad439ee83b91015dc4d0d7cb53b14bf1bc", ) From 2fae2d78930494cd68da6cdb4252ad19d9ef5819 Mon Sep 17 00:00:00 2001 From: Sebastien Vas Date: Sat, 1 Apr 2017 13:40:28 -0700 Subject: [PATCH 74/83] Fixes bazel.rc issues (#212) * Fixes bazel rc issues * Update Jenkins to latest pipeline version --- .travis.yml | 3 ++- Jenkinsfile | 2 +- .bazelrc.jenkins => tools/bazel.rc.jenkins | 0 .bazelrc.travis => tools/bazel.rc.travis | 0 4 files changed, 3 insertions(+), 2 deletions(-) rename .bazelrc.jenkins => tools/bazel.rc.jenkins (100%) rename .bazelrc.travis => tools/bazel.rc.travis (100%) diff --git a/.travis.yml b/.travis.yml index b2c2e63a881..845ac9bbd8e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,7 +37,8 @@ before_install: - sudo dpkg -i bazel_${BAZEL_VERSION}-linux-x86_64.deb - sudo apt-get -f install -qqy uuid-dev - cd ${TRAVIS_BUILD_DIR} - - mv .bazelrc.travis .bazelrc + - mv tools/bazel.rc tools/bazel.rc.orig + - cat tools/bazel.rc.travis tools/bazel.rc.orig > tools/bazel.rc script: - script/check-style diff --git a/Jenkinsfile b/Jenkinsfile index 6575dc498f5..c1aa89e3f25 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,6 +1,6 @@ #!groovy -@Library('testutils@stable-63c264e') +@Library('testutils@stable-3e4d089') import org.istio.testutils.Utilities import org.istio.testutils.GitUtilities diff --git a/.bazelrc.jenkins b/tools/bazel.rc.jenkins similarity index 100% rename from .bazelrc.jenkins rename to tools/bazel.rc.jenkins diff --git a/.bazelrc.travis b/tools/bazel.rc.travis similarity index 100% rename from .bazelrc.travis rename to tools/bazel.rc.travis From f71ef2366321661193a1a300336f54eb791c8fea Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Thu, 6 Apr 2017 17:01:40 -0700 Subject: [PATCH 75/83] Fix go build (#224) --- WORKSPACE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WORKSPACE b/WORKSPACE index 4bc7137304c..6f387192fb9 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -100,7 +100,7 @@ http_file( # Following go repositories are for building go integration test for mixer filter. git_repository( name = "io_bazel_rules_go", - commit = "9496d79880a7d55b8e4a96f04688d70a374eaaf4", # Mar 3, 2017 (v0.4.1) + commit = "2d9f328a9723baf2d037ba9db28d9d0e30683938", # Apr 6, 2017 (buildifier fix) remote = "https://github.com/bazelbuild/rules_go.git", ) From 987223f6ce6998a8af0f3524545bed7f0d0a1ebb Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Thu, 6 Apr 2017 17:30:47 -0700 Subject: [PATCH 76/83] Use TranscoderInputStream to reduce confusion around ByteCount() (#225) * Add TranscoderInputStream to reduce confusion * fix_format --- contrib/endpoints/src/grpc/transcoding/BUILD | 14 +++++ .../src/grpc/transcoding/message_reader.cc | 10 +--- .../src/grpc/transcoding/message_reader.h | 11 +--- .../src/grpc/transcoding/message_stream.cc | 30 ++++------ .../src/grpc/transcoding/message_stream.h | 4 +- .../grpc/transcoding/message_stream_test.cc | 58 +++++++++---------- .../response_to_json_translator.cc | 3 +- .../transcoding/response_to_json_translator.h | 3 +- .../src/grpc/transcoding/test_common.cc | 2 +- .../src/grpc/transcoding/test_common.h | 9 +-- .../src/grpc/transcoding/transcoder.h | 18 +++--- .../grpc/transcoding/transcoder_factory.cc | 20 +++---- .../src/grpc/transcoding/transcoder_factory.h | 3 +- .../transcoding/transcoder_input_stream.h | 35 +++++++++++ .../src/grpc/transcoding/transcoder_test.cc | 2 +- 15 files changed, 125 insertions(+), 97 deletions(-) create mode 100644 contrib/endpoints/src/grpc/transcoding/transcoder_input_stream.h diff --git a/contrib/endpoints/src/grpc/transcoding/BUILD b/contrib/endpoints/src/grpc/transcoding/BUILD index 894fd27b960..dce09d218c4 100644 --- a/contrib/endpoints/src/grpc/transcoding/BUILD +++ b/contrib/endpoints/src/grpc/transcoding/BUILD @@ -67,6 +67,7 @@ cc_library( "message_stream.h", ], deps = [ + ":transcoder_input_stream", "//external:protobuf", ], ) @@ -125,6 +126,7 @@ cc_library( "message_reader.h", ], deps = [ + ":transcoder_input_stream", "//external:protobuf", ], ) @@ -144,6 +146,17 @@ cc_library( ], ) +cc_library( + name = "transcoder_input_stream", + srcs = [ + "transcoder_input_stream.h", + ], + visibility = ["//visibility:public"], + deps = [ + "@protobuf_git//:protobuf", + ], +) + cc_library( name = "transcoding", srcs = [ @@ -223,6 +236,7 @@ cc_library( srcs = ["test_common.cc"], hdrs = ["test_common.h"], deps = [ + ":transcoder_input_stream", "//external:googletest", "//external:protobuf", "//external:service_config", diff --git a/contrib/endpoints/src/grpc/transcoding/message_reader.cc b/contrib/endpoints/src/grpc/transcoding/message_reader.cc index e5e4aeeae14..23b6a0cbad3 100644 --- a/contrib/endpoints/src/grpc/transcoding/message_reader.cc +++ b/contrib/endpoints/src/grpc/transcoding/message_reader.cc @@ -18,7 +18,6 @@ #include -#include "google/protobuf/io/zero_copy_stream.h" #include "google/protobuf/io/zero_copy_stream_impl.h" namespace google { @@ -29,7 +28,7 @@ namespace transcoding { namespace pb = ::google::protobuf; namespace pbio = ::google::protobuf::io; -MessageReader::MessageReader(pbio::ZeroCopyInputStream* in) +MessageReader::MessageReader(TranscoderInputStream* in) : in_(in), current_message_size_(0), have_current_message_size_(false), @@ -99,7 +98,7 @@ std::unique_ptr MessageReader::NextMessage() { // Check if we have the current message size. If not try to read it. if (!have_current_message_size_) { const size_t kDelimiterSize = 5; - if (in_->ByteCount() < static_cast(kDelimiterSize)) { + if (in_->BytesAvailable() < static_cast(kDelimiterSize)) { // We don't have 5 bytes available to read the length of the message. // Find out whether the stream is finished and return false. finished_ = IsStreamFinished(in_); @@ -117,10 +116,7 @@ std::unique_ptr MessageReader::NextMessage() { have_current_message_size_ = true; } - // We interpret ZeroCopyInputStream::ByteCount() as the number of bytes - // available for reading at the moment. Check if we have the full message - // available to read. - if (in_->ByteCount() < static_cast(current_message_size_)) { + if (in_->BytesAvailable() < static_cast(current_message_size_)) { // We don't have a full message return std::unique_ptr(); } diff --git a/contrib/endpoints/src/grpc/transcoding/message_reader.h b/contrib/endpoints/src/grpc/transcoding/message_reader.h index df175561f47..f07408e23af 100644 --- a/contrib/endpoints/src/grpc/transcoding/message_reader.h +++ b/contrib/endpoints/src/grpc/transcoding/message_reader.h @@ -17,7 +17,7 @@ #include -#include "google/protobuf/io/zero_copy_stream.h" +#include "contrib/endpoints/src/grpc/transcoding/transcoder_input_stream.h" #include "google/protobuf/stubs/status.h" namespace google { @@ -49,11 +49,6 @@ namespace transcoding { // } // } // -// NOTE: MesssageReader assumes that ZeroCopyInputStream::ByteCount() returns -// the number of bytes available to read at the moment. That's what -// MessageReader uses to determine whether there is a complete message -// available or not. -// // NOTE: MessageReader is unable to recognize the case when there is an // incomplete message at the end of the input. The callers will need to // detect it and act appropriately. @@ -64,7 +59,7 @@ namespace transcoding { // class MessageReader { public: - MessageReader(::google::protobuf::io::ZeroCopyInputStream* in); + MessageReader(TranscoderInputStream* in); // If a full message is available, NextMessage() returns a ZeroCopyInputStream // over the message. Otherwise returns nullptr - this might be temporary, the @@ -82,7 +77,7 @@ class MessageReader { bool Finished() const { return finished_; } private: - ::google::protobuf::io::ZeroCopyInputStream* in_; + TranscoderInputStream* in_; // The size of the current message. unsigned int current_message_size_; // Whether we have read the current message size or not diff --git a/contrib/endpoints/src/grpc/transcoding/message_stream.cc b/contrib/endpoints/src/grpc/transcoding/message_stream.cc index 11af66bb170..26a09a23361 100644 --- a/contrib/endpoints/src/grpc/transcoding/message_stream.cc +++ b/contrib/endpoints/src/grpc/transcoding/message_stream.cc @@ -19,7 +19,6 @@ #include #include -#include "google/protobuf/io/zero_copy_stream.h" #include "google/protobuf/io/zero_copy_stream_impl_lite.h" namespace google { @@ -32,12 +31,12 @@ namespace pbio = ::google::protobuf::io; namespace { // a ZeroCopyInputStream implementation over a MessageStream implementation -class ZeroCopyStreamOverMessageStream : public pbio::ZeroCopyInputStream { +class InputStreamOverMessageStream : public TranscoderInputStream { public: - // src - the underlying MessageStream. ZeroCopyStreamOverMessageStream doesn't + // src - the underlying MessageStream. InputStreamOverMessageStream doesn't // maintain the ownership of src, the caller must make sure it exists - // throughtout the lifetime of ZeroCopyStreamOverMessageStream. - ZeroCopyStreamOverMessageStream(MessageStream* src) + // throughtout the lifetime of InputStreamOverMessageStream. + InputStreamOverMessageStream(MessageStream* src) : src_(src), message_(), position_(0) {} // ZeroCopyInputStream implementation @@ -72,19 +71,15 @@ class ZeroCopyStreamOverMessageStream : public pbio::ZeroCopyInputStream { bool Skip(int) { return false; } // Not implemented (no need) - ::google::protobuf::int64 ByteCount() const { - // NOTE: we are changing the ByteCount() interpretation. In our case - // ByteCount() returns the number of bytes available for reading at this - // moment. In the original interpretation it is supposed to be the number - // of bytes read so far. - // We need this such that the consumers are able to read the gRPC delimited - // message stream only if there is a full message available. + google::protobuf::int64 ByteCount() const { return 0; } // Not implemented + + int64_t BytesAvailable() const { if (position_ >= message_.size()) { // If the current message is all done, try to read the next message // to make sure we return the correct byte count. - const_cast(this)->ReadNextMessage(); + const_cast(this)->ReadNextMessage(); } - return static_cast<::google::protobuf::int64>(message_.size() - position_); + return static_cast(message_.size() - position_); } private: @@ -109,10 +104,9 @@ class ZeroCopyStreamOverMessageStream : public pbio::ZeroCopyInputStream { } // namespace -std::unique_ptr<::google::protobuf::io::ZeroCopyInputStream> -MessageStream::CreateZeroCopyInputStream() { - return std::unique_ptr<::google::protobuf::io::ZeroCopyInputStream>( - new ZeroCopyStreamOverMessageStream(this)); +std::unique_ptr MessageStream::CreateInputStream() { + return std::unique_ptr( + new InputStreamOverMessageStream(this)); } } // namespace transcoding diff --git a/contrib/endpoints/src/grpc/transcoding/message_stream.h b/contrib/endpoints/src/grpc/transcoding/message_stream.h index 435040332a7..aef90f524f6 100644 --- a/contrib/endpoints/src/grpc/transcoding/message_stream.h +++ b/contrib/endpoints/src/grpc/transcoding/message_stream.h @@ -18,6 +18,7 @@ #include #include +#include "contrib/endpoints/src/grpc/transcoding/transcoder_input_stream.h" #include "google/protobuf/io/zero_copy_stream.h" #include "google/protobuf/stubs/status.h" @@ -73,8 +74,7 @@ class MessageStream { // Virtual destructor virtual ~MessageStream() {} // Creates ZeroCopyInputStream implementation based on this stream - std::unique_ptr<::google::protobuf::io::ZeroCopyInputStream> - CreateZeroCopyInputStream(); + std::unique_ptr CreateInputStream(); }; } // namespace transcoding diff --git a/contrib/endpoints/src/grpc/transcoding/message_stream_test.cc b/contrib/endpoints/src/grpc/transcoding/message_stream_test.cc index 3699b499bfc..81276310f01 100644 --- a/contrib/endpoints/src/grpc/transcoding/message_stream_test.cc +++ b/contrib/endpoints/src/grpc/transcoding/message_stream_test.cc @@ -70,14 +70,14 @@ class ZeroCopyInputStreamOverMessageStreamTest : public ::testing::Test { bool Test(const Messages& messages) { TestMessageStream test_message_stream; - auto zero_copy_stream = test_message_stream.CreateZeroCopyInputStream(); + auto input_stream = test_message_stream.CreateInputStream(); const void* data = nullptr; int size = 0; // Check that Next() returns true and a 0-sized buffer meaning that // nothing is available at the moment. - if (!zero_copy_stream->Next(&data, &size)) { + if (!input_stream->Next(&data, &size)) { ADD_FAILURE() << "The stream finished unexpectedly" << std::endl; return false; } @@ -91,13 +91,13 @@ class ZeroCopyInputStreamOverMessageStreamTest : public ::testing::Test { test_message_stream.AddMessage(message); // message.size() bytes must be available for reading - if (static_cast(message.size()) != zero_copy_stream->ByteCount()) { - EXPECT_EQ(message.size(), zero_copy_stream->ByteCount()); + if (static_cast(message.size()) != input_stream->BytesAvailable()) { + EXPECT_EQ(message.size(), input_stream->BytesAvailable()); return false; } // Now try to read & match the message - if (!zero_copy_stream->Next(&data, &size)) { + if (!input_stream->Next(&data, &size)) { ADD_FAILURE() << "The stream finished unexpectedly" << std::endl; return false; } @@ -120,16 +120,16 @@ class ZeroCopyInputStreamOverMessageStreamTest : public ::testing::Test { // Not a valid test case continue; } - zero_copy_stream->BackUp(backup_size); + input_stream->BackUp(backup_size); // backup_size bytes must be available for reading again - if (static_cast(backup_size) != zero_copy_stream->ByteCount()) { - EXPECT_EQ(message.size(), zero_copy_stream->ByteCount()); + if (static_cast(backup_size) != input_stream->BytesAvailable()) { + EXPECT_EQ(message.size(), input_stream->BytesAvailable()); return false; } // Now Next() must return the backed up data again. - if (!zero_copy_stream->Next(&data, &size)) { + if (!input_stream->Next(&data, &size)) { ADD_FAILURE() << "The stream finished unexpectedly" << std::endl; return false; } @@ -143,7 +143,7 @@ class ZeroCopyInputStreamOverMessageStreamTest : public ::testing::Test { } // At this point no data should be available - if (!zero_copy_stream->Next(&data, &size)) { + if (!input_stream->Next(&data, &size)) { ADD_FAILURE() << "The stream finished unexpectedly" << std::endl; return false; } @@ -156,7 +156,7 @@ class ZeroCopyInputStreamOverMessageStreamTest : public ::testing::Test { // Now finish the MessageStream & make sure the ZeroCopyInputStream has // ended. test_message_stream.Finish(); - if (zero_copy_stream->Next(&data, &size)) { + if (input_stream->Next(&data, &size)) { ADD_FAILURE() << "The stream still hasn't finished" << std::endl; return false; } @@ -201,14 +201,14 @@ TEST_F(ZeroCopyInputStreamOverMessageStreamTest, DifferenteSizesOneStream) { TEST_F(ZeroCopyInputStreamOverMessageStreamTest, DirectTest) { TestMessageStream test_message_stream; - auto zero_copy_stream = test_message_stream.CreateZeroCopyInputStream(); + auto input_stream = test_message_stream.CreateInputStream(); const void* data = nullptr; int size = 0; // Check that Next() returns true and a 0-sized buffer meaning that // nothing is available at the moment. - EXPECT_TRUE(zero_copy_stream->Next(&data, &size)); + EXPECT_TRUE(input_stream->Next(&data, &size)); EXPECT_EQ(0, size); // Test messages @@ -221,16 +221,16 @@ TEST_F(ZeroCopyInputStreamOverMessageStreamTest, DirectTest) { test_message_stream.AddMessage(message1); // message1 is available for reading - EXPECT_EQ(message1.size(), zero_copy_stream->ByteCount()); - EXPECT_TRUE(zero_copy_stream->Next(&data, &size)); + EXPECT_EQ(message1.size(), input_stream->BytesAvailable()); + EXPECT_TRUE(input_stream->Next(&data, &size)); EXPECT_EQ(message1, std::string(reinterpret_cast(data), size)); // Back up a bit - zero_copy_stream->BackUp(5); + input_stream->BackUp(5); // Now read the backed up data again - EXPECT_EQ(5, zero_copy_stream->ByteCount()); - EXPECT_TRUE(zero_copy_stream->Next(&data, &size)); + EXPECT_EQ(5, input_stream->BytesAvailable()); + EXPECT_TRUE(input_stream->Next(&data, &size)); EXPECT_EQ(message1.substr(message1.size() - 5), std::string(reinterpret_cast(data), size)); @@ -238,20 +238,20 @@ TEST_F(ZeroCopyInputStreamOverMessageStreamTest, DirectTest) { test_message_stream.AddMessage(message2); // message2 is available for reading - EXPECT_EQ(message2.size(), zero_copy_stream->ByteCount()); - EXPECT_TRUE(zero_copy_stream->Next(&data, &size)); + EXPECT_EQ(message2.size(), input_stream->BytesAvailable()); + EXPECT_TRUE(input_stream->Next(&data, &size)); EXPECT_EQ(message2, std::string(reinterpret_cast(data), size)); // Back up all of message2 - zero_copy_stream->BackUp(message2.size()); + input_stream->BackUp(message2.size()); // Now read message2 again - EXPECT_EQ(message2.size(), zero_copy_stream->ByteCount()); - EXPECT_TRUE(zero_copy_stream->Next(&data, &size)); + EXPECT_EQ(message2.size(), input_stream->BytesAvailable()); + EXPECT_TRUE(input_stream->Next(&data, &size)); EXPECT_EQ(message2, std::string(reinterpret_cast(data), size)); // At this point no data should be available - EXPECT_TRUE(zero_copy_stream->Next(&data, &size)); + EXPECT_TRUE(input_stream->Next(&data, &size)); EXPECT_EQ(0, size); // Add both message3 & message4 & finish the MessageStream afterwards @@ -260,16 +260,16 @@ TEST_F(ZeroCopyInputStreamOverMessageStreamTest, DirectTest) { test_message_stream.Finish(); // Read & match both message3 & message4 - EXPECT_EQ(message3.size(), zero_copy_stream->ByteCount()); - EXPECT_TRUE(zero_copy_stream->Next(&data, &size)); + EXPECT_EQ(message3.size(), input_stream->BytesAvailable()); + EXPECT_TRUE(input_stream->Next(&data, &size)); EXPECT_EQ(message3, std::string(reinterpret_cast(data), size)); - EXPECT_EQ(message4.size(), zero_copy_stream->ByteCount()); - EXPECT_TRUE(zero_copy_stream->Next(&data, &size)); + EXPECT_EQ(message4.size(), input_stream->BytesAvailable()); + EXPECT_TRUE(input_stream->Next(&data, &size)); EXPECT_EQ(message4, std::string(reinterpret_cast(data), size)); // All done! - EXPECT_FALSE(zero_copy_stream->Next(&data, &size)); + EXPECT_FALSE(input_stream->Next(&data, &size)); } } // namespace diff --git a/contrib/endpoints/src/grpc/transcoding/response_to_json_translator.cc b/contrib/endpoints/src/grpc/transcoding/response_to_json_translator.cc index eb5e54f9bec..8354cc07f25 100644 --- a/contrib/endpoints/src/grpc/transcoding/response_to_json_translator.cc +++ b/contrib/endpoints/src/grpc/transcoding/response_to_json_translator.cc @@ -18,7 +18,6 @@ #include -#include "google/protobuf/io/zero_copy_stream.h" #include "google/protobuf/io/zero_copy_stream_impl_lite.h" #include "google/protobuf/stubs/status.h" #include "google/protobuf/util/json_util.h" @@ -31,7 +30,7 @@ namespace transcoding { ResponseToJsonTranslator::ResponseToJsonTranslator( ::google::protobuf::util::TypeResolver* type_resolver, std::string type_url, - bool streaming, ::google::protobuf::io::ZeroCopyInputStream* in) + bool streaming, TranscoderInputStream* in) : type_resolver_(type_resolver), type_url_(std::move(type_url)), streaming_(streaming), diff --git a/contrib/endpoints/src/grpc/transcoding/response_to_json_translator.h b/contrib/endpoints/src/grpc/transcoding/response_to_json_translator.h index e41791467f8..d674d71d258 100644 --- a/contrib/endpoints/src/grpc/transcoding/response_to_json_translator.h +++ b/contrib/endpoints/src/grpc/transcoding/response_to_json_translator.h @@ -66,8 +66,7 @@ class ResponseToJsonTranslator : public MessageStream { // format (http://www.grpc.io/docs/guides/wire.html) ResponseToJsonTranslator( ::google::protobuf::util::TypeResolver* type_resolver, - std::string type_url, bool streaming, - ::google::protobuf::io::ZeroCopyInputStream* in); + std::string type_url, bool streaming, TranscoderInputStream* in); // MessageStream implementation bool NextMessage(std::string* message); diff --git a/contrib/endpoints/src/grpc/transcoding/test_common.cc b/contrib/endpoints/src/grpc/transcoding/test_common.cc index 269bc1172d7..12a278dbfa3 100644 --- a/contrib/endpoints/src/grpc/transcoding/test_common.cc +++ b/contrib/endpoints/src/grpc/transcoding/test_common.cc @@ -81,7 +81,7 @@ void TestZeroCopyInputStream::BackUp(int count) { position_ -= count; } -pb::int64 TestZeroCopyInputStream::ByteCount() const { +int64_t TestZeroCopyInputStream::BytesAvailable() const { auto total = current_.size() - position_; for (auto chunk : chunks_) { total += chunk.size(); diff --git a/contrib/endpoints/src/grpc/transcoding/test_common.h b/contrib/endpoints/src/grpc/transcoding/test_common.h index bc452e052fc..b6f2532b867 100644 --- a/contrib/endpoints/src/grpc/transcoding/test_common.h +++ b/contrib/endpoints/src/grpc/transcoding/test_common.h @@ -20,6 +20,7 @@ #include #include +#include "contrib/endpoints/src/grpc/transcoding/transcoder_input_stream.h" #include "google/api/service.pb.h" #include "google/protobuf/io/zero_copy_stream.h" #include "google/protobuf/text_format.h" @@ -33,8 +34,7 @@ namespace testing { // An implementation of ZeroCopyInputStream for testing. // The tests define the chunks that TestZeroCopyInputStream produces. -class TestZeroCopyInputStream - : public ::google::protobuf::io::ZeroCopyInputStream { +class TestZeroCopyInputStream : public TranscoderInputStream { public: TestZeroCopyInputStream(); @@ -50,8 +50,9 @@ class TestZeroCopyInputStream // ZeroCopyInputStream methods bool Next(const void** data, int* size); void BackUp(int count); - ::google::protobuf::int64 ByteCount() const; - bool Skip(int) { return false; } // Not implemented + int64_t BytesAvailable() const; + ::google::protobuf::int64 ByteCount() const { return 0; } // Not implemented + bool Skip(int) { return false; } // Not implemented private: std::deque chunks_; diff --git a/contrib/endpoints/src/grpc/transcoding/transcoder.h b/contrib/endpoints/src/grpc/transcoding/transcoder.h index da58386089d..806c519f921 100644 --- a/contrib/endpoints/src/grpc/transcoding/transcoder.h +++ b/contrib/endpoints/src/grpc/transcoding/transcoder.h @@ -15,7 +15,7 @@ #ifndef GRPC_TRANSCODING_TRANSCODER_H_ #define GRPC_TRANSCODING_TRANSCODER_H_ -#include "google/protobuf/io/zero_copy_stream.h" +#include "contrib/endpoints/src/grpc/transcoding/transcoder_input_stream.h" #include "google/protobuf/stubs/status.h" namespace google { @@ -28,11 +28,11 @@ namespace transcoding { // - translated response stream, // - status of response translation. // -// NOTE: Transcoder uses ::google::protobuf::io::ZeroCopyInputStream for -// carrying the payloads both for input and output. It assumes the -// following interpretation of the ZeroCopyInputStream interface: +// NOTE: Transcoder uses TranscoderInputStream for carrying the payloads +// both for input and output. It assumes the following interpretation +// of the TranscoderInputStream interface: // -// bool ZeroCopyInputStream::Next(const void** data, int* size); +// bool TranscoderInputStream::Next(const void** data, int* size); // // Obtains a chunk of data from the stream. // @@ -52,7 +52,7 @@ namespace transcoding { // again later. // // -// void ZeroCopyInputStream::BackUp(int count); +// void TranscoderInputStream::BackUp(int count); // // Backs up a number of bytes, so that the next call to Next() returns // data again that was already returned by the last call to Next(). This @@ -72,12 +72,12 @@ namespace transcoding { // the same data again before producing new data. // // -// bool ZeroCopyInputStream::Skip(int count); +// bool TranscoderInputStream::Skip(int count); // // Not used and not implemented by the Transcoder. // // -// int64 ZeroCopyInputStream::ByteCount() const; +// int64_t TranscoderInputStream::BytesAvailable() const; // // Returns the number of bytes available for reading at this moment // @@ -133,7 +133,7 @@ namespace transcoding { class Transcoder { public: // ZeroCopyInputStream to read the transcoded request. - virtual ::google::protobuf::io::ZeroCopyInputStream* RequestOutput() = 0; + virtual TranscoderInputStream* RequestOutput() = 0; // The status of request transcoding virtual ::google::protobuf::util::Status RequestStatus() = 0; diff --git a/contrib/endpoints/src/grpc/transcoding/transcoder_factory.cc b/contrib/endpoints/src/grpc/transcoding/transcoder_factory.cc index 2dfcf511a17..f42b6d23108 100644 --- a/contrib/endpoints/src/grpc/transcoding/transcoder_factory.cc +++ b/contrib/endpoints/src/grpc/transcoding/transcoder_factory.cc @@ -52,29 +52,23 @@ class TranscoderImpl : public Transcoder { std::unique_ptr response_translator) : request_translator_(std::move(request_translator)), response_translator_(std::move(response_translator)), - request_zero_copy_stream_( - request_translator_->Output().CreateZeroCopyInputStream()), - response_zero_copy_stream_( - response_translator_->CreateZeroCopyInputStream()) {} + request_stream_(request_translator_->Output().CreateInputStream()), + response_stream_(response_translator_->CreateInputStream()) {} // Transcoder implementation - pbio::ZeroCopyInputStream* RequestOutput() { - return request_zero_copy_stream_.get(); - } + TranscoderInputStream* RequestOutput() { return request_stream_.get(); } pbutil::Status RequestStatus() { return request_translator_->Output().Status(); } - pbio::ZeroCopyInputStream* ResponseOutput() { - return response_zero_copy_stream_.get(); - } + pbio::ZeroCopyInputStream* ResponseOutput() { return response_stream_.get(); } pbutil::Status ResponseStatus() { return response_translator_->Status(); } private: std::unique_ptr request_translator_; std::unique_ptr response_translator_; - std::unique_ptr request_zero_copy_stream_; - std::unique_ptr response_zero_copy_stream_; + std::unique_ptr request_stream_; + std::unique_ptr response_stream_; }; // Converts MethodCallInfo into a RequestInfo structure needed by the @@ -132,7 +126,7 @@ TranscoderFactory::TranscoderFactory(const ::google::api::Service& service) pbutil::Status TranscoderFactory::Create( const MethodCallInfo& call_info, pbio::ZeroCopyInputStream* request_input, - pbio::ZeroCopyInputStream* response_input, + TranscoderInputStream* response_input, std::unique_ptr* transcoder) { // Convert MethodCallInfo into RequestInfo RequestInfo request_info; diff --git a/contrib/endpoints/src/grpc/transcoding/transcoder_factory.h b/contrib/endpoints/src/grpc/transcoding/transcoder_factory.h index b7c10865025..d80c93c7f86 100644 --- a/contrib/endpoints/src/grpc/transcoding/transcoder_factory.h +++ b/contrib/endpoints/src/grpc/transcoding/transcoder_factory.h @@ -19,6 +19,7 @@ #include "contrib/endpoints/include/api_manager/method_call_info.h" #include "contrib/endpoints/src/grpc/transcoding/transcoder.h" +#include "contrib/endpoints/src/grpc/transcoding/transcoder_input_stream.h" #include "contrib/endpoints/src/grpc/transcoding/type_helper.h" #include "google/api/service.pb.h" #include "google/protobuf/io/zero_copy_stream.h" @@ -71,7 +72,7 @@ class TranscoderFactory { ::google::protobuf::util::Status Create( const MethodCallInfo& call_info, ::google::protobuf::io::ZeroCopyInputStream* request_input, - ::google::protobuf::io::ZeroCopyInputStream* response_input, + TranscoderInputStream* response_input, std::unique_ptr* transcoder); private: diff --git a/contrib/endpoints/src/grpc/transcoding/transcoder_input_stream.h b/contrib/endpoints/src/grpc/transcoding/transcoder_input_stream.h new file mode 100644 index 00000000000..240067d600b --- /dev/null +++ b/contrib/endpoints/src/grpc/transcoding/transcoder_input_stream.h @@ -0,0 +1,35 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef GRPC_TRANSCODING_TRANSCODER_INPUT_STREAM_H_ +#define GRPC_TRANSCODING_TRANSCODER_INPUT_STREAM_H_ + +#include "google/protobuf/io/zero_copy_stream.h" + +namespace google { +namespace api_manager { +namespace transcoding { + +class TranscoderInputStream + : public virtual google::protobuf::io::ZeroCopyInputStream { + public: + // returns the number of bytes available to read at the moment. + virtual int64_t BytesAvailable() const = 0; +}; + +} // namespace transcoding +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_TRANSCODER_H_ diff --git a/contrib/endpoints/src/grpc/transcoding/transcoder_test.cc b/contrib/endpoints/src/grpc/transcoding/transcoder_test.cc index eb2a38eaf5d..86fcc1911c1 100644 --- a/contrib/endpoints/src/grpc/transcoding/transcoder_test.cc +++ b/contrib/endpoints/src/grpc/transcoding/transcoder_test.cc @@ -139,7 +139,7 @@ class TranscoderTest : public ::testing::Test { } pbutil::Status Build(pbio::ZeroCopyInputStream *request_input, - pbio::ZeroCopyInputStream *response_input, + TranscoderInputStream *response_input, std::unique_ptr *transcoder) { MethodCallInfo call_info; call_info.method_info = method_info_.get(); From 1ef1b4a3aec1f1728a160f8eb6eabf1fdfce8a04 Mon Sep 17 00:00:00 2001 From: Jae Kim Date: Thu, 6 Apr 2017 23:04:52 -0700 Subject: [PATCH 77/83] Merge latest changes from rate_limiting to master (#221) * Point to googleapi in service control client. (#91) * Point to googleapi in service control client. * Use git repository for service-control-client. * Merge latest changes from master (#104) * Get attributes from envoy config. (#87) * Send all attributes. * Remove unused const strings. * Address comment. * updated SHA to point to newer envoy with RDS API feature (#94) * Disable travis on stable branches (#96) * Publish debug binaries (no release yet) (#98) * Copies the binary instead of linking for release (#102) * Extract quota config from service config. (#101) * Add metric_cost in config. * Remove group rules. * Call loadQuotaConfig in config::create. * Update latest update from master branch (#106) * Get attributes from envoy config. (#87) * Send all attributes. * Remove unused const strings. * Address comment. * updated SHA to point to newer envoy with RDS API feature (#94) * Disable travis on stable branches (#96) * Publish debug binaries (no release yet) (#98) * Copies the binary instead of linking for release (#102) * Added quota contoll without the service control client library (#93) * Added quota contoll without the service control client library * Applied code review * Applied code review * Resolve conflicts * Resolve conflicts * Fixed format error reported by script/check-style * Fixed a bug at Aggregated::GetAuthToken that causes Segmentation Fault * Changed usage of template funcion * Applied latest changes from the repo * Applied latest changes from the repo * Applied latest changes from the repo * Adde comments * Updated log information * Applied #101 * Changed metric_cost_map to metric_cost_vector * Fixed test case compilation error * Fixed test case compilation error * Add unit test for quota config. (#108) * Add unit test for quota config. * Add comments. * Update test specifics. * Merge latest changes from master branch (#112) * Get attributes from envoy config. (#87) * Send all attributes. * Remove unused const strings. * Address comment. * updated SHA to point to newer envoy with RDS API feature (#94) * Disable travis on stable branches (#96) * Publish debug binaries (no release yet) (#98) * Copies the binary instead of linking for release (#102) * Not to use api_key if its service is not actived. (#109) * If QuotaControl service is not available, return utils::Status::OK (#113) * If QuotaControl service is not available, return utils::Status::OK * Updated comment * Return HTTP status code 429 on google.rpc.Code.RESOURCE_EXHAUSTED (#119) * Fixed incorrectly resolved conflicts (#123) * Added unit test cases for rate limiting (#124) * Fixed incorrectly resolved conflicts * Added unit test cases for rate limiting * Added unit test cases for rate limiting * Added unit test cases for rate limiting * Added unit test cases for rate limiting * Added unit test cases for rate limiting * Added unit test cases for rate limiting * Rename response.http.code (#125) (#128) * Added handling of error code QUOTA_SYSTEM_UNAVAILABLE (#148) * Integrated service control client library with quota cache aggregation (#149) * Fixed error on merge (#151) * Integrated service control client library with quota cache aggregation * Fixed error on merge * Fixed the compatibility issue with the latest update on esp (#152) * Removed copied proto files (#208) * Set default allocate quota request timeout to 1sec and applied latest service control client library change (#211) * Merged key_restriction related changes from master (#213) * Merge latest changes from master branch (#217) * Not call report if decodeHeaders is not called. (#150) * Update mixerclient with sync-ed grpc write and fail-fast. (#155) * Update mixerclient with sync-ed write and fail-fast. * Update to latest test. * Update again * Update envoy to PR553 (#156) * Update envoy to PR553 * Update libevent to 2.1.8 * Uses a specific version of the Shared Pipeline lib (#158) * Update lyft/envoy commit Id to latest. (#161) * Update lyft/envoy commit Id to latest. * Remove the comment about pull request * Add new line - will delete in next commit. * Update repositories.bzl (#169) * Always set response latency (#172) * Update mixerclient to sync_transport change. (#178) * Use opaque config to turn on/off forward attribute and mixer filter (#179) * Modify mixer filter * Swap defaults * Make the filter decoder only * cache mixer disabled decision * Fix a bug in opaque config change and test it out (#182) * Fix a bug and test it out * Update filter type * Update README.md * Update mixer client to mixer api with gogoproto. (#184) * Move .bazelrc to tools/bazel.rc (#186) * Move .bazelrc to tools/bazel.rc * Update Jenkinsfile with latest version of pipeline * Support apikey based traffic restriction (#189) * b/36368559 support apikey based traffic restriction * Fixed code formatting * Fix crash in unreachable/overloaded RDS (#190) * Add mixer client end to end integration test. (#177) * Add mixer client end to end integration test. * Split some repositories into a separate file. * use real mixer for fake mixer_server. * Test repository * use mixer bzl file. * Use mixer repositories * Not to use mixer repository. * Add return line at the end of WORKSPACE. * Fix broken link (#193) * Make quota call (#192) * hookup quota call * Make quota call. * Update indent. * Update envoy and update configs (#195) * Update envoy and update configs * Use gcc-4.9 for travis * Use bazel 0.4.5 * Fix SHA of lightstep-tracer-common * Enable check cache and refactory mixer config loading (#197) * Refactory the mixer config loading. * fix format * Add integration test. * updated README.md * s/send/sent/ * Split into separate tests. (#201) * Update README on how to enable check cache. (#204) * Update README on how to enable check cache. * Update the comment. * build: support Envoy native Bazel build. (#210) * build: support Envoy native Bazel build. This patch switches the Envoy build from src/envoy/repositories.bzl to using the upstream native build. See https://github.com/lyft/envoy/pull/663 for the corresponding changes on the Envoy side. * Use Envoy master with BUILD.wip rename merged. * Fix clang-format issues. * Fixes bazel.rc issues (#212) * Fixes bazel rc issues * Update Jenkins to latest pipeline version * Updated the commit id of cloudendpoints/service-control-client-cxx (#218) * Update commitid of cloudendpoints/service-control-client-cxx repo (#220) --- .../endpoints/include/api_manager/method.h | 4 + contrib/endpoints/repositories.bzl | 22 +- contrib/endpoints/src/api_manager/BUILD | 2 + .../api_manager/auth/service_account_token.h | 1 + .../src/api_manager/check_workflow.cc | 3 + contrib/endpoints/src/api_manager/config.cc | 20 ++ contrib/endpoints/src/api_manager/config.h | 3 + .../endpoints/src/api_manager/config_test.cc | 84 ++++++++ .../api_manager/context/request_context.cc | 10 + .../src/api_manager/context/request_context.h | 3 + .../endpoints/src/api_manager/method_impl.h | 18 +- .../src/api_manager/mock_method_info.h | 2 + .../src/api_manager/proto/server_config.proto | 18 ++ .../src/api_manager/quota_control.cc | 57 +++++ .../endpoints/src/api_manager/quota_control.h | 33 +++ .../src/api_manager/service_control/BUILD | 13 ++ .../api_manager/service_control/aggregated.cc | 200 ++++++++++++++---- .../api_manager/service_control/aggregated.h | 21 +- .../service_control/aggregated_test.cc | 131 ++++++++++++ .../allocate_quota_response_test.cc | 184 ++++++++++++++++ .../src/api_manager/service_control/info.h | 9 +- .../api_manager/service_control/interface.h | 11 + .../src/api_manager/service_control/proto.cc | 156 ++++++++++++++ .../src/api_manager/service_control/proto.h | 9 + .../api_manager/service_control/proto_test.cc | 50 +++++ .../testdata/allocate_quota_request.golden | 36 ++++ .../allocate_quota_request_android_ios.golden | 48 +++++ ...locate_quota_request_no_method_name.golden | 35 +++ .../src/api_manager/service_control/url.cc | 2 + .../src/api_manager/service_control/url.h | 2 + .../api_manager/service_control/url_test.cc | 4 + .../src/grpc/transcoding/transcoder_test.cc | 5 + 32 files changed, 1145 insertions(+), 51 deletions(-) create mode 100644 contrib/endpoints/src/api_manager/quota_control.cc create mode 100644 contrib/endpoints/src/api_manager/quota_control.h create mode 100644 contrib/endpoints/src/api_manager/service_control/allocate_quota_response_test.cc create mode 100644 contrib/endpoints/src/api_manager/service_control/testdata/allocate_quota_request.golden create mode 100644 contrib/endpoints/src/api_manager/service_control/testdata/allocate_quota_request_android_ios.golden create mode 100644 contrib/endpoints/src/api_manager/service_control/testdata/allocate_quota_request_no_method_name.golden diff --git a/contrib/endpoints/include/api_manager/method.h b/contrib/endpoints/include/api_manager/method.h index 6b7c01072b8..1680b6950ff 100644 --- a/contrib/endpoints/include/api_manager/method.h +++ b/contrib/endpoints/include/api_manager/method.h @@ -89,6 +89,10 @@ class MethodInfo { // Get the names of url system parameters virtual const std::set &system_query_parameter_names() const = 0; + + // Get quota metric cost vector + virtual const std::vector> &metric_cost_vector() + const = 0; }; } // namespace api_manager diff --git a/contrib/endpoints/repositories.bzl b/contrib/endpoints/repositories.bzl index bae14e92d61..19ab3bf402a 100644 --- a/contrib/endpoints/repositories.bzl +++ b/contrib/endpoints/repositories.bzl @@ -211,12 +211,9 @@ def googleapis_repositories(protobuf_repo="@protobuf_git//", bind=True): # ################################################################################ # - licenses(["notice"]) -load("{}:protobuf.bzl", "cc_proto_library") - -exports_files(glob(["google/**"])) +load("@protobuf_git//:protobuf.bzl", "cc_proto_library") cc_proto_library( name = "servicecontrol", @@ -259,9 +256,13 @@ cc_proto_library( "google/api/log.proto", "google/api/logging.proto", "google/api/metric.proto", + "google/api/experimental/experimental.proto", + "google/api/experimental/authorization_config.proto", "google/api/monitored_resource.proto", "google/api/monitoring.proto", + "google/api/quota.proto", "google/api/service.proto", + "google/api/source_info.proto", "google/api/system_parameter.proto", "google/api/usage.proto", ], @@ -290,10 +291,9 @@ cc_proto_library( ) """.format(protobuf_repo) - native.new_git_repository( name = "googleapis_git", - commit = "db1d4547dc56a798915e0eb2c795585385922165", + commit = "2fe0050bd2a6d4c6ba798c0311f0b149b8997314", remote = "https://github.com/googleapis/googleapis.git", build_file_content = BUILD, ) @@ -324,7 +324,7 @@ def servicecontrol_client_repositories(bind=True): native.git_repository( name = "servicecontrol_client_git", - commit = "d739d755365c6a13d0b4164506fd593f53932f5d", + commit = "3d1a30d9221e700542eeaaf20eab69faddb63894", remote = "https://github.com/cloudendpoints/service-control-client-cxx.git", ) @@ -333,3 +333,11 @@ def servicecontrol_client_repositories(bind=True): name = "servicecontrol_client", actual = "@servicecontrol_client_git//:service_control_client_lib", ) + native.bind( + name = "quotacontrol", + actual = "@servicecontrol_client_git//proto:quotacontrol", + ) + native.bind( + name = "quotacontrol_genproto", + actual = "@servicecontrol_client_git//proto:quotacontrol_genproto", + ) diff --git a/contrib/endpoints/src/api_manager/BUILD b/contrib/endpoints/src/api_manager/BUILD index e3d54b46eac..76416aea90e 100644 --- a/contrib/endpoints/src/api_manager/BUILD +++ b/contrib/endpoints/src/api_manager/BUILD @@ -81,6 +81,8 @@ cc_library( "method_impl.cc", "path_matcher.cc", "path_matcher_node.cc", + "quota_control.cc", + "quota_control.h", "request_handler.cc", ], linkopts = select({ diff --git a/contrib/endpoints/src/api_manager/auth/service_account_token.h b/contrib/endpoints/src/api_manager/auth/service_account_token.h index 211377449f4..eca3e148f68 100644 --- a/contrib/endpoints/src/api_manager/auth/service_account_token.h +++ b/contrib/endpoints/src/api_manager/auth/service_account_token.h @@ -64,6 +64,7 @@ class ServiceAccountToken { enum JWT_TOKEN_TYPE { JWT_TOKEN_FOR_SERVICE_CONTROL = 0, JWT_TOKEN_FOR_CLOUD_TRACING, + JWT_TOKEN_FOR_QUOTA_CONTROL, JWT_TOKEN_TYPE_MAX, }; // Set audience. Only calcualtes JWT token with specified audience. diff --git a/contrib/endpoints/src/api_manager/check_workflow.cc b/contrib/endpoints/src/api_manager/check_workflow.cc index 8335d779142..27b9fba60da 100644 --- a/contrib/endpoints/src/api_manager/check_workflow.cc +++ b/contrib/endpoints/src/api_manager/check_workflow.cc @@ -18,6 +18,7 @@ #include "contrib/endpoints/src/api_manager/check_auth.h" #include "contrib/endpoints/src/api_manager/check_service_control.h" #include "contrib/endpoints/src/api_manager/fetch_metadata.h" +#include "contrib/endpoints/src/api_manager/quota_control.h" using ::google::api_manager::utils::Status; @@ -33,6 +34,8 @@ void CheckWorkflow::RegisterAll() { Register(CheckAuth); // Checks service control. Register(CheckServiceControl); + // Quota control + Register(QuotaControl); } void CheckWorkflow::Register(CheckHandler handler) { diff --git a/contrib/endpoints/src/api_manager/config.cc b/contrib/endpoints/src/api_manager/config.cc index b0a1b85d119..bcc0e3babee 100644 --- a/contrib/endpoints/src/api_manager/config.cc +++ b/contrib/endpoints/src/api_manager/config.cc @@ -113,6 +113,23 @@ MethodInfoImpl *Config::GetOrCreateMethodInfoImpl(const string &name, return i->second.get(); } +bool Config::LoadQuotaRule(ApiManagerEnvInterface *env) { + for (const auto &rule : service_.quota().metric_rules()) { + auto method = utils::FindOrNull(method_map_, rule.selector()); + if (method) { + for (auto &metric_cost : rule.metric_costs()) { + (*method)->add_metric_cost(metric_cost.first, metric_cost.second); + } + } else { + env->LogError("Metric rule with selector " + rule.selector() + + "is mismatched."); + return false; + } + } + + return true; +} + bool Config::LoadHttpMethods(ApiManagerEnvInterface *env, PathMatcherBuilder *pmb) { std::set all_urls, urls_with_options; @@ -443,6 +460,9 @@ std::unique_ptr Config::Create(ApiManagerEnvInterface *env, if (!config->LoadBackends(env)) { return nullptr; } + if (!config->LoadQuotaRule(env)) { + return nullptr; + } return config; } diff --git a/contrib/endpoints/src/api_manager/config.h b/contrib/endpoints/src/api_manager/config.h index 9a56d16d745..bdff7e08826 100644 --- a/contrib/endpoints/src/api_manager/config.h +++ b/contrib/endpoints/src/api_manager/config.h @@ -25,6 +25,7 @@ #include "contrib/endpoints/src/api_manager/method_impl.h" #include "contrib/endpoints/src/api_manager/path_matcher.h" #include "contrib/endpoints/src/api_manager/proto/server_config.pb.h" +#include "google/api/quota.pb.h" #include "google/api/service.pb.h" namespace google { @@ -111,6 +112,8 @@ class Config { // Load SystemParameters info to MethodInfo. bool LoadSystemParameters(ApiManagerEnvInterface *env); + bool LoadQuotaRule(ApiManagerEnvInterface *env); + // Gets the MethodInfoImpl creating it if necessary MethodInfoImpl *GetOrCreateMethodInfoImpl(const std::string &name, const std::string &api_name, diff --git a/contrib/endpoints/src/api_manager/config_test.cc b/contrib/endpoints/src/api_manager/config_test.cc index ace0d2afc49..b133df42226 100644 --- a/contrib/endpoints/src/api_manager/config_test.cc +++ b/contrib/endpoints/src/api_manager/config_test.cc @@ -870,6 +870,90 @@ TEST(Config, TestCorsDisabled) { ASSERT_EQ(nullptr, method1); } +TEST(Config, TestInvalidMetricRules) { + MockApiManagerEnvironmentWithLog env; + // There is no http.rule or api.method to match the selector. + static const char config_text[] = R"( +name: "Service.Name" +quota { + metric_rules { + selector: "GetShelves" + metric_costs { + key: "test.googleapis.com/operation/read_book" + value: 100 + } + } +} +)"; + + std::unique_ptr config = Config::Create(&env, config_text, ""); + EXPECT_EQ(nullptr, config); +} + +TEST(Config, TestMetricRules) { + MockApiManagerEnvironmentWithLog env; + static const char config_text[] = R"( +name: "Service.Name" +http { + rules { + selector: "DeleteShelf" + delete: "/shelves" + } + rules { + selector: "GetShelves" + get: "/shelves" + } +} +quota { + metric_rules { + selector: "GetShelves" + metric_costs { + key: "test.googleapis.com/operation/get_shelves" + value: 100 + } + metric_costs { + key: "test.googleapis.com/operation/request" + value: 10 + } + } + metric_rules { + selector: "DeleteShelf" + metric_costs { + key: "test.googleapis.com/operation/delete_shelves" + value: 200 + } + } +} +)"; + + std::unique_ptr config = Config::Create(&env, config_text, ""); + ASSERT_TRUE(config); + + const MethodInfo *method1 = config->GetMethodInfo("GET", "/shelves"); + ASSERT_NE(nullptr, method1); + + std::vector> metric_cost_vector = + method1->metric_cost_vector(); + std::sort(metric_cost_vector.begin(), metric_cost_vector.end()); + ASSERT_EQ(2, metric_cost_vector.size()); + ASSERT_EQ("test.googleapis.com/operation/get_shelves", + metric_cost_vector[0].first); + ASSERT_EQ(100, metric_cost_vector[0].second); + + ASSERT_EQ("test.googleapis.com/operation/request", + metric_cost_vector[1].first); + ASSERT_EQ(10, metric_cost_vector[1].second); + + const MethodInfo *method2 = config->GetMethodInfo("DELETE", "/shelves"); + ASSERT_NE(nullptr, method1); + + metric_cost_vector = method2->metric_cost_vector(); + ASSERT_EQ(1, metric_cost_vector.size()); + ASSERT_EQ("test.googleapis.com/operation/delete_shelves", + metric_cost_vector[0].first); + ASSERT_EQ(200, metric_cost_vector[0].second); +} + } // namespace } // namespace api_manager diff --git a/contrib/endpoints/src/api_manager/context/request_context.cc b/contrib/endpoints/src/api_manager/context/request_context.cc index 24508a4a981..2c0a2985927 100644 --- a/contrib/endpoints/src/api_manager/context/request_context.cc +++ b/contrib/endpoints/src/api_manager/context/request_context.cc @@ -241,6 +241,16 @@ void RequestContext::FillCheckRequestInfo( request_->FindHeader(kXIosBundleId, &info->ios_bundle_id); } +void RequestContext::FillAllocateQuotaRequestInfo( + service_control::QuotaRequestInfo *info) { + FillOperationInfo(info); + + info->client_ip = request_->GetClientIP(); + info->method_name = this->method_call_.method_info->name(); + info->metric_cost_vector = + &this->method_call_.method_info->metric_cost_vector(); +} + void RequestContext::FillReportRequestInfo( Response *response, service_control::ReportRequestInfo *info) { FillOperationInfo(info); diff --git a/contrib/endpoints/src/api_manager/context/request_context.h b/contrib/endpoints/src/api_manager/context/request_context.h index 5b29c271aad..8832b38fc3e 100644 --- a/contrib/endpoints/src/api_manager/context/request_context.h +++ b/contrib/endpoints/src/api_manager/context/request_context.h @@ -66,6 +66,9 @@ class RequestContext { // Fill CheckRequestInfo void FillCheckRequestInfo(service_control::CheckRequestInfo *info); + // FillAllocateQuotaRequestInfo + void FillAllocateQuotaRequestInfo(service_control::QuotaRequestInfo *info); + // Fill ReportRequestInfo void FillReportRequestInfo(Response *response, service_control::ReportRequestInfo *info); diff --git a/contrib/endpoints/src/api_manager/method_impl.h b/contrib/endpoints/src/api_manager/method_impl.h index e5739d639bc..d6eaba33724 100644 --- a/contrib/endpoints/src/api_manager/method_impl.h +++ b/contrib/endpoints/src/api_manager/method_impl.h @@ -18,6 +18,7 @@ #include #include #include +#include #include "contrib/endpoints/include/api_manager/method.h" #include "contrib/endpoints/src/api_manager/utils/stl_util.h" @@ -62,6 +63,10 @@ class MethodInfoImpl : public MethodInfo { const std::string &backend_address() const { return backend_address_; } + const std::vector> &metric_cost_vector() const { + return metric_cost_vector_; + } + const std::string &rpc_method_full_name() const { return rpc_method_full_name_; } @@ -90,6 +95,10 @@ class MethodInfoImpl : public MethodInfo { url_query_parameters_[name].push_back(url_query_parameter); } + void add_metric_cost(const std::string &metric, int64_t cost) { + metric_cost_vector_.push_back(std::make_pair(metric, cost)); + } + // After add all system parameters, lookup some of them to cache // their lookup results. void process_system_parameters(); @@ -139,13 +148,13 @@ class MethodInfoImpl : public MethodInfo { // such as API Key)? bool allow_unregistered_calls_; // Issuers to allowed audiences map. - std::map > issuer_audiences_map_; + std::map> issuer_audiences_map_; // system parameter map of parameter name to http_header name. - std::map > http_header_parameters_; + std::map> http_header_parameters_; // system parameter map of parameter name to url query parameter name. - std::map > url_query_parameters_; + std::map> url_query_parameters_; // all the names of system query parameters std::set system_query_parameter_names_; @@ -175,6 +184,9 @@ class MethodInfoImpl : public MethodInfo { // Whether the response is streaming or not. bool response_streaming_; + + // map of metric and its cost + std::vector> metric_cost_vector_; }; typedef std::unique_ptr MethodInfoImplPtr; diff --git a/contrib/endpoints/src/api_manager/mock_method_info.h b/contrib/endpoints/src/api_manager/mock_method_info.h index a7de28e3422..6b78e86228b 100644 --- a/contrib/endpoints/src/api_manager/mock_method_info.h +++ b/contrib/endpoints/src/api_manager/mock_method_info.h @@ -48,6 +48,8 @@ class MockMethodInfo : public MethodInfo { MOCK_CONST_METHOD0(response_streaming, bool()); MOCK_CONST_METHOD0(system_query_parameter_names, const std::set&()); + MOCK_CONST_METHOD0(metric_cost_vector, + const std::vector>&()); }; } // namespace api_manager diff --git a/contrib/endpoints/src/api_manager/proto/server_config.proto b/contrib/endpoints/src/api_manager/proto/server_config.proto index 1fe04af6d07..7f18460c10b 100644 --- a/contrib/endpoints/src/api_manager/proto/server_config.proto +++ b/contrib/endpoints/src/api_manager/proto/server_config.proto @@ -66,6 +66,13 @@ message ServiceControlConfig { // The intermediate reports for streaming calls should not be more frequent // than this value (in seconds) int32 intermediate_report_min_interval = 7; + + // Quota aggregator config + QuotaAggregatorConfig quota_aggregator_config = 8; + + // Timeout in milliseconds on service control allocate quota requests. + // If the value is <= 0, default timeout is 5000 milliseconds. + int32 quota_timeout_ms = 9; } // Check aggregator config @@ -82,6 +89,17 @@ message CheckAggregatorConfig { int32 response_expiration_ms = 3; } +// Quota aggregator config +message QuotaAggregatorConfig { + // The maximum number of cache entries that can be kept in the aggregation + // cache. Cache is disabled when entries <= 0. + int32 cache_entries = 1; + + // The maximum milliseconds before aggregated quota requests are refreshed to + // the server. + int32 refresh_interval_ms = 2; +} + // Report aggregator config message ReportAggregatorConfig { // The maximum number of cache entries that can be kept in the aggregation diff --git a/contrib/endpoints/src/api_manager/quota_control.cc b/contrib/endpoints/src/api_manager/quota_control.cc new file mode 100644 index 00000000000..8a5b65322d6 --- /dev/null +++ b/contrib/endpoints/src/api_manager/quota_control.cc @@ -0,0 +1,57 @@ +// Copyright 2017 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include + +#include "contrib/endpoints/src/api_manager/cloud_trace/cloud_trace.h" +#include "contrib/endpoints/src/api_manager/quota_control.h" +#include "google/protobuf/stubs/status.h" + +using ::google::api_manager::utils::Status; +using ::google::protobuf::util::error::Code; + +namespace google { +namespace api_manager { + +void QuotaControl(std::shared_ptr context, + std::function continuation) { + std::shared_ptr trace_span( + CreateSpan(context->cloud_trace(), "QuotaControl")); + + if (context->method()->metric_cost_vector().size() == 0) { + TRACE(trace_span) << "Quota control check is not needed"; + continuation(Status::OK); + return; + } + + service_control::QuotaRequestInfo info; + context->FillAllocateQuotaRequestInfo(&info); + context->service_context()->service_control()->Quota( + info, trace_span.get(), + [context, continuation, trace_span](utils::Status status) { + + TRACE(trace_span) << "Quota service control request returned with " + << "status " << status.ToString(); + + // quota control is using "failed open" policy. If the server is not + // available, allow the request to go. + continuation((status.code() == Code::UNAVAILABLE) ? utils::Status::OK + : status); + }); +} + +} // namespace service_control_client +} // namespace google diff --git a/contrib/endpoints/src/api_manager/quota_control.h b/contrib/endpoints/src/api_manager/quota_control.h new file mode 100644 index 00000000000..e4f94d6ac93 --- /dev/null +++ b/contrib/endpoints/src/api_manager/quota_control.h @@ -0,0 +1,33 @@ +// Copyright 2017 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#ifndef API_MANAGER_QUOTA_CONTROL_H_ +#define API_MANAGER_QUOTA_CONTROL_H_ + +#include "contrib/endpoints/include/api_manager/utils/status.h" +#include "contrib/endpoints/src/api_manager/context/request_context.h" + +namespace google { +namespace api_manager { + +// Call service control quota. +void QuotaControl(std::shared_ptr, + std::function); + +} // namespace api_manager +} // namespace google + +#endif // API_MANAGER_QUOTA_CONTROL_H_ diff --git a/contrib/endpoints/src/api_manager/service_control/BUILD b/contrib/endpoints/src/api_manager/service_control/BUILD index 73b722b4808..c0e35adca07 100644 --- a/contrib/endpoints/src/api_manager/service_control/BUILD +++ b/contrib/endpoints/src/api_manager/service_control/BUILD @@ -122,3 +122,16 @@ cc_test( "//external:googletest_main", ], ) + +cc_test( + name = "allocate_quota_response_test", + size = "small", + srcs = [ + "allocate_quota_response_test.cc", + ], + linkstatic = 1, + deps = [ + ":service_control", + "//external:googletest_main", + ], +) diff --git a/contrib/endpoints/src/api_manager/service_control/aggregated.cc b/contrib/endpoints/src/api_manager/service_control/aggregated.cc index e59a69dc8cc..f1ef15e7ee2 100644 --- a/contrib/endpoints/src/api_manager/service_control/aggregated.cc +++ b/contrib/endpoints/src/api_manager/service_control/aggregated.cc @@ -22,6 +22,8 @@ using ::google::api::servicecontrol::v1::CheckRequest; using ::google::api::servicecontrol::v1::CheckResponse; +using ::google::api::servicecontrol::v1::AllocateQuotaRequest; +using ::google::api::servicecontrol::v1::AllocateQuotaResponse; using ::google::api::servicecontrol::v1::ReportRequest; using ::google::api::servicecontrol::v1::ReportResponse; using ::google::api_manager::proto::ServerConfig; @@ -29,6 +31,7 @@ using ::google::api_manager::utils::Status; using ::google::protobuf::util::error::Code; using ::google::service_control_client::CheckAggregationOptions; +using ::google::service_control_client::QuotaAggregationOptions; using ::google::service_control_client::ReportAggregationOptions; using ::google::service_control_client::ServiceControlClient; using ::google::service_control_client::ServiceControlClientOptions; @@ -40,6 +43,9 @@ namespace service_control { namespace { +const int kQuotaAggregationEntries = 10000; +const int kQuotaAggregationRefreshMs = 1000; + // Default config for check aggregator const int kCheckAggregationEntries = 10000; // Check doesn't support quota yet. It is safe to increase @@ -54,6 +60,8 @@ const int kReportAggregationFlushIntervalMs = 1000; // The default connection timeout for check requests. const int kCheckDefaultTimeoutInMs = 5000; +// The default connection timeout for allocate quota requests. +const int kAllocateQuotaDefaultTimeoutInMs = 1000; // The default connection timeout for report requests. const int kReportDefaultTimeoutInMs = 15000; @@ -69,6 +77,10 @@ const char application_proto[] = "application/x-protobuf"; const char servicecontrol_service[] = "/google.api.servicecontrol.v1.ServiceController"; +// The quota_control service name. used for as audience to generate JWT token. +const char quotacontrol_service[] = + "/google.api.servicecontrol.v1.QuotaController"; + // Generates CheckAggregationOptions. CheckAggregationOptions GetCheckAggregationOptions( const ServerConfig* server_config) { @@ -85,6 +97,24 @@ CheckAggregationOptions GetCheckAggregationOptions( kCheckAggregationExpirationMs); } +// Generate QuotaAggregationOptions +QuotaAggregationOptions GetQuotaAggregationOptions( + const ServerConfig* server_config) { + QuotaAggregationOptions option = QuotaAggregationOptions( + kQuotaAggregationEntries, kQuotaAggregationRefreshMs); + + if (server_config && server_config->has_service_control_config() && + server_config->service_control_config().has_quota_aggregator_config()) { + const auto& quota_config = + server_config->service_control_config().quota_aggregator_config(); + + option.num_entries = quota_config.cache_entries(); + option.refresh_interval_ms = quota_config.refresh_interval_ms(); + } + + return option; +} + // Generates ReportAggregationOptions. ReportAggregationOptions GetReportAggregationOptions( const ServerConfig* server_config) { @@ -143,6 +173,9 @@ Aggregated::Aggregated(const ::google::api::Service& service, sa_token_->SetAudience( auth::ServiceAccountToken::JWT_TOKEN_FOR_SERVICE_CONTROL, url_.service_control() + servicecontrol_service); + sa_token_->SetAudience( + auth::ServiceAccountToken::JWT_TOKEN_FOR_QUOTA_CONTROL, + url_.service_control() + quotacontrol_service); } } @@ -171,6 +204,7 @@ Status Aggregated::Init() { // env->StartPeriodicTimer doens't work at constructor. ServiceControlClientOptions options( GetCheckAggregationOptions(server_config_), + GetQuotaAggregationOptions(server_config_), GetReportAggregationOptions(server_config_)); std::stringstream ss; @@ -186,6 +220,11 @@ Status Aggregated::Init() { options.check_transport = [this]( const CheckRequest& request, CheckResponse* response, TransportDoneFunc on_done) { Call(request, response, on_done, nullptr); }; + + options.quota_transport = [this]( + const AllocateQuotaRequest& request, AllocateQuotaResponse* response, + TransportDoneFunc on_done) { Call(request, response, on_done, nullptr); }; + options.report_transport = [this]( const ReportRequest& request, ReportResponse* response, TransportDoneFunc on_done) { Call(request, response, on_done, nullptr); }; @@ -323,6 +362,57 @@ void Aggregated::Check( check_pool_.Free(std::move(request)); } +void Aggregated::Quota(const QuotaRequestInfo& info, + cloud_trace::CloudTraceSpan* parent_span, + std::function on_done) { + std::shared_ptr trace_span( + CreateChildSpan(parent_span, "QuotaServiceControlCache")); + + if (!client_) { + on_done(Status(Code::INTERNAL, "Missing service control client")); + return; + } + + auto request = quota_pool_.Alloc(); + + Status status = + service_control_proto_.FillAllocateQuotaRequest(info, request.get()); + if (!status.ok()) { + on_done(status); + quota_pool_.Free(std::move(request)); + return; + } + + AllocateQuotaResponse* response = new AllocateQuotaResponse(); + + auto quota_on_done = [this, response, on_done, trace_span]( + const ::google::protobuf::util::Status& status) { + TRACE(trace_span) << "AllocateQuotaRequst returned with status: " + << status.ToString(); + + if (status.ok()) { + on_done(Proto::ConvertAllocateQuotaResponse( + *response, service_control_proto_.service_name())); + } else { + on_done(Status(status.error_code(), status.error_message(), + Status::SERVICE_CONTROL)); + } + + delete response; + }; + + client_->Quota(*request, response, quota_on_done, + [trace_span, this](const AllocateQuotaRequest& request, + AllocateQuotaResponse* response, + TransportDoneFunc on_done) { + Call(request, response, on_done, trace_span.get()); + }); + + // There is no reference to request anymore at this point and it is safe to + // free request now. + quota_pool_.Free(std::move(request)); +} + Status Aggregated::GetStatistics(Statistics* esp_stat) const { if (!client_) { return Status(Code::INTERNAL, "Missing service control client"); @@ -347,13 +437,79 @@ Status Aggregated::GetStatistics(Statistics* esp_stat) const { return Status::OK; } +template +const std::string& Aggregated::GetApiReqeustUrl() { + if (typeid(RequestType) == typeid(CheckRequest)) { + return url_.check_url(); + } else if (typeid(RequestType) == typeid(AllocateQuotaRequest)) { + return url_.quota_url(); + } else { + return url_.report_url(); + } +} + +template +int Aggregated::GetHttpRequestTimeout() { + int timeout_ms = 0; + + // Set timeout on the request if it was so configured. + if (typeid(RequestType) == typeid(CheckRequest)) { + timeout_ms = kCheckDefaultTimeoutInMs; + } else if (typeid(RequestType) == typeid(AllocateQuotaRequest)) { + timeout_ms = kAllocateQuotaDefaultTimeoutInMs; + } else { + timeout_ms = kReportDefaultTimeoutInMs; + } + + if (server_config_ != nullptr && + server_config_->has_service_control_config()) { + const auto& config = server_config_->service_control_config(); + if (typeid(RequestType) == typeid(CheckRequest)) { + if (config.check_timeout_ms() > 0) { + timeout_ms = config.check_timeout_ms(); + } + } else if (typeid(RequestType) == typeid(AllocateQuotaRequest)) { + if (config.quota_timeout_ms() > 0) { + timeout_ms = config.quota_timeout_ms(); + } + } else { + if (config.report_timeout_ms() > 0) { + timeout_ms = config.report_timeout_ms(); + } + } + } + + return timeout_ms; +} + +template +const std::string& Aggregated::GetAuthToken() { + if (sa_token_) { + if (typeid(RequestType) == typeid(AllocateQuotaRequest)) { + return sa_token_->GetAuthToken( + auth::ServiceAccountToken::JWT_TOKEN_FOR_QUOTA_CONTROL); + } else { + return sa_token_->GetAuthToken( + auth::ServiceAccountToken::JWT_TOKEN_FOR_SERVICE_CONTROL); + } + } else { + static std::string empty; + return empty; + } +} + template void Aggregated::Call(const RequestType& request, ResponseType* response, TransportDoneFunc on_done, cloud_trace::CloudTraceSpan* parent_span) { std::shared_ptr trace_span( CreateChildSpan(parent_span, "Call ServiceControl server")); - std::unique_ptr http_request(new HTTPRequest([response, on_done, + + const std::string& url = GetApiReqeustUrl(); + TRACE(trace_span) << "Http request URL: " << url; + + std::unique_ptr http_request(new HTTPRequest([url, response, + on_done, trace_span, this]( Status status, std::map&&, std::string&& body) { TRACE(trace_span) << "HTTP response status: " << status.ToString(); @@ -364,9 +520,6 @@ void Aggregated::Call(const RequestType& request, ResponseType* response, Status(Code::INVALID_ARGUMENT, std::string("Invalid response")); } } else { - const std::string& url = typeid(RequestType) == typeid(CheckRequest) - ? url_.check_url() - : url_.report_url(); env_->LogError(std::string("Failed to call ") + url + ", Error: " + status.ToString() + ", Response body: " + body); @@ -384,56 +537,25 @@ void Aggregated::Call(const RequestType& request, ResponseType* response, on_done(status.ToProto()); })); - bool is_check = (typeid(RequestType) == typeid(CheckRequest)); - const std::string& url = is_check ? url_.check_url() : url_.report_url(); - TRACE(trace_span) << "Http request URL: " << url; - std::string request_body; request.SerializeToString(&request_body); - if (!is_check && (request_body.size() > max_report_size_)) { + if ((typeid(RequestType) == typeid(ReportRequest)) && + (request_body.size() > max_report_size_)) { max_report_size_ = request_body.size(); } http_request->set_url(url) .set_method("POST") - .set_auth_token(GetAuthToken()) + .set_auth_token(GetAuthToken()) .set_header("Content-Type", application_proto) .set_body(request_body); - // Set timeout on the request if it was so configured. - if (is_check) { - http_request->set_timeout_ms(kCheckDefaultTimeoutInMs); - } else { - http_request->set_timeout_ms(kReportDefaultTimeoutInMs); - } - if (server_config_ != nullptr && - server_config_->has_service_control_config()) { - const auto& config = server_config_->service_control_config(); - if (is_check) { - if (config.check_timeout_ms() > 0) { - http_request->set_timeout_ms(config.check_timeout_ms()); - } - } else { - if (config.report_timeout_ms() > 0) { - http_request->set_timeout_ms(config.report_timeout_ms()); - } - } - } + http_request->set_timeout_ms(GetHttpRequestTimeout()); env_->RunHTTPRequest(std::move(http_request)); } -const std::string& Aggregated::GetAuthToken() { - if (sa_token_) { - return sa_token_->GetAuthToken( - auth::ServiceAccountToken::JWT_TOKEN_FOR_SERVICE_CONTROL); - } else { - static std::string empty; - return empty; - } -} - Interface* Aggregated::Create(const ::google::api::Service& service, const ServerConfig* server_config, ApiManagerEnvInterface* env, diff --git a/contrib/endpoints/src/api_manager/service_control/aggregated.h b/contrib/endpoints/src/api_manager/service_control/aggregated.h index 27e42833dbe..759cdbb41fd 100644 --- a/contrib/endpoints/src/api_manager/service_control/aggregated.h +++ b/contrib/endpoints/src/api_manager/service_control/aggregated.h @@ -23,6 +23,7 @@ #include "contrib/endpoints/src/api_manager/service_control/proto.h" #include "contrib/endpoints/src/api_manager/service_control/url.h" #include "google/api/service.pb.h" +#include "google/api/servicecontrol/v1/quota_controller.pb.h" #include "google/api/servicecontrol/v1/service_controller.pb.h" #include "include/service_control_client.h" @@ -49,6 +50,10 @@ class Aggregated : public Interface { const CheckRequestInfo& info, cloud_trace::CloudTraceSpan* parent_span, std::function on_done); + virtual void Quota(const QuotaRequestInfo& info, + cloud_trace::CloudTraceSpan* parent_span, + std::function on_done); + virtual utils::Status Init(); virtual utils::Status Close(); @@ -111,7 +116,16 @@ class Aggregated : public Interface { ::google::service_control_client::TransportDoneFunc on_done, cloud_trace::CloudTraceSpan* parent_span); - // Gets the auth token to access service control server. + // Returns API request url based on RequestType + template + const std::string& GetApiReqeustUrl(); + + // Returns API request timeout in ms based on RequestType + template + int GetHttpRequestTimeout(); + + // Returns API request auth token based on RequestType + template const std::string& GetAuthToken(); // the sevice config. @@ -134,6 +148,11 @@ class Aggregated : public Interface { // The service control client instance. std::unique_ptr<::google::service_control_client::ServiceControlClient> client_; + + // The protobuf pool to reuse AllocateQuotaRequest protobuf. + ProtoPool<::google::api::servicecontrol::v1::AllocateQuotaRequest> + quota_pool_; + // The protobuf pool to reuse CheckRequest protobuf. ProtoPool<::google::api::servicecontrol::v1::CheckRequest> check_pool_; // The protobuf pool to reuse ReportRequest protobuf. diff --git a/contrib/endpoints/src/api_manager/service_control/aggregated_test.cc b/contrib/endpoints/src/api_manager/service_control/aggregated_test.cc index 5e9ca38a55d..c51b2b5f6aa 100644 --- a/contrib/endpoints/src/api_manager/service_control/aggregated_test.cc +++ b/contrib/endpoints/src/api_manager/service_control/aggregated_test.cc @@ -19,16 +19,20 @@ #include "contrib/endpoints/src/api_manager/mock_api_manager_environment.h" #include "contrib/endpoints/src/api_manager/service_control/proto.h" #include "gmock/gmock.h" +#include "google/protobuf/text_format.h" #include "gtest/gtest.h" using ::google::api::servicecontrol::v1::CheckRequest; using ::google::api::servicecontrol::v1::CheckResponse; +using ::google::api::servicecontrol::v1::AllocateQuotaRequest; +using ::google::api::servicecontrol::v1::AllocateQuotaResponse; using ::google::api::servicecontrol::v1::ReportRequest; using ::google::api::servicecontrol::v1::ReportResponse; using ::google::api_manager::utils::Status; using ::google::protobuf::util::error::Code; using ::google::service_control_client::ServiceControlClient; using ::google::service_control_client::TransportCheckFunc; +using ::google::service_control_client::TransportQuotaFunc; using ::google::service_control_client::TransportReportFunc; using ::testing::Return; using ::testing::Invoke; @@ -39,6 +43,39 @@ namespace api_manager { namespace service_control { namespace { + +const char kAllocateQuotaResponse[] = R"( +operation_id: "test_service" +quota_metrics { + metric_name: "serviceruntime.googleapis.com/api/consumer/quota_used_count" + metric_values { + labels { + key: "/quota_name" + value: "metric_first" + } + int64_value: 2 + } + metric_values { + labels { + key: "/quota_name" + value: "metric" + } + int64_value: 1 + } +}service_config_id: "2017-02-08r9" + +)"; + +const char kAllocateQuotaResponseErrorExhausted[] = R"( +operation_id: "test_service" +allocate_errors { + code: RESOURCE_EXHAUSTED + description: "Insufficient tokens for quota group and limit \'apiWriteQpsPerProject_LOW\' of service \'jaebonginternal.sandbox.google.com\', using the limit by ID \'container:1002409420961\'." +} +service_config_id: "2017-02-08r9" + +)"; + void FillOperationInfo(OperationInfo* op) { op->operation_id = "operation_id"; op->operation_name = "operation_name"; @@ -53,6 +90,15 @@ class MockServiceControClient : public ServiceControlClient { CheckResponse*)); MOCK_METHOD4(Check, void(const CheckRequest&, CheckResponse*, DoneCallback, TransportCheckFunc)); + + MOCK_METHOD2(Quota, + ::google::protobuf::util::Status(const AllocateQuotaRequest&, + AllocateQuotaResponse*)); + MOCK_METHOD3(Quota, void(const AllocateQuotaRequest&, AllocateQuotaResponse*, + DoneCallback)); + MOCK_METHOD4(Quota, void(const AllocateQuotaRequest&, AllocateQuotaResponse*, + DoneCallback, TransportQuotaFunc)); + MOCK_METHOD3(Report, void(const ReportRequest&, ReportResponse*, DoneCallback)); MOCK_METHOD2(Report, ::google::protobuf::util::Status(const ReportRequest&, @@ -195,6 +241,91 @@ TEST_F(AggregatedTestWithRealClient, CheckOKTest) { EXPECT_EQ(stat.send_report_operations, 0); } +class QuotaAllocationTestWithRealClient : public ::testing::Test { + public: + void SetUp() { + service_.set_name("test_service"); + service_.mutable_control()->set_environment( + "servicecontrol.googleapis.com"); + env_.reset(new ::testing::NiceMock); + sc_lib_.reset(Aggregated::Create(service_, nullptr, env_.get(), nullptr)); + ASSERT_TRUE((bool)(sc_lib_)); + // This is the call actually creating the client. + sc_lib_->Init(); + + metric_cost_vector_ = {{"metric_first", 1}, {"metric_second", 2}}; + } + + std::string getResponseBody(const char* response) { + AllocateQuotaResponse quota_response; + ::google::protobuf::TextFormat::ParseFromString(response, "a_response); + return quota_response.SerializeAsString(); + } + + void DoRunHTTPRequest(HTTPRequest* request) { + std::map headers; + + AllocateQuotaRequest quota_request; + + ASSERT_TRUE(quota_request.ParseFromString(request->body())); + ASSERT_EQ(quota_request.allocate_operation().quota_metrics_size(), 2); + + std::set> expected_costs = { + {"metric_first", 1}, {"metric_second", 2}}; + std::set> actual_costs; + + for (auto rule : quota_request.allocate_operation().quota_metrics()) { + actual_costs.insert(std::make_pair(rule.metric_name(), + rule.metric_values(0).int64_value())); + } + + ASSERT_EQ(actual_costs, expected_costs); + + request->OnComplete(Status::OK, std::move(headers), + std::move(getResponseBody(kAllocateQuotaResponse))); + } + + void DoRunHTTPRequestAllocationFailed(HTTPRequest* request) { + std::map headers; + + request->OnComplete( + Status::OK, std::move(headers), + std::move(getResponseBody(kAllocateQuotaResponseErrorExhausted))); + } + + ::google::api::Service service_; + std::unique_ptr env_; + std::unique_ptr sc_lib_; + std::vector> metric_cost_vector_; +}; + +TEST_F(QuotaAllocationTestWithRealClient, AllocateQuotaTest) { + EXPECT_CALL(*env_, DoRunHTTPRequest(_)) + .WillOnce( + Invoke(this, &QuotaAllocationTestWithRealClient::DoRunHTTPRequest)); + + QuotaRequestInfo info; + info.metric_cost_vector = &metric_cost_vector_; + + FillOperationInfo(&info); + sc_lib_->Quota(info, nullptr, + [](Status status) { ASSERT_TRUE(status.ok()); }); +} + +TEST_F(QuotaAllocationTestWithRealClient, AllocateQuotaFailedTest) { + EXPECT_CALL(*env_, DoRunHTTPRequest(_)) + .WillOnce(Invoke(this, &QuotaAllocationTestWithRealClient:: + DoRunHTTPRequestAllocationFailed)); + + QuotaRequestInfo info; + info.metric_cost_vector = &metric_cost_vector_; + + FillOperationInfo(&info); + sc_lib_->Quota(info, nullptr, [](Status status) { + ASSERT_TRUE(status.code() == Code::RESOURCE_EXHAUSTED); + }); +} + TEST(AggregatedServiceControlTest, Create) { // Verify that invalid service config yields nullptr. ::google::api::Service diff --git a/contrib/endpoints/src/api_manager/service_control/allocate_quota_response_test.cc b/contrib/endpoints/src/api_manager/service_control/allocate_quota_response_test.cc new file mode 100644 index 00000000000..729a7302527 --- /dev/null +++ b/contrib/endpoints/src/api_manager/service_control/allocate_quota_response_test.cc @@ -0,0 +1,184 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//////////////////////////////////////////////////////////////////////////////// +// +#include "contrib/endpoints/include/api_manager/utils/status.h" +#include "contrib/endpoints/src/api_manager/service_control/proto.h" +#include "gtest/gtest.h" + +namespace gasv1 = ::google::api::servicecontrol::v1; + +using ::google::api::servicecontrol::v1::QuotaError; +using ::google::api_manager::utils::Status; +using ::google::protobuf::util::error::Code; + +namespace google { +namespace api_manager { +namespace service_control { + +namespace { + +Status ConvertAllocateQuotaErrorToStatus(gasv1::QuotaError::Code code, + const char* error_detail, + const char* service_name) { + gasv1::AllocateQuotaResponse response; + gasv1::QuotaError* quota_error = response.add_allocate_errors(); + QuotaRequestInfo info; + quota_error->set_code(code); + quota_error->set_description(error_detail); + return Proto::ConvertAllocateQuotaResponse(response, service_name); +} + +Status ConvertAllocateQuotaErrorToStatus(gasv1::QuotaError::Code code) { + gasv1::AllocateQuotaResponse response; + std::string service_name; + response.add_allocate_errors()->set_code(code); + return Proto::ConvertAllocateQuotaResponse(response, service_name); +} + +} // namespace + +TEST(AllocateQuotaResponseTest, + AbortedWithInvalidArgumentWhenRespIsKeyInvalid) { + Status result = + ConvertAllocateQuotaErrorToStatus(QuotaError::API_KEY_INVALID); + EXPECT_EQ(Code::INVALID_ARGUMENT, result.code()); +} + +TEST(AllocateQuotaResponseTest, + AbortedWithInvalidArgumentWhenRespIsKeyExpired) { + Status result = + ConvertAllocateQuotaErrorToStatus(QuotaError::API_KEY_EXPIRED); + EXPECT_EQ(Code::INVALID_ARGUMENT, result.code()); +} + +TEST(AllocateQuotaResponseTest, + AbortedWithInvalidArgumentWhenRespIsBlockedWithResourceExausted) { + Status result = + ConvertAllocateQuotaErrorToStatus(QuotaError::RESOURCE_EXHAUSTED); + EXPECT_EQ(Code::RESOURCE_EXHAUSTED, result.code()); +} + +TEST(AllocateQuotaResponseTest, + AbortedWithInvalidArgumentWhenRespIsBlockedWithProjectSuspended) { + Status result = + ConvertAllocateQuotaErrorToStatus(QuotaError::PROJECT_SUSPENDED); + EXPECT_EQ(Code::PERMISSION_DENIED, result.code()); +} + +TEST(AllocateQuotaResponseTest, + AbortedWithPermissionDeniedWhenRespIsBlockedWithServiceNotEnabled) { + Status result = ConvertAllocateQuotaErrorToStatus( + QuotaError::SERVICE_NOT_ENABLED, + "API api_xxxx is not enabled for the project.", "api_xxxx"); + EXPECT_EQ(Code::PERMISSION_DENIED, result.code()); + EXPECT_EQ(result.message(), "API api_xxxx is not enabled for the project."); +} + +TEST(AllocateQuotaResponseTest, + AbortedWithPermissionDeniedWhenRespIsBlockedWithBillingNotActivated) { + Status result = ConvertAllocateQuotaErrorToStatus( + QuotaError::BILLING_NOT_ACTIVE, + "API api_xxxx has billing disabled. Please enable it..", "api_xxxx"); + EXPECT_EQ(Code::PERMISSION_DENIED, result.code()); + EXPECT_EQ(result.message(), + "API api_xxxx has billing disabled. Please enable it."); +} + +TEST(AllocateQuotaResponseTest, + AbortedWithPermissionDeniedWhenRespIsBlockedWithIpAddressBlocked) { + Status result = + ConvertAllocateQuotaErrorToStatus(QuotaError::IP_ADDRESS_BLOCKED); + EXPECT_EQ(Code::PERMISSION_DENIED, result.code()); +} + +TEST(AllocateQuotaResponseTest, + AbortedWithPermissionDeniedWhenRespIsBlockedWithRefererBlocked) { + Status result = + ConvertAllocateQuotaErrorToStatus(QuotaError::REFERER_BLOCKED); + EXPECT_EQ(Code::PERMISSION_DENIED, result.code()); +} + +TEST(AllocateQuotaResponseTest, + AbortedWithPermissionDeniedWhenRespIsBlockedWithClientAppBlocked) { + Status result = + ConvertAllocateQuotaErrorToStatus(QuotaError::CLIENT_APP_BLOCKED); + EXPECT_EQ(Code::PERMISSION_DENIED, result.code()); +} + +TEST(AllocateQuotaResponseTest, + AbortedWithPermissionDeniedWhenResponseIsBlockedWithProjectInvalid) { + Status result = + ConvertAllocateQuotaErrorToStatus(QuotaError::PROJECT_INVALID); + EXPECT_EQ(Code::INVALID_ARGUMENT, result.code()); +} + +TEST(AllocateQuotaResponseTest, + AbortedWithPermissionDeniedWhenRespIsBlockedWithProjectDeleted) { + Status result = + ConvertAllocateQuotaErrorToStatus(QuotaError::PROJECT_DELETED); + EXPECT_EQ(Code::INVALID_ARGUMENT, result.code()); +} + +TEST(AllocateQuotaResponseTest, + AbortedWithPermissionDeniedWhenRespIsBlockedWithApiKeyInvalid) { + Status result = + ConvertAllocateQuotaErrorToStatus(QuotaError::API_KEY_INVALID); + EXPECT_EQ(Code::INVALID_ARGUMENT, result.code()); +} + +TEST(AllocateQuotaResponseTest, + AbortedWithPermissionDeniedWhenRespIsBlockedWithApiKeyExpiread) { + Status result = + ConvertAllocateQuotaErrorToStatus(QuotaError::API_KEY_EXPIRED); + EXPECT_EQ(Code::INVALID_ARGUMENT, result.code()); +} + +TEST(AllocateQuotaResponseTest, + AcceptOKWhenRespIsBlockedWithProjectStatusUnavailable) { + Status result = + ConvertAllocateQuotaErrorToStatus(QuotaError::PROJECT_STATUS_UNVAILABLE); + EXPECT_EQ(Code::OK, result.code()); +} + +TEST(AllocateQuotaResponseTest, + AcceptOKWhenRespIsBlockedWithServiceStatusUnavailable) { + Status result = + ConvertAllocateQuotaErrorToStatus(QuotaError::SERVICE_STATUS_UNAVAILABLE); + EXPECT_EQ(Code::OK, result.code()); +} + +TEST(AllocateQuotaResponseTest, + AcceptOKWhenRespIsBlockedWithBillingStatusUnavailable) { + Status result = + ConvertAllocateQuotaErrorToStatus(QuotaError::BILLING_STATUS_UNAVAILABLE); + EXPECT_EQ(Code::OK, result.code()); +} + +TEST(AllocateQuotaResponseTest, FailOpenWhenResponseIsUnknownBillingStatus) { + EXPECT_TRUE( + ConvertAllocateQuotaErrorToStatus(QuotaError::BILLING_STATUS_UNAVAILABLE) + .ok()); +} + +TEST(AllocateQuotaResponseTest, FailOpenWhenResponseIsUnknownServiceStatus) { + EXPECT_TRUE( + ConvertAllocateQuotaErrorToStatus(QuotaError::SERVICE_STATUS_UNAVAILABLE) + .ok()); +} + +} // namespace service_control +} // namespace api_manager +} // namespace google diff --git a/contrib/endpoints/src/api_manager/service_control/info.h b/contrib/endpoints/src/api_manager/service_control/info.h index 1a62fdfed13..184d598dae7 100644 --- a/contrib/endpoints/src/api_manager/service_control/info.h +++ b/contrib/endpoints/src/api_manager/service_control/info.h @@ -17,7 +17,8 @@ #include "google/protobuf/stubs/stringpiece.h" -#include +#include "google/api/quota.pb.h" + #include #include #include @@ -94,6 +95,12 @@ struct CheckResponseInfo { CheckResponseInfo() : is_api_key_valid(true), service_is_activated(true) {} }; +struct QuotaRequestInfo : public OperationInfo { + std::string method_name; + + const std::vector>* metric_cost_vector; +}; + // Information to fill Report request protobuf. struct ReportRequestInfo : public OperationInfo { // The HTTP response code. diff --git a/contrib/endpoints/src/api_manager/service_control/interface.h b/contrib/endpoints/src/api_manager/service_control/interface.h index 708acc56a88..a6188e0f73b 100644 --- a/contrib/endpoints/src/api_manager/service_control/interface.h +++ b/contrib/endpoints/src/api_manager/service_control/interface.h @@ -70,6 +70,17 @@ class Interface { const CheckRequestInfo& info, cloud_trace::CloudTraceSpan* parent_span, std::function on_done) = 0; + // on_done() function will be called once it is completed. + // utils::Status in the on_done callback: + // If status.code is more than 100, it is the HTTP response status + // from the service control server. + // If status code is less than 20, within the ranges defined by + // google/protobuf/stubs/status.h, is from parsing error response + // body. + virtual void Quota(const QuotaRequestInfo& info, + cloud_trace::CloudTraceSpan* parent_span, + std::function on_done) = 0; + // Get statistics of ServiceControl library. virtual utils::Status GetStatistics(Statistics* stat) const = 0; }; diff --git a/contrib/endpoints/src/api_manager/service_control/proto.cc b/contrib/endpoints/src/api_manager/service_control/proto.cc index cfaa9a5bf53..aa88fb39eb2 100644 --- a/contrib/endpoints/src/api_manager/service_control/proto.cc +++ b/contrib/endpoints/src/api_manager/service_control/proto.cc @@ -30,6 +30,7 @@ #include "utils/distribution_helper.h" using ::google::api::servicecontrol::v1::CheckError; +using ::google::api::servicecontrol::v1::QuotaError; using ::google::api::servicecontrol::v1::CheckRequest; using ::google::api::servicecontrol::v1::CheckResponse; using ::google::api::servicecontrol::v1::Distribution; @@ -49,6 +50,11 @@ namespace google { namespace api_manager { namespace service_control { +const char kConsumerQuotaUsedCount[] = + "serviceruntime.googleapis.com/api/consumer/quota_used_count"; + +const char kQuotaName[] = "/quota_name"; + struct SupportedMetric { const char* name; ::google::api::MetricDescriptor_MetricKind metric_kind; @@ -911,6 +917,62 @@ Proto::Proto(const std::set& logs, service_name_(service_name), service_config_id_(service_config_id) {} +utils::Status Proto::FillAllocateQuotaRequest( + const QuotaRequestInfo& info, + ::google::api::servicecontrol::v1::AllocateQuotaRequest* request) { + ::google::api::servicecontrol::v1::QuotaOperation* operation = + request->mutable_allocate_operation(); + + // service_name + request->set_service_name(service_name_); + // service_config_id + request->set_service_config_id(service_config_id_); + + // allocate_operation.operation_id + if (!info.operation_id.empty()) { + operation->set_operation_id(info.operation_id); + } + // allocate_operation.method_name + if (!info.method_name.empty()) { + operation->set_method_name(info.method_name); + } + // allocate_operation.consumer_id + if (!info.api_key.empty()) { + operation->set_consumer_id(std::string(kConsumerIdApiKey) + + std::string(info.api_key)); + } + + // allocate_operation.quota_mode + operation->set_quota_mode( + ::google::api::servicecontrol::v1::QuotaOperation_QuotaMode:: + QuotaOperation_QuotaMode_NORMAL); + + // allocate_operation.labels + auto* labels = operation->mutable_labels(); + if (!info.client_ip.empty()) { + (*labels)[kServiceControlCallerIp] = info.client_ip; + } + + if (!info.referer.empty()) { + (*labels)[kServiceControlReferer] = info.referer; + } + (*labels)[kServiceControlUserAgent] = kUserAgent; + (*labels)[kServiceControlServiceAgent] = + kServiceAgentPrefix + utils::Version::instance().get(); + + if (info.metric_cost_vector) { + for (auto metric : *info.metric_cost_vector) { + MetricValueSet* value_set = operation->add_quota_metrics(); + value_set->set_metric_name(metric.first); + MetricValue* value = value_set->add_metric_values(); + const auto& cost = metric.second; + value->set_int64_value(cost <= 0 ? 1 : cost); + } + } + + return Status::OK; +} + Status Proto::FillCheckRequest(const CheckRequestInfo& info, CheckRequest* request) { Status status = VerifyRequiredCheckFields(info); @@ -1010,6 +1072,100 @@ Status Proto::FillReportRequest(const ReportRequestInfo& info, return Status::OK; } +Status Proto::ConvertAllocateQuotaResponse( + const ::google::api::servicecontrol::v1::AllocateQuotaResponse& response, + const std::string& service_name) { + // response.operation_id() + if (response.allocate_errors().size() == 0) { + return Status::OK; + } + + const ::google::api::servicecontrol::v1::QuotaError& error = + response.allocate_errors().Get(0); + + switch (error.code()) { + case ::google::api::servicecontrol::v1::QuotaError::UNSPECIFIED: + // This is never used. + break; + + case ::google::api::servicecontrol::v1::QuotaError::RESOURCE_EXHAUSTED: + // Quota allocation failed. + // Same as [google.rpc.Code.RESOURCE_EXHAUSTED][]. + return Status(Code::RESOURCE_EXHAUSTED, "Quota allocation failed."); + + case ::google::api::servicecontrol::v1::QuotaError::PROJECT_SUSPENDED: + // Consumer project has been suspended. + return Status(Code::PERMISSION_DENIED, "Project suspended."); + + case ::google::api::servicecontrol::v1::QuotaError::SERVICE_NOT_ENABLED: + // Consumer has not enabled the service. + return Status(Code::PERMISSION_DENIED, + std::string("API ") + service_name + + " is not enabled for the project."); + + case ::google::api::servicecontrol::v1::QuotaError::BILLING_NOT_ACTIVE: + // Consumer cannot access the service because billing is disabled. + return Status(Code::PERMISSION_DENIED, + std::string("API ") + service_name + + " has billing disabled. Please enable it."); + + case ::google::api::servicecontrol::v1::QuotaError::PROJECT_DELETED: + // Consumer's project has been marked as deleted (soft deletion). + case ::google::api::servicecontrol::v1::QuotaError::PROJECT_INVALID: + // Consumer's project number or ID does not represent a valid project. + return Status(Code::INVALID_ARGUMENT, + "Client project not valid. Please pass a valid project."); + + case ::google::api::servicecontrol::v1::QuotaError::IP_ADDRESS_BLOCKED: + // IP address of the consumer is invalid for the specific consumer + // project. + return Status(Code::PERMISSION_DENIED, "IP address blocked."); + + case ::google::api::servicecontrol::v1::QuotaError::REFERER_BLOCKED: + // Referer address of the consumer request is invalid for the specific + // consumer project. + return Status(Code::PERMISSION_DENIED, "Referer blocked."); + + case ::google::api::servicecontrol::v1::QuotaError::CLIENT_APP_BLOCKED: + // Client application of the consumer request is invalid for the + // specific consumer project. + return Status(Code::PERMISSION_DENIED, "Client app blocked."); + + case ::google::api::servicecontrol::v1::QuotaError::API_KEY_INVALID: + // Specified API key is invalid. + return Status(Code::INVALID_ARGUMENT, + "API key not valid. Please pass a valid API key."); + + case ::google::api::servicecontrol::v1::QuotaError::API_KEY_EXPIRED: + // Specified API Key has expired. + return Status(Code::INVALID_ARGUMENT, + "API key expired. Please renew the API key."); + + case ::google::api::servicecontrol::v1::QuotaError:: + PROJECT_STATUS_UNVAILABLE: + // The backend server for looking up project id/number is unavailable. + case ::google::api::servicecontrol::v1::QuotaError:: + SERVICE_STATUS_UNAVAILABLE: + // The backend server for checking service status is unavailable. + case ::google::api::servicecontrol::v1::QuotaError:: + BILLING_STATUS_UNAVAILABLE: + // The backend server for checking billing status is unavailable. + // Fail open for internal server errors per recommendation + case ::google::api::servicecontrol::v1::QuotaError:: + QUOTA_SYSTEM_UNAVAILABLE: + // The backend server for checking quota limits is unavailable. + return Status::OK; + + default: + return Status( + Code::INTERNAL, + std::string("Request blocked due to unsupported error code: ") + + std::to_string(error.code())); + } + + return Status::OK; +} + Status Proto::ConvertCheckResponse(const CheckResponse& check_response, const std::string& service_name, CheckResponseInfo* check_response_info) { diff --git a/contrib/endpoints/src/api_manager/service_control/proto.h b/contrib/endpoints/src/api_manager/service_control/proto.h index bea2a948b63..d4fa09592c9 100644 --- a/contrib/endpoints/src/api_manager/service_control/proto.h +++ b/contrib/endpoints/src/api_manager/service_control/proto.h @@ -19,6 +19,7 @@ #include "contrib/endpoints/src/api_manager/service_control/info.h" #include "google/api/label.pb.h" #include "google/api/metric.pb.h" +#include "google/api/servicecontrol/v1/quota_controller.pb.h" #include "google/api/servicecontrol/v1/service_controller.pb.h" namespace google { @@ -48,6 +49,10 @@ class Proto final { const CheckRequestInfo& info, ::google::api::servicecontrol::v1::CheckRequest* request); + utils::Status FillAllocateQuotaRequest( + const QuotaRequestInfo& info, + ::google::api::servicecontrol::v1::AllocateQuotaRequest* request); + // Fills the CheckRequest protobuf from info. // FillReportRequest function should copy the strings pointed by info. // These buffers may be freed after the FillReportRequest call. @@ -64,6 +69,10 @@ class Proto final { const ::google::api::servicecontrol::v1::CheckResponse& response, const std::string& service_name, CheckResponseInfo* check_response_info); + static utils::Status ConvertAllocateQuotaResponse( + const ::google::api::servicecontrol::v1::AllocateQuotaResponse& response, + const std::string& service_name); + static bool IsMetricSupported(const ::google::api::MetricDescriptor& metric); static bool IsLabelSupported(const ::google::api::LabelDescriptor& label); const std::string& service_name() const { return service_name_; } diff --git a/contrib/endpoints/src/api_manager/service_control/proto_test.cc b/contrib/endpoints/src/api_manager/service_control/proto_test.cc index 0a3f05c3668..4e2a0aeb61b 100644 --- a/contrib/endpoints/src/api_manager/service_control/proto_test.cc +++ b/contrib/endpoints/src/api_manager/service_control/proto_test.cc @@ -76,6 +76,12 @@ void FillCheckRequestInfo(CheckRequestInfo* request) { request->referer = "referer"; } +void FillAllocateQuotaRequestInfo(QuotaRequestInfo* request) { + request->client_ip = "1.2.3.4"; + request->referer = "referer"; + request->method_name = "operation_name"; +} + void FillReportRequestInfo(ReportRequestInfo* request) { request->referer = "referer"; request->response_code = 200; @@ -122,6 +128,12 @@ std::string CheckRequestToString(gasv1::CheckRequest* request) { return text; } +std::string AllocateQuotaRequestToString(gasv1::AllocateQuotaRequest* request) { + std::string text; + google::protobuf::TextFormat::PrintToString(*request, &text); + return text; +} + std::string ReportRequestToString(gasv1::ReportRequest* request) { gasv1::Operation* op = request->mutable_operations(0); SetFixTimeStamps(op); @@ -179,6 +191,44 @@ TEST_F(ProtoTest, FillGoodCheckRequestAndroidIosTest) { ASSERT_EQ(expected_text, text); } +TEST_F(ProtoTest, FillGoodAllocateQuotaRequestTest) { + std::vector> metric_cost_vector = { + {"metric_first", 1}, {"metric_second", 2}}; + + google::api_manager::service_control::QuotaRequestInfo info; + info.metric_cost_vector = &metric_cost_vector; + + FillOperationInfo(&info); + FillAllocateQuotaRequestInfo(&info); + + gasv1::AllocateQuotaRequest request; + ASSERT_TRUE(scp_.FillAllocateQuotaRequest(info, &request).ok()); + + std::string text = AllocateQuotaRequestToString(&request); + std::string expected_text = ReadTestBaseline("allocate_quota_request.golden"); + ASSERT_EQ(expected_text, text); +} + +TEST_F(ProtoTest, FillAllocateQuotaRequestNoMethodNameTest) { + std::vector> metric_cost_vector = { + {"metric_first", 1}, {"metric_second", 2}}; + + google::api_manager::service_control::QuotaRequestInfo info; + FillOperationInfo(&info); + info.metric_cost_vector = &metric_cost_vector; + info.client_ip = "1.2.3.4"; + info.referer = "referer"; + info.method_name = ""; + + gasv1::AllocateQuotaRequest request; + ASSERT_TRUE(scp_.FillAllocateQuotaRequest(info, &request).ok()); + + std::string text = AllocateQuotaRequestToString(&request); + std::string expected_text = + ReadTestBaseline("allocate_quota_request_no_method_name.golden"); + ASSERT_EQ(expected_text, text); +} + TEST_F(ProtoTest, FillNoApiKeyCheckRequestTest) { CheckRequestInfo info; info.operation_id = "operation_id"; diff --git a/contrib/endpoints/src/api_manager/service_control/testdata/allocate_quota_request.golden b/contrib/endpoints/src/api_manager/service_control/testdata/allocate_quota_request.golden new file mode 100644 index 00000000000..73a26239238 --- /dev/null +++ b/contrib/endpoints/src/api_manager/service_control/testdata/allocate_quota_request.golden @@ -0,0 +1,36 @@ +service_name: "test_service" +allocate_operation { + operation_id: "operation_id" + method_name: "operation_name" + consumer_id: "api_key:api_key_x" + labels { + key: "servicecontrol.googleapis.com/caller_ip" + value: "1.2.3.4" + } + labels { + key: "servicecontrol.googleapis.com/referer" + value: "referer" + } + labels { + key: "servicecontrol.googleapis.com/service_agent" + value: "ESP/{{service_agent_version}}" + } + labels { + key: "servicecontrol.googleapis.com/user_agent" + value: "ESP" + } + quota_metrics { + metric_name: "metric_first" + metric_values { + int64_value: 1 + } + } + quota_metrics { + metric_name: "metric_second" + metric_values { + int64_value: 2 + } + } + quota_mode: NORMAL +} +service_config_id: "2016-09-19r0" diff --git a/contrib/endpoints/src/api_manager/service_control/testdata/allocate_quota_request_android_ios.golden b/contrib/endpoints/src/api_manager/service_control/testdata/allocate_quota_request_android_ios.golden new file mode 100644 index 00000000000..476bae4e5f5 --- /dev/null +++ b/contrib/endpoints/src/api_manager/service_control/testdata/allocate_quota_request_android_ios.golden @@ -0,0 +1,48 @@ +service_name: "test_service" +allocate_operation { + operation_id: "operation_id" + method_name: "operation_name" + consumer_id: "api_key:api_key_x" + labels { + key: "servicecontrol.googleapis.com/android_cert_fingerprint" + value: "AIzaSyB4Gz8nyaSaWo63IPUcy5d_L8dpKtOTSD0" + } + labels { + key: "servicecontrol.googleapis.com/android_package_name" + value: "com.google.cloud" + } + labels { + key: "servicecontrol.googleapis.com/caller_ip" + value: "1.2.3.4" + } + labels { + key: "servicecontrol.googleapis.com/ios_bundle_id" + value: "5b40ad6af9a806305a0a56d7cb91b82a27c26909" + } + labels { + key: "servicecontrol.googleapis.com/referer" + value: "referer" + } + labels { + key: "servicecontrol.googleapis.com/service_agent" + value: "ESP/{{service_agent_version}}" + } + labels { + key: "servicecontrol.googleapis.com/user_agent" + value: "ESP" + } + quota_metrics { + metric_name: "metric_first" + metric_values { + int64_value: 1 + } + } + quota_metrics { + metric_name: "metric_second" + metric_values { + int64_value: 2 + } + } + quota_mode: NORMAL +} +service_config_id: "2016-09-19r0" diff --git a/contrib/endpoints/src/api_manager/service_control/testdata/allocate_quota_request_no_method_name.golden b/contrib/endpoints/src/api_manager/service_control/testdata/allocate_quota_request_no_method_name.golden new file mode 100644 index 00000000000..34a59f47652 --- /dev/null +++ b/contrib/endpoints/src/api_manager/service_control/testdata/allocate_quota_request_no_method_name.golden @@ -0,0 +1,35 @@ +service_name: "test_service" +allocate_operation { + operation_id: "operation_id" + consumer_id: "api_key:api_key_x" + labels { + key: "servicecontrol.googleapis.com/caller_ip" + value: "1.2.3.4" + } + labels { + key: "servicecontrol.googleapis.com/referer" + value: "referer" + } + labels { + key: "servicecontrol.googleapis.com/service_agent" + value: "ESP/{{service_agent_version}}" + } + labels { + key: "servicecontrol.googleapis.com/user_agent" + value: "ESP" + } + quota_metrics { + metric_name: "metric_first" + metric_values { + int64_value: 1 + } + } + quota_metrics { + metric_name: "metric_second" + metric_values { + int64_value: 2 + } + } + quota_mode: NORMAL +} +service_config_id: "2016-09-19r0" diff --git a/contrib/endpoints/src/api_manager/service_control/url.cc b/contrib/endpoints/src/api_manager/service_control/url.cc index d113d402d1f..f106ccd40fa 100644 --- a/contrib/endpoints/src/api_manager/service_control/url.cc +++ b/contrib/endpoints/src/api_manager/service_control/url.cc @@ -27,6 +27,7 @@ namespace { // /v1/services/{service}:report const char v1_services_path[] = "/v1/services/"; const char check_verb[] = ":check"; +const char quota_verb[] = ":allocateQuota"; const char report_verb[] = ":report"; const char http[] = "http://"; const char https[] = "https://"; @@ -66,6 +67,7 @@ Url::Url(const ::google::api::Service* service, std::string path = service_control_ + v1_services_path + service->name(); check_url_ = path + check_verb; report_url_ = path + report_verb; + quota_url_ = path + quota_verb; } } diff --git a/contrib/endpoints/src/api_manager/service_control/url.h b/contrib/endpoints/src/api_manager/service_control/url.h index 4615864bbdd..fb17aa5e006 100644 --- a/contrib/endpoints/src/api_manager/service_control/url.h +++ b/contrib/endpoints/src/api_manager/service_control/url.h @@ -31,12 +31,14 @@ class Url { // Pre-computed url for service control. const std::string& service_control() const { return service_control_; } const std::string& check_url() const { return check_url_; } + const std::string& quota_url() const { return quota_url_; } const std::string& report_url() const { return report_url_; } private: // Pre-computed url for service control methods. std::string service_control_; std::string check_url_; + std::string quota_url_; std::string report_url_; }; diff --git a/contrib/endpoints/src/api_manager/service_control/url_test.cc b/contrib/endpoints/src/api_manager/service_control/url_test.cc index afbf48bd154..8e0bb2d188a 100644 --- a/contrib/endpoints/src/api_manager/service_control/url_test.cc +++ b/contrib/endpoints/src/api_manager/service_control/url_test.cc @@ -57,6 +57,10 @@ TEST(UrlTest, PrependHttps) { ASSERT_EQ( "https://servicecontrol.googleapis.com/v1/services/https-config:report", url.report_url()); + ASSERT_EQ( + "https://servicecontrol.googleapis.com/v1/services/" + "https-config:allocateQuota", + url.quota_url()); } TEST(UrlTest, ServerControlOverride) { diff --git a/contrib/endpoints/src/grpc/transcoding/transcoder_test.cc b/contrib/endpoints/src/grpc/transcoding/transcoder_test.cc index 86fcc1911c1..a2d09b9a642 100644 --- a/contrib/endpoints/src/grpc/transcoding/transcoder_test.cc +++ b/contrib/endpoints/src/grpc/transcoding/transcoder_test.cc @@ -90,6 +90,10 @@ class TestMethodInfo : public MethodInfo { return dummy; }; + const std::vector> &metric_cost_vector() const { + return metric_cost_vector_; + } + // Methods that the Transcoder does use const std::string &request_type_url() const { return request_type_url_; } bool request_streaming() const { return request_streaming_; } @@ -104,6 +108,7 @@ class TestMethodInfo : public MethodInfo { bool response_streaming_; std::string body_field_path_; std::string empty_; + std::vector> metric_cost_vector_; }; class TranscoderTest : public ::testing::Test { From 1bcfab45692024a612488574650c37e247712816 Mon Sep 17 00:00:00 2001 From: Qian Sun Date: Fri, 7 Apr 2017 10:08:48 -0700 Subject: [PATCH 78/83] Send delta metrics for intermediate reports. (#219) * Send delta metrics for intermediate reports. * Move last_request_bytes/last_response_bytes to RequestContext. * Handle final report. * Address comment. --- .../api_manager/context/request_context.cc | 22 ++++++++++++++----- .../src/api_manager/context/request_context.h | 4 ++++ 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/contrib/endpoints/src/api_manager/context/request_context.cc b/contrib/endpoints/src/api_manager/context/request_context.cc index 2c0a2985927..4703e2e5ee7 100644 --- a/contrib/endpoints/src/api_manager/context/request_context.cc +++ b/contrib/endpoints/src/api_manager/context/request_context.cc @@ -81,7 +81,9 @@ RequestContext::RequestContext(std::shared_ptr service_context, std::unique_ptr request) : service_context_(service_context), request_(std::move(request)), - is_first_report_(true) { + is_first_report_(true), + last_request_bytes_(0), + last_response_bytes_(0) { start_time_ = std::chrono::system_clock::now(); last_report_time_ = std::chrono::steady_clock::now(); operation_id_ = GenerateUUID(); @@ -267,13 +269,23 @@ void RequestContext::FillReportRequestInfo( info->auth_audience = auth_audience_; if (!info->is_final_report) { - info->request_bytes = request_->GetGrpcRequestBytes(); - info->response_bytes = request_->GetGrpcResponseBytes(); + // Make sure we send delta metrics for intermediate reports. + info->request_bytes = request_->GetGrpcRequestBytes() - last_request_bytes_; + info->response_bytes = + request_->GetGrpcResponseBytes() - last_response_bytes_; + last_request_bytes_ += info->request_bytes; + last_response_bytes_ += info->response_bytes; } else { info->request_size = response->GetRequestSize(); info->response_size = response->GetResponseSize(); - info->request_bytes = info->request_size; - info->response_bytes = info->response_size; + info->request_bytes = info->request_size - last_request_bytes_; + if (info->request_bytes < 0) { + info->request_bytes = 0; + } + info->response_bytes = info->response_size - last_response_bytes_; + if (info->response_bytes < 0) { + info->response_bytes = 0; + } info->streaming_request_message_counts = request_->GetGrpcRequestMessageCounts(); diff --git a/contrib/endpoints/src/api_manager/context/request_context.h b/contrib/endpoints/src/api_manager/context/request_context.h index 8832b38fc3e..aef0eecf24d 100644 --- a/contrib/endpoints/src/api_manager/context/request_context.h +++ b/contrib/endpoints/src/api_manager/context/request_context.h @@ -180,6 +180,10 @@ class RequestContext { // The time point of last intermediate report std::chrono::steady_clock::time_point last_report_time_; + + // The accumulated data sent till last intermediate report + int64_t last_request_bytes_; + int64_t last_response_bytes_; }; } // namespace context From e47cfc3038e8a8d63ed4eb3d95662d89e297a551 Mon Sep 17 00:00:00 2001 From: Zack Date: Fri, 7 Apr 2017 15:46:07 -0700 Subject: [PATCH 79/83] Update attributes to match the canonical attribute list. (#232) * Update response.http.code to response.code and response.latency to response.duration to line up with the canonical attributes in istio/istio.github.io/docs/concepts/attributes.md * Format according to clang-format --- src/envoy/mixer/http_control.cc | 11 +++++------ src/envoy/mixer/integration_test/check_report_test.go | 8 ++++---- .../mixer/integration_test/failed_request_test.go | 8 ++++---- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/src/envoy/mixer/http_control.cc b/src/envoy/mixer/http_control.cc index 8283482dff3..61aa3285a62 100644 --- a/src/envoy/mixer/http_control.cc +++ b/src/envoy/mixer/http_control.cc @@ -44,8 +44,8 @@ const std::string kRequestSize = "request.size"; const std::string kRequestTime = "request.time"; const std::string kResponseHeaders = "response.headers"; -const std::string kResponseHttpCode = "response.http.code"; -const std::string kResponseLatency = "response.latency"; +const std::string kResponseCode = "response.code"; +const std::string kResponseDuration = "response.duration"; const std::string kResponseSize = "response.size"; const std::string kResponseTime = "response.time"; @@ -118,15 +118,14 @@ void FillRequestInfoAttributes(const AccessLog::RequestInfo& info, attr->attributes[kResponseSize] = Attributes::Int64Value(info.bytesSent()); } - attr->attributes[kResponseLatency] = Attributes::DurationValue( + attr->attributes[kResponseDuration] = Attributes::DurationValue( std::chrono::duration_cast(info.duration())); if (info.responseCode().valid()) { - attr->attributes[kResponseHttpCode] = + attr->attributes[kResponseCode] = Attributes::Int64Value(info.responseCode().value()); } else { - attr->attributes[kResponseHttpCode] = - Attributes::Int64Value(check_status_code); + attr->attributes[kResponseCode] = Attributes::Int64Value(check_status_code); } } diff --git a/src/envoy/mixer/integration_test/check_report_test.go b/src/envoy/mixer/integration_test/check_report_test.go index 71d8a3f6bb9..9b9708c19b2 100644 --- a/src/envoy/mixer/integration_test/check_report_test.go +++ b/src/envoy/mixer/integration_test/check_report_test.go @@ -61,8 +61,8 @@ const reportAttributesOkGet = ` "request.size": 0, "response.time": "*", "response.size": 0, - "response.latency": "*", - "response.http.code": 200, + "response.duration": "*", + "response.code": 200, "response.headers": { "date": "*", "content-type": "text/plain; charset=utf-8", @@ -115,8 +115,8 @@ const reportAttributesOkPost = ` "request.size": 12, "response.time": "*", "response.size": 12, - "response.latency": "*", - "response.http.code": 200, + "response.duration": "*", + "response.code": 200, "response.headers": { "date": "*", "content-type": "text/plain", diff --git a/src/envoy/mixer/integration_test/failed_request_test.go b/src/envoy/mixer/integration_test/failed_request_test.go index 9ae99bcf81d..011227afa6f 100644 --- a/src/envoy/mixer/integration_test/failed_request_test.go +++ b/src/envoy/mixer/integration_test/failed_request_test.go @@ -67,8 +67,8 @@ const reportAttributesMixerFail = ` "request.size": 0, "response.time": "*", "response.size": 41, - "response.latency": "*", - "response.http.code": 401, + "response.duration": "*", + "response.code": 401, "response.headers": { "date": "*", "content-type": "text/plain", @@ -100,8 +100,8 @@ const reportAttributesBackendFail = ` "request.size": 0, "response.time": "*", "response.size": 25, - "response.latency": "*", - "response.http.code": 400, + "response.duration": "*", + "response.code": 400, "response.headers": { "date": "*", "content-type": "text/plain; charset=utf-8", From 38a15cbf32590ccfde07b865d8c524431935c6cc Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Fri, 7 Apr 2017 17:29:56 -0700 Subject: [PATCH 80/83] Add envoy Buffer based TranscoderInputStream (#231) * Add envoy Buffer based TranscoderInputStream * fix format --- src/envoy/transcoding/BUILD | 40 ++++++++ src/envoy/transcoding/envoy_input_stream.cc | 63 ++++++++++++ src/envoy/transcoding/envoy_input_stream.h | 51 ++++++++++ .../transcoding/envoy_input_stream_test.cc | 97 +++++++++++++++++++ 4 files changed, 251 insertions(+) create mode 100644 src/envoy/transcoding/BUILD create mode 100644 src/envoy/transcoding/envoy_input_stream.cc create mode 100644 src/envoy/transcoding/envoy_input_stream.h create mode 100644 src/envoy/transcoding/envoy_input_stream_test.cc diff --git a/src/envoy/transcoding/BUILD b/src/envoy/transcoding/BUILD new file mode 100644 index 00000000000..8e15a45eadb --- /dev/null +++ b/src/envoy/transcoding/BUILD @@ -0,0 +1,40 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ +# +cc_library( + name = "envoy_input_stream", + srcs = [ + "envoy_input_stream.cc", + ], + hdrs = [ + "envoy_input_stream.h", + ], + deps = [ + "//contrib/endpoints/src/grpc/transcoding:transcoder_input_stream", + "@envoy//source/exe:envoy_common_lib", + ], +) + +cc_test( + name = "envoy_input_stream_test", + srcs = [ + "envoy_input_stream_test.cc", + ], + deps = [ + ":envoy_input_stream", + "@googletest_git//:googletest_main", + ], +) diff --git a/src/envoy/transcoding/envoy_input_stream.cc b/src/envoy/transcoding/envoy_input_stream.cc new file mode 100644 index 00000000000..81334303a7a --- /dev/null +++ b/src/envoy/transcoding/envoy_input_stream.cc @@ -0,0 +1,63 @@ +/* Copyright 2017 Istio Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/envoy/transcoding/envoy_input_stream.h" + +namespace Grpc { + +void EnvoyInputStream::Move(Buffer::Instance &instance) { + if (!finished_) { + buffer_.move(instance); + } +} + +bool EnvoyInputStream::Next(const void **data, int *size) { + if (position_ != 0) { + buffer_.drain(position_); + position_ = 0; + } + + Buffer::RawSlice slice; + uint64_t num_slices = buffer_.getRawSlices(&slice, 1); + + if (num_slices) { + *data = slice.mem_; + *size = slice.len_; + position_ = slice.len_; + byte_count_ += slice.len_; + return true; + } + + if (!finished_) { + *data = nullptr; + *size = 0; + return true; + } + return false; +} + +void EnvoyInputStream::BackUp(int count) { + GOOGLE_CHECK_GE(count, 0); + GOOGLE_CHECK_LE(count, position_); + + position_ -= count; + byte_count_ -= count; +} + +int64_t EnvoyInputStream::BytesAvailable() const { + return buffer_.length() - position_; +} + +} // namespace Grpc \ No newline at end of file diff --git a/src/envoy/transcoding/envoy_input_stream.h b/src/envoy/transcoding/envoy_input_stream.h new file mode 100644 index 00000000000..3e204e21eec --- /dev/null +++ b/src/envoy/transcoding/envoy_input_stream.h @@ -0,0 +1,51 @@ +/* Copyright 2017 Istio Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "precompiled/precompiled.h" + +#include "common/buffer/buffer_impl.h" +#include "contrib/endpoints/src/grpc/transcoding/transcoder_input_stream.h" + +namespace Grpc { + +class EnvoyInputStream + : public google::api_manager::transcoding::TranscoderInputStream { + public: + // Add a buffer to input stream, will consume all buffer from parameter + // if the stream is not finished + void Move(Buffer::Instance &instance); + + // Mark the buffer is finished + void Finish() { finished_ = true; } + + // TranscoderInputStream + virtual bool Next(const void **data, int *size) override; + virtual void BackUp(int count) override; + virtual bool Skip(int count) override { return false; } // Not implemented + virtual google::protobuf::int64 ByteCount() const override { + return byte_count_; + } + virtual int64_t BytesAvailable() const override; + + private: + Buffer::OwnedImpl buffer_; + int position_{0}; + int64_t byte_count_{0}; + bool finished_{false}; +}; + +} // namespace Grpc \ No newline at end of file diff --git a/src/envoy/transcoding/envoy_input_stream_test.cc b/src/envoy/transcoding/envoy_input_stream_test.cc new file mode 100644 index 00000000000..e8604367b91 --- /dev/null +++ b/src/envoy/transcoding/envoy_input_stream_test.cc @@ -0,0 +1,97 @@ +/* Copyright 2017 Istio Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/envoy/transcoding/envoy_input_stream.h" + +#include "gtest/gtest.h" + +namespace Grpc { +namespace { + +class EnvoyInputStreamTest : public testing::Test { + public: + EnvoyInputStreamTest() { + Buffer::OwnedImpl buffer{"abcd"}; + stream_.Move(buffer); + } + + std::string slice_data_{"abcd"}; + EnvoyInputStream stream_; + + const void *data_; + int size_; +}; + +TEST_F(EnvoyInputStreamTest, Move) { + Buffer::OwnedImpl buffer{"abcd"}; + stream_.Move(buffer); + + EXPECT_EQ(0, buffer.length()); + EXPECT_EQ(8, stream_.BytesAvailable()); +} + +TEST_F(EnvoyInputStreamTest, Next) { + EXPECT_TRUE(stream_.Next(&data_, &size_)); + EXPECT_EQ(4, size_); + EXPECT_EQ(0, memcmp(slice_data_.data(), data_, size_)); +} + +TEST_F(EnvoyInputStreamTest, TwoSlices) { + Buffer::OwnedImpl buffer("efgh"); + + stream_.Move(buffer); + + EXPECT_TRUE(stream_.Next(&data_, &size_)); + EXPECT_EQ(4, size_); + EXPECT_EQ(0, memcmp(slice_data_.data(), data_, size_)); + EXPECT_TRUE(stream_.Next(&data_, &size_)); + EXPECT_EQ(4, size_); + EXPECT_EQ(0, memcmp("efgh", data_, size_)); +} + +TEST_F(EnvoyInputStreamTest, BackUp) { + EXPECT_TRUE(stream_.Next(&data_, &size_)); + EXPECT_EQ(4, size_); + + stream_.BackUp(3); + EXPECT_EQ(3, stream_.BytesAvailable()); + EXPECT_EQ(1, stream_.ByteCount()); + + EXPECT_TRUE(stream_.Next(&data_, &size_)); + EXPECT_EQ(3, size_); + EXPECT_EQ(0, memcmp("bcd", data_, size_)); + EXPECT_EQ(4, stream_.ByteCount()); +} + +TEST_F(EnvoyInputStreamTest, ByteCount) { + EXPECT_EQ(0, stream_.ByteCount()); + EXPECT_TRUE(stream_.Next(&data_, &size_)); + EXPECT_EQ(4, stream_.ByteCount()); +} + +TEST_F(EnvoyInputStreamTest, Finish) { + EXPECT_TRUE(stream_.Next(&data_, &size_)); + EXPECT_TRUE(stream_.Next(&data_, &size_)); + EXPECT_EQ(0, size_); + stream_.Finish(); + EXPECT_FALSE(stream_.Next(&data_, &size_)); + + Buffer::OwnedImpl buffer("efgh"); + stream_.Move(buffer); + + EXPECT_EQ(4, buffer.length()); +} +} +} \ No newline at end of file From 71097decafdb9975814c7867992216fd3fa5918c Mon Sep 17 00:00:00 2001 From: Martin Taillefer Date: Sun, 9 Apr 2017 05:37:30 -0700 Subject: [PATCH 81/83] A few doc changes for consistency across repos. (#235) --- CONTRIBUTING.md | 40 ++++------------------------------------ README.md | 13 ++++--------- 2 files changed, 8 insertions(+), 45 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 936e3c8f91b..1c21f8a4311 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,37 +1,5 @@ -# Contributing guidelines +# Contribution guidelines -So, you want to hack on the Istio Proxy? Yay! - -- [Contributor license agreements](#contributor-license-agreements) -- [Contributing a patch](#contributing-a-patch) - -## Contributor license agreements - -We'd love to accept your patches! Before we can take them, you have to jump a -few legal hurdles. - -Please fill out the [Google CLA](https://cla.developers.google.com). - -Once you are CLA'ed, we'll be able to accept your pull requests. This is necessary -because you own the copyright to your changes, even after your contribution -becomes part of this project. So this agreement simply gives us permission -to use and redistribute your contributions as part of the project. - -***NOTE***: Only original source code from you and other people that have -signed the CLA can be accepted into the repository. This policy does not -apply to [third_party](third_party/) and [vendor](vendor/). - -## Contributing a patch - -If you're working on an existing issue, simply respond to the issue and express -interest in working on it. This helps other people know that the issue is -active, and hopefully prevents duplicated efforts. - -If you want to work on a new idea of relatively small scope: - -1. Submit an issue describing your proposed change to the repo in question. -1. The repo owners will respond to your issue promptly. -1. If your proposed change is accepted, and you haven't already done so, sign a - Contributor License Agreement (see details above). -1. Fork the repo, develop, and test your changes. -1. Submit a pull request. +So, you want to hack on the Istio proxy? Yay! Please refer to Istio's overall +[contribution guidelines](https://github.com/istio/istio/blob/master/CONTRIBUTING.md) +to find out how you can help. diff --git a/README.md b/README.md index 26a33ca4b89..9c478e2f3f0 100644 --- a/README.md +++ b/README.md @@ -22,12 +22,7 @@ Server Side Features: - *Monitoring & Logging*. The Proxy can report server-side metrics and logs to the Mixer. -To learn more... - -- User guide coming soon! -- [Contributing to the project](./CONTRIBUTING.md) - -### Filing issues - -If you have a question about an Istio proxy or have a problem using one, please -[file an issue](https://github.com/istio/proxy/issues/new). +Please see [istio.io](https://istio.io) +to learn about the overall Istio project and how to get in touch with us. To learn how you can +contribute to any of the Istio components, including the proxy, please +see the Istio [contribution guidelines](https://github.com/istio/istio/blob/master/CONTRIBUTING.md). From f9eb106f8ae5ba2a39a57239d4a10a57a7316fd1 Mon Sep 17 00:00:00 2001 From: Sarvani Vakkalanka Date: Mon, 10 Apr 2017 11:16:13 -0700 Subject: [PATCH 82/83] Add repositories.bzl --- contrib/endpoints/repositories.bzl | 2 -- script/release-docker | 46 ++++++++++++++++++++++++++++++ src/envoy/mixer/repositories.bzl | 31 ++++++++++++++++++++ 3 files changed, 77 insertions(+), 2 deletions(-) create mode 100755 script/release-docker create mode 100644 src/envoy/mixer/repositories.bzl diff --git a/contrib/endpoints/repositories.bzl b/contrib/endpoints/repositories.bzl index f1865cfbb60..19ab3bf402a 100644 --- a/contrib/endpoints/repositories.bzl +++ b/contrib/endpoints/repositories.bzl @@ -251,8 +251,6 @@ cc_proto_library( "google/api/control.proto", "google/api/documentation.proto", "google/api/endpoint.proto", - "google/api/experimental/authorization_config.proto", - "google/api/experimental/experimental.proto", "google/api/http.proto", "google/api/label.proto", "google/api/log.proto", diff --git a/script/release-docker b/script/release-docker new file mode 100755 index 00000000000..d21caffee55 --- /dev/null +++ b/script/release-docker @@ -0,0 +1,46 @@ +#!/bin/bash +# +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ +# +set -ex + +# This docker image can be used as: +# +# docker run IMAGE -a backend_address -p PORT -m MIXER_SERVER +# + +PROJECT=istio-testing +IMAGE_PREFIX="gcr.io/${PROJECT}/proxy" + +DATE_PART=$(date +"%Y%m%d") +SHA_PART=$(git show -q HEAD --pretty=format:%h) +DOCKER_TAG="${DATE_PART}${SHA_PART}" + +IMAGE_NAME="${IMAGE_PREFIX}:${DOCKER_TAG}" + +gcloud docker --authorize-only + +bazel run --config=release //src/envoy/mixer:proxy "${IMAGE_NAME}" + +gcloud docker -- push "${IMAGE_NAME}" + +IMAGE_LATEST="${IMAGE_PREFIX}:latest" + +docker tag -f "${IMAGE_NAME}" "${IMAGE_LATEST}" + +gcloud docker -- push "${IMAGE_LATEST}" + diff --git a/src/envoy/mixer/repositories.bzl b/src/envoy/mixer/repositories.bzl new file mode 100644 index 00000000000..4382e71f921 --- /dev/null +++ b/src/envoy/mixer/repositories.bzl @@ -0,0 +1,31 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ +# + +MIXER_CLIENT = "c5d857e28bfcc53f20f59466b464f99526737545" + +def mixer_client_repositories(bind=True): + native.git_repository( + name = "mixerclient_git", + commit = MIXER_CLIENT, + remote = "https://github.com/istio/mixerclient.git", + ) + + if bind: + native.bind( + name = "mixer_client_lib", + actual = "@mixerclient_git//:mixer_client_lib", + ) From d602c7aef35399f7d0c74a7b34175ade3066f44c Mon Sep 17 00:00:00 2001 From: Jae Kim Date: Mon, 10 Apr 2017 12:18:20 -0700 Subject: [PATCH 83/83] Added missing export setting in bazel configuration (#236) * Added export missing in bazel configuration * Added export missing in bazel configuration --- contrib/endpoints/repositories.bzl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/contrib/endpoints/repositories.bzl b/contrib/endpoints/repositories.bzl index 19ab3bf402a..0d66babfcc8 100644 --- a/contrib/endpoints/repositories.bzl +++ b/contrib/endpoints/repositories.bzl @@ -211,10 +211,13 @@ def googleapis_repositories(protobuf_repo="@protobuf_git//", bind=True): # ################################################################################ # + licenses(["notice"]) load("@protobuf_git//:protobuf.bzl", "cc_proto_library") +exports_files(glob(["google/**"])) + cc_proto_library( name = "servicecontrol", srcs = [