From efa1dac46259215592da689008b862ed0dc6cc16 Mon Sep 17 00:00:00 2001 From: Sebastien Vas Date: Tue, 10 Jan 2017 15:18:08 -0800 Subject: [PATCH 1/5] Fixed bazel install Change-Id: Ic573e8e4cc2900fa44dfa5c67d6728c58283b649 --- script/tools/linux-install-bazel | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/script/tools/linux-install-bazel b/script/tools/linux-install-bazel index ef5fc51b8..7d9c8925a 100755 --- a/script/tools/linux-install-bazel +++ b/script/tools/linux-install-bazel @@ -51,12 +51,13 @@ function update_bazel() { fi if [[ "${BAZEL_VERSION}" != "${bazel_current}" ]]; then - retry -n 3 wget -nc -P "${BAZEL_DIRECTORY}" "${BAZEL_URL}" \ + retry -n 3 wget -q -nc -P "${BAZEL_DIRECTORY}" "${BAZEL_URL}" \ || error_exit "Could not download bazel" chmod +x "${BAZEL_DIRECTORY}/${BAZEL_SH}" \ || error_exit "Could not mark bazel shell as executable" - sh "${BAZEL_DIRECTORY}/${BAZEL_SH}" --user \ + ${SUDO} "${BAZEL_DIRECTORY}/${BAZEL_SH}" \ || error_exit "Could not install bazel" + set_bazel fi echo 'Bazel up-to-date.' } From 315d07230e8b4baa22ecf2545f855d2a307480ea Mon Sep 17 00:00:00 2001 From: Sebastien Vas Date: Wed, 11 Jan 2017 11:04:09 -0800 Subject: [PATCH 2/5] Make docker slaves consistent with jenkinsci/slave Change-Id: I6251e93ca7f9e588a552fbf5562d0ec24534480a --- jenkins/slaves/debian-8.Dockerfile | 3 ++- jenkins/slaves/ubuntu-16-04.Dockerfile | 3 ++- script/tools/linux-install-bazel | 2 +- script/tools/linux-install-nodejs | 27 +++++++++++++------------- 4 files changed, 19 insertions(+), 16 deletions(-) diff --git a/jenkins/slaves/debian-8.Dockerfile b/jenkins/slaves/debian-8.Dockerfile index 4c7be50c5..05ddb304b 100644 --- a/jenkins/slaves/debian-8.Dockerfile +++ b/jenkins/slaves/debian-8.Dockerfile @@ -45,7 +45,8 @@ ADD jenkins/slaves/entrypoint /usr/local/bin/entrypoint RUN chmod +rx /usr/local/bin/jenkins-slave /usr/local/bin/entrypoint USER jenkins -VOLUME ${HOME} +RUN mkdir ${HOME}/.jenkins +VOLUME ${HOME}/.jenkins WORKDIR ${HOME} ENTRYPOINT ["entrypoint"] diff --git a/jenkins/slaves/ubuntu-16-04.Dockerfile b/jenkins/slaves/ubuntu-16-04.Dockerfile index aa87bc7f0..9c842d594 100644 --- a/jenkins/slaves/ubuntu-16-04.Dockerfile +++ b/jenkins/slaves/ubuntu-16-04.Dockerfile @@ -45,7 +45,8 @@ ADD jenkins/slaves/entrypoint /usr/local/bin/entrypoint RUN chmod +rx /usr/local/bin/jenkins-slave /usr/local/bin/entrypoint USER jenkins -VOLUME ${HOME} +RUN mkdir ${HOME}/.jenkins +VOLUME ${HOME}/.jenkins WORKDIR ${HOME} ENTRYPOINT ["entrypoint"] diff --git a/script/tools/linux-install-bazel b/script/tools/linux-install-bazel index 7d9c8925a..aa2064ae7 100755 --- a/script/tools/linux-install-bazel +++ b/script/tools/linux-install-bazel @@ -53,7 +53,7 @@ function update_bazel() { if [[ "${BAZEL_VERSION}" != "${bazel_current}" ]]; then retry -n 3 wget -q -nc -P "${BAZEL_DIRECTORY}" "${BAZEL_URL}" \ || error_exit "Could not download bazel" - chmod +x "${BAZEL_DIRECTORY}/${BAZEL_SH}" \ + ${SUDO} chmod +x "${BAZEL_DIRECTORY}/${BAZEL_SH}" \ || error_exit "Could not mark bazel shell as executable" ${SUDO} "${BAZEL_DIRECTORY}/${BAZEL_SH}" \ || error_exit "Could not install bazel" diff --git a/script/tools/linux-install-nodejs b/script/tools/linux-install-nodejs index 2bbe383a6..f9aa19934 100755 --- a/script/tools/linux-install-nodejs +++ b/script/tools/linux-install-nodejs @@ -37,23 +37,24 @@ DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" . ${DIR}/all-utilities || { echo "Cannot load Bash utilities" ; exit 1 ; } NODE_JS_VERSION='4.6.1' -NODE_JS_URL="https://nodejs.org/dist/v${NODE_JS_VERSION}/node-v${NODE_JS_VERSION}-linux-x64.tar.xz" - -function install_node_js() { - echo 'Installing Node.js and npm.' - local node_tar="$(mktemp /tmp/XXXX.tar.xz)" - wget "${NODE_JS_URL}" -O "${node_tar}" \ - && ${SUDO} tar -C /usr/local --strip-components 1 -xf "${node_tar}" \ - && rm -rf "${node_tar}" \ - && return 0 - return 1 -} +NODE_JS_ARCHIVE="node-v${NODE_JS_VERSION}-linux-x64.tar.xz" +NODE_JS_URL="https://nodejs.org/dist/v${NODE_JS_VERSION}/${NODE_JS_ARCHIVE}" +NODE_JS_DIRECTORY="${TOOLS_DIR}/nodejs" function update_node_js() { local version="$(node --version | grep -oh '[0-9]*\.[0-9]*\.[0-9]*')" + + if [[ -d "${NODE_JS_DIRECTORY}" ]]; then + mkdir -p "${NODE_JS_DIRECTORY}" + fi + if [[ "${version}" != "${NODE_JS_VERSION}" ]]; then - retry install_node_js \ - || error_exit 'Cannot install node.js.' + echo 'Installing Node.js and npmV.' + retry -n 3 wget -q -nc -P "${NODE_JS_DIRECTORY}" "${NODE_JS_URL}" \ + || error_exit 'Could not download nodejs' + ${SUDO} tar -C /usr/local --strip-components 1 -xf \ + "${NODE_JS_DIRECTORY}/${NODE_JS_ARCHIVE}" \ + || error_exit 'Could not extract nodejs archive' fi echo 'Node.js is up to date.' } From b76ea583d6421c9ccbf96e34343f75b957b5c8ed Mon Sep 17 00:00:00 2001 From: Qian Sun Date: Thu, 12 Jan 2017 15:28:31 -0800 Subject: [PATCH 3/5] Align with latest istio/proxy. (#73) --- WORKSPACE | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/WORKSPACE b/WORKSPACE index 1185129b0..8c58161ab 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -26,7 +26,7 @@ # # A Bazel (http://bazel.io) workspace for the Google Cloud Endpoints runtime. -ISTIO_PROXY = "decf6c670351d4db93858f238f430ab0151da73e" +ISTIO_PROXY = "8b3a4429965ed1e792704e795e5d22782eca505b" git_repository( name = "nginx", @@ -57,6 +57,7 @@ load( "@istio_proxy_git//contrib/endpoints:repositories.bzl", "grpc_repositories", "servicecontrol_client_repositories", + "mixerapi_repositories", ) load( "@istio_proxy_git//:repositories.bzl", @@ -96,6 +97,8 @@ bind( servicecontrol_client_repositories() +mixerapi_repositories() + protobuf_repositories() googletest_repositories() From 8784e6a84bbf4f23e60ba187ab44945b6a9b209a Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Fri, 13 Jan 2017 17:25:25 -0800 Subject: [PATCH 4/5] Support HEAD request in transcoding (#74) * Support HEAD request in transcoding * Removed unused variables --- WORKSPACE | 2 +- src/nginx/grpc_server_call.cc | 5 + src/nginx/t/BUILD | 1 + src/nginx/t/transcoding_head.t | 204 +++++++++++++++++++++++++++++++++ 4 files changed, 211 insertions(+), 1 deletion(-) create mode 100644 src/nginx/t/transcoding_head.t diff --git a/WORKSPACE b/WORKSPACE index 8c58161ab..9c5d68349 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -26,7 +26,7 @@ # # A Bazel (http://bazel.io) workspace for the Google Cloud Endpoints runtime. -ISTIO_PROXY = "8b3a4429965ed1e792704e795e5d22782eca505b" +ISTIO_PROXY = "ee3cdeeb33b1de0b87c80809802de5246cbb3e25" git_repository( name = "nginx", diff --git a/src/nginx/grpc_server_call.cc b/src/nginx/grpc_server_call.cc index cd6bbb9ff..8ad492370 100644 --- a/src/nginx/grpc_server_call.cc +++ b/src/nginx/grpc_server_call.cc @@ -78,6 +78,11 @@ void TrimFront(std::vector *slices, size_t count) { // returns NGX_AGAIN. ngx_int_t ngx_esp_write_output(ngx_http_request_t *r, ngx_chain_t *out, ngx_http_event_handler_pt write_event_handler) { + // Don't write response body if the request is HEAD + if (r->header_only) { + return NGX_OK; + } + ngx_int_t rc = ngx_http_output_filter(r, out); if (rc == NGX_OK) { diff --git a/src/nginx/t/BUILD b/src/nginx/t/BUILD index 1511cbe17..02b2e5926 100644 --- a/src/nginx/t/BUILD +++ b/src/nginx/t/BUILD @@ -280,6 +280,7 @@ nginx_suite( "transcoding_bindings.t", "transcoding_deep_struct.t", "transcoding_errors.t", + "transcoding_head.t", "transcoding_ignore_unknown_fields.t", "transcoding_large.t", "transcoding_metadata.t", diff --git a/src/nginx/t/transcoding_head.t b/src/nginx/t/transcoding_head.t new file mode 100644 index 000000000..cace262e9 --- /dev/null +++ b/src/nginx/t/transcoding_head.t @@ -0,0 +1,204 @@ +# Copyright (C) Extensible Service Proxy Authors +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# +################################################################################ +# +use strict; +use warnings; + +################################################################################ + +use src::nginx::t::ApiManager; # Must be first (sets up import path to the Nginx test module) +use src::nginx::t::HttpServer; +use src::nginx::t::ServiceControl; +use Test::Nginx; # Imports Nginx's test module +use Test::More; # And the test framework +use JSON::PP; + +################################################################################ + +# Port assignments +my $NginxPort = ApiManager::pick_port(); +my $ServiceControlPort = ApiManager::pick_port(); +my $GrpcServerPort = ApiManager::pick_port(); + +my $t = Test::Nginx->new()->has(qw/http proxy/)->plan(13); + +$t->write_file('service.pb.txt', + ApiManager::get_transcoding_test_service_config( + 'endpoints-transcoding-test.cloudendpointsapis.com', + "http://127.0.0.1:${ServiceControlPort}") . + ApiManager::read_test_file('testdata/logs_metrics.pb.txt')); + +$t->write_file('server_config.pb.txt', ApiManager::disable_service_control_cache); + +$t->write_file_expand('nginx.conf', <run_daemon(\&service_control, $t, $ServiceControlPort, 'servicecontrol.log', $report_done); +ApiManager::run_transcoding_test_server($t, 'server.log', "127.0.0.1:${GrpcServerPort}"); + +is($t->waitforsocket("127.0.0.1:${ServiceControlPort}"), 1, "Service control socket ready."); +is($t->waitforsocket("127.0.0.1:${GrpcServerPort}"), 1, "GRPC test server socket ready."); +$t->run(); +is($t->waitforsocket("127.0.0.1:${NginxPort}"), 1, "Nginx socket ready."); + +################################################################################ + +my $shelves_response = ApiManager::http($NginxPort, <waitforfile("$t->{_testdir}/${report_done}"), 1, 'Report body file ready.'); +$t->stop_daemons(); + +# Check the requests that the backend has received +my $shelves_request_expected = {}; + +my $server_output = $t->read_file('server.log'); +my @server_requests = split /\r\n\r\n/, $server_output; + +ok(ApiManager::compare_json($server_requests[0], $shelves_request_expected), 'Server received request'); + +my ($headers, $actual_body) = split /\r\n\r\n/, $shelves_response, 2; +like($headers, qr/HTTP\/1\.1 200 OK/, 'Returned HTTP 200.'); +like($headers, qr/Content-Type: application\/json/, 'Returned Content-Type'); +is('', $actual_body, 'Response body is empty'); + +# Expect 2 service control calls for 1 requests. +my @servicecontrol_requests = ApiManager::read_http_stream($t, 'servicecontrol.log'); +is(scalar @servicecontrol_requests, 2, 'Service control was called 2 times'); + +while (my ($i, $r) = each @servicecontrol_requests) { + if ($i % 2 == 0) { + like($r->{uri}, qr/:check$/, "Check has correct uri for ${i}"); + } else { + like($r->{uri}, qr/:report$/, "Report has correct uri for ${i}"); + } +} + +# :check +my $r = shift @servicecontrol_requests; +my $check_body = ServiceControl::convert_proto($r->{body}, 'check_request', 'json'); +my $expected_check_body = { + 'serviceName' => 'endpoints-transcoding-test.cloudendpointsapis.com', + 'serviceConfigId' => '2016-08-25r1', + 'operation' => { + 'consumerId' => 'api_key:api-key', + 'operationName' => 'endpoints.examples.bookstore.Bookstore.ListShelves', + 'labels' => { + 'servicecontrol.googleapis.com/caller_ip' => '127.0.0.1', + 'servicecontrol.googleapis.com/service_agent' => ServiceControl::service_agent(), + 'servicecontrol.googleapis.com/user_agent' => 'ESP', + } + } +}; +ok(ServiceControl::compare_json($check_body, $expected_check_body), 'Check body is received.'); + +# :report +$r = shift @servicecontrol_requests; +my $report_body = ServiceControl::convert_proto($r->{body}, 'report_request', 'json'); +my $expected_report_body = ServiceControl::gen_report_body({ + 'serviceName' => 'endpoints-transcoding-test.cloudendpointsapis.com', + 'api_method' => 'endpoints.examples.bookstore.Bookstore.ListShelves', + 'serviceConfigId' => '2016-08-25r1', + 'url' => '/shelves?key=api-key', + 'api_key' => 'api-key', + 'producer_project_id' => 'endpoints-transcoding-test', + 'location' => 'us-central1', + 'api_name' => 'endpoints.examples.bookstore.Bookstore', + 'api_version' => 'v1', + 'http_method' => 'HEAD', + 'log_message' => 'Method: endpoints.examples.bookstore.Bookstore.ListShelves', + 'response_code' => '200', + 'request_size' => 52, + 'response_size' => 122, + 'request_bytes' => 52, + 'response_bytes' => 122, + 'streaming_request_message_counts' => 1, + 'streaming_response_message_counts' => 1, + }); +ok(ServiceControl::compare_json($report_body, $expected_report_body), 'Report body is received.'); + +################################################################################ + +sub service_control { + my ($t, $port, $file, $done) = @_; + + my $server = HttpServer->new($port, $t->testdir() . '/' . $file) + or die "Can't create test server socket: $!\n"; + + local $SIG{PIPE} = 'IGNORE'; + + $server->on_sub('POST', '/v1/services/endpoints-transcoding-test.cloudendpointsapis.com:check', sub { + my ($headers, $body, $client) = @_; + print $client <on_sub('POST', '/v1/services/endpoints-transcoding-test.cloudendpointsapis.com:report', sub { + my ($headers, $body, $client) = @_; + print $client <write_file($done, ":report done"); + }); + + $server->run(); +} + +################################################################################ From 3ed939330e2dc2a5d25228a2760b749e3c7ffe9a Mon Sep 17 00:00:00 2001 From: Sebastien Vas Date: Wed, 11 Jan 2017 15:46:30 -0800 Subject: [PATCH 5/5] Fixing bazel install for ubuntu salve Change-Id: I184624d166dd49ccd4242f3bcfc54b1182216909 --- Jenkinsfile | 2 +- script/tools/linux-install-bazel | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 5bf674bed..71358414d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -536,7 +536,7 @@ def buildNewDockerSlave(nodeLabel) { "-T \"${TOOLS_BUCKET}\"") echo("Testing ${testDockerImage}") node(getTestSlaveLabel(nodeLabel)) { - setupNode() + checkoutSourceCode() sh('jenkins/slaves/slave-test') } echo("Retagging ${testDockerImage} to ${dockerImage}") diff --git a/script/tools/linux-install-bazel b/script/tools/linux-install-bazel index aa2064ae7..9ec386c79 100755 --- a/script/tools/linux-install-bazel +++ b/script/tools/linux-install-bazel @@ -58,6 +58,8 @@ function update_bazel() { ${SUDO} "${BAZEL_DIRECTORY}/${BAZEL_SH}" \ || error_exit "Could not install bazel" set_bazel + ${SUDO} ${BAZEL} shutdown + ${SUDO} rm -rf ${HOME}/.cache/bazel fi echo 'Bazel up-to-date.' }