diff --git a/lib/cpp-netlib b/lib/cpp-netlib index b540ff3b..540ed762 160000 --- a/lib/cpp-netlib +++ b/lib/cpp-netlib @@ -1 +1 @@ -Subproject commit b540ff3bb234d9457bb67656a57bedaeb41edd2d +Subproject commit 540ed7622be3f9534709036522f86bde1e84829f diff --git a/src/Makefile b/src/Makefile index afc41b43..c2e27294 100644 --- a/src/Makefile +++ b/src/Makefile @@ -1,6 +1,8 @@ LIBDIR=../lib CPP_NETLIB_DIR=$(LIBDIR)/cpp-netlib CPP_NETLIB_LIBDIR=$(CPP_NETLIB_DIR)/libs/network/src +NETWORK_URI_DIR=$(CPP_NETLIB_DIR)/deps/uri +NETWORK_URI_LIBDIR=$(NETWORK_URI_DIR)/src YAML_CPP_DIR=$(LIBDIR)/yaml-cpp YAML_CPP_LIBDIR=$(YAML_CPP_DIR) SUBMODULE_DIRS=$(CPP_NETLIB_DIR) $(YAML_CPP_DIR) @@ -8,20 +10,23 @@ SUBMODULE_DIRS=$(CPP_NETLIB_DIR) $(YAML_CPP_DIR) SED_I=/usr/bin/env sed -i CMAKE=cmake CXXFLAGS=\ - -std=c++11 -g -Wno-deprecated \ + -std=c++11 -g -Wno-write-strings -Wno-deprecated \ -DENABLE_DOCKER_METADATA -DENABLE_KUBERNETES_METADATA \ - -I$(CPP_NETLIB_DIR) -I$(YAML_CPP_DIR)/include -LDFLAGS=-L$(CPP_NETLIB_LIBDIR) -L$(YAML_CPP_LIBDIR) -LDLIBS=-lcppnetlib-uri -lcppnetlib-client-connections -lboost_program_options \ - -lboost_system -lboost_thread -lpthread -lyajl -lssl -lcrypto -lyaml-cpp -SED_EXTRA= + -I$(CPP_NETLIB_DIR) -I$(NETWORK_URI_DIR)/include -I$(YAML_CPP_DIR)/include +LDFLAGS=-L$(CPP_NETLIB_LIBDIR) -L$(NETWORK_URI_LIBDIR) -L$(YAML_CPP_LIBDIR) +LDLIBS=\ + -lcppnetlib-client-connections -lcppnetlib-server-parsers -lnetwork-uri \ + -lboost_program_options -lboost_system -lboost_thread \ + -lpthread -lyajl -lssl -lcrypto -lyaml-cpp +SED_EXTRA=-e 's/-Wall/-Wall -Wno-deprecated/' UNAME_S=$(shell uname -s) ifeq ($(UNAME_S),Darwin) - CXXFLAGS+= -Wno-deprecated-declarations -I/usr/local/include + CXXFLAGS+= -Wno-deprecated-declarations -Wno-c++14-extensions \ + -I/usr/local/include LDFLAGS+= -L/usr/local/lib SED_I+= '' - SED_EXTRA+=-e \ + SED_EXTRA+= -e \ 's/-Wall/-Wall -Wno-deprecated-declarations -Wno-unused-local-typedef/' endif @@ -46,8 +51,9 @@ SRCS=\ OBJS=$(SRCS:%.cc=%.o) CPP_NETLIB_LIBS=\ - $(CPP_NETLIB_LIBDIR)/libcppnetlib-uri.a \ - $(CPP_NETLIB_LIBDIR)/libcppnetlib-client-connections.a + $(CPP_NETLIB_LIBDIR)/libcppnetlib-client-connections.a \ + $(CPP_NETLIB_LIBDIR)/libcppnetlib-server-parsers.a \ + $(NETWORK_URI_LIBDIR)/libnetwork-uri.a YAML_CPP_LIBS=\ $(YAML_CPP_LIBDIR)/libyaml-cpp.a LIBS=$(CPP_NETLIB_LIBS) $(YAML_CPP_LIBS) @@ -135,7 +141,7 @@ purge: clean git submodule deinit -f $(SUBMODULE_DIRS) init-submodules: - git submodule update --init $(SUBMODULE_DIRS) + git submodule update --init --recursive $(SUBMODULE_DIRS) touch init-submodules $(CPP_NETLIB_DIR)/Makefile: init-submodules @@ -143,7 +149,8 @@ $(CPP_NETLIB_DIR)/Makefile: init-submodules $(SED_I) -e 's/unit_test_framework //' $(SED_EXTRA) CMakeLists.txt && \ $(CMAKE) -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_FLAGS=-std=c++11 \ -DCMAKE_C_COMPILER=gcc -DCMAKE_CXX_COMPILER=g++ \ - -DCPP-NETLIB_BUILD_TESTS=OFF -DCPP-NETLIB_BUILD_EXAMPLES=OFF + -DCPP-NETLIB_BUILD_TESTS=OFF -DCPP-NETLIB_BUILD_EXAMPLES=OFF \ + -DUri_BUILD_TESTS=OFF -DUri_BUILD_DOCS=OFF $(CPP_NETLIB_LIBS): build-cpp-netlib diff --git a/src/api_server.cc b/src/api_server.cc index 6e6fbafb..a0051026 100644 --- a/src/api_server.cc +++ b/src/api_server.cc @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -53,7 +54,7 @@ class MetadataApiServer { public: Handler(const MetadataAgent& agent); void operator()(const HttpServer::request& request, - HttpServer::response& response); + std::shared_ptr conn); void log(const HttpServer::string_type& info); private: const MetadataAgent& agent_; @@ -90,7 +91,7 @@ MetadataApiServer::Handler::Handler(const MetadataAgent& agent) : agent_(agent) {} void MetadataApiServer::Handler::operator()(const HttpServer::request& request, - HttpServer::response& response) { + std::shared_ptr conn) { static const std::string kPrefix = "/monitoredResource/"; // The format for the local metadata API request is: // {host}:{port}/monitoredResource/{id} @@ -111,15 +112,14 @@ void MetadataApiServer::Handler::operator()(const HttpServer::request& request, if (agent_.config_.VerboseLogging()) { LOG(WARNING) << "No matching resource for " << id; } - response = HttpServer::response::stock_reply( - HttpServer::response::not_found, ""); + conn->set_status(HttpServer::connection::not_found); } else { const MonitoredResource& resource = result->second; if (agent_.config_.VerboseLogging()) { LOG(INFO) << "Found resource for " << id << ": " << resource; } - response = HttpServer::response::stock_reply( - HttpServer::response::ok, resource.ToJSON()->ToString()); + conn->set_status(HttpServer::connection::ok); + conn->write(resource.ToJSON()->ToString()); } } } diff --git a/src/asio/local_resolve_op.hpp b/src/asio/local_resolve_op.hpp index cde5f4a0..7d646b9e 100644 --- a/src/asio/local_resolve_op.hpp +++ b/src/asio/local_resolve_op.hpp @@ -31,7 +31,8 @@ #include #include #include -#include +#include +#include #include #include "../logging.h" @@ -85,7 +86,7 @@ class resolve_op : public operation o->ec_ = boost::asio::error::operation_aborted; else { //LOG(ERROR) << "async_resolve() " << o->query_.host_name(); - std::string path = boost::network::uri::decoded(o->query_.host_name()); + std::string path = ::network::detail::decode(o->query_.host_name()); //LOG(ERROR) << "decoded " << path; struct stat buffer; int result = stat(path.c_str(), &buffer); @@ -120,7 +121,7 @@ class resolve_op : public operation p.h = boost::asio::detail::addressof(handler.handler_); if (!o->ec_) { - std::string path = boost::network::uri::decoded(o->query_.host_name()); + std::string path = ::network::detail::decode(o->query_.host_name()); handler.arg2_ = iterator_type::create( Protocol::endpoint(path), path, o->query_.service_name()); diff --git a/src/asio/local_resolver_service.hpp b/src/asio/local_resolver_service.hpp index d6fa4378..38f18fa8 100644 --- a/src/asio/local_resolver_service.hpp +++ b/src/asio/local_resolver_service.hpp @@ -30,7 +30,7 @@ #include #include #include -#include +#include #include #include "../logging.h" @@ -72,7 +72,7 @@ class resolver_service : public resolver_service_base boost::system::error_code& ec) { //LOG(ERROR) << "resolve() " << query.host_name(); - std::string path = boost::network::uri::decoded(query.host_name()); + std::string path = ::network::detail::decode(query.host_name()); //LOG(ERROR) << "decoded " << path; struct stat buffer; int result = stat(path.c_str(), &buffer); diff --git a/src/http/local_async_connection_base.hpp b/src/http/local_async_connection_base.hpp index a416a0ae..866184bc 100644 --- a/src/http/local_async_connection_base.hpp +++ b/src/http/local_async_connection_base.hpp @@ -1,6 +1,10 @@ #ifndef BOOST_NETWORK_PROTOCOL_HTTP_IMPL_LOCAL_ASYNC_CONNECTION_BASE_20170307 #define BOOST_NETWORK_PROTOCOL_HTTP_IMPL_LOCAL_ASYNC_CONNECTION_BASE_20170307 +// Note: This file is mostly a copy of +// boost/network/protocol/http/client/connection/async_base.hpp, +// except for the specialized bits. + // Copyright 2017 Igor Peshansky (igorp@google.com). // Copyright 2013-2017 Google, Inc. // Copyright 2010 Dean Michael Berris @@ -9,18 +13,18 @@ // (See accompanying file LICENSE_1_0.txt or copy at // http://www.boost.org/LICENSE_1_0.txt) -#include #include #include #include "traits/local_resolver.hpp" #include +#include namespace boost { namespace network { namespace http { namespace impl { -template +template struct async_connection_base; template @@ -32,30 +36,32 @@ struct http_async_connection; template<> struct async_connection_base { typedef async_connection_base this_type; - typedef resolver_policy::type resolver_base; - typedef resolver_base::resolver_type resolver_type; - typedef resolver_base::resolve_function resolve_function; - typedef string::type string_type; + typedef typename resolver_policy::type resolver_base; + typedef typename resolver_base::resolver_type resolver_type; + typedef typename resolver_base::resolve_function resolve_function; + typedef typename string::type string_type; typedef basic_request request; typedef basic_response response; - typedef iterator_range char_const_range; - typedef function + typedef typename std::array::type, 1024>::const_iterator const_iterator; + typedef iterator_range char_const_range; + typedef std::function body_callback_function_type; - typedef function body_generator_function_type; - typedef shared_ptr connection_ptr; + typedef std::function body_generator_function_type; + typedef std::shared_ptr connection_ptr; // This is the factory function which constructs the appropriate async - // connection implementation with the correct delegate chosen based on - // the + // connection implementation with the correct delegate chosen based on the // tag. static connection_ptr new_connection( resolve_function resolve, resolver_type &resolver, bool follow_redirect, bool always_verify_peer, bool https, int timeout, + bool remove_chunk_markers, optional certificate_filename = optional(), optional const &verify_path = optional(), optional certificate_file = optional(), optional private_key_file = optional(), optional ciphers = optional(), + optional sni_hostname = optional(), long ssl_options = 0); // This is the pure virtual entry-point for all asynchronous @@ -64,7 +70,7 @@ template<> struct async_connection_base { bool get_body, body_callback_function_type callback, body_generator_function_type generator) = 0; - virtual ~async_connection_base() {} + virtual ~async_connection_base() = default; }; #undef version_minor @@ -72,11 +78,8 @@ template<> struct async_connection_base { #undef Tag } // namespace impl - } // namespace http - } // namespace network - } // namespace boost #endif // BOOST_NETWORK_PROTOCOL_HTTP_IMPL_LOCAL_ASYNC_CONNECTION_BASE_20170307 diff --git a/src/http/local_async_connection_base.ipp b/src/http/local_async_connection_base.ipp index d1beba76..7c712ddd 100644 --- a/src/http/local_async_connection_base.ipp +++ b/src/http/local_async_connection_base.ipp @@ -1,6 +1,10 @@ #ifndef BOOST_NETWORK_PROTOCOL_HTTP_IMPL_LOCAL_ASYNC_CONNECTION_BASE_IPP_20170307 #define BOOST_NETWORK_PROTOCOL_HTTP_IMPL_LOCAL_ASYNC_CONNECTION_BASE_IPP_20170307 +// Note: This file is mostly a copy of +// boost/network/protocol/http/client/connection/async_base.hpp, +// except for the specialized bits. + // Copyright 2017 Igor Peshansky (igorp@google.com). // Copryight 2013-2017 Google, Inc. // Copyright 2010 Dean Michael Berris @@ -27,23 +31,25 @@ using acb = async_connection_base; acb::connection_ptr acb::new_connection( acb::resolve_function resolve, acb::resolver_type &resolver, bool follow_redirect, bool always_verify_peer, bool https, int timeout, + bool remove_chunk_markers, optional certificate_filename, optional const &verify_path, optional certificate_file, optional private_key_file, optional ciphers, + optional sni_hostname, long ssl_options) { typedef http_async_connection async_connection; typedef typename delegate_factory::type delegate_factory_type; - connection_ptr temp; - temp.reset(new async_connection( - resolver, resolve, follow_redirect, timeout, - delegate_factory_type::new_connection_delegate( - resolver.get_io_service(), https, always_verify_peer, - certificate_filename, verify_path, certificate_file, - private_key_file, ciphers, ssl_options))); - BOOST_ASSERT(temp.get() != 0); + auto delegate = delegate_factory_type::new_connection_delegate( + resolver.get_io_service(), https, always_verify_peer, + certificate_filename, verify_path, certificate_file, private_key_file, + ciphers, sni_hostname, ssl_options); + auto temp = std::make_shared( + resolver, resolve, follow_redirect, timeout, remove_chunk_markers, + std::move(delegate)); + BOOST_ASSERT(temp != nullptr); return temp; } @@ -52,11 +58,8 @@ acb::connection_ptr acb::new_connection( #undef Tag } // namespace impl - } // namespace http - } // namespace network - } // namespace boost #endif // BOOST_NETWORK_PROTOCOL_HTTP_IMPL_LOCAL_ASYNC_CONNECTION_BASE_IPP_20170307 diff --git a/src/http/local_async_normal.hpp b/src/http/local_async_normal.hpp index 290ea088..24de0018 100644 --- a/src/http/local_async_normal.hpp +++ b/src/http/local_async_normal.hpp @@ -1,6 +1,10 @@ #ifndef BOOST_NETWORK_PROTOCOL_HTTP_IMPL_HTTP_LOCAL_ASYNC_CONNECTION_HPP_20170307 #define BOOST_NETWORK_PROTOCOL_HTTP_IMPL_HTTP_LOCAL_ASYNC_CONNECTION_HPP_20170307 +// Note: This file is mostly a copy of +// boost/network/protocol/http/client/connection/async_normal.hpp, +// except for the specialized bits. + // Copyright 2010 (C) Dean Michael Berris // Copyright 2010 (C) Sinefunc, Inc. // Copyright 2011 Dean Michael Berris (dberris@google.com). @@ -10,6 +14,7 @@ // (See accompanying file LICENSE_1_0.txt or copy at // http://www.boost.org/LICENSE_1_0.txt) +#include #include "local_normal_delegate.hpp" #include "local_connection_delegate_factory.hpp" #include "../asio/local_resolve_op.hpp" @@ -17,9 +22,18 @@ #include "local_async_connection_base.hpp" // Needed by async_protocol_handler. #include +#include +#include +#include +#include +#include #include #include +#include +#include #include +#include +#include #include "../logging.h" @@ -28,7 +42,7 @@ namespace network { namespace http { namespace impl { -template +template struct http_async_connection; namespace placeholders = boost::asio::placeholders; @@ -41,8 +55,10 @@ template<> struct http_async_connection : async_connection_base, protected http_async_protocol_handler, - boost::enable_shared_from_this< + std::enable_shared_from_this< http_async_connection > { + http_async_connection(http_async_connection const&) = delete; + typedef async_connection_base base; typedef http_async_protocol_handler protocol_base; @@ -54,32 +70,32 @@ struct http_async_connection typedef typename base::string_type string_type; typedef typename base::request request; typedef typename base::resolver_base::resolve_function resolve_function; - typedef typename base::body_callback_function_type - body_callback_function_type; - typedef typename base::body_generator_function_type - body_generator_function_type; + typedef + typename base::body_callback_function_type body_callback_function_type; + typedef + typename base::body_generator_function_type body_generator_function_type; typedef http_async_connection this_type; typedef typename delegate_factory::type delegate_factory_type; typedef typename delegate_factory_type::connection_delegate_ptr connection_delegate_ptr; http_async_connection(resolver_type& resolver, resolve_function resolve, - bool follow_redirect, int timeout, + bool follow_redirect, int64_t timeout, + bool remove_chunk_markers, connection_delegate_ptr delegate) : timeout_(timeout), + remove_chunk_markers_(remove_chunk_markers), timer_(resolver.get_io_service()), is_timedout_(false), follow_redirect_(follow_redirect), resolver_(resolver), - resolve_(resolve), + resolve_(std::move(resolve)), request_strand_(resolver.get_io_service()), - delegate_(delegate) {} + delegate_(std::move(delegate)) {} - // This is the main entry point for the connection/request pipeline. - // We're - // overriding async_connection_base<...>::start(...) here which is - // called - // by the client. + // This is the main entry point for the connection/request pipeline. We're + // overriding async_connection_base<...>::start(...) here which is called by + // the client. virtual response start(request const& request, string_type const& method, bool get_body, body_callback_function_type callback, body_generator_function_type generator) { @@ -92,28 +108,35 @@ struct http_async_connection // << std::string(boost::asio::buffers_begin(command_streambuf.data()), // boost::asio::buffers_end(command_streambuf.data())); this->method = method; - boost::uint16_t port_ = port(request); - string_type host_ = host(request); - boost::uint16_t source_port = request.source_port(); + std::uint16_t port_ = port(request); + string_type host_ = host(request); + std::uint16_t source_port = request.source_port(); - resolve_(resolver_, host_, port_, - request_strand_.wrap(boost::bind( - &this_type::handle_resolved, this_type::shared_from_this(), - host_, port_, source_port, get_body, callback, - generator, boost::arg<1>(), boost::arg<2>()))); + auto self = this->shared_from_this(); if (timeout_ > 0) { - timer_.expires_from_now(boost::posix_time::seconds(timeout_)); - timer_.async_wait(request_strand_.wrap( - boost::bind(&this_type::handle_timeout, this_type::shared_from_this(), - boost::arg<1>()))); +#if defined(BOOST_ASIO_HAS_STD_CHRONO) + timer_.expires_from_now(std::chrono::seconds(timeout_)); +#elif defined(BOOST_ASIO_HAS_BOOST_CHRONO) + timer_.expires_from_now(boost::chrono::seconds(timeout_)); +#else +#error Need a chrono implementation +#endif + timer_.async_wait(request_strand_.wrap([=] (boost::system::error_code const &ec) { + self->handle_timeout(ec); + })); } + resolve_(resolver_, host_, port_, + request_strand_.wrap( + [=] (boost::system::error_code const &ec, + resolver_iterator_pair endpoint_range) { + self->handle_resolved(host_, port_, source_port, get_body, + callback, generator, ec, endpoint_range); + })); return response_; } private: - http_async_connection(http_async_connection const&); // = delete - - void set_errors(boost::system::error_code const& ec) { + void set_errors(boost::system::error_code const& ec, body_callback_function_type callback) { boost::system::system_error error(ec); this->version_promise.set_exception(boost::copy_exception(error)); this->status_promise.set_exception(boost::copy_exception(error)); @@ -122,6 +145,8 @@ struct http_async_connection this->source_promise.set_exception(boost::copy_exception(error)); this->destination_promise.set_exception(boost::copy_exception(error)); this->body_promise.set_exception(boost::copy_exception(error)); + if ( callback ) + callback( boost::iterator_range::type, 1024>::const_iterator>(), ec ); this->timer_.cancel(); } @@ -130,28 +155,27 @@ struct http_async_connection is_timedout_ = true; } - void handle_resolved(string_type host, boost::uint16_t port, boost::uint16_t source_port, bool get_body, + void handle_resolved(string_type host, std::uint16_t port, + std::uint16_t source_port, bool get_body, body_callback_function_type callback, body_generator_function_type generator, boost::system::error_code const& ec, resolver_iterator_pair endpoint_range) { if (!ec && !boost::empty(endpoint_range)) { - // Here we deal with the case that there was an error encountered - // and + // Here we deal with the case that there was an error encountered and // that there's still more endpoints to try connecting to. resolver_iterator iter = boost::begin(endpoint_range); resolver_type::endpoint_type endpoint(iter->endpoint().path()); + auto self = this->shared_from_this(); delegate_->connect( endpoint, - request_strand_.wrap(boost::bind( - &this_type::handle_connected, this_type::shared_from_this(), - port, get_body, callback, generator, - std::make_pair(++iter, resolver_iterator()), - placeholders::error))); + request_strand_.wrap([=] (boost::system::error_code const &ec) { + auto iter_copy = iter; + self->handle_connected(port, get_body, callback, + generator, std::make_pair(++iter_copy, resolver_iterator()), ec); + })); } else { - set_errors(ec ? ec : boost::asio::error::host_not_found); - boost::iterator_range range; - if (callback) callback(range, ec); + set_errors((ec ? ec : boost::asio::error::host_not_found), callback); } } @@ -161,80 +185,76 @@ struct http_async_connection resolver_iterator_pair endpoint_range, boost::system::error_code const& ec) { if (is_timedout_) { - set_errors(asio::error::timed_out); + set_errors(boost::asio::error::timed_out, callback); } else if (!ec) { BOOST_ASSERT(delegate_.get() != 0); + auto self = this->shared_from_this(); delegate_->write( command_streambuf, - request_strand_.wrap(boost::bind( - &this_type::handle_sent_request, this_type::shared_from_this(), - get_body, callback, generator, placeholders::error, - placeholders::bytes_transferred))); + request_strand_.wrap([=] (boost::system::error_code const &ec, + std::size_t bytes_transferred) { + self->handle_sent_request(get_body, callback, generator, + ec, bytes_transferred); + })); } else { if (!boost::empty(endpoint_range)) { resolver_iterator iter = boost::begin(endpoint_range); resolver_type::endpoint_type endpoint(iter->endpoint().path()); + auto self = this->shared_from_this(); delegate_->connect( endpoint, - request_strand_.wrap(boost::bind( - &this_type::handle_connected, this_type::shared_from_this(), - port, get_body, callback, generator, - std::make_pair(++iter, resolver_iterator()), - placeholders::error))); + request_strand_.wrap([=] (boost::system::error_code const &ec) { + auto iter_copy = iter; + self->handle_connected(port, get_body, callback, + generator, std::make_pair(++iter_copy, resolver_iterator()), + ec); + })); } else { - set_errors(ec ? ec : boost::asio::error::host_not_found); - boost::iterator_range range; - if (callback) callback(range, ec); + set_errors((ec ? ec : boost::asio::error::host_not_found), callback); } } } - enum state_t { - version, - status, - status_message, - headers, - body - }; + enum state_t { version, status, status_message, headers, body }; void handle_sent_request(bool get_body, body_callback_function_type callback, body_generator_function_type generator, boost::system::error_code const& ec, std::size_t bytes_transferred) { - // TODO(dberris): review parameter necessity. - (void)bytes_transferred; - if (!is_timedout_ && !ec) { if (generator) { - // Here we write some more data that the generator provides, - // before - // we wait for data from the server. + // Here we write some more data that the generator provides, before we + // wait for data from the server. string_type chunk; if (generator(chunk)) { - // At this point this means we have more data to write, so we - // write + // At this point this means we have more data to write, so we write // it out. std::copy(chunk.begin(), chunk.end(), std::ostreambuf_iterator::type>( &command_streambuf)); + auto self = this->shared_from_this(); delegate_->write( command_streambuf, - request_strand_.wrap(boost::bind( - &this_type::handle_sent_request, - this_type::shared_from_this(), get_body, callback, generator, - placeholders::error, placeholders::bytes_transferred))); + request_strand_.wrap([=] (boost::system::error_code const &ec, + std::size_t bytes_transferred) { + self->handle_sent_request(get_body, callback, generator, + ec, bytes_transferred); + })); return; } } + + auto self = this->shared_from_this(); delegate_->read_some( - boost::asio::mutable_buffers_1(this->part.c_array(), + boost::asio::mutable_buffers_1(this->part.data(), this->part.size()), - request_strand_.wrap(boost::bind( - &this_type::handle_received_data, this_type::shared_from_this(), - version, get_body, callback, placeholders::error, - placeholders::bytes_transferred))); + request_strand_.wrap([=] (boost::system::error_code const &ec, + std::size_t bytes_transferred) { + self->handle_received_data(version, get_body, callback, + ec, bytes_transferred); + })); } else { - set_errors(is_timedout_ ? boost::asio::error::timed_out : ec); + set_errors((is_timedout_ ? boost::asio::error::timed_out : ec), callback); } } @@ -245,7 +265,7 @@ struct http_async_connection static const long short_read_error = 335544539; bool is_ssl_short_read_error = #ifdef BOOST_NETWORK_ENABLE_HTTPS - ec.category() == asio::error::ssl_category && + ec.category() == boost::asio::error::ssl_category && ec.value() == short_read_error; #else false && short_read_error; @@ -254,177 +274,213 @@ struct http_async_connection (!ec || ec == boost::asio::error::eof || is_ssl_short_read_error)) { logic::tribool parsed_ok; size_t remainder; + auto self = this->shared_from_this(); switch (state) { case version: if (ec == boost::asio::error::eof) return; parsed_ok = this->parse_version( delegate_, - request_strand_.wrap(boost::bind( - &this_type::handle_received_data, - this_type::shared_from_this(), version, get_body, callback, - placeholders::error, placeholders::bytes_transferred)), + request_strand_.wrap([=] (boost::system::error_code const &ec, + std::size_t bytes_transferred) { + self->handle_received_data(version, get_body, callback, + ec, bytes_transferred); + }), bytes_transferred); - if (!parsed_ok || indeterminate(parsed_ok)) return; + if (!parsed_ok || indeterminate(parsed_ok)) { + return; + } case status: if (ec == boost::asio::error::eof) return; parsed_ok = this->parse_status( delegate_, - request_strand_.wrap(boost::bind( - &this_type::handle_received_data, - this_type::shared_from_this(), status, get_body, callback, - placeholders::error, placeholders::bytes_transferred)), + request_strand_.wrap([=] (boost::system::error_code const &ec, + std::size_t bytes_transferred) { + self->handle_received_data(status, get_body, callback, + ec, bytes_transferred); + }), bytes_transferred); - if (!parsed_ok || indeterminate(parsed_ok)) return; + if (!parsed_ok || indeterminate(parsed_ok)) { + return; + } case status_message: if (ec == boost::asio::error::eof) return; parsed_ok = this->parse_status_message( - delegate_, request_strand_.wrap(boost::bind( - &this_type::handle_received_data, - this_type::shared_from_this(), status_message, - get_body, callback, placeholders::error, - placeholders::bytes_transferred)), + delegate_, request_strand_.wrap([=] (boost::system::error_code const &, + std::size_t bytes_transferred) { + self->handle_received_data(status_message, get_body, callback, + ec, bytes_transferred); + }), bytes_transferred); - if (!parsed_ok || indeterminate(parsed_ok)) return; + if (!parsed_ok || indeterminate(parsed_ok)) { + return; + } case headers: if (ec == boost::asio::error::eof) return; - // In the following, remainder is the number of bytes that - // remain - // in the buffer. We need this in the body processing to make - // sure - // that the data remaining in the buffer is dealt with before - // another call to get more data for the body is scheduled. - fusion::tie(parsed_ok, remainder) = this->parse_headers( + // In the following, remainder is the number of bytes that remain in + // the buffer. We need this in the body processing to make sure that + // the data remaining in the buffer is dealt with before another call + // to get more data for the body is scheduled. + std::tie(parsed_ok, remainder) = this->parse_headers( delegate_, - request_strand_.wrap(boost::bind( - &this_type::handle_received_data, - this_type::shared_from_this(), headers, get_body, callback, - placeholders::error, placeholders::bytes_transferred)), + request_strand_.wrap([=] (boost::system::error_code const &ec, + std::size_t bytes_transferred) { + self->handle_received_data(headers, get_body, callback, + ec, bytes_transferred); + }), bytes_transferred); - if (!parsed_ok || indeterminate(parsed_ok)) return; + if (!parsed_ok || indeterminate(parsed_ok)) { + return; + } if (!get_body) { - // We short-circuit here because the user does not - // want to get the body (in the case of a HEAD - // request). + // We short-circuit here because the user does not want to get the + // body (in the case of a HEAD request). this->body_promise.set_value(""); + if ( callback ) + callback( boost::iterator_range::type, 1024>::const_iterator>(), boost::asio::error::eof ); this->destination_promise.set_value(""); this->source_promise.set_value(""); - this->part.assign('\0'); + // this->part.assign('\0'); + boost::copy("\0", std::begin(this->part)); this->response_parser_.reset(); return; } if (callback) { - // Here we deal with the spill-over data from the - // headers processing. This means the headers data - // has already been parsed appropriately and we're - // looking to treat everything that remains in the - // buffer. + // Here we deal with the spill-over data from the headers + // processing. This means the headers data has already been parsed + // appropriately and we're looking to treat everything that remains + // in the buffer. typename protocol_base::buffer_type::const_iterator begin = this->part_begin; typename protocol_base::buffer_type::const_iterator end = begin; std::advance(end, remainder); - // We're setting the body promise here to an empty string - // because + // We're setting the body promise here to an empty string because // this can be used as a signaling mechanism for the user to - // determine that the body is now ready for processing, even - // though the callback is already provided. + // determine that the body is now ready for processing, even though + // the callback is already provided. this->body_promise.set_value(""); - // The invocation of the callback is synchronous to allow us - // to + // The invocation of the callback is synchronous to allow us to // wait before scheduling another read. - callback(make_iterator_range(begin, end), ec); - + if (this->is_chunk_encoding && remove_chunk_markers_) { + callback(parse_chunk_encoding(make_iterator_range(begin, end)), ec); + } else { + callback(make_iterator_range(begin, end), ec); + } + auto self = this->shared_from_this(); delegate_->read_some( - boost::asio::mutable_buffers_1(this->part.c_array(), + boost::asio::mutable_buffers_1(this->part.data(), this->part.size()), - request_strand_.wrap(boost::bind( - &this_type::handle_received_data, - this_type::shared_from_this(), body, get_body, callback, - placeholders::error, placeholders::bytes_transferred))); + request_strand_.wrap([=] (boost::system::error_code const &ec, + std::size_t bytes_transferred) { + self->handle_received_data(body, get_body, callback, + ec, bytes_transferred); + })); } else { // Here we handle the body data ourself and append to an // ever-growing string buffer. + auto self = this->shared_from_this(); this->parse_body( delegate_, - request_strand_.wrap(boost::bind( - &this_type::handle_received_data, - this_type::shared_from_this(), body, get_body, callback, - placeholders::error, placeholders::bytes_transferred)), + request_strand_.wrap([=] (boost::system::error_code const &ec, + std::size_t bytes_transferred) { + self->handle_received_data(body, get_body, callback, + ec, bytes_transferred); + }), remainder); } return; case body: if (ec == boost::asio::error::eof || is_ssl_short_read_error) { - // Here we're handling the case when the connection has been - // closed from the server side, or at least that the end of - // file - // has been reached while reading the socket. This signals - // the end - // of the body processing chain. + // Here we're handling the case when the connection has been closed + // from the server side, or at least that the end of file has been + // reached while reading the socket. This signals the end of the + // body processing chain. if (callback) { typename protocol_base::buffer_type::const_iterator begin = this->part.begin(), end = begin; std::advance(end, bytes_transferred); - // We call the callback function synchronously passing the - // error - // condition (in this case, end of file) so that it can - // handle - // it appropriately. - callback(make_iterator_range(begin, end), ec); + // We call the callback function synchronously passing the error + // condition (in this case, end of file) so that it can handle it + // appropriately. + if (this->is_chunk_encoding && remove_chunk_markers_) { + callback(parse_chunk_encoding(make_iterator_range(begin, end)), ec); + } else { + callback(make_iterator_range(begin, end), ec); + } } else { string_type body_string; - std::swap(body_string, this->partial_parsed); - body_string.append(this->part.begin(), bytes_transferred); - if (this->is_chunk_encoding) - this->body_promise.set_value(parse_chunk_encoding(body_string)); - else + if (this->is_chunk_encoding && remove_chunk_markers_) { + for (size_t i = 0; i < this->partial_parsed.size(); i += 1024) { + auto range = parse_chunk_encoding(boost::make_iterator_range( + static_cast::type, 1024>::const_iterator>( + this->partial_parsed.data()) + i, + static_cast::type, 1024>::const_iterator>( + this->partial_parsed.data()) + + std::min(i + 1024, this->partial_parsed.size()))); + body_string.append(boost::begin(range), boost::end(range)); + } + this->partial_parsed.clear(); + auto range = parse_chunk_encoding(boost::make_iterator_range( + this->part.begin(), + this->part.begin() + bytes_transferred)); + body_string.append(boost::begin(range), boost::end(range)); + this->body_promise.set_value(body_string); + } else { + std::swap(body_string, this->partial_parsed); + body_string.append(this->part.begin(), + this->part.begin() + bytes_transferred); this->body_promise.set_value(body_string); + } } - // TODO set the destination value somewhere! + // TODO(dberris): set the destination value somewhere! this->destination_promise.set_value(""); this->source_promise.set_value(""); - this->part.assign('\0'); + // this->part.assign('\0'); + boost::copy("\0", std::begin(this->part)); this->response_parser_.reset(); this->timer_.cancel(); } else { - // This means the connection has not been closed yet and we - // want - // to get more - // data. + // This means the connection has not been closed yet and we want to + // get more data. if (callback) { // Here we have a body_handler callback. Let's invoke the - // callback from here and make sure we're getting more - // data - // right after. + // callback from here and make sure we're getting more data right + // after. typename protocol_base::buffer_type::const_iterator begin = this->part.begin(); typename protocol_base::buffer_type::const_iterator end = begin; std::advance(end, bytes_transferred); - callback(make_iterator_range(begin, end), ec); + if (this->is_chunk_encoding && remove_chunk_markers_) { + callback(parse_chunk_encoding(make_iterator_range(begin, end)), ec); + } else { + callback(make_iterator_range(begin, end), ec); + } + auto self = this->shared_from_this(); delegate_->read_some( - boost::asio::mutable_buffers_1(this->part.c_array(), + boost::asio::mutable_buffers_1(this->part.data(), this->part.size()), - request_strand_.wrap(boost::bind( - &this_type::handle_received_data, - this_type::shared_from_this(), body, get_body, callback, - placeholders::error, placeholders::bytes_transferred))); + request_strand_.wrap([=] (boost::system::error_code const &ec, + std::size_t bytes_transferred) { + self->handle_received_data(body, get_body, callback, + ec, bytes_transferred); + })); } else { - // Here we don't have a body callback. Let's - // make sure that we deal with the remainder - // from the headers part in case we do have data - // that's still in the buffer. + // Here we don't have a body callback. Let's make sure that we + // deal with the remainder from the headers part in case we do + // have data that's still in the buffer. this->parse_body( delegate_, - request_strand_.wrap(boost::bind( - &this_type::handle_received_data, - this_type::shared_from_this(), body, get_body, callback, - placeholders::error, placeholders::bytes_transferred)), + request_strand_.wrap([=] (boost::system::error_code const &ec, + std::size_t bytes_transferred) { + self->handle_received_data(body, get_body, callback, + ec, bytes_transferred); + }), bytes_transferred); } } @@ -433,27 +489,29 @@ struct http_async_connection BOOST_ASSERT(false && "Bug, report this to the developers!"); } } else { - boost::system::system_error error(is_timedout_ ? asio::error::timed_out - : ec); - this->source_promise.set_exception(boost::copy_exception(error)); - this->destination_promise.set_exception(boost::copy_exception(error)); + boost::system::error_code report_code = is_timedout_ ? boost::asio::error::timed_out : ec; + boost::system::system_error error(report_code); + this->source_promise.set_exception(std::make_exception_ptr(error)); + this->destination_promise.set_exception(std::make_exception_ptr(error)); switch (state) { case version: - this->version_promise.set_exception(boost::copy_exception(error)); + this->version_promise.set_exception(std::make_exception_ptr(error)); case status: - this->status_promise.set_exception(boost::copy_exception(error)); + this->status_promise.set_exception(std::make_exception_ptr(error)); case status_message: this->status_message_promise.set_exception( - boost::copy_exception(error)); + std::make_exception_ptr(error)); case headers: - this->headers_promise.set_exception(boost::copy_exception(error)); + this->headers_promise.set_exception(std::make_exception_ptr(error)); case body: if (!callback) { - // N.B. if callback is non-null, then body_promise has - // already been set to value "" to indicate body is - // handled by streaming handler so no exception should be set - this->body_promise.set_exception(boost::copy_exception(error)); + // N.B. if callback is non-null, then body_promise has already been + // set to value "" to indicate body is handled by streaming handler + // so no exception should be set + this->body_promise.set_exception(std::make_exception_ptr(error)); } + else + callback( boost::iterator_range::type, 1024>::const_iterator>(), report_code ); break; default: BOOST_ASSERT(false && "Bug, report this to the developers!"); @@ -461,35 +519,9 @@ struct http_async_connection } } - string_type parse_chunk_encoding(string_type& body_string) { - string_type body; - string_type crlf = "\r\n"; - - typename string_type::iterator begin = body_string.begin(); - for (typename string_type::iterator iter = - std::search(begin, body_string.end(), crlf.begin(), crlf.end()); - iter != body_string.end(); - iter = - std::search(begin, body_string.end(), crlf.begin(), crlf.end())) { - string_type line(begin, iter); - if (line.empty()) break; - std::stringstream stream(line); - int len; - stream >> std::hex >> len; - std::advance(iter, 2); - if (!len) break; - if (len <= body_string.end() - iter) { - body.insert(body.end(), iter, iter + len); - std::advance(iter, len + 2); - } - begin = iter; - } - - return body; - } - - int timeout_; - boost::asio::deadline_timer timer_; + int64_t timeout_; + bool remove_chunk_markers_; + boost::asio::steady_timer timer_; bool is_timedout_; bool follow_redirect_; resolver_type& resolver_; @@ -498,6 +530,7 @@ struct http_async_connection connection_delegate_ptr delegate_; boost::asio::streambuf command_streambuf; string_type method; + chunk_encoding_parser parse_chunk_encoding; }; #undef version_minor diff --git a/src/http/local_connection_delegate.hpp b/src/http/local_connection_delegate.hpp index 7b6ed094..76e9d444 100644 --- a/src/http/local_connection_delegate.hpp +++ b/src/http/local_connection_delegate.hpp @@ -1,6 +1,10 @@ #ifndef BOOST_NETWORK_PROTOCOL_HTTP_CLIENT_CONNECTION_LOCAL_CONNECTION_DELEGATE_HPP_ #define BOOST_NETWORK_PROTOCOL_HTTP_CLIENT_CONNECTION_LOCAL_CONNECTION_DELEGATE_HPP_ +// Note: This file is mostly a copy of +// boost/network/protocol/http/client/connection/connection_delegate.hpp, +// except for the specialized bits. + // Copyright 2011 Dean Michael Berris (dberris@google.com). // Copyright 2017 Igor Peshansky (igorp@google.com). // Copyright 2011-2017 Google, Inc. @@ -9,8 +13,8 @@ // http://www.boost.org/LICENSE_1_0.txt) #include - #include +#include namespace boost { namespace network { @@ -18,26 +22,23 @@ namespace http { namespace impl { struct local_connection_delegate { - // TODO: this is similar enough to connection_delegate that it may be possible to refactor. + // TODO(igorp): this is similar enough to connection_delegate that it may be possible to refactor. virtual void connect(boost::asio::local::stream_protocol::endpoint &endpoint, - std::function handler) = 0; + std::function handler) = 0; virtual void write( - asio::streambuf &command_streambuf, - std::function handler) = 0; + boost::asio::streambuf &command_streambuf, + std::function handler) = 0; virtual void read_some( - asio::mutable_buffers_1 const &read_buffer, - std::function handler) = 0; + boost::asio::mutable_buffers_1 const &read_buffer, + std::function handler) = 0; virtual void disconnect() = 0; - virtual ~local_connection_delegate() {} + virtual ~local_connection_delegate() = default; }; -} /* impl */ - -} /* http */ - -} /* network */ - -} /* boost */ +} // namespace impl +} // namespace http +} // namespace network +} // namespace boost #endif /* BOOST_NETWORK_PROTOCOL_HTTP_CLIENT_CONNECTION_LOCAL_CONNECTION_DELEGATE_HPP_ \ */ diff --git a/src/http/local_connection_delegate_factory.hpp b/src/http/local_connection_delegate_factory.hpp index f1473b44..5f2e60a1 100644 --- a/src/http/local_connection_delegate_factory.hpp +++ b/src/http/local_connection_delegate_factory.hpp @@ -1,6 +1,10 @@ #ifndef BOOST_NETWORK_PROTOCOL_HTTP_CLIENT_LOCAL_CONNECTION_DELEGATE_FACTORY_HPP_20170307 #define BOOST_NETWORK_PROTOCOL_HTTP_CLIENT_LOCAL_CONNECTION_DELEGATE_FACTORY_HPP_20170307 +// Note: This file is mostly a copy of +// boost/network/protocol/http/client/connection/connection_delegate_factory.hpp, +// except for the specialized bits. + // Copyright 2011 Dean Michael Berris (dberris@google.com). // Copyright 2017 Igor Peshansky (igorp@google.com). // Copyright 2011-2017 Google, Inc. @@ -8,8 +12,8 @@ // (See accompanying file LICENSE_1_0.txt or copy at // http://www.boost.org/LICENSE_1_0.txt) -#include #include +#include #include "local_connection_delegate.hpp" #include "local_normal_delegate.hpp" #include "local_tags.hpp" @@ -26,29 +30,28 @@ struct local_normal_delegate; template<> struct connection_delegate_factory { typedef std::shared_ptr connection_delegate_ptr; - typedef string::type string_type; + typedef typename string::type string_type; - // This is the factory method that actually returns the delegate - // instance. - // TODO Support passing in proxy settings when crafting connections. + // This is the factory method that actually returns the delegate instance. + // TODO(dberris): Support passing in proxy settings when crafting connections. static connection_delegate_ptr new_connection_delegate( boost::asio::io_service& service, bool https, bool always_verify_peer, optional certificate_filename, optional verify_path, optional certificate_file, optional private_key_file, optional ciphers, - long ssl_options) { + optional sni_hostname, long ssl_options) { connection_delegate_ptr delegate; - delegate.reset(new local_normal_delegate(service)); + delegate = std::make_shared(service); return delegate; } }; #undef Tag -} /* impl */ -} /* http */ -} /* network */ -} /* boost */ +} // namespace impl +} // namespace http +} // namespace network +} // namespace boost #endif /* BOOST_NETWORK_PROTOCOL_HTTP_CLIENT_LOCAL_CONNECTION_DELEGATE_FACTORY_HPP_20170307 \ */ diff --git a/src/http/local_normal_delegate.hpp b/src/http/local_normal_delegate.hpp index 6984e7ea..63c3efb3 100644 --- a/src/http/local_normal_delegate.hpp +++ b/src/http/local_normal_delegate.hpp @@ -1,6 +1,10 @@ #ifndef BOOST_NETWORK_PROTOCOL_HTTP_CLIENT_CONNECTION_LOCAL_NORMAL_DELEGATE_20170307 #define BOOST_NETWORK_PROTOCOL_HTTP_CLIENT_CONNECTION_LOCAL_NORMAL_DELEGATE_20170307 +// Note: This file is mostly a copy of +// boost/network/protocol/http/client/connection/normal_delegate.hpp, +// except for the specialized bits. + // Copyright 2011 Dean Michael Berris (dberris@google.com). // Copyright 2017 Igor Peshansky (igorp@google.com). // Copyright 2011-2017 Google, Inc. @@ -27,16 +31,14 @@ struct local_normal_delegate : local_connection_delegate { void connect(boost::asio::local::stream_protocol::endpoint &endpoint, std::function handler) override; - void write( - boost::asio::streambuf &command_streambuf, - std::function handler) + void write(boost::asio::streambuf &command_streambuf, + std::function handler) override; - void read_some( - boost::asio::mutable_buffers_1 const &read_buffer, - std::function handler) + void read_some(boost::asio::mutable_buffers_1 const &read_buffer, + std::function handler) override; void disconnect() override; - ~local_normal_delegate() override; + ~local_normal_delegate() override = default; local_normal_delegate(local_normal_delegate const &) = delete; local_normal_delegate &operator=(local_normal_delegate) = delete; @@ -46,17 +48,13 @@ struct local_normal_delegate : local_connection_delegate { std::unique_ptr socket_; }; -} /* impl */ - -} /* http */ - -} /* network */ - -} /* boost */ +} // namespace impl +} // namespace http +} // namespace network +} // namespace boost #ifdef BOOST_NETWORK_NO_LIB #include "local_normal_delegate.ipp" #endif /* BOOST_NETWORK_NO_LIB */ -#endif /* BOOST_NETWORK_PROTOCOL_HTTP_CLIENT_CONNECTION_LOCAL_NORMAL_DELEGATE_20110819 \ - */ +#endif // BOOST_NETWORK_PROTOCOL_HTTP_CLIENT_CONNECTION_LOCAL_NORMAL_DELEGATE_20110819 diff --git a/src/http/local_normal_delegate.ipp b/src/http/local_normal_delegate.ipp index b9a07944..ab663937 100644 --- a/src/http/local_normal_delegate.ipp +++ b/src/http/local_normal_delegate.ipp @@ -12,6 +12,7 @@ #include #include #include +#include #include "local_normal_delegate.hpp" @@ -42,15 +43,11 @@ void boost::network::http::impl::local_normal_delegate::read_some( void boost::network::http::impl::local_normal_delegate::disconnect() { if (socket_.get() && socket_->is_open()) { boost::system::error_code ignored; - socket_->shutdown( - boost::asio::local::stream_protocol::socket::shutdown_both, ignored); + socket_->shutdown(boost::asio::local::stream_protocol::socket::shutdown_both, ignored); if (!ignored) { socket_->close(ignored); } } } -boost::network::http::impl::local_normal_delegate::~local_normal_delegate() {} - -#endif /* BOOST_NETWORK_PROTOCOL_HTTP_CLIENT_CONNECTION_LOCAL_NORMAL_DELEGATE_IPP_20170307 \ - */ +#endif // BOOST_NETWORK_PROTOCOL_HTTP_CLIENT_CONNECTION_LOCAL_NORMAL_DELEGATE_IPP_20170307 diff --git a/src/kubernetes.cc b/src/kubernetes.cc index dab6485e..b8d340a4 100644 --- a/src/kubernetes.cc +++ b/src/kubernetes.cc @@ -547,7 +547,8 @@ json::value KubernetesReader::QueryMaster(const std::string& path) const throw(QueryException, json::Exception) { const std::string endpoint(config_.KubernetesEndpointHost() + path); http::client client( - http::client::options().openssl_certificate(SecretPath("ca.crt"))); + http::client::options() + .openssl_certificate(SecretPath("ca.crt"))); http::client::request request(endpoint); request << boost::network::header( "Authorization", "Bearer " + KubernetesApiToken()); @@ -809,7 +810,9 @@ void KubernetesReader::WatchMaster( const std::string endpoint( config_.KubernetesEndpointHost() + path + watch_param); http::client client( - http::client::options().openssl_certificate(SecretPath("ca.crt"))); + http::client::options() + .remove_chunk_markers(false) + .openssl_certificate(SecretPath("ca.crt"))); http::client::request request(endpoint); request << boost::network::header( "Authorization", "Bearer " + KubernetesApiToken()); @@ -830,7 +833,7 @@ void KubernetesReader::WatchMaster( endpoint, std::bind(&BodyCallback, name, event_callback, std::placeholders::_1), std::move(watch_completion), config_.VerboseLogging()); - http::client::response response = client.get(request, boost::ref(watcher)); + http::client::response response = client.get(request, std::ref(watcher)); if (config_.VerboseLogging()) { LOG(INFO) << "Waiting for completion"; } diff --git a/src/oauth2.cc b/src/oauth2.cc index 6de64e6a..ccd5df37 100644 --- a/src/oauth2.cc +++ b/src/oauth2.cc @@ -18,7 +18,7 @@ #define BOOST_NETWORK_ENABLE_HTTPS #include -#include +#include #include #include #include @@ -187,8 +187,8 @@ json::value OAuth2::ComputeTokenFromCredentials() const { } http::client client; http::client::request request("https://www.googleapis.com/oauth2/v3/token"); - std::string grant_type = boost::network::uri::encoded( - "urn:ietf:params:oauth:grant-type:jwt-bearer"); + std::string grant_type = ::network::detail::encode_fragment( + std::string("urn:ietf:params:oauth:grant-type:jwt-bearer")); //std::string jwt_header = "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9"; //std::string jwt_header = // base64::Encode("{\"alg\":\"RS256\",\"typ\":\"JWT\"}");