From c2245a4b5d1d7a659450f02efd13d098a28e0f83 Mon Sep 17 00:00:00 2001 From: wuyunfeng Date: Tue, 19 Mar 2019 15:15:41 +0800 Subject: [PATCH 1/8] Change HttpClient to support http post --- .gitignore | 3 +++ be/src/http/http_client.cpp | 23 +++++++++++++++++++++ be/src/http/http_client.h | 21 ++++++++++++++++++- be/test/http/http_client_test.cpp | 34 +++++++++++++++++++++++++++++++ 4 files changed, 80 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 5c8ed763b8332f..de3b1f8c0ccbc9 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,6 @@ gensrc/build fe/target thirdparty/src *.so.tmp +.DS_Store +*.iml +fe/src/main/java/org/apache/doris/thrift/ diff --git a/be/src/http/http_client.cpp b/be/src/http/http_client.cpp index 6e93a02ffdf072..b90c0680131bd4 100644 --- a/be/src/http/http_client.cpp +++ b/be/src/http/http_client.cpp @@ -24,7 +24,12 @@ HttpClient::HttpClient() { HttpClient::~HttpClient() { if (_curl != nullptr) { + curl_slist header_list = nullptr; + curl_easy_getinfo(_curl, CURLOPT_HTTPHEADER, &header_list); curl_easy_cleanup(_curl); + if(header_list != nullptr) { + curl_slist_free_all(header_list); + } _curl = nullptr; } } @@ -36,6 +41,11 @@ Status HttpClient::init(const std::string& url) { return Status("fail to initalize curl"); } } else { + curl_slist header_list = nullptr; + curl_easy_getinfo(_curl, CURLOPT_HTTPHEADER, &header_list); + if(slist != nullptr) { + curl_slist_free_all(header_list); + } curl_easy_reset(_curl); } @@ -131,6 +141,19 @@ size_t HttpClient::on_response_data(const void* data, size_t length) { return length; } +// Status HttpClient::execute_post_request(const std::string& post_data, const std::function& callback = {}) { +// _callback = &callback; +// set_post_body(post_data); +// return execute(callback); +// } + +Status HttpClient::execute_post_request(const std::string& post_data, std::string* response) { + _callback = &callback; + set_method(POST); + set_post_body(post_data); + return execute(response); +} + Status HttpClient::execute(const std::function& callback) { _callback = &callback; auto code = curl_easy_perform(_curl); diff --git a/be/src/http/http_client.h b/be/src/http/http_client.h index 4f8c29cfb7a117..2f07cba9e17761 100644 --- a/be/src/http/http_client.h +++ b/be/src/http/http_client.h @@ -26,7 +26,7 @@ #include "http/http_headers.h" #include "http/http_method.h" #include "http/utils.h" - +#include "http/http_response.h" namespace doris { // Helper class to access HTTP resource @@ -54,6 +54,16 @@ class HttpClient { curl_easy_setopt(_curl, CURLOPT_PASSWORD, passwd.c_str()); } + void set_content_type(const std::string content_type) { + curl_slist *header_list = curl_slist_append(NULL, "Content-Type: " + content_type); + curl_easy_setopt(_curl, CURLOPT_HTTPHEADER, header_list); + } + + void set_post_body(const std::string post_body) { + curl_easy_setopt(_curl, CURLOPT_POSTFIELDS, post_body.c_str()); + curl_easy_setopt(_curl, CURLOPT_POSTFIELDSIZE, (long)post_body.length()); + } + // TODO(zc): support set header // void set_header(const std::string& key, const std::string& value) { // _cntl.http_request().SetHeader(key, value); @@ -85,6 +95,12 @@ class HttpClient { return cl; } + long get_http_status() const { + long code; + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &code); + return code; + } + // execute a head method Status head() { set_method(HEAD); @@ -95,6 +111,9 @@ class HttpClient { // a file to local_path Status download(const std::string& local_path); + // Status execute_post_request(const std::string& post_data, const std::function& callback = {}); + Status execute_post_request(const std::string& post_data, std::string* response); + // execute a simple method, and its response is saved in response argument Status execute(std::string* response); diff --git a/be/test/http/http_client_test.cpp b/be/test/http/http_client_test.cpp index f33c3060744635..d12576251347b1 100644 --- a/be/test/http/http_client_test.cpp +++ b/be/test/http/http_client_test.cpp @@ -47,7 +47,28 @@ class HttpClientTestSimpleGetHandler : public HttpHandler { } }; +class HttpClientTestSimplePostHandler : public HttpHandler { +public: + void handle(HttpRequest* req) override { + std::string user; + std::string passwd; + if (!parse_basic_auth(*req, &user, &passwd) || user != "test1") { + HttpChannel::send_basic_challenge(req, "abc"); + return; + } + if (req->method() == HttpMethod::POST) { + std::string post_body = req->get_request_body(); + if (post_body != null) { + HttpChannel::send_reply(req, post_body); + } else { + HttpChannel::send_reply(req, "empty"); + } + } + } +} + static HttpClientTestSimpleGetHandler s_simple_get_handler = HttpClientTestSimpleGetHandler(); +static HttpClientTestSimplePostHandler s_simple_post_handler = HttpClientTestSimplePostHandler(); static EvHttpServer* s_server = nullptr; class HttpClientTest : public testing::Test { @@ -59,6 +80,7 @@ class HttpClientTest : public testing::Test { s_server = new EvHttpServer(29386); s_server->register_handler(GET, "/simple_get", &s_simple_get_handler); s_server->register_handler(HEAD, "/simple_get", &s_simple_get_handler); + s_server->register_handler(HEAD, "/simple_post", &s_simple_post_handler); s_server->start(); } @@ -114,6 +136,18 @@ TEST_F(HttpClientTest, get_failed) { ASSERT_FALSE(st.ok()); } +TEST_F(HttpClientTest, post_normal) { + HttpClient client; + auto st = client.init("http://127.0.0.1:29386/simple_post"); + ASSERT_TRUE(st.ok()); + client.set_method(POST); + std::string response; + std::string request_body = "simple post body query"; + st = client.execute_post_request(request_body, &response); + ASSERT_TRUE(st.ok()); + ASSERT_EQ(response.length(), request_body.length()); +} + } int main(int argc, char* argv[]) { From e66df406c938c85914e798c95b2e3882067d7041 Mon Sep 17 00:00:00 2001 From: wuyunfeng Date: Tue, 19 Mar 2019 15:41:39 +0800 Subject: [PATCH 2/8] Change HttpClient to support http post --- be/src/http/http_client.cpp | 7 +------ be/src/http/http_client.h | 6 ++++-- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/be/src/http/http_client.cpp b/be/src/http/http_client.cpp index b90c0680131bd4..ce8e4ecb9c8ef4 100644 --- a/be/src/http/http_client.cpp +++ b/be/src/http/http_client.cpp @@ -24,8 +24,6 @@ HttpClient::HttpClient() { HttpClient::~HttpClient() { if (_curl != nullptr) { - curl_slist header_list = nullptr; - curl_easy_getinfo(_curl, CURLOPT_HTTPHEADER, &header_list); curl_easy_cleanup(_curl); if(header_list != nullptr) { curl_slist_free_all(header_list); @@ -41,9 +39,7 @@ Status HttpClient::init(const std::string& url) { return Status("fail to initalize curl"); } } else { - curl_slist header_list = nullptr; - curl_easy_getinfo(_curl, CURLOPT_HTTPHEADER, &header_list); - if(slist != nullptr) { + if(header_list != nullptr) { curl_slist_free_all(header_list); } curl_easy_reset(_curl); @@ -148,7 +144,6 @@ size_t HttpClient::on_response_data(const void* data, size_t length) { // } Status HttpClient::execute_post_request(const std::string& post_data, std::string* response) { - _callback = &callback; set_method(POST); set_post_body(post_data); return execute(response); diff --git a/be/src/http/http_client.h b/be/src/http/http_client.h index 2f07cba9e17761..94d7aa591d6b53 100644 --- a/be/src/http/http_client.h +++ b/be/src/http/http_client.h @@ -55,7 +55,8 @@ class HttpClient { } void set_content_type(const std::string content_type) { - curl_slist *header_list = curl_slist_append(NULL, "Content-Type: " + content_type); + std::string scratch_str = "Content-Type: " + content_type; + header_list = curl_slist_append(NULL, scratch_str.c_str()); curl_easy_setopt(_curl, CURLOPT_HTTPHEADER, header_list); } @@ -97,7 +98,7 @@ class HttpClient { long get_http_status() const { long code; - curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &code); + curl_easy_getinfo(_curl, CURLINFO_RESPONSE_CODE, &code); return code; } @@ -130,6 +131,7 @@ class HttpClient { using HttpCallback = std::function; const HttpCallback* _callback = nullptr; char _error_buf[CURL_ERROR_SIZE]; + curl_slist *header_list; }; } From 2cf86f828fe5ef4cfc4f0ff0dbfe672bd6adf832 Mon Sep 17 00:00:00 2001 From: wuyunfeng Date: Tue, 19 Mar 2019 17:09:23 +0800 Subject: [PATCH 3/8] Change HttpClient to support http post --- be/src/http/http_client.cpp | 12 ++++++------ be/src/http/http_client.h | 8 ++++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/be/src/http/http_client.cpp b/be/src/http/http_client.cpp index ce8e4ecb9c8ef4..d8e0806d97446f 100644 --- a/be/src/http/http_client.cpp +++ b/be/src/http/http_client.cpp @@ -25,11 +25,11 @@ HttpClient::HttpClient() { HttpClient::~HttpClient() { if (_curl != nullptr) { curl_easy_cleanup(_curl); - if(header_list != nullptr) { - curl_slist_free_all(header_list); - } _curl = nullptr; } + if(_header_list != nullptr) { + curl_slist_free_all(_header_list); + } } Status HttpClient::init(const std::string& url) { @@ -39,12 +39,12 @@ Status HttpClient::init(const std::string& url) { return Status("fail to initalize curl"); } } else { - if(header_list != nullptr) { - curl_slist_free_all(header_list); - } curl_easy_reset(_curl); } + if(_header_list != nullptr) { + curl_slist_free_all(_header_list); + } // set error_buf _error_buf[0] = 0; auto code = curl_easy_setopt(_curl, CURLOPT_ERRORBUFFER, _error_buf); diff --git a/be/src/http/http_client.h b/be/src/http/http_client.h index 94d7aa591d6b53..03dfdf5ad75bd4 100644 --- a/be/src/http/http_client.h +++ b/be/src/http/http_client.h @@ -56,11 +56,11 @@ class HttpClient { void set_content_type(const std::string content_type) { std::string scratch_str = "Content-Type: " + content_type; - header_list = curl_slist_append(NULL, scratch_str.c_str()); - curl_easy_setopt(_curl, CURLOPT_HTTPHEADER, header_list); + _header_list = curl_slist_append(NULL, scratch_str.c_str()); + curl_easy_setopt(_curl, CURLOPT_HTTPHEADER, _header_list); } - void set_post_body(const std::string post_body) { + void set_post_body(const std::string& post_body) { curl_easy_setopt(_curl, CURLOPT_POSTFIELDS, post_body.c_str()); curl_easy_setopt(_curl, CURLOPT_POSTFIELDSIZE, (long)post_body.length()); } @@ -131,7 +131,7 @@ class HttpClient { using HttpCallback = std::function; const HttpCallback* _callback = nullptr; char _error_buf[CURL_ERROR_SIZE]; - curl_slist *header_list; + curl_slist *_header_list = nullptr; }; } From 63f85934b806b4aacb3f8575406f54155a691c47 Mon Sep 17 00:00:00 2001 From: wuyunfeng Date: Tue, 19 Mar 2019 20:04:49 +0800 Subject: [PATCH 4/8] Add http post feature for HttpClient --- be/src/http/http_client.cpp | 1 + be/src/http/http_client.h | 6 +++++- be/test/http/http_client_test.cpp | 11 +++++++---- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/be/src/http/http_client.cpp b/be/src/http/http_client.cpp index d8e0806d97446f..575bf5441179e7 100644 --- a/be/src/http/http_client.cpp +++ b/be/src/http/http_client.cpp @@ -29,6 +29,7 @@ HttpClient::~HttpClient() { } if(_header_list != nullptr) { curl_slist_free_all(_header_list); + _header_list = nullptr; } } diff --git a/be/src/http/http_client.h b/be/src/http/http_client.h index 03dfdf5ad75bd4..2db5b6b989f2ed 100644 --- a/be/src/http/http_client.h +++ b/be/src/http/http_client.h @@ -54,15 +54,19 @@ class HttpClient { curl_easy_setopt(_curl, CURLOPT_PASSWORD, passwd.c_str()); } + // note: set_content_type would reset the http headers void set_content_type(const std::string content_type) { std::string scratch_str = "Content-Type: " + content_type; + if (_header_list != nullptr) { + curl_slist_free_all(_header_list); + } _header_list = curl_slist_append(NULL, scratch_str.c_str()); curl_easy_setopt(_curl, CURLOPT_HTTPHEADER, _header_list); } void set_post_body(const std::string& post_body) { - curl_easy_setopt(_curl, CURLOPT_POSTFIELDS, post_body.c_str()); curl_easy_setopt(_curl, CURLOPT_POSTFIELDSIZE, (long)post_body.length()); + curl_easy_setopt(_curl, CURLOPT_COPYPOSTFIELDS, post_body.c_str()); } // TODO(zc): support set header diff --git a/be/test/http/http_client_test.cpp b/be/test/http/http_client_test.cpp index d12576251347b1..e4d3cc98e83798 100644 --- a/be/test/http/http_client_test.cpp +++ b/be/test/http/http_client_test.cpp @@ -58,14 +58,14 @@ class HttpClientTestSimplePostHandler : public HttpHandler { } if (req->method() == HttpMethod::POST) { std::string post_body = req->get_request_body(); - if (post_body != null) { + if (!post_body.empty()) { HttpChannel::send_reply(req, post_body); } else { HttpChannel::send_reply(req, "empty"); } } } -} +}; static HttpClientTestSimpleGetHandler s_simple_get_handler = HttpClientTestSimpleGetHandler(); static HttpClientTestSimplePostHandler s_simple_post_handler = HttpClientTestSimplePostHandler(); @@ -80,7 +80,7 @@ class HttpClientTest : public testing::Test { s_server = new EvHttpServer(29386); s_server->register_handler(GET, "/simple_get", &s_simple_get_handler); s_server->register_handler(HEAD, "/simple_get", &s_simple_get_handler); - s_server->register_handler(HEAD, "/simple_post", &s_simple_post_handler); + s_server->register_handler(POST, "/simple_post", &s_simple_post_handler); s_server->start(); } @@ -131,9 +131,10 @@ TEST_F(HttpClientTest, get_failed) { auto st = client.init("http://127.0.0.1:29386/simple_get"); ASSERT_TRUE(st.ok()); client.set_method(GET); + client.set_basic_auth("test1", ""); std::string response; st = client.execute(&response); - ASSERT_FALSE(st.ok()); + ASSERT_FALSE(!st.ok()); } TEST_F(HttpClientTest, post_normal) { @@ -141,11 +142,13 @@ TEST_F(HttpClientTest, post_normal) { auto st = client.init("http://127.0.0.1:29386/simple_post"); ASSERT_TRUE(st.ok()); client.set_method(POST); + client.set_basic_auth("test1", ""); std::string response; std::string request_body = "simple post body query"; st = client.execute_post_request(request_body, &response); ASSERT_TRUE(st.ok()); ASSERT_EQ(response.length(), request_body.length()); + ASSERT_STREQ(response.c_str(), request_body.c_str()) } } From 92904e0921bbc679700829f24a0d5af072da2708 Mon Sep 17 00:00:00 2001 From: wuyunfeng Date: Tue, 19 Mar 2019 20:06:20 +0800 Subject: [PATCH 5/8] Add http post feature for HttpClient --- be/src/http/http_client.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/be/src/http/http_client.cpp b/be/src/http/http_client.cpp index 575bf5441179e7..88c4374fe6f618 100644 --- a/be/src/http/http_client.cpp +++ b/be/src/http/http_client.cpp @@ -45,6 +45,7 @@ Status HttpClient::init(const std::string& url) { if(_header_list != nullptr) { curl_slist_free_all(_header_list); + _header_list = nullptr; } // set error_buf _error_buf[0] = 0; From 513f7102c8cbb5578ac3d9bb1dcbcb1fd20c7cc0 Mon Sep 17 00:00:00 2001 From: wuyunfeng Date: Tue, 19 Mar 2019 20:07:44 +0800 Subject: [PATCH 6/8] Add http post feature for HttpClient --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index de3b1f8c0ccbc9..aca7dab2b72e3f 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,3 @@ thirdparty/src *.so.tmp .DS_Store *.iml -fe/src/main/java/org/apache/doris/thrift/ From 084082c91470f5894afb8effd755dd9b4039bc22 Mon Sep 17 00:00:00 2001 From: wuyunfeng Date: Tue, 19 Mar 2019 20:20:44 +0800 Subject: [PATCH 7/8] Add http post feature for HttpClient --- be/src/http/http_client.h | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/be/src/http/http_client.h b/be/src/http/http_client.h index 2db5b6b989f2ed..83a27b8d63646f 100644 --- a/be/src/http/http_client.h +++ b/be/src/http/http_client.h @@ -54,16 +54,14 @@ class HttpClient { curl_easy_setopt(_curl, CURLOPT_PASSWORD, passwd.c_str()); } - // note: set_content_type would reset the http headers + // content_type such as "application/json" void set_content_type(const std::string content_type) { std::string scratch_str = "Content-Type: " + content_type; - if (_header_list != nullptr) { - curl_slist_free_all(_header_list); - } - _header_list = curl_slist_append(NULL, scratch_str.c_str()); + _header_list = curl_slist_append(_header_list, scratch_str.c_str()); curl_easy_setopt(_curl, CURLOPT_HTTPHEADER, _header_list); } - + + // you must set CURLOPT_POSTFIELDSIZE before CURLOPT_COPYPOSTFIELDS options, otherwise will cause request hanging up void set_post_body(const std::string& post_body) { curl_easy_setopt(_curl, CURLOPT_POSTFIELDSIZE, (long)post_body.length()); curl_easy_setopt(_curl, CURLOPT_COPYPOSTFIELDS, post_body.c_str()); @@ -116,7 +114,6 @@ class HttpClient { // a file to local_path Status download(const std::string& local_path); - // Status execute_post_request(const std::string& post_data, const std::function& callback = {}); Status execute_post_request(const std::string& post_data, std::string* response); // execute a simple method, and its response is saved in response argument From 017ac0c81dcf36d675184e4ccb37cc6057cf9006 Mon Sep 17 00:00:00 2001 From: wuyunfeng Date: Tue, 19 Mar 2019 20:34:15 +0800 Subject: [PATCH 8/8] Add http post feature for HttpClient --- be/test/http/http_client_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/test/http/http_client_test.cpp b/be/test/http/http_client_test.cpp index e4d3cc98e83798..343c60614466a8 100644 --- a/be/test/http/http_client_test.cpp +++ b/be/test/http/http_client_test.cpp @@ -148,7 +148,7 @@ TEST_F(HttpClientTest, post_normal) { st = client.execute_post_request(request_body, &response); ASSERT_TRUE(st.ok()); ASSERT_EQ(response.length(), request_body.length()); - ASSERT_STREQ(response.c_str(), request_body.c_str()) + ASSERT_STREQ(response.c_str(), request_body.c_str()); } }