From 4655c2f689735c879e5ab103bcce5f9e564e3e38 Mon Sep 17 00:00:00 2001 From: Roman Geraskin Date: Tue, 28 Apr 2020 12:19:19 +0000 Subject: [PATCH 1/5] json_array switch to use JSON array data format Signed-off-by: Roman Geraskin --- lib/fluent/plugin/out_http.rb | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/lib/fluent/plugin/out_http.rb b/lib/fluent/plugin/out_http.rb index 431c1382d9..9d46571b76 100644 --- a/lib/fluent/plugin/out_http.rb +++ b/lib/fluent/plugin/out_http.rb @@ -37,6 +37,8 @@ class RetryableResponse < StandardError; end config_param :proxy, :string, default: ENV['HTTP_PROXY'] || ENV['http_proxy'] desc 'Content-Type for HTTP request' config_param :content_type, :string, default: nil + desc 'JSON array data format for HTTP request body' + config_param :json_array, :bool, default: false desc 'Additional headers for HTTP request' config_param :headers, :hash, default: nil @@ -100,6 +102,13 @@ def configure(conf) @proxy_uri = URI.parse(@proxy) if @proxy @formatter = formatter_create @content_type = setup_content_type unless @content_type + + if @json_array + if @formatter_configs.first[:@type] != "json" + raise Fluent::ConfigError, "json_array option could be used with json formatter only" + end + define_singleton_method(:format, method(:format_json_array)) + end end def multi_workers_ready? @@ -114,6 +123,10 @@ def format(tag, time, record) @formatter.format(tag, time, record) end + def format_json_array(tag, time, record) + @formatter.format(tag, time, record).strip() << "," + end + def write(chunk) uri = parse_endpoint(chunk) req = create_request(chunk, uri) @@ -128,7 +141,7 @@ def write(chunk) def setup_content_type case @formatter_configs.first[:@type] when 'json' - 'application/x-ndjson' + @json_array ? 'application/json' : 'application/x-ndjson' when 'csv' 'text/csv' when 'tsv', 'ltsv' @@ -202,7 +215,7 @@ def create_request(chunk, uri) req.basic_auth(@auth.username, @auth.password) end set_headers(req) - req.body = chunk.read + req.body = @json_array ? "[#{chunk.read[0...-1]}]" : chunk.read req end @@ -234,4 +247,4 @@ def send_request(uri, req) end end end -end +end \ No newline at end of file From 3ad87972c92e5d4fc36a4d3b4b7d2a34c8f2a359 Mon Sep 17 00:00:00 2001 From: Roman Geraskin Date: Sat, 2 May 2020 07:00:08 +0000 Subject: [PATCH 2/5] strip() is removed from format_json_array strip call has overhead. Setting add_newline false to json formatter in configure is enough to remove \n from json formatter result. Signed-off-by: Roman Geraskin --- lib/fluent/plugin/out_http.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/fluent/plugin/out_http.rb b/lib/fluent/plugin/out_http.rb index 9d46571b76..b1bfb20afc 100644 --- a/lib/fluent/plugin/out_http.rb +++ b/lib/fluent/plugin/out_http.rb @@ -124,7 +124,7 @@ def format(tag, time, record) end def format_json_array(tag, time, record) - @formatter.format(tag, time, record).strip() << "," + @formatter.format(tag, time, record) << "," end def write(chunk) From 10867e0cfc417df48ba0c4d7cb9b43177f93031d Mon Sep 17 00:00:00 2001 From: Roman Geraskin Date: Sat, 2 May 2020 07:08:05 +0000 Subject: [PATCH 3/5] use chop! method instead of [0...-1] [0...-1] creates new object and it consumes biggerm emory for large str Signed-off-by: Roman Geraskin --- lib/fluent/plugin/out_http.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/fluent/plugin/out_http.rb b/lib/fluent/plugin/out_http.rb index b1bfb20afc..e8dbd7287b 100644 --- a/lib/fluent/plugin/out_http.rb +++ b/lib/fluent/plugin/out_http.rb @@ -215,7 +215,7 @@ def create_request(chunk, uri) req.basic_auth(@auth.username, @auth.password) end set_headers(req) - req.body = @json_array ? "[#{chunk.read[0...-1]}]" : chunk.read + req.body = @json_array ? "[#{chunk.read.chop!}]" : chunk.read req end From 26814a6b39aa1a9173e727ec7b1dda3a789c1ad9 Mon Sep 17 00:00:00 2001 From: Roman Geraskin Date: Sat, 2 May 2020 11:17:46 +0000 Subject: [PATCH 4/5] json_array tests added test_configure_with_json_array_err: check if an exception is raised on not JSON format use test_configure_content_type_json_array: check that json_array setting sets content_type = application/json test_write_with_json_array_setting: check that JSON at HTTP request body is valid Signed-off-by: Roman Geraskin --- test/plugin/test_out_http.rb | 38 ++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/test/plugin/test_out_http.rb b/test/plugin/test_out_http.rb index 757c8d620a..b7b4fe9da8 100644 --- a/test/plugin/test_out_http.rb +++ b/test/plugin/test_out_http.rb @@ -80,6 +80,8 @@ def run_http_server req.body.each_line { |l| data << JSON.parse(l) } + when 'application/json' + data = JSON.parse(req.body) when 'text/plain' # Use single_value in this test req.body.each_line { |line| @@ -181,6 +183,19 @@ def test_configure_without_warn assert_not_match(/Status code 503 is going to be removed/, d.instance.log.out.logs.join) end + # Check if an exception is raised on not JSON format use + data('not_json' => 'msgpack') + def test_configure_with_json_array_err(format_type) + assert_raise(Fluent::ConfigError) do + create_driver(config + %[ + json_array true + + @type #{format_type} + + ]) + end + end + data('json' => ['json', 'application/x-ndjson'], 'ltsv' => ['ltsv', 'text/tab-separated-values'], 'msgpack' => ['msgpack', 'application/x-msgpack'], @@ -195,6 +210,14 @@ def test_configure_content_type(types) assert_equal content_type, d.instance.content_type end + # Check that json_array setting sets content_type = application/json + data('json' => 'application/json') + def test_configure_content_type_json_array(content_type) + d = create_driver(config + "json_array true") + + assert_equal content_type, d.instance.content_type + end + data('PUT' => 'put', 'POST' => 'post') def test_write_with_method(method) d = create_driver(config + "http_method #{method}") @@ -211,6 +234,21 @@ def test_write_with_method(method) assert_not_empty result.headers end + # Check that JSON at HTTP request body is valid + def test_write_with_json_array_setting + d = create_driver(config + "json_array true") + d.run(default_tag: 'test.http') do + test_events.each { |event| + d.feed(event) + } + end + + result = @@result + assert_equal 'application/json', result.content_type + assert_equal test_events, result.data + assert_not_empty result.headers + end + def test_write_with_single_value_format d = create_driver(config + %[ From a3dc3a651ad8953b032b067db641654bef77a267 Mon Sep 17 00:00:00 2001 From: Roman Geraskin Date: Thu, 7 May 2020 11:20:20 +0000 Subject: [PATCH 5/5] nits: indent fix, eof newline return Signed-off-by: Roman Geraskin --- lib/fluent/plugin/out_http.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/fluent/plugin/out_http.rb b/lib/fluent/plugin/out_http.rb index e8dbd7287b..a5de8701d8 100644 --- a/lib/fluent/plugin/out_http.rb +++ b/lib/fluent/plugin/out_http.rb @@ -105,7 +105,7 @@ def configure(conf) if @json_array if @formatter_configs.first[:@type] != "json" - raise Fluent::ConfigError, "json_array option could be used with json formatter only" + raise Fluent::ConfigError, "json_array option could be used with json formatter only" end define_singleton_method(:format, method(:format_json_array)) end @@ -247,4 +247,4 @@ def send_request(uri, req) end end end -end \ No newline at end of file +end