From a991e22e94934a8ad19acd4f52082e42a4799927 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Thu, 7 Jan 2021 20:32:32 -0800 Subject: [PATCH 01/10] wip --- .../scripts/tomlparser-agent-config.rb | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/build/linux/installer/scripts/tomlparser-agent-config.rb b/build/linux/installer/scripts/tomlparser-agent-config.rb index 87c5194ed..f331e29b3 100644 --- a/build/linux/installer/scripts/tomlparser-agent-config.rb +++ b/build/linux/installer/scripts/tomlparser-agent-config.rb @@ -55,6 +55,12 @@ @podsEmitStreamBatchSizeMin = 50 @nodesEmitStreamBatchSizeMin = 50 +# configmap settings related fbit config +@fbitFlushInterval = 0 +@fbitTailBufferChunkSize = 0 +@fbitTailBufferMaxSize = 0 + + def is_number?(value) true if Integer(value) rescue false end @@ -131,6 +137,36 @@ def populateSettingValuesFromConfigMap(parsedConfig) puts "Using config map value: NODES_EMIT_STREAM_BATCH_SIZE = #{@nodesEmitStreamBatchSize}" end end + fbit_config = parsedConfig[:agent_settings][:fbit_config] + if !fbit_config.nil? + fbitFlushInterval = fbit_config[:FBIT_SERVICE_FLUSH_INTERVAL] + if !fbitFlushInterval.nil? && is_number?(fbitFlushInterval) && fbitFlushInterval.to_i > 0 + @fbitFlushInterval = fbitFlushInterval.to_i + puts "Using config map value: FBIT_SERVICE_FLUSH_INTERVAL = #{@fbitFlushInterval}" + end + + fbitTailBufferChunkSize = fbit_config[:FBIT_TAIL_BUFFER_CHUNK_SIZE] + if !fbitTailBufferChunkSize.nil? && is_number?(fbitTailBufferChunkSize) && fbitTailBufferChunkSize.to_i > 0 + @fbitTailBufferChunkSize = fbitTailBufferChunkSize.to_i + puts "Using config map value: FBIT_TAIL_BUFFER_CHUNK_SIZE = #{@fbitTailBufferChunkSize}" + end + + fbitTailBufferMaxSize = fbit_config[:FBIT_TAIL_BUFFER_MAX_SIZE] + if !fbitTailBufferMaxSize.nil? && is_number?(fbitTailBufferMaxSize) && fbitTailBufferMaxSize.to_i > 0 + if fbitTailBufferMaxSize.to_i >= @fbitTailBufferChunkSize + @fbitTailBufferMaxSize = fbitTailBufferMaxSize.to_i + puts "Using config map value: FBIT_TAIL_BUFFER_MAX_SIZE = #{@fbitTailBufferMaxSize}" + else + @fbitTailBufferMaxSize = @fbitTailBufferChunkSize + puts "config::warn: FBIT_TAIL_BUFFER_MAX_SIZE should be greater or equal to value of FBIT_TAIL_BUFFER_CHUNK_SIZE. Using FBIT_TAIL_BUFFER_MAX_SIZE = #{@fbitTailBufferMaxSize} since provodided config value not valid" + end + end + # in scenario - FBIT_TAIL_BUFFER_MAX_SIZE provided but not FBIT_TAIL_BUFFER_CHUNK_SIZE + if @fbitTailBufferMaxSize > 0 && @fbitTailBufferChunkSize == 0 + @fbitTailBufferChunkSize = @fbitTailBufferMaxSize + end + + end end rescue => errorStr puts "config::error:Exception while reading config settings for agent configuration setting - #{errorStr}, using defaults" From 39d6a8fbd6afa14e856e8a766cab39699469db40 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Fri, 8 Jan 2021 08:06:34 -0800 Subject: [PATCH 02/10] fbit config settings --- .../installer/scripts/tomlparser-agent-config.rb | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/build/linux/installer/scripts/tomlparser-agent-config.rb b/build/linux/installer/scripts/tomlparser-agent-config.rb index f331e29b3..444e364a6 100644 --- a/build/linux/installer/scripts/tomlparser-agent-config.rb +++ b/build/linux/installer/scripts/tomlparser-agent-config.rb @@ -137,6 +137,7 @@ def populateSettingValuesFromConfigMap(parsedConfig) puts "Using config map value: NODES_EMIT_STREAM_BATCH_SIZE = #{@nodesEmitStreamBatchSize}" end end + # fbit config settings fbit_config = parsedConfig[:agent_settings][:fbit_config] if !fbit_config.nil? fbitFlushInterval = fbit_config[:FBIT_SERVICE_FLUSH_INTERVAL] @@ -165,7 +166,6 @@ def populateSettingValuesFromConfigMap(parsedConfig) if @fbitTailBufferMaxSize > 0 && @fbitTailBufferChunkSize == 0 @fbitTailBufferChunkSize = @fbitTailBufferMaxSize end - end end rescue => errorStr @@ -200,6 +200,16 @@ def populateSettingValuesFromConfigMap(parsedConfig) file.write("export HPA_CHUNK_SIZE=#{@hpaChunkSize}\n") file.write("export PODS_EMIT_STREAM_BATCH_SIZE=#{@podsEmitStreamBatchSize}\n") file.write("export NODES_EMIT_STREAM_BATCH_SIZE=#{@nodesEmitStreamBatchSize}\n") + # fbit settings + if @fbitFlushInterval > 0 + file.write("export FBIT_SERVICE_FLUSH_INTERVAL=#{@fbitFlushInterval}\n") + end + if @fbitTailBufferChunkSize > 0 + file.write("export FBIT_TAIL_BUFFER_CHUNK_SIZE=#{@fbitTailBufferChunkSize}\n") + end + if @fbitTailBufferMaxSize > 0 + file.write("export FBIT_TAIL_BUFFER_MAX_SIZE=#{@fbitTailBufferMaxSize}\n") + end # Close file after writing all environment variables file.close else From 324902e54914d0fde5a3a4df2bd3e6e02abfb423 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Fri, 8 Jan 2021 08:49:35 -0800 Subject: [PATCH 03/10] add config warn message --- build/linux/installer/scripts/tomlparser-agent-config.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/build/linux/installer/scripts/tomlparser-agent-config.rb b/build/linux/installer/scripts/tomlparser-agent-config.rb index 444e364a6..a640addf5 100644 --- a/build/linux/installer/scripts/tomlparser-agent-config.rb +++ b/build/linux/installer/scripts/tomlparser-agent-config.rb @@ -165,6 +165,7 @@ def populateSettingValuesFromConfigMap(parsedConfig) # in scenario - FBIT_TAIL_BUFFER_MAX_SIZE provided but not FBIT_TAIL_BUFFER_CHUNK_SIZE if @fbitTailBufferMaxSize > 0 && @fbitTailBufferChunkSize == 0 @fbitTailBufferChunkSize = @fbitTailBufferMaxSize + puts "config::warn: since FBIT_TAIL_BUFFER_CHUNK_SIZE not provided hence using FBIT_TAIL_BUFFER_CHUNK_SIZE=#{@fbitTailBufferChunkSize} which is same as the value of FBIT_TAIL_BUFFER_MAX_SIZE" end end end From 056de78bf7bb13ea44a47a8c537207a5a99db892 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Fri, 8 Jan 2021 09:10:46 -0800 Subject: [PATCH 04/10] handle one config provided but not other --- build/linux/installer/scripts/tomlparser-agent-config.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/build/linux/installer/scripts/tomlparser-agent-config.rb b/build/linux/installer/scripts/tomlparser-agent-config.rb index a640addf5..1eddddb2d 100644 --- a/build/linux/installer/scripts/tomlparser-agent-config.rb +++ b/build/linux/installer/scripts/tomlparser-agent-config.rb @@ -162,10 +162,13 @@ def populateSettingValuesFromConfigMap(parsedConfig) puts "config::warn: FBIT_TAIL_BUFFER_MAX_SIZE should be greater or equal to value of FBIT_TAIL_BUFFER_CHUNK_SIZE. Using FBIT_TAIL_BUFFER_MAX_SIZE = #{@fbitTailBufferMaxSize} since provodided config value not valid" end end - # in scenario - FBIT_TAIL_BUFFER_MAX_SIZE provided but not FBIT_TAIL_BUFFER_CHUNK_SIZE + # in scenario - FBIT_TAIL_BUFFER_MAX_SIZE provided but not FBIT_TAIL_BUFFER_CHUNK_SIZE and vice versa if @fbitTailBufferMaxSize > 0 && @fbitTailBufferChunkSize == 0 @fbitTailBufferChunkSize = @fbitTailBufferMaxSize puts "config::warn: since FBIT_TAIL_BUFFER_CHUNK_SIZE not provided hence using FBIT_TAIL_BUFFER_CHUNK_SIZE=#{@fbitTailBufferChunkSize} which is same as the value of FBIT_TAIL_BUFFER_MAX_SIZE" + elsif @fbitTailBufferChunkSize > 0 && @fbitTailBufferMaxSize == 0 + @fbitTailBufferMaxSize = @fbitTailBufferChunkSize + puts "config::warn: since FBIT_TAIL_BUFFER_MAX_SIZE not provided hence using FBIT_TAIL_BUFFER_MAX_SIZE=#{@fbitTailBufferMaxSize} which is same as the value of FBIT_TAIL_BUFFER_CHUNK_SIZE" end end end From e7c95b6c5153373057ebeebc25cc4fdc1439a70b Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Fri, 8 Jan 2021 12:44:41 -0800 Subject: [PATCH 05/10] fixed pr feedback --- .../scripts/tomlparser-agent-config.rb | 62 +++++++++---------- kubernetes/linux/main.sh | 1 + 2 files changed, 32 insertions(+), 31 deletions(-) diff --git a/build/linux/installer/scripts/tomlparser-agent-config.rb b/build/linux/installer/scripts/tomlparser-agent-config.rb index 1eddddb2d..f3bc13c87 100644 --- a/build/linux/installer/scripts/tomlparser-agent-config.rb +++ b/build/linux/installer/scripts/tomlparser-agent-config.rb @@ -56,9 +56,9 @@ @nodesEmitStreamBatchSizeMin = 50 # configmap settings related fbit config -@fbitFlushInterval = 0 -@fbitTailBufferChunkSize = 0 -@fbitTailBufferMaxSize = 0 +@fbitFlushIntervalSecsSecs = 0 +@fbitTailBufferChunkSizeMBs = 0 +@fbitTailBufferMaxSizeMBs = 0 def is_number?(value) @@ -140,35 +140,35 @@ def populateSettingValuesFromConfigMap(parsedConfig) # fbit config settings fbit_config = parsedConfig[:agent_settings][:fbit_config] if !fbit_config.nil? - fbitFlushInterval = fbit_config[:FBIT_SERVICE_FLUSH_INTERVAL] - if !fbitFlushInterval.nil? && is_number?(fbitFlushInterval) && fbitFlushInterval.to_i > 0 - @fbitFlushInterval = fbitFlushInterval.to_i - puts "Using config map value: FBIT_SERVICE_FLUSH_INTERVAL = #{@fbitFlushInterval}" + fbitFlushIntervalSecs = fbit_config[:logflushintervalsecs] + if !fbitFlushIntervalSecs.nil? && is_number?(fbitFlushIntervalSecs) && fbitFlushIntervalSecs.to_i > 0 + @fbitFlushIntervalSecs = fbitFlushIntervalSecs.to_i + puts "Using config map value: logflushintervalsecs = #{@fbitFlushIntervalSecs}" end - fbitTailBufferChunkSize = fbit_config[:FBIT_TAIL_BUFFER_CHUNK_SIZE] - if !fbitTailBufferChunkSize.nil? && is_number?(fbitTailBufferChunkSize) && fbitTailBufferChunkSize.to_i > 0 - @fbitTailBufferChunkSize = fbitTailBufferChunkSize.to_i - puts "Using config map value: FBIT_TAIL_BUFFER_CHUNK_SIZE = #{@fbitTailBufferChunkSize}" + fbitTailBufferChunkSizeMBs = fbit_config[:tailbufchunksizemegabytes ] + if !fbitTailBufferChunkSizeMBs.nil? && is_number?(fbitTailBufferChunkSizeMBs) && fbitTailBufferChunkSizeMBs.to_i > 0 + @fbitTailBufferChunkSizeMBs = fbitTailBufferChunkSizeMBs.to_i + puts "Using config map value: tailbufchunksizemegabytes = #{@fbitTailBufferChunkSizeMBs}" end - fbitTailBufferMaxSize = fbit_config[:FBIT_TAIL_BUFFER_MAX_SIZE] - if !fbitTailBufferMaxSize.nil? && is_number?(fbitTailBufferMaxSize) && fbitTailBufferMaxSize.to_i > 0 - if fbitTailBufferMaxSize.to_i >= @fbitTailBufferChunkSize - @fbitTailBufferMaxSize = fbitTailBufferMaxSize.to_i - puts "Using config map value: FBIT_TAIL_BUFFER_MAX_SIZE = #{@fbitTailBufferMaxSize}" + fbitTailBufferMaxSizeMBs = fbit_config[:tailbufmaxsizemegabytes ] + if !fbitTailBufferMaxSizeMBs.nil? && is_number?(fbitTailBufferMaxSizeMBs) && fbitTailBufferMaxSizeMBs.to_i > 0 + if fbitTailBufferMaxSizeMBs.to_i >= @fbitTailBufferChunkSizeMBs + @fbitTailBufferMaxSizeMBs = fbitTailBufferMaxSizeMBs.to_i + puts "Using config map value: tailbufmaxsizemegabytes = #{@fbitTailBufferMaxSizeMBs}" else - @fbitTailBufferMaxSize = @fbitTailBufferChunkSize - puts "config::warn: FBIT_TAIL_BUFFER_MAX_SIZE should be greater or equal to value of FBIT_TAIL_BUFFER_CHUNK_SIZE. Using FBIT_TAIL_BUFFER_MAX_SIZE = #{@fbitTailBufferMaxSize} since provodided config value not valid" + @fbitTailBufferMaxSizeMBs = @fbitTailBufferChunkSizeMBs + puts "config::warn: tailbufmaxsizemegabytes should be greater or equal to value of tailbufchunksizemegabytes. Using tailbufmaxsizemegabytes = #{@fbitTailBufferMaxSizeMBs} since provodided config value not valid" end end - # in scenario - FBIT_TAIL_BUFFER_MAX_SIZE provided but not FBIT_TAIL_BUFFER_CHUNK_SIZE and vice versa - if @fbitTailBufferMaxSize > 0 && @fbitTailBufferChunkSize == 0 - @fbitTailBufferChunkSize = @fbitTailBufferMaxSize - puts "config::warn: since FBIT_TAIL_BUFFER_CHUNK_SIZE not provided hence using FBIT_TAIL_BUFFER_CHUNK_SIZE=#{@fbitTailBufferChunkSize} which is same as the value of FBIT_TAIL_BUFFER_MAX_SIZE" - elsif @fbitTailBufferChunkSize > 0 && @fbitTailBufferMaxSize == 0 - @fbitTailBufferMaxSize = @fbitTailBufferChunkSize - puts "config::warn: since FBIT_TAIL_BUFFER_MAX_SIZE not provided hence using FBIT_TAIL_BUFFER_MAX_SIZE=#{@fbitTailBufferMaxSize} which is same as the value of FBIT_TAIL_BUFFER_CHUNK_SIZE" + # in scenario - tailbufmaxsizemegabytes provided but not tailbufchunksizemegabytes and vice versa + if @fbitTailBufferMaxSizeMBs > 0 && @fbitTailBufferChunkSizeMBs == 0 + @fbitTailBufferChunkSizeMBs = @fbitTailBufferMaxSizeMBs + puts "config::warn: since tailbufchunksizemegabytes not provided hence using tailbufchunksizemegabytes=#{@fbitTailBufferChunkSizeMBs} which is same as the value of tailbufmaxsizemegabytes" + elsif @fbitTailBufferChunkSizeMBs > 0 && @fbitTailBufferMaxSizeMBs == 0 + @fbitTailBufferMaxSizeMBs = @fbitTailBufferChunkSizeMBs + puts "config::warn: since tailbufmaxsizemegabytes not provided hence using tailbufmaxsizemegabytes=#{@fbitTailBufferMaxSizeMBs} which is same as the value of tailbufchunksizemegabytes" end end end @@ -205,14 +205,14 @@ def populateSettingValuesFromConfigMap(parsedConfig) file.write("export PODS_EMIT_STREAM_BATCH_SIZE=#{@podsEmitStreamBatchSize}\n") file.write("export NODES_EMIT_STREAM_BATCH_SIZE=#{@nodesEmitStreamBatchSize}\n") # fbit settings - if @fbitFlushInterval > 0 - file.write("export FBIT_SERVICE_FLUSH_INTERVAL=#{@fbitFlushInterval}\n") + if @fbitFlushIntervalSecs > 0 + file.write("export FBIT_SERVICE_FLUSH_INTERVAL=#{@fbitFlushIntervalSecs}\n") end - if @fbitTailBufferChunkSize > 0 - file.write("export FBIT_TAIL_BUFFER_CHUNK_SIZE=#{@fbitTailBufferChunkSize}\n") + if @fbitTailBufferChunkSizeMBs > 0 + file.write("export FBIT_TAIL_BUFFER_CHUNK_SIZE=#{@fbitTailBufferChunkSizeMBs}\n") end - if @fbitTailBufferMaxSize > 0 - file.write("export FBIT_TAIL_BUFFER_MAX_SIZE=#{@fbitTailBufferMaxSize}\n") + if @fbitTailBufferMaxSizeMBs > 0 + file.write("export FBIT_TAIL_BUFFER_MAX_SIZE=#{@fbitTailBufferMaxSizeMBs}\n") end # Close file after writing all environment variables file.close diff --git a/kubernetes/linux/main.sh b/kubernetes/linux/main.sh index ed16d3e32..b4df538d4 100644 --- a/kubernetes/linux/main.sh +++ b/kubernetes/linux/main.sh @@ -172,6 +172,7 @@ source config_env_var #Parse the configmap to set the right environment variables for agent config. +#Note > tomlparser-agent-config.rb has to be parsed first before td-agent-bit-conf-customizer.rb for fbit agent settings /opt/microsoft/omsagent/ruby/bin/ruby tomlparser-agent-config.rb cat agent_config_env_var | while read line; do From 335bd7d6ba7069e7b9830bf517986e06bbb0faca Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Fri, 8 Jan 2021 13:14:50 -0800 Subject: [PATCH 06/10] fix copy paste error --- build/linux/installer/scripts/tomlparser-agent-config.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/linux/installer/scripts/tomlparser-agent-config.rb b/build/linux/installer/scripts/tomlparser-agent-config.rb index f3bc13c87..fb1d2fa0b 100644 --- a/build/linux/installer/scripts/tomlparser-agent-config.rb +++ b/build/linux/installer/scripts/tomlparser-agent-config.rb @@ -56,7 +56,7 @@ @nodesEmitStreamBatchSizeMin = 50 # configmap settings related fbit config -@fbitFlushIntervalSecsSecs = 0 +@fbitFlushIntervalSecs = 0 @fbitTailBufferChunkSizeMBs = 0 @fbitTailBufferMaxSizeMBs = 0 From a5fac755fae5f77bd1b55c2fce562e0cab0548d5 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Fri, 8 Jan 2021 17:37:35 -0800 Subject: [PATCH 07/10] rename config parameter names --- .../scripts/tomlparser-agent-config.rb | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/build/linux/installer/scripts/tomlparser-agent-config.rb b/build/linux/installer/scripts/tomlparser-agent-config.rb index fb1d2fa0b..32a04358a 100644 --- a/build/linux/installer/scripts/tomlparser-agent-config.rb +++ b/build/linux/installer/scripts/tomlparser-agent-config.rb @@ -140,35 +140,35 @@ def populateSettingValuesFromConfigMap(parsedConfig) # fbit config settings fbit_config = parsedConfig[:agent_settings][:fbit_config] if !fbit_config.nil? - fbitFlushIntervalSecs = fbit_config[:logflushintervalsecs] + fbitFlushIntervalSecs = fbit_config[:log_flush_interval_secs] if !fbitFlushIntervalSecs.nil? && is_number?(fbitFlushIntervalSecs) && fbitFlushIntervalSecs.to_i > 0 @fbitFlushIntervalSecs = fbitFlushIntervalSecs.to_i - puts "Using config map value: logflushintervalsecs = #{@fbitFlushIntervalSecs}" + puts "Using config map value: log_flush_interval_secs = #{@fbitFlushIntervalSecs}" end - fbitTailBufferChunkSizeMBs = fbit_config[:tailbufchunksizemegabytes ] + fbitTailBufferChunkSizeMBs = fbit_config[:tail_buf_chunksize_megabytes] if !fbitTailBufferChunkSizeMBs.nil? && is_number?(fbitTailBufferChunkSizeMBs) && fbitTailBufferChunkSizeMBs.to_i > 0 @fbitTailBufferChunkSizeMBs = fbitTailBufferChunkSizeMBs.to_i - puts "Using config map value: tailbufchunksizemegabytes = #{@fbitTailBufferChunkSizeMBs}" + puts "Using config map value: tail_buf_chunksize_megabytes = #{@fbitTailBufferChunkSizeMBs}" end - fbitTailBufferMaxSizeMBs = fbit_config[:tailbufmaxsizemegabytes ] + fbitTailBufferMaxSizeMBs = fbit_config[:tail_buf_maxsize_megabytes] if !fbitTailBufferMaxSizeMBs.nil? && is_number?(fbitTailBufferMaxSizeMBs) && fbitTailBufferMaxSizeMBs.to_i > 0 if fbitTailBufferMaxSizeMBs.to_i >= @fbitTailBufferChunkSizeMBs @fbitTailBufferMaxSizeMBs = fbitTailBufferMaxSizeMBs.to_i - puts "Using config map value: tailbufmaxsizemegabytes = #{@fbitTailBufferMaxSizeMBs}" + puts "Using config map value: tail_buf_maxsize_megabytes = #{@fbitTailBufferMaxSizeMBs}" else @fbitTailBufferMaxSizeMBs = @fbitTailBufferChunkSizeMBs - puts "config::warn: tailbufmaxsizemegabytes should be greater or equal to value of tailbufchunksizemegabytes. Using tailbufmaxsizemegabytes = #{@fbitTailBufferMaxSizeMBs} since provodided config value not valid" + puts "config::warn: tail_buf_maxsize_megabytes should be greater or equal to value of tail_buf_chunksize_megabytes. Using tail_buf_maxsize_megabytes = #{@fbitTailBufferMaxSizeMBs} since provodided config value not valid" end end - # in scenario - tailbufmaxsizemegabytes provided but not tailbufchunksizemegabytes and vice versa + # in scenario - tail_buf_maxsize_megabytes provided but not tail_buf_chunksize_megabytes and vice versa if @fbitTailBufferMaxSizeMBs > 0 && @fbitTailBufferChunkSizeMBs == 0 @fbitTailBufferChunkSizeMBs = @fbitTailBufferMaxSizeMBs - puts "config::warn: since tailbufchunksizemegabytes not provided hence using tailbufchunksizemegabytes=#{@fbitTailBufferChunkSizeMBs} which is same as the value of tailbufmaxsizemegabytes" + puts "config::warn: since tail_buf_chunksize_megabytes not provided hence using tail_buf_chunksize_megabytes=#{@fbitTailBufferChunkSizeMBs} which is same as the value of tail_buf_maxsize_megabytes" elsif @fbitTailBufferChunkSizeMBs > 0 && @fbitTailBufferMaxSizeMBs == 0 @fbitTailBufferMaxSizeMBs = @fbitTailBufferChunkSizeMBs - puts "config::warn: since tailbufmaxsizemegabytes not provided hence using tailbufmaxsizemegabytes=#{@fbitTailBufferMaxSizeMBs} which is same as the value of tailbufchunksizemegabytes" + puts "config::warn: since tail_buf_maxsize_megabytes not provided hence using tail_buf_maxsize_megabytes=#{@fbitTailBufferMaxSizeMBs} which is same as the value of tail_buf_chunksize_megabytes" end end end From 5c7ccab96fe8495e5d07669e010099ebfb314c25 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Fri, 8 Jan 2021 18:40:25 -0800 Subject: [PATCH 08/10] fix typo --- build/linux/installer/scripts/tomlparser-agent-config.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/linux/installer/scripts/tomlparser-agent-config.rb b/build/linux/installer/scripts/tomlparser-agent-config.rb index 32a04358a..9a01410f4 100644 --- a/build/linux/installer/scripts/tomlparser-agent-config.rb +++ b/build/linux/installer/scripts/tomlparser-agent-config.rb @@ -159,7 +159,7 @@ def populateSettingValuesFromConfigMap(parsedConfig) puts "Using config map value: tail_buf_maxsize_megabytes = #{@fbitTailBufferMaxSizeMBs}" else @fbitTailBufferMaxSizeMBs = @fbitTailBufferChunkSizeMBs - puts "config::warn: tail_buf_maxsize_megabytes should be greater or equal to value of tail_buf_chunksize_megabytes. Using tail_buf_maxsize_megabytes = #{@fbitTailBufferMaxSizeMBs} since provodided config value not valid" + puts "config::warn: tail_buf_maxsize_megabytes should be greater or equal to value of tail_buf_chunksize_megabytes. Using tail_buf_maxsize_megabytes = #{@fbitTailBufferMaxSizeMBs} since provided config value not valid" end end # in scenario - tail_buf_maxsize_megabytes provided but not tail_buf_chunksize_megabytes and vice versa From e175ad24a017b0f33b6379a895be0069842671ea Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Mon, 11 Jan 2021 00:29:51 -0800 Subject: [PATCH 09/10] fix fbit crash in helm path --- .../scripts/td-agent-bit-conf-customizer.rb | 11 ++++++++--- .../installer/scripts/tomlparser-agent-config.rb | 12 +++++------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/build/common/installer/scripts/td-agent-bit-conf-customizer.rb b/build/common/installer/scripts/td-agent-bit-conf-customizer.rb index fae3acb36..9e9a9b16f 100644 --- a/build/common/installer/scripts/td-agent-bit-conf-customizer.rb +++ b/build/common/installer/scripts/td-agent-bit-conf-customizer.rb @@ -18,12 +18,17 @@ def substituteFluentBitPlaceHolders bufferChunkSize = ENV["FBIT_TAIL_BUFFER_CHUNK_SIZE"] bufferMaxSize = ENV["FBIT_TAIL_BUFFER_MAX_SIZE"] - serviceInterval = (!interval.nil? && is_number?(interval)) ? interval : @default_service_interval + serviceInterval = (!interval.nil? && is_number?(interval) && interval.to_i > 0 ) ? interval : @default_service_interval serviceIntervalSetting = "Flush " + serviceInterval - tailBufferChunkSize = (!bufferChunkSize.nil? && is_number?(bufferChunkSize)) ? bufferChunkSize : nil + tailBufferChunkSize = (!bufferChunkSize.nil? && is_number?(bufferChunkSize) && bufferChunkSize.to_i > 0) ? bufferChunkSize : nil - tailBufferMaxSize = (!bufferMaxSize.nil? && is_number?(bufferMaxSize)) ? bufferMaxSize : nil + tailBufferMaxSize = (!bufferMaxSize.nil? && is_number?(bufferMaxSize) && bufferMaxSize.to_i > 0) ? bufferMaxSize : nil + + if ((!tailBufferChunkSize.nil? && tailBufferMaxSize.nil) || (!tailBufferChunkSize.nil? && !tailBufferMaxSize.nil? && tailBufferChunkSize.to_i > tailBufferMaxSize.to_i)) + puts "config:warn buffer max size must be greater or equal to chunk size" + tailBufferMaxSize = tailBufferChunkSize + end text = File.read(@td_agent_bit_conf_path) new_contents = text.gsub("${SERVICE_FLUSH_INTERVAL}", serviceIntervalSetting) diff --git a/build/linux/installer/scripts/tomlparser-agent-config.rb b/build/linux/installer/scripts/tomlparser-agent-config.rb index 9a01410f4..e587909e5 100644 --- a/build/linux/installer/scripts/tomlparser-agent-config.rb +++ b/build/linux/installer/scripts/tomlparser-agent-config.rb @@ -153,20 +153,18 @@ def populateSettingValuesFromConfigMap(parsedConfig) end fbitTailBufferMaxSizeMBs = fbit_config[:tail_buf_maxsize_megabytes] - if !fbitTailBufferMaxSizeMBs.nil? && is_number?(fbitTailBufferMaxSizeMBs) && fbitTailBufferMaxSizeMBs.to_i > 0 + if !fbitTailBufferMaxSizeMBs.nil? && is_number?(fbitTailBufferMaxSizeMBs) && fbitTailBufferMaxSizeMBs.to_i > 0 if fbitTailBufferMaxSizeMBs.to_i >= @fbitTailBufferChunkSizeMBs @fbitTailBufferMaxSizeMBs = fbitTailBufferMaxSizeMBs.to_i puts "Using config map value: tail_buf_maxsize_megabytes = #{@fbitTailBufferMaxSizeMBs}" else + # tail_buf_maxsize_megabytes has to be greater or equal to tail_buf_chunksize_megabytes @fbitTailBufferMaxSizeMBs = @fbitTailBufferChunkSizeMBs - puts "config::warn: tail_buf_maxsize_megabytes should be greater or equal to value of tail_buf_chunksize_megabytes. Using tail_buf_maxsize_megabytes = #{@fbitTailBufferMaxSizeMBs} since provided config value not valid" + puts "config::warn: tail_buf_maxsize_megabytes must be greater or equal to value of tail_buf_chunksize_megabytes. Using tail_buf_maxsize_megabytes = #{@fbitTailBufferMaxSizeMBs} since provided config value not valid" end end - # in scenario - tail_buf_maxsize_megabytes provided but not tail_buf_chunksize_megabytes and vice versa - if @fbitTailBufferMaxSizeMBs > 0 && @fbitTailBufferChunkSizeMBs == 0 - @fbitTailBufferChunkSizeMBs = @fbitTailBufferMaxSizeMBs - puts "config::warn: since tail_buf_chunksize_megabytes not provided hence using tail_buf_chunksize_megabytes=#{@fbitTailBufferChunkSizeMBs} which is same as the value of tail_buf_maxsize_megabytes" - elsif @fbitTailBufferChunkSizeMBs > 0 && @fbitTailBufferMaxSizeMBs == 0 + # in scenario - tail_buf_chunksize_megabytes provided but not tail_buf_maxsize_megabytes to prevent fbit crash + if @fbitTailBufferChunkSizeMBs > 0 && @fbitTailBufferMaxSizeMBs == 0 @fbitTailBufferMaxSizeMBs = @fbitTailBufferChunkSizeMBs puts "config::warn: since tail_buf_maxsize_megabytes not provided hence using tail_buf_maxsize_megabytes=#{@fbitTailBufferMaxSizeMBs} which is same as the value of tail_buf_chunksize_megabytes" end From 4b46c41abe96b9462cccad4723ff801e1bcff549 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Mon, 11 Jan 2021 08:32:12 -0800 Subject: [PATCH 10/10] fix nil check --- build/common/installer/scripts/td-agent-bit-conf-customizer.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/common/installer/scripts/td-agent-bit-conf-customizer.rb b/build/common/installer/scripts/td-agent-bit-conf-customizer.rb index 9e9a9b16f..35b71e550 100644 --- a/build/common/installer/scripts/td-agent-bit-conf-customizer.rb +++ b/build/common/installer/scripts/td-agent-bit-conf-customizer.rb @@ -25,7 +25,7 @@ def substituteFluentBitPlaceHolders tailBufferMaxSize = (!bufferMaxSize.nil? && is_number?(bufferMaxSize) && bufferMaxSize.to_i > 0) ? bufferMaxSize : nil - if ((!tailBufferChunkSize.nil? && tailBufferMaxSize.nil) || (!tailBufferChunkSize.nil? && !tailBufferMaxSize.nil? && tailBufferChunkSize.to_i > tailBufferMaxSize.to_i)) + if ((!tailBufferChunkSize.nil? && tailBufferMaxSize.nil?) || (!tailBufferChunkSize.nil? && !tailBufferMaxSize.nil? && tailBufferChunkSize.to_i > tailBufferMaxSize.to_i)) puts "config:warn buffer max size must be greater or equal to chunk size" tailBufferMaxSize = tailBufferChunkSize end