From 3d60f7deddbe804ecce65fd4cf3297af0de88324 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Fri, 22 Jul 2022 11:16:29 -0700 Subject: [PATCH 1/3] use ignore_older fbit default and option for configurability --- .../scripts/td-agent-bit-conf-customizer.rb | 11 ++++- .../scripts/tomlparser-agent-config.rb | 46 +++++++++++++------ build/linux/installer/conf/td-agent-bit.conf | 2 +- build/windows/installer/conf/fluent-bit.conf | 2 +- kubernetes/container-azm-ms-agentconfig.yaml | 12 +++-- 5 files changed, 51 insertions(+), 22 deletions(-) diff --git a/build/common/installer/scripts/td-agent-bit-conf-customizer.rb b/build/common/installer/scripts/td-agent-bit-conf-customizer.rb index 995d72b87..5db387911 100644 --- a/build/common/installer/scripts/td-agent-bit-conf-customizer.rb +++ b/build/common/installer/scripts/td-agent-bit-conf-customizer.rb @@ -24,15 +24,16 @@ def substituteFluentBitPlaceHolders bufferChunkSize = ENV["FBIT_TAIL_BUFFER_CHUNK_SIZE"] bufferMaxSize = ENV["FBIT_TAIL_BUFFER_MAX_SIZE"] memBufLimit = ENV["FBIT_TAIL_MEM_BUF_LIMIT"] + ignoreOlder = ENV["FBIT_TAIL_IGNORE_OLDER"] - serviceInterval = (!interval.nil? && is_number?(interval) && interval.to_i > 0 ) ? interval : @default_service_interval + serviceInterval = (!interval.nil? && is_number?(interval) && interval.to_i > 0) ? interval : @default_service_interval serviceIntervalSetting = "Flush " + serviceInterval tailBufferChunkSize = (!bufferChunkSize.nil? && is_number?(bufferChunkSize) && bufferChunkSize.to_i > 0) ? bufferChunkSize : nil tailBufferMaxSize = (!bufferMaxSize.nil? && is_number?(bufferMaxSize) && bufferMaxSize.to_i > 0) ? bufferMaxSize : nil - if ((!tailBufferChunkSize.nil? && tailBufferMaxSize.nil?) || (!tailBufferChunkSize.nil? && !tailBufferMaxSize.nil? && tailBufferChunkSize.to_i > tailBufferMaxSize.to_i)) + if ((!tailBufferChunkSize.nil? && tailBufferMaxSize.nil?) || (!tailBufferChunkSize.nil? && !tailBufferMaxSize.nil? && tailBufferChunkSize.to_i > tailBufferMaxSize.to_i)) puts "config:warn buffer max size must be greater or equal to chunk size" tailBufferMaxSize = tailBufferChunkSize end @@ -54,6 +55,12 @@ def substituteFluentBitPlaceHolders new_contents = new_contents.gsub("\n ${TAIL_BUFFER_MAX_SIZE}\n", "\n") end + if !ignoreOlder.nil? && !ignoreOlder.empty? + new_contents = new_contents.gsub("${TAIL_IGNORE_OLDER}", "Ignore_Older " + ignoreOlder) + else + new_contents = new_contents.gsub("\n ${TAIL_IGNORE_OLDER}\n", "\n") + end + File.open(@td_agent_bit_conf_path, "w") { |file| file.puts new_contents } puts "config::Successfully substituted the placeholders in td-agent-bit.conf file" rescue => errorStr diff --git a/build/common/installer/scripts/tomlparser-agent-config.rb b/build/common/installer/scripts/tomlparser-agent-config.rb index ebe1e3982..b0eede05b 100644 --- a/build/common/installer/scripts/tomlparser-agent-config.rb +++ b/build/common/installer/scripts/tomlparser-agent-config.rb @@ -59,7 +59,7 @@ @fbitTailBufferChunkSizeMBs = 0 @fbitTailBufferMaxSizeMBs = 0 @fbitTailMemBufLimitMBs = 0 - +@fbitTailIgnoreOlder = "" def is_number?(value) true if Integer(value) rescue false @@ -149,7 +149,7 @@ def populateSettingValuesFromConfigMap(parsedConfig) end fbitTailBufferMaxSizeMBs = fbit_config[:tail_buf_maxsize_megabytes] - if !fbitTailBufferMaxSizeMBs.nil? && is_number?(fbitTailBufferMaxSizeMBs) && fbitTailBufferMaxSizeMBs.to_i > 0 + if !fbitTailBufferMaxSizeMBs.nil? && is_number?(fbitTailBufferMaxSizeMBs) && fbitTailBufferMaxSizeMBs.to_i > 0 if fbitTailBufferMaxSizeMBs.to_i >= @fbitTailBufferChunkSizeMBs @fbitTailBufferMaxSizeMBs = fbitTailBufferMaxSizeMBs.to_i puts "Using config map value: tail_buf_maxsize_megabytes = #{@fbitTailBufferMaxSizeMBs}" @@ -160,16 +160,27 @@ def populateSettingValuesFromConfigMap(parsedConfig) end end # in scenario - tail_buf_chunksize_megabytes provided but not tail_buf_maxsize_megabytes to prevent fbit crash - if @fbitTailBufferChunkSizeMBs > 0 && @fbitTailBufferMaxSizeMBs == 0 + if @fbitTailBufferChunkSizeMBs > 0 && @fbitTailBufferMaxSizeMBs == 0 @fbitTailBufferMaxSizeMBs = @fbitTailBufferChunkSizeMBs puts "config::warn: since tail_buf_maxsize_megabytes not provided hence using tail_buf_maxsize_megabytes=#{@fbitTailBufferMaxSizeMBs} which is same as the value of tail_buf_chunksize_megabytes" - end + end fbitTailMemBufLimitMBs = fbit_config[:tail_mem_buf_limit_megabytes] if !fbitTailMemBufLimitMBs.nil? && is_number?(fbitTailMemBufLimitMBs) && fbitTailMemBufLimitMBs.to_i > 0 @fbitTailMemBufLimitMBs = fbitTailMemBufLimitMBs.to_i puts "Using config map value: tail_mem_buf_limit_megabytes = #{@fbitTailMemBufLimitMBs}" end + + fbitTailIgnoreOlder = fbit_config[:tail_ignore_older] + re = /^[0-9]+[mhd]$/ + if !fbitTailIgnoreOlder.nil? && !fbitTailIgnoreOlder.empty? + if !re.match(fbitTailIgnoreOlder).nil? + @fbitTailIgnoreOlder = fbitTailIgnoreOlder + puts "Using config map value: tail_ignore_older = #{@tail_ignore_older}" + else + puts "config:warn: provided tail_ignore_older value is not valid hence using default value" + end + end end end rescue => errorStr @@ -210,10 +221,15 @@ def populateSettingValuesFromConfigMap(parsedConfig) end if @fbitTailBufferMaxSizeMBs > 0 file.write("export FBIT_TAIL_BUFFER_MAX_SIZE=#{@fbitTailBufferMaxSizeMBs}\n") - end + end if @fbitTailMemBufLimitMBs > 0 file.write("export FBIT_TAIL_MEM_BUF_LIMIT=#{@fbitTailMemBufLimitMBs}\n") - end + end + + if !@fbitTailIgnoreOlder.nil? && !@fbitTailIgnoreOlder.empty? + file.write("export FBIT_TAIL_IGNORE_OLDER=#{@fbitTailIgnoreOlder}\n") + end + # Close file after writing all environment variables file.close else @@ -231,21 +247,25 @@ def get_command_windows(env_variable_name, env_variable_value) if !file.nil? if @fbitFlushIntervalSecs > 0 - commands = get_command_windows('FBIT_SERVICE_FLUSH_INTERVAL', @fbitFlushIntervalSecs) + commands = get_command_windows("FBIT_SERVICE_FLUSH_INTERVAL", @fbitFlushIntervalSecs) file.write(commands) end if @fbitTailBufferChunkSizeMBs > 0 - commands = get_command_windows('FBIT_TAIL_BUFFER_CHUNK_SIZE', @fbitTailBufferChunkSizeMBs) + commands = get_command_windows("FBIT_TAIL_BUFFER_CHUNK_SIZE", @fbitTailBufferChunkSizeMBs) file.write(commands) end if @fbitTailBufferMaxSizeMBs > 0 - commands = get_command_windows('FBIT_TAIL_BUFFER_MAX_SIZE', @fbitTailBufferMaxSizeMBs) + commands = get_command_windows("FBIT_TAIL_BUFFER_MAX_SIZE", @fbitTailBufferMaxSizeMBs) file.write(commands) - end + end if @fbitTailMemBufLimitMBs > 0 - commands = get_command_windows('FBIT_TAIL_MEM_BUF_LIMIT', @fbitTailMemBufLimitMBs) + commands = get_command_windows("FBIT_TAIL_MEM_BUF_LIMIT", @fbitTailMemBufLimitMBs) file.write(commands) - end + end + if !@fbitTailIgnoreOlder.nil? && !@fbitTailIgnoreOlder.empty? + commands = get_command_windows("FBIT_TAIL_IGNORE_OLDER", @fbitTailIgnoreOlder) + file.write(commands) + end # Close file after writing all environment variables file.close puts "****************End Config Processing********************" @@ -253,4 +273,4 @@ def get_command_windows(env_variable_name, env_variable_value) puts "Exception while opening file for writing config environment variables for WINDOWS LOG" puts "****************End Config Processing********************" end -end \ No newline at end of file +end diff --git a/build/linux/installer/conf/td-agent-bit.conf b/build/linux/installer/conf/td-agent-bit.conf index beba6a3ca..fe550ab62 100644 --- a/build/linux/installer/conf/td-agent-bit.conf +++ b/build/linux/installer/conf/td-agent-bit.conf @@ -26,7 +26,7 @@ Refresh_Interval 30 Path_Key filepath Skip_Long_Lines On - Ignore_Older 5m + ${TAIL_IGNORE_OLDER} Exclude_Path ${AZMON_CLUSTER_LOG_TAIL_EXCLUDE_PATH} [INPUT] diff --git a/build/windows/installer/conf/fluent-bit.conf b/build/windows/installer/conf/fluent-bit.conf index 1e2d8a93e..b43354e3f 100644 --- a/build/windows/installer/conf/fluent-bit.conf +++ b/build/windows/installer/conf/fluent-bit.conf @@ -25,7 +25,7 @@ Refresh_Interval 30 Path_Key filepath Skip_Long_Lines On - Ignore_Older 5m + ${TAIL_IGNORE_OLDER} Exclude_Path ${AZMON_CLUSTER_LOG_TAIL_EXCLUDE_PATH} [INPUT] diff --git a/kubernetes/container-azm-ms-agentconfig.yaml b/kubernetes/container-azm-ms-agentconfig.yaml index 8b9e2d718..609fd5425 100644 --- a/kubernetes/container-azm-ms-agentconfig.yaml +++ b/kubernetes/container-azm-ms-agentconfig.yaml @@ -46,8 +46,8 @@ data: # In the absence of this configmap, default value for containerlog_schema_version is "v1" # Supported values for this setting are "v1","v2" # See documentation at https://aka.ms/ContainerLogv2 for benefits of v2 schema over v1 schema before opting for "v2" schema - # containerlog_schema_version = "v2" - + # containerlog_schema_version = "v2" + prometheus-data-collection-settings: |- # Custom Prometheus metrics data collection settings @@ -135,7 +135,7 @@ data: # Alertable metrics configuration settings for completed jobs count [alertable_metrics_configuration_settings.job_completion_threshold] - # Threshold for completed job count , metric will be sent only for those jobs which were completed earlier than the following threshold + # Threshold for completed job count , metric will be sent only for those jobs which were completed earlier than the following threshold job_completion_threshold_time_minutes = 360 integrations: |- [integrations.azure_network_policy_manager] @@ -147,19 +147,21 @@ data: # Doc - https://github.com/microsoft/Docker-Provider/blob/ci_prod/Documentation/AgentSettings/ReadMe.md agent-settings: |- # prometheus scrape fluent bit settings for high scale - # buffer size should be greater than or equal to chunk size else we set it to chunk size. + # buffer size should be greater than or equal to chunk size else we set it to chunk size. [agent_settings.prometheus_fbit_settings] tcp_listener_chunk_size = 10 tcp_listener_buffer_size = 10 tcp_listener_mem_buf_limit = 200 - + # The following settings are "undocumented", we don't recommend uncommenting them unless directed by Microsoft. # They increase the maximum stdout/stderr log collection rate but will also cause higher cpu/memory usage. + ## Ref more details about Ignore_Older - https://docs.fluentbit.io/manual/v/1.7/pipeline/inputs/tail # [agent_settings.fbit_config] # log_flush_interval_secs = "1" # default value is 15 # tail_mem_buf_limit_megabytes = "10" # default value is 10 # tail_buf_chunksize_megabytes = "1" # default value is 32kb (comment out this line for default) # tail_buf_maxsize_megabytes = "1" # defautl value is 32kb (comment out this line for default) + # tail_ignore_older = "5m" # default value same as fluent-bit default i.e.0m metadata: name: container-azm-ms-agentconfig From 4c5857612e5671cc074b7807c8b0b897fc589d6a Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Fri, 22 Jul 2022 19:12:22 -0700 Subject: [PATCH 2/3] fix minor comment --- kubernetes/container-azm-ms-agentconfig.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/container-azm-ms-agentconfig.yaml b/kubernetes/container-azm-ms-agentconfig.yaml index 609fd5425..a274724ef 100644 --- a/kubernetes/container-azm-ms-agentconfig.yaml +++ b/kubernetes/container-azm-ms-agentconfig.yaml @@ -155,7 +155,7 @@ data: # The following settings are "undocumented", we don't recommend uncommenting them unless directed by Microsoft. # They increase the maximum stdout/stderr log collection rate but will also cause higher cpu/memory usage. - ## Ref more details about Ignore_Older - https://docs.fluentbit.io/manual/v/1.7/pipeline/inputs/tail + ## Ref for more details about Ignore_Older - https://docs.fluentbit.io/manual/v/1.7/pipeline/inputs/tail # [agent_settings.fbit_config] # log_flush_interval_secs = "1" # default value is 15 # tail_mem_buf_limit_megabytes = "10" # default value is 10 From 8e7f0efa9da88c54eeb3dc3ea2f550fbcbd184af Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Fri, 22 Jul 2022 19:15:51 -0700 Subject: [PATCH 3/3] fix minor comment --- build/common/installer/scripts/tomlparser-agent-config.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/common/installer/scripts/tomlparser-agent-config.rb b/build/common/installer/scripts/tomlparser-agent-config.rb index 168843c88..4cfbc45ee 100644 --- a/build/common/installer/scripts/tomlparser-agent-config.rb +++ b/build/common/installer/scripts/tomlparser-agent-config.rb @@ -172,7 +172,7 @@ def populateSettingValuesFromConfigMap(parsedConfig) if !fbitTailIgnoreOlder.nil? && !fbitTailIgnoreOlder.empty? if !re.match(fbitTailIgnoreOlder).nil? @fbitTailIgnoreOlder = fbitTailIgnoreOlder - puts "Using config map value: tail_ignore_older = #{@tail_ignore_older}" + puts "Using config map value: tail_ignore_older = #{@fbitTailIgnoreOlder}" else puts "config:warn: provided tail_ignore_older value is not valid hence using default value" end