diff --git a/build/common/installer/scripts/td-agent-bit-conf-customizer.rb b/build/common/installer/scripts/td-agent-bit-conf-customizer.rb index 995d72b87..5db387911 100644 --- a/build/common/installer/scripts/td-agent-bit-conf-customizer.rb +++ b/build/common/installer/scripts/td-agent-bit-conf-customizer.rb @@ -24,15 +24,16 @@ def substituteFluentBitPlaceHolders bufferChunkSize = ENV["FBIT_TAIL_BUFFER_CHUNK_SIZE"] bufferMaxSize = ENV["FBIT_TAIL_BUFFER_MAX_SIZE"] memBufLimit = ENV["FBIT_TAIL_MEM_BUF_LIMIT"] + ignoreOlder = ENV["FBIT_TAIL_IGNORE_OLDER"] - serviceInterval = (!interval.nil? && is_number?(interval) && interval.to_i > 0 ) ? interval : @default_service_interval + serviceInterval = (!interval.nil? && is_number?(interval) && interval.to_i > 0) ? interval : @default_service_interval serviceIntervalSetting = "Flush " + serviceInterval tailBufferChunkSize = (!bufferChunkSize.nil? && is_number?(bufferChunkSize) && bufferChunkSize.to_i > 0) ? bufferChunkSize : nil tailBufferMaxSize = (!bufferMaxSize.nil? && is_number?(bufferMaxSize) && bufferMaxSize.to_i > 0) ? bufferMaxSize : nil - if ((!tailBufferChunkSize.nil? && tailBufferMaxSize.nil?) || (!tailBufferChunkSize.nil? && !tailBufferMaxSize.nil? && tailBufferChunkSize.to_i > tailBufferMaxSize.to_i)) + if ((!tailBufferChunkSize.nil? && tailBufferMaxSize.nil?) || (!tailBufferChunkSize.nil? && !tailBufferMaxSize.nil? && tailBufferChunkSize.to_i > tailBufferMaxSize.to_i)) puts "config:warn buffer max size must be greater or equal to chunk size" tailBufferMaxSize = tailBufferChunkSize end @@ -54,6 +55,12 @@ def substituteFluentBitPlaceHolders new_contents = new_contents.gsub("\n ${TAIL_BUFFER_MAX_SIZE}\n", "\n") end + if !ignoreOlder.nil? && !ignoreOlder.empty? + new_contents = new_contents.gsub("${TAIL_IGNORE_OLDER}", "Ignore_Older " + ignoreOlder) + else + new_contents = new_contents.gsub("\n ${TAIL_IGNORE_OLDER}\n", "\n") + end + File.open(@td_agent_bit_conf_path, "w") { |file| file.puts new_contents } puts "config::Successfully substituted the placeholders in td-agent-bit.conf file" rescue => errorStr diff --git a/build/common/installer/scripts/tomlparser-agent-config.rb b/build/common/installer/scripts/tomlparser-agent-config.rb index a6b6ee0a1..4cfbc45ee 100644 --- a/build/common/installer/scripts/tomlparser-agent-config.rb +++ b/build/common/installer/scripts/tomlparser-agent-config.rb @@ -55,7 +55,7 @@ @fbitTailBufferChunkSizeMBs = 0 @fbitTailBufferMaxSizeMBs = 0 @fbitTailMemBufLimitMBs = 0 - +@fbitTailIgnoreOlder = "" def is_number?(value) true if Integer(value) rescue false @@ -145,7 +145,7 @@ def populateSettingValuesFromConfigMap(parsedConfig) end fbitTailBufferMaxSizeMBs = fbit_config[:tail_buf_maxsize_megabytes] - if !fbitTailBufferMaxSizeMBs.nil? && is_number?(fbitTailBufferMaxSizeMBs) && fbitTailBufferMaxSizeMBs.to_i > 0 + if !fbitTailBufferMaxSizeMBs.nil? && is_number?(fbitTailBufferMaxSizeMBs) && fbitTailBufferMaxSizeMBs.to_i > 0 if fbitTailBufferMaxSizeMBs.to_i >= @fbitTailBufferChunkSizeMBs @fbitTailBufferMaxSizeMBs = fbitTailBufferMaxSizeMBs.to_i puts "Using config map value: tail_buf_maxsize_megabytes = #{@fbitTailBufferMaxSizeMBs}" @@ -156,16 +156,27 @@ def populateSettingValuesFromConfigMap(parsedConfig) end end # in scenario - tail_buf_chunksize_megabytes provided but not tail_buf_maxsize_megabytes to prevent fbit crash - if @fbitTailBufferChunkSizeMBs > 0 && @fbitTailBufferMaxSizeMBs == 0 + if @fbitTailBufferChunkSizeMBs > 0 && @fbitTailBufferMaxSizeMBs == 0 @fbitTailBufferMaxSizeMBs = @fbitTailBufferChunkSizeMBs puts "config::warn: since tail_buf_maxsize_megabytes not provided hence using tail_buf_maxsize_megabytes=#{@fbitTailBufferMaxSizeMBs} which is same as the value of tail_buf_chunksize_megabytes" - end + end fbitTailMemBufLimitMBs = fbit_config[:tail_mem_buf_limit_megabytes] if !fbitTailMemBufLimitMBs.nil? && is_number?(fbitTailMemBufLimitMBs) && fbitTailMemBufLimitMBs.to_i > 0 @fbitTailMemBufLimitMBs = fbitTailMemBufLimitMBs.to_i puts "Using config map value: tail_mem_buf_limit_megabytes = #{@fbitTailMemBufLimitMBs}" end + + fbitTailIgnoreOlder = fbit_config[:tail_ignore_older] + re = /^[0-9]+[mhd]$/ + if !fbitTailIgnoreOlder.nil? && !fbitTailIgnoreOlder.empty? + if !re.match(fbitTailIgnoreOlder).nil? + @fbitTailIgnoreOlder = fbitTailIgnoreOlder + puts "Using config map value: tail_ignore_older = #{@fbitTailIgnoreOlder}" + else + puts "config:warn: provided tail_ignore_older value is not valid hence using default value" + end + end end end rescue => errorStr @@ -206,10 +217,15 @@ def populateSettingValuesFromConfigMap(parsedConfig) end if @fbitTailBufferMaxSizeMBs > 0 file.write("export FBIT_TAIL_BUFFER_MAX_SIZE=#{@fbitTailBufferMaxSizeMBs}\n") - end + end if @fbitTailMemBufLimitMBs > 0 file.write("export FBIT_TAIL_MEM_BUF_LIMIT=#{@fbitTailMemBufLimitMBs}\n") - end + end + + if !@fbitTailIgnoreOlder.nil? && !@fbitTailIgnoreOlder.empty? + file.write("export FBIT_TAIL_IGNORE_OLDER=#{@fbitTailIgnoreOlder}\n") + end + # Close file after writing all environment variables file.close else @@ -227,21 +243,25 @@ def get_command_windows(env_variable_name, env_variable_value) if !file.nil? if @fbitFlushIntervalSecs > 0 - commands = get_command_windows('FBIT_SERVICE_FLUSH_INTERVAL', @fbitFlushIntervalSecs) + commands = get_command_windows("FBIT_SERVICE_FLUSH_INTERVAL", @fbitFlushIntervalSecs) file.write(commands) end if @fbitTailBufferChunkSizeMBs > 0 - commands = get_command_windows('FBIT_TAIL_BUFFER_CHUNK_SIZE', @fbitTailBufferChunkSizeMBs) + commands = get_command_windows("FBIT_TAIL_BUFFER_CHUNK_SIZE", @fbitTailBufferChunkSizeMBs) file.write(commands) end if @fbitTailBufferMaxSizeMBs > 0 - commands = get_command_windows('FBIT_TAIL_BUFFER_MAX_SIZE', @fbitTailBufferMaxSizeMBs) + commands = get_command_windows("FBIT_TAIL_BUFFER_MAX_SIZE", @fbitTailBufferMaxSizeMBs) file.write(commands) - end + end if @fbitTailMemBufLimitMBs > 0 - commands = get_command_windows('FBIT_TAIL_MEM_BUF_LIMIT', @fbitTailMemBufLimitMBs) + commands = get_command_windows("FBIT_TAIL_MEM_BUF_LIMIT", @fbitTailMemBufLimitMBs) file.write(commands) - end + end + if !@fbitTailIgnoreOlder.nil? && !@fbitTailIgnoreOlder.empty? + commands = get_command_windows("FBIT_TAIL_IGNORE_OLDER", @fbitTailIgnoreOlder) + file.write(commands) + end # Close file after writing all environment variables file.close puts "****************End Config Processing********************" @@ -249,4 +269,4 @@ def get_command_windows(env_variable_name, env_variable_value) puts "Exception while opening file for writing config environment variables for WINDOWS LOG" puts "****************End Config Processing********************" end -end \ No newline at end of file +end diff --git a/build/linux/installer/conf/td-agent-bit.conf b/build/linux/installer/conf/td-agent-bit.conf index beba6a3ca..fe550ab62 100644 --- a/build/linux/installer/conf/td-agent-bit.conf +++ b/build/linux/installer/conf/td-agent-bit.conf @@ -26,7 +26,7 @@ Refresh_Interval 30 Path_Key filepath Skip_Long_Lines On - Ignore_Older 5m + ${TAIL_IGNORE_OLDER} Exclude_Path ${AZMON_CLUSTER_LOG_TAIL_EXCLUDE_PATH} [INPUT] diff --git a/build/windows/installer/conf/fluent-bit.conf b/build/windows/installer/conf/fluent-bit.conf index 1e2d8a93e..b43354e3f 100644 --- a/build/windows/installer/conf/fluent-bit.conf +++ b/build/windows/installer/conf/fluent-bit.conf @@ -25,7 +25,7 @@ Refresh_Interval 30 Path_Key filepath Skip_Long_Lines On - Ignore_Older 5m + ${TAIL_IGNORE_OLDER} Exclude_Path ${AZMON_CLUSTER_LOG_TAIL_EXCLUDE_PATH} [INPUT] diff --git a/kubernetes/container-azm-ms-agentconfig.yaml b/kubernetes/container-azm-ms-agentconfig.yaml index 8b9e2d718..a274724ef 100644 --- a/kubernetes/container-azm-ms-agentconfig.yaml +++ b/kubernetes/container-azm-ms-agentconfig.yaml @@ -46,8 +46,8 @@ data: # In the absence of this configmap, default value for containerlog_schema_version is "v1" # Supported values for this setting are "v1","v2" # See documentation at https://aka.ms/ContainerLogv2 for benefits of v2 schema over v1 schema before opting for "v2" schema - # containerlog_schema_version = "v2" - + # containerlog_schema_version = "v2" + prometheus-data-collection-settings: |- # Custom Prometheus metrics data collection settings @@ -135,7 +135,7 @@ data: # Alertable metrics configuration settings for completed jobs count [alertable_metrics_configuration_settings.job_completion_threshold] - # Threshold for completed job count , metric will be sent only for those jobs which were completed earlier than the following threshold + # Threshold for completed job count , metric will be sent only for those jobs which were completed earlier than the following threshold job_completion_threshold_time_minutes = 360 integrations: |- [integrations.azure_network_policy_manager] @@ -147,19 +147,21 @@ data: # Doc - https://github.com/microsoft/Docker-Provider/blob/ci_prod/Documentation/AgentSettings/ReadMe.md agent-settings: |- # prometheus scrape fluent bit settings for high scale - # buffer size should be greater than or equal to chunk size else we set it to chunk size. + # buffer size should be greater than or equal to chunk size else we set it to chunk size. [agent_settings.prometheus_fbit_settings] tcp_listener_chunk_size = 10 tcp_listener_buffer_size = 10 tcp_listener_mem_buf_limit = 200 - + # The following settings are "undocumented", we don't recommend uncommenting them unless directed by Microsoft. # They increase the maximum stdout/stderr log collection rate but will also cause higher cpu/memory usage. + ## Ref for more details about Ignore_Older - https://docs.fluentbit.io/manual/v/1.7/pipeline/inputs/tail # [agent_settings.fbit_config] # log_flush_interval_secs = "1" # default value is 15 # tail_mem_buf_limit_megabytes = "10" # default value is 10 # tail_buf_chunksize_megabytes = "1" # default value is 32kb (comment out this line for default) # tail_buf_maxsize_megabytes = "1" # defautl value is 32kb (comment out this line for default) + # tail_ignore_older = "5m" # default value same as fluent-bit default i.e.0m metadata: name: container-azm-ms-agentconfig