Gangams/fix log loss inode reuse

build/common/installer/scripts/td-agent-bit-conf-customizer.rb

@@ -24,15 +24,16 @@ def substituteFluentBitPlaceHolders
     bufferChunkSize = ENV["FBIT_TAIL_BUFFER_CHUNK_SIZE"]
     bufferMaxSize = ENV["FBIT_TAIL_BUFFER_MAX_SIZE"]
     memBufLimit = ENV["FBIT_TAIL_MEM_BUF_LIMIT"]
+    ignoreOlder = ENV["FBIT_TAIL_IGNORE_OLDER"]
 
-    serviceInterval = (!interval.nil? && is_number?(interval) && interval.to_i > 0 ) ? interval : @default_service_interval
+    serviceInterval = (!interval.nil? && is_number?(interval) && interval.to_i > 0) ? interval : @default_service_interval
     serviceIntervalSetting = "Flush         " + serviceInterval
 
     tailBufferChunkSize = (!bufferChunkSize.nil? && is_number?(bufferChunkSize) && bufferChunkSize.to_i > 0) ? bufferChunkSize : nil
 
     tailBufferMaxSize = (!bufferMaxSize.nil? && is_number?(bufferMaxSize) && bufferMaxSize.to_i > 0) ? bufferMaxSize : nil
 
-    if ((!tailBufferChunkSize.nil? && tailBufferMaxSize.nil?) ||  (!tailBufferChunkSize.nil? && !tailBufferMaxSize.nil? && tailBufferChunkSize.to_i > tailBufferMaxSize.to_i))
+    if ((!tailBufferChunkSize.nil? && tailBufferMaxSize.nil?) || (!tailBufferChunkSize.nil? && !tailBufferMaxSize.nil? && tailBufferChunkSize.to_i > tailBufferMaxSize.to_i))
       puts "config:warn buffer max size must be greater or equal to chunk size"
       tailBufferMaxSize = tailBufferChunkSize
     end
@@ -54,6 +55,12 @@ def substituteFluentBitPlaceHolders
       new_contents = new_contents.gsub("\n    ${TAIL_BUFFER_MAX_SIZE}\n", "\n")
     end
 
+    if !ignoreOlder.nil? && !ignoreOlder.empty?
+      new_contents = new_contents.gsub("${TAIL_IGNORE_OLDER}", "Ignore_Older " + ignoreOlder)
+    else
+      new_contents = new_contents.gsub("\n    ${TAIL_IGNORE_OLDER}\n", "\n")
+    end
+
     File.open(@td_agent_bit_conf_path, "w") { |file| file.puts new_contents }
     puts "config::Successfully substituted the placeholders in td-agent-bit.conf file"
   rescue => errorStr

build/common/installer/scripts/tomlparser-agent-config.rb

@@ -55,7 +55,7 @@
 @fbitTailBufferChunkSizeMBs = 0
 @fbitTailBufferMaxSizeMBs = 0
 @fbitTailMemBufLimitMBs = 0
-
+@fbitTailIgnoreOlder = ""
 
 def is_number?(value)
   true if Integer(value) rescue false
@@ -145,7 +145,7 @@ def populateSettingValuesFromConfigMap(parsedConfig)
         end
 
         fbitTailBufferMaxSizeMBs = fbit_config[:tail_buf_maxsize_megabytes]
-        if !fbitTailBufferMaxSizeMBs.nil? && is_number?(fbitTailBufferMaxSizeMBs) && fbitTailBufferMaxSizeMBs.to_i > 0           
+        if !fbitTailBufferMaxSizeMBs.nil? && is_number?(fbitTailBufferMaxSizeMBs) && fbitTailBufferMaxSizeMBs.to_i > 0
           if fbitTailBufferMaxSizeMBs.to_i >= @fbitTailBufferChunkSizeMBs
             @fbitTailBufferMaxSizeMBs = fbitTailBufferMaxSizeMBs.to_i
             puts "Using config map value: tail_buf_maxsize_megabytes = #{@fbitTailBufferMaxSizeMBs}"
@@ -156,16 +156,27 @@ def populateSettingValuesFromConfigMap(parsedConfig)
           end
         end
         # in scenario - tail_buf_chunksize_megabytes provided but not tail_buf_maxsize_megabytes to prevent fbit crash
-        if  @fbitTailBufferChunkSizeMBs > 0  && @fbitTailBufferMaxSizeMBs == 0
+        if @fbitTailBufferChunkSizeMBs > 0 && @fbitTailBufferMaxSizeMBs == 0
           @fbitTailBufferMaxSizeMBs = @fbitTailBufferChunkSizeMBs
           puts "config::warn: since tail_buf_maxsize_megabytes not provided hence using tail_buf_maxsize_megabytes=#{@fbitTailBufferMaxSizeMBs} which is same as the value of tail_buf_chunksize_megabytes"
-        end 
+        end
 
         fbitTailMemBufLimitMBs = fbit_config[:tail_mem_buf_limit_megabytes]
         if !fbitTailMemBufLimitMBs.nil? && is_number?(fbitTailMemBufLimitMBs) && fbitTailMemBufLimitMBs.to_i > 0
           @fbitTailMemBufLimitMBs = fbitTailMemBufLimitMBs.to_i
           puts "Using config map value: tail_mem_buf_limit_megabytes  = #{@fbitTailMemBufLimitMBs}"
         end
+
+        fbitTailIgnoreOlder = fbit_config[:tail_ignore_older]
+        re = /^[0-9]+[mhd]$/
+        if !fbitTailIgnoreOlder.nil? && !fbitTailIgnoreOlder.empty?
+          if !re.match(fbitTailIgnoreOlder).nil?
+            @fbitTailIgnoreOlder = fbitTailIgnoreOlder
+            puts "Using config map value: tail_ignore_older  = #{@fbitTailIgnoreOlder}"
+          else
+            puts "config:warn: provided tail_ignore_older value is not valid hence using default value"
+          end
+        end
       end
     end
   rescue => errorStr
@@ -206,10 +217,15 @@ def populateSettingValuesFromConfigMap(parsedConfig)
   end
   if @fbitTailBufferMaxSizeMBs > 0
     file.write("export FBIT_TAIL_BUFFER_MAX_SIZE=#{@fbitTailBufferMaxSizeMBs}\n")
-  end 
+  end
   if @fbitTailMemBufLimitMBs > 0
     file.write("export FBIT_TAIL_MEM_BUF_LIMIT=#{@fbitTailMemBufLimitMBs}\n")
-  end 
+  end
+
+  if !@fbitTailIgnoreOlder.nil? && !@fbitTailIgnoreOlder.empty?
+    file.write("export FBIT_TAIL_IGNORE_OLDER=#{@fbitTailIgnoreOlder}\n")
+  end
+
   # Close file after writing all environment variables
   file.close
 else
@@ -227,26 +243,30 @@ def get_command_windows(env_variable_name, env_variable_value)
 
   if !file.nil?
     if @fbitFlushIntervalSecs > 0
-      commands = get_command_windows('FBIT_SERVICE_FLUSH_INTERVAL', @fbitFlushIntervalSecs)
+      commands = get_command_windows("FBIT_SERVICE_FLUSH_INTERVAL", @fbitFlushIntervalSecs)
       file.write(commands)
     end
     if @fbitTailBufferChunkSizeMBs > 0
-      commands = get_command_windows('FBIT_TAIL_BUFFER_CHUNK_SIZE', @fbitTailBufferChunkSizeMBs)
+      commands = get_command_windows("FBIT_TAIL_BUFFER_CHUNK_SIZE", @fbitTailBufferChunkSizeMBs)
       file.write(commands)
     end
     if @fbitTailBufferMaxSizeMBs > 0
-      commands = get_command_windows('FBIT_TAIL_BUFFER_MAX_SIZE', @fbitTailBufferMaxSizeMBs)
+      commands = get_command_windows("FBIT_TAIL_BUFFER_MAX_SIZE", @fbitTailBufferMaxSizeMBs)
       file.write(commands)
-    end 
+    end
     if @fbitTailMemBufLimitMBs > 0
-      commands = get_command_windows('FBIT_TAIL_MEM_BUF_LIMIT', @fbitTailMemBufLimitMBs)
+      commands = get_command_windows("FBIT_TAIL_MEM_BUF_LIMIT", @fbitTailMemBufLimitMBs)
       file.write(commands)
-    end 
+    end
+    if !@fbitTailIgnoreOlder.nil? && !@fbitTailIgnoreOlder.empty?
+      commands = get_command_windows("FBIT_TAIL_IGNORE_OLDER", @fbitTailIgnoreOlder)
+      file.write(commands)
+    end
     # Close file after writing all environment variables
     file.close
     puts "****************End Config Processing********************"
   else
     puts "Exception while opening file for writing config environment variables for WINDOWS LOG"
     puts "****************End Config Processing********************"
   end
-end
+end

build/linux/installer/conf/td-agent-bit.conf

@@ -26,7 +26,7 @@
     Refresh_Interval 30
     Path_Key filepath
     Skip_Long_Lines On
-    Ignore_Older 5m
+    ${TAIL_IGNORE_OLDER}
     Exclude_Path ${AZMON_CLUSTER_LOG_TAIL_EXCLUDE_PATH}
 
 [INPUT]

build/windows/installer/conf/fluent-bit.conf

@@ -25,7 +25,7 @@
     Refresh_Interval    30
     Path_Key            filepath
     Skip_Long_Lines     On
-    Ignore_Older        5m
+    ${TAIL_IGNORE_OLDER}
     Exclude_Path        ${AZMON_CLUSTER_LOG_TAIL_EXCLUDE_PATH}
 
 [INPUT]

kubernetes/container-azm-ms-agentconfig.yaml

@@ -46,8 +46,8 @@ data:
           # In the absence of this configmap, default value for containerlog_schema_version is "v1"
           # Supported values for this setting are "v1","v2"
           # See documentation at https://aka.ms/ContainerLogv2 for benefits of v2 schema over v1 schema before opting for "v2" schema
-          # containerlog_schema_version = "v2"   
-       
+          # containerlog_schema_version = "v2"
+
 
   prometheus-data-collection-settings: |-
     # Custom Prometheus metrics data collection settings
@@ -135,7 +135,7 @@ data:
 
     # Alertable metrics configuration settings for completed jobs count
     [alertable_metrics_configuration_settings.job_completion_threshold]
-        # Threshold for completed job count , metric will be sent only for those jobs which were completed earlier than the following threshold 
+        # Threshold for completed job count , metric will be sent only for those jobs which were completed earlier than the following threshold
         job_completion_threshold_time_minutes = 360
   integrations: |-
     [integrations.azure_network_policy_manager]
@@ -147,19 +147,21 @@ data:
 # Doc - https://github.com/microsoft/Docker-Provider/blob/ci_prod/Documentation/AgentSettings/ReadMe.md
   agent-settings: |-
     # prometheus scrape fluent bit settings for high scale
-    # buffer size should be greater than or equal to chunk size else we set it to chunk size. 
+    # buffer size should be greater than or equal to chunk size else we set it to chunk size.
     [agent_settings.prometheus_fbit_settings]
       tcp_listener_chunk_size = 10
       tcp_listener_buffer_size = 10
       tcp_listener_mem_buf_limit = 200
-    
+
     # The following settings are "undocumented", we don't recommend uncommenting them unless directed by Microsoft.
     # They increase the maximum stdout/stderr log collection rate but will also cause higher cpu/memory usage.
+    ## Ref for more details about Ignore_Older -  https://docs.fluentbit.io/manual/v/1.7/pipeline/inputs/tail
     # [agent_settings.fbit_config]
     #   log_flush_interval_secs = "1"                 # default value is 15
     #   tail_mem_buf_limit_megabytes = "10"           # default value is 10
     #   tail_buf_chunksize_megabytes = "1"            # default value is 32kb (comment out this line for default)
     #   tail_buf_maxsize_megabytes = "1"              # defautl value is 32kb (comment out this line for default)
+    #   tail_ignore_older = "5m"                      # default value same as fluent-bit default i.e.0m
 
 metadata:
   name: container-azm-ms-agentconfig