Sarah/fluentbit windows log

build/common/installer/scripts/td-agent-bit-conf-customizer.rb

@@ -3,6 +3,11 @@
 
 @td_agent_bit_conf_path = "/etc/opt/microsoft/docker-cimprov/td-agent-bit.conf"
 
+@os_type = ENV["OS_TYPE"]
+if !@os_type.nil? && !@os_type.empty? && @os_type.strip.casecmp("windows") == 0
+  @td_agent_bit_conf_path = "/etc/fluent-bit/fluent-bit.conf"
+end
+
 @default_service_interval = "15"
 @default_mem_buf_limit = "10"
 

build/linux/installer/scripts/tomlparser-agent-config.rb → build/common/installer/scripts/tomlparser-agent-config.rb

@@ -228,3 +228,37 @@ def populateSettingValuesFromConfigMap(parsedConfig)
   puts "Exception while opening file for writing config environment variables"
   puts "****************End Config Processing********************"
 end
+
+def get_command_windows(env_variable_name, env_variable_value)
+  return "[System.Environment]::SetEnvironmentVariable(\"#{env_variable_name}\", \"#{env_variable_value}\", \"Process\")" + "\n" + "[System.Environment]::SetEnvironmentVariable(\"#{env_variable_name}\", \"#{env_variable_value}\", \"Machine\")" + "\n"
+end
+
+if !@os_type.nil? && !@os_type.empty? && @os_type.strip.casecmp("windows") == 0
+  # Write the settings to file, so that they can be set as environment variables
+  file = File.open("setagentenv.ps1", "w")
+
+  if !file.nil?
+    if @fbitFlushIntervalSecs > 0
+      commands = get_command_windows('FBIT_SERVICE_FLUSH_INTERVAL', @fbitFlushIntervalSecs)
+      file.write(commands)
+    end
+    if @fbitTailBufferChunkSizeMBs > 0
+      commands = get_command_windows('FBIT_TAIL_BUFFER_CHUNK_SIZE', @fbitTailBufferChunkSizeMBs)
+      file.write(commands)
+    end
+    if @fbitTailBufferMaxSizeMBs > 0
+      commands = get_command_windows('FBIT_TAIL_BUFFER_MAX_SIZE', @fbitTailBufferMaxSizeMBs)
+      file.write(commands)
+    end 
+    if @fbitTailMemBufLimitMBs > 0
+      commands = get_command_windows('FBIT_TAIL_MEM_BUF_LIMIT', @fbitTailMemBufLimitMBs)
+      file.write(commands)
+    end 
+    # Close file after writing all environment variables
+    file.close
+    puts "****************End Config Processing********************"
+  else
+    puts "Exception while opening file for writing config environment variables for WINDOWS LOG"
+    puts "****************End Config Processing********************"
+  end
+end

build/common/installer/scripts/tomlparser.rb

@@ -29,6 +29,8 @@
 @adxDatabaseName = "containerinsights" # default for all configurations
 if !@os_type.nil? && !@os_type.empty? && @os_type.strip.casecmp("windows") == 0
   @containerLogsRoute = "v1" # default is v1 for windows until windows agent integrates windows ama
+  # This path format is necessary for fluent-bit in windows
+  @logTailPath = "C:\\var\\log\\containers\\*.log"
 end
 # Use parser to parse the configmap toml file to a ruby structure
 def parseConfigMap

build/linux/installer/datafiles/base_container.data

@@ -34,7 +34,7 @@ MAINTAINER:              'Microsoft Corporation'
 /etc/opt/microsoft/docker-cimprov/td-agent-bit.conf;			            build/linux/installer/conf/td-agent-bit.conf;                    644; root; root
 /etc/opt/microsoft/docker-cimprov/td-agent-bit-prom-side-car.conf;	   build/linux/installer/conf/td-agent-bit-prom-side-car.conf;                    644; root; root
 /etc/opt/microsoft/docker-cimprov/td-agent-bit-rs.conf;			         build/linux/installer/conf/td-agent-bit-rs.conf;                    644; root; root
-/etc/opt/microsoft/docker-cimprov/azm-containers-parser.conf;	         build/linux/installer/conf/azm-containers-parser.conf;                    644; root; root
+/etc/opt/microsoft/docker-cimprov/azm-containers-parser.conf;	         build/common/installer/conf/azm-containers-parser.conf;                    644; root; root
 /etc/opt/microsoft/docker-cimprov/out_oms.conf;			                  build/linux/installer/conf/out_oms.conf;                    644; root; root
 /etc/opt/microsoft/docker-cimprov/test.json;			                     build/linux/installer/conf/test.json;                    644; root; root
 /etc/opt/microsoft/docker-cimprov/telegraf.conf;			               build/linux/installer/conf/telegraf.conf;                    644; root; root
@@ -48,7 +48,7 @@ MAINTAINER:              'Microsoft Corporation'
 /opt/tomlparser-metric-collection-config.rb;                            build/linux/installer/scripts/tomlparser-metric-collection-config.rb;     755; root; root
 
 
-/opt/tomlparser-agent-config.rb;                                build/linux/installer/scripts/tomlparser-agent-config.rb;     755; root; root
+/opt/tomlparser-agent-config.rb;                                build/common/installer/scripts/tomlparser-agent-config.rb;     755; root; root
 /opt/tomlparser.rb;                                             build/common/installer/scripts/tomlparser.rb;     755; root; root
 /opt/td-agent-bit-conf-customizer.rb;                           build/common/installer/scripts/td-agent-bit-conf-customizer.rb;     755; root; root
 /opt/ConfigParseErrorLogger.rb;                                 build/common/installer/scripts/ConfigParseErrorLogger.rb;           755; root; root

build/windows/installer/conf/fluent-bit.conf

@@ -1,25 +1,54 @@
 [SERVICE]
-    Flush        15
-    Daemon       Off
-    Log_Level    info
-    Log_File      /etc/fluent-bit/fluent-bit.log
+    #Default service flush interval is 15 seconds
+    ${SERVICE_FLUSH_INTERVAL}
+    Daemon          Off
+    storage.path  /etc/fluent-bit/flbstore/
+    storage.sync  normal
+    storage.checksum off
+    storage.backlog.mem_limit 10M
+    Log_Level       info
+    Parsers_File    /etc/fluent-bit/azm-containers-parser.conf
+    Log_File        /etc/fluent-bit/fluent-bit.log
 
 [INPUT]
-    Name            forward
-    Listen          127.0.0.1
-    Port            25230
-    Mem_Buf_Limit   10m
-    Chunk_Size      32
-    Buffer_Size     64
+    Name                tail
+    Tag                 oms.container.log.la.*
+    Path                ${AZMON_LOG_TAIL_PATH}
+    Read_from_Head      true
+    DB                  C:\\var\\log\\omsagent-fblogs.db
+    DB.Sync             Off
+    Parser              docker
+    ${TAIL_MEM_BUF_LIMIT}
+    ${TAIL_BUFFER_CHUNK_SIZE}
+    ${TAIL_BUFFER_MAX_SIZE}
+    Rotate_Wait         20
+    Refresh_Interval    30
+    Path_Key            filepath
+    Skip_Long_Lines     On
+    Ignore_Older        5m
+    Exclude_Path        ${AZMON_CLUSTER_LOG_TAIL_EXCLUDE_PATH}
 
 [INPUT]
-    Name        tcp
-    Tag         oms.container.perf.telegraf.*
-    Listen      0.0.0.0
-    Port        25229
-    Chunk_Size  32
-    Buffer_Size 64
-    Mem_Buf_Limit 5m
+    Name            tail
+    Tag             oms.container.log.flbplugin.*
+    Path            C:\\var\\log\\containers\\omsagent*.log
+    Read_from_Head  true
+    DB              C:\\var\\log\\omsagent-fluentbit-containers.db
+    DB.Sync         Off
+    Parser          docker
+    Mem_Buf_Limit   1m
+    Path_Key        filepath
+    Skip_Long_Lines On
+    Ignore_Older    2m
+
+[INPUT]
+    Name                tcp
+    Tag                 oms.container.perf.telegraf.*
+    Listen              0.0.0.0
+    Port                25229
+    Chunk_Size          32
+    Buffer_Size         64
+    Mem_Buf_Limit       5m
 
 [OUTPUT]
     Name  oms

build/windows/installer/conf/fluent.conf

@@ -11,31 +11,6 @@
   @log_level debug
 </source>
 
-<source>
-  @type tail
-  path "#{ENV['AZMON_LOG_TAIL_PATH']}"
-  exclude_path "#{ENV['AZMON_CLUSTER_LOG_TAIL_EXCLUDE_PATH']}"
-  pos_file /var/opt/microsoft/fluent/fluentd-containers.log.pos
-  tag oms.container.log.la
-  @log_level trace
-  path_key tailed_path
-  limit_recently_modified 5m
-  # if the container runtime is non docker then this will be updated to fluent-cri-parser.conf during container startup
-  @include fluent-docker-parser.conf
-</source>
-
-<source>
-  @type tail
-  path  /var/log/containers/omsagent*.log
-  pos_file /opt/microsoft/fluent/omsagent-fluentd-containers.log.pos
-  tag oms.container.log.flbplugin
-  @log_level trace
-  path_key tailed_path
-  read_from_head true
-  # if the container runtime is non docker then this will be updated to fluent-cri-parser.conf during container startup
-  @include fluent-docker-parser.conf
-</source>
-
 #custom_metrics_mdm filter plugin
 <filter mdm.cadvisorperf**>
   @type cadvisor2mdm
@@ -44,23 +19,6 @@
   @log_level info
 </filter>
 
-<filter oms.container.log.la>
-  @type grep
-  <exclude>
-    key stream
-    pattern "#{ENV['AZMON_LOG_EXCLUSION_REGEX_PATTERN']}"
-  </exclude>
-</filter>
-
-<filter  oms.container.**>
-  @type record_transformer
-  # fluent-plugin-record-modifier more light-weight but needs to be installed (dependency worth it?)
-  remove_keys tailed_path
-  <record>
-    filepath ${record["tailed_path"]}
-  </record>
-</filter>
-
 <match mdm.cadvisorperf**>
   @type mdm
   @log_level debug
@@ -77,29 +35,3 @@
   </buffer>
   retry_mdm_post_wait_minutes 30
 </match>
-
-<match oms.container.**>
-  @type forward
-  send_timeout 60s
-  recover_wait 10s
-  hard_timeout 60s
-  heartbeat_type none
-  ignore_network_errors_at_startup true
-  <server>
-    name logaggregationserver
-    host 127.0.0.1
-    port 25230
-    weight 60
-  </server>
-
-  <buffer>
-    overflow_action throw_exception
-    chunk_limit_size 32k
-    queued_chunks_limit_size 256
-    flush_interval 1
-    flush_thread_interval 0.5
-    flush_thread_burst_interval 0.01
-    flush_thread_count 4
-    retry_forever true
-  </buffer>
-</match>

kubernetes/linux/setup.sh

@@ -44,7 +44,7 @@ chmod 777 /opt/telegraf
 wget -qO - https://packages.fluentbit.io/fluentbit.key | sudo apt-key add -
 sudo echo "deb https://packages.fluentbit.io/ubuntu/xenial xenial main" >> /etc/apt/sources.list
 sudo apt-get update
-sudo apt-get install td-agent-bit=1.6.8 -y
+sudo apt-get install td-agent-bit=1.7.8 -y
 
 # install ruby2.6
 sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys F5DA5F09C3173AA6

kubernetes/omsagent.yaml

@@ -783,6 +783,12 @@ spec:
            cpu: 200m
            memory: 600Mi
          env:
+          - name: FBIT_SERVICE_FLUSH_INTERVAL
+            value: "15"
+          - name: FBIT_TAIL_BUFFER_CHUNK_SIZE
+            value: "1"
+          - name: FBIT_TAIL_BUFFER_MAX_SIZE
+            value: "1"
           # azure devops pipeline uses AKS_RESOURCE_ID and AKS_REGION hence ensure to uncomment these
           - name: AKS_RESOURCE_ID
             value: "VALUE_AKS_RESOURCE_ID_VALUE"

kubernetes/windows/Dockerfile

@@ -57,10 +57,8 @@ COPY ./omsagentwindows/out_oms.so /opt/omsagentwindows/out_oms.so
 
 # copy fluent, fluent-bit and out_oms conf files
 COPY ./omsagentwindows/installer/conf/fluent.conf /etc/fluent/
-# copy fluent docker and cri parser conf files
-COPY ./omsagentwindows/installer/conf/fluent-cri-parser.conf /etc/fluent/
-COPY ./omsagentwindows/installer/conf/fluent-docker-parser.conf /etc/fluent/
 COPY ./omsagentwindows/installer/conf/fluent-bit.conf /etc/fluent-bit
+COPY ./omsagentwindows/installer/conf/azm-containers-parser.conf /etc/fluent-bit/
 COPY ./omsagentwindows/installer/conf/out_oms.conf /etc/omsagentwindows
 
 # copy telegraf conf file

kubernetes/windows/Dockerfile-dev-image

@@ -19,10 +19,8 @@ COPY ./omsagentwindows/out_oms.so /opt/omsagentwindows/out_oms.so
 
 # copy fluent, fluent-bit and out_oms conf files
 COPY ./omsagentwindows/installer/conf/fluent.conf /etc/fluent/
-# copy fluent docker and cri parser conf files
-COPY ./omsagentwindows/installer/conf/fluent-cri-parser.conf /etc/fluent/
-COPY ./omsagentwindows/installer/conf/fluent-docker-parser.conf /etc/fluent/
 COPY ./omsagentwindows/installer/conf/fluent-bit.conf /etc/fluent-bit
+COPY ./omsagentwindows/installer/conf/azm-containers-parser.conf /etc/fluent-bit/
 COPY ./omsagentwindows/installer/conf/out_oms.conf /etc/omsagentwindows
 
 # copy telegraf conf file

kubernetes/windows/main.ps1

@@ -297,6 +297,13 @@ function Set-EnvironmentVariables {
     # run config parser
     ruby /opt/omsagentwindows/scripts/ruby/tomlparser.rb
     .\setenv.ps1
+
+    #Parse the configmap to set the right environment variables for agent config.
+    ruby /opt/omsagentwindows/scripts/ruby/tomlparser-agent-config.rb
+    .\setagentenv.ps1
+
+    #Replace placeholders in fluent-bit.conf
+    ruby /opt/omsagentwindows/scripts/ruby/td-agent-bit-conf-customizer.rb
 
     # run mdm config parser
     ruby /opt/omsagentwindows/scripts/ruby/tomlparser-mdm-metrics-config.rb
@@ -418,18 +425,18 @@ function Get-ContainerRuntime {
 
 function Start-Fluent-Telegraf {
 
-    # Run fluent-bit service first so that we do not miss any logs being forwarded by the fluentd service and telegraf service.
+    $containerRuntime = Get-ContainerRuntime
+
+    # Run fluent-bit service first so that we do not miss any logs being forwarded by the telegraf service.
     # Run fluent-bit as a background job. Switch this to a windows service once fluent-bit supports natively running as a windows service
     Start-Job -ScriptBlock { Start-Process -NoNewWindow -FilePath "C:\opt\fluent-bit\bin\fluent-bit.exe" -ArgumentList @("-c", "C:\etc\fluent-bit\fluent-bit.conf", "-e", "C:\opt\omsagentwindows\out_oms.so") }
 
-    $containerRuntime = Get-ContainerRuntime
-
     #register fluentd as a service and start
     # there is a known issues with win32-service https://github.com/chef/win32-service/issues/70
     if (![string]::IsNullOrEmpty($containerRuntime) -and [string]$containerRuntime.StartsWith('docker') -eq $false) {
         # change parser from docker to cri if the container runtime is not docker
         Write-Host "changing parser from Docker to CRI since container runtime : $($containerRuntime) and which is non-docker"
-        (Get-Content -Path C:/etc/fluent/fluent.conf -Raw) -replace 'fluent-docker-parser.conf', 'fluent-cri-parser.conf' | Set-Content C:/etc/fluent/fluent.conf
+        (Get-Content -Path C:/etc/fluent-bit/fluent-bit.conf -Raw) -replace 'docker', 'cri' | Set-Content C:/etc/fluent-bit/fluent-bit.conf
     }
 
     # Start telegraf only in sidecar scraping mode

kubernetes/windows/setup.ps1

@@ -1,8 +1,3 @@
-#
-################# Dangerous to use appveyor links - the builds are removed after 6 months
-#
-#ARG FLUENTBIT_URL=https://ci.appveyor.com/api/buildjobs/37lho3xf8j5i6crj/artifacts/build%2Ftd-agent-bit-1.4.0-win64.zip
-
 Write-Host ('Creating folder structure')
     New-Item -Type Directory -Path /installation -ErrorAction SilentlyContinue
 
@@ -21,7 +16,7 @@ Write-Host ('Creating folder structure')
 Write-Host ('Installing Fluent Bit');
 
     try {
-        $fluentBitUri='https://github.com/microsoft/OMS-docker/releases/download/winakslogagent/td-agent-bit-1.4.0-win64.zip'
+        $fluentBitUri='https://fluentbit.io/releases/1.7/td-agent-bit-1.7.8-win64.zip'
         Invoke-WebRequest -Uri $fluentBitUri -OutFile /installation/td-agent-bit.zip
         Expand-Archive -Path /installation/td-agent-bit.zip -Destination /installation/fluent-bit
         Move-Item -Path /installation/fluent-bit/*/* -Destination /opt/fluent-bit/ -ErrorAction SilentlyContinue