microsoft · pfrcks · Aug 11, 2022 · Aug 7, 2020 · Aug 13, 2020 · Aug 14, 2020
@@ -115,14 +115,14 @@ jobs:
         az acr login -n ${{ variables.containerRegistry }}
 
         if [ "$(Build.Reason)" != "PullRequest" ]; then
-          docker buildx build --platform linux/amd64,linux/arm64 --tag ${{ variables.repoImageName }}:$(linuxImagetag) -f kubernetes/linux/Dockerfile.multiarch --metadata-file $(Build.ArtifactStagingDirectory)/linux/metadata.json --build-arg IMAGE_TAG=$(linuxImagetag) --push .
+          docker buildx build --platform linux/amd64,linux/arm64 --tag ${{ variables.repoImageName }}:$(linuxImagetag) -f kubernetes/linux/Dockerfile.multiarch --metadata-file $(Build.ArtifactStagingDirectory)/linux/metadata.json --build-arg IMAGE_TAG=$(linuxImagetag) --build-arg GOLANG_BASE_IMAGE=$(GOLANG_BASE_IMAGE) --build-arg CI_BASE_IMAGE=$(CI_BASE_IMAGE) --push .
 
           docker pull ${{ variables.repoImageName }}:$(linuxImagetag)
         else
-          docker buildx build --platform linux/amd64,linux/arm64 --tag ${{ variables.repoImageName }}:$(linuxImagetag) -f kubernetes/linux/Dockerfile.multiarch --metadata-file $(Build.ArtifactStagingDirectory)/linux/metadata.json --build-arg IMAGE_TAG=$(linuxImagetag) .
+          docker buildx build --platform linux/amd64,linux/arm64 --tag ${{ variables.repoImageName }}:$(linuxImagetag) -f kubernetes/linux/Dockerfile.multiarch --metadata-file $(Build.ArtifactStagingDirectory)/linux/metadata.json --build-arg IMAGE_TAG=$(linuxImagetag) --build-arg GOLANG_BASE_IMAGE=$(GOLANG_BASE_IMAGE) --build-arg CI_BASE_IMAGE=$(CI_BASE_IMAGE) .
 
           # load the multi-arch image to run tests
-          docker buildx build --tag ${{ variables.repoImageName }}:$(linuxImagetag) -f kubernetes/linux/Dockerfile.multiarch --metadata-file $(Build.ArtifactStagingDirectory)/linux/metadata.json --build-arg IMAGE_TAG=$(linuxImagetag) --load .
+          docker buildx build --tag ${{ variables.repoImageName }}:$(linuxImagetag) -f kubernetes/linux/Dockerfile.multiarch --metadata-file $(Build.ArtifactStagingDirectory)/linux/metadata.json --build-arg IMAGE_TAG=$(linuxImagetag) --build-arg GOLANG_BASE_IMAGE=$(GOLANG_BASE_IMAGE) --build-arg CI_BASE_IMAGE=$(CI_BASE_IMAGE) --load .
         fi
 
         curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin
@@ -135,14 +135,14 @@ jobs:
     condition: eq(variables.IS_PR, true)
     inputs:
       BuildDropPath: '$(Build.ArtifactStagingDirectory)/linux'
-      DockerImagesToScan: 'golang:1.15.14, ubuntu:18.04'
+      DockerImagesToScan: '$(GOLANG_BASE_IMAGE), $(CI_BASE_IMAGE)'
 
   - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0
     displayName: 'Generation Task'
     condition: eq(variables.IS_PR, false)
     inputs:
       BuildDropPath: '$(Build.ArtifactStagingDirectory)/linux'
-      DockerImagesToScan: 'golang:1.15.14, ubuntu:18.04, ${{ variables.repoImageName }}:$(linuxImagetag)'
+      DockerImagesToScan: '$(GOLANG_BASE_IMAGE), $(CI_BASE_IMAGE), ${{ variables.repoImageName }}:$(linuxImagetag)'
 
   - task: PublishBuildArtifacts@1
     inputs:

@@ -119,14 +119,14 @@ jobs:
         az acr login -n ${{ variables.containerRegistry }}
 
         if [ "$(Build.Reason)" != "PullRequest" ]; then
-          docker buildx build --platform linux/amd64,linux/arm64 --tag ${{ variables.repoImageNameLinux }}:$(linuxImagetag) -f kubernetes/linux/Dockerfile.multiarch --metadata-file $(Build.ArtifactStagingDirectory)/linux/metadata.json --push .
+          docker buildx build --platform linux/amd64,linux/arm64 --tag ${{ variables.repoImageNameLinux }}:$(linuxImagetag) -f kubernetes/linux/Dockerfile.multiarch --metadata-file $(Build.ArtifactStagingDirectory)/linux/metadata.json --build-arg GOLANG_BASE_IMAGE=$(GOLANG_BASE_IMAGE) --build-arg CI_BASE_IMAGE=$(CI_BASE_IMAGE) --push .
 
           docker pull ${{ variables.repoImageNameLinux }}:$(linuxImagetag)
         else
-          docker buildx build --platform linux/amd64,linux/arm64 --tag ${{ variables.repoImageNameLinux }}:$(linuxImagetag) -f kubernetes/linux/Dockerfile.multiarch --metadata-file $(Build.ArtifactStagingDirectory)/linux/metadata.json .
+          docker buildx build --platform linux/amd64,linux/arm64 --tag ${{ variables.repoImageNameLinux }}:$(linuxImagetag) -f kubernetes/linux/Dockerfile.multiarch --metadata-file $(Build.ArtifactStagingDirectory)/linux/metadata.json --build-arg GOLANG_BASE_IMAGE=$(GOLANG_BASE_IMAGE) --build-arg CI_BASE_IMAGE=$(CI_BASE_IMAGE) .
 
           # load the multi-arch image to run tests
-          docker buildx build --tag ${{ variables.repoImageNameLinux }}:$(linuxImagetag) -f kubernetes/linux/Dockerfile.multiarch --metadata-file $(Build.ArtifactStagingDirectory)/linux/metadata.json --load .
+          docker buildx build --tag ${{ variables.repoImageNameLinux }}:$(linuxImagetag) -f kubernetes/linux/Dockerfile.multiarch --metadata-file $(Build.ArtifactStagingDirectory)/linux/metadata.json --build-arg GOLANG_BASE_IMAGE=$(GOLANG_BASE_IMAGE) --build-arg CI_BASE_IMAGE=$(CI_BASE_IMAGE) --load .
         fi
 
         curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin
@@ -138,14 +138,14 @@ jobs:
     condition: eq(variables.IS_PR, true)
     inputs:
       BuildDropPath: '$(Build.ArtifactStagingDirectory)/linux'
-      DockerImagesToScan: 'golang:1.15.14, ubuntu:18.04'
+      DockerImagesToScan: '$(GOLANG_BASE_IMAGE), $(CI_BASE_IMAGE)'
 
   - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0
     displayName: 'Generation Task'
     condition: eq(variables.IS_PR, false)
     inputs:
       BuildDropPath: '$(Build.ArtifactStagingDirectory)/linux'
-      DockerImagesToScan: 'golang:1.15.14, ubuntu:18.04, ${{ variables.repoImageNameLinux }}:$(linuxImagetag)'
+      DockerImagesToScan: '$(GOLANG_BASE_IMAGE), $(CI_BASE_IMAGE), ${{ variables.repoImageNameLinux }}:$(linuxImagetag)'
 
   - task: PublishBuildArtifacts@1
     inputs:

@@ -1,27 +1,10 @@
-# related to telegraf
-#[vishwa] - Fix telegraf & test all for next release - see work item #https://msazure.visualstudio.com/InfrastructureInsights/_workitems/edit/13322134
-# Unfixed as of 4/28/2022
+# telegraf vulnerabilities
 CVE-2019-3826
-
-#still present in mdsd telegraf
-CVE-2021-42836
-
-# ruby in /usr/lib
-CVE-2020-36327
-CVE-2021-43809
-CVE-2021-41816
-CVE-2021-41819
-CVE-2021-31799
-CVE-2021-28965
+CVE-2022-29190
+CVE-2022-29222
+CVE-2022-29189
+CVE-2022-1996
 
 #dpkg vulnerability in ubuntu
 CVE-2022-1304
-
-
-# Adding for Hotfix : This needs to be fixed
-CVE-2022-27191
-CVE-2022-29190
-CVE-2022-29222
-CVE-2022-31030  
-CVE-2022-29189 
-CVE-2022-29526
+CVE-2022-2509
@@ -10,7 +10,7 @@ Feel free to contact engineering team owners in case you have any questions abou
 
 ## Common
 - [Visual Studio Code](https://code.visualstudio.com/) for authoring
-- [Go lang](https://golang.org/) for building go code. Go lang version 1.15.14 (both Linux & Windows)
+- [Go lang](https://golang.org/) for building go code. Go lang version 1.18.3 (both Linux & Windows)
 
 > Note: If you are using WSL2, make sure you have cloned the code onto ubuntu not onto windows
 
@@ -90,7 +90,6 @@ The general directory structure is:
 │   │   |   ├── health/                       - code for health feature
 │   │   |   ├── lib/                          - lib for app insights ruby and this code of application_insights gem
 │   │   |   ...                               - plugins in, out and filters code in ruby
-│   ├── toml-parser/                          - code for parsing of toml configuration files
 ├── test/                                     - source code for tests
 │   ├── e2e/                                  - e2e tests to validate agent and e2e workflow(s)
 │   ├── unit-tests/                           - unit tests code
@@ -121,7 +120,7 @@ We recommend using [Visual Studio Code](https://code.visualstudio.com/) for auth
 
 ### Install Pre-requisites
 
-1. Install go1.15.14, dotnet, powershell, docker and build dependencies to build go code for both Linux and Windows platforms
+1. Install go1.18.3, dotnet, powershell, docker and build dependencies to build go code for both Linux and Windows platforms
 ```
 bash ~/Docker-Provider/scripts/build/linux/install-build-pre-requisites.sh
 ```
@@ -143,13 +142,35 @@ bash ~/Docker-Provider/scripts/build/linux/install-build-pre-requisites.sh
 
 > Note: If you are using WSL2, ensure `Docker for windows` running with Linux containers mode on your windows machine to build Linux agent image successfully
 
+> Note: format of the imagetag will be `ci<release><MMDDYYYY>`. possible values for release are test, dev, preview, dogfood, prod etc. Please use MCR urls while building internally.
+
+Preferred Way: You can build and push images for multiple architectures. This is powered by docker buildx
+Directly use the docker buildx commands (the MCR images can be found in our internal wiki to be used as arguments)
+```
+# multiple platforms
+cd ~/Docker-Provider
+docker buildx build --platform linux/arm64/v8,linux/amd64 -t <repo>/<imagename>:<imagetag> --build-arg IMAGE_TAG=<imagetag> --build-arg CI_BASE_IMAGE=<ciimage> --build-arg GOLANG_BASE_IMAGE=<golangimage> -f kubernetes/linux/Dockerfile.multiarch --push .
+
+# single platform
+cd ~/Docker-Provider
+docker buildx build --platform linux/amd64 -t <repo>/<imagename>:<imagetag> --build-arg IMAGE_TAG=<imagetag> --build-arg CI_BASE_IMAGE=<ciimage> --build-arg GOLANG_BASE_IMAGE=<golangimage> -f kubernetes/linux/Dockerfile.multiarch --push .
+```
+
+Using the build and publish script
+
 ```
 cd ~/Docker-Provider/kubernetes/linux/dockerbuild
 sudo docker login # if you want to publish the image to acr then login to acr via `docker login <acr-name>`
 # build provider, docker image and publish to docker image
-bash build-and-publish-docker-image.sh --image <repo>/<imagename>:<imagetag>
+bash build-and-publish-docker-image.sh --image <repo>/<imagename>:<imagetag> --ubuntu <ubuntu image url> --golang <golang image url>
+```
+
+```
+cd ~/Docker-Provider/kubernetes/linux/dockerbuild
+sudo docker login # if you want to publish the image to acr then login to acr via `docker login <acr-name>`
+# build and publish using docker buildx
+bash build-and-publish-docker-image.sh --image <repo>/<imagename>:<imagetag> --ubuntu <ubuntu image url> --golang <golang image url> --multiarch
 ```
-> Note: format of the imagetag will be `ci<release><MMDDYYYY>`. possible values for release are test, dev, preview, dogfood, prod etc.
 
 You can also build and push images for multiple architectures. This is powered by docker buildx
 ```
@@ -182,7 +203,7 @@ make
 
 ```
 cd ~/Docker-Provider/kubernetes/linux/
-docker build -t <repo>/<imagename>:<imagetag> --build-arg IMAGE_TAG=<imagetag> .
+docker build -t <repo>/<imagename>:<imagetag> --build-arg IMAGE_TAG=<imagetag> --build-arg CI_BASE_IMAGE=<ciimage> .
 docker push <repo>/<imagename>:<imagetag>
 ```
 ## Windows Agent

@@ -11,6 +11,24 @@ additional questions or comments.
 
 Note : The agent version(s) below has dates (ciprod<mmddyyyy>), which indicate the agent build dates (not release dates)
 
+### 08/10/2022 -
+##### Version microsoft/oms:ciprod08102022 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod08102022 (linux)
+##### Version microsoft/oms:win-ciprod08102022 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:win-ciprod08102022 (windows)
+##### Code change log
+- Linux Agent
+  - Consume ruby from RVM instead of brightbox ppa
+
+- Common
+  - Update to Ruby 3.1.1
+  - Update telegraf to 1.23.2
+  - Updates fluentd to 1.14.6
+  - Use default JSON gem instead of yajl-json
+  - Consume tomlrb as a gem instead of committed source code
+  - Move from beta.kubernetes.io to kubernetes.io
+  - Bug fixes
+    - Fix bug in processing fractional memory limits
+    - Fix log loss due to inode reuse
+
 ### 07/27/2022 -
 ##### Version microsoft/oms:ciprod06272022-hotfix Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod06272022-hotfix (linux)
 - Fixes for sending the proper node allocatable cpu and memory value for the container which does not specify limits.

@@ -24,15 +24,16 @@ def substituteFluentBitPlaceHolders
     bufferChunkSize = ENV["FBIT_TAIL_BUFFER_CHUNK_SIZE"]
     bufferMaxSize = ENV["FBIT_TAIL_BUFFER_MAX_SIZE"]
     memBufLimit = ENV["FBIT_TAIL_MEM_BUF_LIMIT"]
+    ignoreOlder = ENV["FBIT_TAIL_IGNORE_OLDER"]
 
-    serviceInterval = (!interval.nil? && is_number?(interval) && interval.to_i > 0 ) ? interval : @default_service_interval
+    serviceInterval = (!interval.nil? && is_number?(interval) && interval.to_i > 0) ? interval : @default_service_interval
     serviceIntervalSetting = "Flush         " + serviceInterval
 
     tailBufferChunkSize = (!bufferChunkSize.nil? && is_number?(bufferChunkSize) && bufferChunkSize.to_i > 0) ? bufferChunkSize : nil
 
     tailBufferMaxSize = (!bufferMaxSize.nil? && is_number?(bufferMaxSize) && bufferMaxSize.to_i > 0) ? bufferMaxSize : nil
 
-    if ((!tailBufferChunkSize.nil? && tailBufferMaxSize.nil?) ||  (!tailBufferChunkSize.nil? && !tailBufferMaxSize.nil? && tailBufferChunkSize.to_i > tailBufferMaxSize.to_i))
+    if ((!tailBufferChunkSize.nil? && tailBufferMaxSize.nil?) || (!tailBufferChunkSize.nil? && !tailBufferMaxSize.nil? && tailBufferChunkSize.to_i > tailBufferMaxSize.to_i))
       puts "config:warn buffer max size must be greater or equal to chunk size"
       tailBufferMaxSize = tailBufferChunkSize
     end
@@ -54,6 +55,12 @@ def substituteFluentBitPlaceHolders
       new_contents = new_contents.gsub("\n    ${TAIL_BUFFER_MAX_SIZE}\n", "\n")
     end
 
+    if !ignoreOlder.nil? && !ignoreOlder.empty?
+      new_contents = new_contents.gsub("${TAIL_IGNORE_OLDER}", "Ignore_Older " + ignoreOlder)
+    else
+      new_contents = new_contents.gsub("\n    ${TAIL_IGNORE_OLDER}\n", "\n")
+    end
+
     File.open(@td_agent_bit_conf_path, "w") { |file| file.puts new_contents }
     puts "config::Successfully substituted the placeholders in td-agent-bit.conf file"
   rescue => errorStr

@@ -2,11 +2,7 @@
 
 #this should be require relative in Linux and require in windows, since it is a gem install on windows
 @os_type = ENV["OS_TYPE"]
-if !@os_type.nil? && !@os_type.empty? && @os_type.strip.casecmp("windows") == 0
-  require "tomlrb"
-else
-  require_relative "tomlrb"
-end
+require "tomlrb"
 
 require_relative "ConfigParseErrorLogger"
 
@@ -59,7 +55,7 @@
 @fbitTailBufferChunkSizeMBs = 0
 @fbitTailBufferMaxSizeMBs = 0
 @fbitTailMemBufLimitMBs = 0
-
+@fbitTailIgnoreOlder = ""
 
 def is_number?(value)
   true if Integer(value) rescue false
@@ -149,7 +145,7 @@ def populateSettingValuesFromConfigMap(parsedConfig)
         end
 
         fbitTailBufferMaxSizeMBs = fbit_config[:tail_buf_maxsize_megabytes]
-        if !fbitTailBufferMaxSizeMBs.nil? && is_number?(fbitTailBufferMaxSizeMBs) && fbitTailBufferMaxSizeMBs.to_i > 0           
+        if !fbitTailBufferMaxSizeMBs.nil? && is_number?(fbitTailBufferMaxSizeMBs) && fbitTailBufferMaxSizeMBs.to_i > 0
           if fbitTailBufferMaxSizeMBs.to_i >= @fbitTailBufferChunkSizeMBs
             @fbitTailBufferMaxSizeMBs = fbitTailBufferMaxSizeMBs.to_i
             puts "Using config map value: tail_buf_maxsize_megabytes = #{@fbitTailBufferMaxSizeMBs}"
@@ -160,16 +156,27 @@ def populateSettingValuesFromConfigMap(parsedConfig)
           end
         end
         # in scenario - tail_buf_chunksize_megabytes provided but not tail_buf_maxsize_megabytes to prevent fbit crash
-        if  @fbitTailBufferChunkSizeMBs > 0  && @fbitTailBufferMaxSizeMBs == 0
+        if @fbitTailBufferChunkSizeMBs > 0 && @fbitTailBufferMaxSizeMBs == 0
           @fbitTailBufferMaxSizeMBs = @fbitTailBufferChunkSizeMBs
           puts "config::warn: since tail_buf_maxsize_megabytes not provided hence using tail_buf_maxsize_megabytes=#{@fbitTailBufferMaxSizeMBs} which is same as the value of tail_buf_chunksize_megabytes"
-        end 
+        end
 
         fbitTailMemBufLimitMBs = fbit_config[:tail_mem_buf_limit_megabytes]
         if !fbitTailMemBufLimitMBs.nil? && is_number?(fbitTailMemBufLimitMBs) && fbitTailMemBufLimitMBs.to_i > 0
           @fbitTailMemBufLimitMBs = fbitTailMemBufLimitMBs.to_i
           puts "Using config map value: tail_mem_buf_limit_megabytes  = #{@fbitTailMemBufLimitMBs}"
         end
+
+        fbitTailIgnoreOlder = fbit_config[:tail_ignore_older]
+        re = /^[0-9]+[mhd]$/
+        if !fbitTailIgnoreOlder.nil? && !fbitTailIgnoreOlder.empty?
+          if !re.match(fbitTailIgnoreOlder).nil?
+            @fbitTailIgnoreOlder = fbitTailIgnoreOlder
+            puts "Using config map value: tail_ignore_older  = #{@fbitTailIgnoreOlder}"
+          else
+            puts "config:warn: provided tail_ignore_older value is not valid hence using default value"
+          end
+        end
       end
     end
   rescue => errorStr
@@ -210,10 +217,15 @@ def populateSettingValuesFromConfigMap(parsedConfig)
   end
   if @fbitTailBufferMaxSizeMBs > 0
     file.write("export FBIT_TAIL_BUFFER_MAX_SIZE=#{@fbitTailBufferMaxSizeMBs}\n")
-  end 
+  end
   if @fbitTailMemBufLimitMBs > 0
     file.write("export FBIT_TAIL_MEM_BUF_LIMIT=#{@fbitTailMemBufLimitMBs}\n")
-  end 
+  end
+
+  if !@fbitTailIgnoreOlder.nil? && !@fbitTailIgnoreOlder.empty?
+    file.write("export FBIT_TAIL_IGNORE_OLDER=#{@fbitTailIgnoreOlder}\n")
+  end
+
   # Close file after writing all environment variables
   file.close
 else
@@ -231,26 +243,30 @@ def get_command_windows(env_variable_name, env_variable_value)
 
   if !file.nil?
     if @fbitFlushIntervalSecs > 0
-      commands = get_command_windows('FBIT_SERVICE_FLUSH_INTERVAL', @fbitFlushIntervalSecs)
+      commands = get_command_windows("FBIT_SERVICE_FLUSH_INTERVAL", @fbitFlushIntervalSecs)
       file.write(commands)
     end
     if @fbitTailBufferChunkSizeMBs > 0
-      commands = get_command_windows('FBIT_TAIL_BUFFER_CHUNK_SIZE', @fbitTailBufferChunkSizeMBs)
+      commands = get_command_windows("FBIT_TAIL_BUFFER_CHUNK_SIZE", @fbitTailBufferChunkSizeMBs)
       file.write(commands)
     end
     if @fbitTailBufferMaxSizeMBs > 0
-      commands = get_command_windows('FBIT_TAIL_BUFFER_MAX_SIZE', @fbitTailBufferMaxSizeMBs)
+      commands = get_command_windows("FBIT_TAIL_BUFFER_MAX_SIZE", @fbitTailBufferMaxSizeMBs)
       file.write(commands)
-    end 
+    end
     if @fbitTailMemBufLimitMBs > 0
-      commands = get_command_windows('FBIT_TAIL_MEM_BUF_LIMIT', @fbitTailMemBufLimitMBs)
+      commands = get_command_windows("FBIT_TAIL_MEM_BUF_LIMIT", @fbitTailMemBufLimitMBs)
       file.write(commands)
-    end 
+    end
+    if !@fbitTailIgnoreOlder.nil? && !@fbitTailIgnoreOlder.empty?
+      commands = get_command_windows("FBIT_TAIL_IGNORE_OLDER", @fbitTailIgnoreOlder)
+      file.write(commands)
+    end
     # Close file after writing all environment variables
     file.close
     puts "****************End Config Processing********************"
   else
     puts "Exception while opening file for writing config environment variables for WINDOWS LOG"
     puts "****************End Config Processing********************"
   end
-end
+end
@@ -3,11 +3,7 @@
 
 #this should be require relative in Linux and require in windows, since it is a gem install on windows
 @os_type = ENV["OS_TYPE"]
-if !@os_type.nil? && !@os_type.empty? && @os_type.strip.casecmp("windows") == 0
-  require "tomlrb"
-else
-  require_relative "tomlrb"
-end
+require "tomlrb"
 
 require_relative "/etc/fluent/plugin/constants"
 require_relative "ConfigParseErrorLogger"

@@ -2,11 +2,7 @@
 
 #this should be require relative in Linux and require in windows, since it is a gem install on windows
 @os_type = ENV["OS_TYPE"]
-if !@os_type.nil? && !@os_type.empty? && @os_type.strip.casecmp("windows") == 0
-  require "tomlrb"
-else
-  require_relative "tomlrb"
-end
+require "tomlrb"
 
 require_relative "ConfigParseErrorLogger"