stdout/stderr log filtering for windows

.github/workflows/pr-checker.yml

@@ -56,11 +56,12 @@ jobs:
           format: 'table'
           severity: 'CRITICAL,HIGH'
           vuln-type: 'os,library'
-          skip-dirs: '/usr/sbin'
+          skip-files: '/usr/sbin/telegraf,/opt/telegraf,/opt/td-agent-bit/bin/out_oms.so'
           exit-code: '1'
           timeout: '5m0s'
+          ignore-unfixed: true
   WINDOWS-build:
-    runs-on: windows-latest
+    runs-on: windows-2019
     steps:
       - name: Set-workflow-initiator
         run: echo ("Initiated by -" + $env:GITHUB_ACTOR)

.pipelines/pipeline.user.linux.official.all_tag.all_phase.all_config.ci_prod.yml

@@ -5,7 +5,7 @@ environment:
     version: '16.04'
   runtime:
     provider: 'appcontainer'
-    image: 'cdpxlinux.azurecr.io/user/azure-monitor/container-insights:1.0'
+    image: 'cdpxlinux.azurecr.io/user/azure-monitor/container-insights:latest'
 
 version:
   name: 'DockerProvider'

.pipelines/pipeline.user.linux.yml

@@ -5,7 +5,7 @@ environment:
     version: '16.04'
   runtime:
     provider: 'appcontainer'
-    image: 'cdpxlinux.azurecr.io/user/azure-monitor/container-insights:1.0'
+    image: 'cdpxlinux.azurecr.io/user/azure-monitor/container-insights:latest'
 
 version:
   name: 'DockerProvider'

README.md

@@ -10,7 +10,7 @@ Feel free to contact engineering team owners in case you have any questions abou
 
 ## Common
 - [Visual Studio Code](https://code.visualstudio.com/) for authoring
-- [Go lang](https://golang.org/) for building go code. Go lang version 1.14.1.
+- [Go lang](https://golang.org/) for building go code. Go lang version 1.15.14 (both Linux & Windows)
 
 > Note: If you are using WSL2, make sure you have cloned the code onto ubuntu not onto windows
 
@@ -121,7 +121,7 @@ We recommend using [Visual Studio Code](https://code.visualstudio.com/) for auth
 
 ### Install Pre-requisites
 
-1. Install go1.14.1, dotnet, powershell, docker and build dependencies to build go code for both Linux and Windows platforms
+1. Install go1.15.14, dotnet, powershell, docker and build dependencies to build go code for both Linux and Windows platforms
 ```
 bash ~/Docker-Provider/scripts/build/linux/install-build-pre-requisites.sh
 ```

ReleaseNotes.md

@@ -11,6 +11,17 @@ additional questions or comments.
 
 Note : The agent version(s) below has dates (ciprod<mmddyyyy>), which indicate the agent build dates (not release dates)
 
+### 3/11/2022 -
+##### Version microsoft/oms:ciprod03112022 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod03112022 (linux)
+##### Version microsoft/oms:win-ciprod03112022 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:win-ciprod03112022 (windows)
+##### Code change log
+- Linux Agent
+  - Vulnerability fixes
+- Windows Agent
+  - Bug fix for FluentBit stdout and stderr log filtering
+- Common
+    - Upgrade Go lang version from 1.14.1 to 1.15.14
+
 ### 1/31/2022 -
 ##### Version microsoft/oms:ciprod01312022 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod01312022 (linux)
 ##### Version microsoft/oms:win-ciprod01312022 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:win-ciprod01312022 (windows)

build/common/installer/scripts/tomlparser.rb

@@ -260,6 +260,16 @@ def get_command_windows(env_variable_name, env_variable_value)
   file = File.open("setenv.ps1", "w")
 
   if !file.nil?
+    # This will be used in fluent-bit.conf file to filter out logs
+    if (!@collectStdoutLogs && !@collectStderrLogs)
+      #Stop log tailing completely
+      @logTailPath = "C:\\opt\\nolog*.log"
+      @logExclusionRegexPattern = "stdout|stderr"
+    elsif !@collectStdoutLogs
+      @logExclusionRegexPattern = "stdout"
+    elsif !@collectStderrLogs
+      @logExclusionRegexPattern = "stderr"
+    end
     commands = get_command_windows("AZMON_COLLECT_STDOUT_LOGS", @collectStdoutLogs)
     file.write(commands)
     commands = get_command_windows("AZMON_LOG_TAIL_PATH", @logTailPath)

build/windows/installer/conf/fluent-bit.conf

@@ -59,6 +59,17 @@
     Buffer_Size 64
     Mem_Buf_Limit 5m
 
+[FILTER]
+    Name grep
+    Match oms.container.log.la.*
+    Exclude stream ${AZMON_LOG_EXCLUSION_REGEX_PATTERN}
+
+# Exclude prometheus plugin exceptions that might be caused due to invalid config.(Logs which contain - E! [inputs.prometheus])
+# Excluding these logs from being sent to AI since it can result in high volume of data in telemetry due to invalid config.
+[FILTER]
+    Name grep
+    Match oms.container.log.flbplugin.*
+
 [OUTPUT]
     Name  oms
     EnableTelemetry                 true

charts/azuremonitor-containers/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v1
 appVersion: 7.0.0-1
 description: Helm chart for deploying Azure Monitor container monitoring agent in Kubernetes
 name: azuremonitor-containers
-version: 2.9.1
+version: 2.9.2
 kubeVersion: "^1.10.0-0"
 keywords:
   - monitoring

charts/azuremonitor-containers/values.yaml

@@ -21,8 +21,8 @@ Azure:
 omsagent:
   image:
     repo: "mcr.microsoft.com/azuremonitor/containerinsights/ciprod"
-    tag: "ciprod01312022"
-    tagWindows: "win-ciprod01312022"
+    tag: "ciprod03112022"
+    tagWindows: "win-ciprod03112022"
     pullPolicy: IfNotPresent
     dockerProviderVersion: "16.0.0-0"
     agentVersion: "azure-mdsd-1.14.2"

kubernetes/linux/Dockerfile

@@ -17,7 +17,7 @@ ENV RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR 0.9
 RUN /usr/bin/apt-get update && /usr/bin/apt-get install -y libc-bin wget openssl curl sudo python-ctypes init-system-helpers  net-tools rsyslog cron vim dmidecode apt-transport-https gnupg && rm -rf /var/lib/apt/lists/*
 COPY setup.sh main.sh defaultpromenvvariables defaultpromenvvariables-rs defaultpromenvvariables-sidecar mdsd.xml envmdsd logrotate.conf $tmpdir/
 
-ARG IMAGE_TAG=ciprod01312022
+ARG IMAGE_TAG=ciprod03112022
 ENV AGENT_VERSION ${IMAGE_TAG}
 
 WORKDIR ${tmpdir}

kubernetes/omsagent.yaml

@@ -368,7 +368,7 @@ spec:
             value: "3"
       containers:
         - name: omsagent
-          image: "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod01312022"
+          image: "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod03112022"
           imagePullPolicy: IfNotPresent
           resources:
             limits:
@@ -454,7 +454,7 @@ spec:
             timeoutSeconds: 15
 #Only in sidecar scraping mode
         # - name: omsagent-prometheus
-        #   image: "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod01312022"
+        #   image: "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod03112022"
         #   imagePullPolicy: IfNotPresent
         #   resources:
         #     limits:
@@ -603,7 +603,7 @@ spec:
       serviceAccountName: omsagent
       containers:
         - name: omsagent
-          image: "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod01312022"
+          image: "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod03112022"
           imagePullPolicy: IfNotPresent
           resources:
             limits:
@@ -776,7 +776,7 @@ spec:
             value: "3"
      containers:
        - name: omsagent-win
-         image: "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:win-ciprod01312022"
+         image: "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:win-ciprod03112022"
          imagePullPolicy: IfNotPresent
          resources:
           limits:

kubernetes/windows/Dockerfile

@@ -3,7 +3,7 @@ MAINTAINER OMSContainers@microsoft.com
 LABEL vendor=Microsoft\ Corp \
     com.microsoft.product="Azure Monitor for containers"
 
-ARG IMAGE_TAG=win-ciprod01312022
+ARG IMAGE_TAG=win-ciprod03112022
 
 # Do not split this into multiple RUN!
 # Docker creates a layer for every RUN-Statement

scripts/build/linux/install-build-pre-requisites.sh

@@ -8,17 +8,17 @@ TEMP_DIR=temp-$RANDOM
 install_go_lang()
 {
   export goVersion="$(echo $(go version))"
-  if [[ $goVersion == *go1.14.1* ]] ; then
-    echo "found existing installation of go version 1.14.1 so skipping the installation of go"
+  if [[ $goVersion == *go1.15.14* ]] ; then
+    echo "found existing installation of go version 1.15.14 so skipping the installation of go"
   else
-    echo "installing go 1.14.1 version ..."
-    sudo curl -O https://dl.google.com/go/go1.14.1.linux-amd64.tar.gz
-    sudo tar -xvf go1.14.1.linux-amd64.tar.gz
+    echo "installing go 1.15.14 version ..."
+    sudo curl -O https://dl.google.com/go/go1.15.14.linux-amd64.tar.gz
+    sudo tar -xvf go1.15.14.linux-amd64.tar.gz
     sudo mv -f go /usr/local
     echo "set file permission for go bin"
     sudo chmod 777 /usr/local/go/bin
-    echo "installation of go 1.14.1 completed."
-    echo "installation of go 1.14.1 completed."
+    echo "installation of go 1.15.14 completed."
+    echo "installation of go 1.15.14 completed."
   fi
 
 }
@@ -154,4 +154,4 @@ sudo rm -rf $TEMP_DIR
 # set go env vars
 install_go_env_vars
 
-echo "installing build pre-requisites python, go 1.14.1, dotnet, powershell, build dependencies and docker completed"
+echo "installing build pre-requisites python, go 1.15.14, dotnet, powershell, build dependencies and docker completed"

scripts/build/windows/install-build-pre-requisites.ps1

@@ -133,7 +133,7 @@ function Install-Docker() {
    Write-Host("installing docker for desktop completed")
 }
 
-Write-Host "Install GO 1.14.1 version"
+Write-Host "Install GO 1.15.14 version"
 Install-Go
 Write-Host "Install Build dependencies"
 Build-Dependencies

source/plugins/go/src/go.mod

@@ -9,8 +9,8 @@ require (
 	github.com/dnaeon/go-vcr v1.2.0 // indirect
 	github.com/fluent/fluent-bit-go v0.0.0-20171103221316-c4a158a6e3a7
 	github.com/golang/mock v1.4.1
-	github.com/google/uuid v1.1.2
-	github.com/microsoft/ApplicationInsights-Go v0.4.3
+	github.com/google/uuid v1.3.0
+	github.com/microsoft/ApplicationInsights-Go v0.4.4
 	github.com/philhofer/fwd v1.1.1 // indirect
 	github.com/tinylib/msgp v1.1.2
 	github.com/ugorji/go v1.1.2-0.20180813092308-00b869d2f4a5

source/plugins/go/src/go.sum

@@ -120,6 +120,9 @@ github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL9
 github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo=
 github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
 github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
+github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
+github.com/gofrs/uuid v4.2.0+incompatible h1:yyYWMnhkhrKwwr8gAOcOCYxOOscHgDS9yZgBrnJfGa0=
+github.com/gofrs/uuid v4.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
@@ -169,6 +172,10 @@ github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs=
+github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
+github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/googleapis/gnostic v0.4.1 h1:DLJCy1n/vrD4HPjOvYcT8aYQXpPIzoRZONaYwyycI+I=
@@ -203,6 +210,8 @@ github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149 h1:HfxbT6/JcvIljm
 github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc=
 github.com/microsoft/ApplicationInsights-Go v0.4.3 h1:gBuy5rM3o6Zo69QTkq1Ens8wx6sVf+mpgMjjfayiRcw=
 github.com/microsoft/ApplicationInsights-Go v0.4.3/go.mod h1:ih0t3h84PdzV1qGeUs89o9wL8eCuwf24M7TZp/nyqXk=
+github.com/microsoft/ApplicationInsights-Go v0.4.4 h1:G4+H9WNs6ygSCe6sUyxRc2U81TI5Es90b2t/MwX5KqY=
+github.com/microsoft/ApplicationInsights-Go v0.4.4/go.mod h1:fKRUseBqkw6bDiXTs3ESTiU/4YTIHsQS4W3fP2ieF4U=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
@@ -237,8 +246,6 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
-github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
-github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
 github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=