User/amagraw/release032022/2

.github/workflows/pr-checker.yml

@@ -56,7 +56,8 @@ jobs:
           format: 'table'
           severity: 'CRITICAL,HIGH'
           vuln-type: 'os,library'
-          skip-files: '/usr/sbin/telegraf,/opt/telegraf,/opt/td-agent-bit/bin/out_oms.so'
+          #[vishwa] - Fix telegraf & test all for next release - see work item #https://msazure.visualstudio.com/InfrastructureInsights/_workitems/edit/13322134
+          skip-files: '/usr/sbin/telegraf,/opt/telegraf'
           exit-code: '1'
           timeout: '5m0s'
           ignore-unfixed: true
@@ -95,3 +96,4 @@ jobs:
             cd ./kubernetes/windows/ && docker build . --file Dockerfile -t $env:IMAGETAG --build-arg IMAGE_TAG=$env:IMAGETAG_TELEMETRY
       - name: List-docker-images
         run: docker images --digests --all
+

.pipelines/azure_pipeline_dev.yaml

@@ -0,0 +1,104 @@
+# Starter pipeline
+# Start with a minimal pipeline that you can customize to build and deploy your code.
+# Add steps that build, run tests, deploy, and more:
+# https://aka.ms/yaml
+
+trigger:
+- ci_dev
+
+pool:
+  name: Azure-Pipelines-CI-Test-EO
+
+variables:
+  armServiceConnectionName: 'ci-1es-acr-connection'
+  subscription: '9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb'
+  containerRegistry: 'containerinsightsprod'
+  repoImageName: '${{ variables.containerRegistry }}.azurecr.io/public/azuremonitor/containerinsights/cidev'
+
+steps:
+- bash: |
+    commit=$(git rev-parse --short HEAD)
+    echo "##vso[task.setvariable variable=commit;]$commit"
+
+    datetime=$(date +'%Y%m%d%s')
+    echo "##vso[task.setvariable variable=datetime;]$datetime"
+
+    cd $(Build.SourcesDirectory)/deployment/multiarch-agent-deployment/ServiceGroupRoot/Scripts
+    tar -czvf ../artifacts.tar.gz pushAgentToAcr.sh
+
+    cd $(Build.SourcesDirectory)/deployment/arc-k8s-extension/ServiceGroupRoot/Scripts
+    tar -czvf ../artifacts.tar.gz ../../../../charts/azuremonitor-containers/ pushChartToAcr.sh
+
+- task: CopyFiles@2
+  displayName: "Copy ev2 deployment artifacts"
+  inputs:
+    SourceFolder: "$(Build.SourcesDirectory)/deployment"
+    Contents: |
+      **/*
+    TargetFolder: '$(Build.ArtifactStagingDirectory)/build'
+
+- task: CopyFiles@2
+  displayName: "Copy ev2 deployment scripts"
+  inputs:
+    SourceFolder: "$(Build.SourcesDirectory)/.pipelines"
+    Contents: |
+      *.sh
+    TargetFolder: '$(Build.ArtifactStagingDirectory)/build'
+
+- task: CopyFiles@2
+  displayName: "Copy ev2 deployment scripts"
+  inputs:
+    SourceFolder: "$(Build.SourcesDirectory)/kubernetes"
+    Contents: |
+      *.yaml
+    TargetFolder: '$(Build.ArtifactStagingDirectory)/build'
+
+- task: CopyFiles@2
+  displayName: "Copy ev2 deployment scripts"
+  inputs:
+    SourceFolder: "$(Build.SourcesDirectory)/charts"
+    Contents: |
+      **/*
+    TargetFolder: '$(Build.ArtifactStagingDirectory)/build'
+
+- task: CopyFiles@2
+  displayName: "Copy ev2 deployment scripts"
+  inputs:
+    SourceFolder: "$(Build.SourcesDirectory)/test/e2e"
+    Contents: |
+      *.yaml
+    TargetFolder: '$(Build.ArtifactStagingDirectory)/build'
+
+- task: AzureCLI@2
+  displayName: "Docker multi-arch linux build"
+  inputs:
+    azureSubscription: ${{ variables.armServiceConnectionName }}
+    scriptType: bash
+    scriptLocation: inlineScript
+    inlineScript: |
+
+      sudo apt-get update && sudo apt-get -y install qemu binfmt-support qemu-user-static
+      docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+
+      docker buildx create --name testbuilder
+      docker buildx use testbuilder
+
+      az --version
+      az account show
+      az account set -s ${{ variables.subscription }}
+      az acr login -n ${{ variables.containerRegistry }}
+
+      docker buildx build --platform linux/amd64,linux/arm64 --tag ${{ variables.repoImageName }}:$(datetime)-$(commit) -f kubernetes/linux/Dockerfile.multiarch --metadata-file $(Build.ArtifactStagingDirectory)/metadata.json --build-arg IMAGE_TAG=$(datetime)-$(commit) --push .
+
+      docker pull ${{ variables.repoImageName }}:$(datetime)-$(commit)
+
+- task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0
+  displayName: 'Generation Task'
+  inputs:
+    BuildDropPath: '$(Build.ArtifactStagingDirectory)'
+    DockerImagesToScan: 'golang:1.15.14, ubuntu:18.04, ${{ variables.repoImageName }}:$(datetime)-$(commit)'
+
+- task: PublishBuildArtifacts@1
+  inputs:
+    pathToPublish: '$(Build.ArtifactStagingDirectory)'
+    artifactName: drop

.pipelines/azure_pipeline_prod.yaml

@@ -0,0 +1,104 @@
+# Starter pipeline
+# Start with a minimal pipeline that you can customize to build and deploy your code.
+# Add steps that build, run tests, deploy, and more:
+# https://aka.ms/yaml
+
+trigger:
+- ci_prod
+
+pool:
+  name: Azure-Pipelines-CI-Prod-EO
+
+variables:
+  armServiceConnectionName: 'ci-1es-acr-connection-prod'
+  subscription: '30c56c3a-54da-46ea-b004-06eb33432687'
+  containerRegistry: 'containerinsightsbuild'
+  repoImageName: '${{ variables.containerRegistry }}.azurecr.io/official/linux'
+
+steps:
+- bash: |
+    commit=$(git rev-parse --short HEAD)
+    echo "##vso[task.setvariable variable=commit;]$commit"
+
+    datetime=$(date +'%Y%m%d%s')
+    echo "##vso[task.setvariable variable=datetime;]$datetime"
+
+    cd $(Build.SourcesDirectory)/deployment/multiarch-agent-deployment/ServiceGroupRoot/Scripts
+    tar -czvf ../artifacts.tar.gz pushAgentToAcr.sh
+
+    cd $(Build.SourcesDirectory)/deployment/arc-k8s-extension/ServiceGroupRoot/Scripts
+    tar -czvf ../artifacts.tar.gz ../../../../charts/azuremonitor-containers/ pushChartToAcr.sh
+
+- task: CopyFiles@2
+  displayName: "Copy ev2 deployment artifacts"
+  inputs:
+    SourceFolder: "$(Build.SourcesDirectory)/deployment"
+    Contents: |
+      **/*
+    TargetFolder: '$(Build.ArtifactStagingDirectory)/build'
+
+- task: CopyFiles@2
+  displayName: "Copy ev2 deployment scripts"
+  inputs:
+    SourceFolder: "$(Build.SourcesDirectory)/.pipelines"
+    Contents: |
+      **/*.sh
+    TargetFolder: '$(Build.ArtifactStagingDirectory)/build'
+
+- task: CopyFiles@2
+  displayName: "Copy ev2 deployment scripts"
+  inputs:
+    SourceFolder: "$(Build.SourcesDirectory)/kubernetes"
+    Contents: |
+      *.yaml
+    TargetFolder: '$(Build.ArtifactStagingDirectory)/build'
+
+- task: CopyFiles@2
+  displayName: "Copy ev2 deployment scripts"
+  inputs:
+    SourceFolder: "$(Build.SourcesDirectory)/charts"
+    Contents: |
+      **/*
+    TargetFolder: '$(Build.ArtifactStagingDirectory)/build'
+
+- task: CopyFiles@2
+  displayName: "Copy ev2 deployment scripts"
+  inputs:
+    SourceFolder: "$(Build.SourcesDirectory)/test/e2e"
+    Contents: |
+      *.yaml
+    TargetFolder: '$(Build.ArtifactStagingDirectory)/build'
+
+- task: AzureCLI@2
+  displayName: "Docker multi-arch linux build"
+  inputs:
+    azureSubscription: ${{ variables.armServiceConnectionName }}
+    scriptType: bash
+    scriptLocation: inlineScript
+    inlineScript: |
+
+      sudo apt-get update && sudo apt-get -y install qemu binfmt-support qemu-user-static
+      docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+
+      docker buildx create --name testbuilder
+      docker buildx use testbuilder
+
+      az --version
+      az account show
+      az account set -s ${{ variables.subscription }}
+      az acr login -n ${{ variables.containerRegistry }}
+
+      docker buildx build --platform linux/amd64,linux/arm64 --tag ${{ variables.repoImageName }}:ciprod-$(datetime)-$(commit) -f kubernetes/linux/Dockerfile.multiarch --metadata-file $(Build.ArtifactStagingDirectory)/metadata.json --build-arg IMAGE_TAG=ciprod-$(datetime)-$(commit) --push .
+
+      docker pull ${{ variables.repoImageName }}:ciprod-$(datetime)-$(commit)
+
+- task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0
+  displayName: 'Generation Task'
+  inputs:
+    BuildDropPath: '$(Build.ArtifactStagingDirectory)'
+    DockerImagesToScan: 'golang:1.15.14, ubuntu:18.04, ${{ variables.repoImageName }}:ciprod-$(datetime)-$(commit)'
+
+- task: PublishBuildArtifacts@1
+  inputs:
+    pathToPublish: '$(Build.ArtifactStagingDirectory)'
+    artifactName: drop

.pipelines/build-linux.sh

@@ -24,3 +24,4 @@ echo "------------ Bundle Shell Extension Scripts & HELM chart -----------------
 cd $DIR/../deployment/arc-k8s-extension/ServiceGroupRoot/Scripts
 tar -czvf ../artifacts.tar.gz ../../../../charts/azuremonitor-containers/ pushChartToAcr.sh
 
+

.pipelines/pipeline.user.linux.official.all_tag.all_phase.all_config.ci_prod.yml

@@ -63,7 +63,7 @@ package:
       repository_name: 'cdpxlinux'                        # only supported ones are cdpx acr repos
       tag: 'ciprod'                                       # OPTIONAL: Defaults to latest. The tag for the built image. Final tag will be 1.0.0alpha, 1.0.0-timestamp-commitID.
       latest: false                                       # OPTIONAL: Defaults to false. If tag is not set to latest and this flag is set, then tag as latest as well and push latest as well.
-      publish_unique_tag: true                            # If set, the image in the registry is tagged with the unique tag generated by CDPx
+      publish_unique_tag: true                            # If set, the image in the registry is tagged with the unique tag generated by CDPx 
       metadata_file:
         artifact_path: 'linux-image-meta.json'            # If defined, the drop outputs relative path to the file into which JSON metadata about the created image is emitted.
       export_to_artifact_path: 'agentimage.tar.gz'   # path for exported image and use this instead of fixed tag

.pipelines/pipeline.user.linux.yml

@@ -63,7 +63,7 @@ package:
       repository_name: 'cdpxlinux'                        # only supported ones are cdpx acr repos
       tag: 'cidev'                                        # OPTIONAL: Defaults to latest. The tag for the built image. Final tag will be 1.0.0alpha, 1.0.0-timestamp-commitID.
       latest: false                                       # OPTIONAL: Defaults to false. If tag is not set to latest and this flag is set, then tag as latest as well and push latest as well.
-      publish_unique_tag: true                            # If set, the image in the registry is tagged with the unique tag generated by CDPx
+      publish_unique_tag: true                            # If set, the image in the registry is tagged with the unique tag generated by CDPx 
       metadata_file:
         artifact_path: 'linux-image-meta.json'            # If defined, the drop outputs relative path to the file into which JSON metadata about the created image is emitted.
       export_to_artifact_path: 'agentimage.tar.gz'   # path for exported image and use this instead of fixed tag

.pipelines/pipeline.user.windows.official.all_tag.all_phase.all_config.ci_prod.yml

@@ -53,7 +53,7 @@ package:
       repository_name: 'cdpxwin1809'                      # only supported ones are cdpx acr repos
       tag: 'win-ciprod'                                    # OPTIONAL: Defaults to latest. The tag for the built image. Final tag will be 1.0.0alpha, 1.0.0-timestamp-commitID.
       latest: false                                       # OPTIONAL: Defaults to false. If tag is not set to latest and this flag is set, then tag as latest as well and push latest as well.
-      publish_unique_tag: true                            # If set, the image in the registry is tagged with the unique tag generated by CDPx
+      publish_unique_tag: true                            # If set, the image in the registry is tagged with the unique tag generated by CDPx 
       metadata_file:
         artifact_path: 'windows-image-meta.json'            # If defined, the drop outputs relative path to the file into which JSON metadata about the created image is emitted.
       export_to_artifact_path: 'agentimage.tar.zip'   # path for exported image and use this instead of fixed tag

README.md

@@ -151,6 +151,25 @@ bash build-and-publish-docker-image.sh --image <repo>/<imagename>:<imagetag>
 ```
 > Note: format of the imagetag will be `ci<release><MMDDYYYY>`. possible values for release are test, dev, preview, dogfood, prod etc.
 
+You can also build and push images for multiple architectures. This is powered by docker buildx
+```
+cd ~/Docker-Provider/kubernetes/linux/dockerbuild
+sudo docker login # if you want to publish the image to acr then login to acr via `docker login <acr-name>`
+# build and publish using docker buildx
+bash build-and-publish-docker-image.sh --image <repo>/<imagename>:<imagetag> --multiarch
+```
+
+or directly use the docker buildx commands
+```
+# multiple platforms
+cd ~/Docker-Provider
+docker buildx build --platform linux/arm64/v8,linux/amd64 -t <repo>/<imagename>:<imagetag> --build-arg IMAGE_TAG=<imagetag> -f kubernetes/linux/Dockerfile.multiarch --push .
+
+# single platform
+cd ~/Docker-Provider
+docker buildx build --platform linux/amd64 -t <repo>/<imagename>:<imagetag> --build-arg IMAGE_TAG=<imagetag> -f kubernetes/linux/Dockerfile.multiarch --push .
+```
+
 If you prefer to build docker provider shell bundle and image separately, then you can follow below instructions
 
 ##### Build Docker Provider shell bundle
@@ -211,7 +230,7 @@ powershell -ExecutionPolicy bypass  # switch to powershell if you are not on pow
 ```
 
 ##### Developer Build optimizations
-If you do not want to build the image from scratch every time you make changes during development,you can choose to build the docker images that are separated out by
+If you do not want to build the image from scratch every time you make changes during development,you can choose to build the docker images that are separated out by 
 * Base image and dependencies including agent bootstrap(setup.ps1)
 * Agent conf and plugin changes
 
@@ -229,7 +248,7 @@ And then run the script to build the image consisting of code and conf changes.
 .\build-and-publish-dev-docker-image.ps1 -image <repo>/<imagename>:<imagetag> # trigger build code and image and publish docker hub or acr
 ```
 
-For the subsequent builds, you can just run -
+For the subsequent builds, you can just run - 
 
 ```
 .\build-and-publish-dev-docker-image.ps1 -image <repo>/<imagename>:<imagetag> # trigger build code and image and publish docker hub or acr
@@ -302,13 +321,13 @@ For DEV and PROD branches, automatically deployed latest yaml with latest agent
 ## For executing tests
 
 1. Deploy the omsagent.yaml with your agent image. In the yaml, make sure `ISTEST` environment variable set to `true` if its not set already
-2. Update the Service Principal CLIENT_ID, CLIENT_SECRET and TENANT_ID placeholder values and apply e2e-tests.yaml to execute the tests
+2. Update the Service Principal CLIENT_ID, CLIENT_SECRET and TENANT_ID placeholder values and apply e2e-tests.yaml to execute the tests 
     > Note: Service Principal requires reader role on log analytics workspace and cluster resource to query LA and metrics
    ```
-   cd ~/Docker-Provider/test/e2e # based on your repo path
-   kubectl apply -f e2e-tests.yaml # this will trigger job to run the tests in sonobuoy namespace
-   kubectl get po -n sonobuoy # to check the pods and jobs associated to tests
-   ```
+   cd ~/Docker-Provider/test/e2e # based on your repo path    
+   kubectl apply -f e2e-tests.yaml # this will trigger job to run the tests in sonobuoy namespace 
+   kubectl get po -n sonobuoy # to check the pods and jobs associated to tests   
+   ``` 
 3. Download (sonobuoy)[https://github.com/vmware-tanzu/sonobuoy/releases] on your dev box to view the results of the tests
    ```
    results=$(sonobuoy retrieve) # downloads tar file which has logs and test results
@@ -319,9 +338,9 @@ For DEV and PROD branches, automatically deployed latest yaml with latest agent
 ## For adding new tests
 
 1. Add the test python file with your test code under `tests` directory
-2. Build the docker image, recommended to use ACR & MCR
+2. Build the docker image, recommended to use ACR & MCR 
   ```
-   cd ~/Docker-Provider/test/e2e/src # based on your repo path
+   cd ~/Docker-Provider/test/e2e/src # based on your repo path 
    docker login <acr> -u <user> -p <pwd> # login to acr
    docker build -f ./core/Dockerfile -t <repo>/<imagename>:<imagetag> .
    docker push <repo>/<imagename>:<imagetag>

ReleaseNotes.md

@@ -11,16 +11,28 @@ additional questions or comments.
 
 Note : The agent version(s) below has dates (ciprod<mmddyyyy>), which indicate the agent build dates (not release dates)
 
-### 3/11/2022 -
-##### Version microsoft/oms:ciprod03112022 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod03112022 (linux)
-##### Version microsoft/oms:win-ciprod03112022 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:win-ciprod03112022 (windows)
+### 3/17/2022 -
+##### Version microsoft/oms:ciprod03172022 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod03172022 (linux)
+##### Version microsoft/oms:win-ciprod03172022 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:win-ciprod03172022 (windows)
 ##### Code change log
 - Linux Agent
+  - Multi-Arch Image to support both AMD64 and ARM64
+  - Ruby upgraded to version 2.7 from 2.6
+  - Fix Telegraf Permissions
+  - Fix ADX bug with database name
   - Vulnerability fixes
+  - MDSD updated to 1.17.0
+    - HTTP Proxy support
+    - Retries for Log Analytics Ingestion
+    - ARM64 support
+    - Memory leak fixes for network failure scenario
 - Windows Agent
   - Bug fix for FluentBit stdout and stderr log filtering
 - Common
-    - Upgrade Go lang version from 1.14.1 to 1.15.14
+  - Upgrade Go lang version from 1.14.1 to 1.15.14
+  - MSI onboarding ARM template update
+  - AKS HTTP Proxy support
+  - Go packages upgrade to address vulnerabilities
 
 ### 1/31/2022 -
 ##### Version microsoft/oms:ciprod01312022 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod01312022 (linux)
@@ -100,6 +112,7 @@ Note : The agent version(s) below has dates (ciprod<mmddyyyy>), which indicate t
    - Fix for https://github.com/Azure/AKS/issues/2457
    - In replicaset, tailing of the mdsd.err log file to agent telemetry
 
+
 ### 07/13/2021 -
 ##### Version microsoft/oms:win-ciprod06112021-2 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:win-ciprod06112021-2 (windows)
 ##### Code change log

build/common/installer/scripts/td-agent-bit-conf-customizer.rb

@@ -25,14 +25,14 @@ def substituteFluentBitPlaceHolders
     bufferMaxSize = ENV["FBIT_TAIL_BUFFER_MAX_SIZE"]
     memBufLimit = ENV["FBIT_TAIL_MEM_BUF_LIMIT"]
 
-    serviceInterval = (!interval.nil? && is_number?(interval) && interval.to_i > 0) ? interval : @default_service_interval
+    serviceInterval = (!interval.nil? && is_number?(interval) && interval.to_i > 0 ) ? interval : @default_service_interval
     serviceIntervalSetting = "Flush         " + serviceInterval
 
     tailBufferChunkSize = (!bufferChunkSize.nil? && is_number?(bufferChunkSize) && bufferChunkSize.to_i > 0) ? bufferChunkSize : nil
 
     tailBufferMaxSize = (!bufferMaxSize.nil? && is_number?(bufferMaxSize) && bufferMaxSize.to_i > 0) ? bufferMaxSize : nil
 
-    if ((!tailBufferChunkSize.nil? && tailBufferMaxSize.nil?) || (!tailBufferChunkSize.nil? && !tailBufferMaxSize.nil? && tailBufferChunkSize.to_i > tailBufferMaxSize.to_i))
+    if ((!tailBufferChunkSize.nil? && tailBufferMaxSize.nil?) ||  (!tailBufferChunkSize.nil? && !tailBufferMaxSize.nil? && tailBufferChunkSize.to_i > tailBufferMaxSize.to_i))
       puts "config:warn buffer max size must be greater or equal to chunk size"
       tailBufferMaxSize = tailBufferChunkSize
     end