[Merge] dev to prod for ciprod08072020 release

.pipelines/pipeline.user.linux.official.all_tag.all_phase.all_config.ci_prod.yml

@@ -0,0 +1,44 @@
+environment:
+  host:
+    os: 'linux'
+    flavor: 'ubuntu'
+    version: '16.04'
+  runtime:
+    provider: 'appcontainer'
+    image: 'cdpxlinux.azurecr.io/user/azure-monitor/container-insights:1.0'
+
+version:
+  name: 'DockerProvider'
+  major: 10
+  minor: 0
+  tag: 'beta'
+  system: 'custom'
+  exclude_commit: true
+
+restore:
+  commands:
+    - !!defaultcommand
+      name: 'get go modules'
+      command: '.pipelines/restore-linux.sh'
+      fail_on_stderr: false
+
+build:
+  commands:
+    - !!defaultcommand
+      name: 'Build Docker Provider Shell Bundle'
+      command: '.pipelines/build-linux.sh'
+      fail_on_stderr: false
+
+package:
+  commands:
+    - !!dockerbuildcommand                               # REQUIRED: This maps the command data to a concrete type in the CDPX orchestrator.
+      name: 'Build Docker Image'                          # REQUIRED: All commands have a name field. All console output captured when
+                                                          #           this command runs is tagged with the value of this field.
+      context_folder: 'kubernetes/linux'                  # REQUIRED: The repository root relative path of the folder containing the Dockerfile to build.
+                                                          # In effect, the context folder will be repository_checkout_folder/src/DockerFinal.
+      dockerfile_name: 'Dockerfile'                       # OPTIONAL: The name of the dockerfile. Docker client does allow the Dockerfile
+                                                          # to be named differently. Defaults to Dockerfile.
+                                                          # In effect, the -f option value passed to docker build will be repository_checkout_folder/src/DockerFinal/Foo.dockerfile.
+      repository_name: 'cdpxlinux'                        # only supported ones are cdpx acr repos
+      tag: 'ciprod'                                        # OPTIONAL: Defaults to latest. The tag for the built image. Final tag will be 1.0.0alpha, 1.0.0-timestamp-commitID.
+      latest: false                                       # OPTIONAL: Defaults to false. If tag is not set to latest and this flag is set, then tag as latest as well and push latest as well.

.pipelines/pipeline.user.windows.official.all_tag.all_phase.all_config.ci_prod.yml

@@ -0,0 +1,55 @@
+environment:
+  host:
+    os: 'windows'
+    flavor: 'server'
+    version: '2019'
+  runtime:
+    provider: 'appcontainer'
+    image: 'cdpxwin1809.azurecr.io/user/azure-monitor/container-insights:6.0'
+    source_mode: 'map'
+
+version:
+  name: 'Certificate Generator and Out OMS plugin'
+  major: 10
+  minor: 0
+  tag: 'beta'
+  system: 'custom'
+  exclude_commit: true
+
+signing_options:
+  profile: 'azure'
+  codesign_validation_glob_pattern: 'regex|.+(?:dll|exe|sys|ps1|psm1|ps1xml|psc1|psd1|cdxml|vbs|js|wsf)$;-:file|**\linux\**' #CSV does not currently support binaries built for linux, so we exclude this folder
+
+static_analysis_options:
+  binskim_options:
+    files_to_scan:
+      - from: 'build\windows\installer\certificategenerator\bin\'
+        exclude: # exclude binaries which are referenced via dotnet packages and not built by us
+          - '**/**/**/BouncyCastle.Crypto.dll'
+          - '**/**/**/**/BouncyCastle.Crypto.dll'
+restore:
+  commands:
+    - !!defaultcommand
+       name: 'Restore dotnet packages'
+       command: '.pipelines/restore-windows.cmd'
+
+build:
+  commands:
+    - !!defaultcommand
+      name: 'Build Certificate Generator Source code and Out OMS Go plugin code'
+      command: '.pipelines/build-windows.cmd'
+      fail_on_stderr: false
+
+package:
+  commands:
+    - !!dockerbuildcommand                                # REQUIRED: This maps the command data to a concrete type in the CDPX orchestrator.
+      name: 'Build Docker Image'                          # REQUIRED: All commands have a name field. All console output captured when
+                                                          #           this command runs is tagged with the value of this field.
+      context_folder: 'kubernetes/windows'                # REQUIRED: The repository root relative path of the folder containing the Dockerfile to build.
+                                                          # In effect, the context folder will be repository_checkout_folder/src/DockerFinal.
+      dockerfile_name: 'Dockerfile'                       # OPTIONAL: The name of the dockerfile. Docker client does allow the Dockerfile
+                                                          # to be named differently. Defaults to Dockerfile.
+                                                          # In effect, the -f option value passed to docker build will be repository_checkout_folder/src/DockerFinal/Foo.dockerfile.
+      repository_name: 'cdpxwin1809'                      # only supported ones are cdpx acr repos
+      tag: 'win-ciprod'                                    # OPTIONAL: Defaults to latest. The tag for the built image. Final tag will be 1.0.0alpha, 1.0.0-timestamp-commitID.
+      latest: false                                       # OPTIONAL: Defaults to false. If tag is not set to latest and this flag is set, then tag as latest as well and push latest as well.

README.md

@@ -213,7 +213,7 @@ powershell -ExecutionPolicy bypass  # switch to powershell if you are not on pow
 
 # Azure DevOps Build Pipeline
 
-Navigate to https://github-private.visualstudio.com/microsoft/_build?view=pipelines to see Linux and Windows Agent build pipelines. These pipelines are configured with CI triggers for ci_dev and ci_prod (TBD).
+Navigate to https://github-private.visualstudio.com/microsoft/_build?view=pipelines to see Linux and Windows Agent build pipelines. These pipelines are configured with CI triggers for ci_dev and ci_prod.
 
 Docker Images will be pushed to CDPX ACR repos and these needs to retagged and pushed to corresponding ACR or docker hub. Only onboarded Azure AD AppId has permission to pull the images from CDPx ACRs.
 
@@ -236,7 +236,7 @@ Here are the instructions to onboard the feature branch to Azure Dev Ops pipelin
 
 # Azure DevOps Release Pipeline
 
-Integrated to Azure DevOps release pipeline for the ci_dev and ci_prod (TBD).With this, for every commit to ci_dev branch, latest bits automatically deployded to DEV AKS clusters in Build subscription and similarly for for every commit to ci_prod branch, latest bits automatically deployed to PROD AKS clusters in Build subscription.
+Integrated to Azure DevOps release pipeline for the ci_dev and ci_prod.With this, for every commit to ci_dev branch, latest bits automatically deployded to DEV AKS clusters in Build subscription and similarly for for every commit to ci_prod branch, latest bits automatically deployed to PROD AKS clusters in Build subscription.
 
 For dev, agent image will be in this format mcr.microsoft.com/azuremonitor/containerinsights/cidev:cidev<git-commit-id>.
 For prod, agent will be in this format mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod`<MM><DD><YYYY>`.

ReleaseNotes.md

@@ -11,6 +11,22 @@ additional questions or comments.
 
 Note : The agent version(s) below has dates (ciprod<mmddyyyy>), which indicate the agent build dates (not release dates)
 
+### 08/07/2020 -
+##### Version microsoft/oms:ciprod08072020 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod08072020 (linux)
+##### Version microsoft/oms:win-ciprod08072020 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:win-ciprod08072020 (windows)
+##### Code change log
+- Collection of KubeState metrics for deployments and HPA
+- Add the Proxy support for Windows agent
+- Fix for ContainerState in ContainerInventory to handle Failed state and collection of environment variables for terminated and failed containers
+- Change /spec to /metrics/cadvisor endpoint to collect node capacity metrics
+- Disable Health Plugin by default and can enabled via configmap
+- Pin version of jq to 1.5+dfsg-2
+- Bug fix for showing node as 'not ready' when there is disk pressure
+- oneagent integration (disabled by default)
+- Add region check before sending alertable metrics to MDM
+- Telemetry fix for agent telemetry for sov. clouds
+
+
 ### 07/15/2020 -
 ##### Version microsoft/oms:ciprod07152020 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod07152020 (linux)
 ##### Version microsoft/oms:win-ciprod05262020-2 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:win-ciprod05262020-2 (windows)

ReleaseProcess.md

@@ -5,15 +5,21 @@
 Here are the high-level instructions to get the CIPROD`<MM><DD><YYYY>` image for the production release
 1. create feature branch from ci_dev and make the following updates
       > Note: This required since Azure Dev Ops pipeline doesnt support --build-arg yet to automate this.
-   -  Ensure IMAGE_TAG updated with release candiate image tag in the DockerFile under kubernetes/linux and kubernetes/windows directory
-   - Update omsagent.yaml if there are any changes to the yaml
+   - Ensure IMAGE_TAG updated with release candiate image tag in the DockerFile under kubernetes/linux and kubernetes/windows directory
+   - Update the version file under build directory with build version and date
+   - Update omsagent.yaml for the image tag and dockerProviderVersion, and any other changes
+   - Update the chart version and image tags in values.yaml under charts/azuremonitor-containers
    - Release notes
 2. Make PR to ci_dev branch and once the PR approved, merge the changes to ci_dev
 3. Latest bits of ci_dev automatically deployed to CIDEV cluster in build subscription so just validated E2E to make sure everthing works
 4. If everything validated in DEV, make merge PR from ci_dev and ci_prod and merge once this reviewed by dev team
-5. Merge ci_dev and ci_prod branch which will trigger automatic deployment of latest bits to CIPROD cluster with CIPROD`<MM><DD><YYYY>` image (TBD)
+6. Update following pipeline variables under ReleaseCandiate with version of chart and image tag
+    - CIHELMCHARTVERSION <VersionValue> # For example, 2.7.4
+    - CIImageTagSuffix <ImageTag> # ciprod08072020 or ciprod08072020-1 etc.
+7. Merge ci_dev and ci_prod branch which will trigger automatic deployment of latest bits to CIPROD cluster with CIPROD`<MM><DD><YYYY>` image to test and scale cluters, AKS, AKS-Engine
    > Note: production image automatically pushed to CIPROD Public cloud ACR which will inturn replicated to Public cloud MCR.
-6. Validate all the scenarios against CIPROD cluster in Build subscription
+8. Validate all the scenarios against clusters in build subscription and scale clusters
+
 
 # 2. Perf and scale testing
 
@@ -27,7 +33,7 @@ Image automatically synched to MCR CN from Public cloud MCR.
 
 ## AKS
 
-Make PR against [AKS-RP](https://msazure.visualstudio.com/CloudNativeCompute/_git/aks-rp?version=GBmaster) repo with chart update(s)
+- Refer to internal docs for the release process and instructions.
 
 ## ARO v3
 

build/version

@@ -5,8 +5,8 @@
 CONTAINER_BUILDVERSION_MAJOR=10
 CONTAINER_BUILDVERSION_MINOR=0
 CONTAINER_BUILDVERSION_PATCH=0
-CONTAINER_BUILDVERSION_BUILDNR=1
-CONTAINER_BUILDVERSION_DATE=20200526
+CONTAINER_BUILDVERSION_BUILDNR=4
+CONTAINER_BUILDVERSION_DATE=20200805
 CONTAINER_BUILDVERSION_STATUS=Developer_Build
 
 #-------------------------------- End of File -----------------------------------

charts/azuremonitor-containers/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v1
 appVersion: 7.0.0-1
 description: Helm chart for deploying Azure Monitor container monitoring agent in Kubernetes
 name: azuremonitor-containers
-version: 2.7.3
+version: 2.7.4
 kubeVersion: "^1.10.0-0"
 keywords:
   - monitoring

charts/azuremonitor-containers/values.yaml

@@ -7,10 +7,10 @@
 omsagent:
   image:
     repo: "mcr.microsoft.com/azuremonitor/containerinsights/ciprod"
-    tag: "ciprod07152020"
-    tagWindows: "win-ciprod05262020-2"
+    tag: "ciprod08072020"
+    tagWindows: "win-ciprod08072020"
     pullPolicy: IfNotPresent
-    dockerProviderVersion: "10.0.0-3"
+    dockerProviderVersion: "10.0.0-4"
     agentVersion: "1.10.0.1"
   ## To get your workspace id and key do the following
   ## You can create a Azure Loganalytics workspace from portal.azure.com and get its ID & PRIMARY KEY from 'Advanced Settings' tab in the Ux.

kubernetes/linux/Dockerfile

@@ -2,7 +2,7 @@ FROM ubuntu:18.04
 MAINTAINER OMSContainers@microsoft.com
 LABEL vendor=Microsoft\ Corp \
     com.microsoft.product="Azure Monitor for containers"
-ARG IMAGE_TAG=ciprod07152020
+ARG IMAGE_TAG=ciprod08072020
 ENV AGENT_VERSION ${IMAGE_TAG}
 ENV tmpdir /opt
 ENV APPLICATIONINSIGHTS_AUTH NzAwZGM5OGYtYTdhZC00NThkLWI5NWMtMjA3ZjM3NmM3YmRi

kubernetes/omsagent.yaml

@@ -337,13 +337,13 @@ spec:
         tier: node
       annotations:
         agentVersion: "1.10.0.1"
-        dockerProviderVersion: "10.0.0-3"
+        dockerProviderVersion: "10.0.0-4"
         schema-versions: "v1"
     spec:
       serviceAccountName: omsagent
       containers:
         - name: omsagent
-          image: "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod07152020"
+          image: "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod08072020"
           imagePullPolicy: IfNotPresent
           resources:
             limits:
@@ -480,13 +480,13 @@ spec:
         rsName: "omsagent-rs"
       annotations:
         agentVersion: "1.10.0.1"
-        dockerProviderVersion: "10.0.0-3"
+        dockerProviderVersion: "10.0.0-4"
         schema-versions: "v1"
     spec:
       serviceAccountName: omsagent
       containers:
         - name: omsagent
-          image: "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod07152020"
+          image: "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod08072020"
           imagePullPolicy: IfNotPresent
           resources:
             limits:
@@ -631,13 +631,13 @@ spec:
         tier: node-win
       annotations:
         agentVersion: "1.10.0.1"
-        dockerProviderVersion: "10.0.0-2"
+        dockerProviderVersion: "10.0.0-4"
         schema-versions: "v1"
     spec:
      serviceAccountName: omsagent
      containers:
        - name: omsagent-win
-         image: "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:win-ciprod05262020-2"
+         image: "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:win-ciprod08072020"
          imagePullPolicy: IfNotPresent
          resources:
           limits:

kubernetes/windows/Dockerfile

@@ -1,9 +1,36 @@
-FROM mcr.microsoft.com/azuremonitor/containerinsights/ciprod:winakslogbase-07022020
+FROM mcr.microsoft.com/windows/servercore:ltsc2019
 MAINTAINER OMSContainers@microsoft.com
 LABEL vendor=Microsoft\ Corp \
     com.microsoft.product="Azure Monitor for containers"
 
-ARG IMAGE_TAG=win-ciprod05262020-2
+ARG IMAGE_TAG=win-ciprod08072020
+
+# Do not split this into multiple RUN!
+# Docker creates a layer for every RUN-Statement
+RUN powershell -Command "Set-ExecutionPolicy Bypass -Scope Process -Force; iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))"
+# Fluentd depends on cool.io whose fat gem is only available for Ruby < 2.5, so need to specify --platform ruby when install Ruby > 2.5 and install msys2 to get dev tools
+RUN choco install -y ruby --version 2.6.5.1 --params "'/InstallDir:C:\ruby26'" \
+&& choco install -y msys2 --version 20190524.0.0.20191030 --params "'/NoPath /NoUpdate /InstallDir:C:\ruby26\msys64'" \
+&& choco install -y vim
+
+# gangams - optional MSYS2 update via ridk failing in merged docker file so skipping that since we dont need optional update
+RUN refreshenv \
+&& ridk install 3 \
+&& echo gem: --no-document >> C:\ProgramData\gemrc \
+&& gem install cool.io -v 1.5.4 --platform ruby \
+&& gem install oj -v 3.3.10 \
+&& gem install json -v 2.2.0 \
+&& gem install fluentd -v 1.10.2 \
+&& gem install win32-service -v 1.0.1 \
+&& gem install win32-ipc -v 0.7.0 \
+&& gem install win32-event -v 0.6.3 \
+&& gem install windows-pr -v 1.2.6 \
+&& gem install tomlrb -v 1.3.0 \
+&& gem install gyoku -v 1.3.1 \
+&& gem sources --clear-all
+
+# Remove gem cache and chocolatey
+RUN powershell -Command "Remove-Item -Force C:\ruby26\lib\ruby\gems\2.6.0\cache\*.gem; Remove-Item -Recurse -Force 'C:\ProgramData\chocolatey'"
 
 SHELL ["powershell"]
 

source/plugins/go/src/oms.go

@@ -1323,9 +1323,7 @@ func InitializePlugin(pluginConfPath string, agentVersion string) {
 		ContainerLogsRouteV2 = true
 		Log("Routing container logs thru %s route...", ContainerLogsV2Route)
 		fmt.Fprintf(os.Stdout, "Routing container logs thru %s route... \n", ContainerLogsV2Route)
-	//} else if strings.Compare(ContainerLogsRoute, ContainerLogsADXRoute) == 0 {
-	//making dormant with below comparison for now --
-	} else if strings.Compare("willnot", "match") == 0 {
+	} else if strings.Compare(ContainerLogsRoute, ContainerLogsADXRoute) == 0 {
 		//check if adx clusteruri, clientid & secret are set
 		var err error
 		AdxClusterUri, err = ReadFileContents(PluginConfiguration["adx_cluster_uri_path"])