Merge ci_dev into ci_prod for 02232021 release

.pipelines/update-place-holdres-in-e2e-tests.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+echo "start: update placeholders of e2e-tests.yaml ..."
+
+for ARGUMENT in "$@"
+do
+   KEY=$(echo $ARGUMENT | cut -f1 -d=)
+   VALUE=$(echo $ARGUMENT | cut -f2 -d=)
+
+   case "$KEY" in           
+           TENANT_ID) TENANT_ID=$VALUE ;;           
+           *)
+    esac
+done
+
+echo "start: read appid and appsecret"
+# used the same SP which used for acr
+CLIENT_ID=$(cat ~/acrappid)
+CLIENT_SECRET=$(cat ~/acrappsecret)
+echo "end: read appid and appsecret"
+
+echo "Service Principal CLIENT_ID:$CLIENT_ID"
+echo "replace CLIENT_ID value"
+sed -i "s=SP_CLIENT_ID_VALUE=$CLIENT_ID=g" e2e-tests.yaml
+
+# only uncomment for debug purpose
+# echo "Service Principal CLIENT_SECRET:$CLIENT_SECRET"
+echo "replace CLIENT_SECRET value"
+sed -i "s=SP_CLIENT_SECRET_VALUE=$CLIENT_SECRET=g" e2e-tests.yaml
+
+echo "Service Principal TENANT_ID:$TENANT_ID"
+echo "replace TENANT_ID value"
+sed -i "s=SP_TENANT_ID_VALUE=$TENANT_ID=g" e2e-tests.yaml
+
+echo "end: update placeholders of e2e-tests.yaml."

.pipelines/validate-e2e-tests-results.sh

@@ -0,0 +1,71 @@
+#!/bin/bash
+echo "start: validating results of e2e-tests  ..."
+DEFAULT_SONOBUOY_VERSION="0.20.0"
+DEFAULT_TIME_OUT_IN_MINS=60
+for ARGUMENT in "$@"
+do
+   KEY=$(echo $ARGUMENT | cut -f1 -d=)
+   VALUE=$(echo $ARGUMENT | cut -f2 -d=)
+
+   case "$KEY" in
+           SONOBUOY_VERSION) SONOBUOY_VERSION=$VALUE ;;
+           *)
+    esac
+done
+
+if [ -z $SONOBUOY_VERSION ]; then
+   SONOBUOY_VERSION=$DEFAULT_SONOBUOY_VERSION
+fi
+
+echo "sonobuoy version: ${SONOBUOY_VERSION}"
+
+echo "start: downloading sonobuoy"
+curl -LO https://github.com/vmware-tanzu/sonobuoy/releases/download/v${SONOBUOY_VERSION}/sonobuoy_${SONOBUOY_VERSION}_linux_amd64.tar.gz
+echo "end: downloading sonobuoy"
+
+echo "start: extract sonobuoy tar file"
+mkdir -p sonobuoy-install/
+tar -zxf sonobuoy_${SONOBUOY_VERSION}_*.tar.gz -C sonobuoy-install/
+echo "end: extract sonobuoy tar file"
+
+echo "start: move sonobuoy binaries to /usr/local/bin/"
+mv -f sonobuoy-install/sonobuoy /usr/local/bin/
+echo "end: move sonobuoy binaries to /usr/local/bin/"
+
+rm -rf sonobuoy_${SONOBUOY_VERSION}_*.tar.gz sonobuoy-install/
+
+results=$(sonobuoy retrieve)
+mins=0
+IsSucceeded=true
+while [ $mins -le $DEFAULT_TIME_OUT_IN_MINS ]
+do
+  # check the status 
+  echo "checking test status"
+  status=$(sonobuoy status)
+  status=$(echo $status | sed 's/`//g')
+  if [[ $status == *"completed"* ]]; then
+      echo "test run completed"
+      mins=$DEFAULT_TIME_OUT_IN_MINS
+       if [[ $status == *"failed"* ]]; then
+          IsSucceeded=false          
+       fi
+  else
+    echo "sleep for 1m to check the status again"
+    sleep 1m
+  fi
+  mins=$(( $mins + 1 ))
+done
+echo "status:${IsSucceeded}"
+
+results=$(sonobuoy retrieve)
+sonobuoy results $results
+
+if $IsSucceeded == true; then 
+   echo "all test passed"
+   exit 0
+else
+   echo "tests are failed. please review the results by downloading tar file via sonobuoy retrieve command"
+   exit 1
+fi 
+
+echo "end: validating results of e2e-tests  ..."

README.md

@@ -91,6 +91,7 @@ The general directory structure is:
 │   │   |   ...                               - plugins in, out and filters code in ruby
 │   ├── toml-parser/                          - code for parsing of toml configuration files
 ├── test/                                     - source code for tests
+│   ├── e2e/                                  - e2e tests to validate agent and e2e workflow(s)
 │   ├── unit-tests/                           - unit tests code
 │   ├── scenario/                             - scenario tests code
 ├── !_README.md                               - this file
@@ -271,6 +272,36 @@ For DEV and PROD branches, automatically deployed latest yaml with latest agent
 
 # E2E Tests
 
+## For executing tests
+
+1. Deploy the omsagent.yaml with your agent image. In the yaml, make sure `ISTEST` environment variable set to `true` if its not set already
+2. Update the Service Principal CLIENT_ID, CLIENT_SECRET and TENANT_ID placeholder values and apply e2e-tests.yaml to execute the tests 
+    > Note: Service Principal requires reader role on log analytics workspace and cluster resource to query LA and metrics
+   ```
+   cd ~/Docker-Provider/test/e2e # based on your repo path    
+   kubectl apply -f e2e-tests.yaml # this will trigger job to run the tests in sonobuoy namespace 
+   kubectl get po -n sonobuoy # to check the pods and jobs associated to tests   
+   ``` 
+3. Download (sonobuoy)[https://github.com/vmware-tanzu/sonobuoy/releases] on your dev box to view the results of the tests
+   ```
+   results=$(sonobuoy retrieve) # downloads tar file which has logs and test results
+   sonobuoy results $results # get the summary of the results
+   tar -xzvf <downloaded-tar-file> # extract downloaded tar file and look for pod logs, results and other k8s resources if there are any failures
+   ```
+
+## For adding new tests
+
+1. Add the test python file with your test code under `tests` directory
+2. Build the docker image, recommended to use ACR & MCR 
+  ```
+   cd ~/Docker-Provider/test/e2e/src # based on your repo path 
+   docker login <acr> -u <user> -p <pwd> # login to acr
+   docker build -f ./core/Dockerfile -t <repo>/<imagename>:<imagetag> .
+   docker push <repo>/<imagename>:<imagetag>
+  ```
+3. update existing agentest image tag in e2e-tests.yaml with newly built image tag with MCR repo
+
+# Scenario Tests
 Clusters are used in release pipeline already has the yamls under test\scenario deployed. Make sure to validate these scenarios.
 If you have new interesting scenarios, please add/update them.
 

ReleaseNotes.md

@@ -10,6 +10,20 @@ additional questions or comments.
 ## Release History
 
 Note : The agent version(s) below has dates (ciprod<mmddyyyy>), which indicate the agent build dates (not release dates)
+### 02/23/2021 -
+##### Version microsoft/oms:ciprod02232021 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod02232021 (linux)
+##### Version microsoft/oms:win-ciprod02232021 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:win-ciprod02232021 (windows)
+##### Code change log
+- ContainerLogV2 schema support for LogAnalytics & ADX (not usable externally yet)
+- Fix nodemetrics (cpuusageprecentage & memoryusagepercentage) metrics not flowing. This is fixed upstream for k8s versions >= 1.19.7 and >=1.20.2.
+- Fix cpu & memory usage exceeded threshold container metrics not flowing when requests and/or limits were not set
+- Mute some unused exceptions from going to telemetry
+- Collect containerimage (repository, image & imagetag) from spec (instead of runtime)
+- Add support for extension MSI for k8s arc
+- Use cloud specific instrumentation keys for telemetry
+- Picked up newer version for apt
+- Add priority class to daemonset (in our chart only)
+
 ### 01/11/2021 -
 ##### Version microsoft/oms:ciprod01112021 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod01112021 (linux)
 ##### Version microsoft/oms:win-ciprod01112021 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:win-ciprod01112021 (windows)
@@ -27,68 +41,6 @@ Note : The agent version(s) below has dates (ciprod<mmddyyyy>), which indicate t
 - Enable ADX route for windows container logs
 - Remove logging to termination log in windows agent liveness probe
 
-
-### 11/09/2020 -
-##### Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod11092020 (linux)
-##### Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:win-ciprod11092020 (windows)
-##### Code change log
-- Fix for duplicate windows metrics
-
-### 10/27/2020 -
-##### Version microsoft/oms:ciprod10272020 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod10272020 (linux)
-##### Version microsoft/oms:win-ciprod10272020 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:win-ciprod10052020 (windows)
-##### Code change log
-- Activate oneagent in few AKS regions (koreacentral,norwayeast)
-- Disable syslog
-- Fix timeout for Windows daemonset liveness probe
-- Make request == limit for Windows daemonset resources (cpu & memory)
-- Schema v2 for container log (ADX only - applicable only for select customers for piloting)
-
-### 10/05/2020 -
-##### Version microsoft/oms:ciprod10052020 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod10052020 (linux)
-##### Version microsoft/oms:win-ciprod10052020 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:win-ciprod10052020 (windows)
-##### Code change log
-- Health CRD to version v1 (from v1beta1) for k8s versions >= 1.19.0
-- Collection of PV usage metrics for PVs mounted by pods (kube-system pods excluded by default)(doc-link-needed)
-- Zero fill few custom metrics under a timer, also add zero filling for new PV usage metrics
-- Collection of additional Kubelet metrics ('kubelet_running_pod_count','volume_manager_total_volumes','kubelet_node_config_error','process_resident_memory_bytes','process_cpu_seconds_total','kubelet_runtime_operations_total','kubelet_runtime_operations_errors_total'). This also includes updates to 'kubelet' workbook to include these new metrics
-- Collection of Azure NPM (Network Policy Manager) metrics (basic & advanced. By default, NPM metrics collection is turned OFF)(doc-link-needed)
-- Support log collection when docker root is changed with knode. Tracked by [this](https://github.com/Azure/AKS/issues/1373) issue
-- Support for Pods in 'Terminating' state for nodelost scenarios
-- Fix for reduction in telemetry for custom metrics ingestion failures
-- Fix CPU capacity/limits metrics being 0 for Virtual nodes (VK)
-- Add new custom metric regions (eastus2,westus,australiasoutheast,brazilsouth,germanywestcentral,northcentralus,switzerlandnorth)
-- Enable strict SSL validation for AppInsights Ruby SDK
-- Turn off custom metrics upload for unsupported cluster types
-- Install CA certs from wire server for windows (in certain clouds)
-
-### 09/16/2020 -
-> Note: This agent release targetted ONLY for non-AKS clusters via Azure Monitor for containers HELM chart update
-##### Version microsoft/oms:ciprod09162020 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod09162020 (linux)
-##### Version microsoft/oms:win-ciprod09162020 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:win-ciprod09162020 (windows)
-##### Code change log
-- Collection of Azure Network Policy Manager Basic and Advanced metrics
-- Add support in Windows Agent for Container log collection of CRI runtimes such as ContainerD
-- Alertable metrics support Arc K8s cluster to parity with AKS
-- Support for multiple container log mount paths when docker is updated through knode
-- Bug fix related to MDM telemetry
-
-### 08/07/2020 -
-##### Version microsoft/oms:ciprod08072020 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod08072020 (linux)
-##### Version microsoft/oms:win-ciprod08072020 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:win-ciprod08072020 (windows)
-##### Code change log
-- Collection of KubeState metrics for deployments and HPA
-- Add the Proxy support for Windows agent
-- Fix for ContainerState in ContainerInventory to handle Failed state and collection of environment variables for terminated and failed containers
-- Change /spec to /metrics/cadvisor endpoint to collect node capacity metrics
-- Disable Health Plugin by default and can enabled via configmap
-- Pin version of jq to 1.5+dfsg-2
-- Bug fix for showing node as 'not ready' when there is disk pressure
-- oneagent integration (disabled by default)
-- Add region check before sending alertable metrics to MDM
-- Telemetry fix for agent telemetry for sov. clouds
-
-
 ### 11/09/2020 -
 ##### Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod11092020 (linux)
 ##### Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:win-ciprod11092020 (windows)
@@ -97,7 +49,7 @@ Note : The agent version(s) below has dates (ciprod<mmddyyyy>), which indicate t
 
 ### 10/27/2020 -
 ##### Version microsoft/oms:ciprod10272020 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod10272020 (linux)
-##### Version microsoft/oms:win-ciprod10272020 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:win-ciprod10052020 (windows)
+##### Version microsoft/oms:win-ciprod10272020 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:win-ciprod10272020 (windows)
 ##### Code change log
 - Activate oneagent in few AKS regions (koreacentral,norwayeast)
 - Disable syslog

build/common/installer/scripts/tomlparser.rb

@@ -23,6 +23,7 @@
 @logExclusionRegexPattern = "(^((?!stdout|stderr).)*$)"
 @excludePath = "*.csv2" #some invalid path
 @enrichContainerLogs = false
+@containerLogSchemaVersion = ""
 @collectAllKubeEvents = false
 @containerLogsRoute = ""
 
@@ -138,6 +139,16 @@ def populateSettingValuesFromConfigMap(parsedConfig)
       ConfigParseErrorLogger.logError("Exception while reading config map settings for cluster level container log enrichment - #{errorStr}, using defaults, please check config map for errors")
     end
 
+     #Get container log schema version setting
+     begin
+      if !parsedConfig[:log_collection_settings][:schema].nil? && !parsedConfig[:log_collection_settings][:schema][:containerlog_schema_version].nil?
+        @containerLogSchemaVersion = parsedConfig[:log_collection_settings][:schema][:containerlog_schema_version]
+        puts "config::Using config map setting for container log schema version"
+      end
+    rescue => errorStr
+      ConfigParseErrorLogger.logError("Exception while reading config map settings for container log schema version - #{errorStr}, using defaults, please check config map for errors")
+    end
+
     #Get kube events enrichment setting
     begin
       if !parsedConfig[:log_collection_settings][:collect_all_kube_events].nil? && !parsedConfig[:log_collection_settings][:collect_all_kube_events][:enabled].nil?
@@ -200,6 +211,7 @@ def populateSettingValuesFromConfigMap(parsedConfig)
   file.write("export AZMON_CLUSTER_CONTAINER_LOG_ENRICH=#{@enrichContainerLogs}\n")
   file.write("export AZMON_CLUSTER_COLLECT_ALL_KUBE_EVENTS=#{@collectAllKubeEvents}\n")
   file.write("export AZMON_CONTAINER_LOGS_ROUTE=#{@containerLogsRoute}\n")
+  file.write("export AZMON_CONTAINER_LOG_SCHEMA_VERSION=#{@containerLogSchemaVersion}\n")
   # Close file after writing all environment variables
   file.close
   puts "Both stdout & stderr log collection are turned off for namespaces: '#{@excludePath}' "
@@ -246,6 +258,8 @@ def get_command_windows(env_variable_name, env_variable_value)
     file.write(commands)
     commands = get_command_windows('AZMON_CONTAINER_LOGS_EFFECTIVE_ROUTE', @containerLogsRoute)
     file.write(commands)
+    commands = get_command_windows('AZMON_CONTAINER_LOG_SCHEMA_VERSION', @containerLogSchemaVersion)
+    file.write(commands)
 
     # Close file after writing all environment variables
     file.close

build/version

@@ -2,11 +2,11 @@
 
 # Build Version Information
 
-CONTAINER_BUILDVERSION_MAJOR=12
+CONTAINER_BUILDVERSION_MAJOR=13
 CONTAINER_BUILDVERSION_MINOR=0
 CONTAINER_BUILDVERSION_PATCH=0
 CONTAINER_BUILDVERSION_BUILDNR=0
-CONTAINER_BUILDVERSION_DATE=20210111
+CONTAINER_BUILDVERSION_DATE=20210223
 CONTAINER_BUILDVERSION_STATUS=Developer_Build
 
 #-------------------------------- End of File -----------------------------------

charts/azuremonitor-containers/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v1
 appVersion: 7.0.0-1
 description: Helm chart for deploying Azure Monitor container monitoring agent in Kubernetes
 name: azuremonitor-containers
-version: 2.8.0
+version: 2.8.1
 kubeVersion: "^1.10.0-0"
 keywords:
   - monitoring

charts/azuremonitor-containers/README.md

@@ -93,13 +93,15 @@ The following table lists the configurable parameters of the MSOMS chart and the
 | `omsagent.env.clusterName`   | Name of your cluster                                    | Does not have a default value, needs to be provided                                                                         |
 | `omsagent.rbac`              | rbac enabled/disabled                                   | true  (i.e.enabled)                                                                                                           |
 | `omsagent.proxy`             | Proxy endpoint                                          | Doesnt have default value. Refer to [configure proxy](#Configuring-Proxy-Endpoint) |
+| `omsagent.priority`          | DaemonSet Pod Priority                                  | This is the [priority](https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/) to use for the daemonsets such that they get scheduled onto the node ahead of "normal" pods - must be an integer, defaults to 10 |
 
 > Note: For Azure Manage K8s clusters such as Azure Arc K8s and ARO v4, `omsagent.env.clusterId` with fully qualified azure resource id of the cluster should be used instead of `omsagent.env.clusterName`
 
 ### Note
 
 - Parameter `omsagent.env.doNotCollectKubeSystemLogs` has been removed starting chart version 1.0.0. Refer to 'Agent data collection settings' section below to configure it using configmap.
 - onboarding of multiple clusters with the same cluster name to same log analytics workspace not supported. If need this configuration, use the cluster FQDN name rather than cluster dns prefix to avoid collision with clusterName
+- The `omsagent.priority` parameter sets the priority of the omsagent daemonset priority class.  This pod priority class is used for daemonsets to allow them to have priority over pods that can be scheduled elsewhere.  Without a priority class, it is possible for a node to fill up with "normal" pods before the daemonset pods get to be created for the node or get scheduled.  Note that pods are not "daemonset" pods - they are just pods created by the daemonset controller but they have a specific affinity set during creation to the specific node each pod was created to run on.  You want this value to be greater than 0 (default is 10) and generally greater than pods that have the flexibility to run on different nodes such that they do not block the node specific pods.
 
 ## Agent data collection settings
 

charts/azuremonitor-containers/templates/omsagent-arc-k8s-crd.yaml

@@ -1,9 +1,26 @@
 {{- if or ( contains "microsoft.kubernetes/connectedclusters" (.Values.Azure.Cluster.ResourceId | lower) ) ( contains "microsoft.kubernetes/connectedclusters" (.Values.omsagent.env.clusterId | lower)) }}
+#extension model
+{{- if not (empty .Values.Azure.Extension.Name) }}
+apiVersion: clusterconfig.azure.com/v1beta1
+kind: AzureExtensionIdentity
+metadata:
+  name: {{ .Values.Azure.Extension.Name }}
+  namespace: azure-arc
+spec:
+  serviceAccounts:
+    - name: omsagent
+      namespace: kube-system
+  tokenNamespace: azure-arc
+---
+{{- end }}
 apiVersion:  clusterconfig.azure.com/v1beta1
 kind: AzureClusterIdentityRequest
 metadata:
   name: container-insights-clusteridentityrequest
   namespace: azure-arc
 spec:
   audience: https://monitoring.azure.com/
+  {{- if not (empty .Values.Azure.Extension.Name) }}
+  resourceId: {{ .Values.Azure.Extension.Name }} 
+  {{- end }}
 {{- end }}

charts/azuremonitor-containers/templates/omsagent-daemonset-windows.yaml

@@ -27,10 +27,11 @@ spec:
     checksum/secret: {{ include (print $.Template.BasePath "/omsagent-secret.yaml") . | sha256sum }}
     checksum/config: {{ toYaml .Values.omsagent.resources | sha256sum }}
   spec:
-   dnsConfig:    
+   priorityClassName: omsagent
+   dnsConfig:
      options:
        - name: ndots
-         value: "3"     
+         value: "3"
 {{- if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion }}
    nodeSelector:
       kubernetes.io/os: windows