David/to merge into ciprod 04222021

.pipelines/build-linux.sh

@@ -14,3 +14,8 @@ cd $DIR/../build/linux
 echo "----------- Build Docker Provider -------------------------------"
 make
 cd $DIR
+
+echo "------------ Bundle Shell Extension Scripts & HELM chart -------------------------"
+cd $DIR/../deployment/arc-k8s-extension/ServiceGroupRoot/Scripts
+tar -czvf ../artifacts.tar.gz ../../../../charts/azuremonitor-containers/ pushChartToAcr.sh
+

.pipelines/pipeline.user.linux.yml

@@ -24,10 +24,15 @@ restore:
 
 build:
   commands:
-    - !!defaultcommand
+    - !!buildcommand
       name: 'Build Docker Provider Shell Bundle'
       command: '.pipelines/build-linux.sh'
       fail_on_stderr: false
+      artifacts:
+        - from: 'deployment'
+          to: 'build'
+          include:
+            - '**'
 
 package:
   commands:

.pipelines/pull-from-cdpx-and-push-to-ci-acr-linux-image.sh

@@ -35,12 +35,22 @@ echo "end: read appid and appsecret which has read access on cdpx acr"
 # suffix 00 primary and 01 secondary, and we only use primary
 # This configured via pipeline variable
 echo "login to cdpxlinux acr:${CDPX_ACR}"
-docker login $CDPX_ACR  --username $CDPX_ACR_APP_ID --password $CDPX_ACR_APP_SECRET
-echo "login to cdpxlinux acr completed: ${CDPX_ACR}"
+echo $CDPX_ACR_APP_SECRET | docker login $CDPX_ACR  --username $CDPX_ACR_APP_ID --password-stdin
+if [ $? -eq 0 ]; then         
+   echo "login to cdpxlinux acr: ${CDPX_ACR} completed successfully."
+else     
+   echo "-e error login to cdpxlinux acr: ${CDPX_ACR} failed.Please see release task logs."
+   exit 1
+fi  
 
 echo "pull agent image from cdpxlinux acr: ${CDPX_ACR}"
 docker pull ${CDPX_ACR}/official/${CDPX_REPO_NAME}:${CDPX_AGENT_IMAGE_TAG}
-echo "pull image from cdpxlinux acr completed: ${CDPX_ACR}"
+if [ $? -eq 0 ]; then
+   echo "pulling of agent image from cdpxlinux acr: ${CDPX_ACR} completed successfully."
+else
+   echo "-e error pulling of agent image from cdpxlinux acr: ${CDPX_ACR} failed.Please see release task logs."
+   exit 1
+fi  
 
 echo "CI Release name is:"$CI_RELEASE
 imagetag=$CI_RELEASE$CI_IMAGE_TAG_SUFFIX
@@ -51,13 +61,29 @@ echo "CI AGENT REPOSITORY NAME : ${CI_AGENT_REPO}"
 
 echo "tag linux agent image"
 docker tag ${CDPX_ACR}/official/${CDPX_REPO_NAME}:${CDPX_AGENT_IMAGE_TAG} ${CI_ACR}/public/azuremonitor/containerinsights/${CI_AGENT_REPO}:${imagetag}
+if [ $? -eq 0 ]; then         
+   echo "tagging of linux agent image completed successfully."
+else     
+    echo "-e error tagging of linux agent image failed. Please see release task logs."
+   exit 1
+fi  
 
 echo "login ciprod acr":$CI_ACR
-docker login $CI_ACR --username $ACR_APP_ID --password $ACR_APP_SECRET
-echo "login to ${CI_ACR} acr completed"
+echo $ACR_APP_SECRET | docker login $CI_ACR --username $ACR_APP_ID --password-stdin 
+if [ $? -eq 0 ]; then         
+   echo "login to ciprod acr: ${CI_ACR} completed successfully"
+else     
+   echo "-e error login to ciprod acr: ${CI_ACR} failed. Please see release task logs."
+   exit 1
+fi  
 
 echo "pushing the image to ciprod acr:${CI_ACR}"
 docker push ${CI_ACR}/public/azuremonitor/containerinsights/${CI_AGENT_REPO}:${imagetag}
-echo "pushing the image to ciprod acr completed"
+if [ $? -eq 0 ]; then
+   echo "pushing of the image to ciprod acr completed successfully"
+else     
+   echo "-e error pushing of image to ciprod acr failed. Please see release task logs."
+   exit 1
+fi  
 
 echo "end: pull linux agent image from cdpx and push to ciprod acr"

.pipelines/pull-from-cdpx-and-push-to-ci-acr-windows-image.sh

@@ -34,12 +34,22 @@ echo "end: read appid and appsecret which has read access on cdpx acr"
 # suffix 00 primary and 01 secondary, and we only use primary
 # This configured via pipeline variable
 echo "login to cdpxwindows acr:${CDPX_ACR}"
-docker login $CDPX_ACR  --username $CDPX_ACR_APP_ID --password $CDPX_ACR_APP_SECRET
-echo "login to cdpxwindows acr:${CDPX_ACR} completed"
+echo $CDPX_ACR_APP_SECRET | docker login $CDPX_ACR  --username $CDPX_ACR_APP_ID --password-stdin
+if [ $? -eq 0 ]; then         
+   echo "login to cdpxwindows acr: ${CDPX_ACR} completed successfully."
+else     
+   echo "-e error login to cdpxwindows acr: ${CDPX_ACR} failed.Please see release task logs."
+   exit 1
+fi  
 
 echo "pull image from cdpxwin acr: ${CDPX_ACR}"
 docker pull ${CDPX_ACR}/official/${CDPX_REPO_NAME}:${CDPX_AGENT_IMAGE_TAG}
-echo "pull image from cdpxwin acr completed: ${CDPX_ACR}"
+if [ $? -eq 0 ]; then         
+   echo "pulling of image from cdpxwin acr: ${CDPX_ACR} completed successfully."
+else     
+   echo "pulling of image from cdpxwin acr: ${CDPX_ACR} failed. Please see release task logs."
+   exit 1
+fi  
 
 echo "CI Release name:"$CI_RELEASE
 echo "CI Image Tax suffix:"$CI_IMAGE_TAG_SUFFIX
@@ -49,13 +59,30 @@ echo "agentimagetag="$imagetag
 
 echo "tag windows agent image"
 docker tag ${CDPX_ACR}/official/${CDPX_REPO_NAME}:${CDPX_AGENT_IMAGE_TAG} ${CI_ACR}/public/azuremonitor/containerinsights/${CI_AGENT_REPO}:${imagetag}
+if [ $? -eq 0 ]; then         
+   echo "tagging of windows agent image completed successfully."
+else     
+   echo "-e error tagging of windows agent image failed. Please see release task logs."
+   exit 1
+fi  
 
 echo "login to ${CI_ACR} acr"
-docker login $CI_ACR --username $ACR_APP_ID --password $ACR_APP_SECRET
-echo "login to ${CI_ACR} acr completed"
+echo $ACR_APP_SECRET | docker login $CI_ACR --username $ACR_APP_ID --password-stdin
+if [ $? -eq 0 ]; then
+   echo "login to acr: ${CI_ACR} completed successfully."
+else     
+   echo "login to acr: ${CI_ACR} failed. Please see release task logs."
+   exit 1
+fi  
+
 
 echo "pushing the image to ciprod acr"
 docker push ${CI_ACR}/public/azuremonitor/containerinsights/${CI_AGENT_REPO}:${imagetag}
-echo "pushing the image to ciprod acr completed"
+if [ $? -eq 0 ]; then
+   echo "pushing the image to ciprod acr completed successfully."
+else     
+   echo "pushing the image to ciprod acr failed. Please see release task logs"
+   exit 1
+fi  
 
 echo "end: pull windows agent image from cdpx and push to ciprod acr"

Documentation/OSMPrivatePreview/ReadMe.md

@@ -0,0 +1,71 @@
+Note - This is private preview. For any support issues, please reach out to us at [askcoin@microsoft.com](mailto:askcoin@microsoft.com). Please don't open a support ticket.
+
+# Azure Monitor Container Insights Open Service Mesh Monitoring
+
+Azure Monitor container insights now supporting preview of [Open Service Mesh(OSM)](https://docs.microsoft.com/azure/aks/servicemesh-osm-about) Monitoring. As part of this support, customer can:
+1.	Filter & view inventory of all the services that are part of your service mesh.
+2.	Visualize and monitor requests between services in your service mesh, with request latency, error rate & resource utilization by services.
+3.	Provides connection summary for OSM infrastructure running on AKS.
+
+## How to onboard Container Insights OSM monitoring?
+OSM exposes Prometheus metrics which Container Insights can collect, for container insights agent to collect OSM metrics follow the following steps.
+
+1.  Follow this [link](https://docs.microsoft.com/en-us/azure/aks/servicemesh-osm-about?pivots=client-operating-system-linux#register-the-aks-openservicemesh-preview-feature) as a prereq before enabling the addon. 
+
+2.  Enable AKS OSM addon on your 
+     - [New AKS cluster](https://docs.microsoft.com/en-us/azure/aks/servicemesh-osm-about?pivots=client-operating-system-linux#install-open-service-mesh-osm-azure-kubernetes-service-aks-add-on-for-a-new-aks-cluster)
+     - [Existing AKS cluster](https://docs.microsoft.com/en-us/azure/aks/servicemesh-osm-about?pivots=client-operating-system-linux#enable-open-service-mesh-osm-azure-kubernetes-service-aks-add-on-for-an-existing-aks-cluster)
+2.	Configure OSM to allow Prometheus scraping, follow steps from [here](https://docs.microsoft.com/en-us/azure/aks/servicemesh-osm-about?pivots=client-operating-system-linux#configure-osm-to-allow-prometheus-scraping)
+3.  To enable namespace(s), download the osm client library [here](https://docs.microsoft.com/en-us/azure/aks/servicemesh-osm-about?pivots=client-operating-system-linux#osm-service-quotas-and-limits-preview) & then enable metrics on namespaces
+```bash
+# With osm
+osm metrics enable --namespace test
+osm metrics enable --namespace "test1, test2"
+
+```
+3.	If you are using Azure Monitor Container Insights follow steps below, if not on-board [here.](https://docs.microsoft.com/azure/azure-monitor/containers/container-insights-overview)
+     * Download the configmap from [here](https://github.com/microsoft/Docker-Provider/blob/ci_prod/kubernetes/container-azm-ms-osmconfig.yaml)
+     * Add the namespaces you want to monitor in configmap `monitor_namespaces = ["namespace1", "namespace2"]`
+     * Run the following kubectl command: kubectl apply -f<configmap_yaml_file.yaml>
+         * Example: `kubectl apply -f container-azm-ms-agentconfig.yaml`
+4. The configuration change can take upto 15 mins to finish before taking effect, and all omsagent pods in the cluster will restart. The restart is a rolling restart for all omsagent pods, not all restart at the same time.
+
+
+## Validate the metrics flow
+1.	Query cluster's Log Analytics workspace InsightsMetrics table to see metrics are flowing or not
+```
+InsightsMetrics
+| where Name contains "envoy"
+| summarize count() by Name
+```
+
+## How to consume OSM monitoring dashboard?
+1.	Access your AKS cluster & Container Insights through this [link.](https://aka.ms/azmon/osmux)
+2.	Go to reports tab and access Open Service Mesh (OSM) workbook.
+3.	Select the time-range & namespace to scope your services. By default, we only show services deployed by customers and we exclude internal service communication. In case you want to view that you select Show All in the filter. Please note OSM is managed service mesh, we show all internal connections for transparency. 
+
+![alt text](https://github.com/microsoft/Docker-Provider/blob/saarorOSMdoc/Documentation/OSMPrivatePreview/Image1.jpg)
+### Requests Tab
+1.	This tab provides you the summary of all the http requests sent via service to service in OSM.
+2.	You can view all the services and all the services it is communicating to by selecting the service in grid.
+3.	You can view total requests, request error rate & P90 latency.
+4.	You can drill-down to destination and view trends for HTTP error/success code, success rate, Pods resource utilization, latencies at different percentiles.
+
+### Connections Tab
+1.	This tab provides you a summary of all the connections between your services in Open Service Mesh. 
+2.	Outbound connections: Total number of connections between Source and destination services.
+3.	Outbound active connections: Last count of active connections between source and destination in selected time range.
+4.	Outbound failed connections: Total number of failed connections between source and destination service
+
+### Troubleshooting guidance when Outbound active connections is 0 or failed connection count is >10k.
+1. Please check your connection policy in OSM configuration.
+2. If connection policy is fine, please refer the OSM documentation. https://aka.ms/osm/tsg
+3. From this view as well, you can drill-down to destination and view trends for HTTP error/success code, success rate, Pods resource utilization, latencies at different percentiles.
+
+
+### Known Issues
+1.	The workbook has scale limits of 50 pods per namespace. If you have more than 50 pods in mesh you can have workbook loading issues.
+2.	When source or destination is osmcontroller we show no latency & for internal services we show no resource utilization. 
+3.  When both prometheus scraping using pod annotations and OSM monitoring are enabled on the same set of namespaces, the default set of metrics (envoy_cluster_upstream_cx_total, envoy_cluster_upstream_cx_connect_fail, envoy_cluster_upstream_rq, envoy_cluster_upstream_rq_xx, envoy_cluster_upstream_rq_total, envoy_cluster_upstream_rq_time_bucket, envoy_cluster_upstream_cx_rx_bytes_total, envoy_cluster_upstream_cx_tx_bytes_total, envoy_cluster_upstream_cx_active) will be collected twice. You can follow [this](https://docs.microsoft.com/en-us/azure/azure-monitor/containers/container-insights-prometheus-integration#prometheus-scraping-settings) documentation to exclude these namespaces from pod annotation scraping using the setting monitor_kubernetes_pods_namespaces to work around this issue.
+
+This is private preview, the goal for us is to get feedback. Please feel free to reach out to us at [askcoin@microsoft.com](mailto:askcoin@microsoft.com) for any feedback and questions!

ReleaseNotes.md

@@ -11,6 +11,23 @@ additional questions or comments.
 
 Note : The agent version(s) below has dates (ciprod<mmddyyyy>), which indicate the agent build dates (not release dates)
 
+### 04/22/2021 -
+##### Version microsoft/oms:ciprod04222021 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod04222021 (linux)
+##### Version microsoft/oms:win-ciprod04222021 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:win-ciprod04222021 (windows)
+##### Code change log
+- Bug fixes for metrics cpuUsagePercentage and memoryWorkingSetPercentage for windows nodes
+- Added metrics for threshold violation
+- Made Job completion metric configurable
+- Udated default buffer sizes in fluent-bit
+- Updated recommended alerts
+- Fixed bug where logs written before agent starts up were not collected
+- Fixed bug which kept agent logs from being rotated
+- Bug fix for Windows Containerd container log collection
+- Bug fixes
+- Doc updates
+- Minor telemetry changes
+
+
 ### 03/26/2021 -
 ##### Version microsoft/oms:ciprod03262021 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod03262021 (linux)
 ##### Version microsoft/oms:win-ciprod03262021 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:win-ciprod03262021 (windows)

ReleaseProcess.md

@@ -35,20 +35,21 @@ Image automatically synched to MCR CN from Public cloud MCR.
 
 - Refer to internal docs for the release process and instructions.
 
-## ARO v3
-
-This needs to be co-ordinated with Red hat  and ARO-RP team for the release and Red hat team will pick up the changes for the release.
-
 ## AKS-Engine
 
 Make PR against [AKS-Engine](https://github.com/Azure/aks-engine). Refer PR https://github.com/Azure/aks-engine/pull/2318
 
-## ARO v4, Azure Arc K8s and OpenShift v4 clusters
-
-Make sure azuremonitor-containers chart yamls updates with all changes going with the release and also make sure to bump the chart version, imagetag and docker provider version etc. Similar to agent container image, build pipeline automatically push the chart to container insights prod acr for canary and prod repos accordingly.
-Both the agent and helm chart will be replicated to `mcr.microsoft.com`.
+## Arc for Kubernetes 
 
-The way, customers will be onboard the monitoring to these clusters using onboarding scripts under `onboarding\managed` directory so please bump chart version for prod release. Once we move to Arc K8s Monitoring extension Public preview, these will be taken care so at that point of time no manual changes like this required.
+Ev2 pipeline used to deploy the chart of the Arc K8s Container Insights Extension as per Safe Deployment Process. 
+Here is the high level process
+```
+ 1. Specify chart version of the release candidate and trigger [container-insights-arc-k8s-extension-ci_prod-release](https://github-private.visualstudio.com/microsoft/_release?_a=releases&view=all)
+ 2. Get the approval from one of team member for the release
+ 3. Once the approved, release should be triggered automatically 
+ 4. use `cimon-arck8s-eastus2euap` for validating latest release in canary region
+ 5. TBD - Notify vendor team for the validation on all Arc K8s supported platforms
+```
 
 ## Microsoft Charts Repo release for On-prem K8s
 

build/linux/installer/conf/td-agent-bit-prom-side-car.conf

@@ -11,14 +11,27 @@
     Parsers_File  /etc/opt/microsoft/docker-cimprov/azm-containers-parser.conf
     Log_File      /var/opt/microsoft/docker-cimprov/log/fluent-bit.log
 
+[INPUT]
+    Name tail
+    Tag oms.container.log.flbplugin.terminationlog.*
+    Path /dev/write-to-traces
+    Read_from_Head true
+    DB /var/opt/microsoft/docker-cimprov/state/terminationlog-ai.db
+    DB.Sync Off
+    Parser docker
+    Mem_Buf_Limit 1m
+    Path_Key filepath
+    Skip_Long_Lines On
+    Ignore_Older 2m
+
 [INPUT]
     Name        tcp
     Tag         oms.container.perf.telegraf.*
     Listen      0.0.0.0
     Port        25229
-    Chunk_Size  1m
-    Buffer_Size 1m
-    Mem_Buf_Limit 20m
+    Chunk_Size  10m
+    Buffer_Size 10m
+    Mem_Buf_Limit 200m
 
 [OUTPUT]
     Name                            oms

build/linux/installer/conf/td-agent-bit-rs.conf

@@ -10,6 +10,19 @@
     Parsers_File  /etc/opt/microsoft/docker-cimprov/azm-containers-parser.conf
     Log_File      /var/opt/microsoft/docker-cimprov/log/fluent-bit.log
 
+[INPUT]
+    Name tail
+    Tag oms.container.log.flbplugin.terminationlog.*
+    Path /dev/write-to-traces
+    Read_from_Head true
+    DB /var/opt/microsoft/docker-cimprov/state/terminationlog-ai.db
+    DB.Sync Off
+    Parser docker
+    Mem_Buf_Limit 1m
+    Path_Key filepath
+    Skip_Long_Lines On
+    Ignore_Older 2m
+
 [INPUT]
     Name        tcp
     Tag         oms.container.perf.telegraf.*

build/linux/installer/conf/td-agent-bit.conf

@@ -15,6 +15,7 @@
     Name tail
     Tag oms.container.log.la.*
     Path ${AZMON_LOG_TAIL_PATH}
+    Read_from_Head true
     DB /var/log/omsagent-fblogs.db
     DB.Sync Off
     Parser docker
@@ -32,6 +33,7 @@
     Name tail
     Tag oms.container.log.flbplugin.*
     Path /var/log/containers/omsagent*.log
+    Read_from_Head true
     DB /var/opt/microsoft/docker-cimprov/state/omsagent-ai.db
     DB.Sync Off
     Parser docker
@@ -44,6 +46,7 @@
     Name tail
     Tag oms.container.log.flbplugin.mdsd.*
     Path /var/opt/microsoft/linuxmonagent/log/mdsd.err
+    Read_from_Head true
     DB /var/opt/microsoft/docker-cimprov/state/mdsd-ai.db
     DB.Sync Off
     Parser docker
@@ -52,6 +55,19 @@
     Skip_Long_Lines On
     Ignore_Older 2m
 
+[INPUT]
+    Name tail
+    Tag oms.container.log.flbplugin.terminationlog.*
+    Path /dev/write-to-traces
+    Read_from_Head true
+    DB /var/opt/microsoft/docker-cimprov/state/terminationlog-ai.db
+    DB.Sync Off
+    Parser docker
+    Mem_Buf_Limit 1m
+    Path_Key filepath
+    Skip_Long_Lines On
+    Ignore_Older 2m
+
 [INPUT]
     Name        tcp
     Tag         oms.container.perf.telegraf.*

build/linux/installer/scripts/livenessprobe.sh

@@ -26,6 +26,15 @@ then
  exit 1
 fi
 
+#test to exit non zero value if telegraf is not running
+(ps -ef | grep telegraf | grep -v "grep")
+if [ $? -ne 0 ]
+then
+ echo "Telegraf is not running" > /dev/termination-log
+ echo "Telegraf is not running (controller: ${CONTROLLER_TYPE}, container type: ${CONTAINER_TYPE})" > /dev/write-to-traces  # this file is tailed and sent to traces
+ exit 1
+fi
+
 if [ -s "inotifyoutput.txt" ]
 then
   # inotifyoutput file has data(config map was applied)