David/aad stage 1 release

ReleaseNotes.md

@@ -11,6 +11,14 @@ additional questions or comments.
 
 Note : The agent version(s) below has dates (ciprod<mmddyyyy>), which indicate the agent build dates (not release dates)
 
+
+### 05/12/2021 -
+##### Version microsoft/oms:ciprod00512021 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod05122021 (linux)
+##### No Windows changes with this release, win-ciprod04222021 still current.
+##### Code change log
+- Upgrading oneagent to version 1.8 (only for Linux)
+- Enabling oneagent for container logs for East US 2
+
 ### 04/22/2021 -
 ##### Version microsoft/oms:ciprod04222021 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod04222021 (linux)
 ##### Version microsoft/oms:win-ciprod04222021 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:win-ciprod04222021 (windows)
@@ -27,7 +35,6 @@ Note : The agent version(s) below has dates (ciprod<mmddyyyy>), which indicate t
 - Doc updates
 - Minor telemetry changes
 
-
 ### 03/26/2021 -
 ##### Version microsoft/oms:ciprod03262021 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod03262021 (linux)
 ##### Version microsoft/oms:win-ciprod03262021 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:win-ciprod03262021 (windows)

build/linux/installer/scripts/livenessprobe.sh

@@ -10,7 +10,7 @@ fi
 
 #optionally test to exit non zero value if oneagent is not running
 if [ -e "/opt/AZMON_CONTAINER_LOGS_EFFECTIVE_ROUTE_V2" ]; then
-  (ps -ef | grep "mdsd -l" | grep -v "grep")
+  (ps -ef | grep "mdsd" | grep -v "grep")
   if [ $? -ne 0 ]
   then
    echo "oneagent is not running" > /dev/termination-log

build/version

@@ -3,10 +3,10 @@
 # Build Version Information
 
 CONTAINER_BUILDVERSION_MAJOR=15
-CONTAINER_BUILDVERSION_MINOR=0
+CONTAINER_BUILDVERSION_MINOR=1
 CONTAINER_BUILDVERSION_PATCH=0
 CONTAINER_BUILDVERSION_BUILDNR=0
-CONTAINER_BUILDVERSION_DATE=20210422
+CONTAINER_BUILDVERSION_DATE=20210512
 CONTAINER_BUILDVERSION_STATUS=Developer_Build
 
 #-------------------------------- End of File -----------------------------------

kubernetes/linux/Dockerfile

@@ -2,7 +2,7 @@ FROM ubuntu:18.04
 MAINTAINER OMSContainers@microsoft.com
 LABEL vendor=Microsoft\ Corp \
     com.microsoft.product="Azure Monitor for containers"
-ARG IMAGE_TAG=ciprod04222021
+ARG IMAGE_TAG=ciprod05122021
 ENV AGENT_VERSION ${IMAGE_TAG}
 ENV tmpdir /opt
 ENV APPLICATIONINSIGHTS_AUTH NzAwZGM5OGYtYTdhZC00NThkLWI5NWMtMjA3ZjM3NmM3YmRi

kubernetes/linux/envmdsd

@@ -12,3 +12,7 @@ export HOSTNAME_OVERRIDE="${NODE_NAME}"
 export MDSD_TCMALLOC_RELEASE_FREQ_SEC=1
 export MDSD_COMPRESSION_ALGORITHM=LZ4
 export SSL_CERT_DIR="/etc/ssl/certs"
+# increase the size of msgpack items mdsd will accept, otherwise they will be silently dropped. These values were arbitrairly chosen to be 10 or 100 times larger than the defaults.
+export MDSD_MSGPACK_ARRAY_SIZE_ITEMS=10000000
+export MDSD_MSGPACK_MAP_SIZE_ITEMS=10000000
+export MDSD_MSGPACK_NESTING_LEVEL=100

kubernetes/linux/main.sh

@@ -581,7 +581,7 @@ if [ ! -e "/etc/config/kube.conf" ] && [ "${CONTAINER_TYPE}" != "PrometheusSidec
             dpkg -l | grep mdsd | awk '{print $2 " " $3}'
 
             echo "starting mdsd ..."
-            mdsd -l -e ${MDSD_LOG}/mdsd.err -w ${MDSD_LOG}/mdsd.warn -o ${MDSD_LOG}/mdsd.info -q ${MDSD_LOG}/mdsd.qos &
+            mdsd -e ${MDSD_LOG}/mdsd.err -w ${MDSD_LOG}/mdsd.warn -o ${MDSD_LOG}/mdsd.info -q ${MDSD_LOG}/mdsd.qos &
 
             touch /opt/AZMON_CONTAINER_LOGS_EFFECTIVE_ROUTE_V2
       fi

kubernetes/linux/setup.sh

@@ -31,8 +31,8 @@ mv $TMPDIR/omsbundle* $TMPDIR/omsbundle
 /usr/bin/dpkg -i $TMPDIR/omsbundle/110/omsagent*.deb
 #/usr/bin/dpkg -i $TMPDIR/omsbundle/100/omsconfig*.deb
 
-#install oneagent - Official bits (10/18)
-wget https://github.com/microsoft/Docker-Provider/releases/download/10182020-oneagent/azure-mdsd_1.5.126-build.master.99_x86_64.deb
+#install oneagent - Official bits (05/2021)
+wget https://github.com/microsoft/Docker-Provider/releases/download/05112021-oneagent/azure-mdsd_1.8.0-build.master.189_x86_64.deb
 /usr/bin/dpkg -i $TMPDIR/azure-mdsd*.deb
 cp -f $TMPDIR/mdsd.xml /etc/mdsd.d
 cp -f $TMPDIR/envmdsd /etc/mdsd.d

kubernetes/omsagent.yaml

@@ -358,7 +358,7 @@ spec:
         tier: node
       annotations:
         agentVersion: "1.10.0.1"
-        dockerProviderVersion: "15.0.0-0"
+        dockerProviderVersion: "15.1.0-0"
         schema-versions: "v1"
     spec:
       serviceAccountName: omsagent
@@ -368,7 +368,7 @@ spec:
             value: "3"       
       containers:
         - name: omsagent
-          image: "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod04222021"
+          image: "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod05122021"
           imagePullPolicy: IfNotPresent
           resources:
             limits:
@@ -399,7 +399,7 @@ spec:
             - name: USER_ASSIGNED_IDENTITY_CLIENT_ID
               value: ""
             - name: AZMON_CONTAINERLOGS_ONEAGENT_REGIONS
-              value: "koreacentral,norwayeast"
+              value: "koreacentral,norwayeast,eastus2"
           securityContext:
             privileged: true
           ports:
@@ -446,7 +446,7 @@ spec:
             timeoutSeconds: 15
 #Only in sidecar scraping mode
         - name: omsagent-prometheus
-          image: "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod04222021"
+          image: "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod05122021"
           imagePullPolicy: IfNotPresent
           resources:
             limits:
@@ -583,13 +583,13 @@ spec:
         rsName: "omsagent-rs"
       annotations:
         agentVersion: "1.10.0.1"
-        dockerProviderVersion: "15.0.0-0"
+        dockerProviderVersion: "15.1.0-0"
         schema-versions: "v1"
     spec:
       serviceAccountName: omsagent
       containers:
         - name: omsagent
-          image: "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod04222021"
+          image: "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod05122021"
           imagePullPolicy: IfNotPresent
           resources:
             limits:

test/containerlog-scale-tests/400logspersec-2klogentrysize.yaml

@@ -0,0 +1,60 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: logs-400persec-2kentrysize
+spec:
+  parallelism: 1    
+  completions: 1    
+  template:         
+    metadata:
+      name: logs-400persec-2kentrysize
+    spec:
+      volumes:
+      - name: logs-400persec-2kentrysize-scripts-volume
+        configMap:
+          name: logs-400persec-test-scripts
+      containers:
+      - name: logs-400persec-2kentrysize
+        image: ubuntu
+        volumeMounts:
+          - mountPath: /logs-400persec-test-scripts
+            name: logs-400persec-2kentrysize-scripts-volume
+        env:
+          - name: HOME
+            value: /tmp
+        command:
+        - /bin/sh
+        - -c
+        - |
+          echo "scripts in /logs-400persec-test-scripts"
+          ls -lh /logs-400persec-test-scripts
+          echo "copy scripts to /tmp"
+          cp /logs-400persec-test-scripts/*.sh /tmp
+          echo "apply 'chmod +x' to /tmp/*.sh"
+          chmod +x /tmp/*.sh
+          echo "script.sh in /tmp"
+          ls -lh /tmp
+          /tmp/script.sh    
+      restartPolicy: Never
+---
+apiVersion: v1
+items:
+- apiVersion: v1
+  data:
+    script.sh: |
+      #!/bin/bash
+      logentry=''
+      for var in {1..400..1}
+      do
+        logentry="${logentry}Test-"
+      done
+      for var in {1..200000..1}
+      do
+        echo $(date "+%Y/%m/%d %H:%M:%S.%3N") ${var}: $logentry
+      done
+  kind: ConfigMap
+  metadata:
+    creationTimestamp: null
+    name:  logs-400persec-test-scripts
+kind: List
+metadata: {}

test/containerlog-scale-tests/400logspersec-5klogentrysize.yaml

@@ -0,0 +1,60 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: logs-400persec-5kentrysize
+spec:
+  parallelism: 1    
+  completions: 1    
+  template:         
+    metadata:
+      name: logs-400persec-5kentrysize
+    spec:
+      volumes:
+      - name: logs-400persec-5kentrysize-scripts-volume
+        configMap:
+          name: logs-400persec-5kentrysize-test-scripts
+      containers:
+      - name: logs-400persec-5kentrysize
+        image: ubuntu
+        volumeMounts:
+          - mountPath: /logs-400persec-5kentrysize-test-scripts
+            name: logs-400persec-5kentrysize-scripts-volume
+        env:
+          - name: HOME
+            value: /tmp
+        command:
+        - /bin/sh
+        - -c
+        - |
+          echo "scripts in /logs-400persec-5kentrysize-test-scripts"
+          ls -lh /logs-400persec-5kentrysize-test-scripts
+          echo "copy scripts to /tmp"
+          cp /logs-400persec-5kentrysize-test-scripts/*.sh /tmp
+          echo "apply 'chmod +x' to /tmp/*.sh"
+          chmod +x /tmp/*.sh
+          echo "script.sh in /tmp"
+          ls -lh /tmp
+          /tmp/script.sh
+      restartPolicy: Never
+---
+apiVersion: v1
+items:
+- apiVersion: v1
+  data:
+    script.sh: |
+      #!/bin/bash
+      logentry=''
+      for var in {1..1024..1}
+      do
+        logentry="${logentry}Test-"
+      done
+      for var in {1..200000..1}
+      do
+        echo $(date "+%Y/%m/%d %H:%M:%S.%3N") ${var}: $logentry
+      done
+  kind: ConfigMap
+  metadata:
+    creationTimestamp: null
+    name:  logs-400persec-5kentrysize-test-scripts
+kind: List
+metadata: {}

test/containerlog-scale-tests/ci-log-scale-4kpersec-5klogline.yaml

@@ -0,0 +1,60 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: ci-log-scale
+spec:
+  parallelism: 1    
+  completions: 1    
+  template:         
+    metadata:
+      name: ci-log-scale
+    spec:
+      volumes:
+      - name: ci-log-scale-scripts-volume
+        configMap:
+          name: test-scripts
+      containers:
+      - name: ci-log-scale
+        image: ubuntu
+        volumeMounts:
+          - mountPath: /test-scripts
+            name: ci-log-scale-scripts-volume
+        env:
+          - name: HOME
+            value: /tmp
+        command:
+        - /bin/sh
+        - -c
+        - |
+          echo "scripts in /test-scripts"
+          ls -lh /test-scripts
+          echo "copy scripts to /tmp"
+          cp /test-scripts/*.sh /tmp
+          echo "apply 'chmod +x' to /tmp/*.sh"
+          chmod +x /tmp/*.sh
+          echo "script.sh in /tmp"
+          ls -lh /tmp
+          /tmp/script.sh 
+      restartPolicy: Never
+---
+apiVersion: v1
+items:
+- apiVersion: v1
+  data:
+    script.sh: |
+      #!/bin/bash
+      logentry=''
+      for var in {1..1024..1}
+      do
+        logentry="${logentry}Test-"
+      done
+      for var in {1..200000..1}
+      do
+        echo $(date "+%Y/%m/%d %H:%M:%S.%3N") ${var}: $logentry
+      done
+  kind: ConfigMap
+  metadata:
+    creationTimestamp: null
+    name:  test-scripts
+kind: List
+metadata: {}