Gangams/e2e test framework

.pipelines/update-place-holdres-in-e2e-tests.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+echo "start: update placeholders of e2e-tests.yaml ..."
+
+for ARGUMENT in "$@"
+do
+   KEY=$(echo $ARGUMENT | cut -f1 -d=)
+   VALUE=$(echo $ARGUMENT | cut -f2 -d=)
+
+   case "$KEY" in           
+           TENANT_ID) TENANT_ID=$VALUE ;;           
+           *)
+    esac
+done
+
+echo "start: read appid and appsecret"
+# used the same SP which used for acr
+CLIENT_ID=$(cat ~/acrappid)
+CLIENT_SECRET=$(cat ~/acrappsecret)
+echo "end: read appid and appsecret"
+
+echo "Service Principal CLIENT_ID:$CLIENT_ID"
+echo "replace CLIENT_ID value"
+sed -i "s=SP_CLIENT_ID_VALUE=$CLIENT_ID=g" e2e-tests.yaml
+
+# only uncomment for debug purpose
+# echo "Service Principal CLIENT_SECRET:$CLIENT_SECRET"
+echo "replace CLIENT_SECRET value"
+sed -i "s=SP_CLIENT_SECRET_VALUE=$CLIENT_SECRET=g" e2e-tests.yaml
+
+echo "Service Principal TENANT_ID:$TENANT_ID"
+echo "replace TENANT_ID value"
+sed -i "s=SP_TENANT_ID_VALUE=$TENANT_ID=g" e2e-tests.yaml
+
+echo "end: update placeholders of e2e-tests.yaml."

.pipelines/validate-e2e-tests-results.sh

@@ -0,0 +1,71 @@
+#!/bin/bash
+echo "start: validating results of e2e-tests  ..."
+DEFAULT_SONOBUOY_VERSION="0.20.0"
+DEFAULT_TIME_OUT_IN_MINS=60
+for ARGUMENT in "$@"
+do
+   KEY=$(echo $ARGUMENT | cut -f1 -d=)
+   VALUE=$(echo $ARGUMENT | cut -f2 -d=)
+
+   case "$KEY" in
+           SONOBUOY_VERSION) SONOBUOY_VERSION=$VALUE ;;
+           *)
+    esac
+done
+
+if [ -z $SONOBUOY_VERSION ]; then
+   SONOBUOY_VERSION=$DEFAULT_SONOBUOY_VERSION
+fi
+
+echo "sonobuoy version: ${SONOBUOY_VERSION}"
+
+echo "start: downloading sonobuoy"
+curl -LO https://github.com/vmware-tanzu/sonobuoy/releases/download/v${SONOBUOY_VERSION}/sonobuoy_${SONOBUOY_VERSION}_linux_amd64.tar.gz
+echo "end: downloading sonobuoy"
+
+echo "start: extract sonobuoy tar file"
+mkdir -p sonobuoy-install/
+tar -zxf sonobuoy_${SONOBUOY_VERSION}_*.tar.gz -C sonobuoy-install/
+echo "end: extract sonobuoy tar file"
+
+echo "start: move sonobuoy binaries to /usr/local/bin/"
+mv -f sonobuoy-install/sonobuoy /usr/local/bin/
+echo "end: move sonobuoy binaries to /usr/local/bin/"
+
+rm -rf sonobuoy_${SONOBUOY_VERSION}_*.tar.gz sonobuoy-install/
+
+results=$(sonobuoy retrieve)
+mins=0
+IsSucceeded=true
+while [ $mins -le $DEFAULT_TIME_OUT_IN_MINS ]
+do
+  # check the status 
+  echo "checking test status"
+  status=$(sonobuoy status)
+  status=$(echo $status | sed 's/`//g')
+  if [[ $status == *"completed"* ]]; then
+      echo "test run completed"
+      mins=$DEFAULT_TIME_OUT_IN_MINS
+       if [[ $status == *"failed"* ]]; then
+          IsSucceeded=false          
+       fi
+  else
+    echo "sleep for 1m to check the status again"
+    sleep 1m
+  fi
+  mins=$(( $mins + 1 ))
+done
+echo "status:${IsSucceeded}"
+
+results=$(sonobuoy retrieve)
+sonobuoy results $results
+
+if $IsSucceeded == true; then 
+   echo "all test passed"
+   exit 0
+else
+   echo "tests are failed. please review the results by downloading tar file via sonobuoy retrieve command"
+   exit 1
+fi 
+
+echo "end: validating results of e2e-tests  ..."

README.md

@@ -91,6 +91,7 @@ The general directory structure is:
 │   │   |   ...                               - plugins in, out and filters code in ruby
 │   ├── toml-parser/                          - code for parsing of toml configuration files
 ├── test/                                     - source code for tests
+│   ├── e2e/                                  - e2e tests to validate agent and e2e workflow(s)
 │   ├── unit-tests/                           - unit tests code
 │   ├── scenario/                             - scenario tests code
 ├── !_README.md                               - this file
@@ -271,6 +272,36 @@ For DEV and PROD branches, automatically deployed latest yaml with latest agent
 
 # E2E Tests
 
+## For executing tests
+
+1. Deploy the omsagent.yaml with your agent image. In the yaml, make sure `ISTEST` environment variable set to `true` if its not set already
+2. Update the Service Principal CLIENT_ID, CLIENT_SECRET and TENANT_ID placeholder values and apply e2e-tests.yaml to execute the tests 
+    > Note: Service Principal requires reader role on log analytics workspace and cluster resource to query LA and metrics
+   ```
+   cd ~/Docker-Provider/test/e2e # based on your repo path    
+   kubectl apply -f e2e-tests.yaml # this will trigger job to run the tests in sonobuoy namespace 
+   kubectl get po -n sonobuoy # to check the pods and jobs associated to tests   
+   ``` 
+3. Download (sonobuoy)[https://github.com/vmware-tanzu/sonobuoy/releases] on your dev box to view the results of the tests
+   ```
+   results=$(sonobuoy retrieve) # downloads tar file which has logs and test results
+   sonobuoy results $results # get the summary of the results
+   tar -xzvf <downloaded-tar-file> # extract downloaded tar file and look for pod logs, results and other k8s resources if there are any failures
+   ```
+
+## For adding new tests
+
+1. Add the test python file with your test code under `tests` directory
+2. Build the docker image, recommended to use ACR & MCR 
+  ```
+   cd ~/Docker-Provider/test/e2e/src # based on your repo path 
+   docker login <acr> -u <user> -p <pwd> # login to acr
+   docker build -f ./core/Dockerfile -t <repo>/<imagename>:<imagetag> .
+   docker push <repo>/<imagename>:<imagetag>
+  ```
+3. update existing agentest image tag in e2e-tests.yaml with newly built image tag with MCR repo
+
+# Scenario Tests
 Clusters are used in release pipeline already has the yamls under test\scenario deployed. Make sure to validate these scenarios.
 If you have new interesting scenarios, please add/update them.
 

charts/azuremonitor-containers/templates/omsagent-daemonset.yaml

@@ -89,6 +89,8 @@ spec:
        - name: FBIT_TAIL_BUFFER_MAX_SIZE
          value: {{ .Values.omsagent.logsettings.tailbufmaxsizemegabytes | quote }}
        {{- end }}
+       - name: ISTEST
+         value: {{ .Values.omsagent.ISTEST | quote }}
        securityContext:
          privileged: true
        ports:

charts/azuremonitor-containers/templates/omsagent-deployment.yaml

@@ -72,7 +72,9 @@ spec:
          value: {{ .Values.Azure.Extension.Name | quote }} 
        {{- end }}                       
        - name: USER_ASSIGNED_IDENTITY_CLIENT_ID
-         value: ""   
+         value: ""       
+       - name: ISTEST
+         value: {{ .Values.omsagent.ISTEST | quote }}       
        securityContext:
          privileged: true
        ports:

charts/azuremonitor-containers/values.yaml

@@ -33,6 +33,10 @@ omsagent:
   # chance to build pod for the node and give it to the scheduler)
   # Should be some number greater than default (0)
   priority: 10
+
+  # This used for running agent pods in test mode.
+  # if set to true additional agent workflow logs will be emitted which are used for e2e and arc k8s conformance testing
+  ISTEST: false
 
   ## To get your workspace id and key do the following
   ## You can create a Azure Loganalytics workspace from portal.azure.com and get its ID & PRIMARY KEY from 'Advanced Settings' tab in the Ux.

kubernetes/omsagent.yaml

@@ -383,6 +383,9 @@ spec:
               value: "VALUE_AKS_RESOURCE_ID_VALUE"
             - name: AKS_REGION
               value: "VALUE_AKS_RESOURCE_REGION_VALUE"
+            # this used for e2e test and setting this just emits some additional log statements which used for the e2e tests
+            - name: ISTEST  
+              value: "true"
             #Uncomment below two lines for ACS clusters and set the cluster names manually. Also comment out the above two lines for ACS clusters
             #- name: ACS_RESOURCE_NAME
             #  value: "my_acs_cluster_name"
@@ -541,6 +544,9 @@ spec:
               value: "VALUE_AKS_RESOURCE_ID_VALUE"
             - name: AKS_REGION
               value: "VALUE_AKS_RESOURCE_REGION_VALUE"
+            # this used for e2e test and setting this just emits some additional log statements which used for the e2e tests
+            - name: ISTEST  
+              value: "true"  
             # Uncomment below two lines for ACS clusters and set the cluster names manually. Also comment out the above two lines for ACS clusters
             #- name: ACS_RESOURCE_NAME
             #  value: "my_acs_cluster_name"

source/plugins/ruby/in_kube_events.rb

@@ -129,6 +129,7 @@ def enumerate
     def parse_and_emit_records(events, eventQueryState, newEventQueryState, batchTime = Time.utc.iso8601)
       currentTime = Time.now
       emitTime = currentTime.to_f
+      @@istestvar = ENV["ISTEST"]
       begin
         eventStream = MultiEventStream.new
         events["items"].each do |items|
@@ -171,6 +172,9 @@ def parse_and_emit_records(events, eventQueryState, newEventQueryState, batchTim
           @eventsCount += 1
         end
         router.emit_stream(@tag, eventStream) if eventStream
+        if (!@@istestvar.nil? && !@@istestvar.empty? && @@istestvar.casecmp("true") == 0)
+          $log.info("kubeEventsInventoryEmitStreamSuccess @ #{Time.now.utc.iso8601}")
+        end
       rescue => errorStr
         $log.debug_backtrace(errorStr.backtrace)
         ApplicationInsightsUtility.sendExceptionTelemetry(errorStr)

source/plugins/ruby/in_kube_nodes.rb

@@ -188,6 +188,9 @@ def parse_and_emit_records(nodeInventory, batchTime = Time.utc.iso8601)
             $log.info("in_kube_node::parse_and_emit_records: number of container node inventory records emitted #{@NODES_EMIT_STREAM_BATCH_SIZE} @ #{Time.now.utc.iso8601}")
             router.emit_stream(@@ContainerNodeInventoryTag, containerNodeInventoryEventStream) if containerNodeInventoryEventStream
             containerNodeInventoryEventStream = MultiEventStream.new
+            if (!@@istestvar.nil? && !@@istestvar.empty? && @@istestvar.casecmp("true") == 0)
+              $log.info("containerNodeInventoryEmitStreamSuccess @ #{Time.now.utc.iso8601}")
+            end
           end
 
           # node metrics records
@@ -217,6 +220,9 @@ def parse_and_emit_records(nodeInventory, batchTime = Time.utc.iso8601)
             $log.info("in_kube_nodes::parse_and_emit_records: number of node perf metric records emitted #{@NODES_EMIT_STREAM_BATCH_SIZE} @ #{Time.now.utc.iso8601}")
             router.emit_stream(@@kubeperfTag, kubePerfEventStream) if kubePerfEventStream
             kubePerfEventStream = MultiEventStream.new
+            if (!@@istestvar.nil? && !@@istestvar.empty? && @@istestvar.casecmp("true") == 0)
+              $log.info("kubeNodePerfEmitStreamSuccess @ #{Time.now.utc.iso8601}")
+            end
           end
 
           # node GPU metrics record
@@ -249,6 +255,9 @@ def parse_and_emit_records(nodeInventory, batchTime = Time.utc.iso8601)
             $log.info("in_kube_nodes::parse_and_emit_records: number of GPU node perf metric records emitted #{@NODES_EMIT_STREAM_BATCH_SIZE} @ #{Time.now.utc.iso8601}")
             router.emit_stream(Constants::INSIGHTSMETRICS_FLUENT_TAG, insightsMetricsEventStream) if insightsMetricsEventStream
             insightsMetricsEventStream = MultiEventStream.new
+            if (!@@istestvar.nil? && !@@istestvar.empty? && @@istestvar.casecmp("true") == 0)
+              $log.info("kubeNodeInsightsMetricsEmitStreamSuccess @ #{Time.now.utc.iso8601}")
+            end
           end
           # Adding telemetry to send node telemetry every 10 minutes
           timeDifference = (DateTime.now.to_time.to_i - @@nodeTelemetryTimeTracker).abs
@@ -300,23 +309,35 @@ def parse_and_emit_records(nodeInventory, batchTime = Time.utc.iso8601)
           router.emit_stream(@tag, eventStream) if eventStream
           $log.info("in_kube_node::parse_and_emit_records: number of mdm node inventory records emitted #{eventStream.count} @ #{Time.now.utc.iso8601}")
           router.emit_stream(@@MDMKubeNodeInventoryTag, eventStream) if eventStream
+          if (!@@istestvar.nil? && !@@istestvar.empty? && @@istestvar.casecmp("true") == 0)
+            $log.info("kubeNodeInventoryEmitStreamSuccess @ #{Time.now.utc.iso8601}")
+          end
           eventStream = nil
         end
         if containerNodeInventoryEventStream.count > 0
           $log.info("in_kube_node::parse_and_emit_records: number of container node inventory records emitted #{containerNodeInventoryEventStream.count} @ #{Time.now.utc.iso8601}")
           router.emit_stream(@@ContainerNodeInventoryTag, containerNodeInventoryEventStream) if containerNodeInventoryEventStream
           containerNodeInventoryEventStream = nil
+          if (!@@istestvar.nil? && !@@istestvar.empty? && @@istestvar.casecmp("true") == 0)
+            $log.info("containerNodeInventoryEmitStreamSuccess @ #{Time.now.utc.iso8601}")
+          end
         end
 
         if kubePerfEventStream.count > 0
           $log.info("in_kube_nodes::parse_and_emit_records: number of node perf metric records emitted #{kubePerfEventStream.count} @ #{Time.now.utc.iso8601}")
           router.emit_stream(@@kubeperfTag, kubePerfEventStream) if kubePerfEventStream
           kubePerfEventStream = nil
+          if (!@@istestvar.nil? && !@@istestvar.empty? && @@istestvar.casecmp("true") == 0)
+            $log.info("kubeNodePerfInventoryEmitStreamSuccess @ #{Time.now.utc.iso8601}")
+          end
         end
         if insightsMetricsEventStream.count > 0
           $log.info("in_kube_nodes::parse_and_emit_records: number of GPU node perf metric records emitted #{insightsMetricsEventStream.count} @ #{Time.now.utc.iso8601}")
           router.emit_stream(Constants::INSIGHTSMETRICS_FLUENT_TAG, insightsMetricsEventStream) if insightsMetricsEventStream
           insightsMetricsEventStream = nil
+          if (!@@istestvar.nil? && !@@istestvar.empty? && @@istestvar.casecmp("true") == 0)
+            $log.info("kubeNodeInsightsMetricsEmitStreamSuccess @ #{Time.now.utc.iso8601}")
+          end
         end
       rescue => errorStr
         $log.warn "Failed to retrieve node inventory: #{errorStr}"

source/plugins/ruby/in_kube_podinventory.rb

@@ -265,6 +265,9 @@ def parse_and_emit_records(podInventory, serviceRecords, continuationToken, batc
           if @PODS_EMIT_STREAM_BATCH_SIZE > 0 && kubePerfEventStream.count >= @PODS_EMIT_STREAM_BATCH_SIZE
             $log.info("in_kube_podinventory::parse_and_emit_records: number of container perf records emitted #{@PODS_EMIT_STREAM_BATCH_SIZE} @ #{Time.now.utc.iso8601}")
             router.emit_stream(@@kubeperfTag, kubePerfEventStream) if kubePerfEventStream
+            if (!@@istestvar.nil? && !@@istestvar.empty? && @@istestvar.casecmp("true") == 0)
+              $log.info("kubeContainerPerfEventEmitStreamSuccess @ #{Time.now.utc.iso8601}")
+            end
             kubePerfEventStream = MultiEventStream.new
           end
 
@@ -306,6 +309,9 @@ def parse_and_emit_records(podInventory, serviceRecords, continuationToken, batc
           $log.info("in_kube_podinventory::parse_and_emit_records: number of perf records emitted #{kubePerfEventStream.count} @ #{Time.now.utc.iso8601}")
           router.emit_stream(@@kubeperfTag, kubePerfEventStream) if kubePerfEventStream
           kubePerfEventStream = nil
+          if (!@@istestvar.nil? && !@@istestvar.empty? && @@istestvar.casecmp("true") == 0)
+            $log.info("kubeContainerPerfEventEmitStreamSuccess @ #{Time.now.utc.iso8601}")
+          end
         end
 
         if insightsMetricsEventStream.count > 0
@@ -345,13 +351,19 @@ def parse_and_emit_records(podInventory, serviceRecords, continuationToken, batc
                 $log.info("in_kube_podinventory::parse_and_emit_records: number of service records emitted #{@PODS_EMIT_STREAM_BATCH_SIZE} @ #{Time.now.utc.iso8601}")
                 router.emit_stream(@@kubeservicesTag, kubeServicesEventStream) if kubeServicesEventStream
                 kubeServicesEventStream = MultiEventStream.new
+                if (!@@istestvar.nil? && !@@istestvar.empty? && @@istestvar.casecmp("true") == 0)
+                  $log.info("kubeServicesEventEmitStreamSuccess @ #{Time.now.utc.iso8601}")
+                end
               end
             end
           end
 
           if kubeServicesEventStream.count > 0
             $log.info("in_kube_podinventory::parse_and_emit_records : number of service records emitted #{kubeServicesEventStream.count} @ #{Time.now.utc.iso8601}")
             router.emit_stream(@@kubeservicesTag, kubeServicesEventStream) if kubeServicesEventStream
+            if (!@@istestvar.nil? && !@@istestvar.empty? && @@istestvar.casecmp("true") == 0)
+              $log.info("kubeServicesEventEmitStreamSuccess @ #{Time.now.utc.iso8601}")
+            end
           end
           kubeServicesEventStream = nil
         end

source/plugins/ruby/in_kube_pvinventory.rb

@@ -106,7 +106,7 @@ def parse_and_emit_records(pvInventory, batchTime = Time.utc.iso8601)
       currentTime = Time.now
       emitTime = currentTime.to_f
       eventStream = MultiEventStream.new
-
+      @@istestvar = ENV["ISTEST"]
       begin
         records = []
         pvInventory["items"].each do |item|
@@ -156,6 +156,9 @@ def parse_and_emit_records(pvInventory, batchTime = Time.utc.iso8601)
         end
 
         router.emit_stream(@tag, eventStream) if eventStream
+        if (!@@istestvar.nil? && !@@istestvar.empty? && @@istestvar.casecmp("true") == 0)
+          $log.info("kubePVInventoryEmitStreamSuccess @ #{Time.now.utc.iso8601}")
+        end
 
       rescue => errorStr
         $log.warn "Failed in parse_and_emit_record for in_kube_pvinventory: #{errorStr}"