Gangams/arc k8s conformance test updates

README.md

@@ -326,7 +326,7 @@ For DEV and PROD branches, automatically deployed latest yaml with latest agent
    docker build -f ./core/Dockerfile -t <repo>/<imagename>:<imagetag> .
    docker push <repo>/<imagename>:<imagetag>
   ```
-3. update existing agentest image tag in e2e-tests.yaml with newly built image tag with MCR repo
+3. update existing agentest image tag in e2e-tests.yaml & conformance.yaml with newly built image tag with MCR repo
 
 # Scenario Tests
 Clusters are used in release pipeline already has the yamls under test\scenario deployed. Make sure to validate these scenarios.

test/e2e/conformance.yaml

@@ -0,0 +1,15 @@
+sonobuoy-config:
+  driver: Job
+  plugin-name: azure-arc-ci-conformance
+  result-format: junit
+spec:
+  image: mcr.microsoft.com/azuremonitor/containerinsights/cidev:ciconftest08142021
+  imagePullPolicy: Always
+  name: plugin
+  resources: {}
+  volumes:
+  - name: results
+    emptyDir: {}
+  volumeMounts:
+  - mountPath: /tmp/results
+    name: results

test/e2e/e2e-tests.yaml

@@ -68,7 +68,7 @@ data:
       containers: []
       restartPolicy: Never
       serviceAccountName: sonobuoy-serviceaccount
-      nodeSelector:    
+      nodeSelector:
         kubernetes.io/os: linux
       tolerations:
       - effect: NoSchedule
@@ -84,24 +84,27 @@ data:
       result-format: junit
     spec:
       env:
+      # this should be false if the test environment is non ARC K8s for example AKS
+      - name: IS_NON_ARC_K8S_TEST_ENVIRONMENT
+        value: "true"
       # Update values of CLIENT_ID, CLIENT_SECRET of the service principal which has permission to query LA ad Metrics API
-      # Update value of TENANT_ID corresponding your Azure Service principal          
+      # Update value of TENANT_ID corresponding your Azure Service principal
       - name: CLIENT_ID
         value: "SP_CLIENT_ID_VALUE"
       - name: CLIENT_SECRET
         value: "CLIENT_SECRET_VALUE"
       - name: TENANT_ID
         value: "SP_TENANT_ID_VALUE"
       - name: DEFAULT_QUERY_TIME_INTERVAL_IN_MINUTES
-        value: "10"  
+        value: "10"
       - name: DEFAULT_METRICS_QUERY_TIME_INTERVAL_IN_MINUTES
-        value: "10" 
+        value: "10"
       - name: AGENT_POD_EXPECTED_RESTART_COUNT
-        value: "0"   
+        value: "0"
       - name: AZURE_CLOUD
-        value: "AZURE_PUBLIC_CLOUD"  
-      # image tag should be updated if new tests being added after this image   
-      image: mcr.microsoft.com/azuremonitor/containerinsights/cidev:ciagenttest02152021
+        value: "AZURE_PUBLIC_CLOUD"
+      # image tag should be updated if new tests being added after this image
+      image: mcr.microsoft.com/azuremonitor/containerinsights/cidev:ciconftest08142021
       imagePullPolicy: IfNotPresent
       name: plugin
       resources: {}
@@ -144,7 +147,7 @@ spec:
       name: output-volume
   restartPolicy: Never
   serviceAccountName: sonobuoy-serviceaccount
-  nodeSelector:    
+  nodeSelector:
     kubernetes.io/os: linux
   tolerations:
   - key: "kubernetes.io/e2e-evict-taint-key"

test/e2e/src/common/constants.py

@@ -40,14 +40,18 @@
 
 TIMEOUT = 300
 
+# WAIT TIME BEFORE READING THE AGENT LOGS
+AGENT_WAIT_TIME_SECS = "180"
 # Azure Monitor for Container Extension related
 AGENT_RESOURCES_NAMESPACE = 'kube-system'
 AGENT_DEPLOYMENT_NAME = 'omsagent-rs'
 AGENT_DAEMONSET_NAME = 'omsagent'
 AGENT_WIN_DAEMONSET_NAME = 'omsagent-win'
 
 AGENT_DEPLOYMENT_PODS_LABEL_SELECTOR = 'rsName=omsagent-rs'
-AGENT_DAEMON_SET_PODS_LABEL_SELECTOR = 'component=oms-agent'
+AGENT_DAEMON_SET_PODS_LABEL_SELECTOR = 'dsName=omsagent-ds'
+AGENT_DAEMON_SET_PODS_LABEL_SELECTOR_NON_ARC = 'component=oms-agent'
+AGENT_FLUENTD_LOG_PATH = '/var/opt/microsoft/docker-cimprov/log/fluentd.log'
 AGENT_OMSAGENT_LOG_PATH = '/var/opt/microsoft/omsagent/log/omsagent.log'
 AGENT_REPLICASET_WORKFLOWS = ["kubePodInventoryEmitStreamSuccess", "kubeNodeInventoryEmitStreamSuccess"]
 

test/e2e/src/core/Dockerfile

@@ -1,11 +1,26 @@
 FROM python:3.6
 
-RUN pip install --trusted-host pypi.org --trusted-host files.pythonhosted.org pytest pytest-xdist filelock requests kubernetes adal msrestazure 
+RUN pip install --trusted-host pypi.org --trusted-host files.pythonhosted.org pytest pytest-xdist filelock requests kubernetes adal msrestazure
 
 RUN curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash \
     && helm version
 
+RUN apt-get update && apt-get -y upgrade && \
+    apt-get -f -y install curl apt-transport-https lsb-release gnupg python3-pip python-pip && \
+    curl -sL https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > /etc/apt/trusted.gpg.d/microsoft.asc.gpg && \
+    CLI_REPO=$(lsb_release -cs) && \
+    echo "deb [arch=amd64] https://packages.microsoft.com/repos/azure-cli/ ${CLI_REPO} main" \
+    > /etc/apt/sources.list.d/azure-cli.list && \
+    apt-get update && \
+    apt-get install -y azure-cli && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN python3 -m pip install junit_xml
+
+COPY --from=lachlanevenson/k8s-kubectl:v1.20.5 /usr/local/bin/kubectl /usr/local/bin/kubectl
+
 COPY ./core/e2e_tests.sh /
+COPY ./core/setup_failure_handler.py /
 COPY ./core/pytest.ini /e2etests/
 COPY ./core/conftest.py /e2etests/
 COPY ./core/helper.py /e2etests/

test/e2e/src/core/conftest.py

@@ -22,42 +22,48 @@ def env_dict():
             create_results_dir('/tmp/results')
 
             # Setting some environment variables
-            env_dict['SETUP_LOG_FILE'] = '/tmp/results/setup'            
+            env_dict['SETUP_LOG_FILE'] = '/tmp/results/setup'
             env_dict['TEST_AGENT_LOG_FILE'] = '/tmp/results/containerinsights'
             env_dict['NUM_TESTS_COMPLETED'] = 0
-            
+
             print("Starting setup...")
             append_result_output("Starting setup...\n", env_dict['SETUP_LOG_FILE'])
-            
+
             # Collecting environment variables
             env_dict['TENANT_ID'] = os.getenv('TENANT_ID')
             env_dict['CLIENT_ID'] = os.getenv('CLIENT_ID')
             env_dict['CLIENT_SECRET'] = os.getenv('CLIENT_SECRET')
-
+            env_dict['IS_NON_ARC_K8S_TEST_ENVIRONMENT'] = os.getenv('IS_NON_ARC_K8S_TEST_ENVIRONMENT')
+            # released agent for Arc K8s still uses omsagent and when we rollout the agent with mdsd
+            # this shouldnt set after agent rollout with mdsd
+            env_dict['USING_OMSAGENT_BASE_AGENT'] = os.getenv('USING_OMSAGENT_BASE_AGENT')
+
+            waitTimeInterval = int(os.getenv('AGENT_WAIT_TIME_SECS')) if os.getenv('AGENT_WAIT_TIME_SECS') else constants.AGENT_WAIT_TIME_SECS
+            env_dict['AGENT_WAIT_TIME_SECS'] = waitTimeInterval
             # get default query time interval for log analytics queries
             queryTimeInterval = int(os.getenv('DEFAULT_QUERY_TIME_INTERVAL_IN_MINUTES')) if os.getenv('DEFAULT_QUERY_TIME_INTERVAL_IN_MINUTES') else constants.DEFAULT_QUERY_TIME_INTERVAL_IN_MINUTES
             # add minute suffix since this format required for LA queries
             env_dict['DEFAULT_QUERY_TIME_INTERVAL_IN_MINUTES'] = str(queryTimeInterval) + "m"
-            
+
             # get default query time interval for metrics queries
             env_dict['DEFAULT_METRICS_QUERY_TIME_INTERVAL_IN_MINUTES'] = int(os.getenv('DEFAULT_METRICS_QUERY_TIME_INTERVAL_IN_MINUTES')) if os.getenv('DEFAULT_METRICS_QUERY_TIME_INTERVAL_IN_MINUTES') else constants.DEFAULT_METRICS_QUERY_TIME_INTERVAL_IN_MINUTES
-            
-            
-            # expected agent pod restart count            
+
+
+            # expected agent pod restart count
             env_dict['AGENT_POD_EXPECTED_RESTART_COUNT'] = int(os.getenv('AGENT_POD_EXPECTED_RESTART_COUNT')) if os.getenv('AGENT_POD_EXPECTED_RESTART_COUNT') else constants.AGENT_POD_EXPECTED_RESTART_COUNT
 
             # default to azure public cloud if AZURE_CLOUD not specified
             env_dict['AZURE_ENDPOINTS'] = constants.AZURE_CLOUD_DICT.get(os.getenv('AZURE_CLOUD')) if os.getenv('AZURE_CLOUD') else constants.AZURE_PUBLIC_CLOUD_ENDPOINTS
-                                    
+
             if not env_dict.get('TENANT_ID'):
                 pytest.fail('ERROR: variable TENANT_ID is required.')
-            
+
             if not env_dict.get('CLIENT_ID'):
                 pytest.fail('ERROR: variable CLIENT_ID is required.')
-            
+
             if not env_dict.get('CLIENT_SECRET'):
                 pytest.fail('ERROR: variable CLIENT_SECRET is required.')
-                       
+
             print("Setup Complete.")
             append_result_output("Setup Complete.\n", env_dict['SETUP_LOG_FILE'])
 
@@ -66,22 +72,22 @@ def env_dict():
         else:
             with Path.open(my_file, "rb") as f:
                 env_dict = pickle.load(f)
-        
+
     yield env_dict
-    
+
     my_file = Path("env.pkl")
     with FileLock(str(my_file) + ".lock"):
         with Path.open(my_file, "rb") as f:
             env_dict = pickle.load(f)
 
         env_dict['NUM_TESTS_COMPLETED'] = 1 + env_dict.get('NUM_TESTS_COMPLETED')
-        if env_dict['NUM_TESTS_COMPLETED'] == int(os.getenv('NUM_TESTS')):           
+        if env_dict['NUM_TESTS_COMPLETED'] == int(os.getenv('NUM_TESTS')):
             # Checking if cleanup is required.
             if os.getenv('SKIP_CLEANUP'):
                 return
             print('Starting cleanup...')
             append_result_output("Starting Cleanup...\n", env_dict['SETUP_LOG_FILE'])
-           
+
             print("Cleanup Complete.")
             append_result_output("Cleanup Complete.\n", env_dict['SETUP_LOG_FILE'])
             return