tests: monitored resources tests

deployable/python/snippets.py

@@ -15,10 +15,10 @@
 import logging
 import os
 
-from google.cloud.logging_v2._helpers import retrieve_metadata_server
 
 try:
     import google.cloud.logging
+    from google.cloud.logging_v2._helpers import retrieve_metadata_server
 except ImportError:
     # import at runtime for GAE environments
     import pip
@@ -28,15 +28,16 @@
     pip.main(["install", "-e", "./python-logging"])
     importlib.reload(site)
     import google.cloud.logging
+    from google.cloud.logging_v2._helpers import retrieve_metadata_server
 
 
-def simple_log(log_name=None, log_text="simple_log", **kwargs):
-    client = google.cloud.logging.Client()
-    logger = client.logger(log_name)
-    logger.log_text(log_text)
+# def simple_log(log_name=None, log_text="simple_log", **kwargs):
+#     client = google.cloud.logging.Client()
+#     logger = client.logger(log_name)
+#     logger.log_text(log_text)
 
 
-def pylogging(log_text="pylogging", severity="warning", **kwargs):
+def simplelog(log_text="pylogging", severity="warning", **kwargs):
     # allowed severity: debug, info, warning, error, critical
     if severity == "debug":
         logging.debug(log_text)
@@ -49,18 +50,21 @@ def pylogging(log_text="pylogging", severity="warning", **kwargs):
     else:
         logging.critical(log_text)
 
+
 def print_handlers(**kwargs):
     root_logger = logging.getLogger()
-    handlers_str = ', '.join([type(h).__name__ for h in root_logger.handlers])
+    handlers_str = ", ".join([type(h).__name__ for h in root_logger.handlers])
     logging.info(handlers_str)
 
+
 def remove_stream_handlers(**kwargs):
     logger = logging.getLogger()
     for handler in logger.handlers:
         if isinstance(handler, logging.StreamHandler):
             logging.error(handler)
             logger.removeHandler(handler)
 
+
 def print_env_vars(env_var=None, **kwargs):
     if env_var:
         value = os.environ.get(env_var, None)
@@ -71,6 +75,7 @@ def print_env_vars(env_var=None, **kwargs):
     else:
         logging.error(os.environ)
 
+
 def get_metadata_server(metadata_key=None, **kwargs):
     if metadata_key is None:
         metadata_key = ""

envctl/env_scripts/python/cloudrun.sh

@@ -27,16 +27,16 @@ add_service_accounts() {
   local PROJECT_NUMBER=$(gcloud projects list --filter=$PROJECT_ID --format="value(PROJECT_NUMBER)")
   gcloud projects add-iam-policy-binding $PROJECT_ID \
     --member=serviceAccount:service-$PROJECT_NUMBER@gcp-sa-pubsub.iam.gserviceaccount.com \
-     --role=roles/iam.serviceAccountTokenCreator 2> /dev/null
+     --role=roles/iam.serviceAccountTokenCreator
   gcloud iam service-accounts create $SA_NAME \
-     --display-name "Pub/Sub Invoker" 2> /dev/null
+     --display-name "Pub/Sub Invoker"
   gcloud run services add-iam-policy-binding  $SERVICE_NAME \
      --member=serviceAccount:$SA_NAME@$PROJECT_ID.iam.gserviceaccount.com \
-     --role=roles/run.invoker 2> /dev/null
+     --role=roles/run.invoker
   RUN_URL=$(gcloud run services list --filter=$SERVICE_NAME --format="value(URL)")
   gcloud pubsub subscriptions create $SERVICE_NAME-subscriber --topic $SERVICE_NAME \
     --push-endpoint=$RUN_URL \
-    --push-auth-service-account=$SA_NAME@$PROJECT_ID.iam.gserviceaccount.com 2> /dev/null
+    --push-auth-service-account=$SA_NAME@$PROJECT_ID.iam.gserviceaccount.com
   set -e
 }
 
@@ -46,7 +46,7 @@ destroy() {
   gcloud pubsub topics delete $SERVICE_NAME -q 2> /dev/null
   gcloud pubsub subscriptions delete $SERVICE_NAME-subscriber -q 2> /dev/null
   # delete service account
-  gcloud iam service-accounts delete $SA_NAME -q 2> /dev/null
+  gcloud iam service-accounts delete $SA_NAME@$PROJECT_ID.iam.gserviceaccount.com -q 2> /dev/null
   # delete container images
   export GCR_PATH=gcr.io/$PROJECT_ID/logging:$SERVICE_NAME
   gcloud container images delete $GCR_PATH -q --force-delete-tags 2> /dev/null
@@ -73,7 +73,6 @@ deploy() {
   gcloud config set run/platform managed
   gcloud config set run/region us-west1
   gcloud run deploy  \
-    --allow-unauthenticated \
     --image $GCR_PATH \
     --update-env-vars ENABLE_FLASK=true \
     $SERVICE_NAME

tests/common/appengine.py

@@ -0,0 +1,34 @@
+# Copyright 2016 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import unittest
+import inspect
+
+import google.cloud.logging
+
+from ..common.common import Common
+
+
+class CommonAppEngine:
+    def test_monitored_resource(self):
+        log_text = f"{inspect.currentframe().f_code.co_name}"
+        log_list = self.trigger_and_retrieve(log_text)
+        found_resource = log_list[-1].resource
+
+        self.assertEqual(found_resource.type, "gae_app")
+        self.assertTrue(found_resource.labels["project_id"])
+        self.assertTrue(found_resource.labels["module_id"])
+        self.assertTrue(found_resource.labels["version_id"])
+        self.assertTrue(found_resource.labels["zone"])

tests/common/common.py

@@ -35,10 +35,13 @@
 from .script_utils import ScriptRunner
 from .script_utils import Command
 
+
 class LogsNotFound(RuntimeError):
     """raised when filter returns no logs."""
+
     pass
 
+
 class Common:
     _client = Client()
     # environment name must be set by subclass
@@ -50,7 +53,6 @@ def _add_time_condition_to_filter(self, filter_str, timestamp=None):
             timestamp = datetime.now(timezone.utc) - timedelta(minutes=10)
         return f'"{filter_str}" AND timestamp > "{timestamp.strftime(time_format)}"'
 
-
     def _get_logs(self, filter_str=None):
         if not filter_str:
             _, filter_str = self._script.run_command(Command.GetFilter)
@@ -65,6 +67,25 @@ def _trigger(self, function, **kwargs):
         args_str = ",".join([f'{k}="{v}"' for k, v in kwargs.items()])
         self._script.run_command(Command.Trigger, [function, args_str])
 
+    @RetryErrors(exception=LogsNotFound, delay=2)
+    def trigger_and_retrieve(self, log_text, append_uuid=True, max_tries=6):
+        if append_uuid:
+            log_text = f"{log_text} - {uuid.uuid1()}"
+        self._trigger("simplelog", log_text=log_text)
+        filter_str = self._add_time_condition_to_filter(log_text)
+        # give the command time to be received
+        tries = 0
+        while tries < max_tries:
+            # retrieve resulting logs
+            try:
+                log_list = self._get_logs(filter_str)
+                return log_list
+            except LogsNotFound:
+                sleep(10)
+                tries += 1
+        # log not found
+        raise LogsNotFound
+
     @classmethod
     def setUpClass(cls):
         if not cls.environment:
@@ -95,15 +116,9 @@ def tearDown_class(cls):
         if not os.getenv("NO_CLEAN"):
             cls._script.run_command(Command.Destroy)
 
-    @RetryErrors(exception=LogsNotFound)
     def test_receive_log(self):
-        log_text = f"{inspect.currentframe().f_code.co_name}: {uuid.uuid1()}"
-        self._trigger("simplelog", logtext=log_text)
-        # give the command time to be received
-        sleep(30)
-        filter_str = self._add_time_condition_to_filter(log_text)
-        # retrieve resulting logs
-        log_list = self._get_logs(filter_str)
+        log_text = f"{inspect.currentframe().f_code.co_name}"
+        log_list = self.trigger_and_retrieve(log_text)
 
         found_log = None
         for log in log_list:

tests/python/test_appengine_flex_container.py

@@ -18,9 +18,10 @@
 import google.cloud.logging
 
 from ..common.common import Common
+from ..common.appengine import CommonAppEngine
 
 
-class TestAppEngineFlexContainer(Common, unittest.TestCase):
+class TestAppEngineFlexContainer(Common, CommonAppEngine, unittest.TestCase):
 
     environment = "appengine_flex_container"
     language = "python"

tests/python/test_appengine_flex_python.py

@@ -18,9 +18,10 @@
 import google.cloud.logging
 
 from ..common.common import Common
+from ..common.appengine import CommonAppEngine
 
 
-class TestAppEngineFlex(Common, unittest.TestCase):
+class TestAppEngineFlex(Common, CommonAppEngine, unittest.TestCase):
 
     environment = "appengine_flex_python"
     language = "python"

tests/python/test_appengine_standard.py

@@ -18,9 +18,10 @@
 import google.cloud.logging
 
 from ..common.common import Common
+from ..common.appengine import CommonAppEngine
 
 
-class TestAppEngineStandard(Common, unittest.TestCase):
+class TestAppEngineStandard(Common, CommonAppEngine, unittest.TestCase):
 
     environment = "appengine_standard"
     language = "python"

tests/python/test_cloudrun.py

@@ -14,8 +14,11 @@
 
 import logging
 import unittest
+import inspect
+import uuid
 
 import google.cloud.logging
+from google.cloud.logging_v2.resource import Resource
 
 from ..common.common import Common
 
@@ -24,3 +27,15 @@ class TestCloudRun(Common, unittest.TestCase):
 
     environment = "cloudrun"
     language = "python"
+
+    def test_monitored_resource(self):
+        log_text = f"{inspect.currentframe().f_code.co_name}"
+        log_list = self.trigger_and_retrieve(log_text)
+        found_resource = log_list[0].resource
+
+        self.assertEqual(found_resource.type, "cloud_run_revision")
+        self.assertTrue(found_resource.labels["project_id"])
+        self.assertTrue(found_resource.labels["service_name"])
+        self.assertTrue(found_resource.labels["revision_name"])
+        self.assertTrue(found_resource.labels["location"])
+        self.assertTrue(found_resource.labels["configuration_name"])

tests/python/test_compute.py

@@ -14,6 +14,7 @@
 
 import logging
 import unittest
+import inspect
 
 import google.cloud.logging
 
@@ -24,3 +25,12 @@ class TestComputeEngine(Common, unittest.TestCase):
 
     environment = "compute"
     language = "python"
+
+    def test_monitored_resource(self):
+        log_text = f"{inspect.currentframe().f_code.co_name}"
+        log_list = self.trigger_and_retrieve(log_text)
+        found_resource = log_list[0].resource
+
+        self.assertEqual(found_resource.type, "gce_instance")
+        self.assertTrue(found_resource.labels["zone"])
+        self.assertTrue(found_resource.labels["instance_id"])

tests/python/test_functions.py

@@ -14,6 +14,7 @@
 
 import logging
 import unittest
+import inspect
 
 import google.cloud.logging
 
@@ -24,3 +25,12 @@ class TestCloudFunctions(Common, unittest.TestCase):
 
     environment = "functions"
     language = "python"
+
+    def test_monitored_resource(self):
+        log_text = f"{inspect.currentframe().f_code.co_name}"
+        log_list = self.trigger_and_retrieve(log_text)
+        found_resource = log_list[0].resource
+
+        self.assertEqual(found_resource.type, "cloud_function")
+        self.assertTrue(found_resource.labels["region"])
+        self.assertTrue(found_resource.labels["function_name"])

tests/python/test_kubernetes.py

@@ -14,6 +14,7 @@
 
 import logging
 import unittest
+import inspect
 
 import google.cloud.logging
 
@@ -24,3 +25,16 @@ class TestKubernetesEngine(Common, unittest.TestCase):
 
     environment = "kubernetes"
     language = "python"
+
+    def test_monitored_resource(self):
+        log_text = f"{inspect.currentframe().f_code.co_name}"
+        log_list = self.trigger_and_retrieve(log_text)
+        found_resource = log_list[0].resource
+
+        self.assertEqual(found_resource.type, "k8s_container")
+        self.assertTrue(found_resource.labels["project_id"])
+        self.assertTrue(found_resource.labels["location"])
+        self.assertTrue(found_resource.labels["cluster_name"])
+        self.assertTrue(found_resource.labels["namespace_name"])
+        self.assertTrue(found_resource.labels["pod_name"])
+        self.assertTrue(found_resource.labels["container_name"])