diff --git a/deployable/python/snippets.py b/deployable/python/snippets.py index efaa976..4c76950 100644 --- a/deployable/python/snippets.py +++ b/deployable/python/snippets.py @@ -15,10 +15,10 @@ import logging import os -from google.cloud.logging_v2._helpers import retrieve_metadata_server try: import google.cloud.logging + from google.cloud.logging_v2._helpers import retrieve_metadata_server except ImportError: # import at runtime for GAE environments import pip @@ -28,15 +28,16 @@ pip.main(["install", "-e", "./python-logging"]) importlib.reload(site) import google.cloud.logging + from google.cloud.logging_v2._helpers import retrieve_metadata_server -def simple_log(log_name=None, log_text="simple_log", **kwargs): - client = google.cloud.logging.Client() - logger = client.logger(log_name) - logger.log_text(log_text) +# def simple_log(log_name=None, log_text="simple_log", **kwargs): +# client = google.cloud.logging.Client() +# logger = client.logger(log_name) +# logger.log_text(log_text) -def pylogging(log_text="pylogging", severity="warning", **kwargs): +def simplelog(log_text="pylogging", severity="warning", **kwargs): # allowed severity: debug, info, warning, error, critical if severity == "debug": logging.debug(log_text) @@ -49,11 +50,13 @@ def pylogging(log_text="pylogging", severity="warning", **kwargs): else: logging.critical(log_text) + def print_handlers(**kwargs): root_logger = logging.getLogger() - handlers_str = ', '.join([type(h).__name__ for h in root_logger.handlers]) + handlers_str = ", ".join([type(h).__name__ for h in root_logger.handlers]) logging.info(handlers_str) + def remove_stream_handlers(**kwargs): logger = logging.getLogger() for handler in logger.handlers: @@ -61,6 +64,7 @@ def remove_stream_handlers(**kwargs): logging.error(handler) logger.removeHandler(handler) + def print_env_vars(env_var=None, **kwargs): if env_var: value = os.environ.get(env_var, None) @@ -71,6 +75,7 @@ def print_env_vars(env_var=None, **kwargs): else: logging.error(os.environ) + def get_metadata_server(metadata_key=None, **kwargs): if metadata_key is None: metadata_key = "" diff --git a/envctl/env_scripts/python/cloudrun.sh b/envctl/env_scripts/python/cloudrun.sh index 1fe9461..4da3dc1 100755 --- a/envctl/env_scripts/python/cloudrun.sh +++ b/envctl/env_scripts/python/cloudrun.sh @@ -27,16 +27,16 @@ add_service_accounts() { local PROJECT_NUMBER=$(gcloud projects list --filter=$PROJECT_ID --format="value(PROJECT_NUMBER)") gcloud projects add-iam-policy-binding $PROJECT_ID \ --member=serviceAccount:service-$PROJECT_NUMBER@gcp-sa-pubsub.iam.gserviceaccount.com \ - --role=roles/iam.serviceAccountTokenCreator 2> /dev/null + --role=roles/iam.serviceAccountTokenCreator gcloud iam service-accounts create $SA_NAME \ - --display-name "Pub/Sub Invoker" 2> /dev/null + --display-name "Pub/Sub Invoker" gcloud run services add-iam-policy-binding $SERVICE_NAME \ --member=serviceAccount:$SA_NAME@$PROJECT_ID.iam.gserviceaccount.com \ - --role=roles/run.invoker 2> /dev/null + --role=roles/run.invoker RUN_URL=$(gcloud run services list --filter=$SERVICE_NAME --format="value(URL)") gcloud pubsub subscriptions create $SERVICE_NAME-subscriber --topic $SERVICE_NAME \ --push-endpoint=$RUN_URL \ - --push-auth-service-account=$SA_NAME@$PROJECT_ID.iam.gserviceaccount.com 2> /dev/null + --push-auth-service-account=$SA_NAME@$PROJECT_ID.iam.gserviceaccount.com set -e } @@ -46,7 +46,7 @@ destroy() { gcloud pubsub topics delete $SERVICE_NAME -q 2> /dev/null gcloud pubsub subscriptions delete $SERVICE_NAME-subscriber -q 2> /dev/null # delete service account - gcloud iam service-accounts delete $SA_NAME -q 2> /dev/null + gcloud iam service-accounts delete $SA_NAME@$PROJECT_ID.iam.gserviceaccount.com -q 2> /dev/null # delete container images export GCR_PATH=gcr.io/$PROJECT_ID/logging:$SERVICE_NAME gcloud container images delete $GCR_PATH -q --force-delete-tags 2> /dev/null @@ -73,7 +73,6 @@ deploy() { gcloud config set run/platform managed gcloud config set run/region us-west1 gcloud run deploy \ - --allow-unauthenticated \ --image $GCR_PATH \ --update-env-vars ENABLE_FLASK=true \ $SERVICE_NAME diff --git a/tests/common/appengine.py b/tests/common/appengine.py new file mode 100644 index 0000000..a062af9 --- /dev/null +++ b/tests/common/appengine.py @@ -0,0 +1,34 @@ +# Copyright 2016 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import unittest +import inspect + +import google.cloud.logging + +from ..common.common import Common + + +class CommonAppEngine: + def test_monitored_resource(self): + log_text = f"{inspect.currentframe().f_code.co_name}" + log_list = self.trigger_and_retrieve(log_text) + found_resource = log_list[-1].resource + + self.assertEqual(found_resource.type, "gae_app") + self.assertTrue(found_resource.labels["project_id"]) + self.assertTrue(found_resource.labels["module_id"]) + self.assertTrue(found_resource.labels["version_id"]) + self.assertTrue(found_resource.labels["zone"]) diff --git a/tests/common/common.py b/tests/common/common.py index dcd6c5f..e26c832 100644 --- a/tests/common/common.py +++ b/tests/common/common.py @@ -35,10 +35,13 @@ from .script_utils import ScriptRunner from .script_utils import Command + class LogsNotFound(RuntimeError): """raised when filter returns no logs.""" + pass + class Common: _client = Client() # environment name must be set by subclass @@ -50,7 +53,6 @@ def _add_time_condition_to_filter(self, filter_str, timestamp=None): timestamp = datetime.now(timezone.utc) - timedelta(minutes=10) return f'"{filter_str}" AND timestamp > "{timestamp.strftime(time_format)}"' - def _get_logs(self, filter_str=None): if not filter_str: _, filter_str = self._script.run_command(Command.GetFilter) @@ -65,6 +67,25 @@ def _trigger(self, function, **kwargs): args_str = ",".join([f'{k}="{v}"' for k, v in kwargs.items()]) self._script.run_command(Command.Trigger, [function, args_str]) + @RetryErrors(exception=LogsNotFound, delay=2) + def trigger_and_retrieve(self, log_text, append_uuid=True, max_tries=6): + if append_uuid: + log_text = f"{log_text} - {uuid.uuid1()}" + self._trigger("simplelog", log_text=log_text) + filter_str = self._add_time_condition_to_filter(log_text) + # give the command time to be received + tries = 0 + while tries < max_tries: + # retrieve resulting logs + try: + log_list = self._get_logs(filter_str) + return log_list + except LogsNotFound: + sleep(10) + tries += 1 + # log not found + raise LogsNotFound + @classmethod def setUpClass(cls): if not cls.environment: @@ -95,15 +116,9 @@ def tearDown_class(cls): if not os.getenv("NO_CLEAN"): cls._script.run_command(Command.Destroy) - @RetryErrors(exception=LogsNotFound) def test_receive_log(self): - log_text = f"{inspect.currentframe().f_code.co_name}: {uuid.uuid1()}" - self._trigger("simplelog", logtext=log_text) - # give the command time to be received - sleep(30) - filter_str = self._add_time_condition_to_filter(log_text) - # retrieve resulting logs - log_list = self._get_logs(filter_str) + log_text = f"{inspect.currentframe().f_code.co_name}" + log_list = self.trigger_and_retrieve(log_text) found_log = None for log in log_list: diff --git a/tests/python/test_appengine_flex_container.py b/tests/python/test_appengine_flex_container.py index 16bb2ea..e8a55ad 100644 --- a/tests/python/test_appengine_flex_container.py +++ b/tests/python/test_appengine_flex_container.py @@ -18,9 +18,10 @@ import google.cloud.logging from ..common.common import Common +from ..common.appengine import CommonAppEngine -class TestAppEngineFlexContainer(Common, unittest.TestCase): +class TestAppEngineFlexContainer(Common, CommonAppEngine, unittest.TestCase): environment = "appengine_flex_container" language = "python" diff --git a/tests/python/test_appengine_flex_python.py b/tests/python/test_appengine_flex_python.py index 5c8fdac..23d1e30 100644 --- a/tests/python/test_appengine_flex_python.py +++ b/tests/python/test_appengine_flex_python.py @@ -18,9 +18,10 @@ import google.cloud.logging from ..common.common import Common +from ..common.appengine import CommonAppEngine -class TestAppEngineFlex(Common, unittest.TestCase): +class TestAppEngineFlex(Common, CommonAppEngine, unittest.TestCase): environment = "appengine_flex_python" language = "python" diff --git a/tests/python/test_appengine_standard.py b/tests/python/test_appengine_standard.py index e4bb999..bb93ab8 100644 --- a/tests/python/test_appengine_standard.py +++ b/tests/python/test_appengine_standard.py @@ -18,9 +18,10 @@ import google.cloud.logging from ..common.common import Common +from ..common.appengine import CommonAppEngine -class TestAppEngineStandard(Common, unittest.TestCase): +class TestAppEngineStandard(Common, CommonAppEngine, unittest.TestCase): environment = "appengine_standard" language = "python" diff --git a/tests/python/test_cloudrun.py b/tests/python/test_cloudrun.py index 1ee0826..7c83c11 100644 --- a/tests/python/test_cloudrun.py +++ b/tests/python/test_cloudrun.py @@ -14,8 +14,11 @@ import logging import unittest +import inspect +import uuid import google.cloud.logging +from google.cloud.logging_v2.resource import Resource from ..common.common import Common @@ -24,3 +27,15 @@ class TestCloudRun(Common, unittest.TestCase): environment = "cloudrun" language = "python" + + def test_monitored_resource(self): + log_text = f"{inspect.currentframe().f_code.co_name}" + log_list = self.trigger_and_retrieve(log_text) + found_resource = log_list[0].resource + + self.assertEqual(found_resource.type, "cloud_run_revision") + self.assertTrue(found_resource.labels["project_id"]) + self.assertTrue(found_resource.labels["service_name"]) + self.assertTrue(found_resource.labels["revision_name"]) + self.assertTrue(found_resource.labels["location"]) + self.assertTrue(found_resource.labels["configuration_name"]) diff --git a/tests/python/test_compute.py b/tests/python/test_compute.py index 4b71761..81b7e3a 100644 --- a/tests/python/test_compute.py +++ b/tests/python/test_compute.py @@ -14,6 +14,7 @@ import logging import unittest +import inspect import google.cloud.logging @@ -24,3 +25,12 @@ class TestComputeEngine(Common, unittest.TestCase): environment = "compute" language = "python" + + def test_monitored_resource(self): + log_text = f"{inspect.currentframe().f_code.co_name}" + log_list = self.trigger_and_retrieve(log_text) + found_resource = log_list[0].resource + + self.assertEqual(found_resource.type, "gce_instance") + self.assertTrue(found_resource.labels["zone"]) + self.assertTrue(found_resource.labels["instance_id"]) diff --git a/tests/python/test_functions.py b/tests/python/test_functions.py index b3bb755..de65169 100644 --- a/tests/python/test_functions.py +++ b/tests/python/test_functions.py @@ -14,6 +14,7 @@ import logging import unittest +import inspect import google.cloud.logging @@ -24,3 +25,12 @@ class TestCloudFunctions(Common, unittest.TestCase): environment = "functions" language = "python" + + def test_monitored_resource(self): + log_text = f"{inspect.currentframe().f_code.co_name}" + log_list = self.trigger_and_retrieve(log_text) + found_resource = log_list[0].resource + + self.assertEqual(found_resource.type, "cloud_function") + self.assertTrue(found_resource.labels["region"]) + self.assertTrue(found_resource.labels["function_name"]) diff --git a/tests/python/test_kubernetes.py b/tests/python/test_kubernetes.py index e4e5bf5..77f4d38 100644 --- a/tests/python/test_kubernetes.py +++ b/tests/python/test_kubernetes.py @@ -14,6 +14,7 @@ import logging import unittest +import inspect import google.cloud.logging @@ -24,3 +25,16 @@ class TestKubernetesEngine(Common, unittest.TestCase): environment = "kubernetes" language = "python" + + def test_monitored_resource(self): + log_text = f"{inspect.currentframe().f_code.co_name}" + log_list = self.trigger_and_retrieve(log_text) + found_resource = log_list[0].resource + + self.assertEqual(found_resource.type, "k8s_container") + self.assertTrue(found_resource.labels["project_id"]) + self.assertTrue(found_resource.labels["location"]) + self.assertTrue(found_resource.labels["cluster_name"]) + self.assertTrue(found_resource.labels["namespace_name"]) + self.assertTrue(found_resource.labels["pod_name"]) + self.assertTrue(found_resource.labels["container_name"])