From 7b9d81fddbe9bf8a7c8f53880b1c9b4cf7b28cf9 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 26 Feb 2021 15:07:17 -0800 Subject: [PATCH 01/29] added resource detection for cloud run --- google/cloud/logging_v2/client.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/google/cloud/logging_v2/client.py b/google/cloud/logging_v2/client.py index ee65d288a..cc497d66f 100644 --- a/google/cloud/logging_v2/client.py +++ b/google/cloud/logging_v2/client.py @@ -39,6 +39,7 @@ from google.cloud.logging_v2.handlers import ContainerEngineHandler from google.cloud.logging_v2.handlers import setup_logging from google.cloud.logging_v2.handlers.handlers import EXCLUDED_LOGGER_DEFAULTS +from google.cloud.logging_v2.resource import Resource from google.cloud.logging_v2.logger import Logger from google.cloud.logging_v2.metric import Metric @@ -57,6 +58,13 @@ _GKE_CLUSTER_NAME = "instance/attributes/cluster-name" """Attribute in metadata server when in GKE environment.""" +_CLOUD_RUN_SERVICE_ID = "K_SERVICE" +_CLOUD_RUN_REVISION_ID = "K_REVISION" +_CLOUD_RUN_CONFIGURATION_ID = "K_CONFIGURATION" +"""Environment variables set in Cloud Run environment.""" + +_REGION_ID = "instance/region" +"""Attribute in metadata server for compute region.""" class Client(ClientWithProject): """Client to bundle configuration needed for API requests.""" @@ -349,6 +357,7 @@ def get_default_handler(self, **kw): logging.Handler: The default log handler based on the environment """ gke_cluster_name = retrieve_metadata_server(_GKE_CLUSTER_NAME) + region = retrieve_metadata_server(_REGION_ID) if ( _APPENGINE_FLEXIBLE_ENV_VM in os.environ @@ -357,7 +366,20 @@ def get_default_handler(self, **kw): return AppEngineHandler(self, **kw) elif gke_cluster_name is not None: return ContainerEngineHandler(**kw) + elif all([env in os.environ for env in (_CLOUD_RUN_SERVICE_ID, _CLOUD_RUN_REVISION_ID, _CLOUD_RUN_CONFIGURATION_ID)]): + resource = Resource( + type="cloud_run_revision", + labels={ + "project_id": self.project, + "service_name": os.environ.get(_CLOUD_RUN_SERVICE_ID, ""), + "revision_name": os.environ.get(_CLOUD_RUN_REVISION_ID, ""), + "location": region if region else "", + "configuration_name": os.environ.get(_CLOUD_RUN_CONFIGURATION_ID, ""), + }, + ) + return CloudLoggingHandler(self, resource=resource, **kw) else: + # generic handler. uses global resource return CloudLoggingHandler(self, **kw) def setup_logging( From add39c49f35749f8511dbd2deb2bcb3b49383870 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 26 Feb 2021 16:30:18 -0800 Subject: [PATCH 02/29] added new monitored resource tests --- tests/environment | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/environment b/tests/environment index f35514893..bf3476742 160000 --- a/tests/environment +++ b/tests/environment @@ -1 +1 @@ -Subproject commit f35514893542dfa29f65214eea96b490f04f3d72 +Subproject commit bf34767428a2cabf323316e1f78c3514960106f7 From 2e857550f5ef2d743ddf5f58048f60e07b24b257 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 26 Feb 2021 16:50:19 -0800 Subject: [PATCH 03/29] added gcf resource detection --- google/cloud/logging_v2/client.py | 33 ++++++++++++++++++++++++++++++- tests/environment | 2 +- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/google/cloud/logging_v2/client.py b/google/cloud/logging_v2/client.py index cc497d66f..976f0b329 100644 --- a/google/cloud/logging_v2/client.py +++ b/google/cloud/logging_v2/client.py @@ -61,8 +61,19 @@ _CLOUD_RUN_SERVICE_ID = "K_SERVICE" _CLOUD_RUN_REVISION_ID = "K_REVISION" _CLOUD_RUN_CONFIGURATION_ID = "K_CONFIGURATION" +_CLOUD_RUN_ENV_VARS = [_CLOUD_RUN_SERVICE_ID, _CLOUD_RUN_REVISION_ID, _CLOUD_RUN_CONFIGURATION_ID] """Environment variables set in Cloud Run environment.""" +_FUNCTION_TARGET = "FUNCTION_TARGET" +_FUNCTION_SIGNATURE = "FUNCTION_SIGNATURE_TYPE" +_FUNCTION_NAME = "FUNCTION_NAME" +_FUNCTION_REGION = "FUNCTION_REGION" +_FUNCTION_ENTRY = "ENTRY_POINT" +_FUNCTION_ENV_VARS = [_FUNCTION_TARGET, _FUNCTION_SIGNATURE_TYPE, _CLOUD_RUN_SERVICE_ID] +_LEGACY_FUNCTION_ENV_VARS = [_FUNCTION_NAME, _FUNCTION_REGION, _FUNCTION_ENTRY] +"""Environment variables set in Cloud Functions environments.""" + + _REGION_ID = "instance/region" """Attribute in metadata server for compute region.""" @@ -366,7 +377,27 @@ def get_default_handler(self, **kw): return AppEngineHandler(self, **kw) elif gke_cluster_name is not None: return ContainerEngineHandler(**kw) - elif all([env in os.environ for env in (_CLOUD_RUN_SERVICE_ID, _CLOUD_RUN_REVISION_ID, _CLOUD_RUN_CONFIGURATION_ID)]): + elif all([env in os.environ for env in _LEGACY_FUNCTION_ENV_VARS]): + resource = Resource( + type="cloud_function", + labels={ + "project_id": self.project, + "function_name": os.environ.get(_FUNCTION_NAME, ""), + "region": region if region else "", + }, + ) + return CloudLoggingHandler(self, resource=resource, **kw) + elif all([env in os.environ for env in _FUNCTION_ENV_VARS]): + resource = Resource( + type="cloud_function", + labels={ + "project_id": self.project, + "function_name": os.environ.get(_CLOUD_RUN_SERVICE_ID, ""), + "region": region if region else "", + }, + ) + return CloudLoggingHandler(self, resource=resource, **kw) + elif all([env in os.environ for env in _CLOUD_RUN_ENV_VARS]): resource = Resource( type="cloud_run_revision", labels={ diff --git a/tests/environment b/tests/environment index bf3476742..ff3a2761e 160000 --- a/tests/environment +++ b/tests/environment @@ -1 +1 @@ -Subproject commit bf34767428a2cabf323316e1f78c3514960106f7 +Subproject commit ff3a2761e13dd3efd3910095cf1f0a2b09e57070 From 0e80fecc237f14f628be453f212bf3c51a5fdcb8 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 26 Feb 2021 17:04:35 -0800 Subject: [PATCH 04/29] monitored resource detection for gce --- google/cloud/logging_v2/client.py | 16 +++++++++++++++- tests/environment | 2 +- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/google/cloud/logging_v2/client.py b/google/cloud/logging_v2/client.py index 976f0b329..6a9faf833 100644 --- a/google/cloud/logging_v2/client.py +++ b/google/cloud/logging_v2/client.py @@ -75,7 +75,9 @@ _REGION_ID = "instance/region" -"""Attribute in metadata server for compute region.""" +_ZONE_ID = "instance/zone" +_GCE_INSTANCE_ID = "instance/id" +"""Attribute in metadata server for compute region and instance.""" class Client(ClientWithProject): """Client to bundle configuration needed for API requests.""" @@ -369,6 +371,7 @@ def get_default_handler(self, **kw): """ gke_cluster_name = retrieve_metadata_server(_GKE_CLUSTER_NAME) region = retrieve_metadata_server(_REGION_ID) + instance = retrieve_metadata_server(_GCE_INSTANCE_ID) if ( _APPENGINE_FLEXIBLE_ENV_VM in os.environ @@ -409,6 +412,17 @@ def get_default_handler(self, **kw): }, ) return CloudLoggingHandler(self, resource=resource, **kw) + elif instance is not None: + resource = Resource( + type="gce_instance", + labels={ + "project_id": self.project, + "instance_id": instance, + "zone": retrieve_metadata_server(_ZONE_ID) + }, + ) + return CloudLoggingHandler(self, resource=resource, **kw) + else: # generic handler. uses global resource return CloudLoggingHandler(self, **kw) diff --git a/tests/environment b/tests/environment index ff3a2761e..5cae58574 160000 --- a/tests/environment +++ b/tests/environment @@ -1 +1 @@ -Subproject commit ff3a2761e13dd3efd3910095cf1f0a2b09e57070 +Subproject commit 5cae5857468fee7a24a6b30a70bca3f107a31379 From 97040e8ef5718a51fe3ebb5eafe6d3a41ce70690 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 26 Feb 2021 17:39:44 -0800 Subject: [PATCH 05/29] refactored --- google/cloud/logging_v2/client.py | 104 ++++------------ .../handlers/_monitored_resources.py | 116 ++++++++++++++++++ 2 files changed, 141 insertions(+), 79 deletions(-) create mode 100644 google/cloud/logging_v2/handlers/_monitored_resources.py diff --git a/google/cloud/logging_v2/client.py b/google/cloud/logging_v2/client.py index 6a9faf833..b1a255c90 100644 --- a/google/cloud/logging_v2/client.py +++ b/google/cloud/logging_v2/client.py @@ -40,6 +40,15 @@ from google.cloud.logging_v2.handlers import setup_logging from google.cloud.logging_v2.handlers.handlers import EXCLUDED_LOGGER_DEFAULTS from google.cloud.logging_v2.resource import Resource +from google.cloud.logging_v2.handlers._monitored_resources import GAE_ENV_VARS +from google.cloud.logging_v2.handlers._monitored_resources import CLOUD_RUN_ENV_VARS +from google.cloud.logging_v2.handlers._monitored_resources import FUNCTION_ENV_VARS +from google.cloud.logging_v2.handlers._monitored_resources import LEGACY_FUNCTION_ENV_VARS +from google.cloud.logging_v2.handlers._monitored_resources import GKE_CLUSTER_NAME +from google.cloud.logging_v2.handlers._monitored_resources import GCE_INSTANCE_ID +from google.cloud.logging_v2.handlers._monitored_resources import create_functions_resource +from google.cloud.logging_v2.handlers._monitored_resources import create_compute_resource +from google.cloud.logging_v2.handlers._monitored_resources import create_cloud_run_resource from google.cloud.logging_v2.logger import Logger from google.cloud.logging_v2.metric import Metric @@ -49,36 +58,6 @@ _DISABLE_GRPC = os.getenv(DISABLE_GRPC, False) _USE_GRPC = _HAVE_GRPC and not _DISABLE_GRPC -_APPENGINE_FLEXIBLE_ENV_VM = "GAE_APPENGINE_HOSTNAME" -"""Environment variable set in App Engine when vm:true is set.""" - -_APPENGINE_INSTANCE_ID = "GAE_INSTANCE" -"""Environment variable set in App Engine standard and flexible environment.""" - -_GKE_CLUSTER_NAME = "instance/attributes/cluster-name" -"""Attribute in metadata server when in GKE environment.""" - -_CLOUD_RUN_SERVICE_ID = "K_SERVICE" -_CLOUD_RUN_REVISION_ID = "K_REVISION" -_CLOUD_RUN_CONFIGURATION_ID = "K_CONFIGURATION" -_CLOUD_RUN_ENV_VARS = [_CLOUD_RUN_SERVICE_ID, _CLOUD_RUN_REVISION_ID, _CLOUD_RUN_CONFIGURATION_ID] -"""Environment variables set in Cloud Run environment.""" - -_FUNCTION_TARGET = "FUNCTION_TARGET" -_FUNCTION_SIGNATURE = "FUNCTION_SIGNATURE_TYPE" -_FUNCTION_NAME = "FUNCTION_NAME" -_FUNCTION_REGION = "FUNCTION_REGION" -_FUNCTION_ENTRY = "ENTRY_POINT" -_FUNCTION_ENV_VARS = [_FUNCTION_TARGET, _FUNCTION_SIGNATURE_TYPE, _CLOUD_RUN_SERVICE_ID] -_LEGACY_FUNCTION_ENV_VARS = [_FUNCTION_NAME, _FUNCTION_REGION, _FUNCTION_ENTRY] -"""Environment variables set in Cloud Functions environments.""" - - -_REGION_ID = "instance/region" -_ZONE_ID = "instance/zone" -_GCE_INSTANCE_ID = "instance/id" -"""Attribute in metadata server for compute region and instance.""" - class Client(ClientWithProject): """Client to bundle configuration needed for API requests.""" @@ -370,62 +349,29 @@ def get_default_handler(self, **kw): logging.Handler: The default log handler based on the environment """ gke_cluster_name = retrieve_metadata_server(_GKE_CLUSTER_NAME) - region = retrieve_metadata_server(_REGION_ID) - instance = retrieve_metadata_server(_GCE_INSTANCE_ID) + gce_instance_name = retrieve_metadata_server(_GCE_INSTANCE_ID) + resource = None - if ( - _APPENGINE_FLEXIBLE_ENV_VM in os.environ - or _APPENGINE_INSTANCE_ID in os.environ - ): + if all([env in os.environ for env in _GAE_ENV_VARS]): + # App Engine Flex or Standard return AppEngineHandler(self, **kw) elif gke_cluster_name is not None: + # Kubernetes Engine return ContainerEngineHandler(**kw) - elif all([env in os.environ for env in _LEGACY_FUNCTION_ENV_VARS]): - resource = Resource( - type="cloud_function", - labels={ - "project_id": self.project, - "function_name": os.environ.get(_FUNCTION_NAME, ""), - "region": region if region else "", - }, - ) - return CloudLoggingHandler(self, resource=resource, **kw) - elif all([env in os.environ for env in _FUNCTION_ENV_VARS]): - resource = Resource( - type="cloud_function", - labels={ - "project_id": self.project, - "function_name": os.environ.get(_CLOUD_RUN_SERVICE_ID, ""), - "region": region if region else "", - }, - ) - return CloudLoggingHandler(self, resource=resource, **kw) + elif all([env in os.environ for env in _LEGACY_FUNCTION_ENV_VARS]) + or all([env in os.environ for env in _FUNCTION_ENV_VARS]): + # Cloud Functions + resource = create_functions_resource(self.project) elif all([env in os.environ for env in _CLOUD_RUN_ENV_VARS]): - resource = Resource( - type="cloud_run_revision", - labels={ - "project_id": self.project, - "service_name": os.environ.get(_CLOUD_RUN_SERVICE_ID, ""), - "revision_name": os.environ.get(_CLOUD_RUN_REVISION_ID, ""), - "location": region if region else "", - "configuration_name": os.environ.get(_CLOUD_RUN_CONFIGURATION_ID, ""), - }, - ) - return CloudLoggingHandler(self, resource=resource, **kw) + # Cloud Run + resource = create_cloud_run_resource(self.project) elif instance is not None: - resource = Resource( - type="gce_instance", - labels={ - "project_id": self.project, - "instance_id": instance, - "zone": retrieve_metadata_server(_ZONE_ID) - }, - ) - return CloudLoggingHandler(self, resource=resource, **kw) - + # Compute Engine + resource = create_compute_resource(self.project) else: - # generic handler. uses global resource - return CloudLoggingHandler(self, **kw) + # use generic global resource + resource = create_global_resource(self.project) + return CloudLoggingHandler(self, resource=resource, **kw) def setup_logging( self, *, log_level=logging.INFO, excluded_loggers=EXCLUDED_LOGGER_DEFAULTS, **kw diff --git a/google/cloud/logging_v2/handlers/_monitored_resources.py b/google/cloud/logging_v2/handlers/_monitored_resources.py new file mode 100644 index 000000000..83645263a --- /dev/null +++ b/google/cloud/logging_v2/handlers/_monitored_resources.py @@ -0,0 +1,116 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os + +from google.cloud.logging_v2.resource import Resource +from google.cloud.logging_v2._helpers import retrieve_metadata_server + +_GAE_SERVICE_ENV = "GAE_SERVICE" +_GAE_VERSION_ENV = "GAE_VERSION" +_GAE_INSTANCE_ENV = "GAE_INSTANCE" +_GAE_ENV_VARS = [_GAE_SERVICE_ENV, _GAE_VERSION_ENV, _GAE_INSTANCE_ENV] +"""Environment variables set in App Engine environment.""" + +_CLOUD_RUN_SERVICE_ID = "K_SERVICE" +_CLOUD_RUN_REVISION_ID = "K_REVISION" +_CLOUD_RUN_CONFIGURATION_ID = "K_CONFIGURATION" +_CLOUD_RUN_ENV_VARS = [_CLOUD_RUN_SERVICE_ID, _CLOUD_RUN_REVISION_ID, _CLOUD_RUN_CONFIGURATION_ID] +"""Environment variables set in Cloud Run environment.""" + +_FUNCTION_TARGET = "FUNCTION_TARGET" +_FUNCTION_SIGNATURE = "FUNCTION_SIGNATURE_TYPE" +_FUNCTION_NAME = "FUNCTION_NAME" +_FUNCTION_REGION = "FUNCTION_REGION" +_FUNCTION_ENTRY = "ENTRY_POINT" +_FUNCTION_ENV_VARS = [_FUNCTION_TARGET, _FUNCTION_SIGNATURE_TYPE, _CLOUD_RUN_SERVICE_ID] +_LEGACY_FUNCTION_ENV_VARS = [_FUNCTION_NAME, _FUNCTION_REGION, _FUNCTION_ENTRY] +"""Environment variables set in Cloud Functions environments.""" + + +_REGION_ID = "instance/region" +_ZONE_ID = "instance/zone" +_GCE_INSTANCE_ID = "instance/id" +"""Attribute in metadata server for compute region and instance.""" + +_GKE_CLUSTER_NAME = "instance/attributes/cluster-name" +"""Attribute in metadata server when in GKE environment.""" + + +def create_functions_resource(project): + region = retrieve_metadata_server(_REGION_ID) + if _FUNCTION_NAME in os.environ: + function_name = os.environ.get(_FUNCTION_NAME) + elif _CLOUD_RUN_SERVICE_ID in os.environ: + function_name = os.environ.get(_CLOUD_RUN_SERVICE_ID) + else: + function_name = "" + resource = Resource( + type="cloud_function", + labels={ + "project_id": project, + "function_name": function_name, + "region": region if region else "", + }, + ) + return resource + +def create_compute_resource(project): + instance = retrieve_metadata_server(_GCE_INSTANCE_ID) + zone = retrieve_metadata_server(_ZONE_ID) + resource = Resource( + type="gce_instance", + labels={ + "project_id": project, + "instance_id": instance if instance else "", + "zone": zone if zone else "", + }, + ) + return resource + + +def create_cloud_run_resource(project): + region = retrieve_metadata_server(_REGION_ID) + resource = Resource( + type="cloud_run_revision", + labels={ + "project_id": project, + "service_name": os.environ.get(_CLOUD_RUN_SERVICE_ID, ""), + "revision_name": os.environ.get(_CLOUD_RUN_REVISION_ID, ""), + "location": region if region else "", + "configuration_name": os.environ.get(_CLOUD_RUN_CONFIGURATION_ID, ""), + }, + ) + return resource + +def create_app_engine_resource(project): + resource = Resource( + type="gae_app", + labels={ + "project_id": project, + "module_id": os.environ.get(_GAE_SERVICE_ENV, ""), + "version_id": os.environ.get(_GAE_VERSION_ENV, ""), + }, + ) + return resource + +def create_global_resource(project): + resource = Resource( + type="global", + labels={ + "project_id": project, + }, + ) + return resource From eb8f30957b425fd8f4c89f8231062422e27237a9 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 26 Feb 2021 18:09:46 -0800 Subject: [PATCH 06/29] added parens --- google/cloud/logging_v2/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/google/cloud/logging_v2/client.py b/google/cloud/logging_v2/client.py index b1a255c90..5180b0d6c 100644 --- a/google/cloud/logging_v2/client.py +++ b/google/cloud/logging_v2/client.py @@ -358,8 +358,8 @@ def get_default_handler(self, **kw): elif gke_cluster_name is not None: # Kubernetes Engine return ContainerEngineHandler(**kw) - elif all([env in os.environ for env in _LEGACY_FUNCTION_ENV_VARS]) - or all([env in os.environ for env in _FUNCTION_ENV_VARS]): + elif (all([env in os.environ for env in _LEGACY_FUNCTION_ENV_VARS]) + or all([env in os.environ for env in _FUNCTION_ENV_VARS])): # Cloud Functions resource = create_functions_resource(self.project) elif all([env in os.environ for env in _CLOUD_RUN_ENV_VARS]): From 98ca4d503e7eead4d982219f35bb298273f7611a Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 26 Feb 2021 18:57:14 -0800 Subject: [PATCH 07/29] fixed env var names --- .../logging_v2/handlers/_monitored_resources.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/google/cloud/logging_v2/handlers/_monitored_resources.py b/google/cloud/logging_v2/handlers/_monitored_resources.py index 83645263a..5b33729b5 100644 --- a/google/cloud/logging_v2/handlers/_monitored_resources.py +++ b/google/cloud/logging_v2/handlers/_monitored_resources.py @@ -21,13 +21,13 @@ _GAE_SERVICE_ENV = "GAE_SERVICE" _GAE_VERSION_ENV = "GAE_VERSION" _GAE_INSTANCE_ENV = "GAE_INSTANCE" -_GAE_ENV_VARS = [_GAE_SERVICE_ENV, _GAE_VERSION_ENV, _GAE_INSTANCE_ENV] +GAE_ENV_VARS = [_GAE_SERVICE_ENV, _GAE_VERSION_ENV, _GAE_INSTANCE_ENV] """Environment variables set in App Engine environment.""" _CLOUD_RUN_SERVICE_ID = "K_SERVICE" _CLOUD_RUN_REVISION_ID = "K_REVISION" _CLOUD_RUN_CONFIGURATION_ID = "K_CONFIGURATION" -_CLOUD_RUN_ENV_VARS = [_CLOUD_RUN_SERVICE_ID, _CLOUD_RUN_REVISION_ID, _CLOUD_RUN_CONFIGURATION_ID] +CLOUD_RUN_ENV_VARS = [_CLOUD_RUN_SERVICE_ID, _CLOUD_RUN_REVISION_ID, _CLOUD_RUN_CONFIGURATION_ID] """Environment variables set in Cloud Run environment.""" _FUNCTION_TARGET = "FUNCTION_TARGET" @@ -35,17 +35,17 @@ _FUNCTION_NAME = "FUNCTION_NAME" _FUNCTION_REGION = "FUNCTION_REGION" _FUNCTION_ENTRY = "ENTRY_POINT" -_FUNCTION_ENV_VARS = [_FUNCTION_TARGET, _FUNCTION_SIGNATURE_TYPE, _CLOUD_RUN_SERVICE_ID] -_LEGACY_FUNCTION_ENV_VARS = [_FUNCTION_NAME, _FUNCTION_REGION, _FUNCTION_ENTRY] +FUNCTION_ENV_VARS = [_FUNCTION_TARGET, _FUNCTION_SIGNATURE, _CLOUD_RUN_SERVICE_ID] +LEGACY_FUNCTION_ENV_VARS = [_FUNCTION_NAME, _FUNCTION_REGION, _FUNCTION_ENTRY] """Environment variables set in Cloud Functions environments.""" _REGION_ID = "instance/region" _ZONE_ID = "instance/zone" -_GCE_INSTANCE_ID = "instance/id" +GCE_INSTANCE_ID = "instance/id" """Attribute in metadata server for compute region and instance.""" -_GKE_CLUSTER_NAME = "instance/attributes/cluster-name" +GKE_CLUSTER_NAME = "instance/attributes/cluster-name" """Attribute in metadata server when in GKE environment.""" From d7ddd5b0ccd34392deaea0a7bfa6e1628912dfcd Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 26 Feb 2021 19:04:53 -0800 Subject: [PATCH 08/29] fixed env var names --- google/cloud/logging_v2/client.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/google/cloud/logging_v2/client.py b/google/cloud/logging_v2/client.py index 5180b0d6c..4298815a6 100644 --- a/google/cloud/logging_v2/client.py +++ b/google/cloud/logging_v2/client.py @@ -348,21 +348,21 @@ def get_default_handler(self, **kw): Returns: logging.Handler: The default log handler based on the environment """ - gke_cluster_name = retrieve_metadata_server(_GKE_CLUSTER_NAME) - gce_instance_name = retrieve_metadata_server(_GCE_INSTANCE_ID) + gke_cluster_name = retrieve_metadata_server(GKE_CLUSTER_NAME) + gce_instance_name = retrieve_metadata_server(GCE_INSTANCE_ID) resource = None - if all([env in os.environ for env in _GAE_ENV_VARS]): + if all([env in os.environ for env in GAE_ENV_VARS]): # App Engine Flex or Standard return AppEngineHandler(self, **kw) elif gke_cluster_name is not None: # Kubernetes Engine return ContainerEngineHandler(**kw) - elif (all([env in os.environ for env in _LEGACY_FUNCTION_ENV_VARS]) - or all([env in os.environ for env in _FUNCTION_ENV_VARS])): + elif (all([env in os.environ for env in LEGACY_FUNCTION_ENV_VARS]) + or all([env in os.environ for env in FUNCTION_ENV_VARS])): # Cloud Functions resource = create_functions_resource(self.project) - elif all([env in os.environ for env in _CLOUD_RUN_ENV_VARS]): + elif all([env in os.environ for env in CLOUD_RUN_ENV_VARS]): # Cloud Run resource = create_cloud_run_resource(self.project) elif instance is not None: From 2124310814e428b23546e27cff9ae2662935763c Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 26 Feb 2021 19:14:59 -0800 Subject: [PATCH 09/29] fixed variable name --- google/cloud/logging_v2/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google/cloud/logging_v2/client.py b/google/cloud/logging_v2/client.py index 4298815a6..8cc4a0077 100644 --- a/google/cloud/logging_v2/client.py +++ b/google/cloud/logging_v2/client.py @@ -365,7 +365,7 @@ def get_default_handler(self, **kw): elif all([env in os.environ for env in CLOUD_RUN_ENV_VARS]): # Cloud Run resource = create_cloud_run_resource(self.project) - elif instance is not None: + elif gce_instance_name is not None: # Compute Engine resource = create_compute_resource(self.project) else: From 32b639f7c8a6a6d52413a62eb6a68dcad3b7762e Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 26 Feb 2021 19:24:00 -0800 Subject: [PATCH 10/29] fixed variable name --- google/cloud/logging_v2/handlers/_monitored_resources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google/cloud/logging_v2/handlers/_monitored_resources.py b/google/cloud/logging_v2/handlers/_monitored_resources.py index 5b33729b5..6273b2f62 100644 --- a/google/cloud/logging_v2/handlers/_monitored_resources.py +++ b/google/cloud/logging_v2/handlers/_monitored_resources.py @@ -68,7 +68,7 @@ def create_functions_resource(project): return resource def create_compute_resource(project): - instance = retrieve_metadata_server(_GCE_INSTANCE_ID) + instance = retrieve_metadata_server(GCE_INSTANCE_ID) zone = retrieve_metadata_server(_ZONE_ID) resource = Resource( type="gce_instance", From 5ed2ab03e8bf41877c332cf3958d8f49f1d47cf8 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Mon, 1 Mar 2021 13:56:00 -0800 Subject: [PATCH 11/29] added zone to appengine results --- google/cloud/logging_v2/handlers/_monitored_resources.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/google/cloud/logging_v2/handlers/_monitored_resources.py b/google/cloud/logging_v2/handlers/_monitored_resources.py index 6273b2f62..c79860fcf 100644 --- a/google/cloud/logging_v2/handlers/_monitored_resources.py +++ b/google/cloud/logging_v2/handlers/_monitored_resources.py @@ -96,12 +96,14 @@ def create_cloud_run_resource(project): return resource def create_app_engine_resource(project): + zone = retrieve_metadata_server(_ZONE_ID) resource = Resource( type="gae_app", labels={ "project_id": project, "module_id": os.environ.get(_GAE_SERVICE_ENV, ""), "version_id": os.environ.get(_GAE_VERSION_ENV, ""), + "zone": zone if zone else "", }, ) return resource From 8723cfa233c49bb820369735c4f258fc06550734 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Mon, 1 Mar 2021 13:56:10 -0800 Subject: [PATCH 12/29] upgraded environment test repo --- tests/environment | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/environment b/tests/environment index 5cae58574..bf49d0612 160000 --- a/tests/environment +++ b/tests/environment @@ -1 +1 @@ -Subproject commit 5cae5857468fee7a24a6b30a70bca3f107a31379 +Subproject commit bf49d0612931d5f079600ddfb362991debe8e4a5 From f1bb7f7c1e7f004f6cbb351b014feaacc22e4bd0 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Mon, 1 Mar 2021 14:11:48 -0800 Subject: [PATCH 13/29] updated appengine resource creation --- google/cloud/logging_v2/handlers/app_engine.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/google/cloud/logging_v2/handlers/app_engine.py b/google/cloud/logging_v2/handlers/app_engine.py index a5d57c53e..ac4c389ec 100644 --- a/google/cloud/logging_v2/handlers/app_engine.py +++ b/google/cloud/logging_v2/handlers/app_engine.py @@ -22,6 +22,7 @@ import os from google.cloud.logging_v2.handlers._helpers import get_request_data +from google.cloud.logging_v2.handlers._monitored_resources import create_app_engine_resource from google.cloud.logging_v2.handlers.transports import BackgroundThreadTransport from google.cloud.logging_v2.resource import Resource @@ -75,15 +76,7 @@ def get_gae_resource(self): Returns: google.cloud.logging_v2.resource.Resource: Monitored resource for GAE. """ - gae_resource = Resource( - type="gae_app", - labels={ - "project_id": self.project_id, - "module_id": self.module_id, - "version_id": self.version_id, - }, - ) - return gae_resource + return create_app_engine_resource(self.project_id) def get_gae_labels(self): """Return the labels for GAE app. From 3a63d84e6ef0bd6b66539c392be37d50c598c747 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Mon, 1 Mar 2021 14:47:06 -0800 Subject: [PATCH 14/29] refactored, fixed unit tests --- google/cloud/logging_v2/client.py | 37 +++--------- .../handlers/_monitored_resources.py | 59 +++++++++++++++---- .../cloud/logging_v2/handlers/app_engine.py | 4 +- 3 files changed, 57 insertions(+), 43 deletions(-) diff --git a/google/cloud/logging_v2/client.py b/google/cloud/logging_v2/client.py index 8cc4a0077..e514f6769 100644 --- a/google/cloud/logging_v2/client.py +++ b/google/cloud/logging_v2/client.py @@ -29,7 +29,6 @@ from google.cloud.client import ClientWithProject from google.cloud.environment_vars import DISABLE_GRPC from google.cloud.logging_v2._helpers import _add_defaults_to_filter -from google.cloud.logging_v2._helpers import retrieve_metadata_server from google.cloud.logging_v2._http import Connection from google.cloud.logging_v2._http import _LoggingAPI as JSONLoggingAPI from google.cloud.logging_v2._http import _MetricsAPI as JSONMetricsAPI @@ -40,15 +39,8 @@ from google.cloud.logging_v2.handlers import setup_logging from google.cloud.logging_v2.handlers.handlers import EXCLUDED_LOGGER_DEFAULTS from google.cloud.logging_v2.resource import Resource -from google.cloud.logging_v2.handlers._monitored_resources import GAE_ENV_VARS -from google.cloud.logging_v2.handlers._monitored_resources import CLOUD_RUN_ENV_VARS -from google.cloud.logging_v2.handlers._monitored_resources import FUNCTION_ENV_VARS -from google.cloud.logging_v2.handlers._monitored_resources import LEGACY_FUNCTION_ENV_VARS -from google.cloud.logging_v2.handlers._monitored_resources import GKE_CLUSTER_NAME -from google.cloud.logging_v2.handlers._monitored_resources import GCE_INSTANCE_ID -from google.cloud.logging_v2.handlers._monitored_resources import create_functions_resource -from google.cloud.logging_v2.handlers._monitored_resources import create_compute_resource -from google.cloud.logging_v2.handlers._monitored_resources import create_cloud_run_resource +from google.cloud.logging_v2.handlers._monitored_resources import detect_resource + from google.cloud.logging_v2.logger import Logger from google.cloud.logging_v2.metric import Metric @@ -348,30 +340,15 @@ def get_default_handler(self, **kw): Returns: logging.Handler: The default log handler based on the environment """ - gke_cluster_name = retrieve_metadata_server(GKE_CLUSTER_NAME) - gce_instance_name = retrieve_metadata_server(GCE_INSTANCE_ID) - resource = None + monitored_resource = kw.pop("resource", detect_resource(self.project)) - if all([env in os.environ for env in GAE_ENV_VARS]): - # App Engine Flex or Standard + if isinstance(monitored_resource, Resource) and monitored_resource.type == "gae_app": return AppEngineHandler(self, **kw) - elif gke_cluster_name is not None: - # Kubernetes Engine + elif isinstance(monitored_resource, Resource) and monitored_resource.type == "k8s_container": return ContainerEngineHandler(**kw) - elif (all([env in os.environ for env in LEGACY_FUNCTION_ENV_VARS]) - or all([env in os.environ for env in FUNCTION_ENV_VARS])): - # Cloud Functions - resource = create_functions_resource(self.project) - elif all([env in os.environ for env in CLOUD_RUN_ENV_VARS]): - # Cloud Run - resource = create_cloud_run_resource(self.project) - elif gce_instance_name is not None: - # Compute Engine - resource = create_compute_resource(self.project) else: - # use generic global resource - resource = create_global_resource(self.project) - return CloudLoggingHandler(self, resource=resource, **kw) + return CloudLoggingHandler(self, resource=monitored_resource, **kw) + def setup_logging( self, *, log_level=logging.INFO, excluded_loggers=EXCLUDED_LOGGER_DEFAULTS, **kw diff --git a/google/cloud/logging_v2/handlers/_monitored_resources.py b/google/cloud/logging_v2/handlers/_monitored_resources.py index c79860fcf..de9961d9e 100644 --- a/google/cloud/logging_v2/handlers/_monitored_resources.py +++ b/google/cloud/logging_v2/handlers/_monitored_resources.py @@ -21,13 +21,13 @@ _GAE_SERVICE_ENV = "GAE_SERVICE" _GAE_VERSION_ENV = "GAE_VERSION" _GAE_INSTANCE_ENV = "GAE_INSTANCE" -GAE_ENV_VARS = [_GAE_SERVICE_ENV, _GAE_VERSION_ENV, _GAE_INSTANCE_ENV] +_GAE_ENV_VARS = [_GAE_SERVICE_ENV, _GAE_VERSION_ENV, _GAE_INSTANCE_ENV] """Environment variables set in App Engine environment.""" _CLOUD_RUN_SERVICE_ID = "K_SERVICE" _CLOUD_RUN_REVISION_ID = "K_REVISION" _CLOUD_RUN_CONFIGURATION_ID = "K_CONFIGURATION" -CLOUD_RUN_ENV_VARS = [_CLOUD_RUN_SERVICE_ID, _CLOUD_RUN_REVISION_ID, _CLOUD_RUN_CONFIGURATION_ID] +_CLOUD_RUN_ENV_VARS = [_CLOUD_RUN_SERVICE_ID, _CLOUD_RUN_REVISION_ID, _CLOUD_RUN_CONFIGURATION_ID] """Environment variables set in Cloud Run environment.""" _FUNCTION_TARGET = "FUNCTION_TARGET" @@ -35,21 +35,21 @@ _FUNCTION_NAME = "FUNCTION_NAME" _FUNCTION_REGION = "FUNCTION_REGION" _FUNCTION_ENTRY = "ENTRY_POINT" -FUNCTION_ENV_VARS = [_FUNCTION_TARGET, _FUNCTION_SIGNATURE, _CLOUD_RUN_SERVICE_ID] -LEGACY_FUNCTION_ENV_VARS = [_FUNCTION_NAME, _FUNCTION_REGION, _FUNCTION_ENTRY] +_FUNCTION_ENV_VARS = [_FUNCTION_TARGET, _FUNCTION_SIGNATURE, _CLOUD_RUN_SERVICE_ID] +_LEGACY_FUNCTION_ENV_VARS = [_FUNCTION_NAME, _FUNCTION_REGION, _FUNCTION_ENTRY] """Environment variables set in Cloud Functions environments.""" _REGION_ID = "instance/region" _ZONE_ID = "instance/zone" -GCE_INSTANCE_ID = "instance/id" +_GCE_INSTANCE_ID = "instance/id" """Attribute in metadata server for compute region and instance.""" -GKE_CLUSTER_NAME = "instance/attributes/cluster-name" +_GKE_CLUSTER_NAME = "instance/attributes/cluster-name" """Attribute in metadata server when in GKE environment.""" -def create_functions_resource(project): +def _create_functions_resource(project): region = retrieve_metadata_server(_REGION_ID) if _FUNCTION_NAME in os.environ: function_name = os.environ.get(_FUNCTION_NAME) @@ -67,7 +67,19 @@ def create_functions_resource(project): ) return resource -def create_compute_resource(project): +def _create_kubernetes_resource(project): + zone = retrieve_metadata_server(_ZONE_ID) + + resource = Resource( + type="k8s_container", + labels={ + "project_id": project, + "location": zone if zone else "", + }, + ) + return resource + +def _create_compute_resource(project): instance = retrieve_metadata_server(GCE_INSTANCE_ID) zone = retrieve_metadata_server(_ZONE_ID) resource = Resource( @@ -81,7 +93,7 @@ def create_compute_resource(project): return resource -def create_cloud_run_resource(project): +def _create_cloud_run_resource(project): region = retrieve_metadata_server(_REGION_ID) resource = Resource( type="cloud_run_revision", @@ -95,7 +107,7 @@ def create_cloud_run_resource(project): ) return resource -def create_app_engine_resource(project): +def _create_app_engine_resource(project): zone = retrieve_metadata_server(_ZONE_ID) resource = Resource( type="gae_app", @@ -108,7 +120,7 @@ def create_app_engine_resource(project): ) return resource -def create_global_resource(project): +def _create_global_resource(project): resource = Resource( type="global", labels={ @@ -116,3 +128,28 @@ def create_global_resource(project): }, ) return resource + +def detect_resource(project): + gke_cluster_name = retrieve_metadata_server(_GKE_CLUSTER_NAME) + gce_instance_name = retrieve_metadata_server(_GCE_INSTANCE_ID) + + if all([env in os.environ for env in _GAE_ENV_VARS]): + # App Engine Flex or Standard + return _create_app_engine_resource(project) + elif gke_cluster_name is not None: + # Kubernetes Engine + return _create_kubernetes_resource(project) + elif (all([env in os.environ for env in _LEGACY_FUNCTION_ENV_VARS]) + or all([env in os.environ for env in _FUNCTION_ENV_VARS])): + # Cloud Functions + resource = _create_functions_resource(project) + elif all([env in os.environ for env in _CLOUD_RUN_ENV_VARS]): + # Cloud Run + resource = _create_cloud_run_resource(project) + elif gce_instance_name is not None: + # Compute Engine + resource = _create_compute_resource(project) + else: + # use generic global resource + resource = _create_global_resource(project) + diff --git a/google/cloud/logging_v2/handlers/app_engine.py b/google/cloud/logging_v2/handlers/app_engine.py index ac4c389ec..20790149f 100644 --- a/google/cloud/logging_v2/handlers/app_engine.py +++ b/google/cloud/logging_v2/handlers/app_engine.py @@ -22,7 +22,7 @@ import os from google.cloud.logging_v2.handlers._helpers import get_request_data -from google.cloud.logging_v2.handlers._monitored_resources import create_app_engine_resource +from google.cloud.logging_v2.handlers._monitored_resources import _create_app_engine_resource from google.cloud.logging_v2.handlers.transports import BackgroundThreadTransport from google.cloud.logging_v2.resource import Resource @@ -76,7 +76,7 @@ def get_gae_resource(self): Returns: google.cloud.logging_v2.resource.Resource: Monitored resource for GAE. """ - return create_app_engine_resource(self.project_id) + return _create_app_engine_resource(self.project_id) def get_gae_labels(self): """Return the labels for GAE app. From 3436d06b5db06d581dbc1ba003cf0a36e6d3667b Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Mon, 1 Mar 2021 14:48:22 -0800 Subject: [PATCH 15/29] ran blacken --- google/cloud/logging_v2/client.py | 12 +++- .../handlers/_monitored_resources.py | 69 ++++++++++--------- .../cloud/logging_v2/handlers/app_engine.py | 4 +- tests/unit/test_client.py | 8 ++- 4 files changed, 52 insertions(+), 41 deletions(-) diff --git a/google/cloud/logging_v2/client.py b/google/cloud/logging_v2/client.py index e514f6769..3d696f750 100644 --- a/google/cloud/logging_v2/client.py +++ b/google/cloud/logging_v2/client.py @@ -50,6 +50,7 @@ _DISABLE_GRPC = os.getenv(DISABLE_GRPC, False) _USE_GRPC = _HAVE_GRPC and not _DISABLE_GRPC + class Client(ClientWithProject): """Client to bundle configuration needed for API requests.""" @@ -342,14 +343,19 @@ def get_default_handler(self, **kw): """ monitored_resource = kw.pop("resource", detect_resource(self.project)) - if isinstance(monitored_resource, Resource) and monitored_resource.type == "gae_app": + if ( + isinstance(monitored_resource, Resource) + and monitored_resource.type == "gae_app" + ): return AppEngineHandler(self, **kw) - elif isinstance(monitored_resource, Resource) and monitored_resource.type == "k8s_container": + elif ( + isinstance(monitored_resource, Resource) + and monitored_resource.type == "k8s_container" + ): return ContainerEngineHandler(**kw) else: return CloudLoggingHandler(self, resource=monitored_resource, **kw) - def setup_logging( self, *, log_level=logging.INFO, excluded_loggers=EXCLUDED_LOGGER_DEFAULTS, **kw ): diff --git a/google/cloud/logging_v2/handlers/_monitored_resources.py b/google/cloud/logging_v2/handlers/_monitored_resources.py index de9961d9e..6917fec3c 100644 --- a/google/cloud/logging_v2/handlers/_monitored_resources.py +++ b/google/cloud/logging_v2/handlers/_monitored_resources.py @@ -27,7 +27,11 @@ _CLOUD_RUN_SERVICE_ID = "K_SERVICE" _CLOUD_RUN_REVISION_ID = "K_REVISION" _CLOUD_RUN_CONFIGURATION_ID = "K_CONFIGURATION" -_CLOUD_RUN_ENV_VARS = [_CLOUD_RUN_SERVICE_ID, _CLOUD_RUN_REVISION_ID, _CLOUD_RUN_CONFIGURATION_ID] +_CLOUD_RUN_ENV_VARS = [ + _CLOUD_RUN_SERVICE_ID, + _CLOUD_RUN_REVISION_ID, + _CLOUD_RUN_CONFIGURATION_ID, +] """Environment variables set in Cloud Run environment.""" _FUNCTION_TARGET = "FUNCTION_TARGET" @@ -67,18 +71,17 @@ def _create_functions_resource(project): ) return resource + def _create_kubernetes_resource(project): zone = retrieve_metadata_server(_ZONE_ID) resource = Resource( type="k8s_container", - labels={ - "project_id": project, - "location": zone if zone else "", - }, + labels={"project_id": project, "location": zone if zone else "",}, ) return resource + def _create_compute_resource(project): instance = retrieve_metadata_server(GCE_INSTANCE_ID) zone = retrieve_metadata_server(_ZONE_ID) @@ -107,6 +110,7 @@ def _create_cloud_run_resource(project): ) return resource + def _create_app_engine_resource(project): zone = retrieve_metadata_server(_ZONE_ID) resource = Resource( @@ -120,36 +124,33 @@ def _create_app_engine_resource(project): ) return resource + def _create_global_resource(project): - resource = Resource( - type="global", - labels={ - "project_id": project, - }, - ) + resource = Resource(type="global", labels={"project_id": project,},) return resource -def detect_resource(project): - gke_cluster_name = retrieve_metadata_server(_GKE_CLUSTER_NAME) - gce_instance_name = retrieve_metadata_server(_GCE_INSTANCE_ID) - - if all([env in os.environ for env in _GAE_ENV_VARS]): - # App Engine Flex or Standard - return _create_app_engine_resource(project) - elif gke_cluster_name is not None: - # Kubernetes Engine - return _create_kubernetes_resource(project) - elif (all([env in os.environ for env in _LEGACY_FUNCTION_ENV_VARS]) - or all([env in os.environ for env in _FUNCTION_ENV_VARS])): - # Cloud Functions - resource = _create_functions_resource(project) - elif all([env in os.environ for env in _CLOUD_RUN_ENV_VARS]): - # Cloud Run - resource = _create_cloud_run_resource(project) - elif gce_instance_name is not None: - # Compute Engine - resource = _create_compute_resource(project) - else: - # use generic global resource - resource = _create_global_resource(project) +def detect_resource(project): + gke_cluster_name = retrieve_metadata_server(_GKE_CLUSTER_NAME) + gce_instance_name = retrieve_metadata_server(_GCE_INSTANCE_ID) + + if all([env in os.environ for env in _GAE_ENV_VARS]): + # App Engine Flex or Standard + return _create_app_engine_resource(project) + elif gke_cluster_name is not None: + # Kubernetes Engine + return _create_kubernetes_resource(project) + elif all([env in os.environ for env in _LEGACY_FUNCTION_ENV_VARS]) or all( + [env in os.environ for env in _FUNCTION_ENV_VARS] + ): + # Cloud Functions + resource = _create_functions_resource(project) + elif all([env in os.environ for env in _CLOUD_RUN_ENV_VARS]): + # Cloud Run + resource = _create_cloud_run_resource(project) + elif gce_instance_name is not None: + # Compute Engine + resource = _create_compute_resource(project) + else: + # use generic global resource + resource = _create_global_resource(project) diff --git a/google/cloud/logging_v2/handlers/app_engine.py b/google/cloud/logging_v2/handlers/app_engine.py index 20790149f..8cfbc448c 100644 --- a/google/cloud/logging_v2/handlers/app_engine.py +++ b/google/cloud/logging_v2/handlers/app_engine.py @@ -22,7 +22,9 @@ import os from google.cloud.logging_v2.handlers._helpers import get_request_data -from google.cloud.logging_v2.handlers._monitored_resources import _create_app_engine_resource +from google.cloud.logging_v2.handlers._monitored_resources import ( + _create_app_engine_resource, +) from google.cloud.logging_v2.handlers.transports import BackgroundThreadTransport from google.cloud.logging_v2.resource import Resource diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index 8083e3c56..f33f1cbdc 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -718,7 +718,7 @@ def test_list_metrics_with_paging(self): def test_get_default_handler_app_engine(self): import os from google.cloud._testing import _Monkey - from google.cloud.logging_v2.client import _APPENGINE_FLEXIBLE_ENV_VM + from google.cloud.logging_v2.handlers._monitored_resources import _GAE_ENV_VARS from google.cloud.logging.handlers import AppEngineHandler credentials = _make_credentials() @@ -726,7 +726,9 @@ def test_get_default_handler_app_engine(self): project=self.PROJECT, credentials=credentials, _use_grpc=False ) - with _Monkey(os, environ={_APPENGINE_FLEXIBLE_ENV_VM: "True"}): + gae_env_vars = {var: "TRUE" for var in _GAE_ENV_VARS} + + with _Monkey(os, environ=gae_env_vars): handler = client.get_default_handler() handler.transport.worker.stop() @@ -742,7 +744,7 @@ def test_get_default_handler_container_engine(self): ) patch = mock.patch( - "google.cloud.logging_v2.client.retrieve_metadata_server", + "google.cloud.logging_v2.handlers._monitored_resources.retrieve_metadata_server", return_value="test-gke-cluster", ) From 91aabaad67c644d69607c79842e24c4f733f99cb Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Mon, 1 Mar 2021 14:49:08 -0800 Subject: [PATCH 16/29] blacken env tests --- tests/environment | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/environment b/tests/environment index bf49d0612..d7fd94da8 160000 --- a/tests/environment +++ b/tests/environment @@ -1 +1 @@ -Subproject commit bf49d0612931d5f079600ddfb362991debe8e4a5 +Subproject commit d7fd94da8b18af8db6bd3749fb02f76ee6a03267 From b4828e865368ff2c3db98eb2b74e59ebf5e97756 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Mon, 1 Mar 2021 14:59:58 -0800 Subject: [PATCH 17/29] fixed incorrect variable name --- google/cloud/logging_v2/handlers/_monitored_resources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google/cloud/logging_v2/handlers/_monitored_resources.py b/google/cloud/logging_v2/handlers/_monitored_resources.py index 6917fec3c..757eabc2e 100644 --- a/google/cloud/logging_v2/handlers/_monitored_resources.py +++ b/google/cloud/logging_v2/handlers/_monitored_resources.py @@ -83,7 +83,7 @@ def _create_kubernetes_resource(project): def _create_compute_resource(project): - instance = retrieve_metadata_server(GCE_INSTANCE_ID) + instance = retrieve_metadata_server(_GCE_INSTANCE_ID) zone = retrieve_metadata_server(_ZONE_ID) resource = Resource( type="gce_instance", From 310b72900621fe48e1b547e80504a597317e157e Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Mon, 1 Mar 2021 15:16:16 -0800 Subject: [PATCH 18/29] fixed lint issues --- .../handlers/_monitored_resources.py | 19 +++++++++++-------- .../cloud/logging_v2/handlers/app_engine.py | 1 - 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/google/cloud/logging_v2/handlers/_monitored_resources.py b/google/cloud/logging_v2/handlers/_monitored_resources.py index 757eabc2e..d5d5d33a1 100644 --- a/google/cloud/logging_v2/handlers/_monitored_resources.py +++ b/google/cloud/logging_v2/handlers/_monitored_resources.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging import os from google.cloud.logging_v2.resource import Resource @@ -74,10 +73,15 @@ def _create_functions_resource(project): def _create_kubernetes_resource(project): zone = retrieve_metadata_server(_ZONE_ID) + cluster_name = retrieve_metadata_server(_GKE_CLUSTER_NAME) resource = Resource( type="k8s_container", - labels={"project_id": project, "location": zone if zone else "",}, + labels={ + "project_id": project, + "location": zone if zone else "", + "cluster_name": cluster_name if cluster_name else "", + }, ) return resource @@ -126,8 +130,7 @@ def _create_app_engine_resource(project): def _create_global_resource(project): - resource = Resource(type="global", labels={"project_id": project,},) - return resource + return Resource(type="global", labels={"project_id": project}) def detect_resource(project): @@ -144,13 +147,13 @@ def detect_resource(project): [env in os.environ for env in _FUNCTION_ENV_VARS] ): # Cloud Functions - resource = _create_functions_resource(project) + return _create_functions_resource(project) elif all([env in os.environ for env in _CLOUD_RUN_ENV_VARS]): # Cloud Run - resource = _create_cloud_run_resource(project) + return _create_cloud_run_resource(project) elif gce_instance_name is not None: # Compute Engine - resource = _create_compute_resource(project) + return _create_compute_resource(project) else: # use generic global resource - resource = _create_global_resource(project) + return _create_global_resource(project) diff --git a/google/cloud/logging_v2/handlers/app_engine.py b/google/cloud/logging_v2/handlers/app_engine.py index 8cfbc448c..7d16ab07a 100644 --- a/google/cloud/logging_v2/handlers/app_engine.py +++ b/google/cloud/logging_v2/handlers/app_engine.py @@ -26,7 +26,6 @@ _create_app_engine_resource, ) from google.cloud.logging_v2.handlers.transports import BackgroundThreadTransport -from google.cloud.logging_v2.resource import Resource _DEFAULT_GAE_LOGGER_NAME = "app" From 3507bc6e85f4883ba164aca992c13258fa3a91ee Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Mon, 1 Mar 2021 16:47:58 -0800 Subject: [PATCH 19/29] added unit tests for _monitored_resources --- .../handlers/test__monitored_resources.py | 222 ++++++++++++++++++ 1 file changed, 222 insertions(+) create mode 100644 tests/unit/handlers/test__monitored_resources.py diff --git a/tests/unit/handlers/test__monitored_resources.py b/tests/unit/handlers/test__monitored_resources.py new file mode 100644 index 000000000..a7f1e39f1 --- /dev/null +++ b/tests/unit/handlers/test__monitored_resources.py @@ -0,0 +1,222 @@ +# Copyright 2021 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import mock +import os + + +from google.cloud.logging_v2.handlers._monitored_resources import _create_functions_resource +from google.cloud.logging_v2.handlers._monitored_resources import _create_app_engine_resource +from google.cloud.logging_v2.handlers._monitored_resources import _create_kubernetes_resource +from google.cloud.logging_v2.handlers._monitored_resources import _create_cloud_run_resource +from google.cloud.logging_v2.handlers._monitored_resources import _create_compute_resource +from google.cloud.logging_v2.handlers._monitored_resources import _create_global_resource +from google.cloud.logging_v2.handlers._monitored_resources import detect_resource +from google.cloud.logging_v2.handlers import _monitored_resources +from google.cloud.logging_v2.resource import Resource + +class Test_Create_Resources(unittest.TestCase): + + PROJECT = 'test-project' + LOCATION = 'test-location' + NAME = 'test-name' + CLUSTER = 'test-cluster' + VERSION = '1' + CONFIG = 'test-config' + + def _mock_metadata(self, endpoint): + if (endpoint == _monitored_resources._ZONE_ID + or endpoint == _monitored_resources._REGION_ID): + return self.LOCATION + elif (endpoint == _monitored_resources._GKE_CLUSTER_NAME + or endpoint == _monitored_resources._GCE_INSTANCE_ID): + return self.NAME + else: + return None + + def test_create_legacy_functions_resource(self): + patch = mock.patch( + "google.cloud.logging_v2.handlers._monitored_resources.retrieve_metadata_server", + wraps=self._mock_metadata, + ) + + os.environ[_monitored_resources._CLOUD_RUN_SERVICE_ID] = self.NAME + with patch: + legacy_func_resource = _create_functions_resource(self.PROJECT) + + self.assertIsInstance(legacy_func_resource, Resource) + self.assertEqual(legacy_func_resource.type, 'cloud_function') + self.assertEqual(legacy_func_resource.labels['project_id'], self.PROJECT) + self.assertEqual(legacy_func_resource.labels['function_name'], self.NAME) + self.assertEqual(legacy_func_resource.labels['region'], self.LOCATION) + + def test_create_modern_functions_resource(self): + patch = mock.patch( + "google.cloud.logging_v2.handlers._monitored_resources.retrieve_metadata_server", + wraps=self._mock_metadata, + ) + os.environ[_monitored_resources._FUNCTION_NAME] = self.NAME + with patch: + func_resource = _create_functions_resource(self.PROJECT) + + self.assertIsInstance(func_resource, Resource) + self.assertEqual(func_resource.type, 'cloud_function') + self.assertEqual(func_resource.labels['project_id'], self.PROJECT) + self.assertEqual(func_resource.labels['function_name'], self.NAME) + self.assertEqual(func_resource.labels['region'], self.LOCATION) + + def test_create_kubernetes_resource(self): + + patch = mock.patch( + "google.cloud.logging_v2.handlers._monitored_resources.retrieve_metadata_server", + wraps=self._mock_metadata, + ) + with patch: + resource = _create_kubernetes_resource(self.PROJECT) + + self.assertIsInstance(resource, Resource) + self.assertEqual(resource.type, 'k8s_container') + self.assertEqual(resource.labels['project_id'], self.PROJECT) + self.assertEqual(resource.labels['cluster_name'], self.NAME) + self.assertEqual(resource.labels['location'], self.LOCATION) + + def test_compute_resource(self): + patch = mock.patch( + "google.cloud.logging_v2.handlers._monitored_resources.retrieve_metadata_server", + wraps=self._mock_metadata, + ) + + with patch: + resource = _create_compute_resource(self.PROJECT) + self.assertIsInstance(resource, Resource) + self.assertEqual(resource.type, 'gce_instance') + self.assertEqual(resource.labels['project_id'], self.PROJECT) + self.assertEqual(resource.labels['instance_id'], self.NAME) + self.assertEqual(resource.labels['zone'], self.LOCATION) + + def test_cloud_run_resource(self): + patch = mock.patch( + "google.cloud.logging_v2.handlers._monitored_resources.retrieve_metadata_server", + wraps=self._mock_metadata, + ) + os.environ[_monitored_resources._CLOUD_RUN_SERVICE_ID] = self.NAME + os.environ[_monitored_resources._CLOUD_RUN_REVISION_ID] = self.VERSION + os.environ[_monitored_resources._CLOUD_RUN_CONFIGURATION_ID] = self.CONFIG + with patch: + resource = _create_cloud_run_resource(self.PROJECT) + self.assertIsInstance(resource, Resource) + self.assertEqual(resource.type, 'cloud_run_revision') + self.assertEqual(resource.labels['project_id'], self.PROJECT) + self.assertEqual(resource.labels['service_name'], self.NAME) + self.assertEqual(resource.labels['revision_name'], self.VERSION) + self.assertEqual(resource.labels['configuration_name'], self.CONFIG) + self.assertEqual(resource.labels['location'], self.LOCATION) + + def test_app_engine_resource(self): + patch = mock.patch( + "google.cloud.logging_v2.handlers._monitored_resources.retrieve_metadata_server", + wraps=self._mock_metadata, + ) + os.environ[_monitored_resources._GAE_SERVICE_ENV] = self.NAME + os.environ[_monitored_resources._GAE_VERSION_ENV] = self.VERSION + with patch: + resource = _create_app_engine_resource(self.PROJECT) + self.assertIsInstance(resource, Resource) + self.assertEqual(resource.type, 'gae_app') + self.assertEqual(resource.labels['project_id'], self.PROJECT) + self.assertEqual(resource.labels['module_id'], self.NAME) + self.assertEqual(resource.labels['version_id'], self.VERSION) + self.assertEqual(resource.labels['zone'], self.LOCATION) + + def test_global_resource(self): + resource = _create_global_resource(self.PROJECT) + self.assertIsInstance(resource, Resource) + self.assertEqual(resource.type, 'global') + self.assertEqual(resource.labels['project_id'], self.PROJECT) + +class Test_Resource_Detection(unittest.TestCase): + + PROJECT = 'test-project' + + def _mock_k8s_metadata(self, endpoint): + if (endpoint == _monitored_resources._GKE_CLUSTER_NAME + or endpoint == _monitored_resources._GCE_INSTANCE_ID): + return "TRUE" + else: + return None + + def _mock_gce_metadata(self, endpoint): + if endpoint == _monitored_resources._GCE_INSTANCE_ID: + return "TRUE" + else: + return None + + def setUp(self): + os.environ.clear() + + def test_detect_appengine(self): + for env in _monitored_resources._GAE_ENV_VARS: + os.environ[env] = 'TRUE' + resource = detect_resource(self.PROJECT) + self.assertIsInstance(resource, Resource) + self.assertEqual(resource.type, 'gae_app') + + def test_detect_kubernetes(self): + patch = mock.patch( + "google.cloud.logging_v2.handlers._monitored_resources.retrieve_metadata_server", + wraps=self._mock_k8s_metadata, + ) + with patch: + resource = detect_resource(self.PROJECT) + self.assertIsInstance(resource, Resource) + self.assertEqual(resource.type, 'k8s_container') + + def test_detect_functions(self): + for env in _monitored_resources._FUNCTION_ENV_VARS: + os.environ[env] = 'TRUE' + resource = detect_resource(self.PROJECT) + self.assertIsInstance(resource, Resource) + self.assertEqual(resource.type, 'cloud_function') + + def test_detect_legacy_functions(self): + for env in _monitored_resources._LEGACY_FUNCTION_ENV_VARS: + os.environ[env] = 'TRUE' + resource = detect_resource(self.PROJECT) + self.assertIsInstance(resource, Resource) + self.assertEqual(resource.type, 'cloud_function') + + def test_detect_cloud_run(self): + for env in _monitored_resources._CLOUD_RUN_ENV_VARS: + os.environ[env] = 'TRUE' + resource = detect_resource(self.PROJECT) + self.assertIsInstance(resource, Resource) + self.assertEqual(resource.type, 'cloud_run_revision') + + def test_detect_compute_engine(self): + patch = mock.patch( + "google.cloud.logging_v2.handlers._monitored_resources.retrieve_metadata_server", + wraps=self._mock_gce_metadata, + ) + with patch: + resource = detect_resource(self.PROJECT) + self.assertIsInstance(resource, Resource) + self.assertEqual(resource.type, 'gce_instance') + + def test_detection_unknown(self): + resource = detect_resource(self.PROJECT) + self.assertIsInstance(resource, Resource) + self.assertEqual(resource.type, 'global') + From 7448cf6e667a1f60274e24c88d7561ec211be7b3 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Mon, 1 Mar 2021 16:52:59 -0800 Subject: [PATCH 20/29] added unit tests --- tests/environment | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/environment b/tests/environment index d7fd94da8..a10a6e95e 160000 --- a/tests/environment +++ b/tests/environment @@ -1 +1 @@ -Subproject commit d7fd94da8b18af8db6bd3749fb02f76ee6a03267 +Subproject commit a10a6e95eb10ce50962b9d1c2f8a9c0790bff469 From bd90217cf9ab8a037dd48e1c93da54a36cca5bba Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Mon, 1 Mar 2021 17:05:21 -0800 Subject: [PATCH 21/29] blacken tests --- .../handlers/test__monitored_resources.py | 141 ++++++++++-------- 1 file changed, 80 insertions(+), 61 deletions(-) diff --git a/tests/unit/handlers/test__monitored_resources.py b/tests/unit/handlers/test__monitored_resources.py index a7f1e39f1..0e8a75233 100644 --- a/tests/unit/handlers/test__monitored_resources.py +++ b/tests/unit/handlers/test__monitored_resources.py @@ -18,31 +18,48 @@ import os -from google.cloud.logging_v2.handlers._monitored_resources import _create_functions_resource -from google.cloud.logging_v2.handlers._monitored_resources import _create_app_engine_resource -from google.cloud.logging_v2.handlers._monitored_resources import _create_kubernetes_resource -from google.cloud.logging_v2.handlers._monitored_resources import _create_cloud_run_resource -from google.cloud.logging_v2.handlers._monitored_resources import _create_compute_resource -from google.cloud.logging_v2.handlers._monitored_resources import _create_global_resource +from google.cloud.logging_v2.handlers._monitored_resources import ( + _create_functions_resource, +) +from google.cloud.logging_v2.handlers._monitored_resources import ( + _create_app_engine_resource, +) +from google.cloud.logging_v2.handlers._monitored_resources import ( + _create_kubernetes_resource, +) +from google.cloud.logging_v2.handlers._monitored_resources import ( + _create_cloud_run_resource, +) +from google.cloud.logging_v2.handlers._monitored_resources import ( + _create_compute_resource, +) +from google.cloud.logging_v2.handlers._monitored_resources import ( + _create_global_resource, +) from google.cloud.logging_v2.handlers._monitored_resources import detect_resource from google.cloud.logging_v2.handlers import _monitored_resources from google.cloud.logging_v2.resource import Resource + class Test_Create_Resources(unittest.TestCase): - PROJECT = 'test-project' - LOCATION = 'test-location' - NAME = 'test-name' - CLUSTER = 'test-cluster' - VERSION = '1' - CONFIG = 'test-config' + PROJECT = "test-project" + LOCATION = "test-location" + NAME = "test-name" + CLUSTER = "test-cluster" + VERSION = "1" + CONFIG = "test-config" def _mock_metadata(self, endpoint): - if (endpoint == _monitored_resources._ZONE_ID - or endpoint == _monitored_resources._REGION_ID): + if ( + endpoint == _monitored_resources._ZONE_ID + or endpoint == _monitored_resources._REGION_ID + ): return self.LOCATION - elif (endpoint == _monitored_resources._GKE_CLUSTER_NAME - or endpoint == _monitored_resources._GCE_INSTANCE_ID): + elif ( + endpoint == _monitored_resources._GKE_CLUSTER_NAME + or endpoint == _monitored_resources._GCE_INSTANCE_ID + ): return self.NAME else: return None @@ -58,10 +75,10 @@ def test_create_legacy_functions_resource(self): legacy_func_resource = _create_functions_resource(self.PROJECT) self.assertIsInstance(legacy_func_resource, Resource) - self.assertEqual(legacy_func_resource.type, 'cloud_function') - self.assertEqual(legacy_func_resource.labels['project_id'], self.PROJECT) - self.assertEqual(legacy_func_resource.labels['function_name'], self.NAME) - self.assertEqual(legacy_func_resource.labels['region'], self.LOCATION) + self.assertEqual(legacy_func_resource.type, "cloud_function") + self.assertEqual(legacy_func_resource.labels["project_id"], self.PROJECT) + self.assertEqual(legacy_func_resource.labels["function_name"], self.NAME) + self.assertEqual(legacy_func_resource.labels["region"], self.LOCATION) def test_create_modern_functions_resource(self): patch = mock.patch( @@ -73,10 +90,10 @@ def test_create_modern_functions_resource(self): func_resource = _create_functions_resource(self.PROJECT) self.assertIsInstance(func_resource, Resource) - self.assertEqual(func_resource.type, 'cloud_function') - self.assertEqual(func_resource.labels['project_id'], self.PROJECT) - self.assertEqual(func_resource.labels['function_name'], self.NAME) - self.assertEqual(func_resource.labels['region'], self.LOCATION) + self.assertEqual(func_resource.type, "cloud_function") + self.assertEqual(func_resource.labels["project_id"], self.PROJECT) + self.assertEqual(func_resource.labels["function_name"], self.NAME) + self.assertEqual(func_resource.labels["region"], self.LOCATION) def test_create_kubernetes_resource(self): @@ -88,10 +105,10 @@ def test_create_kubernetes_resource(self): resource = _create_kubernetes_resource(self.PROJECT) self.assertIsInstance(resource, Resource) - self.assertEqual(resource.type, 'k8s_container') - self.assertEqual(resource.labels['project_id'], self.PROJECT) - self.assertEqual(resource.labels['cluster_name'], self.NAME) - self.assertEqual(resource.labels['location'], self.LOCATION) + self.assertEqual(resource.type, "k8s_container") + self.assertEqual(resource.labels["project_id"], self.PROJECT) + self.assertEqual(resource.labels["cluster_name"], self.NAME) + self.assertEqual(resource.labels["location"], self.LOCATION) def test_compute_resource(self): patch = mock.patch( @@ -102,10 +119,10 @@ def test_compute_resource(self): with patch: resource = _create_compute_resource(self.PROJECT) self.assertIsInstance(resource, Resource) - self.assertEqual(resource.type, 'gce_instance') - self.assertEqual(resource.labels['project_id'], self.PROJECT) - self.assertEqual(resource.labels['instance_id'], self.NAME) - self.assertEqual(resource.labels['zone'], self.LOCATION) + self.assertEqual(resource.type, "gce_instance") + self.assertEqual(resource.labels["project_id"], self.PROJECT) + self.assertEqual(resource.labels["instance_id"], self.NAME) + self.assertEqual(resource.labels["zone"], self.LOCATION) def test_cloud_run_resource(self): patch = mock.patch( @@ -118,12 +135,12 @@ def test_cloud_run_resource(self): with patch: resource = _create_cloud_run_resource(self.PROJECT) self.assertIsInstance(resource, Resource) - self.assertEqual(resource.type, 'cloud_run_revision') - self.assertEqual(resource.labels['project_id'], self.PROJECT) - self.assertEqual(resource.labels['service_name'], self.NAME) - self.assertEqual(resource.labels['revision_name'], self.VERSION) - self.assertEqual(resource.labels['configuration_name'], self.CONFIG) - self.assertEqual(resource.labels['location'], self.LOCATION) + self.assertEqual(resource.type, "cloud_run_revision") + self.assertEqual(resource.labels["project_id"], self.PROJECT) + self.assertEqual(resource.labels["service_name"], self.NAME) + self.assertEqual(resource.labels["revision_name"], self.VERSION) + self.assertEqual(resource.labels["configuration_name"], self.CONFIG) + self.assertEqual(resource.labels["location"], self.LOCATION) def test_app_engine_resource(self): patch = mock.patch( @@ -135,31 +152,34 @@ def test_app_engine_resource(self): with patch: resource = _create_app_engine_resource(self.PROJECT) self.assertIsInstance(resource, Resource) - self.assertEqual(resource.type, 'gae_app') - self.assertEqual(resource.labels['project_id'], self.PROJECT) - self.assertEqual(resource.labels['module_id'], self.NAME) - self.assertEqual(resource.labels['version_id'], self.VERSION) - self.assertEqual(resource.labels['zone'], self.LOCATION) + self.assertEqual(resource.type, "gae_app") + self.assertEqual(resource.labels["project_id"], self.PROJECT) + self.assertEqual(resource.labels["module_id"], self.NAME) + self.assertEqual(resource.labels["version_id"], self.VERSION) + self.assertEqual(resource.labels["zone"], self.LOCATION) def test_global_resource(self): resource = _create_global_resource(self.PROJECT) self.assertIsInstance(resource, Resource) - self.assertEqual(resource.type, 'global') - self.assertEqual(resource.labels['project_id'], self.PROJECT) + self.assertEqual(resource.type, "global") + self.assertEqual(resource.labels["project_id"], self.PROJECT) + class Test_Resource_Detection(unittest.TestCase): - PROJECT = 'test-project' + PROJECT = "test-project" def _mock_k8s_metadata(self, endpoint): - if (endpoint == _monitored_resources._GKE_CLUSTER_NAME - or endpoint == _monitored_resources._GCE_INSTANCE_ID): + if ( + endpoint == _monitored_resources._GKE_CLUSTER_NAME + or endpoint == _monitored_resources._GCE_INSTANCE_ID + ): return "TRUE" else: return None def _mock_gce_metadata(self, endpoint): - if endpoint == _monitored_resources._GCE_INSTANCE_ID: + if endpoint == _monitored_resources._GCE_INSTANCE_ID: return "TRUE" else: return None @@ -169,10 +189,10 @@ def setUp(self): def test_detect_appengine(self): for env in _monitored_resources._GAE_ENV_VARS: - os.environ[env] = 'TRUE' + os.environ[env] = "TRUE" resource = detect_resource(self.PROJECT) self.assertIsInstance(resource, Resource) - self.assertEqual(resource.type, 'gae_app') + self.assertEqual(resource.type, "gae_app") def test_detect_kubernetes(self): patch = mock.patch( @@ -182,28 +202,28 @@ def test_detect_kubernetes(self): with patch: resource = detect_resource(self.PROJECT) self.assertIsInstance(resource, Resource) - self.assertEqual(resource.type, 'k8s_container') + self.assertEqual(resource.type, "k8s_container") def test_detect_functions(self): for env in _monitored_resources._FUNCTION_ENV_VARS: - os.environ[env] = 'TRUE' + os.environ[env] = "TRUE" resource = detect_resource(self.PROJECT) self.assertIsInstance(resource, Resource) - self.assertEqual(resource.type, 'cloud_function') + self.assertEqual(resource.type, "cloud_function") def test_detect_legacy_functions(self): for env in _monitored_resources._LEGACY_FUNCTION_ENV_VARS: - os.environ[env] = 'TRUE' + os.environ[env] = "TRUE" resource = detect_resource(self.PROJECT) self.assertIsInstance(resource, Resource) - self.assertEqual(resource.type, 'cloud_function') + self.assertEqual(resource.type, "cloud_function") def test_detect_cloud_run(self): for env in _monitored_resources._CLOUD_RUN_ENV_VARS: - os.environ[env] = 'TRUE' + os.environ[env] = "TRUE" resource = detect_resource(self.PROJECT) self.assertIsInstance(resource, Resource) - self.assertEqual(resource.type, 'cloud_run_revision') + self.assertEqual(resource.type, "cloud_run_revision") def test_detect_compute_engine(self): patch = mock.patch( @@ -213,10 +233,9 @@ def test_detect_compute_engine(self): with patch: resource = detect_resource(self.PROJECT) self.assertIsInstance(resource, Resource) - self.assertEqual(resource.type, 'gce_instance') + self.assertEqual(resource.type, "gce_instance") def test_detection_unknown(self): resource = detect_resource(self.PROJECT) self.assertIsInstance(resource, Resource) - self.assertEqual(resource.type, 'global') - + self.assertEqual(resource.type, "global") From c6dde1ab22fecb508227a9893ce5bfcbaad9a2f8 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Mon, 1 Mar 2021 17:21:40 -0800 Subject: [PATCH 22/29] fixed failing unit tests --- tests/unit/handlers/test__monitored_resources.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/unit/handlers/test__monitored_resources.py b/tests/unit/handlers/test__monitored_resources.py index 0e8a75233..00fade39c 100644 --- a/tests/unit/handlers/test__monitored_resources.py +++ b/tests/unit/handlers/test__monitored_resources.py @@ -64,6 +64,9 @@ def _mock_metadata(self, endpoint): else: return None + def setUp(self): + os.environ.clear() + def test_create_legacy_functions_resource(self): patch = mock.patch( "google.cloud.logging_v2.handlers._monitored_resources.retrieve_metadata_server", @@ -236,6 +239,11 @@ def test_detect_compute_engine(self): self.assertEqual(resource.type, "gce_instance") def test_detection_unknown(self): - resource = detect_resource(self.PROJECT) - self.assertIsInstance(resource, Resource) - self.assertEqual(resource.type, "global") + patch = mock.patch( + "google.cloud.logging_v2.handlers._monitored_resources.retrieve_metadata_server", + return_value=None, + ) + with patch: + resource = detect_resource(self.PROJECT) + self.assertIsInstance(resource, Resource) + self.assertEqual(resource.type, "global") From 93be25c2273978adb95ead3910c04f83d891a942 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Mon, 1 Mar 2021 18:35:27 -0800 Subject: [PATCH 23/29] fixed app engine import path --- tests/environment | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/environment b/tests/environment index a10a6e95e..849ffa979 160000 --- a/tests/environment +++ b/tests/environment @@ -1 +1 @@ -Subproject commit a10a6e95eb10ce50962b9d1c2f8a9c0790bff469 +Subproject commit 849ffa9796d275c222d8e9bc828d63eb7fb75907 From 6c0db74aa250b301445aed879cf120a03379b037 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Tue, 2 Mar 2021 09:37:14 -0800 Subject: [PATCH 24/29] added comments --- .../handlers/_monitored_resources.py | 36 +++++++++++++++++++ tests/environment | 2 +- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/google/cloud/logging_v2/handlers/_monitored_resources.py b/google/cloud/logging_v2/handlers/_monitored_resources.py index d5d5d33a1..4bc30d4fb 100644 --- a/google/cloud/logging_v2/handlers/_monitored_resources.py +++ b/google/cloud/logging_v2/handlers/_monitored_resources.py @@ -53,6 +53,12 @@ def _create_functions_resource(project): + """Create a standardized Cloud Functions resource. + Args: + project (str): The project ID to pass on to the resource + Returns: + google.cloud.logging.Resource + """ region = retrieve_metadata_server(_REGION_ID) if _FUNCTION_NAME in os.environ: function_name = os.environ.get(_FUNCTION_NAME) @@ -72,6 +78,12 @@ def _create_functions_resource(project): def _create_kubernetes_resource(project): + """Create a standardized Kubernetes resource. + Args: + project (str): The project ID to pass on to the resource + Returns: + google.cloud.logging.Resource + """ zone = retrieve_metadata_server(_ZONE_ID) cluster_name = retrieve_metadata_server(_GKE_CLUSTER_NAME) @@ -87,6 +99,12 @@ def _create_kubernetes_resource(project): def _create_compute_resource(project): + """Create a standardized Compute Engine resource. + Args: + project (str): The project ID to pass on to the resource + Returns: + google.cloud.logging.Resource + """ instance = retrieve_metadata_server(_GCE_INSTANCE_ID) zone = retrieve_metadata_server(_ZONE_ID) resource = Resource( @@ -101,6 +119,12 @@ def _create_compute_resource(project): def _create_cloud_run_resource(project): + """Create a standardized Cloud Run resource. + Args: + project (str): The project ID to pass on to the resource + Returns: + google.cloud.logging.Resource + """ region = retrieve_metadata_server(_REGION_ID) resource = Resource( type="cloud_run_revision", @@ -116,6 +140,12 @@ def _create_cloud_run_resource(project): def _create_app_engine_resource(project): + """Create a standardized App Engine resource. + Args: + project (str): The project ID to pass on to the resource + Returns: + google.cloud.logging.Resource + """ zone = retrieve_metadata_server(_ZONE_ID) resource = Resource( type="gae_app", @@ -134,6 +164,12 @@ def _create_global_resource(project): def detect_resource(project): + """Return the default monitored resource based on the local environment. + Args: + project (str): The project ID to pass on to the resource + Returns: + google.cloud.logging.Resource: The default resource based on the environment + """ gke_cluster_name = retrieve_metadata_server(_GKE_CLUSTER_NAME) gce_instance_name = retrieve_metadata_server(_GCE_INSTANCE_ID) diff --git a/tests/environment b/tests/environment index 849ffa979..7183823c0 160000 --- a/tests/environment +++ b/tests/environment @@ -1 +1 @@ -Subproject commit 849ffa9796d275c222d8e9bc828d63eb7fb75907 +Subproject commit 7183823c0619a02c052fea497f2bfe985070dd3f From 2cca768088d08ac64af1a009db5062e2e21c93d0 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Tue, 2 Mar 2021 09:53:57 -0800 Subject: [PATCH 25/29] CloudLoggingHandler should attempt to infer resource type --- google/cloud/logging_v2/handlers/handlers.py | 9 ++++-- tests/unit/handlers/test_handlers.py | 31 +++++++++++++------- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/google/cloud/logging_v2/handlers/handlers.py b/google/cloud/logging_v2/handlers/handlers.py index fd99f7adc..c2ad6f355 100644 --- a/google/cloud/logging_v2/handlers/handlers.py +++ b/google/cloud/logging_v2/handlers/handlers.py @@ -17,7 +17,7 @@ import logging from google.cloud.logging_v2.handlers.transports import BackgroundThreadTransport -from google.cloud.logging_v2.logger import _GLOBAL_RESOURCE +from google.cloud.logging_v2.handlers._monitored_resources import detect_resource DEFAULT_LOGGER_NAME = "python" @@ -59,7 +59,7 @@ def __init__( *, name=DEFAULT_LOGGER_NAME, transport=BackgroundThreadTransport, - resource=_GLOBAL_RESOURCE, + resource=None, labels=None, stream=None, ): @@ -78,12 +78,15 @@ def __init__( :class:`.BackgroundThreadTransport`. The other option is :class:`.SyncTransport`. resource (~logging_v2.resource.Resource): - Resource for this Handler. Defaults to ``GLOBAL_RESOURCE``. + Resource for this Handler. If not given, will be inferred from the environment. labels (Optional[dict]): Monitored resource of the entry, defaults to the global resource type. stream (Optional[IO]): Stream to be used by the handler. """ super(CloudLoggingHandler, self).__init__(stream) + if not resource: + # infer the correct monitored resource from the local environment + resource = detect_resource(client.project) self.name = name self.client = client self.transport = transport(client, name) diff --git a/tests/unit/handlers/test_handlers.py b/tests/unit/handlers/test_handlers.py index d84c19635..0e7c63cc4 100644 --- a/tests/unit/handlers/test_handlers.py +++ b/tests/unit/handlers/test_handlers.py @@ -14,6 +14,7 @@ import logging import unittest +import mock class TestCloudLoggingHandler(unittest.TestCase): @@ -31,19 +32,27 @@ def _make_one(self, *args, **kw): def test_ctor_defaults(self): import sys - from google.cloud.logging_v2.logger import _GLOBAL_RESOURCE + from google.cloud.logging_v2.handlers._monitored_resources import ( + _create_global_resource, + ) from google.cloud.logging_v2.handlers.handlers import DEFAULT_LOGGER_NAME - client = _Client(self.PROJECT) - handler = self._make_one(client, transport=_Transport) - self.assertEqual(handler.name, DEFAULT_LOGGER_NAME) - self.assertIs(handler.client, client) - self.assertIsInstance(handler.transport, _Transport) - self.assertIs(handler.transport.client, client) - self.assertEqual(handler.transport.name, DEFAULT_LOGGER_NAME) - self.assertIs(handler.resource, _GLOBAL_RESOURCE) - self.assertIsNone(handler.labels) - self.assertIs(handler.stream, sys.stderr) + patch = mock.patch( + "google.cloud.logging_v2.handlers._monitored_resources.retrieve_metadata_server", + return_value=None, + ) + with patch: + client = _Client(self.PROJECT) + handler = self._make_one(client, transport=_Transport) + self.assertEqual(handler.name, DEFAULT_LOGGER_NAME) + self.assertIs(handler.client, client) + self.assertIsInstance(handler.transport, _Transport) + self.assertIs(handler.transport.client, client) + self.assertEqual(handler.transport.name, DEFAULT_LOGGER_NAME) + global_resource = _create_global_resource(self.PROJECT) + self.assertEqual(handler.resource, global_resource) + self.assertIsNone(handler.labels) + self.assertIs(handler.stream, sys.stderr) def test_ctor_explicit(self): import io From aeebe3a0b11d74b2b0f5de086666790a2584c16f Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Tue, 2 Mar 2021 10:04:46 -0800 Subject: [PATCH 26/29] use variables for magic strings --- google/cloud/logging_v2/client.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/google/cloud/logging_v2/client.py b/google/cloud/logging_v2/client.py index 3d696f750..60767aca2 100644 --- a/google/cloud/logging_v2/client.py +++ b/google/cloud/logging_v2/client.py @@ -50,6 +50,8 @@ _DISABLE_GRPC = os.getenv(DISABLE_GRPC, False) _USE_GRPC = _HAVE_GRPC and not _DISABLE_GRPC +_GAE_RESOURCE_TYPE = "gae_app" +_GKE_RESOURCE_TYPE = "k8s_container" class Client(ClientWithProject): """Client to bundle configuration needed for API requests.""" @@ -345,12 +347,12 @@ def get_default_handler(self, **kw): if ( isinstance(monitored_resource, Resource) - and monitored_resource.type == "gae_app" + and monitored_resource.type == _GAE_RESOURCE_TYPE ): return AppEngineHandler(self, **kw) elif ( isinstance(monitored_resource, Resource) - and monitored_resource.type == "k8s_container" + and monitored_resource.type == _GKE_RESOURCE_TYPE ): return ContainerEngineHandler(**kw) else: From f4da836a4866112166e10901043b0286eb7c6289 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Tue, 2 Mar 2021 10:35:26 -0800 Subject: [PATCH 27/29] fixed lint issue --- google/cloud/logging_v2/client.py | 1 + 1 file changed, 1 insertion(+) diff --git a/google/cloud/logging_v2/client.py b/google/cloud/logging_v2/client.py index 60767aca2..f196f443a 100644 --- a/google/cloud/logging_v2/client.py +++ b/google/cloud/logging_v2/client.py @@ -53,6 +53,7 @@ _GAE_RESOURCE_TYPE = "gae_app" _GKE_RESOURCE_TYPE = "k8s_container" + class Client(ClientWithProject): """Client to bundle configuration needed for API requests.""" From 2d0ff7e912a90df87bfe500e53a9dd2e0efb25b1 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Tue, 2 Mar 2021 15:12:26 -0800 Subject: [PATCH 28/29] pulled in go environment tests --- tests/environment | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/environment b/tests/environment index 7183823c0..f2783bdc3 160000 --- a/tests/environment +++ b/tests/environment @@ -1 +1 @@ -Subproject commit 7183823c0619a02c052fea497f2bfe985070dd3f +Subproject commit f2783bdc397af92b595caee19a78ccbfc4e64ae0 From 4e0dd3cb3f45612955f0ae9e2335db7a4ea0d9a5 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 5 Mar 2021 14:05:16 -0800 Subject: [PATCH 29/29] merged submodule into main --- tests/environment | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/environment b/tests/environment index f2783bdc3..0e3311158 160000 --- a/tests/environment +++ b/tests/environment @@ -1 +1 @@ -Subproject commit f2783bdc397af92b595caee19a78ccbfc4e64ae0 +Subproject commit 0e331115867ca5a26b1efd9d99c43fbb1cb9363b