diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e1cf8a11..484597332 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ Use `UnknownMetricFamily` for `SumData` instead of `UntypedMetricFamily`. Check if label keys and values match before exporting. - Remove min and max from Distribution. +- Replace stackdriver `gke_container` resources, see the [GKE migration + notes](https://cloud.google.com/monitoring/kubernetes-engine/migration#incompatible) + for details ## 0.2.0 Released 2019-01-18 diff --git a/opencensus/common/monitored_resource/gcp_metadata_config.py b/opencensus/common/monitored_resource/gcp_metadata_config.py index 6fe99739a..6548db475 100644 --- a/opencensus/common/monitored_resource/gcp_metadata_config.py +++ b/opencensus/common/monitored_resource/gcp_metadata_config.py @@ -13,63 +13,32 @@ # limitations under the License. from opencensus.common.http_handler import get_request -import os _GCP_METADATA_URI = 'http://metadata/computeMetadata/v1/' _GCP_METADATA_URI_HEADER = {'Metadata-Flavor': 'Google'} -# GCE common attributes -# See: https://cloud.google.com/appengine/docs/flexible/python/runtime# -# environment_variables -_GCE_ATTRIBUTES = { - # ProjectID is the identifier of the GCP project associated with this - # resource, such as "my-project". - 'project_id': 'project/project-id', - - # instance_id is the numeric VM instance identifier assigned by - # Compute Engine. - 'instance_id': 'instance/id', - - # zone is the Compute Engine zone in which the VM is running. - 'zone': 'instance/zone' -} +# ID of the GCP project associated with this resource, such as "my-project" +PROJECT_ID_KEY = 'project_id' -_GKE_ATTRIBUTES = { - # ProjectID is the identifier of the GCP project associated with this - # resource, such as "my-project". - 'project_id': 'project/project-id', - - # instance_id is the numeric VM instance identifier assigned by - # Compute Engine. - 'instance_id': 'instance/id', - - # zone is the Compute Engine zone in which the VM is running. - 'zone': 'instance/zone', - - # cluster_name is the name for the cluster the container is running in. - 'cluster_name': 'instance/attributes/cluster-name' -} +# Numeric VM instance identifier assigned by GCE +INSTANCE_ID_KEY = 'instance_id' -# Following attributes are derived from environment variables. They are -# configured via yaml file. For details refer to: -# https://cloud.google.com/kubernetes-engine/docs/tutorials/ -# custom-metrics-autoscaling#exporting_metrics_from_the_application -_GKE_ENV_ATTRIBUTES = { - # ContainerName is the name of the container. - 'container_name': 'CONTAINER_NAME', +# The GCE zone in which the VM is running +ZONE_KEY = 'zone' - # namespace_id is the identifier for the cluster namespace the container - # is running in - 'namespace_id': 'NAMESPACE', +# GKE cluster name +CLUSTER_NAME_KEY = 'instance/attributes/cluster-name' - # pod_id is the identifier for the pod the container is running in. - 'pod_id': 'HOSTNAME' +# GCE common attributes +# See: https://cloud.google.com/appengine/docs/flexible/python/runtime#environment_variables # noqa +_GCE_ATTRIBUTES = { + PROJECT_ID_KEY: 'project/project-id', + INSTANCE_ID_KEY: 'instance/id', + ZONE_KEY: 'instance/zone' } -# Kubenertes environment variables -_KUBERNETES_SERVICE_HOST = 'KUBERNETES_SERVICE_HOST' -gcp_metadata_map = {} +_GCP_METADATA_MAP = {} class GcpMetadataConfig(object): @@ -90,23 +59,19 @@ def _initialize_metadata_service(cls): if cls.inited: return - instance_id = cls._get_attribute('instance_id') + instance_id = cls.get_attribute('instance/id') if instance_id is not None: cls.is_running = True - gcp_metadata_map['instance_id'] = instance_id - - attributes = _GCE_ATTRIBUTES - if _KUBERNETES_SERVICE_HOST in os.environ: - attributes = _GKE_ATTRIBUTES + _GCP_METADATA_MAP['instance_id'] = instance_id # fetch attributes from metadata request - for attribute_key, attribute_uri in attributes.items(): - if attribute_key not in gcp_metadata_map: - attribute_value = cls._get_attribute(attribute_key) - if attribute_value is not None: - gcp_metadata_map[attribute_key] = attribute_value + for attribute_key, attribute_uri in _GCE_ATTRIBUTES.items(): + if attribute_key not in _GCP_METADATA_MAP: + attribute_value = cls.get_attribute(attribute_uri) + if attribute_value is not None: # pragma: NO COVER + _GCP_METADATA_MAP[attribute_key] = attribute_value cls.inited = True @@ -118,35 +83,19 @@ def is_running_on_gcp(cls): def get_gce_metadata(self): """for GCP GCE instance""" if self.is_running_on_gcp(): - return gcp_metadata_map + return _GCP_METADATA_MAP return dict() - def get_gke_metadata(self): - """for GCP GKE container.""" - gke_metadata = {} - - if self.is_running_on_gcp(): - gke_metadata = gcp_metadata_map - - # fetch attributes from Environment Variables - for attribute_key, attribute_env in _GKE_ENV_ATTRIBUTES.items(): - attribute_value = os.environ.get(attribute_env) - if attribute_value is not None: - gke_metadata[attribute_key] = attribute_value - - return gke_metadata - @staticmethod - def _get_attribute(attribute_key): + def get_attribute(attribute_uri): """ Fetch the requested instance metadata entry. :param attribute_uri: attribute_uri: attribute name relative to the computeMetadata/v1 prefix :return: The value read from the metadata service or None """ - attribute_value = get_request(_GCP_METADATA_URI + - _GKE_ATTRIBUTES[attribute_key], + attribute_value = get_request(_GCP_METADATA_URI + attribute_uri, _GCP_METADATA_URI_HEADER) if attribute_value is not None and isinstance(attribute_value, bytes): diff --git a/opencensus/common/monitored_resource/k8s_utils.py b/opencensus/common/monitored_resource/k8s_utils.py new file mode 100644 index 000000000..f4eeb51d0 --- /dev/null +++ b/opencensus/common/monitored_resource/k8s_utils.py @@ -0,0 +1,64 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from opencensus.common.monitored_resource import gcp_metadata_config + +# Env var that signals that we're in a kubernetes container +_KUBERNETES_SERVICE_HOST = 'KUBERNETES_SERVICE_HOST' + +# Name of the cluster the container is running in +CLUSTER_NAME_KEY = 'k8s.io/cluster/name' + +# ID of the instance the container is running on +NAMESPACE_NAME_KEY = 'k8s.io/namespace/name' + +# Container pod ID +POD_NAME_KEY = 'k8s.io/pod/name' + +# Container name +CONTAINER_NAME_KEY = 'k8s.io/container/name' + +# Attributes set from environment variables +_K8S_ENV_ATTRIBUTES = { + CONTAINER_NAME_KEY: 'CONTAINER_NAME', + NAMESPACE_NAME_KEY: 'NAMESPACE', + POD_NAME_KEY: 'HOSTNAME' +} + + +def is_k8s_environment(): + """Whether the environment is a kubernetes container. + + The KUBERNETES_SERVICE_HOST environment variable must be set. + """ + return _KUBERNETES_SERVICE_HOST in os.environ + + +def get_k8s_metadata(): + """Get kubernetes container metadata, as on GCP GKE.""" + k8s_metadata = {} + + gcp_cluster = (gcp_metadata_config.GcpMetadataConfig + .get_attribute(gcp_metadata_config.CLUSTER_NAME_KEY)) + if gcp_cluster is not None: + k8s_metadata[CLUSTER_NAME_KEY] = gcp_cluster + + for attribute_key, attribute_env in _K8S_ENV_ATTRIBUTES.items(): + attribute_value = os.environ.get(attribute_env) + if attribute_value is not None: + k8s_metadata[attribute_key] = attribute_value + + return k8s_metadata diff --git a/opencensus/common/monitored_resource/monitored_resource.py b/opencensus/common/monitored_resource/monitored_resource.py index 43b48d59c..c2649113a 100644 --- a/opencensus/common/monitored_resource/monitored_resource.py +++ b/opencensus/common/monitored_resource/monitored_resource.py @@ -12,29 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os - from opencensus.common import resource from opencensus.common.monitored_resource import aws_identity_doc_utils from opencensus.common.monitored_resource import gcp_metadata_config +from opencensus.common.monitored_resource import k8s_utils # Supported environments (resource types) _GCE_INSTANCE = "gce_instance" -_GKE_CONTAINER = "gke_container" +_K8S_CONTAINER = "k8s_container" _AWS_EC2_INSTANCE = "aws_ec2_instance" -# Kubenertes environment variables -_KUBERNETES_SERVICE_HOST = 'KUBERNETES_SERVICE_HOST' - - -def is_gke_environment(): - """Whether the environment is a GKE container instance. - - The KUBERNETES_SERVICE_HOST environment variable must be set. - """ - return _KUBERNETES_SERVICE_HOST in os.environ - def is_gce_environment(): """Whether the environment is a virtual machine on GCE.""" @@ -52,22 +40,11 @@ def get_instance(): Returns a `Resource` configured for the current environment, or None if the environment is unknown or unsupported. - Supported environments include: - - 1. 'gke_container' - - https://cloud.google.com/monitoring/api/resources#tag_gke_container - 2. 'gce_instance' - - https://cloud.google.com/monitoring/api/resources#tag_gce_instance - 3. 'aws_ec2_instance' - - https://cloud.google.com/monitoring/api/resources#tag_aws_ec2_instance - :rtype: :class:`opencensus.common.resource.Resource` or None :return: A `Resource` configured for the current environment. """ - if is_gke_environment(): - return resource.Resource( - _GKE_CONTAINER, - gcp_metadata_config.GcpMetadataConfig().get_gke_metadata()) + if k8s_utils.is_k8s_environment(): + return resource.Resource(_K8S_CONTAINER, k8s_utils.get_k8s_metadata()) if is_gce_environment(): return resource.Resource( _GCE_INSTANCE, diff --git a/tests/unit/common/monitored_resource_util/test_gcp_metadata_config.py b/tests/unit/common/monitored_resource_util/test_gcp_metadata_config.py index 7421b510c..a35da3b1d 100644 --- a/tests/unit/common/monitored_resource_util/test_gcp_metadata_config.py +++ b/tests/unit/common/monitored_resource_util/test_gcp_metadata_config.py @@ -20,6 +20,7 @@ class TestGcpMetadataConfig(unittest.TestCase): + @mock.patch('opencensus.common.monitored_resource.' 'gcp_metadata_config.get_request') def test_get_gce_metadata(self, http_request_mock): @@ -35,7 +36,7 @@ def assign_attribute_value(*args, **kwargs): http_request_mock.side_effect = assign_attribute_value gcp_metadata_config.GcpMetadataConfig.inited = False gcp_metadata_config.GcpMetadataConfig.is_running = False - gcp_metadata_config.gcp_metadata_map = {} + gcp_metadata_config._GCP_METADATA_MAP = {} self.assertTrue( gcp_metadata_config.GcpMetadataConfig.is_running_on_gcp()) @@ -51,7 +52,7 @@ def assign_attribute_value(*args, **kwargs): 'zone': 'us-east1' } - self.assertEquals(labels_list, expected_labels) + self.assertDictEqual(labels_list, expected_labels) @mock.patch('opencensus.common.monitored_resource.' 'gcp_metadata_config.get_request') @@ -72,7 +73,7 @@ def assign_attribute_value(*args, **kwargs): http_request_mock.side_effect = assign_attribute_value gcp_metadata_config.GcpMetadataConfig.inited = False gcp_metadata_config.GcpMetadataConfig.is_running = False - gcp_metadata_config.gcp_metadata_map = {} + gcp_metadata_config._GCP_METADATA_MAP = {} self.assertTrue( gcp_metadata_config.GcpMetadataConfig.is_running_on_gcp()) @@ -88,96 +89,7 @@ def assign_attribute_value(*args, **kwargs): 'zone': 'us-east1' } - self.assertEquals(labels_list, expected_labels) - - @mock.patch.dict( - os.environ, { - 'KUBERNETES_SERVICE_HOST': '127.0.0.1', - 'CONTAINER_NAME': 'container', - 'NAMESPACE': 'namespace', - 'HOSTNAME': 'localhost' - }, - clear=True) - @mock.patch('opencensus.common.monitored_resource.' - 'gcp_metadata_config.get_request') - def test_get_gke_metadata(self, http_request_mock): - def assign_attribute_value(*args, **kwargs): - attribute_uri = args[0].split('/')[-1] - if attribute_uri == 'id': - return 'my-instance' - elif attribute_uri == 'project-id': - return 'my-project' - elif attribute_uri == 'cluster-name': - return 'cluster' - elif attribute_uri == 'zone': - return 'us-east1' - - http_request_mock.side_effect = assign_attribute_value - gcp_metadata_config.GcpMetadataConfig.inited = False - gcp_metadata_config.GcpMetadataConfig.is_running = False - gcp_metadata_config.gcp_metadata_map = {} - - self.assertTrue( - gcp_metadata_config.GcpMetadataConfig.is_running_on_gcp()) - - labels_list = gcp_metadata_config.GcpMetadataConfig().get_gke_metadata( - ) - - self.assertEquals(len(labels_list), 7) - - expected_labels = { - 'instance_id': 'my-instance', - 'cluster_name': 'cluster', - 'project_id': 'my-project', - 'zone': 'us-east1', - 'pod_id': 'localhost', - 'namespace_id': 'namespace', - 'container_name': 'container' - } - - self.assertEquals(labels_list, expected_labels) - - @mock.patch.dict( - os.environ, { - 'KUBERNETES_SERVICE_HOST': '127.0.0.1', - 'NAMESPACE': 'namespace', - 'HOSTNAME': 'localhost' - }, - clear=True) - @mock.patch('opencensus.common.monitored_resource.' - 'gcp_metadata_config.get_request') - def test_get_gke_metadata_container_empty(self, http_request_mock): - def assign_attribute_value(*args, **kwargs): - attribute_uri = args[0].split('/')[-1] - if attribute_uri == 'id': - return 'my-instance' - elif attribute_uri == 'project-id': - return 'my-project' - elif attribute_uri == 'zone': - return 'us-east1' - - http_request_mock.side_effect = assign_attribute_value - gcp_metadata_config.GcpMetadataConfig.inited = False - gcp_metadata_config.GcpMetadataConfig.is_running = False - gcp_metadata_config.gcp_metadata_map = {} - - self.assertTrue( - gcp_metadata_config.GcpMetadataConfig.is_running_on_gcp()) - - labels_list = gcp_metadata_config.GcpMetadataConfig().get_gke_metadata( - ) - - self.assertEquals(len(labels_list), 5) - - expected_labels = { - 'instance_id': 'my-instance', - 'project_id': 'my-project', - 'zone': 'us-east1', - 'pod_id': 'localhost', - 'namespace_id': 'namespace' - } - - self.assertEquals(labels_list, expected_labels) + self.assertDictEqual(labels_list, expected_labels) @mock.patch.dict(os.environ, clear=True) @mock.patch('opencensus.common.monitored_resource.' @@ -186,12 +98,10 @@ def test_gcp_not_running(self, http_request_mock): http_request_mock.return_value = None gcp_metadata_config.GcpMetadataConfig.inited = False gcp_metadata_config.GcpMetadataConfig.is_running = False - gcp_metadata_config.gcp_metadata_map = {} + gcp_metadata_config._GCP_METADATA_MAP = {} self.assertFalse( gcp_metadata_config.GcpMetadataConfig.is_running_on_gcp()) self.assertEquals( len(gcp_metadata_config.GcpMetadataConfig().get_gce_metadata()), 0) - self.assertEquals( - len(gcp_metadata_config.GcpMetadataConfig().get_gke_metadata()), 0) diff --git a/tests/unit/common/monitored_resource_util/test_k8s_utils.py b/tests/unit/common/monitored_resource_util/test_k8s_utils.py new file mode 100644 index 000000000..be6686056 --- /dev/null +++ b/tests/unit/common/monitored_resource_util/test_k8s_utils.py @@ -0,0 +1,68 @@ +# Copyright 2019, OpenCensus Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +try: + import mock +except ImportError: + from unittest import mock + +import os +import unittest + +from opencensus.common.monitored_resource import k8s_utils + + +class TestK8SUtils(unittest.TestCase): + + @mock.patch.dict( + os.environ, { + 'KUBERNETES_SERVICE_HOST': '127.0.0.1', + 'CONTAINER_NAME': 'container', + 'NAMESPACE': 'namespace', + 'HOSTNAME': 'localhost' + }, + clear=True) + @mock.patch('opencensus.common.monitored_resource.' + 'gcp_metadata_config.get_request') + def test_get_k8s_metadata(self, http_request_mock): + def assign_attribute_value(*args, **kwargs): + if args[0].split('/')[-1] == 'cluster-name': + return 'cluster' + raise AssertionError + + http_request_mock.side_effect = assign_attribute_value + labels_list = (k8s_utils.get_k8s_metadata()) + self.assertDictEqual( + labels_list, + {'k8s.io/cluster/name': 'cluster', + 'k8s.io/container/name': 'container', + 'k8s.io/namespace/name': 'namespace', + 'k8s.io/pod/name': 'localhost' + }) + + @mock.patch.dict( + os.environ, { + 'KUBERNETES_SERVICE_HOST': '127.0.0.1', + 'NAMESPACE': 'namespace', + 'HOSTNAME': 'localhost' + }, + clear=True) + def test_get_k8s_metadata_container_empty(self): + labels_list = (k8s_utils.get_k8s_metadata()) + + self.assertDictEqual( + labels_list, + {'k8s.io/namespace/name': 'namespace', + 'k8s.io/pod/name': 'localhost' + }) diff --git a/tests/unit/common/monitored_resource_util/test_monitored_resource.py b/tests/unit/common/monitored_resource_util/test_monitored_resource.py index 23d2c4fdd..ca1317071 100644 --- a/tests/unit/common/monitored_resource_util/test_monitored_resource.py +++ b/tests/unit/common/monitored_resource_util/test_monitored_resource.py @@ -27,8 +27,8 @@ def mock_mr_method(method, use): yield -def mock_use_gke(use): - return mock_mr_method('is_gke_environment', use) +def mock_use_k8s(use): + return mock_mr_method('k8s_utils.is_k8s_environment', use) def mock_use_gce(use): @@ -40,8 +40,8 @@ def mock_use_aws(use): @contextmanager -def mock_gke_env(): - with mock_use_gke(True): +def mock_k8s_env(): + with mock_use_k8s(True): with mock_use_gce(False): with mock_use_aws(False): yield @@ -49,7 +49,7 @@ def mock_gke_env(): @contextmanager def mock_gce_env(): - with mock_use_gke(False): + with mock_use_k8s(False): with mock_use_gce(True): with mock_use_aws(False): yield @@ -57,7 +57,7 @@ def mock_gce_env(): @contextmanager def mock_aws_env(): - with mock_use_gke(False): + with mock_use_k8s(False): with mock_use_gce(False): with mock_use_aws(True): yield @@ -79,13 +79,10 @@ def test_gcp_gce_monitored_resource(self, gcp_metadata_mock): mocked_labels with mock_gce_env(): resource = monitored_resource.get_instance() - self.assertEquals(resource.get_type(), 'gce_instance') - self.assertEquals(resource.get_labels(), mocked_labels) - - @mock.patch('opencensus.common.monitored_resource.monitored_resource' - '.gcp_metadata_config.GcpMetadataConfig') - def test_gcp_gke_monitored_resource(self, gcp_metadata_mock): + self.assertEqual(resource.get_type(), 'gce_instance') + self.assertDictEqual(resource.get_labels(), mocked_labels) + def test_gcp_k8s_monitored_resource(self): mocked_labels = { 'instance_id': 'my-instance', 'cluster_name': 'cluster', @@ -96,13 +93,11 @@ def test_gcp_gke_monitored_resource(self, gcp_metadata_mock): 'container_name': 'container' } - gcp_metadata_mock.return_value = mock.Mock() - gcp_metadata_mock.return_value.get_gke_metadata.return_value =\ - mocked_labels - with mock_gke_env(): - resource = monitored_resource.get_instance() - self.assertEquals(resource.get_type(), 'gke_container') - self.assertEquals(resource.get_labels(), mocked_labels) + with mock_mr_method('k8s_utils.get_k8s_metadata', mocked_labels): + with mock_k8s_env(): + resource = monitored_resource.get_instance() + self.assertEqual(resource.get_type(), 'k8s_container') + self.assertDictEqual(resource.get_labels(), mocked_labels) @mock.patch('opencensus.common.monitored_resource.monitored_resource' '.aws_identity_doc_utils.AwsIdentityDocumentUtils') @@ -120,10 +115,10 @@ def test_aws_monitored_resource(self, aws_metadata_mock): with mock_aws_env(): resource = monitored_resource.get_instance() - self.assertEquals(resource.get_type(), 'aws_ec2_instance') - self.assertEquals(resource.get_labels(), mocked_labels) + self.assertEqual(resource.get_type(), 'aws_ec2_instance') + self.assertDictEqual(resource.get_labels(), mocked_labels) - def test_gke_environment(self): + def test_k8s_environment(self): patch = mock.patch.dict(os.environ, {'KUBERNETES_SERVICE_HOST': '127.0.0.1'}) @@ -131,7 +126,7 @@ def test_gke_environment(self): mr = monitored_resource.get_instance() self.assertIsNotNone(mr) - self.assertEqual(mr.get_type(), "gke_container") + self.assertEqual(mr.get_type(), "k8s_container") def test_gce_environment(self): patch = mock.patch(