From a14305edf5441203e2f25ed8b5928b942a37a1c8 Mon Sep 17 00:00:00 2001 From: Cheng-Lung Sung Date: Wed, 13 Nov 2019 23:44:19 +0800 Subject: [PATCH 01/12] Add stackdriver trace exporter --- ext/opentelemetry-ext-stackdriver/README.rst | 44 +++ .../examples/client.py | 32 ++ .../examples/server.py | 44 +++ .../examples/trace.py | 25 ++ ext/opentelemetry-ext-stackdriver/setup.cfg | 48 +++ ext/opentelemetry-ext-stackdriver/setup.py | 26 ++ .../ext/stackdriver/trace/__init__.py | 281 ++++++++++++++++++ .../opentelemetry/ext/stackdriver/version.py | 15 + .../tests/__init__.py | 0 .../tests/test_stackdriver_exporter.py | 113 +++++++ scripts/coverage.sh | 1 + tox.ini | 1 + 12 files changed, 630 insertions(+) create mode 100644 ext/opentelemetry-ext-stackdriver/README.rst create mode 100644 ext/opentelemetry-ext-stackdriver/examples/client.py create mode 100644 ext/opentelemetry-ext-stackdriver/examples/server.py create mode 100644 ext/opentelemetry-ext-stackdriver/examples/trace.py create mode 100644 ext/opentelemetry-ext-stackdriver/setup.cfg create mode 100644 ext/opentelemetry-ext-stackdriver/setup.py create mode 100644 ext/opentelemetry-ext-stackdriver/src/opentelemetry/ext/stackdriver/trace/__init__.py create mode 100644 ext/opentelemetry-ext-stackdriver/src/opentelemetry/ext/stackdriver/version.py create mode 100644 ext/opentelemetry-ext-stackdriver/tests/__init__.py create mode 100644 ext/opentelemetry-ext-stackdriver/tests/test_stackdriver_exporter.py diff --git a/ext/opentelemetry-ext-stackdriver/README.rst b/ext/opentelemetry-ext-stackdriver/README.rst new file mode 100644 index 00000000000..fcf2a08e014 --- /dev/null +++ b/ext/opentelemetry-ext-stackdriver/README.rst @@ -0,0 +1,44 @@ +OpenTelemetry Stackdriver Exporters +===================================== + +This library provides integration with Google Cloud Stackdriver. + +Installation +------------ + +:: + + pip install opentelemetry-ext-stackdriver + +Usage +----- + +.. code:: python + + from opentelemetry import trace + from opentelemetry.ext import stackdriver + from opentelemetry.sdk.trace import Tracer + from opentelemetry.sdk.trace.export import BatchExportSpanProcessor + + trace.set_preferred_tracer_implementation(lambda T: Tracer()) + tracer = trace.tracer() + + # create a StackdriverSpanExporter + stackdriver_exporter = stackdriver.trace.StackdriverSpanExporter( + project_id='my-helloworld-project', + ) + + # Create a BatchExportSpanProcessor and add the exporter to it + span_processor = BatchExportSpanProcessor(stackdriver_exporter) + + # add to the tracer + tracer.add_span_processor(span_processor) + + with tracer.start_as_current_span('foo'): + print('Hello world!') + +References +---------- + +* `Stackdriver `_ +* `OpenTelemetry Project `_ diff --git a/ext/opentelemetry-ext-stackdriver/examples/client.py b/ext/opentelemetry-ext-stackdriver/examples/client.py new file mode 100644 index 00000000000..ee6206d9470 --- /dev/null +++ b/ext/opentelemetry-ext-stackdriver/examples/client.py @@ -0,0 +1,32 @@ +# Copyright 2019, OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import requests + +from opentelemetry import trace +from opentelemetry.ext import http_requests +from opentelemetry.ext.stackdriver.trace import StackdriverSpanExporter +from opentelemetry.sdk.trace import Tracer +from opentelemetry.sdk.trace.export import SimpleExportSpanProcessor + +trace.set_preferred_tracer_implementation(lambda T: Tracer()) +tracer = trace.tracer() +span_processor = SimpleExportSpanProcessor( + StackdriverSpanExporter(project_id="my-helloworld-project") +) +tracer.add_span_processor(span_processor) + +http_requests.enable(tracer) +response = requests.get(url="http://localhost:7777/hello") +span_processor.shutdown() diff --git a/ext/opentelemetry-ext-stackdriver/examples/server.py b/ext/opentelemetry-ext-stackdriver/examples/server.py new file mode 100644 index 00000000000..8cd74a653e3 --- /dev/null +++ b/ext/opentelemetry-ext-stackdriver/examples/server.py @@ -0,0 +1,44 @@ +# Copyright 2019, OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import flask +import requests + +from opentelemetry import trace +from opentelemetry.ext import http_requests +from opentelemetry.ext.stackdriver.trace import StackdriverSpanExporter +from opentelemetry.ext.wsgi import OpenTelemetryMiddleware +from opentelemetry.sdk.trace import Tracer +from opentelemetry.sdk.trace.export import BatchExportSpanProcessor + +trace.set_preferred_tracer_implementation(lambda T: Tracer()) + +span_processor = BatchExportSpanProcessor(StackdriverSpanExporter()) +http_requests.enable(trace.tracer()) +trace.tracer().add_span_processor(span_processor) + +app = flask.Flask(__name__) +app.wsgi_app = OpenTelemetryMiddleware(app.wsgi_app) + + +@app.route("/") +def hello(): + with trace.tracer().start_as_current_span("parent"): + requests.get("https://www.wikipedia.org/wiki/Rabbit") + return "hello" + + +if __name__ == "__main__": + app.run(debug=True) + span_processor.shutdown() diff --git a/ext/opentelemetry-ext-stackdriver/examples/trace.py b/ext/opentelemetry-ext-stackdriver/examples/trace.py new file mode 100644 index 00000000000..1cdb41ecc0e --- /dev/null +++ b/ext/opentelemetry-ext-stackdriver/examples/trace.py @@ -0,0 +1,25 @@ +# Copyright 2019, OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from opentelemetry import trace +from opentelemetry.ext.stackdriver.trace import StackdriverSpanExporter +from opentelemetry.sdk.trace import Tracer +from opentelemetry.sdk.trace.export import SimpleExportSpanProcessor + +trace.set_preferred_tracer_implementation(lambda T: Tracer()) +tracer = trace.tracer() +tracer.add_span_processor(SimpleExportSpanProcessor(StackdriverSpanExporter())) + +with tracer.start_as_current_span("hello") as span: + print("Hello, World!") diff --git a/ext/opentelemetry-ext-stackdriver/setup.cfg b/ext/opentelemetry-ext-stackdriver/setup.cfg new file mode 100644 index 00000000000..d3307eae772 --- /dev/null +++ b/ext/opentelemetry-ext-stackdriver/setup.cfg @@ -0,0 +1,48 @@ +# Copyright 2019, OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +[metadata] +name = opentelemetry-ext-stackdriver +description = Stackdriver integration for OpenTelemetry +long_description = file: README.rst +long_description_content_type = text/x-rst +author = OpenTelemetry Authors +author_email = cncf-opentelemetry-contributors@lists.cncf.io +url = https://github.com/open-telemetry/opentelemetry-python/ext/opentelemetry-ext-stackdriver +platforms = any +license = Apache-2.0 +classifiers = + Development Status :: 3 - Alpha + Intended Audience :: Developers + License :: OSI Approved :: Apache Software License + Programming Language :: Python + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.4 + Programming Language :: Python :: 3.5 + Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.7 + +[options] +python_requires = >=3.4 +package_dir= + =src +packages=find_namespace: +install_requires = + opentelemetry-api + opentelemetry-sdk + google-cloud-monitoring + google-cloud-trace + +[options.packages.find] +where = src diff --git a/ext/opentelemetry-ext-stackdriver/setup.py b/ext/opentelemetry-ext-stackdriver/setup.py new file mode 100644 index 00000000000..8d43c44ffde --- /dev/null +++ b/ext/opentelemetry-ext-stackdriver/setup.py @@ -0,0 +1,26 @@ +# Copyright 2019, OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os + +import setuptools + +BASE_DIR = os.path.dirname(__file__) +VERSION_FILENAME = os.path.join( + BASE_DIR, "src", "opentelemetry", "ext", "stackdriver", "version.py" +) +PACKAGE_INFO = {} +with open(VERSION_FILENAME) as f: + exec(f.read(), PACKAGE_INFO) + +setuptools.setup(version=PACKAGE_INFO["__version__"]) diff --git a/ext/opentelemetry-ext-stackdriver/src/opentelemetry/ext/stackdriver/trace/__init__.py b/ext/opentelemetry-ext-stackdriver/src/opentelemetry/ext/stackdriver/trace/__init__.py new file mode 100644 index 00000000000..6995cbe8633 --- /dev/null +++ b/ext/opentelemetry-ext-stackdriver/src/opentelemetry/ext/stackdriver/trace/__init__.py @@ -0,0 +1,281 @@ +# Copyright 2019, OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Stackdriver Span Exporter for OpenTelemetry.""" + +import logging +import typing + +from google.cloud.trace import trace_service_client +from google.cloud.trace.client import Client +from google.cloud.trace_v2.proto import trace_pb2 + +import opentelemetry.trace as trace_api +from opentelemetry.context import Context +from opentelemetry.sdk.trace.export import Span, SpanExporter, SpanExportResult +from opentelemetry.sdk.util import ns_to_iso_str +from opentelemetry.util import types + +from ..version import __version__ + +logger = logging.getLogger(__name__) + +AGENT = "opentelemetry-python [{}]".format(__version__) +# Max length is 128 bytes for a truncatable string. +MAX_LENGTH = 128 + + +class StackdriverSpanExporter(SpanExporter): + """Stackdriver span exporter for OpenTelemetry. + + Args: + client: Stackdriver Trace client. + project_id: project_id to create the Trace client. + """ + + def __init__( + self, client=None, project_id=None, + ): + if client is None: + client = Client(project=project_id) + self.client = client + self.project_id = self.client.project + + def export(self, spans: typing.Sequence[Span]) -> SpanExportResult: + """Export the spans to Stackdriver. + + See: https://cloud.google.com/trace/docs/reference/v2/rest/v2/ + projects.traces/batchWrite + + Args: + spans: Tuple of spans to export + """ + stackdriver_spans = self.translate_to_stackdriver(spans) + + try: + self.client.batch_write_spans( + "projects/{}".format(self.project_id), + {"spans": stackdriver_spans}, + ) + except Exception as ex: + logger.warning("Error while writing to stackdriver: %s", ex) + return SpanExportResult.FAILED_RETRYABLE + + return SpanExportResult.SUCCESS + + def translate_to_stackdriver( + self, spans: typing.Sequence[Span] + ) -> typing.List[typing.Dict[str, typing.Any]]: + """Translate the spans to Stackdriver format. + + Args: + spans: Tuple of spans to convert + """ + + stackdriver_spans = [] + + for span in spans: + ctx = span.get_context() + trace_id = "{:032x}".format(ctx.trace_id) + span_id = "{:016x}".format(ctx.span_id) + span_name = "projects/{}/traces/{}/spans/{}".format( + self.project_id, trace_id, span_id + ) + + parent_id = None + if isinstance(span.parent, trace_api.Span): + parent_id = "{:016x}".format(span.parent.get_context().span_id) + elif isinstance(span.parent, trace_api.SpanContext): + parent_id = "{:016x}".format(span.parent.span_id) + + start_time = None + if span.start_time: + start_time = ns_to_iso_str(span.start_time) + end_time = None + if span.end_time: + end_time = ns_to_iso_str(span.end_time) + + span.attributes["g.co/agent"] = AGENT + attr_map = extract_attributes(span.attributes) + + sd_span = { + "name": span_name, + "spanId": span_id, + "parentSpanId": parent_id, + "displayName": get_truncatable_str(span.name), + "attributes": map_attributes(attr_map), + "links": extract_links(span.links), + "status": extract_status(span.status), + "timeEvents": extract_events(span.events), + "startTime": start_time, + "endTime": end_time, + } + + stackdriver_spans.append(sd_span) + + return stackdriver_spans + + def shutdown(self): + pass + + +def get_truncatable_str(str_to_convert): + """Truncate a string if exceed limit and record the truncated bytes + count. + """ + truncated, truncated_byte_count = check_str_length( + str_to_convert, MAX_LENGTH + ) + + result = { + "value": truncated, + "truncated_byte_count": truncated_byte_count, + } + return result + + +def check_str_length(str_to_check, limit=MAX_LENGTH): + """Check the length of a string. If exceeds limit, then truncate it. + """ + str_bytes = str_to_check.encode("utf-8") + str_len = len(str_bytes) + truncated_byte_count = 0 + + if str_len > limit: + truncated_byte_count = str_len - limit + str_bytes = str_bytes[:limit] + + result = str(str_bytes.decode("utf-8", errors="ignore")) + + return (result, truncated_byte_count) + + +def extract_status(status: trace_api.Status): + """Convert a Status object to dict.""" + status_json = {"details": None} + + status_json["code"] = status.canonical_code.value + + if status.description is not None: + status_json["message"] = status.description + + return status_json + + +def extract_links(links): + """Convert span.links to set.""" + if not links: + return None + + links = [] + for link in links: + trace_id = link.context.trace_id + span_id = link.context.span_id + links.append( + {trace_id: trace_id, span_id: span_id, type: "CHILD_LINKED_SPAN"} + ) + return set(links) + + +def extract_events(events): + """Convert span.events to dict.""" + if not events: + return None + + logs = [] + + for event in events: + annotation_json = {"description": get_truncatable_str(event.name)} + if event.attributes is not None: + annotation_json["attributes"] = extract_attributes( + event.attributes + ) + + logs.append( + { + "time": ns_to_iso_str(event.timestamp), + "annotation": annotation_json, + } + ) + return {"timeEvent": logs} + + +def extract_attributes(attrs: types.Attributes): + """Convert span.attributes to dict.""" + attributes_json = {} + + for key, value in attrs.items(): + key = check_str_length(key)[0] + value = _format_attribute_value(value) + + if value is not None: + attributes_json[key] = value + + result = {"attributeMap": attributes_json} + + return result + + +def map_attributes(attribute_map): + """Convert the attributes to stackdriver attributes.""" + if attribute_map is None: + return attribute_map + for (key, value) in attribute_map.items(): + if key != "attributeMap": + continue + for attribute_key in list(value.keys()): + if attribute_key in ATTRIBUTE_MAPPING: + new_key = ATTRIBUTE_MAPPING.get(attribute_key) + value[new_key] = value.pop(attribute_key) + return attribute_map + + +def _format_attribute_value(value): + if isinstance(value, bool): + value_type = "bool_value" + elif isinstance(value, int): + value_type = "int_value" + elif isinstance(value, str): + value_type = "string_value" + value = get_truncatable_str(value) + elif isinstance(value, float): + value_type = "double_value" + else: + return None + + return {value_type: value} + + +ATTRIBUTE_MAPPING = { + "component": "/component", + "error.message": "/error/message", + "error.name": "/error/name", + "http.client_city": "/http/client_city", + "http.client_country": "/http/client_country", + "http.client_protocol": "/http/client_protocol", + "http.client_region": "/http/client_region", + "http.host": "/http/host", + "http.method": "/http/method", + "http.redirected_url": "/http/redirected_url", + "http.request_size": "/http/request/size", + "http.response_size": "/http/response/size", + "http.status_code": "/http/status_code", + "http.url": "/http/url", + "http.user_agent": "/http/user_agent", + "pid": "/pid", + "stacktrace": "/stacktrace", + "tid": "/tid", + "grpc.host_port": "/grpc/host_port", + "grpc.method": "/grpc/method", +} diff --git a/ext/opentelemetry-ext-stackdriver/src/opentelemetry/ext/stackdriver/version.py b/ext/opentelemetry-ext-stackdriver/src/opentelemetry/ext/stackdriver/version.py new file mode 100644 index 00000000000..93ef792d051 --- /dev/null +++ b/ext/opentelemetry-ext-stackdriver/src/opentelemetry/ext/stackdriver/version.py @@ -0,0 +1,15 @@ +# Copyright 2019, OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__version__ = "0.3.dev0" diff --git a/ext/opentelemetry-ext-stackdriver/tests/__init__.py b/ext/opentelemetry-ext-stackdriver/tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ext/opentelemetry-ext-stackdriver/tests/test_stackdriver_exporter.py b/ext/opentelemetry-ext-stackdriver/tests/test_stackdriver_exporter.py new file mode 100644 index 00000000000..d86ef4986a0 --- /dev/null +++ b/ext/opentelemetry-ext-stackdriver/tests/test_stackdriver_exporter.py @@ -0,0 +1,113 @@ +# Copyright 2019, OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +from unittest import mock + +import opentelemetry.ext.stackdriver.trace as sd_exporter +from opentelemetry.sdk.trace import Span +from opentelemetry.trace import SpanContext, SpanKind +from opentelemetry.util.version import __version__ + + +class TestStackdriverSpanExporter(unittest.TestCase): + def setUp(self): + self.client_patcher = mock.patch( + "opentelemetry.ext.stackdriver.trace.Client" + ) + self.client_patcher.start() + + def tearDown(self): + self.client_patcher.stop() + + def test_constructor_default(self): + exporter = sd_exporter.StackdriverSpanExporter() + self.assertEqual(exporter.project_id, exporter.client.project) + + def test_constructor_explicit(self): + client = mock.Mock() + project_id = "PROJECT" + client.project = project_id + + exporter = sd_exporter.StackdriverSpanExporter( + client=client, project_id=project_id + ) + + self.assertIs(exporter.client, client) + self.assertEqual(exporter.project_id, project_id) + + def test_export(self): + trace_id = "6e0c63257de34c92bf9efcd03927272e" + span_id = "95bb5edabd45950f" + # start_times = 683647322 * 10 ** 9 # in ns + # durations = 50 * 10 ** 6 + # end_times = start_times + durations + span_datas = [ + Span( + name="span_name", + context=SpanContext( + trace_id=int(trace_id, 16), span_id=int(span_id, 16) + ), + parent=None, + kind=SpanKind.INTERNAL, + ) + ] + + stackdriver_spans = { + "spans": [ + { + "name": "projects/PROJECT/traces/{}/spans/{}".format( + trace_id, span_id + ), + "spanId": span_id, + "parentSpanId": None, + "displayName": { + "value": "span_name", + "truncated_byte_count": 0, + }, + "attributes": { + "attributeMap": { + "g.co/agent": { + "string_value": { + "value": "opentelemetry-python [{}]".format( + __version__ + ), + "truncated_byte_count": 0, + } + } + } + }, + "links": None, + "status": {"details": None, "code": 0}, + "timeEvents": None, + "startTime": None, + "endTime": None, + } + ] + } + + client = mock.Mock() + project_id = "PROJECT" + client.project = project_id + + exporter = sd_exporter.StackdriverSpanExporter( + client=client, project_id=project_id + ) + + exporter.export(span_datas) + + name = "projects/{}".format(project_id) + + client.batch_write_spans.assert_called_with(name, stackdriver_spans) + self.assertTrue(client.batch_write_spans.called) diff --git a/scripts/coverage.sh b/scripts/coverage.sh index 0b45fbf643b..b90647a470a 100755 --- a/scripts/coverage.sh +++ b/scripts/coverage.sh @@ -36,6 +36,7 @@ cov ext/opentelemetry-ext-flask cov ext/opentelemetry-ext-requests cov ext/opentelemetry-ext-jaeger cov ext/opentelemetry-ext-opentracing-shim +cov ext/opentelemetry-ext-stackdriver cov ext/opentelemetry-ext-wsgi cov ext/opentelemetry-ext-zipkin cov docs/examples/opentelemetry-example-app diff --git a/tox.ini b/tox.ini index 381e8604e1e..70109525483 100644 --- a/tox.ini +++ b/tox.ini @@ -159,6 +159,7 @@ changedir = test-ext-opencensusexporter: ext/opentelemetry-ext-opencensusexporter/tests test-ext-prometheus: ext/opentelemetry-ext-prometheus/tests test-ext-pymongo: ext/opentelemetry-ext-pymongo/tests + test-ext-stackdriver: ext/opentelemetry-ext-stackdriver/tests test-ext-psycopg2: ext/opentelemetry-ext-psycopg2/tests test-ext-pymysql: ext/opentelemetry-ext-pymysql/tests test-ext-asgi: ext/opentelemetry-ext-asgi/tests From a332fe6f9a53dfade44586619dba867f39652aaf Mon Sep 17 00:00:00 2001 From: Andrew Xue Date: Mon, 18 May 2020 22:20:13 -0400 Subject: [PATCH 02/12] Add Cloud Trace Exporter --- docs-requirements.txt | 1 + docs/examples/cloud_trace_exporter/README.rst | 34 ++ .../cloud_trace_exporter/basic_trace.py | 14 + docs/ext/cloud_trace/cloud_trace.rst | 7 + .../README.rst | 43 +++ .../setup.cfg | 11 +- .../setup.py | 4 +- .../exporter/cloud_trace/__init__.py | 299 ++++++++++++++++++ .../exporter/cloud_trace}/version.py | 4 +- .../tests/__init__.py | 0 .../tests/test_cloud_trace_exporter.py | 256 +++++++++++++++ .../src/opentelemetry/ext/datadog/exporter.py | 3 +- .../tests/test_datadog_exporter.py | 1 - .../opentelemetry/ext/requests/__init__.py | 13 +- ext/opentelemetry-ext-stackdriver/README.rst | 44 --- .../examples/client.py | 32 -- .../examples/server.py | 44 --- .../examples/trace.py | 25 -- .../ext/stackdriver/trace/__init__.py | 281 ---------------- .../tests/test_stackdriver_exporter.py | 113 ------- scripts/coverage.sh | 2 +- tox.ini | 2 +- 22 files changed, 677 insertions(+), 556 deletions(-) create mode 100644 docs/examples/cloud_trace_exporter/README.rst create mode 100644 docs/examples/cloud_trace_exporter/basic_trace.py create mode 100644 docs/ext/cloud_trace/cloud_trace.rst create mode 100644 ext/opentelemetry-exporter-cloud-trace/README.rst rename ext/{opentelemetry-ext-stackdriver => opentelemetry-exporter-cloud-trace}/setup.cfg (85%) rename ext/{opentelemetry-ext-stackdriver => opentelemetry-exporter-cloud-trace}/setup.py (87%) create mode 100644 ext/opentelemetry-exporter-cloud-trace/src/opentelemetry/exporter/cloud_trace/__init__.py rename ext/{opentelemetry-ext-stackdriver/src/opentelemetry/ext/stackdriver => opentelemetry-exporter-cloud-trace/src/opentelemetry/exporter/cloud_trace}/version.py (89%) rename ext/{opentelemetry-ext-stackdriver => opentelemetry-exporter-cloud-trace}/tests/__init__.py (100%) create mode 100644 ext/opentelemetry-exporter-cloud-trace/tests/test_cloud_trace_exporter.py delete mode 100644 ext/opentelemetry-ext-stackdriver/README.rst delete mode 100644 ext/opentelemetry-ext-stackdriver/examples/client.py delete mode 100644 ext/opentelemetry-ext-stackdriver/examples/server.py delete mode 100644 ext/opentelemetry-ext-stackdriver/examples/trace.py delete mode 100644 ext/opentelemetry-ext-stackdriver/src/opentelemetry/ext/stackdriver/trace/__init__.py delete mode 100644 ext/opentelemetry-ext-stackdriver/tests/test_stackdriver_exporter.py diff --git a/docs-requirements.txt b/docs-requirements.txt index a61f0beedb3..875fa04d595 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -21,3 +21,4 @@ thrift>=0.10.0 wrapt>=1.0.0,<2.0.0 psutil~=5.7.0 boto~=2.0 +google-cloud-trace >=0.23.0 \ No newline at end of file diff --git a/docs/examples/cloud_trace_exporter/README.rst b/docs/examples/cloud_trace_exporter/README.rst new file mode 100644 index 00000000000..871422356a7 --- /dev/null +++ b/docs/examples/cloud_trace_exporter/README.rst @@ -0,0 +1,34 @@ +Cloud Trace Exporter Example +============================ + +These examples show how to use OpenTelemetry to send tracing data to Cloud Trace. + + +Basic Example +------------- + +To use this exporter you first need to: + * A Google Cloud project. You can `create one here. `_ + * Enable Cloud Trace API (aka StackDriver Trace API) in the project `here. `_ + * Enable `Default Application Credentials. `_ + +* Installation + +.. code-block:: sh + + pip install opentelemetry-api + pip install opentelemetry-sdk + pip install opentelemetry-exporter-cloud-trace + +* Run example + +.. code-block:: sh + + python basic_trace.py + +Checking Output +-------------------------- + +After running any of these examples, you can go to `Cloud Trace overview `_ to see the results. + +* `More information about exporters in general `_ \ No newline at end of file diff --git a/docs/examples/cloud_trace_exporter/basic_trace.py b/docs/examples/cloud_trace_exporter/basic_trace.py new file mode 100644 index 00000000000..76840a291ec --- /dev/null +++ b/docs/examples/cloud_trace_exporter/basic_trace.py @@ -0,0 +1,14 @@ +from opentelemetry import trace +from opentelemetry.exporter.cloud_trace import CloudTraceSpanExporter +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleExportSpanProcessor + +trace.set_tracer_provider(TracerProvider()) + +cloud_trace_exporter = CloudTraceSpanExporter() +trace.get_tracer_provider().add_span_processor( + SimpleExportSpanProcessor(cloud_trace_exporter) +) +tracer = trace.get_tracer(__name__) +with tracer.start_as_current_span("foo"): + print("Hello world!") diff --git a/docs/ext/cloud_trace/cloud_trace.rst b/docs/ext/cloud_trace/cloud_trace.rst new file mode 100644 index 00000000000..5914b00d1a4 --- /dev/null +++ b/docs/ext/cloud_trace/cloud_trace.rst @@ -0,0 +1,7 @@ +OpenTelemetry Cloud Trace Exporter +================================== + +.. automodule:: opentelemetry.exporter.cloud_trace + :members: + :undoc-members: + :show-inheritance: diff --git a/ext/opentelemetry-exporter-cloud-trace/README.rst b/ext/opentelemetry-exporter-cloud-trace/README.rst new file mode 100644 index 00000000000..001f163007e --- /dev/null +++ b/ext/opentelemetry-exporter-cloud-trace/README.rst @@ -0,0 +1,43 @@ +OpenTelemetry Cloud Trace Exporters +=================================== + +This library provides classes for exporting trace data to Google Cloud Trace. + +Installation +------------ + +:: + + pip install opentelemetry-exporter-cloud-trace + +Usage +----- + +.. code:: python + + from opentelemetry import trace + from opentelemetry.exporter.cloud_trace import CloudTraceSpanExporter + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import ( + SimpleExportSpanProcessor, + ) + + trace.set_tracer_provider(TracerProvider()) + + cloud_trace_exporter = CloudTraceSpanExporter( + project_id='my-gcloud-project', + ) + trace.get_tracer_provider().add_span_processor( + SimpleExportSpanProcessor(cloud_trace_exporter) + ) + tracer = trace.get_tracer(__name__) + with tracer.start_as_current_span('foo'): + print('Hello world!') + + + +References +---------- + +* `Cloud Trace `_ +* `OpenTelemetry Project `_ diff --git a/ext/opentelemetry-ext-stackdriver/setup.cfg b/ext/opentelemetry-exporter-cloud-trace/setup.cfg similarity index 85% rename from ext/opentelemetry-ext-stackdriver/setup.cfg rename to ext/opentelemetry-exporter-cloud-trace/setup.cfg index d3307eae772..df6c2ce587b 100644 --- a/ext/opentelemetry-ext-stackdriver/setup.cfg +++ b/ext/opentelemetry-exporter-cloud-trace/setup.cfg @@ -1,4 +1,4 @@ -# Copyright 2019, OpenTelemetry Authors +# Copyright OpenTelemetry Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,17 +13,17 @@ # limitations under the License. # [metadata] -name = opentelemetry-ext-stackdriver -description = Stackdriver integration for OpenTelemetry +name = opentelemetry-exporter-cloud-trace +description = Cloud Trace integration for OpenTelemetry long_description = file: README.rst long_description_content_type = text/x-rst author = OpenTelemetry Authors author_email = cncf-opentelemetry-contributors@lists.cncf.io -url = https://github.com/open-telemetry/opentelemetry-python/ext/opentelemetry-ext-stackdriver +url = https://github.com/open-telemetry/opentelemetry-python/ext/opentelemetry-exporter-cloud-trace platforms = any license = Apache-2.0 classifiers = - Development Status :: 3 - Alpha + Development Status :: 4 - Beta Intended Audience :: Developers License :: OSI Approved :: Apache Software License Programming Language :: Python @@ -41,7 +41,6 @@ packages=find_namespace: install_requires = opentelemetry-api opentelemetry-sdk - google-cloud-monitoring google-cloud-trace [options.packages.find] diff --git a/ext/opentelemetry-ext-stackdriver/setup.py b/ext/opentelemetry-exporter-cloud-trace/setup.py similarity index 87% rename from ext/opentelemetry-ext-stackdriver/setup.py rename to ext/opentelemetry-exporter-cloud-trace/setup.py index 8d43c44ffde..332cf41d01c 100644 --- a/ext/opentelemetry-ext-stackdriver/setup.py +++ b/ext/opentelemetry-exporter-cloud-trace/setup.py @@ -1,4 +1,4 @@ -# Copyright 2019, OpenTelemetry Authors +# Copyright OpenTelemetry Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,7 +17,7 @@ BASE_DIR = os.path.dirname(__file__) VERSION_FILENAME = os.path.join( - BASE_DIR, "src", "opentelemetry", "ext", "stackdriver", "version.py" + BASE_DIR, "src", "opentelemetry", "exporter", "cloud_trace", "version.py" ) PACKAGE_INFO = {} with open(VERSION_FILENAME) as f: diff --git a/ext/opentelemetry-exporter-cloud-trace/src/opentelemetry/exporter/cloud_trace/__init__.py b/ext/opentelemetry-exporter-cloud-trace/src/opentelemetry/exporter/cloud_trace/__init__.py new file mode 100644 index 00000000000..12a0cc01304 --- /dev/null +++ b/ext/opentelemetry-exporter-cloud-trace/src/opentelemetry/exporter/cloud_trace/__init__.py @@ -0,0 +1,299 @@ +# Copyright OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Cloud Trace Span Exporter for OpenTelemetry. Uses Cloud Trace Client's REST +API to export traces and spans for viewing in Cloud Trace. + +Usage +----- + +.. code-block:: python + + from opentelemetry import trace + from opentelemetry.exporter.cloud_trace import CloudTraceSpanExporter + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import SimpleExportSpanProcessor + + trace.set_tracer_provider(TracerProvider()) + + cloud_trace_exporter = CloudTraceSpanExporter() + trace.get_tracer_provider().add_span_processor( + SimpleExportSpanProcessor(cloud_trace_exporter) + ) + tracer = trace.get_tracer(__name__) + with tracer.start_as_current_span("foo"): + print("Hello world!") + + +API +--- +""" + +import logging +from typing import Any, Dict, List, Sequence, Tuple + +import google.auth +from google.cloud.trace_v2 import TraceServiceClient +from google.cloud.trace_v2.proto.trace_pb2 import AttributeValue +from google.cloud.trace_v2.proto.trace_pb2 import Span as ProtoSpan +from google.cloud.trace_v2.proto.trace_pb2 import TruncatableString +from google.rpc.status_pb2 import Status + +import opentelemetry.trace as trace_api +from opentelemetry.sdk.trace import Event +from opentelemetry.sdk.trace.export import Span, SpanExporter, SpanExportResult +from opentelemetry.util import types + +logger = logging.getLogger(__name__) + +# Max length is 128 bytes for a truncatable string. +MAX_LENGTH = 128 + + +class CloudTraceSpanExporter(SpanExporter): + """Cloud Trace span exporter for OpenTelemetry. + + Args: + project_id: ID of the cloud project that will receive the traces. + client: Cloud Trace client. If not given, will be taken from gcloud + default credentials + """ + + def __init__( + self, project_id=None, client=None, + ): + self.client = client or TraceServiceClient() + if not project_id: + _, self.project_id = google.auth.default() + else: + self.project_id = project_id + + def export(self, spans: Sequence[Span]) -> SpanExportResult: + """Export the spans to Cloud Trace. + + See: https://cloud.google.com/trace/docs/reference/v2/rest/v2/projects.traces/batchWrite + + Args: + spans: Tuple of spans to export + """ + cloud_trace_spans = [] + for span in self._translate_to_cloud_trace(spans): + try: + cloud_trace_spans.append(self.client.create_span(**span)) + # pylint: disable=broad-except + except Exception as ex: + logger.error("Error when creating span %s", span, exc_info=ex) + + try: + self.client.batch_write_spans( + "projects/{}".format(self.project_id), cloud_trace_spans, + ) + # pylint: disable=broad-except + except Exception as ex: + logger.error("Error while writing to Cloud Trace", exc_info=ex) + return SpanExportResult.FAILURE + + return SpanExportResult.SUCCESS + + def _translate_to_cloud_trace( + self, spans: Sequence[Span] + ) -> List[Dict[str, Any]]: + """Translate the spans to Cloud Trace format. + + Args: + spans: Tuple of spans to convert + """ + + cloud_trace_spans = [] + + for span in spans: + ctx = span.get_context() + trace_id = _get_hexadecimal_trace_id(ctx.trace_id) + span_id = _get_hexadecimal_span_id(ctx.span_id) + span_name = "projects/{}/traces/{}/spans/{}".format( + self.project_id, trace_id, span_id + ) + + parent_id = None + if span.parent: + parent_id = _get_hexadecimal_span_id(span.parent.span_id) + + start_time = _get_time_from_ns(span.start_time) + end_time = _get_time_from_ns(span.end_time) + + attributes = _extract_attributes(span.attributes) + + cloud_trace_spans.append( + { + "name": span_name, + "span_id": span_id, + "display_name": _get_truncatable_str_object(span.name), + "start_time": start_time, + "end_time": end_time, + "parent_span_id": parent_id, + "attributes": attributes, + "links": _extract_links(span.links), + "status": _extract_status(span.status), + "time_events": _extract_events(span.events), + } + ) + # TODO: Leverage more of the Cloud Trace API, e.g. + # same_process_as_parent_span and child_span_count + + return cloud_trace_spans + + def shutdown(self): + pass + + +def _get_hexadecimal_trace_id(trace_id: int) -> str: + return "{:032x}".format(trace_id) + + +def _get_hexadecimal_span_id(span_id: int) -> str: + return "{:016x}".format(span_id) + + +def _get_time_from_ns(nanoseconds: int) -> Dict: + """Given epoch nanoseconds, split into epoch milliseconds and remaining + nanoseconds""" + if not nanoseconds: + return None + seconds, nanos = divmod(nanoseconds, 1e9) + return {"seconds": int(seconds), "nanos": int(nanos)} + + +def _get_truncatable_str_object( + str_to_convert: str, max_length: int = MAX_LENGTH +): + """Truncate the string if it exceeds the length limit and record the + truncated bytes count.""" + truncated, truncated_byte_count = _truncate_str(str_to_convert, max_length) + + return TruncatableString( + value=truncated, truncated_byte_count=truncated_byte_count + ) + + +def _truncate_str( + str_to_check: str, limit: int = MAX_LENGTH +) -> Tuple[str, int]: + """Check the length of a string. If exceeds limit, then truncate it.""" + str_bytes = str_to_check.encode("utf-8") + str_len = len(str_bytes) + truncated_byte_count = 0 + + if str_len > limit: + truncated_byte_count = str_len - limit + str_bytes = str_bytes[:limit] + + result = str(str_bytes.decode("utf-8", errors="ignore")) + + return result, truncated_byte_count + + +def _extract_status(status: trace_api.Status) -> Status: + """Convert a Status object to protobuf object.""" + if not status: + return None + status_dict = {"details": None, "code": status.canonical_code.value} + + if status.description is not None: + status_dict["message"] = status.description + + return Status(**status_dict) + + +def _extract_links(links: Sequence[trace_api.Link]) -> ProtoSpan.Links: + """Convert span.links""" + if not links: + return None + extracted_links = [] + for link in links: + trace_id = _get_hexadecimal_trace_id(link.context.trace_id) + span_id = _get_hexadecimal_span_id(link.context.span_id) + extracted_links.append( + { + "trace_id": trace_id, + "span_id": span_id, + "type": "TYPE_UNSPECIFIED", + "attributes": _extract_attributes(link.attributes), + } + ) + return ProtoSpan.Links(link=extracted_links, dropped_links_count=0) + + +def _extract_events(events: Sequence[Event]) -> ProtoSpan.TimeEvents: + """Convert span.events to dict.""" + if not events: + return None + logs = [] + for event in events: + if len(event.attributes) > 4: + logger.warning( + "Event %s has more then 4 attributes, some will be truncated", + event.name, + ) + logs.append( + { + "time": _get_time_from_ns(event.timestamp), + "annotation": { + "description": _get_truncatable_str_object( + event.name, 256 + ), + "attributes": _extract_attributes(event.attributes), + }, + } + ) + return ProtoSpan.TimeEvents( + time_event=logs, + dropped_annotations_count=0, + dropped_message_events_count=0, + ) + + +def _extract_attributes(attrs: types.Attributes) -> ProtoSpan.Attributes: + """Convert span.attributes to dict.""" + attributes_dict = {} + + for key, value in attrs.items(): + key = _truncate_str(key)[0] + value = _format_attribute_value(value) + + if value is not None: + attributes_dict[key] = value + return ProtoSpan.Attributes(attribute_map=attributes_dict) + + +def _format_attribute_value(value: types.AttributeValue) -> AttributeValue: + if isinstance(value, bool): + value_type = "bool_value" + elif isinstance(value, int): + value_type = "int_value" + elif isinstance(value, str): + value_type = "string_value" + value = _get_truncatable_str_object(value) + elif isinstance(value, float): + value_type = "string_value" + value = _get_truncatable_str_object("{:0.4f}".format(value)) + else: + logger.warning( + "ignoring attribute value %s of type %s. Values type must be one " + "of bool, int, string or float", + value, + type(value), + ) + return None + + return AttributeValue(**{value_type: value}) diff --git a/ext/opentelemetry-ext-stackdriver/src/opentelemetry/ext/stackdriver/version.py b/ext/opentelemetry-exporter-cloud-trace/src/opentelemetry/exporter/cloud_trace/version.py similarity index 89% rename from ext/opentelemetry-ext-stackdriver/src/opentelemetry/ext/stackdriver/version.py rename to ext/opentelemetry-exporter-cloud-trace/src/opentelemetry/exporter/cloud_trace/version.py index 93ef792d051..ec792e9af10 100644 --- a/ext/opentelemetry-ext-stackdriver/src/opentelemetry/ext/stackdriver/version.py +++ b/ext/opentelemetry-exporter-cloud-trace/src/opentelemetry/exporter/cloud_trace/version.py @@ -1,4 +1,4 @@ -# Copyright 2019, OpenTelemetry Authors +# Copyright OpenTelemetry Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "0.3.dev0" +__version__ = "0.8.dev0" diff --git a/ext/opentelemetry-ext-stackdriver/tests/__init__.py b/ext/opentelemetry-exporter-cloud-trace/tests/__init__.py similarity index 100% rename from ext/opentelemetry-ext-stackdriver/tests/__init__.py rename to ext/opentelemetry-exporter-cloud-trace/tests/__init__.py diff --git a/ext/opentelemetry-exporter-cloud-trace/tests/test_cloud_trace_exporter.py b/ext/opentelemetry-exporter-cloud-trace/tests/test_cloud_trace_exporter.py new file mode 100644 index 00000000000..ce68c842e1e --- /dev/null +++ b/ext/opentelemetry-exporter-cloud-trace/tests/test_cloud_trace_exporter.py @@ -0,0 +1,256 @@ +# Copyright OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +from unittest import mock + +from google.cloud.trace_v2.proto.trace_pb2 import AttributeValue +from google.cloud.trace_v2.proto.trace_pb2 import Span as ProtoSpan +from google.cloud.trace_v2.proto.trace_pb2 import TruncatableString +from google.rpc.status_pb2 import Status + +from opentelemetry.exporter.cloud_trace import ( + CloudTraceSpanExporter, + _extract_attributes, + _extract_events, + _extract_links, + _extract_status, + _truncate_str, +) +from opentelemetry.sdk.trace import Event, Span +from opentelemetry.trace import DefaultSpan, Link, SpanContext, SpanKind +from opentelemetry.trace.status import Status as SpanStatus +from opentelemetry.trace.status import StatusCanonicalCode + + +class TestCloudTraceSpanExporter(unittest.TestCase): + def setUp(self): + self.client_patcher = mock.patch( + "opentelemetry.exporter.cloud_trace.TraceServiceClient" + ) + self.client_patcher.start() + self.project_id = "PROJECT" + self.attributes_variety_pack = { + "str_key": "str_value", + "bool_key": False, + "double_key": 1.421, + "int_key": 123, + "int_key2": 1234, + } + self.extracted_attributes_variety_pack = ProtoSpan.Attributes( + attribute_map={ + "str_key": AttributeValue( + string_value=TruncatableString( + value="str_value", truncated_byte_count=0 + ) + ), + "bool_key": AttributeValue(bool_value=False), + "double_key": AttributeValue( + string_value=TruncatableString( + value="1.421", truncated_byte_count=0 + ) + ), + "int_key": AttributeValue(int_value=123), + "int_key2": AttributeValue(int_value=1234), + } + ) + + def tearDown(self): + self.client_patcher.stop() + + def test_constructor_default(self): + exporter = CloudTraceSpanExporter(self.project_id) + self.assertEqual(exporter.project_id, self.project_id) + + def test_constructor_explicit(self): + client = mock.Mock() + exporter = CloudTraceSpanExporter(self.project_id, client=client) + + self.assertIs(exporter.client, client) + self.assertEqual(exporter.project_id, self.project_id) + + def test_export(self): + trace_id = "6e0c63257de34c92bf9efcd03927272e" + span_id = "95bb5edabd45950f" + span_datas = [ + Span( + name="span_name", + context=SpanContext( + trace_id=int(trace_id, 16), + span_id=int(span_id, 16), + is_remote=False, + ), + parent=None, + kind=SpanKind.INTERNAL, + ) + ] + + cloud_trace_spans = { + "name": "projects/{}/traces/{}/spans/{}".format( + self.project_id, trace_id, span_id + ), + "span_id": span_id, + "parent_span_id": None, + "display_name": TruncatableString( + value="span_name", truncated_byte_count=0 + ), + "attributes": ProtoSpan.Attributes(attribute_map={}), + "links": None, + "status": None, + "time_events": None, + "start_time": None, + "end_time": None, + } + + client = mock.Mock() + + exporter = CloudTraceSpanExporter(self.project_id, client=client) + + exporter.export(span_datas) + + client.create_span.assert_called_with(**cloud_trace_spans) + self.assertTrue(client.create_span.called) + + def test_extract_status(self): + self.assertIsNone(_extract_status(None)) + self.assertEqual( + _extract_status(SpanStatus(canonical_code=StatusCanonicalCode.OK)), + Status(details=None, code=0), + ) + self.assertEqual( + _extract_status( + SpanStatus( + canonical_code=StatusCanonicalCode.UNKNOWN, + description="error_desc", + ) + ), + Status(details=None, code=2, message="error_desc"), + ) + + def test_extract_attributes(self): + self.assertEqual( + _extract_attributes({}), ProtoSpan.Attributes(attribute_map={}) + ) + self.assertEqual( + _extract_attributes(self.attributes_variety_pack), + self.extracted_attributes_variety_pack, + ) + # Test ignoring attributes with illegal value type + self.assertEqual( + _extract_attributes({"illegal_attribute_value": dict()}), + ProtoSpan.Attributes(attribute_map={}), + ) + + def test_extract_events(self): + self.assertIsNone(_extract_events([])) + time_in_ns1 = 1589919268850900051 + time_in_ms_and_ns1 = {"seconds": 1589919268, "nanos": 850899968} + time_in_ns2 = 1589919438550020326 + time_in_ms_and_ns2 = {"seconds": 1589919438, "nanos": 550020352} + event1 = Event( + name="event1", + attributes=self.attributes_variety_pack, + timestamp=time_in_ns1, + ) + event2 = Event( + name="event2", + attributes={"illegal_attr_value": dict()}, + timestamp=time_in_ns2, + ) + self.assertEqual( + _extract_events([event1, event2]), + ProtoSpan.TimeEvents( + time_event=[ + { + "time": time_in_ms_and_ns1, + "annotation": { + "description": TruncatableString( + value="event1", truncated_byte_count=0 + ), + "attributes": self.extracted_attributes_variety_pack, + }, + }, + { + "time": time_in_ms_and_ns2, + "annotation": { + "description": TruncatableString( + value="event2", truncated_byte_count=0 + ), + "attributes": ProtoSpan.Attributes( + attribute_map={} + ), + }, + }, + ] + ), + ) + + def test_extract_links(self): + self.assertIsNone(_extract_links([])) + trace_id = "6e0c63257de34c92bf9efcd03927272e" + span_id1 = "95bb5edabd45950f" + span_id2 = "b6b86ad2915c9ddc" + link1 = Link( + context=SpanContext( + trace_id=int(trace_id, 16), + span_id=int(span_id1, 16), + is_remote=False, + ), + attributes={}, + ) + link2 = Link( + context=SpanContext( + trace_id=int(trace_id, 16), + span_id=int(span_id1, 16), + is_remote=False, + ), + attributes=self.attributes_variety_pack, + ) + link3 = Link( + context=SpanContext( + trace_id=int(trace_id, 16), + span_id=int(span_id2, 16), + is_remote=False, + ), + attributes={"illegal_attr_value": dict(), "int_attr_value": 123}, + ) + self.assertEqual( + _extract_links([link1, link2, link3]), + ProtoSpan.Links( + link=[ + { + "trace_id": trace_id, + "span_id": span_id1, + "type": "TYPE_UNSPECIFIED", + "attributes": ProtoSpan.Attributes(attribute_map={}), + }, + { + "trace_id": trace_id, + "span_id": span_id1, + "type": "TYPE_UNSPECIFIED", + "attributes": self.extracted_attributes_variety_pack, + }, + { + "trace_id": trace_id, + "span_id": span_id2, + "type": "TYPE_UNSPECIFIED", + "attributes": { + "attribute_map": { + "int_attr_value": AttributeValue(int_value=123) + } + }, + }, + ] + ), + ) diff --git a/ext/opentelemetry-ext-datadog/src/opentelemetry/ext/datadog/exporter.py b/ext/opentelemetry-ext-datadog/src/opentelemetry/ext/datadog/exporter.py index 35d0f98203c..0fbedcb5884 100644 --- a/ext/opentelemetry-ext-datadog/src/opentelemetry/ext/datadog/exporter.py +++ b/ext/opentelemetry-ext-datadog/src/opentelemetry/ext/datadog/exporter.py @@ -16,11 +16,10 @@ import os from urllib.parse import urlparse +import opentelemetry.trace as trace_api from ddtrace.ext import SpanTypes as DatadogSpanTypes from ddtrace.internal.writer import AgentWriter from ddtrace.span import Span as DatadogSpan - -import opentelemetry.trace as trace_api from opentelemetry.sdk.trace.export import SpanExporter, SpanExportResult from opentelemetry.trace.status import StatusCanonicalCode diff --git a/ext/opentelemetry-ext-datadog/tests/test_datadog_exporter.py b/ext/opentelemetry-ext-datadog/tests/test_datadog_exporter.py index 378bef4147e..8447058b5e5 100644 --- a/ext/opentelemetry-ext-datadog/tests/test_datadog_exporter.py +++ b/ext/opentelemetry-ext-datadog/tests/test_datadog_exporter.py @@ -19,7 +19,6 @@ from unittest import mock from ddtrace.internal.writer import AgentWriter - from opentelemetry import trace as trace_api from opentelemetry.ext import datadog from opentelemetry.sdk import trace diff --git a/ext/opentelemetry-ext-requests/src/opentelemetry/ext/requests/__init__.py b/ext/opentelemetry-ext-requests/src/opentelemetry/ext/requests/__init__.py index 1621c4a95e6..098fc441220 100644 --- a/ext/opentelemetry-ext-requests/src/opentelemetry/ext/requests/__init__.py +++ b/ext/opentelemetry-ext-requests/src/opentelemetry/ext/requests/__init__.py @@ -50,9 +50,15 @@ from opentelemetry import context, propagators, trace from opentelemetry.auto_instrumentation.instrumentor import BaseInstrumentor from opentelemetry.ext.requests.version import __version__ -from opentelemetry.trace import SpanKind, get_tracer +from opentelemetry.trace import SpanKind from opentelemetry.trace.status import Status, StatusCanonicalCode +# StackDriver exporter spins up a new thread (that doesn't inherit the +# "suppress_instrumentation" context) that makes a request call. We need to +# manually blacklist the url to avoid falling into an infinite loop. +# https://github.com/GoogleCloudPlatform/opentelemetry-operations-python/issues/3 +SUPPRESSION_BLACKLIST = ["https://oauth2.googleapis.com/token"] + # pylint: disable=unused-argument def _instrument(tracer_provider=None, span_callback=None): @@ -73,7 +79,10 @@ def _instrument(tracer_provider=None, span_callback=None): @functools.wraps(wrapped) def instrumented_request(self, method, url, *args, **kwargs): - if context.get_value("suppress_instrumentation"): + if ( + context.get_value("suppress_instrumentation") + or url in SUPPRESSION_BLACKLIST + ): return wrapped(self, method, url, *args, **kwargs) # See diff --git a/ext/opentelemetry-ext-stackdriver/README.rst b/ext/opentelemetry-ext-stackdriver/README.rst deleted file mode 100644 index fcf2a08e014..00000000000 --- a/ext/opentelemetry-ext-stackdriver/README.rst +++ /dev/null @@ -1,44 +0,0 @@ -OpenTelemetry Stackdriver Exporters -===================================== - -This library provides integration with Google Cloud Stackdriver. - -Installation ------------- - -:: - - pip install opentelemetry-ext-stackdriver - -Usage ------ - -.. code:: python - - from opentelemetry import trace - from opentelemetry.ext import stackdriver - from opentelemetry.sdk.trace import Tracer - from opentelemetry.sdk.trace.export import BatchExportSpanProcessor - - trace.set_preferred_tracer_implementation(lambda T: Tracer()) - tracer = trace.tracer() - - # create a StackdriverSpanExporter - stackdriver_exporter = stackdriver.trace.StackdriverSpanExporter( - project_id='my-helloworld-project', - ) - - # Create a BatchExportSpanProcessor and add the exporter to it - span_processor = BatchExportSpanProcessor(stackdriver_exporter) - - # add to the tracer - tracer.add_span_processor(span_processor) - - with tracer.start_as_current_span('foo'): - print('Hello world!') - -References ----------- - -* `Stackdriver `_ -* `OpenTelemetry Project `_ diff --git a/ext/opentelemetry-ext-stackdriver/examples/client.py b/ext/opentelemetry-ext-stackdriver/examples/client.py deleted file mode 100644 index ee6206d9470..00000000000 --- a/ext/opentelemetry-ext-stackdriver/examples/client.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2019, OpenTelemetry Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import requests - -from opentelemetry import trace -from opentelemetry.ext import http_requests -from opentelemetry.ext.stackdriver.trace import StackdriverSpanExporter -from opentelemetry.sdk.trace import Tracer -from opentelemetry.sdk.trace.export import SimpleExportSpanProcessor - -trace.set_preferred_tracer_implementation(lambda T: Tracer()) -tracer = trace.tracer() -span_processor = SimpleExportSpanProcessor( - StackdriverSpanExporter(project_id="my-helloworld-project") -) -tracer.add_span_processor(span_processor) - -http_requests.enable(tracer) -response = requests.get(url="http://localhost:7777/hello") -span_processor.shutdown() diff --git a/ext/opentelemetry-ext-stackdriver/examples/server.py b/ext/opentelemetry-ext-stackdriver/examples/server.py deleted file mode 100644 index 8cd74a653e3..00000000000 --- a/ext/opentelemetry-ext-stackdriver/examples/server.py +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright 2019, OpenTelemetry Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import flask -import requests - -from opentelemetry import trace -from opentelemetry.ext import http_requests -from opentelemetry.ext.stackdriver.trace import StackdriverSpanExporter -from opentelemetry.ext.wsgi import OpenTelemetryMiddleware -from opentelemetry.sdk.trace import Tracer -from opentelemetry.sdk.trace.export import BatchExportSpanProcessor - -trace.set_preferred_tracer_implementation(lambda T: Tracer()) - -span_processor = BatchExportSpanProcessor(StackdriverSpanExporter()) -http_requests.enable(trace.tracer()) -trace.tracer().add_span_processor(span_processor) - -app = flask.Flask(__name__) -app.wsgi_app = OpenTelemetryMiddleware(app.wsgi_app) - - -@app.route("/") -def hello(): - with trace.tracer().start_as_current_span("parent"): - requests.get("https://www.wikipedia.org/wiki/Rabbit") - return "hello" - - -if __name__ == "__main__": - app.run(debug=True) - span_processor.shutdown() diff --git a/ext/opentelemetry-ext-stackdriver/examples/trace.py b/ext/opentelemetry-ext-stackdriver/examples/trace.py deleted file mode 100644 index 1cdb41ecc0e..00000000000 --- a/ext/opentelemetry-ext-stackdriver/examples/trace.py +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2019, OpenTelemetry Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from opentelemetry import trace -from opentelemetry.ext.stackdriver.trace import StackdriverSpanExporter -from opentelemetry.sdk.trace import Tracer -from opentelemetry.sdk.trace.export import SimpleExportSpanProcessor - -trace.set_preferred_tracer_implementation(lambda T: Tracer()) -tracer = trace.tracer() -tracer.add_span_processor(SimpleExportSpanProcessor(StackdriverSpanExporter())) - -with tracer.start_as_current_span("hello") as span: - print("Hello, World!") diff --git a/ext/opentelemetry-ext-stackdriver/src/opentelemetry/ext/stackdriver/trace/__init__.py b/ext/opentelemetry-ext-stackdriver/src/opentelemetry/ext/stackdriver/trace/__init__.py deleted file mode 100644 index 6995cbe8633..00000000000 --- a/ext/opentelemetry-ext-stackdriver/src/opentelemetry/ext/stackdriver/trace/__init__.py +++ /dev/null @@ -1,281 +0,0 @@ -# Copyright 2019, OpenTelemetry Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Stackdriver Span Exporter for OpenTelemetry.""" - -import logging -import typing - -from google.cloud.trace import trace_service_client -from google.cloud.trace.client import Client -from google.cloud.trace_v2.proto import trace_pb2 - -import opentelemetry.trace as trace_api -from opentelemetry.context import Context -from opentelemetry.sdk.trace.export import Span, SpanExporter, SpanExportResult -from opentelemetry.sdk.util import ns_to_iso_str -from opentelemetry.util import types - -from ..version import __version__ - -logger = logging.getLogger(__name__) - -AGENT = "opentelemetry-python [{}]".format(__version__) -# Max length is 128 bytes for a truncatable string. -MAX_LENGTH = 128 - - -class StackdriverSpanExporter(SpanExporter): - """Stackdriver span exporter for OpenTelemetry. - - Args: - client: Stackdriver Trace client. - project_id: project_id to create the Trace client. - """ - - def __init__( - self, client=None, project_id=None, - ): - if client is None: - client = Client(project=project_id) - self.client = client - self.project_id = self.client.project - - def export(self, spans: typing.Sequence[Span]) -> SpanExportResult: - """Export the spans to Stackdriver. - - See: https://cloud.google.com/trace/docs/reference/v2/rest/v2/ - projects.traces/batchWrite - - Args: - spans: Tuple of spans to export - """ - stackdriver_spans = self.translate_to_stackdriver(spans) - - try: - self.client.batch_write_spans( - "projects/{}".format(self.project_id), - {"spans": stackdriver_spans}, - ) - except Exception as ex: - logger.warning("Error while writing to stackdriver: %s", ex) - return SpanExportResult.FAILED_RETRYABLE - - return SpanExportResult.SUCCESS - - def translate_to_stackdriver( - self, spans: typing.Sequence[Span] - ) -> typing.List[typing.Dict[str, typing.Any]]: - """Translate the spans to Stackdriver format. - - Args: - spans: Tuple of spans to convert - """ - - stackdriver_spans = [] - - for span in spans: - ctx = span.get_context() - trace_id = "{:032x}".format(ctx.trace_id) - span_id = "{:016x}".format(ctx.span_id) - span_name = "projects/{}/traces/{}/spans/{}".format( - self.project_id, trace_id, span_id - ) - - parent_id = None - if isinstance(span.parent, trace_api.Span): - parent_id = "{:016x}".format(span.parent.get_context().span_id) - elif isinstance(span.parent, trace_api.SpanContext): - parent_id = "{:016x}".format(span.parent.span_id) - - start_time = None - if span.start_time: - start_time = ns_to_iso_str(span.start_time) - end_time = None - if span.end_time: - end_time = ns_to_iso_str(span.end_time) - - span.attributes["g.co/agent"] = AGENT - attr_map = extract_attributes(span.attributes) - - sd_span = { - "name": span_name, - "spanId": span_id, - "parentSpanId": parent_id, - "displayName": get_truncatable_str(span.name), - "attributes": map_attributes(attr_map), - "links": extract_links(span.links), - "status": extract_status(span.status), - "timeEvents": extract_events(span.events), - "startTime": start_time, - "endTime": end_time, - } - - stackdriver_spans.append(sd_span) - - return stackdriver_spans - - def shutdown(self): - pass - - -def get_truncatable_str(str_to_convert): - """Truncate a string if exceed limit and record the truncated bytes - count. - """ - truncated, truncated_byte_count = check_str_length( - str_to_convert, MAX_LENGTH - ) - - result = { - "value": truncated, - "truncated_byte_count": truncated_byte_count, - } - return result - - -def check_str_length(str_to_check, limit=MAX_LENGTH): - """Check the length of a string. If exceeds limit, then truncate it. - """ - str_bytes = str_to_check.encode("utf-8") - str_len = len(str_bytes) - truncated_byte_count = 0 - - if str_len > limit: - truncated_byte_count = str_len - limit - str_bytes = str_bytes[:limit] - - result = str(str_bytes.decode("utf-8", errors="ignore")) - - return (result, truncated_byte_count) - - -def extract_status(status: trace_api.Status): - """Convert a Status object to dict.""" - status_json = {"details": None} - - status_json["code"] = status.canonical_code.value - - if status.description is not None: - status_json["message"] = status.description - - return status_json - - -def extract_links(links): - """Convert span.links to set.""" - if not links: - return None - - links = [] - for link in links: - trace_id = link.context.trace_id - span_id = link.context.span_id - links.append( - {trace_id: trace_id, span_id: span_id, type: "CHILD_LINKED_SPAN"} - ) - return set(links) - - -def extract_events(events): - """Convert span.events to dict.""" - if not events: - return None - - logs = [] - - for event in events: - annotation_json = {"description": get_truncatable_str(event.name)} - if event.attributes is not None: - annotation_json["attributes"] = extract_attributes( - event.attributes - ) - - logs.append( - { - "time": ns_to_iso_str(event.timestamp), - "annotation": annotation_json, - } - ) - return {"timeEvent": logs} - - -def extract_attributes(attrs: types.Attributes): - """Convert span.attributes to dict.""" - attributes_json = {} - - for key, value in attrs.items(): - key = check_str_length(key)[0] - value = _format_attribute_value(value) - - if value is not None: - attributes_json[key] = value - - result = {"attributeMap": attributes_json} - - return result - - -def map_attributes(attribute_map): - """Convert the attributes to stackdriver attributes.""" - if attribute_map is None: - return attribute_map - for (key, value) in attribute_map.items(): - if key != "attributeMap": - continue - for attribute_key in list(value.keys()): - if attribute_key in ATTRIBUTE_MAPPING: - new_key = ATTRIBUTE_MAPPING.get(attribute_key) - value[new_key] = value.pop(attribute_key) - return attribute_map - - -def _format_attribute_value(value): - if isinstance(value, bool): - value_type = "bool_value" - elif isinstance(value, int): - value_type = "int_value" - elif isinstance(value, str): - value_type = "string_value" - value = get_truncatable_str(value) - elif isinstance(value, float): - value_type = "double_value" - else: - return None - - return {value_type: value} - - -ATTRIBUTE_MAPPING = { - "component": "/component", - "error.message": "/error/message", - "error.name": "/error/name", - "http.client_city": "/http/client_city", - "http.client_country": "/http/client_country", - "http.client_protocol": "/http/client_protocol", - "http.client_region": "/http/client_region", - "http.host": "/http/host", - "http.method": "/http/method", - "http.redirected_url": "/http/redirected_url", - "http.request_size": "/http/request/size", - "http.response_size": "/http/response/size", - "http.status_code": "/http/status_code", - "http.url": "/http/url", - "http.user_agent": "/http/user_agent", - "pid": "/pid", - "stacktrace": "/stacktrace", - "tid": "/tid", - "grpc.host_port": "/grpc/host_port", - "grpc.method": "/grpc/method", -} diff --git a/ext/opentelemetry-ext-stackdriver/tests/test_stackdriver_exporter.py b/ext/opentelemetry-ext-stackdriver/tests/test_stackdriver_exporter.py deleted file mode 100644 index d86ef4986a0..00000000000 --- a/ext/opentelemetry-ext-stackdriver/tests/test_stackdriver_exporter.py +++ /dev/null @@ -1,113 +0,0 @@ -# Copyright 2019, OpenTelemetry Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import unittest -from unittest import mock - -import opentelemetry.ext.stackdriver.trace as sd_exporter -from opentelemetry.sdk.trace import Span -from opentelemetry.trace import SpanContext, SpanKind -from opentelemetry.util.version import __version__ - - -class TestStackdriverSpanExporter(unittest.TestCase): - def setUp(self): - self.client_patcher = mock.patch( - "opentelemetry.ext.stackdriver.trace.Client" - ) - self.client_patcher.start() - - def tearDown(self): - self.client_patcher.stop() - - def test_constructor_default(self): - exporter = sd_exporter.StackdriverSpanExporter() - self.assertEqual(exporter.project_id, exporter.client.project) - - def test_constructor_explicit(self): - client = mock.Mock() - project_id = "PROJECT" - client.project = project_id - - exporter = sd_exporter.StackdriverSpanExporter( - client=client, project_id=project_id - ) - - self.assertIs(exporter.client, client) - self.assertEqual(exporter.project_id, project_id) - - def test_export(self): - trace_id = "6e0c63257de34c92bf9efcd03927272e" - span_id = "95bb5edabd45950f" - # start_times = 683647322 * 10 ** 9 # in ns - # durations = 50 * 10 ** 6 - # end_times = start_times + durations - span_datas = [ - Span( - name="span_name", - context=SpanContext( - trace_id=int(trace_id, 16), span_id=int(span_id, 16) - ), - parent=None, - kind=SpanKind.INTERNAL, - ) - ] - - stackdriver_spans = { - "spans": [ - { - "name": "projects/PROJECT/traces/{}/spans/{}".format( - trace_id, span_id - ), - "spanId": span_id, - "parentSpanId": None, - "displayName": { - "value": "span_name", - "truncated_byte_count": 0, - }, - "attributes": { - "attributeMap": { - "g.co/agent": { - "string_value": { - "value": "opentelemetry-python [{}]".format( - __version__ - ), - "truncated_byte_count": 0, - } - } - } - }, - "links": None, - "status": {"details": None, "code": 0}, - "timeEvents": None, - "startTime": None, - "endTime": None, - } - ] - } - - client = mock.Mock() - project_id = "PROJECT" - client.project = project_id - - exporter = sd_exporter.StackdriverSpanExporter( - client=client, project_id=project_id - ) - - exporter.export(span_datas) - - name = "projects/{}".format(project_id) - - client.batch_write_spans.assert_called_with(name, stackdriver_spans) - self.assertTrue(client.batch_write_spans.called) diff --git a/scripts/coverage.sh b/scripts/coverage.sh index b90647a470a..1794cdf01b7 100755 --- a/scripts/coverage.sh +++ b/scripts/coverage.sh @@ -36,7 +36,7 @@ cov ext/opentelemetry-ext-flask cov ext/opentelemetry-ext-requests cov ext/opentelemetry-ext-jaeger cov ext/opentelemetry-ext-opentracing-shim -cov ext/opentelemetry-ext-stackdriver +cov ext/opentelemetry-exporter-cloud-trace cov ext/opentelemetry-ext-wsgi cov ext/opentelemetry-ext-zipkin cov docs/examples/opentelemetry-example-app diff --git a/tox.ini b/tox.ini index 70109525483..026d01cc0c1 100644 --- a/tox.ini +++ b/tox.ini @@ -159,7 +159,7 @@ changedir = test-ext-opencensusexporter: ext/opentelemetry-ext-opencensusexporter/tests test-ext-prometheus: ext/opentelemetry-ext-prometheus/tests test-ext-pymongo: ext/opentelemetry-ext-pymongo/tests - test-ext-stackdriver: ext/opentelemetry-ext-stackdriver/tests + test-exporter-cloud-trace: ext/opentelemetry-exporter-cloud-trace/tests test-ext-psycopg2: ext/opentelemetry-ext-psycopg2/tests test-ext-pymysql: ext/opentelemetry-ext-pymysql/tests test-ext-asgi: ext/opentelemetry-ext-asgi/tests From 1ec87a8990fff79fa89aac281c6634a700313a0b Mon Sep 17 00:00:00 2001 From: Andrew Xue Date: Mon, 1 Jun 2020 12:53:32 -0400 Subject: [PATCH 03/12] revert requests change --- .../src/opentelemetry/ext/requests/__init__.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/ext/opentelemetry-ext-requests/src/opentelemetry/ext/requests/__init__.py b/ext/opentelemetry-ext-requests/src/opentelemetry/ext/requests/__init__.py index 098fc441220..20412acf671 100644 --- a/ext/opentelemetry-ext-requests/src/opentelemetry/ext/requests/__init__.py +++ b/ext/opentelemetry-ext-requests/src/opentelemetry/ext/requests/__init__.py @@ -53,12 +53,6 @@ from opentelemetry.trace import SpanKind from opentelemetry.trace.status import Status, StatusCanonicalCode -# StackDriver exporter spins up a new thread (that doesn't inherit the -# "suppress_instrumentation" context) that makes a request call. We need to -# manually blacklist the url to avoid falling into an infinite loop. -# https://github.com/GoogleCloudPlatform/opentelemetry-operations-python/issues/3 -SUPPRESSION_BLACKLIST = ["https://oauth2.googleapis.com/token"] - # pylint: disable=unused-argument def _instrument(tracer_provider=None, span_callback=None): @@ -79,10 +73,7 @@ def _instrument(tracer_provider=None, span_callback=None): @functools.wraps(wrapped) def instrumented_request(self, method, url, *args, **kwargs): - if ( - context.get_value("suppress_instrumentation") - or url in SUPPRESSION_BLACKLIST - ): + if context.get_value("suppress_instrumentation"): return wrapped(self, method, url, *args, **kwargs) # See From 1fae33112c704807c2e2b4a78b54457c0e194fbc Mon Sep 17 00:00:00 2001 From: Andrew Xue Date: Mon, 1 Jun 2020 12:55:50 -0400 Subject: [PATCH 04/12] revert --- .../src/opentelemetry/ext/requests/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opentelemetry-ext-requests/src/opentelemetry/ext/requests/__init__.py b/ext/opentelemetry-ext-requests/src/opentelemetry/ext/requests/__init__.py index 20412acf671..1621c4a95e6 100644 --- a/ext/opentelemetry-ext-requests/src/opentelemetry/ext/requests/__init__.py +++ b/ext/opentelemetry-ext-requests/src/opentelemetry/ext/requests/__init__.py @@ -50,7 +50,7 @@ from opentelemetry import context, propagators, trace from opentelemetry.auto_instrumentation.instrumentor import BaseInstrumentor from opentelemetry.ext.requests.version import __version__ -from opentelemetry.trace import SpanKind +from opentelemetry.trace import SpanKind, get_tracer from opentelemetry.trace.status import Status, StatusCanonicalCode From eb91bba2805ccd521736471b607b163992264e3c Mon Sep 17 00:00:00 2001 From: Andrew Xue Date: Mon, 1 Jun 2020 12:56:57 -0400 Subject: [PATCH 05/12] f --- docs-requirements.txt | 2 +- .../src/opentelemetry/ext/datadog/exporter.py | 3 ++- ext/opentelemetry-ext-datadog/tests/test_datadog_exporter.py | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs-requirements.txt b/docs-requirements.txt index 875fa04d595..db10f6f9ee4 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -21,4 +21,4 @@ thrift>=0.10.0 wrapt>=1.0.0,<2.0.0 psutil~=5.7.0 boto~=2.0 -google-cloud-trace >=0.23.0 \ No newline at end of file +google-cloud-trace >=0.23.0 diff --git a/ext/opentelemetry-ext-datadog/src/opentelemetry/ext/datadog/exporter.py b/ext/opentelemetry-ext-datadog/src/opentelemetry/ext/datadog/exporter.py index 0fbedcb5884..35d0f98203c 100644 --- a/ext/opentelemetry-ext-datadog/src/opentelemetry/ext/datadog/exporter.py +++ b/ext/opentelemetry-ext-datadog/src/opentelemetry/ext/datadog/exporter.py @@ -16,10 +16,11 @@ import os from urllib.parse import urlparse -import opentelemetry.trace as trace_api from ddtrace.ext import SpanTypes as DatadogSpanTypes from ddtrace.internal.writer import AgentWriter from ddtrace.span import Span as DatadogSpan + +import opentelemetry.trace as trace_api from opentelemetry.sdk.trace.export import SpanExporter, SpanExportResult from opentelemetry.trace.status import StatusCanonicalCode diff --git a/ext/opentelemetry-ext-datadog/tests/test_datadog_exporter.py b/ext/opentelemetry-ext-datadog/tests/test_datadog_exporter.py index 8447058b5e5..378bef4147e 100644 --- a/ext/opentelemetry-ext-datadog/tests/test_datadog_exporter.py +++ b/ext/opentelemetry-ext-datadog/tests/test_datadog_exporter.py @@ -19,6 +19,7 @@ from unittest import mock from ddtrace.internal.writer import AgentWriter + from opentelemetry import trace as trace_api from opentelemetry.ext import datadog from opentelemetry.sdk import trace From 5c17f91ed2fe88730bab45c193eef89bfcbb323e Mon Sep 17 00:00:00 2001 From: Andrew Xue Date: Mon, 1 Jun 2020 13:25:28 -0400 Subject: [PATCH 06/12] add tests for truncation --- .../cloud_trace_exporter/basic_trace.py | 4 +- .../exporter/cloud_trace/__init__.py | 36 ++++--------- .../tests/test_cloud_trace_exporter.py | 53 ++++++++++++++++++- 3 files changed, 64 insertions(+), 29 deletions(-) diff --git a/docs/examples/cloud_trace_exporter/basic_trace.py b/docs/examples/cloud_trace_exporter/basic_trace.py index 76840a291ec..1626c672142 100644 --- a/docs/examples/cloud_trace_exporter/basic_trace.py +++ b/docs/examples/cloud_trace_exporter/basic_trace.py @@ -10,5 +10,7 @@ SimpleExportSpanProcessor(cloud_trace_exporter) ) tracer = trace.get_tracer(__name__) -with tracer.start_as_current_span("foo"): +with tracer.start_as_current_span("foo") as span: + value = 'a' * 256 + span.set_attribute('key', value) print("Hello world!") diff --git a/ext/opentelemetry-exporter-cloud-trace/src/opentelemetry/exporter/cloud_trace/__init__.py b/ext/opentelemetry-exporter-cloud-trace/src/opentelemetry/exporter/cloud_trace/__init__.py index 12a0cc01304..9afac7bf771 100644 --- a/ext/opentelemetry-exporter-cloud-trace/src/opentelemetry/exporter/cloud_trace/__init__.py +++ b/ext/opentelemetry-exporter-cloud-trace/src/opentelemetry/exporter/cloud_trace/__init__.py @@ -44,22 +44,18 @@ from typing import Any, Dict, List, Sequence, Tuple import google.auth +import opentelemetry.trace as trace_api from google.cloud.trace_v2 import TraceServiceClient from google.cloud.trace_v2.proto.trace_pb2 import AttributeValue from google.cloud.trace_v2.proto.trace_pb2 import Span as ProtoSpan from google.cloud.trace_v2.proto.trace_pb2 import TruncatableString from google.rpc.status_pb2 import Status - -import opentelemetry.trace as trace_api from opentelemetry.sdk.trace import Event from opentelemetry.sdk.trace.export import Span, SpanExporter, SpanExportResult from opentelemetry.util import types logger = logging.getLogger(__name__) -# Max length is 128 bytes for a truncatable string. -MAX_LENGTH = 128 - class CloudTraceSpanExporter(SpanExporter): """Cloud Trace span exporter for OpenTelemetry. @@ -138,7 +134,9 @@ def _translate_to_cloud_trace( { "name": span_name, "span_id": span_id, - "display_name": _get_truncatable_str_object(span.name), + "display_name": _get_truncatable_str_object( + span.name, 128 + ), "start_time": start_time, "end_time": end_time, "parent_span_id": parent_id, @@ -174,9 +172,7 @@ def _get_time_from_ns(nanoseconds: int) -> Dict: return {"seconds": int(seconds), "nanos": int(nanos)} -def _get_truncatable_str_object( - str_to_convert: str, max_length: int = MAX_LENGTH -): +def _get_truncatable_str_object(str_to_convert: str, max_length: int): """Truncate the string if it exceeds the length limit and record the truncated bytes count.""" truncated, truncated_byte_count = _truncate_str(str_to_convert, max_length) @@ -186,21 +182,9 @@ def _get_truncatable_str_object( ) -def _truncate_str( - str_to_check: str, limit: int = MAX_LENGTH -) -> Tuple[str, int]: +def _truncate_str(str_to_check: str, limit: int) -> Tuple[str, int]: """Check the length of a string. If exceeds limit, then truncate it.""" - str_bytes = str_to_check.encode("utf-8") - str_len = len(str_bytes) - truncated_byte_count = 0 - - if str_len > limit: - truncated_byte_count = str_len - limit - str_bytes = str_bytes[:limit] - - result = str(str_bytes.decode("utf-8", errors="ignore")) - - return result, truncated_byte_count + return str_to_check[:limit], max(0, len(str_to_check) - limit) def _extract_status(status: trace_api.Status) -> Status: @@ -268,7 +252,7 @@ def _extract_attributes(attrs: types.Attributes) -> ProtoSpan.Attributes: attributes_dict = {} for key, value in attrs.items(): - key = _truncate_str(key)[0] + key = _truncate_str(key, 128)[0] value = _format_attribute_value(value) if value is not None: @@ -283,10 +267,10 @@ def _format_attribute_value(value: types.AttributeValue) -> AttributeValue: value_type = "int_value" elif isinstance(value, str): value_type = "string_value" - value = _get_truncatable_str_object(value) + value = _get_truncatable_str_object(value, 256) elif isinstance(value, float): value_type = "string_value" - value = _get_truncatable_str_object("{:0.4f}".format(value)) + value = _get_truncatable_str_object("{:0.4f}".format(value), 256) else: logger.warning( "ignoring attribute value %s of type %s. Values type must be one " diff --git a/ext/opentelemetry-exporter-cloud-trace/tests/test_cloud_trace_exporter.py b/ext/opentelemetry-exporter-cloud-trace/tests/test_cloud_trace_exporter.py index ce68c842e1e..cb05395cd8c 100644 --- a/ext/opentelemetry-exporter-cloud-trace/tests/test_cloud_trace_exporter.py +++ b/ext/opentelemetry-exporter-cloud-trace/tests/test_cloud_trace_exporter.py @@ -19,7 +19,6 @@ from google.cloud.trace_v2.proto.trace_pb2 import Span as ProtoSpan from google.cloud.trace_v2.proto.trace_pb2 import TruncatableString from google.rpc.status_pb2 import Status - from opentelemetry.exporter.cloud_trace import ( CloudTraceSpanExporter, _extract_attributes, @@ -27,9 +26,10 @@ _extract_links, _extract_status, _truncate_str, + _format_attribute_value, ) from opentelemetry.sdk.trace import Event, Span -from opentelemetry.trace import DefaultSpan, Link, SpanContext, SpanKind +from opentelemetry.trace import Link, SpanContext, SpanKind from opentelemetry.trace.status import Status as SpanStatus from opentelemetry.trace.status import StatusCanonicalCode @@ -254,3 +254,52 @@ def test_extract_links(self): ] ), ) + + def test_truncate_string(self): + str_300 = "a" * 300 + str_256 = "a" * 256 + str_128 = "a" * 128 + self.assertEqual(_truncate_str("aaaa", 1), ("a", 3)) + self.assertEqual(_truncate_str("aaaa", 5), ("aaaa", 0)) + self.assertEqual(_truncate_str("aaaa", 4), ("aaaa", 0)) + + self.assertEqual( + _format_attribute_value(str_300), + AttributeValue( + string_value=TruncatableString( + value=str_256, truncated_byte_count=300 - 256 + ) + ), + ) + self.assertEqual( + _extract_attributes({str_300: str_300}), + ProtoSpan.Attributes( + attribute_map={ + str_128: AttributeValue( + string_value=TruncatableString( + value=str_256, truncated_byte_count=300 - 256 + ) + ) + } + ), + ) + + time_in_ns1 = 1589919268850900051 + time_in_ms_and_ns1 = {"seconds": 1589919268, "nanos": 850899968} + event1 = Event(name=str_300, attributes={}, timestamp=time_in_ns1) + self.assertEqual( + _extract_events([event1]), + ProtoSpan.TimeEvents( + time_event=[ + { + "time": time_in_ms_and_ns1, + "annotation": { + "description": TruncatableString( + value=str_256, truncated_byte_count=300 - 256 + ), + "attributes": {}, + }, + }, + ] + ), + ) From f64d67f4afb30f6f4608b2c0ba8532f8eaf76775 Mon Sep 17 00:00:00 2001 From: Andrew Xue Date: Mon, 1 Jun 2020 13:27:16 -0400 Subject: [PATCH 07/12] lint --- docs/examples/cloud_trace_exporter/basic_trace.py | 4 +--- .../src/opentelemetry/exporter/cloud_trace/__init__.py | 3 ++- .../tests/test_cloud_trace_exporter.py | 3 ++- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/examples/cloud_trace_exporter/basic_trace.py b/docs/examples/cloud_trace_exporter/basic_trace.py index 1626c672142..76840a291ec 100644 --- a/docs/examples/cloud_trace_exporter/basic_trace.py +++ b/docs/examples/cloud_trace_exporter/basic_trace.py @@ -10,7 +10,5 @@ SimpleExportSpanProcessor(cloud_trace_exporter) ) tracer = trace.get_tracer(__name__) -with tracer.start_as_current_span("foo") as span: - value = 'a' * 256 - span.set_attribute('key', value) +with tracer.start_as_current_span("foo"): print("Hello world!") diff --git a/ext/opentelemetry-exporter-cloud-trace/src/opentelemetry/exporter/cloud_trace/__init__.py b/ext/opentelemetry-exporter-cloud-trace/src/opentelemetry/exporter/cloud_trace/__init__.py index 9afac7bf771..3ee1cf88f04 100644 --- a/ext/opentelemetry-exporter-cloud-trace/src/opentelemetry/exporter/cloud_trace/__init__.py +++ b/ext/opentelemetry-exporter-cloud-trace/src/opentelemetry/exporter/cloud_trace/__init__.py @@ -44,12 +44,13 @@ from typing import Any, Dict, List, Sequence, Tuple import google.auth -import opentelemetry.trace as trace_api from google.cloud.trace_v2 import TraceServiceClient from google.cloud.trace_v2.proto.trace_pb2 import AttributeValue from google.cloud.trace_v2.proto.trace_pb2 import Span as ProtoSpan from google.cloud.trace_v2.proto.trace_pb2 import TruncatableString from google.rpc.status_pb2 import Status + +import opentelemetry.trace as trace_api from opentelemetry.sdk.trace import Event from opentelemetry.sdk.trace.export import Span, SpanExporter, SpanExportResult from opentelemetry.util import types diff --git a/ext/opentelemetry-exporter-cloud-trace/tests/test_cloud_trace_exporter.py b/ext/opentelemetry-exporter-cloud-trace/tests/test_cloud_trace_exporter.py index cb05395cd8c..fc420b18dba 100644 --- a/ext/opentelemetry-exporter-cloud-trace/tests/test_cloud_trace_exporter.py +++ b/ext/opentelemetry-exporter-cloud-trace/tests/test_cloud_trace_exporter.py @@ -19,14 +19,15 @@ from google.cloud.trace_v2.proto.trace_pb2 import Span as ProtoSpan from google.cloud.trace_v2.proto.trace_pb2 import TruncatableString from google.rpc.status_pb2 import Status + from opentelemetry.exporter.cloud_trace import ( CloudTraceSpanExporter, _extract_attributes, _extract_events, _extract_links, _extract_status, - _truncate_str, _format_attribute_value, + _truncate_str, ) from opentelemetry.sdk.trace import Event, Span from opentelemetry.trace import Link, SpanContext, SpanKind From a94fd5aa2da927fffdecabb91619fb4a1255f491 Mon Sep 17 00:00:00 2001 From: Andrew Xue Date: Mon, 1 Jun 2020 13:31:47 -0400 Subject: [PATCH 08/12] done --- .../tests/test_cloud_trace_exporter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opentelemetry-exporter-cloud-trace/tests/test_cloud_trace_exporter.py b/ext/opentelemetry-exporter-cloud-trace/tests/test_cloud_trace_exporter.py index fc420b18dba..18745d6b2dc 100644 --- a/ext/opentelemetry-exporter-cloud-trace/tests/test_cloud_trace_exporter.py +++ b/ext/opentelemetry-exporter-cloud-trace/tests/test_cloud_trace_exporter.py @@ -59,7 +59,7 @@ def setUp(self): "bool_key": AttributeValue(bool_value=False), "double_key": AttributeValue( string_value=TruncatableString( - value="1.421", truncated_byte_count=0 + value="1.4210", truncated_byte_count=0 ) ), "int_key": AttributeValue(int_value=123), From c470c0a000c2cdbe905e390f9051fd1cec8e09a6 Mon Sep 17 00:00:00 2001 From: Andrew Xue Date: Tue, 2 Jun 2020 12:13:29 -0400 Subject: [PATCH 09/12] add limits --- .../exporter/cloud_trace/__init__.py | 72 +++++++++-- .../tests/test_cloud_trace_exporter.py | 115 ++++++++++++++++-- 2 files changed, 165 insertions(+), 22 deletions(-) diff --git a/ext/opentelemetry-exporter-cloud-trace/src/opentelemetry/exporter/cloud_trace/__init__.py b/ext/opentelemetry-exporter-cloud-trace/src/opentelemetry/exporter/cloud_trace/__init__.py index 3ee1cf88f04..b97a964cbfb 100644 --- a/ext/opentelemetry-exporter-cloud-trace/src/opentelemetry/exporter/cloud_trace/__init__.py +++ b/ext/opentelemetry-exporter-cloud-trace/src/opentelemetry/exporter/cloud_trace/__init__.py @@ -53,10 +53,17 @@ import opentelemetry.trace as trace_api from opentelemetry.sdk.trace import Event from opentelemetry.sdk.trace.export import Span, SpanExporter, SpanExportResult +from opentelemetry.sdk.util import BoundedDict from opentelemetry.util import types logger = logging.getLogger(__name__) +MAX_NUM_LINKS = 128 +MAX_NUM_EVENTS = 32 +MAX_EVENT_ATTRS = 4 +MAX_LINK_ATTRS = 32 +MAX_SPAN_ATTRS = 32 + class CloudTraceSpanExporter(SpanExporter): """Cloud Trace span exporter for OpenTelemetry. @@ -129,7 +136,11 @@ def _translate_to_cloud_trace( start_time = _get_time_from_ns(span.start_time) end_time = _get_time_from_ns(span.end_time) - attributes = _extract_attributes(span.attributes) + if len(span.attributes) > MAX_SPAN_ATTRS: + logger.warning( + "Span has more then %s attributes, some will be truncated", + MAX_SPAN_ATTRS, + ) cloud_trace_spans.append( { @@ -141,7 +152,9 @@ def _translate_to_cloud_trace( "start_time": start_time, "end_time": end_time, "parent_span_id": parent_id, - "attributes": attributes, + "attributes": _extract_attributes( + span.attributes, MAX_SPAN_ATTRS + ), "links": _extract_links(span.links), "status": _extract_status(span.status), "time_events": _extract_events(span.events), @@ -185,7 +198,9 @@ def _get_truncatable_str_object(str_to_convert: str, max_length: int): def _truncate_str(str_to_check: str, limit: int) -> Tuple[str, int]: """Check the length of a string. If exceeds limit, then truncate it.""" - return str_to_check[:limit], max(0, len(str_to_check) - limit) + encoded = str_to_check.encode("utf-8") + truncated_str = encoded[:limit].decode("utf-8", errors="ignore") + return truncated_str, len(encoded) - len(truncated_str.encode("utf-8")) def _extract_status(status: trace_api.Status) -> Status: @@ -205,7 +220,20 @@ def _extract_links(links: Sequence[trace_api.Link]) -> ProtoSpan.Links: if not links: return None extracted_links = [] + dropped_links = 0 + if len(links) > MAX_NUM_LINKS: + logger.warning( + "Exporting more then %s links, some will be truncated", + MAX_NUM_LINKS, + ) + dropped_links = len(links) - MAX_NUM_LINKS + links = links[:MAX_NUM_LINKS] for link in links: + if len(link.attributes) > MAX_LINK_ATTRS: + logger.warning( + "Link has more then %s attributes, some will be truncated", + MAX_LINK_ATTRS, + ) trace_id = _get_hexadecimal_trace_id(link.context.trace_id) span_id = _get_hexadecimal_span_id(link.context.span_id) extracted_links.append( @@ -213,10 +241,14 @@ def _extract_links(links: Sequence[trace_api.Link]) -> ProtoSpan.Links: "trace_id": trace_id, "span_id": span_id, "type": "TYPE_UNSPECIFIED", - "attributes": _extract_attributes(link.attributes), + "attributes": _extract_attributes( + link.attributes, MAX_LINK_ATTRS + ), } ) - return ProtoSpan.Links(link=extracted_links, dropped_links_count=0) + return ProtoSpan.Links( + link=extracted_links, dropped_links_count=dropped_links + ) def _extract_events(events: Sequence[Event]) -> ProtoSpan.TimeEvents: @@ -224,11 +256,20 @@ def _extract_events(events: Sequence[Event]) -> ProtoSpan.TimeEvents: if not events: return None logs = [] + dropped_annontations = 0 + if len(events) > MAX_NUM_EVENTS: + logger.warning( + "Exporting more then %s annotations, some will be truncated", + MAX_NUM_EVENTS, + ) + dropped_annontations = len(events) - MAX_NUM_EVENTS + events = events[:MAX_NUM_EVENTS] for event in events: - if len(event.attributes) > 4: + if len(event.attributes) > MAX_EVENT_ATTRS: logger.warning( - "Event %s has more then 4 attributes, some will be truncated", + "Event %s has more then %s attributes, some will be truncated", event.name, + MAX_EVENT_ATTRS, ) logs.append( { @@ -237,20 +278,24 @@ def _extract_events(events: Sequence[Event]) -> ProtoSpan.TimeEvents: "description": _get_truncatable_str_object( event.name, 256 ), - "attributes": _extract_attributes(event.attributes), + "attributes": _extract_attributes( + event.attributes, MAX_EVENT_ATTRS + ), }, } ) return ProtoSpan.TimeEvents( time_event=logs, - dropped_annotations_count=0, + dropped_annotations_count=dropped_annontations, dropped_message_events_count=0, ) -def _extract_attributes(attrs: types.Attributes) -> ProtoSpan.Attributes: +def _extract_attributes( + attrs: types.Attributes, num_attrs_limit: int +) -> ProtoSpan.Attributes: """Convert span.attributes to dict.""" - attributes_dict = {} + attributes_dict = BoundedDict(num_attrs_limit) for key, value in attrs.items(): key = _truncate_str(key, 128)[0] @@ -258,7 +303,10 @@ def _extract_attributes(attrs: types.Attributes) -> ProtoSpan.Attributes: if value is not None: attributes_dict[key] = value - return ProtoSpan.Attributes(attribute_map=attributes_dict) + return ProtoSpan.Attributes( + attribute_map=attributes_dict, + dropped_attributes_count=len(attrs) - len(attributes_dict), + ) def _format_attribute_value(value: types.AttributeValue) -> AttributeValue: diff --git a/ext/opentelemetry-exporter-cloud-trace/tests/test_cloud_trace_exporter.py b/ext/opentelemetry-exporter-cloud-trace/tests/test_cloud_trace_exporter.py index 18745d6b2dc..5ebd5f3b649 100644 --- a/ext/opentelemetry-exporter-cloud-trace/tests/test_cloud_trace_exporter.py +++ b/ext/opentelemetry-exporter-cloud-trace/tests/test_cloud_trace_exporter.py @@ -21,6 +21,10 @@ from google.rpc.status_pb2 import Status from opentelemetry.exporter.cloud_trace import ( + MAX_EVENT_ATTRS, + MAX_LINK_ATTRS, + MAX_NUM_EVENTS, + MAX_NUM_LINKS, CloudTraceSpanExporter, _extract_attributes, _extract_events, @@ -47,7 +51,6 @@ def setUp(self): "bool_key": False, "double_key": 1.421, "int_key": 123, - "int_key2": 1234, } self.extracted_attributes_variety_pack = ProtoSpan.Attributes( attribute_map={ @@ -63,7 +66,6 @@ def setUp(self): ) ), "int_key": AttributeValue(int_value=123), - "int_key2": AttributeValue(int_value=1234), } ) @@ -141,18 +143,24 @@ def test_extract_status(self): def test_extract_attributes(self): self.assertEqual( - _extract_attributes({}), ProtoSpan.Attributes(attribute_map={}) + _extract_attributes({}, 4), ProtoSpan.Attributes(attribute_map={}) ) self.assertEqual( - _extract_attributes(self.attributes_variety_pack), + _extract_attributes(self.attributes_variety_pack, 4), self.extracted_attributes_variety_pack, ) # Test ignoring attributes with illegal value type self.assertEqual( - _extract_attributes({"illegal_attribute_value": dict()}), - ProtoSpan.Attributes(attribute_map={}), + _extract_attributes({"illegal_attribute_value": dict()}, 4), + ProtoSpan.Attributes(attribute_map={}, dropped_attributes_count=1), ) + too_many_attrs = {} + for attr_key in range(5): + too_many_attrs[str(attr_key)] = 0 + proto_attrs = _extract_attributes(too_many_attrs, 4) + self.assertEqual(proto_attrs.dropped_attributes_count, 1) + def test_extract_events(self): self.assertIsNone(_extract_events([])) time_in_ns1 = 1589919268850900051 @@ -189,7 +197,7 @@ def test_extract_events(self): value="event2", truncated_byte_count=0 ), "attributes": ProtoSpan.Attributes( - attribute_map={} + attribute_map={}, dropped_attributes_count=1 ), }, }, @@ -249,20 +257,27 @@ def test_extract_links(self): "attributes": { "attribute_map": { "int_attr_value": AttributeValue(int_value=123) - } + }, + "dropped_attributes_count": 1, }, }, ] ), ) - def test_truncate_string(self): + # pylint:disable=too-many-locals + def test_truncate(self): + """Cloud Trace API imposes limits on the length of many things, + e.g. strings, number of events, number of attributes. We truncate + these things before sending it to the API as an optimization. + """ str_300 = "a" * 300 str_256 = "a" * 256 str_128 = "a" * 128 self.assertEqual(_truncate_str("aaaa", 1), ("a", 3)) self.assertEqual(_truncate_str("aaaa", 5), ("aaaa", 0)) self.assertEqual(_truncate_str("aaaa", 4), ("aaaa", 0)) + self.assertEqual(_truncate_str("中文翻译", 4), ("中", 9)) self.assertEqual( _format_attribute_value(str_300), @@ -272,8 +287,9 @@ def test_truncate_string(self): ) ), ) + self.assertEqual( - _extract_attributes({str_300: str_300}), + _extract_attributes({str_300: str_300}, 4), ProtoSpan.Attributes( attribute_map={ str_128: AttributeValue( @@ -304,3 +320,82 @@ def test_truncate_string(self): ] ), ) + + trace_id = "6e0c63257de34c92bf9efcd03927272e" + span_id = "95bb5edabd45950f" + link = Link( + context=SpanContext( + trace_id=int(trace_id, 16), + span_id=int(span_id, 16), + is_remote=False, + ), + attributes={}, + ) + too_many_links = [link] * (MAX_NUM_LINKS + 1) + self.assertEqual( + _extract_links(too_many_links), + ProtoSpan.Links( + link=[ + { + "trace_id": trace_id, + "span_id": span_id, + "type": "TYPE_UNSPECIFIED", + "attributes": {}, + } + ] + * MAX_NUM_LINKS, + dropped_links_count=len(too_many_links) - MAX_NUM_LINKS, + ), + ) + + link_attrs = {} + for attr_key in range(MAX_LINK_ATTRS + 1): + link_attrs[str(attr_key)] = 0 + attr_link = Link( + context=SpanContext( + trace_id=int(trace_id, 16), + span_id=int(span_id, 16), + is_remote=False, + ), + attributes=link_attrs, + ) + + proto_link = _extract_links([attr_link]) + self.assertEqual( + len(proto_link.link[0].attributes.attribute_map), MAX_LINK_ATTRS + ) + + too_many_events = [event1] * (MAX_NUM_EVENTS + 1) + self.assertEqual( + _extract_events(too_many_events), + ProtoSpan.TimeEvents( + time_event=[ + { + "time": time_in_ms_and_ns1, + "annotation": { + "description": TruncatableString( + value=str_256, truncated_byte_count=300 - 256 + ), + "attributes": {}, + }, + }, + ] + * MAX_NUM_EVENTS, + dropped_annotations_count=len(too_many_events) + - MAX_NUM_EVENTS, + ), + ) + + time_in_ns1 = 1589919268850900051 + event_attrs = {} + for attr_key in range(MAX_EVENT_ATTRS + 1): + event_attrs[str(attr_key)] = 0 + proto_events = _extract_events( + [Event(name="a", attributes=event_attrs, timestamp=time_in_ns1)] + ) + self.assertEqual( + len( + proto_events.time_event[0].annotation.attributes.attribute_map + ), + MAX_EVENT_ATTRS, + ) From a0ffad69169c1918e4941829dbda9cab192724b3 Mon Sep 17 00:00:00 2001 From: Andrew Xue Date: Wed, 3 Jun 2020 13:20:45 -0400 Subject: [PATCH 10/12] f --- .../src/opentelemetry/exporter/cloud_trace/__init__.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/ext/opentelemetry-exporter-cloud-trace/src/opentelemetry/exporter/cloud_trace/__init__.py b/ext/opentelemetry-exporter-cloud-trace/src/opentelemetry/exporter/cloud_trace/__init__.py index b97a964cbfb..8cf5dd1f983 100644 --- a/ext/opentelemetry-exporter-cloud-trace/src/opentelemetry/exporter/cloud_trace/__init__.py +++ b/ext/opentelemetry-exporter-cloud-trace/src/opentelemetry/exporter/cloud_trace/__init__.py @@ -41,16 +41,15 @@ """ import logging -from typing import Any, Dict, List, Sequence, Tuple +from typing import Any, Dict, List, Sequence, Tuple, Optional import google.auth +import opentelemetry.trace as trace_api from google.cloud.trace_v2 import TraceServiceClient from google.cloud.trace_v2.proto.trace_pb2 import AttributeValue from google.cloud.trace_v2.proto.trace_pb2 import Span as ProtoSpan from google.cloud.trace_v2.proto.trace_pb2 import TruncatableString from google.rpc.status_pb2 import Status - -import opentelemetry.trace as trace_api from opentelemetry.sdk.trace import Event from opentelemetry.sdk.trace.export import Span, SpanExporter, SpanExportResult from opentelemetry.sdk.util import BoundedDict @@ -203,7 +202,7 @@ def _truncate_str(str_to_check: str, limit: int) -> Tuple[str, int]: return truncated_str, len(encoded) - len(truncated_str.encode("utf-8")) -def _extract_status(status: trace_api.Status) -> Status: +def _extract_status(status: trace_api.Status) -> Optional[Status]: """Convert a Status object to protobuf object.""" if not status: return None From 7f09868aa632dd01af4df08b0836eeaaa810c5c5 Mon Sep 17 00:00:00 2001 From: Andrew Xue Date: Wed, 3 Jun 2020 14:04:31 -0400 Subject: [PATCH 11/12] final --- .../src/opentelemetry/exporter/cloud_trace/__init__.py | 5 +++-- .../src/opentelemetry/exporter/cloud_trace/version.py | 2 +- .../tests/test_boto_instrumentation.py | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/ext/opentelemetry-exporter-cloud-trace/src/opentelemetry/exporter/cloud_trace/__init__.py b/ext/opentelemetry-exporter-cloud-trace/src/opentelemetry/exporter/cloud_trace/__init__.py index 8cf5dd1f983..7e7aa017cfd 100644 --- a/ext/opentelemetry-exporter-cloud-trace/src/opentelemetry/exporter/cloud_trace/__init__.py +++ b/ext/opentelemetry-exporter-cloud-trace/src/opentelemetry/exporter/cloud_trace/__init__.py @@ -41,15 +41,16 @@ """ import logging -from typing import Any, Dict, List, Sequence, Tuple, Optional +from typing import Any, Dict, List, Optional, Sequence, Tuple import google.auth -import opentelemetry.trace as trace_api from google.cloud.trace_v2 import TraceServiceClient from google.cloud.trace_v2.proto.trace_pb2 import AttributeValue from google.cloud.trace_v2.proto.trace_pb2 import Span as ProtoSpan from google.cloud.trace_v2.proto.trace_pb2 import TruncatableString from google.rpc.status_pb2 import Status + +import opentelemetry.trace as trace_api from opentelemetry.sdk.trace import Event from opentelemetry.sdk.trace.export import Span, SpanExporter, SpanExportResult from opentelemetry.sdk.util import BoundedDict diff --git a/ext/opentelemetry-exporter-cloud-trace/src/opentelemetry/exporter/cloud_trace/version.py b/ext/opentelemetry-exporter-cloud-trace/src/opentelemetry/exporter/cloud_trace/version.py index ec792e9af10..f83f20e7bac 100644 --- a/ext/opentelemetry-exporter-cloud-trace/src/opentelemetry/exporter/cloud_trace/version.py +++ b/ext/opentelemetry-exporter-cloud-trace/src/opentelemetry/exporter/cloud_trace/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "0.8.dev0" +__version__ = "0.9.dev0" diff --git a/ext/opentelemetry-ext-boto/tests/test_boto_instrumentation.py b/ext/opentelemetry-ext-boto/tests/test_boto_instrumentation.py index 492fac5a883..a629b108705 100644 --- a/ext/opentelemetry-ext-boto/tests/test_boto_instrumentation.py +++ b/ext/opentelemetry-ext-boto/tests/test_boto_instrumentation.py @@ -19,13 +19,13 @@ import boto.elasticache import boto.s3 import boto.sts - from moto import ( # pylint: disable=import-error mock_ec2_deprecated, mock_lambda_deprecated, mock_s3_deprecated, mock_sts_deprecated, ) + from opentelemetry.ext.boto import BotoInstrumentor from opentelemetry.test.test_base import TestBase From d4c351ccd4df15dda3f01c61941348083ef807c5 Mon Sep 17 00:00:00 2001 From: Andrew Xue Date: Wed, 3 Jun 2020 14:07:18 -0400 Subject: [PATCH 12/12] final againg --- ext/opentelemetry-ext-boto/tests/test_boto_instrumentation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opentelemetry-ext-boto/tests/test_boto_instrumentation.py b/ext/opentelemetry-ext-boto/tests/test_boto_instrumentation.py index a629b108705..492fac5a883 100644 --- a/ext/opentelemetry-ext-boto/tests/test_boto_instrumentation.py +++ b/ext/opentelemetry-ext-boto/tests/test_boto_instrumentation.py @@ -19,13 +19,13 @@ import boto.elasticache import boto.s3 import boto.sts + from moto import ( # pylint: disable=import-error mock_ec2_deprecated, mock_lambda_deprecated, mock_s3_deprecated, mock_sts_deprecated, ) - from opentelemetry.ext.boto import BotoInstrumentor from opentelemetry.test.test_base import TestBase