diff --git a/ext/opentelemetry-ext-stackdriver/README.rst b/ext/opentelemetry-ext-stackdriver/README.rst new file mode 100644 index 00000000000..fcf2a08e014 --- /dev/null +++ b/ext/opentelemetry-ext-stackdriver/README.rst @@ -0,0 +1,44 @@ +OpenTelemetry Stackdriver Exporters +===================================== + +This library provides integration with Google Cloud Stackdriver. + +Installation +------------ + +:: + + pip install opentelemetry-ext-stackdriver + +Usage +----- + +.. code:: python + + from opentelemetry import trace + from opentelemetry.ext import stackdriver + from opentelemetry.sdk.trace import Tracer + from opentelemetry.sdk.trace.export import BatchExportSpanProcessor + + trace.set_preferred_tracer_implementation(lambda T: Tracer()) + tracer = trace.tracer() + + # create a StackdriverSpanExporter + stackdriver_exporter = stackdriver.trace.StackdriverSpanExporter( + project_id='my-helloworld-project', + ) + + # Create a BatchExportSpanProcessor and add the exporter to it + span_processor = BatchExportSpanProcessor(stackdriver_exporter) + + # add to the tracer + tracer.add_span_processor(span_processor) + + with tracer.start_as_current_span('foo'): + print('Hello world!') + +References +---------- + +* `Stackdriver `_ +* `OpenTelemetry Project `_ diff --git a/ext/opentelemetry-ext-stackdriver/examples/client.py b/ext/opentelemetry-ext-stackdriver/examples/client.py new file mode 100644 index 00000000000..ee6206d9470 --- /dev/null +++ b/ext/opentelemetry-ext-stackdriver/examples/client.py @@ -0,0 +1,32 @@ +# Copyright 2019, OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import requests + +from opentelemetry import trace +from opentelemetry.ext import http_requests +from opentelemetry.ext.stackdriver.trace import StackdriverSpanExporter +from opentelemetry.sdk.trace import Tracer +from opentelemetry.sdk.trace.export import SimpleExportSpanProcessor + +trace.set_preferred_tracer_implementation(lambda T: Tracer()) +tracer = trace.tracer() +span_processor = SimpleExportSpanProcessor( + StackdriverSpanExporter(project_id="my-helloworld-project") +) +tracer.add_span_processor(span_processor) + +http_requests.enable(tracer) +response = requests.get(url="http://localhost:7777/hello") +span_processor.shutdown() diff --git a/ext/opentelemetry-ext-stackdriver/examples/server.py b/ext/opentelemetry-ext-stackdriver/examples/server.py new file mode 100644 index 00000000000..8cd74a653e3 --- /dev/null +++ b/ext/opentelemetry-ext-stackdriver/examples/server.py @@ -0,0 +1,44 @@ +# Copyright 2019, OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import flask +import requests + +from opentelemetry import trace +from opentelemetry.ext import http_requests +from opentelemetry.ext.stackdriver.trace import StackdriverSpanExporter +from opentelemetry.ext.wsgi import OpenTelemetryMiddleware +from opentelemetry.sdk.trace import Tracer +from opentelemetry.sdk.trace.export import BatchExportSpanProcessor + +trace.set_preferred_tracer_implementation(lambda T: Tracer()) + +span_processor = BatchExportSpanProcessor(StackdriverSpanExporter()) +http_requests.enable(trace.tracer()) +trace.tracer().add_span_processor(span_processor) + +app = flask.Flask(__name__) +app.wsgi_app = OpenTelemetryMiddleware(app.wsgi_app) + + +@app.route("/") +def hello(): + with trace.tracer().start_as_current_span("parent"): + requests.get("https://www.wikipedia.org/wiki/Rabbit") + return "hello" + + +if __name__ == "__main__": + app.run(debug=True) + span_processor.shutdown() diff --git a/ext/opentelemetry-ext-stackdriver/examples/trace.py b/ext/opentelemetry-ext-stackdriver/examples/trace.py new file mode 100644 index 00000000000..1cdb41ecc0e --- /dev/null +++ b/ext/opentelemetry-ext-stackdriver/examples/trace.py @@ -0,0 +1,25 @@ +# Copyright 2019, OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from opentelemetry import trace +from opentelemetry.ext.stackdriver.trace import StackdriverSpanExporter +from opentelemetry.sdk.trace import Tracer +from opentelemetry.sdk.trace.export import SimpleExportSpanProcessor + +trace.set_preferred_tracer_implementation(lambda T: Tracer()) +tracer = trace.tracer() +tracer.add_span_processor(SimpleExportSpanProcessor(StackdriverSpanExporter())) + +with tracer.start_as_current_span("hello") as span: + print("Hello, World!") diff --git a/ext/opentelemetry-ext-stackdriver/setup.cfg b/ext/opentelemetry-ext-stackdriver/setup.cfg new file mode 100644 index 00000000000..d3307eae772 --- /dev/null +++ b/ext/opentelemetry-ext-stackdriver/setup.cfg @@ -0,0 +1,48 @@ +# Copyright 2019, OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +[metadata] +name = opentelemetry-ext-stackdriver +description = Stackdriver integration for OpenTelemetry +long_description = file: README.rst +long_description_content_type = text/x-rst +author = OpenTelemetry Authors +author_email = cncf-opentelemetry-contributors@lists.cncf.io +url = https://github.com/open-telemetry/opentelemetry-python/ext/opentelemetry-ext-stackdriver +platforms = any +license = Apache-2.0 +classifiers = + Development Status :: 3 - Alpha + Intended Audience :: Developers + License :: OSI Approved :: Apache Software License + Programming Language :: Python + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.4 + Programming Language :: Python :: 3.5 + Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.7 + +[options] +python_requires = >=3.4 +package_dir= + =src +packages=find_namespace: +install_requires = + opentelemetry-api + opentelemetry-sdk + google-cloud-monitoring + google-cloud-trace + +[options.packages.find] +where = src diff --git a/ext/opentelemetry-ext-stackdriver/setup.py b/ext/opentelemetry-ext-stackdriver/setup.py new file mode 100644 index 00000000000..8d43c44ffde --- /dev/null +++ b/ext/opentelemetry-ext-stackdriver/setup.py @@ -0,0 +1,26 @@ +# Copyright 2019, OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os + +import setuptools + +BASE_DIR = os.path.dirname(__file__) +VERSION_FILENAME = os.path.join( + BASE_DIR, "src", "opentelemetry", "ext", "stackdriver", "version.py" +) +PACKAGE_INFO = {} +with open(VERSION_FILENAME) as f: + exec(f.read(), PACKAGE_INFO) + +setuptools.setup(version=PACKAGE_INFO["__version__"]) diff --git a/ext/opentelemetry-ext-stackdriver/src/opentelemetry/ext/stackdriver/trace/__init__.py b/ext/opentelemetry-ext-stackdriver/src/opentelemetry/ext/stackdriver/trace/__init__.py new file mode 100644 index 00000000000..6995cbe8633 --- /dev/null +++ b/ext/opentelemetry-ext-stackdriver/src/opentelemetry/ext/stackdriver/trace/__init__.py @@ -0,0 +1,281 @@ +# Copyright 2019, OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Stackdriver Span Exporter for OpenTelemetry.""" + +import logging +import typing + +from google.cloud.trace import trace_service_client +from google.cloud.trace.client import Client +from google.cloud.trace_v2.proto import trace_pb2 + +import opentelemetry.trace as trace_api +from opentelemetry.context import Context +from opentelemetry.sdk.trace.export import Span, SpanExporter, SpanExportResult +from opentelemetry.sdk.util import ns_to_iso_str +from opentelemetry.util import types + +from ..version import __version__ + +logger = logging.getLogger(__name__) + +AGENT = "opentelemetry-python [{}]".format(__version__) +# Max length is 128 bytes for a truncatable string. +MAX_LENGTH = 128 + + +class StackdriverSpanExporter(SpanExporter): + """Stackdriver span exporter for OpenTelemetry. + + Args: + client: Stackdriver Trace client. + project_id: project_id to create the Trace client. + """ + + def __init__( + self, client=None, project_id=None, + ): + if client is None: + client = Client(project=project_id) + self.client = client + self.project_id = self.client.project + + def export(self, spans: typing.Sequence[Span]) -> SpanExportResult: + """Export the spans to Stackdriver. + + See: https://cloud.google.com/trace/docs/reference/v2/rest/v2/ + projects.traces/batchWrite + + Args: + spans: Tuple of spans to export + """ + stackdriver_spans = self.translate_to_stackdriver(spans) + + try: + self.client.batch_write_spans( + "projects/{}".format(self.project_id), + {"spans": stackdriver_spans}, + ) + except Exception as ex: + logger.warning("Error while writing to stackdriver: %s", ex) + return SpanExportResult.FAILED_RETRYABLE + + return SpanExportResult.SUCCESS + + def translate_to_stackdriver( + self, spans: typing.Sequence[Span] + ) -> typing.List[typing.Dict[str, typing.Any]]: + """Translate the spans to Stackdriver format. + + Args: + spans: Tuple of spans to convert + """ + + stackdriver_spans = [] + + for span in spans: + ctx = span.get_context() + trace_id = "{:032x}".format(ctx.trace_id) + span_id = "{:016x}".format(ctx.span_id) + span_name = "projects/{}/traces/{}/spans/{}".format( + self.project_id, trace_id, span_id + ) + + parent_id = None + if isinstance(span.parent, trace_api.Span): + parent_id = "{:016x}".format(span.parent.get_context().span_id) + elif isinstance(span.parent, trace_api.SpanContext): + parent_id = "{:016x}".format(span.parent.span_id) + + start_time = None + if span.start_time: + start_time = ns_to_iso_str(span.start_time) + end_time = None + if span.end_time: + end_time = ns_to_iso_str(span.end_time) + + span.attributes["g.co/agent"] = AGENT + attr_map = extract_attributes(span.attributes) + + sd_span = { + "name": span_name, + "spanId": span_id, + "parentSpanId": parent_id, + "displayName": get_truncatable_str(span.name), + "attributes": map_attributes(attr_map), + "links": extract_links(span.links), + "status": extract_status(span.status), + "timeEvents": extract_events(span.events), + "startTime": start_time, + "endTime": end_time, + } + + stackdriver_spans.append(sd_span) + + return stackdriver_spans + + def shutdown(self): + pass + + +def get_truncatable_str(str_to_convert): + """Truncate a string if exceed limit and record the truncated bytes + count. + """ + truncated, truncated_byte_count = check_str_length( + str_to_convert, MAX_LENGTH + ) + + result = { + "value": truncated, + "truncated_byte_count": truncated_byte_count, + } + return result + + +def check_str_length(str_to_check, limit=MAX_LENGTH): + """Check the length of a string. If exceeds limit, then truncate it. + """ + str_bytes = str_to_check.encode("utf-8") + str_len = len(str_bytes) + truncated_byte_count = 0 + + if str_len > limit: + truncated_byte_count = str_len - limit + str_bytes = str_bytes[:limit] + + result = str(str_bytes.decode("utf-8", errors="ignore")) + + return (result, truncated_byte_count) + + +def extract_status(status: trace_api.Status): + """Convert a Status object to dict.""" + status_json = {"details": None} + + status_json["code"] = status.canonical_code.value + + if status.description is not None: + status_json["message"] = status.description + + return status_json + + +def extract_links(links): + """Convert span.links to set.""" + if not links: + return None + + links = [] + for link in links: + trace_id = link.context.trace_id + span_id = link.context.span_id + links.append( + {trace_id: trace_id, span_id: span_id, type: "CHILD_LINKED_SPAN"} + ) + return set(links) + + +def extract_events(events): + """Convert span.events to dict.""" + if not events: + return None + + logs = [] + + for event in events: + annotation_json = {"description": get_truncatable_str(event.name)} + if event.attributes is not None: + annotation_json["attributes"] = extract_attributes( + event.attributes + ) + + logs.append( + { + "time": ns_to_iso_str(event.timestamp), + "annotation": annotation_json, + } + ) + return {"timeEvent": logs} + + +def extract_attributes(attrs: types.Attributes): + """Convert span.attributes to dict.""" + attributes_json = {} + + for key, value in attrs.items(): + key = check_str_length(key)[0] + value = _format_attribute_value(value) + + if value is not None: + attributes_json[key] = value + + result = {"attributeMap": attributes_json} + + return result + + +def map_attributes(attribute_map): + """Convert the attributes to stackdriver attributes.""" + if attribute_map is None: + return attribute_map + for (key, value) in attribute_map.items(): + if key != "attributeMap": + continue + for attribute_key in list(value.keys()): + if attribute_key in ATTRIBUTE_MAPPING: + new_key = ATTRIBUTE_MAPPING.get(attribute_key) + value[new_key] = value.pop(attribute_key) + return attribute_map + + +def _format_attribute_value(value): + if isinstance(value, bool): + value_type = "bool_value" + elif isinstance(value, int): + value_type = "int_value" + elif isinstance(value, str): + value_type = "string_value" + value = get_truncatable_str(value) + elif isinstance(value, float): + value_type = "double_value" + else: + return None + + return {value_type: value} + + +ATTRIBUTE_MAPPING = { + "component": "/component", + "error.message": "/error/message", + "error.name": "/error/name", + "http.client_city": "/http/client_city", + "http.client_country": "/http/client_country", + "http.client_protocol": "/http/client_protocol", + "http.client_region": "/http/client_region", + "http.host": "/http/host", + "http.method": "/http/method", + "http.redirected_url": "/http/redirected_url", + "http.request_size": "/http/request/size", + "http.response_size": "/http/response/size", + "http.status_code": "/http/status_code", + "http.url": "/http/url", + "http.user_agent": "/http/user_agent", + "pid": "/pid", + "stacktrace": "/stacktrace", + "tid": "/tid", + "grpc.host_port": "/grpc/host_port", + "grpc.method": "/grpc/method", +} diff --git a/ext/opentelemetry-ext-stackdriver/src/opentelemetry/ext/stackdriver/version.py b/ext/opentelemetry-ext-stackdriver/src/opentelemetry/ext/stackdriver/version.py new file mode 100644 index 00000000000..93ef792d051 --- /dev/null +++ b/ext/opentelemetry-ext-stackdriver/src/opentelemetry/ext/stackdriver/version.py @@ -0,0 +1,15 @@ +# Copyright 2019, OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__version__ = "0.3.dev0" diff --git a/ext/opentelemetry-ext-stackdriver/tests/__init__.py b/ext/opentelemetry-ext-stackdriver/tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ext/opentelemetry-ext-stackdriver/tests/test_stackdriver_exporter.py b/ext/opentelemetry-ext-stackdriver/tests/test_stackdriver_exporter.py new file mode 100644 index 00000000000..d86ef4986a0 --- /dev/null +++ b/ext/opentelemetry-ext-stackdriver/tests/test_stackdriver_exporter.py @@ -0,0 +1,113 @@ +# Copyright 2019, OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +from unittest import mock + +import opentelemetry.ext.stackdriver.trace as sd_exporter +from opentelemetry.sdk.trace import Span +from opentelemetry.trace import SpanContext, SpanKind +from opentelemetry.util.version import __version__ + + +class TestStackdriverSpanExporter(unittest.TestCase): + def setUp(self): + self.client_patcher = mock.patch( + "opentelemetry.ext.stackdriver.trace.Client" + ) + self.client_patcher.start() + + def tearDown(self): + self.client_patcher.stop() + + def test_constructor_default(self): + exporter = sd_exporter.StackdriverSpanExporter() + self.assertEqual(exporter.project_id, exporter.client.project) + + def test_constructor_explicit(self): + client = mock.Mock() + project_id = "PROJECT" + client.project = project_id + + exporter = sd_exporter.StackdriverSpanExporter( + client=client, project_id=project_id + ) + + self.assertIs(exporter.client, client) + self.assertEqual(exporter.project_id, project_id) + + def test_export(self): + trace_id = "6e0c63257de34c92bf9efcd03927272e" + span_id = "95bb5edabd45950f" + # start_times = 683647322 * 10 ** 9 # in ns + # durations = 50 * 10 ** 6 + # end_times = start_times + durations + span_datas = [ + Span( + name="span_name", + context=SpanContext( + trace_id=int(trace_id, 16), span_id=int(span_id, 16) + ), + parent=None, + kind=SpanKind.INTERNAL, + ) + ] + + stackdriver_spans = { + "spans": [ + { + "name": "projects/PROJECT/traces/{}/spans/{}".format( + trace_id, span_id + ), + "spanId": span_id, + "parentSpanId": None, + "displayName": { + "value": "span_name", + "truncated_byte_count": 0, + }, + "attributes": { + "attributeMap": { + "g.co/agent": { + "string_value": { + "value": "opentelemetry-python [{}]".format( + __version__ + ), + "truncated_byte_count": 0, + } + } + } + }, + "links": None, + "status": {"details": None, "code": 0}, + "timeEvents": None, + "startTime": None, + "endTime": None, + } + ] + } + + client = mock.Mock() + project_id = "PROJECT" + client.project = project_id + + exporter = sd_exporter.StackdriverSpanExporter( + client=client, project_id=project_id + ) + + exporter.export(span_datas) + + name = "projects/{}".format(project_id) + + client.batch_write_spans.assert_called_with(name, stackdriver_spans) + self.assertTrue(client.batch_write_spans.called) diff --git a/scripts/coverage.sh b/scripts/coverage.sh index bddf39a90cd..d93a9fa9f83 100755 --- a/scripts/coverage.sh +++ b/scripts/coverage.sh @@ -20,6 +20,7 @@ cov opentelemetry-sdk cov ext/opentelemetry-ext-http-requests cov ext/opentelemetry-ext-jaeger cov ext/opentelemetry-ext-opentracing-shim +cov ext/opentelemetry-ext-stackdriver cov ext/opentelemetry-ext-wsgi cov examples/opentelemetry-example-app diff --git a/tox.ini b/tox.ini index f5624503b6a..352405e9c31 100644 --- a/tox.ini +++ b/tox.ini @@ -38,6 +38,7 @@ changedir = test-ext-http-requests: ext/opentelemetry-ext-http-requests/tests test-ext-jaeger: ext/opentelemetry-ext-jaeger/tests test-ext-pymongo: ext/opentelemetry-ext-pymongo/tests + test-ext-stackdriver: ext/opentelemetry-ext-stackdriver/tests test-ext-wsgi: ext/opentelemetry-ext-wsgi/tests test-ext-flask: ext/opentelemetry-ext-flask/tests test-example-app: examples/opentelemetry-example-app/tests @@ -80,6 +81,7 @@ commands_pre = coverage: pip install -e {toxinidir}/ext/opentelemetry-ext-http-requests coverage: pip install -e {toxinidir}/ext/opentelemetry-ext-jaeger coverage: pip install -e {toxinidir}/ext/opentelemetry-ext-opentracing-shim + coverage: pip install -e {toxinidir}/ext/opentelemetry-ext-stackdriver coverage: pip install -e {toxinidir}/ext/opentelemetry-ext-testutil coverage: pip install -e {toxinidir}/ext/opentelemetry-ext-wsgi coverage: pip install -e {toxinidir}/ext/opentelemetry-ext-flask[test] @@ -116,6 +118,7 @@ commands_pre = pip install -e {toxinidir}/ext/opentelemetry-ext-http-requests pip install -e {toxinidir}/ext/opentelemetry-ext-jaeger pip install -e {toxinidir}/ext/opentelemetry-ext-pymongo + pip install -e {toxinidir}/ext/opentelemetry-ext-stackdriver pip install -e {toxinidir}/ext/opentelemetry-ext-testutil pip install -e {toxinidir}/ext/opentelemetry-ext-wsgi pip install -e {toxinidir}/ext/opentelemetry-ext-flask[test]