[AO-20302] Add minimal SolarWinds exporter

MANIFEST.in

@@ -0,0 +1,11 @@
+include opentelemetry_distro_solarwinds/extension/oboe_api.hpp
+include opentelemetry_distro_solarwinds/extension/oboe.h
+include opentelemetry_distro_solarwinds/extension/oboe_debug.h
+include opentelemetry_distro_solarwinds/extension/liboboe-1.0-x86_64.so.0.0.0
+include opentelemetry_distro_solarwinds/extension/liboboe-1.0-alpine-x86_64.so.0.0.0
+include opentelemetry_distro_solarwinds/extension/bson/bson.h
+include opentelemetry_distro_solarwinds/extension/bson/platform_hacks.h
+exclude opentelemetry_distro_solarwinds/extension/liboboe-1.0.so.0
+exclude MANIFEST.in
+exclude Makefile
+prune test

README.md

@@ -1,2 +1,46 @@
 # opentelemetry-python-instrumentation-custom-distro
 The custom distro to extend the OpenTelemetry Python agent for compatibility with AO
+
+
+## Prerequisites
+
+### Git Repos and Directory Structure
+
+The code in this repository makes use of code located in the [otel-oboe](https://github.com/librato/otel-oboe) GIT repository. Thus, first make sure that you clone the following repositories into the same root directory. For example, if your development directory is `~/gitrepos/`, please clone the `otel-oboe` and the `opentelemetry-python-instrumentation-custom-distro` repositories under `~/gitrepos`, so that your directory structure looks as shown below:
+```
+~/gitrepos/
+|
+|----otel-oboe/
+|
+|----opentelemetry-python-instrumentation-custom-distro/
+```
+
+### Development (Build) Container
+
+In order to compile the C-extension which is part of this Python module, SWIG needs to be installed. This repository provides a Linux-based Dockerfile which can be used to create a Docker image in which the C-extension can be build easily.
+
+To build the Docker image which provides all necessary build tools, run the following command inside the `dev_tools` directory:
+```bash
+docker build -t dev-container .
+```
+
+Then you can start a build container by running `./run_docker_dev.sh` from within the `dev_tools` location. This will provide you with a docker container which has all volumes mapped as required so you can easily build the agent.
+
+## How to Build and Install the Agent
+* Switch into the build container by running
+    ```bash
+    ./run_docker_dev.sh
+   ```
+  from within the `dev_tools` directory.
+* Inside the docker container, you can now build the agent with the provided Makefile.
+
+### Install Agent from Source in Development Mode
+* Execute `make wrapper` inside the build container. This copies the C-extension artifacts and builds the SWIG bindings.
+* Install the agent in your application (Linux environment only) in development mode by running
+   ```python
+   pip install -Ie ~/gitrepos/opentelemetry-python-instrumentation-custom-distro/
+   ```
+When installing the agent in development mode every change in the Python source code will be reflected in the Python environment directly without re-installation. However, if changes have been made to the C-extension files, you need to reinstall the agent (as described above) to reflect these changes in the Python environment.
+
+### Build Agent Source Distribution Archive
+* Execute `make sdist` inside the build container. This will create a zip archive (source distribution) of the Python module under the `dist` directory.

dev_tools/Dockerfile

@@ -0,0 +1,39 @@
+# build development environment to locally build appoptics_apm Python agent and publish RC versions
+FROM quay.io/pypa/manylinux2014_x86_64
+
+# install packages need to build Ruby and dependencies need to build agent locally
+RUN yum install -y \
+    curl \
+    gpg \
+    gcc \
+    gcc-c++ \
+    make \
+    patch \
+    autoconf \
+    automake \
+    bison \
+    libffi-devel \
+    libtool \
+    patch \
+    readline-devel \
+    sqlite-devel \
+    zlib-devel \
+    openssl-devel \
+    wget \
+    jq \
+    vim \
+    less \
+    zip \
+    && yum clean all && rm -rf /var/cache/yum
+
+# install Ruby2.5 which is needed for package cloud cli
+RUN gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 \
+        7D2BAF1CF37B13E2069D6956105BD0E739499BDB && \
+    curl -sSL https://get.rvm.io | bash -s stable && source /etc/profile.d/rvm.sh && \
+    /usr/local/rvm/bin/rvm install 2.5.1 --disable-binary
+
+# install PackageCloud Cli
+RUN /usr/local/rvm/bin/rvm 2.5.1 do gem install package_cloud
+
+# install boto3 for interatction with AWS and twine to upload to TestPyPi
+RUN python3.8 -m pip install boto3 twine

dev_tools/Makefile

@@ -0,0 +1,85 @@
+#    Copyright 2021 SolarWinds, LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ----The Makefile for automated building (and hopefully) deployment
+SHELL=bash
+
+.DEFAULT_GOAL := wrapper
+
+#----------------------------------------------------------------------------------------------------------------------#
+# variable definitions and recipes for downloading of required header and library files
+#----------------------------------------------------------------------------------------------------------------------#
+OTELOBOEREPO := /code/otel_oboe/liboboe
+
+# Copy the pre-compiled liboboe shared library from source specified in OTELOBOEREPO
+copy-liboboe:
+	@echo -e "Copying shared library.\n"
+	@cd ../opentelemetry_distro_solarwinds/extension; \
+		cp "${OTELOBOEREPO}/liboboe-1.0-x86_64.so.0.0.0" .; \
+		if [ $$? -ne 0 ]; then echo " **** failed to copy shared library ****" ; exit 1; fi;
+
+# Copy liboboe header files (Python wrapper for Oboe c-lib) from source specified in OTELOBOEREPO
+copy-headers: copy-bson-headers
+	@echo -e "Copying header files (.hpp, .h, .i)"
+	@echo "Copying files from ${OTELOBOEREPO}:"
+	@cd ../opentelemetry_distro_solarwinds/extension; \
+		for i in oboe.h oboe_api.hpp oboe_api.cpp oboe_debug.h; do \
+			echo "Copying $$i"; \
+			cp "${OTELOBOEREPO}/$$i" .; \
+			if [ $$? -ne 0 ]; then echo " **** failed to copy $$i ****" ; exit 1; fi; \
+		done
+	@cd ../opentelemetry_distro_solarwinds/extension; \
+		echo "Copying oboe.i"; \
+		cp "${OTELOBOEREPO}/swig/oboe.i" .; \
+		if [ $$? -ne 0 ]; then echo " **** failed to copy oboe.i ****" ; exit 1; fi; \
+
+# Copy bson header files from source specified in OTELOBOEREPO
+copy-bson-headers:
+	@echo -e "Copying bson header files (.hpp, .h)"
+	@if [ ! -d ../opentelemetry_distro_solarwinds/extension/bson ]; then \
+		mkdir ../opentelemetry_distro_solarwinds/extension/bson; \
+		echo "Created ../opentelemetry_distro_solarwinds/extension/bson"; \
+	 fi
+	@echo "Copying files from ${OTELOBOEREPO}:"
+	@cd ../opentelemetry_distro_solarwinds/extension/bson; \
+		for i in bson.h platform_hacks.h; do \
+			echo "Copying $$i"; \
+			cp "${OTELOBOEREPO}/bson/$$i" .; \
+			if [ $$? -ne 0 ]; then echo " **** fail to copy $$i ****" ; exit 1; fi; \
+		done
+
+# copy artifacts from local otel-oboe
+copy-all: copy-headers copy-liboboe
+
+#----------------------------------------------------------------------------------------------------------------------#
+# recipes for building the package distribution
+#----------------------------------------------------------------------------------------------------------------------#
+# Check if SWIG is installed
+check-swig:
+	@echo -e "Is SWIG installed?"
+	@command -v swig >/dev/null 2>&1 || \
+		{ echo >&2 "Swig is required to build the distribution. Aborting."; exit 1;}
+	@echo -e "Yes."
+
+# Build the Python wrapper from liboboe headers inside build container
+wrapper: check-swig copy-all
+	@echo -e "Generating SWIG wrapper for C/C++ headers."
+	@cd ../opentelemetry_distro_solarwinds/extension && ./gen_bindings.sh
+
+# Create package source distribution archive
+sdist: wrapper
+	@echo -e "Generating python agent sdist package"
+	@python3.8 setup.py sdist
+	@echo -e "\nDone."
+
+.PHONY: nothing check-swig download-liboboe download-headers wrapper sdist

dev_tools/run_docker_dev.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+#
+# to build the image:
+# docker build -t dev-container .
+#
+# to run the image:
+# ./run_docker_dev
+
+docker run -it \
+    --net=host \
+    --cap-add SYS_PTRACE \
+    --workdir /code/opentelemetry_distro_solarwinds \
+    -v "$PWD"/..:/code/opentelemetry_distro_solarwinds \
+    -v "$PWD"/../../otel-oboe/:/code/otel_oboe \
+    -v `echo ~`:/home/developer \
+    dev-container bash

opentelemetry_distro_solarwinds/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.0.1"

opentelemetry_distro_solarwinds/distro.py

@@ -1,13 +1,21 @@
 """Module to configure OpenTelemetry agent to work with SolarWinds backend"""
 
+from opentelemetry import trace
 from opentelemetry.instrumentation.distro import BaseDistro
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor
 
+from opentelemetry_distro_solarwinds.exporter import SolarWindsSpanExporter
 
-class AoDistro(BaseDistro):
+
+class SolarWindsDistro(BaseDistro):
     """SolarWinds custom distro for OpenTelemetry agents.
 
     With this custom distro, the following functionality is introduced:
         - no functionality added at this time
     """
     def _configure(self, **kwargs):
-        pass
+        # Automatically configure the SolarWinds Span exporter
+        trace.set_tracer_provider(TracerProvider())
+        span_exporter = BatchSpanProcessor(SolarWindsSpanExporter())
+        trace.get_tracer_provider().add_span_processor(span_exporter)

opentelemetry_distro_solarwinds/exporter.py

@@ -0,0 +1,104 @@
+""" This module provides a SolarWinds-specific exporter.
+
+The exporter translates OpenTelemetry spans into AppOptics events so that the instrumentation data
+generated by an OpenTelemetry-based agent can be processed by the SolarWinds backend.
+"""
+
+import logging
+import os
+import threading
+import time
+
+from opentelemetry.sdk.trace.export import SpanExporter, SpanExportResult
+
+from opentelemetry_distro_solarwinds.extension.oboe import (Context, Event,
+                                                            Metadata, Reporter)
+from opentelemetry_distro_solarwinds.ot_ao_transformer import transform_id
+
+logger = logging.getLogger(__file__)
+
+
+class SolarWindsSpanExporter(SpanExporter):
+    """SolarWinds span exporter.
+
+    Reports instrumentation data to the SolarWinds backend.
+    """
+    def __init__(self, *args, **kw_args):
+        super().__init__(*args, **kw_args)
+        self.reporter = None
+        self._initialize_solarwinds_reporter()
+
+    def export(self, spans):
+        """Export to AO events and report via liboboe.
+
+        Note that OpenTelemetry timestamps are in nanoseconds, whereas AppOptics expects timestamps
+        to be in microseconds, thus all times need to be divided by 1000.
+        """
+        for span in spans:
+            md = self._build_metadata(span.get_span_context())
+            if span.parent and span.parent.is_valid:
+                # If there is a parent, we need to add an edge to this parent to this entry event
+                logger.debug("Continue trace from %s", md.toString())
+                parent_md = self._build_metadata(span.parent)
+                evt = Context.startTrace(md, int(span.start_time / 1000),
+                                         parent_md)
+            else:
+                # In OpenTelemrtry, there are no events with individual IDs, but only a span ID
+                # and trace ID. Thus, the entry event needs to be generated such that it has the
+                # same op ID as the span ID of the OTel span.
+                logger.debug("Start a new trace %s", md.toString())
+                evt = Context.startTrace(md, int(span.start_time / 1000))
+            evt.addInfo('Layer', span.name)
+            evt.addInfo('Language', 'Python')
+            self.reporter.sendReport(evt)
+
+            for event in span.events:
+                if event.name == 'exception':
+                    self._report_exception_event(event)
+                else:
+                    self.reporter().sendReport(event)
+
+            evt = Context.stopTrace(int(span.end_time / 1000))
+            evt.addInfo('Layer', span.name)
+            self.reporter.sendReport(evt)
+
+    def _report_exception_event(self, event):
+        pass
+
+    def _report_info_event(self, event):
+        pass
+
+    def _initialize_solarwinds_reporter(self):
+        """Initialize liboboe."""
+        log_level = os.environ.get('APPOPTICS_DEBUG_LEVEL', 3)
+        try:
+            log_level = int(log_level)
+        except ValueError:
+            log_level = 3
+        self.reporter = Reporter(
+            hostname_alias='',
+            log_level=log_level,
+            log_file_path='',
+            max_transactions=-1,
+            max_flush_wait_time=-1,
+            events_flush_interval=-1,
+            max_request_size_bytes=-1,
+            reporter='ssl',
+            host=os.environ.get('APPOPTICS_COLLECTOR', ''),
+            service_key=os.environ.get('APPOPTICS_SERVICE_KEY', ''),
+            trusted_path='',
+            buffer_size=-1,
+            trace_metrics=-1,
+            histogram_precision=-1,
+            token_bucket_capacity=-1,
+            token_bucket_rate=-1,
+            file_single=0,
+            ec2_metadata_timeout=1000,
+            grpc_proxy='',
+            stdout_clear_nonblocking=0,
+            is_grpc_clean_hack_enabled=False,
+        )
+
+    @staticmethod
+    def _build_metadata(span_context):
+        return Metadata.fromString(transform_id(span_context))

opentelemetry_distro_solarwinds/extension/.gitignore

@@ -0,0 +1,6 @@
+# Ignore everything in this directory
+*
+# Except this file
+!.gitignore
+!__init__.py
+!gen_bindings.sh

opentelemetry_distro_solarwinds/extension/__init__.py

@@ -0,0 +1,14 @@
+#    Copyright 2021 SolarWinds, LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+

opentelemetry_distro_solarwinds/extension/gen_bindings.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+#    Copyright 2021 SolarWinds, LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+rm -f oboe.py oboe_wrap.cxx _oboe.so
+swig -c++ -python oboe.i

opentelemetry_distro_solarwinds/ot_ao_transformer.py

@@ -0,0 +1,16 @@
+"""Provides functionality to transform OpenTelemetry Data to SolarWinds AppOptics data.
+"""
+
+import logging
+
+logger = logging.getLogger(__file__)
+
+
+def transform_id(span_context):
+    """Generates an AppOptics X-Trace ID from the provided OpenTelemetry span context."""
+    xtr = "2B00000000{0:032X}{1:016X}0{2}".format(span_context.trace_id,
+                                                  span_context.span_id,
+                                                  span_context.trace_flags)
+    logger.debug("Generated X-Trace %s from span context %s", xtr,
+                 span_context)
+    return xtr