From 4587a594d1d3d5f5048d5e35b865ada4f9a287da Mon Sep 17 00:00:00 2001 From: Varsha GS Date: Thu, 24 Oct 2024 16:47:08 +0530 Subject: [PATCH 1/3] fix: provide ASGI support with Django - Migrate Django middleware to new-style Signed-off-by: Varsha GS --- .../instrumentation/django/middleware.py | 448 +++++++++--------- 1 file changed, 232 insertions(+), 216 deletions(-) diff --git a/src/instana/instrumentation/django/middleware.py b/src/instana/instrumentation/django/middleware.py index 25ced03e..49b4f393 100644 --- a/src/instana/instrumentation/django/middleware.py +++ b/src/instana/instrumentation/django/middleware.py @@ -1,242 +1,256 @@ # (c) Copyright IBM Corp. 2021 # (c) Copyright Instana Inc. 2018 +try: + import sys + + from django import VERSION as django_version + from opentelemetry import context, trace + from opentelemetry.semconv.trace import SpanAttributes + import wrapt + from typing import TYPE_CHECKING, Dict, Any, Callable, Optional, List, Tuple + + from instana.log import logger + from instana.singletons import agent, tracer + from instana.util.secrets import strip_secrets_from_query + from instana.propagators.format import Format + + if TYPE_CHECKING: + from instana.span.span import InstanaSpan + from django.core.handlers.wsgi import WSGIRequest, WSGIHandler + from django.http import HttpRequest, HttpResponse + + DJ_INSTANA_MIDDLEWARE = "instana.instrumentation.django.middleware.InstanaMiddleware" + + if django_version >= (2, 0): + # Since Django 2.0, only `settings.MIDDLEWARE` is supported, so new-style + # middlewares can be used. + class MiddlewareMixin: + def __init__(self, get_response): + self.get_response = get_response + + def __call__(self, request): + self.process_request(request) + response = self.get_response(request) + return self.process_response(request, response) + + else: + # Note: For 1.11 <= django_version < 2.0 + # Django versions 1.x can use `settings.MIDDLEWARE_CLASSES` and expect + # old-style middlewares, which are created by inheriting from + # `deprecation.MiddlewareMixin` since its creation in Django 1.10 and 1.11 + from django.utils.deprecation import MiddlewareMixin + + + class InstanaMiddleware(MiddlewareMixin): + """Django Middleware to provide request tracing for Instana""" + + def __init__( + self, get_response: Optional[Callable[["HttpRequest"], "HttpResponse"]] = None + ) -> None: + super(InstanaMiddleware, self).__init__(get_response) + self.get_response = get_response + + def _extract_custom_headers( + self, span: "InstanaSpan", headers: Dict[str, Any], format: bool + ) -> None: + if agent.options.extra_http_headers is None: + return -import sys + try: + for custom_header in agent.options.extra_http_headers: + # Headers are available in this format: HTTP_X_CAPTURE_THIS + django_header = ( + ("HTTP_" + custom_header.upper()).replace("-", "_") + if format + else custom_header + ) -from opentelemetry import context, trace -from opentelemetry.semconv.trace import SpanAttributes -import wrapt -from typing import TYPE_CHECKING, Dict, Any, Callable, Optional, List, Tuple + if django_header in headers: + span.set_attribute( + "http.header.%s" % custom_header, headers[django_header] + ) -from instana.log import logger -from instana.singletons import agent, tracer -from instana.util.secrets import strip_secrets_from_query -from instana.propagators.format import Format + except Exception: + logger.debug("extract_custom_headers: ", exc_info=True) -if TYPE_CHECKING: - from instana.span.span import InstanaSpan - from django.core.handlers.wsgi import WSGIRequest, WSGIHandler - from django.http import HttpRequest, HttpResponse + def process_request(self, request: "WSGIRequest") -> None: + try: + env = request.META -DJ_INSTANA_MIDDLEWARE = "instana.instrumentation.django.middleware.InstanaMiddleware" + span_context = tracer.extract(Format.HTTP_HEADERS, env) -try: - from django.utils.deprecation import MiddlewareMixin -except ImportError: - MiddlewareMixin = object + span = tracer.start_span("django", span_context=span_context) + request.span = span + ctx = trace.set_span_in_context(span) + token = context.attach(ctx) + request.token = token -class InstanaMiddleware(MiddlewareMixin): - """Django Middleware to provide request tracing for Instana""" + self._extract_custom_headers(span, env, format=True) - def __init__( - self, get_response: Optional[Callable[["HttpRequest"], "HttpResponse"]] = None - ) -> None: - super(InstanaMiddleware, self).__init__(get_response) - self.get_response = get_response + request.span.set_attribute(SpanAttributes.HTTP_METHOD, request.method) + if "PATH_INFO" in env: + request.span.set_attribute(SpanAttributes.HTTP_URL, env["PATH_INFO"]) + if "QUERY_STRING" in env and len(env["QUERY_STRING"]): + scrubbed_params = strip_secrets_from_query( + env["QUERY_STRING"], + agent.options.secrets_matcher, + agent.options.secrets_list, + ) + request.span.set_attribute("http.params", scrubbed_params) + if "HTTP_HOST" in env: + request.span.set_attribute("http.host", env["HTTP_HOST"]) + except Exception: + logger.debug("Django middleware @ process_request", exc_info=True) + + def process_response( + self, request: "WSGIRequest", response: "HttpResponse" + ) -> "HttpResponse": + try: + if request.span: + if 500 <= response.status_code: + request.span.assure_errored() + # for django >= 2.2 + if request.resolver_match is not None and hasattr( + request.resolver_match, "route" + ): + path_tpl = request.resolver_match.route + # django < 2.2 or in case of 404 + else: + try: + from django.urls import resolve + + view_name = resolve(request.path)._func_path + path_tpl = "".join(url_pattern_route(view_name)) + except Exception: + # the resolve method can fire a Resolver404 exception, in this case there is no matching route + # so the path_tpl is set to None in order not to be added as a tag + path_tpl = None + if path_tpl: + request.span.set_attribute("http.path_tpl", path_tpl) + + request.span.set_attribute( + SpanAttributes.HTTP_STATUS_CODE, response.status_code + ) + self._extract_custom_headers( + request.span, response.headers, format=False + ) + tracer.inject(request.span.context, Format.HTTP_HEADERS, response) + except Exception: + logger.debug("Instana middleware @ process_response", exc_info=True) + finally: + if hasattr(request, "span") and request.span: + if request.span.is_recording(): + request.span.end() + request.span = None + if hasattr(request, "token") and request.token: + context.detach(request.token) + request.token = None + return response + + def process_exception(self, request: "WSGIRequest", exception: Exception) -> None: + from django.http.response import Http404 + + if isinstance(exception, Http404): + return None - def _extract_custom_headers( - self, span: "InstanaSpan", headers: Dict[str, Any], format: bool - ) -> None: - if agent.options.extra_http_headers is None: - return + if request.span: + request.span.record_exception(exception) + + + def url_pattern_route(view_name: str) -> Callable[..., object]: + from django.conf import settings try: - for custom_header in agent.options.extra_http_headers: - # Headers are available in this format: HTTP_X_CAPTURE_THIS - django_header = ( - ("HTTP_" + custom_header.upper()).replace("-", "_") - if format - else custom_header - ) - - if django_header in headers: - span.set_attribute( - "http.header.%s" % custom_header, headers[django_header] + from django.urls import ( + RegexURLPattern as URLPattern, + RegexURLResolver as URLResolver, + ) + except ImportError: + from django.urls import URLPattern, URLResolver + + urlconf = __import__(settings.ROOT_URLCONF, {}, {}, [""]) + + def list_urls( + urlpatterns: List[str], parent_pattern: Optional[List[str]] = None + ) -> Callable[..., object]: + if not urlpatterns: + return + if parent_pattern is None: + parent_pattern = [] + first = urlpatterns[0] + if isinstance(first, URLPattern): + if first.lookup_str == view_name: + if hasattr(first, "regex"): + return parent_pattern + [str(first.regex.pattern)] + else: + return parent_pattern + [str(first.pattern)] + elif isinstance(first, URLResolver): + if hasattr(first, "regex"): + return list_urls( + first.url_patterns, parent_pattern + [str(first.regex.pattern)] + ) + else: + return list_urls( + first.url_patterns, parent_pattern + [str(first.pattern)] ) + return list_urls(urlpatterns[1:], parent_pattern) - except Exception: - logger.debug("extract_custom_headers: ", exc_info=True) + return list_urls(urlconf.urlpatterns) - def process_request(self, request: "WSGIRequest") -> None: - try: - env = request.environ - - span_context = tracer.extract(Format.HTTP_HEADERS, env) - - span = tracer.start_span("django", span_context=span_context) - request.span = span - - ctx = trace.set_span_in_context(span) - token = context.attach(ctx) - request.token = token - - self._extract_custom_headers(span, env, format=True) - - request.span.set_attribute(SpanAttributes.HTTP_METHOD, request.method) - if "PATH_INFO" in env: - request.span.set_attribute(SpanAttributes.HTTP_URL, env["PATH_INFO"]) - if "QUERY_STRING" in env and len(env["QUERY_STRING"]): - scrubbed_params = strip_secrets_from_query( - env["QUERY_STRING"], - agent.options.secrets_matcher, - agent.options.secrets_list, - ) - request.span.set_attribute("http.params", scrubbed_params) - if "HTTP_HOST" in env: - request.span.set_attribute("http.host", env["HTTP_HOST"]) - except Exception: - logger.debug("Django middleware @ process_request", exc_info=True) - def process_response( - self, request: "WSGIRequest", response: "HttpResponse" - ) -> "HttpResponse": + def load_middleware_wrapper( + wrapped: Callable[..., None], + instance: "WSGIHandler", + args: Tuple[object, ...], + kwargs: Dict[str, Any], + ) -> Callable[..., None]: try: - if request.span: - if 500 <= response.status_code: - request.span.assure_errored() - # for django >= 2.2 - if request.resolver_match is not None and hasattr( - request.resolver_match, "route" - ): - path_tpl = request.resolver_match.route - # django < 2.2 or in case of 404 + from django.conf import settings + + # Django >=1.10 to <2.0 support old-style MIDDLEWARE_CLASSES so we + # do as well here + if hasattr(settings, "MIDDLEWARE") and settings.MIDDLEWARE is not None: + if DJ_INSTANA_MIDDLEWARE in settings.MIDDLEWARE: + return wrapped(*args, **kwargs) + + if isinstance(settings.MIDDLEWARE, tuple): + settings.MIDDLEWARE = (DJ_INSTANA_MIDDLEWARE,) + settings.MIDDLEWARE + elif isinstance(settings.MIDDLEWARE, list): + settings.MIDDLEWARE = [DJ_INSTANA_MIDDLEWARE] + settings.MIDDLEWARE else: - try: - from django.urls import resolve - - view_name = resolve(request.path)._func_path - path_tpl = "".join(url_pattern_route(view_name)) - except Exception: - # the resolve method can fire a Resolver404 exception, in this case there is no matching route - # so the path_tpl is set to None in order not to be added as a tag - path_tpl = None - if path_tpl: - request.span.set_attribute("http.path_tpl", path_tpl) - - request.span.set_attribute( - SpanAttributes.HTTP_STATUS_CODE, response.status_code - ) - self._extract_custom_headers( - request.span, response.headers, format=False - ) - tracer.inject(request.span.context, Format.HTTP_HEADERS, response) - except Exception: - logger.debug("Instana middleware @ process_response", exc_info=True) - finally: - if request.span: - if request.span.is_recording(): - request.span.end() - request.span = None - if request.token: - context.detach(request.token) - request.token = None - return response - - def process_exception(self, request: "WSGIRequest", exception: Exception) -> None: - from django.http.response import Http404 - - if isinstance(exception, Http404): - return None + logger.warning("Instana: Couldn't add InstanaMiddleware to Django") + + elif ( + hasattr(settings, "MIDDLEWARE_CLASSES") + and settings.MIDDLEWARE_CLASSES is not None + ): # pragma: no cover + if DJ_INSTANA_MIDDLEWARE in settings.MIDDLEWARE_CLASSES: + return wrapped(*args, **kwargs) + + if isinstance(settings.MIDDLEWARE_CLASSES, tuple): + settings.MIDDLEWARE_CLASSES = ( + DJ_INSTANA_MIDDLEWARE, + ) + settings.MIDDLEWARE_CLASSES + elif isinstance(settings.MIDDLEWARE_CLASSES, list): + settings.MIDDLEWARE_CLASSES = [ + DJ_INSTANA_MIDDLEWARE + ] + settings.MIDDLEWARE_CLASSES + else: + logger.warning("Instana: Couldn't add InstanaMiddleware to Django") - if request.span: - request.span.record_exception(exception) + else: # pragma: no cover + logger.warning("Instana: Couldn't find middleware settings") + return wrapped(*args, **kwargs) + except Exception: + logger.warning( + "Instana: Couldn't add InstanaMiddleware to Django: ", exc_info=True + ) -def url_pattern_route(view_name: str) -> Callable[..., object]: - from django.conf import settings try: - from django.urls import ( - RegexURLPattern as URLPattern, - RegexURLResolver as URLResolver, - ) - except ImportError: - from django.urls import URLPattern, URLResolver - - urlconf = __import__(settings.ROOT_URLCONF, {}, {}, [""]) - - def list_urls( - urlpatterns: List[str], parent_pattern: Optional[List[str]] = None - ) -> Callable[..., object]: - if not urlpatterns: - return - if parent_pattern is None: - parent_pattern = [] - first = urlpatterns[0] - if isinstance(first, URLPattern): - if first.lookup_str == view_name: - if hasattr(first, "regex"): - return parent_pattern + [str(first.regex.pattern)] - else: - return parent_pattern + [str(first.pattern)] - elif isinstance(first, URLResolver): - if hasattr(first, "regex"): - return list_urls( - first.url_patterns, parent_pattern + [str(first.regex.pattern)] - ) - else: - return list_urls( - first.url_patterns, parent_pattern + [str(first.pattern)] - ) - return list_urls(urlpatterns[1:], parent_pattern) - - return list_urls(urlconf.urlpatterns) - - -def load_middleware_wrapper( - wrapped: Callable[..., None], - instance: "WSGIHandler", - args: Tuple[object, ...], - kwargs: Dict[str, Any], -) -> Callable[..., None]: - try: - from django.conf import settings - - # Django >=1.10 to <2.0 support old-style MIDDLEWARE_CLASSES so we - # do as well here - if hasattr(settings, "MIDDLEWARE") and settings.MIDDLEWARE is not None: - if DJ_INSTANA_MIDDLEWARE in settings.MIDDLEWARE: - return wrapped(*args, **kwargs) - - if isinstance(settings.MIDDLEWARE, tuple): - settings.MIDDLEWARE = (DJ_INSTANA_MIDDLEWARE,) + settings.MIDDLEWARE - elif isinstance(settings.MIDDLEWARE, list): - settings.MIDDLEWARE = [DJ_INSTANA_MIDDLEWARE] + settings.MIDDLEWARE - else: - logger.warning("Instana: Couldn't add InstanaMiddleware to Django") - - elif ( - hasattr(settings, "MIDDLEWARE_CLASSES") - and settings.MIDDLEWARE_CLASSES is not None - ): # pragma: no cover - if DJ_INSTANA_MIDDLEWARE in settings.MIDDLEWARE_CLASSES: - return wrapped(*args, **kwargs) - - if isinstance(settings.MIDDLEWARE_CLASSES, tuple): - settings.MIDDLEWARE_CLASSES = ( - DJ_INSTANA_MIDDLEWARE, - ) + settings.MIDDLEWARE_CLASSES - elif isinstance(settings.MIDDLEWARE_CLASSES, list): - settings.MIDDLEWARE_CLASSES = [ - DJ_INSTANA_MIDDLEWARE - ] + settings.MIDDLEWARE_CLASSES - else: - logger.warning("Instana: Couldn't add InstanaMiddleware to Django") - - else: # pragma: no cover - logger.warning("Instana: Couldn't find middleware settings") - - return wrapped(*args, **kwargs) - except Exception: - logger.warning( - "Instana: Couldn't add InstanaMiddleware to Django: ", exc_info=True - ) - - -try: - if "django" in sys.modules: logger.debug("Instrumenting django") wrapt.wrap_function_wrapper( "django.core.handlers.base", @@ -256,6 +270,8 @@ def load_middleware_wrapper( except ImproperlyConfigured: pass -except Exception: - logger.debug("django.middleware:", exc_info=True) - pass + except Exception: + logger.debug("django.middleware:", exc_info=True) + +except ImportError: + pass \ No newline at end of file From 828b7e32c3d07dbe539f7afd3e7afee5aed64f46 Mon Sep 17 00:00:00 2001 From: Varsha GS Date: Tue, 5 Nov 2024 14:43:40 +0530 Subject: [PATCH 2/3] style: Adapt type hints to ASGI support Signed-off-by: Varsha GS --- .../instrumentation/django/middleware.py | 40 ++++++++++--------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/src/instana/instrumentation/django/middleware.py b/src/instana/instrumentation/django/middleware.py index 49b4f393..8953214d 100644 --- a/src/instana/instrumentation/django/middleware.py +++ b/src/instana/instrumentation/django/middleware.py @@ -8,7 +8,7 @@ from opentelemetry import context, trace from opentelemetry.semconv.trace import SpanAttributes import wrapt - from typing import TYPE_CHECKING, Dict, Any, Callable, Optional, List, Tuple + from typing import TYPE_CHECKING, Dict, Any, Callable, Optional, List, Tuple, Type from instana.log import logger from instana.singletons import agent, tracer @@ -17,10 +17,12 @@ if TYPE_CHECKING: from instana.span.span import InstanaSpan - from django.core.handlers.wsgi import WSGIRequest, WSGIHandler + from django.core.handlers.base import BaseHandler from django.http import HttpRequest, HttpResponse - DJ_INSTANA_MIDDLEWARE = "instana.instrumentation.django.middleware.InstanaMiddleware" + DJ_INSTANA_MIDDLEWARE = ( + "instana.instrumentation.django.middleware.InstanaMiddleware" + ) if django_version >= (2, 0): # Since Django 2.0, only `settings.MIDDLEWARE` is supported, so new-style @@ -34,19 +36,19 @@ def __call__(self, request): response = self.get_response(request) return self.process_response(request, response) - else: + else: # Note: For 1.11 <= django_version < 2.0 # Django versions 1.x can use `settings.MIDDLEWARE_CLASSES` and expect # old-style middlewares, which are created by inheriting from # `deprecation.MiddlewareMixin` since its creation in Django 1.10 and 1.11 from django.utils.deprecation import MiddlewareMixin - class InstanaMiddleware(MiddlewareMixin): """Django Middleware to provide request tracing for Instana""" def __init__( - self, get_response: Optional[Callable[["HttpRequest"], "HttpResponse"]] = None + self, + get_response: Optional[Callable[["HttpRequest"], "HttpResponse"]] = None, ) -> None: super(InstanaMiddleware, self).__init__(get_response) self.get_response = get_response @@ -74,7 +76,7 @@ def _extract_custom_headers( except Exception: logger.debug("extract_custom_headers: ", exc_info=True) - def process_request(self, request: "WSGIRequest") -> None: + def process_request(self, request: Type["HttpRequest"]) -> None: try: env = request.META @@ -91,7 +93,9 @@ def process_request(self, request: "WSGIRequest") -> None: request.span.set_attribute(SpanAttributes.HTTP_METHOD, request.method) if "PATH_INFO" in env: - request.span.set_attribute(SpanAttributes.HTTP_URL, env["PATH_INFO"]) + request.span.set_attribute( + SpanAttributes.HTTP_URL, env["PATH_INFO"] + ) if "QUERY_STRING" in env and len(env["QUERY_STRING"]): scrubbed_params = strip_secrets_from_query( env["QUERY_STRING"], @@ -105,7 +109,7 @@ def process_request(self, request: "WSGIRequest") -> None: logger.debug("Django middleware @ process_request", exc_info=True) def process_response( - self, request: "WSGIRequest", response: "HttpResponse" + self, request: Type["HttpRequest"], response: "HttpResponse" ) -> "HttpResponse": try: if request.span: @@ -133,9 +137,10 @@ def process_response( request.span.set_attribute( SpanAttributes.HTTP_STATUS_CODE, response.status_code ) - self._extract_custom_headers( - request.span, response.headers, format=False - ) + if hasattr(response, "headers"): + self._extract_custom_headers( + request.span, response.headers, format=False + ) tracer.inject(request.span.context, Format.HTTP_HEADERS, response) except Exception: logger.debug("Instana middleware @ process_response", exc_info=True) @@ -149,7 +154,9 @@ def process_response( request.token = None return response - def process_exception(self, request: "WSGIRequest", exception: Exception) -> None: + def process_exception( + self, request: Type["HttpRequest"], exception: Exception + ) -> None: from django.http.response import Http404 if isinstance(exception, Http404): @@ -158,7 +165,6 @@ def process_exception(self, request: "WSGIRequest", exception: Exception) -> Non if request.span: request.span.record_exception(exception) - def url_pattern_route(view_name: str) -> Callable[..., object]: from django.conf import settings @@ -199,10 +205,9 @@ def list_urls( return list_urls(urlconf.urlpatterns) - def load_middleware_wrapper( wrapped: Callable[..., None], - instance: "WSGIHandler", + instance: Type["BaseHandler"], args: Tuple[object, ...], kwargs: Dict[str, Any], ) -> Callable[..., None]: @@ -249,7 +254,6 @@ def load_middleware_wrapper( "Instana: Couldn't add InstanaMiddleware to Django: ", exc_info=True ) - try: logger.debug("Instrumenting django") wrapt.wrap_function_wrapper( @@ -274,4 +278,4 @@ def load_middleware_wrapper( logger.debug("django.middleware:", exc_info=True) except ImportError: - pass \ No newline at end of file + pass From ad1861c886ecd2a1853039d8eeb9f8db4099826a Mon Sep 17 00:00:00 2001 From: Varsha GS Date: Mon, 11 Nov 2024 11:58:24 +0530 Subject: [PATCH 3/3] ci: have a single entry for django in requirements file Signed-off-by: Varsha GS --- src/instana/instrumentation/django/middleware.py | 6 +++--- tests/requirements.txt | 4 +--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/instana/instrumentation/django/middleware.py b/src/instana/instrumentation/django/middleware.py index 8953214d..4dc2e621 100644 --- a/src/instana/instrumentation/django/middleware.py +++ b/src/instana/instrumentation/django/middleware.py @@ -70,11 +70,11 @@ def _extract_custom_headers( if django_header in headers: span.set_attribute( - "http.header.%s" % custom_header, headers[django_header] + f"http.header.{custom_header}", headers[django_header] ) except Exception: - logger.debug("extract_custom_headers: ", exc_info=True) + logger.debug("Instana middleware @ extract_custom_headers: ", exc_info=True) def process_request(self, request: Type["HttpRequest"]) -> None: try: @@ -104,7 +104,7 @@ def process_request(self, request: Type["HttpRequest"]) -> None: ) request.span.set_attribute("http.params", scrubbed_params) if "HTTP_HOST" in env: - request.span.set_attribute("http.host", env["HTTP_HOST"]) + request.span.set_attribute(SpanAttributes.HTTP_HOST, env["HTTP_HOST"]) except Exception: logger.debug("Django middleware @ process_request", exc_info=True) diff --git a/tests/requirements.txt b/tests/requirements.txt index 78ed2f68..570be7cd 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -4,9 +4,7 @@ boto3>=1.17.74 bottle>=0.12.25 celery>=5.2.7 coverage>=5.5 -Django>=4.2.4; python_version < "3.10" -Django>=5.0; python_version >= "3.10" and python_version < "3.12" -Django>=5.0a1; python_version >= "3.12" --pre +Django>=4.2.16 fastapi>=0.92.0; python_version < "3.13" fastapi>=0.115.0; python_version >= "3.13" flask>=2.3.2