Skip to content

Commit 65ecb98

Browse files
pvitalpvital
committed
feat: Add instrumentation to kafka-python.
Signed-off-by: Paulo Vital <paulo.vital@ibm.com>
1 parent d1e45ab commit 65ecb98

File tree

11 files changed

+348
-11
lines changed

11 files changed

+348
-11
lines changed

‎.circleci/config.yml‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,15 @@ jobs:
136136
environment:
137137
PUBSUB_EMULATOR_HOST: 0.0.0.0:8681
138138
PUBSUB_PROJECT1: test-project,test-topic
139+
- image: public.ecr.aws/bitnami/kafka:3.9.0
140+
environment:
141+
KAFKA_CFG_NODE_ID: 0
142+
KAFKA_CFG_PROCESS_ROLES: controller,broker
143+
KAFKA_CFG_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093,EXTERNAL://:9094
144+
KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT
145+
KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 0@localhost:9093
146+
KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER
147+
KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092,EXTERNAL://localhost:9094
139148
working_directory: ~/repo
140149
steps:
141150
- checkout
@@ -164,6 +173,15 @@ jobs:
164173
environment:
165174
PUBSUB_EMULATOR_HOST: 0.0.0.0:8681
166175
PUBSUB_PROJECT1: test-project,test-topic
176+
- image: public.ecr.aws/bitnami/kafka:3.9.0
177+
environment:
178+
KAFKA_CFG_NODE_ID: 0
179+
KAFKA_CFG_PROCESS_ROLES: controller,broker
180+
KAFKA_CFG_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093,EXTERNAL://:9094
181+
KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT
182+
KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 0@localhost:9093
183+
KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER
184+
KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092,EXTERNAL://localhost:9094
167185
working_directory: ~/repo
168186
steps:
169187
- checkout
@@ -192,6 +210,15 @@ jobs:
192210
environment:
193211
PUBSUB_EMULATOR_HOST: 0.0.0.0:8681
194212
PUBSUB_PROJECT1: test-project,test-topic
213+
- image: public.ecr.aws/bitnami/kafka:3.9.0
214+
environment:
215+
KAFKA_CFG_NODE_ID: 0
216+
KAFKA_CFG_PROCESS_ROLES: controller,broker
217+
KAFKA_CFG_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093,EXTERNAL://:9094
218+
KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT
219+
KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 0@localhost:9093
220+
KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER
221+
KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092,EXTERNAL://localhost:9094
195222
working_directory: ~/repo
196223
steps:
197224
- checkout
@@ -221,6 +248,15 @@ jobs:
221248
environment:
222249
PUBSUB_EMULATOR_HOST: 0.0.0.0:8681
223250
PUBSUB_PROJECT1: test-project,test-topic
251+
- image: public.ecr.aws/bitnami/kafka:3.9.0
252+
environment:
253+
KAFKA_CFG_NODE_ID: 0
254+
KAFKA_CFG_PROCESS_ROLES: controller,broker
255+
KAFKA_CFG_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093,EXTERNAL://:9094
256+
KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT
257+
KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 0@localhost:9093
258+
KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER
259+
KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092,EXTERNAL://localhost:9094
224260
working_directory: ~/repo
225261
steps:
226262
- checkout
@@ -250,6 +286,15 @@ jobs:
250286
environment:
251287
PUBSUB_EMULATOR_HOST: 0.0.0.0:8681
252288
PUBSUB_PROJECT1: test-project,test-topic
289+
- image: public.ecr.aws/bitnami/kafka:3.9.0
290+
environment:
291+
KAFKA_CFG_NODE_ID: 0
292+
KAFKA_CFG_PROCESS_ROLES: controller,broker
293+
KAFKA_CFG_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093,EXTERNAL://:9094
294+
KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT
295+
KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 0@localhost:9093
296+
KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER
297+
KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092,EXTERNAL://localhost:9094
253298
working_directory: ~/repo
254299
steps:
255300
- checkout
@@ -293,6 +338,15 @@ jobs:
293338
environment:
294339
PUBSUB_EMULATOR_HOST: 0.0.0.0:8681
295340
PUBSUB_PROJECT1: test-project,test-topic
341+
- image: public.ecr.aws/bitnami/kafka:3.9.0
342+
environment:
343+
KAFKA_CFG_NODE_ID: 0
344+
KAFKA_CFG_PROCESS_ROLES: controller,broker
345+
KAFKA_CFG_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093,EXTERNAL://:9094
346+
KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT
347+
KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 0@localhost:9093
348+
KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER
349+
KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092,EXTERNAL://localhost:9094
296350
working_directory: ~/repo
297351
steps:
298352
- checkout
@@ -322,6 +376,15 @@ jobs:
322376
environment:
323377
PUBSUB_EMULATOR_HOST: 0.0.0.0:8681
324378
PUBSUB_PROJECT1: test-project,test-topic
379+
- image: public.ecr.aws/bitnami/kafka:3.9.0
380+
environment:
381+
KAFKA_CFG_NODE_ID: 0
382+
KAFKA_CFG_PROCESS_ROLES: controller,broker
383+
KAFKA_CFG_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093,EXTERNAL://:9094
384+
KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT
385+
KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 0@localhost:9093
386+
KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER
387+
KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092,EXTERNAL://localhost:9094
325388
working_directory: ~/repo
326389
steps:
327390
- checkout

‎.tekton/python-tracer-prepuller.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,10 @@ spec:
4545
# public.ecr.aws/docker/library/postgres:16.2-bookworm
4646
image: public.ecr.aws/docker/library/postgres@sha256:07572430dbcd821f9f978899c3ab3a727f5029be9298a41662e1b5404d5b73e0
4747
command: ["sh", "-c", "'true'"]
48+
- name: prepuller-kafka
49+
# public.ecr.aws/bitnami/kafka:3.9.0
50+
image: public.ecr.aws/docker/library/kafka@sha256:d2890d68f96b36da3c8413fa94294f018b2f95d87cf108cbf71eab510572d9be
51+
command: ["sh", "-c", "'true'"]
4852
- name: prepuller-38
4953
# public.ecr.aws/docker/library/python:3.8.20-bookworm
5054
image: public.ecr.aws/docker/library/python@

‎.tekton/task.yaml‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,25 @@ spec:
131131
- name: rabbitmq
132132
# public.ecr.aws/docker/library/rabbitmq:3.13.0
133133
image: public.ecr.aws/docker/library/rabbitmq@sha256:39de1a4fc6c72d12bd5dfa23e8576536fd1c0cc8418344cd5a51addfc9a1145d
134+
- name: kafka
135+
# public.ecr.aws/bitnami/kafka:3.9.0
136+
image: public.ecr.aws/bitnami/kafka@sha256:d2890d68f96b36da3c8413fa94294f018b2f95d87cf108cbf71eab510572d9be
137+
env:
138+
- name: KAFKA_CFG_NODE_ID
139+
value: 0
140+
- name: KAFKA_CFG_PROCESS_ROLES
141+
value: controller,broker
142+
- name: KAFKA_CFG_LISTENERS
143+
value: PLAINTEXT://:9092,CONTROLLER://:9093,EXTERNAL://:9094
144+
- name: KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP
145+
value: CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT
146+
- name: KAFKA_CFG_CONTROLLER_QUORUM_VOTERS
147+
value: 0@kafka:9093
148+
- name: KAFKA_CFG_CONTROLLER_LISTENER_NAMES
149+
value: CONTROLLER
150+
- name: KAFKA_CFG_ADVERTISED_LISTENERS
151+
value: PLAINTEXT://kafka:9092,EXTERNAL://localhost:9094
152+
134153
params:
135154
- name: imageDigest
136155
type: string

‎docker-compose.yml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,3 +62,17 @@ services:
6262
ports:
6363
- "8681:8681"
6464
- "8682:8682"
65+
66+
kafka:
67+
image: public.ecr.aws/bitnami/kafka:latest
68+
ports:
69+
- '9092:9092'
70+
- '9094:9094'
71+
environment:
72+
- KAFKA_CFG_NODE_ID=0
73+
- KAFKA_CFG_PROCESS_ROLES=controller,broker
74+
- KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093,EXTERNAL://:9094
75+
- KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT
76+
- KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka:9093
77+
- KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER
78+
- KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092,EXTERNAL://localhost:9094

‎src/instana/__init__.py‎

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -169,43 +169,50 @@ def boot_agent() -> None:
169169
asyncio, # noqa: F401
170170
boto3_inst, # noqa: F401
171171
cassandra_inst, # noqa: F401
172+
celery, # noqa: F401
172173
couchbase_inst, # noqa: F401
173174
fastapi_inst, # noqa: F401
174175
flask, # noqa: F401
175176
# gevent_inst, # noqa: F401
176177
grpcio, # noqa: F401
177178
logging, # noqa: F401
178179
mysqlclient, # noqa: F401
179-
pika, # noqa: F401
180180
pep0249, # noqa: F401
181+
pika, # noqa: F401
181182
psycopg2, # noqa: F401
182183
pymongo, # noqa: F401
183184
pymysql, # noqa: F401
184185
pyramid, # noqa: F401
185186
redis, # noqa: F401
187+
sanic_inst, # noqa: F401
186188
sqlalchemy, # noqa: F401
187189
starlette_inst, # noqa: F401
188-
sanic_inst, # noqa: F401
189190
urllib3, # noqa: F401
190191
)
191192
from instana.instrumentation.aiohttp import (
192-
client, # noqa: F401
193-
server, # noqa: F401
193+
client as aiohttp_client, # noqa: F401
194+
)
195+
from instana.instrumentation.aiohttp import (
196+
server as aiohttp_server, # noqa: F401
194197
)
195198
from instana.instrumentation.aws import lambda_inst # noqa: F401
196-
from instana.instrumentation import celery # noqa: F401
197199
from instana.instrumentation.django import middleware # noqa: F401
198200
from instana.instrumentation.google.cloud import (
199201
pubsub, # noqa: F401
200202
storage, # noqa: F401
201203
)
204+
from instana.instrumentation.kafka import (
205+
kafka_python, # noqa: F401
206+
)
207+
from instana.instrumentation.tornado import (
208+
client as tornado_client, # noqa: F401
209+
)
202210
from instana.instrumentation.tornado import (
203-
client, # noqa: F401
204-
server, # noqa: F401
211+
server as tornado_server, # noqa: F401
205212
)
206213

207214
# Hooks
208-
from instana.hooks import hook_uwsgi, hook_gunicorn # noqa: F401
215+
from instana.hooks import hook_gunicorn, hook_uwsgi # noqa: F401
209216

210217

211218
if "INSTANA_DISABLE" not in os.environ:

‎src/instana/instrumentation/kafka/__init__.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
# (c) Copyright IBM Corp. 2025

‎src/instana/instrumentation/kafka/kafka_python.py‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
# (c) Copyright IBM Corp. 2025
2+
3+
try:
4+
from typing import TYPE_CHECKING, Any, Callable, Dict, Tuple
5+
6+
import kafka # noqa: F401
7+
import wrapt
8+
from opentelemetry.trace import SpanKind
9+
10+
from instana.log import logger
11+
from instana.propagators.format import Format
12+
from instana.util.traceutils import (
13+
get_tracer_tuple,
14+
tracing_is_off,
15+
)
16+
17+
if TYPE_CHECKING:
18+
from kafka.producer.future import FutureRecordMetadata
19+
20+
@wrapt.patch_function_wrapper("kafka", "KafkaProducer.send")
21+
def trace_kafka_send(
22+
wrapped: Callable[..., "kafka.KafkaProducer.send"],
23+
instance: "kafka.KafkaProducer",
24+
args: Tuple[int, str, Tuple[Any, ...]],
25+
kwargs: Dict[str, Any],
26+
) -> "FutureRecordMetadata":
27+
if tracing_is_off():
28+
return wrapped(*args, **kwargs)
29+
30+
tracer, parent_span, _ = get_tracer_tuple()
31+
parent_context = parent_span.get_span_context() if parent_span else None
32+
33+
with tracer.start_as_current_span(
34+
"kafka-producer", span_context=parent_context, kind=SpanKind.PRODUCER
35+
) as span:
36+
span.set_attribute("kafka.service", args[0])
37+
span.set_attribute("kafka.access", "send")
38+
39+
# context propagation
40+
tracer.inject(
41+
span.context,
42+
Format.KAFKA_HEADERS,
43+
kwargs.get("headers", {}),
44+
disable_w3c_trace_context=True,
45+
)
46+
47+
try:
48+
res = wrapped(*args, **kwargs)
49+
except Exception as exc:
50+
span.record_exception(exc)
51+
else:
52+
return res
53+
54+
@wrapt.patch_function_wrapper("kafka", "KafkaConsumer.__next__")
55+
def trace_kafka_consume(
56+
wrapped: Callable[..., "kafka.KafkaConsumer.__next__"],
57+
instance: "kafka.KafkaConsumer",
58+
args: Tuple[int, str, Tuple[Any, ...]],
59+
kwargs: Dict[str, Any],
60+
) -> "FutureRecordMetadata":
61+
if tracing_is_off():
62+
return wrapped(*args, **kwargs)
63+
64+
tracer, parent_span, _ = get_tracer_tuple()
65+
66+
parent_context = (
67+
parent_span.get_span_context()
68+
if parent_span
69+
else tracer.extract(
70+
Format.KAFKA_HEADERS, {}, disable_w3c_trace_context=True
71+
)
72+
)
73+
74+
with tracer.start_as_current_span(
75+
"kafka-consumer", span_context=parent_context, kind=SpanKind.CONSUMER
76+
) as span:
77+
topic = list(instance.subscription())[0]
78+
span.set_attribute("kafka.service", topic)
79+
span.set_attribute("kafka.access", "consume")
80+
81+
try:
82+
res = wrapped(*args, **kwargs)
83+
except Exception as exc:
84+
span.record_exception(exc)
85+
else:
86+
return res
87+
88+
logger.debug("Instrumenting Kafka (kafka-python)")
89+
except ImportError:
90+
pass

0 commit comments

Comments
 (0)