From f0389a7d4928a8b31868e4a95e19db55e59793bd Mon Sep 17 00:00:00 2001 From: Santiago Gimeno Date: Wed, 2 Apr 2025 15:39:14 +0200 Subject: [PATCH] agents: optimize span collector message sending Add state tracking to prevent excessive `uv_async_t::send()` calls. Benchmark results for (40k and 50k): ``` === Benchmark Summary === Benchmark Type: wrk2 Old Version: ./nsolid_baseline New Version: ./nsolid_span_collector Iterations: 50 Connections: 10 Duration: 60s Rate: 40k === Latency Percentile Distribution === +------------+------------------+------------------+------------+---------+--------------+ | Percentile | Old Version (ms) | New Version (ms) | Difference | p-value | Significance | +------------+------------------+------------------+------------+---------+--------------+ | P50 | 1.09 | 1.10 | +0.40% | 0.00 | *** | | P75 | 1.67 | 1.67 | +0.03% | 0.73 | | | P90 | 2.22 | 2.22 | -0.23% | 0.01 | ** | | P95 | 2.54 | 2.53 | -0.52% | 0.00 | *** | | P99 | 3.16 | 3.10 | -1.82% | 0.00 | *** | | P99.9 | 4.98 | 4.36 | -12.45% | 0.01 | *** | | P99.99 | 9.97 | 8.96 | -10.09% | 0.13 | | | P99.999 | 12.91 | 11.77 | -8.88% | 0.15 | | +------------+------------------+------------------+------------+---------+--------------+ ``` ``` === Benchmark Summary === Benchmark Type: wrk2 Old Version: ./nsolid_baseline New Version: ./nsolid_span_collector Iterations: 50 Connections: 10 Duration: 60s Rate: 50k === Latency Percentile Distribution === +------------+------------------+------------------+------------+---------+--------------+ | Percentile | Old Version (ms) | New Version (ms) | Difference | p-value | Significance | +------------+------------------+------------------+------------+---------+--------------+ | P50 | 0.98 | 0.95 | -3.76% | 0.16 | | | P75 | 1.78 | 1.54 | -13.31% | 0.23 | | | P90 | 3.36 | 2.49 | -25.71% | 0.18 | | | P95 | 5.37 | 3.51 | -34.60% | 0.09 | * | | P99 | 18.08 | 10.66 | -41.02% | 0.01 | *** | | P99.9 | 50.91 | 30.78 | -39.54% | 0.01 | *** | | P99.99 | 72.19 | 47.71 | -33.91% | 0.01 | *** | | P99.999 | 75.98 | 51.69 | -31.97% | 0.01 | *** | +------------+------------------+------------------+------------+---------+--------------+ ``` TL;DR The new version demonstrates a clear improvement in latency performance, particularly for edge cases and high-load scenarios represented by the tail latencies. This suggests better handling of resource-intensive operations or improved performance under stress conditions. --- agents/src/span_collector.cc | 7 ++++++- agents/src/span_collector.h | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/agents/src/span_collector.cc b/agents/src/span_collector.cc index 3e429c65f8d..8aafe53dee6 100644 --- a/agents/src/span_collector.cc +++ b/agents/src/span_collector.cc @@ -34,7 +34,12 @@ void SpanCollector::do_collect() { // Only notify the loop_ thread if the queue reaches min_span_count_to // avoid too many uv_async_t::send() calls. if (collector->span_msg_q_.enqueue(span) > collector->min_span_count_) { - ASSERT_EQ(0, collector->span_msg_->send()); + if (!collector->sent_) { + ASSERT_EQ(0, collector->span_msg_->send()); + collector->sent_ = true; + } + } else if (collector->sent_) { + collector->sent_ = false; } }, weak_from_this()); diff --git a/agents/src/span_collector.h b/agents/src/span_collector.h index 71e23197bd1..3e666889db6 100644 --- a/agents/src/span_collector.h +++ b/agents/src/span_collector.h @@ -83,6 +83,7 @@ class SpanCollector: public std::enable_shared_from_this { nsuv::ns_timer* span_timer_; std::function callback_ = nullptr; std::function transform_callback_ = nullptr; + bool sent_ = false; }; } // namespace nsolid