From f11718ac18d470d461c774ff44acce7bc7e861af Mon Sep 17 00:00:00 2001 From: Julien Balestra Date: Wed, 6 Mar 2024 13:47:16 +0100 Subject: [PATCH 1/6] otel: create a sampler for verbose calls --- tracing/plugin/otlp.go | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tracing/plugin/otlp.go b/tracing/plugin/otlp.go index 527a998fb2703..04427b218215f 100644 --- a/tracing/plugin/otlp.go +++ b/tracing/plugin/otlp.go @@ -21,6 +21,7 @@ import ( "fmt" "io" "net/url" + "sync" "time" "github.com/containerd/containerd/errdefs" @@ -169,6 +170,11 @@ func newTracer(ctx context.Context, config *TraceConfig, procs []trace.SpanProce opts := []trace.TracerProviderOption{ trace.WithSampler(sampler), trace.WithResource(res), + trace.WithSampler(&readSampling{ + drop: map[string]struct{}{ + "runtime.v1.ImageService/ImageFsInfo": {}, + }, + }), } for _, proc := range procs { @@ -192,6 +198,30 @@ func newTracer(ctx context.Context, config *TraceConfig, procs []trace.SpanProce } +type readSampling struct { + sync.Mutex + drop map[string]struct{} +} + +func (s *readSampling) ShouldSample(p trace.SamplingParameters) trace.SamplingResult { + s.Lock() + _, ok := s.drop[p.Name] + s.Unlock() + if ok { + return trace.SamplingResult{ + Decision: trace.Drop, + } + } + println(p.Name) + return trace.SamplingResult{ + Decision: trace.RecordOnly, + } +} + +func (s *readSampling) Description() string { + return "readSampling" +} + // Returns a composite TestMap propagator func propagators() propagation.TextMapPropagator { return propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{}) From 7a8f3697fdf79d29e6f1c2fd7d849a613dba2a44 Mon Sep 17 00:00:00 2001 From: Julien Balestra Date: Wed, 6 Mar 2024 15:41:25 +0100 Subject: [PATCH 2/6] otel: inventorise all calls --- tracing/plugin/otlp.go | 55 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 51 insertions(+), 4 deletions(-) diff --git a/tracing/plugin/otlp.go b/tracing/plugin/otlp.go index 04427b218215f..f19c821b0d4cc 100644 --- a/tracing/plugin/otlp.go +++ b/tracing/plugin/otlp.go @@ -165,14 +165,62 @@ func newTracer(ctx context.Context, config *TraceConfig, procs []trace.SpanProce return nil, fmt.Errorf("failed to create resource: %w", err) } - sampler := trace.ParentBased(trace.TraceIDRatioBased(config.TraceSamplingRatio)) + //sampler := trace.ParentBased(trace.TraceIDRatioBased(config.TraceSamplingRatio)) opts := []trace.TracerProviderOption{ - trace.WithSampler(sampler), trace.WithResource(res), trace.WithSampler(&readSampling{ drop: map[string]struct{}{ + "containerd.services.containers.v1.Containers/Get": {}, + "containerd.services.leases.v1.Leases/Delete": {}, + "containerd.services.containers.v1.Containers/ListStream": {}, + "containerd.services.content.v1.Content/Info": {}, + "containerd.services.content.v1.Content/Read": {}, + "containerd.services.content.v1.Content/Write": {}, + "containerd.services.events.v1.Events/Subscribe": {}, + "containerd.services.images.v1.Images/Get": {}, + "containerd.services.images.v1.Images/List": {}, + "containerd.services.namespaces.v1.Namespaces/Get": {}, + "containerd.services.namespaces.v1.Namespaces/List": {}, + "containerd.services.snapshots.v1.Snapshots/Stat": {}, + "containerd.services.tasks.v1.Tasks/Get": {}, + "containerd.services.tasks.v1.Tasks/ListPids": {}, + "containerd.services.version.v1.Version/Version": {}, + "pkg.cri.server.ImageFsInfo": {}, + "pkg.cri.server.ImageStatus": {}, + "pkg.cri.server.ListImages": {}, + //"pkg.cri.server.PullImage": {}, + //"pkg.unpack.unpacker.UnpackHandler": {}, + //"pkg.unpack.unpacker.fetchLayer": {}, + //"pkg.unpack.unpacker.unpack": {}, + //"pkg.unpack.unpacker.unpackLayer": {}, + //"pull.Pull": {}, + //"pull.UnpackWait": {}, + //"pull.fetch": {}, + //"pull.pull.createNewImage": {}, + //"remotes.docker.resolver.HTTPRequest": {}, "runtime.v1.ImageService/ImageFsInfo": {}, + "runtime.v1.ImageService/ImageStatus": {}, + "runtime.v1.ImageService/ListImages": {}, + //"runtime.v1.ImageService/PullImage": {}, + "runtime.v1.RuntimeService/ContainerStatus": {}, + //"runtime.v1.RuntimeService/CreateContainer": {}, + "runtime.v1.RuntimeService/ExecSync": {}, + "runtime.v1.RuntimeService/ListContainerStats": {}, + "runtime.v1.RuntimeService/ListContainers": {}, + "runtime.v1.RuntimeService/ListPodSandbox": {}, + "runtime.v1.RuntimeService/PodSandboxStatus": {}, + //"runtime.v1.RuntimeService/RemoveContainer": {}, + //"runtime.v1.RuntimeService/RemovePodSandbox": {}, + //"runtime.v1.RuntimeService/RunPodSandbox": {}, + //"runtime.v1.RuntimeService/StartContainer": {}, + "runtime.v1.RuntimeService/Status": {}, + //"runtime.v1.RuntimeService/StopContainer": {}, + //"runtime.v1.RuntimeService/StopPodSandbox": {}, + "runtime.v1.RuntimeService/UpdateContainerResources": {}, + "runtime.v1.RuntimeService/Version": {}, + "runtime.v1alpha2.RuntimeService/ListPodSandbox": {}, + "runtime.v1alpha2.RuntimeService/PodSandboxStatus": {}, }, }), } @@ -212,9 +260,8 @@ func (s *readSampling) ShouldSample(p trace.SamplingParameters) trace.SamplingRe Decision: trace.Drop, } } - println(p.Name) return trace.SamplingResult{ - Decision: trace.RecordOnly, + Decision: trace.RecordAndSample, } } From 7576b7fa5c331f1a57a48881571ed88c459c86c1 Mon Sep 17 00:00:00 2001 From: Baptiste Girard-Carrabin Date: Fri, 14 Jun 2024 11:58:38 +0200 Subject: [PATCH 3/6] [tracing] Create NameSampler Simple sampler that samples every span having a certain name. It's best used in conjunction with the ParentBased sampler so that the child spans are also sampled. --- tracing/plugin/sampler.go | 46 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 tracing/plugin/sampler.go diff --git a/tracing/plugin/sampler.go b/tracing/plugin/sampler.go new file mode 100644 index 0000000000000..c34c57d01b8d4 --- /dev/null +++ b/tracing/plugin/sampler.go @@ -0,0 +1,46 @@ +package plugin + +import ( + "fmt" + + sdkTrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/trace" +) + +type NameSampler struct { + // allow is a set of names that should be sampled. + // Uses a map of empty structs for O(1) lookups and no memory overhead. + allow map[string]struct{} +} + +// NameBased returns a Sampler that samples every span having a certain name. +// It should be used in conjunction with the ParentBased sampler so that the child spans are also sampled. +func NameBased(allowedNames []string) NameSampler { + allowedNamesMap := make(map[string]struct{}, len(allowedNames)) + for _, name := range allowedNames { + allowedNamesMap[name] = struct{}{} + } + return NameSampler{ + allow: allowedNamesMap, + } +} + +func (ns NameSampler) ShouldSample(parameters sdkTrace.SamplingParameters) sdkTrace.SamplingResult { + psc := trace.SpanContextFromContext(parameters.ParentContext) + + if _, ok := ns.allow[parameters.Name]; ok { + return sdkTrace.SamplingResult{ + Decision: sdkTrace.RecordAndSample, + Tracestate: psc.TraceState(), + } + } + + return sdkTrace.SamplingResult{ + Decision: sdkTrace.Drop, + Tracestate: psc.TraceState(), + } +} + +func (ns NameSampler) Description() string { + return fmt.Sprintf("NameBased:{%v}", ns.allow) +} From f32736b73575a8ecbf3bc582e3a12c127e082b08 Mon Sep 17 00:00:00 2001 From: Baptiste Girard-Carrabin Date: Fri, 14 Jun 2024 12:21:30 +0200 Subject: [PATCH 4/6] [tracing] Remove read sampler --- tracing/plugin/otlp.go | 81 ++---------------------------------------- 1 file changed, 2 insertions(+), 79 deletions(-) diff --git a/tracing/plugin/otlp.go b/tracing/plugin/otlp.go index f19c821b0d4cc..527a998fb2703 100644 --- a/tracing/plugin/otlp.go +++ b/tracing/plugin/otlp.go @@ -21,7 +21,6 @@ import ( "fmt" "io" "net/url" - "sync" "time" "github.com/containerd/containerd/errdefs" @@ -165,64 +164,11 @@ func newTracer(ctx context.Context, config *TraceConfig, procs []trace.SpanProce return nil, fmt.Errorf("failed to create resource: %w", err) } - //sampler := trace.ParentBased(trace.TraceIDRatioBased(config.TraceSamplingRatio)) + sampler := trace.ParentBased(trace.TraceIDRatioBased(config.TraceSamplingRatio)) opts := []trace.TracerProviderOption{ + trace.WithSampler(sampler), trace.WithResource(res), - trace.WithSampler(&readSampling{ - drop: map[string]struct{}{ - "containerd.services.containers.v1.Containers/Get": {}, - "containerd.services.leases.v1.Leases/Delete": {}, - "containerd.services.containers.v1.Containers/ListStream": {}, - "containerd.services.content.v1.Content/Info": {}, - "containerd.services.content.v1.Content/Read": {}, - "containerd.services.content.v1.Content/Write": {}, - "containerd.services.events.v1.Events/Subscribe": {}, - "containerd.services.images.v1.Images/Get": {}, - "containerd.services.images.v1.Images/List": {}, - "containerd.services.namespaces.v1.Namespaces/Get": {}, - "containerd.services.namespaces.v1.Namespaces/List": {}, - "containerd.services.snapshots.v1.Snapshots/Stat": {}, - "containerd.services.tasks.v1.Tasks/Get": {}, - "containerd.services.tasks.v1.Tasks/ListPids": {}, - "containerd.services.version.v1.Version/Version": {}, - "pkg.cri.server.ImageFsInfo": {}, - "pkg.cri.server.ImageStatus": {}, - "pkg.cri.server.ListImages": {}, - //"pkg.cri.server.PullImage": {}, - //"pkg.unpack.unpacker.UnpackHandler": {}, - //"pkg.unpack.unpacker.fetchLayer": {}, - //"pkg.unpack.unpacker.unpack": {}, - //"pkg.unpack.unpacker.unpackLayer": {}, - //"pull.Pull": {}, - //"pull.UnpackWait": {}, - //"pull.fetch": {}, - //"pull.pull.createNewImage": {}, - //"remotes.docker.resolver.HTTPRequest": {}, - "runtime.v1.ImageService/ImageFsInfo": {}, - "runtime.v1.ImageService/ImageStatus": {}, - "runtime.v1.ImageService/ListImages": {}, - //"runtime.v1.ImageService/PullImage": {}, - "runtime.v1.RuntimeService/ContainerStatus": {}, - //"runtime.v1.RuntimeService/CreateContainer": {}, - "runtime.v1.RuntimeService/ExecSync": {}, - "runtime.v1.RuntimeService/ListContainerStats": {}, - "runtime.v1.RuntimeService/ListContainers": {}, - "runtime.v1.RuntimeService/ListPodSandbox": {}, - "runtime.v1.RuntimeService/PodSandboxStatus": {}, - //"runtime.v1.RuntimeService/RemoveContainer": {}, - //"runtime.v1.RuntimeService/RemovePodSandbox": {}, - //"runtime.v1.RuntimeService/RunPodSandbox": {}, - //"runtime.v1.RuntimeService/StartContainer": {}, - "runtime.v1.RuntimeService/Status": {}, - //"runtime.v1.RuntimeService/StopContainer": {}, - //"runtime.v1.RuntimeService/StopPodSandbox": {}, - "runtime.v1.RuntimeService/UpdateContainerResources": {}, - "runtime.v1.RuntimeService/Version": {}, - "runtime.v1alpha2.RuntimeService/ListPodSandbox": {}, - "runtime.v1alpha2.RuntimeService/PodSandboxStatus": {}, - }, - }), } for _, proc := range procs { @@ -246,29 +192,6 @@ func newTracer(ctx context.Context, config *TraceConfig, procs []trace.SpanProce } -type readSampling struct { - sync.Mutex - drop map[string]struct{} -} - -func (s *readSampling) ShouldSample(p trace.SamplingParameters) trace.SamplingResult { - s.Lock() - _, ok := s.drop[p.Name] - s.Unlock() - if ok { - return trace.SamplingResult{ - Decision: trace.Drop, - } - } - return trace.SamplingResult{ - Decision: trace.RecordAndSample, - } -} - -func (s *readSampling) Description() string { - return "readSampling" -} - // Returns a composite TestMap propagator func propagators() propagation.TextMapPropagator { return propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{}) From b279c54c2705f0422073a46ff647e835648ad676 Mon Sep 17 00:00:00 2001 From: Baptiste Girard-Carrabin Date: Fri, 14 Jun 2024 14:30:37 +0200 Subject: [PATCH 5/6] [tracing] Configure otlp plugin to use NameSampler --- tracing/plugin/otlp.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tracing/plugin/otlp.go b/tracing/plugin/otlp.go index 527a998fb2703..a5b56e12d63af 100644 --- a/tracing/plugin/otlp.go +++ b/tracing/plugin/otlp.go @@ -21,6 +21,7 @@ import ( "fmt" "io" "net/url" + "strings" "time" "github.com/containerd/containerd/errdefs" @@ -101,6 +102,7 @@ type OTLPConfig struct { type TraceConfig struct { ServiceName string `toml:"service_name"` TraceSamplingRatio float64 `toml:"sampling_ratio"` + AllowedNames string `toml:"allowed_names"` } type closer struct { @@ -164,7 +166,17 @@ func newTracer(ctx context.Context, config *TraceConfig, procs []trace.SpanProce return nil, fmt.Errorf("failed to create resource: %w", err) } - sampler := trace.ParentBased(trace.TraceIDRatioBased(config.TraceSamplingRatio)) + if config.TraceSamplingRatio < 1 && config.AllowedNames == "" { + return nil, fmt.Errorf("only one of trace_sampling_ratio or allowed_names can be set") + } + + var sampler trace.Sampler + if config.AllowedNames == "" { + sampler = trace.ParentBased(trace.TraceIDRatioBased(config.TraceSamplingRatio)) + } else { + allowedNames := strings.Split(config.AllowedNames, ",") + sampler = trace.ParentBased(NameBased(allowedNames)) + } opts := []trace.TracerProviderOption{ trace.WithSampler(sampler), From 300880b945ae0c06967099f1cbe7cd72924632a4 Mon Sep 17 00:00:00 2001 From: Baptiste Girard-Carrabin Date: Fri, 14 Jun 2024 18:34:39 +0200 Subject: [PATCH 6/6] [tracing] Allow tag configuration fron env var Allow pulling attributes from OTEL_RESOURCE_ATTRIBUTES and OTEL_SERVICE_NAME environment variables . --- tracing/plugin/otlp.go | 1 + 1 file changed, 1 insertion(+) diff --git a/tracing/plugin/otlp.go b/tracing/plugin/otlp.go index a5b56e12d63af..f7b497d5d0af9 100644 --- a/tracing/plugin/otlp.go +++ b/tracing/plugin/otlp.go @@ -161,6 +161,7 @@ func newTracer(ctx context.Context, config *TraceConfig, procs []trace.SpanProce // Service name used to displace traces in backends semconv.ServiceNameKey.String(config.ServiceName), ), + resource.WithFromEnv(), ) if err != nil { return nil, fmt.Errorf("failed to create resource: %w", err)