From 073ad15cf19327a0eacdbf647964e3c666de9f2a Mon Sep 17 00:00:00 2001 From: Tim Mwangi Date: Tue, 24 Jan 2023 10:51:06 -0800 Subject: [PATCH 1/8] feat: otel go metrics setup Set up otel go metrics reporting and add a RequestCount metric for the http handler instrumentation. --- go.mod | 11 +++- go.sum | 11 ++++ .../hypertrace/net/hyperhttp/handler.go | 2 +- instrumentation/opentelemetry/init.go | 65 +++++++++++++++++++ sdk/instrumentation/net/http/handler.go | 40 +++++++++++- sdk/instrumentation/net/http/handler_test.go | 16 +++-- 6 files changed, 132 insertions(+), 13 deletions(-) diff --git a/go.mod b/go.mod index 2e3194bb..e4333f9c 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,9 @@ go 1.17 require ( contrib.go.opencensus.io/exporter/zipkin v0.1.2 github.com/gin-gonic/gin v1.7.2 + github.com/go-logr/stdr v1.2.2 github.com/golang/protobuf v1.5.2 + github.com/google/uuid v1.1.2 github.com/gorilla/mux v1.8.0 github.com/hypertrace/agent-config/gen/go v0.0.0-20221206162312-4a295cabd009 github.com/json-iterator/go v1.1.11 // indirect @@ -23,13 +25,18 @@ require ( go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.10.0 go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.10.0 go.opentelemetry.io/otel/exporters/zipkin v1.10.0 + go.opentelemetry.io/otel/metric v0.31.0 go.opentelemetry.io/otel/sdk v1.10.0 go.opentelemetry.io/otel/trace v1.10.0 google.golang.org/grpc v1.49.0 google.golang.org/protobuf v1.28.1 ) -require github.com/google/uuid v1.1.2 +require ( + go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.31.0 + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.31.0 + go.opentelemetry.io/otel/sdk/metric v0.31.0 +) require ( cloud.google.com/go v0.81.0 // indirect @@ -39,7 +46,6 @@ require ( github.com/ghodss/yaml v1.0.0 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-logr/logr v1.2.3 // indirect - github.com/go-logr/stdr v1.2.2 // indirect github.com/go-playground/locales v0.13.0 // indirect github.com/go-playground/universal-translator v0.17.0 // indirect github.com/go-playground/validator/v10 v10.4.1 // indirect @@ -52,7 +58,6 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/ugorji/go/codec v1.1.7 // indirect go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.10.0 // indirect - go.opentelemetry.io/otel/metric v0.31.0 // indirect go.opentelemetry.io/proto/otlp v0.19.0 // indirect golang.org/x/crypto v0.0.0-20210920023735-84f357641f63 // indirect golang.org/x/net v0.0.0-20211216030914-fe4d6282115f // indirect diff --git a/go.sum b/go.sum index e82e398e..22db73a0 100644 --- a/go.sum +++ b/go.sum @@ -47,6 +47,8 @@ github.com/Shopify/sarama v1.30.0/go.mod h1:zujlQQx1kzHsh4jfV1USnptCQrHAEZ2Hk8fT github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/Shopify/toxiproxy/v2 v2.1.6-0.20210914104332-15ea381dcdae/go.mod h1:/cvHQkZ1fst0EmZnA5dFtiQdWCNCFYzb+uE2vqVgvx0= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= +github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4= github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -315,8 +317,13 @@ go.opentelemetry.io/contrib/propagators/b3 v1.10.0/go.mod h1:oxvamQ/mTDFQVugml/u go.opentelemetry.io/otel v1.8.0/go.mod h1:2pkj+iMj0o03Y+cW6/m8Y4WkRdYN3AvCXCnzRMp9yvM= go.opentelemetry.io/otel v1.10.0 h1:Y7DTJMR6zs1xkS/upamJYk0SxxN4C9AqRd77jmZnyY4= go.opentelemetry.io/otel v1.10.0/go.mod h1:NbvWjCthWHKBEUMpf0/v8ZRZlni86PpGFEMA9pnQSnQ= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.8.0/go.mod h1:78XhIg8Ht9vR4tbLNUhXsiOnE2HOuSeKAiAcoVQEpOY= go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.10.0 h1:TaB+1rQhddO1sF71MpZOZAuSPW1klK2M8XxfrBMfK7Y= go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.10.0/go.mod h1:78XhIg8Ht9vR4tbLNUhXsiOnE2HOuSeKAiAcoVQEpOY= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.31.0 h1:H0+xwv4shKw0gfj/ZqR13qO2N/dBQogB1OcRjJjV39Y= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.31.0/go.mod h1:nkenGD8vcvs0uN6WhR90ZVHQlgDsRmXicnNadMnk+XQ= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.31.0 h1:BaQ2xM5cPmldVCMvbLoy5tcLUhXCtIhItDYBNw83B7Y= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.31.0/go.mod h1:VRr8tlXQEsTdesDCh0qBe2iKDWhpi3ZqDYw6VlZ8MhI= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.10.0 h1:pDDYmo0QadUPal5fwXoY1pmMpFcdyhXOmL5drCrI3vU= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.10.0/go.mod h1:Krqnjl22jUJ0HgMzw5eveuCvFDXY4nSYb4F8t5gdrag= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.10.0 h1:KtiUEhQmj/Pa874bVYKGNVdq8NPKiacPbaRRtgXi+t4= @@ -327,12 +334,16 @@ go.opentelemetry.io/otel/exporters/zipkin v1.10.0 h1:HcPAFsFpEBKF+G5NIOA+gBsxifd go.opentelemetry.io/otel/exporters/zipkin v1.10.0/go.mod h1:HdfvgwcOoCB0+zzrTHycW6btjK0zNpkz2oTGO815SCI= go.opentelemetry.io/otel/metric v0.31.0 h1:6SiklT+gfWAwWUR0meEMxQBtihpiEs4c+vL9spDTqUs= go.opentelemetry.io/otel/metric v0.31.0/go.mod h1:ohmwj9KTSIeBnDBm/ZwH2PSZxZzoOaG2xZeekTRzL5A= +go.opentelemetry.io/otel/sdk v1.8.0/go.mod h1:uPSfc+yfDH2StDM/Rm35WE8gXSNdvCg023J6HeGNO0c= go.opentelemetry.io/otel/sdk v1.10.0 h1:jZ6K7sVn04kk/3DNUdJ4mqRlGDiXAVuIG+MMENpTNdY= go.opentelemetry.io/otel/sdk v1.10.0/go.mod h1:vO06iKzD5baltJz1zarxMCNHFpUlUiOy4s65ECtn6kE= +go.opentelemetry.io/otel/sdk/metric v0.31.0 h1:2sZx4R43ZMhJdteKAlKoHvRgrMp53V1aRxvEf5lCq8Q= +go.opentelemetry.io/otel/sdk/metric v0.31.0/go.mod h1:fl0SmNnX9mN9xgU6OLYLMBMrNAsaZQi7qBwprwO3abk= go.opentelemetry.io/otel/trace v1.8.0/go.mod h1:0Bt3PXY8w+3pheS3hQUt+wow8b1ojPaTBoTCh2zIFI4= go.opentelemetry.io/otel/trace v1.10.0 h1:npQMbR8o7mum8uF95yFbOEJffhs1sbCOfDh8zAJiH5E= go.opentelemetry.io/otel/trace v1.10.0/go.mod h1:Sij3YYczqAdz+EhmGhE6TpTxUO5/F/AzrK+kxfGqySM= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.opentelemetry.io/proto/otlp v0.18.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.opentelemetry.io/proto/otlp v0.19.0 h1:IVN6GR+mhC4s5yfcTbmzHYODqvWAp3ZedA2SJPI1Nnw= go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= diff --git a/instrumentation/hypertrace/net/hyperhttp/handler.go b/instrumentation/hypertrace/net/hyperhttp/handler.go index de075712..2b95a4fe 100644 --- a/instrumentation/hypertrace/net/hyperhttp/handler.go +++ b/instrumentation/hypertrace/net/hyperhttp/handler.go @@ -16,7 +16,7 @@ func NewHandler(base http.Handler, operation string, opts ...Option) http.Handle } return otelhttp.NewHandler( - sdkhttp.WrapHandler(base, opentelemetry.SpanFromContext, o.toSDKOptions(), map[string]string{}), + sdkhttp.WrapHandler(base, operation, opentelemetry.SpanFromContext, o.toSDKOptions(), map[string]string{}), operation, ) } diff --git a/instrumentation/opentelemetry/init.go b/instrumentation/opentelemetry/init.go index 7dfcf079..1ede7654 100644 --- a/instrumentation/opentelemetry/init.go +++ b/instrumentation/opentelemetry/init.go @@ -14,6 +14,7 @@ import ( "sync" "time" + "go.opentelemetry.io/otel/exporters/otlp/otlpmetric" "go.opentelemetry.io/otel/exporters/otlp/otlptrace" "go.opentelemetry.io/otel/trace" @@ -22,10 +23,12 @@ import ( config "github.com/hypertrace/agent-config/gen/go/v1" "go.opentelemetry.io/otel/attribute" + "github.com/go-logr/stdr" sdkconfig "github.com/hypertrace/goagent/sdk/config" "github.com/hypertrace/goagent/version" "go.opentelemetry.io/contrib/propagators/b3" "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc" otlpgrpc "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" "go.opentelemetry.io/otel/exporters/stdout/stdouttrace" "go.opentelemetry.io/otel/exporters/zipkin" @@ -34,6 +37,17 @@ import ( sdktrace "go.opentelemetry.io/otel/sdk/trace" semconv "go.opentelemetry.io/otel/semconv/v1.4.0" "google.golang.org/grpc/credentials" + //"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc" + //"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp" + //"go.opentelemetry.io/otel/exporters/stdout/stdoutmetric" + // otelmetric "go.opentelemetry.io/otel/metric" + otelmetricglobal "go.opentelemetry.io/otel/metric/global" + controller "go.opentelemetry.io/otel/sdk/metric/controller/basic" + processor "go.opentelemetry.io/otel/sdk/metric/processor/basic" + "go.opentelemetry.io/otel/sdk/metric/selector/simple" + // "go.opentelemetry.io/otel/sdk/metric" + // "go.opentelemetry.io/otel/sdk/metric/metricdata" + // "go.opentelemetry.io/otel/sdk/metric/view" ) var batchTimeout = time.Duration(200) * time.Millisecond @@ -167,6 +181,7 @@ func Init(cfg *config.AgentConfig) func() { // InitWithSpanProcessorWrapper initializes opentelemetry tracing with a wrapper over span processor // and returns a shutdown function to flush data immediately on a termination signal. func InitWithSpanProcessorWrapper(cfg *config.AgentConfig, wrapper SpanProcessorWrapper) func() { + stdr.SetVerbosity(5) mu.Lock() defer mu.Unlock() if initialized { @@ -219,6 +234,8 @@ func InitWithSpanProcessorWrapper(cfg *config.AgentConfig, wrapper SpanProcessor otel.SetTextMapPropagator(makePropagator(cfg.PropagationFormats)) + initMetrics() + traceProviders = make(map[string]*sdktrace.TracerProvider) globalSampler = sampler initialized = true @@ -348,3 +365,51 @@ func (sp *spanProcessorWithWrapper) Shutdown(ctx context.Context) error { func (sp *spanProcessorWithWrapper) ForceFlush(ctx context.Context) error { return sp.processor.ForceFlush(ctx) } + +func initMetrics() { + // stdout exporter + // exporter, err := stdoutmetric.New(stdoutmetric.WithPrettyPrint()) + // if err != nil { + // log.Printf("error in init metrics: %v", fmt.Errorf("creating stdoutmetric exporter: %w", err)) + // //return nil, fmt.Errorf("creating stdoutmetric exporter: %w", err) + // return + // } + + // otlp exporter + opts := []otlpmetricgrpc.Option{ + otlpmetricgrpc.WithEndpoint("localhost:4317"), + otlpmetricgrpc.WithInsecure(), + } + + exporter, err := otlpmetric.New( + context.Background(), + otlpmetricgrpc.NewClient(opts...), + ) + if err != nil { + log.Printf("error in init metrics: %v", fmt.Errorf("creating otlpmetric exporter: %w", err)) + return + } + + pusher := controller.New( + processor.NewFactory( + simple.NewWithInexpensiveDistribution(), + exporter, + ), + controller.WithExporter(exporter), + ) + if err := pusher.Start(context.Background()); err != nil { + log.Fatalf("starting push controller: %v", err) + } + + otelmetricglobal.SetMeterProvider(pusher) + + // metricsClient := + // defaultView, _ := view.New(view.MatchInstrumentName("*")) + + // meterProvider := metric.NewMeterProvider(metric.WithReader(metric.NewPeriodicReader(metricsClient, + // metric.WithAggregationSelector(metric.DefaultAggregationSelector), + // metric.WithTemporalitySelector(deltaTemporalitySelector), + // ), defaultView, defaultView)) + + // otelmetricglobal.SetMeterProvider(meterProvider) +} diff --git a/sdk/instrumentation/net/http/handler.go b/sdk/instrumentation/net/http/handler.go index dcb49ead..d449a9a8 100644 --- a/sdk/instrumentation/net/http/handler.go +++ b/sdk/instrumentation/net/http/handler.go @@ -12,14 +12,31 @@ import ( "github.com/hypertrace/goagent/sdk/filter" internalconfig "github.com/hypertrace/goagent/sdk/internal/config" "github.com/hypertrace/goagent/sdk/internal/container" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/metric/global" + "go.opentelemetry.io/otel/metric/instrument/syncint64" + semconv "go.opentelemetry.io/otel/semconv/v1.12.0" +) + +// Server HTTP metrics. +const ( + // Pseudo of go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp#RequestCount since a metric is not + // created for that one for some reason.(annotated with hypertrace to avoid a duplicate if otel go ever implement + // their own) + RequestCount = "hypertrace.http.server.request_count" // Incoming request count total ) type handler struct { delegate http.Handler + operation string defaultAttributes map[string]string spanFromContextRetriever sdk.SpanFromContext dataCaptureConfig *config.DataCapture filter filter.Filter + // Some metrics in here. + counters map[string]syncint64.Counter } // Options for HTTP handler instrumentation @@ -29,7 +46,7 @@ type Options struct { // WrapHandler wraps an uninstrumented handler (e.g. a handleFunc) and returns a new one // that should be used as base to an instrumented handler -func WrapHandler(delegate http.Handler, spanFromContext sdk.SpanFromContext, options *Options, spanAttributes map[string]string) http.Handler { +func WrapHandler(delegate http.Handler, operation string, spanFromContext sdk.SpanFromContext, options *Options, spanAttributes map[string]string) http.Handler { defaultAttributes := make(map[string]string) for k, v := range spanAttributes { defaultAttributes[k] = v @@ -41,10 +58,29 @@ func WrapHandler(delegate http.Handler, spanFromContext sdk.SpanFromContext, opt if options != nil && options.Filter != nil { f = options.Filter } - return &handler{delegate, defaultAttributes, spanFromContext, internalconfig.GetConfig().GetDataCapture(), f} + + mp := global.MeterProvider() + meter := mp.Meter("go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp", + metric.WithInstrumentationVersion(otelhttp.SemVersion())) + counters := make(map[string]syncint64.Counter) + + requestCountCounter, err := meter.SyncInt64().Counter(RequestCount) + if err != nil { + otel.Handle(err) + } + + counters[RequestCount] = requestCountCounter + + return &handler{delegate, operation, defaultAttributes, spanFromContext, internalconfig.GetConfig().GetDataCapture(), f, counters} } func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + // Add metrics using the same logic in go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp#handler.go + ctx := r.Context() + labeler, _ := otelhttp.LabelerFromContext(ctx) + attributes := append(labeler.Get(), semconv.HTTPServerMetricAttributesFromHTTPRequest(h.operation, r)...) + h.counters[RequestCount].Add(ctx, 1, attributes...) + span := h.spanFromContextRetriever(r.Context()) if span.IsNoop() { diff --git a/sdk/instrumentation/net/http/handler_test.go b/sdk/instrumentation/net/http/handler_test.go index d58a0e6a..5ea3fda2 100644 --- a/sdk/instrumentation/net/http/handler_test.go +++ b/sdk/instrumentation/net/http/handler_test.go @@ -16,6 +16,8 @@ import ( "github.com/stretchr/testify/assert" ) +const fooOpName string = "/foo" + var emptyTestConfig = &config.DataCapture{ HttpHeaders: &config.Message{ Request: config.Bool(false), @@ -50,7 +52,7 @@ func TestServerRequestWithNilBodyIsntChanged(t *testing.T) { assert.Nil(t, r.Body) }) - wh, _ := WrapHandler(h, mock.SpanFromContext, &Options{}, map[string]string{}).(*handler) + wh, _ := WrapHandler(h, fooOpName, mock.SpanFromContext, &Options{}, map[string]string{}).(*handler) wh.dataCaptureConfig = emptyTestConfig ih := &mockHandler{baseHandler: wh} @@ -74,7 +76,7 @@ func TestServerRequestIsSuccessfullyTraced(t *testing.T) { rw.Write([]byte("ponse_body")) }) - wh, _ := WrapHandler(h, mock.SpanFromContext, &Options{}, map[string]string{"foo": "bar"}).(*handler) + wh, _ := WrapHandler(h, fooOpName, mock.SpanFromContext, &Options{}, map[string]string{"foo": "bar"}).(*handler) wh.dataCaptureConfig = emptyTestConfig ih := &mockHandler{baseHandler: wh} @@ -103,7 +105,7 @@ func TestHostIsSuccessfullyRecorded(t *testing.T) { assert.Nil(t, r.Body) }) - wh, _ := WrapHandler(h, mock.SpanFromContext, &Options{}, map[string]string{}).(*handler) + wh, _ := WrapHandler(h, fooOpName, mock.SpanFromContext, &Options{}, map[string]string{}).(*handler) wh.dataCaptureConfig = emptyTestConfig ih := &mockHandler{baseHandler: wh} @@ -139,7 +141,7 @@ func TestServerRequestHeadersAreSuccessfullyRecorded(t *testing.T) { rw.WriteHeader(202) }) - wh, _ := WrapHandler(h, mock.SpanFromContext, &Options{}, map[string]string{}).(*handler) + wh, _ := WrapHandler(h, fooOpName, mock.SpanFromContext, &Options{}, map[string]string{}).(*handler) ih := &mockHandler{baseHandler: wh} wh.dataCaptureConfig = emptyTestConfig wh.dataCaptureConfig.HttpHeaders = &config.Message{ @@ -285,7 +287,7 @@ func TestServerRecordsRequestAndResponseBodyAccordingly(t *testing.T) { rw.Write([]byte(tCase.responseBody)) }) - wh, _ := WrapHandler(h, mock.SpanFromContext, &Options{}, map[string]string{}).(*handler) + wh, _ := WrapHandler(h, fooOpName, mock.SpanFromContext, &Options{}, map[string]string{}).(*handler) wh.dataCaptureConfig = emptyTestConfig wh.dataCaptureConfig.HttpBody = &config.Message{ Request: config.Bool(tCase.captureHTTPBodyConfig), @@ -463,7 +465,7 @@ func TestServerRequestFilter(t *testing.T) { rw.WriteHeader(http.StatusOK) }) - wh, _ := WrapHandler(h, mock.SpanFromContext, tCase.options, map[string]string{}).(*handler) + wh, _ := WrapHandler(h, fooOpName, mock.SpanFromContext, tCase.options, map[string]string{}).(*handler) ih := &mockHandler{baseHandler: wh} r, _ := http.NewRequest("POST", tCase.url, strings.NewReader(tCase.body)) for i := 0; i < len(tCase.headerKeys); i++ { @@ -491,7 +493,7 @@ func TestProcessingBodyIsTrimmed(t *testing.T) { h := http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {}) - wh, _ := WrapHandler(h, mock.SpanFromContext, &Options{ + wh, _ := WrapHandler(h, fooOpName, mock.SpanFromContext, &Options{ Filter: mock.Filter{ BodyEvaluator: func(span sdk.Span, body []byte, headers map[string][]string) result.FilterResult { assert.Equal(t, "{", string(body)) // body is truncated From ca7b03cdec6a5689a12dbeacd1da371760aa8d70 Mon Sep 17 00:00:00 2001 From: Tim Mwangi Date: Tue, 24 Jan 2023 13:07:38 -0800 Subject: [PATCH 2/8] cleanup metrics init --- go.mod | 3 +- go.sum | 2 + .../github.com/gin-gonic/hypergin/gin.go | 7 +- .../github.com/gorilla/hypermux/mux.go | 3 +- instrumentation/opentelemetry/init.go | 139 ++++++++++-------- .../opentelemetry/net/hyperhttp/handler.go | 3 +- 6 files changed, 89 insertions(+), 68 deletions(-) diff --git a/go.mod b/go.mod index e4333f9c..594ab06c 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.17 require ( contrib.go.opencensus.io/exporter/zipkin v0.1.2 github.com/gin-gonic/gin v1.7.2 - github.com/go-logr/stdr v1.2.2 + github.com/go-logr/stdr v1.2.2 // indirect github.com/golang/protobuf v1.5.2 github.com/google/uuid v1.1.2 github.com/gorilla/mux v1.8.0 @@ -35,6 +35,7 @@ require ( require ( go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.31.0 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.31.0 + go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v0.31.0 go.opentelemetry.io/otel/sdk/metric v0.31.0 ) diff --git a/go.sum b/go.sum index 22db73a0..4b5f5811 100644 --- a/go.sum +++ b/go.sum @@ -328,6 +328,8 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.10.0 h1:pDDYmo0QadUPal5fwXo go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.10.0/go.mod h1:Krqnjl22jUJ0HgMzw5eveuCvFDXY4nSYb4F8t5gdrag= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.10.0 h1:KtiUEhQmj/Pa874bVYKGNVdq8NPKiacPbaRRtgXi+t4= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.10.0/go.mod h1:OfUCyyIiDvNXHWpcWgbF+MWvqPZiNa3YDEnivcnYsV0= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v0.31.0 h1:fu/wxbXqjgIRZYzQNrF175qtwrJx+oQSFhZpTIbNQLc= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v0.31.0/go.mod h1:a80IJcYgCLVXJurhoyPjMBiNI5gPrWXLBTAwOp8N6Vw= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.10.0 h1:c9UtMu/qnbLlVwTwt+ABrURrioEruapIslTDYZHJe2w= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.10.0/go.mod h1:h3Lrh9t3Dnqp3NPwAZx7i37UFX7xrfnO1D+fuClREOA= go.opentelemetry.io/otel/exporters/zipkin v1.10.0 h1:HcPAFsFpEBKF+G5NIOA+gBsxifd3Ej+wb+KsdBLa15E= diff --git a/instrumentation/opentelemetry/github.com/gin-gonic/hypergin/gin.go b/instrumentation/opentelemetry/github.com/gin-gonic/hypergin/gin.go index 50baa576..438cf657 100644 --- a/instrumentation/opentelemetry/github.com/gin-gonic/hypergin/gin.go +++ b/instrumentation/opentelemetry/github.com/gin-gonic/hypergin/gin.go @@ -83,15 +83,16 @@ func spanNameFormatter(operation string, r *http.Request) (spanName string) { func Middleware(options *sdkhttp.Options) gin.HandlerFunc { return wrap(func(delegate http.Handler) http.Handler { wrappedHandler, ok := delegate.(*nextRequestHandler) - + ginOperationName := "" // if we fail to extract the next request handler from delegate the route template won't be reported if ok { + ginOperationName := wrappedHandler.c.FullPath() rc := wrappedHandler.c.Request.Context() - ctx := context.WithValue(rc, hyperGinKey, ginRoute{route: wrappedHandler.c.FullPath()}) + ctx := context.WithValue(rc, hyperGinKey, ginRoute{route: ginOperationName}) wrappedHandler.c.Request = wrappedHandler.c.Request.WithContext(ctx) } return otelhttp.NewHandler( - sdkhttp.WrapHandler(delegate, opentelemetry.SpanFromContext, options, map[string]string{}), + sdkhttp.WrapHandler(delegate, ginOperationName, opentelemetry.SpanFromContext, options, map[string]string{}), "", otelhttp.WithSpanNameFormatter(spanNameFormatter), ) diff --git a/instrumentation/opentelemetry/github.com/gorilla/hypermux/mux.go b/instrumentation/opentelemetry/github.com/gorilla/hypermux/mux.go index d917e1c6..e27c7725 100644 --- a/instrumentation/opentelemetry/github.com/gorilla/hypermux/mux.go +++ b/instrumentation/opentelemetry/github.com/gorilla/hypermux/mux.go @@ -30,9 +30,10 @@ func spanNameFormatter(operation string, r *http.Request) (spanName string) { // NewMiddleware sets up a handler to start tracing the incoming requests. func NewMiddleware(options *sdkhttp.Options) mux.MiddlewareFunc { + // TODO: Get a proper operation name for http gorilla mux return func(delegate http.Handler) http.Handler { return otelhttp.NewHandler( - sdkhttp.WrapHandler(delegate, opentelemetry.SpanFromContext, options, map[string]string{}), + sdkhttp.WrapHandler(delegate, "", opentelemetry.SpanFromContext, options, map[string]string{}), "", otelhttp.WithSpanNameFormatter(spanNameFormatter), ) diff --git a/instrumentation/opentelemetry/init.go b/instrumentation/opentelemetry/init.go index 1ede7654..1526d389 100644 --- a/instrumentation/opentelemetry/init.go +++ b/instrumentation/opentelemetry/init.go @@ -23,31 +23,25 @@ import ( config "github.com/hypertrace/agent-config/gen/go/v1" "go.opentelemetry.io/otel/attribute" - "github.com/go-logr/stdr" sdkconfig "github.com/hypertrace/goagent/sdk/config" "github.com/hypertrace/goagent/version" "go.opentelemetry.io/contrib/propagators/b3" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc" otlpgrpc "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" + "go.opentelemetry.io/otel/exporters/stdout/stdoutmetric" "go.opentelemetry.io/otel/exporters/stdout/stdouttrace" "go.opentelemetry.io/otel/exporters/zipkin" + otelmetricglobal "go.opentelemetry.io/otel/metric/global" "go.opentelemetry.io/otel/propagation" + controller "go.opentelemetry.io/otel/sdk/metric/controller/basic" + sdkmetricexport "go.opentelemetry.io/otel/sdk/metric/export" + processor "go.opentelemetry.io/otel/sdk/metric/processor/basic" + "go.opentelemetry.io/otel/sdk/metric/selector/simple" "go.opentelemetry.io/otel/sdk/resource" sdktrace "go.opentelemetry.io/otel/sdk/trace" semconv "go.opentelemetry.io/otel/semconv/v1.4.0" "google.golang.org/grpc/credentials" - //"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc" - //"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp" - //"go.opentelemetry.io/otel/exporters/stdout/stdoutmetric" - // otelmetric "go.opentelemetry.io/otel/metric" - otelmetricglobal "go.opentelemetry.io/otel/metric/global" - controller "go.opentelemetry.io/otel/sdk/metric/controller/basic" - processor "go.opentelemetry.io/otel/sdk/metric/processor/basic" - "go.opentelemetry.io/otel/sdk/metric/selector/simple" - // "go.opentelemetry.io/otel/sdk/metric" - // "go.opentelemetry.io/otel/sdk/metric/metricdata" - // "go.opentelemetry.io/otel/sdk/metric/view" ) var batchTimeout = time.Duration(200) * time.Millisecond @@ -89,6 +83,53 @@ func removeProtocolPrefixForOTLP(endpoint string) string { return pieces[1] } +func makeMetricsExporterFactory(cfg *config.AgentConfig) func() (sdkmetricexport.Exporter, error) { + // We are only supporting logging and otlp metric exporters for now. We will add support for prometheus + // metrics later + switch cfg.Reporting.MetricReporterType { + case config.MetricReporterType_METRIC_REPORTER_TYPE_LOGGING: + // stdout exporter + return func() (sdkmetricexport.Exporter, error) { + // TODO: Define if endpoint could be a filepath to write into a file. + return stdoutmetric.New(stdoutmetric.WithPrettyPrint()) + } + default: + endpoint := cfg.GetReporting().GetMetricEndpoint().GetValue() + if len(endpoint) == 0 { + endpoint = cfg.GetReporting().GetEndpoint().GetValue() + } + + opts := []otlpmetricgrpc.Option{ + otlpmetricgrpc.WithEndpoint(removeProtocolPrefixForOTLP(endpoint)), + } + + if !cfg.GetReporting().GetSecure().GetValue() { + opts = append(opts, otlpmetricgrpc.WithInsecure()) + } + + certFile := cfg.GetReporting().GetCertFile().GetValue() + if len(certFile) > 0 { + if tlsCredentials, err := credentials.NewClientTLSFromFile(certFile, ""); err == nil { + opts = append(opts, otlpmetricgrpc.WithTLSCredentials(tlsCredentials)) + } else { + log.Printf("error while creating tls credentials from cert path %s: %v", certFile, err) + } + } + + if cfg.Reporting.GetEnableGrpcLoadbalancing().GetValue() { + resolver.SetDefaultScheme("dns") + opts = append(opts, otlpmetricgrpc.WithServiceConfig(`{"loadBalancingConfig": [ { "round_robin": {} } ]}`)) + } + + return func() (sdkmetricexport.Exporter, error) { + return otlpmetric.New( + context.Background(), + otlpmetricgrpc.NewClient(opts...), + ) + } + } +} + func makeExporterFactory(cfg *config.AgentConfig) func() (sdktrace.SpanExporter, error) { switch cfg.Reporting.TraceReporterType { case config.TraceReporterType_ZIPKIN: @@ -181,7 +222,6 @@ func Init(cfg *config.AgentConfig) func() { // InitWithSpanProcessorWrapper initializes opentelemetry tracing with a wrapper over span processor // and returns a shutdown function to flush data immediately on a termination signal. func InitWithSpanProcessorWrapper(cfg *config.AgentConfig, wrapper SpanProcessorWrapper) func() { - stdr.SetVerbosity(5) mu.Lock() defer mu.Unlock() if initialized { @@ -234,7 +274,28 @@ func InitWithSpanProcessorWrapper(cfg *config.AgentConfig, wrapper SpanProcessor otel.SetTextMapPropagator(makePropagator(cfg.PropagationFormats)) - initMetrics() + // Initialize metrics + metricsExporterFactory := makeMetricsExporterFactory(cfg) + metricsExporter, err := metricsExporterFactory() + if err != nil { + log.Fatal(err) + } + + metricsPusher := controller.New( + processor.NewFactory( + simple.NewWithInexpensiveDistribution(), + metricsExporter, + ), + controller.WithExporter(metricsExporter), + controller.WithResource(resources), + ) + if err := metricsPusher.Start(context.Background()); err != nil { + log.Fatalf("starting metrics push controller: %v", err) + } + + otelmetricglobal.SetMeterProvider(metricsPusher) + + //initMetrics() traceProviders = make(map[string]*sdktrace.TracerProvider) globalSampler = sampler @@ -266,6 +327,8 @@ func InitWithSpanProcessorWrapper(cfg *config.AgentConfig, wrapper SpanProcessor if err != nil { log.Printf("error while shutting down default tracer provider: %v\n", err) } + + metricsPusher.Stop(context.Background()) initialized = false enabled = false sdkconfig.ResetConfig() @@ -365,51 +428,3 @@ func (sp *spanProcessorWithWrapper) Shutdown(ctx context.Context) error { func (sp *spanProcessorWithWrapper) ForceFlush(ctx context.Context) error { return sp.processor.ForceFlush(ctx) } - -func initMetrics() { - // stdout exporter - // exporter, err := stdoutmetric.New(stdoutmetric.WithPrettyPrint()) - // if err != nil { - // log.Printf("error in init metrics: %v", fmt.Errorf("creating stdoutmetric exporter: %w", err)) - // //return nil, fmt.Errorf("creating stdoutmetric exporter: %w", err) - // return - // } - - // otlp exporter - opts := []otlpmetricgrpc.Option{ - otlpmetricgrpc.WithEndpoint("localhost:4317"), - otlpmetricgrpc.WithInsecure(), - } - - exporter, err := otlpmetric.New( - context.Background(), - otlpmetricgrpc.NewClient(opts...), - ) - if err != nil { - log.Printf("error in init metrics: %v", fmt.Errorf("creating otlpmetric exporter: %w", err)) - return - } - - pusher := controller.New( - processor.NewFactory( - simple.NewWithInexpensiveDistribution(), - exporter, - ), - controller.WithExporter(exporter), - ) - if err := pusher.Start(context.Background()); err != nil { - log.Fatalf("starting push controller: %v", err) - } - - otelmetricglobal.SetMeterProvider(pusher) - - // metricsClient := - // defaultView, _ := view.New(view.MatchInstrumentName("*")) - - // meterProvider := metric.NewMeterProvider(metric.WithReader(metric.NewPeriodicReader(metricsClient, - // metric.WithAggregationSelector(metric.DefaultAggregationSelector), - // metric.WithTemporalitySelector(deltaTemporalitySelector), - // ), defaultView, defaultView)) - - // otelmetricglobal.SetMeterProvider(meterProvider) -} diff --git a/instrumentation/opentelemetry/net/hyperhttp/handler.go b/instrumentation/opentelemetry/net/hyperhttp/handler.go index 3704faee..c241adda 100644 --- a/instrumentation/opentelemetry/net/hyperhttp/handler.go +++ b/instrumentation/opentelemetry/net/hyperhttp/handler.go @@ -10,5 +10,6 @@ import ( // WrapHandler returns a new round tripper instrumented that relies on the // needs to be used with OTel instrumentation. func WrapHandler(delegate http.Handler, options *sdkhttp.Options) http.Handler { - return sdkhttp.WrapHandler(delegate, opentelemetry.SpanFromContext, options, map[string]string{}) + // TODO: Find another way to get the operation name + return sdkhttp.WrapHandler(delegate, "", opentelemetry.SpanFromContext, options, map[string]string{}) } From 500d088e7ceb71bf380160d4a6fc7678ca7930d5 Mon Sep 17 00:00:00 2001 From: Tim Mwangi Date: Tue, 24 Jan 2023 13:40:17 -0800 Subject: [PATCH 3/8] fix lint check fail in open census libs --- instrumentation/opencensus/net/hyperhttp/handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/instrumentation/opencensus/net/hyperhttp/handler.go b/instrumentation/opencensus/net/hyperhttp/handler.go index f60b9252..e212c996 100644 --- a/instrumentation/opencensus/net/hyperhttp/handler.go +++ b/instrumentation/opencensus/net/hyperhttp/handler.go @@ -10,5 +10,5 @@ import ( // WrapHandler returns a new http.Handler that should be passed to // the *ochttp.Handler func WrapHandler(delegate http.Handler, options *sdkhttp.Options) http.Handler { - return sdkhttp.WrapHandler(delegate, opencensus.SpanFromContext, options, map[string]string{}) + return sdkhttp.WrapHandler(delegate, "", opencensus.SpanFromContext, options, map[string]string{}) } From 9c8b2cae96399a6e4445d3838d4c229af51c1085 Mon Sep 17 00:00:00 2001 From: Tim Mwangi Date: Wed, 25 Jan 2023 07:09:36 -0800 Subject: [PATCH 4/8] update golangci lint gha to v0.3.4 --- .github/workflows/pr-test.yml | 2 +- instrumentation/opencensus/net/hyperhttp/handler.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index c95c778d..80fd10b9 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -37,7 +37,7 @@ jobs: make deps - name: Lint files - uses: golangci/golangci-lint-action@v3.3.1 + uses: golangci/golangci-lint-action@v3.4.0 with: # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version. version: v1.50.1 diff --git a/instrumentation/opencensus/net/hyperhttp/handler.go b/instrumentation/opencensus/net/hyperhttp/handler.go index e212c996..fb7428e6 100644 --- a/instrumentation/opencensus/net/hyperhttp/handler.go +++ b/instrumentation/opencensus/net/hyperhttp/handler.go @@ -10,5 +10,6 @@ import ( // WrapHandler returns a new http.Handler that should be passed to // the *ochttp.Handler func WrapHandler(delegate http.Handler, options *sdkhttp.Options) http.Handler { + // TODO: If I am doing this then I might have the metrics code in the wrong place. return sdkhttp.WrapHandler(delegate, "", opencensus.SpanFromContext, options, map[string]string{}) } From 48c419bbac44da732f9fbf64dd481dce1e873093 Mon Sep 17 00:00:00 2001 From: Tim Mwangi Date: Wed, 25 Jan 2023 07:49:06 -0800 Subject: [PATCH 5/8] fix failing unit test --- instrumentation/opentelemetry/init_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/instrumentation/opentelemetry/init_test.go b/instrumentation/opentelemetry/init_test.go index 1be88263..bb9bc84b 100644 --- a/instrumentation/opentelemetry/init_test.go +++ b/instrumentation/opentelemetry/init_test.go @@ -178,9 +178,10 @@ func TestMultipleTraceProviders(t *testing.T) { assert.Equal(t, 0, count) }) - t.Run("test 2 requests after flush", func(t *testing.T) { + // 2 requests for spans and 1 for metrics. + t.Run("test 3 requests after flush", func(t *testing.T) { shutdown() - assert.Equal(t, 2, count) + assert.Equal(t, 3, count) assert.Equal(t, 0, len(traceProviders)) }) } From 2c36f66636225cc173d59666481d034d135ed784 Mon Sep 17 00:00:00 2001 From: Tim Mwangi Date: Wed, 25 Jan 2023 12:12:56 -0800 Subject: [PATCH 6/8] add service instance id to metrics resource --- instrumentation/opentelemetry/init.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/instrumentation/opentelemetry/init.go b/instrumentation/opentelemetry/init.go index 1526d389..1534f0d8 100644 --- a/instrumentation/opentelemetry/init.go +++ b/instrumentation/opentelemetry/init.go @@ -23,6 +23,7 @@ import ( config "github.com/hypertrace/agent-config/gen/go/v1" "go.opentelemetry.io/otel/attribute" + "github.com/hypertrace/goagent/instrumentation/opentelemetry/internal/identifier" sdkconfig "github.com/hypertrace/goagent/sdk/config" "github.com/hypertrace/goagent/version" "go.opentelemetry.io/contrib/propagators/b3" @@ -281,13 +282,19 @@ func InitWithSpanProcessorWrapper(cfg *config.AgentConfig, wrapper SpanProcessor log.Fatal(err) } + resourceKvps := createResources(cfg.GetServiceName().GetValue(), cfg.ResourceAttributes) + resourceKvps = append(resourceKvps, identifier.ServiceInstanceKeyValue) + metricResources, err := resource.New(context.Background(), resource.WithAttributes(resourceKvps...)) + if err != nil { + log.Fatal(err) + } metricsPusher := controller.New( processor.NewFactory( simple.NewWithInexpensiveDistribution(), metricsExporter, ), controller.WithExporter(metricsExporter), - controller.WithResource(resources), + controller.WithResource(metricResources), ) if err := metricsPusher.Start(context.Background()); err != nil { log.Fatalf("starting metrics push controller: %v", err) @@ -295,8 +302,6 @@ func InitWithSpanProcessorWrapper(cfg *config.AgentConfig, wrapper SpanProcessor otelmetricglobal.SetMeterProvider(metricsPusher) - //initMetrics() - traceProviders = make(map[string]*sdktrace.TracerProvider) globalSampler = sampler initialized = true From 9bf078d48a021be28515e7aa0ea22f5e46790af1 Mon Sep 17 00:00:00 2001 From: Tim Mwangi Date: Thu, 26 Jan 2023 12:34:14 -0800 Subject: [PATCH 7/8] add metrics enabled config --- config/defaults.go | 1 + go.mod | 2 +- go.sum | 2 + instrumentation/opentelemetry/init.go | 64 ++++++++++++++++----------- 4 files changed, 42 insertions(+), 27 deletions(-) diff --git a/config/defaults.go b/config/defaults.go index 0ce911a0..bf30c521 100644 --- a/config/defaults.go +++ b/config/defaults.go @@ -40,5 +40,6 @@ var defaultConfig = agentconfig.AgentConfig{ }, Telemetry: &agentconfig.Telemetry{ StartupSpanEnabled: agentconfig.Bool(true), + MetricsEnabled: agentconfig.Bool(true), }, } diff --git a/go.mod b/go.mod index 594ab06c..34abf540 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/golang/protobuf v1.5.2 github.com/google/uuid v1.1.2 github.com/gorilla/mux v1.8.0 - github.com/hypertrace/agent-config/gen/go v0.0.0-20221206162312-4a295cabd009 + github.com/hypertrace/agent-config/gen/go v0.0.0-20230126155022-544bf33b6648 github.com/json-iterator/go v1.1.11 // indirect github.com/kr/pretty v0.3.0 // indirect github.com/mattn/go-sqlite3 v1.14.4 diff --git a/go.sum b/go.sum index 4b5f5811..d8068a44 100644 --- a/go.sum +++ b/go.sum @@ -205,6 +205,8 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/hypertrace/agent-config/gen/go v0.0.0-20221206162312-4a295cabd009 h1:a2Y5RfRQC3Fomz+HF3FdqdvNNMCtV/lc/N1Xuf5+au0= github.com/hypertrace/agent-config/gen/go v0.0.0-20221206162312-4a295cabd009/go.mod h1:WRbKE44DNsSbRnHja1VpU+dUSrTIuduePGhZ+bXmvDw= +github.com/hypertrace/agent-config/gen/go v0.0.0-20230126155022-544bf33b6648 h1:ez9iewAbKBCXtVKdG99xtzG7GPCbEJWXP8FUlCIJsSY= +github.com/hypertrace/agent-config/gen/go v0.0.0-20230126155022-544bf33b6648/go.mod h1:WRbKE44DNsSbRnHja1VpU+dUSrTIuduePGhZ+bXmvDw= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= diff --git a/instrumentation/opentelemetry/init.go b/instrumentation/opentelemetry/init.go index 1534f0d8..eeddebdf 100644 --- a/instrumentation/opentelemetry/init.go +++ b/instrumentation/opentelemetry/init.go @@ -276,31 +276,7 @@ func InitWithSpanProcessorWrapper(cfg *config.AgentConfig, wrapper SpanProcessor otel.SetTextMapPropagator(makePropagator(cfg.PropagationFormats)) // Initialize metrics - metricsExporterFactory := makeMetricsExporterFactory(cfg) - metricsExporter, err := metricsExporterFactory() - if err != nil { - log.Fatal(err) - } - - resourceKvps := createResources(cfg.GetServiceName().GetValue(), cfg.ResourceAttributes) - resourceKvps = append(resourceKvps, identifier.ServiceInstanceKeyValue) - metricResources, err := resource.New(context.Background(), resource.WithAttributes(resourceKvps...)) - if err != nil { - log.Fatal(err) - } - metricsPusher := controller.New( - processor.NewFactory( - simple.NewWithInexpensiveDistribution(), - metricsExporter, - ), - controller.WithExporter(metricsExporter), - controller.WithResource(metricResources), - ) - if err := metricsPusher.Start(context.Background()); err != nil { - log.Fatalf("starting metrics push controller: %v", err) - } - - otelmetricglobal.SetMeterProvider(metricsPusher) + metricsShutdownFn := initializeMetrics(cfg) traceProviders = make(map[string]*sdktrace.TracerProvider) globalSampler = sampler @@ -333,7 +309,7 @@ func InitWithSpanProcessorWrapper(cfg *config.AgentConfig, wrapper SpanProcessor log.Printf("error while shutting down default tracer provider: %v\n", err) } - metricsPusher.Stop(context.Background()) + metricsShutdownFn() initialized = false enabled = false sdkconfig.ResetConfig() @@ -406,6 +382,42 @@ func RegisterServiceWithSpanProcessorWrapper(serviceName string, resourceAttribu }), tp, nil } +func initializeMetrics(cfg *config.AgentConfig) func() { + if cfg.GetTelemetry() == nil || !cfg.GetTelemetry().GetMetricsEnabled().GetValue() { + return func() {} + } + + metricsExporterFactory := makeMetricsExporterFactory(cfg) + metricsExporter, err := metricsExporterFactory() + if err != nil { + log.Fatal(err) + } + + resourceKvps := createResources(cfg.GetServiceName().GetValue(), cfg.ResourceAttributes) + resourceKvps = append(resourceKvps, identifier.ServiceInstanceKeyValue) + metricResources, err := resource.New(context.Background(), resource.WithAttributes(resourceKvps...)) + if err != nil { + log.Fatal(err) + } + metricsPusher := controller.New( + processor.NewFactory( + simple.NewWithInexpensiveDistribution(), + metricsExporter, + ), + controller.WithExporter(metricsExporter), + controller.WithResource(metricResources), + ) + if err := metricsPusher.Start(context.Background()); err != nil { + log.Fatalf("starting metrics push controller: %v", err) + } + + otelmetricglobal.SetMeterProvider(metricsPusher) + + return func() { + metricsPusher.Stop(context.Background()) + } +} + // SpanProcessorWrapper wraps otel span processor // and is responsible to delegate calls to the wrapped processor type SpanProcessorWrapper interface { From 1cebe3506a2fc1fcd1cc12faf9bc6dc53255eb76 Mon Sep 17 00:00:00 2001 From: Tim Mwangi Date: Thu, 26 Jan 2023 13:09:36 -0800 Subject: [PATCH 8/8] use main branch for agent-config --- go.mod | 2 +- go.sum | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 34abf540..26feaa70 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/golang/protobuf v1.5.2 github.com/google/uuid v1.1.2 github.com/gorilla/mux v1.8.0 - github.com/hypertrace/agent-config/gen/go v0.0.0-20230126155022-544bf33b6648 + github.com/hypertrace/agent-config/gen/go v0.0.0-20230126205246-bd4d81e696a6 github.com/json-iterator/go v1.1.11 // indirect github.com/kr/pretty v0.3.0 // indirect github.com/mattn/go-sqlite3 v1.14.4 diff --git a/go.sum b/go.sum index d8068a44..61a50b14 100644 --- a/go.sum +++ b/go.sum @@ -203,10 +203,8 @@ github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/hypertrace/agent-config/gen/go v0.0.0-20221206162312-4a295cabd009 h1:a2Y5RfRQC3Fomz+HF3FdqdvNNMCtV/lc/N1Xuf5+au0= -github.com/hypertrace/agent-config/gen/go v0.0.0-20221206162312-4a295cabd009/go.mod h1:WRbKE44DNsSbRnHja1VpU+dUSrTIuduePGhZ+bXmvDw= -github.com/hypertrace/agent-config/gen/go v0.0.0-20230126155022-544bf33b6648 h1:ez9iewAbKBCXtVKdG99xtzG7GPCbEJWXP8FUlCIJsSY= -github.com/hypertrace/agent-config/gen/go v0.0.0-20230126155022-544bf33b6648/go.mod h1:WRbKE44DNsSbRnHja1VpU+dUSrTIuduePGhZ+bXmvDw= +github.com/hypertrace/agent-config/gen/go v0.0.0-20230126205246-bd4d81e696a6 h1:MuiFiuigCk2NwMM5HOvI7FJUTZEsGeqA25c4acBjdEs= +github.com/hypertrace/agent-config/gen/go v0.0.0-20230126205246-bd4d81e696a6/go.mod h1:WRbKE44DNsSbRnHja1VpU+dUSrTIuduePGhZ+bXmvDw= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs=