From 66265c1cbc3b1563131fc3eabf264377cbf522a8 Mon Sep 17 00:00:00 2001 From: nachocano Date: Mon, 26 Aug 2019 08:35:18 -0700 Subject: [PATCH 1/8] Stats reporter refactor --- cmd/broker/filter/main.go | 7 +- cmd/broker/ingress/main.go | 7 + pkg/broker/filter/filter_handler.go | 135 ++++++----- pkg/broker/filter/filter_handler_test.go | 23 +- pkg/broker/filter/metrics.go | 112 --------- pkg/broker/filter/stats_reporter.go | 270 +++++++++++++++++++++ pkg/broker/filter/stats_reporter_test.go | 154 ++++++++++++ pkg/broker/ingress/ingress_handler.go | 45 ++-- pkg/broker/ingress/ingress_handler_test.go | 16 ++ pkg/broker/ingress/metrics.go | 77 ------ pkg/broker/ingress/stats_reporter.go | 160 ++++++++++++ pkg/broker/ingress/stats_reporter_test.go | 86 +++++++ pkg/broker/metrics.go | 26 +- pkg/broker/ttl.go | 6 - pkg/metrics/metricskey/constants.go | 85 +++++++ 15 files changed, 908 insertions(+), 301 deletions(-) delete mode 100644 pkg/broker/filter/metrics.go create mode 100644 pkg/broker/filter/stats_reporter.go create mode 100644 pkg/broker/filter/stats_reporter_test.go delete mode 100644 pkg/broker/ingress/metrics.go create mode 100644 pkg/broker/ingress/stats_reporter.go create mode 100644 pkg/broker/ingress/stats_reporter_test.go create mode 100644 pkg/metrics/metricskey/constants.go diff --git a/cmd/broker/filter/main.go b/cmd/broker/filter/main.go index 89771f398ae..392d3f3ed70 100644 --- a/cmd/broker/filter/main.go +++ b/cmd/broker/filter/main.go @@ -76,9 +76,14 @@ func main() { logger.Fatal("Error setting up trace publishing", zap.Error(err)) } + reporter, err := filter.NewStatsReporter() + if err != nil { + logger.Fatal("Error creating stats reporter", zap.Error(err)) + } + // We are running both the receiver (takes messages in from the Broker) and the dispatcher (send // the messages to the triggers' subscribers) in this binary. - handler, err := filter.NewHandler(logger, mgr.GetClient()) + handler, err := filter.NewHandler(logger, mgr.GetClient(), reporter) if err != nil { logger.Fatal("Error creating Handler", zap.Error(err)) } diff --git a/cmd/broker/ingress/main.go b/cmd/broker/ingress/main.go index 3743eee84a9..81022c07b83 100644 --- a/cmd/broker/ingress/main.go +++ b/cmd/broker/ingress/main.go @@ -104,11 +104,18 @@ func main() { if err != nil { logger.Fatal("Unable to create CE client", zap.Error(err)) } + reporter, err := ingress.NewStatsReporter() + if err != nil { + logger.Fatal("Unable to create StatsReporter", zap.Error(err)) + } + h := &ingress.Handler{ Logger: logger, CeClient: ceClient, ChannelURI: channelURI, BrokerName: env.Broker, + Namespace: env.Namespace, + Reporter: reporter, } // Run the event handler with the manager. diff --git a/pkg/broker/filter/filter_handler.go b/pkg/broker/filter/filter_handler.go index 9c9ae63c0c0..a6c24abfd40 100644 --- a/pkg/broker/filter/filter_handler.go +++ b/pkg/broker/filter/filter_handler.go @@ -25,12 +25,8 @@ import ( "sync/atomic" "time" - "knative.dev/eventing/pkg/logging" - - cloudevents "github.com/cloudevents/sdk-go" + "github.com/cloudevents/sdk-go" cehttp "github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http" - "go.opencensus.io/stats" - "go.opencensus.io/tag" "go.uber.org/zap" eventingv1alpha1 "knative.dev/eventing/pkg/apis/eventing/v1alpha1" "knative.dev/eventing/pkg/broker" @@ -48,11 +44,12 @@ type Handler struct { logger *zap.Logger client client.Client ceClient cloudevents.Client + reporter StatsReporter } // NewHandler creates a new Handler and its associated MessageReceiver. The caller is responsible for // Start()ing the returned Handler. -func NewHandler(logger *zap.Logger, client client.Client) (*Handler, error) { +func NewHandler(logger *zap.Logger, client client.Client, reporter StatsReporter) (*Handler, error) { httpTransport, err := cloudevents.NewHTTPTransport(cloudevents.WithBinaryEncoding(), cehttp.WithMiddleware(tracing.HTTPSpanMiddleware)) if err != nil { return nil, err @@ -81,6 +78,7 @@ func NewHandler(logger *zap.Logger, client client.Client) (*Handler, error) { logger: logger, client: client, ceClient: ceClient, + reporter: reporter, } err = r.initClient() if err != nil { @@ -201,52 +199,56 @@ func (r *Handler) sendEvent(ctx context.Context, tctx cloudevents.HTTPTransportC return nil, err } - // Set up the metrics context - ctx, _ = tag.New(ctx, - tag.Insert(TagTrigger, trigger.String()), - tag.Insert(TagBroker, fmt.Sprintf("%s/%s", trigger.Namespace, t.Spec.Broker)), - ) - // Record event count and filtering time - startTS := time.Now() - defer func() { - var deliveryTime time.Time - now := time.Now() - dispatchTimeMS := int64(now.Sub(startTS) / time.Millisecond) - stats.Record(ctx, MeasureTriggerDispatchTime.M(dispatchTimeMS)) - stats.Record(ctx, MeasureTriggerEventsTotal.M(1)) - if err := event.ExtensionAs(broker.TimeInFlightMetadataName, &deliveryTime); err != nil { - return - } - timeInFlightMS := int64(now.Sub(deliveryTime) / time.Millisecond) - stats.Record(ctx, MeasureDeliveryTime.M(timeInFlightMS)) - }() + reportArgs := &ReportArgs{ + ns: t.Namespace, + trigger: t.Name, + broker: t.Spec.Broker, + eventType: triggerFilterAttribute(t.Spec.Filter, "type"), + eventSource: triggerFilterAttribute(t.Spec.Filter, "source"), + } subscriberURIString := t.Status.SubscriberURI if subscriberURIString == "" { - ctx, _ = tag.New(ctx, tag.Upsert(TagResult, "error")) - return nil, errors.New("unable to read subscriberURI") + err = errors.New("unable to read subscriberURI") + // Record the event count. + r.reporter.ReportEventCount(reportArgs, err) + return nil, err } // We could just send the request to this URI regardless, but let's just check to see if it well // formed first, that way we can generate better error message if it isn't. subscriberURI, err := url.Parse(subscriberURIString) if err != nil { r.logger.Error("Unable to parse subscriberURI", zap.Error(err), zap.String("subscriberURIString", subscriberURIString)) - ctx, _ = tag.New(ctx, tag.Upsert(TagResult, "error")) + // Record the event count. + r.reporter.ReportEventCount(reportArgs, err) return nil, err } - if !r.shouldSendMessage(ctx, &t.Spec, event) { - r.logger.Debug("Message did not pass filter", zap.Any("triggerRef", trigger)) - ctx, _ = tag.New(ctx, tag.Upsert(TagResult, "drop")) + // Check if the event should be sent, and record filtering time. + start := time.Now() + pass, filterResult := r.shouldSendEvent(ctx, &t.Spec, event) + r.reporter.ReportFilterTime(reportArgs, filterResult, time.Since(start)) + + if !pass { + r.logger.Debug("Event did not pass filter", zap.Any("triggerRef", trigger)) + // Record the event count. + r.reporter.ReportEventCount(reportArgs, errors.New("event did not pass filter")) return nil, nil } + start = time.Now() sendingCTX := broker.SendingContext(ctx, tctx, subscriberURI) + // TODO get HTTP status codes and use those. replyEvent, err := r.ceClient.Send(sendingCTX, *event) - if err == nil { - ctx, _ = tag.New(ctx, tag.Upsert(TagResult, "accept")) - } else { - ctx, _ = tag.New(ctx, tag.Upsert(TagResult, "error")) + // Record the dispatch time. + r.reporter.ReportDispatchTime(reportArgs, err, time.Since(start)) + // Record the event count. + r.reporter.ReportEventCount(reportArgs, err) + // Record the event latency. This might be off if the receiver and the filter pods are running in + // different nodes with different clocks. + var arrivalTime time.Time + if extErr := event.ExtensionAs(broker.EventArrivalTime, &arrivalTime); extErr == nil { + r.reporter.ReportEventDeliveryTime(reportArgs, err, time.Since(arrivalTime)) } return replyEvent, err } @@ -263,28 +265,13 @@ func (r *Handler) getTrigger(ctx context.Context, ref path.NamespacedNameUID) (* return t, nil } -// shouldSendMessage determines whether message 'm' should be sent based on the triggerSpec 'ts'. -// Currently it supports exact matching on event context attributes. -// If no filter is present, shouldSendMessage returns false. -// If no filter strategy is present, shouldSendMessage returns true. -func (r *Handler) shouldSendMessage(ctx context.Context, ts *eventingv1alpha1.TriggerSpec, event *cloudevents.Event) bool { - if ts.Filter == nil { - r.logger.Error("No filter specified") - ctx, _ = tag.New(ctx, tag.Upsert(TagFilterResult, "empty-fail")) - return false - } - - // Record event count and filtering time. - startTS := time.Now() - defer func() { - filterTimeMS := int64(time.Now().Sub(startTS) / time.Millisecond) - stats.Record(ctx, MeasureTriggerFilterTime.M(filterTimeMS)) - }() - +// shouldSendEvent determines whether event 'event' should be sent based on the triggerSpec 'ts'. +// Currently it supports exact matching on event context attributes and extension attributes. +// If no filter is present, shouldSendEvent returns true. +func (r *Handler) shouldSendEvent(ctx context.Context, ts *eventingv1alpha1.TriggerSpec, event *cloudevents.Event) (bool, string) { // No filter specified, default to passing everything. - if ts.Filter.DeprecatedSourceAndType == nil && ts.Filter.Attributes == nil { - ctx, _ = tag.New(ctx, tag.Upsert(TagFilterResult, "empty-pass")) - return true + if ts.Filter == nil || (ts.Filter.DeprecatedSourceAndType == nil && ts.Filter.Attributes == nil) { + return true, "no_filter" } attrs := map[string]string{} @@ -297,16 +284,15 @@ func (r *Handler) shouldSendMessage(ctx context.Context, ts *eventingv1alpha1.Tr attrs = map[string]string(*ts.Filter.Attributes) } - result := r.filterEventByAttributes(ctx, attrs, event) + result := r.filterEventByAttributes(attrs, event) + resultStr := "fail" if result { - ctx, _ = tag.New(ctx, tag.Upsert(TagFilterResult, "pass")) - } else { - ctx, _ = tag.New(ctx, tag.Upsert(TagFilterResult, "fail")) + resultStr = "pass" } - return result + return result, resultStr } -func (r *Handler) filterEventByAttributes(ctx context.Context, attrs map[string]string, event *cloudevents.Event) bool { +func (r *Handler) filterEventByAttributes(attrs map[string]string, event *cloudevents.Event) bool { // Set standard context attributes. The attributes available may not be // exactly the same as the attributes defined in the current version of the // CloudEvents spec. @@ -334,14 +320,35 @@ func (r *Handler) filterEventByAttributes(ctx context.Context, attrs map[string] value, ok := ce[k] // If the attribute does not exist in the event, return false. if !ok { - logging.FromContext(ctx).Debug("Attribute not found", zap.String("attribute", k)) + r.logger.Debug("Attribute not found", zap.String("attribute", k)) return false } // If the attribute is not set to any and is different than the one from the event, return false. if v != eventingv1alpha1.TriggerAnyFilter && v != value { - logging.FromContext(ctx).Debug("Attribute had non-matching value", zap.String("attribute", k), zap.String("filter", v), zap.Any("received", value)) + r.logger.Debug("Attribute had non-matching value", zap.String("attribute", k), zap.String("filter", v), zap.Any("received", value)) return false } } return true } + +// triggerFilterAttribute returns the filter attribute value for a given `attributeName`. If it doesn't not exist, +// returns the any value filter. +func triggerFilterAttribute(filter *eventingv1alpha1.TriggerFilter, attributeName string) string { + attributeValue := eventingv1alpha1.TriggerAnyFilter + if filter != nil { + if filter.DeprecatedSourceAndType != nil { + if attributeName == "type" { + attributeValue = filter.DeprecatedSourceAndType.Type + } else if attributeName == "source" { + attributeValue = filter.DeprecatedSourceAndType.Source + } + } else if filter.Attributes != nil { + attrs := map[string]string(*filter.Attributes) + if v, ok := attrs[attributeName]; ok { + attributeValue = v + } + } + } + return attributeValue +} diff --git a/pkg/broker/filter/filter_handler_test.go b/pkg/broker/filter/filter_handler_test.go index c56308f0e24..08832714ac6 100644 --- a/pkg/broker/filter/filter_handler_test.go +++ b/pkg/broker/filter/filter_handler_test.go @@ -25,6 +25,7 @@ import ( "net/url" "strings" "testing" + "time" cloudevents "github.com/cloudevents/sdk-go" cehttp "github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http" @@ -147,6 +148,7 @@ func TestReceiver(t *testing.T) { triggers: []*eventingv1alpha1.Trigger{ makeTriggerWithoutFilter(), }, + expectedDispatch: true, }, "No TTL": { triggers: []*eventingv1alpha1.Trigger{ @@ -293,7 +295,8 @@ func TestReceiver(t *testing.T) { r, err := NewHandler( zap.NewNop(), - getClient(correctURI, tc.mocks)) + getClient(correctURI, tc.mocks), + &mockReporter{}) if tc.expectNewToFail { if err == nil { t.Fatal("Expected New to fail, it didn't") @@ -354,6 +357,24 @@ func TestReceiver(t *testing.T) { } } +type mockReporter struct{} + +func (r *mockReporter) ReportEventCount(args *ReportArgs, err error) error { + return nil +} + +func (r *mockReporter) ReportDispatchTime(args *ReportArgs, err error, d time.Duration) error { + return nil +} + +func (r *mockReporter) ReportFilterTime(args *ReportArgs, filterResult string, d time.Duration) error { + return nil +} + +func (r *mockReporter) ReportEventDeliveryTime(args *ReportArgs, err error, d time.Duration) error { + return nil +} + type fakeHandler struct { failRequest bool requestReceived bool diff --git a/pkg/broker/filter/metrics.go b/pkg/broker/filter/metrics.go deleted file mode 100644 index cf1a0bf189a..00000000000 --- a/pkg/broker/filter/metrics.go +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Copyright 2019 The Knative Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package filter - -import ( - "go.opencensus.io/stats" - "go.opencensus.io/stats/view" - "go.opencensus.io/tag" - utils "knative.dev/eventing/pkg/broker" -) - -var ( - // MeasureTriggerEventsTotal is a counter which records the number of events received - // by a Trigger. - MeasureTriggerEventsTotal = stats.Int64( - "knative.dev/eventing/trigger/measures/events_total", - "Total number of events received by a Trigger", - stats.UnitNone, - ) - - // MeasureTriggerDispatchTime records the time spent dispatching an event for - // a Trigger, in milliseconds. - MeasureTriggerDispatchTime = stats.Int64( - "knative.dev/eventing/trigger/measures/dispatch_time", - "Time spent dispatching an event to a Trigger", - stats.UnitMilliseconds, - ) - - // MeasureTriggerFilterTime records the time spent filtering a message for a - // Trigger, in milliseconds. - MeasureTriggerFilterTime = stats.Int64( - "knative.dev/eventing/trigger/measures/filter_time", - "Time spent filtering a message for a Trigger", - stats.UnitMilliseconds, - ) - - // MeasureDeliveryTime records the time spent between arrival at ingress - // and delivery to the trigger subscriber. - MeasureDeliveryTime = stats.Int64( - "knative.dev/eventing/trigger/measures/delivery_time", - "Time between an event arriving at ingress and delivery to the trigger subscriber", - stats.UnitMilliseconds, - ) - - // Tag keys must conform to the restrictions described in - // go.opencensus.io/tag/validate.go. Currently those restrictions are: - // - length between 1 and 255 inclusive - // - characters are printable US-ASCII - - // TagResult is a tag key referring to the observed result of an operation. - TagResult = utils.MustNewTagKey("result") - - // TagFilterResult is a tag key referring to the observed result of a filter - // operation. - TagFilterResult = utils.MustNewTagKey("filter_result") - - // TagBroker is a tag key referring to the Broker name serviced by this - // filter process. - TagBroker = utils.MustNewTagKey("broker") - - // TagTrigger is a tag key referring to the Trigger name serviced by this - // filter process. - TagTrigger = utils.MustNewTagKey("trigger") -) - -func init() { - // Create views for exporting measurements. This returns an error if a - // previously registered view has the same name with a different value. - err := view.Register( - &view.View{ - Name: "trigger_events_total", - Measure: MeasureTriggerEventsTotal, - Aggregation: view.Count(), - TagKeys: []tag.Key{TagResult, TagBroker, TagTrigger}, - }, - &view.View{ - Name: "trigger_dispatch_time", - Measure: MeasureTriggerDispatchTime, - Aggregation: view.Distribution(utils.Buckets125(1, 100)...), // 1, 2, 5, 10, 20, 50, 100, - TagKeys: []tag.Key{TagResult, TagBroker, TagTrigger}, - }, - &view.View{ - Name: "trigger_filter_time", - Measure: MeasureTriggerFilterTime, - Aggregation: view.Distribution(utils.Buckets125(0.1, 10)...), // 0.1, 0.2, 0.5, 1, 2, 5, 10 - TagKeys: []tag.Key{TagResult, TagFilterResult, TagBroker, TagTrigger}, - }, - &view.View{ - Name: "broker_to_function_delivery_time", - Measure: MeasureDeliveryTime, - Aggregation: view.Distribution(utils.Buckets125(1, 100)...), // 1, 2, 5, 10, 20, 50, 100 - TagKeys: []tag.Key{TagResult, TagBroker, TagTrigger}, - }, - ) - if err != nil { - panic(err) - } -} diff --git a/pkg/broker/filter/stats_reporter.go b/pkg/broker/filter/stats_reporter.go new file mode 100644 index 00000000000..ac50810874a --- /dev/null +++ b/pkg/broker/filter/stats_reporter.go @@ -0,0 +1,270 @@ +/* + * Copyright 2019 The Knative Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package filter + +import ( + "context" + "fmt" + "go.opencensus.io/stats" + "go.opencensus.io/stats/view" + "go.opencensus.io/tag" + utils "knative.dev/eventing/pkg/broker" + "knative.dev/eventing/pkg/metrics/metricskey" + "knative.dev/pkg/metrics" + "time" +) + +var ( + // eventCountM is a counter which records the number of events received + // by a Trigger. + eventCountM = stats.Int64( + "event_count", + "Number of events received by a Trigger", + stats.UnitDimensionless, + ) + + // dispatchTimeInMsecM records the time spent dispatching an event to + // a Trigger subscriber, in milliseconds. + dispatchTimeInMsecM = stats.Float64( + "dispatch_latencies", + "The time spent dispatching an event to a Trigger subscriber", + stats.UnitMilliseconds, + ) + + // filterTimeInMsecM records the time spent filtering an event for a + // Trigger, in milliseconds. + filterTimeInMsecM = stats.Float64( + "filter_latencies", + "The time spent filtering an event for a Trigger", + stats.UnitMilliseconds, + ) + + // deliveryTimeInMsecM records the time spent between arrival at the Broker + // and delivery to the Trigger subscriber. + deliveryTimeInMsecM = stats.Float64( + "event_latencies", + "The time spent routing an event from a Broker to a Trigger subscriber", + stats.UnitMilliseconds, + ) +) + +type ReportArgs struct { + ns string + trigger string + broker string + eventType string + eventSource string +} + +// StatsReporter defines the interface for sending filter metrics. +type StatsReporter interface { + ReportEventCount(args *ReportArgs, err error) error + ReportDispatchTime(args *ReportArgs, err error, d time.Duration) error + ReportFilterTime(args *ReportArgs, filterResult string, d time.Duration) error + ReportEventDeliveryTime(args *ReportArgs, err error, d time.Duration) error +} + +// Reporter holds cached metric objects to report filter metrics. +type Reporter struct { + initialized bool + namespaceTagKey tag.Key + triggerTagKey tag.Key + brokerTagKey tag.Key + triggerTypeKey tag.Key + triggerSourceKey tag.Key + resultKey tag.Key + filterResultKey tag.Key +} + +// NewStatsReporter creates a reporter that collects and reports filter metrics. +func NewStatsReporter() (*Reporter, error) { + var r = &Reporter{} + + // Create the tag keys that will be used to add tags to our measurements. + nsTag, err := tag.NewKey(metricskey.NamespaceName) + if err != nil { + return nil, err + } + r.namespaceTagKey = nsTag + triggerTag, err := tag.NewKey(metricskey.TriggerName) + if err != nil { + return nil, err + } + r.triggerTagKey = triggerTag + brokerTag, err := tag.NewKey(metricskey.BrokerName) + if err != nil { + return nil, err + } + r.brokerTagKey = brokerTag + triggerTypeTag, err := tag.NewKey(metricskey.TriggerType) + if err != nil { + return nil, err + } + r.triggerTypeKey = triggerTypeTag + triggerSourceKey, err := tag.NewKey(metricskey.TriggerSource) + if err != nil { + return nil, err + } + r.triggerSourceKey = triggerSourceKey + filterResultTag, err := tag.NewKey(metricskey.FilterResult) + if err != nil { + return nil, err + } + r.filterResultKey = filterResultTag + resultTag, err := tag.NewKey(metricskey.Result) + if err != nil { + return nil, err + } + r.resultKey = resultTag + + // Create view to see our measurements. + err = view.Register( + &view.View{ + Description: eventCountM.Description(), + Measure: eventCountM, + // TODO count or sum aggregation? + Aggregation: view.Count(), + TagKeys: []tag.Key{r.namespaceTagKey, r.triggerTagKey, r.brokerTagKey, r.triggerTypeKey, r.triggerSourceKey, r.resultKey}, + }, + &view.View{ + Description: dispatchTimeInMsecM.Description(), + Measure: dispatchTimeInMsecM, + Aggregation: view.Distribution(utils.Buckets125(1, 100)...), // 1, 2, 5, 10, 20, 50, 100 + TagKeys: []tag.Key{r.namespaceTagKey, r.triggerTagKey, r.brokerTagKey, r.triggerTypeKey, r.triggerSourceKey, r.resultKey}, + }, + &view.View{ + Description: filterTimeInMsecM.Description(), + Measure: filterTimeInMsecM, + Aggregation: view.Distribution(utils.Buckets125(0.1, 10)...), // 0.1, 0.2, 0.5, 1, 2, 5, 10 + TagKeys: []tag.Key{r.namespaceTagKey, r.triggerTagKey, r.brokerTagKey, r.triggerTypeKey, r.triggerSourceKey, r.filterResultKey}, + }, + &view.View{ + Description: deliveryTimeInMsecM.Description(), + Measure: deliveryTimeInMsecM, + Aggregation: view.Distribution(utils.Buckets125(1, 100)...), // 1, 2, 5, 10, 20, 50, 100 + TagKeys: []tag.Key{r.namespaceTagKey, r.triggerTagKey, r.brokerTagKey, r.triggerTypeKey, r.triggerSourceKey, r.resultKey}, + }, + ) + if err != nil { + return nil, err + } + + r.initialized = true + return r, nil +} + +// ReportEventCount captures the event count. +func (r *Reporter) ReportEventCount(args *ReportArgs, err error) error { + if !r.initialized { + return fmt.Errorf("StatsReporter is not initialized yet") + } + + // Note that eventType and eventSource can be empty strings, so they need a special treatment. + ctx, err := tag.New( + context.Background(), + tag.Insert(r.namespaceTagKey, args.ns), + tag.Insert(r.triggerTagKey, args.trigger), + tag.Insert(r.brokerTagKey, args.broker), + tag.Insert(r.triggerTypeKey, valueOrAny(args.eventType)), + tag.Insert(r.triggerSourceKey, valueOrAny(args.eventSource)), + tag.Insert(r.resultKey, utils.Result(err))) + if err != nil { + return err + } + + metrics.Record(ctx, eventCountM.M(1)) + return nil +} + +// ReportDispatchTime captures dispatch times. +func (r *Reporter) ReportDispatchTime(args *ReportArgs, err error, d time.Duration) error { + if !r.initialized { + return fmt.Errorf("StatsReporter is not initialized yet") + } + + // Note that eventType and eventSource can be empty strings, so they need a special treatment. + ctx, err := tag.New( + context.Background(), + tag.Insert(r.namespaceTagKey, args.ns), + tag.Insert(r.triggerTagKey, args.trigger), + tag.Insert(r.brokerTagKey, args.broker), + tag.Insert(r.triggerTypeKey, valueOrAny(args.eventType)), + tag.Insert(r.triggerSourceKey, valueOrAny(args.eventSource)), + tag.Insert(r.resultKey, utils.Result(err))) + if err != nil { + return err + } + + // convert time.Duration in nanoseconds to milliseconds. + metrics.Record(ctx, dispatchTimeInMsecM.M(float64(d/time.Millisecond))) + return nil +} + +// ReportFilterTime captures filtering times. +func (r *Reporter) ReportFilterTime(args *ReportArgs, filterResult string, d time.Duration) error { + if !r.initialized { + return fmt.Errorf("StatsReporter is not initialized yet") + } + + // Note that eventType and eventSource can be empty strings, so they need a special treatment. + ctx, err := tag.New( + context.Background(), + tag.Insert(r.namespaceTagKey, args.ns), + tag.Insert(r.triggerTagKey, args.trigger), + tag.Insert(r.brokerTagKey, args.broker), + tag.Insert(r.triggerTypeKey, valueOrAny(args.eventType)), + tag.Insert(r.triggerSourceKey, valueOrAny(args.eventSource)), + tag.Insert(r.filterResultKey, filterResult)) + if err != nil { + return err + } + + // convert time.Duration in nanoseconds to milliseconds. + metrics.Record(ctx, filterTimeInMsecM.M(float64(d/time.Millisecond))) + return nil +} + +// ReportEventDeliveryTime captures event delivery times. +func (r *Reporter) ReportEventDeliveryTime(args *ReportArgs, err error, d time.Duration) error { + if !r.initialized { + return fmt.Errorf("StatsReporter is not initialized yet") + } + + // Note that eventType and eventSource can be empty strings, so they need a special treatment. + ctx, err := tag.New( + context.Background(), + tag.Insert(r.namespaceTagKey, args.ns), + tag.Insert(r.triggerTagKey, args.trigger), + tag.Insert(r.brokerTagKey, args.broker), + tag.Insert(r.triggerTypeKey, valueOrAny(args.eventType)), + tag.Insert(r.triggerSourceKey, valueOrAny(args.eventSource)), + tag.Insert(r.resultKey, utils.Result(err))) + if err != nil { + return err + } + + // convert time.Duration in nanoseconds to milliseconds. + metrics.Record(ctx, deliveryTimeInMsecM.M(float64(d/time.Millisecond))) + return nil +} + +func valueOrAny(v string) string { + if v != "" { + return v + } + return metricskey.Any +} diff --git a/pkg/broker/filter/stats_reporter_test.go b/pkg/broker/filter/stats_reporter_test.go new file mode 100644 index 00000000000..4c0dac26d9b --- /dev/null +++ b/pkg/broker/filter/stats_reporter_test.go @@ -0,0 +1,154 @@ +/* +Copyright 2019 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package filter + +import ( + "github.com/pkg/errors" + "knative.dev/eventing/pkg/metrics/metricskey" + "knative.dev/pkg/metrics/metricstest" + "testing" + "time" +) + +// unregister, ehm, unregisters the metrics that were registered, by +// virtue of StatsReporter creation. +// Since golang executes test iterations within the same process, the stats reporter +// returns an error if the metric is already registered and the test panics. +func unregister() { + metricstest.Unregister("event_count", "dispatch_latencies", "filter_latencies", "event_latencies") +} + +func TestStatsReporter(t *testing.T) { + r := &Reporter{} + + args := &ReportArgs{ + ns: "testns", + trigger: "testtrigger", + broker: "testbroker", + eventType: "testeventtype", + eventSource: "testeventsource", + } + if err := r.ReportEventCount(args, errors.New("error")); err == nil { + t.Error("Reporter expected an error for Report call before init. Got success.") + } + + var err error + if r, err = NewStatsReporter(); err != nil { + t.Fatalf("Failed to create a new reporter: %v", err) + } + // Without this `go test ... -count=X`, where X > 1, fails, since + // we get an error about view already being registered. + defer unregister() + + wantTags := map[string]string{ + metricskey.NamespaceName: "testns", + metricskey.TriggerName: "testtrigger", + metricskey.BrokerName: "testbroker", + metricskey.TriggerType: "testeventtype", + metricskey.TriggerSource: "testeventsource", + } + + wantTags1 := map[string]string(wantTags) + wantTags1[metricskey.Result] = "success" + + // test ReportEventCount + expectSuccess(t, func() error { + return r.ReportEventCount(args, nil) + }) + expectSuccess(t, func() error { + return r.ReportEventCount(args, nil) + }) + metricstest.CheckCountData(t, "event_count", wantTags1, 2) + + // test ReportDispatchTime + expectSuccess(t, func() error { + return r.ReportDispatchTime(args, nil, 1100*time.Millisecond) + }) + expectSuccess(t, func() error { + return r.ReportDispatchTime(args, nil, 9100*time.Millisecond) + }) + metricstest.CheckDistributionData(t, "dispatch_latencies", wantTags1, 2, 1100.0, 9100.0) + + // test ReportEventDeliveryTime + expectSuccess(t, func() error { + return r.ReportEventDeliveryTime(args, nil, 1000*time.Millisecond) + }) + expectSuccess(t, func() error { + return r.ReportEventDeliveryTime(args, nil, 8000*time.Millisecond) + }) + metricstest.CheckDistributionData(t, "event_latencies", wantTags1, 2, 1000.0, 8000.0) + + wantTags2 := map[string]string(wantTags) + wantTags2[metricskey.FilterResult] = "pass" + + // test ReportFilterTime + expectSuccess(t, func() error { + return r.ReportFilterTime(args, "pass", 100*time.Millisecond) + }) + expectSuccess(t, func() error { + return r.ReportFilterTime(args, "pass", 500*time.Millisecond) + }) + metricstest.CheckDistributionData(t, "filter_latencies", wantTags1, 2, 100.0, 500.0) +} + +func TestReporterEmptySourceAndType(t *testing.T) { + r, err := NewStatsReporter() + defer unregister() + + if err != nil { + t.Fatalf("Failed to create a new reporter: %v", err) + } + + args := &ReportArgs{ + ns: "testns", + trigger: "testtrigger", + broker: "testbroker", + eventType: "", + eventSource: "", + } + + wantTags := map[string]string{ + metricskey.NamespaceName: "testns", + metricskey.TriggerName: "testtrigger", + metricskey.BrokerName: "testbroker", + metricskey.TriggerType: metricskey.Any, + metricskey.TriggerSource: metricskey.Any, + metricskey.Result: "success", + } + + // test ReportEventCount + expectSuccess(t, func() error { + return r.ReportEventCount(args, nil) + }) + expectSuccess(t, func() error { + return r.ReportEventCount(args, nil) + }) + expectSuccess(t, func() error { + return r.ReportEventCount(args, nil) + }) + expectSuccess(t, func() error { + return r.ReportEventCount(args, nil) + }) + metricstest.CheckCountData(t, "event_count", wantTags, 4) +} + +func expectSuccess(t *testing.T, f func() error) { + t.Helper() + if err := f(); err != nil { + t.Errorf("Reporter expected success but got error: %v", err) + } +} diff --git a/pkg/broker/ingress/ingress_handler.go b/pkg/broker/ingress/ingress_handler.go index 8dc0dcba003..4c0f0a6bc4d 100644 --- a/pkg/broker/ingress/ingress_handler.go +++ b/pkg/broker/ingress/ingress_handler.go @@ -8,9 +8,7 @@ import ( "reflect" "time" - cloudevents "github.com/cloudevents/sdk-go" - "go.opencensus.io/stats" - "go.opencensus.io/tag" + "github.com/cloudevents/sdk-go" "go.uber.org/zap" "knative.dev/eventing/pkg/broker" ) @@ -26,6 +24,8 @@ type Handler struct { CeClient cloudevents.Client ChannelURI *url.URL BrokerName string + Namespace string + Reporter StatsReporter } func (h *Handler) Start(stopCh <-chan struct{}) error { @@ -57,7 +57,7 @@ func (h *Handler) Start(stopCh <-chan struct{}) error { } func (h *Handler) serveHTTP(ctx context.Context, event cloudevents.Event, resp *cloudevents.EventResponse) error { - event.SetExtension(broker.TimeInFlightMetadataName, time.Now()) + event.SetExtension(broker.EventArrivalTime, time.Now()) tctx := cloudevents.HTTPTransportContextFrom(ctx) if tctx.Method != http.MethodPost { resp.Status = http.StatusMethodNotAllowed @@ -70,38 +70,27 @@ func (h *Handler) serveHTTP(ctx context.Context, event cloudevents.Event, resp * return nil } - ctx, _ = tag.New(ctx, tag.Insert(TagBroker, h.BrokerName)) - defer func() { - stats.Record(ctx, MeasureEventsTotal.M(1)) - }() + reporterArgs := &ReportArgs{ + ns: h.Namespace, + broker: h.BrokerName, + eventType: event.Type(), + } send := h.decrementTTL(&event) if !send { - ctx, _ = tag.New(ctx, tag.Insert(TagResult, "droppedDueToTTL")) + // Record the event count. + h.Reporter.ReportEventCount(reporterArgs, errors.New("dropped due to TTL")) return nil } - // TODO Filter. - - ctx, _ = tag.New(ctx, tag.Insert(TagResult, "dispatched")) - return h.sendEvent(ctx, tctx, event) -} - -func (h *Handler) sendEvent(ctx context.Context, tctx cloudevents.HTTPTransportContext, event cloudevents.Event) error { + start := time.Now() sendingCTX := broker.SendingContext(ctx, tctx, h.ChannelURI) - - startTS := time.Now() - defer func() { - dispatchTimeMS := int64(time.Now().Sub(startTS) / time.Millisecond) - stats.Record(sendingCTX, MeasureDispatchTime.M(dispatchTimeMS)) - }() - + // TODO get HTTP status codes and use those. _, err := h.CeClient.Send(sendingCTX, event) - if err != nil { - sendingCTX, _ = tag.New(sendingCTX, tag.Insert(TagResult, "error")) - } else { - sendingCTX, _ = tag.New(sendingCTX, tag.Insert(TagResult, "ok")) - } + // Record the dispatch time. + h.Reporter.ReportDispatchTime(reporterArgs, err, time.Since(start)) + // Record the event count. + h.Reporter.ReportEventCount(reporterArgs, err) return err } diff --git a/pkg/broker/ingress/ingress_handler_test.go b/pkg/broker/ingress/ingress_handler_test.go index 61e7f846005..85162e3a7da 100644 --- a/pkg/broker/ingress/ingress_handler_test.go +++ b/pkg/broker/ingress/ingress_handler_test.go @@ -5,6 +5,7 @@ import ( nethttp "net/http" "net/url" "testing" + "time" cloudevents "github.com/cloudevents/sdk-go" "github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http" @@ -12,6 +13,7 @@ import ( ) const ( + namespace = "testNamespace" brokerName = "testBroker" validURI = "/" urlHost = "testHost" @@ -20,6 +22,16 @@ const ( validHTTPMethod = nethttp.MethodPost ) +type mockReporter struct{} + +func (r *mockReporter) ReportEventCount(args *ReportArgs, err error) error { + return nil +} + +func (r *mockReporter) ReportDispatchTime(args *ReportArgs, err error, d time.Duration) error { + return nil +} + type fakeClient struct{ sent bool } func (f *fakeClient) Send(ctx context.Context, event cloudevents.Event) (*cloudevents.Event, error) { @@ -61,6 +73,8 @@ func TestIngressHandler_ServeHTTP_FAIL(t *testing.T) { Path: urlPath, }, BrokerName: brokerName, + Namespace: namespace, + Reporter: &mockReporter{}, } event := cloudevents.NewEvent() resp := new(cloudevents.EventResponse) @@ -85,6 +99,8 @@ func TestIngressHandler_ServeHTTP_Succeed(t *testing.T) { Path: urlPath, }, BrokerName: brokerName, + Namespace: namespace, + Reporter: &mockReporter{}, } event := cloudevents.NewEvent() resp := new(cloudevents.EventResponse) diff --git a/pkg/broker/ingress/metrics.go b/pkg/broker/ingress/metrics.go deleted file mode 100644 index 67e266fe78e..00000000000 --- a/pkg/broker/ingress/metrics.go +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright 2019 The Knative Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package ingress - -import ( - "go.opencensus.io/stats" - "go.opencensus.io/stats/view" - "go.opencensus.io/tag" - utils "knative.dev/eventing/pkg/broker" -) - -var ( - // MeasureEventsTotal is a counter which records the number of events received - // by the ingress. The value of the Result tag indicates whether the event - // was filtered or dispatched by the ingress. - MeasureEventsTotal = stats.Int64( - "knative.dev/eventing/broker/measures/events_total", - "Total number of events received", - stats.UnitNone, - ) - - // MeasureDispatchTime records the time spent dispatching an event, in - // milliseconds. - MeasureDispatchTime = stats.Int64( - "knative.dev/eventing/broker/measures/dispatch_time", - "Time spent dispatching an event", - stats.UnitMilliseconds, - ) - - // Tag keys must conform to the restrictions described in - // go.opencensus.io/tag/validate.go. Currently those restrictions are: - // - length between 1 and 255 inclusive - // - characters are printable US-ASCII - - // TagResult is a tag key referring to the observed result of an operation. - TagResult = utils.MustNewTagKey("result") - - // TagBroker is a tag key referring to the Broker name serviced by this - // ingress process. - TagBroker = utils.MustNewTagKey("broker") -) - -func init() { - // Create views for exporting measurements. This returns an error if a - // previously registered view has the same name with a different value. - err := view.Register( - &view.View{ - Name: "broker_events_total", - Measure: MeasureEventsTotal, - Aggregation: view.Count(), - TagKeys: []tag.Key{TagResult, TagBroker}, - }, - &view.View{ - Name: "broker_dispatch_time", - Measure: MeasureDispatchTime, - Aggregation: view.Distribution(utils.Buckets125(1, 100)...), // 1, 2, 5, 10, 20, 50, 100 - TagKeys: []tag.Key{TagResult, TagBroker}, - }, - ) - if err != nil { - panic(err) - } -} diff --git a/pkg/broker/ingress/stats_reporter.go b/pkg/broker/ingress/stats_reporter.go new file mode 100644 index 00000000000..a8d22bdb7d0 --- /dev/null +++ b/pkg/broker/ingress/stats_reporter.go @@ -0,0 +1,160 @@ +/* + * Copyright 2019 The Knative Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ingress + +import ( + "context" + "fmt" + "go.opencensus.io/stats" + "go.opencensus.io/stats/view" + "go.opencensus.io/tag" + utils "knative.dev/eventing/pkg/broker" + "knative.dev/eventing/pkg/metrics/metricskey" + "knative.dev/pkg/metrics" + "time" +) + +var ( + // eventCountM is a counter which records the number of events received + // by the Broker. + eventCountM = stats.Int64( + "event_count", + "Number of events received by a Broker", + stats.UnitDimensionless, + ) + + // dispatchTimeInMsecM records the time spent dispatching an event to + // a Trigger, in milliseconds. + dispatchTimeInMsecM = stats.Float64( + "dispatch_latencies", + "The time spent dispatching an event to a Trigger", + stats.UnitMilliseconds, + ) +) + +type ReportArgs struct { + ns string + broker string + eventType string +} + +// StatsReporter defines the interface for sending ingress metrics. +type StatsReporter interface { + ReportEventCount(args *ReportArgs, err error) error + ReportDispatchTime(args *ReportArgs, err error, d time.Duration) error +} + +// Reporter holds cached metric objects to report ingress metrics. +type Reporter struct { + initialized bool + namespaceTagKey tag.Key + brokerTagKey tag.Key + eventTypeKey tag.Key + // TODO add support for EventSource + resultKey tag.Key +} + +// NewStatsReporter creates a reporter that collects and reports ingress metrics. +func NewStatsReporter() (*Reporter, error) { + var r = &Reporter{} + + // Create the tag keys that will be used to add tags to our measurements. + nsTag, err := tag.NewKey(metricskey.NamespaceName) + if err != nil { + return nil, err + } + r.namespaceTagKey = nsTag + brokerTag, err := tag.NewKey(metricskey.BrokerName) + if err != nil { + return nil, err + } + r.brokerTagKey = brokerTag + eventTypeTag, err := tag.NewKey(metricskey.EventType) + if err != nil { + return nil, err + } + r.eventTypeKey = eventTypeTag + resultTag, err := tag.NewKey(metricskey.Result) + if err != nil { + return nil, err + } + r.resultKey = resultTag + + // Create view to see our measurements. + err = view.Register( + &view.View{ + Description: eventCountM.Description(), + Measure: eventCountM, + // TODO count or sum aggregation? + Aggregation: view.Count(), + TagKeys: []tag.Key{r.namespaceTagKey, r.brokerTagKey, r.eventTypeKey, r.resultKey}, + }, + &view.View{ + Description: dispatchTimeInMsecM.Description(), + Measure: dispatchTimeInMsecM, + Aggregation: view.Distribution(utils.Buckets125(1, 100)...), // 1, 2, 5, 10, 20, 50, 100 + TagKeys: []tag.Key{r.namespaceTagKey, r.brokerTagKey, r.eventTypeKey, r.resultKey}, + }, + ) + if err != nil { + return nil, err + } + + r.initialized = true + return r, nil +} + +// ReportEventCount captures the event count. +func (r *Reporter) ReportEventCount(args *ReportArgs, err error) error { + if !r.initialized { + return fmt.Errorf("StatsReporter is not initialized yet") + } + + ctx, err := tag.New( + context.Background(), + tag.Insert(r.namespaceTagKey, args.ns), + tag.Insert(r.brokerTagKey, args.broker), + tag.Insert(r.eventTypeKey, args.eventType), + tag.Insert(r.resultKey, utils.Result(err))) + if err != nil { + return err + } + + metrics.Record(ctx, eventCountM.M(1)) + return nil +} + +// ReportDispatchTime captures dispatch times. +func (r *Reporter) ReportDispatchTime(args *ReportArgs, err error, d time.Duration) error { + if !r.initialized { + return fmt.Errorf("StatsReporter is not initialized yet") + } + + ctx, err := tag.New( + context.Background(), + tag.Insert(r.namespaceTagKey, args.ns), + tag.Insert(r.brokerTagKey, args.broker), + tag.Insert(r.eventTypeKey, args.eventType), + tag.Insert(r.resultKey, utils.Result(err))) + if err != nil { + return err + } + + // convert time.Duration in nanoseconds to milliseconds. + metrics.Record(ctx, dispatchTimeInMsecM.M(float64(d/time.Millisecond))) + return nil +} diff --git a/pkg/broker/ingress/stats_reporter_test.go b/pkg/broker/ingress/stats_reporter_test.go new file mode 100644 index 00000000000..845028193b6 --- /dev/null +++ b/pkg/broker/ingress/stats_reporter_test.go @@ -0,0 +1,86 @@ +/* +Copyright 2019 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ingress + +import ( + "github.com/pkg/errors" + "knative.dev/eventing/pkg/metrics/metricskey" + "knative.dev/pkg/metrics/metricstest" + "testing" + "time" +) + +// unregister, ehm, unregisters the metrics that were registered, by +// virtue of StatsReporter creation. +// Since golang executes test iterations within the same process, the stats reporter +// returns an error if the metric is already registered and the test panics. +func unregister() { + metricstest.Unregister("event_count", "dispatch_latencies") +} + +func TestStatsReporter(t *testing.T) { + r := &Reporter{} + + args := &ReportArgs{ + ns: "testns", + broker: "testbroker", + eventType: "testeventtype", + } + if err := r.ReportEventCount(args, errors.New("error")); err == nil { + t.Error("Reporter expected an error for Report call before init. Got success.") + } + + var err error + if r, err = NewStatsReporter(); err != nil { + t.Fatalf("Failed to create a new reporter: %v", err) + } + // Without this `go test ... -count=X`, where X > 1, fails, since + // we get an error about view already being registered. + defer unregister() + + wantTags := map[string]string{ + metricskey.NamespaceName: "testns", + metricskey.BrokerName: "testbroker", + metricskey.EventType: "testeventtype", + metricskey.Result: "success", + } + + // test ReportEventCount + expectSuccess(t, func() error { + return r.ReportEventCount(args, nil) + }) + expectSuccess(t, func() error { + return r.ReportEventCount(args, nil) + }) + metricstest.CheckCountData(t, "event_count", wantTags, 2) + + // test ReportDispatchTime + expectSuccess(t, func() error { + return r.ReportDispatchTime(args, nil, 1100*time.Millisecond) + }) + expectSuccess(t, func() error { + return r.ReportDispatchTime(args, nil, 9100*time.Millisecond) + }) + metricstest.CheckDistributionData(t, "dispatch_latencies", wantTags, 2, 1100.0, 9100.0) +} + +func expectSuccess(t *testing.T, f func() error) { + t.Helper() + if err := f(); err != nil { + t.Errorf("Reporter expected success but got error: %v", err) + } +} diff --git a/pkg/broker/metrics.go b/pkg/broker/metrics.go index 9e3b7df4d8d..c3e9485a796 100644 --- a/pkg/broker/metrics.go +++ b/pkg/broker/metrics.go @@ -16,18 +16,12 @@ package broker -import "go.opencensus.io/tag" - -// MustNewTagKey creates a Tag or panics. This will only fail if the tag key -// doesn't conform to tag name validations. -// TODO OC library should provide this -func MustNewTagKey(k string) tag.Key { - tagKey, err := tag.NewKey(k) - if err != nil { - panic(err) - } - return tagKey -} +const ( + // EventArrivalTime is used to access the metadata stored on a + // CloudEvent to measure the time difference between when an event is + // received on a broker and when it is dispatched to the trigger function. + EventArrivalTime = "knativearrivaltime" +) // Buckets125 generates an array of buckets with approximate powers-of-two // buckets that also aligns with powers of 10 on every 3rd step. This can @@ -39,3 +33,11 @@ func Buckets125(low, high float64) []float64 { } return buckets } + +// Result converts an error to a result string (either "success" or "error"). +func Result(err error) string { + if err != nil { + return "error" + } + return "success" +} diff --git a/pkg/broker/ttl.go b/pkg/broker/ttl.go index ad6e459896f..33fbf4b89c3 100644 --- a/pkg/broker/ttl.go +++ b/pkg/broker/ttl.go @@ -27,12 +27,6 @@ const ( // Broker's TTL (number of times a single event can reply through a Broker continuously). All // interactions with the attribute should be done through the GetTTL and SetTTL functions. V03TTLAttribute = "knativebrokerttl" - - // TODO rename this extension, and place it somewhere else. - // TimeInFlightMetadataName is used to access the metadata stored on a - // CloudEvent to measure the time difference between when an event is - // received and when it is dispatched to the trigger function. - TimeInFlightMetadataName = "kn00timeinflight" ) // GetTTL finds the TTL in the EventContext using a case insensitive comparison diff --git a/pkg/metrics/metricskey/constants.go b/pkg/metrics/metricskey/constants.go new file mode 100644 index 00000000000..fbfe49b7173 --- /dev/null +++ b/pkg/metrics/metricskey/constants.go @@ -0,0 +1,85 @@ +/* +Copyright 2019 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metricskey + +import "k8s.io/apimachinery/pkg/util/sets" + +const ( + // KnativeTrigger is the Stackdriver resource type for Triggers. + KnativeTrigger = "knative_trigger" + + // Project is the label for the project (e.g., GCP project ID). + Project = "project_id" + + // Location is the label for the location (e.g. GCE zone) where the cluster is deployed. + Location = "location" + + // ClusterName is the label for the immutable name of the cluster. + ClusterName = "cluster_name" + + // NamespaceName is the label for the immutable name of the namespace where the resource type exists. + NamespaceName = "namespace_name" + + // TriggerName is the label for the name of the Trigger. + TriggerName = "trigger_name" + + // BrokerName is the label for the name of the Broker. + BrokerName = "broker_name" + + // TriggerType is the label for the type attribute filter of the Trigger. + TriggerType = "trigger_type" + + // TriggerSource is the label for the source attribute filter of the Trigger. + TriggerSource = "trigger_source" + + // EventType is the label for the CloudEvents type context attribute. + EventType = "event_type" + + // FilterResult is the label for the Trigger filtering result. + FilterResult = "filter_result" + + // Unknown is the default value if the field is unknown, e.g., the project will be unknown if Knative + // is not running on GKE. + Unknown = "unknown" + + // Result is the label for the result of sending an event to a downstream consumer. One of "success", "error". + Result = "result" + + // Any is the default value if the trigger filter attributes are empty. + Any = "any" +) + +var ( + // KnativeTriggerLabels stores the set of resource labels for the resource type knative_trigger. + KnativeTriggerLabels = sets.NewString( + Project, + Location, + ClusterName, + NamespaceName, + TriggerName, + TriggerType, + TriggerSource, + ) + + // KnativeTriggerMetrics stores the set of metric types that are supported by the resource type knative_trigger. + KnativeTriggerMetrics = sets.NewString( + "knative.dev/eventing/trigger/event_count", + "knative.dev/eventing/trigger/dispatch_latencies", + "knative.dev/eventing/trigger/filter_latencies", + // TODO event_latencies should be associated with Broker. + ) +) From 51925250c334a0f397954fb0163031d23c0c0932 Mon Sep 17 00:00:00 2001 From: nachocano Date: Mon, 26 Aug 2019 09:00:45 -0700 Subject: [PATCH 2/8] updating lock --- Gopkg.lock | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Gopkg.lock b/Gopkg.lock index 5a5b5a50eff..185e16ad730 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -1236,6 +1236,7 @@ "logging/testing", "metrics", "metrics/metricskey", + "metrics/metricstest", "reconciler/testing", "signals", "system", @@ -1413,6 +1414,7 @@ "knative.dev/pkg/logging/logkey", "knative.dev/pkg/logging/testing", "knative.dev/pkg/metrics", + "knative.dev/pkg/metrics/metricstest", "knative.dev/pkg/reconciler/testing", "knative.dev/pkg/signals", "knative.dev/pkg/system", From 213f19d88a767057729dd3a83c04ba48eaa9ced7 Mon Sep 17 00:00:00 2001 From: nachocano Date: Mon, 26 Aug 2019 16:45:50 -0700 Subject: [PATCH 3/8] updates after code review --- pkg/broker/filter/filter_handler.go | 32 +++++--- pkg/broker/filter/filter_handler_test.go | 2 +- pkg/broker/filter/stats_reporter.go | 93 +++++++---------------- pkg/broker/filter/stats_reporter_test.go | 10 +-- pkg/broker/ingress/stats_reporter.go | 50 +++++------- pkg/broker/ingress/stats_reporter_test.go | 10 +-- pkg/broker/metrics.go | 2 + pkg/metrics/metricskey/constants.go | 23 ------ 8 files changed, 73 insertions(+), 149 deletions(-) diff --git a/pkg/broker/filter/filter_handler.go b/pkg/broker/filter/filter_handler.go index a6c24abfd40..26f250f88a4 100644 --- a/pkg/broker/filter/filter_handler.go +++ b/pkg/broker/filter/filter_handler.go @@ -30,6 +30,7 @@ import ( "go.uber.org/zap" eventingv1alpha1 "knative.dev/eventing/pkg/apis/eventing/v1alpha1" "knative.dev/eventing/pkg/broker" + "knative.dev/eventing/pkg/logging" "knative.dev/eventing/pkg/reconciler/trigger/path" "knative.dev/pkg/tracing" "sigs.k8s.io/controller-runtime/pkg/client" @@ -37,6 +38,10 @@ import ( const ( writeTimeout = 1 * time.Minute + + passFilter FilterResult = "pass" + failFilter FilterResult = "fail" + noFilter FilterResult = "no_filter" ) // Handler parses Cloud Events, determines if they pass a filter, and sends them to a subscriber. @@ -47,6 +52,9 @@ type Handler struct { reporter StatsReporter } +// FilterResult has the result of the filtering operation. +type FilterResult string + // NewHandler creates a new Handler and its associated MessageReceiver. The caller is responsible for // Start()ing the returned Handler. func NewHandler(logger *zap.Logger, client client.Client, reporter StatsReporter) (*Handler, error) { @@ -226,10 +234,10 @@ func (r *Handler) sendEvent(ctx context.Context, tctx cloudevents.HTTPTransportC // Check if the event should be sent, and record filtering time. start := time.Now() - pass, filterResult := r.shouldSendEvent(ctx, &t.Spec, event) + filterResult := r.shouldSendEvent(ctx, &t.Spec, event) r.reporter.ReportFilterTime(reportArgs, filterResult, time.Since(start)) - if !pass { + if filterResult == failFilter { r.logger.Debug("Event did not pass filter", zap.Any("triggerRef", trigger)) // Record the event count. r.reporter.ReportEventCount(reportArgs, errors.New("event did not pass filter")) @@ -267,11 +275,11 @@ func (r *Handler) getTrigger(ctx context.Context, ref path.NamespacedNameUID) (* // shouldSendEvent determines whether event 'event' should be sent based on the triggerSpec 'ts'. // Currently it supports exact matching on event context attributes and extension attributes. -// If no filter is present, shouldSendEvent returns true. -func (r *Handler) shouldSendEvent(ctx context.Context, ts *eventingv1alpha1.TriggerSpec, event *cloudevents.Event) (bool, string) { +// If no filter is present, shouldSendEvent returns passFilter. +func (r *Handler) shouldSendEvent(ctx context.Context, ts *eventingv1alpha1.TriggerSpec, event *cloudevents.Event) FilterResult { // No filter specified, default to passing everything. if ts.Filter == nil || (ts.Filter.DeprecatedSourceAndType == nil && ts.Filter.Attributes == nil) { - return true, "no_filter" + return noFilter } attrs := map[string]string{} @@ -284,15 +292,15 @@ func (r *Handler) shouldSendEvent(ctx context.Context, ts *eventingv1alpha1.Trig attrs = map[string]string(*ts.Filter.Attributes) } - result := r.filterEventByAttributes(attrs, event) - resultStr := "fail" + result := r.filterEventByAttributes(ctx, attrs, event) + filterResult := failFilter if result { - resultStr = "pass" + filterResult = passFilter } - return result, resultStr + return filterResult } -func (r *Handler) filterEventByAttributes(attrs map[string]string, event *cloudevents.Event) bool { +func (r *Handler) filterEventByAttributes(ctx context.Context, attrs map[string]string, event *cloudevents.Event) bool { // Set standard context attributes. The attributes available may not be // exactly the same as the attributes defined in the current version of the // CloudEvents spec. @@ -320,12 +328,12 @@ func (r *Handler) filterEventByAttributes(attrs map[string]string, event *cloude value, ok := ce[k] // If the attribute does not exist in the event, return false. if !ok { - r.logger.Debug("Attribute not found", zap.String("attribute", k)) + logging.FromContext(ctx).Debug("Attribute not found", zap.String("attribute", k)) return false } // If the attribute is not set to any and is different than the one from the event, return false. if v != eventingv1alpha1.TriggerAnyFilter && v != value { - r.logger.Debug("Attribute had non-matching value", zap.String("attribute", k), zap.String("filter", v), zap.Any("received", value)) + logging.FromContext(ctx).Debug("Attribute had non-matching value", zap.String("attribute", k), zap.String("filter", v), zap.Any("received", value)) return false } } diff --git a/pkg/broker/filter/filter_handler_test.go b/pkg/broker/filter/filter_handler_test.go index 08832714ac6..1a9bef36d55 100644 --- a/pkg/broker/filter/filter_handler_test.go +++ b/pkg/broker/filter/filter_handler_test.go @@ -367,7 +367,7 @@ func (r *mockReporter) ReportDispatchTime(args *ReportArgs, err error, d time.Du return nil } -func (r *mockReporter) ReportFilterTime(args *ReportArgs, filterResult string, d time.Duration) error { +func (r *mockReporter) ReportFilterTime(args *ReportArgs, filterResult FilterResult, d time.Duration) error { return nil } diff --git a/pkg/broker/filter/stats_reporter.go b/pkg/broker/filter/stats_reporter.go index ac50810874a..7b682519671 100644 --- a/pkg/broker/filter/stats_reporter.go +++ b/pkg/broker/filter/stats_reporter.go @@ -18,14 +18,14 @@ package filter import ( "context" - "fmt" + "time" + "go.opencensus.io/stats" "go.opencensus.io/stats/view" "go.opencensus.io/tag" utils "knative.dev/eventing/pkg/broker" "knative.dev/eventing/pkg/metrics/metricskey" "knative.dev/pkg/metrics" - "time" ) var ( @@ -74,13 +74,14 @@ type ReportArgs struct { type StatsReporter interface { ReportEventCount(args *ReportArgs, err error) error ReportDispatchTime(args *ReportArgs, err error, d time.Duration) error - ReportFilterTime(args *ReportArgs, filterResult string, d time.Duration) error + ReportFilterTime(args *ReportArgs, filterResult FilterResult, d time.Duration) error ReportEventDeliveryTime(args *ReportArgs, err error, d time.Duration) error } -// Reporter holds cached metric objects to report filter metrics. -type Reporter struct { - initialized bool +var _ StatsReporter = (*reporter)(nil) + +// reporter holds cached metric objects to report filter metrics. +type reporter struct { namespaceTagKey tag.Key triggerTagKey tag.Key brokerTagKey tag.Key @@ -91,8 +92,8 @@ type Reporter struct { } // NewStatsReporter creates a reporter that collects and reports filter metrics. -func NewStatsReporter() (*Reporter, error) { - var r = &Reporter{} +func NewStatsReporter() (StatsReporter, error) { + var r = &reporter{} // Create the tag keys that will be used to add tags to our measurements. nsTag, err := tag.NewKey(metricskey.NamespaceName) @@ -163,103 +164,63 @@ func NewStatsReporter() (*Reporter, error) { return nil, err } - r.initialized = true return r, nil } // ReportEventCount captures the event count. -func (r *Reporter) ReportEventCount(args *ReportArgs, err error) error { - if !r.initialized { - return fmt.Errorf("StatsReporter is not initialized yet") - } - - // Note that eventType and eventSource can be empty strings, so they need a special treatment. - ctx, err := tag.New( - context.Background(), - tag.Insert(r.namespaceTagKey, args.ns), - tag.Insert(r.triggerTagKey, args.trigger), - tag.Insert(r.brokerTagKey, args.broker), - tag.Insert(r.triggerTypeKey, valueOrAny(args.eventType)), - tag.Insert(r.triggerSourceKey, valueOrAny(args.eventSource)), - tag.Insert(r.resultKey, utils.Result(err))) +func (r *reporter) ReportEventCount(args *ReportArgs, err error) error { + ctx, err := r.generateTag(args, tag.Insert(r.resultKey, utils.Result(err))) if err != nil { return err } - metrics.Record(ctx, eventCountM.M(1)) return nil } // ReportDispatchTime captures dispatch times. -func (r *Reporter) ReportDispatchTime(args *ReportArgs, err error, d time.Duration) error { - if !r.initialized { - return fmt.Errorf("StatsReporter is not initialized yet") - } - - // Note that eventType and eventSource can be empty strings, so they need a special treatment. - ctx, err := tag.New( - context.Background(), - tag.Insert(r.namespaceTagKey, args.ns), - tag.Insert(r.triggerTagKey, args.trigger), - tag.Insert(r.brokerTagKey, args.broker), - tag.Insert(r.triggerTypeKey, valueOrAny(args.eventType)), - tag.Insert(r.triggerSourceKey, valueOrAny(args.eventSource)), - tag.Insert(r.resultKey, utils.Result(err))) +func (r *reporter) ReportDispatchTime(args *ReportArgs, err error, d time.Duration) error { + ctx, err := r.generateTag(args, tag.Insert(r.resultKey, utils.Result(err))) if err != nil { return err } - // convert time.Duration in nanoseconds to milliseconds. metrics.Record(ctx, dispatchTimeInMsecM.M(float64(d/time.Millisecond))) return nil } // ReportFilterTime captures filtering times. -func (r *Reporter) ReportFilterTime(args *ReportArgs, filterResult string, d time.Duration) error { - if !r.initialized { - return fmt.Errorf("StatsReporter is not initialized yet") - } - - // Note that eventType and eventSource can be empty strings, so they need a special treatment. - ctx, err := tag.New( - context.Background(), - tag.Insert(r.namespaceTagKey, args.ns), - tag.Insert(r.triggerTagKey, args.trigger), - tag.Insert(r.brokerTagKey, args.broker), - tag.Insert(r.triggerTypeKey, valueOrAny(args.eventType)), - tag.Insert(r.triggerSourceKey, valueOrAny(args.eventSource)), - tag.Insert(r.filterResultKey, filterResult)) +func (r *reporter) ReportFilterTime(args *ReportArgs, filterResult FilterResult, d time.Duration) error { + ctx, err := r.generateTag(args, tag.Insert(r.filterResultKey, string(filterResult))) if err != nil { return err } - // convert time.Duration in nanoseconds to milliseconds. metrics.Record(ctx, filterTimeInMsecM.M(float64(d/time.Millisecond))) return nil } // ReportEventDeliveryTime captures event delivery times. -func (r *Reporter) ReportEventDeliveryTime(args *ReportArgs, err error, d time.Duration) error { - if !r.initialized { - return fmt.Errorf("StatsReporter is not initialized yet") +func (r *reporter) ReportEventDeliveryTime(args *ReportArgs, err error, d time.Duration) error { + ctx, err := r.generateTag(args, tag.Insert(r.resultKey, utils.Result(err))) + if err != nil { + return err } + // convert time.Duration in nanoseconds to milliseconds. + metrics.Record(ctx, deliveryTimeInMsecM.M(float64(d/time.Millisecond))) + return nil +} + +func (r *reporter) generateTag(args *ReportArgs, t tag.Mutator) (context.Context, error) { // Note that eventType and eventSource can be empty strings, so they need a special treatment. - ctx, err := tag.New( + return tag.New( context.Background(), tag.Insert(r.namespaceTagKey, args.ns), tag.Insert(r.triggerTagKey, args.trigger), tag.Insert(r.brokerTagKey, args.broker), tag.Insert(r.triggerTypeKey, valueOrAny(args.eventType)), tag.Insert(r.triggerSourceKey, valueOrAny(args.eventSource)), - tag.Insert(r.resultKey, utils.Result(err))) - if err != nil { - return err - } - - // convert time.Duration in nanoseconds to milliseconds. - metrics.Record(ctx, deliveryTimeInMsecM.M(float64(d/time.Millisecond))) - return nil + t) } func valueOrAny(v string) string { diff --git a/pkg/broker/filter/stats_reporter_test.go b/pkg/broker/filter/stats_reporter_test.go index 4c0dac26d9b..eddae32f385 100644 --- a/pkg/broker/filter/stats_reporter_test.go +++ b/pkg/broker/filter/stats_reporter_test.go @@ -17,7 +17,6 @@ limitations under the License. package filter import ( - "github.com/pkg/errors" "knative.dev/eventing/pkg/metrics/metricskey" "knative.dev/pkg/metrics/metricstest" "testing" @@ -33,8 +32,6 @@ func unregister() { } func TestStatsReporter(t *testing.T) { - r := &Reporter{} - args := &ReportArgs{ ns: "testns", trigger: "testtrigger", @@ -42,12 +39,9 @@ func TestStatsReporter(t *testing.T) { eventType: "testeventtype", eventSource: "testeventsource", } - if err := r.ReportEventCount(args, errors.New("error")); err == nil { - t.Error("Reporter expected an error for Report call before init. Got success.") - } - var err error - if r, err = NewStatsReporter(); err != nil { + r, err := NewStatsReporter() + if err != nil { t.Fatalf("Failed to create a new reporter: %v", err) } // Without this `go test ... -count=X`, where X > 1, fails, since diff --git a/pkg/broker/ingress/stats_reporter.go b/pkg/broker/ingress/stats_reporter.go index a8d22bdb7d0..e250ab1674d 100644 --- a/pkg/broker/ingress/stats_reporter.go +++ b/pkg/broker/ingress/stats_reporter.go @@ -18,7 +18,6 @@ package ingress import ( "context" - "fmt" "go.opencensus.io/stats" "go.opencensus.io/stats/view" "go.opencensus.io/tag" @@ -38,10 +37,10 @@ var ( ) // dispatchTimeInMsecM records the time spent dispatching an event to - // a Trigger, in milliseconds. + // a Channel, in milliseconds. dispatchTimeInMsecM = stats.Float64( "dispatch_latencies", - "The time spent dispatching an event to a Trigger", + "The time spent dispatching an event to a Channel", stats.UnitMilliseconds, ) ) @@ -58,9 +57,10 @@ type StatsReporter interface { ReportDispatchTime(args *ReportArgs, err error, d time.Duration) error } +var _ StatsReporter = (*reporter)(nil) + // Reporter holds cached metric objects to report ingress metrics. -type Reporter struct { - initialized bool +type reporter struct { namespaceTagKey tag.Key brokerTagKey tag.Key eventTypeKey tag.Key @@ -69,8 +69,8 @@ type Reporter struct { } // NewStatsReporter creates a reporter that collects and reports ingress metrics. -func NewStatsReporter() (*Reporter, error) { - var r = &Reporter{} +func NewStatsReporter() (StatsReporter, error) { + var r = &reporter{} // Create the tag keys that will be used to add tags to our measurements. nsTag, err := tag.NewKey(metricskey.NamespaceName) @@ -114,47 +114,35 @@ func NewStatsReporter() (*Reporter, error) { return nil, err } - r.initialized = true return r, nil } // ReportEventCount captures the event count. -func (r *Reporter) ReportEventCount(args *ReportArgs, err error) error { - if !r.initialized { - return fmt.Errorf("StatsReporter is not initialized yet") - } - - ctx, err := tag.New( - context.Background(), - tag.Insert(r.namespaceTagKey, args.ns), - tag.Insert(r.brokerTagKey, args.broker), - tag.Insert(r.eventTypeKey, args.eventType), - tag.Insert(r.resultKey, utils.Result(err))) +func (r *reporter) ReportEventCount(args *ReportArgs, err error) error { + ctx, err := r.generateTag(args, err) if err != nil { return err } - metrics.Record(ctx, eventCountM.M(1)) return nil } // ReportDispatchTime captures dispatch times. -func (r *Reporter) ReportDispatchTime(args *ReportArgs, err error, d time.Duration) error { - if !r.initialized { - return fmt.Errorf("StatsReporter is not initialized yet") +func (r *reporter) ReportDispatchTime(args *ReportArgs, err error, d time.Duration) error { + ctx, err := r.generateTag(args, err) + if err != nil { + return err } + // convert time.Duration in nanoseconds to milliseconds. + metrics.Record(ctx, dispatchTimeInMsecM.M(float64(d/time.Millisecond))) + return nil +} - ctx, err := tag.New( +func (r *reporter) generateTag(args *ReportArgs, err error) (context.Context, error) { + return tag.New( context.Background(), tag.Insert(r.namespaceTagKey, args.ns), tag.Insert(r.brokerTagKey, args.broker), tag.Insert(r.eventTypeKey, args.eventType), tag.Insert(r.resultKey, utils.Result(err))) - if err != nil { - return err - } - - // convert time.Duration in nanoseconds to milliseconds. - metrics.Record(ctx, dispatchTimeInMsecM.M(float64(d/time.Millisecond))) - return nil } diff --git a/pkg/broker/ingress/stats_reporter_test.go b/pkg/broker/ingress/stats_reporter_test.go index 845028193b6..7a9e734cfb5 100644 --- a/pkg/broker/ingress/stats_reporter_test.go +++ b/pkg/broker/ingress/stats_reporter_test.go @@ -17,7 +17,6 @@ limitations under the License. package ingress import ( - "github.com/pkg/errors" "knative.dev/eventing/pkg/metrics/metricskey" "knative.dev/pkg/metrics/metricstest" "testing" @@ -33,19 +32,14 @@ func unregister() { } func TestStatsReporter(t *testing.T) { - r := &Reporter{} - args := &ReportArgs{ ns: "testns", broker: "testbroker", eventType: "testeventtype", } - if err := r.ReportEventCount(args, errors.New("error")); err == nil { - t.Error("Reporter expected an error for Report call before init. Got success.") - } - var err error - if r, err = NewStatsReporter(); err != nil { + r, err := NewStatsReporter() + if err != nil { t.Fatalf("Failed to create a new reporter: %v", err) } // Without this `go test ... -count=X`, where X > 1, fails, since diff --git a/pkg/broker/metrics.go b/pkg/broker/metrics.go index c3e9485a796..6b08e664ca1 100644 --- a/pkg/broker/metrics.go +++ b/pkg/broker/metrics.go @@ -20,6 +20,8 @@ const ( // EventArrivalTime is used to access the metadata stored on a // CloudEvent to measure the time difference between when an event is // received on a broker and when it is dispatched to the trigger function. + // Should be set using time.Now(), which returns the current local time. + // The format is: 2019-08-26T23:38:17.834384404Z. EventArrivalTime = "knativearrivaltime" ) diff --git a/pkg/metrics/metricskey/constants.go b/pkg/metrics/metricskey/constants.go index fbfe49b7173..c3bd4e9be74 100644 --- a/pkg/metrics/metricskey/constants.go +++ b/pkg/metrics/metricskey/constants.go @@ -16,8 +16,6 @@ limitations under the License. package metricskey -import "k8s.io/apimachinery/pkg/util/sets" - const ( // KnativeTrigger is the Stackdriver resource type for Triggers. KnativeTrigger = "knative_trigger" @@ -62,24 +60,3 @@ const ( // Any is the default value if the trigger filter attributes are empty. Any = "any" ) - -var ( - // KnativeTriggerLabels stores the set of resource labels for the resource type knative_trigger. - KnativeTriggerLabels = sets.NewString( - Project, - Location, - ClusterName, - NamespaceName, - TriggerName, - TriggerType, - TriggerSource, - ) - - // KnativeTriggerMetrics stores the set of metric types that are supported by the resource type knative_trigger. - KnativeTriggerMetrics = sets.NewString( - "knative.dev/eventing/trigger/event_count", - "knative.dev/eventing/trigger/dispatch_latencies", - "knative.dev/eventing/trigger/filter_latencies", - // TODO event_latencies should be associated with Broker. - ) -) From 94fdc0c48dd7b924c203bb93e66ed60efa8ca50d Mon Sep 17 00:00:00 2001 From: nachocano Date: Mon, 26 Aug 2019 16:47:35 -0700 Subject: [PATCH 4/8] go imports --- pkg/broker/filter/stats_reporter_test.go | 5 +++-- pkg/broker/ingress/stats_reporter.go | 3 ++- pkg/broker/ingress/stats_reporter_test.go | 5 +++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/pkg/broker/filter/stats_reporter_test.go b/pkg/broker/filter/stats_reporter_test.go index eddae32f385..71ab7e989f0 100644 --- a/pkg/broker/filter/stats_reporter_test.go +++ b/pkg/broker/filter/stats_reporter_test.go @@ -17,10 +17,11 @@ limitations under the License. package filter import ( - "knative.dev/eventing/pkg/metrics/metricskey" - "knative.dev/pkg/metrics/metricstest" "testing" "time" + + "knative.dev/eventing/pkg/metrics/metricskey" + "knative.dev/pkg/metrics/metricstest" ) // unregister, ehm, unregisters the metrics that were registered, by diff --git a/pkg/broker/ingress/stats_reporter.go b/pkg/broker/ingress/stats_reporter.go index e250ab1674d..a1335087e1c 100644 --- a/pkg/broker/ingress/stats_reporter.go +++ b/pkg/broker/ingress/stats_reporter.go @@ -18,13 +18,14 @@ package ingress import ( "context" + "time" + "go.opencensus.io/stats" "go.opencensus.io/stats/view" "go.opencensus.io/tag" utils "knative.dev/eventing/pkg/broker" "knative.dev/eventing/pkg/metrics/metricskey" "knative.dev/pkg/metrics" - "time" ) var ( diff --git a/pkg/broker/ingress/stats_reporter_test.go b/pkg/broker/ingress/stats_reporter_test.go index 7a9e734cfb5..6d713b8056b 100644 --- a/pkg/broker/ingress/stats_reporter_test.go +++ b/pkg/broker/ingress/stats_reporter_test.go @@ -17,10 +17,11 @@ limitations under the License. package ingress import ( - "knative.dev/eventing/pkg/metrics/metricskey" - "knative.dev/pkg/metrics/metricstest" "testing" "time" + + "knative.dev/eventing/pkg/metrics/metricskey" + "knative.dev/pkg/metrics/metricstest" ) // unregister, ehm, unregisters the metrics that were registered, by From 0c45b55788cda60e90ab65984bc7ece9aed4c3d0 Mon Sep 17 00:00:00 2001 From: nachocano Date: Fri, 30 Aug 2019 18:43:21 -0700 Subject: [PATCH 5/8] not working --- Gopkg.lock | 9 +- Gopkg.toml | 2 +- pkg/broker/filter/filter_handler.go | 16 +- pkg/broker/filter/filter_handler_test.go | 12 +- pkg/broker/filter/stats_reporter.go | 64 ++-- pkg/broker/filter/stats_reporter_test.go | 27 +- pkg/broker/ingress/ingress_handler.go | 8 +- pkg/broker/ingress/stats_reporter.go | 4 +- pkg/broker/metrics.go | 19 - pkg/metrics/config.go | 95 ----- pkg/metrics/config_test.go | 131 ------- pkg/metrics/doc.go | 19 - pkg/metrics/metricskey/constants.go | 7 +- .../testdata/config-observability.yaml | 152 -------- pkg/metrics/zz_generated.deepcopy.go | 37 -- pkg/utils/utils.go | 8 + pkg/utils/utils_test.go | 25 ++ vendor/github.com/cloudevents/sdk-go/alias.go | 83 ++-- .../sdk-go/pkg/cloudevents/client/client.go | 31 +- .../pkg/cloudevents/client/defaulters.go | 10 +- .../sdk-go/pkg/cloudevents/client/options.go | 16 + .../sdk-go/pkg/cloudevents/client/receiver.go | 8 +- .../sdk-go/pkg/cloudevents/codec/doc.go | 5 - .../sdk-go/pkg/cloudevents/codec/jsoncodec.go | 316 ---------------- .../pkg/cloudevents/codec/observability.go | 90 ----- .../sdk-go/pkg/cloudevents/context/context.go | 46 +++ .../sdk-go/pkg/cloudevents/context/logger.go | 1 + .../sdk-go/pkg/cloudevents/datacodec/codec.go | 27 +- .../pkg/cloudevents/datacodec/json/data.go | 21 +- .../pkg/cloudevents/datacodec/xml/data.go | 21 +- .../sdk-go/pkg/cloudevents/event_data.go | 8 +- .../sdk-go/pkg/cloudevents/event_marshal.go | 281 ++++++++++++++ .../pkg/cloudevents/event_observability.go | 94 +++++ .../sdk-go/pkg/cloudevents/event_reader.go | 57 ++- .../pkg/cloudevents/eventcontext_v01.go | 7 +- .../cloudevents/eventcontext_v01_writer.go | 3 +- .../pkg/cloudevents/eventcontext_v02.go | 5 +- .../cloudevents/eventcontext_v02_writer.go | 3 +- .../pkg/cloudevents/eventcontext_v03.go | 9 +- .../cloudevents/eventcontext_v03_writer.go | 3 +- .../pkg/cloudevents/observability/observer.go | 33 +- .../sdk-go/pkg/cloudevents/transport/codec.go | 31 +- .../sdk-go/pkg/cloudevents/transport/doc.go | 11 +- .../sdk-go/pkg/cloudevents/transport/error.go | 30 ++ .../pkg/cloudevents/transport/http/codec.go | 90 ++--- .../transport/http/codec_structured.go | 44 +++ .../cloudevents/transport/http/codec_v01.go | 73 ++-- .../cloudevents/transport/http/codec_v02.go | 76 ++-- .../cloudevents/transport/http/codec_v03.go | 69 ++-- .../pkg/cloudevents/transport/http/context.go | 59 ++- .../cloudevents/transport/http/encoding.go | 60 ++- .../pkg/cloudevents/transport/http/message.go | 81 +++- .../pkg/cloudevents/transport/http/options.go | 80 +++- .../cloudevents/transport/http/transport.go | 357 +++++++++++++----- .../pkg/cloudevents/transport/transport.go | 25 +- .../sdk-go/pkg/cloudevents/types/timestamp.go | 4 +- .../sdk-go/pkg/cloudevents/types/urlref.go | 7 +- 57 files changed, 1540 insertions(+), 1370 deletions(-) delete mode 100644 pkg/metrics/config.go delete mode 100644 pkg/metrics/config_test.go delete mode 100644 pkg/metrics/doc.go delete mode 100644 pkg/metrics/testdata/config-observability.yaml delete mode 100644 pkg/metrics/zz_generated.deepcopy.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/codec/doc.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/codec/jsoncodec.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/codec/observability.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/event_marshal.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/event_observability.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/error.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/codec_structured.go diff --git a/Gopkg.lock b/Gopkg.lock index b7a41ecb942..af70971dd80 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -106,13 +106,13 @@ version = "v0.2.0" [[projects]] - digest = "1:0b2ffe5c17b0a3d49cc0a0cc20a13f7e1223fca1cd26587248a6b65c801dd46e" + branch = "master" + digest = "1:3e07bcc86130bbdd84341656a26746949f32a710e18ae43d5751f6fe16cb7fdb" name = "github.com/cloudevents/sdk-go" packages = [ ".", "pkg/cloudevents", "pkg/cloudevents/client", - "pkg/cloudevents/codec", "pkg/cloudevents/context", "pkg/cloudevents/datacodec", "pkg/cloudevents/datacodec/json", @@ -123,8 +123,7 @@ "pkg/cloudevents/types", ] pruneopts = "NUT" - revision = "56931988abe3adf6792b5c6e575ee5f573e9ccbc" - version = "0.7.0" + revision = "a1c7a759c034d9289981dd8ca42ff3663842a8a6" [[projects]] digest = "1:ffe9824d294da03b391f44e1ae8281281b4afc1bdaa9588c9097785e3af10cec" @@ -1199,7 +1198,6 @@ "codegen/cmd/injection-gen/args", "codegen/cmd/injection-gen/generators", "configmap", - "configmap/testing", "controller", "injection", "injection/clients/apiextclient", @@ -1358,7 +1356,6 @@ "knative.dev/pkg/client/clientset/versioned/fake", "knative.dev/pkg/codegen/cmd/injection-gen", "knative.dev/pkg/configmap", - "knative.dev/pkg/configmap/testing", "knative.dev/pkg/controller", "knative.dev/pkg/injection", "knative.dev/pkg/injection/clients/dynamicclient", diff --git a/Gopkg.toml b/Gopkg.toml index d68aeba9ee4..6c14836240e 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -89,7 +89,7 @@ required = [ [[constraint]] name = "github.com/cloudevents/sdk-go" - version = "=0.7.0" + branch = "master" # needed because pkg upgraded [[override]] diff --git a/pkg/broker/filter/filter_handler.go b/pkg/broker/filter/filter_handler.go index 31b434b71d1..de76b136af0 100644 --- a/pkg/broker/filter/filter_handler.go +++ b/pkg/broker/filter/filter_handler.go @@ -213,7 +213,7 @@ func (r *Handler) sendEvent(ctx context.Context, tctx cloudevents.HTTPTransportC if subscriberURIString == "" { err = errors.New("unable to read subscriberURI") // Record the event count. - r.reporter.ReportEventCount(reportArgs, err) + r.reporter.ReportEventCount(reportArgs, http.StatusNotFound) return nil, err } // We could just send the request to this URI regardless, but let's just check to see if it well @@ -222,7 +222,7 @@ func (r *Handler) sendEvent(ctx context.Context, tctx cloudevents.HTTPTransportC if err != nil { r.logger.Error("Unable to parse subscriberURI", zap.Error(err), zap.String("subscriberURIString", subscriberURIString)) // Record the event count. - r.reporter.ReportEventCount(reportArgs, err) + r.reporter.ReportEventCount(reportArgs, http.StatusNotFound) return nil, err } @@ -232,7 +232,7 @@ func (r *Handler) sendEvent(ctx context.Context, tctx cloudevents.HTTPTransportC if filterResult == failFilter { r.logger.Debug("Event did not pass filter", zap.Any("triggerRef", trigger)) // Record the event count. - r.reporter.ReportEventCount(reportArgs, errors.New("event did not pass filter")) + r.reporter.ReportEventCount(reportArgs, http.StatusForbidden) return nil, nil } @@ -242,18 +242,18 @@ func (r *Handler) sendEvent(ctx context.Context, tctx cloudevents.HTTPTransportC if extErr := event.ExtensionAs(broker.EventArrivalTime, &arrivalTimeStr); extErr == nil { arrivalTime, err := time.Parse(time.RFC3339, arrivalTimeStr) if err != nil { - r.reporter.ReportEventProcessingTime(reportArgs, err, time.Since(arrivalTime)) + r.reporter.ReportEventProcessingTime(reportArgs, time.Since(arrivalTime)) } } start := time.Now() sendingCTX := broker.SendingContext(ctx, tctx, subscriberURI) - // TODO use HTTP codes: https://github.com/cloudevents/sdk-go/pull/177 - replyEvent, err := r.ceClient.Send(sendingCTX, *event) + rctx, replyEvent, err := r.ceClient.Send(sendingCTX, *event) + rtctx := cloudevents.HTTPTransportContextFrom(rctx) // Record the dispatch time. - r.reporter.ReportEventDispatchTime(reportArgs, err, time.Since(start)) + r.reporter.ReportEventDispatchTime(reportArgs, rtctx.StatusCode, time.Since(start)) // Record the event count. - r.reporter.ReportEventCount(reportArgs, err) + r.reporter.ReportEventCount(reportArgs, rtctx.StatusCode) return replyEvent, err } diff --git a/pkg/broker/filter/filter_handler_test.go b/pkg/broker/filter/filter_handler_test.go index 61129340c9c..34cd1d4ad48 100644 --- a/pkg/broker/filter/filter_handler_test.go +++ b/pkg/broker/filter/filter_handler_test.go @@ -28,12 +28,12 @@ import ( "k8s.io/apimachinery/pkg/labels" - cloudevents "github.com/cloudevents/sdk-go" + "github.com/cloudevents/sdk-go" cehttp "github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http" "github.com/google/go-cmp/cmp" "go.uber.org/zap" + "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes/scheme" @@ -350,15 +350,15 @@ func TestReceiver(t *testing.T) { type mockReporter struct{} -func (r *mockReporter) ReportEventCount(args *ReportArgs, err error) error { +func (r *mockReporter) ReportEventCount(args *ReportArgs, responseCode int) error { return nil } -func (r *mockReporter) ReportEventDispatchTime(args *ReportArgs, err error, d time.Duration) error { +func (r *mockReporter) ReportEventDispatchTime(args *ReportArgs, responseCode int, d time.Duration) error { return nil } -func (r *mockReporter) ReportEventProcessingTime(args *ReportArgs, err error, d time.Duration) error { +func (r *mockReporter) ReportEventProcessingTime(args *ReportArgs, d time.Duration) error { return nil } @@ -392,7 +392,7 @@ func (h *fakeHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) { } c := &cehttp.CodecV03{} - m, err := c.Encode(*h.returnedEvent) + m, err := c.Encode(context.Background(), *h.returnedEvent) if err != nil { h.t.Fatalf("Could not encode message: %v", err) } diff --git a/pkg/broker/filter/stats_reporter.go b/pkg/broker/filter/stats_reporter.go index 7c2163667cd..992ad0b8f6a 100644 --- a/pkg/broker/filter/stats_reporter.go +++ b/pkg/broker/filter/stats_reporter.go @@ -18,13 +18,14 @@ package filter import ( "context" + "strconv" "time" "go.opencensus.io/stats" "go.opencensus.io/stats/view" "go.opencensus.io/tag" - utils "knative.dev/eventing/pkg/broker" . "knative.dev/eventing/pkg/metrics/metricskey" + "knative.dev/eventing/pkg/utils" "knative.dev/pkg/metrics" "knative.dev/pkg/metrics/metricskey" ) @@ -65,9 +66,9 @@ type ReportArgs struct { // StatsReporter defines the interface for sending filter metrics. type StatsReporter interface { - ReportEventCount(args *ReportArgs, err error) error - ReportEventDispatchTime(args *ReportArgs, err error, d time.Duration) error - ReportEventProcessingTime(args *ReportArgs, err error, d time.Duration) error + ReportEventCount(args *ReportArgs, responseCode int) error + ReportEventDispatchTime(args *ReportArgs, responseCode int, d time.Duration) error + ReportEventProcessingTime(args *ReportArgs, d time.Duration) error } var _ StatsReporter = (*reporter)(nil) @@ -79,7 +80,8 @@ type reporter struct { brokerTagKey tag.Key triggerFilterTypeKey tag.Key triggerFilterSourceKey tag.Key - resultKey tag.Key + responseCodeKey tag.Key + responseCodeClassKey tag.Key filterResultKey tag.Key } @@ -118,11 +120,16 @@ func NewStatsReporter() (StatsReporter, error) { return nil, err } r.filterResultKey = filterResultTag - resultTag, err := tag.NewKey(LabelResult) + responseCodeTag, err := tag.NewKey(LabelResponseCode) if err != nil { return nil, err } - r.resultKey = resultTag + r.responseCodeKey = responseCodeTag + responseCodeClassTag, err := tag.NewKey(LabelResponseCodeClass) + if err != nil { + return nil, err + } + r.responseCodeClassKey = responseCodeClassTag // Create view to see our measurements. err = view.Register( @@ -130,19 +137,19 @@ func NewStatsReporter() (StatsReporter, error) { Description: eventCountM.Description(), Measure: eventCountM, Aggregation: view.Count(), - TagKeys: []tag.Key{r.namespaceTagKey, r.triggerTagKey, r.brokerTagKey, r.triggerFilterTypeKey, r.triggerFilterSourceKey, r.resultKey}, + TagKeys: []tag.Key{r.namespaceTagKey, r.triggerTagKey, r.brokerTagKey, r.triggerFilterTypeKey, r.triggerFilterSourceKey, r.responseCodeKey, r.responseCodeClassKey}, }, &view.View{ Description: dispatchTimeInMsecM.Description(), Measure: dispatchTimeInMsecM, - Aggregation: view.Distribution(utils.Buckets125(1, 100)...), // 1, 2, 5, 10, 20, 50, 100 - TagKeys: []tag.Key{r.namespaceTagKey, r.triggerTagKey, r.brokerTagKey, r.triggerFilterTypeKey, r.triggerFilterSourceKey, r.resultKey}, + Aggregation: view.Distribution(metrics.Buckets125(1, 100)...), // 1, 2, 5, 10, 20, 50, 100 + TagKeys: []tag.Key{r.namespaceTagKey, r.triggerTagKey, r.brokerTagKey, r.triggerFilterTypeKey, r.triggerFilterSourceKey, r.responseCodeKey, r.responseCodeClassKey}, }, &view.View{ Description: processingTimeInMsecM.Description(), Measure: processingTimeInMsecM, - Aggregation: view.Distribution(utils.Buckets125(1, 100)...), // 1, 2, 5, 10, 20, 50, 100 - TagKeys: []tag.Key{r.namespaceTagKey, r.triggerTagKey, r.brokerTagKey, r.triggerFilterTypeKey, r.triggerFilterSourceKey, r.resultKey}, + Aggregation: view.Distribution(metrics.Buckets125(1, 100)...), // 1, 2, 5, 10, 20, 50, 100 + TagKeys: []tag.Key{r.namespaceTagKey, r.triggerTagKey, r.brokerTagKey, r.triggerFilterTypeKey, r.triggerFilterSourceKey}, }, ) if err != nil { @@ -153,8 +160,10 @@ func NewStatsReporter() (StatsReporter, error) { } // ReportEventCount captures the event count. -func (r *reporter) ReportEventCount(args *ReportArgs, err error) error { - ctx, err := r.generateTag(args, tag.Insert(r.resultKey, utils.Result(err))) +func (r *reporter) ReportEventCount(args *ReportArgs, responseCode int) error { + ctx, err := r.generateTag(args, + tag.Insert(r.responseCodeKey, strconv.Itoa(responseCode)), + tag.Insert(r.responseCodeClassKey, utils.ResponseCodeClass(responseCode))) if err != nil { return err } @@ -163,8 +172,10 @@ func (r *reporter) ReportEventCount(args *ReportArgs, err error) error { } // ReportEventDispatchTime captures dispatch times. -func (r *reporter) ReportEventDispatchTime(args *ReportArgs, err error, d time.Duration) error { - ctx, err := r.generateTag(args, tag.Insert(r.resultKey, utils.Result(err))) +func (r *reporter) ReportEventDispatchTime(args *ReportArgs, responseCode int, d time.Duration) error { + ctx, err := r.generateTag(args, + tag.Insert(r.responseCodeKey, strconv.Itoa(responseCode)), + tag.Insert(r.responseCodeClassKey, utils.ResponseCodeClass(responseCode))) if err != nil { return err } @@ -174,8 +185,8 @@ func (r *reporter) ReportEventDispatchTime(args *ReportArgs, err error, d time.D } // ReportEventProcessingTime captures event processing times. -func (r *reporter) ReportEventProcessingTime(args *ReportArgs, err error, d time.Duration) error { - ctx, err := r.generateTag(args, tag.Insert(r.resultKey, utils.Result(err))) +func (r *reporter) ReportEventProcessingTime(args *ReportArgs, d time.Duration) error { + ctx, err := r.generateTag(args) if err != nil { return err } @@ -185,16 +196,25 @@ func (r *reporter) ReportEventProcessingTime(args *ReportArgs, err error, d time return nil } -func (r *reporter) generateTag(args *ReportArgs, t tag.Mutator) (context.Context, error) { +func (r *reporter) generateTag(args *ReportArgs, tags ...tag.Mutator) (context.Context, error) { // Note that filterType and filterSource can be empty strings, so they need a special treatment. - return tag.New( + ctx, err := tag.New( context.Background(), tag.Insert(r.namespaceTagKey, args.ns), tag.Insert(r.triggerTagKey, args.trigger), tag.Insert(r.brokerTagKey, args.broker), tag.Insert(r.triggerFilterTypeKey, valueOrAny(args.filterType)), - tag.Insert(r.triggerFilterSourceKey, valueOrAny(args.filterSource)), - t) + tag.Insert(r.triggerFilterSourceKey, valueOrAny(args.filterSource))) + if err != nil { + return nil, err + } + for _, t := range tags { + ctx, err = tag.New(ctx, t) + if err != nil { + return nil, err + } + } + return ctx, err } func valueOrAny(v string) string { diff --git a/pkg/broker/filter/stats_reporter_test.go b/pkg/broker/filter/stats_reporter_test.go index 6afb106e39d..f82545a89ae 100644 --- a/pkg/broker/filter/stats_reporter_test.go +++ b/pkg/broker/filter/stats_reporter_test.go @@ -17,6 +17,7 @@ limitations under the License. package filter import ( + "net/http" "testing" "time" @@ -56,33 +57,34 @@ func TestStatsReporter(t *testing.T) { metricskey.LabelBrokerName: "testbroker", metricskey.LabelFilterType: "testeventtype", metricskey.LabelFilterSource: "testeventsource", - LabelResult: "success", + LabelResponseCode: "202", + LabelResponseCodeClass: "2xx", } // test ReportEventCount expectSuccess(t, func() error { - return r.ReportEventCount(args, nil) + return r.ReportEventCount(args, http.StatusAccepted) }) expectSuccess(t, func() error { - return r.ReportEventCount(args, nil) + return r.ReportEventCount(args, http.StatusAccepted) }) metricstest.CheckCountData(t, "event_count", wantTags, 2) // test ReportEventDispatchTime expectSuccess(t, func() error { - return r.ReportEventDispatchTime(args, nil, 1100*time.Millisecond) + return r.ReportEventDispatchTime(args, http.StatusAccepted, 1100*time.Millisecond) }) expectSuccess(t, func() error { - return r.ReportEventDispatchTime(args, nil, 9100*time.Millisecond) + return r.ReportEventDispatchTime(args, http.StatusAccepted, 9100*time.Millisecond) }) metricstest.CheckDistributionData(t, "event_dispatch_latencies", wantTags, 2, 1100.0, 9100.0) // test ReportEventProcessingTime expectSuccess(t, func() error { - return r.ReportEventProcessingTime(args, nil, 1000*time.Millisecond) + return r.ReportEventProcessingTime(args, 1000*time.Millisecond) }) expectSuccess(t, func() error { - return r.ReportEventProcessingTime(args, nil, 8000*time.Millisecond) + return r.ReportEventProcessingTime(args, 8000*time.Millisecond) }) metricstest.CheckDistributionData(t, "event_processing_latencies", wantTags, 2, 1000.0, 8000.0) } @@ -109,21 +111,22 @@ func TestReporterEmptySourceAndTypeFilter(t *testing.T) { metricskey.LabelBrokerName: "testbroker", metricskey.LabelFilterType: AnyValue, metricskey.LabelFilterSource: AnyValue, - LabelResult: "success", + LabelResponseCode: "202", + LabelResponseCodeClass: "2xx", } // test ReportEventCount expectSuccess(t, func() error { - return r.ReportEventCount(args, nil) + return r.ReportEventCount(args, http.StatusAccepted) }) expectSuccess(t, func() error { - return r.ReportEventCount(args, nil) + return r.ReportEventCount(args, http.StatusAccepted) }) expectSuccess(t, func() error { - return r.ReportEventCount(args, nil) + return r.ReportEventCount(args, http.StatusAccepted) }) expectSuccess(t, func() error { - return r.ReportEventCount(args, nil) + return r.ReportEventCount(args, http.StatusAccepted) }) metricstest.CheckCountData(t, "event_count", wantTags, 4) } diff --git a/pkg/broker/ingress/ingress_handler.go b/pkg/broker/ingress/ingress_handler.go index acd7639b549..bd571ae7eb5 100644 --- a/pkg/broker/ingress/ingress_handler.go +++ b/pkg/broker/ingress/ingress_handler.go @@ -94,12 +94,12 @@ func (h *Handler) serveHTTP(ctx context.Context, event cloudevents.Event, resp * start := time.Now() sendingCTX := broker.SendingContext(ctx, tctx, h.ChannelURI) - // TODO use HTTP codes: https://github.com/cloudevents/sdk-go/pull/177 - _, err := h.CeClient.Send(sendingCTX, event) + rctx, _, err := h.CeClient.Send(sendingCTX, event) + rtctx := cloudevents.HTTPTransportContextFrom(rctx) // Record the dispatch time. - h.Reporter.ReportEventDispatchTime(reporterArgs, err, time.Since(start)) + h.Reporter.ReportEventDispatchTime(reporterArgs, rtctx.StatusCode, time.Since(start)) // Record the event count. - h.Reporter.ReportEventCount(reporterArgs, err) + h.Reporter.ReportEventCount(reporterArgs, rtctx.StatusCode) return err } diff --git a/pkg/broker/ingress/stats_reporter.go b/pkg/broker/ingress/stats_reporter.go index f1a86017654..e135223c688 100644 --- a/pkg/broker/ingress/stats_reporter.go +++ b/pkg/broker/ingress/stats_reporter.go @@ -55,8 +55,8 @@ type ReportArgs struct { // StatsReporter defines the interface for sending ingress metrics. type StatsReporter interface { - ReportEventCount(args *ReportArgs, err error) error - ReportEventDispatchTime(args *ReportArgs, err error, d time.Duration) error + ReportEventCount(args *ReportArgs, responseCode int) error + ReportEventDispatchTime(args *ReportArgs, responseCode int, d time.Duration) error } var _ StatsReporter = (*reporter)(nil) diff --git a/pkg/broker/metrics.go b/pkg/broker/metrics.go index 74b6b423f7e..349247b6eb5 100644 --- a/pkg/broker/metrics.go +++ b/pkg/broker/metrics.go @@ -29,22 +29,3 @@ const ( // which stands for version("00" is the current version)-traceID-spanID-trace options TraceParent = "traceparent" ) - -// Buckets125 generates an array of buckets with approximate powers-of-two -// buckets that also aligns with powers of 10 on every 3rd step. This can -// be used to create a view.Distribution. -func Buckets125(low, high float64) []float64 { - buckets := []float64{low} - for last := low; last < high; last = last * 10 { - buckets = append(buckets, 2*last, 5*last, 10*last) - } - return buckets -} - -// Result converts an error to a result string (either "success" or "error"). -func Result(err error) string { - if err != nil { - return "error" - } - return "success" -} diff --git a/pkg/metrics/config.go b/pkg/metrics/config.go deleted file mode 100644 index 51a41cbbcfa..00000000000 --- a/pkg/metrics/config.go +++ /dev/null @@ -1,95 +0,0 @@ -/* -Copyright 2019 The Knative Authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package metrics - -import ( - "fmt" - "strings" - "text/template" - - corev1 "k8s.io/api/core/v1" -) - -const ( - ObservabilityConfigName = "config-observability" - defaultLogURLTemplate = "http://localhost:8001/api/v1/namespaces/knative-monitoring/services/kibana-logging/proxy/app/kibana#/discover?_a=(query:(match:(kubernetes.labels.knative-dev%2FrevisionUID:(query:'${REVISION_UID}',type:phrase))))" -) - -// ObservabilityConfig contains the configuration defined in the observability ConfigMap. -type ObservabilityConfig struct { - // EnableVarLogCollection dedicates whether to set up a fluentd sidecar to - // collect logs under /var/log/. - EnableVarLogCollection bool - - // TODO(https://github.com/knative/serving/issues/818): Use the fluentd - // daemon set to collect /var/log. FluentdSidecarImage is the name of the - // image used for the fluentd sidecar injected into the revision pod. It - // is used only when enableVarLogCollection is true. - FluentdSidecarImage string - - // FluentdSidecarOutputConfig is the config for fluentd sidecar to specify - // logging output destination. - FluentdSidecarOutputConfig string - - // LoggingURLTemplate is a string containing the logging url template where - // the variable REVISION_UID will be replaced with the created revision's UID. - LoggingURLTemplate string - - // RequestLogTemplate is the go template to use to shape the request logs. - RequestLogTemplate string - - // RequestMetricsBackend specifies the request metrics destination, e.g. Prometheus, - // Stackdriver. - RequestMetricsBackend string -} - -// NewObservabilityConfigFromConfigMap creates a Observability from the supplied ConfigMap -func NewObservabilityConfigFromConfigMap(configMap *corev1.ConfigMap) (*ObservabilityConfig, error) { - oc := &ObservabilityConfig{} - if evlc, ok := configMap.Data["logging.enable-var-log-collection"]; ok { - oc.EnableVarLogCollection = strings.ToLower(evlc) == "true" - } - if fsi, ok := configMap.Data["logging.fluentd-sidecar-image"]; ok { - oc.FluentdSidecarImage = fsi - } else if oc.EnableVarLogCollection { - return nil, fmt.Errorf("received bad Observability ConfigMap, want %q when %q is true", - "logging.fluentd-sidecar-image", "logging.enable-var-log-collection") - } - - if fsoc, ok := configMap.Data["logging.fluentd-sidecar-output-config"]; ok { - oc.FluentdSidecarOutputConfig = fsoc - } - if rut, ok := configMap.Data["logging.revision-url-template"]; ok { - oc.LoggingURLTemplate = rut - } else { - oc.LoggingURLTemplate = defaultLogURLTemplate - } - - if rlt, ok := configMap.Data["logging.request-log-template"]; ok { - // Verify that we get valid templates. - if _, err := template.New("requestLog").Parse(rlt); err != nil { - return nil, err - } - oc.RequestLogTemplate = rlt - } - - if mb, ok := configMap.Data["metrics.request-metrics-backend-destination"]; ok { - oc.RequestMetricsBackend = mb - } - - return oc, nil -} diff --git a/pkg/metrics/config_test.go b/pkg/metrics/config_test.go deleted file mode 100644 index ffcb865b507..00000000000 --- a/pkg/metrics/config_test.go +++ /dev/null @@ -1,131 +0,0 @@ -/* -Copyright 2019 The Knative Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package metrics - -import ( - "testing" - - "github.com/google/go-cmp/cmp" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "knative.dev/pkg/system" - - . "knative.dev/pkg/configmap/testing" - _ "knative.dev/pkg/system/testing" -) - -func TestOurObservability(t *testing.T) { - cm, example := ConfigMapsFromTestFile(t, ObservabilityConfigName) - - if _, err := NewObservabilityConfigFromConfigMap(cm); err != nil { - t.Errorf("NewObservabilityFromConfigMap(actual) = %v", err) - } - - if _, err := NewObservabilityConfigFromConfigMap(example); err != nil { - t.Errorf("NewObservabilityFromConfigMap(example) = %v", err) - } -} - -func TestObservabilityConfiguration(t *testing.T) { - observabilityConfigTests := []struct { - name string - wantErr bool - wantController interface{} - config *corev1.ConfigMap - }{{ - name: "observability configuration with all inputs", - wantErr: false, - wantController: &ObservabilityConfig{ - LoggingURLTemplate: "https://logging.io", - FluentdSidecarOutputConfig: "the-config", - FluentdSidecarImage: "gcr.io/log-stuff/fluentd:latest", - EnableVarLogCollection: true, - RequestLogTemplate: `{"requestMethod": "{{.Request.Method}}"}`, - RequestMetricsBackend: "stackdriver", - }, - config: &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: system.Namespace(), - Name: ObservabilityConfigName, - }, - Data: map[string]string{ - "logging.enable-var-log-collection": "true", - "logging.fluentd-sidecar-image": "gcr.io/log-stuff/fluentd:latest", - "logging.fluentd-sidecar-output-config": "the-config", - "logging.revision-url-template": "https://logging.io", - "logging.write-request-logs": "true", - "logging.request-log-template": `{"requestMethod": "{{.Request.Method}}"}`, - "metrics.request-metrics-backend-destination": "stackdriver", - }, - }, - }, { - name: "observability config with no map", - wantErr: false, - wantController: &ObservabilityConfig{ - EnableVarLogCollection: false, - LoggingURLTemplate: defaultLogURLTemplate, - RequestLogTemplate: "", - RequestMetricsBackend: "", - }, - config: &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: system.Namespace(), - Name: ObservabilityConfigName, - }, - }, - }, { - name: "observability configuration with no side car image", - wantErr: true, - wantController: (*ObservabilityConfig)(nil), - config: &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: system.Namespace(), - Name: ObservabilityConfigName, - }, - Data: map[string]string{ - "logging.enable-var-log-collection": "true", - }, - }, - }, { - name: "invalid request log template", - wantErr: true, - wantController: (*ObservabilityConfig)(nil), - config: &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: system.Namespace(), - Name: ObservabilityConfigName, - }, - Data: map[string]string{ - "logging.request-log-template": `{{ something }}`, - }, - }, - }} - - for _, tt := range observabilityConfigTests { - t.Run(tt.name, func(t *testing.T) { - actualController, err := NewObservabilityConfigFromConfigMap(tt.config) - - if (err != nil) != tt.wantErr { - t.Fatalf("Test: %q; NewObservabilityFromConfigMap() error = %v, WantErr %v", tt.name, err, tt.wantErr) - } - - if diff := cmp.Diff(actualController, tt.wantController); diff != "" { - t.Fatalf("Test: %q; want %v, but got %v", tt.name, tt.wantController, actualController) - } - }) - } -} diff --git a/pkg/metrics/doc.go b/pkg/metrics/doc.go deleted file mode 100644 index eb005b410a7..00000000000 --- a/pkg/metrics/doc.go +++ /dev/null @@ -1,19 +0,0 @@ -/* -Copyright 2019 The Knative Authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// +k8s:deepcopy-gen=package - -package metrics diff --git a/pkg/metrics/metricskey/constants.go b/pkg/metrics/metricskey/constants.go index 7419bf71b57..3cf3edb80dd 100644 --- a/pkg/metrics/metricskey/constants.go +++ b/pkg/metrics/metricskey/constants.go @@ -20,8 +20,11 @@ const ( // LabelFilterResult is the label for the Trigger filtering result. LabelFilterResult = "filter_result" - // LabelResult is the label for the result of sending an event to a downstream consumer. One of "success", "error". - LabelResult = "result" + // LabelResponseCode is the label for the HTTP response status code. + LabelResponseCode = "response_code" + + // LabelResponseCodeClass is the label for the HTTP response status code class. For example, "2xx", "3xx", etc. + LabelResponseCodeClass = "response_code_class" // AnyValue is the default value if the trigger filter attributes are empty. AnyValue = "any" diff --git a/pkg/metrics/testdata/config-observability.yaml b/pkg/metrics/testdata/config-observability.yaml deleted file mode 100644 index 8757d3e714d..00000000000 --- a/pkg/metrics/testdata/config-observability.yaml +++ /dev/null @@ -1,152 +0,0 @@ -# Copyright 2019 The Knative Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: ConfigMap -metadata: - name: config-observability - namespace: knative-eventing - labels: - serving.knative.dev/release: devel - -data: - _example: | - ################################ - # # - # EXAMPLE CONFIGURATION # - # # - ################################ - - # This block is not actually functional configuration, - # but serves to illustrate the available configuration - # options and document them in a way that is accessible - # to users that `kubectl edit` this config map. - # - # These sample configuration options may be copied out of - # this example block and unindented to be in the data block - # to actually change the configuration. - - # logging.enable-var-log-collection defaults to false. - # A fluentd sidecar will be set up to collect var log if - # this flag is true. - logging.enable-var-log-collection: false - - # logging.fluentd-sidecar-image provides the fluentd sidecar image - # to inject as a sidecar to collect logs from /var/log. - # Must be presented if logging.enable-var-log-collection is true. - logging.fluentd-sidecar-image: k8s.gcr.io/fluentd-elasticsearch:v2.0.4 - - # logging.fluentd-sidecar-output-config provides the configuration - # for the fluentd sidecar, which will be placed into a configmap and - # mounted into the fluentd sidecar image. - logging.fluentd-sidecar-output-config: | - # Parse json log before sending to Elastic Search - - @type parser - key_name log - - @type multi_format - - format json - time_key fluentd-time # fluentd-time is reserved for structured logs - time_format %Y-%m-%dT%H:%M:%S.%NZ - - - format none - message_key log - - - - # Send to Elastic Search - - @id elasticsearch - @type elasticsearch - @log_level info - include_tag_key true - # Elasticsearch service is in monitoring namespace. - host elasticsearch-logging.knative-monitoring - port 9200 - logstash_format true - - @type file - path /var/log/fluentd-buffers/kubernetes.system.buffer - flush_mode interval - retry_type exponential_backoff - flush_thread_count 2 - flush_interval 5s - retry_forever - retry_max_interval 30 - chunk_limit_size 2M - queue_limit_length 8 - overflow_action block - - - - # logging.revision-url-template provides a template to use for producing the - # logging URL that is injected into the status of each Revision. - # This value is what you might use the the Knative monitoring bundle, and provides - # access to Kibana after setting up kubectl proxy. - logging.revision-url-template: | - http://localhost:8001/api/v1/namespaces/knative-monitoring/services/kibana-logging/proxy/app/kibana#/discover?_a=(query:(match:(kubernetes.labels.knative-dev%2FrevisionUID:(query:'${REVISION_UID}',type:phrase)))) - - # If non-empty, this enables queue proxy writing request logs to stdout. - # The value determines the shape of the request logs and it must be a valid go text/template. - # It is important to keep this as a single line. Multiple lines are parsed as separate entities - # by most collection agents and will split the request logs into multiple records. - # - # The following fields and functions are available to the template: - # - # Request: An http.Request (see https://golang.org/pkg/net/http/#Request) - # representing an HTTP request received by the server. - # - # Response: - # struct { - # Code int // HTTP status code (see https://www.iana.org/assignments/http-status-codes/http-status-codes.xhtml) - # Size int // An int representing the size of the response. - # Latency float64 // A float64 representing the latency of the response in seconds. - # } - # - # Revision: - # struct { - # Name string // Knative revision name - # Namespace string // Knative revision namespace - # Service string // Knative service name - # Configuration string // Knative configuration name - # PodName string // Name of the pod hosting the revision - # PodIP string // IP of the pod hosting the revision - # } - # - logging.request-log-template: '{"httpRequest": {"requestMethod": "{{.Request.Method}}", "requestUrl": "{{js .Request.RequestURI}}", "requestSize": "{{.Request.ContentLength}}", "status": {{.Response.Code}}, "responseSize": "{{.Response.Size}}", "userAgent": "{{js .Request.UserAgent}}", "remoteIp": "{{js .Request.RemoteAddr}}", "serverIp": "{{.Revision.PodIP}}", "referer": "{{js .Request.Referer}}", "latency": "{{.Response.Latency}}s", "protocol": "{{.Request.Proto}}"}, "traceId": "{{index .Request.Header "X-B3-Traceid"}}"}' - - # metrics.backend-destination field specifies the system metrics destination. - # It supports either prometheus (the default) or stackdriver. - # Note: Using stackdriver will incur additional charges - metrics.backend-destination: prometheus - - # metrics.request-metrics-backend-destination specifies the request metrics - # destination. If non-empty, it enables queue proxy to send request metrics. - # Currently supported values: prometheus, stackdriver. - metrics.request-metrics-backend-destination: prometheus - - # metrics.stackdriver-project-id field specifies the stackdriver project ID. This - # field is optional. When running on GCE, application default credentials will be - # used if this field is not provided. - metrics.stackdriver-project-id: "" - - # metrics.allow-stackdriver-custom-metrics indicates whether it is allowed to send metrics to - # Stackdriver using "global" resource type and custom metric type if the - # metrics are not supported by "knative_revision" resource type. Setting this - # flag to "true" could cause extra Stackdriver charge. - # If metrics.backend-destination is not Stackdriver, this is ignored. - metrics.allow-stackdriver-custom-metrics: "false" diff --git a/pkg/metrics/zz_generated.deepcopy.go b/pkg/metrics/zz_generated.deepcopy.go deleted file mode 100644 index 64996c609b4..00000000000 --- a/pkg/metrics/zz_generated.deepcopy.go +++ /dev/null @@ -1,37 +0,0 @@ -// +build !ignore_autogenerated - -/* -Copyright 2019 The Knative Authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by deepcopy-gen. DO NOT EDIT. - -package metrics - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ObservabilityConfig) DeepCopyInto(out *ObservabilityConfig) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObservabilityConfig. -func (in *ObservabilityConfig) DeepCopy() *ObservabilityConfig { - if in == nil { - return nil - } - out := new(ObservabilityConfig) - in.DeepCopyInto(out) - return out -} diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index a1cebaca8a1..c6226ec000b 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -21,6 +21,7 @@ import ( "io" "os" "regexp" + "strconv" "strings" "sync" @@ -128,3 +129,10 @@ func GenerateFixedName(owner metav1.Object, prefix string) string { } return prefix[:pl] + uid } + +// ResponseCodeClass converts an HTTP response code to a string representing its response code class. +// E.g., The response code class is "5xx" for response code 503. +func ResponseCodeClass(responseCode int) string { + // Get the hundred digit of the response code and concatenate "xx". + return strconv.Itoa(responseCode/100) + "xx" +} diff --git a/pkg/utils/utils_test.go b/pkg/utils/utils_test.go index 2b057945c93..7d920687cb7 100644 --- a/pkg/utils/utils_test.go +++ b/pkg/utils/utils_test.go @@ -18,6 +18,7 @@ package utils import ( "fmt" + "net/http" "strings" "testing" @@ -197,3 +198,27 @@ func TestToDNS1123Subdomain(t *testing.T) { }) } } + +func TestResponseCodeClass(t *testing.T) { + testCases := map[string]struct { + responseCode int + expected string + }{ + "2xx": { + responseCode: http.StatusOK, + expected: "2xx", + }, + "4xx": { + responseCode: http.StatusForbidden, + expected: "4xx", + }, + } + for n, tc := range testCases { + t.Run(n, func(t *testing.T) { + a := ResponseCodeClass(tc.responseCode) + if a != tc.expected { + t.Errorf("Expected %q, actually %q", tc.expected, a) + } + }) + } +} diff --git a/vendor/github.com/cloudevents/sdk-go/alias.go b/vendor/github.com/cloudevents/sdk-go/alias.go index f805ec5226f..f97b6473adf 100644 --- a/vendor/github.com/cloudevents/sdk-go/alias.go +++ b/vendor/github.com/cloudevents/sdk-go/alias.go @@ -7,6 +7,7 @@ import ( "github.com/cloudevents/sdk-go/pkg/cloudevents" "github.com/cloudevents/sdk-go/pkg/cloudevents/client" "github.com/cloudevents/sdk-go/pkg/cloudevents/context" + "github.com/cloudevents/sdk-go/pkg/cloudevents/observability" "github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http" "github.com/cloudevents/sdk-go/pkg/cloudevents/types" ) @@ -15,6 +16,7 @@ import ( type ClientOption client.Option type Client = client.Client +type ConvertFn = client.ConvertFn // Event @@ -41,6 +43,37 @@ type HTTPTransportContext = http.TransportContext type HTTPTransportResponseContext = http.TransportResponseContext type HTTPEncoding = http.Encoding +const ( + // Encoding + + ApplicationXML = cloudevents.ApplicationXML + ApplicationJSON = cloudevents.ApplicationJSON + ApplicationCloudEventsJSON = cloudevents.ApplicationCloudEventsJSON + ApplicationCloudEventsBatchJSON = cloudevents.ApplicationCloudEventsBatchJSON + Base64 = cloudevents.Base64 + + // Event Versions + + VersionV01 = cloudevents.CloudEventsVersionV01 + VersionV02 = cloudevents.CloudEventsVersionV02 + VersionV03 = cloudevents.CloudEventsVersionV03 + + // HTTP Transport Encodings + + HTTPBinaryV01 = http.BinaryV01 + HTTPStructuredV01 = http.StructuredV01 + HTTPBinaryV02 = http.BinaryV02 + HTTPStructuredV02 = http.StructuredV02 + HTTPBinaryV03 = http.BinaryV03 + HTTPStructuredV03 = http.StructuredV03 + HTTPBatchedV03 = http.BatchedV03 + + // Context HTTP Transport Encodings + + Binary = http.Binary + Structured = http.Structured +) + var ( // ContentType Helpers @@ -50,8 +83,6 @@ var ( StringOfApplicationCloudEventsBatchJSON = cloudevents.StringOfApplicationCloudEventsBatchJSON StringOfBase64 = cloudevents.StringOfBase64 - Base64 = cloudevents.Base64 - // Client Creation NewClient = client.New @@ -62,18 +93,22 @@ var ( WithEventDefaulter = client.WithEventDefaulter WithUUIDs = client.WithUUIDs WithTimeNow = client.WithTimeNow + WithConverterFn = client.WithConverterFn // Event Creation - NewEvent = cloudevents.New - VersionV01 = cloudevents.CloudEventsVersionV01 - VersionV02 = cloudevents.CloudEventsVersionV02 - VersionV03 = cloudevents.CloudEventsVersionV03 + NewEvent = cloudevents.New + + // Tracing + + EnableTracing = observability.EnableTracing // Context - ContextWithTarget = context.WithTarget - TargetFromContext = context.TargetFrom + ContextWithTarget = context.WithTarget + TargetFromContext = context.TargetFrom + ContextWithEncoding = context.WithEncoding + EncodingFromContext = context.EncodingFrom // Custom Types @@ -86,29 +121,21 @@ var ( // HTTP Transport Options - WithTarget = http.WithTarget - WithMethod = http.WithMethod - WitHHeader = http.WithHeader - WithShutdownTimeout = http.WithShutdownTimeout - WithEncoding = http.WithEncoding - WithBinaryEncoding = http.WithBinaryEncoding - WithStructuredEncoding = http.WithStructuredEncoding - WithPort = http.WithPort - WithPath = http.WithPath - WithMiddleware = http.WithMiddleware + WithTarget = http.WithTarget + WithMethod = http.WithMethod + WitHHeader = http.WithHeader + WithShutdownTimeout = http.WithShutdownTimeout + WithEncoding = http.WithEncoding + WithContextBasedEncoding = http.WithContextBasedEncoding + WithBinaryEncoding = http.WithBinaryEncoding + WithStructuredEncoding = http.WithStructuredEncoding + WithPort = http.WithPort + WithPath = http.WithPath + WithMiddleware = http.WithMiddleware + WithLongPollTarget = http.WithLongPollTarget // HTTP Context HTTPTransportContextFrom = http.TransportContextFrom ContextWithHeader = http.ContextWithHeader - - // HTTP Transport Encodings - - HTTPBinaryV01 = http.BinaryV01 - HTTPStructuredV01 = http.StructuredV01 - HTTPBinaryV02 = http.BinaryV02 - HTTPStructuredV02 = http.StructuredV02 - HTTPBinaryV03 = http.BinaryV03 - HTTPStructuredV03 = http.StructuredV03 - HTTPBatchedV03 = http.BatchedV03 ) diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/client.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/client.go index 9eefa27c8ec..a36fe4718e9 100644 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/client.go +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/client.go @@ -3,17 +3,18 @@ package client import ( "context" "fmt" + "sync" + "github.com/cloudevents/sdk-go/pkg/cloudevents" "github.com/cloudevents/sdk-go/pkg/cloudevents/observability" "github.com/cloudevents/sdk-go/pkg/cloudevents/transport" "github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http" - "sync" ) // Client interface defines the runtime contract the CloudEvents client supports. type Client interface { // Send will transmit the given event over the client's configured transport. - Send(ctx context.Context, event cloudevents.Event) (*cloudevents.Event, error) + Send(ctx context.Context, event cloudevents.Event) (context.Context, *cloudevents.Event, error) // StartReceiver will register the provided function for callback on receipt // of a cloudevent. It will also start the underlying transport as it has @@ -72,6 +73,8 @@ type ceClient struct { transport transport.Transport fn *receiverFn + convertFn ConvertFn + receiverMu sync.Mutex eventDefaulterFns []EventDefaulter } @@ -80,32 +83,32 @@ type ceClient struct { // Send returns a response event if there is a response or an error if there // was an an issue validating the outbound event or the transport returns an // error. -func (c *ceClient) Send(ctx context.Context, event cloudevents.Event) (*cloudevents.Event, error) { +func (c *ceClient) Send(ctx context.Context, event cloudevents.Event) (context.Context, *cloudevents.Event, error) { ctx, r := observability.NewReporter(ctx, reportSend) - resp, err := c.obsSend(ctx, event) + rctx, resp, err := c.obsSend(ctx, event) if err != nil { r.Error() } else { r.OK() } - return resp, err + return rctx, resp, err } -func (c *ceClient) obsSend(ctx context.Context, event cloudevents.Event) (*cloudevents.Event, error) { +func (c *ceClient) obsSend(ctx context.Context, event cloudevents.Event) (context.Context, *cloudevents.Event, error) { // Confirm we have a transport set. if c.transport == nil { - return nil, fmt.Errorf("client not ready, transport not initialized") + return ctx, nil, fmt.Errorf("client not ready, transport not initialized") } // Apply the defaulter chain to the incoming event. if len(c.eventDefaulterFns) > 0 { for _, fn := range c.eventDefaulterFns { - event = fn(event) + event = fn(ctx, event) } } // Validate the event conforms to the CloudEvents Spec. if err := event.Validate(); err != nil { - return nil, err + return ctx, nil, err } // Send the event over the transport. return c.transport.Send(ctx, event) @@ -136,7 +139,7 @@ func (c *ceClient) obsReceive(ctx context.Context, event cloudevents.Event, resp // Apply the defaulter chain to the outgoing event. if err == nil && resp != nil && resp.Event != nil && len(c.eventDefaulterFns) > 0 { for _, fn := range c.eventDefaulterFns { - *resp.Event = fn(*resp.Event) + *resp.Event = fn(ctx, *resp.Event) } // Validate the event conforms to the CloudEvents Spec. if err := resp.Event.Validate(); err != nil { @@ -182,3 +185,11 @@ func (c *ceClient) applyOptions(opts ...Option) error { } return nil } + +// Convert implements transport Converter.Convert. +func (c *ceClient) Convert(ctx context.Context, m transport.Message, err error) (*cloudevents.Event, error) { + if c.convertFn != nil { + return c.convertFn(ctx, m, err) + } + return nil, err +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/defaulters.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/defaulters.go index a5cd591c740..40bd85a9cb5 100644 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/defaulters.go +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/defaulters.go @@ -1,18 +1,20 @@ package client import ( + "context" + "time" + "github.com/cloudevents/sdk-go/pkg/cloudevents" "github.com/google/uuid" - "time" ) // EventDefaulter is the function signature for extensions that are able // to perform event defaulting. -type EventDefaulter func(event cloudevents.Event) cloudevents.Event +type EventDefaulter func(ctx context.Context, event cloudevents.Event) cloudevents.Event // DefaultIDToUUIDIfNotSet will inspect the provided event and assign a UUID to // context.ID if it is found to be empty. -func DefaultIDToUUIDIfNotSet(event cloudevents.Event) cloudevents.Event { +func DefaultIDToUUIDIfNotSet(ctx context.Context, event cloudevents.Event) cloudevents.Event { if event.Context != nil { if event.ID() == "" { event.Context = event.Context.Clone() @@ -24,7 +26,7 @@ func DefaultIDToUUIDIfNotSet(event cloudevents.Event) cloudevents.Event { // DefaultTimeToNowIfNotSet will inspect the provided event and assign a new // Timestamp to context.Time if it is found to be nil or zero. -func DefaultTimeToNowIfNotSet(event cloudevents.Event) cloudevents.Event { +func DefaultTimeToNowIfNotSet(ctx context.Context, event cloudevents.Event) cloudevents.Event { if event.Context != nil { if event.Time().IsZero() { event.Context = event.Context.Clone() diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/options.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/options.go index ec4ac7842f8..6e5051c3eaf 100644 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/options.go +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/options.go @@ -35,3 +35,19 @@ func WithTimeNow() Option { return nil } } + +// WithConverterFn defines the function the transport will use to delegate +// conversion of non-decodable messages. +func WithConverterFn(fn ConvertFn) Option { + return func(c *ceClient) error { + if fn == nil { + return fmt.Errorf("client option was given an nil message converter") + } + if c.transport.HasConverter() { + return fmt.Errorf("transport converter already set") + } + c.convertFn = fn + c.transport.SetConverter(c) + return nil + } +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/receiver.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/receiver.go index 6264a3f79d0..9734341d43f 100644 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/receiver.go +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/receiver.go @@ -4,8 +4,10 @@ import ( "context" "errors" "fmt" - "github.com/cloudevents/sdk-go/pkg/cloudevents" "reflect" + + "github.com/cloudevents/sdk-go/pkg/cloudevents" + "github.com/cloudevents/sdk-go/pkg/cloudevents/transport" ) // Receive is the signature of a fn to be invoked for incoming cloudevents. @@ -25,6 +27,10 @@ type receiverFn struct { hasErrorOut bool } +// ConvertFn defines the signature the client expects to enable conversion +// delegation. +type ConvertFn func(context.Context, transport.Message, error) (*cloudevents.Event, error) + const ( inParamUsage = "expected a function taking either no parameters, one or more of (context.Context, cloudevents.Event, *cloudevents.EventResponse) ordered" outParamUsage = "expected a function returning either nothing or an error" diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/codec/doc.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/codec/doc.go deleted file mode 100644 index f6028398f5a..00000000000 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/codec/doc.go +++ /dev/null @@ -1,5 +0,0 @@ -/* -Package codec holds the encoder/decoder implementation for structured encodings using `application/json` of the -whole CloudEvent. -*/ -package codec diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/codec/jsoncodec.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/codec/jsoncodec.go deleted file mode 100644 index 09efe52d2a8..00000000000 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/codec/jsoncodec.go +++ /dev/null @@ -1,316 +0,0 @@ -package codec - -import ( - "context" - "encoding/json" - "github.com/cloudevents/sdk-go/pkg/cloudevents" - "github.com/cloudevents/sdk-go/pkg/cloudevents/observability" - "strconv" -) - -// JsonEncodeV01 takes in a cloudevent.Event and outputs the byte representation of that event using CloudEvents -// version 0.1 structured json formatting rules. -func JsonEncodeV01(e cloudevents.Event) ([]byte, error) { - _, r := observability.NewReporter(context.Background(), codecObserved{o: reportEncode, v: "v0.1"}) - b, err := obsJsonEncodeV01(e) - if err != nil { - r.Error() - } else { - r.OK() - } - return b, err -} - -func obsJsonEncodeV01(e cloudevents.Event) ([]byte, error) { - ctx := e.Context.AsV01() - if ctx.ContentType == nil { - ctx.ContentType = cloudevents.StringOfApplicationJSON() - } - data, err := e.DataBytes() - if err != nil { - return nil, err - } - return jsonEncode(ctx, data) -} - -// JsonEncodeV02 takes in a cloudevent.Event and outputs the byte representation of that event using CloudEvents -// version 0.2 structured json formatting rules. -func JsonEncodeV02(e cloudevents.Event) ([]byte, error) { - _, r := observability.NewReporter(context.Background(), codecObserved{o: reportEncode, v: "v0.2"}) - b, err := obsJsonEncodeV02(e) - if err != nil { - r.Error() - } else { - r.OK() - } - return b, err -} - -func obsJsonEncodeV02(e cloudevents.Event) ([]byte, error) { - ctx := e.Context.AsV02() - if ctx.ContentType == nil { - ctx.ContentType = cloudevents.StringOfApplicationJSON() - } - data, err := e.DataBytes() - if err != nil { - return nil, err - } - return jsonEncode(ctx, data) -} - -// JsonEncodeV03 takes in a cloudevent.Event and outputs the byte representation of that event using CloudEvents -// version 0.3 structured json formatting rules. -func JsonEncodeV03(e cloudevents.Event) ([]byte, error) { - _, r := observability.NewReporter(context.Background(), codecObserved{o: reportEncode, v: "v0.3"}) - b, err := obsJsonEncodeV03(e) - if err != nil { - r.Error() - } else { - r.OK() - } - return b, err -} - -func obsJsonEncodeV03(e cloudevents.Event) ([]byte, error) { - ctx := e.Context.AsV03() - if ctx.DataContentType == nil { - ctx.DataContentType = cloudevents.StringOfApplicationJSON() - } - - data, err := e.DataBytes() - if err != nil { - return nil, err - } - return jsonEncode(ctx, data) -} - -func jsonEncode(ctx cloudevents.EventContextReader, data []byte) ([]byte, error) { - var b map[string]json.RawMessage - var err error - - if ctx.GetSpecVersion() == cloudevents.CloudEventsVersionV01 { - b, err = marshalEventLegacy(ctx) - } else { - b, err = marshalEvent(ctx, ctx.GetExtensions()) - } - if err != nil { - return nil, err - } - - if data != nil { - // data is passed in as an encoded []byte. That slice might be any - // number of things but for json encoding of the envelope all we care - // is if the payload is either a string or a json object. If it is a - // json object, it can be inserted into the body without modification. - // Otherwise we need to quote it if not already quoted. - mediaType, err := ctx.GetDataMediaType() - if err != nil { - return nil, err - } - isBase64 := ctx.GetDataContentEncoding() == cloudevents.Base64 - isJson := mediaType == "" || mediaType == cloudevents.ApplicationJSON || mediaType == cloudevents.TextJSON - // TODO(#60): we do not support json values at the moment, only objects and lists. - if isJson && !isBase64 { - b["data"] = data - } else if data[0] != byte('"') { - b["data"] = []byte(strconv.QuoteToASCII(string(data))) - } else { - // already quoted - b["data"] = data - } - } - - body, err := json.Marshal(b) - if err != nil { - return nil, err - } - - return body, nil -} - -// JsonDecodeV01 takes in the byte representation of a version 0.1 structured json CloudEvent and returns a -// cloudevent.Event or an error if there are parsing errors. -func JsonDecodeV01(body []byte) (*cloudevents.Event, error) { - _, r := observability.NewReporter(context.Background(), codecObserved{o: reportDecode, v: "v0.1"}) - e, err := obsJsonDecodeV01(body) - if err != nil { - r.Error() - } else { - r.OK() - } - return e, err -} - -func obsJsonDecodeV01(body []byte) (*cloudevents.Event, error) { - ec := cloudevents.EventContextV01{} - if err := json.Unmarshal(body, &ec); err != nil { - return nil, err - } - - raw := make(map[string]json.RawMessage) - - if err := json.Unmarshal(body, &raw); err != nil { - return nil, err - } - var data interface{} - if d, ok := raw["data"]; ok { - data = []byte(d) - } - - return &cloudevents.Event{ - Context: &ec, - Data: data, - DataEncoded: true, - }, nil -} - -// JsonDecodeV02 takes in the byte representation of a version 0.2 structured json CloudEvent and returns a -// cloudevent.Event or an error if there are parsing errors. -func JsonDecodeV02(body []byte) (*cloudevents.Event, error) { - _, r := observability.NewReporter(context.Background(), codecObserved{o: reportDecode, v: "v0.2"}) - e, err := obsJsonDecodeV02(body) - if err != nil { - r.Error() - } else { - r.OK() - } - return e, err -} - -func obsJsonDecodeV02(body []byte) (*cloudevents.Event, error) { - ec := cloudevents.EventContextV02{} - if err := json.Unmarshal(body, &ec); err != nil { - return nil, err - } - - raw := make(map[string]json.RawMessage, 0) - - if err := json.Unmarshal(body, &raw); err != nil { - return nil, err - } - - // TODO: could use reflection to get these. - delete(raw, "specversion") - delete(raw, "type") - delete(raw, "source") - delete(raw, "id") - delete(raw, "time") - delete(raw, "schemaurl") - delete(raw, "contenttype") - - var data interface{} - if d, ok := raw["data"]; ok { - data = []byte(d) - } - delete(raw, "data") - - if len(raw) > 0 { - extensions := make(map[string]interface{}, len(raw)) - for k, v := range raw { - extensions[k] = v - } - ec.Extensions = extensions - } - - return &cloudevents.Event{ - Context: &ec, - Data: data, - DataEncoded: true, - }, nil -} - -// JsonDecodeV03 takes in the byte representation of a version 0.3 structured json CloudEvent and returns a -// cloudevent.Event or an error if there are parsing errors. -func JsonDecodeV03(body []byte) (*cloudevents.Event, error) { - _, r := observability.NewReporter(context.Background(), codecObserved{o: reportDecode, v: "v0.3"}) - e, err := obsJsonDecodeV03(body) - if err != nil { - r.Error() - } else { - r.OK() - } - return e, err -} - -func obsJsonDecodeV03(body []byte) (*cloudevents.Event, error) { - ec := cloudevents.EventContextV03{} - if err := json.Unmarshal(body, &ec); err != nil { - return nil, err - } - - raw := make(map[string]json.RawMessage) - - if err := json.Unmarshal(body, &raw); err != nil { - return nil, err - } - - // TODO: could use reflection to get these. - delete(raw, "specversion") - delete(raw, "type") - delete(raw, "source") - delete(raw, "subject") - delete(raw, "id") - delete(raw, "time") - delete(raw, "schemaurl") - delete(raw, "datacontenttype") - delete(raw, "datacontentencoding") - - var data interface{} - if d, ok := raw["data"]; ok { - data = []byte(d) - } - delete(raw, "data") - - if len(raw) > 0 { - extensions := make(map[string]interface{}, len(raw)) - for k, v := range raw { - extensions[k] = v - } - ec.Extensions = extensions - } - - return &cloudevents.Event{ - Context: &ec, - Data: data, - DataEncoded: true, - }, nil -} - -func marshalEventLegacy(event interface{}) (map[string]json.RawMessage, error) { - b, err := json.Marshal(event) - if err != nil { - return nil, err - } - - brm := map[string]json.RawMessage{} - if err := json.Unmarshal(b, &brm); err != nil { - return nil, err - } - - return brm, nil -} - -func marshalEvent(event interface{}, extensions map[string]interface{}) (map[string]json.RawMessage, error) { - b, err := json.Marshal(event) - if err != nil { - return nil, err - } - - brm := map[string]json.RawMessage{} - if err := json.Unmarshal(b, &brm); err != nil { - return nil, err - } - - for k, v := range extensions { - vb, err := json.Marshal(v) - if err != nil { - return nil, err - } - // Don't overwrite spec keys. - if _, ok := brm[k]; !ok { - brm[k] = vb - } - } - - return brm, nil -} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/codec/observability.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/codec/observability.go deleted file mode 100644 index 49bcb793d81..00000000000 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/codec/observability.go +++ /dev/null @@ -1,90 +0,0 @@ -package codec - -import ( - "fmt" - - "github.com/cloudevents/sdk-go/pkg/cloudevents/observability" - "go.opencensus.io/stats" - "go.opencensus.io/stats/view" -) - -var ( - // LatencyMs measures the latency in milliseconds for the CloudEvents json codec methods. - LatencyMs = stats.Float64("cloudevents.io/sdk-go/codec/json/latency", "The latency in milliseconds for the CloudEvents json codec methods.", "ms") -) - -var ( - // LatencyView is an OpenCensus view that shows codec/json method latency. - LatencyView = &view.View{ - Name: "codec/json/latency", - Measure: LatencyMs, - Description: "The distribution of latency inside of the json codec for CloudEvents.", - Aggregation: view.Distribution(0, .01, .1, 1, 10, 100, 1000, 10000), - TagKeys: observability.LatencyTags(), - } -) - -type observed int32 - -// Adheres to Observable -var _ observability.Observable = observed(0) - -const ( - reportEncode observed = iota - reportDecode -) - -// TraceName implements Observable.TraceName -func (o observed) TraceName() string { - switch o { - case reportEncode: - return "codec/json/encode" - case reportDecode: - return "codec/json/decode" - default: - return "codec/unknown" - } -} - -// MethodName implements Observable.MethodName -func (o observed) MethodName() string { - switch o { - case reportEncode: - return "encode" - case reportDecode: - return "decode" - default: - return "unknown" - } -} - -// LatencyMs implements Observable.LatencyMs -func (o observed) LatencyMs() *stats.Float64Measure { - return LatencyMs -} - -// codecObserved is a wrapper to append version to observed. -type codecObserved struct { - // Method - o observed - // Version - v string -} - -// Adheres to Observable -var _ observability.Observable = (*codecObserved)(nil) - -// TraceName implements Observable.TraceName -func (c codecObserved) TraceName() string { - return fmt.Sprintf("%s/%s", c.o.TraceName(), c.v) -} - -// MethodName implements Observable.MethodName -func (c codecObserved) MethodName() string { - return fmt.Sprintf("%s/%s", c.o.MethodName(), c.v) -} - -// LatencyMs implements Observable.LatencyMs -func (c codecObserved) LatencyMs() *stats.Float64Measure { - return c.o.LatencyMs() -} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/context/context.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/context/context.go index 18afb6282b6..e580360f130 100644 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/context/context.go +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/context/context.go @@ -3,6 +3,7 @@ package context import ( "context" "net/url" + "strings" ) // Opaque key type used to store target @@ -28,3 +29,48 @@ func TargetFrom(ctx context.Context) *url.URL { } return nil } + +// Opaque key type used to store topic +type topicKeyType struct{} + +var topicKey = topicKeyType{} + +// WithTopic returns back a new context with the given topic. Topic is intended to be transport dependent. +// For pubsub transport, `topic` should be a Pub/Sub Topic ID. +func WithTopic(ctx context.Context, topic string) context.Context { + return context.WithValue(ctx, topicKey, topic) +} + +// TopicFrom looks in the given context and returns `topic` as a string if found and valid, otherwise "". +func TopicFrom(ctx context.Context) string { + c := ctx.Value(topicKey) + if c != nil { + if s, ok := c.(string); ok { + return s + } + } + return "" +} + +// Opaque key type used to store encoding +type encodingKeyType struct{} + +var encodingKey = encodingKeyType{} + +// WithEncoding returns back a new context with the given encoding. Encoding is intended to be transport dependent. +// For http transport, `encoding` should be one of [binary, structured] and will be used to override the outbound +// codec encoding setting. If the transport does not understand the encoding, it will be ignored. +func WithEncoding(ctx context.Context, encoding string) context.Context { + return context.WithValue(ctx, encodingKey, strings.ToLower(encoding)) +} + +// EncodingFrom looks in the given context and returns `target` as a parsed url if found and valid, otherwise nil. +func EncodingFrom(ctx context.Context) string { + c := ctx.Value(encodingKey) + if c != nil { + if s, ok := c.(string); ok && s != "" { + return s + } + } + return "" +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/context/logger.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/context/logger.go index 38d46069664..996f720572e 100644 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/context/logger.go +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/context/logger.go @@ -2,6 +2,7 @@ package context import ( "context" + "go.uber.org/zap" ) diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/codec.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/codec.go index 7b88e627ccd..41425c21ff5 100644 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/codec.go +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/codec.go @@ -3,6 +3,7 @@ package datacodec import ( "context" "fmt" + "github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/json" "github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/xml" "github.com/cloudevents/sdk-go/pkg/cloudevents/observability" @@ -11,11 +12,11 @@ import ( // Decoder is the expected function signature for decoding `in` to `out`. What // `in` is could be decoder dependent. For example, `in` could be bytes, or a // base64 string. -type Decoder func(in, out interface{}) error +type Decoder func(ctx context.Context, in, out interface{}) error // Encoder is the expected function signature for encoding `in` to bytes. // Returns an error if the encoder has an issue encoding `in`. -type Encoder func(in interface{}) ([]byte, error) +type Encoder func(ctx context.Context, in interface{}) ([]byte, error) var decoder map[string]Decoder var encoder map[string]Encoder @@ -52,10 +53,9 @@ func AddEncoder(contentType string, fn Encoder) { // Decode looks up and invokes the decoder registered for the given content // type. An error is returned if no decoder is registered for the given // content type. -func Decode(contentType string, in, out interface{}) error { - // TODO: wire in context. - _, r := observability.NewReporter(context.Background(), reportDecode) - err := obsDecode(contentType, in, out) +func Decode(ctx context.Context, contentType string, in, out interface{}) error { + _, r := observability.NewReporter(ctx, reportDecode) + err := obsDecode(ctx, contentType, in, out) if err != nil { r.Error() } else { @@ -64,9 +64,9 @@ func Decode(contentType string, in, out interface{}) error { return err } -func obsDecode(contentType string, in, out interface{}) error { +func obsDecode(ctx context.Context, contentType string, in, out interface{}) error { if fn, ok := decoder[contentType]; ok { - return fn(in, out) + return fn(ctx, in, out) } return fmt.Errorf("[decode] unsupported content type: %q", contentType) } @@ -74,10 +74,9 @@ func obsDecode(contentType string, in, out interface{}) error { // Encode looks up and invokes the encoder registered for the given content // type. An error is returned if no encoder is registered for the given // content type. -func Encode(contentType string, in interface{}) ([]byte, error) { - // TODO: wire in context. - _, r := observability.NewReporter(context.Background(), reportEncode) - b, err := obsEncode(contentType, in) +func Encode(ctx context.Context, contentType string, in interface{}) ([]byte, error) { + _, r := observability.NewReporter(ctx, reportEncode) + b, err := obsEncode(ctx, contentType, in) if err != nil { r.Error() } else { @@ -86,9 +85,9 @@ func Encode(contentType string, in interface{}) ([]byte, error) { return b, err } -func obsEncode(contentType string, in interface{}) ([]byte, error) { +func obsEncode(ctx context.Context, contentType string, in interface{}) ([]byte, error) { if fn, ok := encoder[contentType]; ok { - return fn(in) + return fn(ctx, in) } return nil, fmt.Errorf("[encode] unsupported content type: %q", contentType) } diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/json/data.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/json/data.go index ef2a69eb78c..926c344fed0 100644 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/json/data.go +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/json/data.go @@ -4,18 +4,18 @@ import ( "context" "encoding/json" "fmt" - "github.com/cloudevents/sdk-go/pkg/cloudevents/observability" "reflect" "strconv" + + "github.com/cloudevents/sdk-go/pkg/cloudevents/observability" ) // Decode takes `in` as []byte, or base64 string, normalizes in to unquoted and // base64 decoded []byte if required, and then attempts to use json.Unmarshal // to convert those bytes to `out`. Returns and error if this process fails. -func Decode(in, out interface{}) error { - // TODO: wire in context. - _, r := observability.NewReporter(context.Background(), reportDecode) - err := obsDecode(in, out) +func Decode(ctx context.Context, in, out interface{}) error { + _, r := observability.NewReporter(ctx, reportDecode) + err := obsDecode(ctx, in, out) if err != nil { r.Error() } else { @@ -24,7 +24,7 @@ func Decode(in, out interface{}) error { return err } -func obsDecode(in, out interface{}) error { +func obsDecode(ctx context.Context, in, out interface{}) error { if in == nil { return nil } @@ -62,10 +62,9 @@ func obsDecode(in, out interface{}) error { // Encode attempts to json.Marshal `in` into bytes. Encode will inspect `in` // and returns `in` unmodified if it is detected that `in` is already a []byte; // Or json.Marshal errors. -func Encode(in interface{}) ([]byte, error) { - // TODO: wire in context. - _, r := observability.NewReporter(context.Background(), reportEncode) - b, err := obsEncode(in) +func Encode(ctx context.Context, in interface{}) ([]byte, error) { + _, r := observability.NewReporter(ctx, reportEncode) + b, err := obsEncode(ctx, in) if err != nil { r.Error() } else { @@ -74,7 +73,7 @@ func Encode(in interface{}) ([]byte, error) { return b, err } -func obsEncode(in interface{}) ([]byte, error) { +func obsEncode(ctx context.Context, in interface{}) ([]byte, error) { if in == nil { return nil, nil } diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/xml/data.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/xml/data.go index 047a961e0d2..6339e444337 100644 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/xml/data.go +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/xml/data.go @@ -5,17 +5,17 @@ import ( "encoding/base64" "encoding/xml" "fmt" - "github.com/cloudevents/sdk-go/pkg/cloudevents/observability" "strconv" + + "github.com/cloudevents/sdk-go/pkg/cloudevents/observability" ) // Decode takes `in` as []byte, or base64 string, normalizes in to unquoted and // base64 decoded []byte if required, and then attempts to use xml.Unmarshal // to convert those bytes to `out`. Returns and error if this process fails. -func Decode(in, out interface{}) error { - // TODO: wire in context. - _, r := observability.NewReporter(context.Background(), reportDecode) - err := obsDecode(in, out) +func Decode(ctx context.Context, in, out interface{}) error { + _, r := observability.NewReporter(ctx, reportDecode) + err := obsDecode(ctx, in, out) if err != nil { r.Error() } else { @@ -24,7 +24,7 @@ func Decode(in, out interface{}) error { return err } -func obsDecode(in, out interface{}) error { +func obsDecode(ctx context.Context, in, out interface{}) error { if in == nil { return nil } @@ -67,10 +67,9 @@ func obsDecode(in, out interface{}) error { // Encode attempts to xml.Marshal `in` into bytes. Encode will inspect `in` // and returns `in` unmodified if it is detected that `in` is already a []byte; // Or xml.Marshal errors. -func Encode(in interface{}) ([]byte, error) { - // TODO: wire in context. - _, r := observability.NewReporter(context.Background(), reportEncode) - b, err := obsEncode(in) +func Encode(ctx context.Context, in interface{}) ([]byte, error) { + _, r := observability.NewReporter(ctx, reportEncode) + b, err := obsEncode(ctx, in) if err != nil { r.Error() } else { @@ -79,7 +78,7 @@ func Encode(in interface{}) ([]byte, error) { return b, err } -func obsEncode(in interface{}) ([]byte, error) { +func obsEncode(ctx context.Context, in interface{}) ([]byte, error) { if b, ok := in.([]byte); ok { // check to see if it is a pre-encoded byte string. if len(b) > 0 && b[0] == byte('"') { diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/event_data.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/event_data.go index 3f2b10fbf5f..9e4cb9d2471 100644 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/event_data.go +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/event_data.go @@ -1,18 +1,20 @@ package cloudevents import ( + "context" "encoding/base64" "errors" "fmt" - "github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec" "strconv" + + "github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec" ) // Data is special. Break it out into it's own file. // SetData implements EventWriter.SetData func (e *Event) SetData(obj interface{}) error { - data, err := datacodec.Encode(e.DataMediaType(), obj) + data, err := datacodec.Encode(context.Background(), e.DataMediaType(), obj) if err != nil { return err } @@ -93,5 +95,5 @@ func (e Event) DataAs(data interface{}) error { // TODO: Clean this function up if err != nil { return err } - return datacodec.Decode(mediaType, obj, data) + return datacodec.Decode(context.Background(), mediaType, obj, data) } diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/event_marshal.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/event_marshal.go new file mode 100644 index 00000000000..5e2c1602efa --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/event_marshal.go @@ -0,0 +1,281 @@ +package cloudevents + +import ( + "context" + "encoding/json" + "fmt" + "strconv" + + "github.com/cloudevents/sdk-go/pkg/cloudevents/observability" +) + +// MarshalJSON implements a custom json marshal method used when this type is +// marshaled using json.Marshal. +func (e Event) MarshalJSON() ([]byte, error) { + _, r := observability.NewReporter(context.Background(), eventJSONObserved{o: reportMarshal, v: e.SpecVersion()}) + + if err := e.Validate(); err != nil { + r.Error() + return nil, err + } + + b, err := JsonEncode(e) + + // Report the observable + if err != nil { + r.Error() + return nil, err + } else { + r.OK() + } + + return b, nil +} + +// UnmarshalJSON implements the json unmarshal method used when this type is +// unmarshaled using json.Unmarshal. +func (e *Event) UnmarshalJSON(b []byte) error { + raw := make(map[string]json.RawMessage) + if err := json.Unmarshal(b, &raw); err != nil { + return err + } + + version := versionFromRawMessage(raw) + + _, r := observability.NewReporter(context.Background(), eventJSONObserved{o: reportUnmarshal, v: version}) + + var err error + switch version { + case CloudEventsVersionV01: + err = e.JsonDecodeV01(b, raw) + case CloudEventsVersionV02: + err = e.JsonDecodeV02(b, raw) + case CloudEventsVersionV03: + err = e.JsonDecodeV03(b, raw) + default: + return fmt.Errorf("unnknown spec version: %q", version) + } + + // Report the observable + if err != nil { + r.Error() + return err + } else { + r.OK() + } + return nil +} + +func versionFromRawMessage(raw map[string]json.RawMessage) string { + // v0.1 + if v, ok := raw["cloudEventsVersion"]; ok { + var version string + if err := json.Unmarshal(v, &version); err != nil { + return "" + } + return version + } + + // v0.2 and after + if v, ok := raw["specversion"]; ok { + var version string + if err := json.Unmarshal(v, &version); err != nil { + return "" + } + return version + } + return "" +} + +// JsonEncode +func JsonEncode(e Event) ([]byte, error) { + if e.DataContentType() == "" { + e.SetDataContentType(ApplicationJSON) + } + data, err := e.DataBytes() + if err != nil { + return nil, err + } + return jsonEncode(e.Context, data) +} + +func jsonEncode(ctx EventContextReader, data []byte) ([]byte, error) { + var b map[string]json.RawMessage + var err error + + if ctx.GetSpecVersion() == CloudEventsVersionV01 { + b, err = marshalEventLegacy(ctx) + } else { + b, err = marshalEvent(ctx, ctx.GetExtensions()) + } + if err != nil { + return nil, err + } + + if data != nil { + // data is passed in as an encoded []byte. That slice might be any + // number of things but for json encoding of the envelope all we care + // is if the payload is either a string or a json object. If it is a + // json object, it can be inserted into the body without modification. + // Otherwise we need to quote it if not already quoted. + mediaType, err := ctx.GetDataMediaType() + if err != nil { + return nil, err + } + isBase64 := ctx.GetDataContentEncoding() == Base64 + isJson := mediaType == "" || mediaType == ApplicationJSON || mediaType == TextJSON + // TODO(#60): we do not support json values at the moment, only objects and lists. + if isJson && !isBase64 { + b["data"] = data + } else if data[0] != byte('"') { + b["data"] = []byte(strconv.QuoteToASCII(string(data))) + } else { + // already quoted + b["data"] = data + } + } + + body, err := json.Marshal(b) + if err != nil { + return nil, err + } + + return body, nil +} + +// JsonDecodeV01 takes in the byte representation of a version 0.1 structured json CloudEvent and returns a +// cloudevent.Event or an error if there are parsing errors. +func (e *Event) JsonDecodeV01(body []byte, raw map[string]json.RawMessage) error { + ec := EventContextV01{} + if err := json.Unmarshal(body, &ec); err != nil { + return err + } + + var data interface{} + if d, ok := raw["data"]; ok { + data = []byte(d) + } + + e.Context = &ec + e.Data = data + e.DataEncoded = data != nil + + return nil +} + +// JsonDecodeV02 takes in the byte representation of a version 0.2 structured json CloudEvent and returns a +// cloudevent.Event or an error if there are parsing errors. +func (e *Event) JsonDecodeV02(body []byte, raw map[string]json.RawMessage) error { + ec := EventContextV02{} + if err := json.Unmarshal(body, &ec); err != nil { + return err + } + + // TODO: could use reflection to get these. + delete(raw, "specversion") + delete(raw, "type") + delete(raw, "source") + delete(raw, "id") + delete(raw, "time") + delete(raw, "schemaurl") + delete(raw, "contenttype") + + var data interface{} + if d, ok := raw["data"]; ok { + data = []byte(d) + } + delete(raw, "data") + + if len(raw) > 0 { + extensions := make(map[string]interface{}, len(raw)) + for k, v := range raw { + extensions[k] = v + } + ec.Extensions = extensions + } + + e.Context = &ec + e.Data = data + e.DataEncoded = data != nil + + return nil +} + +// JsonDecodeV03 takes in the byte representation of a version 0.3 structured json CloudEvent and returns a +// cloudevent.Event or an error if there are parsing errors. +func (e *Event) JsonDecodeV03(body []byte, raw map[string]json.RawMessage) error { + ec := EventContextV03{} + if err := json.Unmarshal(body, &ec); err != nil { + return err + } + + // TODO: could use reflection to get these. + delete(raw, "specversion") + delete(raw, "type") + delete(raw, "source") + delete(raw, "subject") + delete(raw, "id") + delete(raw, "time") + delete(raw, "schemaurl") + delete(raw, "datacontenttype") + delete(raw, "datacontentencoding") + + var data interface{} + if d, ok := raw["data"]; ok { + data = []byte(d) + } + delete(raw, "data") + + if len(raw) > 0 { + extensions := make(map[string]interface{}, len(raw)) + for k, v := range raw { + extensions[k] = v + } + ec.Extensions = extensions + } + + e.Context = &ec + e.Data = data + e.DataEncoded = data != nil + + return nil +} + +func marshalEventLegacy(event interface{}) (map[string]json.RawMessage, error) { + b, err := json.Marshal(event) + if err != nil { + return nil, err + } + + brm := map[string]json.RawMessage{} + if err := json.Unmarshal(b, &brm); err != nil { + return nil, err + } + + return brm, nil +} + +func marshalEvent(event interface{}, extensions map[string]interface{}) (map[string]json.RawMessage, error) { + b, err := json.Marshal(event) + if err != nil { + return nil, err + } + + brm := map[string]json.RawMessage{} + if err := json.Unmarshal(b, &brm); err != nil { + return nil, err + } + + for k, v := range extensions { + vb, err := json.Marshal(v) + if err != nil { + return nil, err + } + // Don't overwrite spec keys. + if _, ok := brm[k]; !ok { + brm[k] = vb + } + } + + return brm, nil +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/event_observability.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/event_observability.go new file mode 100644 index 00000000000..bce63f5c600 --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/event_observability.go @@ -0,0 +1,94 @@ +package cloudevents + +import ( + "fmt" + + "github.com/cloudevents/sdk-go/pkg/cloudevents/observability" + "go.opencensus.io/stats" + "go.opencensus.io/stats/view" +) + +var ( + // EventMarshalLatencyMs measures the latency in milliseconds for the + // CloudEvents.Event marshal/unmarshalJSON methods. + EventMarshalLatencyMs = stats.Float64( + "cloudevents.io/sdk-go/event/json/latency", + "The latency in milliseconds of (un)marshalJSON methods for CloudEvents.Event.", + "ms") +) + +var ( + // LatencyView is an OpenCensus view that shows CloudEvents.Event (un)marshalJSON method latency. + EventMarshalLatencyView = &view.View{ + Name: "event/json/latency", + Measure: EventMarshalLatencyMs, + Description: "The distribution of latency inside of (un)marshalJSON methods for CloudEvents.Event.", + Aggregation: view.Distribution(0, .01, .1, 1, 10, 100, 1000, 10000), + TagKeys: observability.LatencyTags(), + } +) + +type observed int32 + +// Adheres to Observable +var _ observability.Observable = observed(0) + +const ( + reportMarshal observed = iota + reportUnmarshal +) + +// TraceName implements Observable.TraceName +func (o observed) TraceName() string { + switch o { + case reportMarshal: + return "cloudevents/event/marshaljson" + case reportUnmarshal: + return "cloudevents/event/unmarshaljson" + default: + return "cloudevents/event/unknwown" + } +} + +// MethodName implements Observable.MethodName +func (o observed) MethodName() string { + switch o { + case reportMarshal: + return "marshaljson" + case reportUnmarshal: + return "unmarshaljson" + default: + return "unknown" + } +} + +// LatencyMs implements Observable.LatencyMs +func (o observed) LatencyMs() *stats.Float64Measure { + return EventMarshalLatencyMs +} + +// eventJSONObserved is a wrapper to append version to observed. +type eventJSONObserved struct { + // Method + o observed + // Version + v string +} + +// Adheres to Observable +var _ observability.Observable = (*eventJSONObserved)(nil) + +// TraceName implements Observable.TraceName +func (c eventJSONObserved) TraceName() string { + return fmt.Sprintf("%s/%s", c.o.TraceName(), c.v) +} + +// MethodName implements Observable.MethodName +func (c eventJSONObserved) MethodName() string { + return fmt.Sprintf("%s/%s", c.o.MethodName(), c.v) +} + +// LatencyMs implements Observable.LatencyMs +func (c eventJSONObserved) LatencyMs() *stats.Float64Measure { + return c.o.LatencyMs() +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/event_reader.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/event_reader.go index 06fc95c838b..a5be4ecf820 100644 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/event_reader.go +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/event_reader.go @@ -8,58 +8,91 @@ var _ EventReader = (*Event)(nil) // SpecVersion implements EventReader.SpecVersion func (e Event) SpecVersion() string { - return e.Context.GetSpecVersion() + if e.Context != nil { + return e.Context.GetSpecVersion() + } + return "" } // Type implements EventReader.Type func (e Event) Type() string { - return e.Context.GetType() + if e.Context != nil { + return e.Context.GetType() + } + return "" } // Source implements EventReader.Source func (e Event) Source() string { - return e.Context.GetSource() + if e.Context != nil { + return e.Context.GetSource() + } + return "" } // Subject implements EventReader.Subject func (e Event) Subject() string { - return e.Context.GetSubject() + if e.Context != nil { + return e.Context.GetSubject() + } + return "" } // ID implements EventReader.ID func (e Event) ID() string { - return e.Context.GetID() + if e.Context != nil { + return e.Context.GetID() + } + return "" } // Time implements EventReader.Time func (e Event) Time() time.Time { - return e.Context.GetTime() + if e.Context != nil { + return e.Context.GetTime() + } + return time.Time{} } // SchemaURL implements EventReader.SchemaURL func (e Event) SchemaURL() string { - return e.Context.GetSchemaURL() + if e.Context != nil { + return e.Context.GetSchemaURL() + } + return "" } // DataContentType implements EventReader.DataContentType func (e Event) DataContentType() string { - return e.Context.GetDataContentType() + if e.Context != nil { + return e.Context.GetDataContentType() + } + return "" } // DataMediaType returns the parsed DataMediaType of the event. If parsing // fails, the empty string is returned. To retrieve the parsing error, use // `Context.GetDataMediaType` instead. func (e Event) DataMediaType() string { - mediaType, _ := e.Context.GetDataMediaType() - return mediaType + if e.Context != nil { + mediaType, _ := e.Context.GetDataMediaType() + return mediaType + } + return "" } // DataContentEncoding implements EventReader.DataContentEncoding func (e Event) DataContentEncoding() string { - return e.Context.GetDataContentEncoding() + if e.Context != nil { + return e.Context.GetDataContentEncoding() + } + return "" } // DataContentEncoding implements EventReader.DataContentEncoding func (e Event) Extensions() map[string]interface{} { - return e.Context.GetExtensions() + if e.Context != nil { + return e.Context.GetExtensions() + } + return map[string]interface{}(nil) } diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/eventcontext_v01.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/eventcontext_v01.go index b8f9ce570d5..d4f416dd12b 100644 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/eventcontext_v01.go +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/eventcontext_v01.go @@ -2,9 +2,10 @@ package cloudevents import ( "fmt" - "github.com/cloudevents/sdk-go/pkg/cloudevents/types" "sort" "strings" + + "github.com/cloudevents/sdk-go/pkg/cloudevents/types" ) const ( @@ -56,7 +57,7 @@ func (ec EventContextV01) ExtensionAs(name string, obj interface{}) error { return fmt.Errorf("invalid type for extension %q", name) } default: - return fmt.Errorf("unkown extension type %T", obj) + return fmt.Errorf("unknown extension type %T", obj) } } @@ -99,7 +100,7 @@ func (ec EventContextV01) AsV02() *EventContextV02 { // eventTypeVersion was retired in v0.2, so put it in an extension. if ec.EventTypeVersion != nil { - ret.SetExtension(EventTypeVersionKey, *ec.EventTypeVersion) + _ = ret.SetExtension(EventTypeVersionKey, *ec.EventTypeVersion) } if ec.Extensions != nil { for k, v := range ec.Extensions { diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/eventcontext_v01_writer.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/eventcontext_v01_writer.go index f3594815614..7c196d939e0 100644 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/eventcontext_v01_writer.go +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/eventcontext_v01_writer.go @@ -3,10 +3,11 @@ package cloudevents import ( "errors" "fmt" - "github.com/cloudevents/sdk-go/pkg/cloudevents/types" "net/url" "strings" "time" + + "github.com/cloudevents/sdk-go/pkg/cloudevents/types" ) // Adhere to EventContextWriter diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/eventcontext_v02.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/eventcontext_v02.go index dd05afb556c..ed4affc388f 100644 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/eventcontext_v02.go +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/eventcontext_v02.go @@ -3,9 +3,10 @@ package cloudevents import ( "encoding/json" "fmt" - "github.com/cloudevents/sdk-go/pkg/cloudevents/types" "sort" "strings" + + "github.com/cloudevents/sdk-go/pkg/cloudevents/types" ) const ( @@ -65,7 +66,7 @@ func (ec EventContextV02) ExtensionAs(name string, obj interface{}) error { return fmt.Errorf("invalid type for extension %q", name) } default: - return fmt.Errorf("unkown extension type %T", obj) + return fmt.Errorf("unknown extension type %T", obj) } } diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/eventcontext_v02_writer.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/eventcontext_v02_writer.go index a67cff01e66..8935e93d79c 100644 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/eventcontext_v02_writer.go +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/eventcontext_v02_writer.go @@ -3,10 +3,11 @@ package cloudevents import ( "errors" "fmt" - "github.com/cloudevents/sdk-go/pkg/cloudevents/types" "net/url" "strings" "time" + + "github.com/cloudevents/sdk-go/pkg/cloudevents/types" ) // Adhere to EventContextWriter diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/eventcontext_v03.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/eventcontext_v03.go index c447b08c846..5f97c043e41 100644 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/eventcontext_v03.go +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/eventcontext_v03.go @@ -3,9 +3,10 @@ package cloudevents import ( "encoding/json" "fmt" - "github.com/cloudevents/sdk-go/pkg/cloudevents/types" "sort" "strings" + + "github.com/cloudevents/sdk-go/pkg/cloudevents/types" ) // WIP: AS OF FEB 19, 2019 @@ -72,7 +73,7 @@ func (ec EventContextV03) ExtensionAs(name string, obj interface{}) error { return fmt.Errorf("invalid type for extension %q", name) } default: - return fmt.Errorf("unkown extension type %T", obj) + return fmt.Errorf("unknown extension type %T", obj) } } @@ -114,11 +115,11 @@ func (ec EventContextV03) AsV02() *EventContextV02 { } // Subject was introduced in 0.3, so put it in an extension for 0.2. if ec.Subject != nil { - ret.SetExtension(SubjectKey, *ec.Subject) + _ = ret.SetExtension(SubjectKey, *ec.Subject) } // DataContentEncoding was introduced in 0.3, so put it in an extension for 0.2. if ec.DataContentEncoding != nil { - ret.SetExtension(DataContentEncodingKey, *ec.DataContentEncoding) + _ = ret.SetExtension(DataContentEncodingKey, *ec.DataContentEncoding) } if ec.Extensions != nil { for k, v := range ec.Extensions { diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/eventcontext_v03_writer.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/eventcontext_v03_writer.go index 16a06803043..9370d2a3d94 100644 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/eventcontext_v03_writer.go +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/eventcontext_v03_writer.go @@ -3,10 +3,11 @@ package cloudevents import ( "errors" "fmt" - "github.com/cloudevents/sdk-go/pkg/cloudevents/types" "net/url" "strings" "time" + + "github.com/cloudevents/sdk-go/pkg/cloudevents/types" ) // Adhere to EventContextWriter diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/observability/observer.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/observability/observer.go index a7936f4b987..76e7b12fda2 100644 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/observability/observer.go +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/observability/observer.go @@ -27,12 +27,11 @@ type Reporter interface { } type reporter struct { - ctx context.Context - span *trace.Span - on Observable - start time.Time - measure stats.Measure - once sync.Once + ctx context.Context + span *trace.Span + on Observable + start time.Time + once sync.Once } // All tags used for Latency measurements. @@ -40,10 +39,26 @@ func LatencyTags() []tag.Key { return []tag.Key{KeyMethod, KeyResult} } +var ( + // Tracing is disabled by default. It is very useful for profiling an + // application. + tracingEnabled = false +) + +// EnableTracing allows control over if tracing is enabled for the sdk. +// Default is false. This applies to all of the +// `github.com/cloudevents/sdk-go/...` package. +func EnableTracing(enabled bool) { + tracingEnabled = enabled +} + // NewReporter creates and returns a reporter wrapping the provided Observable, // and injects a trace span into the context. func NewReporter(ctx context.Context, on Observable) (context.Context, Reporter) { - ctx, span := trace.StartSpan(ctx, on.TraceName()) + var span *trace.Span + if tracingEnabled { + ctx, span = trace.StartSpan(ctx, on.TraceName()) + } r := &reporter{ ctx: ctx, on: on, @@ -65,7 +80,9 @@ func (r *reporter) tagMethod() { func (r *reporter) record() { ms := float64(time.Since(r.start) / time.Millisecond) stats.Record(r.ctx, r.on.LatencyMs().M(ms)) - r.span.End() + if r.span != nil { + r.span.End() + } } // Error records the result as an error. diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/codec.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/codec.go index 6564dc9e502..091064c9155 100644 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/codec.go +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/codec.go @@ -1,10 +1,35 @@ package transport -import "github.com/cloudevents/sdk-go/pkg/cloudevents" +import ( + "context" + "fmt" + + "github.com/cloudevents/sdk-go/pkg/cloudevents" +) // Codec is the interface for transport codecs to convert between transport // specific payloads and the Message interface. type Codec interface { - Encode(cloudevents.Event) (Message, error) - Decode(Message) (*cloudevents.Event, error) + Encode(context.Context, cloudevents.Event) (Message, error) + Decode(context.Context, Message) (*cloudevents.Event, error) +} + +// ErrMessageEncodingUnknown is an error produced when the encoding for an incoming +// message can not be understood. +type ErrMessageEncodingUnknown struct { + codec string + transport string +} + +// NewErrMessageEncodingUnknown makes a new ErrMessageEncodingUnknown. +func NewErrMessageEncodingUnknown(codec, transport string) *ErrMessageEncodingUnknown { + return &ErrMessageEncodingUnknown{ + codec: codec, + transport: transport, + } +} + +// Error implements error.Error +func (e *ErrMessageEncodingUnknown) Error() string { + return fmt.Sprintf("message encoding unknown for %s codec on %s transport", e.codec, e.transport) } diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/doc.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/doc.go index b93cd60a8cf..c2cbadde0d2 100644 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/doc.go +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/doc.go @@ -1,5 +1,12 @@ /* -Package transport is the toplevel package to define interfaces that the client and codec packages use to decouple from -the transport implementations. + +Package transport defines interfaces to decouple the client package +from transport implementations. + +Most event sender and receiver applications should not use this +package, they should use the client package. This package is for +infrastructure developers implementing new transports, or intermediary +components like importers, channels or brokers. + */ package transport diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/error.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/error.go new file mode 100644 index 00000000000..95e0f342e62 --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/error.go @@ -0,0 +1,30 @@ +package transport + +import "fmt" + +// ErrTransportMessageConversion is an error produced when the transport +// message can not be converted. +type ErrTransportMessageConversion struct { + fatal bool + transport string + message string +} + +// NewErrMessageEncodingUnknown makes a new ErrMessageEncodingUnknown. +func NewErrTransportMessageConversion(transport, message string, fatal bool) *ErrTransportMessageConversion { + return &ErrTransportMessageConversion{ + transport: transport, + message: message, + fatal: fatal, + } +} + +// IsFatal reports if this error should be considered fatal. +func (e *ErrTransportMessageConversion) IsFatal() bool { + return e.fatal +} + +// Error implements error.Error +func (e *ErrTransportMessageConversion) Error() string { + return fmt.Sprintf("transport %s failed to convert message: %s", e.transport, e.message) +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/codec.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/codec.go index 513c923f679..6774111e52d 100644 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/codec.go +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/codec.go @@ -1,7 +1,9 @@ package http import ( + "context" "fmt" + "github.com/cloudevents/sdk-go/pkg/cloudevents" "github.com/cloudevents/sdk-go/pkg/cloudevents/transport" ) @@ -24,42 +26,12 @@ type Codec struct { // Adheres to Codec var _ transport.Codec = (*Codec)(nil) -// DefaultBinaryEncodingSelectionStrategy implements a selection process for -// which binary encoding to use based on spec version of the event. -func DefaultBinaryEncodingSelectionStrategy(e cloudevents.Event) Encoding { - switch e.SpecVersion() { - case cloudevents.CloudEventsVersionV01: - return BinaryV01 - case cloudevents.CloudEventsVersionV02: - return BinaryV02 - case cloudevents.CloudEventsVersionV03: - return BinaryV03 - } - // Unknown version, return Default. - return Default -} - -// DefaultStructuredEncodingSelectionStrategy implements a selection process -// for which structured encoding to use based on spec version of the event. -func DefaultStructuredEncodingSelectionStrategy(e cloudevents.Event) Encoding { - switch e.SpecVersion() { - case cloudevents.CloudEventsVersionV01: - return StructuredV01 - case cloudevents.CloudEventsVersionV02: - return StructuredV02 - case cloudevents.CloudEventsVersionV03: - return StructuredV03 - } - // Unknown version, return Default. - return Default -} - // Encode encodes the provided event into a transport message. -func (c *Codec) Encode(e cloudevents.Event) (transport.Message, error) { +func (c *Codec) Encode(ctx context.Context, e cloudevents.Event) (transport.Message, error) { encoding := c.Encoding if encoding == Default && c.DefaultEncodingSelectionFn != nil { - encoding = c.DefaultEncodingSelectionFn(e) + encoding = c.DefaultEncodingSelectionFn(ctx, e) } switch encoding { @@ -71,66 +43,60 @@ func (c *Codec) Encode(e cloudevents.Event) (transport.Message, error) { if c.v01 == nil { c.v01 = &CodecV01{Encoding: encoding} } - return c.v01.Encode(e) + return c.v01.Encode(ctx, e) case BinaryV02: fallthrough case StructuredV02: if c.v02 == nil { c.v02 = &CodecV02{Encoding: encoding} } - return c.v02.Encode(e) + return c.v02.Encode(ctx, e) case BinaryV03: fallthrough case StructuredV03: if c.v03 == nil { c.v03 = &CodecV03{Encoding: encoding} } - return c.v03.Encode(e) + return c.v03.Encode(ctx, e) default: return nil, fmt.Errorf("unknown encoding: %s", encoding) } } // Decode converts a provided transport message into an Event, or error. -func (c *Codec) Decode(msg transport.Message) (*cloudevents.Event, error) { - switch c.inspectEncoding(msg) { - case BinaryV01: - fallthrough - case StructuredV01: +func (c *Codec) Decode(ctx context.Context, msg transport.Message) (*cloudevents.Event, error) { + switch c.inspectEncoding(ctx, msg) { + case BinaryV01, StructuredV01: if c.v01 == nil { c.v01 = &CodecV01{Encoding: c.Encoding} } - if event, err := c.v01.Decode(msg); err != nil { + if event, err := c.v01.Decode(ctx, msg); err != nil { return nil, err } else { return c.convertEvent(event), nil } - case BinaryV02: - fallthrough - case StructuredV02: + + case BinaryV02, StructuredV02: if c.v02 == nil { c.v02 = &CodecV02{Encoding: c.Encoding} } - if event, err := c.v02.Decode(msg); err != nil { + if event, err := c.v02.Decode(ctx, msg); err != nil { return nil, err } else { return c.convertEvent(event), nil } - case BinaryV03: - fallthrough - case StructuredV03: - fallthrough - case BatchedV03: + + case BinaryV03, StructuredV03, BatchedV03: if c.v03 == nil { c.v03 = &CodecV03{Encoding: c.Encoding} } - if event, err := c.v03.Decode(msg); err != nil { + if event, err := c.v03.Decode(ctx, msg); err != nil { return nil, err } else { return c.convertEvent(event), nil } default: - return nil, fmt.Errorf("unknown encoding") + return nil, transport.NewErrMessageEncodingUnknown("wrapper", TransportName) } } @@ -148,8 +114,8 @@ func (c *Codec) convertEvent(event *cloudevents.Event) *cloudevents.Event { if c.v01 == nil { c.v01 = &CodecV01{Encoding: c.Encoding} } - ctx := event.Context.AsV01() - event.Context = ctx + ca := event.Context.AsV01() + event.Context = ca return event case BinaryV02: fallthrough @@ -157,8 +123,8 @@ func (c *Codec) convertEvent(event *cloudevents.Event) *cloudevents.Event { if c.v02 == nil { c.v02 = &CodecV02{Encoding: c.Encoding} } - ctx := event.Context.AsV02() - event.Context = ctx + ca := event.Context.AsV02() + event.Context = ca return event case BinaryV03: fallthrough @@ -168,21 +134,21 @@ func (c *Codec) convertEvent(event *cloudevents.Event) *cloudevents.Event { if c.v03 == nil { c.v03 = &CodecV03{Encoding: c.Encoding} } - ctx := event.Context.AsV03() - event.Context = ctx + ca := event.Context.AsV03() + event.Context = ca return event default: return nil } } -func (c *Codec) inspectEncoding(msg transport.Message) Encoding { +func (c *Codec) inspectEncoding(ctx context.Context, msg transport.Message) Encoding { // TODO: there should be a better way to make the version codecs on demand. if c.v01 == nil { c.v01 = &CodecV01{Encoding: c.Encoding} } // Try v0.1 first. - encoding := c.v01.inspectEncoding(msg) + encoding := c.v01.inspectEncoding(ctx, msg) if encoding != Unknown { return encoding } @@ -191,7 +157,7 @@ func (c *Codec) inspectEncoding(msg transport.Message) Encoding { c.v02 = &CodecV02{Encoding: c.Encoding} } // Try v0.2. - encoding = c.v02.inspectEncoding(msg) + encoding = c.v02.inspectEncoding(ctx, msg) if encoding != Unknown { return encoding } @@ -200,7 +166,7 @@ func (c *Codec) inspectEncoding(msg transport.Message) Encoding { c.v03 = &CodecV03{Encoding: c.Encoding} } // Try v0.3. - encoding = c.v03.inspectEncoding(msg) + encoding = c.v03.inspectEncoding(ctx, msg) if encoding != Unknown { return encoding } diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/codec_structured.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/codec_structured.go new file mode 100644 index 00000000000..098cb5a1569 --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/codec_structured.go @@ -0,0 +1,44 @@ +package http + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + + "github.com/cloudevents/sdk-go/pkg/cloudevents" + "github.com/cloudevents/sdk-go/pkg/cloudevents/transport" +) + +// CodecStructured represents an structured http transport codec for all versions. +// Intended to be used as a base class. +type CodecStructured struct { + Encoding Encoding +} + +func (v CodecStructured) encodeStructured(ctx context.Context, e cloudevents.Event) (transport.Message, error) { + header := http.Header{} + header.Set("Content-Type", cloudevents.ApplicationCloudEventsJSON) + + body, err := json.Marshal(e) + if err != nil { + return nil, err + } + + msg := &Message{ + Header: header, + Body: body, + } + + return msg, nil +} + +func (v CodecStructured) decodeStructured(ctx context.Context, version string, msg transport.Message) (*cloudevents.Event, error) { + m, ok := msg.(*Message) + if !ok { + return nil, fmt.Errorf("failed to convert transport.Message to http.Message") + } + event := cloudevents.New(version) + err := json.Unmarshal(m.Body, &event) + return &event, err +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/codec_v01.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/codec_v01.go index 93eb3539c9d..e414c090a4b 100644 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/codec_v01.go +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/codec_v01.go @@ -4,18 +4,20 @@ import ( "context" "encoding/json" "fmt" + "net/http" + "net/textproto" + "strings" + "github.com/cloudevents/sdk-go/pkg/cloudevents" - "github.com/cloudevents/sdk-go/pkg/cloudevents/codec" "github.com/cloudevents/sdk-go/pkg/cloudevents/observability" "github.com/cloudevents/sdk-go/pkg/cloudevents/transport" "github.com/cloudevents/sdk-go/pkg/cloudevents/types" - "net/http" - "net/textproto" - "strings" ) // CodecV01 represents a http transport codec that uses CloudEvents spec v0.3 type CodecV01 struct { + CodecStructured + Encoding Encoding } @@ -23,10 +25,10 @@ type CodecV01 struct { var _ transport.Codec = (*CodecV01)(nil) // Encode implements Codec.Encode -func (v CodecV01) Encode(e cloudevents.Event) (transport.Message, error) { +func (v CodecV01) Encode(ctx context.Context, e cloudevents.Event) (transport.Message, error) { // TODO: wire context _, r := observability.NewReporter(context.Background(), CodecObserved{o: reportEncode, c: v.Encoding.Codec()}) - m, err := v.obsEncode(e) + m, err := v.obsEncode(ctx, e) if err != nil { r.Error() } else { @@ -35,24 +37,24 @@ func (v CodecV01) Encode(e cloudevents.Event) (transport.Message, error) { return m, err } -func (v CodecV01) obsEncode(e cloudevents.Event) (transport.Message, error) { +func (v CodecV01) obsEncode(ctx context.Context, e cloudevents.Event) (transport.Message, error) { switch v.Encoding { case Default: fallthrough case BinaryV01: - return v.encodeBinary(e) + return v.encodeBinary(ctx, e) case StructuredV01: - return v.encodeStructured(e) + return v.encodeStructured(ctx, e) default: return nil, fmt.Errorf("unknown encoding: %d", v.Encoding) } } // Decode implements Codec.Decode -func (v CodecV01) Decode(msg transport.Message) (*cloudevents.Event, error) { +func (v CodecV01) Decode(ctx context.Context, msg transport.Message) (*cloudevents.Event, error) { // TODO: wire context - _, r := observability.NewReporter(context.Background(), CodecObserved{o: reportDecode, c: v.inspectEncoding(msg).Codec()}) // TODO: inspectEncoding is not free. - e, err := v.obsDecode(msg) + _, r := observability.NewReporter(ctx, CodecObserved{o: reportDecode, c: v.inspectEncoding(ctx, msg).Codec()}) // TODO: inspectEncoding is not free. + e, err := v.obsDecode(ctx, msg) if err != nil { r.Error() } else { @@ -61,18 +63,18 @@ func (v CodecV01) Decode(msg transport.Message) (*cloudevents.Event, error) { return e, err } -func (v CodecV01) obsDecode(msg transport.Message) (*cloudevents.Event, error) { - switch v.inspectEncoding(msg) { +func (v CodecV01) obsDecode(ctx context.Context, msg transport.Message) (*cloudevents.Event, error) { + switch v.inspectEncoding(ctx, msg) { case BinaryV01: - return v.decodeBinary(msg) + return v.decodeBinary(ctx, msg) case StructuredV01: - return v.decodeStructured(msg) + return v.decodeStructured(ctx, cloudevents.CloudEventsVersionV01, msg) default: - return nil, fmt.Errorf("unknown encoding") + return nil, transport.NewErrMessageEncodingUnknown("v01", TransportName) } } -func (v CodecV01) encodeBinary(e cloudevents.Event) (transport.Message, error) { +func (v CodecV01) encodeBinary(ctx context.Context, e cloudevents.Event) (transport.Message, error) { header, err := v.toHeaders(e.Context.AsV01()) if err != nil { return nil, err @@ -129,29 +131,12 @@ func (v CodecV01) toHeaders(ec *cloudevents.EventContextV01) (http.Header, error return h, nil } -func (v CodecV01) encodeStructured(e cloudevents.Event) (transport.Message, error) { - header := http.Header{} - header.Set("Content-Type", cloudevents.ApplicationCloudEventsJSON) - - body, err := codec.JsonEncodeV01(e) - if err != nil { - return nil, err - } - - msg := &Message{ - Header: header, - Body: body, - } - - return msg, nil -} - -func (v CodecV01) decodeBinary(msg transport.Message) (*cloudevents.Event, error) { +func (v CodecV01) decodeBinary(ctx context.Context, msg transport.Message) (*cloudevents.Event, error) { m, ok := msg.(*Message) if !ok { return nil, fmt.Errorf("failed to convert transport.Message to http.Message") } - ctx, err := v.fromHeaders(m.Header) + ca, err := v.fromHeaders(m.Header) if err != nil { return nil, err } @@ -160,9 +145,9 @@ func (v CodecV01) decodeBinary(msg transport.Message) (*cloudevents.Event, error body = m.Body } return &cloudevents.Event{ - Context: &ctx, + Context: &ca, Data: body, - DataEncoded: true, + DataEncoded: body != nil, }, nil } @@ -219,15 +204,7 @@ func (v CodecV01) fromHeaders(h http.Header) (cloudevents.EventContextV01, error return ec, nil } -func (v CodecV01) decodeStructured(msg transport.Message) (*cloudevents.Event, error) { - m, ok := msg.(*Message) - if !ok { - return nil, fmt.Errorf("failed to convert transport.Message to http.Message") - } - return codec.JsonDecodeV01(m.Body) -} - -func (v CodecV01) inspectEncoding(msg transport.Message) Encoding { +func (v CodecV01) inspectEncoding(ctx context.Context, msg transport.Message) Encoding { version := msg.CloudEventsVersion() if version != cloudevents.CloudEventsVersionV01 { return Unknown diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/codec_v02.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/codec_v02.go index 8df9432113c..939e60204ed 100644 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/codec_v02.go +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/codec_v02.go @@ -4,18 +4,20 @@ import ( "context" "encoding/json" "fmt" + "net/http" + "net/textproto" + "strings" + "github.com/cloudevents/sdk-go/pkg/cloudevents" - "github.com/cloudevents/sdk-go/pkg/cloudevents/codec" "github.com/cloudevents/sdk-go/pkg/cloudevents/observability" "github.com/cloudevents/sdk-go/pkg/cloudevents/transport" "github.com/cloudevents/sdk-go/pkg/cloudevents/types" - "net/http" - "net/textproto" - "strings" ) // CodecV02 represents a http transport codec that uses CloudEvents spec v0.2 type CodecV02 struct { + CodecStructured + Encoding Encoding } @@ -23,10 +25,10 @@ type CodecV02 struct { var _ transport.Codec = (*CodecV02)(nil) // Encode implements Codec.Encode -func (v CodecV02) Encode(e cloudevents.Event) (transport.Message, error) { +func (v CodecV02) Encode(ctx context.Context, e cloudevents.Event) (transport.Message, error) { // TODO: wire context - _, r := observability.NewReporter(context.Background(), CodecObserved{o: reportEncode, c: v.Encoding.Codec()}) - m, err := v.obsEncode(e) + _, r := observability.NewReporter(ctx, CodecObserved{o: reportEncode, c: v.Encoding.Codec()}) + m, err := v.obsEncode(ctx, e) if err != nil { r.Error() } else { @@ -35,24 +37,24 @@ func (v CodecV02) Encode(e cloudevents.Event) (transport.Message, error) { return m, err } -func (v CodecV02) obsEncode(e cloudevents.Event) (transport.Message, error) { +func (v CodecV02) obsEncode(ctx context.Context, e cloudevents.Event) (transport.Message, error) { switch v.Encoding { case Default: fallthrough case BinaryV02: - return v.encodeBinary(e) + return v.encodeBinary(ctx, e) case StructuredV02: - return v.encodeStructured(e) + return v.encodeStructured(ctx, e) default: return nil, fmt.Errorf("unknown encoding: %d", v.Encoding) } } // Decode implements Codec.Decode -func (v CodecV02) Decode(msg transport.Message) (*cloudevents.Event, error) { +func (v CodecV02) Decode(ctx context.Context, msg transport.Message) (*cloudevents.Event, error) { // TODO: wire context - _, r := observability.NewReporter(context.Background(), CodecObserved{o: reportDecode, c: v.inspectEncoding(msg).Codec()}) // TODO: inspectEncoding is not free. - e, err := v.obsDecode(msg) + _, r := observability.NewReporter(ctx, CodecObserved{o: reportDecode, c: v.inspectEncoding(ctx, msg).Codec()}) // TODO: inspectEncoding is not free. + e, err := v.obsDecode(ctx, msg) if err != nil { r.Error() } else { @@ -61,18 +63,18 @@ func (v CodecV02) Decode(msg transport.Message) (*cloudevents.Event, error) { return e, err } -func (v CodecV02) obsDecode(msg transport.Message) (*cloudevents.Event, error) { - switch v.inspectEncoding(msg) { +func (v CodecV02) obsDecode(ctx context.Context, msg transport.Message) (*cloudevents.Event, error) { + switch v.inspectEncoding(ctx, msg) { case BinaryV02: - return v.decodeBinary(msg) + return v.decodeBinary(ctx, msg) case StructuredV02: - return v.decodeStructured(msg) + return v.decodeStructured(ctx, cloudevents.CloudEventsVersionV02, msg) default: - return nil, fmt.Errorf("unknown encoding") + return nil, transport.NewErrMessageEncodingUnknown("v02", TransportName) } } -func (v CodecV02) encodeBinary(e cloudevents.Event) (transport.Message, error) { +func (v CodecV02) encodeBinary(ctx context.Context, e cloudevents.Event) (transport.Message, error) { header, err := v.toHeaders(e.Context.AsV02()) if err != nil { return nil, err @@ -133,29 +135,12 @@ func (v CodecV02) toHeaders(ec *cloudevents.EventContextV02) (http.Header, error return h, nil } -func (v CodecV02) encodeStructured(e cloudevents.Event) (transport.Message, error) { - header := http.Header{} - header.Set("Content-Type", "application/cloudevents+json") - - body, err := codec.JsonEncodeV02(e) - if err != nil { - return nil, err - } - - msg := &Message{ - Header: header, - Body: body, - } - - return msg, nil -} - -func (v CodecV02) decodeBinary(msg transport.Message) (*cloudevents.Event, error) { +func (v CodecV02) decodeBinary(ctx context.Context, msg transport.Message) (*cloudevents.Event, error) { m, ok := msg.(*Message) if !ok { return nil, fmt.Errorf("failed to convert transport.Message to http.Message") } - ctx, err := v.fromHeaders(m.Header) + ca, err := v.fromHeaders(m.Header) if err != nil { return nil, err } @@ -164,9 +149,9 @@ func (v CodecV02) decodeBinary(msg transport.Message) (*cloudevents.Event, error body = m.Body } return &cloudevents.Event{ - Context: &ctx, + Context: &ca, Data: body, - DataEncoded: true, + DataEncoded: body != nil, }, nil } @@ -250,16 +235,7 @@ func (v CodecV02) fromHeaders(h http.Header) (cloudevents.EventContextV02, error return ec, nil } -func (v CodecV02) decodeStructured(msg transport.Message) (*cloudevents.Event, error) { - m, ok := msg.(*Message) - if !ok { - return nil, fmt.Errorf("failed to convert transport.Message to http.Message") - } - - return codec.JsonDecodeV02(m.Body) -} - -func (v CodecV02) inspectEncoding(msg transport.Message) Encoding { +func (v CodecV02) inspectEncoding(ctx context.Context, msg transport.Message) Encoding { version := msg.CloudEventsVersion() if version != cloudevents.CloudEventsVersionV02 { return Unknown diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/codec_v03.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/codec_v03.go index c88ec4009d9..43641193366 100644 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/codec_v03.go +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/codec_v03.go @@ -9,7 +9,6 @@ import ( "strings" "github.com/cloudevents/sdk-go/pkg/cloudevents" - "github.com/cloudevents/sdk-go/pkg/cloudevents/codec" "github.com/cloudevents/sdk-go/pkg/cloudevents/observability" "github.com/cloudevents/sdk-go/pkg/cloudevents/transport" "github.com/cloudevents/sdk-go/pkg/cloudevents/types" @@ -17,6 +16,8 @@ import ( // CodecV03 represents a http transport codec that uses CloudEvents spec v0.3 type CodecV03 struct { + CodecStructured + Encoding Encoding } @@ -24,10 +25,10 @@ type CodecV03 struct { var _ transport.Codec = (*CodecV03)(nil) // Encode implements Codec.Encode -func (v CodecV03) Encode(e cloudevents.Event) (transport.Message, error) { +func (v CodecV03) Encode(ctx context.Context, e cloudevents.Event) (transport.Message, error) { // TODO: wire context - _, r := observability.NewReporter(context.Background(), CodecObserved{o: reportEncode, c: v.Encoding.Codec()}) - m, err := v.obsEncode(e) + _, r := observability.NewReporter(ctx, CodecObserved{o: reportEncode, c: v.Encoding.Codec()}) + m, err := v.obsEncode(ctx, e) if err != nil { r.Error() } else { @@ -36,14 +37,14 @@ func (v CodecV03) Encode(e cloudevents.Event) (transport.Message, error) { return m, err } -func (v CodecV03) obsEncode(e cloudevents.Event) (transport.Message, error) { +func (v CodecV03) obsEncode(ctx context.Context, e cloudevents.Event) (transport.Message, error) { switch v.Encoding { case Default: fallthrough case BinaryV03: - return v.encodeBinary(e) + return v.encodeBinary(ctx, e) case StructuredV03: - return v.encodeStructured(e) + return v.encodeStructured(ctx, e) case BatchedV03: return nil, fmt.Errorf("not implemented") default: @@ -52,10 +53,10 @@ func (v CodecV03) obsEncode(e cloudevents.Event) (transport.Message, error) { } // Decode implements Codec.Decode -func (v CodecV03) Decode(msg transport.Message) (*cloudevents.Event, error) { +func (v CodecV03) Decode(ctx context.Context, msg transport.Message) (*cloudevents.Event, error) { // TODO: wire context - _, r := observability.NewReporter(context.Background(), CodecObserved{o: reportDecode, c: v.inspectEncoding(msg).Codec()}) // TODO: inspectEncoding is not free. - e, err := v.obsDecode(msg) + _, r := observability.NewReporter(ctx, CodecObserved{o: reportDecode, c: v.inspectEncoding(ctx, msg).Codec()}) // TODO: inspectEncoding is not free. + e, err := v.obsDecode(ctx, msg) if err != nil { r.Error() } else { @@ -64,20 +65,20 @@ func (v CodecV03) Decode(msg transport.Message) (*cloudevents.Event, error) { return e, err } -func (v CodecV03) obsDecode(msg transport.Message) (*cloudevents.Event, error) { - switch v.inspectEncoding(msg) { +func (v CodecV03) obsDecode(ctx context.Context, msg transport.Message) (*cloudevents.Event, error) { + switch v.inspectEncoding(ctx, msg) { case BinaryV03: - return v.decodeBinary(msg) + return v.decodeBinary(ctx, msg) case StructuredV03: - return v.decodeStructured(msg) + return v.decodeStructured(ctx, cloudevents.CloudEventsVersionV03, msg) case BatchedV03: return nil, fmt.Errorf("not implemented") default: - return nil, fmt.Errorf("unknown encoding") + return nil, transport.NewErrMessageEncodingUnknown("v03", TransportName) } } -func (v CodecV03) encodeBinary(e cloudevents.Event) (transport.Message, error) { +func (v CodecV03) encodeBinary(ctx context.Context, e cloudevents.Event) (transport.Message, error) { header, err := v.toHeaders(e.Context.AsV03()) if err != nil { return nil, err @@ -146,29 +147,12 @@ func (v CodecV03) toHeaders(ec *cloudevents.EventContextV03) (http.Header, error return h, nil } -func (v CodecV03) encodeStructured(e cloudevents.Event) (transport.Message, error) { - header := http.Header{} - header.Set("Content-Type", "application/cloudevents+json") - - body, err := codec.JsonEncodeV03(e) - if err != nil { - return nil, err - } - - msg := &Message{ - Header: header, - Body: body, - } - - return msg, nil -} - -func (v CodecV03) decodeBinary(msg transport.Message) (*cloudevents.Event, error) { +func (v CodecV03) decodeBinary(ctx context.Context, msg transport.Message) (*cloudevents.Event, error) { m, ok := msg.(*Message) if !ok { return nil, fmt.Errorf("failed to convert transport.Message to http.Message") } - ctx, err := v.fromHeaders(m.Header) + ca, err := v.fromHeaders(m.Header) if err != nil { return nil, err } @@ -177,9 +161,9 @@ func (v CodecV03) decodeBinary(msg transport.Message) (*cloudevents.Event, error body = m.Body } return &cloudevents.Event{ - Context: &ctx, + Context: &ca, Data: body, - DataEncoded: true, + DataEncoded: body != nil, }, nil } @@ -275,16 +259,7 @@ func (v CodecV03) fromHeaders(h http.Header) (cloudevents.EventContextV03, error return ec, nil } -func (v CodecV03) decodeStructured(msg transport.Message) (*cloudevents.Event, error) { - m, ok := msg.(*Message) - if !ok { - return nil, fmt.Errorf("failed to convert transport.Message to http.Message") - } - - return codec.JsonDecodeV03(m.Body) -} - -func (v CodecV03) inspectEncoding(msg transport.Message) Encoding { +func (v CodecV03) inspectEncoding(ctx context.Context, msg transport.Message) Encoding { version := msg.CloudEventsVersion() if version != cloudevents.CloudEventsVersionV03 { return Unknown diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/context.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/context.go index 9511ec6d239..cf8b8510d7a 100644 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/context.go +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/context.go @@ -4,15 +4,18 @@ import ( "context" "fmt" "net/http" + "net/url" + "strconv" "strings" ) // TransportContext allows a Receiver to understand the context of a request. type TransportContext struct { - URI string - Host string - Method string - Header http.Header + URI string + Host string + Method string + Header http.Header + StatusCode int // IgnoreHeaderPrefixes controls what comes back from AttendToHeaders. // AttendToHeaders controls what is output for .String() @@ -36,6 +39,22 @@ func NewTransportContext(req *http.Request) TransportContext { return *tx } +// NewTransportContextFromResponse creates a new TransportContext from a http.Response. +// If `res` is nil, it returns a context with a http.StatusInternalServerError status code. +func NewTransportContextFromResponse(res *http.Response) TransportContext { + var tx *TransportContext + if res != nil { + tx = &TransportContext{ + Header: res.Header, + StatusCode: res.StatusCode, + } + } else { + tx = &TransportContext{StatusCode: http.StatusInternalServerError} + } + tx.AddIgnoreHeaderPrefix("accept-encoding", "user-agent", "connection", "content-type") + return *tx +} + // TransportResponseContext allows a Receiver response with http transport specific fields. type TransportResponseContext struct { // Header will be merged with the response headers. @@ -85,6 +104,10 @@ func (tx TransportContext) String() string { b.WriteString(" Method: " + tx.Method + "\n") } + if tx.StatusCode != 0 { + b.WriteString(" StatusCode: " + strconv.Itoa(tx.StatusCode) + "\n") + } + if tx.Header != nil && len(tx.Header) > 0 { b.WriteString(" Header:\n") for _, k := range tx.AttendToHeaders() { @@ -145,7 +168,7 @@ func ContextWithHeader(ctx context.Context, key, value string) context.Context { return context.WithValue(ctx, headerKey, header) } -// HeaderFrom extracts the header oject in the given context. Always returns a non-nil Header. +// HeaderFrom extracts the header object in the given context. Always returns a non-nil Header. func HeaderFrom(ctx context.Context) http.Header { ch := http.Header{} header := ctx.Value(headerKey) @@ -156,3 +179,29 @@ func HeaderFrom(ctx context.Context) http.Header { } return ch } + +// Opaque key type used to store long poll target. +type longPollTargetKeyType struct{} + +var longPollTargetKey = longPollTargetKeyType{} + +// WithLongPollTarget returns a new context with the given long poll target. +// `target` should be a full URL and will be injected into the long polling +// http request within StartReceiver. +func ContextWithLongPollTarget(ctx context.Context, target string) context.Context { + return context.WithValue(ctx, longPollTargetKey, target) +} + +// LongPollTargetFrom looks in the given context and returns `target` as a +// parsed url if found and valid, otherwise nil. +func LongPollTargetFrom(ctx context.Context) *url.URL { + c := ctx.Value(longPollTargetKey) + if c != nil { + if s, ok := c.(string); ok && s != "" { + if target, err := url.Parse(s); err == nil { + return target + } + } + } + return nil +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/encoding.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/encoding.go index ca62e5b5df9..a0d80c49485 100644 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/encoding.go +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/encoding.go @@ -1,8 +1,17 @@ package http +import ( + "context" + + "github.com/cloudevents/sdk-go/pkg/cloudevents" + cecontext "github.com/cloudevents/sdk-go/pkg/cloudevents/context" +) + // Encoding to use for HTTP transport. type Encoding int32 +type EncodingSelector func(context.Context, cloudevents.Event) Encoding + const ( // Default Default Encoding = iota @@ -22,8 +31,57 @@ const ( BatchedV03 // Unknown is unknown. Unknown + + // Binary is used for Context Based Encoding Selections to use the + // DefaultBinaryEncodingSelectionStrategy + Binary = "binary" + + // Structured is used for Context Based Encoding Selections to use the + // DefaultStructuredEncodingSelectionStrategy + Structured = "structured" ) +func ContextBasedEncodingSelectionStrategy(ctx context.Context, e cloudevents.Event) Encoding { + encoding := cecontext.EncodingFrom(ctx) + switch encoding { + case "", Binary: + return DefaultBinaryEncodingSelectionStrategy(ctx, e) + case Structured: + return DefaultStructuredEncodingSelectionStrategy(ctx, e) + } + return Default +} + +// DefaultBinaryEncodingSelectionStrategy implements a selection process for +// which binary encoding to use based on spec version of the event. +func DefaultBinaryEncodingSelectionStrategy(ctx context.Context, e cloudevents.Event) Encoding { + switch e.SpecVersion() { + case cloudevents.CloudEventsVersionV01: + return BinaryV01 + case cloudevents.CloudEventsVersionV02: + return BinaryV02 + case cloudevents.CloudEventsVersionV03: + return BinaryV03 + } + // Unknown version, return Default. + return Default +} + +// DefaultStructuredEncodingSelectionStrategy implements a selection process +// for which structured encoding to use based on spec version of the event. +func DefaultStructuredEncodingSelectionStrategy(ctx context.Context, e cloudevents.Event) Encoding { + switch e.SpecVersion() { + case cloudevents.CloudEventsVersionV01: + return StructuredV01 + case cloudevents.CloudEventsVersionV02: + return StructuredV02 + case cloudevents.CloudEventsVersionV03: + return StructuredV03 + } + // Unknown version, return Default. + return Default +} + // String pretty-prints the encoding as a string. func (e Encoding) String() string { switch e { @@ -101,7 +159,7 @@ func (e Encoding) Codec() string { // Version 0.2 case BinaryV02: - return "binary/v0.3" + return "binary/v0.2" case StructuredV02: return "structured/v0.2" diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/message.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/message.go index 0c5748c0b94..a6cdbecb1c6 100644 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/message.go +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/message.go @@ -1,9 +1,14 @@ package http import ( + "bytes" "encoding/json" - "github.com/cloudevents/sdk-go/pkg/cloudevents/transport" + + "io" + "io/ioutil" "net/http" + + "github.com/cloudevents/sdk-go/pkg/cloudevents/transport" ) // type check that this transport message impl matches the contract @@ -18,8 +23,7 @@ type Message struct { // Response is an http transport response. type Response struct { StatusCode int - Header http.Header - Body []byte + Message } // CloudEventsVersion inspects a message and tries to discover and return the @@ -31,7 +35,10 @@ func (m Message) CloudEventsVersion() string { if m.Header != nil { // Try headers first. // v0.1, cased from the spec - if v := m.Header["CE-CloudEventsVersion"]; len(v) == 1 { + // Note: don't pass literal string direct to m.Header[] so that + // go vet won't complain about non-canonical case. + name := "CE-CloudEventsVersion" + if v := m.Header[name]; len(v) == 1 { return v[0] } // v0.2, canonical casing @@ -40,11 +47,13 @@ func (m Message) CloudEventsVersion() string { } // v0.2, cased from the spec - if v := m.Header["ce-specversion"]; len(v) == 1 { + name = "ce-specversion" + if v := m.Header[name]; len(v) == 1 { return v[0] } // v0.2, canonical casing - if ver := m.Header.Get("ce-specversion"); ver != "" { + name = "ce-specversion" + if ver := m.Header.Get(name); ver != "" { return ver } } @@ -77,3 +86,63 @@ func (m Message) CloudEventsVersion() string { return "" } + +func readAllClose(r io.ReadCloser) ([]byte, error) { + if r != nil { + defer r.Close() + return ioutil.ReadAll(r) + } + return nil, nil +} + +// NewMessage creates a new message from the Header and Body of +// an http.Request or http.Response +func NewMessage(header http.Header, body io.ReadCloser) (*Message, error) { + var m Message + err := m.Init(header, body) + return &m, err +} + +// NewResponse creates a new response from the Header and Body of +// an http.Request or http.Response +func NewResponse(header http.Header, body io.ReadCloser, statusCode int) (*Response, error) { + resp := Response{StatusCode: statusCode} + err := resp.Init(header, body) + return &resp, err +} + +// Copy copies a new Body and Header into a message, replacing any previous data. +func (m *Message) Init(header http.Header, body io.ReadCloser) error { + m.Header = make(http.Header, len(header)) + copyHeadersEnsure(header, &m.Header) + var err error + m.Body, err = readAllClose(body) + return err +} + +func (m *Message) copyOut(header *http.Header, body *io.ReadCloser) { + copyHeadersEnsure(m.Header, header) + *body = nil + if m.Body != nil { + copy := append([]byte(nil), m.Body...) + *body = ioutil.NopCloser(bytes.NewBuffer(copy)) + } +} + +// ToRequest updates a http.Request from a Message. +// Replaces Body, ContentLength and Method, updates Headers. +// Panic if req is nil +func (m *Message) ToRequest(req *http.Request) { + m.copyOut(&req.Header, &req.Body) + req.ContentLength = int64(len(m.Body)) + req.Method = http.MethodPost +} + +// ToResponse updates a http.Response from a Response. +// Replaces Body, updates Headers. +// Panic if resp is nil +func (m *Response) ToResponse(resp *http.Response) { + m.copyOut(&resp.Header, &resp.Body) + resp.ContentLength = int64(len(m.Body)) + resp.StatusCode = m.StatusCode +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/options.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/options.go index 2a86ed385bd..c6e0c20df3b 100644 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/options.go +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/options.go @@ -2,6 +2,7 @@ package http import ( "fmt" + "net" nethttp "net/http" "net/url" "strings" @@ -117,6 +118,21 @@ func WithDefaultEncodingSelector(fn EncodingSelector) Option { } } +// WithContextBasedEncoding sets the encoding selection strategy for +// default encoding selections based context and then on Event, the encoded +// event will be the given version in the encoding specified by the given +// context, or Binary if not set. +func WithContextBasedEncoding() Option { + return func(t *Transport) error { + if t == nil { + return fmt.Errorf("http context based encoding option can not set nil transport") + } + + t.DefaultEncodingSelectionFn = ContextBasedEncodingSelectionStrategy + return nil + } +} + // WithBinaryEncoding sets the encoding selection strategy for // default encoding selections based on Event, the encoded event will be the // given version in Binary form. @@ -145,7 +161,18 @@ func WithStructuredEncoding() Option { } } -// WithPort sets the port for for clients with HTTP transports. +func checkListen(t *Transport, prefix string) error { + switch { + case t.Port != nil: + return fmt.Errorf("%v port already set", prefix) + case t.listener != nil: + return fmt.Errorf("%v listener already set", prefix) + } + return nil +} + +// WithPort sets the listening port for StartReceiver. +// Only one of WithListener or WithPort is allowed. func WithPort(port int) Option { return func(t *Transport) error { if t == nil { @@ -154,11 +181,30 @@ func WithPort(port int) Option { if port < 0 { return fmt.Errorf("http port option was given an invalid port: %d", port) } - t.Port = &port + if err := checkListen(t, "http port option"); err != nil { + return err + } + t.setPort(port) return nil } } +// WithListener sets the listener for StartReceiver. +// Only one of WithListener or WithPort is allowed. +func WithListener(l net.Listener) Option { + return func(t *Transport) error { + if t == nil { + return fmt.Errorf("http listener option can not set nil transport") + } + if err := checkListen(t, "http port option"); err != nil { + return err + } + t.listener = l + _, err := t.listen() + return err + } +} + // WithPath sets the path to receive cloudevents on for HTTP transports. func WithPath(path string) Option { return func(t *Transport) error { @@ -182,7 +228,7 @@ type Middleware func(next nethttp.Handler) nethttp.Handler // Middleware is applied to everything before it. For example // `NewClient(WithMiddleware(foo), WithMiddleware(bar))` would result in `bar(foo(original))`. func WithMiddleware(middleware Middleware) Option { - return func (t *Transport) error { + return func(t *Transport) error { if t == nil { return fmt.Errorf("http middleware option can not set nil transport") } @@ -190,3 +236,31 @@ func WithMiddleware(middleware Middleware) Option { return nil } } + +// WithLongPollTarget sets the receivers URL to perform long polling after +// StartReceiver is called. +func WithLongPollTarget(targetUrl string) Option { + return func(t *Transport) error { + if t == nil { + return fmt.Errorf("http long poll target option can not set nil transport") + } + targetUrl = strings.TrimSpace(targetUrl) + if targetUrl != "" { + var err error + var target *url.URL + target, err = url.Parse(targetUrl) + if err != nil { + return fmt.Errorf("http long poll target option failed to parse target url: %s", err.Error()) + } + + if t.LongPollReq == nil { + t.LongPollReq = &nethttp.Request{ + Method: nethttp.MethodGet, + } + } + t.LongPollReq.URL = target + return nil + } + return fmt.Errorf("http long poll target option was empty string") + } +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/transport.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/transport.go index 8434e8254aa..54d49a16cca 100644 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/transport.go +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/transport.go @@ -1,12 +1,13 @@ package http import ( - "bytes" "context" + "errors" "fmt" "io/ioutil" "net" "net/http" + "net/url" "strconv" "strings" "sync" @@ -20,14 +21,15 @@ import ( "github.com/cloudevents/sdk-go/pkg/cloudevents/transport" ) -type EncodingSelector func(e cloudevents.Event) Encoding - // Transport adheres to transport.Transport. var _ transport.Transport = (*Transport)(nil) const ( // DefaultShutdownTimeout defines the default timeout given to the http.Server when calling Shutdown. DefaultShutdownTimeout = time.Minute * 1 + + // TransportName is the name of this transport. + TransportName = "HTTP" ) // Transport acts as both a http client and a http handler. @@ -57,6 +59,9 @@ type Transport struct { // Receiver is invoked target for incoming events. Receiver transport.Receiver + // Converter is invoked if the incoming transport receives an undecodable + // message. + Converter transport.Converter // Port is the port to bind the receiver to. Defaults to 8080. Port *int // Path is the path to bind the receiver to. Defaults to "/". @@ -65,7 +70,17 @@ type Transport struct { // http server. If nil, the Transport will create a one. Handler *http.ServeMux - realPort int + // LongPollClient is the http client that will be used to long poll. + // If nil and LongPollReq is set, the Transport will create a one. + LongPollClient *http.Client + // LongPollReq is the base http request that is used for long poll. + // Only .Method, .URL, .Close, and .Header is considered. + // If not set, LongPollReq.Method defaults to GET. + // LongPollReq.URL or context.WithLongPollTarget(url) are required to long + // poll on StartReceiver. + LongPollReq *http.Request + + listener net.Listener server *http.Server handlerRegistered bool codec transport.Codec @@ -130,19 +145,29 @@ func copyHeaders(from, to http.Header) { } } +// Ensure to is a non-nil map before copying +func copyHeadersEnsure(from http.Header, to *http.Header) { + if len(from) > 0 { + if *to == nil { + *to = http.Header{} + } + copyHeaders(from, *to) + } +} + // Send implements Transport.Send -func (t *Transport) Send(ctx context.Context, event cloudevents.Event) (*cloudevents.Event, error) { +func (t *Transport) Send(ctx context.Context, event cloudevents.Event) (context.Context, *cloudevents.Event, error) { ctx, r := observability.NewReporter(ctx, reportSend) - resp, err := t.obsSend(ctx, event) + rctx, resp, err := t.obsSend(ctx, event) if err != nil { r.Error() } else { r.OK() } - return resp, err + return rctx, resp, err } -func (t *Transport) obsSend(ctx context.Context, event cloudevents.Event) (*cloudevents.Event, error) { +func (t *Transport) obsSend(ctx context.Context, event cloudevents.Event) (context.Context, *cloudevents.Event, error) { if t.Client == nil { t.crMu.Lock() t.Client = &http.Client{} @@ -156,7 +181,7 @@ func (t *Transport) obsSend(ctx context.Context, event cloudevents.Event) (*clou req.Method = t.Req.Method req.URL = t.Req.URL req.Close = t.Req.Close - copyHeaders(t.Req.Header, req.Header) + copyHeadersEnsure(t.Req.Header, &req.Header) } // Override the default request with target from context. @@ -165,55 +190,77 @@ func (t *Transport) obsSend(ctx context.Context, event cloudevents.Event) (*clou } if ok := t.loadCodec(ctx); !ok { - return nil, fmt.Errorf("unknown encoding set on transport: %d", t.Encoding) + return WithTransportContext(ctx, NewTransportContextFromResponse(nil)), nil, fmt.Errorf("unknown encoding set on transport: %d", t.Encoding) } - msg, err := t.codec.Encode(event) + msg, err := t.codec.Encode(ctx, event) if err != nil { - return nil, err + return WithTransportContext(ctx, NewTransportContextFromResponse(nil)), nil, err } if m, ok := msg.(*Message); ok { - copyHeaders(m.Header, req.Header) - - if m.Body != nil { - req.Body = ioutil.NopCloser(bytes.NewBuffer(m.Body)) - req.ContentLength = int64(len(m.Body)) - } else { - req.ContentLength = 0 - } - - return httpDo(ctx, t.Client, &req, func(resp *http.Response, err error) (*cloudevents.Event, error) { - logger := cecontext.LoggerFrom(ctx) + m.ToRequest(&req) + return httpDo(ctx, t.Client, &req, func(resp *http.Response, err error) (context.Context, *cloudevents.Event, error) { + rctx := WithTransportContext(ctx, NewTransportContextFromResponse(resp)) if err != nil { - return nil, err + return rctx, nil, err } defer resp.Body.Close() body, _ := ioutil.ReadAll(resp.Body) - msg := &Message{ + respEvent, err := t.MessageToEvent(ctx, &Message{ Header: resp.Header, Body: body, - } - - var respEvent *cloudevents.Event - if msg.CloudEventsVersion() != "" { - if ok := t.loadCodec(ctx); !ok { - err := fmt.Errorf("unknown encoding set on transport: %d", t.Encoding) - logger.Error("failed to load codec", zap.Error(err)) + }) + if err != nil { + isErr := true + if txerr, ok := err.(*transport.ErrTransportMessageConversion); ok { + if !txerr.IsFatal() { + isErr = false + } } - if respEvent, err = t.codec.Decode(msg); err != nil { - logger.Error("failed to decode message", zap.Error(err)) + if isErr { + return rctx, nil, err } } - if accepted(resp) { - return respEvent, nil + return rctx, respEvent, nil } - return respEvent, fmt.Errorf("error sending cloudevent: %s", resp.Status) + return rctx, respEvent, fmt.Errorf("error sending cloudevent: %s", resp.Status) }) } - return nil, fmt.Errorf("failed to encode Event into a Message") + return WithTransportContext(ctx, NewTransportContextFromResponse(nil)), nil, fmt.Errorf("failed to encode Event into a Message") +} + +func (t *Transport) MessageToEvent(ctx context.Context, msg *Message) (*cloudevents.Event, error) { + logger := cecontext.LoggerFrom(ctx) + var event *cloudevents.Event + var err error + + if msg.CloudEventsVersion() != "" { + // This is likely a cloudevents encoded message, try to decode it. + if ok := t.loadCodec(ctx); !ok { + err = transport.NewErrTransportMessageConversion("http", fmt.Sprintf("unknown encoding set on transport: %d", t.Encoding), true) + logger.Error("failed to load codec", zap.Error(err)) + } else { + event, err = t.codec.Decode(ctx, msg) + } + } else { + err = transport.NewErrTransportMessageConversion("http", "cloudevents version unknown", false) + } + + // If codec returns and error, or could not load the correct codec, try + // with the converter if it is set. + if err != nil && t.HasConverter() { + event, err = t.Converter.Convert(ctx, msg, err) + } + // If err is still set, it means that there was no converter, or the + // converter failed to convert. + if err != nil { + logger.Debug("failed to decode message", zap.Error(err)) + } + + return event, err } // SetReceiver implements Transport.SetReceiver @@ -221,12 +268,26 @@ func (t *Transport) SetReceiver(r transport.Receiver) { t.Receiver = r } +// SetConverter implements Transport.SetConverter +func (t *Transport) SetConverter(c transport.Converter) { + t.Converter = c +} + +// HasConverter implements Transport.HasConverter +func (t *Transport) HasConverter() bool { + return t.Converter != nil +} + // StartReceiver implements Transport.StartReceiver // NOTE: This is a blocking call. func (t *Transport) StartReceiver(ctx context.Context) error { t.reMu.Lock() defer t.reMu.Unlock() + if t.LongPollReq != nil { + go func() { _ = t.longPollStart(ctx) }() + } + if t.Handler == nil { t.Handler = http.NewServeMux() } @@ -236,28 +297,25 @@ func (t *Transport) StartReceiver(ctx context.Context) error { t.handlerRegistered = true } - addr := fmt.Sprintf(":%d", t.GetPort()) - t.server = &http.Server{ - Addr: addr, - Handler: attachMiddleware(t.Handler, t.middleware), - } - - listener, err := net.Listen("tcp", addr) + addr, err := t.listen() if err != nil { return err } - t.realPort = listener.Addr().(*net.TCPAddr).Port + + t.server = &http.Server{ + Addr: addr.String(), + Handler: attachMiddleware(t.Handler, t.middleware), + } // Shutdown defer func() { - t.realPort = 0 t.server.Close() t.server = nil }() errChan := make(chan error, 1) go func() { - errChan <- t.server.Serve(listener) + errChan <- t.server.Serve(t.listener) }() // wait for the server to return or ctx.Done(). @@ -270,12 +328,103 @@ func (t *Transport) StartReceiver(ctx context.Context) error { } ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() - return t.server.Shutdown(ctx) + err := t.server.Shutdown(ctx) + <-errChan // Wait for server goroutine to exit + return err case err := <-errChan: return err } } +func (t *Transport) longPollStart(ctx context.Context) error { + logger := cecontext.LoggerFrom(ctx) + logger.Info("starting long poll receiver") + + if t.LongPollClient == nil { + t.crMu.Lock() + t.LongPollClient = &http.Client{} + t.crMu.Unlock() + } + req := &http.Request{ + // TODO: decide if it is ok to use HeaderFrom context here. + Header: HeaderFrom(ctx), + } + if t.LongPollReq != nil { + req.Method = t.LongPollReq.Method + req.URL = t.LongPollReq.URL + req.Close = t.LongPollReq.Close + copyHeaders(t.LongPollReq.Header, req.Header) + } + + // Override the default request with target from context. + if target := LongPollTargetFrom(ctx); target != nil { + req.URL = target + } + + if req.URL == nil { + return errors.New("no long poll target found") + } + + req = req.WithContext(ctx) + msgCh := make(chan Message) + defer close(msgCh) + + go func(ch chan<- Message) { + for { + if resp, err := t.LongPollClient.Do(req); err != nil { + logger.Errorw("long poll request returned error", err) + uErr := err.(*url.Error) + if uErr.Temporary() || uErr.Timeout() { + continue + } + // TODO: if the transport is throwing errors, we might want to try again. Maybe with a back-off sleep. + // But this error also might be that there was a done on the context. + } else if resp.StatusCode == http.StatusNotModified { + // Keep polling. + continue + } else if resp.StatusCode == http.StatusOK { + body, _ := ioutil.ReadAll(resp.Body) + if err := resp.Body.Close(); err != nil { + logger.Warnw("error closing long poll response body", zap.Error(err)) + } + msg := Message{ + Header: resp.Header, + Body: body, + } + msgCh <- msg + } else { + // TODO: not sure what to do with upstream errors yet. + logger.Errorw("unhandled long poll response", zap.Any("resp", resp)) + } + } + }(msgCh) + + // Attach the long poll request context to the context. + ctx = WithTransportContext(ctx, TransportContext{ + URI: req.URL.RequestURI(), + Host: req.URL.Host, + Method: req.Method, + }) + + for { + select { + case <-ctx.Done(): + return nil + case msg := <-msgCh: + logger.Debug("got a message", zap.Any("msg", msg)) + if event, err := t.MessageToEvent(ctx, &msg); err != nil { + logger.Errorw("could not convert http message to event", zap.Error(err)) + } else { + logger.Debugw("got an event", zap.Any("event", event)) + // TODO: deliver event. + if _, err := t.invokeReceiver(ctx, *event); err != nil { + logger.Errorw("could not invoke receiver event", zap.Error(err)) + } + } + } + } +} + // attachMiddleware attaches the HTTP middleware to the specified handler. func attachMiddleware(h http.Handler, middleware []Middleware) http.Handler { for _, m := range middleware { @@ -285,23 +434,24 @@ func attachMiddleware(h http.Handler, middleware []Middleware) http.Handler { } type eventError struct { + ctx context.Context event *cloudevents.Event err error } -func httpDo(ctx context.Context, client *http.Client, req *http.Request, fn func(*http.Response, error) (*cloudevents.Event, error)) (*cloudevents.Event, error) { +func httpDo(ctx context.Context, client *http.Client, req *http.Request, fn func(*http.Response, error) (context.Context, *cloudevents.Event, error)) (context.Context, *cloudevents.Event, error) { // Run the HTTP request in a goroutine and pass the response to fn. c := make(chan eventError, 1) req = req.WithContext(ctx) go func() { - event, err := fn(client.Do(req)) - c <- eventError{event: event, err: err} + rctx, event, err := fn(client.Do(req)) + c <- eventError{ctx: rctx, event: event, err: err} }() select { case <-ctx.Done(): - return nil, ctx.Err() + return ctx, nil, ctx.Err() case ee := <-c: - return ee.event, ee.err + return ee.ctx, ee.event, ee.err } } @@ -334,18 +484,17 @@ func (t *Transport) obsInvokeReceiver(ctx context.Context, event cloudevents.Eve err := t.Receiver.Receive(ctx, event, &eventResp) if err != nil { - logger.Warnw("got an error from receiver fn: %s", zap.Error(err)) + logger.Warnw("got an error from receiver fn", zap.Error(err)) resp.StatusCode = http.StatusInternalServerError return &resp, err } if eventResp.Event != nil { if t.loadCodec(ctx) { - if m, err := t.codec.Encode(*eventResp.Event); err != nil { + if m, err := t.codec.Encode(ctx, *eventResp.Event); err != nil { logger.Errorw("failed to encode response from receiver fn", zap.Error(err)) } else if msg, ok := m.(*Message); ok { - resp.Header = msg.Header - resp.Body = msg.Body + resp.Message = *msg } } else { logger.Error("failed to load codec") @@ -363,7 +512,7 @@ func (t *Transport) obsInvokeReceiver(ctx context.Context, event cloudevents.Eve } // If we found a TransportResponseContext, use it. if trx != nil && trx.Header != nil && len(trx.Header) > 0 { - copyHeaders(trx.Header, resp.Header) + copyHeadersEnsure(trx.Header, &resp.Message.Header) } } @@ -380,47 +529,42 @@ func (t *Transport) obsInvokeReceiver(ctx context.Context, event cloudevents.Eve // ServeHTTP implements http.Handler func (t *Transport) ServeHTTP(w http.ResponseWriter, req *http.Request) { ctx, r := observability.NewReporter(req.Context(), reportServeHTTP) + // Add the transport context to ctx. + ctx = WithTransportContext(ctx, NewTransportContext(req)) logger := cecontext.LoggerFrom(ctx) body, err := ioutil.ReadAll(req.Body) if err != nil { logger.Errorw("failed to handle request", zap.Error(err)) w.WriteHeader(http.StatusBadRequest) - w.Write([]byte(`{"error":"Invalid request"}`)) + _, _ = w.Write([]byte(`{"error":"Invalid request"}`)) r.Error() return } - msg := &Message{ + + event, err := t.MessageToEvent(ctx, &Message{ Header: req.Header, Body: body, - } - - if ok := t.loadCodec(ctx); !ok { - err := fmt.Errorf("unknown encoding set on transport: %d", t.Encoding) - logger.Errorw("failed to load codec", zap.Error(err)) - w.WriteHeader(http.StatusBadRequest) - w.Write([]byte(fmt.Sprintf(`{"error":%q}`, err.Error()))) - r.Error() - return - } - event, err := t.codec.Decode(msg) + }) if err != nil { - logger.Errorw("failed to decode message", zap.Error(err)) - w.WriteHeader(http.StatusBadRequest) - w.Write([]byte(fmt.Sprintf(`{"error":%q}`, err.Error()))) - r.Error() - return - } - - if req != nil { - ctx = WithTransportContext(ctx, NewTransportContext(req)) + isFatal := true + if txerr, ok := err.(*transport.ErrTransportMessageConversion); ok { + isFatal = txerr.IsFatal() + } + if isFatal || event == nil { + logger.Errorw("failed to convert http message to event", zap.Error(err)) + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(fmt.Sprintf(`{"error":%q}`, err.Error()))) + r.Error() + return + } } resp, err := t.invokeReceiver(ctx, *event) if err != nil { logger.Warnw("error returned from invokeReceiver", zap.Error(err)) w.WriteHeader(http.StatusBadRequest) - w.Write([]byte(fmt.Sprintf(`{"error":%q}`, err.Error()))) + _, _ = w.Write([]byte(fmt.Sprintf(`{"error":%q}`, err.Error()))) r.Error() return } @@ -429,12 +573,12 @@ func (t *Transport) ServeHTTP(w http.ResponseWriter, req *http.Request) { if t.Req != nil { copyHeaders(t.Req.Header, w.Header()) } - if len(resp.Header) > 0 { - copyHeaders(resp.Header, w.Header()) + if len(resp.Message.Header) > 0 { + copyHeaders(resp.Message.Header, w.Header()) } - w.Header().Add("Content-Length", strconv.Itoa(len(resp.Body))) - if len(resp.Body) > 0 { - if _, err := w.Write(resp.Body); err != nil { + w.Header().Add("Content-Length", strconv.Itoa(len(resp.Message.Body))) + if len(resp.Message.Body) > 0 { + if _, err := w.Write(resp.Message.Body); err != nil { r.Error() return } @@ -453,18 +597,41 @@ func (t *Transport) ServeHTTP(w http.ResponseWriter, req *http.Request) { r.OK() } -// GetPort returns the port the transport is active on. -// .Port can be set to 0, which means the transport selects a port, GetPort -// allows the transport to report back the selected port. +// GetPort returns the listening port. +// Returns -1 if there is a listening error. +// Note this will call net.Listen() if the listener is not already started. func (t *Transport) GetPort() int { - if t.Port != nil && *t.Port == 0 && t.realPort != 0 { - return t.realPort + // Ensure we have a listener and therefore a port. + if _, err := t.listen(); err == nil || t.Port != nil { + return *t.Port } + return -1 +} - if t.Port != nil && *t.Port >= 0 { // 0 means next open port - return *t.Port +func (t *Transport) setPort(port int) { + if t.Port == nil { + t.Port = new(int) + } + *t.Port = port +} + +// listen if not already listening, update t.Port +func (t *Transport) listen() (net.Addr, error) { + if t.listener == nil { + port := 8080 + if t.Port != nil { + port = *t.Port + } + var err error + if t.listener, err = net.Listen("tcp", fmt.Sprintf(":%d", port)); err != nil { + return nil, err + } + } + addr := t.listener.Addr() + if tcpAddr, ok := addr.(*net.TCPAddr); ok { + t.setPort(tcpAddr.Port) } - return 8080 // default + return addr, nil } // GetPath returns the path the transport is hosted on. If the path is '/', diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/transport.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/transport.go index 75c652b9233..a08d5a12e52 100644 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/transport.go +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/transport.go @@ -2,16 +2,23 @@ package transport import ( "context" + "github.com/cloudevents/sdk-go/pkg/cloudevents" ) // Transport is the interface for transport sender to send the converted Message // over the underlying transport. type Transport interface { - Send(context.Context, cloudevents.Event) (*cloudevents.Event, error) + Send(context.Context, cloudevents.Event) (context.Context, *cloudevents.Event, error) SetReceiver(Receiver) StartReceiver(context.Context) error + + // SetConverter sets the delegate to use for converting messages that have + // failed to be decoded from known codecs for this transport. + SetConverter(Converter) + // HasConverter is true when a non-nil converter has been set. + HasConverter() bool } // Receiver is an interface to define how a transport will invoke a listener @@ -19,3 +26,19 @@ type Transport interface { type Receiver interface { Receive(context.Context, cloudevents.Event, *cloudevents.EventResponse) error } + +// ReceiveFunc wraps a function as a Receiver object. +type ReceiveFunc func(ctx context.Context, e cloudevents.Event, er *cloudevents.EventResponse) error + +// Receive implements Receiver.Receive +func (f ReceiveFunc) Receive(ctx context.Context, e cloudevents.Event, er *cloudevents.EventResponse) error { + return f(ctx, e, er) +} + +// Converter is an interface to define how a transport delegate to convert an +// non-understood transport message from the internal codecs. Providing a +// Converter allows incoming requests to be bridged to CloudEvents format if +// they have not been sent as an event in CloudEvents format. +type Converter interface { + Convert(context.Context, Message, error) (*cloudevents.Event, error) +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/types/timestamp.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/types/timestamp.go index c11e4e15a6d..6534aacbb50 100644 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/types/timestamp.go +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/types/timestamp.go @@ -38,7 +38,7 @@ func (t *Timestamp) MarshalJSON() ([]byte, error) { } // UnmarshalJSON implements the json unmarshal method used when this type is -// unmarshed using json.Unmarshal. +// unmarshaled using json.Unmarshal. func (t *Timestamp) UnmarshalJSON(b []byte) error { var timestamp string if err := json.Unmarshal(b, ×tamp); err != nil { @@ -61,7 +61,7 @@ func (t *Timestamp) MarshalXML(e *xml.Encoder, start xml.StartElement) error { } // UnmarshalXML implements the xml unmarshal method used when this type is -// unmarshed using xml.Unmarshal. +// unmarshaled using xml.Unmarshal. func (t *Timestamp) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { var timestamp string if err := d.DecodeElement(×tamp, &start); err != nil { diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/types/urlref.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/types/urlref.go index d1cc2703645..2743c45e2b9 100644 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/types/urlref.go +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/types/urlref.go @@ -35,7 +35,7 @@ func (u URLRef) MarshalJSON() ([]byte, error) { } // UnmarshalJSON implements the json unmarshal method used when this type is -// unmarshed using json.Unmarshal. +// unmarshaled using json.Unmarshal. func (u *URLRef) UnmarshalJSON(b []byte) error { var ref string if err := json.Unmarshal(b, &ref); err != nil { @@ -51,12 +51,11 @@ func (u *URLRef) UnmarshalJSON(b []byte) error { // MarshalXML implements a custom xml marshal method used when this type is // marshaled using xml.Marshal. func (u URLRef) MarshalXML(e *xml.Encoder, start xml.StartElement) error { - v := fmt.Sprintf("%s", u.String()) - return e.EncodeElement(v, start) + return e.EncodeElement(u.String(), start) } // UnmarshalXML implements the xml unmarshal method used when this type is -// unmarshed using xml.Unmarshal. +// unmarshaled using xml.Unmarshal. func (u *URLRef) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { var ref string if err := d.DecodeElement(&ref, &start); err != nil { From cb7694e1d284ec88c68b199d5e2c359980dbe556 Mon Sep 17 00:00:00 2001 From: Nacho Cano Date: Sun, 1 Sep 2019 08:44:16 -0700 Subject: [PATCH 6/8] updates --- pkg/adapter/apiserver/ref.go | 6 +-- pkg/adapter/apiserver/resource.go | 6 +-- pkg/adapter/cronjobevents/adapter.go | 2 +- pkg/broker/ingress/ingress_handler.go | 2 +- pkg/broker/ingress/ingress_handler_test.go | 8 ++-- pkg/broker/ingress/stats_reporter.go | 42 +++++++++++-------- pkg/broker/ingress/stats_reporter_test.go | 12 +++--- pkg/broker/metrics_test.go | 48 ---------------------- pkg/kncloudevents/testing/test_client.go | 4 +- 9 files changed, 46 insertions(+), 84 deletions(-) delete mode 100644 pkg/broker/metrics_test.go diff --git a/pkg/adapter/apiserver/ref.go b/pkg/adapter/apiserver/ref.go index 9fcbcede367..c7c9ac67a22 100644 --- a/pkg/adapter/apiserver/ref.go +++ b/pkg/adapter/apiserver/ref.go @@ -67,7 +67,7 @@ func (a *ref) Add(obj interface{}) error { return err } - if _, err := a.ce.Send(context.Background(), *event); err != nil { + if _, _, err := a.ce.Send(context.Background(), *event); err != nil { a.logger.Info("event delivery failed", zap.Error(err)) return err } @@ -82,7 +82,7 @@ func (a *ref) Update(obj interface{}) error { return err } - if _, err := a.ce.Send(context.Background(), *event); err != nil { + if _, _, err := a.ce.Send(context.Background(), *event); err != nil { a.logger.Info("event delivery failed", zap.Error(err)) return err } @@ -97,7 +97,7 @@ func (a *ref) Delete(obj interface{}) error { return err } - if _, err := a.ce.Send(context.Background(), *event); err != nil { + if _, _, err := a.ce.Send(context.Background(), *event); err != nil { a.logger.Info("event delivery failed", zap.Error(err)) return err } diff --git a/pkg/adapter/apiserver/resource.go b/pkg/adapter/apiserver/resource.go index 043327df91a..567524caf1f 100644 --- a/pkg/adapter/apiserver/resource.go +++ b/pkg/adapter/apiserver/resource.go @@ -41,7 +41,7 @@ func (a *resource) Add(obj interface{}) error { return err } - if _, err := a.ce.Send(context.Background(), *event); err != nil { + if _, _, err := a.ce.Send(context.Background(), *event); err != nil { a.logger.Info("event delivery failed", zap.Error(err)) return err } @@ -56,7 +56,7 @@ func (a *resource) Update(obj interface{}) error { return err } - if _, err := a.ce.Send(context.Background(), *event); err != nil { + if _, _, err := a.ce.Send(context.Background(), *event); err != nil { a.logger.Info("event delivery failed", zap.Error(err)) return err } @@ -71,7 +71,7 @@ func (a *resource) Delete(obj interface{}) error { return err } - if _, err := a.ce.Send(context.Background(), *event); err != nil { + if _, _, err := a.ce.Send(context.Background(), *event); err != nil { a.logger.Info("event delivery failed", zap.Error(err)) return err } diff --git a/pkg/adapter/cronjobevents/adapter.go b/pkg/adapter/cronjobevents/adapter.go index 5c9ec5e9d6d..60c047a9146 100644 --- a/pkg/adapter/cronjobevents/adapter.go +++ b/pkg/adapter/cronjobevents/adapter.go @@ -93,7 +93,7 @@ func (a *Adapter) cronTick() { event.SetSource(sourcesv1alpha1.CronJobEventSource(a.Namespace, a.Name)) event.SetData(message(a.Data)) - if _, err := a.client.Send(context.TODO(), event); err != nil { + if _, _, err := a.client.Send(context.TODO(), event); err != nil { logger.Error("failed to send cloudevent", err) } } diff --git a/pkg/broker/ingress/ingress_handler.go b/pkg/broker/ingress/ingress_handler.go index bd571ae7eb5..90e167ed758 100644 --- a/pkg/broker/ingress/ingress_handler.go +++ b/pkg/broker/ingress/ingress_handler.go @@ -88,7 +88,7 @@ func (h *Handler) serveHTTP(ctx context.Context, event cloudevents.Event, resp * send := h.decrementTTL(&event) if !send { // Record the event count. - h.Reporter.ReportEventCount(reporterArgs, errors.New("dropped due to TTL")) + h.Reporter.ReportEventCount(reporterArgs, http.StatusBadRequest) return nil } diff --git a/pkg/broker/ingress/ingress_handler_test.go b/pkg/broker/ingress/ingress_handler_test.go index 1f2228652a2..bc456f44833 100644 --- a/pkg/broker/ingress/ingress_handler_test.go +++ b/pkg/broker/ingress/ingress_handler_test.go @@ -24,19 +24,19 @@ const ( type mockReporter struct{} -func (r *mockReporter) ReportEventCount(args *ReportArgs, err error) error { +func (r *mockReporter) ReportEventCount(args *ReportArgs, responseCode int) error { return nil } -func (r *mockReporter) ReportEventDispatchTime(args *ReportArgs, err error, d time.Duration) error { +func (r *mockReporter) ReportEventDispatchTime(args *ReportArgs, responseCode int, d time.Duration) error { return nil } type fakeClient struct{ sent bool } -func (f *fakeClient) Send(ctx context.Context, event cloudevents.Event) (*cloudevents.Event, error) { +func (f *fakeClient) Send(ctx context.Context, event cloudevents.Event) (context.Context, *cloudevents.Event, error) { f.sent = true - return &event, nil + return ctx, &event, nil } func (f *fakeClient) StartReceiver(ctx context.Context, fn interface{}) error { diff --git a/pkg/broker/ingress/stats_reporter.go b/pkg/broker/ingress/stats_reporter.go index e135223c688..a122f8a4397 100644 --- a/pkg/broker/ingress/stats_reporter.go +++ b/pkg/broker/ingress/stats_reporter.go @@ -21,10 +21,11 @@ import ( "go.opencensus.io/stats" "go.opencensus.io/stats/view" "go.opencensus.io/tag" - utils "knative.dev/eventing/pkg/broker" . "knative.dev/eventing/pkg/metrics/metricskey" + "knative.dev/eventing/pkg/utils" "knative.dev/pkg/metrics" "knative.dev/pkg/metrics/metricskey" + "strconv" "time" ) @@ -63,11 +64,12 @@ var _ StatsReporter = (*reporter)(nil) // Reporter holds cached metric objects to report ingress metrics. type reporter struct { - namespaceTagKey tag.Key - brokerTagKey tag.Key - eventTypeKey tag.Key - eventSourceKey tag.Key - resultKey tag.Key + namespaceTagKey tag.Key + brokerTagKey tag.Key + eventTypeKey tag.Key + eventSourceKey tag.Key + responseCodeKey tag.Key + responseCodeClassKey tag.Key } // NewStatsReporter creates a reporter that collects and reports ingress metrics. @@ -95,11 +97,16 @@ func NewStatsReporter() (StatsReporter, error) { return nil, err } r.eventSourceKey = eventSourceTag - resultTag, err := tag.NewKey(LabelResult) + responseCodeTag, err := tag.NewKey(LabelResponseCode) if err != nil { return nil, err } - r.resultKey = resultTag + r.responseCodeKey = responseCodeTag + responseCodeClassTag, err := tag.NewKey(LabelResponseCodeClass) + if err != nil { + return nil, err + } + r.responseCodeClassKey = responseCodeClassTag // Create view to see our measurements. err = view.Register( @@ -107,13 +114,13 @@ func NewStatsReporter() (StatsReporter, error) { Description: eventCountM.Description(), Measure: eventCountM, Aggregation: view.Count(), - TagKeys: []tag.Key{r.namespaceTagKey, r.brokerTagKey, r.eventTypeKey, r.eventSourceKey, r.resultKey}, + TagKeys: []tag.Key{r.namespaceTagKey, r.brokerTagKey, r.eventTypeKey, r.eventSourceKey, r.responseCodeKey, r.responseCodeClassKey}, }, &view.View{ Description: dispatchTimeInMsecM.Description(), Measure: dispatchTimeInMsecM, - Aggregation: view.Distribution(utils.Buckets125(1, 100)...), // 1, 2, 5, 10, 20, 50, 100 - TagKeys: []tag.Key{r.namespaceTagKey, r.brokerTagKey, r.eventTypeKey, r.eventSourceKey, r.resultKey}, + Aggregation: view.Distribution(metrics.Buckets125(1, 100)...), // 1, 2, 5, 10, 20, 50, 100 + TagKeys: []tag.Key{r.namespaceTagKey, r.brokerTagKey, r.eventTypeKey, r.eventSourceKey, r.responseCodeKey, r.responseCodeClassKey}, }, ) if err != nil { @@ -124,8 +131,8 @@ func NewStatsReporter() (StatsReporter, error) { } // ReportEventCount captures the event count. -func (r *reporter) ReportEventCount(args *ReportArgs, err error) error { - ctx, err := r.generateTag(args, err) +func (r *reporter) ReportEventCount(args *ReportArgs, responseCode int) error { + ctx, err := r.generateTag(args, responseCode) if err != nil { return err } @@ -134,8 +141,8 @@ func (r *reporter) ReportEventCount(args *ReportArgs, err error) error { } // ReportEventDispatchTime captures dispatch times. -func (r *reporter) ReportEventDispatchTime(args *ReportArgs, err error, d time.Duration) error { - ctx, err := r.generateTag(args, err) +func (r *reporter) ReportEventDispatchTime(args *ReportArgs, responseCode int, d time.Duration) error { + ctx, err := r.generateTag(args, responseCode) if err != nil { return err } @@ -144,12 +151,13 @@ func (r *reporter) ReportEventDispatchTime(args *ReportArgs, err error, d time.D return nil } -func (r *reporter) generateTag(args *ReportArgs, err error) (context.Context, error) { +func (r *reporter) generateTag(args *ReportArgs, responseCode int) (context.Context, error) { return tag.New( context.Background(), tag.Insert(r.namespaceTagKey, args.ns), tag.Insert(r.brokerTagKey, args.broker), tag.Insert(r.eventTypeKey, args.eventType), tag.Insert(r.eventSourceKey, args.eventSource), - tag.Insert(r.resultKey, utils.Result(err))) + tag.Insert(r.responseCodeKey, strconv.Itoa(responseCode)), + tag.Insert(r.responseCodeClassKey, utils.ResponseCodeClass(responseCode))) } diff --git a/pkg/broker/ingress/stats_reporter_test.go b/pkg/broker/ingress/stats_reporter_test.go index 9ce3c25f786..a3c1591958e 100644 --- a/pkg/broker/ingress/stats_reporter_test.go +++ b/pkg/broker/ingress/stats_reporter_test.go @@ -17,6 +17,7 @@ limitations under the License. package ingress import ( + "net/http" "testing" "time" @@ -54,24 +55,25 @@ func TestStatsReporter(t *testing.T) { metricskey.LabelBrokerName: "testbroker", metricskey.LabelEventType: "testeventtype", metricskey.LabelEventSource: "testeventsource", - LabelResult: "success", + LabelResponseCode: "202", + LabelResponseCodeClass: "2xx", } // test ReportEventCount expectSuccess(t, func() error { - return r.ReportEventCount(args, nil) + return r.ReportEventCount(args, http.StatusAccepted) }) expectSuccess(t, func() error { - return r.ReportEventCount(args, nil) + return r.ReportEventCount(args, http.StatusAccepted) }) metricstest.CheckCountData(t, "event_count", wantTags, 2) // test ReportDispatchTime expectSuccess(t, func() error { - return r.ReportEventDispatchTime(args, nil, 1100*time.Millisecond) + return r.ReportEventDispatchTime(args, http.StatusAccepted, 1100*time.Millisecond) }) expectSuccess(t, func() error { - return r.ReportEventDispatchTime(args, nil, 9100*time.Millisecond) + return r.ReportEventDispatchTime(args, http.StatusAccepted, 9100*time.Millisecond) }) metricstest.CheckDistributionData(t, "event_dispatch_latencies", wantTags, 2, 1100.0, 9100.0) } diff --git a/pkg/broker/metrics_test.go b/pkg/broker/metrics_test.go deleted file mode 100644 index 2e8951de719..00000000000 --- a/pkg/broker/metrics_test.go +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright 2019 The Knative Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package broker - -import ( - "testing" - - "github.com/google/go-cmp/cmp" -) - -func TestBuckets125(t *testing.T) { - testCases := []struct { - low float64 - high float64 - want []float64 - }{{ - low: 1, - high: 100, - want: []float64{1, 2, 5, 10, 20, 50, 100}, - }, { - low: 0.1, - high: 10, - want: []float64{0.1, 0.2, 0.5, 1, 2, 5, 10}, - }} - - for i, tc := range testCases { - t.Run(string(i), func(t *testing.T) { - got := Buckets125(tc.low, tc.high) - if diff := cmp.Diff(tc.want, got); diff != "" { - t.Errorf("Unexpected buckets for %f-%f (-want, +got): %s", tc.low, tc.high, diff) - } - }) - } -} diff --git a/pkg/kncloudevents/testing/test_client.go b/pkg/kncloudevents/testing/test_client.go index c1176778275..e56e102a0e0 100644 --- a/pkg/kncloudevents/testing/test_client.go +++ b/pkg/kncloudevents/testing/test_client.go @@ -12,10 +12,10 @@ type TestCloudEventsClient struct { var _ cloudevents.Client = (*TestCloudEventsClient)(nil) -func (c *TestCloudEventsClient) Send(ctx context.Context, event cloudevents.Event) (*cloudevents.Event, error) { +func (c *TestCloudEventsClient) Send(ctx context.Context, event cloudevents.Event) (context.Context, *cloudevents.Event, error) { // TODO: improve later. c.Sent = append(c.Sent, event) - return nil, nil + return ctx, nil, nil } func (c *TestCloudEventsClient) StartReceiver(ctx context.Context, fn interface{}) error { From 6b0334d3cf38061c680347b122df36918bc9cbd4 Mon Sep 17 00:00:00 2001 From: Nacho Cano Date: Sun, 1 Sep 2019 08:45:59 -0700 Subject: [PATCH 7/8] compilation problems --- cmd/sendevent/main.go | 2 +- test/performance/broker-latency/main.go | 2 +- test/test_images/heartbeats/main.go | 2 +- test/test_images/latency/main.go | 2 +- test/test_images/sendevents/main.go | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/sendevent/main.go b/cmd/sendevent/main.go index 4991c8c1c7b..544a743e29c 100644 --- a/cmd/sendevent/main.go +++ b/cmd/sendevent/main.go @@ -91,7 +91,7 @@ func main() { os.Exit(1) } - if resp, err := c.Send(context.Background(), event); err != nil { + if _, resp, err := c.Send(context.Background(), event); err != nil { fmt.Printf("Failed to send event to %s: %s\n", target, err) os.Exit(1) } else if resp != nil { diff --git a/test/performance/broker-latency/main.go b/test/performance/broker-latency/main.go index 3562dcdff7c..9bc23ed1624 100644 --- a/test/performance/broker-latency/main.go +++ b/test/performance/broker-latency/main.go @@ -171,7 +171,7 @@ func main() { defer cancel() sendTime := time.Now() - _, err := c.Send(ctx, event) + _, _, err := c.Send(ctx, event) if err != nil { if qerr := q.AddError(mako.XTime(sendTime), err.Error()); qerr != nil { log.Printf("ERROR AddError: %v", qerr) diff --git a/test/test_images/heartbeats/main.go b/test/test_images/heartbeats/main.go index 3c345e8b9fc..749d81302a3 100644 --- a/test/test_images/heartbeats/main.go +++ b/test/test_images/heartbeats/main.go @@ -111,7 +111,7 @@ func main() { } log.Printf("sending cloudevent to %s", sink) - if _, err := c.Send(context.Background(), event); err != nil { + if _, _, err := c.Send(context.Background(), event); err != nil { log.Printf("failed to send cloudevent: %s", err.Error()) } // Wait for next tick diff --git a/test/test_images/latency/main.go b/test/test_images/latency/main.go index 33eb03d5347..80b423b9559 100644 --- a/test/test_images/latency/main.go +++ b/test/test_images/latency/main.go @@ -146,7 +146,7 @@ func main() { ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeout)*time.Second) defer cancel() sendTime := time.Now() - if _, err := c.Send(ctx, event); err != nil { + if _, _, err := c.Send(ctx, event); err != nil { resultCh <- state{status: undelivered} } if timeCh, ok := eventTimeMap[seqStr]; ok { diff --git a/test/test_images/sendevents/main.go b/test/test_images/sendevents/main.go index 7812013c48b..5b54ad48f72 100644 --- a/test/test_images/sendevents/main.go +++ b/test/test_images/sendevents/main.go @@ -153,7 +153,7 @@ func main() { log.Fatalf("failed to set data, %v", err) } - if resp, err := c.Send(context.Background(), event); err != nil { + if _, resp, err := c.Send(context.Background(), event); err != nil { log.Printf("send returned an error: %v\n", err) } else if resp != nil { log.Printf("Got response from %s\n%s\n", sink, resp) From 113d77f1aa59f5871d65dc95e50e11d47c3e7ade Mon Sep 17 00:00:00 2001 From: nachocano Date: Tue, 3 Sep 2019 08:59:14 -0700 Subject: [PATCH 8/8] updates after code review --- pkg/broker/filter/filter_handler.go | 4 ++-- pkg/broker/filter/filter_handler_test.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/broker/filter/filter_handler.go b/pkg/broker/filter/filter_handler.go index de76b136af0..397620b61a3 100644 --- a/pkg/broker/filter/filter_handler.go +++ b/pkg/broker/filter/filter_handler.go @@ -222,7 +222,7 @@ func (r *Handler) sendEvent(ctx context.Context, tctx cloudevents.HTTPTransportC if err != nil { r.logger.Error("Unable to parse subscriberURI", zap.Error(err), zap.String("subscriberURIString", subscriberURIString)) // Record the event count. - r.reporter.ReportEventCount(reportArgs, http.StatusNotFound) + r.reporter.ReportEventCount(reportArgs, http.StatusInternalServerError) return nil, err } @@ -232,7 +232,7 @@ func (r *Handler) sendEvent(ctx context.Context, tctx cloudevents.HTTPTransportC if filterResult == failFilter { r.logger.Debug("Event did not pass filter", zap.Any("triggerRef", trigger)) // Record the event count. - r.reporter.ReportEventCount(reportArgs, http.StatusForbidden) + r.reporter.ReportEventCount(reportArgs, http.StatusExpectationFailed) return nil, nil } diff --git a/pkg/broker/filter/filter_handler_test.go b/pkg/broker/filter/filter_handler_test.go index 34cd1d4ad48..d89e4b164c0 100644 --- a/pkg/broker/filter/filter_handler_test.go +++ b/pkg/broker/filter/filter_handler_test.go @@ -28,7 +28,7 @@ import ( "k8s.io/apimachinery/pkg/labels" - "github.com/cloudevents/sdk-go" + cloudevents "github.com/cloudevents/sdk-go" cehttp "github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http" "github.com/google/go-cmp/cmp" "go.uber.org/zap"