From 2d86e5c81dd993d610132429bbd87303fb6c5f03 Mon Sep 17 00:00:00 2001 From: slinkydeveloper Date: Tue, 6 Oct 2020 17:31:21 +0200 Subject: [PATCH 01/10] Replacing port-forwarding with k8s events in recordevents (#4171) * WIP Replacing port-forwarding with k8s events in recordevents Signed-off-by: Francesco Guardiani * WIP Events are sent properly from recordevents to k8s events Signed-off-by: Francesco Guardiani * Sending the beast in battle Signed-off-by: Francesco Guardiani * Update Signed-off-by: Francesco Guardiani * Moved resources of recordevents in proper module Signed-off-by: Francesco Guardiani * gofmt Signed-off-by: Francesco Guardiani * Another one Signed-off-by: Francesco Guardiani * Other stuff Signed-off-by: Francesco Guardiani * Copyrights Signed-off-by: Francesco Guardiani * Other copyrights Signed-off-by: Francesco Guardiani * boilerplate check Signed-off-by: Francesco Guardiani * boilerplate check Signed-off-by: Francesco Guardiani * Other lints checks Signed-off-by: Francesco Guardiani * Warn Signed-off-by: Francesco Guardiani * Now the linter Signed-off-by: Francesco Guardiani * Missing decorator in recordevents Signed-off-by: Francesco Guardiani * Simplified the sink definition, just sending events to myself Signed-off-by: Francesco Guardiani * Happy linter, happy me Signed-off-by: Francesco Guardiani * Correct HTTP server shutdown Signed-off-by: Francesco Guardiani * Try fix correlator config Signed-off-by: Francesco Guardiani * Just close the server Signed-off-by: Francesco Guardiani * Still messing up with the Event correlator options Signed-off-by: Francesco Guardiani * lint check Signed-off-by: Francesco Guardiani (cherry picked from commit b7a7126e278b2f5cda968c5724caae7f0fa64bd7) Signed-off-by: Francesco Guardiani --- go.sum | 6 +- .../broker_control_plane_test_helper.go | 6 +- .../helpers/broker_tracing_test_helper.go | 4 +- .../channel_status_subscriber_test_helper.go | 4 +- .../helpers/channel_tracing_test_helper.go | 4 +- .../helpers/tracing_test_helper.go | 4 +- .../helpers/trigger_no_broker_test_helper.go | 5 +- test/lib/client.go | 6 + test/lib/k8s_events.go | 84 +++++ test/lib/recordevents/event_info.go | 232 +----------- test/lib/recordevents/event_info_store.go | 232 +++++------- .../lib/recordevents/event_info_store_test.go | 279 -------------- test/lib/recordevents/event_log.go | 33 ++ test/lib/recordevents/observer/observer.go | 116 ++++++ .../recordevents/recorder_vent/constructor.go | 108 ++++++ test/lib/recordevents/recorder_vent/doc.go | 19 + .../recordevents/recorder_vent/recorder.go | 43 +++ test/lib/recordevents/resources.go | 61 ++++ test/lib/resources/kube.go | 22 -- test/test_images/recordevents/eventstore.go | 191 ---------- .../recordevents/eventstore_test.go | 339 ------------------ test/test_images/recordevents/main.go | 162 ++------- vendor/gopkg.in/yaml.v3/.travis.yml | 22 +- vendor/gopkg.in/yaml.v3/apic.go | 1 + vendor/gopkg.in/yaml.v3/decode.go | 63 ++-- vendor/gopkg.in/yaml.v3/emitterc.go | 54 ++- vendor/gopkg.in/yaml.v3/encode.go | 42 ++- vendor/gopkg.in/yaml.v3/parserc.go | 39 ++ vendor/gopkg.in/yaml.v3/scannerc.go | 180 ++++++---- vendor/gopkg.in/yaml.v3/yaml.go | 35 +- vendor/gopkg.in/yaml.v3/yamlh.go | 4 + vendor/modules.txt | 2 +- 32 files changed, 931 insertions(+), 1471 deletions(-) create mode 100644 test/lib/k8s_events.go delete mode 100644 test/lib/recordevents/event_info_store_test.go create mode 100644 test/lib/recordevents/event_log.go create mode 100644 test/lib/recordevents/observer/observer.go create mode 100644 test/lib/recordevents/recorder_vent/constructor.go create mode 100644 test/lib/recordevents/recorder_vent/doc.go create mode 100644 test/lib/recordevents/recorder_vent/recorder.go create mode 100644 test/lib/recordevents/resources.go delete mode 100644 test/test_images/recordevents/eventstore.go delete mode 100644 test/test_images/recordevents/eventstore_test.go diff --git a/go.sum b/go.sum index 4f024dd92e6..4fd256184e7 100644 --- a/go.sum +++ b/go.sum @@ -863,6 +863,7 @@ github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNx github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= +github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.8/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= @@ -1107,6 +1108,7 @@ github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKv github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v0.0.6/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= +github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= @@ -1758,8 +1760,8 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20190709130402-674ba3eaed22/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20191026110619-0b21df46bc1d h1:LCPbGQ34PMrwad11aMZ+dbz5SAsq/0ySjRwQ8I9Qwd8= -gopkg.in/yaml.v3 v3.0.0-20191026110619-0b21df46bc1d/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= helm.sh/helm/v3 v3.1.1/go.mod h1:WYsFJuMASa/4XUqLyv54s0U/f3mlAaRErGmyy4z921g= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/test/conformance/helpers/broker_control_plane_test_helper.go b/test/conformance/helpers/broker_control_plane_test_helper.go index c9a975fdedb..887a69d87b8 100644 --- a/test/conformance/helpers/broker_control_plane_test_helper.go +++ b/test/conformance/helpers/broker_control_plane_test_helper.go @@ -24,6 +24,7 @@ import ( "knative.dev/pkg/reconciler" eventingv1beta1 "knative.dev/eventing/pkg/apis/eventing/v1beta1" + "knative.dev/eventing/test/lib/recordevents" testlib "knative.dev/eventing/test/lib" "knative.dev/eventing/test/lib/duck" @@ -81,9 +82,8 @@ func triggerV1Beta1BeforeBrokerHelper(triggerName string, client *testlib.Client const etLogger = "logger" const loggerPodName = "logger-pod" - logPod := resources.EventRecordPod(loggerPodName) - client.CreatePodOrFail(logPod, testlib.WithService(loggerPodName)) - client.WaitForAllTestResourcesReadyOrFail() // Can't do this for the trigger because it's not 'ready' yet + _ = recordevents.DeployEventRecordOrFail(context.TODO(), client, loggerPodName) + client.WaitForAllTestResourcesReadyOrFail(context.Background()) // Can't do this for the trigger because it's not 'ready' yet client.CreateTriggerOrFailV1Beta1(triggerName, resources.WithAttributesTriggerFilterV1Beta1(eventingv1beta1.TriggerAnyFilter, etLogger, map[string]interface{}{}), resources.WithSubscriberServiceRefForTriggerV1Beta1(loggerPodName), diff --git a/test/conformance/helpers/broker_tracing_test_helper.go b/test/conformance/helpers/broker_tracing_test_helper.go index 597a2c6884b..ff415db469f 100644 --- a/test/conformance/helpers/broker_tracing_test_helper.go +++ b/test/conformance/helpers/broker_tracing_test_helper.go @@ -29,6 +29,7 @@ import ( "knative.dev/eventing/pkg/utils" tracinghelper "knative.dev/eventing/test/conformance/helpers/tracing" testlib "knative.dev/eventing/test/lib" + "knative.dev/eventing/test/lib/recordevents" "knative.dev/eventing/test/lib/resources" "knative.dev/eventing/test/lib/sender" ) @@ -78,8 +79,7 @@ func setupBrokerTracing(brokerClass string) SetupTracingTestInfrastructureFunc { ) // Create a logger (EventRecord) Pod and a K8s Service that points to it. - logPod := resources.EventRecordPod(loggerPodName) - client.CreatePodOrFail(logPod, testlib.WithService(loggerPodName)) + _ = recordevents.DeployEventRecordOrFail(ctx, client, loggerPodName) // Create a Trigger that receives events (type=bar) and sends them to the logger Pod. loggerTrigger := client.CreateTriggerOrFailV1Beta1( diff --git a/test/conformance/helpers/channel_status_subscriber_test_helper.go b/test/conformance/helpers/channel_status_subscriber_test_helper.go index 125661a16b3..e893c9424b6 100644 --- a/test/conformance/helpers/channel_status_subscriber_test_helper.go +++ b/test/conformance/helpers/channel_status_subscriber_test_helper.go @@ -22,6 +22,7 @@ import ( duckv1beta1 "knative.dev/eventing/pkg/apis/duck/v1beta1" eventingv1beta1 "knative.dev/eventing/pkg/apis/messaging/v1beta1" testlib "knative.dev/eventing/test/lib" + "knative.dev/eventing/test/lib/recordevents" "knative.dev/eventing/test/lib/resources" corev1 "k8s.io/api/core/v1" @@ -56,8 +57,7 @@ func channelHasRequiredSubscriberStatus(st *testing.T, client *testlib.Client, c client.CreateChannelOrFail(channelName, &channel) client.WaitForResourceReadyOrFail(channelName, &channel) - pod := resources.EventRecordPod(subscriberServiceName + "-pod") - client.CreatePodOrFail(pod, testlib.WithService(subscriberServiceName)) + _ = recordevents.DeployEventRecordOrFail(context.TODO(), client, subscriberServiceName+"-pod") subscription := client.CreateSubscriptionOrFail( subscriberServiceName, diff --git a/test/conformance/helpers/channel_tracing_test_helper.go b/test/conformance/helpers/channel_tracing_test_helper.go index 2447971b1cb..b493960ea15 100644 --- a/test/conformance/helpers/channel_tracing_test_helper.go +++ b/test/conformance/helpers/channel_tracing_test_helper.go @@ -28,6 +28,7 @@ import ( tracinghelper "knative.dev/eventing/test/conformance/helpers/tracing" testlib "knative.dev/eventing/test/lib" + "knative.dev/eventing/test/lib/recordevents" "knative.dev/eventing/test/lib/resources" "knative.dev/eventing/test/lib/sender" ) @@ -66,8 +67,7 @@ func setupChannelTracingWithReply( client.CreateChannelOrFail(replyChannelName, channel) // Create the 'sink', a LogEvents Pod and a K8s Service that points to it. - recordEventsPod := resources.EventRecordPod(recordEventsPodName) - client.CreatePodOrFail(recordEventsPod, testlib.WithService(recordEventsPodName)) + recordEventsPod := recordevents.DeployEventRecordOrFail(ctx, client, recordEventsPodName) // Create the subscriber, a Pod that mutates the event. transformerPod := resources.EventTransformationPod( diff --git a/test/conformance/helpers/tracing_test_helper.go b/test/conformance/helpers/tracing_test_helper.go index 9f4d5772f0d..d5d1a84ef45 100644 --- a/test/conformance/helpers/tracing_test_helper.go +++ b/test/conformance/helpers/tracing_test_helper.go @@ -65,7 +65,7 @@ func tracingTest( expectedTestSpan, eventMatcher := setupInfrastructure(t, &channel, client, recordEventsPodName, true) // Start the event info store and assert the event was received correctly - targetTracker, err := recordevents.NewEventInfoStore(client, recordEventsPodName) + targetTracker, err := recordevents.NewEventInfoStore(client, recordEventsPodName, client.Namespace) if err != nil { t.Fatalf("Pod tracker failed: %v", err) } @@ -105,6 +105,6 @@ func getTraceIDHeader(t *testing.T, evInfos []recordevents.EventInfo) string { } } } - t.Fatalf("FAIL: No traceid in %d messages: (%s)", len(evInfos), evInfos) + t.Fatalf("FAIL: No traceid in %d messages: (%v)", len(evInfos), evInfos) return "" } diff --git a/test/e2e/helpers/trigger_no_broker_test_helper.go b/test/e2e/helpers/trigger_no_broker_test_helper.go index 0f7d04f2445..8a4a6764f1e 100644 --- a/test/e2e/helpers/trigger_no_broker_test_helper.go +++ b/test/e2e/helpers/trigger_no_broker_test_helper.go @@ -26,6 +26,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" testlib "knative.dev/eventing/test/lib" + "knative.dev/eventing/test/lib/recordevents" "knative.dev/eventing/test/lib/resources" ) @@ -40,8 +41,8 @@ func TestTriggerNoBroker(t *testing.T, channel string, brokerCreator BrokerCreat brokerName := strings.ToLower(channel) subscriberName := "dumper-empty" - eventRecordPod := resources.EventRecordPod(subscriberName) - client.CreatePodOrFail(eventRecordPod, testlib.WithService(subscriberName)) + recordevents.DeployEventRecordOrFail(context.TODO(), client, subscriberName) + client.CreateTriggerOrFailV1Beta1("testtrigger", resources.WithSubscriberServiceRefForTriggerV1Beta1(subscriberName), resources.WithBrokerV1Beta1(brokerName), diff --git a/test/lib/client.go b/test/lib/client.go index b26e043473a..f8868acdca5 100644 --- a/test/lib/client.go +++ b/test/lib/client.go @@ -47,6 +47,8 @@ type Client struct { Dynamic dynamic.Interface Config *rest.Config + EventListener *EventListener + Namespace string T *testing.T Tracker *Tracker @@ -92,6 +94,10 @@ func NewClient(configPath string, clusterName string, namespace string, t *testi client.T = t client.Tracker = NewTracker(t, client.Dynamic) + // Start informer + client.EventListener = NewEventListener(client.Kube.Kube, client.Namespace, client.T.Logf) + client.Cleanup(client.EventListener.Stop) + client.tracingEnv, err = getTracingConfig(client.Kube.Kube) if err != nil { return nil, err diff --git a/test/lib/k8s_events.go b/test/lib/k8s_events.go new file mode 100644 index 00000000000..1188d5c234f --- /dev/null +++ b/test/lib/k8s_events.go @@ -0,0 +1,84 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package lib + +import ( + "context" + "sync" + + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" +) + +// EventHandler is the callback type for the EventListener +type EventHandler func(event *corev1.Event) + +// EventListener is a type that broadcasts new k8s events +type EventListener struct { + cancel context.CancelFunc + + lock sync.Mutex + handlers []EventHandler +} + +// NewEventListener creates a new event listener +func NewEventListener(client kubernetes.Interface, namespace string, logf func(string, ...interface{})) *EventListener { + ctx, cancelCtx := context.WithCancel(context.Background()) + informerFactory := informers.NewSharedInformerFactoryWithOptions( + client, + 0, + informers.WithNamespace(namespace), + ) + eventsInformer := informerFactory.Core().V1().Events().Informer() + + el := EventListener{ + cancel: cancelCtx, + } + + eventsInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + el.handle(obj.(*corev1.Event)) + }, + }) + + go func() { + eventsInformer.Run(ctx.Done()) + logf("EventListener stopped") + }() + + return &el +} + +func (el *EventListener) handle(event *corev1.Event) { + el.lock.Lock() + defer el.lock.Unlock() + for _, handler := range el.handlers { + handler(event) + } +} + +func (el *EventListener) AddHandler(handler EventHandler) { + el.lock.Lock() + defer el.lock.Unlock() + el.handlers = append(el.handlers, handler) +} + +func (el *EventListener) Stop() { + el.cancel() +} diff --git a/test/lib/recordevents/event_info.go b/test/lib/recordevents/event_info.go index e3fac767959..d9dbbdae819 100644 --- a/test/lib/recordevents/event_info.go +++ b/test/lib/recordevents/event_info.go @@ -17,55 +17,31 @@ limitations under the License. package recordevents import ( - "encoding/json" "fmt" - "io/ioutil" - "math/rand" - "net/http" "strings" "time" cloudevents "github.com/cloudevents/sdk-go/v2" - corev1 "k8s.io/api/core/v1" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/kubernetes" - "knative.dev/pkg/test/monitoring" - - "knative.dev/pkg/test/logging" - - testlib "knative.dev/eventing/test/lib" ) -// Port for the recordevents pod REST listener -const RecordEventsPort = 8392 - -// HTTP path for the GetMinMax REST call -const GetMinMaxPath = "/minmax" - -// HTTP path for the GetEntry REST call -const GetEntryPath = "/entry/" - -// HTTP path for the TrimThrough REST call -const TrimThroughPath = "/trimthrough/" - -// On-wire json rest api format for recordevents GetMinMax calls -// sennt to the recordevents pod. -type MinMaxResponse struct { - MinAvail int - MaxSeen int -} +const ( + // EventReason is the Kubernetes event reason used for observed events. + CloudEventObservedReason = "CloudEventObserved" +) // Structure to hold information about an event seen by recordevents pod. type EventInfo struct { // Set if the http request received by the pod couldn't be decoded or // didn't pass validation - Error string + Error string `json:"error,omitempty"` // Event received if the cloudevent received by the pod passed validation - Event *cloudevents.Event + Event *cloudevents.Event `json:"event,omitempty"` // HTTPHeaders of the connection that delivered the event - HTTPHeaders map[string][]string + HTTPHeaders map[string][]string `json:"httpHeaders,omitempty"` + Origin string `json:"origin,omitempty"` + Observer string `json:"observer,omitempty"` + Time time.Time `json:"time,omitempty"` + Sequence uint64 `json:"sequence"` } // Pretty print the event. Meant for debugging. This formats the validation error @@ -94,189 +70,3 @@ func (s *SearchedInfo) String() string { } return sb.String() } - -// Connection state for a REST connection to a pod -type eventGetter struct { - podName string - podNamespace string - podPort int - kubeClientset kubernetes.Interface - logf logging.FormatLogger - - host string - port int - forwardPID int -} - -// Creates a forwarded port to the specified recordevents pod and waits until -// it can successfully talk to the REST API. Times out after timeoutEvRetry -func newEventGetter(podName string, client *testlib.Client, logf logging.FormatLogger) (eventGetterInterface, error) { - egi := &eventGetter{podName: podName, podNamespace: client.Namespace, - kubeClientset: client.Kube.Kube, podPort: RecordEventsPort, logf: logf} - err := egi.forwardPort() - if err != nil { - return nil, err - } - - err = egi.waitTillUp() - if err != nil { - return nil, err - } - return egi, nil -} - -// Get information about the provided podName. Uses list (rather than get) and -// returns a pod list for compatibility with the monitoring.PortForward -// interface -func (eg *eventGetter) getRunningPodInfo(podName, namespace string) (*v1.PodList, error) { - pods, err := eg.kubeClientset.CoreV1().Pods(namespace).List( - metav1.ListOptions{FieldSelector: fmt.Sprintf("metadata.name=%s", podName)}) - if err == nil && len(pods.Items) != 1 { - err = fmt.Errorf("no %s Pod found on the cluster", podName) - } else if pods.Items[0].Status.Phase != corev1.PodRunning { - err = fmt.Errorf("pod %s in state %s, wanted Running", podName, - pods.Items[0].Status.Phase) - } - - return pods, err -} - -// Try to forward the pod port to a local port somewhere in the range 30000-60000. -// keeps retrying with random ports in that range, timing out after timeoutEvRetry -func (eg *eventGetter) forwardPort() error { - portRand := rand.New(rand.NewSource(time.Now().UnixNano())) - portMin := 30000 - portMax := 60000 - var internalErr error - - wait.PollImmediate(minEvRetryInterval, timeoutEvRetry, func() (bool, error) { - localPort := portMin + portRand.Intn(portMax-portMin) - if err := monitoring.CheckPortAvailability(localPort); err != nil { - internalErr = err - return false, nil - } - pods, err := eg.getRunningPodInfo(eg.podName, eg.podNamespace) - if err != nil { - internalErr = err - return false, nil - } - - pid, err := monitoring.PortForward(eg.logf, pods, localPort, eg.podPort, eg.podNamespace) - if err != nil { - internalErr = err - return false, nil - } - internalErr = nil - - eg.forwardPID = pid - eg.port = localPort - eg.host = "localhost" - return true, nil - }) - if internalErr != nil { - return fmt.Errorf("timeout forwarding port: %v", internalErr) - } - return nil -} - -// Return the min available, max seen by the recordevents pod. -// maxRet is the largest event that has ever been seen (whether it's been trimmed -// or not). minRet is the smallest event still available via Get, or 1+maxRet if -// no events are available. maxRet starts at 0 when no events have been seen. -func (eg *eventGetter) getMinMax() (minRet int, maxRet int, errRet error) { - resp, err := http.Get(fmt.Sprintf("http://%s:%d%s", eg.host, eg.port, GetMinMaxPath)) - if err != nil { - return -1, -1, fmt.Errorf("http get error: %v", err) - } - defer resp.Body.Close() - bodyContents, err := ioutil.ReadAll(resp.Body) - if err != nil { - return -1, -1, fmt.Errorf("error reading response body %w", err) - } - if resp.StatusCode != http.StatusOK { - return -1, -1, fmt.Errorf("error %d reading GetMinMax response", resp.StatusCode) - } - minMaxResponse := MinMaxResponse{} - err = json.Unmarshal(bodyContents, &minMaxResponse) - if err != nil { - return -1, -1, fmt.Errorf("error unmarshalling response %w", err) - } - if minMaxResponse.MinAvail == 0 { - return -1, -1, fmt.Errorf("invalid decoded json: %+v", minMaxResponse) - } - - return minMaxResponse.MinAvail, minMaxResponse.MaxSeen, nil -} - -// Return the event with the provided sequence number. Returns the appropriate -// EventInfo or an error if no such event is known, or the event has already -// been trimmed. -func (eg *eventGetter) getEntry(seqno int) (EventInfo, error) { - resp, err := http.Get(fmt.Sprintf("http://%s:%d%s/%d", eg.host, eg.port, GetEntryPath, seqno)) - if err != nil { - return EventInfo{}, fmt.Errorf("http get err %v", err) - } - defer resp.Body.Close() - bodyContents, err := ioutil.ReadAll(resp.Body) - if err != nil { - return EventInfo{}, fmt.Errorf("error reading response body %w", err) - } - if resp.StatusCode != http.StatusOK { - return EventInfo{}, fmt.Errorf("error %d reading GetEntry response", resp.StatusCode) - } - entryResponse := EventInfo{} - err = json.Unmarshal(bodyContents, &entryResponse) - if err != nil { - return EventInfo{}, fmt.Errorf("error unmarshalling response %w", err) - } - if len(entryResponse.Error) == 0 && entryResponse.Event == nil { - return EventInfo{}, fmt.Errorf("invalid decoded json: %+v", entryResponse) - } - - return entryResponse, nil -} - -// Trim the events up to and including seqno from the recordevents pod. -// Returns an error if a nonsensical seqno is passed in, but does not return -// error for trimming already trimmed regions. -func (eg *eventGetter) trimThrough(seqno int) error { - resp, err := http.Post(fmt.Sprintf("http://%s:%d%s/%d", eg.host, eg.port, TrimThroughPath, seqno), "", nil) - if err != nil { - return fmt.Errorf("http post err %v", err) - } - defer resp.Body.Close() - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return fmt.Errorf("error reading response body %w", err) - } - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("error %d reading TrimThrough response: %s", resp.StatusCode, string(body)) - } - - return nil -} - -// Clean up the getter by tearing down the port forward. -func (eg *eventGetter) cleanup() { - pid := eg.forwardPID - eg.forwardPID = 0 - if pid != 0 { - monitoring.Cleanup(pid) - } -} - -// Wait (up to timeoutEvRetry) for the pod to RestAPI to answer request. -func (eg *eventGetter) waitTillUp() error { - var internalErr error - wait.PollImmediate(minEvRetryInterval, timeoutEvRetry, func() (bool, error) { - _, _, internalErr = eg.getMinMax() - if internalErr != nil { - return false, nil - } - return true, nil - }) - if internalErr != nil { - return fmt.Errorf("timeout waiting for recordevents pod to come up: %v", internalErr) - } - return nil -} diff --git a/test/lib/recordevents/event_info_store.go b/test/lib/recordevents/event_info_store.go index 3b0c46cf44e..007a4c14952 100644 --- a/test/lib/recordevents/event_info_store.go +++ b/test/lib/recordevents/event_info_store.go @@ -17,6 +17,8 @@ limitations under the License. package recordevents import ( + "context" + "encoding/json" "fmt" "strconv" "strings" @@ -26,6 +28,7 @@ import ( "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/util/wait" pkgTest "knative.dev/pkg/test" @@ -35,169 +38,106 @@ import ( const ( // The interval and timeout used for checking events - minEvRetryInterval = 4 * time.Second - timeoutEvRetry = 4 * time.Minute + retryInterval = 4 * time.Second + retryTimeout = 4 * time.Minute ) -// Stateful store of events received by the recordevents pod it is pointed at. -// This pulls events from the pod during any Find or Wait call, storing them -// locally and triming them from the remote pod store. -type EventInfoStore struct { - tb testing.TB - - podName string - getter eventGetterInterface - - lock sync.Mutex - allEvents []EventInfo - firstID int - closeCh chan struct{} - doRefresh chan chan error - timeout time.Duration - retryInterval time.Duration -} - -// Functions used for getting data from the REST api of the recordevents pod. -// The interface exists for use with unit tests of this module. -type eventGetterInterface interface { - getMinMax() (minRet int, maxRet int, errRet error) - getEntry(seqno int) (EventInfo, error) - trimThrough(seqno int) error - cleanup() -} - -// Internal function to create an event store. This is called directly by unit tests of -// this module. -func newTestableEventInfoStore(egi eventGetterInterface, retryInterval time.Duration, - timeout time.Duration) *EventInfoStore { - if timeout == -1 { - timeout = timeoutEvRetry - } - if retryInterval == -1 { - retryInterval = minEvRetryInterval - } - ei := &EventInfoStore{getter: egi, firstID: 1, timeout: timeout, retryInterval: retryInterval} - ei.start() - return ei -} +type EventRecordOption = func(*corev1.Pod, *testlib.Client) error -// Creates an EventInfoStore that is used to iteratively download events recorded by the -// recordevents pod. Calling this forwards the recordevents port to the local machine -// and blocks waiting to connect to that pod. Fails if it cannot connect within -// the expected timeout (4 minutes currently) -func NewEventInfoStore(client *testlib.Client, podName string) (*EventInfoStore, error) { - egi, err := newEventGetter(podName, client, client.T.Logf) +func DeployEventRecordOrFail(ctx context.Context, client *testlib.Client, name string, options ...EventRecordOption) *corev1.Pod { + client.CreateServiceAccountOrFail(name) + client.CreateRoleOrFail(resources.Role(name, + resources.WithRuleForRole(&rbacv1.PolicyRule{ + APIGroups: []string{""}, + Resources: []string{"pods"}, + Verbs: []string{"get"}, + }), + resources.WithRuleForRole(&rbacv1.PolicyRule{ + APIGroups: []string{""}, + Resources: []string{"events"}, + Verbs: []string{rbacv1.VerbAll}, + }), + )) + client.CreateRoleBindingOrFail(name, "Role", name, name, client.Namespace) + + eventRecordPod := EventRecordPod(name, name) + client.CreatePodOrFail(eventRecordPod, append(options, testlib.WithService(name))...) + err := pkgTest.WaitForPodRunning(ctx, client.Kube, name, client.Namespace) if err != nil { - return nil, err + client.T.Fatalf("Failed to start the recordevent pod '%s': %v", name, errors.WithStack(err)) } - ei := newTestableEventInfoStore(egi, -1, -1) - ei.podName = podName - ei.tb = client.T - client.Cleanup(ei.cleanup) - return ei, nil + client.WaitForServiceEndpointsOrFail(ctx, name, 1) + return eventRecordPod } -type EventRecordOption = func(*corev1.Pod, *testlib.Client) error - // Deploys a new recordevents pod and start the associated EventInfoStore -func StartEventRecordOrFail(client *testlib.Client, podName string, options ...EventRecordOption) (*EventInfoStore, *corev1.Pod) { - eventRecordPod := resources.EventRecordPod(podName) - client.CreatePodOrFail(eventRecordPod, append(options, testlib.WithService(podName))...) - err := pkgTest.WaitForPodRunning(client.Kube, podName, client.Namespace) - if err != nil { - client.T.Fatalf("Failed to start the recordevent pod '%s': %v", podName, errors.WithStack(err)) - } - client.WaitForServiceEndpointsOrFail(podName, 1) +func StartEventRecordOrFail(ctx context.Context, client *testlib.Client, podName string, options ...EventRecordOption) (*EventInfoStore, *corev1.Pod) { + eventRecordPod := DeployEventRecordOrFail(ctx, client, podName, options...) - eventTracker, err := NewEventInfoStore(client, podName) + eventTracker, err := NewEventInfoStore(client, podName, client.Namespace) if err != nil { client.T.Fatalf("Failed to start the EventInfoStore associated to pod '%s': %v", podName, err) } return eventTracker, eventRecordPod } -// Starts the single threaded background goroutine used to update local state -// from the remote REST API. -func (ei *EventInfoStore) start() { - ei.closeCh = make(chan struct{}) - ei.doRefresh = make(chan chan error) - go func() { - for { - select { - case <-ei.closeCh: - ei.getter.cleanup() - return - case replyCh := <-ei.doRefresh: - replyCh <- ei.doRetrieveData() - } - } - }() +// Stateful store of events received by the recordevents pod it is pointed at. +// This pulls events from the pod during any Find or Wait call, storing them +// locally and triming them from the remote pod store. +type EventInfoStore struct { + tb testing.TB + + podName string + podNamespace string + + lock sync.Mutex + collected []EventInfo } -// The data update thread used by the single threaded background goroutine -// for updating data from the REST api. -func (ei *EventInfoStore) doRetrieveData() error { - min, max, err := ei.getter.getMinMax() - if err != nil { - return fmt.Errorf("error getting MinMax %v", err) - } - ei.lock.Lock() - curMin := ei.firstID - curMax := curMin + len(ei.allEvents) - 1 - ei.lock.Unlock() - if min == max+1 { - // Nothing to read or trim - return nil - } else { - if min > curMax+1 { - return fmt.Errorf("mismatched stored max/available min: %d, %d", curMax, min) - } - min = curMax + 1 - // We may have data to read, definitely have data to trim. +// Creates an EventInfoStore that is used to iteratively download events recorded by the +// recordevents pod. +func NewEventInfoStore(client *testlib.Client, podName string, podNamespace string) (*EventInfoStore, error) { + store := &EventInfoStore{ + tb: client.T, + podName: podName, + podNamespace: podNamespace, } - var newEvents []EventInfo - for i := min; i <= max; i++ { - e, err := ei.getter.getEntry(i) - if err != nil { - return fmt.Errorf("error calling getEntry of %d %v", i, err) - } - newEvents = append(newEvents, e) - } - ei.lock.Lock() - ei.allEvents = append(ei.allEvents, newEvents...) - ei.lock.Unlock() - err = ei.getter.trimThrough(max) - return err + client.EventListener.AddHandler(store.handle) + return store, nil } -// Clean up any background resources used by the store. Must be called exactly once after -// the last use. -func (ei *EventInfoStore) cleanup() { - close(ei.closeCh) +func (ei *EventInfoStore) getEventInfo() []EventInfo { + ei.lock.Lock() + defer ei.lock.Unlock() + return ei.collected } -//TODO remove it, this is not useful anymore -// Deprecated: you can remove the manual cleanup of the event getter, since now it's done at test tear down automatically -func (ei *EventInfoStore) Cleanup() {} - -// Called internally by functions wanting the current list of all -// known events. This calls for an update from the REST server and -// returns the summary of all locally and remotely known events. -// Returns an error in case of a connection or protocol error. -func (ei *EventInfoStore) refreshData() ([]EventInfo, error) { - var allEvents []EventInfo - replyCh := make(chan error) - ei.doRefresh <- replyCh - err := <-replyCh +func (ei *EventInfoStore) handle(event *corev1.Event) { + // Filter events + if !ei.isMyEvent(event) { + return + } + + eventInfo := EventInfo{} + err := json.Unmarshal([]byte(event.Message), &eventInfo) if err != nil { - return nil, err + ei.tb.Errorf("Received EventInfo that cannot be unmarshalled! %+v", err) + return } + ei.lock.Lock() - allEvents = append(allEvents, ei.allEvents...) - ei.lock.Unlock() - return allEvents, nil + defer ei.lock.Unlock() + ei.collected = append(ei.collected, eventInfo) +} + +func (ei *EventInfoStore) isMyEvent(event *corev1.Event) bool { + return event.Type == corev1.EventTypeNormal && + event.Reason == CloudEventObservedReason && + event.InvolvedObject.Kind == "Pod" && + event.InvolvedObject.Name == ei.podName && + event.InvolvedObject.Namespace == ei.podNamespace } // Find all events received by the recordevents pod that match the provided matchers, @@ -215,10 +155,7 @@ func (ei *EventInfoStore) Find(matchers ...EventInfoMatcher) ([]EventInfo, Searc lastEvents := []EventInfo{} var nonMatchingErrors []error - allEvents, err := ei.refreshData() - if err != nil { - return nil, sInfo, nonMatchingErrors, fmt.Errorf("error getting events %v", err) - } + allEvents := ei.getEventInfo() for i := range allEvents { if err := f(allEvents[i]); err == nil { allMatch = append(allMatch, allEvents[i]) @@ -240,27 +177,31 @@ func (ei *EventInfoStore) Find(matchers ...EventInfoMatcher) ([]EventInfo, Searc // Assert that there are at least min number of match for the provided matchers. // This method fails the test if the assert is not fulfilled. func (ei *EventInfoStore) AssertAtLeast(min int, matchers ...EventInfoMatcher) []EventInfo { + ei.tb.Helper() events, err := ei.waitAtLeastNMatch(AllOf(matchers...), min) if err != nil { ei.tb.Fatalf("Timeout waiting for at least %d matches.\nError: %+v", min, errors.WithStack(err)) } + ei.tb.Logf("Assert passed") return events } // Assert that there are at least min number of matches and at most max number of matches for the provided matchers. // This method fails the test if the assert is not fulfilled. func (ei *EventInfoStore) AssertInRange(min int, max int, matchers ...EventInfoMatcher) []EventInfo { + ei.tb.Helper() events := ei.AssertAtLeast(min, matchers...) if max > 0 && len(events) > max { ei.tb.Fatalf("Assert in range failed: %+v", errors.WithStack(fmt.Errorf("expected <= %d events, saw %d", max, len(events)))) } - + ei.tb.Logf("Assert passed") return events } // Assert that there aren't any matches for the provided matchers. // This method fails the test if the assert is not fulfilled. func (ei *EventInfoStore) AssertNot(matchers ...EventInfoMatcher) []EventInfo { + ei.tb.Helper() res, recentEvents, _, err := ei.Find(matchers...) if err != nil { ei.tb.Fatalf("Unexpected error during find on recordevents '%s': %+v", ei.podName, errors.WithStack(err)) @@ -271,14 +212,17 @@ func (ei *EventInfoStore) AssertNot(matchers ...EventInfoMatcher) []EventInfo { fmt.Errorf("Unexpected matches on recordevents '%s', found: %v. %s", ei.podName, res, &recentEvents)), ) } - + ei.tb.Logf("Assert passed") return res } // Assert that there are exactly n matches for the provided matchers. // This method fails the test if the assert is not fulfilled. func (ei *EventInfoStore) AssertExact(n int, matchers ...EventInfoMatcher) []EventInfo { - return ei.AssertInRange(n, n, matchers...) + ei.tb.Helper() + events := ei.AssertInRange(n, n, matchers...) + ei.tb.Logf("Assert passed") + return events } // Wait a long time (currently 4 minutes) until the provided function matches at least @@ -289,7 +233,7 @@ func (ei *EventInfoStore) waitAtLeastNMatch(f EventInfoMatcher, min int) ([]Even var matchRet []EventInfo var internalErr error - wait.PollImmediate(ei.retryInterval, ei.timeout, func() (bool, error) { + wait.PollImmediate(retryInterval, retryTimeout, func() (bool, error) { allMatch, sInfo, matchErrs, err := ei.Find(f) if err != nil { internalErr = fmt.Errorf("FAIL MATCHING: unexpected error during find: %v", err) diff --git a/test/lib/recordevents/event_info_store_test.go b/test/lib/recordevents/event_info_store_test.go deleted file mode 100644 index 52f09c76fb8..00000000000 --- a/test/lib/recordevents/event_info_store_test.go +++ /dev/null @@ -1,279 +0,0 @@ -/* -Copyright 2020 The Knative Authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package recordevents - -import ( - "fmt" - "strconv" - "sync" - "testing" - "time" - - cloudevents "github.com/cloudevents/sdk-go/v2" -) - -type dummyEventGet struct { - lock sync.Mutex - - minSeq int - allEv []EventInfo - clean bool - tCalls int -} - -func newDummyEventGet() *dummyEventGet { - return &dummyEventGet{minSeq: 1} -} - -func (deg *dummyEventGet) getMinMax() (minRet int, maxRet int, errRet error) { - deg.lock.Lock() - defer deg.lock.Unlock() - if deg.clean { - panic("getminmax called on cleaned event getter") - } - min := deg.minSeq - max := min + len(deg.allEv) - 1 - return min, max, nil -} -func (deg *dummyEventGet) getEntry(seqno int) (EventInfo, error) { - deg.lock.Lock() - defer deg.lock.Unlock() - if deg.clean { - panic("getentry called on cleaned event getter") - } - min := deg.minSeq - max := min + len(deg.allEv) - 1 - if seqno >= min && seqno <= max { - return deg.allEv[seqno-deg.minSeq], nil - } else { - return EventInfo{}, fmt.Errorf("illegal get seqno: %d, range min = %d, max = %d", seqno, min, max) - } -} -func (deg *dummyEventGet) trimThrough(seqno int) error { - deg.lock.Lock() - defer deg.lock.Unlock() - deg.tCalls++ - if deg.clean { - panic("trim called on cleaned event getter") - } - min := deg.minSeq - max := min + len(deg.allEv) - 1 - if seqno < 0 { - return fmt.Errorf("Illegal negative seqno %d", seqno) - } else if seqno > max { - return fmt.Errorf("Illegal negative seqno %d > %d", seqno, max) - } else if seqno < min { - return nil - } - deg.allEv = deg.allEv[seqno-min+1:] - deg.minSeq = seqno + 1 - return nil -} -func (deg *dummyEventGet) trimCalls() int { - deg.lock.Lock() - defer deg.lock.Unlock() - return deg.tCalls -} -func (deg *dummyEventGet) cleanup() { - deg.lock.Lock() - defer deg.lock.Unlock() - if deg.clean { - panic("Unexpected clean") - } else { - deg.clean = true - } -} - -func makeEvents() []EventInfo { - var allEv []EventInfo - for i := 0; i < 30; i++ { - ce := cloudevents.NewEvent(cloudevents.VersionV1) - ce.SetType("knative.dev.test.event.a") - ce.SetSource("https://source.test.event.knative.dev/foo") - ce.SetID(strconv.FormatInt(int64(i), 10)) - allEv = append(allEv, EventInfo{Event: &ce}) - } - return allEv -} - -func checkEvIDEqual(t *testing.T, seen []EventInfo, expected []EventInfo) { - if len(seen) != len(expected) { - t.Fatalf("Seen ev list length %d does not match expected %d", len(seen), len(expected)) - - } - for i := range seen { - if seen[i].Event.ID() != expected[i].Event.ID() { - t.Fatalf("Id entry %d mismatch: seen %s != expected %s", - i, seen[i].Event.ID(), expected[i].Event.ID()) - } - } -} - -func (deg *dummyEventGet) setEv(firstID int, allEv []EventInfo) { - deg.lock.Lock() - defer deg.lock.Unlock() - deg.minSeq = firstID - deg.allEv = allEv -} - -// Test that Finds where the server gives sequential updates that -// don't overlap see the expected events. -func TestSequentialAndTrim(t *testing.T) { - totalEv := makeEvents() - expectedFull := makeEvents() - deg := newDummyEventGet() - subEv := totalEv[:10] - deg.setEv(1, subEv) - ei := newTestableEventInfoStore(deg, -1, -1) - allData, _, _, err := ei.Find(func(EventInfo) error { return nil }) - if err != nil { - t.Fatalf("Unexpected error from find: %v", err) - } - checkEvIDEqual(t, allData, expectedFull[:10]) - min, _, _ := deg.getMinMax() - if min != 11 { - t.Fatalf("Expected trim to have moved min to %d, saw %d", 11, min) - } - - subEv = totalEv[10:19] - deg.setEv(11, subEv) - - allData, _, _, err = ei.Find(func(EventInfo) error { return nil }) - if err != nil { - t.Fatalf("Unexpected error from find: %v", err) - } - checkEvIDEqual(t, allData, expectedFull[:19]) - ei.cleanup() -} - -// Test that Finds where the server gives overlapping updates -// see the expected events (no double events) -func TestOverlap(t *testing.T) { - totalEv := makeEvents() - expectedFull := makeEvents() - deg := newDummyEventGet() - subEv := totalEv[:10] - deg.setEv(1, subEv) - ei := newTestableEventInfoStore(deg, -1, -1) - allData, _, _, err := ei.Find(func(EventInfo) error { return nil }) - if err != nil { - t.Fatalf("Unexpected error from find: %v", err) - } - checkEvIDEqual(t, allData, expectedFull[:10]) - subEv = totalEv[6:19] - deg.setEv(7, subEv) - allData, _, _, err = ei.Find(func(EventInfo) error { return nil }) - if err != nil { - t.Fatalf("Unexpected error from find: %v", err) - } - checkEvIDEqual(t, allData, expectedFull[:19]) - ei.cleanup() -} - -// Test that we see an error if repeated Finds see a gap in the sequence -// space -func TestGap(t *testing.T) { - totalEv := makeEvents() - expectedFull := makeEvents() - deg := newDummyEventGet() - subEv := totalEv[:10] - deg.setEv(1, subEv) - ei := newTestableEventInfoStore(deg, -1, -1) - allData, _, _, err := ei.Find(func(EventInfo) error { return nil }) - if err != nil { - t.Fatalf("Unexpected error from find: %v", err) - } - checkEvIDEqual(t, allData, expectedFull[:10]) - subEv = totalEv[11:19] - deg.setEv(12, subEv) - _, _, _, err = ei.Find(func(EventInfo) error { return nil }) - if err == nil { - t.Fatalf("Unexpected success from find") - } - ei.cleanup() -} - -// Test that two finds, with the second one having -// no new events, returns the right events. -func TestSequentialNoOp(t *testing.T) { - totalEv := makeEvents() - expectedFull := makeEvents() - deg := newDummyEventGet() - subEv := totalEv[:10] - deg.setEv(1, subEv) - ei := newTestableEventInfoStore(deg, -1, -1) - allData, _, _, err := ei.Find(func(EventInfo) error { return nil }) - if err != nil { - t.Fatalf("Unexpected error from find: %v", err) - } - checkEvIDEqual(t, allData, expectedFull[:10]) - subEv = []EventInfo{} - deg.setEv(11, subEv) - allData, _, _, err = ei.Find(func(EventInfo) error { return nil }) - if err != nil { - t.Fatalf("Unexpected error from find: %v", err) - } - checkEvIDEqual(t, allData, expectedFull[:10]) - ei.cleanup() -} - -// Test that wait for N Works -func TestWaitForN(t *testing.T) { - deg := newDummyEventGet() - subEv := makeEvents()[:10] - deg.setEv(1, subEv) - ei := newTestableEventInfoStore(deg, time.Second/8, -1) - var wg sync.WaitGroup - wg.Add(1) - var waitErr error - var allMatch []EventInfo - go func() { - matchFunc := func(ev cloudevents.Event) error { - if ev.ID() == "3" { - return nil - } else { - return fmt.Errorf("mismatch %s %s", ev.ID(), "3") - } - } - - allMatch, waitErr = ei.waitAtLeastNMatch(MatchEvent(matchFunc), 2) - wg.Done() - }() - var tCalls int - for tCalls == 0 { - time.Sleep(time.Second / 100) - tCalls = deg.trimCalls() - } - subEv = makeEvents()[:10] - deg.setEv(11, subEv) - - wg.Wait() - if waitErr != nil { - t.Fatalf("Unexpected error from find: %v", waitErr) - } - if len(allMatch) != 2 { - t.Fatalf("Unexpected match length: %d != %d", len(allMatch), 2) - } - if allMatch[0].Event.ID() != "3" || allMatch[1].Event.ID() != "3" { - t.Fatalf("Unexpected IDs %s, %s, expected both %s", allMatch[0].Event.ID(), allMatch[1].Event.ID(), "3") - } - tCalls = deg.trimCalls() - if tCalls < 2 { - t.Fatalf("Expected at least %d trim calls, saw %d", 2, tCalls) - } - ei.cleanup() -} diff --git a/test/lib/recordevents/event_log.go b/test/lib/recordevents/event_log.go new file mode 100644 index 00000000000..62aba6113b4 --- /dev/null +++ b/test/lib/recordevents/event_log.go @@ -0,0 +1,33 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package recordevents + +// EventLog is the contract for an event logger to vent an event. +type EventLog interface { + Vent(observed EventInfo) error +} + +type EventLogs []EventLog + +func (e EventLogs) Vent(observed EventInfo) error { + for _, el := range e { + if err := el.Vent(observed); err != nil { + return err + } + } + return nil +} diff --git a/test/lib/recordevents/observer/observer.go b/test/lib/recordevents/observer/observer.go new file mode 100644 index 00000000000..e9bd4224aac --- /dev/null +++ b/test/lib/recordevents/observer/observer.go @@ -0,0 +1,116 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observer + +import ( + "context" + "net/http" + "sync/atomic" + "time" + + cloudeventsbindings "github.com/cloudevents/sdk-go/v2/binding" + cloudeventshttp "github.com/cloudevents/sdk-go/v2/protocol/http" + "github.com/kelseyhightower/envconfig" + "github.com/prometheus/common/log" + "knative.dev/pkg/logging" + + "knative.dev/eventing/test/lib/recordevents" +) + +// Observer is the entry point for sinking events into the event log. +type Observer struct { + // Name is the name of this Observer, used to filter if multiple observers. + Name string + // EventLogs is the list of EventLog implementors to vent observed events. + EventLogs recordevents.EventLogs + + seq uint64 +} + +// New returns an observer that will vent observations to the list of provided +// EventLog instances. It will listen on :8080. +func New(name string, eventLogs ...recordevents.EventLog) *Observer { + return &Observer{ + Name: name, + EventLogs: eventLogs, + } +} + +type envConfig struct { + // ObserverName is used to identify this instance of the observer. + ObserverName string `envconfig:"OBSERVER_NAME" default:"observer-default" required:"true"` +} + +func NewFromEnv(eventLogs ...recordevents.EventLog) *Observer { + var env envConfig + if err := envconfig.Process("", &env); err != nil { + log.Fatal("Failed to process env var", err) + } + + return New(env.ObserverName, eventLogs...) +} + +// Start will create the CloudEvents client and start listening for inbound +// HTTP requests. This is a is a blocking call. +func (o *Observer) Start(ctx context.Context, handlerFuncs ...func(handler http.Handler) http.Handler) error { + var handler http.Handler = o + + for _, dec := range handlerFuncs { + handler = dec(handler) + } + + server := &http.Server{Addr: ":8080", Handler: handler} + + go func() { + if err := server.ListenAndServe(); err != nil { + logging.FromContext(ctx).Fatal("Error while starting the HTTP server", err) + } + }() + + <-ctx.Done() + + logging.FromContext(ctx).Info("Closing the HTTP server") + + return server.Close() +} + +func (o *Observer) ServeHTTP(writer http.ResponseWriter, request *http.Request) { + m := cloudeventshttp.NewMessageFromHttpRequest(request) + defer m.Finish(nil) + + event, eventErr := cloudeventsbindings.ToEvent(context.TODO(), m) + header := request.Header + + eventErrStr := "" + if eventErr != nil { + eventErrStr = eventErr.Error() + } + err := o.EventLogs.Vent(recordevents.EventInfo{ + Error: eventErrStr, + Event: event, + HTTPHeaders: header, + Origin: request.RemoteAddr, + Observer: o.Name, + Time: time.Now(), + Sequence: atomic.AddUint64(&o.seq, 1), + }) + if err != nil { + log.Warn("Error while venting the recorded event", err) + } + + writer.WriteHeader(http.StatusAccepted) +} diff --git a/test/lib/recordevents/recorder_vent/constructor.go b/test/lib/recordevents/recorder_vent/constructor.go new file mode 100644 index 00000000000..cc73a9671f0 --- /dev/null +++ b/test/lib/recordevents/recorder_vent/constructor.go @@ -0,0 +1,108 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package recorder_vent + +import ( + "context" + "log" + "strings" + + "github.com/kelseyhightower/envconfig" + "knative.dev/pkg/system" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/kubernetes/scheme" + v1 "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/client-go/tools/record" + + kubeclient "knative.dev/pkg/client/injection/kube/client" + "knative.dev/pkg/controller" + "knative.dev/pkg/logging" + + "knative.dev/eventing/test/lib/recordevents" +) + +type envConfig struct { + AgentName string `envconfig:"AGENT_NAME" default:"observer-default" required:"true"` + PodName string `envconfig:"POD_NAME" required:"true"` + Port int `envconfig:"PORT" default:"8080" required:"true"` +} + +func NewFromEnv(ctx context.Context) recordevents.EventLog { + var env envConfig + if err := envconfig.Process("", &env); err != nil { + log.Fatal("Failed to process env var", err) + } + + logging.FromContext(ctx).Infof("Environment configuration: %+v", env) + + return NewEventLog(ctx, env.AgentName, env.PodName) +} + +func NewEventLog(ctx context.Context, agentName string, podName string) recordevents.EventLog { + on, err := kubeclient.Get(ctx).CoreV1().Pods(system.Namespace()).Get(ctx, podName, metav1.GetOptions{}) + if err != nil { + logging.FromContext(ctx).Fatal("Error while trying to retrieve the pod", err) + } + + logging.FromContext(ctx).Infof("Going to send events to pod '%s' in namespace '%s'", on.Name, on.Namespace) + + return &recorder{out: createRecorder(ctx, agentName), on: on} +} + +func createRecorder(ctx context.Context, agentName string) record.EventRecorder { + logger := logging.FromContext(ctx) + + recorder := controller.GetEventRecorder(ctx) + if recorder == nil { + // Create event broadcaster + logger.Debug("Creating event broadcaster") + eventBroadcaster := record.NewBroadcasterWithCorrelatorOptions(record.CorrelatorOptions{ + KeyFunc: func(event *corev1.Event) (aggregateKey string, localKey string) { + return strings.Join([]string{ + event.Source.Component, + event.Source.Host, + event.InvolvedObject.Kind, + event.InvolvedObject.Namespace, + event.InvolvedObject.Name, + string(event.InvolvedObject.UID), + event.InvolvedObject.APIVersion, + event.Type, + event.Reason, + }, ""), string(event.UID) + }, + }) + watches := []watch.Interface{ + eventBroadcaster.StartLogging(logger.Named("event-broadcaster").Infof), + eventBroadcaster.StartRecordingToSink( + &v1.EventSinkImpl{Interface: kubeclient.Get(ctx).CoreV1().Events(system.Namespace())}, + ), + } + recorder = eventBroadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: agentName}) + go func() { + <-ctx.Done() + for _, w := range watches { + w.Stop() + } + logging.FromContext(ctx).Debug("Closed event-broadcaster") + }() + } + + return recorder +} diff --git a/test/lib/recordevents/recorder_vent/doc.go b/test/lib/recordevents/recorder_vent/doc.go new file mode 100644 index 00000000000..1a4bcc2907f --- /dev/null +++ b/test/lib/recordevents/recorder_vent/doc.go @@ -0,0 +1,19 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package recorder_vent implements an recordevents.EventLog backed by Kubernetes +// Events using an event recorder. +package recorder_vent diff --git a/test/lib/recordevents/recorder_vent/recorder.go b/test/lib/recordevents/recorder_vent/recorder.go new file mode 100644 index 00000000000..9669741f9b6 --- /dev/null +++ b/test/lib/recordevents/recorder_vent/recorder.go @@ -0,0 +1,43 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package recorder_vent + +import ( + "encoding/json" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/record" + + "knative.dev/eventing/test/lib/recordevents" +) + +type recorder struct { + out record.EventRecorder + on runtime.Object +} + +func (r *recorder) Vent(observed recordevents.EventInfo) error { + b, err := json.Marshal(observed) + if err != nil { + return err + } + + r.out.Eventf(r.on, corev1.EventTypeNormal, recordevents.CloudEventObservedReason, "%s", string(b)) + + return nil +} diff --git a/test/lib/recordevents/resources.go b/test/lib/recordevents/resources.go new file mode 100644 index 00000000000..54726bedfdc --- /dev/null +++ b/test/lib/recordevents/resources.go @@ -0,0 +1,61 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package recordevents + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/uuid" + "knative.dev/pkg/test" +) + +// EventRecordPod creates a Pod that stores received events for test retrieval. +func EventRecordPod(name string, serviceAccountName string) *corev1.Pod { + return recordEventsPod("recordevents", name, serviceAccountName) +} + +func recordEventsPod(imageName string, name string, serviceAccountName string) *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: map[string]string{"e2etest": string(uuid.NewUUID())}, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: imageName, + Image: test.ImagePath(imageName), + ImagePullPolicy: corev1.PullAlways, + Env: []corev1.EnvVar{{ + Name: "SYSTEM_NAMESPACE", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}, + }, + }, { + Name: "OBSERVER", + Value: "recorder-" + name, + }, { + Name: "POD_NAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}, + }, + }}, + }}, + ServiceAccountName: serviceAccountName, + RestartPolicy: corev1.RestartPolicyAlways, + }, + } +} diff --git a/test/lib/resources/kube.go b/test/lib/resources/kube.go index 2b154fe2b20..bc415d10873 100644 --- a/test/lib/resources/kube.go +++ b/test/lib/resources/kube.go @@ -37,28 +37,6 @@ type PodOption func(*corev1.Pod) // Option enables further configuration of a Role. type RoleOption func(*rbacv1.Role) -// EventRecordPod creates a Pod that stores received events for test retrieval. -func EventRecordPod(name string) *corev1.Pod { - return eventLoggerPod("recordevents", name) -} - -func eventLoggerPod(imageName string, name string) *corev1.Pod { - return &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Labels: map[string]string{"e2etest": string(uuid.NewUUID())}, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{{ - Name: imageName, - Image: pkgTest.ImagePath(imageName), - ImagePullPolicy: corev1.PullAlways, - }}, - RestartPolicy: corev1.RestartPolicyAlways, - }, - } -} - // EventTransformationPod creates a Pod that transforms events received receiving as arg a cloudevents sdk2 Event func EventTransformationPod(name string, newEventType string, newEventSource string, newEventData []byte) *corev1.Pod { const imageName = "transformevents" diff --git a/test/test_images/recordevents/eventstore.go b/test/test_images/recordevents/eventstore.go deleted file mode 100644 index 470e5adc21d..00000000000 --- a/test/test_images/recordevents/eventstore.go +++ /dev/null @@ -1,191 +0,0 @@ -/* -Copyright 2020 The Knative Authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "encoding/json" - "fmt" - "sync" - - cloudevents "github.com/cloudevents/sdk-go/v2" - - "knative.dev/eventing/test/lib/recordevents" -) - -// Number of EventInfo per block -const evBlockSize = 100 - -// Block of stored EventInfo -type eventBlock struct { - // seqno of [0] evInfoBytes entry - firstIndex int - // offset inside block for newly appended event - firstOffsetFree int - // offset inside block of first non-trimmed event - firstValid int - // serialized EventInfo structures for each seqno. We enforce - // that there is always at least one block. - evInfoBytes [evBlockSize][]byte -} - -// All events currently seen -type eventStore struct { - // Blocks of events in increasing sequency number order - evBlocks []*eventBlock - evBlocksLock sync.Mutex -} - -// Create a new event store. -func newEventStore() *eventStore { - es := &eventStore{} - es.evBlocks = []*eventBlock{{}} - - // One block with no entries starting at sequence number 1 - es.evBlocks[0].firstIndex = 1 - es.evBlocks[0].firstOffsetFree = 0 - es.evBlocks[0].firstValid = 0 - return es -} - -// See if there's enough room to append to the current last block. If not, -// append an extra block. -func (es *eventStore) checkAppendBlock() { - if es.evBlocks[len(es.evBlocks)-1].firstOffsetFree == evBlockSize { - newEVBlock := &eventBlock{ - firstIndex: es.evBlocks[len(es.evBlocks)-1].firstIndex + evBlockSize, - } - es.evBlocks = append(es.evBlocks, newEVBlock) - } -} - -// Store the specified event. -func (es *eventStore) StoreEvent(event *cloudevents.Event, evErr error, httpHeaders map[string][]string) { - var evInfo recordevents.EventInfo - var err error - var evInfoBytes []byte - if evErr != nil { - evInfo.HTTPHeaders = httpHeaders - evInfo.Error = evErr.Error() - if evInfo.Error == "" { - evInfo.Error = "Unknown Incoming Error" - } - evInfoBytes, err = json.Marshal(&evInfo) - if err != nil { - panic(fmt.Errorf("unexpected marshal error (%v) (%+v)", err, evInfo)) - } - } else { - evInfo.Event = event - evInfo.HTTPHeaders = httpHeaders - evInfoBytes, err = json.Marshal(&evInfo) - - if err != nil { - evInfo.Event = nil - evInfo.Error = err.Error() - if evInfo.Error == "" { - evInfo.Error = "Unknown Error" - } - evInfoBytes, err = json.Marshal(&evInfo) - if err != nil { - panic(fmt.Errorf("unexpected marshal error (%v) (%+v)", err, evInfo)) - } - } - } - - es.evBlocksLock.Lock() - // Add a new block if we're out of space - es.checkAppendBlock() - - evBlock := es.evBlocks[len(es.evBlocks)-1] - if evBlock.firstOffsetFree < evBlockSize { - evBlock.evInfoBytes[evBlock.firstOffsetFree] = evInfoBytes - evBlock.firstOffsetFree++ - } - - es.evBlocksLock.Unlock() -} - -// Logically trim all events up to and include the provided -// sequence number. Returns error for patently incorrect -// values (negative sequence number or sequence number larger -// than the largest event seen). Trimming already trimmed -// regions is legal. -func (es *eventStore) TrimThrough(through int) error { - es.evBlocksLock.Lock() - defer es.evBlocksLock.Unlock() - minAvail, maxSeen := es.minMaxUnlocked() - if through > maxSeen { - return fmt.Errorf("invalid trim through %d, maxSeen %d", through, maxSeen) - } else if through < 0 { - return fmt.Errorf("invalid trim less than zero %d", through) - } else if through < minAvail { - return nil - } - // Completely remove blocks if they are full and all events in them are less than - // the specified value. - for len(es.evBlocks) > 1 && (es.evBlocks[0].firstIndex+evBlockSize-1) <= through { - es.evBlocks = es.evBlocks[1:] - } - // Logically trim the block split by through. - es.evBlocks[0].firstValid = through - es.evBlocks[0].firstIndex + 1 - return nil -} - -// return min/max untrimmed value when already holding the lock -func (es *eventStore) minMaxUnlocked() (minAvail int, maxSeen int) { - minBlock := es.evBlocks[0] - minAvail = minBlock.firstIndex + (minBlock.firstValid) - - maxBlock := es.evBlocks[len(es.evBlocks)-1] - maxSeen = maxBlock.firstIndex + maxBlock.firstOffsetFree - 1 - return minAvail, maxSeen -} - -// Returns min available value and max seen value for the store. -// min is the minimum value that can be retrieved via GetEntry, or -// if no values can be retrieved, min == max+1. Max starts at 0 -// when no events have been seen. -func (es *eventStore) MinMax() (minAvail int, maxSeen int) { - es.evBlocksLock.Lock() - minAvail, maxSeen = es.minMaxUnlocked() - - es.evBlocksLock.Unlock() - return minAvail, maxSeen -} - -// Get the already serialized EventInfo structure for the provided sequence -// number. -func (es *eventStore) GetEventInfoBytes(seq int) ([]byte, error) { - var evInfoBytes []byte - found := false - - es.evBlocksLock.Lock() - for _, block := range es.evBlocks { - if seq < block.firstIndex+block.firstValid { - break - } - if seq < block.firstIndex+block.firstOffsetFree { - found = true - evInfoBytes = block.evInfoBytes[seq-block.firstIndex] - break - } - } - es.evBlocksLock.Unlock() - if !found { - return evInfoBytes, fmt.Errorf("Invalid sequence number %d", seq) - } - return evInfoBytes, nil -} diff --git a/test/test_images/recordevents/eventstore_test.go b/test/test_images/recordevents/eventstore_test.go deleted file mode 100644 index 4642fe7b02a..00000000000 --- a/test/test_images/recordevents/eventstore_test.go +++ /dev/null @@ -1,339 +0,0 @@ -/* -Copyright 2020 The Knative Authors -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "encoding/json" - "fmt" - "strconv" - "testing" - - cloudevents "github.com/cloudevents/sdk-go/v2" - - "knative.dev/eventing/test/lib/recordevents" -) - -func helperGetEvInfo(es *eventStore, entry int) (*recordevents.EventInfo, error) { - evInfoBytes, err := es.GetEventInfoBytes(entry) - if err != nil { - return nil, fmt.Errorf("error calling get on item %d: %v", entry, err) - } - if len(evInfoBytes) == 0 { - return nil, fmt.Errorf("empty info bytes") - } - - var evInfo recordevents.EventInfo - err = json.Unmarshal(evInfoBytes, &evInfo) - if err != nil { - return nil, fmt.Errorf("error unmarshalling stored JSON: %v", err) - } - return &evInfo, nil -} - -// Test that adding and getting a bunch of events stores them all -// and that retrieving the events retrieves the correct events. -func TestAddGetMany(t *testing.T) { - es := newEventStore() - - count := 10009 - for i := 0; i < count; i++ { - ce := cloudevents.NewEvent(cloudevents.VersionV1) - ce.SetType("knative.dev.test.event.a") - ce.SetSource("https://source.test.event.knative.dev/foo") - ce.SetID(strconv.FormatInt(int64(i), 10)) - es.StoreEvent(&ce, nil, nil) - minAvail, maxSeen := es.MinMax() - if minAvail != 1 { - t.Fatalf("Pass %d Bad min: %d, expected %d", i, minAvail, 1) - } - if maxSeen != i+1 { - t.Fatalf("Pass %d Bad max: %d, expected %d", i, maxSeen, i+1) - } - - } - for i := 1; i <= count; i++ { - evInfo, err := helperGetEvInfo(es, i) - if err != nil { - t.Fatalf("Count %d error: %v", count, err) - } - - if evInfo.Event == nil { - t.Fatalf("Unexpected empty event info event %d: %+v", i, evInfo) - } - if len(evInfo.Error) != 0 { - t.Fatalf("Unexpected error for stored event %d: %s", i, evInfo.Error) - } - - // Make sure it's the expected event - seenID := evInfo.Event.ID() - expectedID := strconv.FormatInt(int64(i-1), 10) - if seenID != expectedID { - t.Errorf("Incorrect id on retrieval: %s, expected %s", seenID, expectedID) - } - } - _, err := es.GetEventInfoBytes(count + 1) - if err == nil { - t.Errorf("Unexpected non-error return for getinfo of %d", count+1) - } - - _, err = es.GetEventInfoBytes(0) - if err == nil { - t.Errorf("Unexpected non-error return for getinfo of %d", 0) - } - -} - -func TestEmpty(t *testing.T) { - es := newEventStore() - min, max := es.MinMax() - if min != 1 { - t.Errorf("Invalid min: %d, expected %d", min, 1) - } - if max != 0 { - t.Errorf("Invalid max: %d, expected %d", max, 0) - } - - for i := -2; i < 2; i++ { - _, err := es.GetEventInfoBytes(0) - if err == nil { - t.Errorf("Unexpected non-error return for getinfo of %d", i) - } - } - -} - -func TestAddGetSingleValid(t *testing.T) { - expectedType := "knative.dev.test.event.a" - expectedSource := "https://source.test.event.knative.dev/foo" - expectedID := "111" - es := newEventStore() - - headers := make(map[string][]string) - headers["foo"] = []string{"bar", "baz"} - ce := cloudevents.NewEvent(cloudevents.VersionV1) - ce.SetType(expectedType) - ce.SetSource(expectedSource) - ce.SetID(expectedID) - es.StoreEvent(&ce, nil, headers) - minAvail, maxSeen := es.MinMax() - if minAvail != maxSeen { - t.Fatalf("Expected match, saw %d, %d", minAvail, maxSeen) - } - - evInfoBytes, err := es.GetEventInfoBytes(minAvail) - if err != nil { - t.Fatalf("Error calling get: %v", err) - } - var evInfo recordevents.EventInfo - err = json.Unmarshal(evInfoBytes, &evInfo) - if err != nil { - t.Fatalf("Error unmarshalling stored JSON: %v", err) - } - - if evInfo.Event == nil { - t.Fatalf("Unexpected empty event info event: %+v", evInfo) - } - if len(evInfo.Error) != 0 { - t.Fatalf("Unexpected error for stored event: %s", evInfo.Error) - } - if len(evInfo.HTTPHeaders) != 1 { - t.Fatalf("Unexpected header contents for stored event: %+v", evInfo.HTTPHeaders) - } - if len(evInfo.HTTPHeaders["foo"]) != 2 { - t.Fatalf("Unexpected header contents for stored event: %+v", evInfo.HTTPHeaders) - } - if evInfo.HTTPHeaders["foo"][0] != "bar" || evInfo.HTTPHeaders["foo"][1] != "baz" { - t.Fatalf("Unexpected header contents for stored event: %+v", evInfo.HTTPHeaders) - } - seenID := evInfo.Event.ID() - if seenID != expectedID { - t.Errorf("Incorrect id on retrieval: %s, expected %s", seenID, expectedID) - } - seenSource := evInfo.Event.Source() - if seenSource != expectedSource { - t.Errorf("Incorrect source on retrieval: %s, expected %s", seenSource, expectedSource) - } - seenType := evInfo.Event.Type() - if seenType != expectedType { - t.Errorf("Incorrect type on retrieval: %s, expected %s", seenType, expectedType) - } -} - -func TestAddGetSingleInvalid(t *testing.T) { - es := newEventStore() - - headers := make(map[string][]string) - headers["foo"] = []string{"bar", "baz"} - ce := cloudevents.NewEvent(cloudevents.VersionV1) - ce.SetType("knative.dev.test.event.a") - // No source - ce.SetID("111") - es.StoreEvent(&ce, nil, headers) - minAvail, maxSeen := es.MinMax() - if minAvail != maxSeen { - t.Fatalf("Expected match, saw %d, %d", minAvail, maxSeen) - } - - evInfoBytes, err := es.GetEventInfoBytes(minAvail) - if err != nil { - t.Fatalf("Error calling get: %v", err) - } - var evInfo recordevents.EventInfo - err = json.Unmarshal(evInfoBytes, &evInfo) - if err != nil { - t.Fatalf("Error unmarshalling stored JSON: %v", err) - } - if evInfo.Event != nil { - t.Fatalf("Unexpected event info: %+v", evInfo) - } - if len(evInfo.Error) == 0 { - t.Fatalf("Unexpected empty error for stored event: %s", evInfo.Error) - } - if len(evInfo.HTTPHeaders) != 1 { - t.Fatalf("Unexpected header contents for stored event: %+v", evInfo.HTTPHeaders) - } - if len(evInfo.HTTPHeaders["foo"]) != 2 { - t.Fatalf("Unexpected header contents for stored event: %+v", evInfo.HTTPHeaders) - } - if evInfo.HTTPHeaders["foo"][0] != "bar" || evInfo.HTTPHeaders["foo"][1] != "baz" { - t.Fatalf("Unexpected header contents for stored event: %+v", evInfo.HTTPHeaders) - } -} - -func TestAddGetSingleInvalidError(t *testing.T) { - es := newEventStore() - - headers := make(map[string][]string) - headers["foo"] = []string{"bar", "baz"} - ce := cloudevents.NewEvent(cloudevents.VersionV1) - ce.SetType("knative.dev.test.event.a") - ce.SetID("111") - ce.SetSource("nnn") - es.StoreEvent(&ce, fmt.Errorf("Error passed in"), headers) - minAvail, maxSeen := es.MinMax() - if minAvail != maxSeen { - t.Fatalf("Expected match, saw %d, %d", minAvail, maxSeen) - } - - evInfoBytes, err := es.GetEventInfoBytes(minAvail) - if err != nil { - t.Fatalf("Error calling get: %v", err) - } - var evInfo recordevents.EventInfo - err = json.Unmarshal(evInfoBytes, &evInfo) - if err != nil { - t.Fatalf("Error unmarshalling stored JSON: %v", err) - } - if evInfo.Event != nil { - t.Fatalf("Unexpected event info: %+v", evInfo) - } - if len(evInfo.Error) == 0 { - t.Fatalf("Unexpected empty error for stored event: %s", evInfo.Error) - } - if len(evInfo.HTTPHeaders) != 1 { - t.Fatalf("Unexpected header contents for stored event: %+v", evInfo.HTTPHeaders) - } - if len(evInfo.HTTPHeaders["foo"]) != 2 { - t.Fatalf("Unexpected header contents for stored event: %+v", evInfo.HTTPHeaders) - } - if evInfo.HTTPHeaders["foo"][0] != "bar" || evInfo.HTTPHeaders["foo"][1] != "baz" { - t.Fatalf("Unexpected header contents for stored event: %+v", evInfo.HTTPHeaders) - } -} - -func helperFillCount(es *eventStore, count int) { - for i := 0; i < count; i++ { - ce := cloudevents.NewEvent(cloudevents.VersionV1) - ce.SetType("knative.dev.test.event.a") - ce.SetSource("https://source.test.event.knative.dev/foo") - ce.SetID(strconv.FormatInt(int64(i), 10)) - es.StoreEvent(&ce, nil, nil) - } -} - -// Test that adding and getting a bunch of events stores them all -// and that retrieving the events retrieves the correct events. -func TestTrim(t *testing.T) { - count := evBlockSize + 10 - - validTrimPoints := []int{0, 1, 2, evBlockSize - 1, evBlockSize, evBlockSize + 1, evBlockSize + 2, count - 1, count} - invalidTrimPoints := []int{-2, -1, count + 1, count + evBlockSize} - - for _, testVal := range validTrimPoints { - es := newEventStore() - helperFillCount(es, count) - err := es.TrimThrough(testVal) - if err != nil { - t.Fatalf("Unexpected error trimming to %d: %v", testVal, err) - } - minAvail, maxSeen := es.MinMax() - if testVal == count { - if minAvail != maxSeen+1 { - t.Errorf("Incorrect minAvail (%d != %d+1) trimming to %d", minAvail, maxSeen, testVal) - } - } else if minAvail != testVal+1 { - t.Errorf("Incorrect minAvail %d trimming to %d", minAvail, testVal) - } - if maxSeen != count { - t.Errorf("Incorrect maxSeen %d trimming to %d", maxSeen, testVal) - } - if minAvail <= maxSeen { - evInfo, err := helperGetEvInfo(es, minAvail) - if err != nil { - t.Fatalf("Couldn't get min avail %d, trim %d error: %v", minAvail, testVal, err) - } - seenID := evInfo.Event.ID() - expectedID := strconv.FormatInt(int64(minAvail-1), 10) - if seenID != expectedID { - t.Fatalf("Expected ID %s, saw %s for ev %d, trim %d", expectedID, seenID, minAvail, testVal) - } - } - ce := cloudevents.NewEvent(cloudevents.VersionV1) - ce.SetType("knative.dev.test.event.a") - ce.SetSource("https://source.test.event.knative.dev/foo") - ce.SetID(strconv.FormatInt(int64(count), 10)) - es.StoreEvent(&ce, nil, nil) - addedMinAvail, addedMaxSeen := es.MinMax() - if addedMaxSeen != maxSeen+1 { - t.Fatalf("Add after trim resulted in bad maxSeen: expected %d, saw %d for trim %d", - maxSeen, addedMaxSeen, testVal) - } - if minAvail == -1 && addedMinAvail != addedMaxSeen { - t.Errorf("Add after full trim resulted in bad minAvail: expected %d, saw %d for trim %d", - addedMaxSeen, addedMinAvail, testVal) - } else if minAvail != -1 && minAvail != addedMinAvail { - t.Errorf("Add after partial trim resulted in bad minAvail: expected %d, saw %d for trim %d", - minAvail, addedMinAvail, testVal) - - } - - } - for _, testVal := range invalidTrimPoints { - es := newEventStore() - helperFillCount(es, count) - err := es.TrimThrough(testVal) - if err == nil { - t.Fatalf("Incorrect missing error trimming to %d", testVal) - } - minAvail, maxSeen := es.MinMax() - if minAvail != 1 { - t.Errorf("Incorrect minAvail %d trimming to %d", minAvail, testVal) - } - if maxSeen != count { - t.Errorf("Incorrect maxSeen %d trimming to %d", maxSeen, testVal) - } - } -} diff --git a/test/test_images/recordevents/main.go b/test/test_images/recordevents/main.go index 8dd74116338..51fde078e9b 100644 --- a/test/test_images/recordevents/main.go +++ b/test/test_images/recordevents/main.go @@ -17,164 +17,60 @@ limitations under the License. package main import ( - "context" - "encoding/json" - "fmt" "log" "net/http" "os" - "strconv" - "strings" - cloudeventsbindings "github.com/cloudevents/sdk-go/v2/binding" - cloudeventshttp "github.com/cloudevents/sdk-go/v2/protocol/http" - "go.uber.org/zap" + "k8s.io/client-go/rest" + "knative.dev/pkg/injection/sharedmain" + "knative.dev/pkg/logging" + _ "knative.dev/pkg/system/testing" "knative.dev/eventing/pkg/kncloudevents" - testlib "knative.dev/eventing/test/lib" "knative.dev/eventing/test/lib/dropevents" - "knative.dev/eventing/test/lib/recordevents" + "knative.dev/eventing/test/lib/recordevents/observer" + "knative.dev/eventing/test/lib/recordevents/recorder_vent" "knative.dev/eventing/test/test_images" ) -type eventRecorder struct { - es *eventStore -} - -func newEventRecorder() *eventRecorder { - return &eventRecorder{es: newEventStore()} -} - -// Start the recordevents REST api server -func (er *eventRecorder) StartServer(port int) { - http.HandleFunc(recordevents.GetMinMaxPath, er.handleMinMax) - http.HandleFunc(recordevents.GetEntryPath, er.handleGetEntry) - http.HandleFunc(recordevents.TrimThroughPath, er.handleTrim) - go http.ListenAndServe(fmt.Sprintf(":%d", port), nil) -} - -// HTTP handler for GetMinMax requests -func (er *eventRecorder) handleMinMax(w http.ResponseWriter, r *http.Request) { - minAvail, maxSeen := er.es.MinMax() - minMax := recordevents.MinMaxResponse{ - MinAvail: minAvail, - MaxSeen: maxSeen, - } - respBytes, err := json.Marshal(minMax) - if err != nil { - log.Panicf("Internal error: json marshal shouldn't fail: (%v) (%+v)", err, minMax) - } - - w.Header().Set("Content-Type", "text/json") - w.WriteHeader(http.StatusOK) - w.Write(respBytes) -} - -// HTTP handler for TrimThrough requests -func (er *eventRecorder) handleTrim(w http.ResponseWriter, r *http.Request) { - // If we extend this much at all we should vendor a better mux(gorilla, etc) - path := strings.TrimLeft(r.URL.Path, "/") - getPrefix := strings.TrimLeft(recordevents.TrimThroughPath, "/") - suffix := strings.TrimLeft(strings.TrimPrefix(path, getPrefix), "/") - - seqNum, err := strconv.ParseInt(suffix, 10, 32) - if err != nil { - http.Error(w, "Can't parse event sequence number in request", http.StatusBadRequest) - return - } - - err = er.es.TrimThrough(int(seqNum)) - if err != nil { - http.Error(w, "Invalid sequence number in request to trim", http.StatusNotFound) - return - } - - w.Header().Set("Content-Type", "text/json") - w.WriteHeader(http.StatusOK) -} - -// HTTP handler for GetEntry requests -func (er *eventRecorder) handleGetEntry(w http.ResponseWriter, r *http.Request) { - // If we extend this much at all we should vendor a better mux(gorilla, etc) - path := strings.TrimLeft(r.URL.Path, "/") - getPrefix := strings.TrimLeft(recordevents.GetEntryPath, "/") - suffix := strings.TrimLeft(strings.TrimPrefix(path, getPrefix), "/") - - seqNum, err := strconv.ParseInt(suffix, 10, 32) +func main() { + cfg, err := rest.InClusterConfig() if err != nil { - http.Error(w, "Can't parse event sequence number in request", http.StatusBadRequest) - return + log.Fatal("Error while reading the cfg", err) } + //nolint // nil ctx is fine here, look at the code of EnableInjectionOrDie + ctx := sharedmain.EnableInjectionOrDie(nil, cfg) - entryBytes, err := er.es.GetEventInfoBytes(int(seqNum)) - if err != nil { - http.Error(w, "Couldn't find requested event", http.StatusNotFound) - return + if err := test_images.ConfigureTracing(logging.FromContext(ctx), ""); err != nil { + logging.FromContext(ctx).Fatal("Unable to setup trace publishing", err) } - w.Header().Set("Content-Type", "text/json") - w.WriteHeader(http.StatusOK) - w.Write(entryBytes) -} - -func (er *eventRecorder) ServeHTTP(writer http.ResponseWriter, request *http.Request) { - m := cloudeventshttp.NewMessageFromHttpRequest(request) - defer m.Finish(nil) - - event, eventErr := cloudeventsbindings.ToEvent(context.TODO(), m) - header := request.Header - - er.es.StoreEvent(event, eventErr, map[string][]string(header)) - - headerNameList := testlib.InterestingHeaders() - for _, headerName := range headerNameList { - if headerValue := header.Get(headerName); headerValue != "" { - log.Printf("Header %s: %s\n", headerName, headerValue) - } - } - - if eventErr != nil { - log.Printf("error receiving the event: %v", eventErr) - } else { - valErr := event.Validate() - if valErr == nil { - log.Printf("eventdetails:\n%s", event.String()) - } else { - log.Printf("error validating the event: %v", valErr) - } - } - - writer.WriteHeader(http.StatusAccepted) -} - -func main() { - er := newEventRecorder() - er.StartServer(recordevents.RecordEventsPort) - - logger, _ := zap.NewDevelopment() - if err := test_images.ConfigureTracing(logger.Sugar(), ""); err != nil { - log.Fatalf("Unable to setup trace publishing: %v", err) - } + obs := observer.NewFromEnv( + recorder_vent.NewFromEnv(ctx), + ) algorithm, ok := os.LookupEnv(dropevents.SkipAlgorithmKey) - handler := kncloudevents.CreateHandler(er) if ok { skipper := dropevents.SkipperAlgorithm(algorithm) counter := dropevents.CounterHandler{ Skipper: skipper, } - next := handler - handler = http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { - if counter.Skip() { - writer.WriteHeader(http.StatusConflict) - return - } - next.ServeHTTP(writer, request) + err = obs.Start(ctx, kncloudevents.CreateHandler, func(handler http.Handler) http.Handler { + return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + if counter.Skip() { + writer.WriteHeader(http.StatusConflict) + return + } + handler.ServeHTTP(writer, request) + }) }) + } else { + err = obs.Start(ctx, kncloudevents.CreateHandler) } - err := http.ListenAndServe(":8080", handler) if err != nil { - panic(err) + logging.FromContext(ctx).Fatal("Error during start", err) } + + logging.FromContext(ctx).Info("Closing the recordevents process") } diff --git a/vendor/gopkg.in/yaml.v3/.travis.yml b/vendor/gopkg.in/yaml.v3/.travis.yml index 1bc5c1cd20b..a130fe883ce 100644 --- a/vendor/gopkg.in/yaml.v3/.travis.yml +++ b/vendor/gopkg.in/yaml.v3/.travis.yml @@ -1,15 +1,17 @@ language: go go: - - "1.4" - - "1.5" - - "1.6" - - "1.7" - - "1.8" - - "1.9" - - "1.10" - - "1.11" - - "1.12" - - tip + - "1.4.x" + - "1.5.x" + - "1.6.x" + - "1.7.x" + - "1.8.x" + - "1.9.x" + - "1.10.x" + - "1.11.x" + - "1.12.x" + - "1.13.x" + - "1.14.x" + - "tip" go_import_path: gopkg.in/yaml.v3 diff --git a/vendor/gopkg.in/yaml.v3/apic.go b/vendor/gopkg.in/yaml.v3/apic.go index 65846e67497..ae7d049f182 100644 --- a/vendor/gopkg.in/yaml.v3/apic.go +++ b/vendor/gopkg.in/yaml.v3/apic.go @@ -108,6 +108,7 @@ func yaml_emitter_initialize(emitter *yaml_emitter_t) { raw_buffer: make([]byte, 0, output_raw_buffer_size), states: make([]yaml_emitter_state_t, 0, initial_stack_size), events: make([]yaml_event_t, 0, initial_queue_size), + best_width: -1, } } diff --git a/vendor/gopkg.in/yaml.v3/decode.go b/vendor/gopkg.in/yaml.v3/decode.go index be63169b719..21c0dacfdff 100644 --- a/vendor/gopkg.in/yaml.v3/decode.go +++ b/vendor/gopkg.in/yaml.v3/decode.go @@ -35,6 +35,7 @@ type parser struct { doc *Node anchors map[string]*Node doneInit bool + textless bool } func newParser(b []byte) *parser { @@ -108,14 +109,18 @@ func (p *parser) peek() yaml_event_type_t { func (p *parser) fail() { var where string var line int - if p.parser.problem_mark.line != 0 { + if p.parser.context_mark.line != 0 { + line = p.parser.context_mark.line + // Scanner errors don't iterate line before returning error + if p.parser.error == yaml_SCANNER_ERROR { + line++ + } + } else if p.parser.problem_mark.line != 0 { line = p.parser.problem_mark.line // Scanner errors don't iterate line before returning error if p.parser.error == yaml_SCANNER_ERROR { line++ } - } else if p.parser.context_mark.line != 0 { - line = p.parser.context_mark.line } if line != 0 { where = "line " + strconv.Itoa(line) + ": " @@ -169,17 +174,20 @@ func (p *parser) node(kind Kind, defaultTag, tag, value string) *Node { } else if kind == ScalarNode { tag, _ = resolve("", value) } - return &Node{ - Kind: kind, - Tag: tag, - Value: value, - Style: style, - Line: p.event.start_mark.line + 1, - Column: p.event.start_mark.column + 1, - HeadComment: string(p.event.head_comment), - LineComment: string(p.event.line_comment), - FootComment: string(p.event.foot_comment), + n := &Node{ + Kind: kind, + Tag: tag, + Value: value, + Style: style, + } + if !p.textless { + n.Line = p.event.start_mark.line + 1 + n.Column = p.event.start_mark.column + 1 + n.HeadComment = string(p.event.head_comment) + n.LineComment = string(p.event.line_comment) + n.FootComment = string(p.event.foot_comment) } + return n } func (p *parser) parseChild(parent *Node) *Node { @@ -391,7 +399,7 @@ func (d *decoder) callObsoleteUnmarshaler(n *Node, u obsoleteUnmarshaler) (good // // If n holds a null value, prepare returns before doing anything. func (d *decoder) prepare(n *Node, out reflect.Value) (newout reflect.Value, unmarshaled, good bool) { - if n.ShortTag() == nullTag { + if n.ShortTag() == nullTag || n.Kind == 0 && n.IsZero() { return out, false, false } again := true @@ -497,8 +505,13 @@ func (d *decoder) unmarshal(n *Node, out reflect.Value) (good bool) { good = d.mapping(n, out) case SequenceNode: good = d.sequence(n, out) + case 0: + if n.IsZero() { + return d.null(out) + } + fallthrough default: - panic("internal error: unknown node kind: " + strconv.Itoa(int(n.Kind))) + failf("cannot decode node with unknown kind %d", n.Kind) } return good } @@ -533,6 +546,17 @@ func resetMap(out reflect.Value) { } } +func (d *decoder) null(out reflect.Value) bool { + if out.CanAddr() { + switch out.Kind() { + case reflect.Interface, reflect.Ptr, reflect.Map, reflect.Slice: + out.Set(reflect.Zero(out.Type())) + return true + } + } + return false +} + func (d *decoder) scalar(n *Node, out reflect.Value) bool { var tag string var resolved interface{} @@ -550,14 +574,7 @@ func (d *decoder) scalar(n *Node, out reflect.Value) bool { } } if resolved == nil { - if out.CanAddr() { - switch out.Kind() { - case reflect.Interface, reflect.Ptr, reflect.Map, reflect.Slice: - out.Set(reflect.Zero(out.Type())) - return true - } - } - return false + return d.null(out) } if resolvedv := reflect.ValueOf(resolved); out.Type() == resolvedv.Type() { // We've resolved to exactly the type we want, so use that. diff --git a/vendor/gopkg.in/yaml.v3/emitterc.go b/vendor/gopkg.in/yaml.v3/emitterc.go index ab2a066194c..c29217ef54b 100644 --- a/vendor/gopkg.in/yaml.v3/emitterc.go +++ b/vendor/gopkg.in/yaml.v3/emitterc.go @@ -235,10 +235,13 @@ func yaml_emitter_increase_indent(emitter *yaml_emitter_t, flow, indentless bool emitter.indent = 0 } } else if !indentless { - emitter.indent += emitter.best_indent - // [Go] If inside a block sequence item, discount the space taken by the indicator. - if emitter.best_indent > 2 && emitter.states[len(emitter.states)-1] == yaml_EMIT_BLOCK_SEQUENCE_ITEM_STATE { - emitter.indent -= 2 + // [Go] This was changed so that indentations are more regular. + if emitter.states[len(emitter.states)-1] == yaml_EMIT_BLOCK_SEQUENCE_ITEM_STATE { + // The first indent inside a sequence will just skip the "- " indicator. + emitter.indent += 2 + } else { + // Everything else aligns to the chosen indentation. + emitter.indent = emitter.best_indent*((emitter.indent+emitter.best_indent)/emitter.best_indent) } } return true @@ -725,16 +728,9 @@ func yaml_emitter_emit_flow_mapping_value(emitter *yaml_emitter_t, event *yaml_e // Expect a block item node. func yaml_emitter_emit_block_sequence_item(emitter *yaml_emitter_t, event *yaml_event_t, first bool) bool { if first { - // [Go] The original logic here would not indent the sequence when inside a mapping. - // In Go we always indent it, but take the sequence indicator out of the indentation. - indentless := emitter.best_indent == 2 && emitter.mapping_context && (emitter.column == 0 || !emitter.indention) - original := emitter.indent - if !yaml_emitter_increase_indent(emitter, false, indentless) { + if !yaml_emitter_increase_indent(emitter, false, false) { return false } - if emitter.indent > original+2 { - emitter.indent -= 2 - } } if event.typ == yaml_SEQUENCE_END_EVENT { emitter.indent = emitter.indents[len(emitter.indents)-1] @@ -785,6 +781,13 @@ func yaml_emitter_emit_block_mapping_key(emitter *yaml_emitter_t, event *yaml_ev if !yaml_emitter_write_indent(emitter) { return false } + if len(emitter.line_comment) > 0 { + // [Go] A line comment was provided for the key. That's unusual as the + // scanner associates line comments with the value. Either way, + // save the line comment and render it appropriately later. + emitter.key_line_comment = emitter.line_comment + emitter.line_comment = nil + } if yaml_emitter_check_simple_key(emitter) { emitter.states = append(emitter.states, yaml_EMIT_BLOCK_MAPPING_SIMPLE_VALUE_STATE) return yaml_emitter_emit_node(emitter, event, false, false, true, true) @@ -810,6 +813,29 @@ func yaml_emitter_emit_block_mapping_value(emitter *yaml_emitter_t, event *yaml_ return false } } + if len(emitter.key_line_comment) > 0 { + // [Go] A line comment was previously provided for the key. Handle it before + // the value so the inline comments are placed correctly. + if yaml_emitter_silent_nil_event(emitter, event) && len(emitter.line_comment) == 0 { + // Nothing other than the line comment will be written on the line. + emitter.line_comment = emitter.key_line_comment + emitter.key_line_comment = nil + } else { + // An actual value is coming, so emit the comment line. + emitter.line_comment, emitter.key_line_comment = emitter.key_line_comment, emitter.line_comment + if !yaml_emitter_process_line_comment(emitter) { + return false + } + emitter.line_comment, emitter.key_line_comment = emitter.key_line_comment, emitter.line_comment + // Indent in unless it's a block that will reindent anyway. + if event.sequence_style() == yaml_FLOW_SEQUENCE_STYLE || (event.typ != yaml_MAPPING_START_EVENT && event.typ != yaml_SEQUENCE_START_EVENT) { + emitter.indent = emitter.best_indent*((emitter.indent+emitter.best_indent)/emitter.best_indent) + if !yaml_emitter_write_indent(emitter) { + return false + } + } + } + } emitter.states = append(emitter.states, yaml_EMIT_BLOCK_MAPPING_KEY_STATE) if !yaml_emitter_emit_node(emitter, event, false, false, true, false) { return false @@ -823,6 +849,10 @@ func yaml_emitter_emit_block_mapping_value(emitter *yaml_emitter_t, event *yaml_ return true } +func yaml_emitter_silent_nil_event(emitter *yaml_emitter_t, event *yaml_event_t) bool { + return event.typ == yaml_SCALAR_EVENT && event.implicit && !emitter.canonical && len(emitter.scalar_data.value) == 0 +} + // Expect a node. func yaml_emitter_emit_node(emitter *yaml_emitter_t, event *yaml_event_t, root bool, sequence bool, mapping bool, simple_key bool) bool { diff --git a/vendor/gopkg.in/yaml.v3/encode.go b/vendor/gopkg.in/yaml.v3/encode.go index eee3667eabc..45e8d1e1b9f 100644 --- a/vendor/gopkg.in/yaml.v3/encode.go +++ b/vendor/gopkg.in/yaml.v3/encode.go @@ -119,6 +119,9 @@ func (e *encoder) marshal(tag string, in reflect.Value) { case *Node: e.nodev(in) return + case Node: + e.nodev(in.Addr()) + return case time.Time: e.timev(tag, in) return @@ -298,6 +301,21 @@ func isBase60Float(s string) (result bool) { // is bogus. In practice parsers do not enforce the "\.[0-9_]*" suffix. var base60float = regexp.MustCompile(`^[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+(?:\.[0-9_]*)?$`) +// isOldBool returns whether s is bool notation as defined in YAML 1.1. +// +// We continue to force strings that YAML 1.1 would interpret as booleans to be +// rendered as quotes strings so that the marshalled output valid for YAML 1.1 +// parsing. +func isOldBool(s string) (result bool) { + switch s { + case "y", "Y", "yes", "Yes", "YES", "on", "On", "ON", + "n", "N", "no", "No", "NO", "off", "Off", "OFF": + return true + default: + return false + } +} + func (e *encoder) stringv(tag string, in reflect.Value) { var style yaml_scalar_style_t s := in.String() @@ -319,7 +337,7 @@ func (e *encoder) stringv(tag string, in reflect.Value) { // tag when encoded unquoted. If it doesn't, // there's no need to quote it. rtag, _ := resolve("", s) - canUsePlain = rtag == strTag && !isBase60Float(s) + canUsePlain = rtag == strTag && !(isBase60Float(s) || isOldBool(s)) } // Note: it's possible for user code to emit invalid YAML // if they explicitly specify a tag and a string containing @@ -407,18 +425,23 @@ func (e *encoder) nodev(in reflect.Value) { } func (e *encoder) node(node *Node, tail string) { + // Zero nodes behave as nil. + if node.Kind == 0 && node.IsZero() { + e.nilv() + return + } + // If the tag was not explicitly requested, and dropping it won't change the // implicit tag of the value, don't include it in the presentation. var tag = node.Tag var stag = shortTag(tag) - var rtag string var forceQuoting bool if tag != "" && node.Style&TaggedStyle == 0 { if node.Kind == ScalarNode { if stag == strTag && node.Style&(SingleQuotedStyle|DoubleQuotedStyle|LiteralStyle|FoldedStyle) != 0 { tag = "" } else { - rtag, _ = resolve("", node.Value) + rtag, _ := resolve("", node.Value) if rtag == stag { tag = "" } else if stag == strTag { @@ -427,6 +450,7 @@ func (e *encoder) node(node *Node, tail string) { } } } else { + var rtag string switch node.Kind { case MappingNode: rtag = mapTag @@ -456,7 +480,7 @@ func (e *encoder) node(node *Node, tail string) { if node.Style&FlowStyle != 0 { style = yaml_FLOW_SEQUENCE_STYLE } - e.must(yaml_sequence_start_event_initialize(&e.event, []byte(node.Anchor), []byte(tag), tag == "", style)) + e.must(yaml_sequence_start_event_initialize(&e.event, []byte(node.Anchor), []byte(longTag(tag)), tag == "", style)) e.event.head_comment = []byte(node.HeadComment) e.emit() for _, node := range node.Content { @@ -472,7 +496,7 @@ func (e *encoder) node(node *Node, tail string) { if node.Style&FlowStyle != 0 { style = yaml_FLOW_MAPPING_STYLE } - yaml_mapping_start_event_initialize(&e.event, []byte(node.Anchor), []byte(tag), tag == "", style) + yaml_mapping_start_event_initialize(&e.event, []byte(node.Anchor), []byte(longTag(tag)), tag == "", style) e.event.tail_comment = []byte(tail) e.event.head_comment = []byte(node.HeadComment) e.emit() @@ -513,11 +537,11 @@ func (e *encoder) node(node *Node, tail string) { case ScalarNode: value := node.Value if !utf8.ValidString(value) { - if tag == binaryTag { + if stag == binaryTag { failf("explicitly tagged !!binary data must be base64-encoded") } - if tag != "" { - failf("cannot marshal invalid UTF-8 data as %s", shortTag(tag)) + if stag != "" { + failf("cannot marshal invalid UTF-8 data as %s", stag) } // It can't be encoded directly as YAML so use a binary tag // and encode it as base64. @@ -542,5 +566,7 @@ func (e *encoder) node(node *Node, tail string) { } e.emitScalar(value, node.Anchor, tag, style, []byte(node.HeadComment), []byte(node.LineComment), []byte(node.FootComment), []byte(tail)) + default: + failf("cannot encode node with unknown kind %d", node.Kind) } } diff --git a/vendor/gopkg.in/yaml.v3/parserc.go b/vendor/gopkg.in/yaml.v3/parserc.go index ec25faabc4e..ac66fccc059 100644 --- a/vendor/gopkg.in/yaml.v3/parserc.go +++ b/vendor/gopkg.in/yaml.v3/parserc.go @@ -423,6 +423,7 @@ func yaml_parser_set_event_comments(parser *yaml_parser_t, event *yaml_event_t) parser.line_comment = nil parser.foot_comment = nil parser.tail_comment = nil + parser.stem_comment = nil } // Parse the productions: @@ -629,6 +630,10 @@ func yaml_parser_parse_node(parser *yaml_parser_t, event *yaml_event_t, block, i implicit: implicit, style: yaml_style_t(yaml_BLOCK_SEQUENCE_STYLE), } + if parser.stem_comment != nil { + event.head_comment = parser.stem_comment + parser.stem_comment = nil + } return true } if block && token.typ == yaml_BLOCK_MAPPING_START_TOKEN { @@ -643,6 +648,10 @@ func yaml_parser_parse_node(parser *yaml_parser_t, event *yaml_event_t, block, i implicit: implicit, style: yaml_style_t(yaml_BLOCK_MAPPING_STYLE), } + if parser.stem_comment != nil { + event.head_comment = parser.stem_comment + parser.stem_comment = nil + } return true } if len(anchor) > 0 || len(tag) > 0 { @@ -689,7 +698,9 @@ func yaml_parser_parse_block_sequence_entry(parser *yaml_parser_t, event *yaml_e if token.typ == yaml_BLOCK_ENTRY_TOKEN { mark := token.end_mark + prior_head_len := len(parser.head_comment) skip_token(parser) + yaml_parser_split_stem_comment(parser, prior_head_len) token = peek_token(parser) if token == nil { return false @@ -735,7 +746,9 @@ func yaml_parser_parse_indentless_sequence_entry(parser *yaml_parser_t, event *y if token.typ == yaml_BLOCK_ENTRY_TOKEN { mark := token.end_mark + prior_head_len := len(parser.head_comment) skip_token(parser) + yaml_parser_split_stem_comment(parser, prior_head_len) token = peek_token(parser) if token == nil { return false @@ -761,6 +774,32 @@ func yaml_parser_parse_indentless_sequence_entry(parser *yaml_parser_t, event *y return true } +// Split stem comment from head comment. +// +// When a sequence or map is found under a sequence entry, the former head comment +// is assigned to the underlying sequence or map as a whole, not the individual +// sequence or map entry as would be expected otherwise. To handle this case the +// previous head comment is moved aside as the stem comment. +func yaml_parser_split_stem_comment(parser *yaml_parser_t, stem_len int) { + if stem_len == 0 { + return + } + + token := peek_token(parser) + if token.typ != yaml_BLOCK_SEQUENCE_START_TOKEN && token.typ != yaml_BLOCK_MAPPING_START_TOKEN { + return + } + + parser.stem_comment = parser.head_comment[:stem_len] + if len(parser.head_comment) == stem_len { + parser.head_comment = nil + } else { + // Copy suffix to prevent very strange bugs if someone ever appends + // further bytes to the prefix in the stem_comment slice above. + parser.head_comment = append([]byte(nil), parser.head_comment[stem_len+1:]...) + } +} + // Parse the productions: // block_mapping ::= BLOCK-MAPPING_START // ******************* diff --git a/vendor/gopkg.in/yaml.v3/scannerc.go b/vendor/gopkg.in/yaml.v3/scannerc.go index e33f4959065..d9a539c39ae 100644 --- a/vendor/gopkg.in/yaml.v3/scannerc.go +++ b/vendor/gopkg.in/yaml.v3/scannerc.go @@ -657,34 +657,22 @@ func trace(args ...interface{}) func() { func yaml_parser_fetch_more_tokens(parser *yaml_parser_t) bool { // While we need more tokens to fetch, do it. for { - // Check if we really need to fetch more tokens. - need_more_tokens := false - - // [Go] The comment parsing logic requires a lookahead of one token - // in block style or two tokens in flow style so that the foot - // comments may be parsed in time of associating them with the tokens - // that are parsed before them. - if parser.tokens_head >= len(parser.tokens)-1 || parser.flow_level > 0 && parser.tokens_head >= len(parser.tokens)-2 { - need_more_tokens = true - } else { - // Check if any potential simple key may occupy the head position. - if !yaml_parser_stale_simple_keys(parser) { + // [Go] The comment parsing logic requires a lookahead of two tokens + // so that foot comments may be parsed in time of associating them + // with the tokens that are parsed before them, and also for line + // comments to be transformed into head comments in some edge cases. + if parser.tokens_head < len(parser.tokens)-2 { + // If a potential simple key is at the head position, we need to fetch + // the next token to disambiguate it. + head_tok_idx, ok := parser.simple_keys_by_tok[parser.tokens_parsed] + if !ok { + break + } else if valid, ok := yaml_simple_key_is_valid(parser, &parser.simple_keys[head_tok_idx]); !ok { return false - } - - for i := range parser.simple_keys { - simple_key := &parser.simple_keys[i] - if simple_key.possible && simple_key.token_number == parser.tokens_parsed { - need_more_tokens = true - break - } + } else if !valid { + break } } - - // We are finished. - if !need_more_tokens { - break - } // Fetch the next token. if !yaml_parser_fetch_next_token(parser) { return false @@ -714,11 +702,6 @@ func yaml_parser_fetch_next_token(parser *yaml_parser_t) (ok bool) { return false } - // Remove obsolete potential simple keys. - if !yaml_parser_stale_simple_keys(parser) { - return false - } - // [Go] While unrolling indents, transform the head comments of prior // indentation levels observed after scan_start into foot comments at // the respective indexes. @@ -766,6 +749,11 @@ func yaml_parser_fetch_next_token(parser *yaml_parser_t) (ok bool) { if !ok { return } + if len(parser.tokens) > 0 && parser.tokens[len(parser.tokens)-1].typ == yaml_BLOCK_ENTRY_TOKEN { + // Sequence indicators alone have no line comments. It becomes + // a head comment for whatever follows. + return + } if !yaml_parser_scan_line_comment(parser, comment_mark) { ok = false return @@ -892,29 +880,30 @@ func yaml_parser_fetch_next_token(parser *yaml_parser_t) (ok bool) { "found character that cannot start any token") } -// Check the list of potential simple keys and remove the positions that -// cannot contain simple keys anymore. -func yaml_parser_stale_simple_keys(parser *yaml_parser_t) bool { - // Check for a potential simple key for each flow level. - for i := range parser.simple_keys { - simple_key := &parser.simple_keys[i] +func yaml_simple_key_is_valid(parser *yaml_parser_t, simple_key *yaml_simple_key_t) (valid, ok bool) { + if !simple_key.possible { + return false, true + } - // The specification requires that a simple key - // - // - is limited to a single line, - // - is shorter than 1024 characters. - if simple_key.possible && (simple_key.mark.line < parser.mark.line || simple_key.mark.index+1024 < parser.mark.index) { - - // Check if the potential simple key to be removed is required. - if simple_key.required { - return yaml_parser_set_scanner_error(parser, - "while scanning a simple key", simple_key.mark, - "could not find expected ':'") - } - simple_key.possible = false + // The 1.2 specification says: + // + // "If the ? indicator is omitted, parsing needs to see past the + // implicit key to recognize it as such. To limit the amount of + // lookahead required, the “:” indicator must appear at most 1024 + // Unicode characters beyond the start of the key. In addition, the key + // is restricted to a single line." + // + if simple_key.mark.line < parser.mark.line || simple_key.mark.index+1024 < parser.mark.index { + // Check if the potential simple key to be removed is required. + if simple_key.required { + return false, yaml_parser_set_scanner_error(parser, + "while scanning a simple key", simple_key.mark, + "could not find expected ':'") } + simple_key.possible = false + return false, true } - return true + return true, true } // Check if a simple key may start at the current position and add it if @@ -934,13 +923,14 @@ func yaml_parser_save_simple_key(parser *yaml_parser_t) bool { possible: true, required: required, token_number: parser.tokens_parsed + (len(parser.tokens) - parser.tokens_head), + mark: parser.mark, } - simple_key.mark = parser.mark if !yaml_parser_remove_simple_key(parser) { return false } parser.simple_keys[len(parser.simple_keys)-1] = simple_key + parser.simple_keys_by_tok[simple_key.token_number] = len(parser.simple_keys) - 1 } return true } @@ -955,9 +945,10 @@ func yaml_parser_remove_simple_key(parser *yaml_parser_t) bool { "while scanning a simple key", parser.simple_keys[i].mark, "could not find expected ':'") } + // Remove the key from the stack. + parser.simple_keys[i].possible = false + delete(parser.simple_keys_by_tok, parser.simple_keys[i].token_number) } - // Remove the key from the stack. - parser.simple_keys[i].possible = false return true } @@ -967,7 +958,12 @@ const max_flow_level = 10000 // Increase the flow level and resize the simple key list if needed. func yaml_parser_increase_flow_level(parser *yaml_parser_t) bool { // Reset the simple key on the next level. - parser.simple_keys = append(parser.simple_keys, yaml_simple_key_t{}) + parser.simple_keys = append(parser.simple_keys, yaml_simple_key_t{ + possible: false, + required: false, + token_number: parser.tokens_parsed + (len(parser.tokens) - parser.tokens_head), + mark: parser.mark, + }) // Increase the flow level. parser.flow_level++ @@ -983,7 +979,9 @@ func yaml_parser_increase_flow_level(parser *yaml_parser_t) bool { func yaml_parser_decrease_flow_level(parser *yaml_parser_t) bool { if parser.flow_level > 0 { parser.flow_level-- - parser.simple_keys = parser.simple_keys[:len(parser.simple_keys)-1] + last := len(parser.simple_keys) - 1 + delete(parser.simple_keys_by_tok, parser.simple_keys[last].token_number) + parser.simple_keys = parser.simple_keys[:last] } return true } @@ -1090,6 +1088,8 @@ func yaml_parser_fetch_stream_start(parser *yaml_parser_t) bool { // Initialize the simple key stack. parser.simple_keys = append(parser.simple_keys, yaml_simple_key_t{}) + parser.simple_keys_by_tok = make(map[int]int) + // A simple key is allowed at the beginning of the stream. parser.simple_key_allowed = true @@ -1372,7 +1372,11 @@ func yaml_parser_fetch_value(parser *yaml_parser_t) bool { simple_key := &parser.simple_keys[len(parser.simple_keys)-1] // Have we found a simple key? - if simple_key.possible { + if valid, ok := yaml_simple_key_is_valid(parser, simple_key); !ok { + return false + + } else if valid { + // Create the KEY token and insert it into the queue. token := yaml_token_t{ typ: yaml_KEY_TOKEN, @@ -1390,6 +1394,7 @@ func yaml_parser_fetch_value(parser *yaml_parser_t) bool { // Remove the simple key. simple_key.possible = false + delete(parser.simple_keys_by_tok, simple_key.token_number) // A simple key cannot follow another simple key. parser.simple_key_allowed = false @@ -1557,6 +1562,29 @@ func yaml_parser_scan_to_next_token(parser *yaml_parser_t) bool { } } + // Check if we just had a line comment under a sequence entry that + // looks more like a header to the following content. Similar to this: + // + // - # The comment + // - Some data + // + // If so, transform the line comment to a head comment and reposition. + if len(parser.comments) > 0 && len(parser.tokens) > 1 { + tokenA := parser.tokens[len(parser.tokens)-2] + tokenB := parser.tokens[len(parser.tokens)-1] + comment := &parser.comments[len(parser.comments)-1] + if tokenA.typ == yaml_BLOCK_SEQUENCE_START_TOKEN && tokenB.typ == yaml_BLOCK_ENTRY_TOKEN && len(comment.line) > 0 && !is_break(parser.buffer, parser.buffer_pos) { + // If it was in the prior line, reposition so it becomes a + // header of the follow up token. Otherwise, keep it in place + // so it becomes a header of the former. + comment.head = comment.line + comment.line = nil + if comment.start_mark.line == parser.mark.line-1 { + comment.token_mark = parser.mark + } + } + } + // Eat a comment until a line break. if parser.buffer[parser.buffer_pos] == '#' { if !yaml_parser_scan_comments(parser, scan_mark) { @@ -2232,8 +2260,15 @@ func yaml_parser_scan_block_scalar(parser *yaml_parser_t, token *yaml_token_t, l } } if parser.buffer[parser.buffer_pos] == '#' { - if !yaml_parser_scan_line_comment(parser, start_mark) { - return false + // TODO Test this and then re-enable it. + //if !yaml_parser_scan_line_comment(parser, start_mark) { + // return false + //} + for !is_breakz(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } } } @@ -2802,8 +2837,8 @@ func yaml_parser_scan_line_comment(parser *yaml_parser_t, token_mark yaml_mark_t return true } - parser.comments = append(parser.comments, yaml_comment_t{token_mark: token_mark}) - comment := &parser.comments[len(parser.comments)-1].line + var start_mark yaml_mark_t + var text []byte for peek := 0; peek < 512; peek++ { if parser.unread < peek+1 && !yaml_parser_update_buffer(parser, peek+1) { @@ -2813,11 +2848,6 @@ func yaml_parser_scan_line_comment(parser *yaml_parser_t, token_mark yaml_mark_t continue } if parser.buffer[parser.buffer_pos+peek] == '#' { - if len(*comment) > 0 { - *comment = append(*comment, '\n') - } - - // Consume until after the consumed comment line. seen := parser.mark.index+peek for { if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { @@ -2831,16 +2861,25 @@ func yaml_parser_scan_line_comment(parser *yaml_parser_t, token_mark yaml_mark_t return false } skip_line(parser) - } else { - if parser.mark.index >= seen { - *comment = append(*comment, parser.buffer[parser.buffer_pos]) + } else if parser.mark.index >= seen { + if len(text) == 0 { + start_mark = parser.mark } + text = read(parser, text) + } else { skip(parser) } } } break } + if len(text) > 0 { + parser.comments = append(parser.comments, yaml_comment_t{ + token_mark: token_mark, + start_mark: start_mark, + line: text, + }) + } return true } @@ -2964,10 +3003,9 @@ func yaml_parser_scan_comments(parser *yaml_parser_t, scan_mark yaml_mark_t) boo return false } skip_line(parser) + } else if parser.mark.index >= seen { + text = read(parser, text) } else { - if parser.mark.index >= seen { - text = append(text, parser.buffer[parser.buffer_pos]) - } skip(parser) } } diff --git a/vendor/gopkg.in/yaml.v3/yaml.go b/vendor/gopkg.in/yaml.v3/yaml.go index b5d35a50ded..56e8a849031 100644 --- a/vendor/gopkg.in/yaml.v3/yaml.go +++ b/vendor/gopkg.in/yaml.v3/yaml.go @@ -89,7 +89,7 @@ func Unmarshal(in []byte, out interface{}) (err error) { return unmarshal(in, out, false) } -// A Decorder reads and decodes YAML values from an input stream. +// A Decoder reads and decodes YAML values from an input stream. type Decoder struct { parser *parser knownFields bool @@ -194,7 +194,7 @@ func unmarshal(in []byte, out interface{}, strict bool) (err error) { // Zero valued structs will be omitted if all their public // fields are zero, unless they implement an IsZero // method (see the IsZeroer interface type), in which -// case the field will be included if that method returns true. +// case the field will be excluded if IsZero returns true. // // flow Marshal using a flow style (useful for structs, // sequences and maps). @@ -252,6 +252,24 @@ func (e *Encoder) Encode(v interface{}) (err error) { return nil } +// Encode encodes value v and stores its representation in n. +// +// See the documentation for Marshal for details about the +// conversion of Go values into YAML. +func (n *Node) Encode(v interface{}) (err error) { + defer handleErr(&err) + e := newEncoder() + defer e.destroy() + e.marshalDoc("", reflect.ValueOf(v)) + e.finish() + p := newParser(e.out) + p.textless = true + defer p.destroy() + doc := p.parse() + *n = *doc.Content[0] + return nil +} + // SetIndent changes the used indentation used when encoding. func (e *Encoder) SetIndent(spaces int) { if spaces < 0 { @@ -328,6 +346,12 @@ const ( // and maps, Node is an intermediate representation that allows detailed // control over the content being decoded or encoded. // +// It's worth noting that although Node offers access into details such as +// line numbers, colums, and comments, the content when re-encoded will not +// have its original textual representation preserved. An effort is made to +// render the data plesantly, and to preserve comments near the data they +// describe, though. +// // Values that make use of the Node type interact with the yaml package in the // same way any other type would do, by encoding and decoding yaml data // directly or indirectly into them. @@ -391,6 +415,13 @@ type Node struct { Column int } +// IsZero returns whether the node has all of its fields unset. +func (n *Node) IsZero() bool { + return n.Kind == 0 && n.Style == 0 && n.Tag == "" && n.Value == "" && n.Anchor == "" && n.Alias == nil && n.Content == nil && + n.HeadComment == "" && n.LineComment == "" && n.FootComment == "" && n.Line == 0 && n.Column == 0 +} + + // LongTag returns the long form of the tag that indicates the data type for // the node. If the Tag field isn't explicitly defined, one will be computed // based on the node properties. diff --git a/vendor/gopkg.in/yaml.v3/yamlh.go b/vendor/gopkg.in/yaml.v3/yamlh.go index 65fb0df3b17..7c6d0077061 100644 --- a/vendor/gopkg.in/yaml.v3/yamlh.go +++ b/vendor/gopkg.in/yaml.v3/yamlh.go @@ -600,6 +600,7 @@ type yaml_parser_t struct { line_comment []byte // The current line comments foot_comment []byte // The current foot comments tail_comment []byte // Foot comment that happens at the end of a block. + stem_comment []byte // Comment in item preceding a nested structure (list inside list item, etc) comments []yaml_comment_t // The folded comments for all parsed tokens comments_head int @@ -621,6 +622,7 @@ type yaml_parser_t struct { simple_key_allowed bool // May a simple key occur at the current position? simple_keys []yaml_simple_key_t // The stack of simple keys. + simple_keys_by_tok map[int]int // possible simple_key indexes indexed by token_number // Parser stuff @@ -785,6 +787,8 @@ type yaml_emitter_t struct { foot_comment []byte tail_comment []byte + key_line_comment []byte + // Dumper stuff opened bool // If the stream was already opened? diff --git a/vendor/modules.txt b/vendor/modules.txt index dcfacff1f58..3819bb2548e 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -600,7 +600,7 @@ gopkg.in/inf.v0 # gopkg.in/yaml.v2 v2.3.0 ## explicit gopkg.in/yaml.v2 -# gopkg.in/yaml.v3 v3.0.0-20191026110619-0b21df46bc1d +# gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 ## explicit gopkg.in/yaml.v3 # honnef.co/go/tools v0.0.1-2020.1.4 From d10281140ac876786a2b75a12f329a623291b029 Mon Sep 17 00:00:00 2001 From: slinkydeveloper Date: Tue, 13 Oct 2020 18:17:31 +0200 Subject: [PATCH 02/10] Reuse recordevents as transformevents (#4291) * transformevents done Signed-off-by: Francesco Guardiani * transformevents done Signed-off-by: Francesco Guardiani * Test Signed-off-by: Francesco Guardiani * Fix copyright Signed-off-by: Francesco Guardiani * Did something Signed-off-by: Francesco Guardiani (cherry picked from commit d5d4f4239bc8f1186debd527edfe8404a8ff8a8c) Signed-off-by: Francesco Guardiani --- go.mod | 1 - ...channel_event_tranformation_test_helper.go | 13 +-- test/lib/recordevents/event_info_store.go | 31 -------- test/lib/recordevents/observer/observer.go | 58 +++++++++----- test/lib/recordevents/observer/reply.go | 70 ++++++++++++++++ .../recordevents/recorder_vent/constructor.go | 2 +- test/lib/recordevents/resources.go | 79 ++++++++++++++++++- test/test_images/recordevents/main.go | 6 +- vendor/modules.txt | 1 - 9 files changed, 200 insertions(+), 61 deletions(-) create mode 100644 test/lib/recordevents/observer/reply.go diff --git a/go.mod b/go.mod index 4d125c2fd9d..88fca4c3603 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,6 @@ require ( github.com/pelletier/go-toml v1.8.0 github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 github.com/pkg/errors v0.9.1 - github.com/prometheus/common v0.9.1 github.com/rickb777/date v1.13.0 github.com/robfig/cron/v3 v3.0.1 github.com/rogpeppe/fastuuid v1.2.0 diff --git a/test/e2e/helpers/channel_event_tranformation_test_helper.go b/test/e2e/helpers/channel_event_tranformation_test_helper.go index 9b252ff4f3c..c00d0991cda 100644 --- a/test/e2e/helpers/channel_event_tranformation_test_helper.go +++ b/test/e2e/helpers/channel_event_tranformation_test_helper.go @@ -62,13 +62,16 @@ func EventTransformationForSubscriptionTestHelper(t *testing.T, if err := eventAfterTransformation.SetData(cloudevents.ApplicationJSON, []byte(transformedEventBody)); err != nil { t.Fatalf("Cannot set the payload of the event: %s", err.Error()) } - transformationPod := resources.EventTransformationPod( + recordevents.DeployEventRecordOrFail( + ctx, + client, transformationPodName, - eventAfterTransformation.Type(), - eventAfterTransformation.Source(), - eventAfterTransformation.Data(), + recordevents.ReplyWithTransformedEvent( + eventAfterTransformation.Type(), + eventAfterTransformation.Source(), + string(eventAfterTransformation.Data()), + ), ) - client.CreatePodOrFail(transformationPod, testlib.WithService(transformationPodName)) // create event logger pod and service as the subscriber eventTracker, _ := recordevents.StartEventRecordOrFail(client, recordEventsPodName) diff --git a/test/lib/recordevents/event_info_store.go b/test/lib/recordevents/event_info_store.go index 007a4c14952..2a3d5fda9fe 100644 --- a/test/lib/recordevents/event_info_store.go +++ b/test/lib/recordevents/event_info_store.go @@ -28,12 +28,9 @@ import ( "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" - rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/util/wait" - pkgTest "knative.dev/pkg/test" testlib "knative.dev/eventing/test/lib" - "knative.dev/eventing/test/lib/resources" ) const ( @@ -42,34 +39,6 @@ const ( retryTimeout = 4 * time.Minute ) -type EventRecordOption = func(*corev1.Pod, *testlib.Client) error - -func DeployEventRecordOrFail(ctx context.Context, client *testlib.Client, name string, options ...EventRecordOption) *corev1.Pod { - client.CreateServiceAccountOrFail(name) - client.CreateRoleOrFail(resources.Role(name, - resources.WithRuleForRole(&rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"pods"}, - Verbs: []string{"get"}, - }), - resources.WithRuleForRole(&rbacv1.PolicyRule{ - APIGroups: []string{""}, - Resources: []string{"events"}, - Verbs: []string{rbacv1.VerbAll}, - }), - )) - client.CreateRoleBindingOrFail(name, "Role", name, name, client.Namespace) - - eventRecordPod := EventRecordPod(name, name) - client.CreatePodOrFail(eventRecordPod, append(options, testlib.WithService(name))...) - err := pkgTest.WaitForPodRunning(ctx, client.Kube, name, client.Namespace) - if err != nil { - client.T.Fatalf("Failed to start the recordevent pod '%s': %v", name, errors.WithStack(err)) - } - client.WaitForServiceEndpointsOrFail(ctx, name, 1) - return eventRecordPod -} - // Deploys a new recordevents pod and start the associated EventInfoStore func StartEventRecordOrFail(ctx context.Context, client *testlib.Client, podName string, options ...EventRecordOption) (*EventInfoStore, *corev1.Pod) { eventRecordPod := DeployEventRecordOrFail(ctx, client, podName, options...) diff --git a/test/lib/recordevents/observer/observer.go b/test/lib/recordevents/observer/observer.go index e9bd4224aac..badc5048802 100644 --- a/test/lib/recordevents/observer/observer.go +++ b/test/lib/recordevents/observer/observer.go @@ -25,7 +25,6 @@ import ( cloudeventsbindings "github.com/cloudevents/sdk-go/v2/binding" cloudeventshttp "github.com/cloudevents/sdk-go/v2/protocol/http" "github.com/kelseyhightower/envconfig" - "github.com/prometheus/common/log" "knative.dev/pkg/logging" "knative.dev/eventing/test/lib/recordevents" @@ -33,35 +32,57 @@ import ( // Observer is the entry point for sinking events into the event log. type Observer struct { + // Name is the name of this Observer, used to filter if multiple observers. Name string // EventLogs is the list of EventLog implementors to vent observed events. EventLogs recordevents.EventLogs - seq uint64 -} - -// New returns an observer that will vent observations to the list of provided -// EventLog instances. It will listen on :8080. -func New(name string, eventLogs ...recordevents.EventLog) *Observer { - return &Observer{ - Name: name, - EventLogs: eventLogs, - } + ctx context.Context + seq uint64 + replyFunc func(context.Context, http.ResponseWriter, recordevents.EventInfo) } type envConfig struct { // ObserverName is used to identify this instance of the observer. ObserverName string `envconfig:"OBSERVER_NAME" default:"observer-default" required:"true"` + + // Reply is used to define if the observer should reply back + Reply bool `envconfig:"REPLY" default:"false" required:"false"` + + // The event type to use in the reply, if enabled + ReplyEventType string `envconfig:"REPLY_EVENT_TYPE" default:"" required:"false"` + + // The event source to use in the reply, if enabled + ReplyEventSource string `envconfig:"REPLY_EVENT_SOURCE" default:"" required:"false"` + + // The event data to use in the reply, if enabled + ReplyEventData string `envconfig:"REPLY_EVENT_DATA" default:"" required:"false"` } -func NewFromEnv(eventLogs ...recordevents.EventLog) *Observer { +func NewFromEnv(ctx context.Context, eventLogs ...recordevents.EventLog) *Observer { var env envConfig if err := envconfig.Process("", &env); err != nil { - log.Fatal("Failed to process env var", err) + logging.FromContext(ctx).Fatal("Failed to process env var", err) + } + + logging.FromContext(ctx).Infof("Observer environment configuration: %+v", env) + + var replyFunc func(context.Context, http.ResponseWriter, recordevents.EventInfo) + if env.Reply { + logging.FromContext(ctx).Info("Observer will reply with an event") + replyFunc = ReplyTransformerFunc(env.ReplyEventType, env.ReplyEventSource, env.ReplyEventData) + } else { + logging.FromContext(ctx).Info("Observer won't reply with an event") + replyFunc = NoOpReply } - return New(env.ObserverName, eventLogs...) + return &Observer{ + Name: env.ObserverName, + EventLogs: eventLogs, + ctx: ctx, + replyFunc: replyFunc, + } } // Start will create the CloudEvents client and start listening for inbound @@ -99,7 +120,7 @@ func (o *Observer) ServeHTTP(writer http.ResponseWriter, request *http.Request) if eventErr != nil { eventErrStr = eventErr.Error() } - err := o.EventLogs.Vent(recordevents.EventInfo{ + eventInfo := recordevents.EventInfo{ Error: eventErrStr, Event: event, HTTPHeaders: header, @@ -107,10 +128,11 @@ func (o *Observer) ServeHTTP(writer http.ResponseWriter, request *http.Request) Observer: o.Name, Time: time.Now(), Sequence: atomic.AddUint64(&o.seq, 1), - }) + } + err := o.EventLogs.Vent(eventInfo) if err != nil { - log.Warn("Error while venting the recorded event", err) + logging.FromContext(o.ctx).Warn("Error while venting the recorded event", err) } - writer.WriteHeader(http.StatusAccepted) + o.replyFunc(o.ctx, writer, eventInfo) } diff --git a/test/lib/recordevents/observer/reply.go b/test/lib/recordevents/observer/reply.go new file mode 100644 index 00000000000..96456977f0c --- /dev/null +++ b/test/lib/recordevents/observer/reply.go @@ -0,0 +1,70 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observer + +import ( + "context" + "net/http" + + cloudevents "github.com/cloudevents/sdk-go/v2" + "github.com/cloudevents/sdk-go/v2/binding" + cehttp "github.com/cloudevents/sdk-go/v2/protocol/http" + "knative.dev/pkg/logging" + + "knative.dev/eventing/test/lib/recordevents" +) + +func NoOpReply(_ context.Context, writer http.ResponseWriter, _ recordevents.EventInfo) { + writer.WriteHeader(http.StatusAccepted) +} + +func ReplyTransformerFunc(replyEventType string, replyEventSource string, replyEventData string) func(context.Context, http.ResponseWriter, recordevents.EventInfo) { + return func(ctx context.Context, writer http.ResponseWriter, info recordevents.EventInfo) { + if info.Error != "" { + writer.WriteHeader(http.StatusBadRequest) + _, _ = writer.Write([]byte(info.Error)) + logging.FromContext(ctx).Warn("Conversion error in the event to send back", info.Error) + return + } + + if info.Event == nil { + writer.WriteHeader(http.StatusBadRequest) + _, _ = writer.Write([]byte("No event!")) + logging.FromContext(ctx).Warn("No event to send back") + return + } + + outputEvent := info.Event.Clone() + + if replyEventSource != "" { + outputEvent.SetSource(replyEventSource) + } + if replyEventType != "" { + outputEvent.SetType(replyEventType) + } + if replyEventData != "" { + if err := outputEvent.SetData(cloudevents.ApplicationJSON, []byte(replyEventData)); err != nil { + logging.FromContext(ctx).Warn("Cannot set the event data") + } + } + + err := cehttp.WriteResponseWriter(ctx, binding.ToMessage(&outputEvent), 200, writer) + if err != nil { + logging.FromContext(ctx).Warn("Error while writing the event as response", err) + } + } +} diff --git a/test/lib/recordevents/recorder_vent/constructor.go b/test/lib/recordevents/recorder_vent/constructor.go index cc73a9671f0..c4743c67ae8 100644 --- a/test/lib/recordevents/recorder_vent/constructor.go +++ b/test/lib/recordevents/recorder_vent/constructor.go @@ -50,7 +50,7 @@ func NewFromEnv(ctx context.Context) recordevents.EventLog { log.Fatal("Failed to process env var", err) } - logging.FromContext(ctx).Infof("Environment configuration: %+v", env) + logging.FromContext(ctx).Infof("Recorder vent environment configuration: %+v", env) return NewEventLog(ctx, env.AgentName, env.PodName) } diff --git a/test/lib/recordevents/resources.go b/test/lib/recordevents/resources.go index 54726bedfdc..4fa3dc88f40 100644 --- a/test/lib/recordevents/resources.go +++ b/test/lib/recordevents/resources.go @@ -17,12 +17,89 @@ limitations under the License. package recordevents import ( + "context" + + "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/uuid" - "knative.dev/pkg/test" + pkgtest "knative.dev/pkg/test" + + testlib "knative.dev/eventing/test/lib" + "knative.dev/eventing/test/lib/resources" ) +type EventRecordOption = func(*corev1.Pod, *testlib.Client) error + +// EchoEvent is an option to let the recordevents reply with the received event +func EchoEvent(pod *corev1.Pod, client *testlib.Client) error { + pod.Spec.Containers[0].Env = append( + pod.Spec.Containers[0].Env, + corev1.EnvVar{Name: "REPLY", Value: "true"}, + ) + return nil +} + +var _ EventRecordOption = EchoEvent + +// ReplyWithTransformedEvent is an option to let the recordevents reply with the transformed event +func ReplyWithTransformedEvent(replyEventType string, replyEventSource string, replyEventData string) EventRecordOption { + return func(pod *corev1.Pod, client *testlib.Client) error { + pod.Spec.Containers[0].Env = append( + pod.Spec.Containers[0].Env, + corev1.EnvVar{Name: "REPLY", Value: "true"}, + ) + if replyEventType != "" { + pod.Spec.Containers[0].Env = append( + pod.Spec.Containers[0].Env, + corev1.EnvVar{Name: "REPLY_EVENT_TYPE", Value: replyEventType}, + ) + } + if replyEventSource != "" { + pod.Spec.Containers[0].Env = append( + pod.Spec.Containers[0].Env, + corev1.EnvVar{Name: "REPLY_EVENT_SOURCE", Value: replyEventSource}, + ) + } + if replyEventData != "" { + pod.Spec.Containers[0].Env = append( + pod.Spec.Containers[0].Env, + corev1.EnvVar{Name: "REPLY_EVENT_DATA", Value: replyEventData}, + ) + } + + return nil + } +} + +// DeployEventRecordOrFail deploys the recordevents image with necessary sa, roles, rb to execute the image +func DeployEventRecordOrFail(ctx context.Context, client *testlib.Client, name string, options ...EventRecordOption) *corev1.Pod { + client.CreateServiceAccountOrFail(name) + client.CreateRoleOrFail(resources.Role(name, + resources.WithRuleForRole(&rbacv1.PolicyRule{ + APIGroups: []string{""}, + Resources: []string{"pods"}, + Verbs: []string{"get"}, + }), + resources.WithRuleForRole(&rbacv1.PolicyRule{ + APIGroups: []string{""}, + Resources: []string{"events"}, + Verbs: []string{rbacv1.VerbAll}, + }), + )) + client.CreateRoleBindingOrFail(name, "Role", name, name, client.Namespace) + + eventRecordPod := EventRecordPod(name, name) + client.CreatePodOrFail(eventRecordPod, append(options, testlib.WithService(name))...) + err := pkgtest.WaitForPodRunning(ctx, client.Kube, name, client.Namespace) + if err != nil { + client.T.Fatalf("Failed to start the recordevent pod '%s': %v", name, errors.WithStack(err)) + } + client.WaitForServiceEndpointsOrFail(ctx, name, 1) + return eventRecordPod +} + // EventRecordPod creates a Pod that stores received events for test retrieval. func EventRecordPod(name string, serviceAccountName string) *corev1.Pod { return recordEventsPod("recordevents", name, serviceAccountName) diff --git a/test/test_images/recordevents/main.go b/test/test_images/recordevents/main.go index 51fde078e9b..38445c45bc8 100644 --- a/test/test_images/recordevents/main.go +++ b/test/test_images/recordevents/main.go @@ -22,7 +22,7 @@ import ( "os" "k8s.io/client-go/rest" - "knative.dev/pkg/injection/sharedmain" + "knative.dev/pkg/injection" "knative.dev/pkg/logging" _ "knative.dev/pkg/system/testing" @@ -39,13 +39,13 @@ func main() { log.Fatal("Error while reading the cfg", err) } //nolint // nil ctx is fine here, look at the code of EnableInjectionOrDie - ctx := sharedmain.EnableInjectionOrDie(nil, cfg) + ctx, _ := injection.EnableInjectionOrDie(nil, cfg) if err := test_images.ConfigureTracing(logging.FromContext(ctx), ""); err != nil { logging.FromContext(ctx).Fatal("Unable to setup trace publishing", err) } - obs := observer.NewFromEnv( + obs := observer.NewFromEnv(ctx, recorder_vent.NewFromEnv(ctx), ) diff --git a/vendor/modules.txt b/vendor/modules.txt index 3819bb2548e..2423edd4666 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -256,7 +256,6 @@ github.com/prometheus/client_golang/prometheus/promhttp # github.com/prometheus/client_model v0.2.0 github.com/prometheus/client_model/go # github.com/prometheus/common v0.9.1 -## explicit github.com/prometheus/common/expfmt github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg github.com/prometheus/common/log From 4c5cef6e8facfde4887fa37adea0e89abdaf2c94 Mon Sep 17 00:00:00 2001 From: Francesco Guardiani Date: Wed, 14 Oct 2020 14:39:30 +0200 Subject: [PATCH 03/10] Using recordevents as sequencestepper (#4313) Signed-off-by: Francesco Guardiani (cherry picked from commit 656ab81f8708177f0aae3423dd5beb1b92f41f19) --- test/e2e/helpers/sequence_test_helper.go | 24 +++++++++++++--------- test/lib/recordevents/observer/observer.go | 6 +++++- test/lib/recordevents/observer/reply.go | 22 +++++++++++++++++++- test/lib/recordevents/resources.go | 18 ++++++++++++++++ 4 files changed, 58 insertions(+), 12 deletions(-) diff --git a/test/e2e/helpers/sequence_test_helper.go b/test/e2e/helpers/sequence_test_helper.go index 321957d96bb..fdaadb5700f 100644 --- a/test/e2e/helpers/sequence_test_helper.go +++ b/test/e2e/helpers/sequence_test_helper.go @@ -73,9 +73,11 @@ func SequenceTestHelper(t *testing.T, // create a stepper Pod with Service podName := config.podName msgAppender := config.msgAppender - stepperPod := resources.SequenceStepperPod(podName, msgAppender) + recordevents.DeployEventRecordOrFail( + ctx, client, podName, + recordevents.ReplyWithAppendedData(msgAppender), + ) - client.CreatePodOrFail(stepperPod, testlib.WithService(podName)) // create a new step step := v1beta1.SequenceStep{ Destination: duckv1.Destination{ @@ -128,8 +130,7 @@ func SequenceTestHelper(t *testing.T, event.SetSource(eventSource) event.SetType(testlib.DefaultEventType) msg := fmt.Sprintf("TestSequence %s", uuid.New().String()) - body := fmt.Sprintf(`{"msg":"%s"}`, msg) - if err := event.SetData(cloudevents.ApplicationJSON, []byte(body)); err != nil { + if err := event.SetData(cloudevents.TextPlain, msg); err != nil { st.Fatalf("Cannot set the payload of the event: %s", err.Error()) } client.SendEventToAddressable( @@ -145,7 +146,8 @@ func SequenceTestHelper(t *testing.T, } eventTracker.AssertAtLeast(1, recordevents.MatchEvent( cetest.HasSource(eventSource), - cetest.DataContains(expectedMsg), + cetest.HasDataContentType(cloudevents.TextPlain), + cetest.HasData([]byte(expectedMsg)), )) }) } @@ -186,9 +188,11 @@ func SequenceV1TestHelper(t *testing.T, // create a stepper Pod with Service podName := config.podName msgAppender := config.msgAppender - stepperPod := resources.SequenceStepperPod(podName, msgAppender) + recordevents.DeployEventRecordOrFail( + ctx, client, podName, + recordevents.ReplyWithAppendedData(msgAppender), + ) - client.CreatePodOrFail(stepperPod, testlib.WithService(podName)) // create a new step step := flowsv1.SequenceStep{ Destination: duckv1.Destination{ @@ -241,8 +245,7 @@ func SequenceV1TestHelper(t *testing.T, event.SetSource(eventSource) event.SetType(testlib.DefaultEventType) msg := fmt.Sprintf("TestSequence %s", uuid.New().String()) - body := fmt.Sprintf(`{"msg":"%s"}`, msg) - if err := event.SetData(cloudevents.ApplicationJSON, []byte(body)); err != nil { + if err := event.SetData(cloudevents.TextPlain, msg); err != nil { st.Fatalf("Cannot set the payload of the event: %s", err.Error()) } client.SendEventToAddressable( @@ -258,7 +261,8 @@ func SequenceV1TestHelper(t *testing.T, } eventTracker.AssertAtLeast(1, recordevents.MatchEvent( cetest.HasSource(eventSource), - cetest.DataContains(expectedMsg), + cetest.HasDataContentType(cloudevents.TextPlain), + cetest.HasData([]byte(expectedMsg)), )) }) } diff --git a/test/lib/recordevents/observer/observer.go b/test/lib/recordevents/observer/observer.go index badc5048802..b57dac9c718 100644 --- a/test/lib/recordevents/observer/observer.go +++ b/test/lib/recordevents/observer/observer.go @@ -58,6 +58,10 @@ type envConfig struct { // The event data to use in the reply, if enabled ReplyEventData string `envconfig:"REPLY_EVENT_DATA" default:"" required:"false"` + + // This string to append in the data field in the reply, if enabled. + // This will threat the data as text/plain field + ReplyAppendData string `envconfig:"REPLY_APPEND_DATA" default:"" required:"false"` } func NewFromEnv(ctx context.Context, eventLogs ...recordevents.EventLog) *Observer { @@ -71,7 +75,7 @@ func NewFromEnv(ctx context.Context, eventLogs ...recordevents.EventLog) *Observ var replyFunc func(context.Context, http.ResponseWriter, recordevents.EventInfo) if env.Reply { logging.FromContext(ctx).Info("Observer will reply with an event") - replyFunc = ReplyTransformerFunc(env.ReplyEventType, env.ReplyEventSource, env.ReplyEventData) + replyFunc = ReplyTransformerFunc(env.ReplyEventType, env.ReplyEventSource, env.ReplyEventData, env.ReplyAppendData) } else { logging.FromContext(ctx).Info("Observer won't reply with an event") replyFunc = NoOpReply diff --git a/test/lib/recordevents/observer/reply.go b/test/lib/recordevents/observer/reply.go index 96456977f0c..f60945d1b00 100644 --- a/test/lib/recordevents/observer/reply.go +++ b/test/lib/recordevents/observer/reply.go @@ -23,6 +23,7 @@ import ( cloudevents "github.com/cloudevents/sdk-go/v2" "github.com/cloudevents/sdk-go/v2/binding" cehttp "github.com/cloudevents/sdk-go/v2/protocol/http" + "go.uber.org/zap" "knative.dev/pkg/logging" "knative.dev/eventing/test/lib/recordevents" @@ -32,7 +33,7 @@ func NoOpReply(_ context.Context, writer http.ResponseWriter, _ recordevents.Eve writer.WriteHeader(http.StatusAccepted) } -func ReplyTransformerFunc(replyEventType string, replyEventSource string, replyEventData string) func(context.Context, http.ResponseWriter, recordevents.EventInfo) { +func ReplyTransformerFunc(replyEventType string, replyEventSource string, replyEventData string, replyAppendData string) func(context.Context, http.ResponseWriter, recordevents.EventInfo) { return func(ctx context.Context, writer http.ResponseWriter, info recordevents.EventInfo) { if info.Error != "" { writer.WriteHeader(http.StatusBadRequest) @@ -51,17 +52,36 @@ func ReplyTransformerFunc(replyEventType string, replyEventSource string, replyE outputEvent := info.Event.Clone() if replyEventSource != "" { + logging.FromContext(ctx).Infof("Setting reply event source '%s'", replyEventSource) outputEvent.SetSource(replyEventSource) } if replyEventType != "" { + logging.FromContext(ctx).Infof("Setting reply event type '%s'", replyEventType) outputEvent.SetType(replyEventType) } if replyEventData != "" { + logging.FromContext(ctx).Infof("Setting reply event data '%s'", replyAppendData) if err := outputEvent.SetData(cloudevents.ApplicationJSON, []byte(replyEventData)); err != nil { logging.FromContext(ctx).Warn("Cannot set the event data") } } + if replyAppendData != "" { + var d string + if outputEvent.Data() == nil { + d = replyAppendData + } else { + if err := info.Event.DataAs(&d); err != nil { + logging.FromContext(ctx).Warn("Cannot read the event data as text/plain") + } + d = d + replyAppendData + } + logging.FromContext(ctx).Infof("Setting appended event data '%s'", d) + if err := outputEvent.SetData(cloudevents.TextPlain, d); err != nil { + logging.FromContext(ctx).Warn("Cannot set the event data") + } + } + logging.FromContext(ctx).Infow("Replying with", zap.Stringer("event", outputEvent)) err := cehttp.WriteResponseWriter(ctx, binding.ToMessage(&outputEvent), 200, writer) if err != nil { logging.FromContext(ctx).Warn("Error while writing the event as response", err) diff --git a/test/lib/recordevents/resources.go b/test/lib/recordevents/resources.go index 4fa3dc88f40..f5f0dc87e5d 100644 --- a/test/lib/recordevents/resources.go +++ b/test/lib/recordevents/resources.go @@ -73,6 +73,24 @@ func ReplyWithTransformedEvent(replyEventType string, replyEventSource string, r } } +// ReplyWithAppendedData is an option to let the recordevents reply with the transformed event with appended data +func ReplyWithAppendedData(appendData string) EventRecordOption { + return func(pod *corev1.Pod, client *testlib.Client) error { + pod.Spec.Containers[0].Env = append( + pod.Spec.Containers[0].Env, + corev1.EnvVar{Name: "REPLY", Value: "true"}, + ) + if appendData != "" { + pod.Spec.Containers[0].Env = append( + pod.Spec.Containers[0].Env, + corev1.EnvVar{Name: "REPLY_APPEND_DATA", Value: appendData}, + ) + } + + return nil + } +} + // DeployEventRecordOrFail deploys the recordevents image with necessary sa, roles, rb to execute the image func DeployEventRecordOrFail(ctx context.Context, client *testlib.Client, name string, options ...EventRecordOption) *corev1.Pod { client.CreateServiceAccountOrFail(name) From 311666957e8d520f16f3ebad1313c97c6eadd60e Mon Sep 17 00:00:00 2001 From: slinkydeveloper Date: Wed, 14 Oct 2020 16:09:30 +0200 Subject: [PATCH 04/10] Remove transformevents usages (#4309) * Remove transformevents Signed-off-by: Francesco Guardiani * Comments Signed-off-by: Francesco Guardiani * Let's be less breaky: brought back the image Signed-off-by: Francesco Guardiani (cherry picked from commit 3e7f942d21ca15ffc8a63b72952d9c0ad7183816) Signed-off-by: Francesco Guardiani --- .../helpers/broker_tracing_test_helper.go | 15 ++++++----- .../helpers/channel_tracing_test_helper.go | 13 +++++---- .../e2e/helpers/broker_channel_flow_helper.go | 13 +++++---- ...broker_event_transformation_test_helper.go | 13 +++++---- test/lib/recordevents/resources.go | 6 ++--- test/lib/resources/kube.go | 27 ------------------- 6 files changed, 36 insertions(+), 51 deletions(-) diff --git a/test/conformance/helpers/broker_tracing_test_helper.go b/test/conformance/helpers/broker_tracing_test_helper.go index ff415db469f..e4236ccbf4f 100644 --- a/test/conformance/helpers/broker_tracing_test_helper.go +++ b/test/conformance/helpers/broker_tracing_test_helper.go @@ -89,15 +89,18 @@ func setupBrokerTracing(brokerClass string) SetupTracingTestInfrastructureFunc { resources.WithSubscriberServiceRefForTriggerV1Beta1(loggerPodName), ) - // Create a transformer (EventTransfrmer) Pod that replies with the same event as the input, + // Create a transformer Pod (recordevents with transform reply) that replies with the same event as the input, // except the reply's event's type is changed to bar. - eventTransformerPod := resources.EventTransformationPod( + eventTransformerPod := recordevents.DeployEventRecordOrFail( + ctx, + client, "transformer", - etLogger, - senderName, - []byte(eventBody), + recordevents.ReplyWithTransformedEvent( + etLogger, + senderName, + eventBody, + ), ) - client.CreatePodOrFail(eventTransformerPod, testlib.WithService(eventTransformerPod.Name)) // Create a Trigger that receives events (type=foo) and sends them to the transformer Pod. transformerTrigger := client.CreateTriggerOrFailV1Beta1( diff --git a/test/conformance/helpers/channel_tracing_test_helper.go b/test/conformance/helpers/channel_tracing_test_helper.go index b493960ea15..ae854d184e3 100644 --- a/test/conformance/helpers/channel_tracing_test_helper.go +++ b/test/conformance/helpers/channel_tracing_test_helper.go @@ -70,13 +70,16 @@ func setupChannelTracingWithReply( recordEventsPod := recordevents.DeployEventRecordOrFail(ctx, client, recordEventsPodName) // Create the subscriber, a Pod that mutates the event. - transformerPod := resources.EventTransformationPod( + transformerPod := recordevents.DeployEventRecordOrFail( + ctx, + client, "transformer", - "mutated", - eventSource, - nil, + recordevents.ReplyWithTransformedEvent( + "mutated", + eventSource, + "", + ), ) - client.CreatePodOrFail(transformerPod, testlib.WithService(transformerPod.Name)) // Create the Subscription linking the Channel to the mutator. client.CreateSubscriptionOrFail( diff --git a/test/e2e/helpers/broker_channel_flow_helper.go b/test/e2e/helpers/broker_channel_flow_helper.go index ce216c5ce79..4e3eb047fe7 100644 --- a/test/e2e/helpers/broker_channel_flow_helper.go +++ b/test/e2e/helpers/broker_channel_flow_helper.go @@ -104,13 +104,16 @@ func BrokerChannelFlowWithTransformation(t *testing.T, } // create the transformation service for trigger1 - transformationPod := resources.EventTransformationPod( + recordevents.DeployEventRecordOrFail( + ctx, + client, transformationPodName, - transformedEventType, - transformedEventSource, - []byte(transformedBody), + recordevents.ReplyWithTransformedEvent( + transformedEventType, + transformedEventSource, + transformedBody, + ), ) - client.CreatePodOrFail(transformationPod, testlib.WithService(transformationPodName)) // create trigger1 to receive the original event, and do event transformation if triggerVersion == "v1" { diff --git a/test/e2e/helpers/broker_event_transformation_test_helper.go b/test/e2e/helpers/broker_event_transformation_test_helper.go index dae341369b8..fc034809198 100644 --- a/test/e2e/helpers/broker_event_transformation_test_helper.go +++ b/test/e2e/helpers/broker_event_transformation_test_helper.go @@ -70,13 +70,16 @@ func EventTransformationForTriggerTestHelper(t *testing.T, client.WaitForResourceReadyOrFail(brokerName, testlib.BrokerTypeMeta) // create the transformation service - transformationPod := resources.EventTransformationPod( + recordevents.DeployEventRecordOrFail( + ctx, + client, transformationPodName, - transformedEventType, - transformedEventSource, - []byte(transformedBody), + recordevents.ReplyWithTransformedEvent( + transformedEventType, + transformedEventSource, + transformedBody, + ), ) - client.CreatePodOrFail(transformationPod, testlib.WithService(transformationPodName)) // create trigger1 for event transformation if triggerVersion == "v1" { diff --git a/test/lib/recordevents/resources.go b/test/lib/recordevents/resources.go index f5f0dc87e5d..481b1936fc7 100644 --- a/test/lib/recordevents/resources.go +++ b/test/lib/recordevents/resources.go @@ -108,7 +108,7 @@ func DeployEventRecordOrFail(ctx context.Context, client *testlib.Client, name s )) client.CreateRoleBindingOrFail(name, "Role", name, name, client.Namespace) - eventRecordPod := EventRecordPod(name, name) + eventRecordPod := eventRecordPod(name, name) client.CreatePodOrFail(eventRecordPod, append(options, testlib.WithService(name))...) err := pkgtest.WaitForPodRunning(ctx, client.Kube, name, client.Namespace) if err != nil { @@ -118,8 +118,8 @@ func DeployEventRecordOrFail(ctx context.Context, client *testlib.Client, name s return eventRecordPod } -// EventRecordPod creates a Pod that stores received events for test retrieval. -func EventRecordPod(name string, serviceAccountName string) *corev1.Pod { +// eventRecordPod creates a Pod that stores received events for test retrieval. +func eventRecordPod(name string, serviceAccountName string) *corev1.Pod { return recordEventsPod("recordevents", name, serviceAccountName) } diff --git a/test/lib/resources/kube.go b/test/lib/resources/kube.go index bc415d10873..b504f6eb91d 100644 --- a/test/lib/resources/kube.go +++ b/test/lib/resources/kube.go @@ -37,33 +37,6 @@ type PodOption func(*corev1.Pod) // Option enables further configuration of a Role. type RoleOption func(*rbacv1.Role) -// EventTransformationPod creates a Pod that transforms events received receiving as arg a cloudevents sdk2 Event -func EventTransformationPod(name string, newEventType string, newEventSource string, newEventData []byte) *corev1.Pod { - const imageName = "transformevents" - return &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Labels: map[string]string{"e2etest": string(uuid.NewUUID())}, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{{ - Name: imageName, - Image: pkgTest.ImagePath(imageName), - ImagePullPolicy: corev1.PullAlways, - Args: []string{ - "-event-type", - newEventType, - "-event-source", - newEventSource, - "-event-data", - string(newEventData), - }, - }}, - RestartPolicy: corev1.RestartPolicyAlways, - }, - } -} - // HelloWorldPod creates a Pod that logs "Hello, World!". func HelloWorldPod(name string, options ...PodOption) *corev1.Pod { const imageName = "print" From 0090b58891237ff007f84368df8469b4c8beb63e Mon Sep 17 00:00:00 2001 From: slinkydeveloper Date: Tue, 20 Oct 2020 10:09:54 +0200 Subject: [PATCH 05/10] Fixes after cherry pick Signed-off-by: Francesco Guardiani --- go.mod | 1 + go.sum | 8 +- .../broker_control_plane_test_helper.go | 4 +- .../helpers/broker_data_plane_test_helper.go | 11 +- .../helpers/broker_tracing_test_helper.go | 3 +- .../channel_status_subscriber_test_helper.go | 2 +- .../helpers/channel_tracing_test_helper.go | 3 +- .../e2e/helpers/broker_channel_flow_helper.go | 1 - ...broker_event_transformation_test_helper.go | 1 - ...channel_event_tranformation_test_helper.go | 1 - test/e2e/helpers/sequence_test_helper.go | 4 +- .../helpers/trigger_no_broker_test_helper.go | 2 +- test/lib/recordevents/event_info_store.go | 5 +- .../recordevents/recorder_vent/constructor.go | 2 +- test/lib/recordevents/resources.go | 10 +- test/test_images/recordevents/main.go | 4 +- vendor/gopkg.in/yaml.v3/.travis.yml | 22 +-- vendor/gopkg.in/yaml.v3/apic.go | 1 - vendor/gopkg.in/yaml.v3/decode.go | 63 +++--- vendor/gopkg.in/yaml.v3/emitterc.go | 54 ++---- vendor/gopkg.in/yaml.v3/encode.go | 42 +--- vendor/gopkg.in/yaml.v3/parserc.go | 39 ---- vendor/gopkg.in/yaml.v3/scannerc.go | 180 +++++++----------- vendor/gopkg.in/yaml.v3/yaml.go | 35 +--- vendor/gopkg.in/yaml.v3/yamlh.go | 4 - vendor/modules.txt | 3 +- 26 files changed, 159 insertions(+), 346 deletions(-) diff --git a/go.mod b/go.mod index 88fca4c3603..4d125c2fd9d 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( github.com/pelletier/go-toml v1.8.0 github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 github.com/pkg/errors v0.9.1 + github.com/prometheus/common v0.9.1 github.com/rickb777/date v1.13.0 github.com/robfig/cron/v3 v3.0.1 github.com/rogpeppe/fastuuid v1.2.0 diff --git a/go.sum b/go.sum index 4fd256184e7..18ecf80d894 100644 --- a/go.sum +++ b/go.sum @@ -373,6 +373,7 @@ github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLi github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/fortytw2/leaktest v1.2.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= @@ -773,6 +774,7 @@ github.com/jmoiron/sqlx v1.2.1-0.20190826204134-d7d95172beb5/go.mod h1:1FEQNm3xl github.com/joefitzgerald/rainbow-reporter v0.1.0/go.mod h1:481CNgqmVHQZzdIbN52CupLJyoVwB10FQ/IQlF1pdL8= github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -854,6 +856,7 @@ github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaO github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.6 h1:6Su7aK7lXmJ/U79bYtBjLNaha4Fs1Rg9plHpcH+vvnE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= @@ -1760,8 +1763,9 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20190709130402-674ba3eaed22/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20191026110619-0b21df46bc1d h1:LCPbGQ34PMrwad11aMZ+dbz5SAsq/0ySjRwQ8I9Qwd8= +gopkg.in/yaml.v3 v3.0.0-20191026110619-0b21df46bc1d/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= helm.sh/helm/v3 v3.1.1/go.mod h1:WYsFJuMASa/4XUqLyv54s0U/f3mlAaRErGmyy4z921g= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/test/conformance/helpers/broker_control_plane_test_helper.go b/test/conformance/helpers/broker_control_plane_test_helper.go index 887a69d87b8..87f58ac1518 100644 --- a/test/conformance/helpers/broker_control_plane_test_helper.go +++ b/test/conformance/helpers/broker_control_plane_test_helper.go @@ -82,8 +82,8 @@ func triggerV1Beta1BeforeBrokerHelper(triggerName string, client *testlib.Client const etLogger = "logger" const loggerPodName = "logger-pod" - _ = recordevents.DeployEventRecordOrFail(context.TODO(), client, loggerPodName) - client.WaitForAllTestResourcesReadyOrFail(context.Background()) // Can't do this for the trigger because it's not 'ready' yet + _ = recordevents.DeployEventRecordOrFail(client, loggerPodName) + client.WaitForAllTestResourcesReadyOrFail() // Can't do this for the trigger because it's not 'ready' yet client.CreateTriggerOrFailV1Beta1(triggerName, resources.WithAttributesTriggerFilterV1Beta1(eventingv1beta1.TriggerAnyFilter, etLogger, map[string]interface{}{}), resources.WithSubscriberServiceRefForTriggerV1Beta1(loggerPodName), diff --git a/test/conformance/helpers/broker_data_plane_test_helper.go b/test/conformance/helpers/broker_data_plane_test_helper.go index 664342c8eed..a3abff1e496 100644 --- a/test/conformance/helpers/broker_data_plane_test_helper.go +++ b/test/conformance/helpers/broker_data_plane_test_helper.go @@ -316,11 +316,14 @@ func BrokerV1Beta1ConsumerDataPlaneTestHelper( source := "origin-for-reply" event.SetSource(source) msg := []byte(`{"msg":"Transformed!"}`) - transformPod := resources.EventTransformationPod( + transformPod := recordevents.DeployEventRecordOrFail( + client, "transformer-pod", - "reply-check-type", - "reply-check-source", - msg, + recordevents.ReplyWithTransformedEvent( + "reply-check-type", + "reply-check-source", + string(msg), + ), ) client.CreatePodOrFail(transformPod, testlib.WithService("transformer-pod")) client.WaitForServiceEndpointsOrFail("transformer-pod", 1) diff --git a/test/conformance/helpers/broker_tracing_test_helper.go b/test/conformance/helpers/broker_tracing_test_helper.go index e4236ccbf4f..1d3f9f63e5d 100644 --- a/test/conformance/helpers/broker_tracing_test_helper.go +++ b/test/conformance/helpers/broker_tracing_test_helper.go @@ -79,7 +79,7 @@ func setupBrokerTracing(brokerClass string) SetupTracingTestInfrastructureFunc { ) // Create a logger (EventRecord) Pod and a K8s Service that points to it. - _ = recordevents.DeployEventRecordOrFail(ctx, client, loggerPodName) + _ = recordevents.DeployEventRecordOrFail(client, loggerPodName) // Create a Trigger that receives events (type=bar) and sends them to the logger Pod. loggerTrigger := client.CreateTriggerOrFailV1Beta1( @@ -92,7 +92,6 @@ func setupBrokerTracing(brokerClass string) SetupTracingTestInfrastructureFunc { // Create a transformer Pod (recordevents with transform reply) that replies with the same event as the input, // except the reply's event's type is changed to bar. eventTransformerPod := recordevents.DeployEventRecordOrFail( - ctx, client, "transformer", recordevents.ReplyWithTransformedEvent( diff --git a/test/conformance/helpers/channel_status_subscriber_test_helper.go b/test/conformance/helpers/channel_status_subscriber_test_helper.go index e893c9424b6..05c7e0f4eed 100644 --- a/test/conformance/helpers/channel_status_subscriber_test_helper.go +++ b/test/conformance/helpers/channel_status_subscriber_test_helper.go @@ -57,7 +57,7 @@ func channelHasRequiredSubscriberStatus(st *testing.T, client *testlib.Client, c client.CreateChannelOrFail(channelName, &channel) client.WaitForResourceReadyOrFail(channelName, &channel) - _ = recordevents.DeployEventRecordOrFail(context.TODO(), client, subscriberServiceName+"-pod") + _ = recordevents.DeployEventRecordOrFail(client, subscriberServiceName+"-pod") subscription := client.CreateSubscriptionOrFail( subscriberServiceName, diff --git a/test/conformance/helpers/channel_tracing_test_helper.go b/test/conformance/helpers/channel_tracing_test_helper.go index ae854d184e3..f7c7058e42c 100644 --- a/test/conformance/helpers/channel_tracing_test_helper.go +++ b/test/conformance/helpers/channel_tracing_test_helper.go @@ -67,11 +67,10 @@ func setupChannelTracingWithReply( client.CreateChannelOrFail(replyChannelName, channel) // Create the 'sink', a LogEvents Pod and a K8s Service that points to it. - recordEventsPod := recordevents.DeployEventRecordOrFail(ctx, client, recordEventsPodName) + recordEventsPod := recordevents.DeployEventRecordOrFail(client, recordEventsPodName) // Create the subscriber, a Pod that mutates the event. transformerPod := recordevents.DeployEventRecordOrFail( - ctx, client, "transformer", recordevents.ReplyWithTransformedEvent( diff --git a/test/e2e/helpers/broker_channel_flow_helper.go b/test/e2e/helpers/broker_channel_flow_helper.go index 4e3eb047fe7..2ee1dd9df8e 100644 --- a/test/e2e/helpers/broker_channel_flow_helper.go +++ b/test/e2e/helpers/broker_channel_flow_helper.go @@ -105,7 +105,6 @@ func BrokerChannelFlowWithTransformation(t *testing.T, // create the transformation service for trigger1 recordevents.DeployEventRecordOrFail( - ctx, client, transformationPodName, recordevents.ReplyWithTransformedEvent( diff --git a/test/e2e/helpers/broker_event_transformation_test_helper.go b/test/e2e/helpers/broker_event_transformation_test_helper.go index fc034809198..6c73a639a39 100644 --- a/test/e2e/helpers/broker_event_transformation_test_helper.go +++ b/test/e2e/helpers/broker_event_transformation_test_helper.go @@ -71,7 +71,6 @@ func EventTransformationForTriggerTestHelper(t *testing.T, // create the transformation service recordevents.DeployEventRecordOrFail( - ctx, client, transformationPodName, recordevents.ReplyWithTransformedEvent( diff --git a/test/e2e/helpers/channel_event_tranformation_test_helper.go b/test/e2e/helpers/channel_event_tranformation_test_helper.go index c00d0991cda..2d80f981bba 100644 --- a/test/e2e/helpers/channel_event_tranformation_test_helper.go +++ b/test/e2e/helpers/channel_event_tranformation_test_helper.go @@ -63,7 +63,6 @@ func EventTransformationForSubscriptionTestHelper(t *testing.T, t.Fatalf("Cannot set the payload of the event: %s", err.Error()) } recordevents.DeployEventRecordOrFail( - ctx, client, transformationPodName, recordevents.ReplyWithTransformedEvent( diff --git a/test/e2e/helpers/sequence_test_helper.go b/test/e2e/helpers/sequence_test_helper.go index fdaadb5700f..9f9b5e60b68 100644 --- a/test/e2e/helpers/sequence_test_helper.go +++ b/test/e2e/helpers/sequence_test_helper.go @@ -74,7 +74,7 @@ func SequenceTestHelper(t *testing.T, podName := config.podName msgAppender := config.msgAppender recordevents.DeployEventRecordOrFail( - ctx, client, podName, + client, podName, recordevents.ReplyWithAppendedData(msgAppender), ) @@ -189,7 +189,7 @@ func SequenceV1TestHelper(t *testing.T, podName := config.podName msgAppender := config.msgAppender recordevents.DeployEventRecordOrFail( - ctx, client, podName, + client, podName, recordevents.ReplyWithAppendedData(msgAppender), ) diff --git a/test/e2e/helpers/trigger_no_broker_test_helper.go b/test/e2e/helpers/trigger_no_broker_test_helper.go index 8a4a6764f1e..cd1e97160de 100644 --- a/test/e2e/helpers/trigger_no_broker_test_helper.go +++ b/test/e2e/helpers/trigger_no_broker_test_helper.go @@ -41,7 +41,7 @@ func TestTriggerNoBroker(t *testing.T, channel string, brokerCreator BrokerCreat brokerName := strings.ToLower(channel) subscriberName := "dumper-empty" - recordevents.DeployEventRecordOrFail(context.TODO(), client, subscriberName) + recordevents.DeployEventRecordOrFail(client, subscriberName) client.CreateTriggerOrFailV1Beta1("testtrigger", resources.WithSubscriberServiceRefForTriggerV1Beta1(subscriberName), diff --git a/test/lib/recordevents/event_info_store.go b/test/lib/recordevents/event_info_store.go index 2a3d5fda9fe..bedd6249bec 100644 --- a/test/lib/recordevents/event_info_store.go +++ b/test/lib/recordevents/event_info_store.go @@ -17,7 +17,6 @@ limitations under the License. package recordevents import ( - "context" "encoding/json" "fmt" "strconv" @@ -40,8 +39,8 @@ const ( ) // Deploys a new recordevents pod and start the associated EventInfoStore -func StartEventRecordOrFail(ctx context.Context, client *testlib.Client, podName string, options ...EventRecordOption) (*EventInfoStore, *corev1.Pod) { - eventRecordPod := DeployEventRecordOrFail(ctx, client, podName, options...) +func StartEventRecordOrFail(client *testlib.Client, podName string, options ...EventRecordOption) (*EventInfoStore, *corev1.Pod) { + eventRecordPod := DeployEventRecordOrFail(client, podName, options...) eventTracker, err := NewEventInfoStore(client, podName, client.Namespace) if err != nil { diff --git a/test/lib/recordevents/recorder_vent/constructor.go b/test/lib/recordevents/recorder_vent/constructor.go index c4743c67ae8..011c075a9d3 100644 --- a/test/lib/recordevents/recorder_vent/constructor.go +++ b/test/lib/recordevents/recorder_vent/constructor.go @@ -56,7 +56,7 @@ func NewFromEnv(ctx context.Context) recordevents.EventLog { } func NewEventLog(ctx context.Context, agentName string, podName string) recordevents.EventLog { - on, err := kubeclient.Get(ctx).CoreV1().Pods(system.Namespace()).Get(ctx, podName, metav1.GetOptions{}) + on, err := kubeclient.Get(ctx).CoreV1().Pods(system.Namespace()).Get(podName, metav1.GetOptions{}) if err != nil { logging.FromContext(ctx).Fatal("Error while trying to retrieve the pod", err) } diff --git a/test/lib/recordevents/resources.go b/test/lib/recordevents/resources.go index 481b1936fc7..21bfc885cc4 100644 --- a/test/lib/recordevents/resources.go +++ b/test/lib/recordevents/resources.go @@ -17,8 +17,6 @@ limitations under the License. package recordevents import ( - "context" - "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -92,7 +90,7 @@ func ReplyWithAppendedData(appendData string) EventRecordOption { } // DeployEventRecordOrFail deploys the recordevents image with necessary sa, roles, rb to execute the image -func DeployEventRecordOrFail(ctx context.Context, client *testlib.Client, name string, options ...EventRecordOption) *corev1.Pod { +func DeployEventRecordOrFail(client *testlib.Client, name string, options ...EventRecordOption) *corev1.Pod { client.CreateServiceAccountOrFail(name) client.CreateRoleOrFail(resources.Role(name, resources.WithRuleForRole(&rbacv1.PolicyRule{ @@ -110,11 +108,11 @@ func DeployEventRecordOrFail(ctx context.Context, client *testlib.Client, name s eventRecordPod := eventRecordPod(name, name) client.CreatePodOrFail(eventRecordPod, append(options, testlib.WithService(name))...) - err := pkgtest.WaitForPodRunning(ctx, client.Kube, name, client.Namespace) + err := pkgtest.WaitForPodRunning(client.Kube, name, client.Namespace) if err != nil { client.T.Fatalf("Failed to start the recordevent pod '%s': %v", name, errors.WithStack(err)) } - client.WaitForServiceEndpointsOrFail(ctx, name, 1) + client.WaitForServiceEndpointsOrFail(name, 1) return eventRecordPod } @@ -132,7 +130,7 @@ func recordEventsPod(imageName string, name string, serviceAccountName string) * Spec: corev1.PodSpec{ Containers: []corev1.Container{{ Name: imageName, - Image: test.ImagePath(imageName), + Image: pkgtest.ImagePath(imageName), ImagePullPolicy: corev1.PullAlways, Env: []corev1.EnvVar{{ Name: "SYSTEM_NAMESPACE", diff --git a/test/test_images/recordevents/main.go b/test/test_images/recordevents/main.go index 38445c45bc8..f2427c40c7b 100644 --- a/test/test_images/recordevents/main.go +++ b/test/test_images/recordevents/main.go @@ -17,6 +17,7 @@ limitations under the License. package main import ( + "context" "log" "net/http" "os" @@ -38,8 +39,7 @@ func main() { if err != nil { log.Fatal("Error while reading the cfg", err) } - //nolint // nil ctx is fine here, look at the code of EnableInjectionOrDie - ctx, _ := injection.EnableInjectionOrDie(nil, cfg) + ctx, _ := injection.Default.SetupInformers(context.TODO(), cfg) if err := test_images.ConfigureTracing(logging.FromContext(ctx), ""); err != nil { logging.FromContext(ctx).Fatal("Unable to setup trace publishing", err) diff --git a/vendor/gopkg.in/yaml.v3/.travis.yml b/vendor/gopkg.in/yaml.v3/.travis.yml index a130fe883ce..1bc5c1cd20b 100644 --- a/vendor/gopkg.in/yaml.v3/.travis.yml +++ b/vendor/gopkg.in/yaml.v3/.travis.yml @@ -1,17 +1,15 @@ language: go go: - - "1.4.x" - - "1.5.x" - - "1.6.x" - - "1.7.x" - - "1.8.x" - - "1.9.x" - - "1.10.x" - - "1.11.x" - - "1.12.x" - - "1.13.x" - - "1.14.x" - - "tip" + - "1.4" + - "1.5" + - "1.6" + - "1.7" + - "1.8" + - "1.9" + - "1.10" + - "1.11" + - "1.12" + - tip go_import_path: gopkg.in/yaml.v3 diff --git a/vendor/gopkg.in/yaml.v3/apic.go b/vendor/gopkg.in/yaml.v3/apic.go index ae7d049f182..65846e67497 100644 --- a/vendor/gopkg.in/yaml.v3/apic.go +++ b/vendor/gopkg.in/yaml.v3/apic.go @@ -108,7 +108,6 @@ func yaml_emitter_initialize(emitter *yaml_emitter_t) { raw_buffer: make([]byte, 0, output_raw_buffer_size), states: make([]yaml_emitter_state_t, 0, initial_stack_size), events: make([]yaml_event_t, 0, initial_queue_size), - best_width: -1, } } diff --git a/vendor/gopkg.in/yaml.v3/decode.go b/vendor/gopkg.in/yaml.v3/decode.go index 21c0dacfdff..be63169b719 100644 --- a/vendor/gopkg.in/yaml.v3/decode.go +++ b/vendor/gopkg.in/yaml.v3/decode.go @@ -35,7 +35,6 @@ type parser struct { doc *Node anchors map[string]*Node doneInit bool - textless bool } func newParser(b []byte) *parser { @@ -109,18 +108,14 @@ func (p *parser) peek() yaml_event_type_t { func (p *parser) fail() { var where string var line int - if p.parser.context_mark.line != 0 { - line = p.parser.context_mark.line - // Scanner errors don't iterate line before returning error - if p.parser.error == yaml_SCANNER_ERROR { - line++ - } - } else if p.parser.problem_mark.line != 0 { + if p.parser.problem_mark.line != 0 { line = p.parser.problem_mark.line // Scanner errors don't iterate line before returning error if p.parser.error == yaml_SCANNER_ERROR { line++ } + } else if p.parser.context_mark.line != 0 { + line = p.parser.context_mark.line } if line != 0 { where = "line " + strconv.Itoa(line) + ": " @@ -174,20 +169,17 @@ func (p *parser) node(kind Kind, defaultTag, tag, value string) *Node { } else if kind == ScalarNode { tag, _ = resolve("", value) } - n := &Node{ - Kind: kind, - Tag: tag, - Value: value, - Style: style, - } - if !p.textless { - n.Line = p.event.start_mark.line + 1 - n.Column = p.event.start_mark.column + 1 - n.HeadComment = string(p.event.head_comment) - n.LineComment = string(p.event.line_comment) - n.FootComment = string(p.event.foot_comment) + return &Node{ + Kind: kind, + Tag: tag, + Value: value, + Style: style, + Line: p.event.start_mark.line + 1, + Column: p.event.start_mark.column + 1, + HeadComment: string(p.event.head_comment), + LineComment: string(p.event.line_comment), + FootComment: string(p.event.foot_comment), } - return n } func (p *parser) parseChild(parent *Node) *Node { @@ -399,7 +391,7 @@ func (d *decoder) callObsoleteUnmarshaler(n *Node, u obsoleteUnmarshaler) (good // // If n holds a null value, prepare returns before doing anything. func (d *decoder) prepare(n *Node, out reflect.Value) (newout reflect.Value, unmarshaled, good bool) { - if n.ShortTag() == nullTag || n.Kind == 0 && n.IsZero() { + if n.ShortTag() == nullTag { return out, false, false } again := true @@ -505,13 +497,8 @@ func (d *decoder) unmarshal(n *Node, out reflect.Value) (good bool) { good = d.mapping(n, out) case SequenceNode: good = d.sequence(n, out) - case 0: - if n.IsZero() { - return d.null(out) - } - fallthrough default: - failf("cannot decode node with unknown kind %d", n.Kind) + panic("internal error: unknown node kind: " + strconv.Itoa(int(n.Kind))) } return good } @@ -546,17 +533,6 @@ func resetMap(out reflect.Value) { } } -func (d *decoder) null(out reflect.Value) bool { - if out.CanAddr() { - switch out.Kind() { - case reflect.Interface, reflect.Ptr, reflect.Map, reflect.Slice: - out.Set(reflect.Zero(out.Type())) - return true - } - } - return false -} - func (d *decoder) scalar(n *Node, out reflect.Value) bool { var tag string var resolved interface{} @@ -574,7 +550,14 @@ func (d *decoder) scalar(n *Node, out reflect.Value) bool { } } if resolved == nil { - return d.null(out) + if out.CanAddr() { + switch out.Kind() { + case reflect.Interface, reflect.Ptr, reflect.Map, reflect.Slice: + out.Set(reflect.Zero(out.Type())) + return true + } + } + return false } if resolvedv := reflect.ValueOf(resolved); out.Type() == resolvedv.Type() { // We've resolved to exactly the type we want, so use that. diff --git a/vendor/gopkg.in/yaml.v3/emitterc.go b/vendor/gopkg.in/yaml.v3/emitterc.go index c29217ef54b..ab2a066194c 100644 --- a/vendor/gopkg.in/yaml.v3/emitterc.go +++ b/vendor/gopkg.in/yaml.v3/emitterc.go @@ -235,13 +235,10 @@ func yaml_emitter_increase_indent(emitter *yaml_emitter_t, flow, indentless bool emitter.indent = 0 } } else if !indentless { - // [Go] This was changed so that indentations are more regular. - if emitter.states[len(emitter.states)-1] == yaml_EMIT_BLOCK_SEQUENCE_ITEM_STATE { - // The first indent inside a sequence will just skip the "- " indicator. - emitter.indent += 2 - } else { - // Everything else aligns to the chosen indentation. - emitter.indent = emitter.best_indent*((emitter.indent+emitter.best_indent)/emitter.best_indent) + emitter.indent += emitter.best_indent + // [Go] If inside a block sequence item, discount the space taken by the indicator. + if emitter.best_indent > 2 && emitter.states[len(emitter.states)-1] == yaml_EMIT_BLOCK_SEQUENCE_ITEM_STATE { + emitter.indent -= 2 } } return true @@ -728,9 +725,16 @@ func yaml_emitter_emit_flow_mapping_value(emitter *yaml_emitter_t, event *yaml_e // Expect a block item node. func yaml_emitter_emit_block_sequence_item(emitter *yaml_emitter_t, event *yaml_event_t, first bool) bool { if first { - if !yaml_emitter_increase_indent(emitter, false, false) { + // [Go] The original logic here would not indent the sequence when inside a mapping. + // In Go we always indent it, but take the sequence indicator out of the indentation. + indentless := emitter.best_indent == 2 && emitter.mapping_context && (emitter.column == 0 || !emitter.indention) + original := emitter.indent + if !yaml_emitter_increase_indent(emitter, false, indentless) { return false } + if emitter.indent > original+2 { + emitter.indent -= 2 + } } if event.typ == yaml_SEQUENCE_END_EVENT { emitter.indent = emitter.indents[len(emitter.indents)-1] @@ -781,13 +785,6 @@ func yaml_emitter_emit_block_mapping_key(emitter *yaml_emitter_t, event *yaml_ev if !yaml_emitter_write_indent(emitter) { return false } - if len(emitter.line_comment) > 0 { - // [Go] A line comment was provided for the key. That's unusual as the - // scanner associates line comments with the value. Either way, - // save the line comment and render it appropriately later. - emitter.key_line_comment = emitter.line_comment - emitter.line_comment = nil - } if yaml_emitter_check_simple_key(emitter) { emitter.states = append(emitter.states, yaml_EMIT_BLOCK_MAPPING_SIMPLE_VALUE_STATE) return yaml_emitter_emit_node(emitter, event, false, false, true, true) @@ -813,29 +810,6 @@ func yaml_emitter_emit_block_mapping_value(emitter *yaml_emitter_t, event *yaml_ return false } } - if len(emitter.key_line_comment) > 0 { - // [Go] A line comment was previously provided for the key. Handle it before - // the value so the inline comments are placed correctly. - if yaml_emitter_silent_nil_event(emitter, event) && len(emitter.line_comment) == 0 { - // Nothing other than the line comment will be written on the line. - emitter.line_comment = emitter.key_line_comment - emitter.key_line_comment = nil - } else { - // An actual value is coming, so emit the comment line. - emitter.line_comment, emitter.key_line_comment = emitter.key_line_comment, emitter.line_comment - if !yaml_emitter_process_line_comment(emitter) { - return false - } - emitter.line_comment, emitter.key_line_comment = emitter.key_line_comment, emitter.line_comment - // Indent in unless it's a block that will reindent anyway. - if event.sequence_style() == yaml_FLOW_SEQUENCE_STYLE || (event.typ != yaml_MAPPING_START_EVENT && event.typ != yaml_SEQUENCE_START_EVENT) { - emitter.indent = emitter.best_indent*((emitter.indent+emitter.best_indent)/emitter.best_indent) - if !yaml_emitter_write_indent(emitter) { - return false - } - } - } - } emitter.states = append(emitter.states, yaml_EMIT_BLOCK_MAPPING_KEY_STATE) if !yaml_emitter_emit_node(emitter, event, false, false, true, false) { return false @@ -849,10 +823,6 @@ func yaml_emitter_emit_block_mapping_value(emitter *yaml_emitter_t, event *yaml_ return true } -func yaml_emitter_silent_nil_event(emitter *yaml_emitter_t, event *yaml_event_t) bool { - return event.typ == yaml_SCALAR_EVENT && event.implicit && !emitter.canonical && len(emitter.scalar_data.value) == 0 -} - // Expect a node. func yaml_emitter_emit_node(emitter *yaml_emitter_t, event *yaml_event_t, root bool, sequence bool, mapping bool, simple_key bool) bool { diff --git a/vendor/gopkg.in/yaml.v3/encode.go b/vendor/gopkg.in/yaml.v3/encode.go index 45e8d1e1b9f..eee3667eabc 100644 --- a/vendor/gopkg.in/yaml.v3/encode.go +++ b/vendor/gopkg.in/yaml.v3/encode.go @@ -119,9 +119,6 @@ func (e *encoder) marshal(tag string, in reflect.Value) { case *Node: e.nodev(in) return - case Node: - e.nodev(in.Addr()) - return case time.Time: e.timev(tag, in) return @@ -301,21 +298,6 @@ func isBase60Float(s string) (result bool) { // is bogus. In practice parsers do not enforce the "\.[0-9_]*" suffix. var base60float = regexp.MustCompile(`^[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+(?:\.[0-9_]*)?$`) -// isOldBool returns whether s is bool notation as defined in YAML 1.1. -// -// We continue to force strings that YAML 1.1 would interpret as booleans to be -// rendered as quotes strings so that the marshalled output valid for YAML 1.1 -// parsing. -func isOldBool(s string) (result bool) { - switch s { - case "y", "Y", "yes", "Yes", "YES", "on", "On", "ON", - "n", "N", "no", "No", "NO", "off", "Off", "OFF": - return true - default: - return false - } -} - func (e *encoder) stringv(tag string, in reflect.Value) { var style yaml_scalar_style_t s := in.String() @@ -337,7 +319,7 @@ func (e *encoder) stringv(tag string, in reflect.Value) { // tag when encoded unquoted. If it doesn't, // there's no need to quote it. rtag, _ := resolve("", s) - canUsePlain = rtag == strTag && !(isBase60Float(s) || isOldBool(s)) + canUsePlain = rtag == strTag && !isBase60Float(s) } // Note: it's possible for user code to emit invalid YAML // if they explicitly specify a tag and a string containing @@ -425,23 +407,18 @@ func (e *encoder) nodev(in reflect.Value) { } func (e *encoder) node(node *Node, tail string) { - // Zero nodes behave as nil. - if node.Kind == 0 && node.IsZero() { - e.nilv() - return - } - // If the tag was not explicitly requested, and dropping it won't change the // implicit tag of the value, don't include it in the presentation. var tag = node.Tag var stag = shortTag(tag) + var rtag string var forceQuoting bool if tag != "" && node.Style&TaggedStyle == 0 { if node.Kind == ScalarNode { if stag == strTag && node.Style&(SingleQuotedStyle|DoubleQuotedStyle|LiteralStyle|FoldedStyle) != 0 { tag = "" } else { - rtag, _ := resolve("", node.Value) + rtag, _ = resolve("", node.Value) if rtag == stag { tag = "" } else if stag == strTag { @@ -450,7 +427,6 @@ func (e *encoder) node(node *Node, tail string) { } } } else { - var rtag string switch node.Kind { case MappingNode: rtag = mapTag @@ -480,7 +456,7 @@ func (e *encoder) node(node *Node, tail string) { if node.Style&FlowStyle != 0 { style = yaml_FLOW_SEQUENCE_STYLE } - e.must(yaml_sequence_start_event_initialize(&e.event, []byte(node.Anchor), []byte(longTag(tag)), tag == "", style)) + e.must(yaml_sequence_start_event_initialize(&e.event, []byte(node.Anchor), []byte(tag), tag == "", style)) e.event.head_comment = []byte(node.HeadComment) e.emit() for _, node := range node.Content { @@ -496,7 +472,7 @@ func (e *encoder) node(node *Node, tail string) { if node.Style&FlowStyle != 0 { style = yaml_FLOW_MAPPING_STYLE } - yaml_mapping_start_event_initialize(&e.event, []byte(node.Anchor), []byte(longTag(tag)), tag == "", style) + yaml_mapping_start_event_initialize(&e.event, []byte(node.Anchor), []byte(tag), tag == "", style) e.event.tail_comment = []byte(tail) e.event.head_comment = []byte(node.HeadComment) e.emit() @@ -537,11 +513,11 @@ func (e *encoder) node(node *Node, tail string) { case ScalarNode: value := node.Value if !utf8.ValidString(value) { - if stag == binaryTag { + if tag == binaryTag { failf("explicitly tagged !!binary data must be base64-encoded") } - if stag != "" { - failf("cannot marshal invalid UTF-8 data as %s", stag) + if tag != "" { + failf("cannot marshal invalid UTF-8 data as %s", shortTag(tag)) } // It can't be encoded directly as YAML so use a binary tag // and encode it as base64. @@ -566,7 +542,5 @@ func (e *encoder) node(node *Node, tail string) { } e.emitScalar(value, node.Anchor, tag, style, []byte(node.HeadComment), []byte(node.LineComment), []byte(node.FootComment), []byte(tail)) - default: - failf("cannot encode node with unknown kind %d", node.Kind) } } diff --git a/vendor/gopkg.in/yaml.v3/parserc.go b/vendor/gopkg.in/yaml.v3/parserc.go index ac66fccc059..ec25faabc4e 100644 --- a/vendor/gopkg.in/yaml.v3/parserc.go +++ b/vendor/gopkg.in/yaml.v3/parserc.go @@ -423,7 +423,6 @@ func yaml_parser_set_event_comments(parser *yaml_parser_t, event *yaml_event_t) parser.line_comment = nil parser.foot_comment = nil parser.tail_comment = nil - parser.stem_comment = nil } // Parse the productions: @@ -630,10 +629,6 @@ func yaml_parser_parse_node(parser *yaml_parser_t, event *yaml_event_t, block, i implicit: implicit, style: yaml_style_t(yaml_BLOCK_SEQUENCE_STYLE), } - if parser.stem_comment != nil { - event.head_comment = parser.stem_comment - parser.stem_comment = nil - } return true } if block && token.typ == yaml_BLOCK_MAPPING_START_TOKEN { @@ -648,10 +643,6 @@ func yaml_parser_parse_node(parser *yaml_parser_t, event *yaml_event_t, block, i implicit: implicit, style: yaml_style_t(yaml_BLOCK_MAPPING_STYLE), } - if parser.stem_comment != nil { - event.head_comment = parser.stem_comment - parser.stem_comment = nil - } return true } if len(anchor) > 0 || len(tag) > 0 { @@ -698,9 +689,7 @@ func yaml_parser_parse_block_sequence_entry(parser *yaml_parser_t, event *yaml_e if token.typ == yaml_BLOCK_ENTRY_TOKEN { mark := token.end_mark - prior_head_len := len(parser.head_comment) skip_token(parser) - yaml_parser_split_stem_comment(parser, prior_head_len) token = peek_token(parser) if token == nil { return false @@ -746,9 +735,7 @@ func yaml_parser_parse_indentless_sequence_entry(parser *yaml_parser_t, event *y if token.typ == yaml_BLOCK_ENTRY_TOKEN { mark := token.end_mark - prior_head_len := len(parser.head_comment) skip_token(parser) - yaml_parser_split_stem_comment(parser, prior_head_len) token = peek_token(parser) if token == nil { return false @@ -774,32 +761,6 @@ func yaml_parser_parse_indentless_sequence_entry(parser *yaml_parser_t, event *y return true } -// Split stem comment from head comment. -// -// When a sequence or map is found under a sequence entry, the former head comment -// is assigned to the underlying sequence or map as a whole, not the individual -// sequence or map entry as would be expected otherwise. To handle this case the -// previous head comment is moved aside as the stem comment. -func yaml_parser_split_stem_comment(parser *yaml_parser_t, stem_len int) { - if stem_len == 0 { - return - } - - token := peek_token(parser) - if token.typ != yaml_BLOCK_SEQUENCE_START_TOKEN && token.typ != yaml_BLOCK_MAPPING_START_TOKEN { - return - } - - parser.stem_comment = parser.head_comment[:stem_len] - if len(parser.head_comment) == stem_len { - parser.head_comment = nil - } else { - // Copy suffix to prevent very strange bugs if someone ever appends - // further bytes to the prefix in the stem_comment slice above. - parser.head_comment = append([]byte(nil), parser.head_comment[stem_len+1:]...) - } -} - // Parse the productions: // block_mapping ::= BLOCK-MAPPING_START // ******************* diff --git a/vendor/gopkg.in/yaml.v3/scannerc.go b/vendor/gopkg.in/yaml.v3/scannerc.go index d9a539c39ae..e33f4959065 100644 --- a/vendor/gopkg.in/yaml.v3/scannerc.go +++ b/vendor/gopkg.in/yaml.v3/scannerc.go @@ -657,22 +657,34 @@ func trace(args ...interface{}) func() { func yaml_parser_fetch_more_tokens(parser *yaml_parser_t) bool { // While we need more tokens to fetch, do it. for { - // [Go] The comment parsing logic requires a lookahead of two tokens - // so that foot comments may be parsed in time of associating them - // with the tokens that are parsed before them, and also for line - // comments to be transformed into head comments in some edge cases. - if parser.tokens_head < len(parser.tokens)-2 { - // If a potential simple key is at the head position, we need to fetch - // the next token to disambiguate it. - head_tok_idx, ok := parser.simple_keys_by_tok[parser.tokens_parsed] - if !ok { - break - } else if valid, ok := yaml_simple_key_is_valid(parser, &parser.simple_keys[head_tok_idx]); !ok { + // Check if we really need to fetch more tokens. + need_more_tokens := false + + // [Go] The comment parsing logic requires a lookahead of one token + // in block style or two tokens in flow style so that the foot + // comments may be parsed in time of associating them with the tokens + // that are parsed before them. + if parser.tokens_head >= len(parser.tokens)-1 || parser.flow_level > 0 && parser.tokens_head >= len(parser.tokens)-2 { + need_more_tokens = true + } else { + // Check if any potential simple key may occupy the head position. + if !yaml_parser_stale_simple_keys(parser) { return false - } else if !valid { - break + } + + for i := range parser.simple_keys { + simple_key := &parser.simple_keys[i] + if simple_key.possible && simple_key.token_number == parser.tokens_parsed { + need_more_tokens = true + break + } } } + + // We are finished. + if !need_more_tokens { + break + } // Fetch the next token. if !yaml_parser_fetch_next_token(parser) { return false @@ -702,6 +714,11 @@ func yaml_parser_fetch_next_token(parser *yaml_parser_t) (ok bool) { return false } + // Remove obsolete potential simple keys. + if !yaml_parser_stale_simple_keys(parser) { + return false + } + // [Go] While unrolling indents, transform the head comments of prior // indentation levels observed after scan_start into foot comments at // the respective indexes. @@ -749,11 +766,6 @@ func yaml_parser_fetch_next_token(parser *yaml_parser_t) (ok bool) { if !ok { return } - if len(parser.tokens) > 0 && parser.tokens[len(parser.tokens)-1].typ == yaml_BLOCK_ENTRY_TOKEN { - // Sequence indicators alone have no line comments. It becomes - // a head comment for whatever follows. - return - } if !yaml_parser_scan_line_comment(parser, comment_mark) { ok = false return @@ -880,30 +892,29 @@ func yaml_parser_fetch_next_token(parser *yaml_parser_t) (ok bool) { "found character that cannot start any token") } -func yaml_simple_key_is_valid(parser *yaml_parser_t, simple_key *yaml_simple_key_t) (valid, ok bool) { - if !simple_key.possible { - return false, true - } +// Check the list of potential simple keys and remove the positions that +// cannot contain simple keys anymore. +func yaml_parser_stale_simple_keys(parser *yaml_parser_t) bool { + // Check for a potential simple key for each flow level. + for i := range parser.simple_keys { + simple_key := &parser.simple_keys[i] - // The 1.2 specification says: - // - // "If the ? indicator is omitted, parsing needs to see past the - // implicit key to recognize it as such. To limit the amount of - // lookahead required, the “:” indicator must appear at most 1024 - // Unicode characters beyond the start of the key. In addition, the key - // is restricted to a single line." - // - if simple_key.mark.line < parser.mark.line || simple_key.mark.index+1024 < parser.mark.index { - // Check if the potential simple key to be removed is required. - if simple_key.required { - return false, yaml_parser_set_scanner_error(parser, - "while scanning a simple key", simple_key.mark, - "could not find expected ':'") + // The specification requires that a simple key + // + // - is limited to a single line, + // - is shorter than 1024 characters. + if simple_key.possible && (simple_key.mark.line < parser.mark.line || simple_key.mark.index+1024 < parser.mark.index) { + + // Check if the potential simple key to be removed is required. + if simple_key.required { + return yaml_parser_set_scanner_error(parser, + "while scanning a simple key", simple_key.mark, + "could not find expected ':'") + } + simple_key.possible = false } - simple_key.possible = false - return false, true } - return true, true + return true } // Check if a simple key may start at the current position and add it if @@ -923,14 +934,13 @@ func yaml_parser_save_simple_key(parser *yaml_parser_t) bool { possible: true, required: required, token_number: parser.tokens_parsed + (len(parser.tokens) - parser.tokens_head), - mark: parser.mark, } + simple_key.mark = parser.mark if !yaml_parser_remove_simple_key(parser) { return false } parser.simple_keys[len(parser.simple_keys)-1] = simple_key - parser.simple_keys_by_tok[simple_key.token_number] = len(parser.simple_keys) - 1 } return true } @@ -945,10 +955,9 @@ func yaml_parser_remove_simple_key(parser *yaml_parser_t) bool { "while scanning a simple key", parser.simple_keys[i].mark, "could not find expected ':'") } - // Remove the key from the stack. - parser.simple_keys[i].possible = false - delete(parser.simple_keys_by_tok, parser.simple_keys[i].token_number) } + // Remove the key from the stack. + parser.simple_keys[i].possible = false return true } @@ -958,12 +967,7 @@ const max_flow_level = 10000 // Increase the flow level and resize the simple key list if needed. func yaml_parser_increase_flow_level(parser *yaml_parser_t) bool { // Reset the simple key on the next level. - parser.simple_keys = append(parser.simple_keys, yaml_simple_key_t{ - possible: false, - required: false, - token_number: parser.tokens_parsed + (len(parser.tokens) - parser.tokens_head), - mark: parser.mark, - }) + parser.simple_keys = append(parser.simple_keys, yaml_simple_key_t{}) // Increase the flow level. parser.flow_level++ @@ -979,9 +983,7 @@ func yaml_parser_increase_flow_level(parser *yaml_parser_t) bool { func yaml_parser_decrease_flow_level(parser *yaml_parser_t) bool { if parser.flow_level > 0 { parser.flow_level-- - last := len(parser.simple_keys) - 1 - delete(parser.simple_keys_by_tok, parser.simple_keys[last].token_number) - parser.simple_keys = parser.simple_keys[:last] + parser.simple_keys = parser.simple_keys[:len(parser.simple_keys)-1] } return true } @@ -1088,8 +1090,6 @@ func yaml_parser_fetch_stream_start(parser *yaml_parser_t) bool { // Initialize the simple key stack. parser.simple_keys = append(parser.simple_keys, yaml_simple_key_t{}) - parser.simple_keys_by_tok = make(map[int]int) - // A simple key is allowed at the beginning of the stream. parser.simple_key_allowed = true @@ -1372,11 +1372,7 @@ func yaml_parser_fetch_value(parser *yaml_parser_t) bool { simple_key := &parser.simple_keys[len(parser.simple_keys)-1] // Have we found a simple key? - if valid, ok := yaml_simple_key_is_valid(parser, simple_key); !ok { - return false - - } else if valid { - + if simple_key.possible { // Create the KEY token and insert it into the queue. token := yaml_token_t{ typ: yaml_KEY_TOKEN, @@ -1394,7 +1390,6 @@ func yaml_parser_fetch_value(parser *yaml_parser_t) bool { // Remove the simple key. simple_key.possible = false - delete(parser.simple_keys_by_tok, simple_key.token_number) // A simple key cannot follow another simple key. parser.simple_key_allowed = false @@ -1562,29 +1557,6 @@ func yaml_parser_scan_to_next_token(parser *yaml_parser_t) bool { } } - // Check if we just had a line comment under a sequence entry that - // looks more like a header to the following content. Similar to this: - // - // - # The comment - // - Some data - // - // If so, transform the line comment to a head comment and reposition. - if len(parser.comments) > 0 && len(parser.tokens) > 1 { - tokenA := parser.tokens[len(parser.tokens)-2] - tokenB := parser.tokens[len(parser.tokens)-1] - comment := &parser.comments[len(parser.comments)-1] - if tokenA.typ == yaml_BLOCK_SEQUENCE_START_TOKEN && tokenB.typ == yaml_BLOCK_ENTRY_TOKEN && len(comment.line) > 0 && !is_break(parser.buffer, parser.buffer_pos) { - // If it was in the prior line, reposition so it becomes a - // header of the follow up token. Otherwise, keep it in place - // so it becomes a header of the former. - comment.head = comment.line - comment.line = nil - if comment.start_mark.line == parser.mark.line-1 { - comment.token_mark = parser.mark - } - } - } - // Eat a comment until a line break. if parser.buffer[parser.buffer_pos] == '#' { if !yaml_parser_scan_comments(parser, scan_mark) { @@ -2260,15 +2232,8 @@ func yaml_parser_scan_block_scalar(parser *yaml_parser_t, token *yaml_token_t, l } } if parser.buffer[parser.buffer_pos] == '#' { - // TODO Test this and then re-enable it. - //if !yaml_parser_scan_line_comment(parser, start_mark) { - // return false - //} - for !is_breakz(parser.buffer, parser.buffer_pos) { - skip(parser) - if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { - return false - } + if !yaml_parser_scan_line_comment(parser, start_mark) { + return false } } @@ -2837,8 +2802,8 @@ func yaml_parser_scan_line_comment(parser *yaml_parser_t, token_mark yaml_mark_t return true } - var start_mark yaml_mark_t - var text []byte + parser.comments = append(parser.comments, yaml_comment_t{token_mark: token_mark}) + comment := &parser.comments[len(parser.comments)-1].line for peek := 0; peek < 512; peek++ { if parser.unread < peek+1 && !yaml_parser_update_buffer(parser, peek+1) { @@ -2848,6 +2813,11 @@ func yaml_parser_scan_line_comment(parser *yaml_parser_t, token_mark yaml_mark_t continue } if parser.buffer[parser.buffer_pos+peek] == '#' { + if len(*comment) > 0 { + *comment = append(*comment, '\n') + } + + // Consume until after the consumed comment line. seen := parser.mark.index+peek for { if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { @@ -2861,25 +2831,16 @@ func yaml_parser_scan_line_comment(parser *yaml_parser_t, token_mark yaml_mark_t return false } skip_line(parser) - } else if parser.mark.index >= seen { - if len(text) == 0 { - start_mark = parser.mark - } - text = read(parser, text) } else { + if parser.mark.index >= seen { + *comment = append(*comment, parser.buffer[parser.buffer_pos]) + } skip(parser) } } } break } - if len(text) > 0 { - parser.comments = append(parser.comments, yaml_comment_t{ - token_mark: token_mark, - start_mark: start_mark, - line: text, - }) - } return true } @@ -3003,9 +2964,10 @@ func yaml_parser_scan_comments(parser *yaml_parser_t, scan_mark yaml_mark_t) boo return false } skip_line(parser) - } else if parser.mark.index >= seen { - text = read(parser, text) } else { + if parser.mark.index >= seen { + text = append(text, parser.buffer[parser.buffer_pos]) + } skip(parser) } } diff --git a/vendor/gopkg.in/yaml.v3/yaml.go b/vendor/gopkg.in/yaml.v3/yaml.go index 56e8a849031..b5d35a50ded 100644 --- a/vendor/gopkg.in/yaml.v3/yaml.go +++ b/vendor/gopkg.in/yaml.v3/yaml.go @@ -89,7 +89,7 @@ func Unmarshal(in []byte, out interface{}) (err error) { return unmarshal(in, out, false) } -// A Decoder reads and decodes YAML values from an input stream. +// A Decorder reads and decodes YAML values from an input stream. type Decoder struct { parser *parser knownFields bool @@ -194,7 +194,7 @@ func unmarshal(in []byte, out interface{}, strict bool) (err error) { // Zero valued structs will be omitted if all their public // fields are zero, unless they implement an IsZero // method (see the IsZeroer interface type), in which -// case the field will be excluded if IsZero returns true. +// case the field will be included if that method returns true. // // flow Marshal using a flow style (useful for structs, // sequences and maps). @@ -252,24 +252,6 @@ func (e *Encoder) Encode(v interface{}) (err error) { return nil } -// Encode encodes value v and stores its representation in n. -// -// See the documentation for Marshal for details about the -// conversion of Go values into YAML. -func (n *Node) Encode(v interface{}) (err error) { - defer handleErr(&err) - e := newEncoder() - defer e.destroy() - e.marshalDoc("", reflect.ValueOf(v)) - e.finish() - p := newParser(e.out) - p.textless = true - defer p.destroy() - doc := p.parse() - *n = *doc.Content[0] - return nil -} - // SetIndent changes the used indentation used when encoding. func (e *Encoder) SetIndent(spaces int) { if spaces < 0 { @@ -346,12 +328,6 @@ const ( // and maps, Node is an intermediate representation that allows detailed // control over the content being decoded or encoded. // -// It's worth noting that although Node offers access into details such as -// line numbers, colums, and comments, the content when re-encoded will not -// have its original textual representation preserved. An effort is made to -// render the data plesantly, and to preserve comments near the data they -// describe, though. -// // Values that make use of the Node type interact with the yaml package in the // same way any other type would do, by encoding and decoding yaml data // directly or indirectly into them. @@ -415,13 +391,6 @@ type Node struct { Column int } -// IsZero returns whether the node has all of its fields unset. -func (n *Node) IsZero() bool { - return n.Kind == 0 && n.Style == 0 && n.Tag == "" && n.Value == "" && n.Anchor == "" && n.Alias == nil && n.Content == nil && - n.HeadComment == "" && n.LineComment == "" && n.FootComment == "" && n.Line == 0 && n.Column == 0 -} - - // LongTag returns the long form of the tag that indicates the data type for // the node. If the Tag field isn't explicitly defined, one will be computed // based on the node properties. diff --git a/vendor/gopkg.in/yaml.v3/yamlh.go b/vendor/gopkg.in/yaml.v3/yamlh.go index 7c6d0077061..65fb0df3b17 100644 --- a/vendor/gopkg.in/yaml.v3/yamlh.go +++ b/vendor/gopkg.in/yaml.v3/yamlh.go @@ -600,7 +600,6 @@ type yaml_parser_t struct { line_comment []byte // The current line comments foot_comment []byte // The current foot comments tail_comment []byte // Foot comment that happens at the end of a block. - stem_comment []byte // Comment in item preceding a nested structure (list inside list item, etc) comments []yaml_comment_t // The folded comments for all parsed tokens comments_head int @@ -622,7 +621,6 @@ type yaml_parser_t struct { simple_key_allowed bool // May a simple key occur at the current position? simple_keys []yaml_simple_key_t // The stack of simple keys. - simple_keys_by_tok map[int]int // possible simple_key indexes indexed by token_number // Parser stuff @@ -787,8 +785,6 @@ type yaml_emitter_t struct { foot_comment []byte tail_comment []byte - key_line_comment []byte - // Dumper stuff opened bool // If the stream was already opened? diff --git a/vendor/modules.txt b/vendor/modules.txt index 2423edd4666..dcfacff1f58 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -256,6 +256,7 @@ github.com/prometheus/client_golang/prometheus/promhttp # github.com/prometheus/client_model v0.2.0 github.com/prometheus/client_model/go # github.com/prometheus/common v0.9.1 +## explicit github.com/prometheus/common/expfmt github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg github.com/prometheus/common/log @@ -599,7 +600,7 @@ gopkg.in/inf.v0 # gopkg.in/yaml.v2 v2.3.0 ## explicit gopkg.in/yaml.v2 -# gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 +# gopkg.in/yaml.v3 v3.0.0-20191026110619-0b21df46bc1d ## explicit gopkg.in/yaml.v3 # honnef.co/go/tools v0.0.1-2020.1.4 From ce07370f2e2d4f07b44a2bde8072b9f744dad854 Mon Sep 17 00:00:00 2001 From: slinkydeveloper Date: Tue, 20 Oct 2020 14:33:59 +0200 Subject: [PATCH 06/10] Remove sequence stepper usages (#4344) Signed-off-by: Francesco Guardiani (cherry picked from commit 515f4169a864e664f2bb91998cd201b10f3a10d0) Signed-off-by: Francesco Guardiani --- test/e2e/helpers/parallel_test_helper.go | 16 +++++++++++---- test/lib/resources/kube.go | 25 ------------------------ 2 files changed, 12 insertions(+), 29 deletions(-) diff --git a/test/e2e/helpers/parallel_test_helper.go b/test/e2e/helpers/parallel_test_helper.go index daf2edfda68..14e69c56349 100644 --- a/test/e2e/helpers/parallel_test_helper.go +++ b/test/e2e/helpers/parallel_test_helper.go @@ -76,8 +76,12 @@ func ParallelTestHelper(t *testing.T, // construct branch subscriber subPodName := fmt.Sprintf("parallel-%s-branch-%d-sub", tc.name, branchNumber) - subPod := resources.SequenceStepperPod(subPodName, subPodName) - client.CreatePodOrFail(subPod, testlib.WithService(subPodName)) + recordevents.DeployEventRecordOrFail( + ctx, + client, + subPodName, + recordevents.ReplyWithAppendedData(subPodName), + ) parallelBranches[branchNumber] = v1beta1.ParallelBranch{ Filter: &duckv1.Destination{ @@ -182,8 +186,12 @@ func ParallelV1TestHelper(t *testing.T, // construct branch subscriber subPodName := fmt.Sprintf("parallel-%s-branch-%d-sub", tc.name, branchNumber) - subPod := resources.SequenceStepperPod(subPodName, subPodName) - client.CreatePodOrFail(subPod, testlib.WithService(subPodName)) + recordevents.DeployEventRecordOrFail( + ctx, + client, + subPodName, + recordevents.ReplyWithAppendedData(subPodName), + ) parallelBranches[branchNumber] = flowsv1.ParallelBranch{ Filter: &duckv1.Destination{ diff --git a/test/lib/resources/kube.go b/test/lib/resources/kube.go index b504f6eb91d..3389f1c82ca 100644 --- a/test/lib/resources/kube.go +++ b/test/lib/resources/kube.go @@ -66,31 +66,6 @@ func WithLabelsForPod(labels map[string]string) PodOption { } } -// SequenceStepperPod creates a Pod that can be used as a step in testing Sequence. -// Note event data used in the test must be BaseData, and this Pod as a Subscriber will receive the event, -// and return a new event with eventMsgAppender added to data.Message. -func SequenceStepperPod(name, eventMsgAppender string) *corev1.Pod { - const imageName = "sequencestepper" - return &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Labels: map[string]string{"e2etest": string(uuid.NewUUID())}, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{{ - Name: imageName, - Image: pkgTest.ImagePath(imageName), - ImagePullPolicy: corev1.PullAlways, - Args: []string{ - "-msg-appender", - eventMsgAppender, - }, - }}, - RestartPolicy: corev1.RestartPolicyAlways, - }, - } -} - // EventFilteringPod creates a Pod that either filter or send the received CloudEvent func EventFilteringPod(name string, filter bool) *corev1.Pod { const imageName = "filterevents" From a013d346077c8e92f23be9debdef851d313e322a Mon Sep 17 00:00:00 2001 From: slinkydeveloper Date: Tue, 20 Oct 2020 20:28:03 +0200 Subject: [PATCH 07/10] Remove filterevents usages (#4347) Signed-off-by: Francesco Guardiani (cherry picked from commit ca85d86bb9a4a845db48cceb9ab2e75e4c66356b) Signed-off-by: Francesco Guardiani # Conflicts: # test/lib/resources/kube.go --- test/e2e/helpers/parallel_test_helper.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/e2e/helpers/parallel_test_helper.go b/test/e2e/helpers/parallel_test_helper.go index 14e69c56349..aac6437df65 100644 --- a/test/e2e/helpers/parallel_test_helper.go +++ b/test/e2e/helpers/parallel_test_helper.go @@ -77,7 +77,6 @@ func ParallelTestHelper(t *testing.T, // construct branch subscriber subPodName := fmt.Sprintf("parallel-%s-branch-%d-sub", tc.name, branchNumber) recordevents.DeployEventRecordOrFail( - ctx, client, subPodName, recordevents.ReplyWithAppendedData(subPodName), @@ -187,7 +186,6 @@ func ParallelV1TestHelper(t *testing.T, // construct branch subscriber subPodName := fmt.Sprintf("parallel-%s-branch-%d-sub", tc.name, branchNumber) recordevents.DeployEventRecordOrFail( - ctx, client, subPodName, recordevents.ReplyWithAppendedData(subPodName), From 543812dc99a98873e7a2c1e365e7e9273c896b40 Mon Sep 17 00:00:00 2001 From: Francesco Guardiani Date: Tue, 20 Oct 2020 20:29:48 +0200 Subject: [PATCH 08/10] Update Signed-off-by: Francesco Guardiani --- test/e2e/helpers/parallel_test_helper.go | 14 ++++++++++---- test/lib/resources/kube.go | 24 ------------------------ 2 files changed, 10 insertions(+), 28 deletions(-) diff --git a/test/e2e/helpers/parallel_test_helper.go b/test/e2e/helpers/parallel_test_helper.go index aac6437df65..180cc907297 100644 --- a/test/e2e/helpers/parallel_test_helper.go +++ b/test/e2e/helpers/parallel_test_helper.go @@ -71,8 +71,11 @@ func ParallelTestHelper(t *testing.T, for branchNumber, cse := range tc.branchesConfig { // construct filter services filterPodName := fmt.Sprintf("parallel-%s-branch-%d-filter", tc.name, branchNumber) - filterPod := resources.EventFilteringPod(filterPodName, cse.filter) - client.CreatePodOrFail(filterPod, testlib.WithService(filterPodName)) + if cse.filter { + recordevents.DeployEventRecordOrFail(client, filterPodName) + } else { + recordevents.DeployEventRecordOrFail(client, filterPodName, recordevents.EchoEvent) + } // construct branch subscriber subPodName := fmt.Sprintf("parallel-%s-branch-%d-sub", tc.name, branchNumber) @@ -180,8 +183,11 @@ func ParallelV1TestHelper(t *testing.T, for branchNumber, cse := range tc.branchesConfig { // construct filter services filterPodName := fmt.Sprintf("parallel-%s-branch-%d-filter", tc.name, branchNumber) - filterPod := resources.EventFilteringPod(filterPodName, cse.filter) - client.CreatePodOrFail(filterPod, testlib.WithService(filterPodName)) + if cse.filter { + recordevents.DeployEventRecordOrFail(client, filterPodName) + } else { + recordevents.DeployEventRecordOrFail(client, filterPodName, recordevents.EchoEvent) + } // construct branch subscriber subPodName := fmt.Sprintf("parallel-%s-branch-%d-sub", tc.name, branchNumber) diff --git a/test/lib/resources/kube.go b/test/lib/resources/kube.go index 3389f1c82ca..d9dc37b90b8 100644 --- a/test/lib/resources/kube.go +++ b/test/lib/resources/kube.go @@ -27,7 +27,6 @@ import ( rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/apimachinery/pkg/util/uuid" pkgTest "knative.dev/pkg/test" ) @@ -66,29 +65,6 @@ func WithLabelsForPod(labels map[string]string) PodOption { } } -// EventFilteringPod creates a Pod that either filter or send the received CloudEvent -func EventFilteringPod(name string, filter bool) *corev1.Pod { - const imageName = "filterevents" - pod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Labels: map[string]string{"e2etest": string(uuid.NewUUID())}, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{{ - Name: imageName, - Image: pkgTest.ImagePath(imageName), - ImagePullPolicy: corev1.PullAlways, - }}, - RestartPolicy: corev1.RestartPolicyAlways, - }, - } - if filter { - pod.Spec.Containers[0].Args = []string{"-filter"} - } - return pod -} - const ( PerfConsumerService = "perf-consumer" PerfAggregatorService = "perf-aggregator" From a69f2435a3a60e1267886d0129d70ce744edd82e Mon Sep 17 00:00:00 2001 From: slinkydeveloper Date: Tue, 20 Oct 2020 21:16:53 +0200 Subject: [PATCH 09/10] Not sure why SYSTEM_NAMESPACE doesn't work in 0.17 Signed-off-by: Francesco Guardiani --- .../recordevents/recorder_vent/constructor.go | 21 +++++++++---------- test/lib/recordevents/resources.go | 12 +++++------ 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/test/lib/recordevents/recorder_vent/constructor.go b/test/lib/recordevents/recorder_vent/constructor.go index 011c075a9d3..0f1a32572fd 100644 --- a/test/lib/recordevents/recorder_vent/constructor.go +++ b/test/lib/recordevents/recorder_vent/constructor.go @@ -22,8 +22,6 @@ import ( "strings" "github.com/kelseyhightower/envconfig" - "knative.dev/pkg/system" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/watch" @@ -39,9 +37,10 @@ import ( ) type envConfig struct { - AgentName string `envconfig:"AGENT_NAME" default:"observer-default" required:"true"` - PodName string `envconfig:"POD_NAME" required:"true"` - Port int `envconfig:"PORT" default:"8080" required:"true"` + AgentName string `envconfig:"AGENT_NAME" default:"observer-default" required:"true"` + PodName string `envconfig:"POD_NAME" required:"true"` + PodNamespace string `envconfig:"POD_NAMESPACE" required:"true"` + Port int `envconfig:"PORT" default:"8080" required:"true"` } func NewFromEnv(ctx context.Context) recordevents.EventLog { @@ -52,21 +51,21 @@ func NewFromEnv(ctx context.Context) recordevents.EventLog { logging.FromContext(ctx).Infof("Recorder vent environment configuration: %+v", env) - return NewEventLog(ctx, env.AgentName, env.PodName) + return NewEventLog(ctx, env.AgentName, env.PodName, env.PodNamespace) } -func NewEventLog(ctx context.Context, agentName string, podName string) recordevents.EventLog { - on, err := kubeclient.Get(ctx).CoreV1().Pods(system.Namespace()).Get(podName, metav1.GetOptions{}) +func NewEventLog(ctx context.Context, agentName string, podName string, podNamespace string) recordevents.EventLog { + on, err := kubeclient.Get(ctx).CoreV1().Pods(podNamespace).Get(podName, metav1.GetOptions{}) if err != nil { logging.FromContext(ctx).Fatal("Error while trying to retrieve the pod", err) } logging.FromContext(ctx).Infof("Going to send events to pod '%s' in namespace '%s'", on.Name, on.Namespace) - return &recorder{out: createRecorder(ctx, agentName), on: on} + return &recorder{out: createRecorder(ctx, agentName, podNamespace), on: on} } -func createRecorder(ctx context.Context, agentName string) record.EventRecorder { +func createRecorder(ctx context.Context, agentName string, namespace string) record.EventRecorder { logger := logging.FromContext(ctx) recorder := controller.GetEventRecorder(ctx) @@ -91,7 +90,7 @@ func createRecorder(ctx context.Context, agentName string) record.EventRecorder watches := []watch.Interface{ eventBroadcaster.StartLogging(logger.Named("event-broadcaster").Infof), eventBroadcaster.StartRecordingToSink( - &v1.EventSinkImpl{Interface: kubeclient.Get(ctx).CoreV1().Events(system.Namespace())}, + &v1.EventSinkImpl{Interface: kubeclient.Get(ctx).CoreV1().Events(namespace)}, ), } recorder = eventBroadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: agentName}) diff --git a/test/lib/recordevents/resources.go b/test/lib/recordevents/resources.go index 21bfc885cc4..e3eb191ff4a 100644 --- a/test/lib/recordevents/resources.go +++ b/test/lib/recordevents/resources.go @@ -106,7 +106,7 @@ func DeployEventRecordOrFail(client *testlib.Client, name string, options ...Eve )) client.CreateRoleBindingOrFail(name, "Role", name, name, client.Namespace) - eventRecordPod := eventRecordPod(name, name) + eventRecordPod := recordEventsPod("recordevents", name, name, client.Namespace) client.CreatePodOrFail(eventRecordPod, append(options, testlib.WithService(name))...) err := pkgtest.WaitForPodRunning(client.Kube, name, client.Namespace) if err != nil { @@ -116,12 +116,7 @@ func DeployEventRecordOrFail(client *testlib.Client, name string, options ...Eve return eventRecordPod } -// eventRecordPod creates a Pod that stores received events for test retrieval. -func eventRecordPod(name string, serviceAccountName string) *corev1.Pod { - return recordEventsPod("recordevents", name, serviceAccountName) -} - -func recordEventsPod(imageName string, name string, serviceAccountName string) *corev1.Pod { +func recordEventsPod(imageName string, name string, serviceAccountName string, namespace string) *corev1.Pod { return &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -137,6 +132,9 @@ func recordEventsPod(imageName string, name string, serviceAccountName string) * ValueFrom: &corev1.EnvVarSource{ FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}, }, + }, { + Name: "POD_NAMESPACE", + Value: namespace, }, { Name: "OBSERVER", Value: "recorder-" + name, From a3c7fdca266e0b6263170470a4fbbe09bd500e42 Mon Sep 17 00:00:00 2001 From: slinkydeveloper Date: Tue, 20 Oct 2020 22:16:39 +0200 Subject: [PATCH 10/10] Did the same error 2 times Signed-off-by: Francesco Guardiani --- test/conformance/helpers/broker_data_plane_test_helper.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/conformance/helpers/broker_data_plane_test_helper.go b/test/conformance/helpers/broker_data_plane_test_helper.go index a3abff1e496..29db37c6119 100644 --- a/test/conformance/helpers/broker_data_plane_test_helper.go +++ b/test/conformance/helpers/broker_data_plane_test_helper.go @@ -316,7 +316,7 @@ func BrokerV1Beta1ConsumerDataPlaneTestHelper( source := "origin-for-reply" event.SetSource(source) msg := []byte(`{"msg":"Transformed!"}`) - transformPod := recordevents.DeployEventRecordOrFail( + recordevents.DeployEventRecordOrFail( client, "transformer-pod", recordevents.ReplyWithTransformedEvent( @@ -325,7 +325,6 @@ func BrokerV1Beta1ConsumerDataPlaneTestHelper( string(msg), ), ) - client.CreatePodOrFail(transformPod, testlib.WithService("transformer-pod")) client.WaitForServiceEndpointsOrFail("transformer-pod", 1) transformTrigger := client.CreateTriggerOrFailV1Beta1( "transform-trigger",