From 7f2175eaef95cc485e0edf81021329b9d1a9443b Mon Sep 17 00:00:00 2001 From: slinkydeveloper Date: Wed, 30 Sep 2020 15:36:58 +0200 Subject: [PATCH 01/23] WIP Replacing port-forwarding with k8s events in recordevents Signed-off-by: Francesco Guardiani --- test/lib/recordevents/event_info.go | 9 +- test/lib/recordevents/event_log.go | 22 ++ test/lib/recordevents/observer/observer.go | 85 +++++ .../recordevents/recorder_vent/constructor.go | 113 ++++++ test/lib/recordevents/recorder_vent/doc.go | 19 + .../recordevents/recorder_vent/recorder.go | 48 +++ test/lib/recordevents/writer_vent/doc.go | 19 + test/lib/recordevents/writer_vent/writer.go | 50 +++ test/test_images/recordevents/eventstore.go | 191 ---------- .../recordevents/eventstore_test.go | 339 ------------------ test/test_images/recordevents/main.go | 152 ++------ 11 files changed, 383 insertions(+), 664 deletions(-) create mode 100644 test/lib/recordevents/event_log.go create mode 100644 test/lib/recordevents/observer/observer.go create mode 100644 test/lib/recordevents/recorder_vent/constructor.go create mode 100644 test/lib/recordevents/recorder_vent/doc.go create mode 100644 test/lib/recordevents/recorder_vent/recorder.go create mode 100644 test/lib/recordevents/writer_vent/doc.go create mode 100644 test/lib/recordevents/writer_vent/writer.go delete mode 100644 test/test_images/recordevents/eventstore.go delete mode 100644 test/test_images/recordevents/eventstore_test.go diff --git a/test/lib/recordevents/event_info.go b/test/lib/recordevents/event_info.go index dc0ffd1f0b6..b52ef675718 100644 --- a/test/lib/recordevents/event_info.go +++ b/test/lib/recordevents/event_info.go @@ -62,11 +62,14 @@ type MinMaxResponse struct { type EventInfo struct { // Set if the http request received by the pod couldn't be decoded or // didn't pass validation - Error string + Error string `json:"error"` // Event received if the cloudevent received by the pod passed validation - Event *cloudevents.Event + Event *cloudevents.Event `json:"event"` // HTTPHeaders of the connection that delivered the event - HTTPHeaders map[string][]string + HTTPHeaders map[string][]string `json:"httpHeaders"` + Origin string `json:"origin"` + Observer string `json:"observer"` + Time time.Time `json:"time"` } // Pretty print the event. Meant for debugging. This formats the validation error diff --git a/test/lib/recordevents/event_log.go b/test/lib/recordevents/event_log.go new file mode 100644 index 00000000000..fcef14a9f12 --- /dev/null +++ b/test/lib/recordevents/event_log.go @@ -0,0 +1,22 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package recordevents + +// EventLog is the contract for an event logger to vent an event. +type EventLog interface { + Vent(observed EventInfo) error +} diff --git a/test/lib/recordevents/observer/observer.go b/test/lib/recordevents/observer/observer.go new file mode 100644 index 00000000000..a5012480c66 --- /dev/null +++ b/test/lib/recordevents/observer/observer.go @@ -0,0 +1,85 @@ +package observer + +import ( + "context" + "net/http" + "time" + + cloudeventsbindings "github.com/cloudevents/sdk-go/v2/binding" + cloudeventshttp "github.com/cloudevents/sdk-go/v2/protocol/http" + "github.com/kelseyhightower/envconfig" + "github.com/prometheus/common/log" + + "knative.dev/eventing/test/lib/recordevents" +) + +// Observer is the entry point for sinking events into the event log. +type Observer struct { + // Name is the name of this Observer, used to filter if multiple observers. + Name string + // EventLogs is the list of EventLog implementors to vent observed events. + EventLogs []recordevents.EventLog +} + +// New returns an observer that will vent observations to the list of provided +// EventLog instances. It will listen on :8080. +func New(name string, eventLogs ...recordevents.EventLog) *Observer { + return &Observer{ + Name: name, + EventLogs: eventLogs, + } +} + +type envConfig struct { + // ObserverName is used to identify this instance of the observer. + ObserverName string `envconfig:"OBSERVER_NAME" default:"observer-default" required:"true"` +} + +func NewFromEnv(eventLogs ...recordevents.EventLog) *Observer { + var env envConfig + if err := envconfig.Process("", &env); err != nil { + log.Fatalf("[ERROR] Failed to process env var: %s", err) + } + + return New(env.ObserverName, eventLogs...) +} + +// Start will create the CloudEvents client and start listening for inbound +// HTTP requests. This is a is a blocking call. +func (o *Observer) Start(ctx context.Context, handlerFuncs ...func(handler http.Handler) http.Handler) error { + var handler http.Handler = o + + for _, dec := range handlerFuncs { + handler = dec(handler) + } + + return http.ListenAndServe(":8080", handler) +} + +func (o *Observer) ServeHTTP(writer http.ResponseWriter, request *http.Request) { + m := cloudeventshttp.NewMessageFromHttpRequest(request) + defer m.Finish(nil) + + event, eventErr := cloudeventsbindings.ToEvent(context.TODO(), m) + header := request.Header + + for _, el := range o.EventLogs { + eventErrStr := "" + if eventErr != nil { + eventErrStr = eventErr.Error() + } + err := el.Vent(recordevents.EventInfo{ + Error: eventErrStr, + Event: event, + HTTPHeaders: header, + Origin: request.RemoteAddr, + Observer: "", + Time: time.Now(), + }) + if err != nil { + log.Warn("Error while venting the recorded event %s", err) + } + } + + writer.WriteHeader(http.StatusAccepted) +} diff --git a/test/lib/recordevents/recorder_vent/constructor.go b/test/lib/recordevents/recorder_vent/constructor.go new file mode 100644 index 00000000000..c01217715d5 --- /dev/null +++ b/test/lib/recordevents/recorder_vent/constructor.go @@ -0,0 +1,113 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package recorder_vent + +import ( + "context" + "encoding/json" + "log" + + "github.com/kelseyhightower/envconfig" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/kubernetes/scheme" + v1 "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/client-go/tools/record" + + duckv1 "knative.dev/pkg/apis/duck/v1" + kubeclient "knative.dev/pkg/client/injection/kube/client" + "knative.dev/pkg/controller" + "knative.dev/pkg/injection/clients/dynamicclient" + "knative.dev/pkg/logging" + + "knative.dev/eventing/test/lib/recordevents" +) + +type envConfig struct { + AgentName string `envconfig:"AGENT_NAME" default:"observer-default" required:"true"` + EventOn string `envconfig:"K8S_EVENT_SINK" required:"true"` + + Port int `envconfig:"PORT" default:"8080" required:"true"` + Sink string `envconfig:"K_SINK"` +} + +func NewFromEnv(ctx context.Context) recordevents.EventLog { + var env envConfig + if err := envconfig.Process("", &env); err != nil { + log.Fatalf("[ERROR] Failed to process env var: %s", err) + } + + var ref duckv1.KReference + if err := json.Unmarshal([]byte(env.EventOn), &ref); err != nil { + log.Fatalf("[ERROR] Failed to process env var [K8S_EVENT_SINK]: %s", err) + } + + return NewEventLog(ctx, env.AgentName, ref) +} + +func NewEventLog(ctx context.Context, agentName string, ref duckv1.KReference) recordevents.EventLog { + + gv, err := schema.ParseGroupVersion(ref.APIVersion) + if err != nil { + logging.FromContext(ctx).Fatalf("failed to parse group version, %s", err) + } + + gvr, _ := meta.UnsafeGuessKindToResource(gv.WithKind(ref.Kind)) + + var on runtime.Object + if ref.Namespace == "" { + on, err = dynamicclient.Get(ctx).Resource(gvr).Get(ctx, ref.Name, metav1.GetOptions{}) + } else { + on, err = dynamicclient.Get(ctx).Resource(gvr).Get(ctx, ref.Name, metav1.GetOptions{}) + } + if err != nil { + logging.FromContext(ctx).Fatalf("failed to fetch object ref, %+v, %s", ref, err) + + } + + return &recorder{out: createRecorder(ctx, agentName), on: on} +} + +func createRecorder(ctx context.Context, agentName string) record.EventRecorder { + logger := logging.FromContext(ctx) + + recorder := controller.GetEventRecorder(ctx) + if recorder == nil { + // Create event broadcaster + logger.Debug("Creating event broadcaster") + eventBroadcaster := record.NewBroadcaster() + watches := []watch.Interface{ + eventBroadcaster.StartLogging(logger.Named("event-broadcaster").Infof), + eventBroadcaster.StartRecordingToSink( + &v1.EventSinkImpl{Interface: kubeclient.Get(ctx).CoreV1().Events("")}), + } + recorder = eventBroadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: agentName}) + go func() { + <-ctx.Done() + for _, w := range watches { + w.Stop() + } + }() + } + + return recorder +} diff --git a/test/lib/recordevents/recorder_vent/doc.go b/test/lib/recordevents/recorder_vent/doc.go new file mode 100644 index 00000000000..1a4bcc2907f --- /dev/null +++ b/test/lib/recordevents/recorder_vent/doc.go @@ -0,0 +1,19 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package recorder_vent implements an recordevents.EventLog backed by Kubernetes +// Events using an event recorder. +package recorder_vent diff --git a/test/lib/recordevents/recorder_vent/recorder.go b/test/lib/recordevents/recorder_vent/recorder.go new file mode 100644 index 00000000000..517a22c2341 --- /dev/null +++ b/test/lib/recordevents/recorder_vent/recorder.go @@ -0,0 +1,48 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package recorder_vent + +import ( + "encoding/json" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/record" + + "knative.dev/eventing/test/lib/recordevents" +) + +const ( + // EventReason is the Kubernetes event reason used for observed events. + EventReason = "EventObserved" +) + +type recorder struct { + out record.EventRecorder + on runtime.Object +} + +func (r *recorder) Vent(observed recordevents.EventInfo) error { + b, err := json.Marshal(observed) + if err != nil { + return err + } + + r.out.Eventf(r.on, corev1.EventTypeNormal, EventReason, "%s", string(b)) + + return nil +} diff --git a/test/lib/recordevents/writer_vent/doc.go b/test/lib/recordevents/writer_vent/doc.go new file mode 100644 index 00000000000..6fc9065ad1c --- /dev/null +++ b/test/lib/recordevents/writer_vent/doc.go @@ -0,0 +1,19 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package writer_vent implements an observer.EventLog backed by a io.Writer +// instance, events are converted to json and written out to the writer. +package writer_vent diff --git a/test/lib/recordevents/writer_vent/writer.go b/test/lib/recordevents/writer_vent/writer.go new file mode 100644 index 00000000000..5457f64c59e --- /dev/null +++ b/test/lib/recordevents/writer_vent/writer.go @@ -0,0 +1,50 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package writer_vent + +import ( + "context" + "encoding/json" + "io" + + "knative.dev/eventing/test/lib/recordevents" +) + +func NewEventLog(ctx context.Context, out io.Writer) recordevents.EventLog { + return &writer{out: out} +} + +type writer struct { + out io.Writer +} + +var newline = []byte("\n") + +func (w *writer) Vent(observed recordevents.EventInfo) error { + b, err := json.Marshal(observed) + if err != nil { + return err + } + if _, err := w.out.Write(b); err != nil { + return err + } + if _, err := w.out.Write(newline); err != nil { + return err + } + + return nil +} diff --git a/test/test_images/recordevents/eventstore.go b/test/test_images/recordevents/eventstore.go deleted file mode 100644 index 470e5adc21d..00000000000 --- a/test/test_images/recordevents/eventstore.go +++ /dev/null @@ -1,191 +0,0 @@ -/* -Copyright 2020 The Knative Authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "encoding/json" - "fmt" - "sync" - - cloudevents "github.com/cloudevents/sdk-go/v2" - - "knative.dev/eventing/test/lib/recordevents" -) - -// Number of EventInfo per block -const evBlockSize = 100 - -// Block of stored EventInfo -type eventBlock struct { - // seqno of [0] evInfoBytes entry - firstIndex int - // offset inside block for newly appended event - firstOffsetFree int - // offset inside block of first non-trimmed event - firstValid int - // serialized EventInfo structures for each seqno. We enforce - // that there is always at least one block. - evInfoBytes [evBlockSize][]byte -} - -// All events currently seen -type eventStore struct { - // Blocks of events in increasing sequency number order - evBlocks []*eventBlock - evBlocksLock sync.Mutex -} - -// Create a new event store. -func newEventStore() *eventStore { - es := &eventStore{} - es.evBlocks = []*eventBlock{{}} - - // One block with no entries starting at sequence number 1 - es.evBlocks[0].firstIndex = 1 - es.evBlocks[0].firstOffsetFree = 0 - es.evBlocks[0].firstValid = 0 - return es -} - -// See if there's enough room to append to the current last block. If not, -// append an extra block. -func (es *eventStore) checkAppendBlock() { - if es.evBlocks[len(es.evBlocks)-1].firstOffsetFree == evBlockSize { - newEVBlock := &eventBlock{ - firstIndex: es.evBlocks[len(es.evBlocks)-1].firstIndex + evBlockSize, - } - es.evBlocks = append(es.evBlocks, newEVBlock) - } -} - -// Store the specified event. -func (es *eventStore) StoreEvent(event *cloudevents.Event, evErr error, httpHeaders map[string][]string) { - var evInfo recordevents.EventInfo - var err error - var evInfoBytes []byte - if evErr != nil { - evInfo.HTTPHeaders = httpHeaders - evInfo.Error = evErr.Error() - if evInfo.Error == "" { - evInfo.Error = "Unknown Incoming Error" - } - evInfoBytes, err = json.Marshal(&evInfo) - if err != nil { - panic(fmt.Errorf("unexpected marshal error (%v) (%+v)", err, evInfo)) - } - } else { - evInfo.Event = event - evInfo.HTTPHeaders = httpHeaders - evInfoBytes, err = json.Marshal(&evInfo) - - if err != nil { - evInfo.Event = nil - evInfo.Error = err.Error() - if evInfo.Error == "" { - evInfo.Error = "Unknown Error" - } - evInfoBytes, err = json.Marshal(&evInfo) - if err != nil { - panic(fmt.Errorf("unexpected marshal error (%v) (%+v)", err, evInfo)) - } - } - } - - es.evBlocksLock.Lock() - // Add a new block if we're out of space - es.checkAppendBlock() - - evBlock := es.evBlocks[len(es.evBlocks)-1] - if evBlock.firstOffsetFree < evBlockSize { - evBlock.evInfoBytes[evBlock.firstOffsetFree] = evInfoBytes - evBlock.firstOffsetFree++ - } - - es.evBlocksLock.Unlock() -} - -// Logically trim all events up to and include the provided -// sequence number. Returns error for patently incorrect -// values (negative sequence number or sequence number larger -// than the largest event seen). Trimming already trimmed -// regions is legal. -func (es *eventStore) TrimThrough(through int) error { - es.evBlocksLock.Lock() - defer es.evBlocksLock.Unlock() - minAvail, maxSeen := es.minMaxUnlocked() - if through > maxSeen { - return fmt.Errorf("invalid trim through %d, maxSeen %d", through, maxSeen) - } else if through < 0 { - return fmt.Errorf("invalid trim less than zero %d", through) - } else if through < minAvail { - return nil - } - // Completely remove blocks if they are full and all events in them are less than - // the specified value. - for len(es.evBlocks) > 1 && (es.evBlocks[0].firstIndex+evBlockSize-1) <= through { - es.evBlocks = es.evBlocks[1:] - } - // Logically trim the block split by through. - es.evBlocks[0].firstValid = through - es.evBlocks[0].firstIndex + 1 - return nil -} - -// return min/max untrimmed value when already holding the lock -func (es *eventStore) minMaxUnlocked() (minAvail int, maxSeen int) { - minBlock := es.evBlocks[0] - minAvail = minBlock.firstIndex + (minBlock.firstValid) - - maxBlock := es.evBlocks[len(es.evBlocks)-1] - maxSeen = maxBlock.firstIndex + maxBlock.firstOffsetFree - 1 - return minAvail, maxSeen -} - -// Returns min available value and max seen value for the store. -// min is the minimum value that can be retrieved via GetEntry, or -// if no values can be retrieved, min == max+1. Max starts at 0 -// when no events have been seen. -func (es *eventStore) MinMax() (minAvail int, maxSeen int) { - es.evBlocksLock.Lock() - minAvail, maxSeen = es.minMaxUnlocked() - - es.evBlocksLock.Unlock() - return minAvail, maxSeen -} - -// Get the already serialized EventInfo structure for the provided sequence -// number. -func (es *eventStore) GetEventInfoBytes(seq int) ([]byte, error) { - var evInfoBytes []byte - found := false - - es.evBlocksLock.Lock() - for _, block := range es.evBlocks { - if seq < block.firstIndex+block.firstValid { - break - } - if seq < block.firstIndex+block.firstOffsetFree { - found = true - evInfoBytes = block.evInfoBytes[seq-block.firstIndex] - break - } - } - es.evBlocksLock.Unlock() - if !found { - return evInfoBytes, fmt.Errorf("Invalid sequence number %d", seq) - } - return evInfoBytes, nil -} diff --git a/test/test_images/recordevents/eventstore_test.go b/test/test_images/recordevents/eventstore_test.go deleted file mode 100644 index 1216db83ccd..00000000000 --- a/test/test_images/recordevents/eventstore_test.go +++ /dev/null @@ -1,339 +0,0 @@ -/* -Copyright 2020 The Knative Authors -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "encoding/json" - "fmt" - "strconv" - "testing" - - cloudevents "github.com/cloudevents/sdk-go/v2" - - "knative.dev/eventing/test/lib/recordevents" -) - -func helperGetEvInfo(es *eventStore, entry int) (*recordevents.EventInfo, error) { - evInfoBytes, err := es.GetEventInfoBytes(entry) - if err != nil { - return nil, fmt.Errorf("error calling get on item %d: %v", entry, err) - } - if len(evInfoBytes) == 0 { - return nil, fmt.Errorf("empty info bytes") - } - - var evInfo recordevents.EventInfo - err = json.Unmarshal(evInfoBytes, &evInfo) - if err != nil { - return nil, fmt.Errorf("error unmarshalling stored JSON: %v", err) - } - return &evInfo, nil -} - -// Test that adding and getting a bunch of events stores them all -// and that retrieving the events retrieves the correct events. -func TestAddGetMany(t *testing.T) { - es := newEventStore() - - count := 10009 - for i := 0; i < count; i++ { - ce := cloudevents.NewEvent(cloudevents.VersionV1) - ce.SetType("knative.dev.test.event.a") - ce.SetSource("https://source.test.event.knative.dev/foo") - ce.SetID(strconv.FormatInt(int64(i), 10)) - es.StoreEvent(&ce, nil, nil) - minAvail, maxSeen := es.MinMax() - if minAvail != 1 { - t.Fatalf("Pass %d Bad min: %d, expected %d", i, minAvail, 1) - } - if maxSeen != i+1 { - t.Fatalf("Pass %d Bad max: %d, expected %d", i, maxSeen, i+1) - } - - } - for i := 1; i <= count; i++ { - evInfo, err := helperGetEvInfo(es, i) - if err != nil { - t.Fatalf("Count %d error: %v", count, err) - } - - if evInfo.Event == nil { - t.Fatalf("Unexpected empty event info event %d: %+v", i, evInfo) - } - if len(evInfo.Error) != 0 { - t.Fatalf("Unexpected error for stored event %d: %s", i, evInfo.Error) - } - - // Make sure it's the expected event - seenID := evInfo.Event.ID() - expectedID := strconv.FormatInt(int64(i-1), 10) - if seenID != expectedID { - t.Errorf("Incorrect id on retrieval: %s, expected %s", seenID, expectedID) - } - } - _, err := es.GetEventInfoBytes(count + 1) - if err == nil { - t.Error("Unexpected non-error return for getinfo of", count+1) - } - - _, err = es.GetEventInfoBytes(0) - if err == nil { - t.Error("Unexpected non-error return for getinfo of", 0) - } - -} - -func TestEmpty(t *testing.T) { - es := newEventStore() - min, max := es.MinMax() - if min != 1 { - t.Errorf("Invalid min: %d, expected %d", min, 1) - } - if max != 0 { - t.Errorf("Invalid max: %d, expected %d", max, 0) - } - - for i := -2; i < 2; i++ { - _, err := es.GetEventInfoBytes(0) - if err == nil { - t.Error("Unexpected non-error return for getinfo of", i) - } - } - -} - -func TestAddGetSingleValid(t *testing.T) { - expectedType := "knative.dev.test.event.a" - expectedSource := "https://source.test.event.knative.dev/foo" - expectedID := "111" - es := newEventStore() - - headers := make(map[string][]string) - headers["foo"] = []string{"bar", "baz"} - ce := cloudevents.NewEvent(cloudevents.VersionV1) - ce.SetType(expectedType) - ce.SetSource(expectedSource) - ce.SetID(expectedID) - es.StoreEvent(&ce, nil, headers) - minAvail, maxSeen := es.MinMax() - if minAvail != maxSeen { - t.Fatalf("Expected match, saw %d, %d", minAvail, maxSeen) - } - - evInfoBytes, err := es.GetEventInfoBytes(minAvail) - if err != nil { - t.Fatal("Error calling get:", err) - } - var evInfo recordevents.EventInfo - err = json.Unmarshal(evInfoBytes, &evInfo) - if err != nil { - t.Fatal("Error unmarshalling stored JSON:", err) - } - - if evInfo.Event == nil { - t.Fatalf("Unexpected empty event info event: %+v", evInfo) - } - if len(evInfo.Error) != 0 { - t.Fatal("Unexpected error for stored event:", evInfo.Error) - } - if len(evInfo.HTTPHeaders) != 1 { - t.Fatalf("Unexpected header contents for stored event: %+v", evInfo.HTTPHeaders) - } - if len(evInfo.HTTPHeaders["foo"]) != 2 { - t.Fatalf("Unexpected header contents for stored event: %+v", evInfo.HTTPHeaders) - } - if evInfo.HTTPHeaders["foo"][0] != "bar" || evInfo.HTTPHeaders["foo"][1] != "baz" { - t.Fatalf("Unexpected header contents for stored event: %+v", evInfo.HTTPHeaders) - } - seenID := evInfo.Event.ID() - if seenID != expectedID { - t.Errorf("Incorrect id on retrieval: %s, expected %s", seenID, expectedID) - } - seenSource := evInfo.Event.Source() - if seenSource != expectedSource { - t.Errorf("Incorrect source on retrieval: %s, expected %s", seenSource, expectedSource) - } - seenType := evInfo.Event.Type() - if seenType != expectedType { - t.Errorf("Incorrect type on retrieval: %s, expected %s", seenType, expectedType) - } -} - -func TestAddGetSingleInvalid(t *testing.T) { - es := newEventStore() - - headers := make(map[string][]string) - headers["foo"] = []string{"bar", "baz"} - ce := cloudevents.NewEvent(cloudevents.VersionV1) - ce.SetType("knative.dev.test.event.a") - // No source - ce.SetID("111") - es.StoreEvent(&ce, nil, headers) - minAvail, maxSeen := es.MinMax() - if minAvail != maxSeen { - t.Fatalf("Expected match, saw %d, %d", minAvail, maxSeen) - } - - evInfoBytes, err := es.GetEventInfoBytes(minAvail) - if err != nil { - t.Fatal("Error calling get:", err) - } - var evInfo recordevents.EventInfo - err = json.Unmarshal(evInfoBytes, &evInfo) - if err != nil { - t.Fatal("Error unmarshalling stored JSON:", err) - } - if evInfo.Event != nil { - t.Fatalf("Unexpected event info: %+v", evInfo) - } - if len(evInfo.Error) == 0 { - t.Fatal("Unexpected empty error for stored event:", evInfo.Error) - } - if len(evInfo.HTTPHeaders) != 1 { - t.Fatalf("Unexpected header contents for stored event: %+v", evInfo.HTTPHeaders) - } - if len(evInfo.HTTPHeaders["foo"]) != 2 { - t.Fatalf("Unexpected header contents for stored event: %+v", evInfo.HTTPHeaders) - } - if evInfo.HTTPHeaders["foo"][0] != "bar" || evInfo.HTTPHeaders["foo"][1] != "baz" { - t.Fatalf("Unexpected header contents for stored event: %+v", evInfo.HTTPHeaders) - } -} - -func TestAddGetSingleInvalidError(t *testing.T) { - es := newEventStore() - - headers := make(map[string][]string) - headers["foo"] = []string{"bar", "baz"} - ce := cloudevents.NewEvent(cloudevents.VersionV1) - ce.SetType("knative.dev.test.event.a") - ce.SetID("111") - ce.SetSource("nnn") - es.StoreEvent(&ce, fmt.Errorf("Error passed in"), headers) - minAvail, maxSeen := es.MinMax() - if minAvail != maxSeen { - t.Fatalf("Expected match, saw %d, %d", minAvail, maxSeen) - } - - evInfoBytes, err := es.GetEventInfoBytes(minAvail) - if err != nil { - t.Fatal("Error calling get:", err) - } - var evInfo recordevents.EventInfo - err = json.Unmarshal(evInfoBytes, &evInfo) - if err != nil { - t.Fatal("Error unmarshalling stored JSON:", err) - } - if evInfo.Event != nil { - t.Fatalf("Unexpected event info: %+v", evInfo) - } - if len(evInfo.Error) == 0 { - t.Fatal("Unexpected empty error for stored event:", evInfo.Error) - } - if len(evInfo.HTTPHeaders) != 1 { - t.Fatalf("Unexpected header contents for stored event: %+v", evInfo.HTTPHeaders) - } - if len(evInfo.HTTPHeaders["foo"]) != 2 { - t.Fatalf("Unexpected header contents for stored event: %+v", evInfo.HTTPHeaders) - } - if evInfo.HTTPHeaders["foo"][0] != "bar" || evInfo.HTTPHeaders["foo"][1] != "baz" { - t.Fatalf("Unexpected header contents for stored event: %+v", evInfo.HTTPHeaders) - } -} - -func helperFillCount(es *eventStore, count int) { - for i := 0; i < count; i++ { - ce := cloudevents.NewEvent(cloudevents.VersionV1) - ce.SetType("knative.dev.test.event.a") - ce.SetSource("https://source.test.event.knative.dev/foo") - ce.SetID(strconv.FormatInt(int64(i), 10)) - es.StoreEvent(&ce, nil, nil) - } -} - -// Test that adding and getting a bunch of events stores them all -// and that retrieving the events retrieves the correct events. -func TestTrim(t *testing.T) { - count := evBlockSize + 10 - - validTrimPoints := []int{0, 1, 2, evBlockSize - 1, evBlockSize, evBlockSize + 1, evBlockSize + 2, count - 1, count} - invalidTrimPoints := []int{-2, -1, count + 1, count + evBlockSize} - - for _, testVal := range validTrimPoints { - es := newEventStore() - helperFillCount(es, count) - err := es.TrimThrough(testVal) - if err != nil { - t.Fatalf("Unexpected error trimming to %d: %v", testVal, err) - } - minAvail, maxSeen := es.MinMax() - if testVal == count { - if minAvail != maxSeen+1 { - t.Errorf("Incorrect minAvail (%d != %d+1) trimming to %d", minAvail, maxSeen, testVal) - } - } else if minAvail != testVal+1 { - t.Errorf("Incorrect minAvail %d trimming to %d", minAvail, testVal) - } - if maxSeen != count { - t.Errorf("Incorrect maxSeen %d trimming to %d", maxSeen, testVal) - } - if minAvail <= maxSeen { - evInfo, err := helperGetEvInfo(es, minAvail) - if err != nil { - t.Fatalf("Couldn't get min avail %d, trim %d error: %v", minAvail, testVal, err) - } - seenID := evInfo.Event.ID() - expectedID := strconv.FormatInt(int64(minAvail-1), 10) - if seenID != expectedID { - t.Fatalf("Expected ID %s, saw %s for ev %d, trim %d", expectedID, seenID, minAvail, testVal) - } - } - ce := cloudevents.NewEvent(cloudevents.VersionV1) - ce.SetType("knative.dev.test.event.a") - ce.SetSource("https://source.test.event.knative.dev/foo") - ce.SetID(strconv.FormatInt(int64(count), 10)) - es.StoreEvent(&ce, nil, nil) - addedMinAvail, addedMaxSeen := es.MinMax() - if addedMaxSeen != maxSeen+1 { - t.Fatalf("Add after trim resulted in bad maxSeen: expected %d, saw %d for trim %d", - maxSeen, addedMaxSeen, testVal) - } - if minAvail == -1 && addedMinAvail != addedMaxSeen { - t.Errorf("Add after full trim resulted in bad minAvail: expected %d, saw %d for trim %d", - addedMaxSeen, addedMinAvail, testVal) - } else if minAvail != -1 && minAvail != addedMinAvail { - t.Errorf("Add after partial trim resulted in bad minAvail: expected %d, saw %d for trim %d", - minAvail, addedMinAvail, testVal) - - } - - } - for _, testVal := range invalidTrimPoints { - es := newEventStore() - helperFillCount(es, count) - err := es.TrimThrough(testVal) - if err == nil { - t.Fatal("Incorrect missing error trimming to", testVal) - } - minAvail, maxSeen := es.MinMax() - if minAvail != 1 { - t.Errorf("Incorrect minAvail %d trimming to %d", minAvail, testVal) - } - if maxSeen != count { - t.Errorf("Incorrect maxSeen %d trimming to %d", maxSeen, testVal) - } - } -} diff --git a/test/test_images/recordevents/main.go b/test/test_images/recordevents/main.go index 8dd74116338..7af5bc1c64d 100644 --- a/test/test_images/recordevents/main.go +++ b/test/test_images/recordevents/main.go @@ -18,163 +18,53 @@ package main import ( "context" - "encoding/json" - "fmt" "log" "net/http" "os" - "strconv" - "strings" - cloudeventsbindings "github.com/cloudevents/sdk-go/v2/binding" - cloudeventshttp "github.com/cloudevents/sdk-go/v2/protocol/http" "go.uber.org/zap" - "knative.dev/eventing/pkg/kncloudevents" - testlib "knative.dev/eventing/test/lib" "knative.dev/eventing/test/lib/dropevents" - "knative.dev/eventing/test/lib/recordevents" + "knative.dev/eventing/test/lib/recordevents/observer" + "knative.dev/eventing/test/lib/recordevents/recorder_vent" + "knative.dev/eventing/test/lib/recordevents/writer_vent" "knative.dev/eventing/test/test_images" ) -type eventRecorder struct { - es *eventStore -} - -func newEventRecorder() *eventRecorder { - return &eventRecorder{es: newEventStore()} -} - -// Start the recordevents REST api server -func (er *eventRecorder) StartServer(port int) { - http.HandleFunc(recordevents.GetMinMaxPath, er.handleMinMax) - http.HandleFunc(recordevents.GetEntryPath, er.handleGetEntry) - http.HandleFunc(recordevents.TrimThroughPath, er.handleTrim) - go http.ListenAndServe(fmt.Sprintf(":%d", port), nil) -} - -// HTTP handler for GetMinMax requests -func (er *eventRecorder) handleMinMax(w http.ResponseWriter, r *http.Request) { - minAvail, maxSeen := er.es.MinMax() - minMax := recordevents.MinMaxResponse{ - MinAvail: minAvail, - MaxSeen: maxSeen, - } - respBytes, err := json.Marshal(minMax) - if err != nil { - log.Panicf("Internal error: json marshal shouldn't fail: (%v) (%+v)", err, minMax) - } - - w.Header().Set("Content-Type", "text/json") - w.WriteHeader(http.StatusOK) - w.Write(respBytes) -} - -// HTTP handler for TrimThrough requests -func (er *eventRecorder) handleTrim(w http.ResponseWriter, r *http.Request) { - // If we extend this much at all we should vendor a better mux(gorilla, etc) - path := strings.TrimLeft(r.URL.Path, "/") - getPrefix := strings.TrimLeft(recordevents.TrimThroughPath, "/") - suffix := strings.TrimLeft(strings.TrimPrefix(path, getPrefix), "/") - - seqNum, err := strconv.ParseInt(suffix, 10, 32) - if err != nil { - http.Error(w, "Can't parse event sequence number in request", http.StatusBadRequest) - return - } - - err = er.es.TrimThrough(int(seqNum)) - if err != nil { - http.Error(w, "Invalid sequence number in request to trim", http.StatusNotFound) - return - } - - w.Header().Set("Content-Type", "text/json") - w.WriteHeader(http.StatusOK) -} - -// HTTP handler for GetEntry requests -func (er *eventRecorder) handleGetEntry(w http.ResponseWriter, r *http.Request) { - // If we extend this much at all we should vendor a better mux(gorilla, etc) - path := strings.TrimLeft(r.URL.Path, "/") - getPrefix := strings.TrimLeft(recordevents.GetEntryPath, "/") - suffix := strings.TrimLeft(strings.TrimPrefix(path, getPrefix), "/") - - seqNum, err := strconv.ParseInt(suffix, 10, 32) - if err != nil { - http.Error(w, "Can't parse event sequence number in request", http.StatusBadRequest) - return - } - - entryBytes, err := er.es.GetEventInfoBytes(int(seqNum)) - if err != nil { - http.Error(w, "Couldn't find requested event", http.StatusNotFound) - return - } - - w.Header().Set("Content-Type", "text/json") - w.WriteHeader(http.StatusOK) - w.Write(entryBytes) -} - -func (er *eventRecorder) ServeHTTP(writer http.ResponseWriter, request *http.Request) { - m := cloudeventshttp.NewMessageFromHttpRequest(request) - defer m.Finish(nil) - - event, eventErr := cloudeventsbindings.ToEvent(context.TODO(), m) - header := request.Header - - er.es.StoreEvent(event, eventErr, map[string][]string(header)) - - headerNameList := testlib.InterestingHeaders() - for _, headerName := range headerNameList { - if headerValue := header.Get(headerName); headerValue != "" { - log.Printf("Header %s: %s\n", headerName, headerValue) - } - } - - if eventErr != nil { - log.Printf("error receiving the event: %v", eventErr) - } else { - valErr := event.Validate() - if valErr == nil { - log.Printf("eventdetails:\n%s", event.String()) - } else { - log.Printf("error validating the event: %v", valErr) - } - } - - writer.WriteHeader(http.StatusAccepted) -} - func main() { - er := newEventRecorder() - er.StartServer(recordevents.RecordEventsPort) + ctx := context.TODO() logger, _ := zap.NewDevelopment() if err := test_images.ConfigureTracing(logger.Sugar(), ""); err != nil { log.Fatalf("Unable to setup trace publishing: %v", err) } + obs := observer.NewFromEnv( + recorder_vent.NewFromEnv(ctx), + writer_vent.NewEventLog(ctx, os.Stdout), + ) + + var err error algorithm, ok := os.LookupEnv(dropevents.SkipAlgorithmKey) - handler := kncloudevents.CreateHandler(er) if ok { skipper := dropevents.SkipperAlgorithm(algorithm) counter := dropevents.CounterHandler{ Skipper: skipper, } - next := handler - handler = http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { - if counter.Skip() { - writer.WriteHeader(http.StatusConflict) - return - } - next.ServeHTTP(writer, request) + err = obs.Start(ctx, func(handler http.Handler) http.Handler { + return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + if counter.Skip() { + writer.WriteHeader(http.StatusConflict) + return + } + handler.ServeHTTP(writer, request) + }) }) + } else { + err = obs.Start(ctx) } - err := http.ListenAndServe(":8080", handler) if err != nil { - panic(err) + log.Fatalf("Problem with start %s", err) } } From 6b62c94246cf5db9be1943381ce347726ab8cd29 Mon Sep 17 00:00:00 2001 From: slinkydeveloper Date: Fri, 2 Oct 2020 12:22:45 +0200 Subject: [PATCH 02/23] WIP Events are sent properly from recordevents to k8s events Signed-off-by: Francesco Guardiani --- .../broker_control_plane_test_helper.go | 4 +-- .../helpers/broker_tracing_test_helper.go | 4 +-- .../channel_status_subscriber_test_helper.go | 4 +-- .../helpers/channel_tracing_test_helper.go | 4 +-- .../helpers/trigger_no_broker_test_helper.go | 5 +-- test/lib/recordevents/event_info.go | 15 ++------ test/lib/recordevents/event_info_store.go | 36 ++++++++++++++----- test/lib/recordevents/observer/observer.go | 2 +- .../recordevents/recorder_vent/constructor.go | 10 ++++-- .../recordevents/recorder_vent/recorder.go | 2 +- test/lib/resources/kube.go | 17 +++++++-- test/test_images/recordevents/main.go | 13 ++++--- 12 files changed, 74 insertions(+), 42 deletions(-) diff --git a/test/conformance/helpers/broker_control_plane_test_helper.go b/test/conformance/helpers/broker_control_plane_test_helper.go index d82789a575b..0121f0471ab 100644 --- a/test/conformance/helpers/broker_control_plane_test_helper.go +++ b/test/conformance/helpers/broker_control_plane_test_helper.go @@ -25,6 +25,7 @@ import ( "knative.dev/pkg/reconciler" eventingv1beta1 "knative.dev/eventing/pkg/apis/eventing/v1beta1" + "knative.dev/eventing/test/lib/recordevents" testlib "knative.dev/eventing/test/lib" "knative.dev/eventing/test/lib/duck" @@ -82,8 +83,7 @@ func triggerV1Beta1BeforeBrokerHelper(triggerName string, client *testlib.Client const etLogger = "logger" const loggerPodName = "logger-pod" - logPod := resources.EventRecordPod(loggerPodName) - client.CreatePodOrFail(logPod, testlib.WithService(loggerPodName)) + _ = recordevents.DeployEventRecordOrFail(context.TODO(), client, loggerPodName) client.WaitForAllTestResourcesReadyOrFail(context.Background()) // Can't do this for the trigger because it's not 'ready' yet client.CreateTriggerOrFailV1Beta1(triggerName, resources.WithAttributesTriggerFilterV1Beta1(eventingv1beta1.TriggerAnyFilter, etLogger, map[string]interface{}{}), diff --git a/test/conformance/helpers/broker_tracing_test_helper.go b/test/conformance/helpers/broker_tracing_test_helper.go index 41db7080d2e..9c09109c9fa 100644 --- a/test/conformance/helpers/broker_tracing_test_helper.go +++ b/test/conformance/helpers/broker_tracing_test_helper.go @@ -31,6 +31,7 @@ import ( "knative.dev/eventing/pkg/apis/eventing/v1beta1" tracinghelper "knative.dev/eventing/test/conformance/helpers/tracing" testlib "knative.dev/eventing/test/lib" + "knative.dev/eventing/test/lib/recordevents" "knative.dev/eventing/test/lib/resources" "knative.dev/eventing/test/lib/sender" ) @@ -82,8 +83,7 @@ func setupBrokerTracing(ctx context.Context, brokerClass string) SetupTracingTes ) // Create a logger (EventRecord) Pod and a K8s Service that points to it. - logPod := resources.EventRecordPod(loggerPodName) - client.CreatePodOrFail(logPod, testlib.WithService(loggerPodName)) + _ = recordevents.DeployEventRecordOrFail(ctx, client, loggerPodName) // Create a Trigger that receives events (type=bar) and sends them to the logger Pod. loggerTrigger := client.CreateTriggerOrFailV1Beta1( diff --git a/test/conformance/helpers/channel_status_subscriber_test_helper.go b/test/conformance/helpers/channel_status_subscriber_test_helper.go index d148e26d682..f27b8525098 100644 --- a/test/conformance/helpers/channel_status_subscriber_test_helper.go +++ b/test/conformance/helpers/channel_status_subscriber_test_helper.go @@ -24,6 +24,7 @@ import ( duckv1beta1 "knative.dev/eventing/pkg/apis/duck/v1beta1" eventingv1beta1 "knative.dev/eventing/pkg/apis/messaging/v1beta1" testlib "knative.dev/eventing/test/lib" + "knative.dev/eventing/test/lib/recordevents" "knative.dev/eventing/test/lib/resources" corev1 "k8s.io/api/core/v1" @@ -59,8 +60,7 @@ func channelHasRequiredSubscriberStatus(ctx context.Context, st *testing.T, clie client.CreateChannelOrFail(channelName, &channel) client.WaitForResourceReadyOrFail(channelName, &channel) - pod := resources.EventRecordPod(subscriberServiceName + "-pod") - client.CreatePodOrFail(pod, testlib.WithService(subscriberServiceName)) + _ = recordevents.DeployEventRecordOrFail(context.TODO(), client, subscriberServiceName + "-pod") subscription := client.CreateSubscriptionOrFail( subscriberServiceName, diff --git a/test/conformance/helpers/channel_tracing_test_helper.go b/test/conformance/helpers/channel_tracing_test_helper.go index 38afa62cfa2..c8d95f5677b 100644 --- a/test/conformance/helpers/channel_tracing_test_helper.go +++ b/test/conformance/helpers/channel_tracing_test_helper.go @@ -29,6 +29,7 @@ import ( tracinghelper "knative.dev/eventing/test/conformance/helpers/tracing" testlib "knative.dev/eventing/test/lib" + "knative.dev/eventing/test/lib/recordevents" "knative.dev/eventing/test/lib/resources" "knative.dev/eventing/test/lib/sender" ) @@ -69,8 +70,7 @@ func setupChannelTracingWithReply( client.CreateChannelOrFail(replyChannelName, channel) // Create the 'sink', a LogEvents Pod and a K8s Service that points to it. - recordEventsPod := resources.EventRecordPod(recordEventsPodName) - client.CreatePodOrFail(recordEventsPod, testlib.WithService(recordEventsPodName)) + recordEventsPod := recordevents.DeployEventRecordOrFail(ctx, client, recordEventsPodName) // Create the subscriber, a Pod that mutates the event. transformerPod := resources.EventTransformationPod( diff --git a/test/e2e/helpers/trigger_no_broker_test_helper.go b/test/e2e/helpers/trigger_no_broker_test_helper.go index ce8a4d9c454..0303be7e8c0 100644 --- a/test/e2e/helpers/trigger_no_broker_test_helper.go +++ b/test/e2e/helpers/trigger_no_broker_test_helper.go @@ -27,6 +27,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" testlib "knative.dev/eventing/test/lib" + "knative.dev/eventing/test/lib/recordevents" "knative.dev/eventing/test/lib/resources" ) @@ -41,8 +42,8 @@ func TestTriggerNoBroker(ctx context.Context, t *testing.T, channel string, brok brokerName := strings.ToLower(channel) subscriberName := "dumper-empty" - eventRecordPod := resources.EventRecordPod(subscriberName) - client.CreatePodOrFail(eventRecordPod, testlib.WithService(subscriberName)) + recordevents.DeployEventRecordOrFail(context.TODO(), client, subscriberName) + client.CreateTriggerOrFailV1Beta1("testtrigger", resources.WithSubscriberServiceRefForTriggerV1Beta1(subscriberName), resources.WithBrokerV1Beta1(brokerName), diff --git a/test/lib/recordevents/event_info.go b/test/lib/recordevents/event_info.go index b52ef675718..2f530394585 100644 --- a/test/lib/recordevents/event_info.go +++ b/test/lib/recordevents/event_info.go @@ -67,9 +67,9 @@ type EventInfo struct { Event *cloudevents.Event `json:"event"` // HTTPHeaders of the connection that delivered the event HTTPHeaders map[string][]string `json:"httpHeaders"` - Origin string `json:"origin"` - Observer string `json:"observer"` - Time time.Time `json:"time"` + Origin string `json:"origin"` + Observer string `json:"observer"` + Time time.Time `json:"time"` } // Pretty print the event. Meant for debugging. This formats the validation error @@ -117,15 +117,6 @@ type eventGetter struct { func newEventGetter(podName string, client *testlib.Client, logf logging.FormatLogger) (eventGetterInterface, error) { egi := &eventGetter{podName: podName, podNamespace: client.Namespace, kubeClientset: client.Kube.Kube, podPort: RecordEventsPort, logf: logf} - err := egi.forwardPort() - if err != nil { - return nil, err - } - - err = egi.waitTillUp() - if err != nil { - return nil, err - } return egi, nil } diff --git a/test/lib/recordevents/event_info_store.go b/test/lib/recordevents/event_info_store.go index 52fcbae3cdb..768933492bd 100644 --- a/test/lib/recordevents/event_info_store.go +++ b/test/lib/recordevents/event_info_store.go @@ -27,6 +27,7 @@ import ( "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/util/wait" pkgTest "knative.dev/pkg/test" @@ -78,7 +79,6 @@ func newTestableEventInfoStore(egi eventGetterInterface, retryInterval time.Dura retryInterval = minEvRetryInterval } ei := &EventInfoStore{getter: egi, firstID: 1, timeout: timeout, retryInterval: retryInterval} - ei.start() return ei } @@ -100,15 +100,35 @@ func NewEventInfoStore(client *testlib.Client, podName string) (*EventInfoStore, type EventRecordOption = func(*corev1.Pod, *testlib.Client) error -// Deploys a new recordevents pod and start the associated EventInfoStore -func StartEventRecordOrFail(ctx context.Context, client *testlib.Client, podName string, options ...EventRecordOption) (*EventInfoStore, *corev1.Pod) { - eventRecordPod := resources.EventRecordPod(podName) - client.CreatePodOrFail(eventRecordPod, append(options, testlib.WithService(podName))...) - err := pkgTest.WaitForPodRunning(ctx, client.Kube, podName, client.Namespace) +func DeployEventRecordOrFail(ctx context.Context, client *testlib.Client, name string, options ...EventRecordOption) *corev1.Pod { + client.CreateServiceAccountOrFail(name) + client.CreateRoleOrFail(resources.Role(name, + resources.WithRuleForRole(&rbacv1.PolicyRule{ + APIGroups: []string{""}, + Resources: []string{"pods"}, + Verbs: []string{"get", "list", "watch"}, + }), + resources.WithRuleForRole(&rbacv1.PolicyRule{ + APIGroups: []string{""}, + Resources: []string{"events"}, + Verbs: []string{rbacv1.VerbAll}, + }), + )) + client.CreateRoleBindingOrFail(name, "Role", name, name, client.Namespace) + + eventRecordPod := resources.EventRecordPod(name, client.Namespace, name) + client.CreatePodOrFail(eventRecordPod, append(options, testlib.WithService(name))...) + err := pkgTest.WaitForPodRunning(ctx, client.Kube, name, client.Namespace) if err != nil { - client.T.Fatalf("Failed to start the recordevent pod '%s': %v", podName, errors.WithStack(err)) + client.T.Fatalf("Failed to start the recordevent pod '%s': %v", name, errors.WithStack(err)) } - client.WaitForServiceEndpointsOrFail(ctx, podName, 1) + client.WaitForServiceEndpointsOrFail(ctx, name, 1) + return eventRecordPod +} + +// Deploys a new recordevents pod and start the associated EventInfoStore +func StartEventRecordOrFail(ctx context.Context, client *testlib.Client, podName string, options ...EventRecordOption) (*EventInfoStore, *corev1.Pod) { + eventRecordPod := DeployEventRecordOrFail(ctx, client, podName, options...) eventTracker, err := NewEventInfoStore(client, podName) if err != nil { diff --git a/test/lib/recordevents/observer/observer.go b/test/lib/recordevents/observer/observer.go index a5012480c66..312ef07596e 100644 --- a/test/lib/recordevents/observer/observer.go +++ b/test/lib/recordevents/observer/observer.go @@ -25,7 +25,7 @@ type Observer struct { // EventLog instances. It will listen on :8080. func New(name string, eventLogs ...recordevents.EventLog) *Observer { return &Observer{ - Name: name, + Name: name, EventLogs: eventLogs, } } diff --git a/test/lib/recordevents/recorder_vent/constructor.go b/test/lib/recordevents/recorder_vent/constructor.go index c01217715d5..302a1e27863 100644 --- a/test/lib/recordevents/recorder_vent/constructor.go +++ b/test/lib/recordevents/recorder_vent/constructor.go @@ -23,6 +23,8 @@ import ( "github.com/kelseyhightower/envconfig" + "knative.dev/pkg/system" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -71,17 +73,18 @@ func NewEventLog(ctx context.Context, agentName string, ref duckv1.KReference) r logging.FromContext(ctx).Fatalf("failed to parse group version, %s", err) } + logging.FromContext(ctx).Infof("Going to send events to %+v", ref) + gvr, _ := meta.UnsafeGuessKindToResource(gv.WithKind(ref.Kind)) var on runtime.Object if ref.Namespace == "" { on, err = dynamicclient.Get(ctx).Resource(gvr).Get(ctx, ref.Name, metav1.GetOptions{}) } else { - on, err = dynamicclient.Get(ctx).Resource(gvr).Get(ctx, ref.Name, metav1.GetOptions{}) + on, err = dynamicclient.Get(ctx).Resource(gvr).Namespace(ref.Namespace).Get(ctx, ref.Name, metav1.GetOptions{}) } if err != nil { logging.FromContext(ctx).Fatalf("failed to fetch object ref, %+v, %s", ref, err) - } return &recorder{out: createRecorder(ctx, agentName), on: on} @@ -98,7 +101,8 @@ func createRecorder(ctx context.Context, agentName string) record.EventRecorder watches := []watch.Interface{ eventBroadcaster.StartLogging(logger.Named("event-broadcaster").Infof), eventBroadcaster.StartRecordingToSink( - &v1.EventSinkImpl{Interface: kubeclient.Get(ctx).CoreV1().Events("")}), + &v1.EventSinkImpl{Interface: kubeclient.Get(ctx).CoreV1().Events(system.Namespace())}, + ), } recorder = eventBroadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: agentName}) go func() { diff --git a/test/lib/recordevents/recorder_vent/recorder.go b/test/lib/recordevents/recorder_vent/recorder.go index 517a22c2341..e2e96e288f7 100644 --- a/test/lib/recordevents/recorder_vent/recorder.go +++ b/test/lib/recordevents/recorder_vent/recorder.go @@ -28,7 +28,7 @@ import ( const ( // EventReason is the Kubernetes event reason used for observed events. - EventReason = "EventObserved" + EventReason = "CloudEventObserved" ) type recorder struct { diff --git a/test/lib/resources/kube.go b/test/lib/resources/kube.go index 2b154fe2b20..7b43881c752 100644 --- a/test/lib/resources/kube.go +++ b/test/lib/resources/kube.go @@ -38,11 +38,11 @@ type PodOption func(*corev1.Pod) type RoleOption func(*rbacv1.Role) // EventRecordPod creates a Pod that stores received events for test retrieval. -func EventRecordPod(name string) *corev1.Pod { - return eventLoggerPod("recordevents", name) +func EventRecordPod(name string, namespace string, serviceAccountName string) *corev1.Pod { + return recordEventsPod("recordevents", name, namespace, serviceAccountName) } -func eventLoggerPod(imageName string, name string) *corev1.Pod { +func recordEventsPod(imageName string, name string, namespace string, serviceAccountName string) *corev1.Pod { return &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -53,7 +53,18 @@ func eventLoggerPod(imageName string, name string) *corev1.Pod { Name: imageName, Image: pkgTest.ImagePath(imageName), ImagePullPolicy: corev1.PullAlways, + Env: []corev1.EnvVar{{ + Name: "SYSTEM_NAMESPACE", + Value: namespace, + }, { + Name: "OBSERVER", + Value: "recorder-" + name, + }, { + Name: "K8S_EVENT_SINK", + Value: fmt.Sprintf("{\"apiVersion\": \"v1\", \"kind\": \"Pod\", \"name\": \"%s\", \"namespace\": \"%s\"}", name, namespace), + }}, }}, + ServiceAccountName: serviceAccountName, RestartPolicy: corev1.RestartPolicyAlways, }, } diff --git a/test/test_images/recordevents/main.go b/test/test_images/recordevents/main.go index 7af5bc1c64d..055a645ce83 100644 --- a/test/test_images/recordevents/main.go +++ b/test/test_images/recordevents/main.go @@ -17,12 +17,14 @@ limitations under the License. package main import ( - "context" "log" "net/http" "os" "go.uber.org/zap" + "k8s.io/client-go/rest" + "knative.dev/pkg/injection/sharedmain" + _ "knative.dev/pkg/system/testing" "knative.dev/eventing/test/lib/dropevents" "knative.dev/eventing/test/lib/recordevents/observer" @@ -32,7 +34,11 @@ import ( ) func main() { - ctx := context.TODO() + cfg, err := rest.InClusterConfig() + if err != nil { + log.Fatalf("Error while reading the cfg: %s", err) + } + ctx := sharedmain.EnableInjectionOrDie(nil, cfg) logger, _ := zap.NewDevelopment() if err := test_images.ConfigureTracing(logger.Sugar(), ""); err != nil { @@ -44,7 +50,6 @@ func main() { writer_vent.NewEventLog(ctx, os.Stdout), ) - var err error algorithm, ok := os.LookupEnv(dropevents.SkipAlgorithmKey) if ok { skipper := dropevents.SkipperAlgorithm(algorithm) @@ -65,6 +70,6 @@ func main() { } if err != nil { - log.Fatalf("Problem with start %s", err) + log.Fatalf("Error during start: %s", err) } } From a06f4675a315ca3f337e975e3eafc9f94292da60 Mon Sep 17 00:00:00 2001 From: slinkydeveloper Date: Mon, 5 Oct 2020 15:32:20 +0200 Subject: [PATCH 03/23] Sending the beast in battle Signed-off-by: Francesco Guardiani --- .../helpers/tracing_test_helper.go | 2 +- test/lib/client.go | 6 + test/lib/k8s_events.go | 68 +++++ test/lib/recordevents/event_info.go | 214 +------------- test/lib/recordevents/event_info_store.go | 190 ++++-------- .../lib/recordevents/event_info_store_test.go | 279 ------------------ .../recordevents/recorder_vent/recorder.go | 7 +- 7 files changed, 133 insertions(+), 633 deletions(-) create mode 100644 test/lib/k8s_events.go delete mode 100644 test/lib/recordevents/event_info_store_test.go diff --git a/test/conformance/helpers/tracing_test_helper.go b/test/conformance/helpers/tracing_test_helper.go index 1b2f9af79bb..0eb6b16995f 100644 --- a/test/conformance/helpers/tracing_test_helper.go +++ b/test/conformance/helpers/tracing_test_helper.go @@ -67,7 +67,7 @@ func tracingTest( expectedTestSpan, eventMatcher := setupInfrastructure(ctx, t, &channel, client, recordEventsPodName, true) // Start the event info store and assert the event was received correctly - targetTracker, err := recordevents.NewEventInfoStore(client, recordEventsPodName) + targetTracker, err := recordevents.NewEventInfoStore(client, recordEventsPodName, client.Namespace) if err != nil { t.Fatal("Pod tracker failed:", err) } diff --git a/test/lib/client.go b/test/lib/client.go index 1eaef6ececd..d1c5782cc0f 100644 --- a/test/lib/client.go +++ b/test/lib/client.go @@ -48,6 +48,8 @@ type Client struct { Dynamic dynamic.Interface Config *rest.Config + EventListener *EventListener + Namespace string T *testing.T Tracker *Tracker @@ -93,6 +95,10 @@ func NewClient(configPath string, clusterName string, namespace string, t *testi client.T = t client.Tracker = NewTracker(t, client.Dynamic) + // Start informer + client.EventListener = NewEventListener(client.Kube.Kube, client.Namespace, client.T.Logf) + client.Cleanup(client.EventListener.Stop) + client.tracingEnv, err = getTracingConfig(client.Kube.Kube) if err != nil { return nil, err diff --git a/test/lib/k8s_events.go b/test/lib/k8s_events.go new file mode 100644 index 00000000000..296a2ef2b80 --- /dev/null +++ b/test/lib/k8s_events.go @@ -0,0 +1,68 @@ +package lib + +import ( + "context" + "sync" + + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" +) + +// EventHandler is the callback type for the EventListener +type EventHandler func(event *corev1.Event) + +// EventListener is a type that broadcasts new k8s events +type EventListener struct { + cancel context.CancelFunc + + lock sync.Mutex + handlers []EventHandler +} + +// NewEventListener creates a new event listener +func NewEventListener(client kubernetes.Interface, namespace string, logf func(string, ...interface{})) *EventListener { + ctx, cancelCtx := context.WithCancel(context.Background()) + informerFactory := informers.NewSharedInformerFactoryWithOptions( + client, + 0, + informers.WithNamespace(namespace), + ) + eventsInformer := informerFactory.Core().V1().Events().Informer() + + el := EventListener{ + cancel: cancelCtx, + } + + eventsInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + el.handle(obj.(*corev1.Event)) + }, + }) + + go func() { + eventsInformer.Run(ctx.Done()) + logf("EventListener stopped") + }() + + return &el +} + +func (el *EventListener) handle(event *corev1.Event) { + el.lock.Lock() + defer el.lock.Unlock() + for _, handler := range el.handlers { + handler(event) + } +} + +func (el *EventListener) AddHandler(handler EventHandler) { + el.lock.Lock() + defer el.lock.Unlock() + el.handlers = append(el.handlers, handler) +} + +func (el *EventListener) Stop() { + el.cancel() +} diff --git a/test/lib/recordevents/event_info.go b/test/lib/recordevents/event_info.go index 2f530394585..fd77a362568 100644 --- a/test/lib/recordevents/event_info.go +++ b/test/lib/recordevents/event_info.go @@ -17,46 +17,17 @@ limitations under the License. package recordevents import ( - "context" - "encoding/json" "fmt" - "io/ioutil" - "math/rand" - "net/http" "strings" "time" cloudevents "github.com/cloudevents/sdk-go/v2" - corev1 "k8s.io/api/core/v1" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/kubernetes" - "knative.dev/pkg/test/monitoring" - - "knative.dev/pkg/test/logging" - - testlib "knative.dev/eventing/test/lib" ) -// Port for the recordevents pod REST listener -const RecordEventsPort = 8392 - -// HTTP path for the GetMinMax REST call -const GetMinMaxPath = "/minmax" - -// HTTP path for the GetEntry REST call -const GetEntryPath = "/entry/" - -// HTTP path for the TrimThrough REST call -const TrimThroughPath = "/trimthrough/" - -// On-wire json rest api format for recordevents GetMinMax calls -// sennt to the recordevents pod. -type MinMaxResponse struct { - MinAvail int - MaxSeen int -} +const ( + // EventReason is the Kubernetes event reason used for observed events. + CloudEventObservedReason = "CloudEventObserved" +) // Structure to hold information about an event seen by recordevents pod. type EventInfo struct { @@ -98,180 +69,3 @@ func (s *SearchedInfo) String() string { } return sb.String() } - -// Connection state for a REST connection to a pod -type eventGetter struct { - podName string - podNamespace string - podPort int - kubeClientset kubernetes.Interface - logf logging.FormatLogger - - host string - port int - forwardPID int -} - -// Creates a forwarded port to the specified recordevents pod and waits until -// it can successfully talk to the REST API. Times out after timeoutEvRetry -func newEventGetter(podName string, client *testlib.Client, logf logging.FormatLogger) (eventGetterInterface, error) { - egi := &eventGetter{podName: podName, podNamespace: client.Namespace, - kubeClientset: client.Kube.Kube, podPort: RecordEventsPort, logf: logf} - return egi, nil -} - -// Get information about the provided podName. Uses list (rather than get) and -// returns a pod list for compatibility with the monitoring.PortForward -// interface -func (eg *eventGetter) getRunningPodInfo(podName, namespace string) (*v1.PodList, error) { - pods, err := eg.kubeClientset.CoreV1().Pods(namespace).List(context.Background(), - metav1.ListOptions{FieldSelector: fmt.Sprintf("metadata.name=%s", podName)}) - if err == nil && len(pods.Items) != 1 { - err = fmt.Errorf("no %s Pod found on the cluster", podName) - } else if pods.Items[0].Status.Phase != corev1.PodRunning { - err = fmt.Errorf("pod %s in state %s, wanted Running", podName, - pods.Items[0].Status.Phase) - } - - return pods, err -} - -// Try to forward the pod port to a local port somewhere in the range 30000-60000. -// keeps retrying with random ports in that range, timing out after timeoutEvRetry -func (eg *eventGetter) forwardPort() error { - portRand := rand.New(rand.NewSource(time.Now().UnixNano())) - portMin := 30000 - portMax := 60000 - var internalErr error - - wait.PollImmediate(minEvRetryInterval, timeoutEvRetry, func() (bool, error) { - localPort := portMin + portRand.Intn(portMax-portMin) - if err := monitoring.CheckPortAvailability(localPort); err != nil { - internalErr = err - return false, nil - } - pods, err := eg.getRunningPodInfo(eg.podName, eg.podNamespace) - if err != nil { - internalErr = err - return false, nil - } - - pid, err := monitoring.PortForward(eg.logf, pods, localPort, eg.podPort, eg.podNamespace) - if err != nil { - internalErr = err - return false, nil - } - internalErr = nil - - eg.forwardPID = pid - eg.port = localPort - eg.host = "localhost" - return true, nil - }) - if internalErr != nil { - return fmt.Errorf("timeout forwarding port: %v", internalErr) - } - return nil -} - -// Return the min available, max seen by the recordevents pod. -// maxRet is the largest event that has ever been seen (whether it's been trimmed -// or not). minRet is the smallest event still available via Get, or 1+maxRet if -// no events are available. maxRet starts at 0 when no events have been seen. -func (eg *eventGetter) getMinMax() (minRet int, maxRet int, errRet error) { - resp, err := http.Get(fmt.Sprintf("http://%s:%d%s", eg.host, eg.port, GetMinMaxPath)) - if err != nil { - return -1, -1, fmt.Errorf("http get error: %v", err) - } - defer resp.Body.Close() - bodyContents, err := ioutil.ReadAll(resp.Body) - if err != nil { - return -1, -1, fmt.Errorf("error reading response body %w", err) - } - if resp.StatusCode != http.StatusOK { - return -1, -1, fmt.Errorf("error %d reading GetMinMax response", resp.StatusCode) - } - minMaxResponse := MinMaxResponse{} - err = json.Unmarshal(bodyContents, &minMaxResponse) - if err != nil { - return -1, -1, fmt.Errorf("error unmarshalling response %w", err) - } - if minMaxResponse.MinAvail == 0 { - return -1, -1, fmt.Errorf("invalid decoded json: %+v", minMaxResponse) - } - - return minMaxResponse.MinAvail, minMaxResponse.MaxSeen, nil -} - -// Return the event with the provided sequence number. Returns the appropriate -// EventInfo or an error if no such event is known, or the event has already -// been trimmed. -func (eg *eventGetter) getEntry(seqno int) (EventInfo, error) { - resp, err := http.Get(fmt.Sprintf("http://%s:%d%s/%d", eg.host, eg.port, GetEntryPath, seqno)) - if err != nil { - return EventInfo{}, fmt.Errorf("http get err %v", err) - } - defer resp.Body.Close() - bodyContents, err := ioutil.ReadAll(resp.Body) - if err != nil { - return EventInfo{}, fmt.Errorf("error reading response body %w", err) - } - if resp.StatusCode != http.StatusOK { - return EventInfo{}, fmt.Errorf("error %d reading GetEntry response", resp.StatusCode) - } - entryResponse := EventInfo{} - err = json.Unmarshal(bodyContents, &entryResponse) - if err != nil { - return EventInfo{}, fmt.Errorf("error unmarshalling response %w", err) - } - if len(entryResponse.Error) == 0 && entryResponse.Event == nil { - return EventInfo{}, fmt.Errorf("invalid decoded json: %+v", entryResponse) - } - - return entryResponse, nil -} - -// Trim the events up to and including seqno from the recordevents pod. -// Returns an error if a nonsensical seqno is passed in, but does not return -// error for trimming already trimmed regions. -func (eg *eventGetter) trimThrough(seqno int) error { - resp, err := http.Post(fmt.Sprintf("http://%s:%d%s/%d", eg.host, eg.port, TrimThroughPath, seqno), "", nil) - if err != nil { - return fmt.Errorf("http post err %v", err) - } - defer resp.Body.Close() - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return fmt.Errorf("error reading response body %w", err) - } - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("error %d reading TrimThrough response: %s", resp.StatusCode, string(body)) - } - - return nil -} - -// Clean up the getter by tearing down the port forward. -func (eg *eventGetter) cleanup() { - pid := eg.forwardPID - eg.forwardPID = 0 - if pid != 0 { - monitoring.Cleanup(pid) - } -} - -// Wait (up to timeoutEvRetry) for the pod to RestAPI to answer request. -func (eg *eventGetter) waitTillUp() error { - var internalErr error - wait.PollImmediate(minEvRetryInterval, timeoutEvRetry, func() (bool, error) { - _, _, internalErr = eg.getMinMax() - if internalErr != nil { - return false, nil - } - return true, nil - }) - if internalErr != nil { - return fmt.Errorf("timeout waiting for recordevents pod to come up: %v", internalErr) - } - return nil -} diff --git a/test/lib/recordevents/event_info_store.go b/test/lib/recordevents/event_info_store.go index 768933492bd..6e721cd8956 100644 --- a/test/lib/recordevents/event_info_store.go +++ b/test/lib/recordevents/event_info_store.go @@ -18,6 +18,7 @@ package recordevents import ( "context" + "encoding/json" "fmt" "strconv" "strings" @@ -37,67 +38,10 @@ import ( const ( // The interval and timeout used for checking events - minEvRetryInterval = 4 * time.Second - timeoutEvRetry = 4 * time.Minute + retryInterval = 4 * time.Second + retryTimeout = 4 * time.Minute ) -// Stateful store of events received by the recordevents pod it is pointed at. -// This pulls events from the pod during any Find or Wait call, storing them -// locally and triming them from the remote pod store. -type EventInfoStore struct { - tb testing.TB - - podName string - getter eventGetterInterface - - lock sync.Mutex - allEvents []EventInfo - firstID int - closeCh chan struct{} - doRefresh chan chan error - timeout time.Duration - retryInterval time.Duration -} - -// Functions used for getting data from the REST api of the recordevents pod. -// The interface exists for use with unit tests of this module. -type eventGetterInterface interface { - getMinMax() (minRet int, maxRet int, errRet error) - getEntry(seqno int) (EventInfo, error) - trimThrough(seqno int) error - cleanup() -} - -// Internal function to create an event store. This is called directly by unit tests of -// this module. -func newTestableEventInfoStore(egi eventGetterInterface, retryInterval time.Duration, - timeout time.Duration) *EventInfoStore { - if timeout == -1 { - timeout = timeoutEvRetry - } - if retryInterval == -1 { - retryInterval = minEvRetryInterval - } - ei := &EventInfoStore{getter: egi, firstID: 1, timeout: timeout, retryInterval: retryInterval} - return ei -} - -// Creates an EventInfoStore that is used to iteratively download events recorded by the -// recordevents pod. Calling this forwards the recordevents port to the local machine -// and blocks waiting to connect to that pod. Fails if it cannot connect within -// the expected timeout (4 minutes currently) -func NewEventInfoStore(client *testlib.Client, podName string) (*EventInfoStore, error) { - egi, err := newEventGetter(podName, client, client.T.Logf) - if err != nil { - return nil, err - } - ei := newTestableEventInfoStore(egi, -1, -1) - ei.podName = podName - ei.tb = client.T - client.Cleanup(ei.cleanup) - return ei, nil -} - type EventRecordOption = func(*corev1.Pod, *testlib.Client) error func DeployEventRecordOrFail(ctx context.Context, client *testlib.Client, name string, options ...EventRecordOption) *corev1.Pod { @@ -106,12 +50,12 @@ func DeployEventRecordOrFail(ctx context.Context, client *testlib.Client, name s resources.WithRuleForRole(&rbacv1.PolicyRule{ APIGroups: []string{""}, Resources: []string{"pods"}, - Verbs: []string{"get", "list", "watch"}, + Verbs: []string{"get", "list", "watch"}, }), resources.WithRuleForRole(&rbacv1.PolicyRule{ APIGroups: []string{""}, Resources: []string{"events"}, - Verbs: []string{rbacv1.VerbAll}, + Verbs: []string{rbacv1.VerbAll}, }), )) client.CreateRoleBindingOrFail(name, "Role", name, name, client.Namespace) @@ -130,95 +74,70 @@ func DeployEventRecordOrFail(ctx context.Context, client *testlib.Client, name s func StartEventRecordOrFail(ctx context.Context, client *testlib.Client, podName string, options ...EventRecordOption) (*EventInfoStore, *corev1.Pod) { eventRecordPod := DeployEventRecordOrFail(ctx, client, podName, options...) - eventTracker, err := NewEventInfoStore(client, podName) + eventTracker, err := NewEventInfoStore(client, podName, client.Namespace) if err != nil { client.T.Fatalf("Failed to start the EventInfoStore associated to pod '%s': %v", podName, err) } return eventTracker, eventRecordPod } -// Starts the single threaded background goroutine used to update local state -// from the remote REST API. -func (ei *EventInfoStore) start() { - ei.closeCh = make(chan struct{}) - ei.doRefresh = make(chan chan error) - go func() { - for { - select { - case <-ei.closeCh: - ei.getter.cleanup() - return - case replyCh := <-ei.doRefresh: - replyCh <- ei.doRetrieveData() - } - } - }() +// Stateful store of events received by the recordevents pod it is pointed at. +// This pulls events from the pod during any Find or Wait call, storing them +// locally and triming them from the remote pod store. +type EventInfoStore struct { + tb testing.TB + + podName string + podNamespace string + + lock sync.Mutex + collected []EventInfo } -// The data update thread used by the single threaded background goroutine -// for updating data from the REST api. -func (ei *EventInfoStore) doRetrieveData() error { - min, max, err := ei.getter.getMinMax() - if err != nil { - return fmt.Errorf("error getting MinMax %v", err) - } - ei.lock.Lock() - curMin := ei.firstID - curMax := curMin + len(ei.allEvents) - 1 - ei.lock.Unlock() - if min == max+1 { - // Nothing to read or trim - return nil - } else { - if min > curMax+1 { - return fmt.Errorf("mismatched stored max/available min: %d, %d", curMax, min) - } - min = curMax + 1 - // We may have data to read, definitely have data to trim. +// Creates an EventInfoStore that is used to iteratively download events recorded by the +// recordevents pod. +func NewEventInfoStore(client *testlib.Client, podName string, podNamespace string) (*EventInfoStore, error) { + store := &EventInfoStore{ + tb: client.T, + podName: podName, + podNamespace: podNamespace, } - var newEvents []EventInfo - for i := min; i <= max; i++ { - e, err := ei.getter.getEntry(i) - if err != nil { - return fmt.Errorf("error calling getEntry of %d %v", i, err) - } - newEvents = append(newEvents, e) - } - ei.lock.Lock() - ei.allEvents = append(ei.allEvents, newEvents...) - ei.lock.Unlock() - err = ei.getter.trimThrough(max) - return err + client.EventListener.AddHandler(store.handle) + return store, nil } -// Clean up any background resources used by the store. Must be called exactly once after -// the last use. -func (ei *EventInfoStore) cleanup() { - close(ei.closeCh) +func (ei *EventInfoStore) getEventInfo() []EventInfo { + ei.lock.Lock() + defer ei.lock.Unlock() + return ei.collected } -//TODO remove it, this is not useful anymore -// Deprecated: you can remove the manual cleanup of the event getter, since now it's done at test tear down automatically -func (ei *EventInfoStore) Cleanup() {} - -// Called internally by functions wanting the current list of all -// known events. This calls for an update from the REST server and -// returns the summary of all locally and remotely known events. -// Returns an error in case of a connection or protocol error. -func (ei *EventInfoStore) refreshData() ([]EventInfo, error) { - var allEvents []EventInfo - replyCh := make(chan error) - ei.doRefresh <- replyCh - err := <-replyCh +func (ei *EventInfoStore) handle(event *corev1.Event) { + // Filter events + if !ei.isMyEvent(event) { + return + } + + eventInfo := EventInfo{} + err := json.Unmarshal([]byte(event.Message), &eventInfo) if err != nil { - return nil, err + ei.tb.Errorf("Received EventInfo that cannot be unmarshalled! %+v", err) + return } + ei.lock.Lock() - allEvents = append(allEvents, ei.allEvents...) - ei.lock.Unlock() - return allEvents, nil + defer ei.lock.Unlock() + ei.collected = append(ei.collected, eventInfo) +} + +func (ei *EventInfoStore) isMyEvent(event *corev1.Event) bool { + return event.Type == corev1.EventTypeNormal && + event.Reason == CloudEventObservedReason && + event.InvolvedObject.Kind == "Pod" && + event.InvolvedObject.Name == ei.podName && + event.InvolvedObject.Namespace == ei.podNamespace } // Find all events received by the recordevents pod that match the provided matchers, @@ -236,10 +155,7 @@ func (ei *EventInfoStore) Find(matchers ...EventInfoMatcher) ([]EventInfo, Searc lastEvents := []EventInfo{} var nonMatchingErrors []error - allEvents, err := ei.refreshData() - if err != nil { - return nil, sInfo, nonMatchingErrors, fmt.Errorf("error getting events %v", err) - } + allEvents := ei.getEventInfo() for i := range allEvents { if err := f(allEvents[i]); err == nil { allMatch = append(allMatch, allEvents[i]) @@ -310,7 +226,7 @@ func (ei *EventInfoStore) waitAtLeastNMatch(f EventInfoMatcher, min int) ([]Even var matchRet []EventInfo var internalErr error - wait.PollImmediate(ei.retryInterval, ei.timeout, func() (bool, error) { + wait.PollImmediate(retryInterval, retryTimeout, func() (bool, error) { allMatch, sInfo, matchErrs, err := ei.Find(f) if err != nil { internalErr = fmt.Errorf("FAIL MATCHING: unexpected error during find: %v", err) diff --git a/test/lib/recordevents/event_info_store_test.go b/test/lib/recordevents/event_info_store_test.go deleted file mode 100644 index 492b6d84e9b..00000000000 --- a/test/lib/recordevents/event_info_store_test.go +++ /dev/null @@ -1,279 +0,0 @@ -/* -Copyright 2020 The Knative Authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package recordevents - -import ( - "fmt" - "strconv" - "sync" - "testing" - "time" - - cloudevents "github.com/cloudevents/sdk-go/v2" -) - -type dummyEventGet struct { - lock sync.Mutex - - minSeq int - allEv []EventInfo - clean bool - tCalls int -} - -func newDummyEventGet() *dummyEventGet { - return &dummyEventGet{minSeq: 1} -} - -func (deg *dummyEventGet) getMinMax() (minRet int, maxRet int, errRet error) { - deg.lock.Lock() - defer deg.lock.Unlock() - if deg.clean { - panic("getminmax called on cleaned event getter") - } - min := deg.minSeq - max := min + len(deg.allEv) - 1 - return min, max, nil -} -func (deg *dummyEventGet) getEntry(seqno int) (EventInfo, error) { - deg.lock.Lock() - defer deg.lock.Unlock() - if deg.clean { - panic("getentry called on cleaned event getter") - } - min := deg.minSeq - max := min + len(deg.allEv) - 1 - if seqno >= min && seqno <= max { - return deg.allEv[seqno-deg.minSeq], nil - } else { - return EventInfo{}, fmt.Errorf("illegal get seqno: %d, range min = %d, max = %d", seqno, min, max) - } -} -func (deg *dummyEventGet) trimThrough(seqno int) error { - deg.lock.Lock() - defer deg.lock.Unlock() - deg.tCalls++ - if deg.clean { - panic("trim called on cleaned event getter") - } - min := deg.minSeq - max := min + len(deg.allEv) - 1 - if seqno < 0 { - return fmt.Errorf("Illegal negative seqno %d", seqno) - } else if seqno > max { - return fmt.Errorf("Illegal negative seqno %d > %d", seqno, max) - } else if seqno < min { - return nil - } - deg.allEv = deg.allEv[seqno-min+1:] - deg.minSeq = seqno + 1 - return nil -} -func (deg *dummyEventGet) trimCalls() int { - deg.lock.Lock() - defer deg.lock.Unlock() - return deg.tCalls -} -func (deg *dummyEventGet) cleanup() { - deg.lock.Lock() - defer deg.lock.Unlock() - if deg.clean { - panic("Unexpected clean") - } else { - deg.clean = true - } -} - -func makeEvents() []EventInfo { - var allEv []EventInfo - for i := 0; i < 30; i++ { - ce := cloudevents.NewEvent(cloudevents.VersionV1) - ce.SetType("knative.dev.test.event.a") - ce.SetSource("https://source.test.event.knative.dev/foo") - ce.SetID(strconv.FormatInt(int64(i), 10)) - allEv = append(allEv, EventInfo{Event: &ce}) - } - return allEv -} - -func checkEvIDEqual(t *testing.T, seen []EventInfo, expected []EventInfo) { - if len(seen) != len(expected) { - t.Fatalf("Seen ev list length %d does not match expected %d", len(seen), len(expected)) - - } - for i := range seen { - if seen[i].Event.ID() != expected[i].Event.ID() { - t.Fatalf("Id entry %d mismatch: seen %s != expected %s", - i, seen[i].Event.ID(), expected[i].Event.ID()) - } - } -} - -func (deg *dummyEventGet) setEv(firstID int, allEv []EventInfo) { - deg.lock.Lock() - defer deg.lock.Unlock() - deg.minSeq = firstID - deg.allEv = allEv -} - -// Test that Finds where the server gives sequential updates that -// don't overlap see the expected events. -func TestSequentialAndTrim(t *testing.T) { - totalEv := makeEvents() - expectedFull := makeEvents() - deg := newDummyEventGet() - subEv := totalEv[:10] - deg.setEv(1, subEv) - ei := newTestableEventInfoStore(deg, -1, -1) - allData, _, _, err := ei.Find(func(EventInfo) error { return nil }) - if err != nil { - t.Fatal("Unexpected error from find:", err) - } - checkEvIDEqual(t, allData, expectedFull[:10]) - min, _, _ := deg.getMinMax() - if min != 11 { - t.Fatalf("Expected trim to have moved min to %d, saw %d", 11, min) - } - - subEv = totalEv[10:19] - deg.setEv(11, subEv) - - allData, _, _, err = ei.Find(func(EventInfo) error { return nil }) - if err != nil { - t.Fatal("Unexpected error from find:", err) - } - checkEvIDEqual(t, allData, expectedFull[:19]) - ei.cleanup() -} - -// Test that Finds where the server gives overlapping updates -// see the expected events (no double events) -func TestOverlap(t *testing.T) { - totalEv := makeEvents() - expectedFull := makeEvents() - deg := newDummyEventGet() - subEv := totalEv[:10] - deg.setEv(1, subEv) - ei := newTestableEventInfoStore(deg, -1, -1) - allData, _, _, err := ei.Find(func(EventInfo) error { return nil }) - if err != nil { - t.Fatal("Unexpected error from find:", err) - } - checkEvIDEqual(t, allData, expectedFull[:10]) - subEv = totalEv[6:19] - deg.setEv(7, subEv) - allData, _, _, err = ei.Find(func(EventInfo) error { return nil }) - if err != nil { - t.Fatal("Unexpected error from find:", err) - } - checkEvIDEqual(t, allData, expectedFull[:19]) - ei.cleanup() -} - -// Test that we see an error if repeated Finds see a gap in the sequence -// space -func TestGap(t *testing.T) { - totalEv := makeEvents() - expectedFull := makeEvents() - deg := newDummyEventGet() - subEv := totalEv[:10] - deg.setEv(1, subEv) - ei := newTestableEventInfoStore(deg, -1, -1) - allData, _, _, err := ei.Find(func(EventInfo) error { return nil }) - if err != nil { - t.Fatal("Unexpected error from find:", err) - } - checkEvIDEqual(t, allData, expectedFull[:10]) - subEv = totalEv[11:19] - deg.setEv(12, subEv) - _, _, _, err = ei.Find(func(EventInfo) error { return nil }) - if err == nil { - t.Fatalf("Unexpected success from find") - } - ei.cleanup() -} - -// Test that two finds, with the second one having -// no new events, returns the right events. -func TestSequentialNoOp(t *testing.T) { - totalEv := makeEvents() - expectedFull := makeEvents() - deg := newDummyEventGet() - subEv := totalEv[:10] - deg.setEv(1, subEv) - ei := newTestableEventInfoStore(deg, -1, -1) - allData, _, _, err := ei.Find(func(EventInfo) error { return nil }) - if err != nil { - t.Fatal("Unexpected error from find:", err) - } - checkEvIDEqual(t, allData, expectedFull[:10]) - subEv = []EventInfo{} - deg.setEv(11, subEv) - allData, _, _, err = ei.Find(func(EventInfo) error { return nil }) - if err != nil { - t.Fatal("Unexpected error from find:", err) - } - checkEvIDEqual(t, allData, expectedFull[:10]) - ei.cleanup() -} - -// Test that wait for N Works -func TestWaitForN(t *testing.T) { - deg := newDummyEventGet() - subEv := makeEvents()[:10] - deg.setEv(1, subEv) - ei := newTestableEventInfoStore(deg, time.Second/8, -1) - var wg sync.WaitGroup - wg.Add(1) - var waitErr error - var allMatch []EventInfo - go func() { - matchFunc := func(ev cloudevents.Event) error { - if ev.ID() == "3" { - return nil - } else { - return fmt.Errorf("mismatch %s %s", ev.ID(), "3") - } - } - - allMatch, waitErr = ei.waitAtLeastNMatch(MatchEvent(matchFunc), 2) - wg.Done() - }() - var tCalls int - for tCalls == 0 { - time.Sleep(time.Second / 100) - tCalls = deg.trimCalls() - } - subEv = makeEvents()[:10] - deg.setEv(11, subEv) - - wg.Wait() - if waitErr != nil { - t.Fatal("Unexpected error from find:", waitErr) - } - if len(allMatch) != 2 { - t.Fatalf("Unexpected match length: %d != %d", len(allMatch), 2) - } - if allMatch[0].Event.ID() != "3" || allMatch[1].Event.ID() != "3" { - t.Fatalf("Unexpected IDs %s, %s, expected both %s", allMatch[0].Event.ID(), allMatch[1].Event.ID(), "3") - } - tCalls = deg.trimCalls() - if tCalls < 2 { - t.Fatalf("Expected at least %d trim calls, saw %d", 2, tCalls) - } - ei.cleanup() -} diff --git a/test/lib/recordevents/recorder_vent/recorder.go b/test/lib/recordevents/recorder_vent/recorder.go index e2e96e288f7..9669741f9b6 100644 --- a/test/lib/recordevents/recorder_vent/recorder.go +++ b/test/lib/recordevents/recorder_vent/recorder.go @@ -26,11 +26,6 @@ import ( "knative.dev/eventing/test/lib/recordevents" ) -const ( - // EventReason is the Kubernetes event reason used for observed events. - EventReason = "CloudEventObserved" -) - type recorder struct { out record.EventRecorder on runtime.Object @@ -42,7 +37,7 @@ func (r *recorder) Vent(observed recordevents.EventInfo) error { return err } - r.out.Eventf(r.on, corev1.EventTypeNormal, EventReason, "%s", string(b)) + r.out.Eventf(r.on, corev1.EventTypeNormal, recordevents.CloudEventObservedReason, "%s", string(b)) return nil } From 46cd1bb1e0b341bab153a6564c04d5c43a048159 Mon Sep 17 00:00:00 2001 From: slinkydeveloper Date: Mon, 5 Oct 2020 15:35:09 +0200 Subject: [PATCH 04/23] Update Signed-off-by: Francesco Guardiani --- go.mod | 3 +- go.sum | 7 +- vendor/gopkg.in/yaml.v3/.travis.yml | 22 ++-- vendor/gopkg.in/yaml.v3/apic.go | 1 + vendor/gopkg.in/yaml.v3/decode.go | 63 ++++++---- vendor/gopkg.in/yaml.v3/emitterc.go | 54 +++++++-- vendor/gopkg.in/yaml.v3/encode.go | 42 +++++-- vendor/gopkg.in/yaml.v3/parserc.go | 39 ++++++ vendor/gopkg.in/yaml.v3/scannerc.go | 180 +++++++++++++++++----------- vendor/gopkg.in/yaml.v3/yaml.go | 35 +++++- vendor/gopkg.in/yaml.v3/yamlh.go | 4 + vendor/modules.txt | 3 +- 12 files changed, 323 insertions(+), 130 deletions(-) diff --git a/go.mod b/go.mod index 3da0104f00a..67e66846039 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( github.com/pelletier/go-toml v1.8.0 github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 github.com/pkg/errors v0.9.1 + github.com/prometheus/common v0.9.1 github.com/rickb777/date v1.13.0 github.com/robfig/cron/v3 v3.0.1 github.com/rogpeppe/fastuuid v1.2.0 @@ -31,7 +32,7 @@ require ( golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 google.golang.org/grpc v1.31.1 gopkg.in/yaml.v2 v2.3.0 - gopkg.in/yaml.v3 v3.0.0-20191026110619-0b21df46bc1d // indirect + gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 // indirect k8s.io/api v0.18.8 k8s.io/apiextensions-apiserver v0.18.4 k8s.io/apimachinery v0.18.8 diff --git a/go.sum b/go.sum index cb7b590594d..05c93c3c118 100644 --- a/go.sum +++ b/go.sum @@ -705,6 +705,7 @@ github.com/imdario/mergo v0.3.7/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJ github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.9 h1:UauaLniWCFHWd+Jp9oCEkTBj8VO/9DKg3PV3VCNMDIg= github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb v0.0.0-20161215172503-049f9b42e9a5/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= github.com/influxdata/tdigest v0.0.0-20181121200506-bf2b5ad3c0a9/go.mod h1:Js0mqiSBE6Ffsg94weZZ2c+v/ciT8QRHFOap7EKDrR0= @@ -821,6 +822,7 @@ github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNx github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= +github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.8/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= @@ -1062,6 +1064,7 @@ github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKv github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v0.0.6/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= +github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= @@ -1726,8 +1729,8 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20190709130402-674ba3eaed22/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20191026110619-0b21df46bc1d h1:LCPbGQ34PMrwad11aMZ+dbz5SAsq/0ySjRwQ8I9Qwd8= -gopkg.in/yaml.v3 v3.0.0-20191026110619-0b21df46bc1d/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= helm.sh/helm/v3 v3.1.1/go.mod h1:WYsFJuMASa/4XUqLyv54s0U/f3mlAaRErGmyy4z921g= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/vendor/gopkg.in/yaml.v3/.travis.yml b/vendor/gopkg.in/yaml.v3/.travis.yml index 1bc5c1cd20b..a130fe883ce 100644 --- a/vendor/gopkg.in/yaml.v3/.travis.yml +++ b/vendor/gopkg.in/yaml.v3/.travis.yml @@ -1,15 +1,17 @@ language: go go: - - "1.4" - - "1.5" - - "1.6" - - "1.7" - - "1.8" - - "1.9" - - "1.10" - - "1.11" - - "1.12" - - tip + - "1.4.x" + - "1.5.x" + - "1.6.x" + - "1.7.x" + - "1.8.x" + - "1.9.x" + - "1.10.x" + - "1.11.x" + - "1.12.x" + - "1.13.x" + - "1.14.x" + - "tip" go_import_path: gopkg.in/yaml.v3 diff --git a/vendor/gopkg.in/yaml.v3/apic.go b/vendor/gopkg.in/yaml.v3/apic.go index 65846e67497..ae7d049f182 100644 --- a/vendor/gopkg.in/yaml.v3/apic.go +++ b/vendor/gopkg.in/yaml.v3/apic.go @@ -108,6 +108,7 @@ func yaml_emitter_initialize(emitter *yaml_emitter_t) { raw_buffer: make([]byte, 0, output_raw_buffer_size), states: make([]yaml_emitter_state_t, 0, initial_stack_size), events: make([]yaml_event_t, 0, initial_queue_size), + best_width: -1, } } diff --git a/vendor/gopkg.in/yaml.v3/decode.go b/vendor/gopkg.in/yaml.v3/decode.go index be63169b719..21c0dacfdff 100644 --- a/vendor/gopkg.in/yaml.v3/decode.go +++ b/vendor/gopkg.in/yaml.v3/decode.go @@ -35,6 +35,7 @@ type parser struct { doc *Node anchors map[string]*Node doneInit bool + textless bool } func newParser(b []byte) *parser { @@ -108,14 +109,18 @@ func (p *parser) peek() yaml_event_type_t { func (p *parser) fail() { var where string var line int - if p.parser.problem_mark.line != 0 { + if p.parser.context_mark.line != 0 { + line = p.parser.context_mark.line + // Scanner errors don't iterate line before returning error + if p.parser.error == yaml_SCANNER_ERROR { + line++ + } + } else if p.parser.problem_mark.line != 0 { line = p.parser.problem_mark.line // Scanner errors don't iterate line before returning error if p.parser.error == yaml_SCANNER_ERROR { line++ } - } else if p.parser.context_mark.line != 0 { - line = p.parser.context_mark.line } if line != 0 { where = "line " + strconv.Itoa(line) + ": " @@ -169,17 +174,20 @@ func (p *parser) node(kind Kind, defaultTag, tag, value string) *Node { } else if kind == ScalarNode { tag, _ = resolve("", value) } - return &Node{ - Kind: kind, - Tag: tag, - Value: value, - Style: style, - Line: p.event.start_mark.line + 1, - Column: p.event.start_mark.column + 1, - HeadComment: string(p.event.head_comment), - LineComment: string(p.event.line_comment), - FootComment: string(p.event.foot_comment), + n := &Node{ + Kind: kind, + Tag: tag, + Value: value, + Style: style, + } + if !p.textless { + n.Line = p.event.start_mark.line + 1 + n.Column = p.event.start_mark.column + 1 + n.HeadComment = string(p.event.head_comment) + n.LineComment = string(p.event.line_comment) + n.FootComment = string(p.event.foot_comment) } + return n } func (p *parser) parseChild(parent *Node) *Node { @@ -391,7 +399,7 @@ func (d *decoder) callObsoleteUnmarshaler(n *Node, u obsoleteUnmarshaler) (good // // If n holds a null value, prepare returns before doing anything. func (d *decoder) prepare(n *Node, out reflect.Value) (newout reflect.Value, unmarshaled, good bool) { - if n.ShortTag() == nullTag { + if n.ShortTag() == nullTag || n.Kind == 0 && n.IsZero() { return out, false, false } again := true @@ -497,8 +505,13 @@ func (d *decoder) unmarshal(n *Node, out reflect.Value) (good bool) { good = d.mapping(n, out) case SequenceNode: good = d.sequence(n, out) + case 0: + if n.IsZero() { + return d.null(out) + } + fallthrough default: - panic("internal error: unknown node kind: " + strconv.Itoa(int(n.Kind))) + failf("cannot decode node with unknown kind %d", n.Kind) } return good } @@ -533,6 +546,17 @@ func resetMap(out reflect.Value) { } } +func (d *decoder) null(out reflect.Value) bool { + if out.CanAddr() { + switch out.Kind() { + case reflect.Interface, reflect.Ptr, reflect.Map, reflect.Slice: + out.Set(reflect.Zero(out.Type())) + return true + } + } + return false +} + func (d *decoder) scalar(n *Node, out reflect.Value) bool { var tag string var resolved interface{} @@ -550,14 +574,7 @@ func (d *decoder) scalar(n *Node, out reflect.Value) bool { } } if resolved == nil { - if out.CanAddr() { - switch out.Kind() { - case reflect.Interface, reflect.Ptr, reflect.Map, reflect.Slice: - out.Set(reflect.Zero(out.Type())) - return true - } - } - return false + return d.null(out) } if resolvedv := reflect.ValueOf(resolved); out.Type() == resolvedv.Type() { // We've resolved to exactly the type we want, so use that. diff --git a/vendor/gopkg.in/yaml.v3/emitterc.go b/vendor/gopkg.in/yaml.v3/emitterc.go index ab2a066194c..c29217ef54b 100644 --- a/vendor/gopkg.in/yaml.v3/emitterc.go +++ b/vendor/gopkg.in/yaml.v3/emitterc.go @@ -235,10 +235,13 @@ func yaml_emitter_increase_indent(emitter *yaml_emitter_t, flow, indentless bool emitter.indent = 0 } } else if !indentless { - emitter.indent += emitter.best_indent - // [Go] If inside a block sequence item, discount the space taken by the indicator. - if emitter.best_indent > 2 && emitter.states[len(emitter.states)-1] == yaml_EMIT_BLOCK_SEQUENCE_ITEM_STATE { - emitter.indent -= 2 + // [Go] This was changed so that indentations are more regular. + if emitter.states[len(emitter.states)-1] == yaml_EMIT_BLOCK_SEQUENCE_ITEM_STATE { + // The first indent inside a sequence will just skip the "- " indicator. + emitter.indent += 2 + } else { + // Everything else aligns to the chosen indentation. + emitter.indent = emitter.best_indent*((emitter.indent+emitter.best_indent)/emitter.best_indent) } } return true @@ -725,16 +728,9 @@ func yaml_emitter_emit_flow_mapping_value(emitter *yaml_emitter_t, event *yaml_e // Expect a block item node. func yaml_emitter_emit_block_sequence_item(emitter *yaml_emitter_t, event *yaml_event_t, first bool) bool { if first { - // [Go] The original logic here would not indent the sequence when inside a mapping. - // In Go we always indent it, but take the sequence indicator out of the indentation. - indentless := emitter.best_indent == 2 && emitter.mapping_context && (emitter.column == 0 || !emitter.indention) - original := emitter.indent - if !yaml_emitter_increase_indent(emitter, false, indentless) { + if !yaml_emitter_increase_indent(emitter, false, false) { return false } - if emitter.indent > original+2 { - emitter.indent -= 2 - } } if event.typ == yaml_SEQUENCE_END_EVENT { emitter.indent = emitter.indents[len(emitter.indents)-1] @@ -785,6 +781,13 @@ func yaml_emitter_emit_block_mapping_key(emitter *yaml_emitter_t, event *yaml_ev if !yaml_emitter_write_indent(emitter) { return false } + if len(emitter.line_comment) > 0 { + // [Go] A line comment was provided for the key. That's unusual as the + // scanner associates line comments with the value. Either way, + // save the line comment and render it appropriately later. + emitter.key_line_comment = emitter.line_comment + emitter.line_comment = nil + } if yaml_emitter_check_simple_key(emitter) { emitter.states = append(emitter.states, yaml_EMIT_BLOCK_MAPPING_SIMPLE_VALUE_STATE) return yaml_emitter_emit_node(emitter, event, false, false, true, true) @@ -810,6 +813,29 @@ func yaml_emitter_emit_block_mapping_value(emitter *yaml_emitter_t, event *yaml_ return false } } + if len(emitter.key_line_comment) > 0 { + // [Go] A line comment was previously provided for the key. Handle it before + // the value so the inline comments are placed correctly. + if yaml_emitter_silent_nil_event(emitter, event) && len(emitter.line_comment) == 0 { + // Nothing other than the line comment will be written on the line. + emitter.line_comment = emitter.key_line_comment + emitter.key_line_comment = nil + } else { + // An actual value is coming, so emit the comment line. + emitter.line_comment, emitter.key_line_comment = emitter.key_line_comment, emitter.line_comment + if !yaml_emitter_process_line_comment(emitter) { + return false + } + emitter.line_comment, emitter.key_line_comment = emitter.key_line_comment, emitter.line_comment + // Indent in unless it's a block that will reindent anyway. + if event.sequence_style() == yaml_FLOW_SEQUENCE_STYLE || (event.typ != yaml_MAPPING_START_EVENT && event.typ != yaml_SEQUENCE_START_EVENT) { + emitter.indent = emitter.best_indent*((emitter.indent+emitter.best_indent)/emitter.best_indent) + if !yaml_emitter_write_indent(emitter) { + return false + } + } + } + } emitter.states = append(emitter.states, yaml_EMIT_BLOCK_MAPPING_KEY_STATE) if !yaml_emitter_emit_node(emitter, event, false, false, true, false) { return false @@ -823,6 +849,10 @@ func yaml_emitter_emit_block_mapping_value(emitter *yaml_emitter_t, event *yaml_ return true } +func yaml_emitter_silent_nil_event(emitter *yaml_emitter_t, event *yaml_event_t) bool { + return event.typ == yaml_SCALAR_EVENT && event.implicit && !emitter.canonical && len(emitter.scalar_data.value) == 0 +} + // Expect a node. func yaml_emitter_emit_node(emitter *yaml_emitter_t, event *yaml_event_t, root bool, sequence bool, mapping bool, simple_key bool) bool { diff --git a/vendor/gopkg.in/yaml.v3/encode.go b/vendor/gopkg.in/yaml.v3/encode.go index eee3667eabc..45e8d1e1b9f 100644 --- a/vendor/gopkg.in/yaml.v3/encode.go +++ b/vendor/gopkg.in/yaml.v3/encode.go @@ -119,6 +119,9 @@ func (e *encoder) marshal(tag string, in reflect.Value) { case *Node: e.nodev(in) return + case Node: + e.nodev(in.Addr()) + return case time.Time: e.timev(tag, in) return @@ -298,6 +301,21 @@ func isBase60Float(s string) (result bool) { // is bogus. In practice parsers do not enforce the "\.[0-9_]*" suffix. var base60float = regexp.MustCompile(`^[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+(?:\.[0-9_]*)?$`) +// isOldBool returns whether s is bool notation as defined in YAML 1.1. +// +// We continue to force strings that YAML 1.1 would interpret as booleans to be +// rendered as quotes strings so that the marshalled output valid for YAML 1.1 +// parsing. +func isOldBool(s string) (result bool) { + switch s { + case "y", "Y", "yes", "Yes", "YES", "on", "On", "ON", + "n", "N", "no", "No", "NO", "off", "Off", "OFF": + return true + default: + return false + } +} + func (e *encoder) stringv(tag string, in reflect.Value) { var style yaml_scalar_style_t s := in.String() @@ -319,7 +337,7 @@ func (e *encoder) stringv(tag string, in reflect.Value) { // tag when encoded unquoted. If it doesn't, // there's no need to quote it. rtag, _ := resolve("", s) - canUsePlain = rtag == strTag && !isBase60Float(s) + canUsePlain = rtag == strTag && !(isBase60Float(s) || isOldBool(s)) } // Note: it's possible for user code to emit invalid YAML // if they explicitly specify a tag and a string containing @@ -407,18 +425,23 @@ func (e *encoder) nodev(in reflect.Value) { } func (e *encoder) node(node *Node, tail string) { + // Zero nodes behave as nil. + if node.Kind == 0 && node.IsZero() { + e.nilv() + return + } + // If the tag was not explicitly requested, and dropping it won't change the // implicit tag of the value, don't include it in the presentation. var tag = node.Tag var stag = shortTag(tag) - var rtag string var forceQuoting bool if tag != "" && node.Style&TaggedStyle == 0 { if node.Kind == ScalarNode { if stag == strTag && node.Style&(SingleQuotedStyle|DoubleQuotedStyle|LiteralStyle|FoldedStyle) != 0 { tag = "" } else { - rtag, _ = resolve("", node.Value) + rtag, _ := resolve("", node.Value) if rtag == stag { tag = "" } else if stag == strTag { @@ -427,6 +450,7 @@ func (e *encoder) node(node *Node, tail string) { } } } else { + var rtag string switch node.Kind { case MappingNode: rtag = mapTag @@ -456,7 +480,7 @@ func (e *encoder) node(node *Node, tail string) { if node.Style&FlowStyle != 0 { style = yaml_FLOW_SEQUENCE_STYLE } - e.must(yaml_sequence_start_event_initialize(&e.event, []byte(node.Anchor), []byte(tag), tag == "", style)) + e.must(yaml_sequence_start_event_initialize(&e.event, []byte(node.Anchor), []byte(longTag(tag)), tag == "", style)) e.event.head_comment = []byte(node.HeadComment) e.emit() for _, node := range node.Content { @@ -472,7 +496,7 @@ func (e *encoder) node(node *Node, tail string) { if node.Style&FlowStyle != 0 { style = yaml_FLOW_MAPPING_STYLE } - yaml_mapping_start_event_initialize(&e.event, []byte(node.Anchor), []byte(tag), tag == "", style) + yaml_mapping_start_event_initialize(&e.event, []byte(node.Anchor), []byte(longTag(tag)), tag == "", style) e.event.tail_comment = []byte(tail) e.event.head_comment = []byte(node.HeadComment) e.emit() @@ -513,11 +537,11 @@ func (e *encoder) node(node *Node, tail string) { case ScalarNode: value := node.Value if !utf8.ValidString(value) { - if tag == binaryTag { + if stag == binaryTag { failf("explicitly tagged !!binary data must be base64-encoded") } - if tag != "" { - failf("cannot marshal invalid UTF-8 data as %s", shortTag(tag)) + if stag != "" { + failf("cannot marshal invalid UTF-8 data as %s", stag) } // It can't be encoded directly as YAML so use a binary tag // and encode it as base64. @@ -542,5 +566,7 @@ func (e *encoder) node(node *Node, tail string) { } e.emitScalar(value, node.Anchor, tag, style, []byte(node.HeadComment), []byte(node.LineComment), []byte(node.FootComment), []byte(tail)) + default: + failf("cannot encode node with unknown kind %d", node.Kind) } } diff --git a/vendor/gopkg.in/yaml.v3/parserc.go b/vendor/gopkg.in/yaml.v3/parserc.go index ec25faabc4e..ac66fccc059 100644 --- a/vendor/gopkg.in/yaml.v3/parserc.go +++ b/vendor/gopkg.in/yaml.v3/parserc.go @@ -423,6 +423,7 @@ func yaml_parser_set_event_comments(parser *yaml_parser_t, event *yaml_event_t) parser.line_comment = nil parser.foot_comment = nil parser.tail_comment = nil + parser.stem_comment = nil } // Parse the productions: @@ -629,6 +630,10 @@ func yaml_parser_parse_node(parser *yaml_parser_t, event *yaml_event_t, block, i implicit: implicit, style: yaml_style_t(yaml_BLOCK_SEQUENCE_STYLE), } + if parser.stem_comment != nil { + event.head_comment = parser.stem_comment + parser.stem_comment = nil + } return true } if block && token.typ == yaml_BLOCK_MAPPING_START_TOKEN { @@ -643,6 +648,10 @@ func yaml_parser_parse_node(parser *yaml_parser_t, event *yaml_event_t, block, i implicit: implicit, style: yaml_style_t(yaml_BLOCK_MAPPING_STYLE), } + if parser.stem_comment != nil { + event.head_comment = parser.stem_comment + parser.stem_comment = nil + } return true } if len(anchor) > 0 || len(tag) > 0 { @@ -689,7 +698,9 @@ func yaml_parser_parse_block_sequence_entry(parser *yaml_parser_t, event *yaml_e if token.typ == yaml_BLOCK_ENTRY_TOKEN { mark := token.end_mark + prior_head_len := len(parser.head_comment) skip_token(parser) + yaml_parser_split_stem_comment(parser, prior_head_len) token = peek_token(parser) if token == nil { return false @@ -735,7 +746,9 @@ func yaml_parser_parse_indentless_sequence_entry(parser *yaml_parser_t, event *y if token.typ == yaml_BLOCK_ENTRY_TOKEN { mark := token.end_mark + prior_head_len := len(parser.head_comment) skip_token(parser) + yaml_parser_split_stem_comment(parser, prior_head_len) token = peek_token(parser) if token == nil { return false @@ -761,6 +774,32 @@ func yaml_parser_parse_indentless_sequence_entry(parser *yaml_parser_t, event *y return true } +// Split stem comment from head comment. +// +// When a sequence or map is found under a sequence entry, the former head comment +// is assigned to the underlying sequence or map as a whole, not the individual +// sequence or map entry as would be expected otherwise. To handle this case the +// previous head comment is moved aside as the stem comment. +func yaml_parser_split_stem_comment(parser *yaml_parser_t, stem_len int) { + if stem_len == 0 { + return + } + + token := peek_token(parser) + if token.typ != yaml_BLOCK_SEQUENCE_START_TOKEN && token.typ != yaml_BLOCK_MAPPING_START_TOKEN { + return + } + + parser.stem_comment = parser.head_comment[:stem_len] + if len(parser.head_comment) == stem_len { + parser.head_comment = nil + } else { + // Copy suffix to prevent very strange bugs if someone ever appends + // further bytes to the prefix in the stem_comment slice above. + parser.head_comment = append([]byte(nil), parser.head_comment[stem_len+1:]...) + } +} + // Parse the productions: // block_mapping ::= BLOCK-MAPPING_START // ******************* diff --git a/vendor/gopkg.in/yaml.v3/scannerc.go b/vendor/gopkg.in/yaml.v3/scannerc.go index e33f4959065..d9a539c39ae 100644 --- a/vendor/gopkg.in/yaml.v3/scannerc.go +++ b/vendor/gopkg.in/yaml.v3/scannerc.go @@ -657,34 +657,22 @@ func trace(args ...interface{}) func() { func yaml_parser_fetch_more_tokens(parser *yaml_parser_t) bool { // While we need more tokens to fetch, do it. for { - // Check if we really need to fetch more tokens. - need_more_tokens := false - - // [Go] The comment parsing logic requires a lookahead of one token - // in block style or two tokens in flow style so that the foot - // comments may be parsed in time of associating them with the tokens - // that are parsed before them. - if parser.tokens_head >= len(parser.tokens)-1 || parser.flow_level > 0 && parser.tokens_head >= len(parser.tokens)-2 { - need_more_tokens = true - } else { - // Check if any potential simple key may occupy the head position. - if !yaml_parser_stale_simple_keys(parser) { + // [Go] The comment parsing logic requires a lookahead of two tokens + // so that foot comments may be parsed in time of associating them + // with the tokens that are parsed before them, and also for line + // comments to be transformed into head comments in some edge cases. + if parser.tokens_head < len(parser.tokens)-2 { + // If a potential simple key is at the head position, we need to fetch + // the next token to disambiguate it. + head_tok_idx, ok := parser.simple_keys_by_tok[parser.tokens_parsed] + if !ok { + break + } else if valid, ok := yaml_simple_key_is_valid(parser, &parser.simple_keys[head_tok_idx]); !ok { return false - } - - for i := range parser.simple_keys { - simple_key := &parser.simple_keys[i] - if simple_key.possible && simple_key.token_number == parser.tokens_parsed { - need_more_tokens = true - break - } + } else if !valid { + break } } - - // We are finished. - if !need_more_tokens { - break - } // Fetch the next token. if !yaml_parser_fetch_next_token(parser) { return false @@ -714,11 +702,6 @@ func yaml_parser_fetch_next_token(parser *yaml_parser_t) (ok bool) { return false } - // Remove obsolete potential simple keys. - if !yaml_parser_stale_simple_keys(parser) { - return false - } - // [Go] While unrolling indents, transform the head comments of prior // indentation levels observed after scan_start into foot comments at // the respective indexes. @@ -766,6 +749,11 @@ func yaml_parser_fetch_next_token(parser *yaml_parser_t) (ok bool) { if !ok { return } + if len(parser.tokens) > 0 && parser.tokens[len(parser.tokens)-1].typ == yaml_BLOCK_ENTRY_TOKEN { + // Sequence indicators alone have no line comments. It becomes + // a head comment for whatever follows. + return + } if !yaml_parser_scan_line_comment(parser, comment_mark) { ok = false return @@ -892,29 +880,30 @@ func yaml_parser_fetch_next_token(parser *yaml_parser_t) (ok bool) { "found character that cannot start any token") } -// Check the list of potential simple keys and remove the positions that -// cannot contain simple keys anymore. -func yaml_parser_stale_simple_keys(parser *yaml_parser_t) bool { - // Check for a potential simple key for each flow level. - for i := range parser.simple_keys { - simple_key := &parser.simple_keys[i] +func yaml_simple_key_is_valid(parser *yaml_parser_t, simple_key *yaml_simple_key_t) (valid, ok bool) { + if !simple_key.possible { + return false, true + } - // The specification requires that a simple key - // - // - is limited to a single line, - // - is shorter than 1024 characters. - if simple_key.possible && (simple_key.mark.line < parser.mark.line || simple_key.mark.index+1024 < parser.mark.index) { - - // Check if the potential simple key to be removed is required. - if simple_key.required { - return yaml_parser_set_scanner_error(parser, - "while scanning a simple key", simple_key.mark, - "could not find expected ':'") - } - simple_key.possible = false + // The 1.2 specification says: + // + // "If the ? indicator is omitted, parsing needs to see past the + // implicit key to recognize it as such. To limit the amount of + // lookahead required, the “:” indicator must appear at most 1024 + // Unicode characters beyond the start of the key. In addition, the key + // is restricted to a single line." + // + if simple_key.mark.line < parser.mark.line || simple_key.mark.index+1024 < parser.mark.index { + // Check if the potential simple key to be removed is required. + if simple_key.required { + return false, yaml_parser_set_scanner_error(parser, + "while scanning a simple key", simple_key.mark, + "could not find expected ':'") } + simple_key.possible = false + return false, true } - return true + return true, true } // Check if a simple key may start at the current position and add it if @@ -934,13 +923,14 @@ func yaml_parser_save_simple_key(parser *yaml_parser_t) bool { possible: true, required: required, token_number: parser.tokens_parsed + (len(parser.tokens) - parser.tokens_head), + mark: parser.mark, } - simple_key.mark = parser.mark if !yaml_parser_remove_simple_key(parser) { return false } parser.simple_keys[len(parser.simple_keys)-1] = simple_key + parser.simple_keys_by_tok[simple_key.token_number] = len(parser.simple_keys) - 1 } return true } @@ -955,9 +945,10 @@ func yaml_parser_remove_simple_key(parser *yaml_parser_t) bool { "while scanning a simple key", parser.simple_keys[i].mark, "could not find expected ':'") } + // Remove the key from the stack. + parser.simple_keys[i].possible = false + delete(parser.simple_keys_by_tok, parser.simple_keys[i].token_number) } - // Remove the key from the stack. - parser.simple_keys[i].possible = false return true } @@ -967,7 +958,12 @@ const max_flow_level = 10000 // Increase the flow level and resize the simple key list if needed. func yaml_parser_increase_flow_level(parser *yaml_parser_t) bool { // Reset the simple key on the next level. - parser.simple_keys = append(parser.simple_keys, yaml_simple_key_t{}) + parser.simple_keys = append(parser.simple_keys, yaml_simple_key_t{ + possible: false, + required: false, + token_number: parser.tokens_parsed + (len(parser.tokens) - parser.tokens_head), + mark: parser.mark, + }) // Increase the flow level. parser.flow_level++ @@ -983,7 +979,9 @@ func yaml_parser_increase_flow_level(parser *yaml_parser_t) bool { func yaml_parser_decrease_flow_level(parser *yaml_parser_t) bool { if parser.flow_level > 0 { parser.flow_level-- - parser.simple_keys = parser.simple_keys[:len(parser.simple_keys)-1] + last := len(parser.simple_keys) - 1 + delete(parser.simple_keys_by_tok, parser.simple_keys[last].token_number) + parser.simple_keys = parser.simple_keys[:last] } return true } @@ -1090,6 +1088,8 @@ func yaml_parser_fetch_stream_start(parser *yaml_parser_t) bool { // Initialize the simple key stack. parser.simple_keys = append(parser.simple_keys, yaml_simple_key_t{}) + parser.simple_keys_by_tok = make(map[int]int) + // A simple key is allowed at the beginning of the stream. parser.simple_key_allowed = true @@ -1372,7 +1372,11 @@ func yaml_parser_fetch_value(parser *yaml_parser_t) bool { simple_key := &parser.simple_keys[len(parser.simple_keys)-1] // Have we found a simple key? - if simple_key.possible { + if valid, ok := yaml_simple_key_is_valid(parser, simple_key); !ok { + return false + + } else if valid { + // Create the KEY token and insert it into the queue. token := yaml_token_t{ typ: yaml_KEY_TOKEN, @@ -1390,6 +1394,7 @@ func yaml_parser_fetch_value(parser *yaml_parser_t) bool { // Remove the simple key. simple_key.possible = false + delete(parser.simple_keys_by_tok, simple_key.token_number) // A simple key cannot follow another simple key. parser.simple_key_allowed = false @@ -1557,6 +1562,29 @@ func yaml_parser_scan_to_next_token(parser *yaml_parser_t) bool { } } + // Check if we just had a line comment under a sequence entry that + // looks more like a header to the following content. Similar to this: + // + // - # The comment + // - Some data + // + // If so, transform the line comment to a head comment and reposition. + if len(parser.comments) > 0 && len(parser.tokens) > 1 { + tokenA := parser.tokens[len(parser.tokens)-2] + tokenB := parser.tokens[len(parser.tokens)-1] + comment := &parser.comments[len(parser.comments)-1] + if tokenA.typ == yaml_BLOCK_SEQUENCE_START_TOKEN && tokenB.typ == yaml_BLOCK_ENTRY_TOKEN && len(comment.line) > 0 && !is_break(parser.buffer, parser.buffer_pos) { + // If it was in the prior line, reposition so it becomes a + // header of the follow up token. Otherwise, keep it in place + // so it becomes a header of the former. + comment.head = comment.line + comment.line = nil + if comment.start_mark.line == parser.mark.line-1 { + comment.token_mark = parser.mark + } + } + } + // Eat a comment until a line break. if parser.buffer[parser.buffer_pos] == '#' { if !yaml_parser_scan_comments(parser, scan_mark) { @@ -2232,8 +2260,15 @@ func yaml_parser_scan_block_scalar(parser *yaml_parser_t, token *yaml_token_t, l } } if parser.buffer[parser.buffer_pos] == '#' { - if !yaml_parser_scan_line_comment(parser, start_mark) { - return false + // TODO Test this and then re-enable it. + //if !yaml_parser_scan_line_comment(parser, start_mark) { + // return false + //} + for !is_breakz(parser.buffer, parser.buffer_pos) { + skip(parser) + if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { + return false + } } } @@ -2802,8 +2837,8 @@ func yaml_parser_scan_line_comment(parser *yaml_parser_t, token_mark yaml_mark_t return true } - parser.comments = append(parser.comments, yaml_comment_t{token_mark: token_mark}) - comment := &parser.comments[len(parser.comments)-1].line + var start_mark yaml_mark_t + var text []byte for peek := 0; peek < 512; peek++ { if parser.unread < peek+1 && !yaml_parser_update_buffer(parser, peek+1) { @@ -2813,11 +2848,6 @@ func yaml_parser_scan_line_comment(parser *yaml_parser_t, token_mark yaml_mark_t continue } if parser.buffer[parser.buffer_pos+peek] == '#' { - if len(*comment) > 0 { - *comment = append(*comment, '\n') - } - - // Consume until after the consumed comment line. seen := parser.mark.index+peek for { if parser.unread < 1 && !yaml_parser_update_buffer(parser, 1) { @@ -2831,16 +2861,25 @@ func yaml_parser_scan_line_comment(parser *yaml_parser_t, token_mark yaml_mark_t return false } skip_line(parser) - } else { - if parser.mark.index >= seen { - *comment = append(*comment, parser.buffer[parser.buffer_pos]) + } else if parser.mark.index >= seen { + if len(text) == 0 { + start_mark = parser.mark } + text = read(parser, text) + } else { skip(parser) } } } break } + if len(text) > 0 { + parser.comments = append(parser.comments, yaml_comment_t{ + token_mark: token_mark, + start_mark: start_mark, + line: text, + }) + } return true } @@ -2964,10 +3003,9 @@ func yaml_parser_scan_comments(parser *yaml_parser_t, scan_mark yaml_mark_t) boo return false } skip_line(parser) + } else if parser.mark.index >= seen { + text = read(parser, text) } else { - if parser.mark.index >= seen { - text = append(text, parser.buffer[parser.buffer_pos]) - } skip(parser) } } diff --git a/vendor/gopkg.in/yaml.v3/yaml.go b/vendor/gopkg.in/yaml.v3/yaml.go index b5d35a50ded..56e8a849031 100644 --- a/vendor/gopkg.in/yaml.v3/yaml.go +++ b/vendor/gopkg.in/yaml.v3/yaml.go @@ -89,7 +89,7 @@ func Unmarshal(in []byte, out interface{}) (err error) { return unmarshal(in, out, false) } -// A Decorder reads and decodes YAML values from an input stream. +// A Decoder reads and decodes YAML values from an input stream. type Decoder struct { parser *parser knownFields bool @@ -194,7 +194,7 @@ func unmarshal(in []byte, out interface{}, strict bool) (err error) { // Zero valued structs will be omitted if all their public // fields are zero, unless they implement an IsZero // method (see the IsZeroer interface type), in which -// case the field will be included if that method returns true. +// case the field will be excluded if IsZero returns true. // // flow Marshal using a flow style (useful for structs, // sequences and maps). @@ -252,6 +252,24 @@ func (e *Encoder) Encode(v interface{}) (err error) { return nil } +// Encode encodes value v and stores its representation in n. +// +// See the documentation for Marshal for details about the +// conversion of Go values into YAML. +func (n *Node) Encode(v interface{}) (err error) { + defer handleErr(&err) + e := newEncoder() + defer e.destroy() + e.marshalDoc("", reflect.ValueOf(v)) + e.finish() + p := newParser(e.out) + p.textless = true + defer p.destroy() + doc := p.parse() + *n = *doc.Content[0] + return nil +} + // SetIndent changes the used indentation used when encoding. func (e *Encoder) SetIndent(spaces int) { if spaces < 0 { @@ -328,6 +346,12 @@ const ( // and maps, Node is an intermediate representation that allows detailed // control over the content being decoded or encoded. // +// It's worth noting that although Node offers access into details such as +// line numbers, colums, and comments, the content when re-encoded will not +// have its original textual representation preserved. An effort is made to +// render the data plesantly, and to preserve comments near the data they +// describe, though. +// // Values that make use of the Node type interact with the yaml package in the // same way any other type would do, by encoding and decoding yaml data // directly or indirectly into them. @@ -391,6 +415,13 @@ type Node struct { Column int } +// IsZero returns whether the node has all of its fields unset. +func (n *Node) IsZero() bool { + return n.Kind == 0 && n.Style == 0 && n.Tag == "" && n.Value == "" && n.Anchor == "" && n.Alias == nil && n.Content == nil && + n.HeadComment == "" && n.LineComment == "" && n.FootComment == "" && n.Line == 0 && n.Column == 0 +} + + // LongTag returns the long form of the tag that indicates the data type for // the node. If the Tag field isn't explicitly defined, one will be computed // based on the node properties. diff --git a/vendor/gopkg.in/yaml.v3/yamlh.go b/vendor/gopkg.in/yaml.v3/yamlh.go index 65fb0df3b17..7c6d0077061 100644 --- a/vendor/gopkg.in/yaml.v3/yamlh.go +++ b/vendor/gopkg.in/yaml.v3/yamlh.go @@ -600,6 +600,7 @@ type yaml_parser_t struct { line_comment []byte // The current line comments foot_comment []byte // The current foot comments tail_comment []byte // Foot comment that happens at the end of a block. + stem_comment []byte // Comment in item preceding a nested structure (list inside list item, etc) comments []yaml_comment_t // The folded comments for all parsed tokens comments_head int @@ -621,6 +622,7 @@ type yaml_parser_t struct { simple_key_allowed bool // May a simple key occur at the current position? simple_keys []yaml_simple_key_t // The stack of simple keys. + simple_keys_by_tok map[int]int // possible simple_key indexes indexed by token_number // Parser stuff @@ -785,6 +787,8 @@ type yaml_emitter_t struct { foot_comment []byte tail_comment []byte + key_line_comment []byte + // Dumper stuff opened bool // If the stream was already opened? diff --git a/vendor/modules.txt b/vendor/modules.txt index 8896ef3df7c..3eeee508b8d 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -254,6 +254,7 @@ github.com/prometheus/client_golang/prometheus/promhttp # github.com/prometheus/client_model v0.2.0 github.com/prometheus/client_model/go # github.com/prometheus/common v0.9.1 +## explicit github.com/prometheus/common/expfmt github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg github.com/prometheus/common/log @@ -573,7 +574,7 @@ gopkg.in/inf.v0 # gopkg.in/yaml.v2 v2.3.0 ## explicit gopkg.in/yaml.v2 -# gopkg.in/yaml.v3 v3.0.0-20191026110619-0b21df46bc1d +# gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 ## explicit gopkg.in/yaml.v3 # honnef.co/go/tools v0.0.1-2020.1.4 From 73d18dba931d9b172988e2fdf8fb5a149855e77c Mon Sep 17 00:00:00 2001 From: slinkydeveloper Date: Mon, 5 Oct 2020 15:39:33 +0200 Subject: [PATCH 05/23] Moved resources of recordevents in proper module Signed-off-by: Francesco Guardiani --- test/lib/recordevents/event_info_store.go | 2 +- test/lib/recordevents/resources.go | 43 +++++++++++++++++++++++ test/lib/resources/kube.go | 33 ----------------- 3 files changed, 44 insertions(+), 34 deletions(-) create mode 100644 test/lib/recordevents/resources.go diff --git a/test/lib/recordevents/event_info_store.go b/test/lib/recordevents/event_info_store.go index 6e721cd8956..a72d9c6574f 100644 --- a/test/lib/recordevents/event_info_store.go +++ b/test/lib/recordevents/event_info_store.go @@ -60,7 +60,7 @@ func DeployEventRecordOrFail(ctx context.Context, client *testlib.Client, name s )) client.CreateRoleBindingOrFail(name, "Role", name, name, client.Namespace) - eventRecordPod := resources.EventRecordPod(name, client.Namespace, name) + eventRecordPod := EventRecordPod(name, client.Namespace, name) client.CreatePodOrFail(eventRecordPod, append(options, testlib.WithService(name))...) err := pkgTest.WaitForPodRunning(ctx, client.Kube, name, client.Namespace) if err != nil { diff --git a/test/lib/recordevents/resources.go b/test/lib/recordevents/resources.go new file mode 100644 index 00000000000..22f1281edbf --- /dev/null +++ b/test/lib/recordevents/resources.go @@ -0,0 +1,43 @@ +package recordevents + +import ( + "fmt" + + "k8s.io/api/core/v1" + v12 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/uuid" + "knative.dev/pkg/test" +) + +// EventRecordPod creates a Pod that stores received events for test retrieval. +func EventRecordPod(name string, namespace string, serviceAccountName string) *v1.Pod { + return recordEventsPod("recordevents", name, namespace, serviceAccountName) +} + +func recordEventsPod(imageName string, name string, namespace string, serviceAccountName string) *v1.Pod { + return &v1.Pod{ + ObjectMeta: v12.ObjectMeta{ + Name: name, + Labels: map[string]string{"e2etest": string(uuid.NewUUID())}, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{{ + Name: imageName, + Image: test.ImagePath(imageName), + ImagePullPolicy: v1.PullAlways, + Env: []v1.EnvVar{{ + Name: "SYSTEM_NAMESPACE", + Value: namespace, + }, { + Name: "OBSERVER", + Value: "recorder-" + name, + }, { + Name: "K8S_EVENT_SINK", + Value: fmt.Sprintf("{\"apiVersion\": \"v1\", \"kind\": \"Pod\", \"name\": \"%s\", \"namespace\": \"%s\"}", name, namespace), + }}, + }}, + ServiceAccountName: serviceAccountName, + RestartPolicy: v1.RestartPolicyAlways, + }, + } +} diff --git a/test/lib/resources/kube.go b/test/lib/resources/kube.go index 7b43881c752..bc415d10873 100644 --- a/test/lib/resources/kube.go +++ b/test/lib/resources/kube.go @@ -37,39 +37,6 @@ type PodOption func(*corev1.Pod) // Option enables further configuration of a Role. type RoleOption func(*rbacv1.Role) -// EventRecordPod creates a Pod that stores received events for test retrieval. -func EventRecordPod(name string, namespace string, serviceAccountName string) *corev1.Pod { - return recordEventsPod("recordevents", name, namespace, serviceAccountName) -} - -func recordEventsPod(imageName string, name string, namespace string, serviceAccountName string) *corev1.Pod { - return &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Labels: map[string]string{"e2etest": string(uuid.NewUUID())}, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{{ - Name: imageName, - Image: pkgTest.ImagePath(imageName), - ImagePullPolicy: corev1.PullAlways, - Env: []corev1.EnvVar{{ - Name: "SYSTEM_NAMESPACE", - Value: namespace, - }, { - Name: "OBSERVER", - Value: "recorder-" + name, - }, { - Name: "K8S_EVENT_SINK", - Value: fmt.Sprintf("{\"apiVersion\": \"v1\", \"kind\": \"Pod\", \"name\": \"%s\", \"namespace\": \"%s\"}", name, namespace), - }}, - }}, - ServiceAccountName: serviceAccountName, - RestartPolicy: corev1.RestartPolicyAlways, - }, - } -} - // EventTransformationPod creates a Pod that transforms events received receiving as arg a cloudevents sdk2 Event func EventTransformationPod(name string, newEventType string, newEventSource string, newEventData []byte) *corev1.Pod { const imageName = "transformevents" From e6eb27daf78e86172d17a3f07c44dabcf7f60ddb Mon Sep 17 00:00:00 2001 From: slinkydeveloper Date: Mon, 5 Oct 2020 15:45:06 +0200 Subject: [PATCH 06/23] gofmt Signed-off-by: Francesco Guardiani --- .../helpers/channel_status_subscriber_test_helper.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/conformance/helpers/channel_status_subscriber_test_helper.go b/test/conformance/helpers/channel_status_subscriber_test_helper.go index f27b8525098..4a0c2e79834 100644 --- a/test/conformance/helpers/channel_status_subscriber_test_helper.go +++ b/test/conformance/helpers/channel_status_subscriber_test_helper.go @@ -60,7 +60,7 @@ func channelHasRequiredSubscriberStatus(ctx context.Context, st *testing.T, clie client.CreateChannelOrFail(channelName, &channel) client.WaitForResourceReadyOrFail(channelName, &channel) - _ = recordevents.DeployEventRecordOrFail(context.TODO(), client, subscriberServiceName + "-pod") + _ = recordevents.DeployEventRecordOrFail(context.TODO(), client, subscriberServiceName+"-pod") subscription := client.CreateSubscriptionOrFail( subscriberServiceName, From bcbd28b014625bdc69fdfdfc8f75e5763a286b45 Mon Sep 17 00:00:00 2001 From: slinkydeveloper Date: Mon, 5 Oct 2020 15:48:33 +0200 Subject: [PATCH 07/23] Another one Signed-off-by: Francesco Guardiani --- test/lib/recordevents/resources.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/lib/recordevents/resources.go b/test/lib/recordevents/resources.go index 22f1281edbf..963fa732f41 100644 --- a/test/lib/recordevents/resources.go +++ b/test/lib/recordevents/resources.go @@ -4,7 +4,7 @@ import ( "fmt" "k8s.io/api/core/v1" - v12 "k8s.io/apimachinery/pkg/apis/meta/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/uuid" "knative.dev/pkg/test" ) @@ -16,7 +16,7 @@ func EventRecordPod(name string, namespace string, serviceAccountName string) *v func recordEventsPod(imageName string, name string, namespace string, serviceAccountName string) *v1.Pod { return &v1.Pod{ - ObjectMeta: v12.ObjectMeta{ + ObjectMeta: metav1.ObjectMeta{ Name: name, Labels: map[string]string{"e2etest": string(uuid.NewUUID())}, }, From 4a589d36eb0742710a8b90c7a26067d3c59f357f Mon Sep 17 00:00:00 2001 From: slinkydeveloper Date: Mon, 5 Oct 2020 15:49:31 +0200 Subject: [PATCH 08/23] Other stuff Signed-off-by: Francesco Guardiani --- test/lib/recordevents/resources.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/test/lib/recordevents/resources.go b/test/lib/recordevents/resources.go index 963fa732f41..8ddcb73ce23 100644 --- a/test/lib/recordevents/resources.go +++ b/test/lib/recordevents/resources.go @@ -3,29 +3,29 @@ package recordevents import ( "fmt" - "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/uuid" "knative.dev/pkg/test" ) // EventRecordPod creates a Pod that stores received events for test retrieval. -func EventRecordPod(name string, namespace string, serviceAccountName string) *v1.Pod { +func EventRecordPod(name string, namespace string, serviceAccountName string) *corev1.Pod { return recordEventsPod("recordevents", name, namespace, serviceAccountName) } -func recordEventsPod(imageName string, name string, namespace string, serviceAccountName string) *v1.Pod { - return &v1.Pod{ +func recordEventsPod(imageName string, name string, namespace string, serviceAccountName string) *corev1.Pod { + return &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: name, Labels: map[string]string{"e2etest": string(uuid.NewUUID())}, }, - Spec: v1.PodSpec{ - Containers: []v1.Container{{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ Name: imageName, Image: test.ImagePath(imageName), - ImagePullPolicy: v1.PullAlways, - Env: []v1.EnvVar{{ + ImagePullPolicy: corev1.PullAlways, + Env: []corev1.EnvVar{{ Name: "SYSTEM_NAMESPACE", Value: namespace, }, { @@ -33,11 +33,11 @@ func recordEventsPod(imageName string, name string, namespace string, serviceAcc Value: "recorder-" + name, }, { Name: "K8S_EVENT_SINK", - Value: fmt.Sprintf("{\"apiVersion\": \"v1\", \"kind\": \"Pod\", \"name\": \"%s\", \"namespace\": \"%s\"}", name, namespace), + Value: fmt.Sprintf("{\"apiVersion\": \"corev1\", \"kind\": \"Pod\", \"name\": \"%s\", \"namespace\": \"%s\"}", name, namespace), }}, }}, ServiceAccountName: serviceAccountName, - RestartPolicy: v1.RestartPolicyAlways, + RestartPolicy: corev1.RestartPolicyAlways, }, } } From 65ac13b7b2d839dc65f42f96ef51df625276d521 Mon Sep 17 00:00:00 2001 From: slinkydeveloper Date: Mon, 5 Oct 2020 15:54:54 +0200 Subject: [PATCH 09/23] Copyrights Signed-off-by: Francesco Guardiani --- test/lib/recordevents/observer/observer.go | 16 ++++++++++++++++ test/lib/recordevents/resources.go | 16 ++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/test/lib/recordevents/observer/observer.go b/test/lib/recordevents/observer/observer.go index 312ef07596e..e742241e442 100644 --- a/test/lib/recordevents/observer/observer.go +++ b/test/lib/recordevents/observer/observer.go @@ -1,3 +1,19 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package observer import ( diff --git a/test/lib/recordevents/resources.go b/test/lib/recordevents/resources.go index 8ddcb73ce23..52ae5bbd3ed 100644 --- a/test/lib/recordevents/resources.go +++ b/test/lib/recordevents/resources.go @@ -1,3 +1,19 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package recordevents import ( From 559bd6b30537b0a3f1d0e212802c1c87fb04c881 Mon Sep 17 00:00:00 2001 From: slinkydeveloper Date: Mon, 5 Oct 2020 15:56:36 +0200 Subject: [PATCH 10/23] Other copyrights Signed-off-by: Francesco Guardiani --- test/lib/k8s_events.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test/lib/k8s_events.go b/test/lib/k8s_events.go index 296a2ef2b80..b2871877397 100644 --- a/test/lib/k8s_events.go +++ b/test/lib/k8s_events.go @@ -1,3 +1,18 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ package lib import ( From b5107ebb98415a01140dc02a709686559eae35ef Mon Sep 17 00:00:00 2001 From: slinkydeveloper Date: Mon, 5 Oct 2020 16:03:19 +0200 Subject: [PATCH 11/23] boilerplate check Signed-off-by: Francesco Guardiani --- test/lib/k8s_events.go | 1 + 1 file changed, 1 insertion(+) diff --git a/test/lib/k8s_events.go b/test/lib/k8s_events.go index b2871877397..5cde94cede1 100644 --- a/test/lib/k8s_events.go +++ b/test/lib/k8s_events.go @@ -13,6 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ + package lib import ( From 9699197d7101e6a956dca76b90487340366c7dcd Mon Sep 17 00:00:00 2001 From: slinkydeveloper Date: Mon, 5 Oct 2020 16:07:00 +0200 Subject: [PATCH 12/23] boilerplate check Signed-off-by: Francesco Guardiani --- test/lib/k8s_events.go | 2 +- test/lib/recordevents/observer/observer.go | 2 +- test/lib/recordevents/resources.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/lib/k8s_events.go b/test/lib/k8s_events.go index 5cde94cede1..1188d5c234f 100644 --- a/test/lib/k8s_events.go +++ b/test/lib/k8s_events.go @@ -5,7 +5,7 @@ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - https://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, diff --git a/test/lib/recordevents/observer/observer.go b/test/lib/recordevents/observer/observer.go index e742241e442..a561a693175 100644 --- a/test/lib/recordevents/observer/observer.go +++ b/test/lib/recordevents/observer/observer.go @@ -5,7 +5,7 @@ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - https://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, diff --git a/test/lib/recordevents/resources.go b/test/lib/recordevents/resources.go index 52ae5bbd3ed..9dba1233a1a 100644 --- a/test/lib/recordevents/resources.go +++ b/test/lib/recordevents/resources.go @@ -5,7 +5,7 @@ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - https://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, From 6aed65bf328b3c5a3c854c359dbaa1464a43ad8c Mon Sep 17 00:00:00 2001 From: slinkydeveloper Date: Mon, 5 Oct 2020 16:23:12 +0200 Subject: [PATCH 13/23] Other lints checks Signed-off-by: Francesco Guardiani --- test/lib/recordevents/observer/observer.go | 2 +- test/lib/recordevents/recorder_vent/constructor.go | 7 ++++--- test/test_images/recordevents/main.go | 6 +++--- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/test/lib/recordevents/observer/observer.go b/test/lib/recordevents/observer/observer.go index a561a693175..1218cbcd5fa 100644 --- a/test/lib/recordevents/observer/observer.go +++ b/test/lib/recordevents/observer/observer.go @@ -54,7 +54,7 @@ type envConfig struct { func NewFromEnv(eventLogs ...recordevents.EventLog) *Observer { var env envConfig if err := envconfig.Process("", &env); err != nil { - log.Fatalf("[ERROR] Failed to process env var: %s", err) + log.Fatal("Failed to process env var", err) } return New(env.ObserverName, eventLogs...) diff --git a/test/lib/recordevents/recorder_vent/constructor.go b/test/lib/recordevents/recorder_vent/constructor.go index 302a1e27863..d061b64b8d4 100644 --- a/test/lib/recordevents/recorder_vent/constructor.go +++ b/test/lib/recordevents/recorder_vent/constructor.go @@ -22,6 +22,7 @@ import ( "log" "github.com/kelseyhightower/envconfig" + "go.uber.org/zap" "knative.dev/pkg/system" @@ -55,12 +56,12 @@ type envConfig struct { func NewFromEnv(ctx context.Context) recordevents.EventLog { var env envConfig if err := envconfig.Process("", &env); err != nil { - log.Fatalf("[ERROR] Failed to process env var: %s", err) + log.Fatal("Failed to process env var", err) } var ref duckv1.KReference if err := json.Unmarshal([]byte(env.EventOn), &ref); err != nil { - log.Fatalf("[ERROR] Failed to process env var [K8S_EVENT_SINK]: %s", err) + log.Fatal("Failed to process env var [K8S_EVENT_SINK]", err) } return NewEventLog(ctx, env.AgentName, ref) @@ -70,7 +71,7 @@ func NewEventLog(ctx context.Context, agentName string, ref duckv1.KReference) r gv, err := schema.ParseGroupVersion(ref.APIVersion) if err != nil { - logging.FromContext(ctx).Fatalf("failed to parse group version, %s", err) + logging.FromContext(ctx).Fatalw("Failed to parse group version", zap.Error(err)) } logging.FromContext(ctx).Infof("Going to send events to %+v", ref) diff --git a/test/test_images/recordevents/main.go b/test/test_images/recordevents/main.go index 055a645ce83..c0a953ee5c6 100644 --- a/test/test_images/recordevents/main.go +++ b/test/test_images/recordevents/main.go @@ -36,13 +36,13 @@ import ( func main() { cfg, err := rest.InClusterConfig() if err != nil { - log.Fatalf("Error while reading the cfg: %s", err) + log.Fatal("Error while reading the cfg", err) } ctx := sharedmain.EnableInjectionOrDie(nil, cfg) logger, _ := zap.NewDevelopment() if err := test_images.ConfigureTracing(logger.Sugar(), ""); err != nil { - log.Fatalf("Unable to setup trace publishing: %v", err) + log.Fatal("Unable to setup trace publishing", err) } obs := observer.NewFromEnv( @@ -70,6 +70,6 @@ func main() { } if err != nil { - log.Fatalf("Error during start: %s", err) + log.Fatal("Error during start", err) } } From 9f80ef2bf1d7afa2fe00daa2267dc937dbd8e370 Mon Sep 17 00:00:00 2001 From: slinkydeveloper Date: Mon, 5 Oct 2020 16:31:31 +0200 Subject: [PATCH 14/23] Warn Signed-off-by: Francesco Guardiani --- test/lib/recordevents/observer/observer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/lib/recordevents/observer/observer.go b/test/lib/recordevents/observer/observer.go index 1218cbcd5fa..ca6111800da 100644 --- a/test/lib/recordevents/observer/observer.go +++ b/test/lib/recordevents/observer/observer.go @@ -93,7 +93,7 @@ func (o *Observer) ServeHTTP(writer http.ResponseWriter, request *http.Request) Time: time.Now(), }) if err != nil { - log.Warn("Error while venting the recorded event %s", err) + log.Warn("Error while venting the recorded event", err) } } From 7451b7566acdddb904f58fe28575e8814c091dd0 Mon Sep 17 00:00:00 2001 From: slinkydeveloper Date: Mon, 5 Oct 2020 16:47:38 +0200 Subject: [PATCH 15/23] Now the linter Signed-off-by: Francesco Guardiani --- test/test_images/recordevents/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_images/recordevents/main.go b/test/test_images/recordevents/main.go index c0a953ee5c6..329e9e9f87c 100644 --- a/test/test_images/recordevents/main.go +++ b/test/test_images/recordevents/main.go @@ -38,6 +38,7 @@ func main() { if err != nil { log.Fatal("Error while reading the cfg", err) } + //nolint // nil ctx is fine here, look at the code of EnableInjectionOrDie ctx := sharedmain.EnableInjectionOrDie(nil, cfg) logger, _ := zap.NewDevelopment() From ab503eb35a34857787c996297fa2ff6632a0d7b5 Mon Sep 17 00:00:00 2001 From: slinkydeveloper Date: Mon, 5 Oct 2020 17:32:06 +0200 Subject: [PATCH 16/23] Missing decorator in recordevents Signed-off-by: Francesco Guardiani --- test/test_images/recordevents/main.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/test_images/recordevents/main.go b/test/test_images/recordevents/main.go index 329e9e9f87c..561b3befd0f 100644 --- a/test/test_images/recordevents/main.go +++ b/test/test_images/recordevents/main.go @@ -26,6 +26,7 @@ import ( "knative.dev/pkg/injection/sharedmain" _ "knative.dev/pkg/system/testing" + "knative.dev/eventing/pkg/kncloudevents" "knative.dev/eventing/test/lib/dropevents" "knative.dev/eventing/test/lib/recordevents/observer" "knative.dev/eventing/test/lib/recordevents/recorder_vent" @@ -57,7 +58,7 @@ func main() { counter := dropevents.CounterHandler{ Skipper: skipper, } - err = obs.Start(ctx, func(handler http.Handler) http.Handler { + err = obs.Start(ctx, kncloudevents.CreateHandler, func(handler http.Handler) http.Handler { return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { if counter.Skip() { writer.WriteHeader(http.StatusConflict) @@ -67,7 +68,7 @@ func main() { }) }) } else { - err = obs.Start(ctx) + err = obs.Start(ctx, kncloudevents.CreateHandler) } if err != nil { From 4fc111a0e79f62bfe67d34b181c7c8a78afc86b0 Mon Sep 17 00:00:00 2001 From: slinkydeveloper Date: Tue, 6 Oct 2020 09:53:44 +0200 Subject: [PATCH 17/23] Simplified the sink definition, just sending events to myself Signed-off-by: Francesco Guardiani --- test/lib/recordevents/event_info.go | 12 +++--- test/lib/recordevents/event_info_store.go | 4 +- .../recordevents/recorder_vent/constructor.go | 42 ++++--------------- test/lib/recordevents/resources.go | 20 +++++---- 4 files changed, 27 insertions(+), 51 deletions(-) diff --git a/test/lib/recordevents/event_info.go b/test/lib/recordevents/event_info.go index fd77a362568..35490108f73 100644 --- a/test/lib/recordevents/event_info.go +++ b/test/lib/recordevents/event_info.go @@ -33,14 +33,14 @@ const ( type EventInfo struct { // Set if the http request received by the pod couldn't be decoded or // didn't pass validation - Error string `json:"error"` + Error string `json:"error,omitempty"` // Event received if the cloudevent received by the pod passed validation - Event *cloudevents.Event `json:"event"` + Event *cloudevents.Event `json:"event,omitempty"` // HTTPHeaders of the connection that delivered the event - HTTPHeaders map[string][]string `json:"httpHeaders"` - Origin string `json:"origin"` - Observer string `json:"observer"` - Time time.Time `json:"time"` + HTTPHeaders map[string][]string `json:"httpHeaders,omitempty"` + Origin string `json:"origin,omitempty"` + Observer string `json:"observer,omitempty"` + Time time.Time `json:"time,omitempty"` } // Pretty print the event. Meant for debugging. This formats the validation error diff --git a/test/lib/recordevents/event_info_store.go b/test/lib/recordevents/event_info_store.go index a72d9c6574f..ba80a19c255 100644 --- a/test/lib/recordevents/event_info_store.go +++ b/test/lib/recordevents/event_info_store.go @@ -50,7 +50,7 @@ func DeployEventRecordOrFail(ctx context.Context, client *testlib.Client, name s resources.WithRuleForRole(&rbacv1.PolicyRule{ APIGroups: []string{""}, Resources: []string{"pods"}, - Verbs: []string{"get", "list", "watch"}, + Verbs: []string{"get"}, }), resources.WithRuleForRole(&rbacv1.PolicyRule{ APIGroups: []string{""}, @@ -60,7 +60,7 @@ func DeployEventRecordOrFail(ctx context.Context, client *testlib.Client, name s )) client.CreateRoleBindingOrFail(name, "Role", name, name, client.Namespace) - eventRecordPod := EventRecordPod(name, client.Namespace, name) + eventRecordPod := EventRecordPod(name, name) client.CreatePodOrFail(eventRecordPod, append(options, testlib.WithService(name))...) err := pkgTest.WaitForPodRunning(ctx, client.Kube, name, client.Namespace) if err != nil { diff --git a/test/lib/recordevents/recorder_vent/constructor.go b/test/lib/recordevents/recorder_vent/constructor.go index d061b64b8d4..3777762f7cb 100644 --- a/test/lib/recordevents/recorder_vent/constructor.go +++ b/test/lib/recordevents/recorder_vent/constructor.go @@ -18,28 +18,20 @@ package recorder_vent import ( "context" - "encoding/json" "log" "github.com/kelseyhightower/envconfig" - "go.uber.org/zap" - "knative.dev/pkg/system" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/kubernetes/scheme" v1 "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/tools/record" - duckv1 "knative.dev/pkg/apis/duck/v1" kubeclient "knative.dev/pkg/client/injection/kube/client" "knative.dev/pkg/controller" - "knative.dev/pkg/injection/clients/dynamicclient" "knative.dev/pkg/logging" "knative.dev/eventing/test/lib/recordevents" @@ -47,10 +39,8 @@ import ( type envConfig struct { AgentName string `envconfig:"AGENT_NAME" default:"observer-default" required:"true"` - EventOn string `envconfig:"K8S_EVENT_SINK" required:"true"` - - Port int `envconfig:"PORT" default:"8080" required:"true"` - Sink string `envconfig:"K_SINK"` + PodName string `envconfig:"POD_NAME" required:"true"` + Port int `envconfig:"PORT" default:"8080" required:"true"` } func NewFromEnv(ctx context.Context) recordevents.EventLog { @@ -59,34 +49,18 @@ func NewFromEnv(ctx context.Context) recordevents.EventLog { log.Fatal("Failed to process env var", err) } - var ref duckv1.KReference - if err := json.Unmarshal([]byte(env.EventOn), &ref); err != nil { - log.Fatal("Failed to process env var [K8S_EVENT_SINK]", err) - } + logging.FromContext(ctx).Infof("Environment configuration: %+v", env) - return NewEventLog(ctx, env.AgentName, ref) + return NewEventLog(ctx, env.AgentName, env.PodName) } -func NewEventLog(ctx context.Context, agentName string, ref duckv1.KReference) recordevents.EventLog { - - gv, err := schema.ParseGroupVersion(ref.APIVersion) +func NewEventLog(ctx context.Context, agentName string, podName string) recordevents.EventLog { + on, err := kubeclient.Get(ctx).CoreV1().Pods(system.Namespace()).Get(ctx, podName, metav1.GetOptions{}) if err != nil { - logging.FromContext(ctx).Fatalw("Failed to parse group version", zap.Error(err)) + logging.FromContext(ctx).Fatal("Error while trying to retrieve the pod", err) } - logging.FromContext(ctx).Infof("Going to send events to %+v", ref) - - gvr, _ := meta.UnsafeGuessKindToResource(gv.WithKind(ref.Kind)) - - var on runtime.Object - if ref.Namespace == "" { - on, err = dynamicclient.Get(ctx).Resource(gvr).Get(ctx, ref.Name, metav1.GetOptions{}) - } else { - on, err = dynamicclient.Get(ctx).Resource(gvr).Namespace(ref.Namespace).Get(ctx, ref.Name, metav1.GetOptions{}) - } - if err != nil { - logging.FromContext(ctx).Fatalf("failed to fetch object ref, %+v, %s", ref, err) - } + logging.FromContext(ctx).Infof("Going to send events to %s", on.ObjectMeta.String()) return &recorder{out: createRecorder(ctx, agentName), on: on} } diff --git a/test/lib/recordevents/resources.go b/test/lib/recordevents/resources.go index 9dba1233a1a..54726bedfdc 100644 --- a/test/lib/recordevents/resources.go +++ b/test/lib/recordevents/resources.go @@ -17,8 +17,6 @@ limitations under the License. package recordevents import ( - "fmt" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/uuid" @@ -26,11 +24,11 @@ import ( ) // EventRecordPod creates a Pod that stores received events for test retrieval. -func EventRecordPod(name string, namespace string, serviceAccountName string) *corev1.Pod { - return recordEventsPod("recordevents", name, namespace, serviceAccountName) +func EventRecordPod(name string, serviceAccountName string) *corev1.Pod { + return recordEventsPod("recordevents", name, serviceAccountName) } -func recordEventsPod(imageName string, name string, namespace string, serviceAccountName string) *corev1.Pod { +func recordEventsPod(imageName string, name string, serviceAccountName string) *corev1.Pod { return &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -42,14 +40,18 @@ func recordEventsPod(imageName string, name string, namespace string, serviceAcc Image: test.ImagePath(imageName), ImagePullPolicy: corev1.PullAlways, Env: []corev1.EnvVar{{ - Name: "SYSTEM_NAMESPACE", - Value: namespace, + Name: "SYSTEM_NAMESPACE", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}, + }, }, { Name: "OBSERVER", Value: "recorder-" + name, }, { - Name: "K8S_EVENT_SINK", - Value: fmt.Sprintf("{\"apiVersion\": \"corev1\", \"kind\": \"Pod\", \"name\": \"%s\", \"namespace\": \"%s\"}", name, namespace), + Name: "POD_NAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}, + }, }}, }}, ServiceAccountName: serviceAccountName, From 9a2cb2763e3cb454488a5ff9f6068353751cb0c2 Mon Sep 17 00:00:00 2001 From: slinkydeveloper Date: Tue, 6 Oct 2020 09:58:56 +0200 Subject: [PATCH 18/23] Happy linter, happy me Signed-off-by: Francesco Guardiani --- test/lib/recordevents/recorder_vent/constructor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/lib/recordevents/recorder_vent/constructor.go b/test/lib/recordevents/recorder_vent/constructor.go index 3777762f7cb..67275dad2eb 100644 --- a/test/lib/recordevents/recorder_vent/constructor.go +++ b/test/lib/recordevents/recorder_vent/constructor.go @@ -60,7 +60,7 @@ func NewEventLog(ctx context.Context, agentName string, podName string) recordev logging.FromContext(ctx).Fatal("Error while trying to retrieve the pod", err) } - logging.FromContext(ctx).Infof("Going to send events to %s", on.ObjectMeta.String()) + logging.FromContext(ctx).Infof("Going to send events to ", on.ObjectMeta.String()) return &recorder{out: createRecorder(ctx, agentName), on: on} } From 1196010d68a052311399b527d4ac0ea65a206f91 Mon Sep 17 00:00:00 2001 From: slinkydeveloper Date: Tue, 6 Oct 2020 11:44:17 +0200 Subject: [PATCH 19/23] Correct HTTP server shutdown Signed-off-by: Francesco Guardiani --- test/lib/recordevents/observer/observer.go | 17 ++++++- .../recordevents/recorder_vent/constructor.go | 3 +- test/lib/recordevents/writer_vent/doc.go | 19 ------- test/lib/recordevents/writer_vent/writer.go | 50 ------------------- test/test_images/recordevents/main.go | 13 +++-- 5 files changed, 23 insertions(+), 79 deletions(-) delete mode 100644 test/lib/recordevents/writer_vent/doc.go delete mode 100644 test/lib/recordevents/writer_vent/writer.go diff --git a/test/lib/recordevents/observer/observer.go b/test/lib/recordevents/observer/observer.go index ca6111800da..d124959955b 100644 --- a/test/lib/recordevents/observer/observer.go +++ b/test/lib/recordevents/observer/observer.go @@ -25,6 +25,7 @@ import ( cloudeventshttp "github.com/cloudevents/sdk-go/v2/protocol/http" "github.com/kelseyhightower/envconfig" "github.com/prometheus/common/log" + "knative.dev/pkg/logging" "knative.dev/eventing/test/lib/recordevents" ) @@ -69,7 +70,19 @@ func (o *Observer) Start(ctx context.Context, handlerFuncs ...func(handler http. handler = dec(handler) } - return http.ListenAndServe(":8080", handler) + server := &http.Server{Addr: ":8080", Handler: handler} + + go func() { + if err := server.ListenAndServe(); err != nil { + logging.FromContext(ctx).Fatal("Error while starting the HTTP server", err) + } + }() + + <-ctx.Done() + + logging.FromContext(ctx).Info("Closing the HTTP server") + + return server.Shutdown(ctx) } func (o *Observer) ServeHTTP(writer http.ResponseWriter, request *http.Request) { @@ -89,7 +102,7 @@ func (o *Observer) ServeHTTP(writer http.ResponseWriter, request *http.Request) Event: event, HTTPHeaders: header, Origin: request.RemoteAddr, - Observer: "", + Observer: o.Name, Time: time.Now(), }) if err != nil { diff --git a/test/lib/recordevents/recorder_vent/constructor.go b/test/lib/recordevents/recorder_vent/constructor.go index 67275dad2eb..99cdae09c11 100644 --- a/test/lib/recordevents/recorder_vent/constructor.go +++ b/test/lib/recordevents/recorder_vent/constructor.go @@ -60,7 +60,7 @@ func NewEventLog(ctx context.Context, agentName string, podName string) recordev logging.FromContext(ctx).Fatal("Error while trying to retrieve the pod", err) } - logging.FromContext(ctx).Infof("Going to send events to ", on.ObjectMeta.String()) + logging.FromContext(ctx).Infof("Going to send events to pod '%s' in namespace '%s'", on.Name, on.Namespace) return &recorder{out: createRecorder(ctx, agentName), on: on} } @@ -85,6 +85,7 @@ func createRecorder(ctx context.Context, agentName string) record.EventRecorder for _, w := range watches { w.Stop() } + logging.FromContext(ctx).Debug("Closed event-broadcaster") }() } diff --git a/test/lib/recordevents/writer_vent/doc.go b/test/lib/recordevents/writer_vent/doc.go deleted file mode 100644 index 6fc9065ad1c..00000000000 --- a/test/lib/recordevents/writer_vent/doc.go +++ /dev/null @@ -1,19 +0,0 @@ -/* -Copyright 2020 The Knative Authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package writer_vent implements an observer.EventLog backed by a io.Writer -// instance, events are converted to json and written out to the writer. -package writer_vent diff --git a/test/lib/recordevents/writer_vent/writer.go b/test/lib/recordevents/writer_vent/writer.go deleted file mode 100644 index 5457f64c59e..00000000000 --- a/test/lib/recordevents/writer_vent/writer.go +++ /dev/null @@ -1,50 +0,0 @@ -/* -Copyright 2020 The Knative Authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package writer_vent - -import ( - "context" - "encoding/json" - "io" - - "knative.dev/eventing/test/lib/recordevents" -) - -func NewEventLog(ctx context.Context, out io.Writer) recordevents.EventLog { - return &writer{out: out} -} - -type writer struct { - out io.Writer -} - -var newline = []byte("\n") - -func (w *writer) Vent(observed recordevents.EventInfo) error { - b, err := json.Marshal(observed) - if err != nil { - return err - } - if _, err := w.out.Write(b); err != nil { - return err - } - if _, err := w.out.Write(newline); err != nil { - return err - } - - return nil -} diff --git a/test/test_images/recordevents/main.go b/test/test_images/recordevents/main.go index 561b3befd0f..51fde078e9b 100644 --- a/test/test_images/recordevents/main.go +++ b/test/test_images/recordevents/main.go @@ -21,16 +21,15 @@ import ( "net/http" "os" - "go.uber.org/zap" "k8s.io/client-go/rest" "knative.dev/pkg/injection/sharedmain" + "knative.dev/pkg/logging" _ "knative.dev/pkg/system/testing" "knative.dev/eventing/pkg/kncloudevents" "knative.dev/eventing/test/lib/dropevents" "knative.dev/eventing/test/lib/recordevents/observer" "knative.dev/eventing/test/lib/recordevents/recorder_vent" - "knative.dev/eventing/test/lib/recordevents/writer_vent" "knative.dev/eventing/test/test_images" ) @@ -42,14 +41,12 @@ func main() { //nolint // nil ctx is fine here, look at the code of EnableInjectionOrDie ctx := sharedmain.EnableInjectionOrDie(nil, cfg) - logger, _ := zap.NewDevelopment() - if err := test_images.ConfigureTracing(logger.Sugar(), ""); err != nil { - log.Fatal("Unable to setup trace publishing", err) + if err := test_images.ConfigureTracing(logging.FromContext(ctx), ""); err != nil { + logging.FromContext(ctx).Fatal("Unable to setup trace publishing", err) } obs := observer.NewFromEnv( recorder_vent.NewFromEnv(ctx), - writer_vent.NewEventLog(ctx, os.Stdout), ) algorithm, ok := os.LookupEnv(dropevents.SkipAlgorithmKey) @@ -72,6 +69,8 @@ func main() { } if err != nil { - log.Fatal("Error during start", err) + logging.FromContext(ctx).Fatal("Error during start", err) } + + logging.FromContext(ctx).Info("Closing the recordevents process") } From 624ccaaa76bc0714a2b5bd7396f45798f5010314 Mon Sep 17 00:00:00 2001 From: slinkydeveloper Date: Tue, 6 Oct 2020 13:45:57 +0200 Subject: [PATCH 20/23] Try fix correlator config Signed-off-by: Francesco Guardiani --- test/lib/recordevents/recorder_vent/constructor.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/lib/recordevents/recorder_vent/constructor.go b/test/lib/recordevents/recorder_vent/constructor.go index 99cdae09c11..ccf58f8e116 100644 --- a/test/lib/recordevents/recorder_vent/constructor.go +++ b/test/lib/recordevents/recorder_vent/constructor.go @@ -72,7 +72,12 @@ func createRecorder(ctx context.Context, agentName string) record.EventRecorder if recorder == nil { // Create event broadcaster logger.Debug("Creating event broadcaster") - eventBroadcaster := record.NewBroadcaster() + eventBroadcaster := record.NewBroadcasterWithCorrelatorOptions(record.CorrelatorOptions{ + MessageFunc: func(event *corev1.Event) string { + return event.Message + }, + MaxEvents: 1, + }) watches := []watch.Interface{ eventBroadcaster.StartLogging(logger.Named("event-broadcaster").Infof), eventBroadcaster.StartRecordingToSink( From 1a54bb2970f758a1ef2d744e02de84538d4851c3 Mon Sep 17 00:00:00 2001 From: slinkydeveloper Date: Tue, 6 Oct 2020 14:28:04 +0200 Subject: [PATCH 21/23] Just close the server Signed-off-by: Francesco Guardiani --- test/lib/recordevents/observer/observer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/lib/recordevents/observer/observer.go b/test/lib/recordevents/observer/observer.go index d124959955b..9c9f3c611ab 100644 --- a/test/lib/recordevents/observer/observer.go +++ b/test/lib/recordevents/observer/observer.go @@ -82,7 +82,7 @@ func (o *Observer) Start(ctx context.Context, handlerFuncs ...func(handler http. logging.FromContext(ctx).Info("Closing the HTTP server") - return server.Shutdown(ctx) + return server.Close() } func (o *Observer) ServeHTTP(writer http.ResponseWriter, request *http.Request) { From 82e138be20dc7516ca1e511220eab87d59226342 Mon Sep 17 00:00:00 2001 From: slinkydeveloper Date: Tue, 6 Oct 2020 16:41:19 +0200 Subject: [PATCH 22/23] Still messing up with the Event correlator options Signed-off-by: Francesco Guardiani --- test/lib/recordevents/event_info.go | 1 + test/lib/recordevents/event_log.go | 11 ++++++ test/lib/recordevents/observer/observer.go | 36 ++++++++++--------- .../recordevents/recorder_vent/constructor.go | 16 +++++++-- 4 files changed, 44 insertions(+), 20 deletions(-) diff --git a/test/lib/recordevents/event_info.go b/test/lib/recordevents/event_info.go index 35490108f73..d9dbbdae819 100644 --- a/test/lib/recordevents/event_info.go +++ b/test/lib/recordevents/event_info.go @@ -41,6 +41,7 @@ type EventInfo struct { Origin string `json:"origin,omitempty"` Observer string `json:"observer,omitempty"` Time time.Time `json:"time,omitempty"` + Sequence uint64 `json:"sequence"` } // Pretty print the event. Meant for debugging. This formats the validation error diff --git a/test/lib/recordevents/event_log.go b/test/lib/recordevents/event_log.go index fcef14a9f12..62aba6113b4 100644 --- a/test/lib/recordevents/event_log.go +++ b/test/lib/recordevents/event_log.go @@ -20,3 +20,14 @@ package recordevents type EventLog interface { Vent(observed EventInfo) error } + +type EventLogs []EventLog + +func (e EventLogs) Vent(observed EventInfo) error { + for _, el := range e { + if err := el.Vent(observed); err != nil { + return err + } + } + return nil +} diff --git a/test/lib/recordevents/observer/observer.go b/test/lib/recordevents/observer/observer.go index 9c9f3c611ab..e9bd4224aac 100644 --- a/test/lib/recordevents/observer/observer.go +++ b/test/lib/recordevents/observer/observer.go @@ -19,6 +19,7 @@ package observer import ( "context" "net/http" + "sync/atomic" "time" cloudeventsbindings "github.com/cloudevents/sdk-go/v2/binding" @@ -35,7 +36,9 @@ type Observer struct { // Name is the name of this Observer, used to filter if multiple observers. Name string // EventLogs is the list of EventLog implementors to vent observed events. - EventLogs []recordevents.EventLog + EventLogs recordevents.EventLogs + + seq uint64 } // New returns an observer that will vent observations to the list of provided @@ -92,22 +95,21 @@ func (o *Observer) ServeHTTP(writer http.ResponseWriter, request *http.Request) event, eventErr := cloudeventsbindings.ToEvent(context.TODO(), m) header := request.Header - for _, el := range o.EventLogs { - eventErrStr := "" - if eventErr != nil { - eventErrStr = eventErr.Error() - } - err := el.Vent(recordevents.EventInfo{ - Error: eventErrStr, - Event: event, - HTTPHeaders: header, - Origin: request.RemoteAddr, - Observer: o.Name, - Time: time.Now(), - }) - if err != nil { - log.Warn("Error while venting the recorded event", err) - } + eventErrStr := "" + if eventErr != nil { + eventErrStr = eventErr.Error() + } + err := o.EventLogs.Vent(recordevents.EventInfo{ + Error: eventErrStr, + Event: event, + HTTPHeaders: header, + Origin: request.RemoteAddr, + Observer: o.Name, + Time: time.Now(), + Sequence: atomic.AddUint64(&o.seq, 1), + }) + if err != nil { + log.Warn("Error while venting the recorded event", err) } writer.WriteHeader(http.StatusAccepted) diff --git a/test/lib/recordevents/recorder_vent/constructor.go b/test/lib/recordevents/recorder_vent/constructor.go index ccf58f8e116..cc73a9671f0 100644 --- a/test/lib/recordevents/recorder_vent/constructor.go +++ b/test/lib/recordevents/recorder_vent/constructor.go @@ -19,6 +19,7 @@ package recorder_vent import ( "context" "log" + "strings" "github.com/kelseyhightower/envconfig" "knative.dev/pkg/system" @@ -73,10 +74,19 @@ func createRecorder(ctx context.Context, agentName string) record.EventRecorder // Create event broadcaster logger.Debug("Creating event broadcaster") eventBroadcaster := record.NewBroadcasterWithCorrelatorOptions(record.CorrelatorOptions{ - MessageFunc: func(event *corev1.Event) string { - return event.Message + KeyFunc: func(event *corev1.Event) (aggregateKey string, localKey string) { + return strings.Join([]string{ + event.Source.Component, + event.Source.Host, + event.InvolvedObject.Kind, + event.InvolvedObject.Namespace, + event.InvolvedObject.Name, + string(event.InvolvedObject.UID), + event.InvolvedObject.APIVersion, + event.Type, + event.Reason, + }, ""), string(event.UID) }, - MaxEvents: 1, }) watches := []watch.Interface{ eventBroadcaster.StartLogging(logger.Named("event-broadcaster").Infof), From 58a8542a6788249b65a52e581320fbf74974ec91 Mon Sep 17 00:00:00 2001 From: slinkydeveloper Date: Tue, 6 Oct 2020 16:49:50 +0200 Subject: [PATCH 23/23] lint check Signed-off-by: Francesco Guardiani --- test/conformance/helpers/tracing_test_helper.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/conformance/helpers/tracing_test_helper.go b/test/conformance/helpers/tracing_test_helper.go index 0eb6b16995f..f3e476f9f5b 100644 --- a/test/conformance/helpers/tracing_test_helper.go +++ b/test/conformance/helpers/tracing_test_helper.go @@ -107,6 +107,6 @@ func getTraceIDHeader(t *testing.T, evInfos []recordevents.EventInfo) string { } } } - t.Fatalf("FAIL: No traceid in %d messages: (%s)", len(evInfos), evInfos) + t.Fatalf("FAIL: No traceid in %d messages: (%v)", len(evInfos), evInfos) return "" }