From d7ed39beabb105c5e91e1ab233c83e75d08faadc Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Mon, 28 Jan 2019 09:13:25 -0800 Subject: [PATCH 001/128] Initial API outline for Broker. --- pkg/apis/eventing/v1alpha1/broker_defaults.go | 71 +++++++ .../eventing/v1alpha1/broker_defaults_test.go | 25 +++ pkg/apis/eventing/v1alpha1/broker_types.go | 137 ++++++++++++++ .../eventing/v1alpha1/broker_validation.go | 92 +++++++++ .../v1alpha1/zz_generated.deepcopy.go | 170 ++++++++++++++++- .../typed/eventing/v1alpha1/broker.go | 174 ++++++++++++++++++ .../eventing/v1alpha1/eventing_client.go | 5 + .../eventing/v1alpha1/fake/fake_broker.go | 140 ++++++++++++++ .../v1alpha1/fake/fake_eventing_client.go | 4 + .../eventing/v1alpha1/generated_expansion.go | 2 + .../eventing/v1alpha1/broker.go | 89 +++++++++ .../eventing/v1alpha1/interface.go | 7 + .../informers/externalversions/generic.go | 2 + .../listers/eventing/v1alpha1/broker.go | 94 ++++++++++ .../eventing/v1alpha1/expansion_generated.go | 8 + 15 files changed, 1010 insertions(+), 10 deletions(-) create mode 100644 pkg/apis/eventing/v1alpha1/broker_defaults.go create mode 100644 pkg/apis/eventing/v1alpha1/broker_defaults_test.go create mode 100644 pkg/apis/eventing/v1alpha1/broker_types.go create mode 100644 pkg/apis/eventing/v1alpha1/broker_validation.go create mode 100644 pkg/client/clientset/versioned/typed/eventing/v1alpha1/broker.go create mode 100644 pkg/client/clientset/versioned/typed/eventing/v1alpha1/fake/fake_broker.go create mode 100644 pkg/client/informers/externalversions/eventing/v1alpha1/broker.go create mode 100644 pkg/client/listers/eventing/v1alpha1/broker.go diff --git a/pkg/apis/eventing/v1alpha1/broker_defaults.go b/pkg/apis/eventing/v1alpha1/broker_defaults.go new file mode 100644 index 00000000000..fe795921b50 --- /dev/null +++ b/pkg/apis/eventing/v1alpha1/broker_defaults.go @@ -0,0 +1,71 @@ +/* +Copyright 2019 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + +const ( + brokerLabel = "eventing.knative.dev/broker" +) + +func (b *Broker) SetDefaults() { + b.Spec.SetDefaults(b.Name) +} + +func (bs *BrokerSpec) SetDefaults(brokerName string) { + if bs.Selector == nil { + bs.Selector = defaultBrokerSpecSelector(brokerName) + } + if bs.ChannelTemplate == nil { + bs.ChannelTemplate = defaultBrokerSpecChannelTemplate(brokerName) + } + if len(bs.SubscribableResources) == 0 { + bs.SubscribableResources = defaultBrokerSpecSubscribableResources() + } +} + +func defaultBrokerLabels(brokerName string) map[string]string { + return map[string]string{ + brokerLabel: brokerName, + } +} + +func defaultBrokerSpecSelector(brokerName string) *v1.LabelSelector { + return &v1.LabelSelector{ + MatchLabels: defaultBrokerLabels(brokerName), + } +} + +func defaultBrokerSpecChannelTemplate(brokerName string) *ChannelTemplateSpec { + return &ChannelTemplateSpec{ + metadata: v1.ObjectMeta{ + Labels: defaultBrokerLabels(brokerName), + }, + // Spec is left blank so that the created Channel defaulter will default the provisioner + // and arguments when the Channel is created. + } +} + +func defaultBrokerSpecSubscribableResources() []v1.GroupVersionKind { + return []v1.GroupVersionKind{ + { + Group: "eventing.knative.dev", + Version: "v1alpha1", + Kind: "Channel", + }, + } +} diff --git a/pkg/apis/eventing/v1alpha1/broker_defaults_test.go b/pkg/apis/eventing/v1alpha1/broker_defaults_test.go new file mode 100644 index 00000000000..b27c6b2d7c6 --- /dev/null +++ b/pkg/apis/eventing/v1alpha1/broker_defaults_test.go @@ -0,0 +1,25 @@ +/* +Copyright 2019 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import "testing" + +// No-op test because method does nothing. +func TestBrokerDefaults(t *testing.T) { + b := Broker{} + b.SetDefaults() +} diff --git a/pkg/apis/eventing/v1alpha1/broker_types.go b/pkg/apis/eventing/v1alpha1/broker_types.go new file mode 100644 index 00000000000..51216de7f89 --- /dev/null +++ b/pkg/apis/eventing/v1alpha1/broker_types.go @@ -0,0 +1,137 @@ +/* + * Copyright 2019 The Knative Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package v1alpha1 + +import ( + "github.com/knative/pkg/apis" + duckv1alpha1 "github.com/knative/pkg/apis/duck/v1alpha1" + "github.com/knative/pkg/webhook" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +// +genclient +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// Broker is an abstract resource that implements the Addressable contract. +// The Provisioner provisions infrastructure to accepts events and +// deliver to Subscriptions. +type Broker struct { + metav1.TypeMeta `json:",inline"` + // +optional + metav1.ObjectMeta `json:"metadata,omitempty"` + + // Spec defines the desired state of the Broker. + Spec BrokerSpec `json:"spec,omitempty"` + + // Status represents the current state of the Broker. This data may be out of + // date. + // +optional + Status BrokerStatus `json:"status,omitempty"` +} + +// Check that Broker can be validated, can be defaulted, and has immutable fields. +var _ apis.Validatable = (*Broker)(nil) +var _ apis.Defaultable = (*Broker)(nil) +var _ apis.Immutable = (*Broker)(nil) +var _ runtime.Object = (*Broker)(nil) +var _ webhook.GenericCRD = (*Broker)(nil) + +// BrokerSpec specifies the Provisioner backing a channel and the configuration +// arguments for a Broker. +type BrokerSpec struct { + Selector *metav1.LabelSelector `json:"selector,omitempty"` + ChannelTemplate *ChannelTemplateSpec `json:"channelTemplate,omitempty"` + SubscribableResources []metav1.GroupVersionKind `json:"subscribableResources,omitempty"` +} + +type ChannelTemplateSpec struct { + metadata metav1.ObjectMeta `json:"metadata,omitempty"` + Spec *ChannelSpec `json:"spec,omitempty"` +} + +var brokerCondSet = duckv1alpha1.NewLivingConditionSet(BrokerConditionChannelTemplateSelector, BrokerConditionSubscribableResourcesExist) + +// BrokerStatus represents the current state of a Broker. +type BrokerStatus struct { + // ObservedGeneration is the most recent generation observed for this Broker. + // It corresponds to the Broker's generation, which is updated on mutation by + // the API Server. + // TODO: The above comment is only true once + // https://github.com/kubernetes/kubernetes/issues/58778 is fixed. + // +optional + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + + // Represents the latest available observations of a channel's current state. + // +optional + // +patchMergeKey=type + // +patchStrategy=merge + Conditions duckv1alpha1.Conditions `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"` +} + +const ( + // BrokerConditionReady has status True when the Broker is ready to + // accept traffic. + BrokerConditionReady = duckv1alpha1.ConditionReady + + BrokerConditionChannelTemplateSelector duckv1alpha1.ConditionType = "ChannelTemplateSelector" + + BrokerConditionSubscribableResourcesExist duckv1alpha1.ConditionType = "SubscribableResourcesExist" +) + +// GetCondition returns the condition currently associated with the given type, or nil. +func (bs *BrokerStatus) GetCondition(t duckv1alpha1.ConditionType) *duckv1alpha1.Condition { + return chanCondSet.Manage(bs).GetCondition(t) +} + +// IsReady returns true if the resource is ready overall. +func (bs *BrokerStatus) IsReady() bool { + return chanCondSet.Manage(bs).IsHappy() +} + +// InitializeConditions sets relevant unset conditions to Unknown state. +func (bs *BrokerStatus) InitializeConditions() { + chanCondSet.Manage(bs).InitializeConditions() +} + +func (bs *BrokerStatus) MarkChannelTemplateMatchesSelector() { + chanCondSet.Manage(bs).MarkTrue(BrokerConditionChannelTemplateSelector) +} + +func (bs *BrokerStatus) MarkChannelTemplateDoesNotMatchSelector() { + chanCondSet.Manage(bs).MarkFalse(BrokerConditionChannelTemplateSelector, "selectorDoesNotMatchTemplate", "`spec.selector` does not match `spec.channelTempalte.meta.labels`") +} + +// MarkProvisioned sets BrokerConditionProvisioned condition to True state. +func (bs *BrokerStatus) MarkSubscribableResourcesExist() { + chanCondSet.Manage(bs).MarkTrue(BrokerConditionSubscribableResourcesExist) +} + +// MarkNotProvisioned sets BrokerConditionProvisioned condition to False state. +func (bs *BrokerStatus) MarkSubscribableResourcesDoNotExist(reason, messageFormat string, messageA ...interface{}) { + chanCondSet.Manage(bs).MarkFalse(BrokerConditionSubscribableResourcesExist, reason, messageFormat, messageA...) +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// BrokerList is a collection of Brokers. +type BrokerList struct { + metav1.TypeMeta `json:",inline"` + // +optional + metav1.ListMeta `json:"metadata,omitempty"` + Items []Broker `json:"items"` +} diff --git a/pkg/apis/eventing/v1alpha1/broker_validation.go b/pkg/apis/eventing/v1alpha1/broker_validation.go new file mode 100644 index 00000000000..72c7fa3c7a2 --- /dev/null +++ b/pkg/apis/eventing/v1alpha1/broker_validation.go @@ -0,0 +1,92 @@ +/* +Copyright 2019 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "fmt" + + "github.com/google/go-cmp/cmp" + "github.com/knative/pkg/apis" + "k8s.io/apimachinery/pkg/api/equality" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func (b *Broker) Validate() *apis.FieldError { + return b.Spec.Validate().ViaField("spec") +} + +func (bs *BrokerSpec) Validate() *apis.FieldError { + var errs *apis.FieldError + if isSelectorNotPresentOrEmpty(bs.Selector) { + fe := apis.ErrMissingField("selector") + fe.Details = "the Broker must have a selector" + errs = errs.Also(fe) + } + + if bs.ChannelTemplate == nil { + fe := apis.ErrMissingField("channelTemplate") + fe.Details = "the Broker must have a channelTemplate" + errs = errs.Also(fe) + } else if !selectorMatchesTemplateLabels(bs.Selector, bs.ChannelTemplate) { + fe := apis.ErrInvalidValue(fmt.Sprint(bs.ChannelTemplate.metadata.Labels), "channelTemplate.metadata.labels") + errs = errs.Also(fe) + } + + if !channelsInSubscribableResources(bs.SubscribableResources) { + fe := apis.ErrInvalidValue(fmt.Sprint(bs.SubscribableResources), "subscribableResources") + errs = errs.Also(fe) + } + + return errs +} + +func isSelectorNotPresentOrEmpty(s *v1.LabelSelector) bool { + return s == nil || equality.Semantic.DeepEqual(s, &v1.LabelSelector{}) +} + +func selectorMatchesTemplateLabels(s *v1.LabelSelector, ct *ChannelTemplateSpec) bool { + // TODO Improve this so it supports something other than direct label equality. + return equality.Semantic.DeepEqual(s.MatchLabels, ct.metadata.Labels) +} + +func channelsInSubscribableResources(sr []v1.GroupVersionKind) bool { + for _, r := range sr { + if r.Group == "eventing.knative.dev" && r.Version == "v1alpha1" && r.Kind == "Channel" { + return true + } + } + return false +} + +func (b *Broker) CheckImmutableFields(og apis.Immutable) *apis.FieldError { + original, ok := og.(*Broker) + if !ok { + return &apis.FieldError{Message: "The provided original was not a Broker"} + } + if original == nil { + return nil + } + + if diff := cmp.Diff(original.Spec.Selector, b.Spec.Selector); diff != "" { + return &apis.FieldError{ + Message: "Immutable fields changed (-old +new)", + Paths: []string{"spec", "selector"}, + Details: diff, + } + } + return nil +} diff --git a/pkg/apis/eventing/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/eventing/v1alpha1/zz_generated.deepcopy.go index d43df63bdb2..6481321c20e 100644 --- a/pkg/apis/eventing/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/eventing/v1alpha1/zz_generated.deepcopy.go @@ -21,12 +21,136 @@ limitations under the License. package v1alpha1 import ( - duck_v1alpha1 "github.com/knative/eventing/pkg/apis/duck/v1alpha1" - apis_duck_v1alpha1 "github.com/knative/pkg/apis/duck/v1alpha1" - v1 "k8s.io/api/core/v1" + apis_duck_v1alpha1 "github.com/knative/eventing/pkg/apis/duck/v1alpha1" + duck_v1alpha1 "github.com/knative/pkg/apis/duck/v1alpha1" + core_v1 "k8s.io/api/core/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Broker) DeepCopyInto(out *Broker) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Broker. +func (in *Broker) DeepCopy() *Broker { + if in == nil { + return nil + } + out := new(Broker) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Broker) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BrokerList) DeepCopyInto(out *BrokerList) { + *out = *in + out.TypeMeta = in.TypeMeta + out.ListMeta = in.ListMeta + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Broker, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BrokerList. +func (in *BrokerList) DeepCopy() *BrokerList { + if in == nil { + return nil + } + out := new(BrokerList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *BrokerList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BrokerSpec) DeepCopyInto(out *BrokerSpec) { + *out = *in + if in.Selector != nil { + in, out := &in.Selector, &out.Selector + if *in == nil { + *out = nil + } else { + *out = new(v1.LabelSelector) + (*in).DeepCopyInto(*out) + } + } + if in.ChannelTemplate != nil { + in, out := &in.ChannelTemplate, &out.ChannelTemplate + if *in == nil { + *out = nil + } else { + *out = new(ChannelTemplateSpec) + (*in).DeepCopyInto(*out) + } + } + if in.SubscribableResources != nil { + in, out := &in.SubscribableResources, &out.SubscribableResources + *out = make([]v1.GroupVersionKind, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BrokerSpec. +func (in *BrokerSpec) DeepCopy() *BrokerSpec { + if in == nil { + return nil + } + out := new(BrokerSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BrokerStatus) DeepCopyInto(out *BrokerStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make(duck_v1alpha1.Conditions, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BrokerStatus. +func (in *BrokerStatus) DeepCopy() *BrokerStatus { + if in == nil { + return nil + } + out := new(BrokerStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Channel) DeepCopyInto(out *Channel) { *out = *in @@ -96,7 +220,7 @@ func (in *ChannelSpec) DeepCopyInto(out *ChannelSpec) { if *in == nil { *out = nil } else { - *out = new(v1.ObjectReference) + *out = new(core_v1.ObjectReference) **out = **in } } @@ -114,7 +238,7 @@ func (in *ChannelSpec) DeepCopyInto(out *ChannelSpec) { if *in == nil { *out = nil } else { - *out = new(duck_v1alpha1.Subscribable) + *out = new(apis_duck_v1alpha1.Subscribable) (*in).DeepCopyInto(*out) } } @@ -137,7 +261,7 @@ func (in *ChannelStatus) DeepCopyInto(out *ChannelStatus) { out.Address = in.Address if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions - *out = make(apis_duck_v1alpha1.Conditions, len(*in)) + *out = make(duck_v1alpha1.Conditions, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } @@ -164,6 +288,32 @@ func (in *ChannelStatus) DeepCopy() *ChannelStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ChannelTemplateSpec) DeepCopyInto(out *ChannelTemplateSpec) { + *out = *in + in.metadata.DeepCopyInto(&out.metadata) + if in.Spec != nil { + in, out := &in.Spec, &out.Spec + if *in == nil { + *out = nil + } else { + *out = new(ChannelSpec) + (*in).DeepCopyInto(*out) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ChannelTemplateSpec. +func (in *ChannelTemplateSpec) DeepCopy() *ChannelTemplateSpec { + if in == nil { + return nil + } + out := new(ChannelTemplateSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ClusterChannelProvisioner) DeepCopyInto(out *ClusterChannelProvisioner) { *out = *in @@ -246,7 +396,7 @@ func (in *ClusterChannelProvisionerStatus) DeepCopyInto(out *ClusterChannelProvi *out = *in if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions - *out = make(apis_duck_v1alpha1.Conditions, len(*in)) + *out = make(duck_v1alpha1.Conditions, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } @@ -272,7 +422,7 @@ func (in *ReplyStrategy) DeepCopyInto(out *ReplyStrategy) { if *in == nil { *out = nil } else { - *out = new(v1.ObjectReference) + *out = new(core_v1.ObjectReference) **out = **in } } @@ -297,7 +447,7 @@ func (in *SubscriberSpec) DeepCopyInto(out *SubscriberSpec) { if *in == nil { *out = nil } else { - *out = new(v1.ObjectReference) + *out = new(core_v1.ObjectReference) **out = **in } } @@ -424,7 +574,7 @@ func (in *SubscriptionStatus) DeepCopyInto(out *SubscriptionStatus) { *out = *in if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions - *out = make(apis_duck_v1alpha1.Conditions, len(*in)) + *out = make(duck_v1alpha1.Conditions, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } diff --git a/pkg/client/clientset/versioned/typed/eventing/v1alpha1/broker.go b/pkg/client/clientset/versioned/typed/eventing/v1alpha1/broker.go new file mode 100644 index 00000000000..7ae965feb0e --- /dev/null +++ b/pkg/client/clientset/versioned/typed/eventing/v1alpha1/broker.go @@ -0,0 +1,174 @@ +/* +Copyright 2018 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + v1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" + scheme "github.com/knative/eventing/pkg/client/clientset/versioned/scheme" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + rest "k8s.io/client-go/rest" +) + +// BrokersGetter has a method to return a BrokerInterface. +// A group's client should implement this interface. +type BrokersGetter interface { + Brokers(namespace string) BrokerInterface +} + +// BrokerInterface has methods to work with Broker resources. +type BrokerInterface interface { + Create(*v1alpha1.Broker) (*v1alpha1.Broker, error) + Update(*v1alpha1.Broker) (*v1alpha1.Broker, error) + UpdateStatus(*v1alpha1.Broker) (*v1alpha1.Broker, error) + Delete(name string, options *v1.DeleteOptions) error + DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error + Get(name string, options v1.GetOptions) (*v1alpha1.Broker, error) + List(opts v1.ListOptions) (*v1alpha1.BrokerList, error) + Watch(opts v1.ListOptions) (watch.Interface, error) + Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1alpha1.Broker, err error) + BrokerExpansion +} + +// brokers implements BrokerInterface +type brokers struct { + client rest.Interface + ns string +} + +// newBrokers returns a Brokers +func newBrokers(c *EventingV1alpha1Client, namespace string) *brokers { + return &brokers{ + client: c.RESTClient(), + ns: namespace, + } +} + +// Get takes name of the broker, and returns the corresponding broker object, and an error if there is any. +func (c *brokers) Get(name string, options v1.GetOptions) (result *v1alpha1.Broker, err error) { + result = &v1alpha1.Broker{} + err = c.client.Get(). + Namespace(c.ns). + Resource("brokers"). + Name(name). + VersionedParams(&options, scheme.ParameterCodec). + Do(). + Into(result) + return +} + +// List takes label and field selectors, and returns the list of Brokers that match those selectors. +func (c *brokers) List(opts v1.ListOptions) (result *v1alpha1.BrokerList, err error) { + result = &v1alpha1.BrokerList{} + err = c.client.Get(). + Namespace(c.ns). + Resource("brokers"). + VersionedParams(&opts, scheme.ParameterCodec). + Do(). + Into(result) + return +} + +// Watch returns a watch.Interface that watches the requested brokers. +func (c *brokers) Watch(opts v1.ListOptions) (watch.Interface, error) { + opts.Watch = true + return c.client.Get(). + Namespace(c.ns). + Resource("brokers"). + VersionedParams(&opts, scheme.ParameterCodec). + Watch() +} + +// Create takes the representation of a broker and creates it. Returns the server's representation of the broker, and an error, if there is any. +func (c *brokers) Create(broker *v1alpha1.Broker) (result *v1alpha1.Broker, err error) { + result = &v1alpha1.Broker{} + err = c.client.Post(). + Namespace(c.ns). + Resource("brokers"). + Body(broker). + Do(). + Into(result) + return +} + +// Update takes the representation of a broker and updates it. Returns the server's representation of the broker, and an error, if there is any. +func (c *brokers) Update(broker *v1alpha1.Broker) (result *v1alpha1.Broker, err error) { + result = &v1alpha1.Broker{} + err = c.client.Put(). + Namespace(c.ns). + Resource("brokers"). + Name(broker.Name). + Body(broker). + Do(). + Into(result) + return +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). + +func (c *brokers) UpdateStatus(broker *v1alpha1.Broker) (result *v1alpha1.Broker, err error) { + result = &v1alpha1.Broker{} + err = c.client.Put(). + Namespace(c.ns). + Resource("brokers"). + Name(broker.Name). + SubResource("status"). + Body(broker). + Do(). + Into(result) + return +} + +// Delete takes name of the broker and deletes it. Returns an error if one occurs. +func (c *brokers) Delete(name string, options *v1.DeleteOptions) error { + return c.client.Delete(). + Namespace(c.ns). + Resource("brokers"). + Name(name). + Body(options). + Do(). + Error() +} + +// DeleteCollection deletes a collection of objects. +func (c *brokers) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + return c.client.Delete(). + Namespace(c.ns). + Resource("brokers"). + VersionedParams(&listOptions, scheme.ParameterCodec). + Body(options). + Do(). + Error() +} + +// Patch applies the patch and returns the patched broker. +func (c *brokers) Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1alpha1.Broker, err error) { + result = &v1alpha1.Broker{} + err = c.client.Patch(pt). + Namespace(c.ns). + Resource("brokers"). + SubResource(subresources...). + Name(name). + Body(data). + Do(). + Into(result) + return +} diff --git a/pkg/client/clientset/versioned/typed/eventing/v1alpha1/eventing_client.go b/pkg/client/clientset/versioned/typed/eventing/v1alpha1/eventing_client.go index ef337f70bf0..8a2796c7417 100644 --- a/pkg/client/clientset/versioned/typed/eventing/v1alpha1/eventing_client.go +++ b/pkg/client/clientset/versioned/typed/eventing/v1alpha1/eventing_client.go @@ -27,6 +27,7 @@ import ( type EventingV1alpha1Interface interface { RESTClient() rest.Interface + BrokersGetter ChannelsGetter ClusterChannelProvisionersGetter SubscriptionsGetter @@ -37,6 +38,10 @@ type EventingV1alpha1Client struct { restClient rest.Interface } +func (c *EventingV1alpha1Client) Brokers(namespace string) BrokerInterface { + return newBrokers(c, namespace) +} + func (c *EventingV1alpha1Client) Channels(namespace string) ChannelInterface { return newChannels(c, namespace) } diff --git a/pkg/client/clientset/versioned/typed/eventing/v1alpha1/fake/fake_broker.go b/pkg/client/clientset/versioned/typed/eventing/v1alpha1/fake/fake_broker.go new file mode 100644 index 00000000000..c7b30557ec1 --- /dev/null +++ b/pkg/client/clientset/versioned/typed/eventing/v1alpha1/fake/fake_broker.go @@ -0,0 +1,140 @@ +/* +Copyright 2018 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + v1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + labels "k8s.io/apimachinery/pkg/labels" + schema "k8s.io/apimachinery/pkg/runtime/schema" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + testing "k8s.io/client-go/testing" +) + +// FakeBrokers implements BrokerInterface +type FakeBrokers struct { + Fake *FakeEventingV1alpha1 + ns string +} + +var brokersResource = schema.GroupVersionResource{Group: "eventing.knative.dev", Version: "v1alpha1", Resource: "brokers"} + +var brokersKind = schema.GroupVersionKind{Group: "eventing.knative.dev", Version: "v1alpha1", Kind: "Broker"} + +// Get takes name of the broker, and returns the corresponding broker object, and an error if there is any. +func (c *FakeBrokers) Get(name string, options v1.GetOptions) (result *v1alpha1.Broker, err error) { + obj, err := c.Fake. + Invokes(testing.NewGetAction(brokersResource, c.ns, name), &v1alpha1.Broker{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.Broker), err +} + +// List takes label and field selectors, and returns the list of Brokers that match those selectors. +func (c *FakeBrokers) List(opts v1.ListOptions) (result *v1alpha1.BrokerList, err error) { + obj, err := c.Fake. + Invokes(testing.NewListAction(brokersResource, brokersKind, c.ns, opts), &v1alpha1.BrokerList{}) + + if obj == nil { + return nil, err + } + + label, _, _ := testing.ExtractFromListOptions(opts) + if label == nil { + label = labels.Everything() + } + list := &v1alpha1.BrokerList{ListMeta: obj.(*v1alpha1.BrokerList).ListMeta} + for _, item := range obj.(*v1alpha1.BrokerList).Items { + if label.Matches(labels.Set(item.Labels)) { + list.Items = append(list.Items, item) + } + } + return list, err +} + +// Watch returns a watch.Interface that watches the requested brokers. +func (c *FakeBrokers) Watch(opts v1.ListOptions) (watch.Interface, error) { + return c.Fake. + InvokesWatch(testing.NewWatchAction(brokersResource, c.ns, opts)) + +} + +// Create takes the representation of a broker and creates it. Returns the server's representation of the broker, and an error, if there is any. +func (c *FakeBrokers) Create(broker *v1alpha1.Broker) (result *v1alpha1.Broker, err error) { + obj, err := c.Fake. + Invokes(testing.NewCreateAction(brokersResource, c.ns, broker), &v1alpha1.Broker{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.Broker), err +} + +// Update takes the representation of a broker and updates it. Returns the server's representation of the broker, and an error, if there is any. +func (c *FakeBrokers) Update(broker *v1alpha1.Broker) (result *v1alpha1.Broker, err error) { + obj, err := c.Fake. + Invokes(testing.NewUpdateAction(brokersResource, c.ns, broker), &v1alpha1.Broker{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.Broker), err +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *FakeBrokers) UpdateStatus(broker *v1alpha1.Broker) (*v1alpha1.Broker, error) { + obj, err := c.Fake. + Invokes(testing.NewUpdateSubresourceAction(brokersResource, "status", c.ns, broker), &v1alpha1.Broker{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.Broker), err +} + +// Delete takes name of the broker and deletes it. Returns an error if one occurs. +func (c *FakeBrokers) Delete(name string, options *v1.DeleteOptions) error { + _, err := c.Fake. + Invokes(testing.NewDeleteAction(brokersResource, c.ns, name), &v1alpha1.Broker{}) + + return err +} + +// DeleteCollection deletes a collection of objects. +func (c *FakeBrokers) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + action := testing.NewDeleteCollectionAction(brokersResource, c.ns, listOptions) + + _, err := c.Fake.Invokes(action, &v1alpha1.BrokerList{}) + return err +} + +// Patch applies the patch and returns the patched broker. +func (c *FakeBrokers) Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1alpha1.Broker, err error) { + obj, err := c.Fake. + Invokes(testing.NewPatchSubresourceAction(brokersResource, c.ns, name, data, subresources...), &v1alpha1.Broker{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.Broker), err +} diff --git a/pkg/client/clientset/versioned/typed/eventing/v1alpha1/fake/fake_eventing_client.go b/pkg/client/clientset/versioned/typed/eventing/v1alpha1/fake/fake_eventing_client.go index 0080c07b8d4..68998cd28f5 100644 --- a/pkg/client/clientset/versioned/typed/eventing/v1alpha1/fake/fake_eventing_client.go +++ b/pkg/client/clientset/versioned/typed/eventing/v1alpha1/fake/fake_eventing_client.go @@ -28,6 +28,10 @@ type FakeEventingV1alpha1 struct { *testing.Fake } +func (c *FakeEventingV1alpha1) Brokers(namespace string) v1alpha1.BrokerInterface { + return &FakeBrokers{c, namespace} +} + func (c *FakeEventingV1alpha1) Channels(namespace string) v1alpha1.ChannelInterface { return &FakeChannels{c, namespace} } diff --git a/pkg/client/clientset/versioned/typed/eventing/v1alpha1/generated_expansion.go b/pkg/client/clientset/versioned/typed/eventing/v1alpha1/generated_expansion.go index fcef70e765c..94933cfbc74 100644 --- a/pkg/client/clientset/versioned/typed/eventing/v1alpha1/generated_expansion.go +++ b/pkg/client/clientset/versioned/typed/eventing/v1alpha1/generated_expansion.go @@ -18,6 +18,8 @@ limitations under the License. package v1alpha1 +type BrokerExpansion interface{} + type ChannelExpansion interface{} type ClusterChannelProvisionerExpansion interface{} diff --git a/pkg/client/informers/externalversions/eventing/v1alpha1/broker.go b/pkg/client/informers/externalversions/eventing/v1alpha1/broker.go new file mode 100644 index 00000000000..f9dcaf534b3 --- /dev/null +++ b/pkg/client/informers/externalversions/eventing/v1alpha1/broker.go @@ -0,0 +1,89 @@ +/* +Copyright 2018 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by informer-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + time "time" + + eventing_v1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" + versioned "github.com/knative/eventing/pkg/client/clientset/versioned" + internalinterfaces "github.com/knative/eventing/pkg/client/informers/externalversions/internalinterfaces" + v1alpha1 "github.com/knative/eventing/pkg/client/listers/eventing/v1alpha1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + watch "k8s.io/apimachinery/pkg/watch" + cache "k8s.io/client-go/tools/cache" +) + +// BrokerInformer provides access to a shared informer and lister for +// Brokers. +type BrokerInformer interface { + Informer() cache.SharedIndexInformer + Lister() v1alpha1.BrokerLister +} + +type brokerInformer struct { + factory internalinterfaces.SharedInformerFactory + tweakListOptions internalinterfaces.TweakListOptionsFunc + namespace string +} + +// NewBrokerInformer constructs a new informer for Broker type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewBrokerInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { + return NewFilteredBrokerInformer(client, namespace, resyncPeriod, indexers, nil) +} + +// NewFilteredBrokerInformer constructs a new informer for Broker type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFilteredBrokerInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { + return cache.NewSharedIndexInformer( + &cache.ListWatch{ + ListFunc: func(options v1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.EventingV1alpha1().Brokers(namespace).List(options) + }, + WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.EventingV1alpha1().Brokers(namespace).Watch(options) + }, + }, + &eventing_v1alpha1.Broker{}, + resyncPeriod, + indexers, + ) +} + +func (f *brokerInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { + return NewFilteredBrokerInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) +} + +func (f *brokerInformer) Informer() cache.SharedIndexInformer { + return f.factory.InformerFor(&eventing_v1alpha1.Broker{}, f.defaultInformer) +} + +func (f *brokerInformer) Lister() v1alpha1.BrokerLister { + return v1alpha1.NewBrokerLister(f.Informer().GetIndexer()) +} diff --git a/pkg/client/informers/externalversions/eventing/v1alpha1/interface.go b/pkg/client/informers/externalversions/eventing/v1alpha1/interface.go index 15bebacbe7f..9d7b654e259 100644 --- a/pkg/client/informers/externalversions/eventing/v1alpha1/interface.go +++ b/pkg/client/informers/externalversions/eventing/v1alpha1/interface.go @@ -24,6 +24,8 @@ import ( // Interface provides access to all the informers in this group version. type Interface interface { + // Brokers returns a BrokerInformer. + Brokers() BrokerInformer // Channels returns a ChannelInformer. Channels() ChannelInformer // ClusterChannelProvisioners returns a ClusterChannelProvisionerInformer. @@ -43,6 +45,11 @@ func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakList return &version{factory: f, namespace: namespace, tweakListOptions: tweakListOptions} } +// Brokers returns a BrokerInformer. +func (v *version) Brokers() BrokerInformer { + return &brokerInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} +} + // Channels returns a ChannelInformer. func (v *version) Channels() ChannelInformer { return &channelInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} diff --git a/pkg/client/informers/externalversions/generic.go b/pkg/client/informers/externalversions/generic.go index eb07d8f78ef..8e99e8692d4 100644 --- a/pkg/client/informers/externalversions/generic.go +++ b/pkg/client/informers/externalversions/generic.go @@ -53,6 +53,8 @@ func (f *genericInformer) Lister() cache.GenericLister { func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource) (GenericInformer, error) { switch resource { // Group=eventing.knative.dev, Version=v1alpha1 + case v1alpha1.SchemeGroupVersion.WithResource("brokers"): + return &genericInformer{resource: resource.GroupResource(), informer: f.Eventing().V1alpha1().Brokers().Informer()}, nil case v1alpha1.SchemeGroupVersion.WithResource("channels"): return &genericInformer{resource: resource.GroupResource(), informer: f.Eventing().V1alpha1().Channels().Informer()}, nil case v1alpha1.SchemeGroupVersion.WithResource("clusterchannelprovisioners"): diff --git a/pkg/client/listers/eventing/v1alpha1/broker.go b/pkg/client/listers/eventing/v1alpha1/broker.go new file mode 100644 index 00000000000..4916e953b2f --- /dev/null +++ b/pkg/client/listers/eventing/v1alpha1/broker.go @@ -0,0 +1,94 @@ +/* +Copyright 2018 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by lister-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + v1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/tools/cache" +) + +// BrokerLister helps list Brokers. +type BrokerLister interface { + // List lists all Brokers in the indexer. + List(selector labels.Selector) (ret []*v1alpha1.Broker, err error) + // Brokers returns an object that can list and get Brokers. + Brokers(namespace string) BrokerNamespaceLister + BrokerListerExpansion +} + +// brokerLister implements the BrokerLister interface. +type brokerLister struct { + indexer cache.Indexer +} + +// NewBrokerLister returns a new BrokerLister. +func NewBrokerLister(indexer cache.Indexer) BrokerLister { + return &brokerLister{indexer: indexer} +} + +// List lists all Brokers in the indexer. +func (s *brokerLister) List(selector labels.Selector) (ret []*v1alpha1.Broker, err error) { + err = cache.ListAll(s.indexer, selector, func(m interface{}) { + ret = append(ret, m.(*v1alpha1.Broker)) + }) + return ret, err +} + +// Brokers returns an object that can list and get Brokers. +func (s *brokerLister) Brokers(namespace string) BrokerNamespaceLister { + return brokerNamespaceLister{indexer: s.indexer, namespace: namespace} +} + +// BrokerNamespaceLister helps list and get Brokers. +type BrokerNamespaceLister interface { + // List lists all Brokers in the indexer for a given namespace. + List(selector labels.Selector) (ret []*v1alpha1.Broker, err error) + // Get retrieves the Broker from the indexer for a given namespace and name. + Get(name string) (*v1alpha1.Broker, error) + BrokerNamespaceListerExpansion +} + +// brokerNamespaceLister implements the BrokerNamespaceLister +// interface. +type brokerNamespaceLister struct { + indexer cache.Indexer + namespace string +} + +// List lists all Brokers in the indexer for a given namespace. +func (s brokerNamespaceLister) List(selector labels.Selector) (ret []*v1alpha1.Broker, err error) { + err = cache.ListAllByNamespace(s.indexer, s.namespace, selector, func(m interface{}) { + ret = append(ret, m.(*v1alpha1.Broker)) + }) + return ret, err +} + +// Get retrieves the Broker from the indexer for a given namespace and name. +func (s brokerNamespaceLister) Get(name string) (*v1alpha1.Broker, error) { + obj, exists, err := s.indexer.GetByKey(s.namespace + "/" + name) + if err != nil { + return nil, err + } + if !exists { + return nil, errors.NewNotFound(v1alpha1.Resource("broker"), name) + } + return obj.(*v1alpha1.Broker), nil +} diff --git a/pkg/client/listers/eventing/v1alpha1/expansion_generated.go b/pkg/client/listers/eventing/v1alpha1/expansion_generated.go index ce1c8af2399..2b4ca79dad1 100644 --- a/pkg/client/listers/eventing/v1alpha1/expansion_generated.go +++ b/pkg/client/listers/eventing/v1alpha1/expansion_generated.go @@ -18,6 +18,14 @@ limitations under the License. package v1alpha1 +// BrokerListerExpansion allows custom methods to be added to +// BrokerLister. +type BrokerListerExpansion interface{} + +// BrokerNamespaceListerExpansion allows custom methods to be added to +// BrokerNamespaceLister. +type BrokerNamespaceListerExpansion interface{} + // ChannelListerExpansion allows custom methods to be added to // ChannelLister. type ChannelListerExpansion interface{} From 080f5eb8c30b2a93c49d264777262d38d99ff3e3 Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Mon, 28 Jan 2019 09:27:12 -0800 Subject: [PATCH 002/128] Metadata is exported --- pkg/apis/eventing/v1alpha1/broker_defaults.go | 2 +- pkg/apis/eventing/v1alpha1/broker_types.go | 16 ++++++++-------- pkg/apis/eventing/v1alpha1/broker_validation.go | 4 ++-- .../eventing/v1alpha1/zz_generated.deepcopy.go | 2 +- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/pkg/apis/eventing/v1alpha1/broker_defaults.go b/pkg/apis/eventing/v1alpha1/broker_defaults.go index fe795921b50..c8a23a6eda5 100644 --- a/pkg/apis/eventing/v1alpha1/broker_defaults.go +++ b/pkg/apis/eventing/v1alpha1/broker_defaults.go @@ -52,7 +52,7 @@ func defaultBrokerSpecSelector(brokerName string) *v1.LabelSelector { func defaultBrokerSpecChannelTemplate(brokerName string) *ChannelTemplateSpec { return &ChannelTemplateSpec{ - metadata: v1.ObjectMeta{ + Metadata: v1.ObjectMeta{ Labels: defaultBrokerLabels(brokerName), }, // Spec is left blank so that the created Channel defaulter will default the provisioner diff --git a/pkg/apis/eventing/v1alpha1/broker_types.go b/pkg/apis/eventing/v1alpha1/broker_types.go index 51216de7f89..08462a1fded 100644 --- a/pkg/apis/eventing/v1alpha1/broker_types.go +++ b/pkg/apis/eventing/v1alpha1/broker_types.go @@ -60,7 +60,7 @@ type BrokerSpec struct { } type ChannelTemplateSpec struct { - metadata metav1.ObjectMeta `json:"metadata,omitempty"` + Metadata metav1.ObjectMeta `json:"metadata,omitempty"` Spec *ChannelSpec `json:"spec,omitempty"` } @@ -95,35 +95,35 @@ const ( // GetCondition returns the condition currently associated with the given type, or nil. func (bs *BrokerStatus) GetCondition(t duckv1alpha1.ConditionType) *duckv1alpha1.Condition { - return chanCondSet.Manage(bs).GetCondition(t) + return brokerCondSet.Manage(bs).GetCondition(t) } // IsReady returns true if the resource is ready overall. func (bs *BrokerStatus) IsReady() bool { - return chanCondSet.Manage(bs).IsHappy() + return brokerCondSet.Manage(bs).IsHappy() } // InitializeConditions sets relevant unset conditions to Unknown state. func (bs *BrokerStatus) InitializeConditions() { - chanCondSet.Manage(bs).InitializeConditions() + brokerCondSet.Manage(bs).InitializeConditions() } func (bs *BrokerStatus) MarkChannelTemplateMatchesSelector() { - chanCondSet.Manage(bs).MarkTrue(BrokerConditionChannelTemplateSelector) + brokerCondSet.Manage(bs).MarkTrue(BrokerConditionChannelTemplateSelector) } func (bs *BrokerStatus) MarkChannelTemplateDoesNotMatchSelector() { - chanCondSet.Manage(bs).MarkFalse(BrokerConditionChannelTemplateSelector, "selectorDoesNotMatchTemplate", "`spec.selector` does not match `spec.channelTempalte.meta.labels`") + brokerCondSet.Manage(bs).MarkFalse(BrokerConditionChannelTemplateSelector, "selectorDoesNotMatchTemplate", "`spec.selector` does not match `spec.channelTempalte.meta.labels`") } // MarkProvisioned sets BrokerConditionProvisioned condition to True state. func (bs *BrokerStatus) MarkSubscribableResourcesExist() { - chanCondSet.Manage(bs).MarkTrue(BrokerConditionSubscribableResourcesExist) + brokerCondSet.Manage(bs).MarkTrue(BrokerConditionSubscribableResourcesExist) } // MarkNotProvisioned sets BrokerConditionProvisioned condition to False state. func (bs *BrokerStatus) MarkSubscribableResourcesDoNotExist(reason, messageFormat string, messageA ...interface{}) { - chanCondSet.Manage(bs).MarkFalse(BrokerConditionSubscribableResourcesExist, reason, messageFormat, messageA...) + brokerCondSet.Manage(bs).MarkFalse(BrokerConditionSubscribableResourcesExist, reason, messageFormat, messageA...) } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/pkg/apis/eventing/v1alpha1/broker_validation.go b/pkg/apis/eventing/v1alpha1/broker_validation.go index 72c7fa3c7a2..98d31a4b9cd 100644 --- a/pkg/apis/eventing/v1alpha1/broker_validation.go +++ b/pkg/apis/eventing/v1alpha1/broker_validation.go @@ -42,7 +42,7 @@ func (bs *BrokerSpec) Validate() *apis.FieldError { fe.Details = "the Broker must have a channelTemplate" errs = errs.Also(fe) } else if !selectorMatchesTemplateLabels(bs.Selector, bs.ChannelTemplate) { - fe := apis.ErrInvalidValue(fmt.Sprint(bs.ChannelTemplate.metadata.Labels), "channelTemplate.metadata.labels") + fe := apis.ErrInvalidValue(fmt.Sprint(bs.ChannelTemplate.Metadata.Labels), "channelTemplate.metadata.labels") errs = errs.Also(fe) } @@ -60,7 +60,7 @@ func isSelectorNotPresentOrEmpty(s *v1.LabelSelector) bool { func selectorMatchesTemplateLabels(s *v1.LabelSelector, ct *ChannelTemplateSpec) bool { // TODO Improve this so it supports something other than direct label equality. - return equality.Semantic.DeepEqual(s.MatchLabels, ct.metadata.Labels) + return equality.Semantic.DeepEqual(s.MatchLabels, ct.Metadata.Labels) } func channelsInSubscribableResources(sr []v1.GroupVersionKind) bool { diff --git a/pkg/apis/eventing/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/eventing/v1alpha1/zz_generated.deepcopy.go index 6481321c20e..8cbb1f99119 100644 --- a/pkg/apis/eventing/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/eventing/v1alpha1/zz_generated.deepcopy.go @@ -291,7 +291,7 @@ func (in *ChannelStatus) DeepCopy() *ChannelStatus { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ChannelTemplateSpec) DeepCopyInto(out *ChannelTemplateSpec) { *out = *in - in.metadata.DeepCopyInto(&out.metadata) + in.Metadata.DeepCopyInto(&out.Metadata) if in.Spec != nil { in, out := &in.Spec, &out.Spec if *in == nil { From f20a2df95e4b66212fb671a55b0b92d56687853a Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Mon, 28 Jan 2019 10:00:33 -0800 Subject: [PATCH 003/128] Initial Trigger API. --- .../eventing/v1alpha1/trigger_defaults.go | 27 +++ pkg/apis/eventing/v1alpha1/trigger_types.go | 138 ++++++++++++++ .../eventing/v1alpha1/trigger_validation.go | 88 +++++++++ .../v1alpha1/zz_generated.deepcopy.go | 148 +++++++++++++++ .../eventing/v1alpha1/eventing_client.go | 5 + .../v1alpha1/fake/fake_eventing_client.go | 4 + .../eventing/v1alpha1/fake/fake_trigger.go | 140 ++++++++++++++ .../eventing/v1alpha1/generated_expansion.go | 2 + .../typed/eventing/v1alpha1/trigger.go | 174 ++++++++++++++++++ .../eventing/v1alpha1/interface.go | 7 + .../eventing/v1alpha1/trigger.go | 89 +++++++++ .../informers/externalversions/generic.go | 2 + .../eventing/v1alpha1/expansion_generated.go | 8 + .../listers/eventing/v1alpha1/trigger.go | 94 ++++++++++ 14 files changed, 926 insertions(+) create mode 100644 pkg/apis/eventing/v1alpha1/trigger_defaults.go create mode 100644 pkg/apis/eventing/v1alpha1/trigger_types.go create mode 100644 pkg/apis/eventing/v1alpha1/trigger_validation.go create mode 100644 pkg/client/clientset/versioned/typed/eventing/v1alpha1/fake/fake_trigger.go create mode 100644 pkg/client/clientset/versioned/typed/eventing/v1alpha1/trigger.go create mode 100644 pkg/client/informers/externalversions/eventing/v1alpha1/trigger.go create mode 100644 pkg/client/listers/eventing/v1alpha1/trigger.go diff --git a/pkg/apis/eventing/v1alpha1/trigger_defaults.go b/pkg/apis/eventing/v1alpha1/trigger_defaults.go new file mode 100644 index 00000000000..732d9fde924 --- /dev/null +++ b/pkg/apis/eventing/v1alpha1/trigger_defaults.go @@ -0,0 +1,27 @@ +/* +Copyright 2019 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +func (t *Trigger) SetDefaults() { + t.Spec.SetDefaults() +} + +func (ts *TriggerSpec) SetDefaults() { + if ts.Broker == "" { + ts.Broker = "default" + } +} diff --git a/pkg/apis/eventing/v1alpha1/trigger_types.go b/pkg/apis/eventing/v1alpha1/trigger_types.go new file mode 100644 index 00000000000..8eecf20e6fc --- /dev/null +++ b/pkg/apis/eventing/v1alpha1/trigger_types.go @@ -0,0 +1,138 @@ +/* + * Copyright 2019 The Knative Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package v1alpha1 + +import ( + "github.com/knative/pkg/apis" + duckv1alpha1 "github.com/knative/pkg/apis/duck/v1alpha1" + "github.com/knative/pkg/webhook" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +// +genclient +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// Trigger is an abstract resource that implements the Addressable contract. +// The Provisioner provisions infrastructure to accepts events and +// deliver to Subscriptions. +type Trigger struct { + metav1.TypeMeta `json:",inline"` + // +optional + metav1.ObjectMeta `json:"metadata,omitempty"` + + // Spec defines the desired state of the Trigger. + Spec TriggerSpec `json:"spec,omitempty"` + + // Status represents the current state of the Trigger. This data may be out of + // date. + // +optional + Status TriggerStatus `json:"status,omitempty"` +} + +// Check that Trigger can be validated, can be defaulted, and has immutable fields. +var _ apis.Validatable = (*Trigger)(nil) +var _ apis.Defaultable = (*Trigger)(nil) +var _ apis.Immutable = (*Trigger)(nil) +var _ runtime.Object = (*Trigger)(nil) +var _ webhook.GenericCRD = (*Trigger)(nil) + +// TriggerSpec specifies the Provisioner backing a channel and the configuration +// arguments for a Trigger. +type TriggerSpec struct { + Broker string `json:"broker,omitempty"` + Selector *TriggerSelectorSpec `json:"selector,omitempty"` + Subscriber *SubscriberSpec `json:"subscriber,omitempty"` +} + +type TriggerSelectorSpec struct { + Header map[string]string `json:"header,omitempty"` + HeaderExpression []metav1.LabelSelectorRequirement `json:"headerExpression,omitEmpty"` + OPAPolicy string `json:"opaPolicy,omitEmpty"` +} + +var triggerCondSet = duckv1alpha1.NewLivingConditionSet(TriggerConditionBrokerExists, TriggerConditionSubscriberFound) + +// TriggerStatus represents the current state of a Trigger. +type TriggerStatus struct { + // ObservedGeneration is the most recent generation observed for this Trigger. + // It corresponds to the Trigger's generation, which is updated on mutation by + // the API Server. + // TODO: The above comment is only true once + // https://github.com/kubernetes/kubernetes/issues/58778 is fixed. + // +optional + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + + // Represents the latest available observations of a channel's current state. + // +optional + // +patchMergeKey=type + // +patchStrategy=merge + Conditions duckv1alpha1.Conditions `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"` +} + +const ( + // TriggerConditionReady has status True when the Trigger is ready to + // accept traffic. + TriggerConditionReady = duckv1alpha1.ConditionReady + + TriggerConditionBrokerExists duckv1alpha1.ConditionType = "BrokerExists" + + TriggerConditionSubscriberFound duckv1alpha1.ConditionType = "SubscriberFound" +) + +// GetCondition returns the condition currently associated with the given type, or nil. +func (ts *TriggerStatus) GetCondition(t duckv1alpha1.ConditionType) *duckv1alpha1.Condition { + return triggerCondSet.Manage(ts).GetCondition(t) +} + +// IsReady returns true if the resource is ready overall. +func (ts *TriggerStatus) IsReady() bool { + return triggerCondSet.Manage(ts).IsHappy() +} + +// InitializeConditions sets relevant unset conditions to Unknown state. +func (ts *TriggerStatus) InitializeConditions() { + triggerCondSet.Manage(ts).InitializeConditions() +} + +func (ts *TriggerStatus) MarkBrokerExists() { + triggerCondSet.Manage(ts).MarkTrue(TriggerConditionBrokerExists) +} + +func (ts *TriggerStatus) MarkBrokerDoesNotExists() { + triggerCondSet.Manage(ts).MarkFalse(TriggerConditionBrokerExists, "doesNotExist", "Broker does not exist") +} + +// MarkProvisioned sets TriggerConditionProvisioned condition to True state. +func (ts *TriggerStatus) MarkSubscriberFound() { + triggerCondSet.Manage(ts).MarkTrue(TriggerConditionSubscriberFound) +} + +// MarkNotProvisioned sets TriggerConditionProvisioned condition to False state. +func (ts *TriggerStatus) MarkSubscriberNotFound(reason, messageFormat string, messageA ...interface{}) { + triggerCondSet.Manage(ts).MarkFalse(TriggerConditionSubscriberFound, reason, messageFormat, messageA...) +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// TriggerList is a collection of Triggers. +type TriggerList struct { + metav1.TypeMeta `json:",inline"` + // +optional + metav1.ListMeta `json:"metadata,omitempty"` + Items []Trigger `json:"items"` +} diff --git a/pkg/apis/eventing/v1alpha1/trigger_validation.go b/pkg/apis/eventing/v1alpha1/trigger_validation.go new file mode 100644 index 00000000000..6ff7301e244 --- /dev/null +++ b/pkg/apis/eventing/v1alpha1/trigger_validation.go @@ -0,0 +1,88 @@ +/* +Copyright 2019 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "github.com/google/go-cmp/cmp" + "github.com/knative/pkg/apis" + "k8s.io/apimachinery/pkg/api/equality" +) + +func (t *Trigger) Validate() *apis.FieldError { + return t.Spec.Validate().ViaField("spec") +} + +func (ts *TriggerSpec) Validate() *apis.FieldError { + var errs *apis.FieldError + if ts.Broker == "" { + fe := apis.ErrMissingField("broker") + errs = errs.Also(fe) + } + + if ts.Selector == nil { + fe := apis.ErrMissingField("selector") + errs = errs.Also(fe) + } else if fe := multipleSelectorsSet(ts.Selector); fe != nil { + errs = errs.Also(fe) + } + + if isSubscriberSpecNilOrEmpty(ts.Subscriber) { + fe := apis.ErrMissingField("subscriber") + errs = errs.Also(fe) + } else if fe := isValidSubscriberSpec(*ts.Subscriber); fe != nil { + errs = errs.Also(fe.ViaField("subscriber")) + } + + return errs +} + +func multipleSelectorsSet(s *TriggerSelectorSpec) *apis.FieldError { + var fields []string + if !equality.Semantic.DeepEqual(s.Header, map[string]string{}) { + fields = append(fields, "header") + } + if len(s.HeaderExpression) > 0 { + fields = append(fields, "headerExpression") + } + if s.OPAPolicy != "" { + fields = append(fields, "opaPolicy") + } + + if len(fields) != 1 { + return apis.ErrMultipleOneOf(fields...) + } + return nil +} + +func (t *Trigger) CheckImmutableFields(og apis.Immutable) *apis.FieldError { + original, ok := og.(*Trigger) + if !ok { + return &apis.FieldError{Message: "The provided original was not a Trigger"} + } + if original == nil { + return nil + } + + if diff := cmp.Diff(original.Spec.Broker, t.Spec.Broker); diff != "" { + return &apis.FieldError{ + Message: "Immutable fields changed (-old +new)", + Paths: []string{"spec", "broker"}, + Details: diff, + } + } + return nil +} diff --git a/pkg/apis/eventing/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/eventing/v1alpha1/zz_generated.deepcopy.go index 8cbb1f99119..77046af1c1d 100644 --- a/pkg/apis/eventing/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/eventing/v1alpha1/zz_generated.deepcopy.go @@ -608,3 +608,151 @@ func (in *SubscriptionStatusPhysicalSubscription) DeepCopy() *SubscriptionStatus in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Trigger) DeepCopyInto(out *Trigger) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Trigger. +func (in *Trigger) DeepCopy() *Trigger { + if in == nil { + return nil + } + out := new(Trigger) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Trigger) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TriggerList) DeepCopyInto(out *TriggerList) { + *out = *in + out.TypeMeta = in.TypeMeta + out.ListMeta = in.ListMeta + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Trigger, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TriggerList. +func (in *TriggerList) DeepCopy() *TriggerList { + if in == nil { + return nil + } + out := new(TriggerList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *TriggerList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TriggerSelectorSpec) DeepCopyInto(out *TriggerSelectorSpec) { + *out = *in + if in.Header != nil { + in, out := &in.Header, &out.Header + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.HeaderExpression != nil { + in, out := &in.HeaderExpression, &out.HeaderExpression + *out = make([]v1.LabelSelectorRequirement, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TriggerSelectorSpec. +func (in *TriggerSelectorSpec) DeepCopy() *TriggerSelectorSpec { + if in == nil { + return nil + } + out := new(TriggerSelectorSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TriggerSpec) DeepCopyInto(out *TriggerSpec) { + *out = *in + if in.Selector != nil { + in, out := &in.Selector, &out.Selector + if *in == nil { + *out = nil + } else { + *out = new(TriggerSelectorSpec) + (*in).DeepCopyInto(*out) + } + } + if in.Subscriber != nil { + in, out := &in.Subscriber, &out.Subscriber + if *in == nil { + *out = nil + } else { + *out = new(SubscriberSpec) + (*in).DeepCopyInto(*out) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TriggerSpec. +func (in *TriggerSpec) DeepCopy() *TriggerSpec { + if in == nil { + return nil + } + out := new(TriggerSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TriggerStatus) DeepCopyInto(out *TriggerStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make(duck_v1alpha1.Conditions, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TriggerStatus. +func (in *TriggerStatus) DeepCopy() *TriggerStatus { + if in == nil { + return nil + } + out := new(TriggerStatus) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/client/clientset/versioned/typed/eventing/v1alpha1/eventing_client.go b/pkg/client/clientset/versioned/typed/eventing/v1alpha1/eventing_client.go index 8a2796c7417..753d1081b84 100644 --- a/pkg/client/clientset/versioned/typed/eventing/v1alpha1/eventing_client.go +++ b/pkg/client/clientset/versioned/typed/eventing/v1alpha1/eventing_client.go @@ -31,6 +31,7 @@ type EventingV1alpha1Interface interface { ChannelsGetter ClusterChannelProvisionersGetter SubscriptionsGetter + TriggersGetter } // EventingV1alpha1Client is used to interact with features provided by the eventing.knative.dev group. @@ -54,6 +55,10 @@ func (c *EventingV1alpha1Client) Subscriptions(namespace string) SubscriptionInt return newSubscriptions(c, namespace) } +func (c *EventingV1alpha1Client) Triggers(namespace string) TriggerInterface { + return newTriggers(c, namespace) +} + // NewForConfig creates a new EventingV1alpha1Client for the given config. func NewForConfig(c *rest.Config) (*EventingV1alpha1Client, error) { config := *c diff --git a/pkg/client/clientset/versioned/typed/eventing/v1alpha1/fake/fake_eventing_client.go b/pkg/client/clientset/versioned/typed/eventing/v1alpha1/fake/fake_eventing_client.go index 68998cd28f5..4362e785f5a 100644 --- a/pkg/client/clientset/versioned/typed/eventing/v1alpha1/fake/fake_eventing_client.go +++ b/pkg/client/clientset/versioned/typed/eventing/v1alpha1/fake/fake_eventing_client.go @@ -44,6 +44,10 @@ func (c *FakeEventingV1alpha1) Subscriptions(namespace string) v1alpha1.Subscrip return &FakeSubscriptions{c, namespace} } +func (c *FakeEventingV1alpha1) Triggers(namespace string) v1alpha1.TriggerInterface { + return &FakeTriggers{c, namespace} +} + // RESTClient returns a RESTClient that is used to communicate // with API server by this client implementation. func (c *FakeEventingV1alpha1) RESTClient() rest.Interface { diff --git a/pkg/client/clientset/versioned/typed/eventing/v1alpha1/fake/fake_trigger.go b/pkg/client/clientset/versioned/typed/eventing/v1alpha1/fake/fake_trigger.go new file mode 100644 index 00000000000..5e4b588c6b1 --- /dev/null +++ b/pkg/client/clientset/versioned/typed/eventing/v1alpha1/fake/fake_trigger.go @@ -0,0 +1,140 @@ +/* +Copyright 2018 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + v1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + labels "k8s.io/apimachinery/pkg/labels" + schema "k8s.io/apimachinery/pkg/runtime/schema" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + testing "k8s.io/client-go/testing" +) + +// FakeTriggers implements TriggerInterface +type FakeTriggers struct { + Fake *FakeEventingV1alpha1 + ns string +} + +var triggersResource = schema.GroupVersionResource{Group: "eventing.knative.dev", Version: "v1alpha1", Resource: "triggers"} + +var triggersKind = schema.GroupVersionKind{Group: "eventing.knative.dev", Version: "v1alpha1", Kind: "Trigger"} + +// Get takes name of the trigger, and returns the corresponding trigger object, and an error if there is any. +func (c *FakeTriggers) Get(name string, options v1.GetOptions) (result *v1alpha1.Trigger, err error) { + obj, err := c.Fake. + Invokes(testing.NewGetAction(triggersResource, c.ns, name), &v1alpha1.Trigger{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.Trigger), err +} + +// List takes label and field selectors, and returns the list of Triggers that match those selectors. +func (c *FakeTriggers) List(opts v1.ListOptions) (result *v1alpha1.TriggerList, err error) { + obj, err := c.Fake. + Invokes(testing.NewListAction(triggersResource, triggersKind, c.ns, opts), &v1alpha1.TriggerList{}) + + if obj == nil { + return nil, err + } + + label, _, _ := testing.ExtractFromListOptions(opts) + if label == nil { + label = labels.Everything() + } + list := &v1alpha1.TriggerList{ListMeta: obj.(*v1alpha1.TriggerList).ListMeta} + for _, item := range obj.(*v1alpha1.TriggerList).Items { + if label.Matches(labels.Set(item.Labels)) { + list.Items = append(list.Items, item) + } + } + return list, err +} + +// Watch returns a watch.Interface that watches the requested triggers. +func (c *FakeTriggers) Watch(opts v1.ListOptions) (watch.Interface, error) { + return c.Fake. + InvokesWatch(testing.NewWatchAction(triggersResource, c.ns, opts)) + +} + +// Create takes the representation of a trigger and creates it. Returns the server's representation of the trigger, and an error, if there is any. +func (c *FakeTriggers) Create(trigger *v1alpha1.Trigger) (result *v1alpha1.Trigger, err error) { + obj, err := c.Fake. + Invokes(testing.NewCreateAction(triggersResource, c.ns, trigger), &v1alpha1.Trigger{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.Trigger), err +} + +// Update takes the representation of a trigger and updates it. Returns the server's representation of the trigger, and an error, if there is any. +func (c *FakeTriggers) Update(trigger *v1alpha1.Trigger) (result *v1alpha1.Trigger, err error) { + obj, err := c.Fake. + Invokes(testing.NewUpdateAction(triggersResource, c.ns, trigger), &v1alpha1.Trigger{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.Trigger), err +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *FakeTriggers) UpdateStatus(trigger *v1alpha1.Trigger) (*v1alpha1.Trigger, error) { + obj, err := c.Fake. + Invokes(testing.NewUpdateSubresourceAction(triggersResource, "status", c.ns, trigger), &v1alpha1.Trigger{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.Trigger), err +} + +// Delete takes name of the trigger and deletes it. Returns an error if one occurs. +func (c *FakeTriggers) Delete(name string, options *v1.DeleteOptions) error { + _, err := c.Fake. + Invokes(testing.NewDeleteAction(triggersResource, c.ns, name), &v1alpha1.Trigger{}) + + return err +} + +// DeleteCollection deletes a collection of objects. +func (c *FakeTriggers) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + action := testing.NewDeleteCollectionAction(triggersResource, c.ns, listOptions) + + _, err := c.Fake.Invokes(action, &v1alpha1.TriggerList{}) + return err +} + +// Patch applies the patch and returns the patched trigger. +func (c *FakeTriggers) Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1alpha1.Trigger, err error) { + obj, err := c.Fake. + Invokes(testing.NewPatchSubresourceAction(triggersResource, c.ns, name, data, subresources...), &v1alpha1.Trigger{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.Trigger), err +} diff --git a/pkg/client/clientset/versioned/typed/eventing/v1alpha1/generated_expansion.go b/pkg/client/clientset/versioned/typed/eventing/v1alpha1/generated_expansion.go index 94933cfbc74..2f88fb3320b 100644 --- a/pkg/client/clientset/versioned/typed/eventing/v1alpha1/generated_expansion.go +++ b/pkg/client/clientset/versioned/typed/eventing/v1alpha1/generated_expansion.go @@ -25,3 +25,5 @@ type ChannelExpansion interface{} type ClusterChannelProvisionerExpansion interface{} type SubscriptionExpansion interface{} + +type TriggerExpansion interface{} diff --git a/pkg/client/clientset/versioned/typed/eventing/v1alpha1/trigger.go b/pkg/client/clientset/versioned/typed/eventing/v1alpha1/trigger.go new file mode 100644 index 00000000000..72207e79d34 --- /dev/null +++ b/pkg/client/clientset/versioned/typed/eventing/v1alpha1/trigger.go @@ -0,0 +1,174 @@ +/* +Copyright 2018 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + v1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" + scheme "github.com/knative/eventing/pkg/client/clientset/versioned/scheme" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + rest "k8s.io/client-go/rest" +) + +// TriggersGetter has a method to return a TriggerInterface. +// A group's client should implement this interface. +type TriggersGetter interface { + Triggers(namespace string) TriggerInterface +} + +// TriggerInterface has methods to work with Trigger resources. +type TriggerInterface interface { + Create(*v1alpha1.Trigger) (*v1alpha1.Trigger, error) + Update(*v1alpha1.Trigger) (*v1alpha1.Trigger, error) + UpdateStatus(*v1alpha1.Trigger) (*v1alpha1.Trigger, error) + Delete(name string, options *v1.DeleteOptions) error + DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error + Get(name string, options v1.GetOptions) (*v1alpha1.Trigger, error) + List(opts v1.ListOptions) (*v1alpha1.TriggerList, error) + Watch(opts v1.ListOptions) (watch.Interface, error) + Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1alpha1.Trigger, err error) + TriggerExpansion +} + +// triggers implements TriggerInterface +type triggers struct { + client rest.Interface + ns string +} + +// newTriggers returns a Triggers +func newTriggers(c *EventingV1alpha1Client, namespace string) *triggers { + return &triggers{ + client: c.RESTClient(), + ns: namespace, + } +} + +// Get takes name of the trigger, and returns the corresponding trigger object, and an error if there is any. +func (c *triggers) Get(name string, options v1.GetOptions) (result *v1alpha1.Trigger, err error) { + result = &v1alpha1.Trigger{} + err = c.client.Get(). + Namespace(c.ns). + Resource("triggers"). + Name(name). + VersionedParams(&options, scheme.ParameterCodec). + Do(). + Into(result) + return +} + +// List takes label and field selectors, and returns the list of Triggers that match those selectors. +func (c *triggers) List(opts v1.ListOptions) (result *v1alpha1.TriggerList, err error) { + result = &v1alpha1.TriggerList{} + err = c.client.Get(). + Namespace(c.ns). + Resource("triggers"). + VersionedParams(&opts, scheme.ParameterCodec). + Do(). + Into(result) + return +} + +// Watch returns a watch.Interface that watches the requested triggers. +func (c *triggers) Watch(opts v1.ListOptions) (watch.Interface, error) { + opts.Watch = true + return c.client.Get(). + Namespace(c.ns). + Resource("triggers"). + VersionedParams(&opts, scheme.ParameterCodec). + Watch() +} + +// Create takes the representation of a trigger and creates it. Returns the server's representation of the trigger, and an error, if there is any. +func (c *triggers) Create(trigger *v1alpha1.Trigger) (result *v1alpha1.Trigger, err error) { + result = &v1alpha1.Trigger{} + err = c.client.Post(). + Namespace(c.ns). + Resource("triggers"). + Body(trigger). + Do(). + Into(result) + return +} + +// Update takes the representation of a trigger and updates it. Returns the server's representation of the trigger, and an error, if there is any. +func (c *triggers) Update(trigger *v1alpha1.Trigger) (result *v1alpha1.Trigger, err error) { + result = &v1alpha1.Trigger{} + err = c.client.Put(). + Namespace(c.ns). + Resource("triggers"). + Name(trigger.Name). + Body(trigger). + Do(). + Into(result) + return +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). + +func (c *triggers) UpdateStatus(trigger *v1alpha1.Trigger) (result *v1alpha1.Trigger, err error) { + result = &v1alpha1.Trigger{} + err = c.client.Put(). + Namespace(c.ns). + Resource("triggers"). + Name(trigger.Name). + SubResource("status"). + Body(trigger). + Do(). + Into(result) + return +} + +// Delete takes name of the trigger and deletes it. Returns an error if one occurs. +func (c *triggers) Delete(name string, options *v1.DeleteOptions) error { + return c.client.Delete(). + Namespace(c.ns). + Resource("triggers"). + Name(name). + Body(options). + Do(). + Error() +} + +// DeleteCollection deletes a collection of objects. +func (c *triggers) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + return c.client.Delete(). + Namespace(c.ns). + Resource("triggers"). + VersionedParams(&listOptions, scheme.ParameterCodec). + Body(options). + Do(). + Error() +} + +// Patch applies the patch and returns the patched trigger. +func (c *triggers) Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1alpha1.Trigger, err error) { + result = &v1alpha1.Trigger{} + err = c.client.Patch(pt). + Namespace(c.ns). + Resource("triggers"). + SubResource(subresources...). + Name(name). + Body(data). + Do(). + Into(result) + return +} diff --git a/pkg/client/informers/externalversions/eventing/v1alpha1/interface.go b/pkg/client/informers/externalversions/eventing/v1alpha1/interface.go index 9d7b654e259..29ad6f191f0 100644 --- a/pkg/client/informers/externalversions/eventing/v1alpha1/interface.go +++ b/pkg/client/informers/externalversions/eventing/v1alpha1/interface.go @@ -32,6 +32,8 @@ type Interface interface { ClusterChannelProvisioners() ClusterChannelProvisionerInformer // Subscriptions returns a SubscriptionInformer. Subscriptions() SubscriptionInformer + // Triggers returns a TriggerInformer. + Triggers() TriggerInformer } type version struct { @@ -64,3 +66,8 @@ func (v *version) ClusterChannelProvisioners() ClusterChannelProvisionerInformer func (v *version) Subscriptions() SubscriptionInformer { return &subscriptionInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} } + +// Triggers returns a TriggerInformer. +func (v *version) Triggers() TriggerInformer { + return &triggerInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} +} diff --git a/pkg/client/informers/externalversions/eventing/v1alpha1/trigger.go b/pkg/client/informers/externalversions/eventing/v1alpha1/trigger.go new file mode 100644 index 00000000000..c1b01ef002b --- /dev/null +++ b/pkg/client/informers/externalversions/eventing/v1alpha1/trigger.go @@ -0,0 +1,89 @@ +/* +Copyright 2018 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by informer-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + time "time" + + eventing_v1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" + versioned "github.com/knative/eventing/pkg/client/clientset/versioned" + internalinterfaces "github.com/knative/eventing/pkg/client/informers/externalversions/internalinterfaces" + v1alpha1 "github.com/knative/eventing/pkg/client/listers/eventing/v1alpha1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + watch "k8s.io/apimachinery/pkg/watch" + cache "k8s.io/client-go/tools/cache" +) + +// TriggerInformer provides access to a shared informer and lister for +// Triggers. +type TriggerInformer interface { + Informer() cache.SharedIndexInformer + Lister() v1alpha1.TriggerLister +} + +type triggerInformer struct { + factory internalinterfaces.SharedInformerFactory + tweakListOptions internalinterfaces.TweakListOptionsFunc + namespace string +} + +// NewTriggerInformer constructs a new informer for Trigger type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewTriggerInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { + return NewFilteredTriggerInformer(client, namespace, resyncPeriod, indexers, nil) +} + +// NewFilteredTriggerInformer constructs a new informer for Trigger type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFilteredTriggerInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { + return cache.NewSharedIndexInformer( + &cache.ListWatch{ + ListFunc: func(options v1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.EventingV1alpha1().Triggers(namespace).List(options) + }, + WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.EventingV1alpha1().Triggers(namespace).Watch(options) + }, + }, + &eventing_v1alpha1.Trigger{}, + resyncPeriod, + indexers, + ) +} + +func (f *triggerInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { + return NewFilteredTriggerInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) +} + +func (f *triggerInformer) Informer() cache.SharedIndexInformer { + return f.factory.InformerFor(&eventing_v1alpha1.Trigger{}, f.defaultInformer) +} + +func (f *triggerInformer) Lister() v1alpha1.TriggerLister { + return v1alpha1.NewTriggerLister(f.Informer().GetIndexer()) +} diff --git a/pkg/client/informers/externalversions/generic.go b/pkg/client/informers/externalversions/generic.go index 8e99e8692d4..6c22f720af9 100644 --- a/pkg/client/informers/externalversions/generic.go +++ b/pkg/client/informers/externalversions/generic.go @@ -61,6 +61,8 @@ func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource return &genericInformer{resource: resource.GroupResource(), informer: f.Eventing().V1alpha1().ClusterChannelProvisioners().Informer()}, nil case v1alpha1.SchemeGroupVersion.WithResource("subscriptions"): return &genericInformer{resource: resource.GroupResource(), informer: f.Eventing().V1alpha1().Subscriptions().Informer()}, nil + case v1alpha1.SchemeGroupVersion.WithResource("triggers"): + return &genericInformer{resource: resource.GroupResource(), informer: f.Eventing().V1alpha1().Triggers().Informer()}, nil } diff --git a/pkg/client/listers/eventing/v1alpha1/expansion_generated.go b/pkg/client/listers/eventing/v1alpha1/expansion_generated.go index 2b4ca79dad1..2a49e1b434b 100644 --- a/pkg/client/listers/eventing/v1alpha1/expansion_generated.go +++ b/pkg/client/listers/eventing/v1alpha1/expansion_generated.go @@ -45,3 +45,11 @@ type SubscriptionListerExpansion interface{} // SubscriptionNamespaceListerExpansion allows custom methods to be added to // SubscriptionNamespaceLister. type SubscriptionNamespaceListerExpansion interface{} + +// TriggerListerExpansion allows custom methods to be added to +// TriggerLister. +type TriggerListerExpansion interface{} + +// TriggerNamespaceListerExpansion allows custom methods to be added to +// TriggerNamespaceLister. +type TriggerNamespaceListerExpansion interface{} diff --git a/pkg/client/listers/eventing/v1alpha1/trigger.go b/pkg/client/listers/eventing/v1alpha1/trigger.go new file mode 100644 index 00000000000..56c323a8238 --- /dev/null +++ b/pkg/client/listers/eventing/v1alpha1/trigger.go @@ -0,0 +1,94 @@ +/* +Copyright 2018 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by lister-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + v1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/tools/cache" +) + +// TriggerLister helps list Triggers. +type TriggerLister interface { + // List lists all Triggers in the indexer. + List(selector labels.Selector) (ret []*v1alpha1.Trigger, err error) + // Triggers returns an object that can list and get Triggers. + Triggers(namespace string) TriggerNamespaceLister + TriggerListerExpansion +} + +// triggerLister implements the TriggerLister interface. +type triggerLister struct { + indexer cache.Indexer +} + +// NewTriggerLister returns a new TriggerLister. +func NewTriggerLister(indexer cache.Indexer) TriggerLister { + return &triggerLister{indexer: indexer} +} + +// List lists all Triggers in the indexer. +func (s *triggerLister) List(selector labels.Selector) (ret []*v1alpha1.Trigger, err error) { + err = cache.ListAll(s.indexer, selector, func(m interface{}) { + ret = append(ret, m.(*v1alpha1.Trigger)) + }) + return ret, err +} + +// Triggers returns an object that can list and get Triggers. +func (s *triggerLister) Triggers(namespace string) TriggerNamespaceLister { + return triggerNamespaceLister{indexer: s.indexer, namespace: namespace} +} + +// TriggerNamespaceLister helps list and get Triggers. +type TriggerNamespaceLister interface { + // List lists all Triggers in the indexer for a given namespace. + List(selector labels.Selector) (ret []*v1alpha1.Trigger, err error) + // Get retrieves the Trigger from the indexer for a given namespace and name. + Get(name string) (*v1alpha1.Trigger, error) + TriggerNamespaceListerExpansion +} + +// triggerNamespaceLister implements the TriggerNamespaceLister +// interface. +type triggerNamespaceLister struct { + indexer cache.Indexer + namespace string +} + +// List lists all Triggers in the indexer for a given namespace. +func (s triggerNamespaceLister) List(selector labels.Selector) (ret []*v1alpha1.Trigger, err error) { + err = cache.ListAllByNamespace(s.indexer, s.namespace, selector, func(m interface{}) { + ret = append(ret, m.(*v1alpha1.Trigger)) + }) + return ret, err +} + +// Get retrieves the Trigger from the indexer for a given namespace and name. +func (s triggerNamespaceLister) Get(name string) (*v1alpha1.Trigger, error) { + obj, exists, err := s.indexer.GetByKey(s.namespace + "/" + name) + if err != nil { + return nil, err + } + if !exists { + return nil, errors.NewNotFound(v1alpha1.Resource("trigger"), name) + } + return obj.(*v1alpha1.Trigger), nil +} From 163474d1e414e44bfbeb29bb6c4a32a00232e123 Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Mon, 28 Jan 2019 10:05:45 -0800 Subject: [PATCH 004/128] Remove old comments. --- pkg/apis/eventing/v1alpha1/broker_types.go | 11 +---------- pkg/apis/eventing/v1alpha1/trigger_types.go | 11 +---------- 2 files changed, 2 insertions(+), 20 deletions(-) diff --git a/pkg/apis/eventing/v1alpha1/broker_types.go b/pkg/apis/eventing/v1alpha1/broker_types.go index 08462a1fded..16a41df1c44 100644 --- a/pkg/apis/eventing/v1alpha1/broker_types.go +++ b/pkg/apis/eventing/v1alpha1/broker_types.go @@ -27,9 +27,6 @@ import ( // +genclient // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object -// Broker is an abstract resource that implements the Addressable contract. -// The Provisioner provisions infrastructure to accepts events and -// deliver to Subscriptions. type Broker struct { metav1.TypeMeta `json:",inline"` // +optional @@ -51,8 +48,6 @@ var _ apis.Immutable = (*Broker)(nil) var _ runtime.Object = (*Broker)(nil) var _ webhook.GenericCRD = (*Broker)(nil) -// BrokerSpec specifies the Provisioner backing a channel and the configuration -// arguments for a Broker. type BrokerSpec struct { Selector *metav1.LabelSelector `json:"selector,omitempty"` ChannelTemplate *ChannelTemplateSpec `json:"channelTemplate,omitempty"` @@ -76,7 +71,7 @@ type BrokerStatus struct { // +optional ObservedGeneration int64 `json:"observedGeneration,omitempty"` - // Represents the latest available observations of a channel's current state. + // Represents the latest available observations of a broker's current state. // +optional // +patchMergeKey=type // +patchStrategy=merge @@ -84,8 +79,6 @@ type BrokerStatus struct { } const ( - // BrokerConditionReady has status True when the Broker is ready to - // accept traffic. BrokerConditionReady = duckv1alpha1.ConditionReady BrokerConditionChannelTemplateSelector duckv1alpha1.ConditionType = "ChannelTemplateSelector" @@ -116,12 +109,10 @@ func (bs *BrokerStatus) MarkChannelTemplateDoesNotMatchSelector() { brokerCondSet.Manage(bs).MarkFalse(BrokerConditionChannelTemplateSelector, "selectorDoesNotMatchTemplate", "`spec.selector` does not match `spec.channelTempalte.meta.labels`") } -// MarkProvisioned sets BrokerConditionProvisioned condition to True state. func (bs *BrokerStatus) MarkSubscribableResourcesExist() { brokerCondSet.Manage(bs).MarkTrue(BrokerConditionSubscribableResourcesExist) } -// MarkNotProvisioned sets BrokerConditionProvisioned condition to False state. func (bs *BrokerStatus) MarkSubscribableResourcesDoNotExist(reason, messageFormat string, messageA ...interface{}) { brokerCondSet.Manage(bs).MarkFalse(BrokerConditionSubscribableResourcesExist, reason, messageFormat, messageA...) } diff --git a/pkg/apis/eventing/v1alpha1/trigger_types.go b/pkg/apis/eventing/v1alpha1/trigger_types.go index 8eecf20e6fc..82ac3aa648f 100644 --- a/pkg/apis/eventing/v1alpha1/trigger_types.go +++ b/pkg/apis/eventing/v1alpha1/trigger_types.go @@ -27,9 +27,6 @@ import ( // +genclient // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object -// Trigger is an abstract resource that implements the Addressable contract. -// The Provisioner provisions infrastructure to accepts events and -// deliver to Subscriptions. type Trigger struct { metav1.TypeMeta `json:",inline"` // +optional @@ -51,8 +48,6 @@ var _ apis.Immutable = (*Trigger)(nil) var _ runtime.Object = (*Trigger)(nil) var _ webhook.GenericCRD = (*Trigger)(nil) -// TriggerSpec specifies the Provisioner backing a channel and the configuration -// arguments for a Trigger. type TriggerSpec struct { Broker string `json:"broker,omitempty"` Selector *TriggerSelectorSpec `json:"selector,omitempty"` @@ -77,7 +72,7 @@ type TriggerStatus struct { // +optional ObservedGeneration int64 `json:"observedGeneration,omitempty"` - // Represents the latest available observations of a channel's current state. + // Represents the latest available observations of a trigger's current state. // +optional // +patchMergeKey=type // +patchStrategy=merge @@ -85,8 +80,6 @@ type TriggerStatus struct { } const ( - // TriggerConditionReady has status True when the Trigger is ready to - // accept traffic. TriggerConditionReady = duckv1alpha1.ConditionReady TriggerConditionBrokerExists duckv1alpha1.ConditionType = "BrokerExists" @@ -117,12 +110,10 @@ func (ts *TriggerStatus) MarkBrokerDoesNotExists() { triggerCondSet.Manage(ts).MarkFalse(TriggerConditionBrokerExists, "doesNotExist", "Broker does not exist") } -// MarkProvisioned sets TriggerConditionProvisioned condition to True state. func (ts *TriggerStatus) MarkSubscriberFound() { triggerCondSet.Manage(ts).MarkTrue(TriggerConditionSubscriberFound) } -// MarkNotProvisioned sets TriggerConditionProvisioned condition to False state. func (ts *TriggerStatus) MarkSubscriberNotFound(reason, messageFormat string, messageA ...interface{}) { triggerCondSet.Manage(ts).MarkFalse(TriggerConditionSubscriberFound, reason, messageFormat, messageA...) } From 5f53e05385bf1450eb33453369973a49ad05f7df Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Mon, 28 Jan 2019 10:12:25 -0800 Subject: [PATCH 005/128] Register the types. --- cmd/webhook/main.go | 2 ++ config/300-broker.yaml | 38 ++++++++++++++++++++++++++ config/300-trigger.yaml | 38 ++++++++++++++++++++++++++ pkg/apis/eventing/v1alpha1/register.go | 4 +++ 4 files changed, 82 insertions(+) create mode 100644 config/300-broker.yaml create mode 100644 config/300-trigger.yaml diff --git a/cmd/webhook/main.go b/cmd/webhook/main.go index e35ff3d7304..400a452cd9a 100644 --- a/cmd/webhook/main.go +++ b/cmd/webhook/main.go @@ -96,9 +96,11 @@ func main() { Options: options, Handlers: map[schema.GroupVersionKind]webhook.GenericCRD{ // For group eventing.knative.dev, + eventingv1alpha1.SchemeGroupVersion.WithKind("Broker"): &eventingv1alpha1.Broker{}, eventingv1alpha1.SchemeGroupVersion.WithKind("Channel"): &eventingv1alpha1.Channel{}, eventingv1alpha1.SchemeGroupVersion.WithKind("ClusterChannelProvisioner"): &eventingv1alpha1.ClusterChannelProvisioner{}, eventingv1alpha1.SchemeGroupVersion.WithKind("Subscription"): &eventingv1alpha1.Subscription{}, + eventingv1alpha1.SchemeGroupVersion.WithKind("Trigger"): &eventingv1alpha1.Trigger{}, }, Logger: logger, } diff --git a/config/300-broker.yaml b/config/300-broker.yaml new file mode 100644 index 00000000000..ce0421a7b66 --- /dev/null +++ b/config/300-broker.yaml @@ -0,0 +1,38 @@ +# Copyright 2019 The Knative Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: brokers.eventing.knative.dev +spec: + group: eventing.knative.dev + version: v1alpha1 + names: + kind: Broker + plural: brokers + singular: broker + categories: + - all + - knative + - eventing + scope: Namespaced + subresources: + status: {} + additionalPrinterColumns: + - name: Ready + type: string + JSONPath: ".status.conditions[?(@.type==\"Ready\")].status" + - name: Reason + type: string + JSONPath: ".status.conditions[?(@.type==\"Ready\")].reason" diff --git a/config/300-trigger.yaml b/config/300-trigger.yaml new file mode 100644 index 00000000000..5ca0c8beb24 --- /dev/null +++ b/config/300-trigger.yaml @@ -0,0 +1,38 @@ +# Copyright 2019 The Knative Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: triggers.eventing.knative.dev +spec: + group: eventing.knative.dev + version: v1alpha1 + names: + kind: Trigger + plural: triggers + singular: trigger + categories: + - all + - knative + - eventing + scope: Namespaced + subresources: + status: {} + additionalPrinterColumns: + - name: Ready + type: string + JSONPath: ".status.conditions[?(@.type==\"Ready\")].status" + - name: Reason + type: string + JSONPath: ".status.conditions[?(@.type==\"Ready\")].reason" diff --git a/pkg/apis/eventing/v1alpha1/register.go b/pkg/apis/eventing/v1alpha1/register.go index df6338f66c1..fb3a5292623 100644 --- a/pkg/apis/eventing/v1alpha1/register.go +++ b/pkg/apis/eventing/v1alpha1/register.go @@ -45,12 +45,16 @@ var ( // Adds the list of known types to Scheme. func addKnownTypes(scheme *runtime.Scheme) error { scheme.AddKnownTypes(SchemeGroupVersion, + &Broker{}, + &BrokerList{}, &Channel{}, &ChannelList{}, &ClusterChannelProvisioner{}, &ClusterChannelProvisionerList{}, &Subscription{}, &SubscriptionList{}, + &Trigger{}, + &TriggerList{}, ) metav1.AddToGroupVersion(scheme, SchemeGroupVersion) return nil From 2f6a03f7ab3c8192367a1ae8f0c7c70eadd1f606 Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Mon, 28 Jan 2019 15:33:47 -0800 Subject: [PATCH 006/128] Initial work on the Broker controller. --- cmd/controller/main.go | 3 + pkg/apis/eventing/v1alpha1/broker_types.go | 32 +- pkg/controller/eventing/broker/provider.go | 89 ++++ pkg/controller/eventing/broker/reconcile.go | 419 ++++++++++++++++++ .../eventing/broker/resources/activator.go | 75 ++++ .../eventing/broker/resources/router.go | 80 ++++ 6 files changed, 695 insertions(+), 3 deletions(-) create mode 100644 pkg/controller/eventing/broker/provider.go create mode 100644 pkg/controller/eventing/broker/reconcile.go create mode 100644 pkg/controller/eventing/broker/resources/activator.go create mode 100644 pkg/controller/eventing/broker/resources/router.go diff --git a/cmd/controller/main.go b/cmd/controller/main.go index be41cfc443a..ea76ced13c4 100644 --- a/cmd/controller/main.go +++ b/cmd/controller/main.go @@ -23,6 +23,8 @@ import ( "net/http" "time" + "github.com/knative/eventing/pkg/controller/eventing/broker" + "github.com/knative/eventing/pkg/controller/eventing/subscription" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/controller" @@ -119,6 +121,7 @@ func main() { // manager run it. providers := []ProvideFunc{ subscription.ProvideController, + broker.ProvideController(logger.Desugar()), } for _, provider := range providers { if _, err := provider(mgr); err != nil { diff --git a/pkg/apis/eventing/v1alpha1/broker_types.go b/pkg/apis/eventing/v1alpha1/broker_types.go index 16a41df1c44..20a6a945c3f 100644 --- a/pkg/apis/eventing/v1alpha1/broker_types.go +++ b/pkg/apis/eventing/v1alpha1/broker_types.go @@ -59,7 +59,7 @@ type ChannelTemplateSpec struct { Spec *ChannelSpec `json:"spec,omitempty"` } -var brokerCondSet = duckv1alpha1.NewLivingConditionSet(BrokerConditionChannelTemplateSelector, BrokerConditionSubscribableResourcesExist) +var brokerCondSet = duckv1alpha1.NewLivingConditionSet(BrokerConditionChannelTemplateSelector, BrokerConditionSubscribableResourcesExist, BrokerConditionSubscribableResourcesExist, BrokerConditionAddressable) // BrokerStatus represents the current state of a Broker. type BrokerStatus struct { @@ -76,6 +76,13 @@ type BrokerStatus struct { // +patchMergeKey=type // +patchStrategy=merge Conditions duckv1alpha1.Conditions `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"` + + // Broker is Addressable. It currently exposes the endpoint as a + // fully-qualified DNS name which will distribute traffic over the + // provided targets from inside the cluster. + // + // It generally has the form {broker}-router.{namespace}.svc.{cluster domain name} + Address duckv1alpha1.Addressable `json:"address,omitempty"` } const ( @@ -84,6 +91,10 @@ const ( BrokerConditionChannelTemplateSelector duckv1alpha1.ConditionType = "ChannelTemplateSelector" BrokerConditionSubscribableResourcesExist duckv1alpha1.ConditionType = "SubscribableResourcesExist" + + BrokerConditionRouterAndActivatorExist duckv1alpha1.ConditionType = "RouterAndActivatorCreated" + + BrokerConditionAddressable duckv1alpha1.ConditionType = "Addressable" ) // GetCondition returns the condition currently associated with the given type, or nil. @@ -113,8 +124,23 @@ func (bs *BrokerStatus) MarkSubscribableResourcesExist() { brokerCondSet.Manage(bs).MarkTrue(BrokerConditionSubscribableResourcesExist) } -func (bs *BrokerStatus) MarkSubscribableResourcesDoNotExist(reason, messageFormat string, messageA ...interface{}) { - brokerCondSet.Manage(bs).MarkFalse(BrokerConditionSubscribableResourcesExist, reason, messageFormat, messageA...) +func (bs *BrokerStatus) MarkSubscribableResourcesDoNotExist(dontExist []metav1.GroupVersionKind) { + brokerCondSet.Manage(bs).MarkFalse(BrokerConditionSubscribableResourcesExist, "resourcesDontExist", "The following resources do not exist: %v", dontExist) +} + +func (bs *BrokerStatus) MarkRouterAndActivatorExist() { + brokerCondSet.Manage(bs).MarkTrue(BrokerConditionRouterAndActivatorExist) +} + +// SetAddress makes this Channel addressable by setting the hostname. It also +// sets the ChannelConditionAddressable to true. +func (bs *BrokerStatus) SetAddress(hostname string) { + bs.Address.Hostname = hostname + if hostname != "" { + chanCondSet.Manage(bs).MarkTrue(BrokerConditionAddressable) + } else { + chanCondSet.Manage(bs).MarkFalse(BrokerConditionAddressable, "emptyHostname", "hostname is the empty string") + } } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/pkg/controller/eventing/broker/provider.go b/pkg/controller/eventing/broker/provider.go new file mode 100644 index 00000000000..9315366245c --- /dev/null +++ b/pkg/controller/eventing/broker/provider.go @@ -0,0 +1,89 @@ +/* +Copyright 2018 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package broker + +import ( + "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" + "go.uber.org/zap" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" +) + +const ( + // controllerAgentName is the string used by this controller to identify + // itself when creating events. + controllerAgentName = "broker-controller" +) + +type reconciler struct { + client client.Client + restConfig *rest.Config + dynamicClient dynamic.Interface + recorder record.EventRecorder + + logger *zap.Logger + + routerImage string + routerServiceAccountName string + activatorImage string + activatorServiceAccountName string +} + +// Verify the struct implements reconcile.Reconciler +var _ reconcile.Reconciler = &reconciler{} + +// ProvideController returns a function that returns a Broker controller. +func ProvideController(logger *zap.Logger) func(manager.Manager) (controller.Controller, error) { + return func(mgr manager.Manager) (controller.Controller, error) { + // Setup a new controller to Reconcile Brokers. + c, err := controller.New(controllerAgentName, mgr, controller.Options{ + Reconciler: &reconciler{ + recorder: mgr.GetRecorder(controllerAgentName), + logger: logger, + }, + }) + if err != nil { + return nil, err + } + + // Watch Subscription events and enqueue Subscription object key. + if err = c.Watch(&source.Kind{Type: &v1alpha1.Broker{}}, &handler.EnqueueRequestForObject{}); err != nil { + return nil, err + } + + return c, nil + } +} + +func (r *reconciler) InjectClient(c client.Client) error { + r.client = c + return nil +} + +func (r *reconciler) InjectConfig(c *rest.Config) error { + r.restConfig = c + var err error + r.dynamicClient, err = dynamic.NewForConfig(c) + return err +} diff --git a/pkg/controller/eventing/broker/reconcile.go b/pkg/controller/eventing/broker/reconcile.go new file mode 100644 index 00000000000..e46d4b263d1 --- /dev/null +++ b/pkg/controller/eventing/broker/reconcile.go @@ -0,0 +1,419 @@ +/* +Copyright 2018 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package broker + +import ( + "context" + "fmt" + + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime/schema" + + "github.com/knative/eventing/pkg/controller/eventing/broker/resources" + + servingv1alpha1 "github.com/knative/serving/pkg/apis/serving/v1alpha1" + + "github.com/knative/eventing/pkg/provisioners/gcppubsub/util/logging" + "go.uber.org/zap" + + "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/equality" + "k8s.io/apimachinery/pkg/api/errors" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + runtimeclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +const ( + // Name of the corev1.Events emitted from the reconciliation process + brokerReconciled = "BrokerReconciled" + brokerUpdateStatusFailed = "BrokerUpdateStatusFailed" +) + +// Reconcile compares the actual state with the desired, and attempts to +// converge the two. It then updates the Status block of the Broker resource +// with the current status of the resource. +func (r *reconciler) Reconcile(request reconcile.Request) (reconcile.Result, error) { + ctx := context.TODO() + ctx = logging.WithLogger(ctx, r.logger.With(zap.Any("request", request))) + + broker := &v1alpha1.Broker{} + err := r.client.Get(context.TODO(), request.NamespacedName, broker) + + if errors.IsNotFound(err) { + logging.FromContext(ctx).Info("Could not find Broker") + return reconcile.Result{}, nil + } + + if err != nil { + logging.FromContext(ctx).Error("Could not Get Broker", zap.Error(err)) + return reconcile.Result{}, err + } + + // Reconcile this copy of the Broker and then write back any status updates regardless of + // whether the reconcile error out. + reconcileErr := r.reconcile(ctx, broker) + if reconcileErr != nil { + logging.FromContext(ctx).Error("Error reconciling Broker", zap.Error(reconcileErr)) + } else { + logging.FromContext(ctx).Debug("Broker reconciled") + r.recorder.Event(broker, corev1.EventTypeNormal, brokerReconciled, "Broker reconciled") + } + + if _, err := r.updateStatus(broker.DeepCopy()); err != nil { + logging.FromContext(ctx).Error("Failed to update Broker status", zap.Error(err)) + r.recorder.Eventf(broker, corev1.EventTypeWarning, brokerUpdateStatusFailed, "Failed to update Broker's status: %v", err) + return reconcile.Result{}, err + } + + // Requeue if the resource is not ready: + return reconcile.Result{}, err +} + +func (r *reconciler) reconcile(ctx context.Context, b *v1alpha1.Broker) error { + b.Status.InitializeConditions() + + // 1. Create the router. + // 2. Create the activator. + // 3. Create the filter [do not do for the prototype]. + // 4. Create the 'needs-activation' Channel. + // 5. Create Subscription from 'needs-activation' Channel to the activator. + + if b.DeletionTimestamp != nil { + // Everything is cleaned up by the garbage collector. + return nil + } + + // TODO Actually check this, for now the webhook ensures they are identical. + b.Status.MarkChannelTemplateMatchesSelector() + + dontExist, err := r.verifyResourcesExist(ctx, b.Spec.SubscribableResources) + if err != nil { + logging.FromContext(ctx).Error("Unable to determine if resources exist", zap.Error(err)) + return err + } else if len(dontExist) > 0 { + b.Status.MarkSubscribableResourcesDoNotExist(dontExist) + } else { + b.Status.MarkSubscribableResourcesExist() + } + + activator, err := r.reconcileActivator(ctx, b) + if err != nil { + logging.FromContext(ctx).Error("Problem reconciling activator", zap.Error(err)) + return err + } + + // TODO Add the filter reconciliation here. + + c, err := r.reconcileNeedsActivationChannel(ctx, b) + if err != nil { + logging.FromContext(ctx).Error("Problem reconciling the needs activation channel", zap.Error(err)) + return err + } + + _, err = r.reconcileNeedsActivationSubscription(ctx, b, activator, c) + if err != nil { + logging.FromContext(ctx).Error("Problem reconciling the needs activation subscription", zap.Error(err)) + return err + } + b.Status.MarkRouterAndActivatorExist() + + router, err := r.reconcileRouter(ctx, b, c) + if err != nil { + logging.FromContext(ctx).Error("Problem reconciling router", zap.Error(err)) + return err + } + b.Status.SetAddress(router.Status.Address.Hostname) + + return nil +} + +// updateStatus may in fact update the broker's finalizers in addition to the status +func (r *reconciler) updateStatus(broker *v1alpha1.Broker) (*v1alpha1.Broker, error) { + objectKey := client.ObjectKey{Namespace: broker.Namespace, Name: broker.Name} + latestBroker := &v1alpha1.Broker{} + + if err := r.client.Get(context.TODO(), objectKey, latestBroker); err != nil { + return nil, err + } + + brokerChanged := false + + if !equality.Semantic.DeepEqual(latestBroker.Finalizers, broker.Finalizers) { + latestBroker.SetFinalizers(broker.ObjectMeta.Finalizers) + if err := r.client.Update(context.TODO(), latestBroker); err != nil { + return nil, err + } + brokerChanged = true + } + + if equality.Semantic.DeepEqual(latestBroker.Status, broker.Status) { + return latestBroker, nil + } + + if brokerChanged { + // Refetch + latestBroker = &v1alpha1.Broker{} + if err := r.client.Get(context.TODO(), objectKey, latestBroker); err != nil { + return nil, err + } + } + + latestBroker.Status = broker.Status + if err := r.client.Status().Update(context.TODO(), latestBroker); err != nil { + return nil, err + } + + return latestBroker, nil +} + +func (r *reconciler) verifyResourcesExist(ctx context.Context, resources []metav1.GroupVersionKind) ([]metav1.GroupVersionKind, error) { + dontExist := make([]metav1.GroupVersionKind, 0, len(resources)) + // TODO Implement, for now the webhook asserts it is only Channel, which we assume exists. + return dontExist, nil +} + +func (r *reconciler) reconcileActivator(ctx context.Context, b *v1alpha1.Broker) (*servingv1alpha1.Service, error) { + expected, err := resources.MakeActivator(&resources.ActivatorArgs{ + Broker: b, + Image: r.activatorImage, + ServiceAccountName: r.activatorServiceAccountName, + }) + if err != nil { + return nil, err + } + return r.reconcileKSvc(ctx, expected) +} + +func (r *reconciler) reconcileNeedsActivationChannel(ctx context.Context, b *v1alpha1.Broker) (*v1alpha1.Channel, error) { + expected := newNeedsActivationChannel(b) + + c, err := r.getNeedsActivationChannel(ctx, b) + // If the resource doesn't exist, we'll create it + if k8serrors.IsNotFound(err) { + c = expected + err = r.client.Create(ctx, c) + if err != nil { + return nil, err + } + return c, nil + } else if err != nil { + return nil, err + } + + // Update Channel if it has changed. Note that we need to both ignore the real Channel's + // subscribable section and if we need to update the real Channel, retain it. + expected.Spec.Subscribable = c.Spec.Subscribable + if !equality.Semantic.DeepDerivative(expected.Spec, c.Spec) { + c.Spec = expected.Spec + err = r.client.Update(ctx, c) + if err != nil { + return nil, err + } + } + return c, nil +} + +func (r *reconciler) getNeedsActivationChannel(ctx context.Context, b *v1alpha1.Broker) (*v1alpha1.Channel, error) { + list := &v1alpha1.ChannelList{} + opts := &runtimeclient.ListOptions{ + Namespace: b.Namespace, + LabelSelector: labels.SelectorFromSet(needsActivationLabels(b)), + // TODO this is here because the fake client needs it. Remove this when it's no longer + // needed. + Raw: &metav1.ListOptions{ + TypeMeta: metav1.TypeMeta{ + APIVersion: v1alpha1.SchemeGroupVersion.String(), + Kind: "Channel", + }, + }, + } + + err := r.client.List(ctx, opts, list) + if err != nil { + return nil, err + } + for _, c := range list.Items { + if metav1.IsControlledBy(&c, b) { + return &c, nil + } + } + + return nil, k8serrors.NewNotFound(schema.GroupResource{}, "") +} + +func newNeedsActivationChannel(b *v1alpha1.Broker) *v1alpha1.Channel { + var spec v1alpha1.ChannelSpec + if b.Spec.ChannelTemplate != nil && b.Spec.ChannelTemplate.Spec != nil { + spec = *b.Spec.ChannelTemplate.Spec + } + + return &v1alpha1.Channel{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: b.Namespace, + GenerateName: fmt.Sprintf("%s-broker-needs-activation-", b.Name), + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(b, schema.GroupVersionKind{ + Group: b.GroupVersionKind().Group, + Version: b.GroupVersionKind().Version, + Kind: b.GroupVersionKind().Kind, + }), + }, + }, + Spec: spec, + } +} + +func needsActivationLabels(b *v1alpha1.Broker) map[string]string { + return map[string]string{ + "eventing.knative.dev/broker": b.Name, + "eventing.knative.dev/broker/needsActivation": "true", + } +} + +func (r *reconciler) reconcileNeedsActivationSubscription(ctx context.Context, b *v1alpha1.Broker, activator *servingv1alpha1.Service, c *v1alpha1.Channel) (*v1alpha1.Subscription, error) { + expected := newNeedsActivationSubscription(b, c, activator) + + sub, err := r.getNeedsActivationSubscription(ctx, b) + // If the resource doesn't exist, we'll create it + if k8serrors.IsNotFound(err) { + sub = expected + err = r.client.Create(ctx, sub) + if err != nil { + return nil, err + } + return sub, nil + } else if err != nil { + return nil, err + } + + // Update Subscription if it has changed. + if !equality.Semantic.DeepDerivative(expected.Spec, sub.Spec) { + sub.Spec = expected.Spec + err = r.client.Update(ctx, sub) + if err != nil { + return nil, err + } + } + return sub, nil +} + +func (r *reconciler) getNeedsActivationSubscription(ctx context.Context, b *v1alpha1.Broker) (*v1alpha1.Subscription, error) { + list := &v1alpha1.SubscriptionList{} + opts := &runtimeclient.ListOptions{ + Namespace: b.Namespace, + LabelSelector: labels.SelectorFromSet(needsActivationLabels(b)), + // TODO this is here because the fake client needs it. Remove this when it's no longer + // needed. + Raw: &metav1.ListOptions{ + TypeMeta: metav1.TypeMeta{ + APIVersion: v1alpha1.SchemeGroupVersion.String(), + Kind: "Subscription", + }, + }, + } + + err := r.client.List(ctx, opts, list) + if err != nil { + return nil, err + } + for _, sub := range list.Items { + if metav1.IsControlledBy(&sub, b) { + return &sub, nil + } + } + + return nil, k8serrors.NewNotFound(schema.GroupResource{}, "") +} + +func newNeedsActivationSubscription(b *v1alpha1.Broker, c *v1alpha1.Channel, activator *servingv1alpha1.Service) *v1alpha1.Subscription { + return &v1alpha1.Subscription{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: b.Namespace, + GenerateName: fmt.Sprintf("%s-broker-needs-activation-", b.Name), + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(b, schema.GroupVersionKind{ + Group: b.GroupVersionKind().Group, + Version: b.GroupVersionKind().Version, + Kind: b.GroupVersionKind().Kind, + }), + }, + }, + Spec: v1alpha1.SubscriptionSpec{ + Channel: corev1.ObjectReference{ + APIVersion: c.APIVersion, + Kind: c.Kind, + Namespace: c.Namespace, + Name: c.Name, + UID: c.UID, + }, + Subscriber: &v1alpha1.SubscriberSpec{ + Ref: &corev1.ObjectReference{ + APIVersion: activator.APIVersion, + Kind: activator.Kind, + Namespace: activator.Namespace, + Name: activator.Name, + UID: activator.UID, + }, + }, + }, + } +} + +func (r *reconciler) reconcileKSvc(ctx context.Context, ksvc *servingv1alpha1.Service) (*servingv1alpha1.Service, error) { + name := types.NamespacedName{ + Namespace: ksvc.Namespace, + Name: ksvc.Name, + } + current := &servingv1alpha1.Service{} + err := r.client.Get(ctx, name, current) + if k8serrors.IsNotFound(err) { + err = r.client.Create(ctx, ksvc) + if err != nil { + return nil, err + } + return ksvc, nil + } else if err != nil { + return nil, err + } + + if !equality.Semantic.DeepDerivative(ksvc.Spec, current.Spec) { + current.Spec = ksvc.Spec + err = r.client.Update(ctx, current) + if err != nil { + return nil, err + } + } + return current, nil +} + +func (r *reconciler) reconcileRouter(ctx context.Context, b *v1alpha1.Broker, c *v1alpha1.Channel) (*servingv1alpha1.Service, error) { + expected, err := resources.MakeRouter(&resources.RouterArgs{ + Broker: b, + Image: r.routerImage, + ServiceAccountName: r.routerServiceAccountName, + NeedsActivationHost: c.Status.Address.Hostname, + }) + if err != nil { + return nil, err + } + return r.reconcileKSvc(ctx, expected) +} diff --git a/pkg/controller/eventing/broker/resources/activator.go b/pkg/controller/eventing/broker/resources/activator.go new file mode 100644 index 00000000000..1de0820eef1 --- /dev/null +++ b/pkg/controller/eventing/broker/resources/activator.go @@ -0,0 +1,75 @@ +/* +Copyright 2018 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resources + +import ( + "encoding/json" + "fmt" + + eventingv1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" + servingv1alpha1 "github.com/knative/serving/pkg/apis/serving/v1alpha1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +type ActivatorArgs struct { + Broker *eventingv1alpha1.Broker + Image string + ServiceAccountName string +} + +func MakeActivator(args *ActivatorArgs) (*servingv1alpha1.Service, error) { + templateJson, err := json.Marshal(args.Broker.Spec.ChannelTemplate) + if err != nil { + return nil, err + } + + return &servingv1alpha1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: args.Broker.Namespace, + Name: fmt.Sprintf("%s-broker-activator", args.Broker.Name), + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(args.Broker, schema.GroupVersionKind{ + Group: args.Broker.GroupVersionKind().Group, + Version: args.Broker.GroupVersionKind().Version, + Kind: args.Broker.GroupVersionKind().Kind, + }), + }, + }, + Spec: servingv1alpha1.ServiceSpec{ + RunLatest: &servingv1alpha1.RunLatestType{ + Configuration: servingv1alpha1.ConfigurationSpec{ + RevisionTemplate: servingv1alpha1.RevisionTemplateSpec{ + Spec: servingv1alpha1.RevisionSpec{ + ServiceAccountName: args.ServiceAccountName, + Container: v1.Container{ + Image: args.Image, + Env: []v1.EnvVar{ + { + Name: "CHANNEL_TEMPLATE", + Value: string(templateJson), + }, + }, + }, + }, + }, + }, + }, + }, + }, nil +} diff --git a/pkg/controller/eventing/broker/resources/router.go b/pkg/controller/eventing/broker/resources/router.go new file mode 100644 index 00000000000..baf8ac4b39c --- /dev/null +++ b/pkg/controller/eventing/broker/resources/router.go @@ -0,0 +1,80 @@ +/* +Copyright 2018 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resources + +import ( + "encoding/json" + "fmt" + + eventingv1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" + servingv1alpha1 "github.com/knative/serving/pkg/apis/serving/v1alpha1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +type RouterArgs struct { + Broker *eventingv1alpha1.Broker + Image string + ServiceAccountName string + NeedsActivationHost string +} + +func MakeRouter(args *RouterArgs) (*servingv1alpha1.Service, error) { + selectorJson, err := json.Marshal(args.Broker.Spec.Selector) + if err != nil { + return nil, err + } + + return &servingv1alpha1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: args.Broker.Namespace, + Name: fmt.Sprintf("%s-broker", args.Broker.Name), + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(args.Broker, schema.GroupVersionKind{ + Group: args.Broker.GroupVersionKind().Group, + Version: args.Broker.GroupVersionKind().Version, + Kind: args.Broker.GroupVersionKind().Kind, + }), + }, + }, + Spec: servingv1alpha1.ServiceSpec{ + RunLatest: &servingv1alpha1.RunLatestType{ + Configuration: servingv1alpha1.ConfigurationSpec{ + RevisionTemplate: servingv1alpha1.RevisionTemplateSpec{ + Spec: servingv1alpha1.RevisionSpec{ + ServiceAccountName: args.ServiceAccountName, + Container: v1.Container{ + Image: args.Image, + Env: []v1.EnvVar{ + { + Name: "NEEDS_ACTIVATION_HOST", + Value: args.NeedsActivationHost, + }, + { + Name: "LABEL_SELECTOR", + Value: string(selectorJson), + }, + }, + }, + }, + }, + }, + }, + }, + }, nil +} From 3ee0b8b6573e7a36ba757ae534e45a07d13371e2 Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Thu, 31 Jan 2019 13:54:59 -0800 Subject: [PATCH 007/128] Initial work on the Trigger controller. --- cmd/controller/main.go | 5 +- pkg/apis/eventing/v1alpha1/trigger_types.go | 15 +- pkg/controller/eventing/broker/reconcile.go | 2 +- pkg/controller/eventing/trigger/provider.go | 113 ++++++ pkg/controller/eventing/trigger/reconcile.go | 366 ++++++++++++++++++ .../eventing/trigger/resources/activator.go | 75 ++++ .../eventing/trigger/resources/router.go | 80 ++++ pkg/provisioners/channel_util.go | 14 +- 8 files changed, 655 insertions(+), 15 deletions(-) create mode 100644 pkg/controller/eventing/trigger/provider.go create mode 100644 pkg/controller/eventing/trigger/reconcile.go create mode 100644 pkg/controller/eventing/trigger/resources/activator.go create mode 100644 pkg/controller/eventing/trigger/resources/router.go diff --git a/cmd/controller/main.go b/cmd/controller/main.go index ea76ced13c4..22212fc6e6c 100644 --- a/cmd/controller/main.go +++ b/cmd/controller/main.go @@ -23,6 +23,8 @@ import ( "net/http" "time" + "github.com/knative/eventing/pkg/controller/eventing/trigger" + "github.com/knative/eventing/pkg/controller/eventing/broker" "github.com/knative/eventing/pkg/controller/eventing/subscription" @@ -112,7 +114,7 @@ func main() { eventingv1alpha1.AddToScheme, } for _, schemeFunc := range schemeFuncs { - if err := schemeFunc(mgr.GetScheme()); err != nil { + if err = schemeFunc(mgr.GetScheme()); err != nil { logger.Fatalf("Error adding type to manager's scheme: %v", err) } } @@ -122,6 +124,7 @@ func main() { providers := []ProvideFunc{ subscription.ProvideController, broker.ProvideController(logger.Desugar()), + trigger.ProvideController(logger.Desugar()), } for _, provider := range providers { if _, err := provider(mgr); err != nil { diff --git a/pkg/apis/eventing/v1alpha1/trigger_types.go b/pkg/apis/eventing/v1alpha1/trigger_types.go index 82ac3aa648f..432c2d3eb32 100644 --- a/pkg/apis/eventing/v1alpha1/trigger_types.go +++ b/pkg/apis/eventing/v1alpha1/trigger_types.go @@ -52,6 +52,9 @@ type TriggerSpec struct { Broker string `json:"broker,omitempty"` Selector *TriggerSelectorSpec `json:"selector,omitempty"` Subscriber *SubscriberSpec `json:"subscriber,omitempty"` + + Type string `json:"type,omitempty"` + Source string `json:"source,omitempty"` } type TriggerSelectorSpec struct { @@ -60,7 +63,7 @@ type TriggerSelectorSpec struct { OPAPolicy string `json:"opaPolicy,omitEmpty"` } -var triggerCondSet = duckv1alpha1.NewLivingConditionSet(TriggerConditionBrokerExists, TriggerConditionSubscriberFound) +var triggerCondSet = duckv1alpha1.NewLivingConditionSet(TriggerConditionBrokerExists, TriggerConditionSubscribed) // TriggerStatus represents the current state of a Trigger. type TriggerStatus struct { @@ -84,7 +87,7 @@ const ( TriggerConditionBrokerExists duckv1alpha1.ConditionType = "BrokerExists" - TriggerConditionSubscriberFound duckv1alpha1.ConditionType = "SubscriberFound" + TriggerConditionSubscribed duckv1alpha1.ConditionType = "Subscribed" ) // GetCondition returns the condition currently associated with the given type, or nil. @@ -110,12 +113,12 @@ func (ts *TriggerStatus) MarkBrokerDoesNotExists() { triggerCondSet.Manage(ts).MarkFalse(TriggerConditionBrokerExists, "doesNotExist", "Broker does not exist") } -func (ts *TriggerStatus) MarkSubscriberFound() { - triggerCondSet.Manage(ts).MarkTrue(TriggerConditionSubscriberFound) +func (ts *TriggerStatus) MarkSubscribed() { + triggerCondSet.Manage(ts).MarkTrue(TriggerConditionSubscribed) } -func (ts *TriggerStatus) MarkSubscriberNotFound(reason, messageFormat string, messageA ...interface{}) { - triggerCondSet.Manage(ts).MarkFalse(TriggerConditionSubscriberFound, reason, messageFormat, messageA...) +func (ts *TriggerStatus) MarkNotSubscribed(reason, messageFormat string, messageA ...interface{}) { + triggerCondSet.Manage(ts).MarkFalse(TriggerConditionSubscribed, reason, messageFormat, messageA...) } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/pkg/controller/eventing/broker/reconcile.go b/pkg/controller/eventing/broker/reconcile.go index e46d4b263d1..b9daa891a69 100644 --- a/pkg/controller/eventing/broker/reconcile.go +++ b/pkg/controller/eventing/broker/reconcile.go @@ -78,7 +78,7 @@ func (r *reconciler) Reconcile(request reconcile.Request) (reconcile.Result, err r.recorder.Event(broker, corev1.EventTypeNormal, brokerReconciled, "Broker reconciled") } - if _, err := r.updateStatus(broker.DeepCopy()); err != nil { + if _, err = r.updateStatus(broker.DeepCopy()); err != nil { logging.FromContext(ctx).Error("Failed to update Broker status", zap.Error(err)) r.recorder.Eventf(broker, corev1.EventTypeWarning, brokerUpdateStatusFailed, "Failed to update Broker's status: %v", err) return reconcile.Result{}, err diff --git a/pkg/controller/eventing/trigger/provider.go b/pkg/controller/eventing/trigger/provider.go new file mode 100644 index 00000000000..316f3312766 --- /dev/null +++ b/pkg/controller/eventing/trigger/provider.go @@ -0,0 +1,113 @@ +/* +Copyright 2018 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package trigger + +import ( + "sync" + + "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" + "go.uber.org/zap" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" +) + +const ( + // controllerAgentName is the string used by this controller to identify + // itself when creating events. + controllerAgentName = "trigger-controller" +) + +type reconciler struct { + client client.Client + restConfig *rest.Config + dynamicClient dynamic.Interface + recorder record.EventRecorder + + triggersLock sync.RWMutex + triggers map[string]map[reconcile.Request]bool + + logger *zap.Logger +} + +// Verify the struct implements reconcile.Reconciler +var _ reconcile.Reconciler = &reconciler{} + +// ProvideController returns a function that returns a Broker controller. +func ProvideController(logger *zap.Logger) func(manager.Manager) (controller.Controller, error) { + return func(mgr manager.Manager) (controller.Controller, error) { + // Setup a new controller to Reconcile Brokers. + r := &reconciler{ + recorder: mgr.GetRecorder(controllerAgentName), + logger: logger, + } + c, err := controller.New(controllerAgentName, mgr, controller.Options{ + Reconciler: r, + }) + if err != nil { + return nil, err + } + + // Watch Subscription events and enqueue Subscription object key. + if err = c.Watch(&source.Kind{Type: &v1alpha1.Trigger{}}, &handler.EnqueueRequestForObject{}); err != nil { + return nil, err + } + + // TODO Make this much, much more efficient. + if err = c.Watch(&source.Kind{Type: &v1alpha1.Channel{}}, &handler.EnqueueRequestsFromMapFunc{ToRequests: &mapAllTriggers{r}}); err != nil { + return nil, err + } + + return c, nil + } +} + +func (r *reconciler) InjectClient(c client.Client) error { + r.client = c + return nil +} + +func (r *reconciler) InjectConfig(c *rest.Config) error { + r.restConfig = c + var err error + r.dynamicClient, err = dynamic.NewForConfig(c) + return err +} + +type mapAllTriggers struct { + r *reconciler +} + +func (m *mapAllTriggers) Map(o handler.MapObject) []reconcile.Request { + m.r.triggersLock.RLock() + defer m.r.triggersLock.RUnlock() + triggersInNamespace := m.r.triggers[o.Meta.GetNamespace()] + if triggersInNamespace == nil { + return []reconcile.Request{} + } + reqs := make([]reconcile.Request, 0, len(triggersInNamespace)) + for name := range triggersInNamespace { + reqs = append(reqs, name) + } + return reqs +} diff --git a/pkg/controller/eventing/trigger/reconcile.go b/pkg/controller/eventing/trigger/reconcile.go new file mode 100644 index 00000000000..b993beeed1c --- /dev/null +++ b/pkg/controller/eventing/trigger/reconcile.go @@ -0,0 +1,366 @@ +/* +Copyright 2018 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package trigger + +import ( + "context" + "fmt" + + "k8s.io/apimachinery/pkg/labels" + + "k8s.io/apimachinery/pkg/runtime/schema" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/knative/eventing/pkg/provisioners" + "k8s.io/apimachinery/pkg/types" + + "github.com/knative/eventing/pkg/provisioners/gcppubsub/util/logging" + "go.uber.org/zap" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + + "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/equality" + "k8s.io/apimachinery/pkg/api/errors" + "sigs.k8s.io/controller-runtime/pkg/client" + runtimeclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +const ( + finalizerName = controllerAgentName + + eventTypeKey = "eventing.knative.dev/broker/eventType" + + // Name of the corev1.Events emitted from the reconciliation process + triggerReconciled = "TriggerReconciled" + triggerUpdateStatusFailed = "TriggerUpdateStatusFailed" +) + +// Reconcile compares the actual state with the desired, and attempts to +// converge the two. It then updates the Status block of the Trigger resource +// with the current status of the resource. +func (r *reconciler) Reconcile(request reconcile.Request) (reconcile.Result, error) { + ctx := context.TODO() + ctx = logging.WithLogger(ctx, r.logger.With(zap.Any("request", request))) + + trigger := &v1alpha1.Trigger{} + err := r.client.Get(context.TODO(), request.NamespacedName, trigger) + + if errors.IsNotFound(err) { + logging.FromContext(ctx).Info("Could not find Trigger") + return reconcile.Result{}, nil + } + + if err != nil { + logging.FromContext(ctx).Error("Could not Get Trigger", zap.Error(err)) + return reconcile.Result{}, err + } + + // Reconcile this copy of the Trigger and then write back any status updates regardless of + // whether the reconcile error out. + reconcileErr := r.reconcile(ctx, trigger) + if reconcileErr != nil { + logging.FromContext(ctx).Error("Error reconciling Trigger", zap.Error(reconcileErr)) + } else { + logging.FromContext(ctx).Debug("Trigger reconciled") + r.recorder.Event(trigger, corev1.EventTypeNormal, triggerReconciled, "Trigger reconciled") + } + + if _, err = r.updateStatus(trigger.DeepCopy()); err != nil { + logging.FromContext(ctx).Error("Failed to update Trigger status", zap.Error(err)) + r.recorder.Eventf(trigger, corev1.EventTypeWarning, triggerUpdateStatusFailed, "Failed to update Trigger's status: %v", err) + return reconcile.Result{}, err + } + + // Requeue if the resource is not ready: + return reconcile.Result{}, err +} + +func (r *reconciler) reconcile(ctx context.Context, t *v1alpha1.Trigger) error { + t.Status.InitializeConditions() + + // 1. Verify the Broker exists. + // 2. Determine subscribers. + // 3. Look over all subscribable resources and subscribe to the correct ones. + // 4. [Do not do for the prototype] Inject a filter on every subscription. + + if t.DeletionTimestamp != nil { + // Everything is cleaned up by the garbage collector. + r.removeFromTriggers(t) + provisioners.RemoveFinalizer(t, finalizerName) + return nil + } + + provisioners.AddFinalizer(t, finalizerName) + r.AddToTriggers(t) + + broker, err := r.getBroker(ctx, t) + if err != nil { + logging.FromContext(ctx).Error("Unable to get the Broker", zap.Error(err)) + t.Status.MarkBrokerDoesNotExists() + return err + } + t.Status.MarkBrokerExists() + + subscribables, err := r.getRelevantSubscribables(ctx, t, broker.Spec.Selector) + if err != nil { + logging.FromContext(ctx).Error("Unable to get relevant subscribables", zap.Error(err)) + return err + } + + err = r.subscribeAll(ctx, t, subscribables) + if err != nil { + logging.FromContext(ctx).Error("Unable to Subscribe", zap.Error(err)) + t.Status.MarkNotSubscribed("notSubscribed", "%v", err) + return err + } + t.Status.MarkSubscribed() + + return nil +} + +func (r *reconciler) AddToTriggers(t *v1alpha1.Trigger) { + name := reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: t.Namespace, + Name: t.Name, + }, + } + + // We will be reconciling an already existing Trigger far more often than adding a new one, so + // check with a read lock before using the write lock. + r.triggersLock.RLock() + triggersInNamespace := r.triggers[t.Namespace] + var present bool + if triggersInNamespace != nil { + _, present = triggersInNamespace[name] + } else { + present = false + } + r.triggersLock.RUnlock() + + if present { + // Already present in the map. + return + } + + r.triggersLock.Lock() + triggersInNamespace = r.triggers[t.Namespace] + if triggersInNamespace == nil { + r.triggers[t.Namespace] = make(map[reconcile.Request]bool) + triggersInNamespace = r.triggers[t.Namespace] + } + triggersInNamespace[name] = false + r.triggersLock.Unlock() +} + +func (r *reconciler) removeFromTriggers(t *v1alpha1.Trigger) { + name := reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: t.Namespace, + Name: t.Name, + }, + } + + r.triggersLock.Lock() + triggersInNamespace := r.triggers[t.Namespace] + if triggersInNamespace != nil { + delete(triggersInNamespace, name) + } + r.triggersLock.Unlock() +} + +// updateStatus may in fact update the trigger's finalizers in addition to the status +func (r *reconciler) updateStatus(trigger *v1alpha1.Trigger) (*v1alpha1.Trigger, error) { + objectKey := client.ObjectKey{Namespace: trigger.Namespace, Name: trigger.Name} + latestTrigger := &v1alpha1.Trigger{} + + if err := r.client.Get(context.TODO(), objectKey, latestTrigger); err != nil { + return nil, err + } + + triggerChanged := false + + if !equality.Semantic.DeepEqual(latestTrigger.Finalizers, trigger.Finalizers) { + latestTrigger.SetFinalizers(trigger.ObjectMeta.Finalizers) + if err := r.client.Update(context.TODO(), latestTrigger); err != nil { + return nil, err + } + triggerChanged = true + } + + if equality.Semantic.DeepEqual(latestTrigger.Status, trigger.Status) { + return latestTrigger, nil + } + + if triggerChanged { + // Refetch + latestTrigger = &v1alpha1.Trigger{} + if err := r.client.Get(context.TODO(), objectKey, latestTrigger); err != nil { + return nil, err + } + } + + latestTrigger.Status = trigger.Status + if err := r.client.Status().Update(context.TODO(), latestTrigger); err != nil { + return nil, err + } + + return latestTrigger, nil +} + +func (r *reconciler) getBroker(ctx context.Context, t *v1alpha1.Trigger) (*v1alpha1.Broker, error) { + broker := &v1alpha1.Broker{} + name := types.NamespacedName{ + Namespace: t.Namespace, + Name: t.Spec.Broker, + } + err := r.client.Get(ctx, name, broker) + return broker, err +} + +func (r *reconciler) getRelevantSubscribables(ctx context.Context, t *v1alpha1.Trigger, selector *metav1.LabelSelector) ([]v1alpha1.Channel, error) { + selector.MatchLabels[eventTypeKey] = t.Spec.Type + + subscribables := make([]v1alpha1.Channel, 0) + opts := &client.ListOptions{ + // TODO this is here because the fake client needs it. Remove this when it's no longer + // needed. + Raw: &metav1.ListOptions{ + TypeMeta: metav1.TypeMeta{ + APIVersion: v1alpha1.SchemeGroupVersion.String(), + Kind: "Channel", + }, + }, + Namespace: t.Namespace, + } + for { + list := v1alpha1.ChannelList{} + err := r.client.List(ctx, opts, &list) + if err != nil { + return nil, err + } + for _, s := range list.Items { + subscribables = append(subscribables, s) + if list.Continue != "" { + opts.Raw.Continue = list.Continue + } else { + return subscribables, nil + } + } + } +} + +func (r *reconciler) subscribeAll(ctx context.Context, t *v1alpha1.Trigger, subscribables []v1alpha1.Channel) error { + for _, subscribable := range subscribables { + _, err := r.subscribe(ctx, t, &subscribable) + if err != nil { + return err + } + } + return nil +} + +func (r *reconciler) subscribe(ctx context.Context, t *v1alpha1.Trigger, subscribable *v1alpha1.Channel) (*v1alpha1.Subscription, error) { + expected := makeSubscription(t, subscribable) + + sub, err := r.getSubscription(ctx, t, subscribable) + // If the resource doesn't exist, we'll create it + if k8serrors.IsNotFound(err) { + sub = expected + err = r.client.Create(ctx, sub) + if err != nil { + return nil, err + } + return sub, nil + } else if err != nil { + return nil, err + } + + // Update Subscription if it has changed. + if !equality.Semantic.DeepDerivative(expected.Spec, sub.Spec) { + sub.Spec = expected.Spec + err = r.client.Update(ctx, sub) + if err != nil { + return nil, err + } + } + return sub, nil +} + +func (r *reconciler) getSubscription(ctx context.Context, t *v1alpha1.Trigger, subscribable *v1alpha1.Channel) (*v1alpha1.Subscription, error) { + list := &v1alpha1.SubscriptionList{} + opts := &runtimeclient.ListOptions{ + Namespace: t.Namespace, + LabelSelector: labels.SelectorFromSet(subscriptionLabels(t)), + // TODO this is here because the fake client needs it. Remove this when it's no longer + // needed. + Raw: &metav1.ListOptions{ + TypeMeta: metav1.TypeMeta{ + APIVersion: v1alpha1.SchemeGroupVersion.String(), + Kind: "Subscription", + }, + }, + } + + err := r.client.List(ctx, opts, list) + if err != nil { + return nil, err + } + for _, s := range list.Items { + if metav1.IsControlledBy(&s, t) { + return &s, nil + } + } + + return nil, k8serrors.NewNotFound(schema.GroupResource{}, "") +} + +func makeSubscription(t *v1alpha1.Trigger, subscribable *v1alpha1.Channel) *v1alpha1.Subscription { + return &v1alpha1.Subscription{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: t.Namespace, + GenerateName: fmt.Sprintf("%s-%s-", t.Spec.Broker, t.Name), + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(t, schema.GroupVersionKind{ + Group: t.GroupVersionKind().Group, + Version: t.GroupVersionKind().Version, + Kind: t.GroupVersionKind().Kind, + }), + }, + Labels: subscriptionLabels(t), + }, + Spec: v1alpha1.SubscriptionSpec{ + Channel: corev1.ObjectReference{ + APIVersion: subscribable.APIVersion, + Kind: subscribable.Kind, + Namespace: subscribable.Namespace, + Name: subscribable.Name, + }, + Subscriber: t.Spec.Subscriber, + }, + } +} + +func subscriptionLabels(t *v1alpha1.Trigger) map[string]string { + return map[string]string{ + "eventing.knative.dev/broker": t.Spec.Broker, + "eventing.knative.dev/trigger": t.Name, + } +} diff --git a/pkg/controller/eventing/trigger/resources/activator.go b/pkg/controller/eventing/trigger/resources/activator.go new file mode 100644 index 00000000000..1de0820eef1 --- /dev/null +++ b/pkg/controller/eventing/trigger/resources/activator.go @@ -0,0 +1,75 @@ +/* +Copyright 2018 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resources + +import ( + "encoding/json" + "fmt" + + eventingv1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" + servingv1alpha1 "github.com/knative/serving/pkg/apis/serving/v1alpha1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +type ActivatorArgs struct { + Broker *eventingv1alpha1.Broker + Image string + ServiceAccountName string +} + +func MakeActivator(args *ActivatorArgs) (*servingv1alpha1.Service, error) { + templateJson, err := json.Marshal(args.Broker.Spec.ChannelTemplate) + if err != nil { + return nil, err + } + + return &servingv1alpha1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: args.Broker.Namespace, + Name: fmt.Sprintf("%s-broker-activator", args.Broker.Name), + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(args.Broker, schema.GroupVersionKind{ + Group: args.Broker.GroupVersionKind().Group, + Version: args.Broker.GroupVersionKind().Version, + Kind: args.Broker.GroupVersionKind().Kind, + }), + }, + }, + Spec: servingv1alpha1.ServiceSpec{ + RunLatest: &servingv1alpha1.RunLatestType{ + Configuration: servingv1alpha1.ConfigurationSpec{ + RevisionTemplate: servingv1alpha1.RevisionTemplateSpec{ + Spec: servingv1alpha1.RevisionSpec{ + ServiceAccountName: args.ServiceAccountName, + Container: v1.Container{ + Image: args.Image, + Env: []v1.EnvVar{ + { + Name: "CHANNEL_TEMPLATE", + Value: string(templateJson), + }, + }, + }, + }, + }, + }, + }, + }, + }, nil +} diff --git a/pkg/controller/eventing/trigger/resources/router.go b/pkg/controller/eventing/trigger/resources/router.go new file mode 100644 index 00000000000..baf8ac4b39c --- /dev/null +++ b/pkg/controller/eventing/trigger/resources/router.go @@ -0,0 +1,80 @@ +/* +Copyright 2018 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resources + +import ( + "encoding/json" + "fmt" + + eventingv1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" + servingv1alpha1 "github.com/knative/serving/pkg/apis/serving/v1alpha1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +type RouterArgs struct { + Broker *eventingv1alpha1.Broker + Image string + ServiceAccountName string + NeedsActivationHost string +} + +func MakeRouter(args *RouterArgs) (*servingv1alpha1.Service, error) { + selectorJson, err := json.Marshal(args.Broker.Spec.Selector) + if err != nil { + return nil, err + } + + return &servingv1alpha1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: args.Broker.Namespace, + Name: fmt.Sprintf("%s-broker", args.Broker.Name), + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(args.Broker, schema.GroupVersionKind{ + Group: args.Broker.GroupVersionKind().Group, + Version: args.Broker.GroupVersionKind().Version, + Kind: args.Broker.GroupVersionKind().Kind, + }), + }, + }, + Spec: servingv1alpha1.ServiceSpec{ + RunLatest: &servingv1alpha1.RunLatestType{ + Configuration: servingv1alpha1.ConfigurationSpec{ + RevisionTemplate: servingv1alpha1.RevisionTemplateSpec{ + Spec: servingv1alpha1.RevisionSpec{ + ServiceAccountName: args.ServiceAccountName, + Container: v1.Container{ + Image: args.Image, + Env: []v1.EnvVar{ + { + Name: "NEEDS_ACTIVATION_HOST", + Value: args.NeedsActivationHost, + }, + { + Name: "LABEL_SELECTOR", + Value: string(selectorJson), + }, + }, + }, + }, + }, + }, + }, + }, + }, nil +} diff --git a/pkg/provisioners/channel_util.go b/pkg/provisioners/channel_util.go index d84992e5664..ca5dc675fbc 100644 --- a/pkg/provisioners/channel_util.go +++ b/pkg/provisioners/channel_util.go @@ -46,21 +46,21 @@ const ( FinalizerAdded AddFinalizerResult = true ) -// AddFinalizer adds finalizerName to the Channel. -func AddFinalizer(c *eventingv1alpha1.Channel, finalizerName string) AddFinalizerResult { - finalizers := sets.NewString(c.Finalizers...) +// AddFinalizer adds finalizerName to the Object. +func AddFinalizer(o metav1.Object, finalizerName string) AddFinalizerResult { + finalizers := sets.NewString(o.GetFinalizers()...) if finalizers.Has(finalizerName) { return FinalizerAlreadyPresent } finalizers.Insert(finalizerName) - c.Finalizers = finalizers.List() + o.SetFinalizers(finalizers.List()) return FinalizerAdded } -func RemoveFinalizer(c *eventingv1alpha1.Channel, finalizerName string) { - finalizers := sets.NewString(c.Finalizers...) +func RemoveFinalizer(o metav1.Object, finalizerName string) { + finalizers := sets.NewString(o.GetFinalizers()...) finalizers.Delete(finalizerName) - c.Finalizers = finalizers.List() + o.SetFinalizers(finalizers.List()) } func CreateK8sService(ctx context.Context, client runtimeClient.Client, c *eventingv1alpha1.Channel) (*corev1.Service, error) { From da9fe36fd6d07709fe099630a3609b663291f050 Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Mon, 4 Feb 2019 15:58:52 -0800 Subject: [PATCH 008/128] Initial work on the new model and its corresponding broker controller. --- Gopkg.lock | 1 + pkg/apis/eventing/v1alpha1/broker_defaults.go | 42 +- pkg/apis/eventing/v1alpha1/broker_types.go | 37 +- .../eventing/v1alpha1/broker_validation.go | 60 +- .../eventing/v1alpha1/trigger_defaults.go | 3 + pkg/apis/eventing/v1alpha1/trigger_types.go | 21 +- .../eventing/v1alpha1/trigger_validation.go | 25 +- .../v1alpha1/zz_generated.deepcopy.go | 91 +-- pkg/controller/eventing/broker/provider.go | 8 +- pkg/controller/eventing/broker/reconcile.go | 153 +++-- .../eventing/broker/resources/activator.go | 75 --- .../eventing/broker/resources/filter.go | 107 +++ .../eventing/broker/resources/ingress.go | 112 ++++ .../eventing/broker/resources/router.go | 80 --- third_party/VENDOR-LICENSE | 625 +++++++++++++++++- 15 files changed, 980 insertions(+), 460 deletions(-) delete mode 100644 pkg/controller/eventing/broker/resources/activator.go create mode 100644 pkg/controller/eventing/broker/resources/filter.go create mode 100644 pkg/controller/eventing/broker/resources/ingress.go delete mode 100644 pkg/controller/eventing/broker/resources/router.go diff --git a/Gopkg.lock b/Gopkg.lock index 78ddd9acdc2..69eb95c7f07 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -1259,6 +1259,7 @@ "golang.org/x/sync/errgroup", "google.golang.org/api/option", "gopkg.in/yaml.v2", + "k8s.io/api/apps/v1", "k8s.io/api/core/v1", "k8s.io/api/rbac/v1", "k8s.io/apimachinery/pkg/api/equality", diff --git a/pkg/apis/eventing/v1alpha1/broker_defaults.go b/pkg/apis/eventing/v1alpha1/broker_defaults.go index c8a23a6eda5..890e4fbecf0 100644 --- a/pkg/apis/eventing/v1alpha1/broker_defaults.go +++ b/pkg/apis/eventing/v1alpha1/broker_defaults.go @@ -16,8 +16,6 @@ limitations under the License. package v1alpha1 -import v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - const ( brokerLabel = "eventing.knative.dev/broker" ) @@ -27,45 +25,7 @@ func (b *Broker) SetDefaults() { } func (bs *BrokerSpec) SetDefaults(brokerName string) { - if bs.Selector == nil { - bs.Selector = defaultBrokerSpecSelector(brokerName) - } - if bs.ChannelTemplate == nil { - bs.ChannelTemplate = defaultBrokerSpecChannelTemplate(brokerName) - } - if len(bs.SubscribableResources) == 0 { - bs.SubscribableResources = defaultBrokerSpecSubscribableResources() - } -} - -func defaultBrokerLabels(brokerName string) map[string]string { - return map[string]string{ - brokerLabel: brokerName, - } + // None } -func defaultBrokerSpecSelector(brokerName string) *v1.LabelSelector { - return &v1.LabelSelector{ - MatchLabels: defaultBrokerLabels(brokerName), - } -} -func defaultBrokerSpecChannelTemplate(brokerName string) *ChannelTemplateSpec { - return &ChannelTemplateSpec{ - Metadata: v1.ObjectMeta{ - Labels: defaultBrokerLabels(brokerName), - }, - // Spec is left blank so that the created Channel defaulter will default the provisioner - // and arguments when the Channel is created. - } -} - -func defaultBrokerSpecSubscribableResources() []v1.GroupVersionKind { - return []v1.GroupVersionKind{ - { - Group: "eventing.knative.dev", - Version: "v1alpha1", - Kind: "Channel", - }, - } -} diff --git a/pkg/apis/eventing/v1alpha1/broker_types.go b/pkg/apis/eventing/v1alpha1/broker_types.go index 20a6a945c3f..d3c2c5b4e2a 100644 --- a/pkg/apis/eventing/v1alpha1/broker_types.go +++ b/pkg/apis/eventing/v1alpha1/broker_types.go @@ -49,17 +49,10 @@ var _ runtime.Object = (*Broker)(nil) var _ webhook.GenericCRD = (*Broker)(nil) type BrokerSpec struct { - Selector *metav1.LabelSelector `json:"selector,omitempty"` - ChannelTemplate *ChannelTemplateSpec `json:"channelTemplate,omitempty"` - SubscribableResources []metav1.GroupVersionKind `json:"subscribableResources,omitempty"` + ChannelTemplate *ChannelSpec `json:"channelTemplate,omitempty"` } -type ChannelTemplateSpec struct { - Metadata metav1.ObjectMeta `json:"metadata,omitempty"` - Spec *ChannelSpec `json:"spec,omitempty"` -} - -var brokerCondSet = duckv1alpha1.NewLivingConditionSet(BrokerConditionChannelTemplateSelector, BrokerConditionSubscribableResourcesExist, BrokerConditionSubscribableResourcesExist, BrokerConditionAddressable) +var brokerCondSet = duckv1alpha1.NewLivingConditionSet(BrokerConditionIngress, BrokerConditionChannel, BrokerConditionFilter, BrokerConditionAddressable) // BrokerStatus represents the current state of a Broker. type BrokerStatus struct { @@ -88,11 +81,11 @@ type BrokerStatus struct { const ( BrokerConditionReady = duckv1alpha1.ConditionReady - BrokerConditionChannelTemplateSelector duckv1alpha1.ConditionType = "ChannelTemplateSelector" + BrokerConditionIngress duckv1alpha1.ConditionType = "Ingress" - BrokerConditionSubscribableResourcesExist duckv1alpha1.ConditionType = "SubscribableResourcesExist" + BrokerConditionChannel duckv1alpha1.ConditionType = "Channel" - BrokerConditionRouterAndActivatorExist duckv1alpha1.ConditionType = "RouterAndActivatorCreated" + BrokerConditionFilter duckv1alpha1.ConditionType = "Filter" BrokerConditionAddressable duckv1alpha1.ConditionType = "Addressable" ) @@ -112,24 +105,24 @@ func (bs *BrokerStatus) InitializeConditions() { brokerCondSet.Manage(bs).InitializeConditions() } -func (bs *BrokerStatus) MarkChannelTemplateMatchesSelector() { - brokerCondSet.Manage(bs).MarkTrue(BrokerConditionChannelTemplateSelector) +func (bs *BrokerStatus) MarkIngressReady() { + brokerCondSet.Manage(bs).MarkTrue(BrokerConditionIngress) } -func (bs *BrokerStatus) MarkChannelTemplateDoesNotMatchSelector() { - brokerCondSet.Manage(bs).MarkFalse(BrokerConditionChannelTemplateSelector, "selectorDoesNotMatchTemplate", "`spec.selector` does not match `spec.channelTempalte.meta.labels`") +func (bs *BrokerStatus) MarkIngressFailed(err error) { + brokerCondSet.Manage(bs).MarkFalse(BrokerConditionIngress, "failed", "%v", err) } -func (bs *BrokerStatus) MarkSubscribableResourcesExist() { - brokerCondSet.Manage(bs).MarkTrue(BrokerConditionSubscribableResourcesExist) +func (bs *BrokerStatus) MarkChannelReady() { + brokerCondSet.Manage(bs).MarkTrue(BrokerConditionChannel) } -func (bs *BrokerStatus) MarkSubscribableResourcesDoNotExist(dontExist []metav1.GroupVersionKind) { - brokerCondSet.Manage(bs).MarkFalse(BrokerConditionSubscribableResourcesExist, "resourcesDontExist", "The following resources do not exist: %v", dontExist) +func (bs *BrokerStatus) MarkChannelFailed(err error) { + brokerCondSet.Manage(bs).MarkFalse(BrokerConditionChannel, "failed", "%v", err) } -func (bs *BrokerStatus) MarkRouterAndActivatorExist() { - brokerCondSet.Manage(bs).MarkTrue(BrokerConditionRouterAndActivatorExist) +func (bs *BrokerStatus) MarkFilterReady() { + brokerCondSet.Manage(bs).MarkTrue(BrokerConditionFilter) } // SetAddress makes this Channel addressable by setting the hostname. It also diff --git a/pkg/apis/eventing/v1alpha1/broker_validation.go b/pkg/apis/eventing/v1alpha1/broker_validation.go index 98d31a4b9cd..c7954c26d15 100644 --- a/pkg/apis/eventing/v1alpha1/broker_validation.go +++ b/pkg/apis/eventing/v1alpha1/broker_validation.go @@ -17,12 +17,7 @@ limitations under the License. package v1alpha1 import ( - "fmt" - - "github.com/google/go-cmp/cmp" "github.com/knative/pkg/apis" - "k8s.io/apimachinery/pkg/api/equality" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func (b *Broker) Validate() *apis.FieldError { @@ -30,63 +25,10 @@ func (b *Broker) Validate() *apis.FieldError { } func (bs *BrokerSpec) Validate() *apis.FieldError { - var errs *apis.FieldError - if isSelectorNotPresentOrEmpty(bs.Selector) { - fe := apis.ErrMissingField("selector") - fe.Details = "the Broker must have a selector" - errs = errs.Also(fe) - } - - if bs.ChannelTemplate == nil { - fe := apis.ErrMissingField("channelTemplate") - fe.Details = "the Broker must have a channelTemplate" - errs = errs.Also(fe) - } else if !selectorMatchesTemplateLabels(bs.Selector, bs.ChannelTemplate) { - fe := apis.ErrInvalidValue(fmt.Sprint(bs.ChannelTemplate.Metadata.Labels), "channelTemplate.metadata.labels") - errs = errs.Also(fe) - } - - if !channelsInSubscribableResources(bs.SubscribableResources) { - fe := apis.ErrInvalidValue(fmt.Sprint(bs.SubscribableResources), "subscribableResources") - errs = errs.Also(fe) - } - - return errs -} - -func isSelectorNotPresentOrEmpty(s *v1.LabelSelector) bool { - return s == nil || equality.Semantic.DeepEqual(s, &v1.LabelSelector{}) + return nil } -func selectorMatchesTemplateLabels(s *v1.LabelSelector, ct *ChannelTemplateSpec) bool { - // TODO Improve this so it supports something other than direct label equality. - return equality.Semantic.DeepEqual(s.MatchLabels, ct.Metadata.Labels) -} - -func channelsInSubscribableResources(sr []v1.GroupVersionKind) bool { - for _, r := range sr { - if r.Group == "eventing.knative.dev" && r.Version == "v1alpha1" && r.Kind == "Channel" { - return true - } - } - return false -} func (b *Broker) CheckImmutableFields(og apis.Immutable) *apis.FieldError { - original, ok := og.(*Broker) - if !ok { - return &apis.FieldError{Message: "The provided original was not a Broker"} - } - if original == nil { - return nil - } - - if diff := cmp.Diff(original.Spec.Selector, b.Spec.Selector); diff != "" { - return &apis.FieldError{ - Message: "Immutable fields changed (-old +new)", - Paths: []string{"spec", "selector"}, - Details: diff, - } - } return nil } diff --git a/pkg/apis/eventing/v1alpha1/trigger_defaults.go b/pkg/apis/eventing/v1alpha1/trigger_defaults.go index 732d9fde924..ac2dfdf843f 100644 --- a/pkg/apis/eventing/v1alpha1/trigger_defaults.go +++ b/pkg/apis/eventing/v1alpha1/trigger_defaults.go @@ -24,4 +24,7 @@ func (ts *TriggerSpec) SetDefaults() { if ts.Broker == "" { ts.Broker = "default" } + if ts.Type == "" { + ts.Type = "ANY" + } } diff --git a/pkg/apis/eventing/v1alpha1/trigger_types.go b/pkg/apis/eventing/v1alpha1/trigger_types.go index 432c2d3eb32..e6dc53bb9a8 100644 --- a/pkg/apis/eventing/v1alpha1/trigger_types.go +++ b/pkg/apis/eventing/v1alpha1/trigger_types.go @@ -50,20 +50,13 @@ var _ webhook.GenericCRD = (*Trigger)(nil) type TriggerSpec struct { Broker string `json:"broker,omitempty"` - Selector *TriggerSelectorSpec `json:"selector,omitempty"` Subscriber *SubscriberSpec `json:"subscriber,omitempty"` Type string `json:"type,omitempty"` Source string `json:"source,omitempty"` } -type TriggerSelectorSpec struct { - Header map[string]string `json:"header,omitempty"` - HeaderExpression []metav1.LabelSelectorRequirement `json:"headerExpression,omitEmpty"` - OPAPolicy string `json:"opaPolicy,omitEmpty"` -} - -var triggerCondSet = duckv1alpha1.NewLivingConditionSet(TriggerConditionBrokerExists, TriggerConditionSubscribed) +var triggerCondSet = duckv1alpha1.NewLivingConditionSet(TriggerConditionBrokerExists, TriggerConditionKubernetesService, TriggerConditionVirtualService, TriggerConditionSubscribed) // TriggerStatus represents the current state of a Trigger. type TriggerStatus struct { @@ -87,6 +80,10 @@ const ( TriggerConditionBrokerExists duckv1alpha1.ConditionType = "BrokerExists" + TriggerConditionKubernetesService duckv1alpha1.ConditionType = "KubernetesService" + + TriggerConditionVirtualService duckv1alpha1.ConditionType = "VirtualService" + TriggerConditionSubscribed duckv1alpha1.ConditionType = "Subscribed" ) @@ -113,6 +110,14 @@ func (ts *TriggerStatus) MarkBrokerDoesNotExists() { triggerCondSet.Manage(ts).MarkFalse(TriggerConditionBrokerExists, "doesNotExist", "Broker does not exist") } +func (ts *TriggerSpec) MarkKubernetesServiceExists() { + triggerCondSet.Manage(ts).MarkTrue(TriggerConditionKubernetesService) +} + +func (ts *TriggerSpec) MarkVirtualServiceExists() { + triggerCondSet.Manage(ts).MarkTrue(TriggerConditionVirtualService) +} + func (ts *TriggerStatus) MarkSubscribed() { triggerCondSet.Manage(ts).MarkTrue(TriggerConditionSubscribed) } diff --git a/pkg/apis/eventing/v1alpha1/trigger_validation.go b/pkg/apis/eventing/v1alpha1/trigger_validation.go index 6ff7301e244..9d0178a464f 100644 --- a/pkg/apis/eventing/v1alpha1/trigger_validation.go +++ b/pkg/apis/eventing/v1alpha1/trigger_validation.go @@ -19,7 +19,6 @@ package v1alpha1 import ( "github.com/google/go-cmp/cmp" "github.com/knative/pkg/apis" - "k8s.io/apimachinery/pkg/api/equality" ) func (t *Trigger) Validate() *apis.FieldError { @@ -33,10 +32,8 @@ func (ts *TriggerSpec) Validate() *apis.FieldError { errs = errs.Also(fe) } - if ts.Selector == nil { - fe := apis.ErrMissingField("selector") - errs = errs.Also(fe) - } else if fe := multipleSelectorsSet(ts.Selector); fe != nil { + if ts.Type == "" { + fe := apis.ErrMissingField("type") errs = errs.Also(fe) } @@ -50,24 +47,6 @@ func (ts *TriggerSpec) Validate() *apis.FieldError { return errs } -func multipleSelectorsSet(s *TriggerSelectorSpec) *apis.FieldError { - var fields []string - if !equality.Semantic.DeepEqual(s.Header, map[string]string{}) { - fields = append(fields, "header") - } - if len(s.HeaderExpression) > 0 { - fields = append(fields, "headerExpression") - } - if s.OPAPolicy != "" { - fields = append(fields, "opaPolicy") - } - - if len(fields) != 1 { - return apis.ErrMultipleOneOf(fields...) - } - return nil -} - func (t *Trigger) CheckImmutableFields(og apis.Immutable) *apis.FieldError { original, ok := og.(*Trigger) if !ok { diff --git a/pkg/apis/eventing/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/eventing/v1alpha1/zz_generated.deepcopy.go index 77046af1c1d..fc8f77d0aee 100644 --- a/pkg/apis/eventing/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/eventing/v1alpha1/zz_generated.deepcopy.go @@ -23,8 +23,7 @@ package v1alpha1 import ( apis_duck_v1alpha1 "github.com/knative/eventing/pkg/apis/duck/v1alpha1" duck_v1alpha1 "github.com/knative/pkg/apis/duck/v1alpha1" - core_v1 "k8s.io/api/core/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + v1 "k8s.io/api/core/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -92,29 +91,15 @@ func (in *BrokerList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BrokerSpec) DeepCopyInto(out *BrokerSpec) { *out = *in - if in.Selector != nil { - in, out := &in.Selector, &out.Selector - if *in == nil { - *out = nil - } else { - *out = new(v1.LabelSelector) - (*in).DeepCopyInto(*out) - } - } if in.ChannelTemplate != nil { in, out := &in.ChannelTemplate, &out.ChannelTemplate if *in == nil { *out = nil } else { - *out = new(ChannelTemplateSpec) + *out = new(ChannelSpec) (*in).DeepCopyInto(*out) } } - if in.SubscribableResources != nil { - in, out := &in.SubscribableResources, &out.SubscribableResources - *out = make([]v1.GroupVersionKind, len(*in)) - copy(*out, *in) - } return } @@ -138,6 +123,7 @@ func (in *BrokerStatus) DeepCopyInto(out *BrokerStatus) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + out.Address = in.Address return } @@ -220,7 +206,7 @@ func (in *ChannelSpec) DeepCopyInto(out *ChannelSpec) { if *in == nil { *out = nil } else { - *out = new(core_v1.ObjectReference) + *out = new(v1.ObjectReference) **out = **in } } @@ -288,32 +274,6 @@ func (in *ChannelStatus) DeepCopy() *ChannelStatus { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ChannelTemplateSpec) DeepCopyInto(out *ChannelTemplateSpec) { - *out = *in - in.Metadata.DeepCopyInto(&out.Metadata) - if in.Spec != nil { - in, out := &in.Spec, &out.Spec - if *in == nil { - *out = nil - } else { - *out = new(ChannelSpec) - (*in).DeepCopyInto(*out) - } - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ChannelTemplateSpec. -func (in *ChannelTemplateSpec) DeepCopy() *ChannelTemplateSpec { - if in == nil { - return nil - } - out := new(ChannelTemplateSpec) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ClusterChannelProvisioner) DeepCopyInto(out *ClusterChannelProvisioner) { *out = *in @@ -422,7 +382,7 @@ func (in *ReplyStrategy) DeepCopyInto(out *ReplyStrategy) { if *in == nil { *out = nil } else { - *out = new(core_v1.ObjectReference) + *out = new(v1.ObjectReference) **out = **in } } @@ -447,7 +407,7 @@ func (in *SubscriberSpec) DeepCopyInto(out *SubscriberSpec) { if *in == nil { *out = nil } else { - *out = new(core_v1.ObjectReference) + *out = new(v1.ObjectReference) **out = **in } } @@ -670,48 +630,9 @@ func (in *TriggerList) DeepCopyObject() runtime.Object { return nil } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *TriggerSelectorSpec) DeepCopyInto(out *TriggerSelectorSpec) { - *out = *in - if in.Header != nil { - in, out := &in.Header, &out.Header - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - if in.HeaderExpression != nil { - in, out := &in.HeaderExpression, &out.HeaderExpression - *out = make([]v1.LabelSelectorRequirement, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TriggerSelectorSpec. -func (in *TriggerSelectorSpec) DeepCopy() *TriggerSelectorSpec { - if in == nil { - return nil - } - out := new(TriggerSelectorSpec) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TriggerSpec) DeepCopyInto(out *TriggerSpec) { *out = *in - if in.Selector != nil { - in, out := &in.Selector, &out.Selector - if *in == nil { - *out = nil - } else { - *out = new(TriggerSelectorSpec) - (*in).DeepCopyInto(*out) - } - } if in.Subscriber != nil { in, out := &in.Subscriber, &out.Subscriber if *in == nil { diff --git a/pkg/controller/eventing/broker/provider.go b/pkg/controller/eventing/broker/provider.go index 9315366245c..8b78d982685 100644 --- a/pkg/controller/eventing/broker/provider.go +++ b/pkg/controller/eventing/broker/provider.go @@ -44,10 +44,10 @@ type reconciler struct { logger *zap.Logger - routerImage string - routerServiceAccountName string - activatorImage string - activatorServiceAccountName string + routerImage string + routerServiceAccountName string + filterImage string + filterServiceAccountName string } // Verify the struct implements reconcile.Reconciler diff --git a/pkg/controller/eventing/broker/reconcile.go b/pkg/controller/eventing/broker/reconcile.go index b9daa891a69..720ef29a266 100644 --- a/pkg/controller/eventing/broker/reconcile.go +++ b/pkg/controller/eventing/broker/reconcile.go @@ -19,6 +19,8 @@ package broker import ( "context" "fmt" + "github.com/knative/eventing/pkg/controller" + "k8s.io/api/apps/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime/schema" @@ -91,57 +93,50 @@ func (r *reconciler) Reconcile(request reconcile.Request) (reconcile.Result, err func (r *reconciler) reconcile(ctx context.Context, b *v1alpha1.Broker) error { b.Status.InitializeConditions() - // 1. Create the router. - // 2. Create the activator. - // 3. Create the filter [do not do for the prototype]. - // 4. Create the 'needs-activation' Channel. - // 5. Create Subscription from 'needs-activation' Channel to the activator. + // 1. Channel is created for all events. + // 2. Filter Deployment. + // 3. Ingress Deployment. + // 4. K8s Service that points at the Deployment. if b.DeletionTimestamp != nil { // Everything is cleaned up by the garbage collector. return nil } - // TODO Actually check this, for now the webhook ensures they are identical. - b.Status.MarkChannelTemplateMatchesSelector() - - dontExist, err := r.verifyResourcesExist(ctx, b.Spec.SubscribableResources) + c, err := r.reconcileChannel(ctx, b) if err != nil { - logging.FromContext(ctx).Error("Unable to determine if resources exist", zap.Error(err)) + logging.FromContext(ctx).Error("Problem reconciling the channel", zap.Error(err)) + b.Status.MarkChannelFailed(err) return err - } else if len(dontExist) > 0 { - b.Status.MarkSubscribableResourcesDoNotExist(dontExist) - } else { - b.Status.MarkSubscribableResourcesExist() } + b.Status.MarkChannelReady() - activator, err := r.reconcileActivator(ctx, b) + _, err = r.reconcileFilterDeployment(ctx, b) if err != nil { - logging.FromContext(ctx).Error("Problem reconciling activator", zap.Error(err)) + logging.FromContext(ctx).Error("Problem reconciling filter deployment", zap.Error(err)) return err } - - // TODO Add the filter reconciliation here. - - c, err := r.reconcileNeedsActivationChannel(ctx, b) + _, err = r.reconcileFilterService(ctx, b) if err != nil { - logging.FromContext(ctx).Error("Problem reconciling the needs activation channel", zap.Error(err)) + logging.FromContext(ctx).Error("Problem reconciling filter service", zap.Error(err)) return err } + b.Status.MarkFilterReady() - _, err = r.reconcileNeedsActivationSubscription(ctx, b, activator, c) + _, err = r.reconcileIngressDeployment(ctx, b, c) if err != nil { - logging.FromContext(ctx).Error("Problem reconciling the needs activation subscription", zap.Error(err)) + logging.FromContext(ctx).Error("Problem reconciling ingress deployment", zap.Error(err)) return err } - b.Status.MarkRouterAndActivatorExist() - router, err := r.reconcileRouter(ctx, b, c) + + svc, err := r.reconcileIngressService(ctx, b) if err != nil { - logging.FromContext(ctx).Error("Problem reconciling router", zap.Error(err)) + logging.FromContext(ctx).Error("Problem reconciling ingress Service", zap.Error(err)) return err } - b.Status.SetAddress(router.Status.Address.Hostname) + b.Status.MarkIngressReady() + b.Status.SetAddress(controller.ServiceHostName(svc.Name, svc.Namespace)) return nil } @@ -185,28 +180,27 @@ func (r *reconciler) updateStatus(broker *v1alpha1.Broker) (*v1alpha1.Broker, er return latestBroker, nil } -func (r *reconciler) verifyResourcesExist(ctx context.Context, resources []metav1.GroupVersionKind) ([]metav1.GroupVersionKind, error) { - dontExist := make([]metav1.GroupVersionKind, 0, len(resources)) - // TODO Implement, for now the webhook asserts it is only Channel, which we assume exists. - return dontExist, nil -} - -func (r *reconciler) reconcileActivator(ctx context.Context, b *v1alpha1.Broker) (*servingv1alpha1.Service, error) { - expected, err := resources.MakeActivator(&resources.ActivatorArgs{ +func (r *reconciler) reconcileFilterDeployment(ctx context.Context, b *v1alpha1.Broker) (*v1.Deployment, error) { + expected, err := resources.MakeFilterDeployment(&resources.FilterArgs{ Broker: b, - Image: r.activatorImage, - ServiceAccountName: r.activatorServiceAccountName, + Image: r.filterImage, + ServiceAccountName: r.filterServiceAccountName, }) if err != nil { return nil, err } - return r.reconcileKSvc(ctx, expected) + return r.reconcileDeployment(ctx, expected) +} + +func (r *reconciler) reconcileFilterService(ctx context.Context, b *v1alpha1.Broker) (*corev1.Service, error) { + expected := resources.MakeFilterService(b) + return r.reconcileService(ctx, expected) } -func (r *reconciler) reconcileNeedsActivationChannel(ctx context.Context, b *v1alpha1.Broker) (*v1alpha1.Channel, error) { - expected := newNeedsActivationChannel(b) +func (r *reconciler) reconcileChannel(ctx context.Context, b *v1alpha1.Broker) (*v1alpha1.Channel, error) { + expected := newChannel(b) - c, err := r.getNeedsActivationChannel(ctx, b) + c, err := r.getChannel(ctx, b) // If the resource doesn't exist, we'll create it if k8serrors.IsNotFound(err) { c = expected @@ -232,11 +226,11 @@ func (r *reconciler) reconcileNeedsActivationChannel(ctx context.Context, b *v1a return c, nil } -func (r *reconciler) getNeedsActivationChannel(ctx context.Context, b *v1alpha1.Broker) (*v1alpha1.Channel, error) { +func (r *reconciler) getChannel(ctx context.Context, b *v1alpha1.Broker) (*v1alpha1.Channel, error) { list := &v1alpha1.ChannelList{} opts := &runtimeclient.ListOptions{ Namespace: b.Namespace, - LabelSelector: labels.SelectorFromSet(needsActivationLabels(b)), + LabelSelector: labels.SelectorFromSet(channelLabels(b)), // TODO this is here because the fake client needs it. Remove this when it's no longer // needed. Raw: &metav1.ListOptions{ @@ -260,16 +254,16 @@ func (r *reconciler) getNeedsActivationChannel(ctx context.Context, b *v1alpha1. return nil, k8serrors.NewNotFound(schema.GroupResource{}, "") } -func newNeedsActivationChannel(b *v1alpha1.Broker) *v1alpha1.Channel { +func newChannel(b *v1alpha1.Broker) *v1alpha1.Channel { var spec v1alpha1.ChannelSpec - if b.Spec.ChannelTemplate != nil && b.Spec.ChannelTemplate.Spec != nil { - spec = *b.Spec.ChannelTemplate.Spec + if b.Spec.ChannelTemplate != nil { + spec = *b.Spec.ChannelTemplate } return &v1alpha1.Channel{ ObjectMeta: metav1.ObjectMeta{ Namespace: b.Namespace, - GenerateName: fmt.Sprintf("%s-broker-needs-activation-", b.Name), + GenerateName: fmt.Sprintf("%s-broker-", b.Name), OwnerReferences: []metav1.OwnerReference{ *metav1.NewControllerRef(b, schema.GroupVersionKind{ Group: b.GroupVersionKind().Group, @@ -282,10 +276,10 @@ func newNeedsActivationChannel(b *v1alpha1.Broker) *v1alpha1.Channel { } } -func needsActivationLabels(b *v1alpha1.Broker) map[string]string { +func channelLabels(b *v1alpha1.Broker) map[string]string { return map[string]string{ "eventing.knative.dev/broker": b.Name, - "eventing.knative.dev/broker/needsActivation": "true", + "eventing.knative.dev/broker/everything": "true", } } @@ -320,7 +314,7 @@ func (r *reconciler) getNeedsActivationSubscription(ctx context.Context, b *v1al list := &v1alpha1.SubscriptionList{} opts := &runtimeclient.ListOptions{ Namespace: b.Namespace, - LabelSelector: labels.SelectorFromSet(needsActivationLabels(b)), + LabelSelector: labels.SelectorFromSet(channelLabels(b)), // TODO this is here because the fake client needs it. Remove this when it's no longer // needed. Raw: &metav1.ListOptions{ @@ -378,25 +372,25 @@ func newNeedsActivationSubscription(b *v1alpha1.Broker, c *v1alpha1.Channel, act } } -func (r *reconciler) reconcileKSvc(ctx context.Context, ksvc *servingv1alpha1.Service) (*servingv1alpha1.Service, error) { +func (r *reconciler) reconcileDeployment(ctx context.Context, d *v1.Deployment) (*v1.Deployment, error) { name := types.NamespacedName{ - Namespace: ksvc.Namespace, - Name: ksvc.Name, + Namespace: d.Namespace, + Name: d.Name, } - current := &servingv1alpha1.Service{} + current := &v1.Deployment{} err := r.client.Get(ctx, name, current) if k8serrors.IsNotFound(err) { - err = r.client.Create(ctx, ksvc) + err = r.client.Create(ctx, d) if err != nil { return nil, err } - return ksvc, nil + return d, nil } else if err != nil { return nil, err } - if !equality.Semantic.DeepDerivative(ksvc.Spec, current.Spec) { - current.Spec = ksvc.Spec + if !equality.Semantic.DeepDerivative(d.Spec, current.Spec) { + current.Spec = d.Spec err = r.client.Update(ctx, current) if err != nil { return nil, err @@ -405,15 +399,50 @@ func (r *reconciler) reconcileKSvc(ctx context.Context, ksvc *servingv1alpha1.Se return current, nil } -func (r *reconciler) reconcileRouter(ctx context.Context, b *v1alpha1.Broker, c *v1alpha1.Channel) (*servingv1alpha1.Service, error) { - expected, err := resources.MakeRouter(&resources.RouterArgs{ +func (r *reconciler) reconcileService(ctx context.Context, svc *corev1.Service) (*corev1.Service, error) { + name := types.NamespacedName{ + Namespace: svc.Namespace, + Name: svc.Name, + } + current := &corev1.Service{} + err := r.client.Get(ctx, name, current) + if k8serrors.IsNotFound(err) { + err = r.client.Create(ctx, svc) + if err != nil { + return nil, err + } + return svc, nil + } else if err != nil { + return nil, err + } + + // spec.clusterIP is immutable and is set on existing services. If we don't set this to the same value, we will + // encounter an error while updating. + svc.Spec.ClusterIP = current.Spec.ClusterIP + if !equality.Semantic.DeepDerivative(svc.Spec, current.Spec) { + current.Spec = svc.Spec + err = r.client.Update(ctx, current) + if err != nil { + return nil, err + } + } + return current, nil +} + +func (r *reconciler) reconcileIngressDeployment(ctx context.Context, b *v1alpha1.Broker, c *v1alpha1.Channel) (*v1.Deployment, error) { + expected, err := resources.MakeIngress(&resources.IngressArgs{ Broker: b, Image: r.routerImage, ServiceAccountName: r.routerServiceAccountName, - NeedsActivationHost: c.Status.Address.Hostname, + ChannelAddress: c.Status.Address.Hostname, }) if err != nil { return nil, err } - return r.reconcileKSvc(ctx, expected) + return r.reconcileDeployment(ctx, expected) +} + +func (r *reconciler) reconcileIngressService(ctx context.Context, b *v1alpha1.Broker) (*corev1.Service, error) { + expected := resources.MakeIngressService(b) + return r.reconcileService(ctx, expected) } diff --git a/pkg/controller/eventing/broker/resources/activator.go b/pkg/controller/eventing/broker/resources/activator.go deleted file mode 100644 index 1de0820eef1..00000000000 --- a/pkg/controller/eventing/broker/resources/activator.go +++ /dev/null @@ -1,75 +0,0 @@ -/* -Copyright 2018 The Knative Authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package resources - -import ( - "encoding/json" - "fmt" - - eventingv1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" - servingv1alpha1 "github.com/knative/serving/pkg/apis/serving/v1alpha1" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" -) - -type ActivatorArgs struct { - Broker *eventingv1alpha1.Broker - Image string - ServiceAccountName string -} - -func MakeActivator(args *ActivatorArgs) (*servingv1alpha1.Service, error) { - templateJson, err := json.Marshal(args.Broker.Spec.ChannelTemplate) - if err != nil { - return nil, err - } - - return &servingv1alpha1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: args.Broker.Namespace, - Name: fmt.Sprintf("%s-broker-activator", args.Broker.Name), - OwnerReferences: []metav1.OwnerReference{ - *metav1.NewControllerRef(args.Broker, schema.GroupVersionKind{ - Group: args.Broker.GroupVersionKind().Group, - Version: args.Broker.GroupVersionKind().Version, - Kind: args.Broker.GroupVersionKind().Kind, - }), - }, - }, - Spec: servingv1alpha1.ServiceSpec{ - RunLatest: &servingv1alpha1.RunLatestType{ - Configuration: servingv1alpha1.ConfigurationSpec{ - RevisionTemplate: servingv1alpha1.RevisionTemplateSpec{ - Spec: servingv1alpha1.RevisionSpec{ - ServiceAccountName: args.ServiceAccountName, - Container: v1.Container{ - Image: args.Image, - Env: []v1.EnvVar{ - { - Name: "CHANNEL_TEMPLATE", - Value: string(templateJson), - }, - }, - }, - }, - }, - }, - }, - }, - }, nil -} diff --git a/pkg/controller/eventing/broker/resources/filter.go b/pkg/controller/eventing/broker/resources/filter.go new file mode 100644 index 00000000000..62a25a4103c --- /dev/null +++ b/pkg/controller/eventing/broker/resources/filter.go @@ -0,0 +1,107 @@ +/* +Copyright 2018 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resources + +import ( + "fmt" + appsv1 "k8s.io/api/apps/v1" + + eventingv1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +type FilterArgs struct { + Broker *eventingv1alpha1.Broker + Image string + ServiceAccountName string +} + +func MakeFilterDeployment(args *FilterArgs) (*appsv1.Deployment, error) { + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: args.Broker.Namespace, + Name: fmt.Sprintf("%s-broker-filter", args.Broker.Name), + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(args.Broker, schema.GroupVersionKind{ + Group: args.Broker.GroupVersionKind().Group, + Version: args.Broker.GroupVersionKind().Version, + Kind: args.Broker.GroupVersionKind().Kind, + }), + }, + }, + Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: filterLabels(args.Broker), + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: filterLabels(args.Broker), + }, + Spec: corev1.PodSpec{ + ServiceAccountName: args.ServiceAccountName, + Containers: []corev1.Container{ + { + Image: args.Image, + Name: "ingress", + Env: []corev1.EnvVar{ + { + Name: "BROKER", + Value: args.Broker.Name, + }, + }, + }, + }, + }, + }, + }, + }, nil +} + +func MakeFilterService(b *eventingv1alpha1.Broker) *corev1.Service { + return &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: b.Namespace, + Name: fmt.Sprintf("%s-broker-filter", b.Name), + Labels: filterLabels(b), + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(b, schema.GroupVersionKind{ + Group: b.GroupVersionKind().Group, + Version: b.GroupVersionKind().Version, + Kind: b.GroupVersionKind().Kind, + }), + }, + }, + Spec: corev1.ServiceSpec{ + Selector: filterLabels(b), + Ports: []corev1.ServicePort{ + { + Name: "http", + Port: 80, + }, + }, + }, + } +} + +func filterLabels(b *eventingv1alpha1.Broker) map[string]string { + return map[string]string{ + "eventing.knative.dev/broker": b.Name, + "eventing.knative.dev/broker/role": "filter", + } +} diff --git a/pkg/controller/eventing/broker/resources/ingress.go b/pkg/controller/eventing/broker/resources/ingress.go new file mode 100644 index 00000000000..d129361f55f --- /dev/null +++ b/pkg/controller/eventing/broker/resources/ingress.go @@ -0,0 +1,112 @@ +/* +Copyright 2018 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resources + +import ( + "fmt" + appsv1 "k8s.io/api/apps/v1" + + eventingv1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +type IngressArgs struct { + Broker *eventingv1alpha1.Broker + Image string + ServiceAccountName string + ChannelAddress string +} + +func MakeIngress(args *IngressArgs) (*appsv1.Deployment, error) { + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: args.Broker.Namespace, + Name: fmt.Sprintf("%s-broker-ingress", args.Broker.Name), + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(args.Broker, schema.GroupVersionKind{ + Group: args.Broker.GroupVersionKind().Group, + Version: args.Broker.GroupVersionKind().Version, + Kind: args.Broker.GroupVersionKind().Kind, + }), + }, + }, + Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: ingressLabels(args.Broker), + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: ingressLabels(args.Broker), + }, + Spec: corev1.PodSpec{ + ServiceAccountName: args.ServiceAccountName, + Containers: []corev1.Container{ + { + Image: args.Image, + Name: "ingress", + Env: []corev1.EnvVar{ + { + Name: "FILTER", + Value: "", // TODO Add one. + }, + { + Name: "CHANNEL", + Value: args.ChannelAddress, + }, + }, + }, + }, + }, + }, + }, + }, nil +} + +func MakeIngressService(b *eventingv1alpha1.Broker) *corev1.Service { + return &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: b.Namespace, + Name: fmt.Sprintf("%s-broker", b.Name), + Labels: ingressLabels(b), + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(b, schema.GroupVersionKind{ + Group: b.GroupVersionKind().Group, + Version: b.GroupVersionKind().Version, + Kind: b.GroupVersionKind().Kind, + }), + }, + }, + Spec: corev1.ServiceSpec{ + Selector: ingressLabels(b), + Ports: []corev1.ServicePort{ + { + Name: "http", + Port: 80, + }, + }, + }, + } +} + +func ingressLabels(b *eventingv1alpha1.Broker) map[string]string { + return map[string]string{ + "eventing.knative.dev/broker": b.Name, + "eventing.knative.dev/broker/role": "ingress", + } +} diff --git a/pkg/controller/eventing/broker/resources/router.go b/pkg/controller/eventing/broker/resources/router.go deleted file mode 100644 index baf8ac4b39c..00000000000 --- a/pkg/controller/eventing/broker/resources/router.go +++ /dev/null @@ -1,80 +0,0 @@ -/* -Copyright 2018 The Knative Authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package resources - -import ( - "encoding/json" - "fmt" - - eventingv1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" - servingv1alpha1 "github.com/knative/serving/pkg/apis/serving/v1alpha1" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" -) - -type RouterArgs struct { - Broker *eventingv1alpha1.Broker - Image string - ServiceAccountName string - NeedsActivationHost string -} - -func MakeRouter(args *RouterArgs) (*servingv1alpha1.Service, error) { - selectorJson, err := json.Marshal(args.Broker.Spec.Selector) - if err != nil { - return nil, err - } - - return &servingv1alpha1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: args.Broker.Namespace, - Name: fmt.Sprintf("%s-broker", args.Broker.Name), - OwnerReferences: []metav1.OwnerReference{ - *metav1.NewControllerRef(args.Broker, schema.GroupVersionKind{ - Group: args.Broker.GroupVersionKind().Group, - Version: args.Broker.GroupVersionKind().Version, - Kind: args.Broker.GroupVersionKind().Kind, - }), - }, - }, - Spec: servingv1alpha1.ServiceSpec{ - RunLatest: &servingv1alpha1.RunLatestType{ - Configuration: servingv1alpha1.ConfigurationSpec{ - RevisionTemplate: servingv1alpha1.RevisionTemplateSpec{ - Spec: servingv1alpha1.RevisionSpec{ - ServiceAccountName: args.ServiceAccountName, - Container: v1.Container{ - Image: args.Image, - Env: []v1.EnvVar{ - { - Name: "NEEDS_ACTIVATION_HOST", - Value: args.NeedsActivationHost, - }, - { - Name: "LABEL_SELECTOR", - Value: string(selectorJson), - }, - }, - }, - }, - }, - }, - }, - }, - }, nil -} diff --git a/third_party/VENDOR-LICENSE b/third_party/VENDOR-LICENSE index fea782378a7..f2dcb874614 100644 --- a/third_party/VENDOR-LICENSE +++ b/third_party/VENDOR-LICENSE @@ -1685,6 +1685,213 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +=========================================================== +Import: github.com/knative/eventing/vendor/github.com/google/go-containerregistry + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + =========================================================== Import: github.com/knative/eventing/vendor/github.com/google/gofuzz @@ -2639,7 +2846,423 @@ SOFTWARE. =========================================================== -Import: github.com/knative/eventing/vendor/github.com/knative/pkg +Import: github.com/knative/eventing/vendor/github.com/knative/build + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + + +=========================================================== +Import: github.com/knative/eventing/vendor/github.com/knative/pkg + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + + +=========================================================== +Import: github.com/knative/eventing/vendor/github.com/knative/serving + Apache License Version 2.0, January 2004 From f8cca2eaa8c03568bb7de579e9d2f84703db00e2 Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Mon, 4 Feb 2019 21:12:46 -0800 Subject: [PATCH 009/128] Initial work on the Trigger controller. --- cmd/controller/main.go | 11 +- config/500-controller.yaml | 9 + pkg/controller/eventing/broker/provider.go | 11 +- pkg/controller/eventing/broker/reconcile.go | 99 +-- pkg/controller/eventing/trigger/provider.go | 28 - pkg/controller/eventing/trigger/reconcile.go | 363 +++++++---- third_party/VENDOR-LICENSE | 623 ------------------- 7 files changed, 280 insertions(+), 864 deletions(-) diff --git a/cmd/controller/main.go b/cmd/controller/main.go index 22212fc6e6c..52661d99c5e 100644 --- a/cmd/controller/main.go +++ b/cmd/controller/main.go @@ -21,6 +21,7 @@ import ( "flag" "log" "net/http" + "os" "time" "github.com/knative/eventing/pkg/controller/eventing/trigger" @@ -123,7 +124,7 @@ func main() { // manager run it. providers := []ProvideFunc{ subscription.ProvideController, - broker.ProvideController(logger.Desugar()), + broker.ProvideController(logger.Desugar(), getRequiredEnv("INGRESS_IMAGE"), getRequiredEnv("INGRESS_SERVICE_ACCOUNT"), getRequiredEnv("FILTER_IMAGE"), getRequiredEnv("FILTER_SERVICE_ACCOUNT")), trigger.ProvideController(logger.Desugar()), } for _, provider := range providers { @@ -194,3 +195,11 @@ func getLoggingConfigOrDie() map[string]string { return cm } } + +func getRequiredEnv(envKey string) string { + val, defined := os.LookupEnv(envKey) + if !defined { + log.Fatalf("required environment variable not defined '%s'", envKey) + } + return val +} diff --git a/config/500-controller.yaml b/config/500-controller.yaml index 40da3a7be30..1c6e9336154 100644 --- a/config/500-controller.yaml +++ b/config/500-controller.yaml @@ -35,6 +35,15 @@ spec: "-logtostderr", "-stderrthreshold", "INFO" ] + env: + - name: INGRESS_IMAGE + value: github.com/knative/eventing/cmd/broker/ingress + - name: INGRESS_SERVICE_ACCOUNT + value: default + - name: FILTER_IMAGE + value: github.com/knative/eventing/cmd/broker/filter + - name: FILTER_SERVICE_ACCOUNT + value: default volumeMounts: - name: config-logging mountPath: /etc/config-logging diff --git a/pkg/controller/eventing/broker/provider.go b/pkg/controller/eventing/broker/provider.go index 8b78d982685..b3655a72ba1 100644 --- a/pkg/controller/eventing/broker/provider.go +++ b/pkg/controller/eventing/broker/provider.go @@ -44,8 +44,8 @@ type reconciler struct { logger *zap.Logger - routerImage string - routerServiceAccountName string + ingressImage string + ingressServiceAccountName string filterImage string filterServiceAccountName string } @@ -54,13 +54,18 @@ type reconciler struct { var _ reconcile.Reconciler = &reconciler{} // ProvideController returns a function that returns a Broker controller. -func ProvideController(logger *zap.Logger) func(manager.Manager) (controller.Controller, error) { +func ProvideController(logger *zap.Logger, ingressImage, ingressServiceAccount, filterImage, filterServiceAccount string) func(manager.Manager) (controller.Controller, error) { return func(mgr manager.Manager) (controller.Controller, error) { // Setup a new controller to Reconcile Brokers. c, err := controller.New(controllerAgentName, mgr, controller.Options{ Reconciler: &reconciler{ recorder: mgr.GetRecorder(controllerAgentName), logger: logger, + + ingressImage: ingressImage, + ingressServiceAccountName: ingressServiceAccount, + filterImage: filterImage, + filterServiceAccountName: filterServiceAccount, }, }) if err != nil { diff --git a/pkg/controller/eventing/broker/reconcile.go b/pkg/controller/eventing/broker/reconcile.go index 720ef29a266..8f7b1fb7f0c 100644 --- a/pkg/controller/eventing/broker/reconcile.go +++ b/pkg/controller/eventing/broker/reconcile.go @@ -27,8 +27,6 @@ import ( "github.com/knative/eventing/pkg/controller/eventing/broker/resources" - servingv1alpha1 "github.com/knative/serving/pkg/apis/serving/v1alpha1" - "github.com/knative/eventing/pkg/provisioners/gcppubsub/util/logging" "go.uber.org/zap" @@ -230,7 +228,7 @@ func (r *reconciler) getChannel(ctx context.Context, b *v1alpha1.Broker) (*v1alp list := &v1alpha1.ChannelList{} opts := &runtimeclient.ListOptions{ Namespace: b.Namespace, - LabelSelector: labels.SelectorFromSet(channelLabels(b)), + LabelSelector: labels.SelectorFromSet(ChannelLabels(b)), // TODO this is here because the fake client needs it. Remove this when it's no longer // needed. Raw: &metav1.ListOptions{ @@ -276,102 +274,13 @@ func newChannel(b *v1alpha1.Broker) *v1alpha1.Channel { } } -func channelLabels(b *v1alpha1.Broker) map[string]string { +func ChannelLabels(b *v1alpha1.Broker) map[string]string { return map[string]string{ "eventing.knative.dev/broker": b.Name, "eventing.knative.dev/broker/everything": "true", } } -func (r *reconciler) reconcileNeedsActivationSubscription(ctx context.Context, b *v1alpha1.Broker, activator *servingv1alpha1.Service, c *v1alpha1.Channel) (*v1alpha1.Subscription, error) { - expected := newNeedsActivationSubscription(b, c, activator) - - sub, err := r.getNeedsActivationSubscription(ctx, b) - // If the resource doesn't exist, we'll create it - if k8serrors.IsNotFound(err) { - sub = expected - err = r.client.Create(ctx, sub) - if err != nil { - return nil, err - } - return sub, nil - } else if err != nil { - return nil, err - } - - // Update Subscription if it has changed. - if !equality.Semantic.DeepDerivative(expected.Spec, sub.Spec) { - sub.Spec = expected.Spec - err = r.client.Update(ctx, sub) - if err != nil { - return nil, err - } - } - return sub, nil -} - -func (r *reconciler) getNeedsActivationSubscription(ctx context.Context, b *v1alpha1.Broker) (*v1alpha1.Subscription, error) { - list := &v1alpha1.SubscriptionList{} - opts := &runtimeclient.ListOptions{ - Namespace: b.Namespace, - LabelSelector: labels.SelectorFromSet(channelLabels(b)), - // TODO this is here because the fake client needs it. Remove this when it's no longer - // needed. - Raw: &metav1.ListOptions{ - TypeMeta: metav1.TypeMeta{ - APIVersion: v1alpha1.SchemeGroupVersion.String(), - Kind: "Subscription", - }, - }, - } - - err := r.client.List(ctx, opts, list) - if err != nil { - return nil, err - } - for _, sub := range list.Items { - if metav1.IsControlledBy(&sub, b) { - return &sub, nil - } - } - - return nil, k8serrors.NewNotFound(schema.GroupResource{}, "") -} - -func newNeedsActivationSubscription(b *v1alpha1.Broker, c *v1alpha1.Channel, activator *servingv1alpha1.Service) *v1alpha1.Subscription { - return &v1alpha1.Subscription{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: b.Namespace, - GenerateName: fmt.Sprintf("%s-broker-needs-activation-", b.Name), - OwnerReferences: []metav1.OwnerReference{ - *metav1.NewControllerRef(b, schema.GroupVersionKind{ - Group: b.GroupVersionKind().Group, - Version: b.GroupVersionKind().Version, - Kind: b.GroupVersionKind().Kind, - }), - }, - }, - Spec: v1alpha1.SubscriptionSpec{ - Channel: corev1.ObjectReference{ - APIVersion: c.APIVersion, - Kind: c.Kind, - Namespace: c.Namespace, - Name: c.Name, - UID: c.UID, - }, - Subscriber: &v1alpha1.SubscriberSpec{ - Ref: &corev1.ObjectReference{ - APIVersion: activator.APIVersion, - Kind: activator.Kind, - Namespace: activator.Namespace, - Name: activator.Name, - UID: activator.UID, - }, - }, - }, - } -} - func (r *reconciler) reconcileDeployment(ctx context.Context, d *v1.Deployment) (*v1.Deployment, error) { name := types.NamespacedName{ Namespace: d.Namespace, @@ -432,8 +341,8 @@ func (r *reconciler) reconcileService(ctx context.Context, svc *corev1.Service) func (r *reconciler) reconcileIngressDeployment(ctx context.Context, b *v1alpha1.Broker, c *v1alpha1.Channel) (*v1.Deployment, error) { expected, err := resources.MakeIngress(&resources.IngressArgs{ Broker: b, - Image: r.routerImage, - ServiceAccountName: r.routerServiceAccountName, + Image: r.ingressImage, + ServiceAccountName: r.ingressServiceAccountName, ChannelAddress: c.Status.Address.Hostname, }) if err != nil { diff --git a/pkg/controller/eventing/trigger/provider.go b/pkg/controller/eventing/trigger/provider.go index 316f3312766..1bd62841435 100644 --- a/pkg/controller/eventing/trigger/provider.go +++ b/pkg/controller/eventing/trigger/provider.go @@ -17,8 +17,6 @@ limitations under the License. package trigger import ( - "sync" - "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" "go.uber.org/zap" "k8s.io/client-go/dynamic" @@ -44,9 +42,6 @@ type reconciler struct { dynamicClient dynamic.Interface recorder record.EventRecorder - triggersLock sync.RWMutex - triggers map[string]map[reconcile.Request]bool - logger *zap.Logger } @@ -73,11 +68,6 @@ func ProvideController(logger *zap.Logger) func(manager.Manager) (controller.Con return nil, err } - // TODO Make this much, much more efficient. - if err = c.Watch(&source.Kind{Type: &v1alpha1.Channel{}}, &handler.EnqueueRequestsFromMapFunc{ToRequests: &mapAllTriggers{r}}); err != nil { - return nil, err - } - return c, nil } } @@ -93,21 +83,3 @@ func (r *reconciler) InjectConfig(c *rest.Config) error { r.dynamicClient, err = dynamic.NewForConfig(c) return err } - -type mapAllTriggers struct { - r *reconciler -} - -func (m *mapAllTriggers) Map(o handler.MapObject) []reconcile.Request { - m.r.triggersLock.RLock() - defer m.r.triggersLock.RUnlock() - triggersInNamespace := m.r.triggers[o.Meta.GetNamespace()] - if triggersInNamespace == nil { - return []reconcile.Request{} - } - reqs := make([]reconcile.Request, 0, len(triggersInNamespace)) - for name := range triggersInNamespace { - reqs = append(reqs, name) - } - return reqs -} diff --git a/pkg/controller/eventing/trigger/reconcile.go b/pkg/controller/eventing/trigger/reconcile.go index b993beeed1c..68433ae56f9 100644 --- a/pkg/controller/eventing/trigger/reconcile.go +++ b/pkg/controller/eventing/trigger/reconcile.go @@ -19,34 +19,26 @@ package trigger import ( "context" "fmt" - - "k8s.io/apimachinery/pkg/labels" - - "k8s.io/apimachinery/pkg/runtime/schema" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "github.com/knative/eventing/pkg/provisioners" - "k8s.io/apimachinery/pkg/types" - + "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" + "github.com/knative/eventing/pkg/controller" + "github.com/knative/eventing/pkg/controller/eventing/broker" "github.com/knative/eventing/pkg/provisioners/gcppubsub/util/logging" + istiov1alpha3 "github.com/knative/pkg/apis/istio/v1alpha3" "go.uber.org/zap" - k8serrors "k8s.io/apimachinery/pkg/api/errors" - - "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/errors" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" runtimeclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" ) const ( - finalizerName = controllerAgentName - - eventTypeKey = "eventing.knative.dev/broker/eventType" - // Name of the corev1.Events emitted from the reconciliation process triggerReconciled = "TriggerReconciled" triggerUpdateStatusFailed = "TriggerUpdateStatusFailed" @@ -96,21 +88,16 @@ func (r *reconciler) reconcile(ctx context.Context, t *v1alpha1.Trigger) error { t.Status.InitializeConditions() // 1. Verify the Broker exists. - // 2. Determine subscribers. - // 3. Look over all subscribable resources and subscribe to the correct ones. - // 4. [Do not do for the prototype] Inject a filter on every subscription. + // 2. Creates a K8s Service uniquely named for this Trigger. + // 3. Creates a VirtualService that routes the K8s Service to the Broker's filter service on an identifiable host name. + // 4. Creates a Subscription from the Broker's single Channel to this Trigger's K8s Service, with reply set to the Broker. if t.DeletionTimestamp != nil { // Everything is cleaned up by the garbage collector. - r.removeFromTriggers(t) - provisioners.RemoveFinalizer(t, finalizerName) return nil } - provisioners.AddFinalizer(t, finalizerName) - r.AddToTriggers(t) - - broker, err := r.getBroker(ctx, t) + b, err := r.getBroker(ctx, t) if err != nil { logging.FromContext(ctx).Error("Unable to get the Broker", zap.Error(err)) t.Status.MarkBrokerDoesNotExists() @@ -118,72 +105,33 @@ func (r *reconciler) reconcile(ctx context.Context, t *v1alpha1.Trigger) error { } t.Status.MarkBrokerExists() - subscribables, err := r.getRelevantSubscribables(ctx, t, broker.Spec.Selector) + c, err := r.getBrokerChannel(ctx, b) if err != nil { - logging.FromContext(ctx).Error("Unable to get relevant subscribables", zap.Error(err)) + logging.FromContext(ctx).Error("Unable to get the Broker's Channel", zap.Error(err)) return err } - err = r.subscribeAll(ctx, t, subscribables) + svc, err := r.reconcileK8sService(ctx, t) if err != nil { - logging.FromContext(ctx).Error("Unable to Subscribe", zap.Error(err)) - t.Status.MarkNotSubscribed("notSubscribed", "%v", err) + logging.FromContext(ctx).Error("Unable to reconcile the K8s Service", zap.Error(err)) return err } - t.Status.MarkSubscribed() - - return nil -} - -func (r *reconciler) AddToTriggers(t *v1alpha1.Trigger) { - name := reconcile.Request{ - NamespacedName: types.NamespacedName{ - Namespace: t.Namespace, - Name: t.Name, - }, - } - - // We will be reconciling an already existing Trigger far more often than adding a new one, so - // check with a read lock before using the write lock. - r.triggersLock.RLock() - triggersInNamespace := r.triggers[t.Namespace] - var present bool - if triggersInNamespace != nil { - _, present = triggersInNamespace[name] - } else { - present = false - } - r.triggersLock.RUnlock() - if present { - // Already present in the map. - return - } - - r.triggersLock.Lock() - triggersInNamespace = r.triggers[t.Namespace] - if triggersInNamespace == nil { - r.triggers[t.Namespace] = make(map[reconcile.Request]bool) - triggersInNamespace = r.triggers[t.Namespace] + _, err = r.reconcileVirtualService(ctx, t, svc) + if err != nil { + logging.FromContext(ctx).Error("Unable to reconcile the VirtualService", zap.Error(err)) + return err } - triggersInNamespace[name] = false - r.triggersLock.Unlock() -} -func (r *reconciler) removeFromTriggers(t *v1alpha1.Trigger) { - name := reconcile.Request{ - NamespacedName: types.NamespacedName{ - Namespace: t.Namespace, - Name: t.Name, - }, + _, err = r.subscribeToBrokerChannel(ctx, t, c, svc) + if err != nil { + logging.FromContext(ctx).Error("Unable to Subscribe", zap.Error(err)) + t.Status.MarkNotSubscribed("notSubscribed", "%v", err) + return err } + t.Status.MarkSubscribed() - r.triggersLock.Lock() - triggersInNamespace := r.triggers[t.Namespace] - if triggersInNamespace != nil { - delete(triggersInNamespace, name) - } - r.triggersLock.Unlock() + return nil } // updateStatus may in fact update the trigger's finalizers in addition to the status @@ -226,20 +174,20 @@ func (r *reconciler) updateStatus(trigger *v1alpha1.Trigger) (*v1alpha1.Trigger, } func (r *reconciler) getBroker(ctx context.Context, t *v1alpha1.Trigger) (*v1alpha1.Broker, error) { - broker := &v1alpha1.Broker{} + b := &v1alpha1.Broker{} name := types.NamespacedName{ Namespace: t.Namespace, Name: t.Spec.Broker, } - err := r.client.Get(ctx, name, broker) - return broker, err + err := r.client.Get(ctx, name, b) + return b, err } -func (r *reconciler) getRelevantSubscribables(ctx context.Context, t *v1alpha1.Trigger, selector *metav1.LabelSelector) ([]v1alpha1.Channel, error) { - selector.MatchLabels[eventTypeKey] = t.Spec.Type - - subscribables := make([]v1alpha1.Channel, 0) - opts := &client.ListOptions{ +func (r *reconciler) getBrokerChannel(ctx context.Context, b *v1alpha1.Broker) (*v1alpha1.Channel, error) { + list := &v1alpha1.ChannelList{} + opts := &runtimeclient.ListOptions{ + Namespace: b.Namespace, + LabelSelector: labels.SelectorFromSet(broker.ChannelLabels(b)), // TODO this is here because the fake client needs it. Remove this when it's no longer // needed. Raw: &metav1.ListOptions{ @@ -248,39 +196,211 @@ func (r *reconciler) getRelevantSubscribables(ctx context.Context, t *v1alpha1.T Kind: "Channel", }, }, - Namespace: t.Namespace, } - for { - list := v1alpha1.ChannelList{} - err := r.client.List(ctx, opts, &list) + + err := r.client.List(ctx, opts, list) + if err != nil { + return nil, err + } + for _, c := range list.Items { + if metav1.IsControlledBy(&c, b) { + return &c, nil + } + } + + return nil, k8serrors.NewNotFound(schema.GroupResource{}, "") +} + +func (r *reconciler) getK8sService(ctx context.Context, t *v1alpha1.Trigger) (*corev1.Service, error) { + list := &corev1.ServiceList{} + opts := &runtimeclient.ListOptions{ + Namespace: t.Namespace, + LabelSelector: labels.SelectorFromSet(k8sServiceLabels(t)), + // TODO this is here because the fake client needs it. Remove this when it's no longer + // needed. + Raw: &metav1.ListOptions{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "Service", + }, + }, + } + + err := r.client.List(ctx, opts, list) + if err != nil { + return nil, err + } + for _, svc := range list.Items { + if metav1.IsControlledBy(&svc, t) { + return &svc, nil + } + } + + return nil, k8serrors.NewNotFound(schema.GroupResource{}, "") +} + +func (r *reconciler) reconcileK8sService(ctx context.Context, t *v1alpha1.Trigger) (*corev1.Service, error) { + current, err := r.getK8sService(ctx, t) + + // If the resource doesn't exist, we'll create it + if k8serrors.IsNotFound(err) { + svc := newK8sService(t) + err = r.client.Create(ctx, svc) + if err != nil { + return nil, err + } + return svc, nil + } else if err != nil { + return nil, err + } + + expected := newK8sService(t) + // spec.clusterIP is immutable and is set on existing services. If we don't set this to the same value, we will + // encounter an error while updating. + expected.Spec.ClusterIP = current.Spec.ClusterIP + if !equality.Semantic.DeepDerivative(expected.Spec, current.Spec) { + current.Spec = expected.Spec + err := r.client.Update(ctx, current) if err != nil { return nil, err } - for _, s := range list.Items { - subscribables = append(subscribables, s) - if list.Continue != "" { - opts.Raw.Continue = list.Continue - } else { - return subscribables, nil - } + } + return current, nil +} + +func newK8sService(t *v1alpha1.Trigger) *corev1.Service { + return &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: t.Namespace, + GenerateName: fmt.Sprintf("trigger-%s-", t.Name), + Labels: k8sServiceLabels(t), + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(t, schema.GroupVersionKind{ + Group: t.GroupVersionKind().Group, + Version: t.GroupVersionKind().Version, + Kind: t.GroupVersionKind().Kind, + }), + }, + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "http", + Port: 80, + }, + }, + }, + } +} + +func k8sServiceLabels(t *v1alpha1.Trigger) map[string]string { + return map[string]string{ + "eventing.knative.dev/trigger": t.Name, + } +} + +func (r *reconciler) getVirtualService(ctx context.Context, t *v1alpha1.Trigger) (*istiov1alpha3.VirtualService, error) { + list := &istiov1alpha3.VirtualServiceList{} + opts := &runtimeclient.ListOptions{ + Namespace: t.Namespace, + LabelSelector: labels.SelectorFromSet(virtualServiceLabels(t)), + // TODO this is here because the fake client needs it. Remove this when it's no longer + // needed. + Raw: &metav1.ListOptions{ + TypeMeta: metav1.TypeMeta{ + APIVersion: istiov1alpha3.SchemeGroupVersion.String(), + Kind: "VirtualService", + }, + }, + } + + err := r.client.List(ctx, opts, list) + if err != nil { + return nil, err + } + for _, vs := range list.Items { + if metav1.IsControlledBy(&vs, t) { + return &vs, nil } } + + return nil, k8serrors.NewNotFound(schema.GroupResource{}, "") } -func (r *reconciler) subscribeAll(ctx context.Context, t *v1alpha1.Trigger, subscribables []v1alpha1.Channel) error { - for _, subscribable := range subscribables { - _, err := r.subscribe(ctx, t, &subscribable) +func (r *reconciler) reconcileVirtualService(ctx context.Context, t *v1alpha1.Trigger, svc *corev1.Service) (*istiov1alpha3.VirtualService, error) { + virtualService, err := r.getVirtualService(ctx, t) + + // If the resource doesn't exist, we'll create it + if k8serrors.IsNotFound(err) { + virtualService = newVirtualService(t, svc) + err = r.client.Create(ctx, virtualService) if err != nil { - return err + return nil, err } + return virtualService, nil + } else if err != nil { + return nil, err } - return nil + + expected := newVirtualService(t, svc) + if !equality.Semantic.DeepDerivative(expected.Spec, virtualService.Spec) { + virtualService.Spec = expected.Spec + err := r.client.Update(ctx, virtualService) + if err != nil { + return nil, err + } + } + return virtualService, nil } -func (r *reconciler) subscribe(ctx context.Context, t *v1alpha1.Trigger, subscribable *v1alpha1.Channel) (*v1alpha1.Subscription, error) { - expected := makeSubscription(t, subscribable) +func newVirtualService(t *v1alpha1.Trigger, svc *corev1.Service) *istiov1alpha3.VirtualService { + // TODO Make this work with endings other than cluster.local + destinationHost := fmt.Sprintf("%s-broker-filter.%s.svc.cluster.local", t.Spec.Broker, t.Namespace) + return &istiov1alpha3.VirtualService{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: fmt.Sprintf("%s-", t.Name), + Namespace: t.Namespace, + Labels: virtualServiceLabels(t), + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(t, schema.GroupVersionKind{ + Group: t.GroupVersionKind().Group, + Version: t.GroupVersionKind().Version, + Kind: t.GroupVersionKind().Kind, + }), + }, + }, + Spec: istiov1alpha3.VirtualServiceSpec{ + Hosts: []string{ + controller.ServiceHostName(svc.Name, svc.Namespace), + }, + Http: []istiov1alpha3.HTTPRoute{{ + Rewrite: &istiov1alpha3.HTTPRewrite{ + // Never really used, so cluster.local should be a good enough ending everywhere. + Authority: fmt.Sprintf("%s.%s.triggers.cluster.local", t.Name, t.Namespace), + }, + Route: []istiov1alpha3.DestinationWeight{{ + Destination: istiov1alpha3.Destination{ + Host: destinationHost, + Port: istiov1alpha3.PortSelector{ + Number: 80, + }, + }}, + }}, + }, + }, + } +} + +func virtualServiceLabels(t *v1alpha1.Trigger) map[string]string { + return map[string]string{ + "eventing.knative.dev/trigger": t.Name, + } +} + +func (r *reconciler) subscribeToBrokerChannel(ctx context.Context, t *v1alpha1.Trigger, c *v1alpha1.Channel, svc *corev1.Service) (*v1alpha1.Subscription, error) { + expected := makeSubscription(t, c, svc) - sub, err := r.getSubscription(ctx, t, subscribable) + sub, err := r.getSubscription(ctx, t, c) // If the resource doesn't exist, we'll create it if k8serrors.IsNotFound(err) { sub = expected @@ -304,7 +424,7 @@ func (r *reconciler) subscribe(ctx context.Context, t *v1alpha1.Trigger, subscri return sub, nil } -func (r *reconciler) getSubscription(ctx context.Context, t *v1alpha1.Trigger, subscribable *v1alpha1.Channel) (*v1alpha1.Subscription, error) { +func (r *reconciler) getSubscription(ctx context.Context, t *v1alpha1.Trigger, c *v1alpha1.Channel) (*v1alpha1.Subscription, error) { list := &v1alpha1.SubscriptionList{} opts := &runtimeclient.ListOptions{ Namespace: t.Namespace, @@ -332,7 +452,7 @@ func (r *reconciler) getSubscription(ctx context.Context, t *v1alpha1.Trigger, s return nil, k8serrors.NewNotFound(schema.GroupResource{}, "") } -func makeSubscription(t *v1alpha1.Trigger, subscribable *v1alpha1.Channel) *v1alpha1.Subscription { +func makeSubscription(t *v1alpha1.Trigger, c *v1alpha1.Channel, svc *corev1.Service) *v1alpha1.Subscription { return &v1alpha1.Subscription{ ObjectMeta: metav1.ObjectMeta{ Namespace: t.Namespace, @@ -348,12 +468,27 @@ func makeSubscription(t *v1alpha1.Trigger, subscribable *v1alpha1.Channel) *v1al }, Spec: v1alpha1.SubscriptionSpec{ Channel: corev1.ObjectReference{ - APIVersion: subscribable.APIVersion, - Kind: subscribable.Kind, - Namespace: subscribable.Namespace, - Name: subscribable.Name, + APIVersion: c.APIVersion, + Kind: c.Kind, + Namespace: c.Namespace, + Name: c.Name, + }, + Subscriber: &v1alpha1.SubscriberSpec{ + Ref: &corev1.ObjectReference{ + APIVersion: svc.APIVersion, + Kind: svc.Kind, + Namespace: svc.Namespace, + Name: svc.Name, + }, + }, + Reply: &v1alpha1.ReplyStrategy{ + Channel: &corev1.ObjectReference{ + APIVersion: c.APIVersion, + Kind: c.Kind, + Namespace: c.Namespace, + Name: c.Name, + }, }, - Subscriber: t.Spec.Subscriber, }, } } diff --git a/third_party/VENDOR-LICENSE b/third_party/VENDOR-LICENSE index f2dcb874614..fea782378a7 100644 --- a/third_party/VENDOR-LICENSE +++ b/third_party/VENDOR-LICENSE @@ -1685,213 +1685,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -=========================================================== -Import: github.com/knative/eventing/vendor/github.com/google/go-containerregistry - - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - =========================================================== Import: github.com/knative/eventing/vendor/github.com/google/gofuzz @@ -2845,214 +2638,6 @@ SOFTWARE. -=========================================================== -Import: github.com/knative/eventing/vendor/github.com/knative/build - - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - =========================================================== Import: github.com/knative/eventing/vendor/github.com/knative/pkg @@ -3260,214 +2845,6 @@ Import: github.com/knative/eventing/vendor/github.com/knative/pkg -=========================================================== -Import: github.com/knative/eventing/vendor/github.com/knative/serving - - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - =========================================================== Import: github.com/knative/eventing/vendor/github.com/markbates/inflect From d7ebdcba67d6dca906c44b1644dc07d2a7fb6646 Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Mon, 4 Feb 2019 22:50:29 -0800 Subject: [PATCH 010/128] Add simple mains. --- cmd/broker/filter/kodata/HEAD | 1 + cmd/broker/filter/kodata/LICENSE | 1 + cmd/broker/filter/kodata/VENDOR-LICENSE | 1 + cmd/broker/filter/main.go | 67 +++++++ cmd/broker/ingress/kodata/HEAD | 1 + cmd/broker/ingress/kodata/LICENSE | 1 + cmd/broker/ingress/kodata/VENDOR-LICENSE | 1 + cmd/broker/ingress/main.go | 142 +++++++++++++ pkg/apis/eventing/v1alpha1/trigger_types.go | 2 + pkg/broker/filter.go | 110 +++++++++++ pkg/broker/filter_test.go | 187 ++++++++++++++++++ .../eventing/trigger/resources/activator.go | 75 ------- .../eventing/trigger/resources/router.go | 80 -------- 13 files changed, 514 insertions(+), 155 deletions(-) create mode 120000 cmd/broker/filter/kodata/HEAD create mode 120000 cmd/broker/filter/kodata/LICENSE create mode 120000 cmd/broker/filter/kodata/VENDOR-LICENSE create mode 100644 cmd/broker/filter/main.go create mode 120000 cmd/broker/ingress/kodata/HEAD create mode 120000 cmd/broker/ingress/kodata/LICENSE create mode 120000 cmd/broker/ingress/kodata/VENDOR-LICENSE create mode 100644 cmd/broker/ingress/main.go create mode 100644 pkg/broker/filter.go create mode 100644 pkg/broker/filter_test.go delete mode 100644 pkg/controller/eventing/trigger/resources/activator.go delete mode 100644 pkg/controller/eventing/trigger/resources/router.go diff --git a/cmd/broker/filter/kodata/HEAD b/cmd/broker/filter/kodata/HEAD new file mode 120000 index 00000000000..481bd4eff49 --- /dev/null +++ b/cmd/broker/filter/kodata/HEAD @@ -0,0 +1 @@ +../../../../.git/HEAD \ No newline at end of file diff --git a/cmd/broker/filter/kodata/LICENSE b/cmd/broker/filter/kodata/LICENSE new file mode 120000 index 00000000000..14776154326 --- /dev/null +++ b/cmd/broker/filter/kodata/LICENSE @@ -0,0 +1 @@ +../../../../LICENSE \ No newline at end of file diff --git a/cmd/broker/filter/kodata/VENDOR-LICENSE b/cmd/broker/filter/kodata/VENDOR-LICENSE new file mode 120000 index 00000000000..7322c09d957 --- /dev/null +++ b/cmd/broker/filter/kodata/VENDOR-LICENSE @@ -0,0 +1 @@ +../../../../third_party/VENDOR-LICENSE \ No newline at end of file diff --git a/cmd/broker/filter/main.go b/cmd/broker/filter/main.go new file mode 100644 index 00000000000..a8f2f8536a8 --- /dev/null +++ b/cmd/broker/filter/main.go @@ -0,0 +1,67 @@ +/* + * Copyright 2018 The Knative Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "flag" + eventingv1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" + "github.com/knative/eventing/pkg/broker" + "github.com/knative/eventing/pkg/provisioners" + "github.com/knative/pkg/signals" + "go.uber.org/zap" + "sigs.k8s.io/controller-runtime/pkg/client/config" + "sigs.k8s.io/controller-runtime/pkg/manager" +) + +func main() { + logConfig := provisioners.NewLoggingConfig() + logger := provisioners.NewProvisionerLoggerFromConfig(logConfig).Desugar() + defer logger.Sync() + + flag.Parse() + + logger.Info("Starting...") + + mgr, err := manager.New(config.GetConfigOrDie(), manager.Options{}) + if err != nil { + logger.Fatal("Error starting up.", zap.Error(err)) + } + + // Add custom types to this array to get them into the manager's scheme. + eventingv1alpha1.AddToScheme(mgr.GetScheme()) + + // We are running both the receiver (takes messages in from the cluster and writes them to + // PubSub) and the dispatcher (takes messages in PubSub and sends them in cluster) in this + // binary. + + _, runnable := broker.New(logger, mgr.GetClient()) + err = mgr.Add(runnable) + if err != nil { + logger.Fatal("Unable to start the receivers runnable", zap.Error(err), zap.Any("runnable", runnable)) + } + + // set up signals so we handle the first shutdown signal gracefully + stopCh := signals.SetupSignalHandler() + + // Start blocks forever. + logger.Info("Manager starting...") + err = mgr.Start(stopCh) + if err != nil { + logger.Fatal("Manager.Start() returned an error", zap.Error(err)) + } +} + diff --git a/cmd/broker/ingress/kodata/HEAD b/cmd/broker/ingress/kodata/HEAD new file mode 120000 index 00000000000..481bd4eff49 --- /dev/null +++ b/cmd/broker/ingress/kodata/HEAD @@ -0,0 +1 @@ +../../../../.git/HEAD \ No newline at end of file diff --git a/cmd/broker/ingress/kodata/LICENSE b/cmd/broker/ingress/kodata/LICENSE new file mode 120000 index 00000000000..14776154326 --- /dev/null +++ b/cmd/broker/ingress/kodata/LICENSE @@ -0,0 +1 @@ +../../../../LICENSE \ No newline at end of file diff --git a/cmd/broker/ingress/kodata/VENDOR-LICENSE b/cmd/broker/ingress/kodata/VENDOR-LICENSE new file mode 120000 index 00000000000..7322c09d957 --- /dev/null +++ b/cmd/broker/ingress/kodata/VENDOR-LICENSE @@ -0,0 +1 @@ +../../../../third_party/VENDOR-LICENSE \ No newline at end of file diff --git a/cmd/broker/ingress/main.go b/cmd/broker/ingress/main.go new file mode 100644 index 00000000000..865d07db82e --- /dev/null +++ b/cmd/broker/ingress/main.go @@ -0,0 +1,142 @@ +/* + * Copyright 2018 The Knative Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "context" + "flag" + "golang.org/x/sync/errgroup" + "log" + "net/http" + "os" + "time" + + eventingv1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" + "github.com/knative/eventing/pkg/provisioners" + "github.com/knative/pkg/signals" + "go.uber.org/zap" + "sigs.k8s.io/controller-runtime/pkg/client/config" + "sigs.k8s.io/controller-runtime/pkg/manager" +) + + +var ( + readTimeout = 1 * time.Minute + writeTimeout = 1 * time.Minute +) + +func main() { + logConfig := provisioners.NewLoggingConfig() + logger := provisioners.NewProvisionerLoggerFromConfig(logConfig).Desugar() + defer logger.Sync() + flag.Parse() + + logger.Info("Starting...") + + mgr, err := manager.New(config.GetConfigOrDie(), manager.Options{}) + if err != nil { + logger.Fatal("Error starting up.", zap.Error(err)) + } + + // Add custom types to this array to get them into the manager's scheme. + err = eventingv1alpha1.AddToScheme(mgr.GetScheme()) + if err != nil { + logger.Fatal("Unable to add scheme", zap.Error(err)) + } + + + c := getRequiredEnv("CHANNEL") + + h := NewHandler(logger, c) + + s := &http.Server{ + Addr: ":8080", + Handler: h, + ErrorLog: zap.NewStdLog(logger), + ReadTimeout: readTimeout, + WriteTimeout: writeTimeout, + } + + // Start both the manager (which notices ConfigMap changes) and the HTTP server. + var g errgroup.Group + g.Go(func() error { + // set up signals so we handle the first shutdown signal gracefully + stopCh := signals.SetupSignalHandler() + // Start blocks forever, so run it in a goroutine. + return mgr.Start(stopCh) + }) + logger.Info("Ingress Listening...", zap.String("Address", s.Addr)) + g.Go(s.ListenAndServe) + err = g.Wait() + if err != nil { + logger.Error("HTTP server failed.", zap.Error(err)) + } + + ctx, cancel := context.WithTimeout(context.Background(), writeTimeout) + defer cancel() + s.Shutdown(ctx) +} + +func getRequiredEnv(envKey string) string { + val, defined := os.LookupEnv(envKey) + if !defined { + log.Fatalf("required environment variable not defined '%s'", envKey) + } + return val +} + +// http.Handler that takes a single request in and fans it out to N other servers. +type Handler struct { + receiver *provisioners.MessageReceiver + dispatcher *provisioners.MessageDispatcher + destination string + + logger *zap.Logger +} + +// NewHandler creates a new fanout.Handler. +func NewHandler(logger *zap.Logger, destination string) *Handler { + handler := &Handler{ + logger: logger, + dispatcher: provisioners.NewMessageDispatcher(logger.Sugar()), + destination: destination, + } + // The receiver function needs to point back at the handler itself, so set it up after + // initialization. + handler.receiver = provisioners.NewMessageReceiver(createReceiverFunction(handler), logger.Sugar()) + + return handler +} + +func createReceiverFunction(f *Handler) func(provisioners.ChannelReference, *provisioners.Message) error { + return func(_ provisioners.ChannelReference, m *provisioners.Message) error { + // TODO Filter. + return f.dispatch(m) + } +} + +func (f *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + f.receiver.HandleRequest(w, r) +} + +// dispatch takes the request, fans it out to each subscription in f.config. If all the fanned out +// requests return successfully, then return nil. Else, return an error. +func (f *Handler) dispatch(msg *provisioners.Message) error { + return f.dispatcher.DispatchMessage(msg, f.destination, "", provisioners.DispatchDefaults{}) +} + + diff --git a/pkg/apis/eventing/v1alpha1/trigger_types.go b/pkg/apis/eventing/v1alpha1/trigger_types.go index e6dc53bb9a8..47d1ab5f5fe 100644 --- a/pkg/apis/eventing/v1alpha1/trigger_types.go +++ b/pkg/apis/eventing/v1alpha1/trigger_types.go @@ -73,6 +73,8 @@ type TriggerStatus struct { // +patchMergeKey=type // +patchStrategy=merge Conditions duckv1alpha1.Conditions `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"` + + SubscriberURI string `json:"subscriberURI,omitempty"` } const ( diff --git a/pkg/broker/filter.go b/pkg/broker/filter.go new file mode 100644 index 00000000000..cb542a2e060 --- /dev/null +++ b/pkg/broker/filter.go @@ -0,0 +1,110 @@ +/* + * Copyright 2018 The Knative Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package broker + +import ( + "context" + "errors" + + eventingv1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" + "github.com/knative/eventing/pkg/provisioners" + "github.com/knative/eventing/pkg/provisioners/gcppubsub/util/logging" + "go.uber.org/zap" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/manager" +) + +const ( + Any = "Any" +) + +// Receiver parses Cloud Events and sends them to GCP PubSub. +type Receiver struct { + logger *zap.Logger + client client.Client + + dispatcher provisioners.Dispatcher +} + +// New creates a new Receiver and its associated MessageReceiver. The caller is responsible for +// Start()ing the returned MessageReceiver. +func New(logger *zap.Logger, client client.Client) (*Receiver, manager.Runnable) { + r := &Receiver{ + logger: logger, + client: client, + dispatcher: provisioners.NewMessageDispatcher(logger.Sugar()), + } + return r, r.newMessageReceiver() +} + +func (r *Receiver) newMessageReceiver() *provisioners.MessageReceiver { + return provisioners.NewMessageReceiver(r.sendEventToTopic, r.logger.Sugar()) +} + +// sendEventToTopic sends a message to the Cloud Pub/Sub Topic backing the Channel. +func (r *Receiver) sendEventToTopic(channel provisioners.ChannelReference, message *provisioners.Message) error { + r.logger.Debug("received message") + ctx := context.Background() + + t, err := r.getTrigger(ctx, channel) + if err != nil { + logging.FromContext(ctx).Info("Unable to get the Trigger", zap.Error(err), zap.Any("channelRef", channel)) + return err + } + + subscriberURI := t.Status.SubscriberURI + if subscriberURI == "" { + logging.FromContext(ctx).Error("Unable to read subscriberURI") + return errors.New("unable to read subscriberURI") + } + + if !shouldSendMessage(t.Spec, message) { + logging.FromContext(ctx).Debug("Message did not pass filter") + return nil + } + + err = r.dispatcher.DispatchMessage(message, subscriberURI, "", provisioners.DispatchDefaults{}) + if err != nil { + logging.FromContext(ctx).Info("Failed to dispatch message", zap.Error(err)) + return err + } + logging.FromContext(ctx).Debug("Successfully sent message") + return nil +} + +func (r *Receiver) getTrigger(ctx context.Context, ref provisioners.ChannelReference) (*eventingv1alpha1.Trigger, error) { + t := &eventingv1alpha1.Trigger{} + err := r.client.Get(ctx, + types.NamespacedName{ + Namespace: ref.Namespace, + Name: ref.Name, + }, + t) + return t, err +} + +func shouldSendMessage(t eventingv1alpha1.TriggerSpec, m *provisioners.Message) bool { + // TODO More filtering! + if t.Type != Any && t.Type != m.Headers["type"] { + return false + } + if t.Source != "" && t.Source != m.Headers["source"] { + return false + } + return true +} diff --git a/pkg/broker/filter_test.go b/pkg/broker/filter_test.go new file mode 100644 index 00000000000..77d9b0d0923 --- /dev/null +++ b/pkg/broker/filter_test.go @@ -0,0 +1,187 @@ +/* + * Copyright 2018 The Knative Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package broker + +import ( + "context" + "errors" + "net/http/httptest" + "strings" + "testing" + + "github.com/knative/eventing/pkg/provisioners/gcppubsub/util" + + eventingv1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" + "k8s.io/client-go/kubernetes/scheme" + + "github.com/knative/eventing/pkg/provisioners/gcppubsub/util/fakepubsub" + "go.uber.org/zap" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/knative/eventing/pkg/provisioners/gcppubsub/util/testcreds" +) + +const ( + validMessage = `{ + "cloudEventsVersion" : "0.1", + "eventType" : "com.example.someevent", + "eventTypeVersion" : "1.0", + "source" : "/mycontext", + "eventID" : "A234-1234-1234", + "eventTime" : "2018-04-05T17:31:00Z", + "extensions" : { + "comExampleExtension" : "value" + }, + "contentType" : "text/xml", + "data" : "" +}` +) + +func init() { + // Add types to scheme. + eventingv1alpha1.AddToScheme(scheme.Scheme) +} + +func TestReceiver(t *testing.T) { + testCases := map[string]struct { + initialState []runtime.Object + pubSubData fakepubsub.CreatorData + expectedErr bool + }{ + "can't get channel": { + initialState: []runtime.Object{ + testcreds.MakeSecretWithInvalidCreds(), + }, + expectedErr: true, + }, + "can't read status": { + initialState: []runtime.Object{ + testcreds.MakeSecretWithInvalidCreds(), + makeChannelWithBadStatus(), + }, + expectedErr: true, + }, + "blank status": { + initialState: []runtime.Object{ + testcreds.MakeSecretWithInvalidCreds(), + makeChannelWithBlankStatus(), + }, + expectedErr: true, + }, + "credential fails": { + initialState: []runtime.Object{ + testcreds.MakeSecretWithInvalidCreds(), + makeChannel(), + }, + expectedErr: true, + }, + "PubSub client fails": { + initialState: []runtime.Object{ + testcreds.MakeSecretWithCreds(), + makeChannel(), + }, + pubSubData: fakepubsub.CreatorData{ + ClientCreateErr: errors.New("testInducedError"), + }, + expectedErr: true, + }, + "Publish fails": { + initialState: []runtime.Object{ + testcreds.MakeSecretWithCreds(), + makeChannel(), + }, + pubSubData: fakepubsub.CreatorData{ + ClientData: fakepubsub.ClientData{ + TopicData: fakepubsub.TopicData{ + Publish: fakepubsub.PublishResultData{ + Err: errors.New("testInducedError"), + }, + }, + }, + }, + expectedErr: true, + }, + "Publish succeeds": { + initialState: []runtime.Object{ + testcreds.MakeSecretWithCreds(), + makeChannel(), + }, + }, + } + for n, tc := range testCases { + t.Run(n, func(t *testing.T) { + mr, _ := New( + zap.NewNop(), + fake.NewFakeClient(tc.initialState...), + fakepubsub.Creator(tc.pubSubData)) + resp := httptest.NewRecorder() + req := httptest.NewRequest("POST", "/", strings.NewReader(validMessage)) + req.Host = "test-channel.test-namespace.channels.cluster.local" + mr.newMessageReceiver().HandleRequest(resp, req) + if tc.expectedErr { + if resp.Result().StatusCode >= 200 && resp.Result().StatusCode < 300 { + t.Errorf("Expected an error. Actual: %v", resp.Result()) + } + } else { + if resp.Result().StatusCode < 200 || resp.Result().StatusCode >= 300 { + t.Errorf("Expected success. Actual: %v", resp.Result()) + } + } + }) + } +} + +func makeChannel() *eventingv1alpha1.Channel { + c := &eventingv1alpha1.Channel{ + TypeMeta: v1.TypeMeta{ + APIVersion: "eventing.knative.dev/v1alpha1", + Kind: "Channel", + }, + ObjectMeta: v1.ObjectMeta{ + Namespace: "test-namespace", + Name: "test-channel", + }, + } + pcs := &util.GcpPubSubChannelStatus{ + GCPProject: "project", + Secret: testcreds.Secret, + SecretKey: testcreds.SecretKey, + } + if err := util.SetInternalStatus(context.Background(), c, pcs); err != nil { + panic(err) + } + return c +} + +func makeChannelWithBlankStatus() *eventingv1alpha1.Channel { + c := makeChannel() + c.Status = eventingv1alpha1.ChannelStatus{} + return c +} + +func makeChannelWithBadStatus() *eventingv1alpha1.Channel { + c := makeChannel() + c.Status.Internal = &runtime.RawExtension{ + // SecretKey must be a string, not an integer, so this will fail during json.Unmarshal. + Raw: []byte(`{"secretKey": 123}`), + } + return c +} diff --git a/pkg/controller/eventing/trigger/resources/activator.go b/pkg/controller/eventing/trigger/resources/activator.go deleted file mode 100644 index 1de0820eef1..00000000000 --- a/pkg/controller/eventing/trigger/resources/activator.go +++ /dev/null @@ -1,75 +0,0 @@ -/* -Copyright 2018 The Knative Authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package resources - -import ( - "encoding/json" - "fmt" - - eventingv1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" - servingv1alpha1 "github.com/knative/serving/pkg/apis/serving/v1alpha1" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" -) - -type ActivatorArgs struct { - Broker *eventingv1alpha1.Broker - Image string - ServiceAccountName string -} - -func MakeActivator(args *ActivatorArgs) (*servingv1alpha1.Service, error) { - templateJson, err := json.Marshal(args.Broker.Spec.ChannelTemplate) - if err != nil { - return nil, err - } - - return &servingv1alpha1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: args.Broker.Namespace, - Name: fmt.Sprintf("%s-broker-activator", args.Broker.Name), - OwnerReferences: []metav1.OwnerReference{ - *metav1.NewControllerRef(args.Broker, schema.GroupVersionKind{ - Group: args.Broker.GroupVersionKind().Group, - Version: args.Broker.GroupVersionKind().Version, - Kind: args.Broker.GroupVersionKind().Kind, - }), - }, - }, - Spec: servingv1alpha1.ServiceSpec{ - RunLatest: &servingv1alpha1.RunLatestType{ - Configuration: servingv1alpha1.ConfigurationSpec{ - RevisionTemplate: servingv1alpha1.RevisionTemplateSpec{ - Spec: servingv1alpha1.RevisionSpec{ - ServiceAccountName: args.ServiceAccountName, - Container: v1.Container{ - Image: args.Image, - Env: []v1.EnvVar{ - { - Name: "CHANNEL_TEMPLATE", - Value: string(templateJson), - }, - }, - }, - }, - }, - }, - }, - }, - }, nil -} diff --git a/pkg/controller/eventing/trigger/resources/router.go b/pkg/controller/eventing/trigger/resources/router.go deleted file mode 100644 index baf8ac4b39c..00000000000 --- a/pkg/controller/eventing/trigger/resources/router.go +++ /dev/null @@ -1,80 +0,0 @@ -/* -Copyright 2018 The Knative Authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package resources - -import ( - "encoding/json" - "fmt" - - eventingv1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" - servingv1alpha1 "github.com/knative/serving/pkg/apis/serving/v1alpha1" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" -) - -type RouterArgs struct { - Broker *eventingv1alpha1.Broker - Image string - ServiceAccountName string - NeedsActivationHost string -} - -func MakeRouter(args *RouterArgs) (*servingv1alpha1.Service, error) { - selectorJson, err := json.Marshal(args.Broker.Spec.Selector) - if err != nil { - return nil, err - } - - return &servingv1alpha1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: args.Broker.Namespace, - Name: fmt.Sprintf("%s-broker", args.Broker.Name), - OwnerReferences: []metav1.OwnerReference{ - *metav1.NewControllerRef(args.Broker, schema.GroupVersionKind{ - Group: args.Broker.GroupVersionKind().Group, - Version: args.Broker.GroupVersionKind().Version, - Kind: args.Broker.GroupVersionKind().Kind, - }), - }, - }, - Spec: servingv1alpha1.ServiceSpec{ - RunLatest: &servingv1alpha1.RunLatestType{ - Configuration: servingv1alpha1.ConfigurationSpec{ - RevisionTemplate: servingv1alpha1.RevisionTemplateSpec{ - Spec: servingv1alpha1.RevisionSpec{ - ServiceAccountName: args.ServiceAccountName, - Container: v1.Container{ - Image: args.Image, - Env: []v1.EnvVar{ - { - Name: "NEEDS_ACTIVATION_HOST", - Value: args.NeedsActivationHost, - }, - { - Name: "LABEL_SELECTOR", - Value: string(selectorJson), - }, - }, - }, - }, - }, - }, - }, - }, - }, nil -} From c5f7a837f12d9ec4b0906f5677826e1b267e95b5 Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Mon, 4 Feb 2019 23:07:52 -0800 Subject: [PATCH 011/128] Small fixes, still not working. --- b.yaml | 11 +++++++++++ pkg/controller/eventing/broker/reconcile.go | 6 +++--- .../eventing/broker/resources/filter.go | 12 ++++++------ .../eventing/broker/resources/ingress.go | 12 ++++++------ pkg/controller/eventing/trigger/reconcile.go | 18 +++++++++--------- 5 files changed, 35 insertions(+), 24 deletions(-) create mode 100644 b.yaml diff --git a/b.yaml b/b.yaml new file mode 100644 index 00000000000..595b6eb20ac --- /dev/null +++ b/b.yaml @@ -0,0 +1,11 @@ +apiVersion: eventing.knative.dev/v1alpha1 +kind: Broker +metadata: + name: default +spec: + channelTemplate: + provisioner: + apiVersion: eventing.knative.dev/v1alpha1 + kind: ClusterChannelProvisioner + name: gcp-pubsub + diff --git a/pkg/controller/eventing/broker/reconcile.go b/pkg/controller/eventing/broker/reconcile.go index 8f7b1fb7f0c..4a380fc0bfe 100644 --- a/pkg/controller/eventing/broker/reconcile.go +++ b/pkg/controller/eventing/broker/reconcile.go @@ -264,9 +264,9 @@ func newChannel(b *v1alpha1.Broker) *v1alpha1.Channel { GenerateName: fmt.Sprintf("%s-broker-", b.Name), OwnerReferences: []metav1.OwnerReference{ *metav1.NewControllerRef(b, schema.GroupVersionKind{ - Group: b.GroupVersionKind().Group, - Version: b.GroupVersionKind().Version, - Kind: b.GroupVersionKind().Kind, + Group: v1alpha1.SchemeGroupVersion.Group, + Version: v1alpha1.SchemeGroupVersion.Version, + Kind: "Broker", }), }, }, diff --git a/pkg/controller/eventing/broker/resources/filter.go b/pkg/controller/eventing/broker/resources/filter.go index 62a25a4103c..8e3aad3b12b 100644 --- a/pkg/controller/eventing/broker/resources/filter.go +++ b/pkg/controller/eventing/broker/resources/filter.go @@ -39,9 +39,9 @@ func MakeFilterDeployment(args *FilterArgs) (*appsv1.Deployment, error) { Name: fmt.Sprintf("%s-broker-filter", args.Broker.Name), OwnerReferences: []metav1.OwnerReference{ *metav1.NewControllerRef(args.Broker, schema.GroupVersionKind{ - Group: args.Broker.GroupVersionKind().Group, - Version: args.Broker.GroupVersionKind().Version, - Kind: args.Broker.GroupVersionKind().Kind, + Group: eventingv1alpha1.SchemeGroupVersion.Group, + Version: eventingv1alpha1.SchemeGroupVersion.Version, + Kind: "Broker", }), }, }, @@ -81,9 +81,9 @@ func MakeFilterService(b *eventingv1alpha1.Broker) *corev1.Service { Labels: filterLabels(b), OwnerReferences: []metav1.OwnerReference{ *metav1.NewControllerRef(b, schema.GroupVersionKind{ - Group: b.GroupVersionKind().Group, - Version: b.GroupVersionKind().Version, - Kind: b.GroupVersionKind().Kind, + Group: eventingv1alpha1.SchemeGroupVersion.Group, + Version: eventingv1alpha1.SchemeGroupVersion.Version, + Kind: "Broker", }), }, }, diff --git a/pkg/controller/eventing/broker/resources/ingress.go b/pkg/controller/eventing/broker/resources/ingress.go index d129361f55f..33777f28c21 100644 --- a/pkg/controller/eventing/broker/resources/ingress.go +++ b/pkg/controller/eventing/broker/resources/ingress.go @@ -40,9 +40,9 @@ func MakeIngress(args *IngressArgs) (*appsv1.Deployment, error) { Name: fmt.Sprintf("%s-broker-ingress", args.Broker.Name), OwnerReferences: []metav1.OwnerReference{ *metav1.NewControllerRef(args.Broker, schema.GroupVersionKind{ - Group: args.Broker.GroupVersionKind().Group, - Version: args.Broker.GroupVersionKind().Version, - Kind: args.Broker.GroupVersionKind().Kind, + Group: eventingv1alpha1.SchemeGroupVersion.Group, + Version: eventingv1alpha1.SchemeGroupVersion.Version, + Kind: "Broker", }), }, }, @@ -86,9 +86,9 @@ func MakeIngressService(b *eventingv1alpha1.Broker) *corev1.Service { Labels: ingressLabels(b), OwnerReferences: []metav1.OwnerReference{ *metav1.NewControllerRef(b, schema.GroupVersionKind{ - Group: b.GroupVersionKind().Group, - Version: b.GroupVersionKind().Version, - Kind: b.GroupVersionKind().Kind, + Group: eventingv1alpha1.SchemeGroupVersion.Group, + Version: eventingv1alpha1.SchemeGroupVersion.Version, + Kind: "Broker", }), }, }, diff --git a/pkg/controller/eventing/trigger/reconcile.go b/pkg/controller/eventing/trigger/reconcile.go index 68433ae56f9..b1805336922 100644 --- a/pkg/controller/eventing/trigger/reconcile.go +++ b/pkg/controller/eventing/trigger/reconcile.go @@ -276,9 +276,9 @@ func newK8sService(t *v1alpha1.Trigger) *corev1.Service { Labels: k8sServiceLabels(t), OwnerReferences: []metav1.OwnerReference{ *metav1.NewControllerRef(t, schema.GroupVersionKind{ - Group: t.GroupVersionKind().Group, - Version: t.GroupVersionKind().Version, - Kind: t.GroupVersionKind().Kind, + Group: v1alpha1.SchemeGroupVersion.Group, + Version: v1alpha1.SchemeGroupVersion.Version, + Kind: "Trigger", }), }, }, @@ -363,9 +363,9 @@ func newVirtualService(t *v1alpha1.Trigger, svc *corev1.Service) *istiov1alpha3. Labels: virtualServiceLabels(t), OwnerReferences: []metav1.OwnerReference{ *metav1.NewControllerRef(t, schema.GroupVersionKind{ - Group: t.GroupVersionKind().Group, - Version: t.GroupVersionKind().Version, - Kind: t.GroupVersionKind().Kind, + Group: v1alpha1.SchemeGroupVersion.Group, + Version: v1alpha1.SchemeGroupVersion.Version, + Kind: "Trigger", }), }, }, @@ -459,9 +459,9 @@ func makeSubscription(t *v1alpha1.Trigger, c *v1alpha1.Channel, svc *corev1.Serv GenerateName: fmt.Sprintf("%s-%s-", t.Spec.Broker, t.Name), OwnerReferences: []metav1.OwnerReference{ *metav1.NewControllerRef(t, schema.GroupVersionKind{ - Group: t.GroupVersionKind().Group, - Version: t.GroupVersionKind().Version, - Kind: t.GroupVersionKind().Kind, + Group: v1alpha1.SchemeGroupVersion.Group, + Version: v1alpha1.SchemeGroupVersion.Version, + Kind: "Trigger", }), }, Labels: subscriptionLabels(t), From 054c10d15dc1bd71eddd62c4ea337f8ae9e23f56 Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Tue, 5 Feb 2019 08:29:24 -0800 Subject: [PATCH 012/128] Add the Istio injection annotation. --- pkg/controller/eventing/broker/reconcile.go | 2 +- pkg/controller/eventing/broker/resources/filter.go | 5 ++++- pkg/controller/eventing/broker/resources/ingress.go | 5 ++++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/pkg/controller/eventing/broker/reconcile.go b/pkg/controller/eventing/broker/reconcile.go index 4a380fc0bfe..095b0a01846 100644 --- a/pkg/controller/eventing/broker/reconcile.go +++ b/pkg/controller/eventing/broker/reconcile.go @@ -277,7 +277,7 @@ func newChannel(b *v1alpha1.Broker) *v1alpha1.Channel { func ChannelLabels(b *v1alpha1.Broker) map[string]string { return map[string]string{ "eventing.knative.dev/broker": b.Name, - "eventing.knative.dev/broker/everything": "true", + "eventing.knative.dev/brokerEverything": "true", } } diff --git a/pkg/controller/eventing/broker/resources/filter.go b/pkg/controller/eventing/broker/resources/filter.go index 8e3aad3b12b..a87af55f5a6 100644 --- a/pkg/controller/eventing/broker/resources/filter.go +++ b/pkg/controller/eventing/broker/resources/filter.go @@ -52,6 +52,9 @@ func MakeFilterDeployment(args *FilterArgs) (*appsv1.Deployment, error) { Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: filterLabels(args.Broker), + Annotations: map[string]string{ + "sidecar.istio.io/inject": "true", + }, }, Spec: corev1.PodSpec{ ServiceAccountName: args.ServiceAccountName, @@ -102,6 +105,6 @@ func MakeFilterService(b *eventingv1alpha1.Broker) *corev1.Service { func filterLabels(b *eventingv1alpha1.Broker) map[string]string { return map[string]string{ "eventing.knative.dev/broker": b.Name, - "eventing.knative.dev/broker/role": "filter", + "eventing.knative.dev/brokerRole": "filter", } } diff --git a/pkg/controller/eventing/broker/resources/ingress.go b/pkg/controller/eventing/broker/resources/ingress.go index 33777f28c21..3a068f8b8c3 100644 --- a/pkg/controller/eventing/broker/resources/ingress.go +++ b/pkg/controller/eventing/broker/resources/ingress.go @@ -53,6 +53,9 @@ func MakeIngress(args *IngressArgs) (*appsv1.Deployment, error) { Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: ingressLabels(args.Broker), + Annotations: map[string]string{ + "sidecar.istio.io/inject": "true", + }, }, Spec: corev1.PodSpec{ ServiceAccountName: args.ServiceAccountName, @@ -107,6 +110,6 @@ func MakeIngressService(b *eventingv1alpha1.Broker) *corev1.Service { func ingressLabels(b *eventingv1alpha1.Broker) map[string]string { return map[string]string{ "eventing.knative.dev/broker": b.Name, - "eventing.knative.dev/broker/role": "ingress", + "eventing.knative.dev/brokerRole": "ingress", } } From 942b97987bee3264a599dfcc273ff32b49ae5364 Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Tue, 5 Feb 2019 09:11:54 -0800 Subject: [PATCH 013/128] Resolve subscriber in the Trigger controller. --- .../eventing/subscription/reconcile.go | 26 +++++++++---------- pkg/controller/eventing/trigger/reconcile.go | 9 +++++++ t.yaml | 13 ++++++++++ 3 files changed, 35 insertions(+), 13 deletions(-) create mode 100644 t.yaml diff --git a/pkg/controller/eventing/subscription/reconcile.go b/pkg/controller/eventing/subscription/reconcile.go index def95e2a1a0..251982b9d3f 100644 --- a/pkg/controller/eventing/subscription/reconcile.go +++ b/pkg/controller/eventing/subscription/reconcile.go @@ -118,14 +118,14 @@ func (r *reconciler) reconcile(subscription *v1alpha1.Subscription) error { } // Verify that `channel` exists. - _, err = r.fetchObjectReference(subscription.Namespace, &subscription.Spec.Channel) + _, err = fetchObjectReference(r.dynamicClient, subscription.Namespace, &subscription.Spec.Channel) if err != nil { glog.Warningf("Failed to validate `channel` exists: %+v, %v", subscription.Spec.Channel, err) r.recorder.Eventf(subscription, corev1.EventTypeWarning, channelReferenceFetchFailed, "Failed to validate spec.channel exists: %v", err) return err } - if subscriberURI, err := r.resolveSubscriberSpec(subscription.Namespace, subscription.Spec.Subscriber); err != nil { + if subscriberURI, err := ResolveSubscriberSpec(context.TODO(), r.client, r.dynamicClient, subscription.Namespace, subscription.Spec.Subscriber); err != nil { glog.Warningf("Failed to resolve Subscriber %+v : %s", *subscription.Spec.Subscriber, err) r.recorder.Eventf(subscription, corev1.EventTypeWarning, subscriberResolveFailed, "Failed to resolve spec.subscriber: %v", err) return err @@ -207,12 +207,12 @@ func (r *reconciler) updateStatus(subscription *v1alpha1.Subscription) (*v1alpha return latestSubscription, nil } -// resolveSubscriberSpec resolves the Spec.Call object. If it's an +// ResolveSubscriberSpec resolves the Spec.Call object. If it's an // ObjectReference will resolve the object and treat it as a Callable. If // it's DNSName then it's used as is. // TODO: Once Service Routes, etc. support Callable, use that. // -func (r *reconciler) resolveSubscriberSpec(namespace string, s *v1alpha1.SubscriberSpec) (string, error) { +func ResolveSubscriberSpec(ctx context.Context, client client.Client, dynamicClient dynamic.Interface, namespace string, s *v1alpha1.SubscriberSpec) (string, error) { if isNilOrEmptySubscriber(s) { return "", nil } @@ -228,7 +228,7 @@ func (r *reconciler) resolveSubscriberSpec(namespace string, s *v1alpha1.Subscri Namespace: namespace, Name: s.Ref.Name, } - err := r.client.Get(context.TODO(), svcKey, svc) + err := client.Get(ctx, svcKey, svc) if err != nil { glog.Warningf("Failed to fetch SubscriberSpec target as a K8s Service %+v: %s", s.Ref, err) return "", err @@ -236,7 +236,7 @@ func (r *reconciler) resolveSubscriberSpec(namespace string, s *v1alpha1.Subscri return domainToURL(controller.ServiceHostName(svc.Name, svc.Namespace)), nil } - obj, err := r.fetchObjectReference(namespace, s.Ref) + obj, err := fetchObjectReference(dynamicClient, namespace, s.Ref) if err != nil { glog.Warningf("Failed to fetch SubscriberSpec target %+v: %s", s.Ref, err) return "", err @@ -263,7 +263,7 @@ func (r *reconciler) resolveResult(namespace string, replyStrategy *v1alpha1.Rep if isNilOrEmptyReply(replyStrategy) { return "", nil } - obj, err := r.fetchObjectReference(namespace, replyStrategy.Channel) + obj, err := fetchObjectReference(r.dynamicClient, namespace, replyStrategy.Channel) if err != nil { glog.Warningf("Failed to fetch ReplyStrategy channel %+v: %s", replyStrategy, err) return "", err @@ -281,8 +281,8 @@ func (r *reconciler) resolveResult(namespace string, replyStrategy *v1alpha1.Rep } // fetchObjectReference fetches an object based on ObjectReference. -func (r *reconciler) fetchObjectReference(namespace string, ref *corev1.ObjectReference) (duck.Marshalable, error) { - resourceClient, err := r.CreateResourceInterface(namespace, ref) +func fetchObjectReference(dynamicClient dynamic.Interface, namespace string, ref *corev1.ObjectReference) (duck.Marshalable, error) { + resourceClient, err := createResourceInterface(dynamicClient, namespace, ref) if err != nil { glog.Warningf("failed to create dynamic client resource: %v", err) return nil, err @@ -387,7 +387,7 @@ func (r *reconciler) createSubscribable(subs []v1alpha1.Subscription) *eventingd func (r *reconciler) patchPhysicalFrom(namespace string, physicalFrom corev1.ObjectReference, subs *eventingduck.Subscribable) error { // First get the original object and convert it to only the bits we care about - s, err := r.fetchObjectReference(namespace, &physicalFrom) + s, err := fetchObjectReference(r.dynamicClient, namespace, &physicalFrom) if err != nil { return err } @@ -411,7 +411,7 @@ func (r *reconciler) patchPhysicalFrom(namespace string, physicalFrom corev1.Obj return err } - resourceClient, err := r.CreateResourceInterface(namespace, &physicalFrom) + resourceClient, err := createResourceInterface(r.dynamicClient, namespace, &physicalFrom) if err != nil { glog.Warningf("failed to create dynamic client resource: %v", err) return err @@ -426,8 +426,8 @@ func (r *reconciler) patchPhysicalFrom(namespace string, physicalFrom corev1.Obj return nil } -func (r *reconciler) CreateResourceInterface(namespace string, ref *corev1.ObjectReference) (dynamic.ResourceInterface, error) { - rc := r.dynamicClient.Resource(duckapis.KindToResource(ref.GroupVersionKind())) +func createResourceInterface(dynamicClient dynamic.Interface, namespace string, ref *corev1.ObjectReference) (dynamic.ResourceInterface, error) { + rc := dynamicClient.Resource(duckapis.KindToResource(ref.GroupVersionKind())) if rc == nil { return nil, fmt.Errorf("failed to create dynamic client resource") diff --git a/pkg/controller/eventing/trigger/reconcile.go b/pkg/controller/eventing/trigger/reconcile.go index b1805336922..9d22f7620f6 100644 --- a/pkg/controller/eventing/trigger/reconcile.go +++ b/pkg/controller/eventing/trigger/reconcile.go @@ -22,6 +22,7 @@ import ( "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" "github.com/knative/eventing/pkg/controller" "github.com/knative/eventing/pkg/controller/eventing/broker" + "github.com/knative/eventing/pkg/controller/eventing/subscription" "github.com/knative/eventing/pkg/provisioners/gcppubsub/util/logging" istiov1alpha3 "github.com/knative/pkg/apis/istio/v1alpha3" "go.uber.org/zap" @@ -88,6 +89,7 @@ func (r *reconciler) reconcile(ctx context.Context, t *v1alpha1.Trigger) error { t.Status.InitializeConditions() // 1. Verify the Broker exists. + // 2. Find the Subscriber's URI. // 2. Creates a K8s Service uniquely named for this Trigger. // 3. Creates a VirtualService that routes the K8s Service to the Broker's filter service on an identifiable host name. // 4. Creates a Subscription from the Broker's single Channel to this Trigger's K8s Service, with reply set to the Broker. @@ -111,6 +113,13 @@ func (r *reconciler) reconcile(ctx context.Context, t *v1alpha1.Trigger) error { return err } + subscriberURI, err := subscription.ResolveSubscriberSpec(ctx, r.client, r.dynamicClient, t.Namespace, t.Spec.Subscriber) + if err != nil { + logging.FromContext(ctx).Error("Unable to get the Subscriber's URI", zap.Error(err)) + return err + } + t.Status.SubscriberURI = subscriberURI + svc, err := r.reconcileK8sService(ctx, t) if err != nil { logging.FromContext(ctx).Error("Unable to reconcile the K8s Service", zap.Error(err)) diff --git a/t.yaml b/t.yaml new file mode 100644 index 00000000000..aa125c7f99e --- /dev/null +++ b/t.yaml @@ -0,0 +1,13 @@ +apiVersion: eventing.knative.dev/v1alpha1 +kind: Trigger +metadata: + name: t +spec: + type: Any + source: Any + subscriber: + ref: + apiVersion: serving.knative.dev/v1alpha1 + kind: Service + name: message-dumper + From afd71af66d38cc4e665035bcd49264b511023753 Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Tue, 5 Feb 2019 09:18:46 -0800 Subject: [PATCH 014/128] Standardize on 'Any'. --- pkg/apis/eventing/v1alpha1/trigger_defaults.go | 2 +- t.yaml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/apis/eventing/v1alpha1/trigger_defaults.go b/pkg/apis/eventing/v1alpha1/trigger_defaults.go index ac2dfdf843f..fd785c8bf38 100644 --- a/pkg/apis/eventing/v1alpha1/trigger_defaults.go +++ b/pkg/apis/eventing/v1alpha1/trigger_defaults.go @@ -25,6 +25,6 @@ func (ts *TriggerSpec) SetDefaults() { ts.Broker = "default" } if ts.Type == "" { - ts.Type = "ANY" + ts.Type = "Any" } } diff --git a/t.yaml b/t.yaml index aa125c7f99e..a53dcc1db33 100644 --- a/t.yaml +++ b/t.yaml @@ -4,7 +4,6 @@ metadata: name: t spec: type: Any - source: Any subscriber: ref: apiVersion: serving.knative.dev/v1alpha1 From 7a1f69b3cad704a95dad553de6ad719db5325806 Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Tue, 5 Feb 2019 11:23:36 -0800 Subject: [PATCH 015/128] Make Broker and Trigger generational --- cmd/webhook/main.go | 3 ++- pkg/apis/eventing/v1alpha1/broker_types.go | 9 +++++++++ pkg/apis/eventing/v1alpha1/trigger_types.go | 9 +++++++++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/cmd/webhook/main.go b/cmd/webhook/main.go index 400a452cd9a..b0e25711735 100644 --- a/cmd/webhook/main.go +++ b/cmd/webhook/main.go @@ -107,5 +107,6 @@ func main() { if err != nil { logger.Fatal("Failed to create the admission controller", zap.Error(err)) } - controller.Run(stopCh) + err = controller.Run(stopCh) + logger.Errorw("Webhook stopping", zap.Error(err)) } diff --git a/pkg/apis/eventing/v1alpha1/broker_types.go b/pkg/apis/eventing/v1alpha1/broker_types.go index d3c2c5b4e2a..fa0ba9b7c93 100644 --- a/pkg/apis/eventing/v1alpha1/broker_types.go +++ b/pkg/apis/eventing/v1alpha1/broker_types.go @@ -49,6 +49,15 @@ var _ runtime.Object = (*Broker)(nil) var _ webhook.GenericCRD = (*Broker)(nil) type BrokerSpec struct { + // TODO By enabling the status subresource metadata.generation should increment + // thus making this property obsolete. + // + // We should be able to drop this property with a CRD conversion webhook + // in the future + // + // +optional + DeprecatedGeneration int64 `json:"generation,omitempty"` + ChannelTemplate *ChannelSpec `json:"channelTemplate,omitempty"` } diff --git a/pkg/apis/eventing/v1alpha1/trigger_types.go b/pkg/apis/eventing/v1alpha1/trigger_types.go index 47d1ab5f5fe..e2c3529b418 100644 --- a/pkg/apis/eventing/v1alpha1/trigger_types.go +++ b/pkg/apis/eventing/v1alpha1/trigger_types.go @@ -49,6 +49,15 @@ var _ runtime.Object = (*Trigger)(nil) var _ webhook.GenericCRD = (*Trigger)(nil) type TriggerSpec struct { + // TODO By enabling the status subresource metadata.generation should increment + // thus making this property obsolete. + // + // We should be able to drop this property with a CRD conversion webhook + // in the future + // + // +optional + DeprecatedGeneration int64 `json:"generation,omitempty"` + Broker string `json:"broker,omitempty"` Subscriber *SubscriberSpec `json:"subscriber,omitempty"` From 2be56bd789b6896d12c24cdcefaf9f2f3666254e Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Tue, 5 Feb 2019 12:16:35 -0800 Subject: [PATCH 016/128] Happy control path. --- pkg/apis/eventing/v1alpha1/trigger_types.go | 4 ++-- pkg/controller/eventing/broker/reconcile.go | 18 +++++++++-------- pkg/controller/eventing/trigger/reconcile.go | 21 ++++++++++---------- 3 files changed, 23 insertions(+), 20 deletions(-) diff --git a/pkg/apis/eventing/v1alpha1/trigger_types.go b/pkg/apis/eventing/v1alpha1/trigger_types.go index e2c3529b418..3e2c1d0c654 100644 --- a/pkg/apis/eventing/v1alpha1/trigger_types.go +++ b/pkg/apis/eventing/v1alpha1/trigger_types.go @@ -121,11 +121,11 @@ func (ts *TriggerStatus) MarkBrokerDoesNotExists() { triggerCondSet.Manage(ts).MarkFalse(TriggerConditionBrokerExists, "doesNotExist", "Broker does not exist") } -func (ts *TriggerSpec) MarkKubernetesServiceExists() { +func (ts *TriggerStatus) MarkKubernetesServiceExists() { triggerCondSet.Manage(ts).MarkTrue(TriggerConditionKubernetesService) } -func (ts *TriggerSpec) MarkVirtualServiceExists() { +func (ts *TriggerStatus) MarkVirtualServiceExists() { triggerCondSet.Manage(ts).MarkTrue(TriggerConditionVirtualService) } diff --git a/pkg/controller/eventing/broker/reconcile.go b/pkg/controller/eventing/broker/reconcile.go index 095b0a01846..2ce00bb10f9 100644 --- a/pkg/controller/eventing/broker/reconcile.go +++ b/pkg/controller/eventing/broker/reconcile.go @@ -211,16 +211,17 @@ func (r *reconciler) reconcileChannel(ctx context.Context, b *v1alpha1.Broker) ( return nil, err } + // TODO Determine if we want to update spec (maybe just args?). // Update Channel if it has changed. Note that we need to both ignore the real Channel's // subscribable section and if we need to update the real Channel, retain it. - expected.Spec.Subscribable = c.Spec.Subscribable - if !equality.Semantic.DeepDerivative(expected.Spec, c.Spec) { - c.Spec = expected.Spec - err = r.client.Update(ctx, c) - if err != nil { - return nil, err - } - } + //expected.Spec.Subscribable = c.Spec.Subscribable + //if !equality.Semantic.DeepDerivative(expected.Spec, c.Spec) { + // c.Spec = expected.Spec + // err = r.client.Update(ctx, c) + // if err != nil { + // return nil, err + // } + //} return c, nil } @@ -262,6 +263,7 @@ func newChannel(b *v1alpha1.Broker) *v1alpha1.Channel { ObjectMeta: metav1.ObjectMeta{ Namespace: b.Namespace, GenerateName: fmt.Sprintf("%s-broker-", b.Name), + Labels: ChannelLabels(b), OwnerReferences: []metav1.OwnerReference{ *metav1.NewControllerRef(b, schema.GroupVersionKind{ Group: v1alpha1.SchemeGroupVersion.Group, diff --git a/pkg/controller/eventing/trigger/reconcile.go b/pkg/controller/eventing/trigger/reconcile.go index 9d22f7620f6..4e461d340b1 100644 --- a/pkg/controller/eventing/trigger/reconcile.go +++ b/pkg/controller/eventing/trigger/reconcile.go @@ -125,12 +125,14 @@ func (r *reconciler) reconcile(ctx context.Context, t *v1alpha1.Trigger) error { logging.FromContext(ctx).Error("Unable to reconcile the K8s Service", zap.Error(err)) return err } + t.Status.MarkKubernetesServiceExists() _, err = r.reconcileVirtualService(ctx, t, svc) if err != nil { logging.FromContext(ctx).Error("Unable to reconcile the VirtualService", zap.Error(err)) return err } + t.Status.MarkVirtualServiceExists() _, err = r.subscribeToBrokerChannel(ctx, t, c, svc) if err != nil { @@ -422,7 +424,8 @@ func (r *reconciler) subscribeToBrokerChannel(ctx context.Context, t *v1alpha1.T return nil, err } - // Update Subscription if it has changed. + // Update Subscription if it has changed. Ignore the generation. + expected.Spec.DeprecatedGeneration = sub.Spec.DeprecatedGeneration if !equality.Semantic.DeepDerivative(expected.Spec, sub.Spec) { sub.Spec = expected.Spec err = r.client.Update(ctx, sub) @@ -477,24 +480,22 @@ func makeSubscription(t *v1alpha1.Trigger, c *v1alpha1.Channel, svc *corev1.Serv }, Spec: v1alpha1.SubscriptionSpec{ Channel: corev1.ObjectReference{ - APIVersion: c.APIVersion, - Kind: c.Kind, - Namespace: c.Namespace, + APIVersion: v1alpha1.SchemeGroupVersion.String(), + Kind: "Channel", Name: c.Name, }, Subscriber: &v1alpha1.SubscriberSpec{ Ref: &corev1.ObjectReference{ - APIVersion: svc.APIVersion, - Kind: svc.Kind, - Namespace: svc.Namespace, + APIVersion: "v1", + Kind: "Service", Name: svc.Name, }, }, + // TODO This pushes directly into the Channel, it should probably point at the Broker ingress instead. Reply: &v1alpha1.ReplyStrategy{ Channel: &corev1.ObjectReference{ - APIVersion: c.APIVersion, - Kind: c.Kind, - Namespace: c.Namespace, + APIVersion: v1alpha1.SchemeGroupVersion.String(), + Kind: "Channel", Name: c.Name, }, }, From 10acfa388a43dba6b2b438e1fae065b6dda2b0c9 Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Tue, 5 Feb 2019 13:05:16 -0800 Subject: [PATCH 017/128] TargetPort is 8080 --- cmd/broker/ingress/main.go | 9 +++++++-- pkg/controller/eventing/broker/reconcile.go | 4 +--- pkg/controller/eventing/broker/resources/filter.go | 2 +- pkg/controller/eventing/broker/resources/ingress.go | 2 ++ 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/cmd/broker/ingress/main.go b/cmd/broker/ingress/main.go index 865d07db82e..37b96da1412 100644 --- a/cmd/broker/ingress/main.go +++ b/cmd/broker/ingress/main.go @@ -19,6 +19,7 @@ package main import ( "context" "flag" + "fmt" "golang.org/x/sync/errgroup" "log" "net/http" @@ -113,7 +114,7 @@ func NewHandler(logger *zap.Logger, destination string) *Handler { handler := &Handler{ logger: logger, dispatcher: provisioners.NewMessageDispatcher(logger.Sugar()), - destination: destination, + destination: fmt.Sprintf("http://%s", destination), } // The receiver function needs to point back at the handler itself, so set it up after // initialization. @@ -136,7 +137,11 @@ func (f *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // dispatch takes the request, fans it out to each subscription in f.config. If all the fanned out // requests return successfully, then return nil. Else, return an error. func (f *Handler) dispatch(msg *provisioners.Message) error { - return f.dispatcher.DispatchMessage(msg, f.destination, "", provisioners.DispatchDefaults{}) + err := f.dispatcher.DispatchMessage(msg, f.destination, "", provisioners.DispatchDefaults{}) + if err != nil { + f.logger.Error("Error dispatching message", zap.String("destination", f.destination)) + } + return err } diff --git a/pkg/controller/eventing/broker/reconcile.go b/pkg/controller/eventing/broker/reconcile.go index 2ce00bb10f9..ea0fef23c83 100644 --- a/pkg/controller/eventing/broker/reconcile.go +++ b/pkg/controller/eventing/broker/reconcile.go @@ -196,12 +196,10 @@ func (r *reconciler) reconcileFilterService(ctx context.Context, b *v1alpha1.Bro } func (r *reconciler) reconcileChannel(ctx context.Context, b *v1alpha1.Broker) (*v1alpha1.Channel, error) { - expected := newChannel(b) - c, err := r.getChannel(ctx, b) // If the resource doesn't exist, we'll create it if k8serrors.IsNotFound(err) { - c = expected + c = newChannel(b) err = r.client.Create(ctx, c) if err != nil { return nil, err diff --git a/pkg/controller/eventing/broker/resources/filter.go b/pkg/controller/eventing/broker/resources/filter.go index a87af55f5a6..c05cc960e96 100644 --- a/pkg/controller/eventing/broker/resources/filter.go +++ b/pkg/controller/eventing/broker/resources/filter.go @@ -61,7 +61,7 @@ func MakeFilterDeployment(args *FilterArgs) (*appsv1.Deployment, error) { Containers: []corev1.Container{ { Image: args.Image, - Name: "ingress", + Name: "filter", Env: []corev1.EnvVar{ { Name: "BROKER", diff --git a/pkg/controller/eventing/broker/resources/ingress.go b/pkg/controller/eventing/broker/resources/ingress.go index 3a068f8b8c3..4f6933116f9 100644 --- a/pkg/controller/eventing/broker/resources/ingress.go +++ b/pkg/controller/eventing/broker/resources/ingress.go @@ -19,6 +19,7 @@ package resources import ( "fmt" appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/util/intstr" eventingv1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" corev1 "k8s.io/api/core/v1" @@ -101,6 +102,7 @@ func MakeIngressService(b *eventingv1alpha1.Broker) *corev1.Service { { Name: "http", Port: 80, + TargetPort: intstr.FromInt(8080), }, }, }, From 2108d837ee01123b22073538a5ca002256bad960 Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Tue, 5 Feb 2019 13:30:24 -0800 Subject: [PATCH 018/128] Watch channels. --- pkg/controller/eventing/broker/provider.go | 7 +++++++ pkg/controller/eventing/broker/reconcile.go | 4 ++++ pkg/controller/eventing/broker/resources/filter.go | 2 ++ 3 files changed, 13 insertions(+) diff --git a/pkg/controller/eventing/broker/provider.go b/pkg/controller/eventing/broker/provider.go index b3655a72ba1..5d94c8cbcfb 100644 --- a/pkg/controller/eventing/broker/provider.go +++ b/pkg/controller/eventing/broker/provider.go @@ -77,6 +77,13 @@ func ProvideController(logger *zap.Logger, ingressImage, ingressServiceAccount, return nil, err } + err = c.Watch(&source.Kind{ + Type: &v1alpha1.Channel{}, + }, &handler.EnqueueRequestForOwner{OwnerType: &v1alpha1.Broker{}, IsController: true}) + if err != nil { + return nil, err + } + return c, nil } } diff --git a/pkg/controller/eventing/broker/reconcile.go b/pkg/controller/eventing/broker/reconcile.go index ea0fef23c83..5837fcbfddb 100644 --- a/pkg/controller/eventing/broker/reconcile.go +++ b/pkg/controller/eventing/broker/reconcile.go @@ -106,6 +106,10 @@ func (r *reconciler) reconcile(ctx context.Context, b *v1alpha1.Broker) error { logging.FromContext(ctx).Error("Problem reconciling the channel", zap.Error(err)) b.Status.MarkChannelFailed(err) return err + } else if c.Status.Address.Hostname == "" { + logging.FromContext(ctx).Info("Channel is not yet ready", zap.Any("c", c)) + // TODO Just re-enqueue, don't return an error + return nil } b.Status.MarkChannelReady() diff --git a/pkg/controller/eventing/broker/resources/filter.go b/pkg/controller/eventing/broker/resources/filter.go index c05cc960e96..dcb330c5570 100644 --- a/pkg/controller/eventing/broker/resources/filter.go +++ b/pkg/controller/eventing/broker/resources/filter.go @@ -19,6 +19,7 @@ package resources import ( "fmt" appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/util/intstr" eventingv1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" corev1 "k8s.io/api/core/v1" @@ -96,6 +97,7 @@ func MakeFilterService(b *eventingv1alpha1.Broker) *corev1.Service { { Name: "http", Port: 80, + TargetPort: intstr.FromInt(8080), }, }, }, From 6aa70860a553a05903536be0664347addb7764a8 Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Tue, 5 Feb 2019 13:45:53 -0800 Subject: [PATCH 019/128] Custom service account for filter (needs trigger watch). --- config/500-controller.yaml | 2 +- sa.yaml | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 sa.yaml diff --git a/config/500-controller.yaml b/config/500-controller.yaml index 1c6e9336154..607707fa3f9 100644 --- a/config/500-controller.yaml +++ b/config/500-controller.yaml @@ -43,7 +43,7 @@ spec: - name: FILTER_IMAGE value: github.com/knative/eventing/cmd/broker/filter - name: FILTER_SERVICE_ACCOUNT - value: default + value: broker-filter volumeMounts: - name: config-logging mountPath: /etc/config-logging diff --git a/sa.yaml b/sa.yaml new file mode 100644 index 00000000000..124edb3f340 --- /dev/null +++ b/sa.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: broker-filter + namespace: default + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: broker-filter +subjects: + - kind: ServiceAccount + name: broker-filter + namespace: default +roleRef: + kind: ClusterRole + name: cluster-admin + apiGroup: rbac.authorization.k8s.io From 3cd236f0606014ec17deca0db843b230bd0d77aa Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Tue, 5 Feb 2019 15:39:39 -0800 Subject: [PATCH 020/128] Increase logging in the Filter to debug level. --- cmd/broker/filter/main.go | 2 ++ pkg/broker/filter.go | 21 +++++++++++---------- t2.yaml | 14 ++++++++++++++ 3 files changed, 27 insertions(+), 10 deletions(-) create mode 100644 t2.yaml diff --git a/cmd/broker/filter/main.go b/cmd/broker/filter/main.go index a8f2f8536a8..f31dcee677f 100644 --- a/cmd/broker/filter/main.go +++ b/cmd/broker/filter/main.go @@ -23,12 +23,14 @@ import ( "github.com/knative/eventing/pkg/provisioners" "github.com/knative/pkg/signals" "go.uber.org/zap" + "go.uber.org/zap/zapcore" "sigs.k8s.io/controller-runtime/pkg/client/config" "sigs.k8s.io/controller-runtime/pkg/manager" ) func main() { logConfig := provisioners.NewLoggingConfig() + logConfig.LoggingLevel["provisioner"] = zapcore.DebugLevel logger := provisioners.NewProvisionerLoggerFromConfig(logConfig).Desugar() defer logger.Sync() diff --git a/pkg/broker/filter.go b/pkg/broker/filter.go index cb542a2e060..a0e467dd942 100644 --- a/pkg/broker/filter.go +++ b/pkg/broker/filter.go @@ -22,7 +22,6 @@ import ( eventingv1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" "github.com/knative/eventing/pkg/provisioners" - "github.com/knative/eventing/pkg/provisioners/gcppubsub/util/logging" "go.uber.org/zap" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" @@ -63,27 +62,27 @@ func (r *Receiver) sendEventToTopic(channel provisioners.ChannelReference, messa t, err := r.getTrigger(ctx, channel) if err != nil { - logging.FromContext(ctx).Info("Unable to get the Trigger", zap.Error(err), zap.Any("channelRef", channel)) + r.logger.Info("Unable to get the Trigger", zap.Error(err), zap.Any("channelRef", channel)) return err } subscriberURI := t.Status.SubscriberURI if subscriberURI == "" { - logging.FromContext(ctx).Error("Unable to read subscriberURI") + r.logger.Error("Unable to read subscriberURI") return errors.New("unable to read subscriberURI") } - if !shouldSendMessage(t.Spec, message) { - logging.FromContext(ctx).Debug("Message did not pass filter") + if !r.shouldSendMessage(&t.Spec, message) { + r.logger.Debug("Message did not pass filter") return nil } err = r.dispatcher.DispatchMessage(message, subscriberURI, "", provisioners.DispatchDefaults{}) if err != nil { - logging.FromContext(ctx).Info("Failed to dispatch message", zap.Error(err)) + r.logger.Info("Failed to dispatch message", zap.Error(err)) return err } - logging.FromContext(ctx).Debug("Successfully sent message") + r.logger.Debug("Successfully sent message") return nil } @@ -98,12 +97,14 @@ func (r *Receiver) getTrigger(ctx context.Context, ref provisioners.ChannelRefer return t, err } -func shouldSendMessage(t eventingv1alpha1.TriggerSpec, m *provisioners.Message) bool { +func (r *Receiver) shouldSendMessage(t *eventingv1alpha1.TriggerSpec, m *provisioners.Message) bool { // TODO More filtering! - if t.Type != Any && t.Type != m.Headers["type"] { + if t.Type != Any && t.Type != m.Headers["Ce-Eventtype"] { + r.logger.Debug("Wrong type", zap.String("trigger.spec.type", t.Type), zap.String("message.type", m.Headers["Ce-Eventtype"]), zap.Any("m", m)) return false } - if t.Source != "" && t.Source != m.Headers["source"] { + if t.Source != "" && t.Source != m.Headers["Ce-Source"] { + r.logger.Debug("Wrong source", zap.String("trigger.spec.source", t.Source), zap.String("message.source", m.Headers["Ce-Source"])) return false } return true diff --git a/t2.yaml b/t2.yaml new file mode 100644 index 00000000000..0e217789a03 --- /dev/null +++ b/t2.yaml @@ -0,0 +1,14 @@ +apiVersion: eventing.knative.dev/v1alpha1 +kind: Trigger +metadata: + name: t +spec: + # Our Cloud Event parsing library seems to have a bug and forces to put type and source in double + # quotes (as it thinks the actual value is `"foo"`, not `foo`). + type: '"com.example.someevent"' + subscriber: + ref: + apiVersion: serving.knative.dev/v1alpha1 + kind: Service + name: message-dumper + From be350a00d705448bf4cbcf7d9253166589db4e72 Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Tue, 5 Feb 2019 16:06:24 -0800 Subject: [PATCH 021/128] Use the default channel provisioner. --- b2.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 b2.yaml diff --git a/b2.yaml b/b2.yaml new file mode 100644 index 00000000000..1c0f7b43a82 --- /dev/null +++ b/b2.yaml @@ -0,0 +1,5 @@ +apiVersion: eventing.knative.dev/v1alpha1 +kind: Broker +metadata: + name: default +spec: {} From 0068232867f6e5f4ccc7326369ec5d954a43d4d8 Mon Sep 17 00:00:00 2001 From: nachocano Date: Tue, 5 Feb 2019 16:35:17 -0800 Subject: [PATCH 022/128] Adding filtering using k8s label selectors --- .../eventing/v1alpha1/trigger_defaults.go | 10 +++++-- pkg/apis/eventing/v1alpha1/trigger_types.go | 7 +++-- .../eventing/v1alpha1/trigger_validation.go | 4 +-- .../v1alpha1/zz_generated.deepcopy.go | 10 +++++++ pkg/broker/filter.go | 26 +++++++++++-------- 5 files changed, 38 insertions(+), 19 deletions(-) diff --git a/pkg/apis/eventing/v1alpha1/trigger_defaults.go b/pkg/apis/eventing/v1alpha1/trigger_defaults.go index fd785c8bf38..334be735fa6 100644 --- a/pkg/apis/eventing/v1alpha1/trigger_defaults.go +++ b/pkg/apis/eventing/v1alpha1/trigger_defaults.go @@ -16,6 +16,8 @@ limitations under the License. package v1alpha1 +import v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + func (t *Trigger) SetDefaults() { t.Spec.SetDefaults() } @@ -24,7 +26,11 @@ func (ts *TriggerSpec) SetDefaults() { if ts.Broker == "" { ts.Broker = "default" } - if ts.Type == "" { - ts.Type = "Any" + // Make an empty LabelSelector so that we allow everything. + if ts.Filters == nil { + ts.Filters = &v1.LabelSelector{ + MatchLabels: make(map[string]string, 0), + MatchExpressions: make([]v1.LabelSelectorRequirement, 0), + } } } diff --git a/pkg/apis/eventing/v1alpha1/trigger_types.go b/pkg/apis/eventing/v1alpha1/trigger_types.go index 3e2c1d0c654..303f57cb8d2 100644 --- a/pkg/apis/eventing/v1alpha1/trigger_types.go +++ b/pkg/apis/eventing/v1alpha1/trigger_types.go @@ -58,11 +58,10 @@ type TriggerSpec struct { // +optional DeprecatedGeneration int64 `json:"generation,omitempty"` - Broker string `json:"broker,omitempty"` - Subscriber *SubscriberSpec `json:"subscriber,omitempty"` + Broker string `json:"broker,omitempty"` + Subscriber *SubscriberSpec `json:"subscriber,omitempty"` - Type string `json:"type,omitempty"` - Source string `json:"source,omitempty"` + Filters *metav1.LabelSelector `json:"filters,omitempty"` } var triggerCondSet = duckv1alpha1.NewLivingConditionSet(TriggerConditionBrokerExists, TriggerConditionKubernetesService, TriggerConditionVirtualService, TriggerConditionSubscribed) diff --git a/pkg/apis/eventing/v1alpha1/trigger_validation.go b/pkg/apis/eventing/v1alpha1/trigger_validation.go index 9d0178a464f..e93b2f60195 100644 --- a/pkg/apis/eventing/v1alpha1/trigger_validation.go +++ b/pkg/apis/eventing/v1alpha1/trigger_validation.go @@ -32,8 +32,8 @@ func (ts *TriggerSpec) Validate() *apis.FieldError { errs = errs.Also(fe) } - if ts.Type == "" { - fe := apis.ErrMissingField("type") + if ts.Filters == nil { + fe := apis.ErrMissingField("filters") errs = errs.Also(fe) } diff --git a/pkg/apis/eventing/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/eventing/v1alpha1/zz_generated.deepcopy.go index fc8f77d0aee..79fb64c6e3a 100644 --- a/pkg/apis/eventing/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/eventing/v1alpha1/zz_generated.deepcopy.go @@ -24,6 +24,7 @@ import ( apis_duck_v1alpha1 "github.com/knative/eventing/pkg/apis/duck/v1alpha1" duck_v1alpha1 "github.com/knative/pkg/apis/duck/v1alpha1" v1 "k8s.io/api/core/v1" + meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -642,6 +643,15 @@ func (in *TriggerSpec) DeepCopyInto(out *TriggerSpec) { (*in).DeepCopyInto(*out) } } + if in.Filters != nil { + in, out := &in.Filters, &out.Filters + if *in == nil { + *out = nil + } else { + *out = new(meta_v1.LabelSelector) + (*in).DeepCopyInto(*out) + } + } return } diff --git a/pkg/broker/filter.go b/pkg/broker/filter.go index cb542a2e060..449e65b5f20 100644 --- a/pkg/broker/filter.go +++ b/pkg/broker/filter.go @@ -20,6 +20,10 @@ import ( "context" "errors" + "k8s.io/apimachinery/pkg/labels" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + eventingv1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" "github.com/knative/eventing/pkg/provisioners" "github.com/knative/eventing/pkg/provisioners/gcppubsub/util/logging" @@ -45,8 +49,8 @@ type Receiver struct { // Start()ing the returned MessageReceiver. func New(logger *zap.Logger, client client.Client) (*Receiver, manager.Runnable) { r := &Receiver{ - logger: logger, - client: client, + logger: logger, + client: client, dispatcher: provisioners.NewMessageDispatcher(logger.Sugar()), } return r, r.newMessageReceiver() @@ -73,8 +77,8 @@ func (r *Receiver) sendEventToTopic(channel provisioners.ChannelReference, messa return errors.New("unable to read subscriberURI") } - if !shouldSendMessage(t.Spec, message) { - logging.FromContext(ctx).Debug("Message did not pass filter") + if !shouldSendMessage(ctx, t.Spec, message) { + logging.FromContext(ctx).Info("Message did not pass filter") return nil } @@ -98,13 +102,13 @@ func (r *Receiver) getTrigger(ctx context.Context, ref provisioners.ChannelRefer return t, err } -func shouldSendMessage(t eventingv1alpha1.TriggerSpec, m *provisioners.Message) bool { - // TODO More filtering! - if t.Type != Any && t.Type != m.Headers["type"] { - return false - } - if t.Source != "" && t.Source != m.Headers["source"] { +func shouldSendMessage(ctx context.Context, t eventingv1alpha1.TriggerSpec, m *provisioners.Message) bool { + // TODO, this conversion to selector should be done only once, possibly upon creation of the trigger + selector, err := v1.LabelSelectorAsSelector(t.Filters) + if err != nil { + logging.FromContext(ctx).Error("Invalid label selector for filter", zap.Error(err)) return false } - return true + l := labels.Set(m.Headers) + return selector.Matches(l) } From b4e2358745ea6ec453ba6ce43d8febe7359529b1 Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Tue, 5 Feb 2019 16:40:01 -0800 Subject: [PATCH 023/128] Watch namespaces and create a default Broker. --- cmd/controller/main.go | 2 + pkg/controller/eventing/namespace/provider.go | 85 +++++++++++ .../eventing/namespace/reconcile.go | 135 ++++++++++++++++++ 3 files changed, 222 insertions(+) create mode 100644 pkg/controller/eventing/namespace/provider.go create mode 100644 pkg/controller/eventing/namespace/reconcile.go diff --git a/cmd/controller/main.go b/cmd/controller/main.go index 52661d99c5e..5eee4da37d1 100644 --- a/cmd/controller/main.go +++ b/cmd/controller/main.go @@ -19,6 +19,7 @@ package main import ( "context" "flag" + "github.com/knative/eventing/pkg/controller/eventing/namespace" "log" "net/http" "os" @@ -126,6 +127,7 @@ func main() { subscription.ProvideController, broker.ProvideController(logger.Desugar(), getRequiredEnv("INGRESS_IMAGE"), getRequiredEnv("INGRESS_SERVICE_ACCOUNT"), getRequiredEnv("FILTER_IMAGE"), getRequiredEnv("FILTER_SERVICE_ACCOUNT")), trigger.ProvideController(logger.Desugar()), + namespace.ProvideController(logger.Desugar()), } for _, provider := range providers { if _, err := provider(mgr); err != nil { diff --git a/pkg/controller/eventing/namespace/provider.go b/pkg/controller/eventing/namespace/provider.go new file mode 100644 index 00000000000..01648c6918f --- /dev/null +++ b/pkg/controller/eventing/namespace/provider.go @@ -0,0 +1,85 @@ +/* +Copyright 2018 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package namespace + +import ( + "go.uber.org/zap" + "k8s.io/api/core/v1" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" +) + +const ( + // controllerAgentName is the string used by this controller to identify + // itself when creating events. + controllerAgentName = "knative-eventing-namespace-controller" +) + +type reconciler struct { + client client.Client + restConfig *rest.Config + dynamicClient dynamic.Interface + recorder record.EventRecorder + + logger *zap.Logger +} + +// Verify the struct implements reconcile.Reconciler +var _ reconcile.Reconciler = &reconciler{} + +// ProvideController returns a function that returns a Broker controller. +func ProvideController(logger *zap.Logger) func(manager.Manager) (controller.Controller, error) { + return func(mgr manager.Manager) (controller.Controller, error) { + // Setup a new controller to Reconcile Brokers. + r := &reconciler{ + recorder: mgr.GetRecorder(controllerAgentName), + logger: logger, + } + c, err := controller.New(controllerAgentName, mgr, controller.Options{ + Reconciler: r, + }) + if err != nil { + return nil, err + } + + // Watch Subscription events and enqueue Subscription object key. + if err = c.Watch(&source.Kind{Type: &v1.Namespace{}}, &handler.EnqueueRequestForObject{}); err != nil { + return nil, err + } + + return c, nil + } +} + +func (r *reconciler) InjectClient(c client.Client) error { + r.client = c + return nil +} + +func (r *reconciler) InjectConfig(c *rest.Config) error { + r.restConfig = c + var err error + r.dynamicClient, err = dynamic.NewForConfig(c) + return err +} diff --git a/pkg/controller/eventing/namespace/reconcile.go b/pkg/controller/eventing/namespace/reconcile.go new file mode 100644 index 00000000000..3185b04ca27 --- /dev/null +++ b/pkg/controller/eventing/namespace/reconcile.go @@ -0,0 +1,135 @@ +/* +Copyright 2018 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package namespace + +import ( + "context" + "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" + "github.com/knative/eventing/pkg/provisioners/gcppubsub/util/logging" + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +const ( + defaultBroker = "default" + KnativeEventingAnnotation = "eventing.knative.dev/injection" + + // Name of the corev1.Events emitted from the reconciliation process + brokerCreated = "BrokerCreated" +) + +// Reconcile compares the actual state with the desired, and attempts to +// converge the two. It then updates the Status block of the Trigger resource +// with the current status of the resource. +func (r *reconciler) Reconcile(request reconcile.Request) (reconcile.Result, error) { + ctx := context.TODO() + ctx = logging.WithLogger(ctx, r.logger.With(zap.Any("request", request))) + + ns := &corev1.Namespace{} + err := r.client.Get(context.TODO(), request.NamespacedName, ns) + + if errors.IsNotFound(err) { + logging.FromContext(ctx).Info("Could not find Namespace") + return reconcile.Result{}, nil + } + + if err != nil { + logging.FromContext(ctx).Error("Could not Get Namespace", zap.Error(err)) + return reconcile.Result{}, err + } + + if ns.Annotations[KnativeEventingAnnotation] != "true" { + logging.FromContext(ctx).Debug("Not reconciling Namespace") + return reconcile.Result{}, nil + } + + // Reconcile this copy of the Trigger and then write back any status updates regardless of + // whether the reconcile error out. + reconcileErr := r.reconcile(ctx, ns) + if reconcileErr != nil { + logging.FromContext(ctx).Error("Error reconciling Namespace", zap.Error(reconcileErr)) + } else { + logging.FromContext(ctx).Debug("Namespace reconciled") + } + + // Requeue if the resource is not ready: + return reconcile.Result{}, err +} + +func (r *reconciler) reconcile(ctx context.Context, ns *corev1.Namespace) error { + if ns.DeletionTimestamp != nil { + return nil + } + + _, err := r.reconcileBroker(ctx, ns) + if err != nil { + logging.FromContext(ctx).Error("Unable to reconcile broker for the namespace", zap.Error(err)) + return err + } + + return nil +} + +func (r *reconciler) getBroker(ctx context.Context, ns *corev1.Namespace) (*v1alpha1.Broker, error) { + b := &v1alpha1.Broker{} + name := types.NamespacedName{ + Namespace: ns.Name, + Name: defaultBroker, + } + err := r.client.Get(ctx, name, b) + return b, err +} + +func (r *reconciler) reconcileBroker(ctx context.Context, ns *corev1.Namespace) (*v1alpha1.Broker, error) { + current, err := r.getBroker(ctx, ns) + + // If the resource doesn't exist, we'll create it + if k8serrors.IsNotFound(err) { + b := newBroker(ns) + err = r.client.Create(ctx, b) + if err != nil { + return nil, err + } + r.recorder.Event(ns, corev1.EventTypeNormal, brokerCreated, "Default eventing.knative.dev Broker created.") + return b, nil + } else if err != nil { + return nil, err + } + // Don't update anything that is already present. + return current, nil +} + +func newBroker(ns *corev1.Namespace) *v1alpha1.Broker { + return &v1alpha1.Broker{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: ns.Name, + Name: defaultBroker, + Labels: brokerLabels(), + }, + } +} + +func brokerLabels() map[string]string { + return map[string]string{ + "eventing.knative.dev/brokerForNamespace": "true", + } +} From 5ca1b9f3003d35bc87e07f2c96a7976731103590 Mon Sep 17 00:00:00 2001 From: nachocano Date: Tue, 5 Feb 2019 16:47:04 -0800 Subject: [PATCH 024/128] Updating trigger example with filters --- t2.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/t2.yaml b/t2.yaml index 0e217789a03..b912ba497a4 100644 --- a/t2.yaml +++ b/t2.yaml @@ -3,12 +3,14 @@ kind: Trigger metadata: name: t spec: - # Our Cloud Event parsing library seems to have a bug and forces to put type and source in double - # quotes (as it thinks the actual value is `"foo"`, not `foo`). - type: '"com.example.someevent"' subscriber: ref: apiVersion: serving.knative.dev/v1alpha1 kind: Service name: message-dumper - + # Our Cloud Event parsing library seems to have a bug and forces to put type and source in double + # quotes (as it thinks the actual value is `"foo"`, not `foo`). + filters: + matchLabels: + Ce-Eventtype: com.example.someevent + Ce-Source: mycontext From bd53cda0d17edabbad6b72e907331dd65463c8cf Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Tue, 5 Feb 2019 21:35:41 -0800 Subject: [PATCH 025/128] Broker changes cause the namespace watcher to reconcile. --- pkg/controller/eventing/broker/reconcile.go | 2 +- pkg/controller/eventing/namespace/provider.go | 20 +++++++++++++++++++ .../eventing/namespace/reconcile.go | 6 +++--- pkg/controller/eventing/trigger/reconcile.go | 2 +- 4 files changed, 25 insertions(+), 5 deletions(-) diff --git a/pkg/controller/eventing/broker/reconcile.go b/pkg/controller/eventing/broker/reconcile.go index 5837fcbfddb..95d602f31c6 100644 --- a/pkg/controller/eventing/broker/reconcile.go +++ b/pkg/controller/eventing/broker/reconcile.go @@ -85,7 +85,7 @@ func (r *reconciler) Reconcile(request reconcile.Request) (reconcile.Result, err } // Requeue if the resource is not ready: - return reconcile.Result{}, err + return reconcile.Result{}, reconcileErr } func (r *reconciler) reconcile(ctx context.Context, b *v1alpha1.Broker) error { diff --git a/pkg/controller/eventing/namespace/provider.go b/pkg/controller/eventing/namespace/provider.go index 01648c6918f..40602be6062 100644 --- a/pkg/controller/eventing/namespace/provider.go +++ b/pkg/controller/eventing/namespace/provider.go @@ -17,8 +17,10 @@ limitations under the License. package namespace import ( + "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" "go.uber.org/zap" "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/dynamic" "k8s.io/client-go/rest" "k8s.io/client-go/tools/record" @@ -68,10 +70,28 @@ func ProvideController(logger *zap.Logger) func(manager.Manager) (controller.Con return nil, err } + if err = c.Watch(&source.Kind{Type: &v1alpha1.Broker{}}, &handler.EnqueueRequestsFromMapFunc{ToRequests: &namespaceMapper{}}); err != nil { + return nil, err + } + return c, nil } } +type namespaceMapper struct {} +var _ handler.Mapper = &namespaceMapper{} + +func (namespaceMapper) Map(o handler.MapObject) []reconcile.Request { + return []reconcile.Request{ + { + NamespacedName: types.NamespacedName{ + Namespace: "", + Name: o.Meta.GetNamespace(), + }, + }, + } +} + func (r *reconciler) InjectClient(c client.Client) error { r.client = c return nil diff --git a/pkg/controller/eventing/namespace/reconcile.go b/pkg/controller/eventing/namespace/reconcile.go index 3185b04ca27..a9a8a064a64 100644 --- a/pkg/controller/eventing/namespace/reconcile.go +++ b/pkg/controller/eventing/namespace/reconcile.go @@ -31,7 +31,7 @@ import ( const ( defaultBroker = "default" - KnativeEventingAnnotation = "eventing.knative.dev/injection" + knativeEventingAnnotation = "eventing.knative.dev/injection" // Name of the corev1.Events emitted from the reconciliation process brokerCreated = "BrokerCreated" @@ -57,7 +57,7 @@ func (r *reconciler) Reconcile(request reconcile.Request) (reconcile.Result, err return reconcile.Result{}, err } - if ns.Annotations[KnativeEventingAnnotation] != "true" { + if ns.Annotations[knativeEventingAnnotation] != "true" { logging.FromContext(ctx).Debug("Not reconciling Namespace") return reconcile.Result{}, nil } @@ -72,7 +72,7 @@ func (r *reconciler) Reconcile(request reconcile.Request) (reconcile.Result, err } // Requeue if the resource is not ready: - return reconcile.Result{}, err + return reconcile.Result{}, reconcileErr } func (r *reconciler) reconcile(ctx context.Context, ns *corev1.Namespace) error { diff --git a/pkg/controller/eventing/trigger/reconcile.go b/pkg/controller/eventing/trigger/reconcile.go index 4e461d340b1..9cc17f8bd43 100644 --- a/pkg/controller/eventing/trigger/reconcile.go +++ b/pkg/controller/eventing/trigger/reconcile.go @@ -82,7 +82,7 @@ func (r *reconciler) Reconcile(request reconcile.Request) (reconcile.Result, err } // Requeue if the resource is not ready: - return reconcile.Result{}, err + return reconcile.Result{}, reconcileErr } func (r *reconciler) reconcile(ctx context.Context, t *v1alpha1.Trigger) error { From 4f803a2faeeb8dbe113f7c30a061d8967b6dc410 Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Thu, 7 Feb 2019 11:02:13 -0800 Subject: [PATCH 026/128] Move the Logging package import and format. --- cmd/broker/filter/main.go | 2 +- cmd/broker/ingress/main.go | 15 ++++----- cmd/controller/main.go | 3 +- pkg/apis/eventing/v1alpha1/broker_defaults.go | 2 -- pkg/apis/eventing/v1alpha1/broker_types.go | 2 +- .../eventing/v1alpha1/broker_validation.go | 1 - pkg/apis/eventing/v1alpha1/trigger_types.go | 4 +-- pkg/broker/filter.go | 4 +-- pkg/controller/eventing/broker/provider.go | 10 +++--- pkg/controller/eventing/broker/reconcile.go | 20 ++++++------ .../eventing/broker/resources/filter.go | 27 ++++++++-------- .../eventing/broker/resources/ingress.go | 31 ++++++++++--------- pkg/controller/eventing/namespace/provider.go | 5 +-- .../eventing/namespace/reconcile.go | 11 ++++--- pkg/controller/eventing/trigger/reconcile.go | 9 +++--- 15 files changed, 73 insertions(+), 73 deletions(-) diff --git a/cmd/broker/filter/main.go b/cmd/broker/filter/main.go index f31dcee677f..0c39dec0550 100644 --- a/cmd/broker/filter/main.go +++ b/cmd/broker/filter/main.go @@ -18,6 +18,7 @@ package main import ( "flag" + eventingv1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" "github.com/knative/eventing/pkg/broker" "github.com/knative/eventing/pkg/provisioners" @@ -66,4 +67,3 @@ func main() { logger.Fatal("Manager.Start() returned an error", zap.Error(err)) } } - diff --git a/cmd/broker/ingress/main.go b/cmd/broker/ingress/main.go index 37b96da1412..1be24c382bb 100644 --- a/cmd/broker/ingress/main.go +++ b/cmd/broker/ingress/main.go @@ -20,12 +20,13 @@ import ( "context" "flag" "fmt" - "golang.org/x/sync/errgroup" "log" "net/http" "os" "time" + "golang.org/x/sync/errgroup" + eventingv1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" "github.com/knative/eventing/pkg/provisioners" "github.com/knative/pkg/signals" @@ -34,7 +35,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" ) - var ( readTimeout = 1 * time.Minute writeTimeout = 1 * time.Minute @@ -59,7 +59,6 @@ func main() { logger.Fatal("Unable to add scheme", zap.Error(err)) } - c := getRequiredEnv("CHANNEL") h := NewHandler(logger, c) @@ -102,8 +101,8 @@ func getRequiredEnv(envKey string) string { // http.Handler that takes a single request in and fans it out to N other servers. type Handler struct { - receiver *provisioners.MessageReceiver - dispatcher *provisioners.MessageDispatcher + receiver *provisioners.MessageReceiver + dispatcher *provisioners.MessageDispatcher destination string logger *zap.Logger @@ -112,8 +111,8 @@ type Handler struct { // NewHandler creates a new fanout.Handler. func NewHandler(logger *zap.Logger, destination string) *Handler { handler := &Handler{ - logger: logger, - dispatcher: provisioners.NewMessageDispatcher(logger.Sugar()), + logger: logger, + dispatcher: provisioners.NewMessageDispatcher(logger.Sugar()), destination: fmt.Sprintf("http://%s", destination), } // The receiver function needs to point back at the handler itself, so set it up after @@ -143,5 +142,3 @@ func (f *Handler) dispatch(msg *provisioners.Message) error { } return err } - - diff --git a/cmd/controller/main.go b/cmd/controller/main.go index 5eee4da37d1..966139ae6e6 100644 --- a/cmd/controller/main.go +++ b/cmd/controller/main.go @@ -19,12 +19,13 @@ package main import ( "context" "flag" - "github.com/knative/eventing/pkg/controller/eventing/namespace" "log" "net/http" "os" "time" + "github.com/knative/eventing/pkg/controller/eventing/namespace" + "github.com/knative/eventing/pkg/controller/eventing/trigger" "github.com/knative/eventing/pkg/controller/eventing/broker" diff --git a/pkg/apis/eventing/v1alpha1/broker_defaults.go b/pkg/apis/eventing/v1alpha1/broker_defaults.go index 890e4fbecf0..2efdd4e3f79 100644 --- a/pkg/apis/eventing/v1alpha1/broker_defaults.go +++ b/pkg/apis/eventing/v1alpha1/broker_defaults.go @@ -27,5 +27,3 @@ func (b *Broker) SetDefaults() { func (bs *BrokerSpec) SetDefaults(brokerName string) { // None } - - diff --git a/pkg/apis/eventing/v1alpha1/broker_types.go b/pkg/apis/eventing/v1alpha1/broker_types.go index fa0ba9b7c93..b50cd43101a 100644 --- a/pkg/apis/eventing/v1alpha1/broker_types.go +++ b/pkg/apis/eventing/v1alpha1/broker_types.go @@ -58,7 +58,7 @@ type BrokerSpec struct { // +optional DeprecatedGeneration int64 `json:"generation,omitempty"` - ChannelTemplate *ChannelSpec `json:"channelTemplate,omitempty"` + ChannelTemplate *ChannelSpec `json:"channelTemplate,omitempty"` } var brokerCondSet = duckv1alpha1.NewLivingConditionSet(BrokerConditionIngress, BrokerConditionChannel, BrokerConditionFilter, BrokerConditionAddressable) diff --git a/pkg/apis/eventing/v1alpha1/broker_validation.go b/pkg/apis/eventing/v1alpha1/broker_validation.go index c7954c26d15..028b8db11f6 100644 --- a/pkg/apis/eventing/v1alpha1/broker_validation.go +++ b/pkg/apis/eventing/v1alpha1/broker_validation.go @@ -28,7 +28,6 @@ func (bs *BrokerSpec) Validate() *apis.FieldError { return nil } - func (b *Broker) CheckImmutableFields(og apis.Immutable) *apis.FieldError { return nil } diff --git a/pkg/apis/eventing/v1alpha1/trigger_types.go b/pkg/apis/eventing/v1alpha1/trigger_types.go index 3e2c1d0c654..86b7c121bf6 100644 --- a/pkg/apis/eventing/v1alpha1/trigger_types.go +++ b/pkg/apis/eventing/v1alpha1/trigger_types.go @@ -58,8 +58,8 @@ type TriggerSpec struct { // +optional DeprecatedGeneration int64 `json:"generation,omitempty"` - Broker string `json:"broker,omitempty"` - Subscriber *SubscriberSpec `json:"subscriber,omitempty"` + Broker string `json:"broker,omitempty"` + Subscriber *SubscriberSpec `json:"subscriber,omitempty"` Type string `json:"type,omitempty"` Source string `json:"source,omitempty"` diff --git a/pkg/broker/filter.go b/pkg/broker/filter.go index a0e467dd942..60e2ae3d6ac 100644 --- a/pkg/broker/filter.go +++ b/pkg/broker/filter.go @@ -44,8 +44,8 @@ type Receiver struct { // Start()ing the returned MessageReceiver. func New(logger *zap.Logger, client client.Client) (*Receiver, manager.Runnable) { r := &Receiver{ - logger: logger, - client: client, + logger: logger, + client: client, dispatcher: provisioners.NewMessageDispatcher(logger.Sugar()), } return r, r.newMessageReceiver() diff --git a/pkg/controller/eventing/broker/provider.go b/pkg/controller/eventing/broker/provider.go index 5d94c8cbcfb..3136269a5df 100644 --- a/pkg/controller/eventing/broker/provider.go +++ b/pkg/controller/eventing/broker/provider.go @@ -46,8 +46,8 @@ type reconciler struct { ingressImage string ingressServiceAccountName string - filterImage string - filterServiceAccountName string + filterImage string + filterServiceAccountName string } // Verify the struct implements reconcile.Reconciler @@ -62,10 +62,10 @@ func ProvideController(logger *zap.Logger, ingressImage, ingressServiceAccount, recorder: mgr.GetRecorder(controllerAgentName), logger: logger, - ingressImage: ingressImage, + ingressImage: ingressImage, ingressServiceAccountName: ingressServiceAccount, - filterImage: filterImage, - filterServiceAccountName: filterServiceAccount, + filterImage: filterImage, + filterServiceAccountName: filterServiceAccount, }, }) if err != nil { diff --git a/pkg/controller/eventing/broker/reconcile.go b/pkg/controller/eventing/broker/reconcile.go index 95d602f31c6..f26b9ab2fb8 100644 --- a/pkg/controller/eventing/broker/reconcile.go +++ b/pkg/controller/eventing/broker/reconcile.go @@ -19,15 +19,16 @@ package broker import ( "context" "fmt" + + "github.com/knative/eventing/contrib/gcppubsub/pkg/util/logging" "github.com/knative/eventing/pkg/controller" - "k8s.io/api/apps/v1" + v1 "k8s.io/api/apps/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime/schema" "github.com/knative/eventing/pkg/controller/eventing/broker/resources" - "github.com/knative/eventing/pkg/provisioners/gcppubsub/util/logging" "go.uber.org/zap" "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" @@ -131,7 +132,6 @@ func (r *reconciler) reconcile(ctx context.Context, b *v1alpha1.Broker) error { return err } - svc, err := r.reconcileIngressService(ctx, b) if err != nil { logging.FromContext(ctx).Error("Problem reconciling ingress Service", zap.Error(err)) @@ -265,12 +265,12 @@ func newChannel(b *v1alpha1.Broker) *v1alpha1.Channel { ObjectMeta: metav1.ObjectMeta{ Namespace: b.Namespace, GenerateName: fmt.Sprintf("%s-broker-", b.Name), - Labels: ChannelLabels(b), + Labels: ChannelLabels(b), OwnerReferences: []metav1.OwnerReference{ *metav1.NewControllerRef(b, schema.GroupVersionKind{ Group: v1alpha1.SchemeGroupVersion.Group, Version: v1alpha1.SchemeGroupVersion.Version, - Kind: "Broker", + Kind: "Broker", }), }, }, @@ -280,7 +280,7 @@ func newChannel(b *v1alpha1.Broker) *v1alpha1.Channel { func ChannelLabels(b *v1alpha1.Broker) map[string]string { return map[string]string{ - "eventing.knative.dev/broker": b.Name, + "eventing.knative.dev/broker": b.Name, "eventing.knative.dev/brokerEverything": "true", } } @@ -344,10 +344,10 @@ func (r *reconciler) reconcileService(ctx context.Context, svc *corev1.Service) func (r *reconciler) reconcileIngressDeployment(ctx context.Context, b *v1alpha1.Broker, c *v1alpha1.Channel) (*v1.Deployment, error) { expected, err := resources.MakeIngress(&resources.IngressArgs{ - Broker: b, - Image: r.ingressImage, - ServiceAccountName: r.ingressServiceAccountName, - ChannelAddress: c.Status.Address.Hostname, + Broker: b, + Image: r.ingressImage, + ServiceAccountName: r.ingressServiceAccountName, + ChannelAddress: c.Status.Address.Hostname, }) if err != nil { return nil, err diff --git a/pkg/controller/eventing/broker/resources/filter.go b/pkg/controller/eventing/broker/resources/filter.go index dcb330c5570..4de9be6d0df 100644 --- a/pkg/controller/eventing/broker/resources/filter.go +++ b/pkg/controller/eventing/broker/resources/filter.go @@ -18,6 +18,7 @@ package resources import ( "fmt" + appsv1 "k8s.io/api/apps/v1" "k8s.io/apimachinery/pkg/util/intstr" @@ -28,9 +29,9 @@ import ( ) type FilterArgs struct { - Broker *eventingv1alpha1.Broker - Image string - ServiceAccountName string + Broker *eventingv1alpha1.Broker + Image string + ServiceAccountName string } func MakeFilterDeployment(args *FilterArgs) (*appsv1.Deployment, error) { @@ -42,7 +43,7 @@ func MakeFilterDeployment(args *FilterArgs) (*appsv1.Deployment, error) { *metav1.NewControllerRef(args.Broker, schema.GroupVersionKind{ Group: eventingv1alpha1.SchemeGroupVersion.Group, Version: eventingv1alpha1.SchemeGroupVersion.Version, - Kind: "Broker", + Kind: "Broker", }), }, }, @@ -62,10 +63,10 @@ func MakeFilterDeployment(args *FilterArgs) (*appsv1.Deployment, error) { Containers: []corev1.Container{ { Image: args.Image, - Name: "filter", + Name: "filter", Env: []corev1.EnvVar{ { - Name: "BROKER", + Name: "BROKER", Value: args.Broker.Name, }, }, @@ -80,14 +81,14 @@ func MakeFilterDeployment(args *FilterArgs) (*appsv1.Deployment, error) { func MakeFilterService(b *eventingv1alpha1.Broker) *corev1.Service { return &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - Namespace: b.Namespace, - Name: fmt.Sprintf("%s-broker-filter", b.Name), - Labels: filterLabels(b), + Namespace: b.Namespace, + Name: fmt.Sprintf("%s-broker-filter", b.Name), + Labels: filterLabels(b), OwnerReferences: []metav1.OwnerReference{ *metav1.NewControllerRef(b, schema.GroupVersionKind{ Group: eventingv1alpha1.SchemeGroupVersion.Group, Version: eventingv1alpha1.SchemeGroupVersion.Version, - Kind: "Broker", + Kind: "Broker", }), }, }, @@ -95,8 +96,8 @@ func MakeFilterService(b *eventingv1alpha1.Broker) *corev1.Service { Selector: filterLabels(b), Ports: []corev1.ServicePort{ { - Name: "http", - Port: 80, + Name: "http", + Port: 80, TargetPort: intstr.FromInt(8080), }, }, @@ -106,7 +107,7 @@ func MakeFilterService(b *eventingv1alpha1.Broker) *corev1.Service { func filterLabels(b *eventingv1alpha1.Broker) map[string]string { return map[string]string{ - "eventing.knative.dev/broker": b.Name, + "eventing.knative.dev/broker": b.Name, "eventing.knative.dev/brokerRole": "filter", } } diff --git a/pkg/controller/eventing/broker/resources/ingress.go b/pkg/controller/eventing/broker/resources/ingress.go index 4f6933116f9..a5444b5cf68 100644 --- a/pkg/controller/eventing/broker/resources/ingress.go +++ b/pkg/controller/eventing/broker/resources/ingress.go @@ -18,6 +18,7 @@ package resources import ( "fmt" + appsv1 "k8s.io/api/apps/v1" "k8s.io/apimachinery/pkg/util/intstr" @@ -28,10 +29,10 @@ import ( ) type IngressArgs struct { - Broker *eventingv1alpha1.Broker - Image string - ServiceAccountName string - ChannelAddress string + Broker *eventingv1alpha1.Broker + Image string + ServiceAccountName string + ChannelAddress string } func MakeIngress(args *IngressArgs) (*appsv1.Deployment, error) { @@ -43,7 +44,7 @@ func MakeIngress(args *IngressArgs) (*appsv1.Deployment, error) { *metav1.NewControllerRef(args.Broker, schema.GroupVersionKind{ Group: eventingv1alpha1.SchemeGroupVersion.Group, Version: eventingv1alpha1.SchemeGroupVersion.Version, - Kind: "Broker", + Kind: "Broker", }), }, }, @@ -63,14 +64,14 @@ func MakeIngress(args *IngressArgs) (*appsv1.Deployment, error) { Containers: []corev1.Container{ { Image: args.Image, - Name: "ingress", + Name: "ingress", Env: []corev1.EnvVar{ { - Name: "FILTER", + Name: "FILTER", Value: "", // TODO Add one. }, { - Name: "CHANNEL", + Name: "CHANNEL", Value: args.ChannelAddress, }, }, @@ -85,14 +86,14 @@ func MakeIngress(args *IngressArgs) (*appsv1.Deployment, error) { func MakeIngressService(b *eventingv1alpha1.Broker) *corev1.Service { return &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - Namespace: b.Namespace, - Name: fmt.Sprintf("%s-broker", b.Name), - Labels: ingressLabels(b), + Namespace: b.Namespace, + Name: fmt.Sprintf("%s-broker", b.Name), + Labels: ingressLabels(b), OwnerReferences: []metav1.OwnerReference{ *metav1.NewControllerRef(b, schema.GroupVersionKind{ Group: eventingv1alpha1.SchemeGroupVersion.Group, Version: eventingv1alpha1.SchemeGroupVersion.Version, - Kind: "Broker", + Kind: "Broker", }), }, }, @@ -100,8 +101,8 @@ func MakeIngressService(b *eventingv1alpha1.Broker) *corev1.Service { Selector: ingressLabels(b), Ports: []corev1.ServicePort{ { - Name: "http", - Port: 80, + Name: "http", + Port: 80, TargetPort: intstr.FromInt(8080), }, }, @@ -111,7 +112,7 @@ func MakeIngressService(b *eventingv1alpha1.Broker) *corev1.Service { func ingressLabels(b *eventingv1alpha1.Broker) map[string]string { return map[string]string{ - "eventing.knative.dev/broker": b.Name, + "eventing.knative.dev/broker": b.Name, "eventing.knative.dev/brokerRole": "ingress", } } diff --git a/pkg/controller/eventing/namespace/provider.go b/pkg/controller/eventing/namespace/provider.go index 40602be6062..2107c50fb5d 100644 --- a/pkg/controller/eventing/namespace/provider.go +++ b/pkg/controller/eventing/namespace/provider.go @@ -19,7 +19,7 @@ package namespace import ( "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" "go.uber.org/zap" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/dynamic" "k8s.io/client-go/rest" @@ -78,7 +78,8 @@ func ProvideController(logger *zap.Logger) func(manager.Manager) (controller.Con } } -type namespaceMapper struct {} +type namespaceMapper struct{} + var _ handler.Mapper = &namespaceMapper{} func (namespaceMapper) Map(o handler.MapObject) []reconcile.Request { diff --git a/pkg/controller/eventing/namespace/reconcile.go b/pkg/controller/eventing/namespace/reconcile.go index a9a8a064a64..cc2b2913010 100644 --- a/pkg/controller/eventing/namespace/reconcile.go +++ b/pkg/controller/eventing/namespace/reconcile.go @@ -18,8 +18,9 @@ package namespace import ( "context" + + "github.com/knative/eventing/contrib/gcppubsub/pkg/util/logging" "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" - "github.com/knative/eventing/pkg/provisioners/gcppubsub/util/logging" "go.uber.org/zap" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -30,7 +31,7 @@ import ( ) const ( - defaultBroker = "default" + defaultBroker = "default" knativeEventingAnnotation = "eventing.knative.dev/injection" // Name of the corev1.Events emitted from the reconciliation process @@ -121,9 +122,9 @@ func (r *reconciler) reconcileBroker(ctx context.Context, ns *corev1.Namespace) func newBroker(ns *corev1.Namespace) *v1alpha1.Broker { return &v1alpha1.Broker{ ObjectMeta: metav1.ObjectMeta{ - Namespace: ns.Name, - Name: defaultBroker, - Labels: brokerLabels(), + Namespace: ns.Name, + Name: defaultBroker, + Labels: brokerLabels(), }, } } diff --git a/pkg/controller/eventing/trigger/reconcile.go b/pkg/controller/eventing/trigger/reconcile.go index 9cc17f8bd43..3652f34ae57 100644 --- a/pkg/controller/eventing/trigger/reconcile.go +++ b/pkg/controller/eventing/trigger/reconcile.go @@ -19,11 +19,12 @@ package trigger import ( "context" "fmt" + + "github.com/knative/eventing/contrib/gcppubsub/pkg/util/logging" "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" "github.com/knative/eventing/pkg/controller" "github.com/knative/eventing/pkg/controller/eventing/broker" "github.com/knative/eventing/pkg/controller/eventing/subscription" - "github.com/knative/eventing/pkg/provisioners/gcppubsub/util/logging" istiov1alpha3 "github.com/knative/pkg/apis/istio/v1alpha3" "go.uber.org/zap" corev1 "k8s.io/api/core/v1" @@ -481,13 +482,13 @@ func makeSubscription(t *v1alpha1.Trigger, c *v1alpha1.Channel, svc *corev1.Serv Spec: v1alpha1.SubscriptionSpec{ Channel: corev1.ObjectReference{ APIVersion: v1alpha1.SchemeGroupVersion.String(), - Kind: "Channel", + Kind: "Channel", Name: c.Name, }, Subscriber: &v1alpha1.SubscriberSpec{ Ref: &corev1.ObjectReference{ APIVersion: "v1", - Kind: "Service", + Kind: "Service", Name: svc.Name, }, }, @@ -495,7 +496,7 @@ func makeSubscription(t *v1alpha1.Trigger, c *v1alpha1.Channel, svc *corev1.Serv Reply: &v1alpha1.ReplyStrategy{ Channel: &corev1.ObjectReference{ APIVersion: v1alpha1.SchemeGroupVersion.String(), - Kind: "Channel", + Kind: "Channel", Name: c.Name, }, }, From eec0f8d70001e3903c364df72ffc612553648c2f Mon Sep 17 00:00:00 2001 From: nachocano Date: Thu, 7 Feb 2019 16:31:22 -0800 Subject: [PATCH 027/128] Updating after review comments. Only doing exact header matching. In a follow up PR, I plan to include matching of expressions. It turns our that they are somewhat more involved than expected. --- .../eventing/v1alpha1/trigger_defaults.go | 11 ++---- pkg/apis/eventing/v1alpha1/trigger_types.go | 10 ++++-- .../eventing/v1alpha1/trigger_validation.go | 2 +- .../v1alpha1/zz_generated.deepcopy.go | 36 +++++++++++++++---- pkg/broker/filter.go | 20 +++++------ t2.yaml | 12 +++---- 6 files changed, 55 insertions(+), 36 deletions(-) diff --git a/pkg/apis/eventing/v1alpha1/trigger_defaults.go b/pkg/apis/eventing/v1alpha1/trigger_defaults.go index 334be735fa6..1869815a849 100644 --- a/pkg/apis/eventing/v1alpha1/trigger_defaults.go +++ b/pkg/apis/eventing/v1alpha1/trigger_defaults.go @@ -16,8 +16,6 @@ limitations under the License. package v1alpha1 -import v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - func (t *Trigger) SetDefaults() { t.Spec.SetDefaults() } @@ -26,11 +24,8 @@ func (ts *TriggerSpec) SetDefaults() { if ts.Broker == "" { ts.Broker = "default" } - // Make an empty LabelSelector so that we allow everything. - if ts.Filters == nil { - ts.Filters = &v1.LabelSelector{ - MatchLabels: make(map[string]string, 0), - MatchExpressions: make([]v1.LabelSelectorRequirement, 0), - } + // Make empty filter selector so that we allow everything. + if ts.Filter == nil { + ts.Filter = &FilterSelector{map[string]string{}} } } diff --git a/pkg/apis/eventing/v1alpha1/trigger_types.go b/pkg/apis/eventing/v1alpha1/trigger_types.go index 303f57cb8d2..39e199ea6cf 100644 --- a/pkg/apis/eventing/v1alpha1/trigger_types.go +++ b/pkg/apis/eventing/v1alpha1/trigger_types.go @@ -58,10 +58,16 @@ type TriggerSpec struct { // +optional DeprecatedGeneration int64 `json:"generation,omitempty"` - Broker string `json:"broker,omitempty"` + Broker string `json:"broker,omitempty"` + + // +optional + Filter *FilterSelector `json:"filter,omitempty"` + Subscriber *SubscriberSpec `json:"subscriber,omitempty"` +} - Filters *metav1.LabelSelector `json:"filters,omitempty"` +type FilterSelector struct { + Headers map[string]string `json:"headers,omitempty" protobuf:"bytes,1,rep,name=headers"` } var triggerCondSet = duckv1alpha1.NewLivingConditionSet(TriggerConditionBrokerExists, TriggerConditionKubernetesService, TriggerConditionVirtualService, TriggerConditionSubscribed) diff --git a/pkg/apis/eventing/v1alpha1/trigger_validation.go b/pkg/apis/eventing/v1alpha1/trigger_validation.go index e93b2f60195..42057cf9bef 100644 --- a/pkg/apis/eventing/v1alpha1/trigger_validation.go +++ b/pkg/apis/eventing/v1alpha1/trigger_validation.go @@ -32,7 +32,7 @@ func (ts *TriggerSpec) Validate() *apis.FieldError { errs = errs.Also(fe) } - if ts.Filters == nil { + if ts.Filter == nil { fe := apis.ErrMissingField("filters") errs = errs.Also(fe) } diff --git a/pkg/apis/eventing/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/eventing/v1alpha1/zz_generated.deepcopy.go index 79fb64c6e3a..167f6d00748 100644 --- a/pkg/apis/eventing/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/eventing/v1alpha1/zz_generated.deepcopy.go @@ -24,7 +24,6 @@ import ( apis_duck_v1alpha1 "github.com/knative/eventing/pkg/apis/duck/v1alpha1" duck_v1alpha1 "github.com/knative/pkg/apis/duck/v1alpha1" v1 "k8s.io/api/core/v1" - meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -375,6 +374,29 @@ func (in *ClusterChannelProvisionerStatus) DeepCopy() *ClusterChannelProvisioner return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FilterSelector) DeepCopyInto(out *FilterSelector) { + *out = *in + if in.Headers != nil { + in, out := &in.Headers, &out.Headers + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FilterSelector. +func (in *FilterSelector) DeepCopy() *FilterSelector { + if in == nil { + return nil + } + out := new(FilterSelector) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ReplyStrategy) DeepCopyInto(out *ReplyStrategy) { *out = *in @@ -634,21 +656,21 @@ func (in *TriggerList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TriggerSpec) DeepCopyInto(out *TriggerSpec) { *out = *in - if in.Subscriber != nil { - in, out := &in.Subscriber, &out.Subscriber + if in.Filter != nil { + in, out := &in.Filter, &out.Filter if *in == nil { *out = nil } else { - *out = new(SubscriberSpec) + *out = new(FilterSelector) (*in).DeepCopyInto(*out) } } - if in.Filters != nil { - in, out := &in.Filters, &out.Filters + if in.Subscriber != nil { + in, out := &in.Subscriber, &out.Subscriber if *in == nil { *out = nil } else { - *out = new(meta_v1.LabelSelector) + *out = new(SubscriberSpec) (*in).DeepCopyInto(*out) } } diff --git a/pkg/broker/filter.go b/pkg/broker/filter.go index 2e80d8ef31e..78002d23b36 100644 --- a/pkg/broker/filter.go +++ b/pkg/broker/filter.go @@ -20,13 +20,10 @@ import ( "context" "errors" - "k8s.io/apimachinery/pkg/labels" - - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - eventingv1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" "github.com/knative/eventing/pkg/provisioners" "go.uber.org/zap" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" @@ -102,13 +99,12 @@ func (r *Receiver) getTrigger(ctx context.Context, ref provisioners.ChannelRefer } func (r *Receiver) shouldSendMessage(t *eventingv1alpha1.TriggerSpec, m *provisioners.Message) bool { - // TODO, this conversion to selector should be done only once, possibly upon creation of the trigger - // in case the filters are immutable - selector, err := v1.LabelSelectorAsSelector(t.Filters) - if err != nil { - r.logger.Error("Invalid label selector for filter", zap.Error(err)) - return false + // This conversion to selector should be done only once, possibly upon creation of the trigger + // in case the filters are immutable. All validations should be performed then. + selector := labels.SelectorFromValidatedSet(labels.Set(t.Filter.Headers)) + matched := selector.Matches(labels.Set(m.Headers)) + if !matched { + r.logger.Debug("Selector did not match message headers", zap.String("selector", selector.String()), zap.Any("headers", m.Headers)) } - l := labels.Set(m.Headers) - return selector.Matches(l) + return matched } diff --git a/t2.yaml b/t2.yaml index b912ba497a4..02b2f7ba64d 100644 --- a/t2.yaml +++ b/t2.yaml @@ -3,14 +3,14 @@ kind: Trigger metadata: name: t spec: + # Our Cloud Event parsing library seems to have a bug and forces to put type and source in double + # quotes (as it thinks the actual value is `"foo"`, not `foo`). + filter: + headers: + Ce-Eventtype: '"com.example.someevent"' + Ce-Source: '"/mycontext/subcontext"' subscriber: ref: apiVersion: serving.knative.dev/v1alpha1 kind: Service name: message-dumper - # Our Cloud Event parsing library seems to have a bug and forces to put type and source in double - # quotes (as it thinks the actual value is `"foo"`, not `foo`). - filters: - matchLabels: - Ce-Eventtype: com.example.someevent - Ce-Source: mycontext From 34d42e0e1f68ff5646041f4e9c3acccd4d0c5ac8 Mon Sep 17 00:00:00 2001 From: Nacho Cano Date: Thu, 7 Feb 2019 23:38:59 -0800 Subject: [PATCH 028/128] Adding filtering expressions. Currently using LabelSelectors without validations. Using reflection to set some unexposed fields for now. --- Gopkg.lock | 1 + .../eventing/v1alpha1/trigger_defaults.go | 5 +- pkg/apis/eventing/v1alpha1/trigger_types.go | 3 +- .../v1alpha1/zz_generated.deepcopy.go | 8 +++ pkg/broker/filter.go | 55 +++++++++++++++++-- t3.yaml | 17 ++++++ 6 files changed, 82 insertions(+), 7 deletions(-) create mode 100644 t3.yaml diff --git a/Gopkg.lock b/Gopkg.lock index bf228ba34d9..96b06a0de20 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -1271,6 +1271,7 @@ "k8s.io/apimachinery/pkg/runtime", "k8s.io/apimachinery/pkg/runtime/schema", "k8s.io/apimachinery/pkg/runtime/serializer", + "k8s.io/apimachinery/pkg/selection", "k8s.io/apimachinery/pkg/types", "k8s.io/apimachinery/pkg/util/intstr", "k8s.io/apimachinery/pkg/util/sets", diff --git a/pkg/apis/eventing/v1alpha1/trigger_defaults.go b/pkg/apis/eventing/v1alpha1/trigger_defaults.go index 1869815a849..d996d5ea30d 100644 --- a/pkg/apis/eventing/v1alpha1/trigger_defaults.go +++ b/pkg/apis/eventing/v1alpha1/trigger_defaults.go @@ -16,6 +16,8 @@ limitations under the License. package v1alpha1 +import v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + func (t *Trigger) SetDefaults() { t.Spec.SetDefaults() } @@ -26,6 +28,7 @@ func (ts *TriggerSpec) SetDefaults() { } // Make empty filter selector so that we allow everything. if ts.Filter == nil { - ts.Filter = &FilterSelector{map[string]string{}} + ts.Filter = &FilterSelector{map[string]string{}, + []v1.LabelSelectorRequirement{}} } } diff --git a/pkg/apis/eventing/v1alpha1/trigger_types.go b/pkg/apis/eventing/v1alpha1/trigger_types.go index 39e199ea6cf..11b86e52b23 100644 --- a/pkg/apis/eventing/v1alpha1/trigger_types.go +++ b/pkg/apis/eventing/v1alpha1/trigger_types.go @@ -67,7 +67,8 @@ type TriggerSpec struct { } type FilterSelector struct { - Headers map[string]string `json:"headers,omitempty" protobuf:"bytes,1,rep,name=headers"` + Headers map[string]string `json:"headers,omitempty" protobuf:"bytes,1,rep,name=headers"` + HeaderExpressions []metav1.LabelSelectorRequirement `json:"headerExpressions,omitempty" protobuf:"bytes,2,rep,name=headerExpressions"` } var triggerCondSet = duckv1alpha1.NewLivingConditionSet(TriggerConditionBrokerExists, TriggerConditionKubernetesService, TriggerConditionVirtualService, TriggerConditionSubscribed) diff --git a/pkg/apis/eventing/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/eventing/v1alpha1/zz_generated.deepcopy.go index 167f6d00748..2e9b4cc6e90 100644 --- a/pkg/apis/eventing/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/eventing/v1alpha1/zz_generated.deepcopy.go @@ -24,6 +24,7 @@ import ( apis_duck_v1alpha1 "github.com/knative/eventing/pkg/apis/duck/v1alpha1" duck_v1alpha1 "github.com/knative/pkg/apis/duck/v1alpha1" v1 "k8s.io/api/core/v1" + meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -384,6 +385,13 @@ func (in *FilterSelector) DeepCopyInto(out *FilterSelector) { (*out)[key] = val } } + if in.HeaderExpressions != nil { + in, out := &in.HeaderExpressions, &out.HeaderExpressions + *out = make([]meta_v1.LabelSelectorRequirement, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } return } diff --git a/pkg/broker/filter.go b/pkg/broker/filter.go index 78002d23b36..eaeb633525f 100644 --- a/pkg/broker/filter.go +++ b/pkg/broker/filter.go @@ -19,6 +19,12 @@ package broker import ( "context" "errors" + "fmt" + "reflect" + "unsafe" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/selection" eventingv1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" "github.com/knative/eventing/pkg/provisioners" @@ -29,10 +35,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" ) -const ( - Any = "Any" -) - // Receiver parses Cloud Events and sends them to GCP PubSub. type Receiver struct { logger *zap.Logger @@ -101,10 +103,53 @@ func (r *Receiver) getTrigger(ctx context.Context, ref provisioners.ChannelRefer func (r *Receiver) shouldSendMessage(t *eventingv1alpha1.TriggerSpec, m *provisioners.Message) bool { // This conversion to selector should be done only once, possibly upon creation of the trigger // in case the filters are immutable. All validations should be performed then. - selector := labels.SelectorFromValidatedSet(labels.Set(t.Filter.Headers)) + selector, err := r.buildSelector(t) + if err != nil { + r.logger.Error("Invalid selector for trigger spec", zap.Any("triggerSpec", t)) + return false + } matched := selector.Matches(labels.Set(m.Headers)) if !matched { r.logger.Debug("Selector did not match message headers", zap.String("selector", selector.String()), zap.Any("headers", m.Headers)) } return matched } + +func (r *Receiver) buildSelector(ts *eventingv1alpha1.TriggerSpec) (labels.Selector, error) { + // Avoid validation of keys and values, otherwise we cannot use LabelSelectors. + // Eventually, we will need to create our own Selector implementation with our own Requirement struct. + selector := labels.SelectorFromValidatedSet(labels.Set(ts.Filter.Headers)) + for _, expr := range ts.Filter.HeaderExpressions { + var op selection.Operator + switch expr.Operator { + case v1.LabelSelectorOpIn: + op = selection.In + case v1.LabelSelectorOpNotIn: + op = selection.NotIn + case v1.LabelSelectorOpExists: + op = selection.Exists + case v1.LabelSelectorOpDoesNotExist: + op = selection.DoesNotExist + default: + return nil, fmt.Errorf("%q is not a valid filter selector operator", expr.Operator) + } + // Hack to set Requirement's unexposed fields to easily support expressions using k8s LabelSelectors. + // We should change this once we agree on a filter API. + r := labels.Requirement{} + rr := reflect.ValueOf(&r).Elem() + // Setting key + rrKey := rr.FieldByName("key") + rrKey = reflect.NewAt(rrKey.Type(), unsafe.Pointer(rrKey.UnsafeAddr())).Elem() + rrKey.SetString(expr.Key) + // Setting operator + rrOperator := rr.FieldByName("operator") + rrOperator = reflect.NewAt(rrOperator.Type(), unsafe.Pointer(rrOperator.UnsafeAddr())).Elem() + rrOperator.Set(reflect.ValueOf(op)) + // Setting strValues + rrStrValues := rr.FieldByName("strValues") + rrStrValues = reflect.NewAt(rrStrValues.Type(), unsafe.Pointer(rrStrValues.UnsafeAddr())).Elem() + rrStrValues.Set(reflect.ValueOf(expr.Values)) + selector = selector.Add(r) + } + return selector, nil +} diff --git a/t3.yaml b/t3.yaml new file mode 100644 index 00000000000..0ed3e969020 --- /dev/null +++ b/t3.yaml @@ -0,0 +1,17 @@ +apiVersion: eventing.knative.dev/v1alpha1 +kind: Trigger +metadata: + name: t +spec: + # Our Cloud Event parsing library seems to have a bug and forces to put type and source in double + # quotes (as it thinks the actual value is `"foo"`, not `foo`). + filter: + headers: + Ce-Source: '"/mycontext/subcontext"' + headerExpressions: + - {key: Ce-Eventtype, operator: In, values: ['"com.example.someevent"', '"com.example.someevent1"']} + subscriber: + ref: + apiVersion: serving.knative.dev/v1alpha1 + kind: Service + name: message-dumper From ad4e6f55d917a039d159cf529bcce3619d6eb960 Mon Sep 17 00:00:00 2001 From: Nacho Cano Date: Mon, 11 Feb 2019 10:47:02 -0800 Subject: [PATCH 029/128] Changes to compile --- pkg/broker/filter_test.go | 9 ++++----- pkg/provisioners/broker/reconcile.go | 7 ++++--- pkg/provisioners/trigger/reconcile.go | 9 +++++---- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/pkg/broker/filter_test.go b/pkg/broker/filter_test.go index 77d9b0d0923..2dead920105 100644 --- a/pkg/broker/filter_test.go +++ b/pkg/broker/filter_test.go @@ -23,12 +23,12 @@ import ( "strings" "testing" - "github.com/knative/eventing/pkg/provisioners/gcppubsub/util" + "github.com/knative/eventing/contrib/gcppubsub/pkg/util" eventingv1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" "k8s.io/client-go/kubernetes/scheme" - "github.com/knative/eventing/pkg/provisioners/gcppubsub/util/fakepubsub" + "github.com/knative/eventing/contrib/gcppubsub/pkg/util/fakepubsub" "go.uber.org/zap" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -36,7 +36,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" - "github.com/knative/eventing/pkg/provisioners/gcppubsub/util/testcreds" + "github.com/knative/eventing/contrib/gcppubsub/pkg/util/testcreds" ) const ( @@ -130,8 +130,7 @@ func TestReceiver(t *testing.T) { t.Run(n, func(t *testing.T) { mr, _ := New( zap.NewNop(), - fake.NewFakeClient(tc.initialState...), - fakepubsub.Creator(tc.pubSubData)) + fake.NewFakeClient(tc.initialState...)) resp := httptest.NewRecorder() req := httptest.NewRequest("POST", "/", strings.NewReader(validMessage)) req.Host = "test-channel.test-namespace.channels.cluster.local" diff --git a/pkg/provisioners/broker/reconcile.go b/pkg/provisioners/broker/reconcile.go index f26b9ab2fb8..7abd4cf8d6e 100644 --- a/pkg/provisioners/broker/reconcile.go +++ b/pkg/provisioners/broker/reconcile.go @@ -20,14 +20,15 @@ import ( "context" "fmt" + "github.com/knative/eventing/pkg/reconciler/names" + "github.com/knative/eventing/contrib/gcppubsub/pkg/util/logging" - "github.com/knative/eventing/pkg/controller" v1 "k8s.io/api/apps/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime/schema" - "github.com/knative/eventing/pkg/controller/eventing/broker/resources" + "github.com/knative/eventing/pkg/provisioners/broker/resources" "go.uber.org/zap" @@ -138,7 +139,7 @@ func (r *reconciler) reconcile(ctx context.Context, b *v1alpha1.Broker) error { return err } b.Status.MarkIngressReady() - b.Status.SetAddress(controller.ServiceHostName(svc.Name, svc.Namespace)) + b.Status.SetAddress(names.ServiceHostName(svc.Name, svc.Namespace)) return nil } diff --git a/pkg/provisioners/trigger/reconcile.go b/pkg/provisioners/trigger/reconcile.go index 3652f34ae57..668742d1308 100644 --- a/pkg/provisioners/trigger/reconcile.go +++ b/pkg/provisioners/trigger/reconcile.go @@ -20,11 +20,12 @@ import ( "context" "fmt" + "github.com/knative/eventing/pkg/reconciler/names" + "github.com/knative/eventing/contrib/gcppubsub/pkg/util/logging" "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" - "github.com/knative/eventing/pkg/controller" - "github.com/knative/eventing/pkg/controller/eventing/broker" - "github.com/knative/eventing/pkg/controller/eventing/subscription" + "github.com/knative/eventing/pkg/provisioners/broker" + "github.com/knative/eventing/pkg/reconciler/v1alpha1/subscription" istiov1alpha3 "github.com/knative/pkg/apis/istio/v1alpha3" "go.uber.org/zap" corev1 "k8s.io/api/core/v1" @@ -383,7 +384,7 @@ func newVirtualService(t *v1alpha1.Trigger, svc *corev1.Service) *istiov1alpha3. }, Spec: istiov1alpha3.VirtualServiceSpec{ Hosts: []string{ - controller.ServiceHostName(svc.Name, svc.Namespace), + names.ServiceHostName(svc.Name, svc.Namespace), }, Http: []istiov1alpha3.HTTPRoute{{ Rewrite: &istiov1alpha3.HTTPRewrite{ From bc519d8986f2c060f205c0932f9805adbb0ba497 Mon Sep 17 00:00:00 2001 From: Nacho Cano Date: Mon, 11 Feb 2019 11:01:26 -0800 Subject: [PATCH 030/128] moving filter --- pkg/{ => provisioners}/broker/filter.go | 0 pkg/{ => provisioners}/broker/filter_test.go | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename pkg/{ => provisioners}/broker/filter.go (100%) rename pkg/{ => provisioners}/broker/filter_test.go (100%) diff --git a/pkg/broker/filter.go b/pkg/provisioners/broker/filter.go similarity index 100% rename from pkg/broker/filter.go rename to pkg/provisioners/broker/filter.go diff --git a/pkg/broker/filter_test.go b/pkg/provisioners/broker/filter_test.go similarity index 100% rename from pkg/broker/filter_test.go rename to pkg/provisioners/broker/filter_test.go From 3f47238e75195d56872ddd61822ac1ea3222758f Mon Sep 17 00:00:00 2001 From: Nacho Cano Date: Mon, 11 Feb 2019 11:13:13 -0800 Subject: [PATCH 031/128] Moving cmds to broker --- {cmd => pkg/provisioners}/broker/filter/kodata/HEAD | 0 {cmd => pkg/provisioners}/broker/filter/kodata/LICENSE | 0 {cmd => pkg/provisioners}/broker/filter/kodata/VENDOR-LICENSE | 0 {cmd => pkg/provisioners}/broker/filter/main.go | 2 +- {cmd => pkg/provisioners}/broker/ingress/kodata/HEAD | 0 {cmd => pkg/provisioners}/broker/ingress/kodata/LICENSE | 0 {cmd => pkg/provisioners}/broker/ingress/kodata/VENDOR-LICENSE | 0 {cmd => pkg/provisioners}/broker/ingress/main.go | 0 pkg/provisioners/broker/{filter.go => receiver.go} | 0 pkg/provisioners/broker/{filter_test.go => receiver_test.go} | 0 10 files changed, 1 insertion(+), 1 deletion(-) rename {cmd => pkg/provisioners}/broker/filter/kodata/HEAD (100%) rename {cmd => pkg/provisioners}/broker/filter/kodata/LICENSE (100%) rename {cmd => pkg/provisioners}/broker/filter/kodata/VENDOR-LICENSE (100%) rename {cmd => pkg/provisioners}/broker/filter/main.go (97%) rename {cmd => pkg/provisioners}/broker/ingress/kodata/HEAD (100%) rename {cmd => pkg/provisioners}/broker/ingress/kodata/LICENSE (100%) rename {cmd => pkg/provisioners}/broker/ingress/kodata/VENDOR-LICENSE (100%) rename {cmd => pkg/provisioners}/broker/ingress/main.go (100%) rename pkg/provisioners/broker/{filter.go => receiver.go} (100%) rename pkg/provisioners/broker/{filter_test.go => receiver_test.go} (100%) diff --git a/cmd/broker/filter/kodata/HEAD b/pkg/provisioners/broker/filter/kodata/HEAD similarity index 100% rename from cmd/broker/filter/kodata/HEAD rename to pkg/provisioners/broker/filter/kodata/HEAD diff --git a/cmd/broker/filter/kodata/LICENSE b/pkg/provisioners/broker/filter/kodata/LICENSE similarity index 100% rename from cmd/broker/filter/kodata/LICENSE rename to pkg/provisioners/broker/filter/kodata/LICENSE diff --git a/cmd/broker/filter/kodata/VENDOR-LICENSE b/pkg/provisioners/broker/filter/kodata/VENDOR-LICENSE similarity index 100% rename from cmd/broker/filter/kodata/VENDOR-LICENSE rename to pkg/provisioners/broker/filter/kodata/VENDOR-LICENSE diff --git a/cmd/broker/filter/main.go b/pkg/provisioners/broker/filter/main.go similarity index 97% rename from cmd/broker/filter/main.go rename to pkg/provisioners/broker/filter/main.go index 0c39dec0550..3a394be949e 100644 --- a/cmd/broker/filter/main.go +++ b/pkg/provisioners/broker/filter/main.go @@ -20,8 +20,8 @@ import ( "flag" eventingv1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" - "github.com/knative/eventing/pkg/broker" "github.com/knative/eventing/pkg/provisioners" + "github.com/knative/eventing/pkg/provisioners/broker" "github.com/knative/pkg/signals" "go.uber.org/zap" "go.uber.org/zap/zapcore" diff --git a/cmd/broker/ingress/kodata/HEAD b/pkg/provisioners/broker/ingress/kodata/HEAD similarity index 100% rename from cmd/broker/ingress/kodata/HEAD rename to pkg/provisioners/broker/ingress/kodata/HEAD diff --git a/cmd/broker/ingress/kodata/LICENSE b/pkg/provisioners/broker/ingress/kodata/LICENSE similarity index 100% rename from cmd/broker/ingress/kodata/LICENSE rename to pkg/provisioners/broker/ingress/kodata/LICENSE diff --git a/cmd/broker/ingress/kodata/VENDOR-LICENSE b/pkg/provisioners/broker/ingress/kodata/VENDOR-LICENSE similarity index 100% rename from cmd/broker/ingress/kodata/VENDOR-LICENSE rename to pkg/provisioners/broker/ingress/kodata/VENDOR-LICENSE diff --git a/cmd/broker/ingress/main.go b/pkg/provisioners/broker/ingress/main.go similarity index 100% rename from cmd/broker/ingress/main.go rename to pkg/provisioners/broker/ingress/main.go diff --git a/pkg/provisioners/broker/filter.go b/pkg/provisioners/broker/receiver.go similarity index 100% rename from pkg/provisioners/broker/filter.go rename to pkg/provisioners/broker/receiver.go diff --git a/pkg/provisioners/broker/filter_test.go b/pkg/provisioners/broker/receiver_test.go similarity index 100% rename from pkg/provisioners/broker/filter_test.go rename to pkg/provisioners/broker/receiver_test.go From 3070157a8cbba100fd31fa3cbaf9982f67e8c030 Mon Sep 17 00:00:00 2001 From: Nacho Cano Date: Mon, 11 Feb 2019 11:48:15 -0800 Subject: [PATCH 032/128] updating controller --- config/500-controller.yaml | 4 ++-- pkg/provisioners/broker/filter/kodata/HEAD | 1 - pkg/provisioners/broker/filter/kodata/LICENSE | 1 - pkg/provisioners/broker/filter/kodata/VENDOR-LICENSE | 1 - pkg/provisioners/broker/ingress/kodata/HEAD | 1 - pkg/provisioners/broker/ingress/kodata/LICENSE | 1 - pkg/provisioners/broker/ingress/kodata/VENDOR-LICENSE | 1 - 7 files changed, 2 insertions(+), 8 deletions(-) delete mode 120000 pkg/provisioners/broker/filter/kodata/HEAD delete mode 120000 pkg/provisioners/broker/filter/kodata/LICENSE delete mode 120000 pkg/provisioners/broker/filter/kodata/VENDOR-LICENSE delete mode 120000 pkg/provisioners/broker/ingress/kodata/HEAD delete mode 120000 pkg/provisioners/broker/ingress/kodata/LICENSE delete mode 120000 pkg/provisioners/broker/ingress/kodata/VENDOR-LICENSE diff --git a/config/500-controller.yaml b/config/500-controller.yaml index 15d76a8cf3a..92ca02e7e19 100644 --- a/config/500-controller.yaml +++ b/config/500-controller.yaml @@ -36,11 +36,11 @@ spec: ] env: - name: INGRESS_IMAGE - value: github.com/knative/eventing/cmd/broker/ingress + value: github.com/knative/eventing/pkg/provisioners/broker/ingress - name: INGRESS_SERVICE_ACCOUNT value: default - name: FILTER_IMAGE - value: github.com/knative/eventing/cmd/broker/filter + value: github.com/knative/eventing/pkg/provisioners/broker/filter - name: FILTER_SERVICE_ACCOUNT value: broker-filter volumeMounts: diff --git a/pkg/provisioners/broker/filter/kodata/HEAD b/pkg/provisioners/broker/filter/kodata/HEAD deleted file mode 120000 index 481bd4eff49..00000000000 --- a/pkg/provisioners/broker/filter/kodata/HEAD +++ /dev/null @@ -1 +0,0 @@ -../../../../.git/HEAD \ No newline at end of file diff --git a/pkg/provisioners/broker/filter/kodata/LICENSE b/pkg/provisioners/broker/filter/kodata/LICENSE deleted file mode 120000 index 14776154326..00000000000 --- a/pkg/provisioners/broker/filter/kodata/LICENSE +++ /dev/null @@ -1 +0,0 @@ -../../../../LICENSE \ No newline at end of file diff --git a/pkg/provisioners/broker/filter/kodata/VENDOR-LICENSE b/pkg/provisioners/broker/filter/kodata/VENDOR-LICENSE deleted file mode 120000 index 7322c09d957..00000000000 --- a/pkg/provisioners/broker/filter/kodata/VENDOR-LICENSE +++ /dev/null @@ -1 +0,0 @@ -../../../../third_party/VENDOR-LICENSE \ No newline at end of file diff --git a/pkg/provisioners/broker/ingress/kodata/HEAD b/pkg/provisioners/broker/ingress/kodata/HEAD deleted file mode 120000 index 481bd4eff49..00000000000 --- a/pkg/provisioners/broker/ingress/kodata/HEAD +++ /dev/null @@ -1 +0,0 @@ -../../../../.git/HEAD \ No newline at end of file diff --git a/pkg/provisioners/broker/ingress/kodata/LICENSE b/pkg/provisioners/broker/ingress/kodata/LICENSE deleted file mode 120000 index 14776154326..00000000000 --- a/pkg/provisioners/broker/ingress/kodata/LICENSE +++ /dev/null @@ -1 +0,0 @@ -../../../../LICENSE \ No newline at end of file diff --git a/pkg/provisioners/broker/ingress/kodata/VENDOR-LICENSE b/pkg/provisioners/broker/ingress/kodata/VENDOR-LICENSE deleted file mode 120000 index 7322c09d957..00000000000 --- a/pkg/provisioners/broker/ingress/kodata/VENDOR-LICENSE +++ /dev/null @@ -1 +0,0 @@ -../../../../third_party/VENDOR-LICENSE \ No newline at end of file From 0957fda7be1696e91311d5b5a29fb010a2fff95b Mon Sep 17 00:00:00 2001 From: Nacho Cano Date: Mon, 11 Feb 2019 12:10:50 -0800 Subject: [PATCH 033/128] Moving provider and reconciler to reconciler folder, and merging them into one --- cmd/controller/main.go | 4 +- pkg/provisioners/broker/provider.go | 101 ------------------ pkg/provisioners/trigger/provider.go | 85 --------------- .../v1alpha1/broker/broker.go} | 77 +++++++++++++ .../v1alpha1/trigger/trigger.go} | 63 ++++++++++- 5 files changed, 141 insertions(+), 189 deletions(-) delete mode 100644 pkg/provisioners/broker/provider.go delete mode 100644 pkg/provisioners/trigger/provider.go rename pkg/{provisioners/broker/reconcile.go => reconciler/v1alpha1/broker/broker.go} (82%) rename pkg/{provisioners/trigger/reconcile.go => reconciler/v1alpha1/trigger/trigger.go} (89%) diff --git a/cmd/controller/main.go b/cmd/controller/main.go index 02694da187d..fdc4acc6d69 100644 --- a/cmd/controller/main.go +++ b/cmd/controller/main.go @@ -24,10 +24,10 @@ import ( "os" "time" - "github.com/knative/eventing/pkg/provisioners/broker" "github.com/knative/eventing/pkg/provisioners/namespace" - "github.com/knative/eventing/pkg/provisioners/trigger" + "github.com/knative/eventing/pkg/reconciler/v1alpha1/broker" "github.com/knative/eventing/pkg/reconciler/v1alpha1/subscription" + "github.com/knative/eventing/pkg/reconciler/v1alpha1/trigger" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/manager" diff --git a/pkg/provisioners/broker/provider.go b/pkg/provisioners/broker/provider.go deleted file mode 100644 index 3136269a5df..00000000000 --- a/pkg/provisioners/broker/provider.go +++ /dev/null @@ -1,101 +0,0 @@ -/* -Copyright 2018 The Knative Authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package broker - -import ( - "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" - "go.uber.org/zap" - "k8s.io/client-go/dynamic" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/record" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/handler" - "sigs.k8s.io/controller-runtime/pkg/manager" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/controller-runtime/pkg/source" -) - -const ( - // controllerAgentName is the string used by this controller to identify - // itself when creating events. - controllerAgentName = "broker-controller" -) - -type reconciler struct { - client client.Client - restConfig *rest.Config - dynamicClient dynamic.Interface - recorder record.EventRecorder - - logger *zap.Logger - - ingressImage string - ingressServiceAccountName string - filterImage string - filterServiceAccountName string -} - -// Verify the struct implements reconcile.Reconciler -var _ reconcile.Reconciler = &reconciler{} - -// ProvideController returns a function that returns a Broker controller. -func ProvideController(logger *zap.Logger, ingressImage, ingressServiceAccount, filterImage, filterServiceAccount string) func(manager.Manager) (controller.Controller, error) { - return func(mgr manager.Manager) (controller.Controller, error) { - // Setup a new controller to Reconcile Brokers. - c, err := controller.New(controllerAgentName, mgr, controller.Options{ - Reconciler: &reconciler{ - recorder: mgr.GetRecorder(controllerAgentName), - logger: logger, - - ingressImage: ingressImage, - ingressServiceAccountName: ingressServiceAccount, - filterImage: filterImage, - filterServiceAccountName: filterServiceAccount, - }, - }) - if err != nil { - return nil, err - } - - // Watch Subscription events and enqueue Subscription object key. - if err = c.Watch(&source.Kind{Type: &v1alpha1.Broker{}}, &handler.EnqueueRequestForObject{}); err != nil { - return nil, err - } - - err = c.Watch(&source.Kind{ - Type: &v1alpha1.Channel{}, - }, &handler.EnqueueRequestForOwner{OwnerType: &v1alpha1.Broker{}, IsController: true}) - if err != nil { - return nil, err - } - - return c, nil - } -} - -func (r *reconciler) InjectClient(c client.Client) error { - r.client = c - return nil -} - -func (r *reconciler) InjectConfig(c *rest.Config) error { - r.restConfig = c - var err error - r.dynamicClient, err = dynamic.NewForConfig(c) - return err -} diff --git a/pkg/provisioners/trigger/provider.go b/pkg/provisioners/trigger/provider.go deleted file mode 100644 index 1bd62841435..00000000000 --- a/pkg/provisioners/trigger/provider.go +++ /dev/null @@ -1,85 +0,0 @@ -/* -Copyright 2018 The Knative Authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package trigger - -import ( - "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" - "go.uber.org/zap" - "k8s.io/client-go/dynamic" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/record" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/handler" - "sigs.k8s.io/controller-runtime/pkg/manager" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/controller-runtime/pkg/source" -) - -const ( - // controllerAgentName is the string used by this controller to identify - // itself when creating events. - controllerAgentName = "trigger-controller" -) - -type reconciler struct { - client client.Client - restConfig *rest.Config - dynamicClient dynamic.Interface - recorder record.EventRecorder - - logger *zap.Logger -} - -// Verify the struct implements reconcile.Reconciler -var _ reconcile.Reconciler = &reconciler{} - -// ProvideController returns a function that returns a Broker controller. -func ProvideController(logger *zap.Logger) func(manager.Manager) (controller.Controller, error) { - return func(mgr manager.Manager) (controller.Controller, error) { - // Setup a new controller to Reconcile Brokers. - r := &reconciler{ - recorder: mgr.GetRecorder(controllerAgentName), - logger: logger, - } - c, err := controller.New(controllerAgentName, mgr, controller.Options{ - Reconciler: r, - }) - if err != nil { - return nil, err - } - - // Watch Subscription events and enqueue Subscription object key. - if err = c.Watch(&source.Kind{Type: &v1alpha1.Trigger{}}, &handler.EnqueueRequestForObject{}); err != nil { - return nil, err - } - - return c, nil - } -} - -func (r *reconciler) InjectClient(c client.Client) error { - r.client = c - return nil -} - -func (r *reconciler) InjectConfig(c *rest.Config) error { - r.restConfig = c - var err error - r.dynamicClient, err = dynamic.NewForConfig(c) - return err -} diff --git a/pkg/provisioners/broker/reconcile.go b/pkg/reconciler/v1alpha1/broker/broker.go similarity index 82% rename from pkg/provisioners/broker/reconcile.go rename to pkg/reconciler/v1alpha1/broker/broker.go index 7abd4cf8d6e..ff9f03c13f3 100644 --- a/pkg/provisioners/broker/reconcile.go +++ b/pkg/reconciler/v1alpha1/broker/broker.go @@ -20,6 +20,14 @@ import ( "context" "fmt" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/source" + "github.com/knative/eventing/pkg/reconciler/names" "github.com/knative/eventing/contrib/gcppubsub/pkg/util/logging" @@ -41,15 +49,84 @@ import ( "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" runtimeclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" ) const ( + // controllerAgentName is the string used by this controller to identify + // itself when creating events. + controllerAgentName = "broker-controller" + // Name of the corev1.Events emitted from the reconciliation process brokerReconciled = "BrokerReconciled" brokerUpdateStatusFailed = "BrokerUpdateStatusFailed" ) +type reconciler struct { + client client.Client + restConfig *rest.Config + dynamicClient dynamic.Interface + recorder record.EventRecorder + + logger *zap.Logger + + ingressImage string + ingressServiceAccountName string + filterImage string + filterServiceAccountName string +} + +// Verify the struct implements reconcile.Reconciler +var _ reconcile.Reconciler = &reconciler{} + +// ProvideController returns a function that returns a Broker controller. +func ProvideController(logger *zap.Logger, ingressImage, ingressServiceAccount, filterImage, filterServiceAccount string) func(manager.Manager) (controller.Controller, error) { + return func(mgr manager.Manager) (controller.Controller, error) { + // Setup a new controller to Reconcile Brokers. + c, err := controller.New(controllerAgentName, mgr, controller.Options{ + Reconciler: &reconciler{ + recorder: mgr.GetRecorder(controllerAgentName), + logger: logger, + + ingressImage: ingressImage, + ingressServiceAccountName: ingressServiceAccount, + filterImage: filterImage, + filterServiceAccountName: filterServiceAccount, + }, + }) + if err != nil { + return nil, err + } + + // Watch Subscription events and enqueue Subscription object key. + if err = c.Watch(&source.Kind{Type: &v1alpha1.Broker{}}, &handler.EnqueueRequestForObject{}); err != nil { + return nil, err + } + + err = c.Watch(&source.Kind{ + Type: &v1alpha1.Channel{}, + }, &handler.EnqueueRequestForOwner{OwnerType: &v1alpha1.Broker{}, IsController: true}) + if err != nil { + return nil, err + } + + return c, nil + } +} + +func (r *reconciler) InjectClient(c client.Client) error { + r.client = c + return nil +} + +func (r *reconciler) InjectConfig(c *rest.Config) error { + r.restConfig = c + var err error + r.dynamicClient, err = dynamic.NewForConfig(c) + return err +} + // Reconcile compares the actual state with the desired, and attempts to // converge the two. It then updates the Status block of the Broker resource // with the current status of the resource. diff --git a/pkg/provisioners/trigger/reconcile.go b/pkg/reconciler/v1alpha1/trigger/trigger.go similarity index 89% rename from pkg/provisioners/trigger/reconcile.go rename to pkg/reconciler/v1alpha1/trigger/trigger.go index 668742d1308..d7ce3cf2c37 100644 --- a/pkg/provisioners/trigger/reconcile.go +++ b/pkg/reconciler/v1alpha1/trigger/trigger.go @@ -20,11 +20,19 @@ import ( "context" "fmt" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/source" + "github.com/knative/eventing/pkg/reconciler/names" "github.com/knative/eventing/contrib/gcppubsub/pkg/util/logging" "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" - "github.com/knative/eventing/pkg/provisioners/broker" + "github.com/knative/eventing/pkg/reconciler/v1alpha1/broker" "github.com/knative/eventing/pkg/reconciler/v1alpha1/subscription" istiov1alpha3 "github.com/knative/pkg/apis/istio/v1alpha3" "go.uber.org/zap" @@ -38,15 +46,68 @@ import ( "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" runtimeclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" ) const ( + // controllerAgentName is the string used by this controller to identify + // itself when creating events. + controllerAgentName = "trigger-controller" + // Name of the corev1.Events emitted from the reconciliation process triggerReconciled = "TriggerReconciled" triggerUpdateStatusFailed = "TriggerUpdateStatusFailed" ) +type reconciler struct { + client client.Client + restConfig *rest.Config + dynamicClient dynamic.Interface + recorder record.EventRecorder + + logger *zap.Logger +} + +// Verify the struct implements reconcile.Reconciler +var _ reconcile.Reconciler = &reconciler{} + +// ProvideController returns a function that returns a Broker controller. +func ProvideController(logger *zap.Logger) func(manager.Manager) (controller.Controller, error) { + return func(mgr manager.Manager) (controller.Controller, error) { + // Setup a new controller to Reconcile Brokers. + r := &reconciler{ + recorder: mgr.GetRecorder(controllerAgentName), + logger: logger, + } + c, err := controller.New(controllerAgentName, mgr, controller.Options{ + Reconciler: r, + }) + if err != nil { + return nil, err + } + + // Watch Subscription events and enqueue Subscription object key. + if err = c.Watch(&source.Kind{Type: &v1alpha1.Trigger{}}, &handler.EnqueueRequestForObject{}); err != nil { + return nil, err + } + + return c, nil + } +} + +func (r *reconciler) InjectClient(c client.Client) error { + r.client = c + return nil +} + +func (r *reconciler) InjectConfig(c *rest.Config) error { + r.restConfig = c + var err error + r.dynamicClient, err = dynamic.NewForConfig(c) + return err +} + // Reconcile compares the actual state with the desired, and attempts to // converge the two. It then updates the Status block of the Trigger resource // with the current status of the resource. From 509c8a1741ae03f21324906b2d9160adece17095 Mon Sep 17 00:00:00 2001 From: Nacho Cano Date: Mon, 11 Feb 2019 16:10:40 -0800 Subject: [PATCH 034/128] Adding verbs to dispatcher --- contrib/gcppubsub/config/gcppubsub.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/contrib/gcppubsub/config/gcppubsub.yaml b/contrib/gcppubsub/config/gcppubsub.yaml index 02359d6d10c..7e373b50280 100644 --- a/contrib/gcppubsub/config/gcppubsub.yaml +++ b/contrib/gcppubsub/config/gcppubsub.yaml @@ -161,6 +161,14 @@ rules: - get - list - watch + - apiGroups: + - "" # Core API Group. + resources: + - events + verbs: + - create + - patch + - update --- From d33308b1077b2a46832618d5f6263a422318a7de Mon Sep 17 00:00:00 2001 From: Nacho Cano Date: Mon, 11 Feb 2019 16:51:18 -0800 Subject: [PATCH 035/128] Moving back mains to cmd --- cmd/broker/filter/kodata/HEAD | 1 + cmd/broker/filter/kodata/LICENSE | 1 + cmd/broker/filter/kodata/VENDOR-LICENSE | 1 + {pkg/provisioners => cmd}/broker/filter/main.go | 0 cmd/broker/ingress/kodata/HEAD | 1 + cmd/broker/ingress/kodata/LICENSE | 1 + cmd/broker/ingress/kodata/VENDOR-LICENSE | 1 + {pkg/provisioners => cmd}/broker/ingress/main.go | 0 8 files changed, 6 insertions(+) create mode 120000 cmd/broker/filter/kodata/HEAD create mode 120000 cmd/broker/filter/kodata/LICENSE create mode 120000 cmd/broker/filter/kodata/VENDOR-LICENSE rename {pkg/provisioners => cmd}/broker/filter/main.go (100%) create mode 120000 cmd/broker/ingress/kodata/HEAD create mode 120000 cmd/broker/ingress/kodata/LICENSE create mode 120000 cmd/broker/ingress/kodata/VENDOR-LICENSE rename {pkg/provisioners => cmd}/broker/ingress/main.go (100%) diff --git a/cmd/broker/filter/kodata/HEAD b/cmd/broker/filter/kodata/HEAD new file mode 120000 index 00000000000..481bd4eff49 --- /dev/null +++ b/cmd/broker/filter/kodata/HEAD @@ -0,0 +1 @@ +../../../../.git/HEAD \ No newline at end of file diff --git a/cmd/broker/filter/kodata/LICENSE b/cmd/broker/filter/kodata/LICENSE new file mode 120000 index 00000000000..14776154326 --- /dev/null +++ b/cmd/broker/filter/kodata/LICENSE @@ -0,0 +1 @@ +../../../../LICENSE \ No newline at end of file diff --git a/cmd/broker/filter/kodata/VENDOR-LICENSE b/cmd/broker/filter/kodata/VENDOR-LICENSE new file mode 120000 index 00000000000..7322c09d957 --- /dev/null +++ b/cmd/broker/filter/kodata/VENDOR-LICENSE @@ -0,0 +1 @@ +../../../../third_party/VENDOR-LICENSE \ No newline at end of file diff --git a/pkg/provisioners/broker/filter/main.go b/cmd/broker/filter/main.go similarity index 100% rename from pkg/provisioners/broker/filter/main.go rename to cmd/broker/filter/main.go diff --git a/cmd/broker/ingress/kodata/HEAD b/cmd/broker/ingress/kodata/HEAD new file mode 120000 index 00000000000..481bd4eff49 --- /dev/null +++ b/cmd/broker/ingress/kodata/HEAD @@ -0,0 +1 @@ +../../../../.git/HEAD \ No newline at end of file diff --git a/cmd/broker/ingress/kodata/LICENSE b/cmd/broker/ingress/kodata/LICENSE new file mode 120000 index 00000000000..14776154326 --- /dev/null +++ b/cmd/broker/ingress/kodata/LICENSE @@ -0,0 +1 @@ +../../../../LICENSE \ No newline at end of file diff --git a/cmd/broker/ingress/kodata/VENDOR-LICENSE b/cmd/broker/ingress/kodata/VENDOR-LICENSE new file mode 120000 index 00000000000..7322c09d957 --- /dev/null +++ b/cmd/broker/ingress/kodata/VENDOR-LICENSE @@ -0,0 +1 @@ +../../../../third_party/VENDOR-LICENSE \ No newline at end of file diff --git a/pkg/provisioners/broker/ingress/main.go b/cmd/broker/ingress/main.go similarity index 100% rename from pkg/provisioners/broker/ingress/main.go rename to cmd/broker/ingress/main.go From af83baa8822c8de06445604a7763d688afe69a17 Mon Sep 17 00:00:00 2001 From: Nacho Cano Date: Mon, 11 Feb 2019 16:52:26 -0800 Subject: [PATCH 036/128] Updating config --- config/500-controller.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/500-controller.yaml b/config/500-controller.yaml index 92ca02e7e19..15d76a8cf3a 100644 --- a/config/500-controller.yaml +++ b/config/500-controller.yaml @@ -36,11 +36,11 @@ spec: ] env: - name: INGRESS_IMAGE - value: github.com/knative/eventing/pkg/provisioners/broker/ingress + value: github.com/knative/eventing/cmd/broker/ingress - name: INGRESS_SERVICE_ACCOUNT value: default - name: FILTER_IMAGE - value: github.com/knative/eventing/pkg/provisioners/broker/filter + value: github.com/knative/eventing/cmd/broker/filter - name: FILTER_SERVICE_ACCOUNT value: broker-filter volumeMounts: From 8df699cdc1fe199ee61b3f96f764594326b62a3c Mon Sep 17 00:00:00 2001 From: Nacho Cano Date: Mon, 11 Feb 2019 17:10:40 -0800 Subject: [PATCH 037/128] Moving resources to reconcilers --- pkg/reconciler/v1alpha1/broker/broker.go | 2 +- .../v1alpha1}/broker/resources/filter.go | 0 .../v1alpha1}/broker/resources/ingress.go | 0 3 files changed, 1 insertion(+), 1 deletion(-) rename pkg/{provisioners => reconciler/v1alpha1}/broker/resources/filter.go (100%) rename pkg/{provisioners => reconciler/v1alpha1}/broker/resources/ingress.go (100%) diff --git a/pkg/reconciler/v1alpha1/broker/broker.go b/pkg/reconciler/v1alpha1/broker/broker.go index ff9f03c13f3..21961fb06f3 100644 --- a/pkg/reconciler/v1alpha1/broker/broker.go +++ b/pkg/reconciler/v1alpha1/broker/broker.go @@ -36,7 +36,7 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime/schema" - "github.com/knative/eventing/pkg/provisioners/broker/resources" + "github.com/knative/eventing/pkg/reconciler/v1alpha1/broker/resources" "go.uber.org/zap" diff --git a/pkg/provisioners/broker/resources/filter.go b/pkg/reconciler/v1alpha1/broker/resources/filter.go similarity index 100% rename from pkg/provisioners/broker/resources/filter.go rename to pkg/reconciler/v1alpha1/broker/resources/filter.go diff --git a/pkg/provisioners/broker/resources/ingress.go b/pkg/reconciler/v1alpha1/broker/resources/ingress.go similarity index 100% rename from pkg/provisioners/broker/resources/ingress.go rename to pkg/reconciler/v1alpha1/broker/resources/ingress.go From bd6a8fa9c62460ee696854fb44a898e72e249ae2 Mon Sep 17 00:00:00 2001 From: Nacho Cano Date: Mon, 11 Feb 2019 17:16:45 -0800 Subject: [PATCH 038/128] Moving broker folder back where it belongs --- cmd/broker/filter/main.go | 2 +- pkg/{provisioners => }/broker/receiver.go | 0 pkg/{provisioners => }/broker/receiver_test.go | 0 3 files changed, 1 insertion(+), 1 deletion(-) rename pkg/{provisioners => }/broker/receiver.go (100%) rename pkg/{provisioners => }/broker/receiver_test.go (100%) diff --git a/cmd/broker/filter/main.go b/cmd/broker/filter/main.go index 3a394be949e..0c39dec0550 100644 --- a/cmd/broker/filter/main.go +++ b/cmd/broker/filter/main.go @@ -20,8 +20,8 @@ import ( "flag" eventingv1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" + "github.com/knative/eventing/pkg/broker" "github.com/knative/eventing/pkg/provisioners" - "github.com/knative/eventing/pkg/provisioners/broker" "github.com/knative/pkg/signals" "go.uber.org/zap" "go.uber.org/zap/zapcore" diff --git a/pkg/provisioners/broker/receiver.go b/pkg/broker/receiver.go similarity index 100% rename from pkg/provisioners/broker/receiver.go rename to pkg/broker/receiver.go diff --git a/pkg/provisioners/broker/receiver_test.go b/pkg/broker/receiver_test.go similarity index 100% rename from pkg/provisioners/broker/receiver_test.go rename to pkg/broker/receiver_test.go From 3b84c915e20e6dd600537f38b916e715688fed81 Mon Sep 17 00:00:00 2001 From: Nacho Cano Date: Tue, 12 Feb 2019 09:16:37 -0800 Subject: [PATCH 039/128] Revert "Merge branch 'broker-new-model' into broker-new" This reverts commit d567412b062233463335c7feea94be795f7f2e15, reversing changes made to bd6a8fa9c62460ee696854fb44a898e72e249ae2. --- .../eventing/v1alpha1/trigger_defaults.go | 5 ++- pkg/apis/eventing/v1alpha1/trigger_types.go | 11 ++----- .../eventing/v1alpha1/trigger_validation.go | 4 +-- .../v1alpha1/zz_generated.deepcopy.go | 32 ------------------- pkg/broker/receiver.go | 17 +++++----- t2.yaml | 6 ++-- 6 files changed, 18 insertions(+), 57 deletions(-) diff --git a/pkg/apis/eventing/v1alpha1/trigger_defaults.go b/pkg/apis/eventing/v1alpha1/trigger_defaults.go index 1869815a849..fd785c8bf38 100644 --- a/pkg/apis/eventing/v1alpha1/trigger_defaults.go +++ b/pkg/apis/eventing/v1alpha1/trigger_defaults.go @@ -24,8 +24,7 @@ func (ts *TriggerSpec) SetDefaults() { if ts.Broker == "" { ts.Broker = "default" } - // Make empty filter selector so that we allow everything. - if ts.Filter == nil { - ts.Filter = &FilterSelector{map[string]string{}} + if ts.Type == "" { + ts.Type = "Any" } } diff --git a/pkg/apis/eventing/v1alpha1/trigger_types.go b/pkg/apis/eventing/v1alpha1/trigger_types.go index 39e199ea6cf..86b7c121bf6 100644 --- a/pkg/apis/eventing/v1alpha1/trigger_types.go +++ b/pkg/apis/eventing/v1alpha1/trigger_types.go @@ -58,16 +58,11 @@ type TriggerSpec struct { // +optional DeprecatedGeneration int64 `json:"generation,omitempty"` - Broker string `json:"broker,omitempty"` - - // +optional - Filter *FilterSelector `json:"filter,omitempty"` - + Broker string `json:"broker,omitempty"` Subscriber *SubscriberSpec `json:"subscriber,omitempty"` -} -type FilterSelector struct { - Headers map[string]string `json:"headers,omitempty" protobuf:"bytes,1,rep,name=headers"` + Type string `json:"type,omitempty"` + Source string `json:"source,omitempty"` } var triggerCondSet = duckv1alpha1.NewLivingConditionSet(TriggerConditionBrokerExists, TriggerConditionKubernetesService, TriggerConditionVirtualService, TriggerConditionSubscribed) diff --git a/pkg/apis/eventing/v1alpha1/trigger_validation.go b/pkg/apis/eventing/v1alpha1/trigger_validation.go index 42057cf9bef..9d0178a464f 100644 --- a/pkg/apis/eventing/v1alpha1/trigger_validation.go +++ b/pkg/apis/eventing/v1alpha1/trigger_validation.go @@ -32,8 +32,8 @@ func (ts *TriggerSpec) Validate() *apis.FieldError { errs = errs.Also(fe) } - if ts.Filter == nil { - fe := apis.ErrMissingField("filters") + if ts.Type == "" { + fe := apis.ErrMissingField("type") errs = errs.Also(fe) } diff --git a/pkg/apis/eventing/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/eventing/v1alpha1/zz_generated.deepcopy.go index 167f6d00748..fc8f77d0aee 100644 --- a/pkg/apis/eventing/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/eventing/v1alpha1/zz_generated.deepcopy.go @@ -374,29 +374,6 @@ func (in *ClusterChannelProvisionerStatus) DeepCopy() *ClusterChannelProvisioner return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *FilterSelector) DeepCopyInto(out *FilterSelector) { - *out = *in - if in.Headers != nil { - in, out := &in.Headers, &out.Headers - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FilterSelector. -func (in *FilterSelector) DeepCopy() *FilterSelector { - if in == nil { - return nil - } - out := new(FilterSelector) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ReplyStrategy) DeepCopyInto(out *ReplyStrategy) { *out = *in @@ -656,15 +633,6 @@ func (in *TriggerList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TriggerSpec) DeepCopyInto(out *TriggerSpec) { *out = *in - if in.Filter != nil { - in, out := &in.Filter, &out.Filter - if *in == nil { - *out = nil - } else { - *out = new(FilterSelector) - (*in).DeepCopyInto(*out) - } - } if in.Subscriber != nil { in, out := &in.Subscriber, &out.Subscriber if *in == nil { diff --git a/pkg/broker/receiver.go b/pkg/broker/receiver.go index 78002d23b36..60e2ae3d6ac 100644 --- a/pkg/broker/receiver.go +++ b/pkg/broker/receiver.go @@ -23,7 +23,6 @@ import ( eventingv1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" "github.com/knative/eventing/pkg/provisioners" "go.uber.org/zap" - "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" @@ -99,12 +98,14 @@ func (r *Receiver) getTrigger(ctx context.Context, ref provisioners.ChannelRefer } func (r *Receiver) shouldSendMessage(t *eventingv1alpha1.TriggerSpec, m *provisioners.Message) bool { - // This conversion to selector should be done only once, possibly upon creation of the trigger - // in case the filters are immutable. All validations should be performed then. - selector := labels.SelectorFromValidatedSet(labels.Set(t.Filter.Headers)) - matched := selector.Matches(labels.Set(m.Headers)) - if !matched { - r.logger.Debug("Selector did not match message headers", zap.String("selector", selector.String()), zap.Any("headers", m.Headers)) + // TODO More filtering! + if t.Type != Any && t.Type != m.Headers["Ce-Eventtype"] { + r.logger.Debug("Wrong type", zap.String("trigger.spec.type", t.Type), zap.String("message.type", m.Headers["Ce-Eventtype"]), zap.Any("m", m)) + return false } - return matched + if t.Source != "" && t.Source != m.Headers["Ce-Source"] { + r.logger.Debug("Wrong source", zap.String("trigger.spec.source", t.Source), zap.String("message.source", m.Headers["Ce-Source"])) + return false + } + return true } diff --git a/t2.yaml b/t2.yaml index 02b2f7ba64d..0e217789a03 100644 --- a/t2.yaml +++ b/t2.yaml @@ -5,12 +5,10 @@ metadata: spec: # Our Cloud Event parsing library seems to have a bug and forces to put type and source in double # quotes (as it thinks the actual value is `"foo"`, not `foo`). - filter: - headers: - Ce-Eventtype: '"com.example.someevent"' - Ce-Source: '"/mycontext/subcontext"' + type: '"com.example.someevent"' subscriber: ref: apiVersion: serving.knative.dev/v1alpha1 kind: Service name: message-dumper + From 2df6bdc6502aedf2310afc73e15f94e713ef2c8a Mon Sep 17 00:00:00 2001 From: Nacho Cano Date: Tue, 12 Feb 2019 09:56:26 -0800 Subject: [PATCH 040/128] Updating headers and headerExpressions to attributes and attributeExpressions --- pkg/apis/eventing/v1alpha1/trigger_types.go | 5 +++-- pkg/apis/eventing/v1alpha1/zz_generated.deepcopy.go | 8 ++++---- pkg/broker/receiver.go | 12 ++++++------ t.yaml | 2 +- t2.yaml | 2 +- t3.yaml | 4 ++-- 6 files changed, 17 insertions(+), 16 deletions(-) diff --git a/pkg/apis/eventing/v1alpha1/trigger_types.go b/pkg/apis/eventing/v1alpha1/trigger_types.go index 11b86e52b23..75131366d6d 100644 --- a/pkg/apis/eventing/v1alpha1/trigger_types.go +++ b/pkg/apis/eventing/v1alpha1/trigger_types.go @@ -67,8 +67,9 @@ type TriggerSpec struct { } type FilterSelector struct { - Headers map[string]string `json:"headers,omitempty" protobuf:"bytes,1,rep,name=headers"` - HeaderExpressions []metav1.LabelSelectorRequirement `json:"headerExpressions,omitempty" protobuf:"bytes,2,rep,name=headerExpressions"` + Attributes map[string]string `json:"attributes,omitempty" protobuf:"bytes,1,rep,name=attributes"` + // TODO create our own FilterSelectorRequirement + AttributeExpressions []metav1.LabelSelectorRequirement `json:"attributeExpressions,omitempty" protobuf:"bytes,2,rep,name=attributeExpressions"` } var triggerCondSet = duckv1alpha1.NewLivingConditionSet(TriggerConditionBrokerExists, TriggerConditionKubernetesService, TriggerConditionVirtualService, TriggerConditionSubscribed) diff --git a/pkg/apis/eventing/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/eventing/v1alpha1/zz_generated.deepcopy.go index 2e9b4cc6e90..1672ffe6dc9 100644 --- a/pkg/apis/eventing/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/eventing/v1alpha1/zz_generated.deepcopy.go @@ -378,15 +378,15 @@ func (in *ClusterChannelProvisionerStatus) DeepCopy() *ClusterChannelProvisioner // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *FilterSelector) DeepCopyInto(out *FilterSelector) { *out = *in - if in.Headers != nil { - in, out := &in.Headers, &out.Headers + if in.Attributes != nil { + in, out := &in.Attributes, &out.Attributes *out = make(map[string]string, len(*in)) for key, val := range *in { (*out)[key] = val } } - if in.HeaderExpressions != nil { - in, out := &in.HeaderExpressions, &out.HeaderExpressions + if in.AttributeExpressions != nil { + in, out := &in.AttributeExpressions, &out.AttributeExpressions *out = make([]meta_v1.LabelSelectorRequirement, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) diff --git a/pkg/broker/receiver.go b/pkg/broker/receiver.go index eaeb633525f..ea009b05f6b 100644 --- a/pkg/broker/receiver.go +++ b/pkg/broker/receiver.go @@ -35,7 +35,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" ) -// Receiver parses Cloud Events and sends them to GCP PubSub. +// Receiver parses Cloud Events and sends them to the channel. type Receiver struct { logger *zap.Logger client client.Client @@ -55,11 +55,11 @@ func New(logger *zap.Logger, client client.Client) (*Receiver, manager.Runnable) } func (r *Receiver) newMessageReceiver() *provisioners.MessageReceiver { - return provisioners.NewMessageReceiver(r.sendEventToTopic, r.logger.Sugar()) + return provisioners.NewMessageReceiver(r.sendEvent, r.logger.Sugar()) } -// sendEventToTopic sends a message to the Cloud Pub/Sub Topic backing the Channel. -func (r *Receiver) sendEventToTopic(channel provisioners.ChannelReference, message *provisioners.Message) error { +// sendEvent sends a message to the Channel. +func (r *Receiver) sendEvent(channel provisioners.ChannelReference, message *provisioners.Message) error { r.logger.Debug("received message") ctx := context.Background() @@ -118,8 +118,8 @@ func (r *Receiver) shouldSendMessage(t *eventingv1alpha1.TriggerSpec, m *provisi func (r *Receiver) buildSelector(ts *eventingv1alpha1.TriggerSpec) (labels.Selector, error) { // Avoid validation of keys and values, otherwise we cannot use LabelSelectors. // Eventually, we will need to create our own Selector implementation with our own Requirement struct. - selector := labels.SelectorFromValidatedSet(labels.Set(ts.Filter.Headers)) - for _, expr := range ts.Filter.HeaderExpressions { + selector := labels.SelectorFromValidatedSet(labels.Set(ts.Filter.Attributes)) + for _, expr := range ts.Filter.AttributeExpressions { var op selection.Operator switch expr.Operator { case v1.LabelSelectorOpIn: diff --git a/t.yaml b/t.yaml index a53dcc1db33..d9427cc9552 100644 --- a/t.yaml +++ b/t.yaml @@ -3,7 +3,7 @@ kind: Trigger metadata: name: t spec: - type: Any + filter: {} subscriber: ref: apiVersion: serving.knative.dev/v1alpha1 diff --git a/t2.yaml b/t2.yaml index 02b2f7ba64d..c95018d4bfb 100644 --- a/t2.yaml +++ b/t2.yaml @@ -6,7 +6,7 @@ spec: # Our Cloud Event parsing library seems to have a bug and forces to put type and source in double # quotes (as it thinks the actual value is `"foo"`, not `foo`). filter: - headers: + attributes: Ce-Eventtype: '"com.example.someevent"' Ce-Source: '"/mycontext/subcontext"' subscriber: diff --git a/t3.yaml b/t3.yaml index 0ed3e969020..1a90d86926b 100644 --- a/t3.yaml +++ b/t3.yaml @@ -6,9 +6,9 @@ spec: # Our Cloud Event parsing library seems to have a bug and forces to put type and source in double # quotes (as it thinks the actual value is `"foo"`, not `foo`). filter: - headers: + attributes: Ce-Source: '"/mycontext/subcontext"' - headerExpressions: + attributeExpressions: - {key: Ce-Eventtype, operator: In, values: ['"com.example.someevent"', '"com.example.someevent1"']} subscriber: ref: From 22efbe0220b4fcced4e263e1d8e602c18c62b71f Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Tue, 12 Feb 2019 16:05:36 -0800 Subject: [PATCH 041/128] Reconcilers notice when mroe things change. --- cmd/controller/main.go | 2 +- pkg/provisioners/namespace/provider.go | 106 ------------------ pkg/reconciler/v1alpha1/broker/broker.go | 19 ++-- .../v1alpha1/broker/resources/filter.go | 4 +- .../v1alpha1/namespace/namespace.go} | 94 +++++++++++++++- pkg/reconciler/v1alpha1/trigger/trigger.go | 15 ++- 6 files changed, 117 insertions(+), 123 deletions(-) delete mode 100644 pkg/provisioners/namespace/provider.go rename pkg/{provisioners/namespace/reconcile.go => reconciler/v1alpha1/namespace/namespace.go} (60%) diff --git a/cmd/controller/main.go b/cmd/controller/main.go index 33319ad6169..c4975b4314e 100644 --- a/cmd/controller/main.go +++ b/cmd/controller/main.go @@ -24,7 +24,7 @@ import ( "os" "time" - "github.com/knative/eventing/pkg/provisioners/namespace" + "github.com/knative/eventing/pkg/reconciler/v1alpha1/namespace" "github.com/knative/eventing/pkg/reconciler/v1alpha1/broker" "github.com/knative/eventing/pkg/reconciler/v1alpha1/subscription" "github.com/knative/eventing/pkg/reconciler/v1alpha1/trigger" diff --git a/pkg/provisioners/namespace/provider.go b/pkg/provisioners/namespace/provider.go deleted file mode 100644 index 2107c50fb5d..00000000000 --- a/pkg/provisioners/namespace/provider.go +++ /dev/null @@ -1,106 +0,0 @@ -/* -Copyright 2018 The Knative Authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package namespace - -import ( - "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" - "go.uber.org/zap" - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/dynamic" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/record" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/handler" - "sigs.k8s.io/controller-runtime/pkg/manager" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/controller-runtime/pkg/source" -) - -const ( - // controllerAgentName is the string used by this controller to identify - // itself when creating events. - controllerAgentName = "knative-eventing-namespace-controller" -) - -type reconciler struct { - client client.Client - restConfig *rest.Config - dynamicClient dynamic.Interface - recorder record.EventRecorder - - logger *zap.Logger -} - -// Verify the struct implements reconcile.Reconciler -var _ reconcile.Reconciler = &reconciler{} - -// ProvideController returns a function that returns a Broker controller. -func ProvideController(logger *zap.Logger) func(manager.Manager) (controller.Controller, error) { - return func(mgr manager.Manager) (controller.Controller, error) { - // Setup a new controller to Reconcile Brokers. - r := &reconciler{ - recorder: mgr.GetRecorder(controllerAgentName), - logger: logger, - } - c, err := controller.New(controllerAgentName, mgr, controller.Options{ - Reconciler: r, - }) - if err != nil { - return nil, err - } - - // Watch Subscription events and enqueue Subscription object key. - if err = c.Watch(&source.Kind{Type: &v1.Namespace{}}, &handler.EnqueueRequestForObject{}); err != nil { - return nil, err - } - - if err = c.Watch(&source.Kind{Type: &v1alpha1.Broker{}}, &handler.EnqueueRequestsFromMapFunc{ToRequests: &namespaceMapper{}}); err != nil { - return nil, err - } - - return c, nil - } -} - -type namespaceMapper struct{} - -var _ handler.Mapper = &namespaceMapper{} - -func (namespaceMapper) Map(o handler.MapObject) []reconcile.Request { - return []reconcile.Request{ - { - NamespacedName: types.NamespacedName{ - Namespace: "", - Name: o.Meta.GetNamespace(), - }, - }, - } -} - -func (r *reconciler) InjectClient(c client.Client) error { - r.client = c - return nil -} - -func (r *reconciler) InjectConfig(c *rest.Config) error { - r.restConfig = c - var err error - r.dynamicClient, err = dynamic.NewForConfig(c) - return err -} diff --git a/pkg/reconciler/v1alpha1/broker/broker.go b/pkg/reconciler/v1alpha1/broker/broker.go index 21961fb06f3..8a79adfd902 100644 --- a/pkg/reconciler/v1alpha1/broker/broker.go +++ b/pkg/reconciler/v1alpha1/broker/broker.go @@ -19,6 +19,7 @@ package broker import ( "context" "fmt" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/dynamic" "k8s.io/client-go/rest" @@ -99,16 +100,17 @@ func ProvideController(logger *zap.Logger, ingressImage, ingressServiceAccount, return nil, err } - // Watch Subscription events and enqueue Subscription object key. + // Watch Brokers. if err = c.Watch(&source.Kind{Type: &v1alpha1.Broker{}}, &handler.EnqueueRequestForObject{}); err != nil { return nil, err } - err = c.Watch(&source.Kind{ - Type: &v1alpha1.Channel{}, - }, &handler.EnqueueRequestForOwner{OwnerType: &v1alpha1.Broker{}, IsController: true}) - if err != nil { - return nil, err + // Watch all the resources that the Broker reconciles. + for _, t := range []runtime.Object{ &v1alpha1.Channel{}, &corev1.Service{}, &v1.Deployment{} } { + err = c.Watch(&source.Kind{Type: t}, &handler.EnqueueRequestForOwner{OwnerType: &v1alpha1.Broker{}, IsController: true}) + if err != nil { + return nil, err + } } return c, nil @@ -261,14 +263,11 @@ func (r *reconciler) updateStatus(broker *v1alpha1.Broker) (*v1alpha1.Broker, er } func (r *reconciler) reconcileFilterDeployment(ctx context.Context, b *v1alpha1.Broker) (*v1.Deployment, error) { - expected, err := resources.MakeFilterDeployment(&resources.FilterArgs{ + expected := resources.MakeFilterDeployment(&resources.FilterArgs{ Broker: b, Image: r.filterImage, ServiceAccountName: r.filterServiceAccountName, }) - if err != nil { - return nil, err - } return r.reconcileDeployment(ctx, expected) } diff --git a/pkg/reconciler/v1alpha1/broker/resources/filter.go b/pkg/reconciler/v1alpha1/broker/resources/filter.go index 4de9be6d0df..38d049f068e 100644 --- a/pkg/reconciler/v1alpha1/broker/resources/filter.go +++ b/pkg/reconciler/v1alpha1/broker/resources/filter.go @@ -34,7 +34,7 @@ type FilterArgs struct { ServiceAccountName string } -func MakeFilterDeployment(args *FilterArgs) (*appsv1.Deployment, error) { +func MakeFilterDeployment(args *FilterArgs) *appsv1.Deployment { return &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Namespace: args.Broker.Namespace, @@ -75,7 +75,7 @@ func MakeFilterDeployment(args *FilterArgs) (*appsv1.Deployment, error) { }, }, }, - }, nil + } } func MakeFilterService(b *eventingv1alpha1.Broker) *corev1.Service { diff --git a/pkg/provisioners/namespace/reconcile.go b/pkg/reconciler/v1alpha1/namespace/namespace.go similarity index 60% rename from pkg/provisioners/namespace/reconcile.go rename to pkg/reconciler/v1alpha1/namespace/namespace.go index cc2b2913010..8fad590ecce 100644 --- a/pkg/provisioners/namespace/reconcile.go +++ b/pkg/reconciler/v1alpha1/namespace/namespace.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Knative Authors +Copyright 2019 The Knative Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -18,19 +18,32 @@ package namespace import ( "context" - "github.com/knative/eventing/contrib/gcppubsub/pkg/util/logging" "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" "go.uber.org/zap" + "k8s.io/api/core/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" ) const ( + // controllerAgentName is the string used by this controller to identify + // itself when creating events. + controllerAgentName = "knative-eventing-namespace-controller" + defaultBroker = "default" knativeEventingAnnotation = "eventing.knative.dev/injection" @@ -38,6 +51,81 @@ const ( brokerCreated = "BrokerCreated" ) +type reconciler struct { + client client.Client + restConfig *rest.Config + dynamicClient dynamic.Interface + recorder record.EventRecorder + + logger *zap.Logger +} + +// Verify the struct implements reconcile.Reconciler +var _ reconcile.Reconciler = &reconciler{} + +// ProvideController returns a function that returns a Broker controller. +func ProvideController(logger *zap.Logger) func(manager.Manager) (controller.Controller, error) { + return func(mgr manager.Manager) (controller.Controller, error) { + // Setup a new controller to Reconcile Brokers. + r := &reconciler{ + recorder: mgr.GetRecorder(controllerAgentName), + logger: logger, + } + c, err := controller.New(controllerAgentName, mgr, controller.Options{ + Reconciler: r, + }) + if err != nil { + return nil, err + } + + // Watch Namespaces. + if err = c.Watch(&source.Kind{Type: &v1.Namespace{}}, &handler.EnqueueRequestForObject{}); err != nil { + return nil, err + } + + // Watch all the resources that this reconciler reconciles. + if err = c.Watch(&source.Kind{Type: &v1alpha1.Broker{}}, &handler.EnqueueRequestsFromMapFunc{ToRequests: &namespaceMapper{}}); err != nil { + return nil, err + } + + for _, t := range []runtime.Object{ &v1alpha1.Broker{} } { + err = c.Watch(&source.Kind{Type: t}, &handler.EnqueueRequestsFromMapFunc{ToRequests: &namespaceMapper{}}); + if err != nil { + return nil, err + } + } + + return c, nil + } +} + +type namespaceMapper struct{} + +var _ handler.Mapper = &namespaceMapper{} + +func (namespaceMapper) Map(o handler.MapObject) []reconcile.Request { + return []reconcile.Request{ + { + NamespacedName: types.NamespacedName{ + Namespace: "", + Name: o.Meta.GetNamespace(), + }, + }, + } +} + +func (r *reconciler) InjectClient(c client.Client) error { + r.client = c + return nil +} + +func (r *reconciler) InjectConfig(c *rest.Config) error { + r.restConfig = c + var err error + r.dynamicClient, err = dynamic.NewForConfig(c) + return err +} + // Reconcile compares the actual state with the desired, and attempts to // converge the two. It then updates the Status block of the Trigger resource // with the current status of the resource. @@ -103,7 +191,7 @@ func (r *reconciler) getBroker(ctx context.Context, ns *corev1.Namespace) (*v1al func (r *reconciler) reconcileBroker(ctx context.Context, ns *corev1.Namespace) (*v1alpha1.Broker, error) { current, err := r.getBroker(ctx, ns) - // If the resource doesn't exist, we'll create it + // If the resource doesn't exist, we'll create it. if k8serrors.IsNotFound(err) { b := newBroker(ns) err = r.client.Create(ctx, b) diff --git a/pkg/reconciler/v1alpha1/trigger/trigger.go b/pkg/reconciler/v1alpha1/trigger/trigger.go index d7ce3cf2c37..8c2d57b26d9 100644 --- a/pkg/reconciler/v1alpha1/trigger/trigger.go +++ b/pkg/reconciler/v1alpha1/trigger/trigger.go @@ -19,6 +19,7 @@ package trigger import ( "context" "fmt" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/dynamic" "k8s.io/client-go/rest" @@ -87,11 +88,23 @@ func ProvideController(logger *zap.Logger) func(manager.Manager) (controller.Con return nil, err } - // Watch Subscription events and enqueue Subscription object key. + // Watch Triggers. if err = c.Watch(&source.Kind{Type: &v1alpha1.Trigger{}}, &handler.EnqueueRequestForObject{}); err != nil { return nil, err } + // Watch all the resources that the Trigger reconciles. + for _, t := range []runtime.Object{ &corev1.Service{}, &istiov1alpha3.VirtualService{}, &v1alpha1.Subscription{} } { + err = c.Watch(&source.Kind{Type: t}, &handler.EnqueueRequestForOwner{OwnerType: &v1alpha1.Broker{}, IsController: true}) + if err != nil { + return nil, err + } + } + + // TODO reconcile after a Broker change. This might require keeping a map from Broker -> []Trigger. + // TODO reconcile after a change to the subscriber. I'm not sure how this is possible, but we should do it if we + // can find a way. + return c, nil } } From 3a3c13c853f6c572fb78409c95a97c835a2ac21c Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Tue, 12 Feb 2019 16:25:11 -0800 Subject: [PATCH 042/128] PR comment. --- pkg/reconciler/v1alpha1/namespace/namespace.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/reconciler/v1alpha1/namespace/namespace.go b/pkg/reconciler/v1alpha1/namespace/namespace.go index 8fad590ecce..078a3590a2d 100644 --- a/pkg/reconciler/v1alpha1/namespace/namespace.go +++ b/pkg/reconciler/v1alpha1/namespace/namespace.go @@ -47,7 +47,7 @@ const ( defaultBroker = "default" knativeEventingAnnotation = "eventing.knative.dev/injection" - // Name of the corev1.Events emitted from the reconciliation process + // Name of the corev1.Events emitted from the reconciliation process. brokerCreated = "BrokerCreated" ) From f834f8927622ba3784381977683048d8667bb04a Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Tue, 12 Feb 2019 16:50:12 -0800 Subject: [PATCH 043/128] Remove redundant watch. --- pkg/reconciler/v1alpha1/namespace/namespace.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pkg/reconciler/v1alpha1/namespace/namespace.go b/pkg/reconciler/v1alpha1/namespace/namespace.go index 078a3590a2d..6862a1a8c5c 100644 --- a/pkg/reconciler/v1alpha1/namespace/namespace.go +++ b/pkg/reconciler/v1alpha1/namespace/namespace.go @@ -84,10 +84,6 @@ func ProvideController(logger *zap.Logger) func(manager.Manager) (controller.Con } // Watch all the resources that this reconciler reconciles. - if err = c.Watch(&source.Kind{Type: &v1alpha1.Broker{}}, &handler.EnqueueRequestsFromMapFunc{ToRequests: &namespaceMapper{}}); err != nil { - return nil, err - } - for _, t := range []runtime.Object{ &v1alpha1.Broker{} } { err = c.Watch(&source.Kind{Type: t}, &handler.EnqueueRequestsFromMapFunc{ToRequests: &namespaceMapper{}}); if err != nil { From 6b817ef8db4894a837aab7b2d27da13ddb0f804c Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Tue, 12 Feb 2019 17:10:39 -0800 Subject: [PATCH 044/128] Unit test scaffold. --- pkg/broker/receiver_test.go | 25 +---- pkg/reconciler/v1alpha1/broker/broker_test.go | 91 +++++++++++++++++++ .../v1alpha1/namespace/namespace_test.go | 91 +++++++++++++++++++ .../v1alpha1/trigger/trigger_test.go | 91 +++++++++++++++++++ 4 files changed, 275 insertions(+), 23 deletions(-) create mode 100644 pkg/reconciler/v1alpha1/broker/broker_test.go create mode 100644 pkg/reconciler/v1alpha1/namespace/namespace_test.go create mode 100644 pkg/reconciler/v1alpha1/trigger/trigger_test.go diff --git a/pkg/broker/receiver_test.go b/pkg/broker/receiver_test.go index 2dead920105..aa553399351 100644 --- a/pkg/broker/receiver_test.go +++ b/pkg/broker/receiver_test.go @@ -18,7 +18,6 @@ package broker import ( "context" - "errors" "net/http/httptest" "strings" "testing" @@ -28,10 +27,9 @@ import ( eventingv1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" "k8s.io/client-go/kubernetes/scheme" - "github.com/knative/eventing/contrib/gcppubsub/pkg/util/fakepubsub" "go.uber.org/zap" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -63,7 +61,6 @@ func init() { func TestReceiver(t *testing.T) { testCases := map[string]struct { initialState []runtime.Object - pubSubData fakepubsub.CreatorData expectedErr bool }{ "can't get channel": { @@ -98,9 +95,6 @@ func TestReceiver(t *testing.T) { testcreds.MakeSecretWithCreds(), makeChannel(), }, - pubSubData: fakepubsub.CreatorData{ - ClientCreateErr: errors.New("testInducedError"), - }, expectedErr: true, }, "Publish fails": { @@ -108,23 +102,8 @@ func TestReceiver(t *testing.T) { testcreds.MakeSecretWithCreds(), makeChannel(), }, - pubSubData: fakepubsub.CreatorData{ - ClientData: fakepubsub.ClientData{ - TopicData: fakepubsub.TopicData{ - Publish: fakepubsub.PublishResultData{ - Err: errors.New("testInducedError"), - }, - }, - }, - }, expectedErr: true, }, - "Publish succeeds": { - initialState: []runtime.Object{ - testcreds.MakeSecretWithCreds(), - makeChannel(), - }, - }, } for n, tc := range testCases { t.Run(n, func(t *testing.T) { @@ -133,7 +112,7 @@ func TestReceiver(t *testing.T) { fake.NewFakeClient(tc.initialState...)) resp := httptest.NewRecorder() req := httptest.NewRequest("POST", "/", strings.NewReader(validMessage)) - req.Host = "test-channel.test-namespace.channels.cluster.local" + req.Host = "test-trigger.test-namespace.triggers.cluster.local" mr.newMessageReceiver().HandleRequest(resp, req) if tc.expectedErr { if resp.Result().StatusCode >= 200 && resp.Result().StatusCode < 300 { diff --git a/pkg/reconciler/v1alpha1/broker/broker_test.go b/pkg/reconciler/v1alpha1/broker/broker_test.go new file mode 100644 index 00000000000..7a887c44284 --- /dev/null +++ b/pkg/reconciler/v1alpha1/broker/broker_test.go @@ -0,0 +1,91 @@ +/* +Copyright 2019 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package broker + +import ( + "github.com/google/go-cmp/cmp" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "testing" +) + +func TestProvideController(t *testing.T) { + //TODO(grantr) This needs a mock of manager.Manager. Creating a manager + // with a fake Config fails because the Manager tries to contact the + // apiserver. + + // cfg := &rest.Config{ + // Host: "http://foo:80", + // } + // + // mgr, err := manager.New(cfg, manager.Options{}) + // if err != nil { + // t.Fatalf("Error creating manager: %v", err) + // } + // + // _, err = ProvideController(mgr) + // if err != nil { + // t.Fatalf("Error in ProvideController: %v", err) + // } +} + +func TestInjectClient(t *testing.T) { + r := &reconciler{} + orig := r.client + n := fake.NewFakeClient() + if orig == n { + t.Errorf("Original and new clients are identical: %v", orig) + } + err := r.InjectClient(n) + if err != nil { + t.Errorf("Unexpected error injecting the client: %v", err) + } + if n != r.client { + t.Errorf("Unexpected client. Expected: '%v'. Actual: '%v'", n, r.client) + } +} + +func TestInjectConfig(t *testing.T) { + r := &reconciler{} + wantCfg := &rest.Config{ + Host: "http://foo", + } + + err := r.InjectConfig(wantCfg) + if err != nil { + t.Fatalf("Unexpected error injecting the config: %v", err) + } + + gotCfg := r.restConfig + if diff := cmp.Diff(wantCfg, gotCfg); diff != "" { + t.Errorf("Unexpected config (-want, +got): %v", diff) + } + + wantDynClient, err := dynamic.NewForConfig(wantCfg) + if err != nil { + t.Fatalf("Unexpected error generating dynamic client: %v", err) + } + + // Since dynamicClient doesn't export any fields, we can only test its type. + switch r.dynamicClient.(type) { + case dynamic.Interface: + // ok + default: + t.Errorf("Unexpected dynamicClient type. Expected: %T, Got: %T", wantDynClient, r.dynamicClient) + } +} diff --git a/pkg/reconciler/v1alpha1/namespace/namespace_test.go b/pkg/reconciler/v1alpha1/namespace/namespace_test.go new file mode 100644 index 00000000000..29885f24d97 --- /dev/null +++ b/pkg/reconciler/v1alpha1/namespace/namespace_test.go @@ -0,0 +1,91 @@ +/* +Copyright 2019 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package namespace + +import ( + "github.com/google/go-cmp/cmp" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "testing" +) + +func TestProvideController(t *testing.T) { + //TODO(grantr) This needs a mock of manager.Manager. Creating a manager + // with a fake Config fails because the Manager tries to contact the + // apiserver. + + // cfg := &rest.Config{ + // Host: "http://foo:80", + // } + // + // mgr, err := manager.New(cfg, manager.Options{}) + // if err != nil { + // t.Fatalf("Error creating manager: %v", err) + // } + // + // _, err = ProvideController(mgr) + // if err != nil { + // t.Fatalf("Error in ProvideController: %v", err) + // } +} + +func TestInjectClient(t *testing.T) { + r := &reconciler{} + orig := r.client + n := fake.NewFakeClient() + if orig == n { + t.Errorf("Original and new clients are identical: %v", orig) + } + err := r.InjectClient(n) + if err != nil { + t.Errorf("Unexpected error injecting the client: %v", err) + } + if n != r.client { + t.Errorf("Unexpected client. Expected: '%v'. Actual: '%v'", n, r.client) + } +} + +func TestInjectConfig(t *testing.T) { + r := &reconciler{} + wantCfg := &rest.Config{ + Host: "http://foo", + } + + err := r.InjectConfig(wantCfg) + if err != nil { + t.Fatalf("Unexpected error injecting the config: %v", err) + } + + gotCfg := r.restConfig + if diff := cmp.Diff(wantCfg, gotCfg); diff != "" { + t.Errorf("Unexpected config (-want, +got): %v", diff) + } + + wantDynClient, err := dynamic.NewForConfig(wantCfg) + if err != nil { + t.Fatalf("Unexpected error generating dynamic client: %v", err) + } + + // Since dynamicClient doesn't export any fields, we can only test its type. + switch r.dynamicClient.(type) { + case dynamic.Interface: + // ok + default: + t.Errorf("Unexpected dynamicClient type. Expected: %T, Got: %T", wantDynClient, r.dynamicClient) + } +} diff --git a/pkg/reconciler/v1alpha1/trigger/trigger_test.go b/pkg/reconciler/v1alpha1/trigger/trigger_test.go new file mode 100644 index 00000000000..298a3a48f5c --- /dev/null +++ b/pkg/reconciler/v1alpha1/trigger/trigger_test.go @@ -0,0 +1,91 @@ +/* +Copyright 2019 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package trigger + +import ( + "github.com/google/go-cmp/cmp" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "testing" +) + +func TestProvideController(t *testing.T) { + //TODO(grantr) This needs a mock of manager.Manager. Creating a manager + // with a fake Config fails because the Manager tries to contact the + // apiserver. + + // cfg := &rest.Config{ + // Host: "http://foo:80", + // } + // + // mgr, err := manager.New(cfg, manager.Options{}) + // if err != nil { + // t.Fatalf("Error creating manager: %v", err) + // } + // + // _, err = ProvideController(mgr) + // if err != nil { + // t.Fatalf("Error in ProvideController: %v", err) + // } +} + +func TestInjectClient(t *testing.T) { + r := &reconciler{} + orig := r.client + n := fake.NewFakeClient() + if orig == n { + t.Errorf("Original and new clients are identical: %v", orig) + } + err := r.InjectClient(n) + if err != nil { + t.Errorf("Unexpected error injecting the client: %v", err) + } + if n != r.client { + t.Errorf("Unexpected client. Expected: '%v'. Actual: '%v'", n, r.client) + } +} + +func TestInjectConfig(t *testing.T) { + r := &reconciler{} + wantCfg := &rest.Config{ + Host: "http://foo", + } + + err := r.InjectConfig(wantCfg) + if err != nil { + t.Fatalf("Unexpected error injecting the config: %v", err) + } + + gotCfg := r.restConfig + if diff := cmp.Diff(wantCfg, gotCfg); diff != "" { + t.Errorf("Unexpected config (-want, +got): %v", diff) + } + + wantDynClient, err := dynamic.NewForConfig(wantCfg) + if err != nil { + t.Fatalf("Unexpected error generating dynamic client: %v", err) + } + + // Since dynamicClient doesn't export any fields, we can only test its type. + switch r.dynamicClient.(type) { + case dynamic.Interface: + // ok + default: + t.Errorf("Unexpected dynamicClient type. Expected: %T, Got: %T", wantDynClient, r.dynamicClient) + } +} From 97e642d7dc7b7fca81a24610a973c4c3ebe052b3 Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Tue, 12 Feb 2019 22:19:29 -0800 Subject: [PATCH 045/128] Tests for the namespace reconciler. --- .../v1alpha1/namespace/namespace.go | 2 +- .../v1alpha1/namespace/namespace_test.go | 225 ++++++++++++++++++ 2 files changed, 226 insertions(+), 1 deletion(-) diff --git a/pkg/reconciler/v1alpha1/namespace/namespace.go b/pkg/reconciler/v1alpha1/namespace/namespace.go index 6862a1a8c5c..1c2c31ca618 100644 --- a/pkg/reconciler/v1alpha1/namespace/namespace.go +++ b/pkg/reconciler/v1alpha1/namespace/namespace.go @@ -45,7 +45,7 @@ const ( controllerAgentName = "knative-eventing-namespace-controller" defaultBroker = "default" - knativeEventingAnnotation = "eventing.knative.dev/injection" + knativeEventingAnnotation = "eventing.knative.dev/inject" // Name of the corev1.Events emitted from the reconciliation process. brokerCreated = "BrokerCreated" diff --git a/pkg/reconciler/v1alpha1/namespace/namespace_test.go b/pkg/reconciler/v1alpha1/namespace/namespace_test.go index 29885f24d97..cdf1876ac29 100644 --- a/pkg/reconciler/v1alpha1/namespace/namespace_test.go +++ b/pkg/reconciler/v1alpha1/namespace/namespace_test.go @@ -17,13 +17,46 @@ limitations under the License. package namespace import ( + "context" + "errors" + "fmt" "github.com/google/go-cmp/cmp" + "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" + controllertesting "github.com/knative/eventing/pkg/reconciler/testing" + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" "testing" ) +const ( + testNS = "test-namespace" + brokerName = "default" +) + +var ( + falseString = "false" + trueString = "true" + + // deletionTime is used when objects are marked as deleted. Rfc3339Copy() + // truncates to seconds to match the loss of precision during serialization. + deletionTime = metav1.Now().Rfc3339Copy() +) + +func init() { + // Add types to scheme + v1alpha1.AddToScheme(scheme.Scheme) +} + func TestProvideController(t *testing.T) { //TODO(grantr) This needs a mock of manager.Manager. Creating a manager // with a fake Config fails because the Manager tries to contact the @@ -89,3 +122,195 @@ func TestInjectConfig(t *testing.T) { t.Errorf("Unexpected dynamicClient type. Expected: %T, Got: %T", wantDynClient, r.dynamicClient) } } + +func TestNamespaceMapper_Map(t *testing.T) { + m := &namespaceMapper{} + + req := handler.MapObject{ + Meta: makeBroker().GetObjectMeta(), + Object: makeBroker(), + } + actual := m.Map(req) + expected := []reconcile.Request{ + { + NamespacedName: types.NamespacedName{ + Namespace: "", + Name: testNS, + }, + }, + } + if diff := cmp.Diff(expected, actual); diff != "" { + t.Errorf("Unexpected reconcile requests (-want +got): %v", diff) + } +} + +func TestReconcile(t *testing.T) { + testCases := []controllertesting.TestCase{ + { + Name: "Namespace not found", + }, + { + Name: "Namespace get fails", + Scheme: scheme.Scheme, + Mocks: controllertesting.Mocks{ + MockGets: []controllertesting.MockGet{ + func(_ client.Client, _ context.Context, _ client.ObjectKey, obj runtime.Object) (controllertesting.MockHandled, error) { + if _, ok := obj.(*corev1.Namespace); ok { + return controllertesting.Handled, errors.New("test error getting the NS") + } + return controllertesting.Unhandled, nil + }, + }, + }, + WantErrMsg: "test error getting the NS", + }, + { + Name: "Namespace is not annotated", + Scheme: scheme.Scheme, + InitialState: []runtime.Object{ + makeNamespace(nil), + }, + WantAbsent: []runtime.Object{ + makeBroker(), + }, + }, + { + Name: "Namespace is annotated off", + Scheme: scheme.Scheme, + InitialState: []runtime.Object{ + makeNamespace(&falseString), + }, + WantAbsent: []runtime.Object{ + makeBroker(), + }, + }, + { + Name: "Namespace is being deleted", + Scheme: scheme.Scheme, + InitialState: []runtime.Object{ + makeDeletingNamespace(), + }, + WantAbsent: []runtime.Object{ + makeBroker(), + }, + }, + { + Name: "Broker.Get fails", + Scheme: scheme.Scheme, + InitialState: []runtime.Object{ + makeNamespace(&trueString), + }, + Mocks: controllertesting.Mocks{ + MockGets: []controllertesting.MockGet{ + func(_ client.Client, _ context.Context, _ client.ObjectKey, obj runtime.Object) (controllertesting.MockHandled, error) { + if _, ok := obj.(*v1alpha1.Broker); ok { + return controllertesting.Handled, errors.New("test error getting the Broker") + } + return controllertesting.Unhandled, nil + }, + }, + }, + WantErrMsg: "test error getting the Broker", + WantAbsent: []runtime.Object{ + makeBroker(), + }, + }, + { + Name: "Broker Found", + Scheme: scheme.Scheme, + InitialState: []runtime.Object{ + makeNamespace(&trueString), + makeBroker(), + }, + }, + { + Name: "Broker.Create fails", + Scheme: scheme.Scheme, + InitialState: []runtime.Object{ + makeNamespace(&trueString), + }, + Mocks: controllertesting.Mocks{ + MockCreates: []controllertesting.MockCreate{ + func(_ client.Client, _ context.Context, obj runtime.Object) (controllertesting.MockHandled, error) { + if _, ok := obj.(*v1alpha1.Broker); ok { + return controllertesting.Handled, errors.New("test error creating the Broker") + } + return controllertesting.Unhandled, nil + }, + }, + }, + WantErrMsg: "test error creating the Broker", + }, + { + Name: "Broker created", + Scheme: scheme.Scheme, + InitialState: []runtime.Object{ + makeNamespace(&trueString), + }, + WantPresent: []runtime.Object{ + makeBroker(), + }, + WantEvent: []corev1.Event{ + { + Reason: brokerCreated, Type: corev1.EventTypeNormal, + }, + }, + }, + } + for _, tc := range testCases { + c := tc.GetClient() + dc := tc.GetDynamicClient() + recorder := tc.GetEventRecorder() + + r := &reconciler{ + client: c, + dynamicClient: dc, + restConfig: &rest.Config{}, + recorder: recorder, + logger: zap.NewNop(), + } + tc.ReconcileKey = fmt.Sprintf("%s/%s", "", testNS) + tc.IgnoreTimes = true + t.Run(tc.Name, tc.Runner(t, r, c, recorder)) + } +} + +func makeNamespace(annotationValue *string) *corev1.Namespace { + annotations := map[string]string{} + if annotationValue != nil { + annotations["eventing.knative.dev/inject"] = *annotationValue + } + + return &corev1.Namespace{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Namespace", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: testNS, + Annotations: annotations, + }, + } +} + +func makeDeletingNamespace() *corev1.Namespace { + ns := makeNamespace(&trueString) + ns.DeletionTimestamp = &deletionTime + return ns +} + +func makeBroker() *v1alpha1.Broker { + return &v1alpha1.Broker{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "eventing.knative.dev/v1alpha1", + Kind: "Broker", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: testNS, + Name: brokerName, + Labels: map[string]string{ + "eventing.knative.dev/brokerForNamespace": "true", + }, + }, + } +} From da32baeecd280940dffdacd931ea6952cb8d1e49 Mon Sep 17 00:00:00 2001 From: Nacho Cano Date: Tue, 12 Feb 2019 23:28:23 -0800 Subject: [PATCH 046/128] Exact matching for filters without using k8s selectors-based syntax. The matching can only be done on source and type for now. --- Gopkg.lock | 1 - .../eventing/v1alpha1/trigger_defaults.go | 7 +- pkg/apis/eventing/v1alpha1/trigger_types.go | 13 ++-- .../eventing/v1alpha1/trigger_validation.go | 2 +- .../v1alpha1/zz_generated.deepcopy.go | 68 +++++++++--------- pkg/broker/receiver.go | 69 ++++--------------- t.yaml | 1 - t2.yaml | 6 +- t3.yaml | 6 +- 9 files changed, 64 insertions(+), 109 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 288846af8fd..5238ff27d50 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -1255,7 +1255,6 @@ "k8s.io/apimachinery/pkg/runtime", "k8s.io/apimachinery/pkg/runtime/schema", "k8s.io/apimachinery/pkg/runtime/serializer", - "k8s.io/apimachinery/pkg/selection", "k8s.io/apimachinery/pkg/types", "k8s.io/apimachinery/pkg/util/intstr", "k8s.io/apimachinery/pkg/util/sets", diff --git a/pkg/apis/eventing/v1alpha1/trigger_defaults.go b/pkg/apis/eventing/v1alpha1/trigger_defaults.go index d996d5ea30d..fe5e7bbf797 100644 --- a/pkg/apis/eventing/v1alpha1/trigger_defaults.go +++ b/pkg/apis/eventing/v1alpha1/trigger_defaults.go @@ -16,8 +16,6 @@ limitations under the License. package v1alpha1 -import v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - func (t *Trigger) SetDefaults() { t.Spec.SetDefaults() } @@ -26,9 +24,8 @@ func (ts *TriggerSpec) SetDefaults() { if ts.Broker == "" { ts.Broker = "default" } - // Make empty filter selector so that we allow everything. + // Make a default filter that allows anything. if ts.Filter == nil { - ts.Filter = &FilterSelector{map[string]string{}, - []v1.LabelSelectorRequirement{}} + ts.Filter = &TriggerFilter{TriggerFilterAttributes{Type: "", Source: ""}} } } diff --git a/pkg/apis/eventing/v1alpha1/trigger_types.go b/pkg/apis/eventing/v1alpha1/trigger_types.go index 75131366d6d..63be349e13d 100644 --- a/pkg/apis/eventing/v1alpha1/trigger_types.go +++ b/pkg/apis/eventing/v1alpha1/trigger_types.go @@ -61,15 +61,18 @@ type TriggerSpec struct { Broker string `json:"broker,omitempty"` // +optional - Filter *FilterSelector `json:"filter,omitempty"` + Filter *TriggerFilter `json:"filter,omitempty"` Subscriber *SubscriberSpec `json:"subscriber,omitempty"` } -type FilterSelector struct { - Attributes map[string]string `json:"attributes,omitempty" protobuf:"bytes,1,rep,name=attributes"` - // TODO create our own FilterSelectorRequirement - AttributeExpressions []metav1.LabelSelectorRequirement `json:"attributeExpressions,omitempty" protobuf:"bytes,2,rep,name=attributeExpressions"` +type TriggerFilter struct { + ExactMatch TriggerFilterAttributes `json:"exactMatch,omitempty"` +} + +type TriggerFilterAttributes struct { + Type string `json:"type,omitempty"` + Source string `json:"source,omitempty"` } var triggerCondSet = duckv1alpha1.NewLivingConditionSet(TriggerConditionBrokerExists, TriggerConditionKubernetesService, TriggerConditionVirtualService, TriggerConditionSubscribed) diff --git a/pkg/apis/eventing/v1alpha1/trigger_validation.go b/pkg/apis/eventing/v1alpha1/trigger_validation.go index 42057cf9bef..fdf3a9ff08b 100644 --- a/pkg/apis/eventing/v1alpha1/trigger_validation.go +++ b/pkg/apis/eventing/v1alpha1/trigger_validation.go @@ -33,7 +33,7 @@ func (ts *TriggerSpec) Validate() *apis.FieldError { } if ts.Filter == nil { - fe := apis.ErrMissingField("filters") + fe := apis.ErrMissingField("filter") errs = errs.Also(fe) } diff --git a/pkg/apis/eventing/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/eventing/v1alpha1/zz_generated.deepcopy.go index 1672ffe6dc9..88e1c446671 100644 --- a/pkg/apis/eventing/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/eventing/v1alpha1/zz_generated.deepcopy.go @@ -24,7 +24,6 @@ import ( apis_duck_v1alpha1 "github.com/knative/eventing/pkg/apis/duck/v1alpha1" duck_v1alpha1 "github.com/knative/pkg/apis/duck/v1alpha1" v1 "k8s.io/api/core/v1" - meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -375,36 +374,6 @@ func (in *ClusterChannelProvisionerStatus) DeepCopy() *ClusterChannelProvisioner return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *FilterSelector) DeepCopyInto(out *FilterSelector) { - *out = *in - if in.Attributes != nil { - in, out := &in.Attributes, &out.Attributes - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - if in.AttributeExpressions != nil { - in, out := &in.AttributeExpressions, &out.AttributeExpressions - *out = make([]meta_v1.LabelSelectorRequirement, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FilterSelector. -func (in *FilterSelector) DeepCopy() *FilterSelector { - if in == nil { - return nil - } - out := new(FilterSelector) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ReplyStrategy) DeepCopyInto(out *ReplyStrategy) { *out = *in @@ -628,6 +597,39 @@ func (in *Trigger) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TriggerFilter) DeepCopyInto(out *TriggerFilter) { + *out = *in + out.ExactMatch = in.ExactMatch + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TriggerFilter. +func (in *TriggerFilter) DeepCopy() *TriggerFilter { + if in == nil { + return nil + } + out := new(TriggerFilter) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TriggerFilterAttributes) DeepCopyInto(out *TriggerFilterAttributes) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TriggerFilterAttributes. +func (in *TriggerFilterAttributes) DeepCopy() *TriggerFilterAttributes { + if in == nil { + return nil + } + out := new(TriggerFilterAttributes) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TriggerList) DeepCopyInto(out *TriggerList) { *out = *in @@ -669,8 +671,8 @@ func (in *TriggerSpec) DeepCopyInto(out *TriggerSpec) { if *in == nil { *out = nil } else { - *out = new(FilterSelector) - (*in).DeepCopyInto(*out) + *out = new(TriggerFilter) + **out = **in } } if in.Subscriber != nil { diff --git a/pkg/broker/receiver.go b/pkg/broker/receiver.go index ea009b05f6b..4b4cfe89136 100644 --- a/pkg/broker/receiver.go +++ b/pkg/broker/receiver.go @@ -19,22 +19,19 @@ package broker import ( "context" "errors" - "fmt" - "reflect" - "unsafe" - - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/selection" eventingv1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" "github.com/knative/eventing/pkg/provisioners" "go.uber.org/zap" - "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" ) +const ( + Any = "" +) + // Receiver parses Cloud Events and sends them to the channel. type Receiver struct { logger *zap.Logger @@ -100,56 +97,16 @@ func (r *Receiver) getTrigger(ctx context.Context, ref provisioners.ChannelRefer return t, err } -func (r *Receiver) shouldSendMessage(t *eventingv1alpha1.TriggerSpec, m *provisioners.Message) bool { - // This conversion to selector should be done only once, possibly upon creation of the trigger - // in case the filters are immutable. All validations should be performed then. - selector, err := r.buildSelector(t) - if err != nil { - r.logger.Error("Invalid selector for trigger spec", zap.Any("triggerSpec", t)) +func (r *Receiver) shouldSendMessage(ts *eventingv1alpha1.TriggerSpec, m *provisioners.Message) bool { + filterType := ts.Filter.ExactMatch.Type + if filterType != Any && filterType != m.Headers["Ce-Eventtype"] { + r.logger.Debug("Wrong type", zap.String("trigger.spec.filter.exactMatch.type", filterType), zap.String("message.type", m.Headers["Ce-Eventtype"])) return false } - matched := selector.Matches(labels.Set(m.Headers)) - if !matched { - r.logger.Debug("Selector did not match message headers", zap.String("selector", selector.String()), zap.Any("headers", m.Headers)) - } - return matched -} - -func (r *Receiver) buildSelector(ts *eventingv1alpha1.TriggerSpec) (labels.Selector, error) { - // Avoid validation of keys and values, otherwise we cannot use LabelSelectors. - // Eventually, we will need to create our own Selector implementation with our own Requirement struct. - selector := labels.SelectorFromValidatedSet(labels.Set(ts.Filter.Attributes)) - for _, expr := range ts.Filter.AttributeExpressions { - var op selection.Operator - switch expr.Operator { - case v1.LabelSelectorOpIn: - op = selection.In - case v1.LabelSelectorOpNotIn: - op = selection.NotIn - case v1.LabelSelectorOpExists: - op = selection.Exists - case v1.LabelSelectorOpDoesNotExist: - op = selection.DoesNotExist - default: - return nil, fmt.Errorf("%q is not a valid filter selector operator", expr.Operator) - } - // Hack to set Requirement's unexposed fields to easily support expressions using k8s LabelSelectors. - // We should change this once we agree on a filter API. - r := labels.Requirement{} - rr := reflect.ValueOf(&r).Elem() - // Setting key - rrKey := rr.FieldByName("key") - rrKey = reflect.NewAt(rrKey.Type(), unsafe.Pointer(rrKey.UnsafeAddr())).Elem() - rrKey.SetString(expr.Key) - // Setting operator - rrOperator := rr.FieldByName("operator") - rrOperator = reflect.NewAt(rrOperator.Type(), unsafe.Pointer(rrOperator.UnsafeAddr())).Elem() - rrOperator.Set(reflect.ValueOf(op)) - // Setting strValues - rrStrValues := rr.FieldByName("strValues") - rrStrValues = reflect.NewAt(rrStrValues.Type(), unsafe.Pointer(rrStrValues.UnsafeAddr())).Elem() - rrStrValues.Set(reflect.ValueOf(expr.Values)) - selector = selector.Add(r) + filterSource := ts.Filter.ExactMatch.Source + if filterSource != Any && filterSource != m.Headers["Ce-Source"] { + r.logger.Debug("Wrong source", zap.String("trigger.spec.filter.exactMatch.source", filterSource), zap.String("message.source", m.Headers["Ce-Source"])) + return false } - return selector, nil + return true } diff --git a/t.yaml b/t.yaml index d9427cc9552..0e2604dd595 100644 --- a/t.yaml +++ b/t.yaml @@ -3,7 +3,6 @@ kind: Trigger metadata: name: t spec: - filter: {} subscriber: ref: apiVersion: serving.knative.dev/v1alpha1 diff --git a/t2.yaml b/t2.yaml index c95018d4bfb..6274a6166b4 100644 --- a/t2.yaml +++ b/t2.yaml @@ -6,9 +6,9 @@ spec: # Our Cloud Event parsing library seems to have a bug and forces to put type and source in double # quotes (as it thinks the actual value is `"foo"`, not `foo`). filter: - attributes: - Ce-Eventtype: '"com.example.someevent"' - Ce-Source: '"/mycontext/subcontext"' + exactMatch: + type: '"com.example.someevent"' + source: '"/mycontext/subcontext"' subscriber: ref: apiVersion: serving.knative.dev/v1alpha1 diff --git a/t3.yaml b/t3.yaml index 1a90d86926b..f286ea7d1ec 100644 --- a/t3.yaml +++ b/t3.yaml @@ -6,10 +6,8 @@ spec: # Our Cloud Event parsing library seems to have a bug and forces to put type and source in double # quotes (as it thinks the actual value is `"foo"`, not `foo`). filter: - attributes: - Ce-Source: '"/mycontext/subcontext"' - attributeExpressions: - - {key: Ce-Eventtype, operator: In, values: ['"com.example.someevent"', '"com.example.someevent1"']} + exactMatch: + source: '"/mycontext/subcontext"' subscriber: ref: apiVersion: serving.knative.dev/v1alpha1 From 8ccfcca4e5261fbee61ffdb3dd61d320f0057c9b Mon Sep 17 00:00:00 2001 From: Nacho Cano Date: Tue, 12 Feb 2019 23:34:30 -0800 Subject: [PATCH 047/128] Removing t3 as we don't have set expressions --- t3.yaml | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 t3.yaml diff --git a/t3.yaml b/t3.yaml deleted file mode 100644 index f286ea7d1ec..00000000000 --- a/t3.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: eventing.knative.dev/v1alpha1 -kind: Trigger -metadata: - name: t -spec: - # Our Cloud Event parsing library seems to have a bug and forces to put type and source in double - # quotes (as it thinks the actual value is `"foo"`, not `foo`). - filter: - exactMatch: - source: '"/mycontext/subcontext"' - subscriber: - ref: - apiVersion: serving.knative.dev/v1alpha1 - kind: Service - name: message-dumper From 29acf36937d69245dddf39f84cdf0e7f434ea96b Mon Sep 17 00:00:00 2001 From: nachocano Date: Wed, 13 Feb 2019 11:34:51 -0800 Subject: [PATCH 048/128] Updates after code review --- pkg/apis/eventing/v1alpha1/trigger_defaults.go | 2 +- pkg/apis/eventing/v1alpha1/trigger_types.go | 5 ++++- pkg/apis/eventing/v1alpha1/trigger_validation.go | 5 +++++ pkg/apis/eventing/v1alpha1/zz_generated.deepcopy.go | 12 ++++++++++-- pkg/broker/receiver.go | 8 ++------ 5 files changed, 22 insertions(+), 10 deletions(-) diff --git a/pkg/apis/eventing/v1alpha1/trigger_defaults.go b/pkg/apis/eventing/v1alpha1/trigger_defaults.go index fe5e7bbf797..e3dd6c32818 100644 --- a/pkg/apis/eventing/v1alpha1/trigger_defaults.go +++ b/pkg/apis/eventing/v1alpha1/trigger_defaults.go @@ -26,6 +26,6 @@ func (ts *TriggerSpec) SetDefaults() { } // Make a default filter that allows anything. if ts.Filter == nil { - ts.Filter = &TriggerFilter{TriggerFilterAttributes{Type: "", Source: ""}} + ts.Filter = &TriggerFilter{&TriggerFilterAttributes{Type: TriggerAnyFilter, Source: TriggerAnyFilter}} } } diff --git a/pkg/apis/eventing/v1alpha1/trigger_types.go b/pkg/apis/eventing/v1alpha1/trigger_types.go index 63be349e13d..c83873698bd 100644 --- a/pkg/apis/eventing/v1alpha1/trigger_types.go +++ b/pkg/apis/eventing/v1alpha1/trigger_types.go @@ -67,7 +67,7 @@ type TriggerSpec struct { } type TriggerFilter struct { - ExactMatch TriggerFilterAttributes `json:"exactMatch,omitempty"` + ExactMatch *TriggerFilterAttributes `json:"exactMatch,omitempty"` } type TriggerFilterAttributes struct { @@ -106,6 +106,9 @@ const ( TriggerConditionVirtualService duckv1alpha1.ConditionType = "VirtualService" TriggerConditionSubscribed duckv1alpha1.ConditionType = "Subscribed" + + // Constant to represent that we should allow anything. + TriggerAnyFilter = "Any" ) // GetCondition returns the condition currently associated with the given type, or nil. diff --git a/pkg/apis/eventing/v1alpha1/trigger_validation.go b/pkg/apis/eventing/v1alpha1/trigger_validation.go index fdf3a9ff08b..4a149dabea5 100644 --- a/pkg/apis/eventing/v1alpha1/trigger_validation.go +++ b/pkg/apis/eventing/v1alpha1/trigger_validation.go @@ -37,6 +37,11 @@ func (ts *TriggerSpec) Validate() *apis.FieldError { errs = errs.Also(fe) } + if ts.Filter.ExactMatch == nil { + fe := apis.ErrMissingField("filter.exactMatch") + errs = errs.Also(fe) + } + if isSubscriberSpecNilOrEmpty(ts.Subscriber) { fe := apis.ErrMissingField("subscriber") errs = errs.Also(fe) diff --git a/pkg/apis/eventing/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/eventing/v1alpha1/zz_generated.deepcopy.go index 88e1c446671..89bd9d76f0b 100644 --- a/pkg/apis/eventing/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/eventing/v1alpha1/zz_generated.deepcopy.go @@ -600,7 +600,15 @@ func (in *Trigger) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TriggerFilter) DeepCopyInto(out *TriggerFilter) { *out = *in - out.ExactMatch = in.ExactMatch + if in.ExactMatch != nil { + in, out := &in.ExactMatch, &out.ExactMatch + if *in == nil { + *out = nil + } else { + *out = new(TriggerFilterAttributes) + **out = **in + } + } return } @@ -672,7 +680,7 @@ func (in *TriggerSpec) DeepCopyInto(out *TriggerSpec) { *out = nil } else { *out = new(TriggerFilter) - **out = **in + (*in).DeepCopyInto(*out) } } if in.Subscriber != nil { diff --git a/pkg/broker/receiver.go b/pkg/broker/receiver.go index 4b4cfe89136..c6b6107b289 100644 --- a/pkg/broker/receiver.go +++ b/pkg/broker/receiver.go @@ -28,10 +28,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" ) -const ( - Any = "" -) - // Receiver parses Cloud Events and sends them to the channel. type Receiver struct { logger *zap.Logger @@ -99,12 +95,12 @@ func (r *Receiver) getTrigger(ctx context.Context, ref provisioners.ChannelRefer func (r *Receiver) shouldSendMessage(ts *eventingv1alpha1.TriggerSpec, m *provisioners.Message) bool { filterType := ts.Filter.ExactMatch.Type - if filterType != Any && filterType != m.Headers["Ce-Eventtype"] { + if filterType != eventingv1alpha1.TriggerAnyFilter && filterType != m.Headers["Ce-Eventtype"] { r.logger.Debug("Wrong type", zap.String("trigger.spec.filter.exactMatch.type", filterType), zap.String("message.type", m.Headers["Ce-Eventtype"])) return false } filterSource := ts.Filter.ExactMatch.Source - if filterSource != Any && filterSource != m.Headers["Ce-Source"] { + if filterSource != eventingv1alpha1.TriggerAnyFilter && filterSource != m.Headers["Ce-Source"] { r.logger.Debug("Wrong source", zap.String("trigger.spec.filter.exactMatch.source", filterSource), zap.String("message.source", m.Headers["Ce-Source"])) return false } From e75141c81ec3182ca37e17b167bb91ea9b833f36 Mon Sep 17 00:00:00 2001 From: nachocano Date: Wed, 13 Feb 2019 13:38:03 -0800 Subject: [PATCH 049/128] Attempt to reconcile broker in trigger controller --- pkg/reconciler/v1alpha1/trigger/trigger.go | 110 +++++++++++++++++++-- 1 file changed, 101 insertions(+), 9 deletions(-) diff --git a/pkg/reconciler/v1alpha1/trigger/trigger.go b/pkg/reconciler/v1alpha1/trigger/trigger.go index 8c2d57b26d9..c61900d04f1 100644 --- a/pkg/reconciler/v1alpha1/trigger/trigger.go +++ b/pkg/reconciler/v1alpha1/trigger/trigger.go @@ -19,6 +19,10 @@ package trigger import ( "context" "fmt" + "sync" + + "github.com/knative/eventing/pkg/provisioners" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/dynamic" @@ -56,8 +60,11 @@ const ( // itself when creating events. controllerAgentName = "trigger-controller" + finalizerName = controllerAgentName + // Name of the corev1.Events emitted from the reconciliation process triggerReconciled = "TriggerReconciled" + triggerReconcileFailed = "TriggerReconcileFailed" triggerUpdateStatusFailed = "TriggerUpdateStatusFailed" ) @@ -67,19 +74,23 @@ type reconciler struct { dynamicClient dynamic.Interface recorder record.EventRecorder + triggersLock sync.RWMutex + triggers map[string]map[reconcile.Request]bool + logger *zap.Logger } // Verify the struct implements reconcile.Reconciler var _ reconcile.Reconciler = &reconciler{} -// ProvideController returns a function that returns a Broker controller. +// ProvideController returns a function that returns a Trigger controller. func ProvideController(logger *zap.Logger) func(manager.Manager) (controller.Controller, error) { return func(mgr manager.Manager) (controller.Controller, error) { - // Setup a new controller to Reconcile Brokers. + // Setup a new controller to Reconcile Triggers. r := &reconciler{ recorder: mgr.GetRecorder(controllerAgentName), logger: logger, + triggers: make(map[string]map[reconcile.Request]bool), } c, err := controller.New(controllerAgentName, mgr, controller.Options{ Reconciler: r, @@ -94,14 +105,17 @@ func ProvideController(logger *zap.Logger) func(manager.Manager) (controller.Con } // Watch all the resources that the Trigger reconciles. - for _, t := range []runtime.Object{ &corev1.Service{}, &istiov1alpha3.VirtualService{}, &v1alpha1.Subscription{} } { - err = c.Watch(&source.Kind{Type: t}, &handler.EnqueueRequestForOwner{OwnerType: &v1alpha1.Broker{}, IsController: true}) + for _, t := range []runtime.Object{&corev1.Service{}, &istiov1alpha3.VirtualService{}, &v1alpha1.Subscription{}} { + err = c.Watch(&source.Kind{Type: t}, &handler.EnqueueRequestForOwner{OwnerType: &v1alpha1.Trigger{}, IsController: true}) if err != nil { return nil, err } } - // TODO reconcile after a Broker change. This might require keeping a map from Broker -> []Trigger. + if err = c.Watch(&source.Kind{Type: &v1alpha1.Channel{}}, &handler.EnqueueRequestsFromMapFunc{ToRequests: &mapAllTriggers{r}}); err != nil { + return nil, err + } + // TODO reconcile after a change to the subscriber. I'm not sure how this is possible, but we should do it if we // can find a way. @@ -109,6 +123,24 @@ func ProvideController(logger *zap.Logger) func(manager.Manager) (controller.Con } } +type mapAllTriggers struct { + r *reconciler +} + +func (m *mapAllTriggers) Map(o handler.MapObject) []reconcile.Request { + m.r.triggersLock.RLock() + defer m.r.triggersLock.RUnlock() + triggersInNamespace := m.r.triggers[o.Meta.GetNamespace()] + if triggersInNamespace == nil { + return []reconcile.Request{} + } + reqs := make([]reconcile.Request, 0, len(triggersInNamespace)) + for name := range triggersInNamespace { + reqs = append(reqs, name) + } + return reqs +} + func (r *reconciler) InjectClient(c client.Client) error { r.client = c return nil @@ -137,27 +169,31 @@ func (r *reconciler) Reconcile(request reconcile.Request) (reconcile.Result, err } if err != nil { - logging.FromContext(ctx).Error("Could not Get Trigger", zap.Error(err)) + logging.FromContext(ctx).Error("Could not get Trigger", zap.Error(err)) return reconcile.Result{}, err } + // Modify a copy, not the original. + trigger = trigger.DeepCopy() + // Reconcile this copy of the Trigger and then write back any status updates regardless of // whether the reconcile error out. reconcileErr := r.reconcile(ctx, trigger) if reconcileErr != nil { logging.FromContext(ctx).Error("Error reconciling Trigger", zap.Error(reconcileErr)) + r.recorder.Eventf(trigger, corev1.EventTypeWarning, triggerReconcileFailed, "Trigger reconciliation failed: %v", reconcileErr) } else { logging.FromContext(ctx).Debug("Trigger reconciled") - r.recorder.Event(trigger, corev1.EventTypeNormal, triggerReconciled, "Trigger reconciled") + r.recorder.Eventf(trigger, corev1.EventTypeNormal, triggerReconciled, "Trigger reconciled: %q", trigger.Name) } - if _, err = r.updateStatus(trigger.DeepCopy()); err != nil { + if _, err = r.updateStatus(trigger); err != nil { logging.FromContext(ctx).Error("Failed to update Trigger status", zap.Error(err)) r.recorder.Eventf(trigger, corev1.EventTypeWarning, triggerUpdateStatusFailed, "Failed to update Trigger's status: %v", err) return reconcile.Result{}, err } - // Requeue if the resource is not ready: + // Requeue if the resource is not ready return reconcile.Result{}, reconcileErr } @@ -172,9 +208,14 @@ func (r *reconciler) reconcile(ctx context.Context, t *v1alpha1.Trigger) error { if t.DeletionTimestamp != nil { // Everything is cleaned up by the garbage collector. + r.removeFromTriggers(t) + provisioners.RemoveFinalizer(t, finalizerName) return nil } + provisioners.AddFinalizer(t, finalizerName) + r.AddToTriggers(t) + b, err := r.getBroker(ctx, t) if err != nil { logging.FromContext(ctx).Error("Unable to get the Broker", zap.Error(err)) @@ -221,6 +262,57 @@ func (r *reconciler) reconcile(ctx context.Context, t *v1alpha1.Trigger) error { return nil } +func (r *reconciler) AddToTriggers(t *v1alpha1.Trigger) { + name := reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: t.Namespace, + Name: t.Name, + }, + } + + // We will be reconciling an already existing Trigger far more often than adding a new one, so + // check with a read lock before using the write lock. + r.triggersLock.RLock() + triggersInNamespace := r.triggers[t.Namespace] + var present bool + if triggersInNamespace != nil { + _, present = triggersInNamespace[name] + } else { + present = false + } + r.triggersLock.RUnlock() + + if present { + // Already present in the map. + return + } + + r.triggersLock.Lock() + triggersInNamespace = r.triggers[t.Namespace] + if triggersInNamespace == nil { + r.triggers[t.Namespace] = make(map[reconcile.Request]bool) + triggersInNamespace = r.triggers[t.Namespace] + } + triggersInNamespace[name] = false + r.triggersLock.Unlock() +} + +func (r *reconciler) removeFromTriggers(t *v1alpha1.Trigger) { + name := reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: t.Namespace, + Name: t.Name, + }, + } + + r.triggersLock.Lock() + triggersInNamespace := r.triggers[t.Namespace] + if triggersInNamespace != nil { + delete(triggersInNamespace, name) + } + r.triggersLock.Unlock() +} + // updateStatus may in fact update the trigger's finalizers in addition to the status func (r *reconciler) updateStatus(trigger *v1alpha1.Trigger) (*v1alpha1.Trigger, error) { objectKey := client.ObjectKey{Namespace: trigger.Namespace, Name: trigger.Name} From e79b8f70180ef3f90e9af4ce43bf2e95bc772adb Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Tue, 12 Feb 2019 23:02:47 -0800 Subject: [PATCH 050/128] Broker controller unit tests. --- pkg/apis/eventing/v1alpha1/broker_types.go | 4 +- pkg/reconciler/v1alpha1/broker/broker.go | 34 +- pkg/reconciler/v1alpha1/broker/broker_test.go | 658 ++++++++++++++++++ .../v1alpha1/broker/resources/filter.go | 1 + .../v1alpha1/broker/resources/ingress.go | 5 +- .../v1alpha1/namespace/namespace_test.go | 2 +- 6 files changed, 682 insertions(+), 22 deletions(-) diff --git a/pkg/apis/eventing/v1alpha1/broker_types.go b/pkg/apis/eventing/v1alpha1/broker_types.go index b50cd43101a..6cad4d3efbd 100644 --- a/pkg/apis/eventing/v1alpha1/broker_types.go +++ b/pkg/apis/eventing/v1alpha1/broker_types.go @@ -139,9 +139,9 @@ func (bs *BrokerStatus) MarkFilterReady() { func (bs *BrokerStatus) SetAddress(hostname string) { bs.Address.Hostname = hostname if hostname != "" { - chanCondSet.Manage(bs).MarkTrue(BrokerConditionAddressable) + brokerCondSet.Manage(bs).MarkTrue(BrokerConditionAddressable) } else { - chanCondSet.Manage(bs).MarkFalse(BrokerConditionAddressable, "emptyHostname", "hostname is the empty string") + brokerCondSet.Manage(bs).MarkFalse(BrokerConditionAddressable, "emptyHostname", "hostname is the empty string") } } diff --git a/pkg/reconciler/v1alpha1/broker/broker.go b/pkg/reconciler/v1alpha1/broker/broker.go index 8a79adfd902..1e4c52b2735 100644 --- a/pkg/reconciler/v1alpha1/broker/broker.go +++ b/pkg/reconciler/v1alpha1/broker/broker.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "k8s.io/apimachinery/pkg/runtime" + "time" "k8s.io/client-go/dynamic" "k8s.io/client-go/rest" @@ -106,7 +107,7 @@ func ProvideController(logger *zap.Logger, ingressImage, ingressServiceAccount, } // Watch all the resources that the Broker reconciles. - for _, t := range []runtime.Object{ &v1alpha1.Channel{}, &corev1.Service{}, &v1.Deployment{} } { + for _, t := range []runtime.Object{&v1alpha1.Channel{}, &corev1.Service{}, &v1.Deployment{}} { err = c.Watch(&source.Kind{Type: t}, &handler.EnqueueRequestForOwner{OwnerType: &v1alpha1.Broker{}, IsController: true}) if err != nil { return nil, err @@ -151,9 +152,11 @@ func (r *reconciler) Reconcile(request reconcile.Request) (reconcile.Result, err // Reconcile this copy of the Broker and then write back any status updates regardless of // whether the reconcile error out. - reconcileErr := r.reconcile(ctx, broker) + result, reconcileErr := r.reconcile(ctx, broker) if reconcileErr != nil { logging.FromContext(ctx).Error("Error reconciling Broker", zap.Error(reconcileErr)) + } else if result.Requeue || result.RequeueAfter > 0 { + logging.FromContext(ctx).Debug("Broker reconcile requeuing") } else { logging.FromContext(ctx).Debug("Broker reconciled") r.recorder.Event(broker, corev1.EventTypeNormal, brokerReconciled, "Broker reconciled") @@ -166,10 +169,10 @@ func (r *reconciler) Reconcile(request reconcile.Request) (reconcile.Result, err } // Requeue if the resource is not ready: - return reconcile.Result{}, reconcileErr + return result, reconcileErr } -func (r *reconciler) reconcile(ctx context.Context, b *v1alpha1.Broker) error { +func (r *reconciler) reconcile(ctx context.Context, b *v1alpha1.Broker) (reconcile.Result, error) { b.Status.InitializeConditions() // 1. Channel is created for all events. @@ -179,48 +182,48 @@ func (r *reconciler) reconcile(ctx context.Context, b *v1alpha1.Broker) error { if b.DeletionTimestamp != nil { // Everything is cleaned up by the garbage collector. - return nil + return reconcile.Result{}, nil } c, err := r.reconcileChannel(ctx, b) if err != nil { logging.FromContext(ctx).Error("Problem reconciling the channel", zap.Error(err)) b.Status.MarkChannelFailed(err) - return err + return reconcile.Result{}, err } else if c.Status.Address.Hostname == "" { logging.FromContext(ctx).Info("Channel is not yet ready", zap.Any("c", c)) - // TODO Just re-enqueue, don't return an error - return nil + // Give the Channel some time to get its address. One second was chosen arbitrarily. + return reconcile.Result{RequeueAfter: time.Second}, nil } b.Status.MarkChannelReady() _, err = r.reconcileFilterDeployment(ctx, b) if err != nil { logging.FromContext(ctx).Error("Problem reconciling filter deployment", zap.Error(err)) - return err + return reconcile.Result{}, err } _, err = r.reconcileFilterService(ctx, b) if err != nil { logging.FromContext(ctx).Error("Problem reconciling filter service", zap.Error(err)) - return err + return reconcile.Result{}, err } b.Status.MarkFilterReady() _, err = r.reconcileIngressDeployment(ctx, b, c) if err != nil { logging.FromContext(ctx).Error("Problem reconciling ingress deployment", zap.Error(err)) - return err + return reconcile.Result{}, err } svc, err := r.reconcileIngressService(ctx, b) if err != nil { logging.FromContext(ctx).Error("Problem reconciling ingress Service", zap.Error(err)) - return err + return reconcile.Result{}, err } b.Status.MarkIngressReady() b.Status.SetAddress(names.ServiceHostName(svc.Name, svc.Namespace)) - return nil + return reconcile.Result{}, nil } // updateStatus may in fact update the broker's finalizers in addition to the status @@ -420,15 +423,12 @@ func (r *reconciler) reconcileService(ctx context.Context, svc *corev1.Service) } func (r *reconciler) reconcileIngressDeployment(ctx context.Context, b *v1alpha1.Broker, c *v1alpha1.Channel) (*v1.Deployment, error) { - expected, err := resources.MakeIngress(&resources.IngressArgs{ + expected := resources.MakeIngress(&resources.IngressArgs{ Broker: b, Image: r.ingressImage, ServiceAccountName: r.ingressServiceAccountName, ChannelAddress: c.Status.Address.Hostname, }) - if err != nil { - return nil, err - } return r.reconcileDeployment(ctx, expected) } diff --git a/pkg/reconciler/v1alpha1/broker/broker_test.go b/pkg/reconciler/v1alpha1/broker/broker_test.go index 7a887c44284..992bbe5708e 100644 --- a/pkg/reconciler/v1alpha1/broker/broker_test.go +++ b/pkg/reconciler/v1alpha1/broker/broker_test.go @@ -17,13 +17,60 @@ limitations under the License. package broker import ( + "context" + "errors" + "fmt" "github.com/google/go-cmp/cmp" + "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" + controllertesting "github.com/knative/eventing/pkg/reconciler/testing" + "github.com/knative/eventing/pkg/reconciler/v1alpha1/broker/resources" + duckv1alpha1 "github.com/knative/pkg/apis/duck/v1alpha1" + "go.uber.org/zap" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "strings" "testing" + "time" ) +const ( + testNS = "test-namespace" + brokerName = "test-broker" + channelHostname = "foo.bar.svc.cluster.local" + + filterImage = "filter-image" + filterSA = "filter-SA" + ingressImage = "ingress-image" + ingressSA = "ingress-SA" +) + +var ( + trueVal = true + + channelProvisioner = &corev1.ObjectReference{ + APIVersion: "eventing.knative.dev/v1alpha1", + Kind: "ClusterChannelProvisioner", + Name: "my-provisioner", + } + + // deletionTime is used when objects are marked as deleted. Rfc3339Copy() + // truncates to seconds to match the loss of precision during serialization. + deletionTime = metav1.Now().Rfc3339Copy() +) + +func init() { + // Add types to scheme + v1alpha1.AddToScheme(scheme.Scheme) +} + func TestProvideController(t *testing.T) { //TODO(grantr) This needs a mock of manager.Manager. Creating a manager // with a fake Config fails because the Manager tries to contact the @@ -89,3 +136,614 @@ func TestInjectConfig(t *testing.T) { t.Errorf("Unexpected dynamicClient type. Expected: %T, Got: %T", wantDynClient, r.dynamicClient) } } + +func TestReconcile(t *testing.T) { + testCases := []controllertesting.TestCase{ + { + Name: "Broker not found", + }, + { + Name: "Broker.Get fails", + Scheme: scheme.Scheme, + Mocks: controllertesting.Mocks{ + MockGets: []controllertesting.MockGet{ + func(_ client.Client, _ context.Context, _ client.ObjectKey, obj runtime.Object) (controllertesting.MockHandled, error) { + if _, ok := obj.(*v1alpha1.Broker); ok { + return controllertesting.Handled, errors.New("test error getting the Broker") + } + return controllertesting.Unhandled, nil + }, + }, + }, + WantErrMsg: "test error getting the Broker", + }, + { + Name: "Broker is being deleted", + Scheme: scheme.Scheme, + InitialState: []runtime.Object{ + makeDeletingBroker(), + }, + WantEvent: []corev1.Event{ + { + Reason: brokerReconciled, Type: corev1.EventTypeNormal, + }, + }, + }, + { + Name: "Channel.List error", + Scheme: scheme.Scheme, + InitialState: []runtime.Object{ + makeBroker(), + }, + Mocks: controllertesting.Mocks{ + MockLists: []controllertesting.MockList{ + func(_ client.Client, _ context.Context, _ *client.ListOptions, list runtime.Object) (controllertesting.MockHandled, error) { + if _, ok := list.(*v1alpha1.ChannelList); ok { + return controllertesting.Handled, errors.New("test error listing channels") + } + return controllertesting.Unhandled, nil + }, + }, + }, + WantErrMsg: "test error listing channels", + }, + { + Name: "Channel.Create error", + Scheme: scheme.Scheme, + InitialState: []runtime.Object{ + makeBroker(), + }, + Mocks: controllertesting.Mocks{ + MockCreates: []controllertesting.MockCreate{ + func(_ client.Client, _ context.Context, obj runtime.Object) (controllertesting.MockHandled, error) { + if _, ok := obj.(*v1alpha1.Channel); ok { + return controllertesting.Handled, errors.New("test error creating Channel") + } + return controllertesting.Unhandled, nil + }, + }, + }, + WantErrMsg: "test error creating Channel", + }, + { + Name: "Channel is different than expected", + Scheme: scheme.Scheme, + InitialState: []runtime.Object{ + makeBroker(), + makeDifferentChannel(), + }, + WantPresent: []runtime.Object{ + // This is special because the Channel is not updated, unlike most things that + // differ from expected. + // TODO uncomment the following line once our test framework supports searching for + // GenerateName. + //makeDifferentChannel(), + }, + WantEvent: []corev1.Event{ + { + Reason: brokerReconciled, Type: corev1.EventTypeNormal, + }, + }, + }, + { + Name: "Channel is not yet Addressable", + Scheme: scheme.Scheme, + InitialState: []runtime.Object{ + makeBroker(), + makeNonAddressableChannel(), + }, + WantResult: reconcile.Result{RequeueAfter: time.Second}, + }, + { + Name: "Filter Deployment.Get error", + Scheme: scheme.Scheme, + InitialState: []runtime.Object{ + makeBroker(), + makeChannel(), + }, + Mocks: controllertesting.Mocks{ + MockGets: []controllertesting.MockGet{ + func(_ client.Client, _ context.Context, key client.ObjectKey, obj runtime.Object) (controllertesting.MockHandled, error) { + if _, ok := obj.(*appsv1.Deployment); ok { + if strings.Contains(key.Name, "filter") { + return controllertesting.Handled, errors.New("test error getting filter Deployment") + } + } + return controllertesting.Unhandled, nil + }, + }, + }, + WantErrMsg: "test error getting filter Deployment", + }, + { + Name: "Filter Deployment.Create error", + Scheme: scheme.Scheme, + InitialState: []runtime.Object{ + makeBroker(), + makeChannel(), + }, + Mocks: controllertesting.Mocks{ + MockCreates: []controllertesting.MockCreate{ + func(_ client.Client, _ context.Context, obj runtime.Object) (controllertesting.MockHandled, error) { + if d, ok := obj.(*appsv1.Deployment); ok { + if d.Labels["eventing.knative.dev/brokerRole"] == "filter" { + return controllertesting.Handled, errors.New("test error creating filter Deployment") + } + } + return controllertesting.Unhandled, nil + }, + }, + }, + WantErrMsg: "test error creating filter Deployment", + }, + { + Name: "Filter Deployment.Update error", + Scheme: scheme.Scheme, + InitialState: []runtime.Object{ + makeBroker(), + makeChannel(), + makeDifferentFilterDeployment(), + }, + Mocks: controllertesting.Mocks{ + MockUpdates: []controllertesting.MockUpdate{ + func(_ client.Client, _ context.Context, obj runtime.Object) (controllertesting.MockHandled, error) { + if d, ok := obj.(*appsv1.Deployment); ok { + if d.Labels["eventing.knative.dev/brokerRole"] == "filter" { + return controllertesting.Handled, errors.New("test error updating filter Deployment") + } + } + return controllertesting.Unhandled, nil + }, + }, + }, + WantErrMsg: "test error updating filter Deployment", + }, + { + Name: "Filter Service.Get error", + Scheme: scheme.Scheme, + InitialState: []runtime.Object{ + makeBroker(), + makeChannel(), + }, + Mocks: controllertesting.Mocks{ + MockGets: []controllertesting.MockGet{ + func(_ client.Client, _ context.Context, key client.ObjectKey, obj runtime.Object) (controllertesting.MockHandled, error) { + if _, ok := obj.(*corev1.Service); ok { + if strings.Contains(key.Name, "filter") { + return controllertesting.Handled, errors.New("test error getting filter Service") + } + } + return controllertesting.Unhandled, nil + }, + }, + }, + WantErrMsg: "test error getting filter Service", + }, + { + Name: "Filter Service.Create error", + Scheme: scheme.Scheme, + InitialState: []runtime.Object{ + makeBroker(), + makeChannel(), + }, + Mocks: controllertesting.Mocks{ + MockCreates: []controllertesting.MockCreate{ + func(_ client.Client, _ context.Context, obj runtime.Object) (controllertesting.MockHandled, error) { + if svc, ok := obj.(*corev1.Service); ok { + if svc.Labels["eventing.knative.dev/brokerRole"] == "filter" { + return controllertesting.Handled, errors.New("test error creating filter Service") + } + } + return controllertesting.Unhandled, nil + }, + }, + }, + WantErrMsg: "test error creating filter Service", + }, + { + Name: "Filter Service.Update error", + Scheme: scheme.Scheme, + InitialState: []runtime.Object{ + makeBroker(), + makeChannel(), + makeDifferentFilterService(), + }, + Mocks: controllertesting.Mocks{ + MockUpdates: []controllertesting.MockUpdate{ + func(_ client.Client, _ context.Context, obj runtime.Object) (controllertesting.MockHandled, error) { + if svc, ok := obj.(*corev1.Service); ok { + if svc.Labels["eventing.knative.dev/brokerRole"] == "filter" { + return controllertesting.Handled, errors.New("test error updating filter Service") + } + } + return controllertesting.Unhandled, nil + }, + }, + }, + WantErrMsg: "test error updating filter Service", + }, + { + Name: "Ingress Deployment.Get error", + Scheme: scheme.Scheme, + InitialState: []runtime.Object{ + makeBroker(), + makeChannel(), + }, + Mocks: controllertesting.Mocks{ + MockGets: []controllertesting.MockGet{ + func(_ client.Client, _ context.Context, key client.ObjectKey, obj runtime.Object) (controllertesting.MockHandled, error) { + if _, ok := obj.(*appsv1.Deployment); ok { + if strings.Contains(key.Name, "ingress") { + return controllertesting.Handled, errors.New("test error getting ingress Deployment") + } + } + return controllertesting.Unhandled, nil + }, + }, + }, + WantErrMsg: "test error getting ingress Deployment", + }, + { + Name: "Ingress Deployment.Create error", + Scheme: scheme.Scheme, + InitialState: []runtime.Object{ + makeBroker(), + makeChannel(), + }, + Mocks: controllertesting.Mocks{ + MockCreates: []controllertesting.MockCreate{ + func(_ client.Client, _ context.Context, obj runtime.Object) (controllertesting.MockHandled, error) { + if d, ok := obj.(*appsv1.Deployment); ok { + if d.Labels["eventing.knative.dev/brokerRole"] == "ingress" { + return controllertesting.Handled, errors.New("test error creating ingress Deployment") + } + } + return controllertesting.Unhandled, nil + }, + }, + }, + WantErrMsg: "test error creating ingress Deployment", + }, + { + Name: "Ingress Deployment.Update error", + Scheme: scheme.Scheme, + InitialState: []runtime.Object{ + makeBroker(), + makeChannel(), + makeDifferentIngressDeployment(), + }, + Mocks: controllertesting.Mocks{ + MockUpdates: []controllertesting.MockUpdate{ + func(_ client.Client, _ context.Context, obj runtime.Object) (controllertesting.MockHandled, error) { + if d, ok := obj.(*appsv1.Deployment); ok { + if d.Labels["eventing.knative.dev/brokerRole"] == "ingress" { + return controllertesting.Handled, errors.New("test error updating ingress Deployment") + } + } + return controllertesting.Unhandled, nil + }, + }, + }, + WantErrMsg: "test error updating ingress Deployment", + }, + { + Name: "Ingress Service.Get error", + Scheme: scheme.Scheme, + InitialState: []runtime.Object{ + makeBroker(), + makeChannel(), + }, + Mocks: controllertesting.Mocks{ + MockGets: []controllertesting.MockGet{ + func(_ client.Client, _ context.Context, key client.ObjectKey, obj runtime.Object) (controllertesting.MockHandled, error) { + if _, ok := obj.(*corev1.Service); ok { + if key.Name == fmt.Sprintf("%s-broker", brokerName) { + return controllertesting.Handled, errors.New("test error getting ingress Service") + } + } + return controllertesting.Unhandled, nil + }, + }, + }, + WantErrMsg: "test error getting ingress Service", + }, + { + Name: "Ingress Service.Create error", + Scheme: scheme.Scheme, + InitialState: []runtime.Object{ + makeBroker(), + makeChannel(), + }, + Mocks: controllertesting.Mocks{ + MockCreates: []controllertesting.MockCreate{ + func(_ client.Client, _ context.Context, obj runtime.Object) (controllertesting.MockHandled, error) { + if svc, ok := obj.(*corev1.Service); ok { + if svc.Labels["eventing.knative.dev/brokerRole"] == "ingress" { + return controllertesting.Handled, errors.New("test error creating ingress Service") + } + } + return controllertesting.Unhandled, nil + }, + }, + }, + WantErrMsg: "test error creating ingress Service", + }, + { + Name: "Ingress Service.Update error", + Scheme: scheme.Scheme, + InitialState: []runtime.Object{ + makeBroker(), + makeChannel(), + makeDifferentIngressService(), + }, + Mocks: controllertesting.Mocks{ + MockUpdates: []controllertesting.MockUpdate{ + func(_ client.Client, _ context.Context, obj runtime.Object) (controllertesting.MockHandled, error) { + if svc, ok := obj.(*corev1.Service); ok { + if svc.Labels["eventing.knative.dev/brokerRole"] == "ingress" { + return controllertesting.Handled, errors.New("test error updating ingress Service") + } + } + return controllertesting.Unhandled, nil + }, + }, + }, + WantErrMsg: "test error updating ingress Service", + }, + { + Name: "Broker.Get for status update fails", + Scheme: scheme.Scheme, + InitialState: []runtime.Object{ + makeBroker(), + makeChannel(), + }, + Mocks: controllertesting.Mocks{ + MockGets: []controllertesting.MockGet{ + // The first Get works. + func(innerClient client.Client, ctx context.Context, key client.ObjectKey, obj runtime.Object) (controllertesting.MockHandled, error) { + if _, ok := obj.(*v1alpha1.Broker); ok { + return controllertesting.Handled, innerClient.Get(ctx, key, obj) + } + return controllertesting.Unhandled, nil + }, + // The second Get fails. + func(_ client.Client, _ context.Context, _ client.ObjectKey, obj runtime.Object) (controllertesting.MockHandled, error) { + if _, ok := obj.(*v1alpha1.Broker); ok { + return controllertesting.Handled, errors.New("test error getting the Broker for status update") + } + return controllertesting.Unhandled, nil + }, + }, + }, + WantErrMsg: "test error getting the Broker for status update", + WantEvent: []corev1.Event{ + { + Reason: brokerReconciled, Type: corev1.EventTypeNormal, + }, + { + Reason: brokerUpdateStatusFailed, Type: corev1.EventTypeWarning, + }, + }, + }, + { + Name: "Broker.Status.Update error", + Scheme: scheme.Scheme, + InitialState: []runtime.Object{ + makeBroker(), + makeChannel(), + }, + Mocks: controllertesting.Mocks{ + MockStatusUpdates: []controllertesting.MockStatusUpdate{ + func(_ client.Client, _ context.Context, obj runtime.Object) (controllertesting.MockHandled, error) { + if _, ok := obj.(*v1alpha1.Broker); ok { + return controllertesting.Handled, errors.New("test error updating the Broker status") + } + return controllertesting.Unhandled, nil + }, + }, + }, + WantErrMsg: "test error updating the Broker status", + WantEvent: []corev1.Event{ + { + Reason: brokerReconciled, Type: corev1.EventTypeNormal, + }, + { + Reason: brokerUpdateStatusFailed, Type: corev1.EventTypeWarning, + }, + }, + }, + { + Name: "Successful reconcile", + Scheme: scheme.Scheme, + InitialState: []runtime.Object{ + makeBroker(), + // The Channel needs to be addressable for the reconcile to succeed. + makeChannel(), + }, + WantPresent: []runtime.Object{ + makeReadyBroker(), + // TODO Uncomment makeChannel() when our test framework handles generateName. + //makeChannel(), + makeFilterDeployment(), + makeFilterService(), + makeIngressDeployment(), + makeIngressService(), + }, + WantEvent: []corev1.Event{ + { + Reason: brokerReconciled, Type: corev1.EventTypeNormal, + }, + }, + }, + } + for _, tc := range testCases { + c := tc.GetClient() + dc := tc.GetDynamicClient() + recorder := tc.GetEventRecorder() + + r := &reconciler{ + client: c, + dynamicClient: dc, + restConfig: &rest.Config{}, + recorder: recorder, + logger: zap.NewNop(), + + filterImage: filterImage, + filterServiceAccountName: filterSA, + ingressImage: ingressImage, + ingressServiceAccountName: ingressSA, + } + tc.ReconcileKey = fmt.Sprintf("%s/%s", testNS, brokerName) + tc.IgnoreTimes = true + t.Run(tc.Name, tc.Runner(t, r, c, recorder)) + } +} + +func makeBroker() *v1alpha1.Broker { + return &v1alpha1.Broker{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "eventing.knative.dev/v1alpha1", + Kind: "Broker", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: testNS, + Name: brokerName, + }, + Spec: v1alpha1.BrokerSpec{ + ChannelTemplate: &v1alpha1.ChannelSpec{ + Provisioner: channelProvisioner, + }, + }, + } +} + +func makeReadyBroker() *v1alpha1.Broker { + b := makeBroker() + b.Status.InitializeConditions() + b.Status.MarkChannelReady() + b.Status.SetAddress(fmt.Sprintf("%s-broker.%s.svc.cluster.local", brokerName, testNS)) + b.Status.MarkFilterReady() + b.Status.MarkIngressReady() + return b +} + +func makeDeletingBroker() *v1alpha1.Broker { + b := makeReadyBroker() + b.DeletionTimestamp = &deletionTime + return b +} + +func makeChannel() *v1alpha1.Channel { + return &v1alpha1.Channel{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testNS, + GenerateName: fmt.Sprintf("%s-broker-", brokerName), + Labels: map[string]string{ + "eventing.knative.dev/broker": brokerName, + "eventing.knative.dev/brokerEverything": "true", + }, + OwnerReferences: []metav1.OwnerReference{ + getOwnerReference(), + }, + }, + Spec: v1alpha1.ChannelSpec{ + Provisioner: channelProvisioner, + }, + Status: v1alpha1.ChannelStatus{ + Address: duckv1alpha1.Addressable{ + Hostname: channelHostname, + }, + }, + } +} + +func makeNonAddressableChannel() *v1alpha1.Channel { + c := makeChannel() + c.Status.Address = duckv1alpha1.Addressable{} + return c +} + +func makeDifferentChannel() *v1alpha1.Channel { + c := makeChannel() + c.Spec.Provisioner.Name = "some-other-provisioner" + return c +} + +func makeFilterDeployment() *appsv1.Deployment { + d := resources.MakeFilterDeployment(&resources.FilterArgs{ + Broker: makeBroker(), + Image: filterImage, + ServiceAccountName: filterSA, + }) + d.TypeMeta = metav1.TypeMeta{ + APIVersion: "apps/v1", + Kind: "Deployment", + } + return d +} + +func makeDifferentFilterDeployment() *appsv1.Deployment { + d := makeFilterDeployment() + d.Spec.Template.Spec.Containers[0].Image = "some-other-image" + return d +} + +func makeFilterService() *corev1.Service { + svc := resources.MakeFilterService(makeBroker()) + svc.TypeMeta = metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Service", + } + return svc +} + +func makeDifferentFilterService() *corev1.Service { + s := makeFilterService() + s.Spec.Selector["eventing.knative.dev/broker"] = "some-other-value" + return s +} + +func makeIngressDeployment() *appsv1.Deployment { + d := resources.MakeIngress(&resources.IngressArgs{ + Broker: makeBroker(), + Image: ingressImage, + ServiceAccountName: ingressSA, + ChannelAddress: channelHostname, + }) + d.TypeMeta = metav1.TypeMeta{ + APIVersion: "apps/v1", + Kind: "Deployment", + } + return d +} + +func makeDifferentIngressDeployment() *appsv1.Deployment { + d := makeIngressDeployment() + d.Spec.Template.Spec.Containers[0].Image = "some-other-image" + return d +} + +func makeIngressService() *corev1.Service { + svc := resources.MakeIngressService(makeBroker()) + svc.TypeMeta = metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Service", + } + return svc +} + +func makeDifferentIngressService() *corev1.Service { + s := makeIngressService() + s.Spec.Selector["eventing.knative.dev/broker"] = "some-other-value" + return s +} + +func getOwnerReference() metav1.OwnerReference { + return metav1.OwnerReference{ + APIVersion: v1alpha1.SchemeGroupVersion.String(), + Kind: "Broker", + Name: brokerName, + Controller: &trueVal, + BlockOwnerDeletion: &trueVal, + } +} diff --git a/pkg/reconciler/v1alpha1/broker/resources/filter.go b/pkg/reconciler/v1alpha1/broker/resources/filter.go index 38d049f068e..428d0e4a38e 100644 --- a/pkg/reconciler/v1alpha1/broker/resources/filter.go +++ b/pkg/reconciler/v1alpha1/broker/resources/filter.go @@ -46,6 +46,7 @@ func MakeFilterDeployment(args *FilterArgs) *appsv1.Deployment { Kind: "Broker", }), }, + Labels: filterLabels(args.Broker), }, Spec: appsv1.DeploymentSpec{ Selector: &metav1.LabelSelector{ diff --git a/pkg/reconciler/v1alpha1/broker/resources/ingress.go b/pkg/reconciler/v1alpha1/broker/resources/ingress.go index a5444b5cf68..406f1b85daa 100644 --- a/pkg/reconciler/v1alpha1/broker/resources/ingress.go +++ b/pkg/reconciler/v1alpha1/broker/resources/ingress.go @@ -35,7 +35,7 @@ type IngressArgs struct { ChannelAddress string } -func MakeIngress(args *IngressArgs) (*appsv1.Deployment, error) { +func MakeIngress(args *IngressArgs) *appsv1.Deployment { return &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Namespace: args.Broker.Namespace, @@ -47,6 +47,7 @@ func MakeIngress(args *IngressArgs) (*appsv1.Deployment, error) { Kind: "Broker", }), }, + Labels: ingressLabels(args.Broker), }, Spec: appsv1.DeploymentSpec{ Selector: &metav1.LabelSelector{ @@ -80,7 +81,7 @@ func MakeIngress(args *IngressArgs) (*appsv1.Deployment, error) { }, }, }, - }, nil + } } func MakeIngressService(b *eventingv1alpha1.Broker) *corev1.Service { diff --git a/pkg/reconciler/v1alpha1/namespace/namespace_test.go b/pkg/reconciler/v1alpha1/namespace/namespace_test.go index cdf1876ac29..a3ce6a05615 100644 --- a/pkg/reconciler/v1alpha1/namespace/namespace_test.go +++ b/pkg/reconciler/v1alpha1/namespace/namespace_test.go @@ -150,7 +150,7 @@ func TestReconcile(t *testing.T) { Name: "Namespace not found", }, { - Name: "Namespace get fails", + Name: "Namespace.Get fails", Scheme: scheme.Scheme, Mocks: controllertesting.Mocks{ MockGets: []controllertesting.MockGet{ From 30f1c7478c8ec7680b6f9259f6d45ab45f78db25 Mon Sep 17 00:00:00 2001 From: nachocano Date: Wed, 13 Feb 2019 16:15:47 -0800 Subject: [PATCH 051/128] Deleting and re-creating subscription object as the backing channel spec is immutable. --- pkg/reconciler/v1alpha1/trigger/trigger.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/pkg/reconciler/v1alpha1/trigger/trigger.go b/pkg/reconciler/v1alpha1/trigger/trigger.go index c61900d04f1..17159cf0776 100644 --- a/pkg/reconciler/v1alpha1/trigger/trigger.go +++ b/pkg/reconciler/v1alpha1/trigger/trigger.go @@ -112,7 +112,7 @@ func ProvideController(logger *zap.Logger) func(manager.Manager) (controller.Con } } - if err = c.Watch(&source.Kind{Type: &v1alpha1.Channel{}}, &handler.EnqueueRequestsFromMapFunc{ToRequests: &mapAllTriggers{r}}); err != nil { + if err = c.Watch(&source.Kind{Type: &v1alpha1.Broker{}}, &handler.EnqueueRequestsFromMapFunc{ToRequests: &mapAllTriggers{r}}); err != nil { return nil, err } @@ -595,9 +595,17 @@ func (r *reconciler) subscribeToBrokerChannel(ctx context.Context, t *v1alpha1.T // Update Subscription if it has changed. Ignore the generation. expected.Spec.DeprecatedGeneration = sub.Spec.DeprecatedGeneration if !equality.Semantic.DeepDerivative(expected.Spec, sub.Spec) { - sub.Spec = expected.Spec - err = r.client.Update(ctx, sub) + // Given that the backing channel spec is immutable, we cannot just update the subscription. + // We delete it instead, and re-create it. + err = r.client.Delete(ctx, sub) if err != nil { + logging.FromContext(ctx).Info("Cannot delete subscription", zap.Error(err)) + return nil, err + } + sub = expected + err = r.client.Create(ctx, sub) + if err != nil { + logging.FromContext(ctx).Info("Cannot create subscription", zap.Error(err)) return nil, err } } From e45d19a9e5cd78fc0fa3484b17959d415fc34cf3 Mon Sep 17 00:00:00 2001 From: nachocano Date: Wed, 13 Feb 2019 16:27:15 -0800 Subject: [PATCH 052/128] Adding comment --- pkg/reconciler/v1alpha1/trigger/trigger.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/reconciler/v1alpha1/trigger/trigger.go b/pkg/reconciler/v1alpha1/trigger/trigger.go index 17159cf0776..6258ca7195d 100644 --- a/pkg/reconciler/v1alpha1/trigger/trigger.go +++ b/pkg/reconciler/v1alpha1/trigger/trigger.go @@ -112,6 +112,7 @@ func ProvideController(logger *zap.Logger) func(manager.Manager) (controller.Con } } + // Watch for Broker changes. if err = c.Watch(&source.Kind{Type: &v1alpha1.Broker{}}, &handler.EnqueueRequestsFromMapFunc{ToRequests: &mapAllTriggers{r}}); err != nil { return nil, err } From 1628333455022889ccd03701a2a930185632dfd6 Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Wed, 13 Feb 2019 17:13:13 -0800 Subject: [PATCH 053/128] Rename Trigger.Spec.Filter.ExactMatch to Trigger.Spec.Filter.SourceAndType. Add unit tests for the filter binary. --- .../eventing/v1alpha1/trigger_defaults.go | 2 +- pkg/apis/eventing/v1alpha1/trigger_types.go | 4 +- .../eventing/v1alpha1/trigger_validation.go | 2 +- .../v1alpha1/zz_generated.deepcopy.go | 14 +- pkg/broker/receiver.go | 8 +- pkg/broker/receiver_test.go | 168 +++++++++++------- t2.yaml | 2 +- 7 files changed, 117 insertions(+), 83 deletions(-) diff --git a/pkg/apis/eventing/v1alpha1/trigger_defaults.go b/pkg/apis/eventing/v1alpha1/trigger_defaults.go index e3dd6c32818..b95ce930497 100644 --- a/pkg/apis/eventing/v1alpha1/trigger_defaults.go +++ b/pkg/apis/eventing/v1alpha1/trigger_defaults.go @@ -26,6 +26,6 @@ func (ts *TriggerSpec) SetDefaults() { } // Make a default filter that allows anything. if ts.Filter == nil { - ts.Filter = &TriggerFilter{&TriggerFilterAttributes{Type: TriggerAnyFilter, Source: TriggerAnyFilter}} + ts.Filter = &TriggerFilter{&TriggerFilterSourceAndType{Type: TriggerAnyFilter, Source: TriggerAnyFilter}} } } diff --git a/pkg/apis/eventing/v1alpha1/trigger_types.go b/pkg/apis/eventing/v1alpha1/trigger_types.go index c83873698bd..794a91b4dc3 100644 --- a/pkg/apis/eventing/v1alpha1/trigger_types.go +++ b/pkg/apis/eventing/v1alpha1/trigger_types.go @@ -67,10 +67,10 @@ type TriggerSpec struct { } type TriggerFilter struct { - ExactMatch *TriggerFilterAttributes `json:"exactMatch,omitempty"` + SourceAndType *TriggerFilterSourceAndType `json:"sourceAndType,omitempty"` } -type TriggerFilterAttributes struct { +type TriggerFilterSourceAndType struct { Type string `json:"type,omitempty"` Source string `json:"source,omitempty"` } diff --git a/pkg/apis/eventing/v1alpha1/trigger_validation.go b/pkg/apis/eventing/v1alpha1/trigger_validation.go index 4a149dabea5..3aca17c6581 100644 --- a/pkg/apis/eventing/v1alpha1/trigger_validation.go +++ b/pkg/apis/eventing/v1alpha1/trigger_validation.go @@ -37,7 +37,7 @@ func (ts *TriggerSpec) Validate() *apis.FieldError { errs = errs.Also(fe) } - if ts.Filter.ExactMatch == nil { + if ts.Filter.SourceAndType == nil { fe := apis.ErrMissingField("filter.exactMatch") errs = errs.Also(fe) } diff --git a/pkg/apis/eventing/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/eventing/v1alpha1/zz_generated.deepcopy.go index 89bd9d76f0b..44ba2ee9196 100644 --- a/pkg/apis/eventing/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/eventing/v1alpha1/zz_generated.deepcopy.go @@ -600,12 +600,12 @@ func (in *Trigger) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TriggerFilter) DeepCopyInto(out *TriggerFilter) { *out = *in - if in.ExactMatch != nil { - in, out := &in.ExactMatch, &out.ExactMatch + if in.SourceAndType != nil { + in, out := &in.SourceAndType, &out.SourceAndType if *in == nil { *out = nil } else { - *out = new(TriggerFilterAttributes) + *out = new(TriggerFilterSourceAndType) **out = **in } } @@ -623,17 +623,17 @@ func (in *TriggerFilter) DeepCopy() *TriggerFilter { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *TriggerFilterAttributes) DeepCopyInto(out *TriggerFilterAttributes) { +func (in *TriggerFilterSourceAndType) DeepCopyInto(out *TriggerFilterSourceAndType) { *out = *in return } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TriggerFilterAttributes. -func (in *TriggerFilterAttributes) DeepCopy() *TriggerFilterAttributes { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TriggerFilterSourceAndType. +func (in *TriggerFilterSourceAndType) DeepCopy() *TriggerFilterSourceAndType { if in == nil { return nil } - out := new(TriggerFilterAttributes) + out := new(TriggerFilterSourceAndType) in.DeepCopyInto(out) return out } diff --git a/pkg/broker/receiver.go b/pkg/broker/receiver.go index c6b6107b289..d51c3d6c755 100644 --- a/pkg/broker/receiver.go +++ b/pkg/broker/receiver.go @@ -94,12 +94,16 @@ func (r *Receiver) getTrigger(ctx context.Context, ref provisioners.ChannelRefer } func (r *Receiver) shouldSendMessage(ts *eventingv1alpha1.TriggerSpec, m *provisioners.Message) bool { - filterType := ts.Filter.ExactMatch.Type + if ts.Filter == nil || ts.Filter.SourceAndType == nil { + r.logger.Error("No filter specified") + return false + } + filterType := ts.Filter.SourceAndType.Type if filterType != eventingv1alpha1.TriggerAnyFilter && filterType != m.Headers["Ce-Eventtype"] { r.logger.Debug("Wrong type", zap.String("trigger.spec.filter.exactMatch.type", filterType), zap.String("message.type", m.Headers["Ce-Eventtype"])) return false } - filterSource := ts.Filter.ExactMatch.Source + filterSource := ts.Filter.SourceAndType.Source if filterSource != eventingv1alpha1.TriggerAnyFilter && filterSource != m.Headers["Ce-Source"] { r.logger.Debug("Wrong source", zap.String("trigger.spec.filter.exactMatch.source", filterSource), zap.String("message.source", m.Headers["Ce-Source"])) return false diff --git a/pkg/broker/receiver_test.go b/pkg/broker/receiver_test.go index aa553399351..d841d895f6b 100644 --- a/pkg/broker/receiver_test.go +++ b/pkg/broker/receiver_test.go @@ -17,13 +17,14 @@ package broker import ( - "context" + "errors" + "fmt" + "github.com/knative/eventing/pkg/provisioners" + "net/http" "net/http/httptest" "strings" "testing" - "github.com/knative/eventing/contrib/gcppubsub/pkg/util" - eventingv1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" "k8s.io/client-go/kubernetes/scheme" @@ -33,24 +34,13 @@ import ( "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client/fake" - - "github.com/knative/eventing/contrib/gcppubsub/pkg/util/testcreds" ) const ( - validMessage = `{ - "cloudEventsVersion" : "0.1", - "eventType" : "com.example.someevent", - "eventTypeVersion" : "1.0", - "source" : "/mycontext", - "eventID" : "A234-1234-1234", - "eventTime" : "2018-04-05T17:31:00Z", - "extensions" : { - "comExampleExtension" : "value" - }, - "contentType" : "text/xml", - "data" : "" -}` + testNS = "test-namespace" + triggerName = "test-trigger" + eventType = `"com.example.someevent"` + eventSource = `"/mycontext"` ) func init() { @@ -60,49 +50,55 @@ func init() { func TestReceiver(t *testing.T) { testCases := map[string]struct { - initialState []runtime.Object - expectedErr bool + initialState []runtime.Object + dispatchErr error + expectedErr bool + expectedDispatch bool }{ - "can't get channel": { + "Trigger.Get fails": { + // No trigger exists, so the Get will fail. + expectedErr: true, + }, + "Trigger doesn't have SubscriberURI": { initialState: []runtime.Object{ - testcreds.MakeSecretWithInvalidCreds(), + makeTriggerWithoutSubscriberURI(), }, expectedErr: true, }, - "can't read status": { + "Trigger without a Filter": { initialState: []runtime.Object{ - testcreds.MakeSecretWithInvalidCreds(), - makeChannelWithBadStatus(), + makeTriggerWithoutFilter(), }, - expectedErr: true, }, - "blank status": { + "Wrong type": { initialState: []runtime.Object{ - testcreds.MakeSecretWithInvalidCreds(), - makeChannelWithBlankStatus(), + makeTrigger("some-other-type", "Any"), }, - expectedErr: true, }, - "credential fails": { + "Wrong source": { initialState: []runtime.Object{ - testcreds.MakeSecretWithInvalidCreds(), - makeChannel(), + makeTrigger("Any", "some-other-source"), }, - expectedErr: true, }, - "PubSub client fails": { + "Dispatch failed": { initialState: []runtime.Object{ - testcreds.MakeSecretWithCreds(), - makeChannel(), + makeTrigger("Any", "Any"), }, - expectedErr: true, + dispatchErr: errors.New("test error dispatching"), + expectedErr: true, + expectedDispatch: true, }, - "Publish fails": { + "Dispatch succeeded - Any": { initialState: []runtime.Object{ - testcreds.MakeSecretWithCreds(), - makeChannel(), + makeTrigger("Any", "Any"), }, - expectedErr: true, + expectedDispatch: true, + }, + "Dispatch succeeded - Specific": { + initialState: []runtime.Object{ + makeTrigger(eventType, eventSource), + }, + expectedDispatch: true, }, } for n, tc := range testCases { @@ -110,10 +106,13 @@ func TestReceiver(t *testing.T) { mr, _ := New( zap.NewNop(), fake.NewFakeClient(tc.initialState...)) + fd := &fakeDispatcher{ + err: tc.dispatchErr, + } + mr.dispatcher = fd + resp := httptest.NewRecorder() - req := httptest.NewRequest("POST", "/", strings.NewReader(validMessage)) - req.Host = "test-trigger.test-namespace.triggers.cluster.local" - mr.newMessageReceiver().HandleRequest(resp, req) + mr.newMessageReceiver().HandleRequest(resp, makeRequest()) if tc.expectedErr { if resp.Result().StatusCode >= 200 && resp.Result().StatusCode < 300 { t.Errorf("Expected an error. Actual: %v", resp.Result()) @@ -123,43 +122,74 @@ func TestReceiver(t *testing.T) { t.Errorf("Expected success. Actual: %v", resp.Result()) } } + if tc.expectedDispatch != fd.requestReceived { + t.Errorf("Incorrect dispatch. Expected %v, Actual %v", tc.expectedDispatch, fd.requestReceived) + } }) } } -func makeChannel() *eventingv1alpha1.Channel { - c := &eventingv1alpha1.Channel{ +type fakeDispatcher struct { + err error + requestReceived bool +} + +func (d *fakeDispatcher) DispatchMessage(_ *provisioners.Message, _, _ string, _ provisioners.DispatchDefaults) error { + d.requestReceived = true + return d.err +} + +func makeTrigger(t, s string) *eventingv1alpha1.Trigger { + return &eventingv1alpha1.Trigger{ TypeMeta: v1.TypeMeta{ APIVersion: "eventing.knative.dev/v1alpha1", - Kind: "Channel", + Kind: "Trigger", }, ObjectMeta: v1.ObjectMeta{ - Namespace: "test-namespace", - Name: "test-channel", + Namespace: testNS, + Name: triggerName, + }, + Spec: eventingv1alpha1.TriggerSpec{ + Filter: &eventingv1alpha1.TriggerFilter{ + SourceAndType: &eventingv1alpha1.TriggerFilterSourceAndType{ + Type: t, + Source: s, + }, + }, + }, + Status: eventingv1alpha1.TriggerStatus{ + SubscriberURI: "subscriberURI", }, } - pcs := &util.GcpPubSubChannelStatus{ - GCPProject: "project", - Secret: testcreds.Secret, - SecretKey: testcreds.SecretKey, - } - if err := util.SetInternalStatus(context.Background(), c, pcs); err != nil { - panic(err) - } - return c } -func makeChannelWithBlankStatus() *eventingv1alpha1.Channel { - c := makeChannel() - c.Status = eventingv1alpha1.ChannelStatus{} - return c +func makeTriggerWithoutFilter() *eventingv1alpha1.Trigger { + t := makeTrigger("Any", "Any") + t.Spec.Filter = nil + return t } -func makeChannelWithBadStatus() *eventingv1alpha1.Channel { - c := makeChannel() - c.Status.Internal = &runtime.RawExtension{ - // SecretKey must be a string, not an integer, so this will fail during json.Unmarshal. - Raw: []byte(`{"secretKey": 123}`), +func makeTriggerWithoutSubscriberURI() *eventingv1alpha1.Trigger { + t := makeTrigger("Any", "Any") + t.Status = eventingv1alpha1.TriggerStatus{} + return t +} + +func makeRequest() *http.Request { + req := httptest.NewRequest("POST", "/", strings.NewReader(``)) + req.Host = fmt.Sprintf("%s.%s.triggers.cluster.local", triggerName, testNS) + + eventAttributes := map[string]string{ + "CE-CloudEventsVersion": `"0.1"`, + "CE-EventType": eventType, + "CE-EventTypeVersion": `"1.0"`, + "CE-Source": eventSource, + "CE-EventID": `"A234-1234-1234"`, + "CE-EventTime": `"2018-04-05T17:31:00Z"`, + "contentType": "text/xml", + } + for k, v := range eventAttributes { + req.Header.Set(k, v) } - return c + return req } diff --git a/t2.yaml b/t2.yaml index 6274a6166b4..105b912796e 100644 --- a/t2.yaml +++ b/t2.yaml @@ -6,7 +6,7 @@ spec: # Our Cloud Event parsing library seems to have a bug and forces to put type and source in double # quotes (as it thinks the actual value is `"foo"`, not `foo`). filter: - exactMatch: + sourceAndType: type: '"com.example.someevent"' source: '"/mycontext/subcontext"' subscriber: From cbbd06615687a6012261e4ecaadef702a7dd5649 Mon Sep 17 00:00:00 2001 From: Nacho Cano Date: Wed, 13 Feb 2019 23:18:46 -0800 Subject: [PATCH 054/128] Adding event messages --- pkg/apis/eventing/v1alpha1/trigger_validation.go | 2 +- pkg/reconciler/v1alpha1/trigger/trigger.go | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/apis/eventing/v1alpha1/trigger_validation.go b/pkg/apis/eventing/v1alpha1/trigger_validation.go index 3aca17c6581..073784972b1 100644 --- a/pkg/apis/eventing/v1alpha1/trigger_validation.go +++ b/pkg/apis/eventing/v1alpha1/trigger_validation.go @@ -38,7 +38,7 @@ func (ts *TriggerSpec) Validate() *apis.FieldError { } if ts.Filter.SourceAndType == nil { - fe := apis.ErrMissingField("filter.exactMatch") + fe := apis.ErrMissingField("filter.sourceAndType") errs = errs.Also(fe) } diff --git a/pkg/reconciler/v1alpha1/trigger/trigger.go b/pkg/reconciler/v1alpha1/trigger/trigger.go index 6258ca7195d..b86d694a525 100644 --- a/pkg/reconciler/v1alpha1/trigger/trigger.go +++ b/pkg/reconciler/v1alpha1/trigger/trigger.go @@ -66,6 +66,8 @@ const ( triggerReconciled = "TriggerReconciled" triggerReconcileFailed = "TriggerReconcileFailed" triggerUpdateStatusFailed = "TriggerUpdateStatusFailed" + subscriptionDeleteFailed = "SubscriptionDeleteFailed" + subscriptionCreateFailed = "SubscriptionCreateFailed" ) type reconciler struct { @@ -601,12 +603,14 @@ func (r *reconciler) subscribeToBrokerChannel(ctx context.Context, t *v1alpha1.T err = r.client.Delete(ctx, sub) if err != nil { logging.FromContext(ctx).Info("Cannot delete subscription", zap.Error(err)) + r.recorder.Eventf(t, corev1.EventTypeWarning, subscriptionDeleteFailed, "Delete Trigger's subscription failed: %v", err) return nil, err } sub = expected err = r.client.Create(ctx, sub) if err != nil { logging.FromContext(ctx).Info("Cannot create subscription", zap.Error(err)) + r.recorder.Eventf(t, corev1.EventTypeWarning, subscriptionCreateFailed, "Create Trigger's subscription failed: %v", err) return nil, err } } From c33c5691e7c0dde9447d391408689b5ef9d320ba Mon Sep 17 00:00:00 2001 From: Nacho Cano Date: Wed, 13 Feb 2019 23:52:47 -0800 Subject: [PATCH 055/128] Using the broker's namespacedNamed as key to the triggers map. With this we allow to reconcile only the triggers that belong to the particular broker that changed --- pkg/reconciler/v1alpha1/trigger/trigger.go | 48 ++++++++++++++-------- 1 file changed, 31 insertions(+), 17 deletions(-) diff --git a/pkg/reconciler/v1alpha1/trigger/trigger.go b/pkg/reconciler/v1alpha1/trigger/trigger.go index b86d694a525..b7c3252c3df 100644 --- a/pkg/reconciler/v1alpha1/trigger/trigger.go +++ b/pkg/reconciler/v1alpha1/trigger/trigger.go @@ -70,6 +70,8 @@ const ( subscriptionCreateFailed = "SubscriptionCreateFailed" ) +var dummyValue struct{} + type reconciler struct { client client.Client restConfig *rest.Config @@ -77,7 +79,10 @@ type reconciler struct { recorder record.EventRecorder triggersLock sync.RWMutex - triggers map[string]map[reconcile.Request]bool + // Contains the triggers that correspond to a particular broker. + // We use this to reconcile only the triggers that correspond to a certain broker. + // brokerNamespacedName -> triggerReconcileRequest -> dummy struct + triggers map[types.NamespacedName]map[reconcile.Request]struct{} logger *zap.Logger } @@ -92,7 +97,7 @@ func ProvideController(logger *zap.Logger) func(manager.Manager) (controller.Con r := &reconciler{ recorder: mgr.GetRecorder(controllerAgentName), logger: logger, - triggers: make(map[string]map[reconcile.Request]bool), + triggers: make(map[types.NamespacedName]map[reconcile.Request]struct{}), } c, err := controller.New(controllerAgentName, mgr, controller.Options{ Reconciler: r, @@ -133,12 +138,13 @@ type mapAllTriggers struct { func (m *mapAllTriggers) Map(o handler.MapObject) []reconcile.Request { m.r.triggersLock.RLock() defer m.r.triggersLock.RUnlock() - triggersInNamespace := m.r.triggers[o.Meta.GetNamespace()] - if triggersInNamespace == nil { + brokerNamespacedName := types.NamespacedName{Namespace: o.Meta.GetNamespace(), Name: o.Meta.GetName()} + triggersInBrokerNamespacedName := m.r.triggers[brokerNamespacedName] + if triggersInBrokerNamespacedName == nil { return []reconcile.Request{} } - reqs := make([]reconcile.Request, 0, len(triggersInNamespace)) - for name := range triggersInNamespace { + reqs := make([]reconcile.Request, 0, len(triggersInBrokerNamespacedName)) + for name := range triggersInBrokerNamespacedName { reqs = append(reqs, name) } return reqs @@ -273,13 +279,17 @@ func (r *reconciler) AddToTriggers(t *v1alpha1.Trigger) { }, } + brokerNamespacedName := types.NamespacedName{ + Namespace: t.Namespace, + Name: t.Spec.Broker} + // We will be reconciling an already existing Trigger far more often than adding a new one, so // check with a read lock before using the write lock. r.triggersLock.RLock() - triggersInNamespace := r.triggers[t.Namespace] + triggersInBrokerNamespacedName := r.triggers[brokerNamespacedName] var present bool - if triggersInNamespace != nil { - _, present = triggersInNamespace[name] + if triggersInBrokerNamespacedName != nil { + _, present = triggersInBrokerNamespacedName[name] } else { present = false } @@ -291,12 +301,12 @@ func (r *reconciler) AddToTriggers(t *v1alpha1.Trigger) { } r.triggersLock.Lock() - triggersInNamespace = r.triggers[t.Namespace] - if triggersInNamespace == nil { - r.triggers[t.Namespace] = make(map[reconcile.Request]bool) - triggersInNamespace = r.triggers[t.Namespace] + triggersInBrokerNamespacedName = r.triggers[brokerNamespacedName] + if triggersInBrokerNamespacedName == nil { + r.triggers[brokerNamespacedName] = make(map[reconcile.Request]struct{}) + triggersInBrokerNamespacedName = r.triggers[brokerNamespacedName] } - triggersInNamespace[name] = false + triggersInBrokerNamespacedName[name] = dummyValue r.triggersLock.Unlock() } @@ -308,10 +318,14 @@ func (r *reconciler) removeFromTriggers(t *v1alpha1.Trigger) { }, } + brokerNamespacedName := types.NamespacedName{ + Namespace: t.Namespace, + Name: t.Spec.Broker} + r.triggersLock.Lock() - triggersInNamespace := r.triggers[t.Namespace] - if triggersInNamespace != nil { - delete(triggersInNamespace, name) + triggersInBrokerNamespacedName := r.triggers[brokerNamespacedName] + if triggersInBrokerNamespacedName != nil { + delete(triggersInBrokerNamespacedName, name) } r.triggersLock.Unlock() } From 40ca8da7bf7d9a02c4bcba32e566d75124d6518d Mon Sep 17 00:00:00 2001 From: nachocano Date: Thu, 14 Feb 2019 14:44:18 -0800 Subject: [PATCH 056/128] Adding some tests to trigger --- pkg/reconciler/v1alpha1/trigger/trigger.go | 2 +- .../v1alpha1/trigger/trigger_test.go | 364 +++++++++++++++++- 2 files changed, 364 insertions(+), 2 deletions(-) diff --git a/pkg/reconciler/v1alpha1/trigger/trigger.go b/pkg/reconciler/v1alpha1/trigger/trigger.go index b7c3252c3df..42404fed7fa 100644 --- a/pkg/reconciler/v1alpha1/trigger/trigger.go +++ b/pkg/reconciler/v1alpha1/trigger/trigger.go @@ -193,7 +193,7 @@ func (r *reconciler) Reconcile(request reconcile.Request) (reconcile.Result, err r.recorder.Eventf(trigger, corev1.EventTypeWarning, triggerReconcileFailed, "Trigger reconciliation failed: %v", reconcileErr) } else { logging.FromContext(ctx).Debug("Trigger reconciled") - r.recorder.Eventf(trigger, corev1.EventTypeNormal, triggerReconciled, "Trigger reconciled: %q", trigger.Name) + r.recorder.Event(trigger, corev1.EventTypeNormal, triggerReconciled, "Trigger reconciled") } if _, err = r.updateStatus(trigger); err != nil { diff --git a/pkg/reconciler/v1alpha1/trigger/trigger_test.go b/pkg/reconciler/v1alpha1/trigger/trigger_test.go index 298a3a48f5c..454875e81a1 100644 --- a/pkg/reconciler/v1alpha1/trigger/trigger_test.go +++ b/pkg/reconciler/v1alpha1/trigger/trigger_test.go @@ -17,13 +17,67 @@ limitations under the License. package trigger import ( + "context" + "errors" + "fmt" + "testing" + + "k8s.io/apimachinery/pkg/runtime/schema" + + "sigs.k8s.io/controller-runtime/pkg/client" + + "k8s.io/apimachinery/pkg/util/intstr" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "github.com/google/go-cmp/cmp" + "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" + controllertesting "github.com/knative/eventing/pkg/reconciler/testing" + duckv1alpha1 "github.com/knative/pkg/apis/duck/v1alpha1" + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client/fake" - "testing" ) +const ( + testNS = "test-namespace" + triggerName = "test-trigger" + brokerName = "test-broker" + + subscriberAPIVersion = "v1" + subscriberKind = "Service" + subscriberName = "subscriberName" + + channelHostname = "foo.bar.svc.cluster.local" + + k8sServiceName = "k8sServiceName" +) + +var ( + trueVal = true + // deletionTime is used when objects are marked as deleted. Rfc3339Copy() + // truncates to seconds to match the loss of precision during serialization. + deletionTime = metav1.Now().Rfc3339Copy() + + // Map of events to set test cases' expectations easier. + events = map[string]corev1.Event{ + triggerReconciled: {Reason: triggerReconciled, Type: corev1.EventTypeNormal}, + triggerUpdateStatusFailed: {Reason: triggerUpdateStatusFailed, Type: corev1.EventTypeWarning}, + triggerReconcileFailed: {Reason: triggerReconcileFailed, Type: corev1.EventTypeWarning}, + } +) + +func init() { + // Add types to scheme + v1alpha1.AddToScheme(scheme.Scheme) +} + func TestProvideController(t *testing.T) { //TODO(grantr) This needs a mock of manager.Manager. Creating a manager // with a fake Config fails because the Manager tries to contact the @@ -89,3 +143,311 @@ func TestInjectConfig(t *testing.T) { t.Errorf("Unexpected dynamicClient type. Expected: %T, Got: %T", wantDynClient, r.dynamicClient) } } + +func TestReconcile(t *testing.T) { + testCases := []controllertesting.TestCase{ + { + Name: "Trigger not found", + }, + { + Name: "Get Trigger error", + Scheme: scheme.Scheme, + Mocks: controllertesting.Mocks{ + MockGets: []controllertesting.MockGet{ + func(_ client.Client, _ context.Context, _ client.ObjectKey, obj runtime.Object) (controllertesting.MockHandled, error) { + if _, ok := obj.(*v1alpha1.Trigger); ok { + return controllertesting.Handled, errors.New("test error getting the Trigger") + } + return controllertesting.Unhandled, nil + }, + }, + }, + WantErrMsg: "test error getting the Trigger", + }, + { + Name: "Trigger being deleted", + Scheme: scheme.Scheme, + InitialState: []runtime.Object{ + makeDeletingTrigger(), + }, + WantEvent: []corev1.Event{events[triggerReconciled]}, + }, + { + Name: "Get Broker error", + Scheme: scheme.Scheme, + InitialState: []runtime.Object{ + makeTrigger(), + }, + Mocks: controllertesting.Mocks{ + MockGets: []controllertesting.MockGet{ + func(_ client.Client, _ context.Context, _ client.ObjectKey, obj runtime.Object) (controllertesting.MockHandled, error) { + if _, ok := obj.(*v1alpha1.Broker); ok { + return controllertesting.Handled, errors.New("test error getting broker") + } + return controllertesting.Unhandled, nil + }, + }, + }, + WantErrMsg: "test error getting broker", + WantEvent: []corev1.Event{events[triggerReconcileFailed]}, + }, + { + Name: "Get Broker channel error", + Scheme: scheme.Scheme, + InitialState: []runtime.Object{ + makeTrigger(), + makeBroker(), + }, + Mocks: controllertesting.Mocks{ + MockLists: []controllertesting.MockList{ + func(_ client.Client, _ context.Context, _ *client.ListOptions, list runtime.Object) (controllertesting.MockHandled, error) { + if _, ok := list.(*v1alpha1.ChannelList); ok { + return controllertesting.Handled, errors.New("test error getting broker's channel") + } + return controllertesting.Unhandled, nil + }, + }, + }, + WantErrMsg: "test error getting broker's channel", + WantEvent: []corev1.Event{events[triggerReconcileFailed]}, + }, + { + Name: "Resolve subscriberURI error", + Scheme: scheme.Scheme, + InitialState: []runtime.Object{ + makeTrigger(), + makeBroker(), + makeChannel(), + }, + Mocks: controllertesting.Mocks{ + MockGets: []controllertesting.MockGet{ + func(_ client.Client, _ context.Context, key client.ObjectKey, obj runtime.Object) (controllertesting.MockHandled, error) { + if _, ok := obj.(*corev1.Service); ok { + return controllertesting.Handled, errors.New("test error resolving subscriber URI") + } + return controllertesting.Unhandled, nil + }, + }, + }, + WantErrMsg: "test error resolving subscriber URI", + WantEvent: []corev1.Event{events[triggerReconcileFailed]}, + }, + { + Name: "Create K8s Service error", + Scheme: scheme.Scheme, + InitialState: []runtime.Object{ + makeTrigger(), + makeBroker(), + makeChannel(), + makeSubscriberService(), + }, + Mocks: controllertesting.Mocks{ + MockCreates: []controllertesting.MockCreate{ + func(_ client.Client, _ context.Context, obj runtime.Object) (controllertesting.MockHandled, error) { + if _, ok := obj.(*corev1.Service); ok { + return controllertesting.Handled, errors.New("test error creating k8s service") + } + return controllertesting.Unhandled, nil + }, + }, + }, + WantErrMsg: "test error creating k8s service", + WantEvent: []corev1.Event{events[triggerReconcileFailed]}, + }, + { + Name: "Update K8s Service error", + Scheme: scheme.Scheme, + InitialState: []runtime.Object{ + makeTrigger(), + makeBroker(), + makeChannel(), + makeSubscriberService(), + makeK8sService(), + }, + Mocks: controllertesting.Mocks{ + MockUpdates: []controllertesting.MockUpdate{ + func(_ client.Client, _ context.Context, obj runtime.Object) (controllertesting.MockHandled, error) { + if _, ok := obj.(*corev1.Service); ok { + return controllertesting.Handled, errors.New("test error updating k8s service") + } + return controllertesting.Unhandled, nil + }, + }, + }, + WantErrMsg: "test error updating k8s service", + WantEvent: []corev1.Event{events[triggerReconcileFailed]}, + }, + } + for _, tc := range testCases { + c := tc.GetClient() + dc := tc.GetDynamicClient() + recorder := tc.GetEventRecorder() + + r := &reconciler{ + client: c, + dynamicClient: dc, + restConfig: &rest.Config{}, + recorder: recorder, + logger: zap.NewNop(), + triggers: make(map[types.NamespacedName]map[reconcile.Request]struct{}), + } + tc.ReconcileKey = fmt.Sprintf("%s/%s", testNS, triggerName) + tc.IgnoreTimes = true + t.Run(tc.Name, tc.Runner(t, r, c, recorder)) + } +} + +func makeTrigger() *v1alpha1.Trigger { + return &v1alpha1.Trigger{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "eventing.knative.dev/v1alpha1", + Kind: "Trigger", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: testNS, + Name: triggerName, + }, + Spec: v1alpha1.TriggerSpec{ + Broker: brokerName, + Filter: &v1alpha1.TriggerFilter{ + SourceAndType: &v1alpha1.TriggerFilterSourceAndType{ + Source: "Any", + Type: "Any", + }, + }, + Subscriber: &v1alpha1.SubscriberSpec{ + Ref: &corev1.ObjectReference{ + Name: subscriberName, + Kind: subscriberKind, + APIVersion: subscriberAPIVersion, + }, + }, + }, + } +} + +func makeReadyTrigger() *v1alpha1.Trigger { + t := makeTrigger() + t.Status.InitializeConditions() + t.Status.MarkBrokerExists() + t.Status.SubscriberURI = fmt.Sprintf("%s-trigger.%s.svc.cluster.local", triggerName, testNS) + t.Status.MarkKubernetesServiceExists() + t.Status.MarkVirtualServiceExists() + t.Status.MarkSubscribed() + return t +} + +func makeDeletingTrigger() *v1alpha1.Trigger { + b := makeReadyTrigger() + b.DeletionTimestamp = &deletionTime + return b +} + +func makeBroker() *v1alpha1.Broker { + return &v1alpha1.Broker{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "eventing.knative.dev/v1alpha1", + Kind: "Broker", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: testNS, + Name: brokerName, + }, + Spec: v1alpha1.BrokerSpec{ + ChannelTemplate: &v1alpha1.ChannelSpec{ + Provisioner: makeChannelProvisioner(), + }, + }, + } +} + +func makeChannelProvisioner() *corev1.ObjectReference { + return &corev1.ObjectReference{ + APIVersion: "eventing.knative.dev/v1alpha1", + Kind: "ClusterChannelProvisioner", + Name: "my-provisioner", + } +} + +func makeChannel() *v1alpha1.Channel { + return &v1alpha1.Channel{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testNS, + GenerateName: fmt.Sprintf("%s-broker-", brokerName), + Labels: map[string]string{ + "eventing.knative.dev/broker": brokerName, + "eventing.knative.dev/brokerEverything": "true", + }, + OwnerReferences: []metav1.OwnerReference{ + getOwnerReference(), + }, + }, + Spec: v1alpha1.ChannelSpec{ + Provisioner: makeChannelProvisioner(), + }, + Status: v1alpha1.ChannelStatus{ + Address: duckv1alpha1.Addressable{ + Hostname: channelHostname, + }, + }, + } +} + +func makeSubscriberService() *corev1.Service { + return &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testNS, + Name: subscriberName, + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "http", + Port: 80, + TargetPort: intstr.FromInt(8080), + }, + }, + }, + } +} + +func makeK8sService() *corev1.Service { + return &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testNS, + Name: k8sServiceName, + Labels: map[string]string{ + "eventing.knative.dev/trigger": triggerName, + }, + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(makeTrigger(), schema.GroupVersionKind{ + Group: v1alpha1.SchemeGroupVersion.Group, + Version: v1alpha1.SchemeGroupVersion.Version, + Kind: "Trigger", + }), + }, + }, + Spec: corev1.ServiceSpec{ + Selector: map[string]string{ + "eventing.knative.dev/trigger": triggerName, + }, + Ports: []corev1.ServicePort{ + { + Name: "http", + Port: 80, + TargetPort: intstr.FromInt(8080), + }, + }, + }, + } +} + +func getOwnerReference() metav1.OwnerReference { + return metav1.OwnerReference{ + APIVersion: v1alpha1.SchemeGroupVersion.String(), + Kind: "Broker", + Name: brokerName, + Controller: &trueVal, + BlockOwnerDeletion: &trueVal, + } +} From 650ef475f3b8c29db19fb290cf996d78ca3e422f Mon Sep 17 00:00:00 2001 From: nachocano Date: Thu, 14 Feb 2019 16:22:52 -0800 Subject: [PATCH 057/128] More UTs --- .../v1alpha1/trigger/trigger_test.go | 105 ++++++++++++------ 1 file changed, 74 insertions(+), 31 deletions(-) diff --git a/pkg/reconciler/v1alpha1/trigger/trigger_test.go b/pkg/reconciler/v1alpha1/trigger/trigger_test.go index 454875e81a1..9b873d3c479 100644 --- a/pkg/reconciler/v1alpha1/trigger/trigger_test.go +++ b/pkg/reconciler/v1alpha1/trigger/trigger_test.go @@ -22,13 +22,13 @@ import ( "fmt" "testing" - "k8s.io/apimachinery/pkg/runtime/schema" + "github.com/knative/eventing/pkg/reconciler/names" + "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -36,6 +36,7 @@ import ( "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" controllertesting "github.com/knative/eventing/pkg/reconciler/testing" duckv1alpha1 "github.com/knative/pkg/apis/duck/v1alpha1" + istiov1alpha3 "github.com/knative/pkg/apis/istio/v1alpha3" "go.uber.org/zap" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -55,8 +56,6 @@ const ( subscriberName = "subscriberName" channelHostname = "foo.bar.svc.cluster.local" - - k8sServiceName = "k8sServiceName" ) var ( @@ -76,6 +75,7 @@ var ( func init() { // Add types to scheme v1alpha1.AddToScheme(scheme.Scheme) + istiov1alpha3.AddToScheme(scheme.Scheme) } func TestProvideController(t *testing.T) { @@ -262,7 +262,7 @@ func TestReconcile(t *testing.T) { makeBroker(), makeChannel(), makeSubscriberService(), - makeK8sService(), + makeDifferentK8sService(), }, Mocks: controllertesting.Mocks{ MockUpdates: []controllertesting.MockUpdate{ @@ -277,6 +277,53 @@ func TestReconcile(t *testing.T) { WantErrMsg: "test error updating k8s service", WantEvent: []corev1.Event{events[triggerReconcileFailed]}, }, + { + Name: "Create Virtual Service error", + Scheme: scheme.Scheme, + InitialState: []runtime.Object{ + makeTrigger(), + makeBroker(), + makeChannel(), + makeSubscriberService(), + makeK8sService(), + }, + Mocks: controllertesting.Mocks{ + MockCreates: []controllertesting.MockCreate{ + func(_ client.Client, _ context.Context, obj runtime.Object) (controllertesting.MockHandled, error) { + if _, ok := obj.(*istiov1alpha3.VirtualService); ok { + return controllertesting.Handled, errors.New("test error creating virtual service") + } + return controllertesting.Unhandled, nil + }, + }, + }, + WantErrMsg: "test error creating virtual service", + WantEvent: []corev1.Event{events[triggerReconcileFailed]}, + }, + { + Name: "Update Virtual Service error", + Scheme: scheme.Scheme, + InitialState: []runtime.Object{ + makeTrigger(), + makeBroker(), + makeChannel(), + makeSubscriberService(), + makeK8sService(), + makeDifferentVirtualService(), + }, + Mocks: controllertesting.Mocks{ + MockUpdates: []controllertesting.MockUpdate{ + func(_ client.Client, _ context.Context, obj runtime.Object) (controllertesting.MockHandled, error) { + if _, ok := obj.(*istiov1alpha3.VirtualService); ok { + return controllertesting.Handled, errors.New("test error updating virtual service") + } + return controllertesting.Unhandled, nil + }, + }, + }, + WantErrMsg: "test error updating virtual service", + WantEvent: []corev1.Event{events[triggerReconcileFailed]}, + }, } for _, tc := range testCases { c := tc.GetClient() @@ -412,34 +459,30 @@ func makeSubscriberService() *corev1.Service { } func makeK8sService() *corev1.Service { - return &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: testNS, - Name: k8sServiceName, - Labels: map[string]string{ - "eventing.knative.dev/trigger": triggerName, - }, - OwnerReferences: []metav1.OwnerReference{ - *metav1.NewControllerRef(makeTrigger(), schema.GroupVersionKind{ - Group: v1alpha1.SchemeGroupVersion.Group, - Version: v1alpha1.SchemeGroupVersion.Version, - Kind: "Trigger", - }), - }, - }, - Spec: corev1.ServiceSpec{ - Selector: map[string]string{ - "eventing.knative.dev/trigger": triggerName, - }, - Ports: []corev1.ServicePort{ - { - Name: "http", - Port: 80, - TargetPort: intstr.FromInt(8080), - }, - }, + return newK8sService(makeTrigger()) +} + +func makeDifferentK8sService() *corev1.Service { + svc := makeK8sService() + svc.Spec.Ports = []corev1.ServicePort{ + { + Name: "http", + Port: 9999, }, } + return svc +} + +func makeVirtualService() *istiov1alpha3.VirtualService { + return newVirtualService(makeTrigger(), makeK8sService()) +} + +func makeDifferentVirtualService() *istiov1alpha3.VirtualService { + vsvc := makeVirtualService() + vsvc.Spec.Hosts = []string{ + names.ServiceHostName("other_svc_name", "other_svc_namespace"), + } + return vsvc } func getOwnerReference() metav1.OwnerReference { From d2c1ef8495c8aa041277f64ce1359815913b63ee Mon Sep 17 00:00:00 2001 From: nachocano Date: Thu, 14 Feb 2019 17:45:25 -0800 Subject: [PATCH 058/128] More UTs --- .../v1alpha1/trigger/trigger_test.go | 75 +++++++++++++++++-- 1 file changed, 69 insertions(+), 6 deletions(-) diff --git a/pkg/reconciler/v1alpha1/trigger/trigger_test.go b/pkg/reconciler/v1alpha1/trigger/trigger_test.go index 9b873d3c479..c5c6a59b1fc 100644 --- a/pkg/reconciler/v1alpha1/trigger/trigger_test.go +++ b/pkg/reconciler/v1alpha1/trigger/trigger_test.go @@ -25,11 +25,10 @@ import ( "github.com/knative/eventing/pkg/reconciler/names" "k8s.io/apimachinery/pkg/runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" "github.com/google/go-cmp/cmp" @@ -55,7 +54,8 @@ const ( subscriberKind = "Service" subscriberName = "subscriberName" - channelHostname = "foo.bar.svc.cluster.local" + channelHostname = "foo.bar.svc.cluster.local" + channelProvisioner = "my-channel-provisioner" ) var ( @@ -69,6 +69,8 @@ var ( triggerReconciled: {Reason: triggerReconciled, Type: corev1.EventTypeNormal}, triggerUpdateStatusFailed: {Reason: triggerUpdateStatusFailed, Type: corev1.EventTypeWarning}, triggerReconcileFailed: {Reason: triggerReconcileFailed, Type: corev1.EventTypeWarning}, + subscriptionDeleteFailed: {Reason: subscriptionDeleteFailed, Type: corev1.EventTypeWarning}, + subscriptionCreateFailed: {Reason: subscriptionCreateFailed, Type: corev1.EventTypeWarning}, } ) @@ -324,6 +326,55 @@ func TestReconcile(t *testing.T) { WantErrMsg: "test error updating virtual service", WantEvent: []corev1.Event{events[triggerReconcileFailed]}, }, + { + Name: "Create Subscription error", + Scheme: scheme.Scheme, + InitialState: []runtime.Object{ + makeTrigger(), + makeBroker(), + makeChannel(), + makeSubscriberService(), + makeK8sService(), + makeVirtualService(), + }, + Mocks: controllertesting.Mocks{ + MockCreates: []controllertesting.MockCreate{ + func(_ client.Client, _ context.Context, obj runtime.Object) (controllertesting.MockHandled, error) { + if _, ok := obj.(*v1alpha1.Subscription); ok { + return controllertesting.Handled, errors.New("test error creating subscription") + } + return controllertesting.Unhandled, nil + }, + }, + }, + WantErrMsg: "test error creating subscription", + WantEvent: []corev1.Event{events[triggerReconcileFailed]}, + }, + { + Name: "Delete Subscription error", + Scheme: scheme.Scheme, + InitialState: []runtime.Object{ + makeTrigger(), + makeBroker(), + makeChannel(), + makeSubscriberService(), + makeK8sService(), + makeVirtualService(), + makeDifferentSubscription(), + }, + Mocks: controllertesting.Mocks{ + MockDeletes: []controllertesting.MockDelete{ + func(_ client.Client, _ context.Context, obj runtime.Object) (controllertesting.MockHandled, error) { + if _, ok := obj.(*v1alpha1.Subscription); ok { + return controllertesting.Handled, errors.New("test error deleting subscription") + } + return controllertesting.Unhandled, nil + }, + }, + }, + WantErrMsg: "test error deleting subscription", + WantEvent: []corev1.Event{events[subscriptionDeleteFailed], events[triggerReconcileFailed]}, + }, } for _, tc := range testCases { c := tc.GetClient() @@ -416,11 +467,11 @@ func makeChannelProvisioner() *corev1.ObjectReference { } } -func makeChannel() *v1alpha1.Channel { +func newChannel(name string) *v1alpha1.Channel { return &v1alpha1.Channel{ ObjectMeta: metav1.ObjectMeta{ - Namespace: testNS, - GenerateName: fmt.Sprintf("%s-broker-", brokerName), + Namespace: testNS, + Name: name, Labels: map[string]string{ "eventing.knative.dev/broker": brokerName, "eventing.knative.dev/brokerEverything": "true", @@ -440,6 +491,14 @@ func makeChannel() *v1alpha1.Channel { } } +func makeChannel() *v1alpha1.Channel { + return newChannel(fmt.Sprintf("%s-broker", brokerName)) +} + +func makeDifferentChannel() *v1alpha1.Channel { + return newChannel(fmt.Sprintf("%s-broker-different", brokerName)) +} + func makeSubscriberService() *corev1.Service { return &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -485,6 +544,10 @@ func makeDifferentVirtualService() *istiov1alpha3.VirtualService { return vsvc } +func makeDifferentSubscription() *v1alpha1.Subscription { + return makeSubscription(makeTrigger(), makeDifferentChannel(), makeK8sService()) +} + func getOwnerReference() metav1.OwnerReference { return metav1.OwnerReference{ APIVersion: v1alpha1.SchemeGroupVersion.String(), From ae040072a7d2ef575cb5bf9716ae2b97e0b0a6f4 Mon Sep 17 00:00:00 2001 From: nachocano Date: Thu, 14 Feb 2019 18:01:30 -0800 Subject: [PATCH 059/128] More UTs --- .../v1alpha1/trigger/trigger_test.go | 76 ++++++++++++++++++- 1 file changed, 75 insertions(+), 1 deletion(-) diff --git a/pkg/reconciler/v1alpha1/trigger/trigger_test.go b/pkg/reconciler/v1alpha1/trigger/trigger_test.go index c5c6a59b1fc..bb1b8e0336e 100644 --- a/pkg/reconciler/v1alpha1/trigger/trigger_test.go +++ b/pkg/reconciler/v1alpha1/trigger/trigger_test.go @@ -22,6 +22,8 @@ import ( "fmt" "testing" + "github.com/knative/eventing/pkg/provisioners" + "github.com/knative/eventing/pkg/reconciler/names" "k8s.io/apimachinery/pkg/runtime" @@ -375,6 +377,73 @@ func TestReconcile(t *testing.T) { WantErrMsg: "test error deleting subscription", WantEvent: []corev1.Event{events[subscriptionDeleteFailed], events[triggerReconcileFailed]}, }, + { + Name: "Re-create Subscription error", + Scheme: scheme.Scheme, + InitialState: []runtime.Object{ + makeTrigger(), + makeBroker(), + makeChannel(), + makeSubscriberService(), + makeK8sService(), + makeVirtualService(), + makeDifferentSubscription(), + }, + Mocks: controllertesting.Mocks{ + MockCreates: []controllertesting.MockCreate{ + func(_ client.Client, _ context.Context, obj runtime.Object) (controllertesting.MockHandled, error) { + if _, ok := obj.(*v1alpha1.Subscription); ok { + return controllertesting.Handled, errors.New("test error re-creating subscription") + } + return controllertesting.Unhandled, nil + }, + }, + }, + WantErrMsg: "test error re-creating subscription", + WantEvent: []corev1.Event{events[subscriptionCreateFailed], events[triggerReconcileFailed]}, + }, + { + Name: "Update status error", + Scheme: scheme.Scheme, + InitialState: []runtime.Object{ + makeTrigger(), + makeBroker(), + makeChannel(), + makeSubscriberService(), + makeK8sService(), + makeVirtualService(), + makeSameSubscription(), + }, + Mocks: controllertesting.Mocks{ + MockStatusUpdates: []controllertesting.MockStatusUpdate{ + func(_ client.Client, _ context.Context, obj runtime.Object) (controllertesting.MockHandled, error) { + if _, ok := obj.(*v1alpha1.Trigger); ok { + return controllertesting.Handled, errors.New("test error updating trigger status") + } + return controllertesting.Unhandled, nil + }, + }, + }, + WantErrMsg: "test error updating trigger status", + WantEvent: []corev1.Event{events[triggerReconciled], events[triggerUpdateStatusFailed]}, + }, + { + Name: "Trigger reconciliation success", + Scheme: scheme.Scheme, + InitialState: []runtime.Object{ + makeTrigger(), + makeBroker(), + makeChannel(), + makeSubscriberService(), + makeK8sService(), + makeVirtualService(), + makeSameSubscription(), + }, + WantEvent: []corev1.Event{events[triggerReconciled]}, + WantPresent: []runtime.Object{ + makeReadyTrigger(), + }, + }, } for _, tc := range testCases { c := tc.GetClient() @@ -426,9 +495,10 @@ func makeTrigger() *v1alpha1.Trigger { func makeReadyTrigger() *v1alpha1.Trigger { t := makeTrigger() + provisioners.AddFinalizer(t, finalizerName) t.Status.InitializeConditions() t.Status.MarkBrokerExists() - t.Status.SubscriberURI = fmt.Sprintf("%s-trigger.%s.svc.cluster.local", triggerName, testNS) + t.Status.SubscriberURI = fmt.Sprintf("http://%s.%s.svc.cluster.local/", subscriberName, testNS) t.Status.MarkKubernetesServiceExists() t.Status.MarkVirtualServiceExists() t.Status.MarkSubscribed() @@ -544,6 +614,10 @@ func makeDifferentVirtualService() *istiov1alpha3.VirtualService { return vsvc } +func makeSameSubscription() *v1alpha1.Subscription { + return makeSubscription(makeTrigger(), makeChannel(), makeK8sService()) +} + func makeDifferentSubscription() *v1alpha1.Subscription { return makeSubscription(makeTrigger(), makeDifferentChannel(), makeK8sService()) } From 9c63dbcc5a35ec861d5b14fe828ab8e67a9c88c6 Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Thu, 14 Feb 2019 18:32:32 -0800 Subject: [PATCH 060/128] Namespace reconciler automatically creates the Broker Filter's ServiceAccount and RBAC. Sadly this doesn't work well because we have such an old version of controller-runtime that the Filter ends up trying to watch _all_ Triggers, not just those in its namespace. And it only gets permission for the Triggers in its own namespace. --- config/200-broker-clusterrole.yaml | 28 ++++ config/500-controller.yaml | 2 +- pkg/broker/receiver.go | 4 + .../v1alpha1/namespace/namespace.go | 130 +++++++++++++++++- .../v1alpha1/namespace/namespace_test.go | 1 + sa.yaml | 20 --- t3.yaml | 16 +++ t4.yaml | 11 ++ 8 files changed, 186 insertions(+), 26 deletions(-) create mode 100644 config/200-broker-clusterrole.yaml delete mode 100644 sa.yaml create mode 100644 t3.yaml create mode 100644 t4.yaml diff --git a/config/200-broker-clusterrole.yaml b/config/200-broker-clusterrole.yaml new file mode 100644 index 00000000000..fc430c0dc8f --- /dev/null +++ b/config/200-broker-clusterrole.yaml @@ -0,0 +1,28 @@ +# Copyright 2019 The Knative Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: eventing-broker-filter +rules: + - apiGroups: + - eventing.knative.dev + resources: + - triggers + - triggers/status + verbs: + - get + - list + - watch diff --git a/config/500-controller.yaml b/config/500-controller.yaml index 15d76a8cf3a..55303c24745 100644 --- a/config/500-controller.yaml +++ b/config/500-controller.yaml @@ -42,7 +42,7 @@ spec: - name: FILTER_IMAGE value: github.com/knative/eventing/cmd/broker/filter - name: FILTER_SERVICE_ACCOUNT - value: broker-filter + value: eventing-broker-filter volumeMounts: - name: config-logging mountPath: /etc/config-logging diff --git a/pkg/broker/receiver.go b/pkg/broker/receiver.go index d51c3d6c755..9dd264337ee 100644 --- a/pkg/broker/receiver.go +++ b/pkg/broker/receiver.go @@ -83,6 +83,10 @@ func (r *Receiver) sendEvent(channel provisioners.ChannelReference, message *pro } func (r *Receiver) getTrigger(ctx context.Context, ref provisioners.ChannelReference) (*eventingv1alpha1.Trigger, error) { + // Sadly this doesn't work well because we do not yet have + // https://github.com/kubernetes-sigs/controller-runtime/pull/136, so controller runtime watches + // all Triggers, not just those in this namespace. And it doesn't have the RBAC (by default) for + // that to work. t := &eventingv1alpha1.Trigger{} err := r.client.Get(ctx, types.NamespacedName{ diff --git a/pkg/reconciler/v1alpha1/namespace/namespace.go b/pkg/reconciler/v1alpha1/namespace/namespace.go index 1c2c31ca618..a0742b8a4e9 100644 --- a/pkg/reconciler/v1alpha1/namespace/namespace.go +++ b/pkg/reconciler/v1alpha1/namespace/namespace.go @@ -18,11 +18,13 @@ package namespace import ( "context" + "fmt" "github.com/knative/eventing/contrib/gcppubsub/pkg/util/logging" "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" "go.uber.org/zap" "k8s.io/api/core/v1" corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/api/errors" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -45,10 +47,16 @@ const ( controllerAgentName = "knative-eventing-namespace-controller" defaultBroker = "default" + brokerFilterSA = "eventing-broker-filter" + brokerFilterRB = "eventing-broker-filter" + brokerFilterClusterRole = "eventing-broker-filter" + knativeEventingAnnotation = "eventing.knative.dev/inject" // Name of the corev1.Events emitted from the reconciliation process. brokerCreated = "BrokerCreated" + serviceAccountCreated = "BrokerFilterServiceAccountCreated" + serviceAccountRBACCreated = "BrokerFilterServiceAccountRBACCreated" ) type reconciler struct { @@ -84,7 +92,7 @@ func ProvideController(logger *zap.Logger) func(manager.Manager) (controller.Con } // Watch all the resources that this reconciler reconciles. - for _, t := range []runtime.Object{ &v1alpha1.Broker{} } { + for _, t := range []runtime.Object{ &corev1.ServiceAccount{}, &rbacv1.RoleBinding{}, &v1alpha1.Broker{} } { err = c.Watch(&source.Kind{Type: t}, &handler.EnqueueRequestsFromMapFunc{ToRequests: &namespaceMapper{}}); if err != nil { return nil, err @@ -165,7 +173,17 @@ func (r *reconciler) reconcile(ctx context.Context, ns *corev1.Namespace) error return nil } - _, err := r.reconcileBroker(ctx, ns) + sa, err := r.reconcileBrokerFilterServiceAccount(ctx, ns) + if err != nil { + logging.FromContext(ctx).Error("Unable to reconcile the Broker Filter Service Account for the namespace", zap.Error(err)) + return err + } + _, err = r.reconcileBrokerFilterRBAC(ctx, ns, sa) + if err != nil { + logging.FromContext(ctx).Error("Unable to reconcile the Broker Filter Service Account RBAC for the namespace", zap.Error(err)) + return err + } + _, err = r.reconcileBroker(ctx, ns) if err != nil { logging.FromContext(ctx).Error("Unable to reconcile broker for the namespace", zap.Error(err)) return err @@ -174,6 +192,108 @@ func (r *reconciler) reconcile(ctx context.Context, ns *corev1.Namespace) error return nil } +func (r *reconciler) reconcileBrokerFilterServiceAccount(ctx context.Context, ns *corev1.Namespace) (*corev1.ServiceAccount, error) { + current, err := r.getBrokerFilterServiceAccount(ctx, ns) + + // If the resource doesn't exist, we'll create it. + if k8serrors.IsNotFound(err) { + sa := newBrokerFilterServiceAccount(ns) + err = r.client.Create(ctx, sa) + if err != nil { + return nil, err + } + r.recorder.Event(ns, + corev1.EventTypeNormal, + serviceAccountCreated, + fmt.Sprintf("Service account created for the Broker '%s'", sa.Name)) + return sa, nil + } else if err != nil { + return nil, err + } + // Don't update anything that is already present. + return current, nil +} + +func (r *reconciler) getBrokerFilterServiceAccount(ctx context.Context, ns *corev1.Namespace) (*corev1.ServiceAccount, error) { + sa := &corev1.ServiceAccount{} + name := types.NamespacedName{ + Namespace: ns.Name, + Name: brokerFilterSA, + } + err := r.client.Get(ctx, name, sa) + return sa, err +} + +func newBrokerFilterServiceAccount(ns *corev1.Namespace) *corev1.ServiceAccount { + return &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: ns.Name, + Name: brokerFilterSA, + Labels: injectedLabels(), + }, + } +} + +func injectedLabels() map[string]string { + return map[string]string{ + "eventing.knative.dev/namespaceInjected": "true", + } +} + +func (r *reconciler) reconcileBrokerFilterRBAC(ctx context.Context, ns *corev1.Namespace, sa *corev1.ServiceAccount) (*rbacv1.RoleBinding, error) { + current, err := r.getBrokerFilterRBAC(ctx, ns) + + // If the resource doesn't exist, we'll create it. + if k8serrors.IsNotFound(err) { + rbac := newBrokerFilterRBAC(ns, sa) + err = r.client.Create(ctx, rbac) + if err != nil { + return nil, err + } + r.recorder.Event(ns, + corev1.EventTypeNormal, + serviceAccountRBACCreated, + fmt.Sprintf("Service account RBAC created for the Broker Filter '%s'", rbac.Name)) + return rbac, nil + } else if err != nil { + return nil, err + } + // Don't update anything that is already present. + return current, nil +} + +func (r *reconciler) getBrokerFilterRBAC(ctx context.Context, ns *corev1.Namespace) (*rbacv1.RoleBinding, error) { + rb := &rbacv1.RoleBinding{} + name := types.NamespacedName{ + Namespace: ns.Name, + Name: brokerFilterRB, + } + err := r.client.Get(ctx, name, rb) + return rb, err +} + +func newBrokerFilterRBAC(ns *corev1.Namespace, sa *corev1.ServiceAccount) *rbacv1.RoleBinding { + return &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: ns.Name, + Name: brokerFilterRB, + Labels: injectedLabels(), + }, + RoleRef:rbacv1.RoleRef{ + APIGroup:"rbac.authorization.k8s.io", + Kind: "ClusterRole", + Name: brokerFilterClusterRole, + }, + Subjects:[]rbacv1.Subject{ + { + Kind: "ServiceAccount", + Namespace: ns.Name, + Name: sa.Name, + }, + }, + } +} + func (r *reconciler) getBroker(ctx context.Context, ns *corev1.Namespace) (*v1alpha1.Broker, error) { b := &v1alpha1.Broker{} name := types.NamespacedName{ @@ -214,7 +334,7 @@ func newBroker(ns *corev1.Namespace) *v1alpha1.Broker { } func brokerLabels() map[string]string { - return map[string]string{ - "eventing.knative.dev/brokerForNamespace": "true", - } + l := injectedLabels() + l["eventing.knative.dev/brokerForNamespace"] = "true" + return l } diff --git a/pkg/reconciler/v1alpha1/namespace/namespace_test.go b/pkg/reconciler/v1alpha1/namespace/namespace_test.go index a3ce6a05615..14b1f6997aa 100644 --- a/pkg/reconciler/v1alpha1/namespace/namespace_test.go +++ b/pkg/reconciler/v1alpha1/namespace/namespace_test.go @@ -309,6 +309,7 @@ func makeBroker() *v1alpha1.Broker { Namespace: testNS, Name: brokerName, Labels: map[string]string{ + "eventing.knative.dev/namespaceInjected": "true", "eventing.knative.dev/brokerForNamespace": "true", }, }, diff --git a/sa.yaml b/sa.yaml deleted file mode 100644 index 124edb3f340..00000000000 --- a/sa.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: broker-filter - namespace: default - ---- - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: broker-filter -subjects: - - kind: ServiceAccount - name: broker-filter - namespace: default -roleRef: - kind: ClusterRole - name: cluster-admin - apiGroup: rbac.authorization.k8s.io diff --git a/t3.yaml b/t3.yaml new file mode 100644 index 00000000000..0ea783d5675 --- /dev/null +++ b/t3.yaml @@ -0,0 +1,16 @@ +apiVersion: eventing.knative.dev/v1alpha1 +kind: Trigger +metadata: + name: t +spec: + # Our Cloud Event parsing library seems to have a bug and forces to put type and source in double + # quotes (as it thinks the actual value is `"foo"`, not `foo`). + filter: + sourceAndType: + type: '"com.example.someevent"' + source: '"/mycontext/subcontext"' + subscriber: + ref: + apiVersion: v1 + kind: Service + name: svc diff --git a/t4.yaml b/t4.yaml new file mode 100644 index 00000000000..a6a981bfc0b --- /dev/null +++ b/t4.yaml @@ -0,0 +1,11 @@ +apiVersion: eventing.knative.dev/v1alpha1 +kind: Trigger +metadata: + name: t +spec: + subscriber: + ref: + apiVersion: v1 + kind: Service + name: svc + From 3ad193f767303ad86128382dc10ef5d7eea8d24d Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Thu, 14 Feb 2019 22:12:44 -0800 Subject: [PATCH 061/128] Remove no longer needed label. --- pkg/reconciler/v1alpha1/namespace/namespace.go | 8 +------- pkg/reconciler/v1alpha1/namespace/namespace_test.go | 1 - 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/pkg/reconciler/v1alpha1/namespace/namespace.go b/pkg/reconciler/v1alpha1/namespace/namespace.go index a0742b8a4e9..483d659604d 100644 --- a/pkg/reconciler/v1alpha1/namespace/namespace.go +++ b/pkg/reconciler/v1alpha1/namespace/namespace.go @@ -328,13 +328,7 @@ func newBroker(ns *corev1.Namespace) *v1alpha1.Broker { ObjectMeta: metav1.ObjectMeta{ Namespace: ns.Name, Name: defaultBroker, - Labels: brokerLabels(), + Labels: injectedLabels(), }, } } - -func brokerLabels() map[string]string { - l := injectedLabels() - l["eventing.knative.dev/brokerForNamespace"] = "true" - return l -} diff --git a/pkg/reconciler/v1alpha1/namespace/namespace_test.go b/pkg/reconciler/v1alpha1/namespace/namespace_test.go index 14b1f6997aa..9f3364caff1 100644 --- a/pkg/reconciler/v1alpha1/namespace/namespace_test.go +++ b/pkg/reconciler/v1alpha1/namespace/namespace_test.go @@ -310,7 +310,6 @@ func makeBroker() *v1alpha1.Broker { Name: brokerName, Labels: map[string]string{ "eventing.knative.dev/namespaceInjected": "true", - "eventing.knative.dev/brokerForNamespace": "true", }, }, } From d18acab2c478a9566c4ff749c08da8a5b9c07683 Mon Sep 17 00:00:00 2001 From: nachocano Date: Fri, 15 Feb 2019 12:10:25 -0800 Subject: [PATCH 062/128] Broker and trigger types UTs --- .../eventing/v1alpha1/broker_types_test.go | 271 +++++++++++++++++ .../eventing/v1alpha1/trigger_types_test.go | 272 ++++++++++++++++++ 2 files changed, 543 insertions(+) create mode 100644 pkg/apis/eventing/v1alpha1/broker_types_test.go create mode 100644 pkg/apis/eventing/v1alpha1/trigger_types_test.go diff --git a/pkg/apis/eventing/v1alpha1/broker_types_test.go b/pkg/apis/eventing/v1alpha1/broker_types_test.go new file mode 100644 index 00000000000..c5745150009 --- /dev/null +++ b/pkg/apis/eventing/v1alpha1/broker_types_test.go @@ -0,0 +1,271 @@ +/* +Copyright 2018 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + duckv1alpha1 "github.com/knative/pkg/apis/duck/v1alpha1" + corev1 "k8s.io/api/core/v1" +) + +var brokerConditionReady = duckv1alpha1.Condition{ + Type: BrokerConditionReady, + Status: corev1.ConditionTrue, +} + +var brokerConditionIngress = duckv1alpha1.Condition{ + Type: BrokerConditionIngress, + Status: corev1.ConditionTrue, +} + +var brokerConditionChannel = duckv1alpha1.Condition{ + Type: BrokerConditionChannel, + Status: corev1.ConditionTrue, +} + +var brokerConditionFilter = duckv1alpha1.Condition{ + Type: BrokerConditionFilter, + Status: corev1.ConditionTrue, +} + +var brokerConditionAddressable = duckv1alpha1.Condition{ + Type: BrokerConditionAddressable, + Status: corev1.ConditionFalse, +} + +func TestBrokerGetCondition(t *testing.T) { + tests := []struct { + name string + bs *BrokerStatus + condQuery duckv1alpha1.ConditionType + want *duckv1alpha1.Condition + }{{ + name: "single condition", + bs: &BrokerStatus{ + Conditions: []duckv1alpha1.Condition{ + brokerConditionReady, + }, + }, + condQuery: duckv1alpha1.ConditionReady, + want: &brokerConditionReady, + }, { + name: "multiple conditions", + bs: &BrokerStatus{ + Conditions: []duckv1alpha1.Condition{ + brokerConditionIngress, + brokerConditionChannel, + brokerConditionFilter, + }, + }, + condQuery: BrokerConditionFilter, + want: &brokerConditionFilter, + }, { + name: "multiple conditions, condition false", + bs: &BrokerStatus{ + Conditions: []duckv1alpha1.Condition{ + brokerConditionChannel, + brokerConditionFilter, + brokerConditionAddressable, + }, + }, + condQuery: BrokerConditionAddressable, + want: &brokerConditionAddressable, + }, { + name: "unknown condition", + bs: &BrokerStatus{ + Conditions: []duckv1alpha1.Condition{ + brokerConditionAddressable, + brokerConditionReady, + }, + }, + condQuery: duckv1alpha1.ConditionType("foo"), + want: nil, + }} + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got := test.bs.GetCondition(test.condQuery) + if diff := cmp.Diff(test.want, got); diff != "" { + t.Errorf("unexpected condition (-want, +got) = %v", diff) + } + }) + } +} + +func TestBrokerInitializeConditions(t *testing.T) { + tests := []struct { + name string + bs *BrokerStatus + want *BrokerStatus + }{{ + name: "empty", + bs: &BrokerStatus{}, + want: &BrokerStatus{ + Conditions: []duckv1alpha1.Condition{{ + Type: BrokerConditionAddressable, + Status: corev1.ConditionUnknown, + }, { + Type: BrokerConditionChannel, + Status: corev1.ConditionUnknown, + }, { + Type: BrokerConditionFilter, + Status: corev1.ConditionUnknown, + }, { + Type: BrokerConditionIngress, + Status: corev1.ConditionUnknown, + }, { + Type: BrokerConditionReady, + Status: corev1.ConditionUnknown, + }}, + }, + }, { + name: "one false", + bs: &BrokerStatus{ + Conditions: []duckv1alpha1.Condition{{ + Type: BrokerConditionChannel, + Status: corev1.ConditionFalse, + }}, + }, + want: &BrokerStatus{ + Conditions: []duckv1alpha1.Condition{{ + Type: BrokerConditionAddressable, + Status: corev1.ConditionUnknown, + }, { + Type: BrokerConditionChannel, + Status: corev1.ConditionFalse, + }, { + Type: BrokerConditionFilter, + Status: corev1.ConditionUnknown, + }, { + Type: BrokerConditionIngress, + Status: corev1.ConditionUnknown, + }, { + Type: BrokerConditionReady, + Status: corev1.ConditionUnknown, + }}, + }, + }, { + name: "one true", + bs: &BrokerStatus{ + Conditions: []duckv1alpha1.Condition{{ + Type: BrokerConditionFilter, + Status: corev1.ConditionTrue, + }}, + }, + want: &BrokerStatus{ + Conditions: []duckv1alpha1.Condition{{ + Type: BrokerConditionAddressable, + Status: corev1.ConditionUnknown, + }, { + Type: BrokerConditionChannel, + Status: corev1.ConditionUnknown, + }, { + Type: BrokerConditionFilter, + Status: corev1.ConditionTrue, + }, { + Type: BrokerConditionIngress, + Status: corev1.ConditionUnknown, + }, { + Type: BrokerConditionReady, + Status: corev1.ConditionUnknown, + }}, + }, + }} + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + test.bs.InitializeConditions() + if diff := cmp.Diff(test.want, test.bs, ignoreAllButTypeAndStatus); diff != "" { + t.Errorf("unexpected conditions (-want, +got) = %v", diff) + } + }) + } +} + +func TestBrokerIsReady(t *testing.T) { + tests := []struct { + name string + markChannelReady bool + markFilterReady bool + markIngressReady bool + address string + wantReady bool + }{{ + name: "all happy", + markChannelReady: true, + markFilterReady: true, + markIngressReady: true, + address: "hostname", + wantReady: true, + }, { + name: "channel sad", + markChannelReady: false, + markFilterReady: true, + markIngressReady: true, + address: "hostname", + wantReady: false, + }, { + name: "filter sad", + markChannelReady: true, + markFilterReady: false, + markIngressReady: true, + address: "hostname", + wantReady: false, + }, { + name: "ingress sad", + markChannelReady: true, + markFilterReady: true, + markIngressReady: false, + address: "hostname", + wantReady: false, + }, { + name: "addressable sad", + markChannelReady: true, + markFilterReady: true, + markIngressReady: true, + address: "", + wantReady: false, + }, { + name: "all sad", + markChannelReady: false, + markFilterReady: false, + markIngressReady: false, + address: "", + wantReady: false, + }} + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ts := &BrokerStatus{} + if test.markChannelReady { + ts.MarkChannelReady() + } + if test.markFilterReady { + ts.MarkFilterReady() + } + if test.markIngressReady { + ts.MarkIngressReady() + } + ts.SetAddress(test.address) + got := ts.IsReady() + if test.wantReady != got { + t.Errorf("unexpected readiness: want %v, got %v", test.wantReady, got) + } + }) + } +} diff --git a/pkg/apis/eventing/v1alpha1/trigger_types_test.go b/pkg/apis/eventing/v1alpha1/trigger_types_test.go new file mode 100644 index 00000000000..fc08fd0d6a0 --- /dev/null +++ b/pkg/apis/eventing/v1alpha1/trigger_types_test.go @@ -0,0 +1,272 @@ +/* +Copyright 2018 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + duckv1alpha1 "github.com/knative/pkg/apis/duck/v1alpha1" + corev1 "k8s.io/api/core/v1" +) + +var triggerConditionReady = duckv1alpha1.Condition{ + Type: TriggerConditionReady, + Status: corev1.ConditionTrue, +} + +var triggerConditionBrokerExists = duckv1alpha1.Condition{ + Type: TriggerConditionBrokerExists, + Status: corev1.ConditionTrue, +} + +var triggerConditionKubernetesService = duckv1alpha1.Condition{ + Type: TriggerConditionKubernetesService, + Status: corev1.ConditionTrue, +} + +var triggerConditionVirtualService = duckv1alpha1.Condition{ + Type: TriggerConditionVirtualService, + Status: corev1.ConditionTrue, +} + +var triggerConditionSubscribed = duckv1alpha1.Condition{ + Type: TriggerConditionSubscribed, + Status: corev1.ConditionFalse, +} + +func TestTriggerGetCondition(t *testing.T) { + tests := []struct { + name string + ts *TriggerStatus + condQuery duckv1alpha1.ConditionType + want *duckv1alpha1.Condition + }{{ + name: "single condition", + ts: &TriggerStatus{ + Conditions: []duckv1alpha1.Condition{ + triggerConditionReady, + }, + }, + condQuery: duckv1alpha1.ConditionReady, + want: &triggerConditionReady, + }, { + name: "multiple conditions", + ts: &TriggerStatus{ + Conditions: []duckv1alpha1.Condition{ + triggerConditionBrokerExists, + triggerConditionKubernetesService, + }, + }, + condQuery: TriggerConditionKubernetesService, + want: &triggerConditionKubernetesService, + }, { + name: "multiple conditions, condition false", + ts: &TriggerStatus{ + Conditions: []duckv1alpha1.Condition{ + triggerConditionBrokerExists, + triggerConditionKubernetesService, + triggerConditionSubscribed, + }, + }, + condQuery: TriggerConditionSubscribed, + want: &triggerConditionSubscribed, + }, { + name: "unknown condition", + ts: &TriggerStatus{ + Conditions: []duckv1alpha1.Condition{ + triggerConditionVirtualService, + triggerConditionSubscribed, + }, + }, + condQuery: duckv1alpha1.ConditionType("foo"), + want: nil, + }} + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got := test.ts.GetCondition(test.condQuery) + if diff := cmp.Diff(test.want, got); diff != "" { + t.Errorf("unexpected condition (-want, +got) = %v", diff) + } + }) + } +} + +func TestTriggerInitializeConditions(t *testing.T) { + tests := []struct { + name string + ts *TriggerStatus + want *TriggerStatus + }{{ + name: "empty", + ts: &TriggerStatus{}, + want: &TriggerStatus{ + Conditions: []duckv1alpha1.Condition{{ + Type: TriggerConditionBrokerExists, + Status: corev1.ConditionUnknown, + }, { + Type: TriggerConditionKubernetesService, + Status: corev1.ConditionUnknown, + }, { + Type: TriggerConditionReady, + Status: corev1.ConditionUnknown, + }, { + Type: TriggerConditionSubscribed, + Status: corev1.ConditionUnknown, + }, { + Type: TriggerConditionVirtualService, + Status: corev1.ConditionUnknown, + }}, + }, + }, { + name: "one false", + ts: &TriggerStatus{ + Conditions: []duckv1alpha1.Condition{{ + Type: TriggerConditionVirtualService, + Status: corev1.ConditionFalse, + }}, + }, + want: &TriggerStatus{ + Conditions: []duckv1alpha1.Condition{{ + Type: TriggerConditionBrokerExists, + Status: corev1.ConditionUnknown, + }, { + Type: TriggerConditionKubernetesService, + Status: corev1.ConditionUnknown, + }, { + Type: TriggerConditionReady, + Status: corev1.ConditionUnknown, + }, { + Type: TriggerConditionSubscribed, + Status: corev1.ConditionUnknown, + }, { + Type: TriggerConditionVirtualService, + Status: corev1.ConditionFalse, + }}, + }, + }, { + name: "one true", + ts: &TriggerStatus{ + Conditions: []duckv1alpha1.Condition{{ + Type: TriggerConditionSubscribed, + Status: corev1.ConditionTrue, + }}, + }, + want: &TriggerStatus{ + Conditions: []duckv1alpha1.Condition{{ + Type: TriggerConditionBrokerExists, + Status: corev1.ConditionUnknown, + }, { + Type: TriggerConditionKubernetesService, + Status: corev1.ConditionUnknown, + }, { + Type: TriggerConditionReady, + Status: corev1.ConditionUnknown, + }, { + Type: TriggerConditionSubscribed, + Status: corev1.ConditionTrue, + }, { + Type: TriggerConditionVirtualService, + Status: corev1.ConditionUnknown, + }}, + }, + }} + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + test.ts.InitializeConditions() + if diff := cmp.Diff(test.want, test.ts, ignoreAllButTypeAndStatus); diff != "" { + t.Errorf("unexpected conditions (-want, +got) = %v", diff) + } + }) + } +} + +func TestTriggerIsReady(t *testing.T) { + tests := []struct { + name string + markBrokerExists bool + markKubernetesServiceExists bool + markVirtualServiceExists bool + markSubscribed bool + wantReady bool + }{{ + name: "all happy", + markBrokerExists: true, + markKubernetesServiceExists: true, + markVirtualServiceExists: true, + markSubscribed: true, + wantReady: true, + }, { + name: "broker sad", + markBrokerExists: false, + markKubernetesServiceExists: true, + markVirtualServiceExists: true, + markSubscribed: true, + wantReady: false, + }, { + name: "k8s service sad", + markBrokerExists: true, + markKubernetesServiceExists: false, + markVirtualServiceExists: true, + markSubscribed: true, + wantReady: false, + }, { + name: "virtual service sad", + markBrokerExists: true, + markKubernetesServiceExists: true, + markVirtualServiceExists: false, + markSubscribed: true, + wantReady: false, + }, { + name: "subscribed sad", + markBrokerExists: true, + markKubernetesServiceExists: true, + markVirtualServiceExists: true, + markSubscribed: false, + wantReady: false, + }, { + name: "all sad", + markBrokerExists: false, + markKubernetesServiceExists: false, + markVirtualServiceExists: false, + markSubscribed: false, + wantReady: false, + }} + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ts := &TriggerStatus{} + if test.markBrokerExists { + ts.MarkBrokerExists() + } + if test.markKubernetesServiceExists { + ts.MarkKubernetesServiceExists() + } + if test.markVirtualServiceExists { + ts.MarkVirtualServiceExists() + } + if test.markSubscribed { + ts.MarkSubscribed() + } + got := ts.IsReady() + if test.wantReady != got { + t.Errorf("unexpected readiness: want %v, got %v", test.wantReady, got) + } + }) + } +} From 2a02ba5dbe30f01344de5f828ef9d6deeffd62ad Mon Sep 17 00:00:00 2001 From: Grant Rodgers Date: Fri, 15 Feb 2019 12:58:09 -0800 Subject: [PATCH 063/128] WIP early E2E test --- test/crd.go | 27 ++++ test/crd_checks.go | 36 +++++ test/e2e/broker_filter_test.go | 231 +++++++++++++++++++++++++++++++++ test/e2e/builder/broker.go | 48 +++++++ test/e2e/builder/trigger.go | 74 +++++++++++ test/e2e/e2e.go | 22 ++++ test/e2e/fixtures.go | 108 +++++++++++++++ test/states.go | 14 ++ 8 files changed, 560 insertions(+) create mode 100644 test/e2e/broker_filter_test.go create mode 100644 test/e2e/builder/broker.go create mode 100644 test/e2e/builder/trigger.go create mode 100644 test/e2e/fixtures.go diff --git a/test/crd.go b/test/crd.go index cd365342a0e..0806b221a70 100644 --- a/test/crd.go +++ b/test/crd.go @@ -18,6 +18,7 @@ package test // crd contains functions that construct boilerplate CRD definitions. import ( + "fmt" "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" servingv1alpha1 "github.com/knative/serving/pkg/apis/serving/v1alpha1" corev1 "k8s.io/api/core/v1" @@ -166,6 +167,32 @@ func Subscription(name string, namespace string, channel *corev1.ObjectReference } } +func Broker(name string, namespace string) *v1alpha1.Broker { + return &v1alpha1.Broker{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: v1alpha1.BrokerSpec{}, + } +} + +func Trigger(name string, namespace string, eventType string, subscriberRef *corev1.ObjectReference, brokerName string) *v1alpha1.Trigger { + return &v1alpha1.Trigger{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: v1alpha1.TriggerSpec{ + Broker: brokerName, + Type: fmt.Sprintf("%q", eventType), + Subscriber: &v1alpha1.SubscriberSpec{ + Ref: subscriberRef, + }, + }, + } +} + // CloudEvent specifies the arguments for a CloudEvent sent by the sendevent // binary. type CloudEvent struct { diff --git a/test/crd_checks.go b/test/crd_checks.go index e66c7722024..f996b8e5fd0 100644 --- a/test/crd_checks.go +++ b/test/crd_checks.go @@ -91,3 +91,39 @@ func WaitForSubscriptionState(client eventingclient.SubscriptionInterface, name return inState(r) }) } + +// WaitForBrokerState polls the status of the Broker called name from client +// every interval until inState returns `true` indicating it is done, returns an +// error or timeout. desc will be used to name the metric that is emitted to +// track how long it took for name to get into the state checked by inState. +func WaitForBrokerState(client eventingclient.BrokerInterface, name string, inState func(r *eventingv1alpha1.Broker) (bool, error), desc string) error { + metricName := fmt.Sprintf("WaitForBrokerState/%s/%s", name, desc) + _, span := trace.StartSpan(context.Background(), metricName) + defer span.End() + + return wait.PollImmediate(interval, timeout, func() (bool, error) { + r, err := client.Get(name, metav1.GetOptions{}) + if err != nil { + return true, err + } + return inState(r) + }) +} + +// WaitForTriggerState polls the status of the Trigger called name from client +// every interval until inState returns `true` indicating it is done, returns an +// error or timeout. desc will be used to name the metric that is emitted to +// track how long it took for name to get into the state checked by inState. +func WaitForTriggerState(client eventingclient.TriggerInterface, name string, inState func(r *eventingv1alpha1.Trigger) (bool, error), desc string) error { + metricName := fmt.Sprintf("WaitForTriggerState/%s/%s", name, desc) + _, span := trace.StartSpan(context.Background(), metricName) + defer span.End() + + return wait.PollImmediate(interval, timeout, func() (bool, error) { + r, err := client.Get(name, metav1.GetOptions{}) + if err != nil { + return true, err + } + return inState(r) + }) +} diff --git a/test/e2e/broker_filter_test.go b/test/e2e/broker_filter_test.go new file mode 100644 index 00000000000..f42436dae2b --- /dev/null +++ b/test/e2e/broker_filter_test.go @@ -0,0 +1,231 @@ +// +build e2e + +/* +Copyright 2019 The Knative Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "fmt" + "testing" + "time" + + "github.com/knative/eventing/test" + pkgTest "github.com/knative/pkg/test" + "github.com/knative/pkg/test/logging" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/uuid" +) + +const ( + defaultBrokerName = "default" + altBrokerName = "alternate" + anyEvent = "Any" + untypedEvent = "test.untyped" + typedEvent = "test.typed" +) + +func triggerName(broker, eventType string) string { + fmt.Sprintf("%s-dump-%s", broker, eventType) +} + +func namespaceExists(t *testing.T, clients *test.Clients) (string, func()) { + logger := logging.GetContextLogger("TestBrokerFilter") + shutdown := func() {} + ns := pkgTest.Flags.Namespace + logger.Infof("Namespace: %s", ns) + + nsSpec, err := clients.Kube.Kube.CoreV1().Namespaces().Get(ns, metav1.GetOptions{}) + + if err != nil && errors.IsNotFound(err) { + nsSpec = &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: ns}} + logger.Infof("Creating Namespace: %s", ns) + nsSpec, err = clients.Kube.Kube.CoreV1().Namespaces().Create(nsSpec) + if err != nil { + t.Fatalf("Failed to create Namespace: %s; %v", ns, err) + } else { + shutdown = func() { + clients.Kube.Kube.CoreV1().Namespaces().Delete(nsSpec.Name, nil) + // TODO: this is a bit hacky but in order for the tests to work + // correctly for a clean namespace to be created we need to also + // wait for it to be removed. + // To fix this we could generate namespace names. + // This only happens when the namespace provided does not exist. + // + // wait up to 120 seconds for the namespace to be removed. + logger.Infof("Deleting Namespace: %s", ns) + for i := 0; i < 120; i++ { + time.Sleep(1 * time.Second) + if _, err := clients.Kube.Kube.CoreV1().Namespaces().Get(ns, metav1.GetOptions{}); err != nil && errors.IsNotFound(err) { + logger.Info("Namespace has been deleted") + // the namespace is gone. + break + } + } + } + } + } + return ns, shutdown +} + +func TestBrokerFilter(t *testing.T) { + logger := logging.GetContextLogger("TestBrokerFilter") + + clients, cleaner := Setup(t, logger) + defer TearDown(clients, cleaner, logger) + + // verify namespace + ns, cleanupNS := namespaceExists(t, clients) + defer cleanupNS() + + // Fixtures + + // TODO label namespace to get default Broker + + fixtures := []Fixture{ + // Default Any Trigger + &KnativeFixture{ + Object: builder.Trigger("default-dump-any"), pkgTest.Flags.Namespace). + Type(anyEvent). + SubscriberSvc("default-any-dumper"), + }, + // Default Typed Trigger + &KnativeFixture{ + Object: builder.Trigger("default-dump-typed"), pkgTest.Flags.Namespace). + Type(typedEvent). + SubscriberSvc("default-typed-dumper"), + }, + + // Alternate Broker + &KnativeFixture{ + Object: builder.Broker(altBrokerName, pkgTest.Flags.Namespace), + }, + // Alternate Any Trigger + &KnativeFixture{ + Object: builder.Trigger("alt-dump-any"), pkgTest.Flags.Namespace). + Broker(altBrokerName). + Type(anyEvent). + SubscriberSvc(fmt.Sprintf("alt-any-dumper"), + }, + // Alternate Typed Trigger + &KnativeFixture{ + Object: builder.Trigger("alt-dump-typed"), pkgTest.Flags.Namespace). + Broker(altBrokerName). + Type(typedEvent). + SubscriberSvc("alt-typed-dumper"), + }, + } + + ctx := context.Background() + + //TODO move this to runners + + for _, f := range fixtures { + f.Create(ctx, client) + } + + for _, f := range fixtures { + f.Verify(ctx, client) + } + + // Create message dumper services + // Create: pod + // Verify: pod status is ready + // + + pods := []Fixture{ + &PodSuccess{ + + } + } + + // create alt Broker + + // For each broker, create a typed and untyped trigger. + // For each trigger, create a logevents pod. + // Wait for Broker, triggers, and pods to become ready. + + // Take Action + + // For each tuple of typed and untyped, default and alt broker: + // create a sendevents pod to send an event of the type to the broker address + + // Verify + + // For each logevents pod: + // check logs to ensure the correct message(s) got there + + // create logger pod + + logger.Infof("creating subscriber pod") + selector := map[string]string{"e2etest": string(uuid.NewUUID())} + subscriberPod := test.EventLoggerPod(routeName, ns, selector) + if err := CreatePod(clients, subscriberPod, logger, cleaner); err != nil { + t.Fatalf("Failed to create event logger pod: %v", err) + } + if err := WaitForAllPodsRunning(clients, logger, ns); err != nil { + t.Fatalf("Error waiting for logger pod to become running: %v", err) + } + logger.Infof("subscriber pod running") + + subscriberSvc := test.Service(routeName, ns, selector) + if err := CreateService(clients, subscriberSvc, logger, cleaner); err != nil { + t.Fatalf("Failed to create event logger service: %v", err) + } + + // Reload subscriberPod to get IP + subscriberPod, err := clients.Kube.Kube.CoreV1().Pods(subscriberPod.Namespace).Get(subscriberPod.Name, metav1.GetOptions{}) + if err != nil { + t.Fatalf("Failed to get subscriber pod: %v", err) + } + + // create channel + + logger.Infof("Creating Channel and Subscription") + if test.EventingFlags.Provisioner == "" { + t.Fatal("ClusterChannelProvisioner must be set to a non-empty string. Either do not specify --clusterChannelProvisioner or set to something other than the empty string") + } + channel := test.Channel(channelName, ns, test.ClusterChannelProvisioner(test.EventingFlags.Provisioner)) + logger.Infof("channel: %#v", channel) + sub := test.Subscription(subscriptionName, ns, test.ChannelRef(channelName), test.SubscriberSpecForService(routeName), nil) + logger.Infof("sub: %#v", sub) + + if err := WithChannelAndSubscriptionReady(clients, channel, sub, logger, cleaner); err != nil { + t.Fatalf("The Channel or Subscription were not marked as Ready: %v", err) + } + + // create sender pod + + logger.Infof("Creating event sender") + body := fmt.Sprintf("TestSingleEvent %s", uuid.NewUUID()) + event := test.CloudEvent{ + Source: senderName, + Type: "test.eventing.knative.dev", + Data: fmt.Sprintf(`{"msg":%q}`, body), + Encoding: encoding, + } + url := fmt.Sprintf("http://%s", channel.Status.Address.Hostname) + pod := test.EventSenderPod(senderName, ns, url, event) + logger.Infof("sender pod: %#v", pod) + if err := CreatePod(clients, pod, logger, cleaner); err != nil { + t.Fatalf("Failed to create event sender pod: %v", err) + } + + if err := WaitForLogContent(clients, logger, routeName, subscriberPod.Spec.Containers[0].Name, ns, body); err != nil { + t.Fatalf("String %q not found in logs of subscriber pod %q: %v", body, routeName, err) + } +} diff --git a/test/e2e/builder/broker.go b/test/e2e/builder/broker.go new file mode 100644 index 00000000000..dc00c6245d3 --- /dev/null +++ b/test/e2e/builder/broker.go @@ -0,0 +1,48 @@ +/* +Copyright 2019 The Knative Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package builder + +import ( + eventingv1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +type BrokerBuilder struct { + *eventingv1alpha1.Broker +} + +func Broker(name, namespace string) *BrokerBuilder { + broker := &eventingv1alpha1.Broker{ + TypeMeta: metav1.TypeMeta{ + APIVersion: eventingv1alpha1.SchemeGroupVersion.String(), + Kind: "Broker", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: eventingv1alpha1.BrokerSpec{}, + } + + return &BrokerBuilder{ + Broker: broker, + } +} + +func (b *BrokerBuilder) Build() runtime.Object { + return b.Broker.DeepCopy() +} \ No newline at end of file diff --git a/test/e2e/builder/trigger.go b/test/e2e/builder/trigger.go new file mode 100644 index 00000000000..745a71a34ad --- /dev/null +++ b/test/e2e/builder/trigger.go @@ -0,0 +1,74 @@ +/* +Copyright 2019 The Knative Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package builder + +import ( + eventingv1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +type TriggerBuilder struct { + *eventingv1alpha1.Trigger +} + +func Trigger(name, namespace string) *TriggerBuilder { + trigger := &eventingv1alpha1.Trigger{ + TypeMeta: metav1.TypeMeta{ + APIVersion: eventingv1alpha1.SchemeGroupVersion.String(), + Kind: "Trigger", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: eventingv1alpha1.TriggerSpec{}, + } + + return &TriggerBuilder{ + Trigger: trigger, + } +} + +func (b *TriggerBuilder) Build() runtime.Object { + return b.Trigger.DeepCopy() +} + +func (b *TriggerBuilder) Type(eventType string) *TriggerBuilder { + b.Trigger.Spec.Type = eventType + return b +} + +func (b *TriggerBuilder) Broker(brokerName string) *TriggerBuilder { + b.Trigger.Spec.Broker = brokerName + return b +} + +func (b *TriggerBuilder) Subscriber(ref *corev1.ObjectReference) *TriggerBuilder { + b.Trigger.Spec.Subscriber.Ref = ref + return b +} + +func (b *TriggerBuilder) SubscriberSvc(svcName string) *TriggerBuilder { + b.Trigger.Spec.Subscriber.Ref = &corev1.ObjectReference{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "Service", + Name: svcName, + Namespace: b.Trigger.GetNamespace(), + } + return b +} \ No newline at end of file diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go index a183ec5cb9c..453c0c69ea6 100644 --- a/test/e2e/e2e.go +++ b/test/e2e/e2e.go @@ -159,6 +159,28 @@ func WithChannelAndSubscriptionReady(clients *test.Clients, channel *v1alpha1.Ch return nil } +// CreateBroker will create a Broker +func CreateBroker(clients *test.Clients, broker *v1alpha1.Broker, logger *logging.BaseLogger, cleaner *test.Cleaner) error { + brokers := clients.Eventing.EventingV1alpha1().Brokers(pkgTest.Flags.Namespace) + res, err := brokers.Create(broker) + if err != nil { + return err + } + cleaner.Add(v1alpha1.SchemeGroupVersion.Group, v1alpha1.SchemeGroupVersion.Version, "brokers", pkgTest.Flags.Namespace, res.ObjectMeta.Name) + return nil +} + +// CreateTrigger will create a Trigger +func CreateTrigger(clients *test.Clients, trigger *v1alpha1.Trigger, logger *logging.BaseLogger, cleaner *test.Cleaner) error { + triggers := clients.Eventing.EventingV1alpha1().Triggers(pkgTest.Flags.Namespace) + res, err := triggers.Create(trigger) + if err != nil { + return err + } + cleaner.Add(v1alpha1.SchemeGroupVersion.Group, v1alpha1.SchemeGroupVersion.Version, "triggers", pkgTest.Flags.Namespace, res.ObjectMeta.Name) + return nil +} + // CreateServiceAccount will create a service account func CreateServiceAccount(clients *test.Clients, sa *corev1.ServiceAccount, logger *logging.BaseLogger, cleaner *test.Cleaner) error { sas := clients.Kube.Kube.CoreV1().ServiceAccounts(pkgTest.Flags.Namespace) diff --git a/test/e2e/fixtures.go b/test/e2e/fixtures.go new file mode 100644 index 00000000000..f2174c58048 --- /dev/null +++ b/test/e2e/fixtures.go @@ -0,0 +1,108 @@ +/* +Copyright 2019 The Knative Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package e2e + +import ( + "context" + "github.com/knative/eventing/pkg/reconciler/testing" + "github.com/knative/pkg/apis/duck" + duckv1alpha1 "github.com/knative/pkg/apis/duck/v1alpha1" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type Fixture interface { + // Create attempts to instantiate the fixture. It may be called multiple times if earlier attempts return an error. + Create(context.Context, client.Client) error + + // Verify determines if the fixture was successfully instantiated once. It may be called multiple times if earlier + // attempts return false or an error. + Verify(context.Context, client.Client) (bool, error) + + // Debug surfaces context about a failure during Create or Verify. + Debug(context.Context, client.Client) + + // Teardown removes any state instantiated by Create or Verify. + Teardown(context.Context, client.Client) error +} + +type KnativeFixture struct { + Object testing.Buildable +} + +type CreateFunc func(context.Context, client.Client) error +type VerifyFunc func(context.Context, client.Client) (bool, error) +type DebugFunc func(context.Context, client.Client) +type TeardownFunc func(context.Context, client.Client) error + +type FixtureFuncs struct { + Create CreateFunc + Verify VerifyFunc + Debug DebugFunc + Teardown TeardownFunc +} + +//Fixtures: +// - Broker, Trigger Verify checks status ready +// - Sendevent Verify checks that pod completed successfully +// - Logevent Verify checks that the expected message was logged + +func (f *KnativeFixture) Create(ctx context.Context, cl client.Client) error { + obj := f.Object.Build() + err := cl.Create(ctx, obj) + if err != nil { + return err + } + return nil +} + +var standardCondSet = duckv1alpha1.NewLivingConditionSet() + +// TODO the runner should do this with backoff and retry. +// This method only verifies Knative objects +func (f *KnativeFixture) Verify(ctx context.Context, cl client.Client) (bool, error) { + obj := f.Object.Build() + u := &unstructured.Unstructured{} + acc, err := meta.Accessor(obj) + if err != nil { + return false, err + } + if err := cl.Get(ctx, client.ObjectKey{Name: acc.GetName(), Namespace: acc.GetNamespace()}, u); err != nil { + return false, err + } + + kr := &duckv1alpha1.KResource{} + if err := duck.FromUnstructured(u, kr); err != nil { + return false, err + } + + if !standardCondSet.Manage(kr.Status).IsHappy() { + return false, nil + } + + return true, nil +} + +// TODO If Create or Verify return an error, run this method, then Cleanup +func (f *KnativeFixture) Debug(ctx context.Context, cl client.Client) { +} + +// TODO if Teardown returns an error, log it and give up (maybe retry for a while?) +func (f *KnativeFixture) Teardownctx context.Context, cl client.Client) error { + obj := f.Object.Build() + return cl.Delete(ctx, obj) +} \ No newline at end of file diff --git a/test/states.go b/test/states.go index 49f4256435e..e5a8dd0cdcf 100644 --- a/test/states.go +++ b/test/states.go @@ -54,6 +54,20 @@ func IsSubscriptionReady(s *eventingv1alpha1.Subscription) (bool, error) { return s.Status.IsReady(), nil } +/ IsBrokerReady will check the status conditions of the Broker and return true +// if the Broker is ready. +func IsBrokerReady(b *eventingv1alpha1.Broker) (bool, error) { + return b.Status.IsReady(), nil +} + +// IsTriggerReady will check the status conditions of the Trigger and +// return true if the Trigger is ready. +func IsTriggerReady(t *eventingv1alpha1.Trigger) (bool, error) { + return t.Status.IsReady(), nil +} + + + // PodsRunning will check the status conditions of the pod list and return true // if all pods are Running. func PodsRunning(podList *corev1.PodList) (bool, error) { From 54cdb3cad27fb9806984aa0965fdeefe43226fcc Mon Sep 17 00:00:00 2001 From: nachocano Date: Tue, 19 Feb 2019 11:26:26 -0800 Subject: [PATCH 064/128] Changes after code review. Adding trigger defaults and validation tests --- .../eventing/v1alpha1/broker_types_test.go | 42 +-- .../v1alpha1/trigger_defaults_test.go | 72 +++++ .../eventing/v1alpha1/trigger_types_test.go | 42 +-- .../eventing/v1alpha1/trigger_validation.go | 2 +- .../v1alpha1/trigger_validation_test.go | 245 ++++++++++++++++++ 5 files changed, 362 insertions(+), 41 deletions(-) create mode 100644 pkg/apis/eventing/v1alpha1/trigger_defaults_test.go create mode 100644 pkg/apis/eventing/v1alpha1/trigger_validation_test.go diff --git a/pkg/apis/eventing/v1alpha1/broker_types_test.go b/pkg/apis/eventing/v1alpha1/broker_types_test.go index c5745150009..623a002b642 100644 --- a/pkg/apis/eventing/v1alpha1/broker_types_test.go +++ b/pkg/apis/eventing/v1alpha1/broker_types_test.go @@ -24,30 +24,32 @@ import ( corev1 "k8s.io/api/core/v1" ) -var brokerConditionReady = duckv1alpha1.Condition{ - Type: BrokerConditionReady, - Status: corev1.ConditionTrue, -} +var ( + brokerConditionReady = duckv1alpha1.Condition{ + Type: BrokerConditionReady, + Status: corev1.ConditionTrue, + } -var brokerConditionIngress = duckv1alpha1.Condition{ - Type: BrokerConditionIngress, - Status: corev1.ConditionTrue, -} + brokerConditionIngress = duckv1alpha1.Condition{ + Type: BrokerConditionIngress, + Status: corev1.ConditionTrue, + } -var brokerConditionChannel = duckv1alpha1.Condition{ - Type: BrokerConditionChannel, - Status: corev1.ConditionTrue, -} + brokerConditionChannel = duckv1alpha1.Condition{ + Type: BrokerConditionChannel, + Status: corev1.ConditionTrue, + } -var brokerConditionFilter = duckv1alpha1.Condition{ - Type: BrokerConditionFilter, - Status: corev1.ConditionTrue, -} + brokerConditionFilter = duckv1alpha1.Condition{ + Type: BrokerConditionFilter, + Status: corev1.ConditionTrue, + } -var brokerConditionAddressable = duckv1alpha1.Condition{ - Type: BrokerConditionAddressable, - Status: corev1.ConditionFalse, -} + brokerConditionAddressable = duckv1alpha1.Condition{ + Type: BrokerConditionAddressable, + Status: corev1.ConditionFalse, + } +) func TestBrokerGetCondition(t *testing.T) { tests := []struct { diff --git a/pkg/apis/eventing/v1alpha1/trigger_defaults_test.go b/pkg/apis/eventing/v1alpha1/trigger_defaults_test.go new file mode 100644 index 00000000000..a3dab313fb7 --- /dev/null +++ b/pkg/apis/eventing/v1alpha1/trigger_defaults_test.go @@ -0,0 +1,72 @@ +/* +Copyright 2019 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "testing" + + "github.com/google/go-cmp/cmp" +) + +var ( + defaultBroker = "default" + otherBroker = "other_broker" + defaultTriggerFilter = &TriggerFilter{ + SourceAndType: &TriggerFilterSourceAndType{ + Type: TriggerAnyFilter, + Source: TriggerAnyFilter}, + } + otherTriggerFilter = &TriggerFilter{ + SourceAndType: &TriggerFilterSourceAndType{ + Type: "other_type", + Source: "other_source"}, + } + defaultTrigger = Trigger{ + Spec: TriggerSpec{ + Broker: defaultBroker, + Filter: defaultTriggerFilter, + }, + } +) + +func TestTriggerDefaults(t *testing.T) { + testCases := map[string]struct { + initial Trigger + expected Trigger + }{ + "nil broker": { + initial: Trigger{Spec: TriggerSpec{Filter: otherTriggerFilter}}, + expected: Trigger{Spec: TriggerSpec{Broker: defaultBroker, Filter: otherTriggerFilter}}, + }, + "nil filter": { + initial: Trigger{Spec: TriggerSpec{Broker: otherBroker}}, + expected: Trigger{Spec: TriggerSpec{Broker: otherBroker, Filter: defaultTriggerFilter}}, + }, + "nil broker and nil filter": { + initial: Trigger{}, + expected: defaultTrigger, + }, + } + for n, tc := range testCases { + t.Run(n, func(t *testing.T) { + tc.initial.SetDefaults() + if diff := cmp.Diff(tc.expected, tc.initial); diff != "" { + t.Fatalf("Unexpected defaults (-want, +got): %s", diff) + } + }) + } +} diff --git a/pkg/apis/eventing/v1alpha1/trigger_types_test.go b/pkg/apis/eventing/v1alpha1/trigger_types_test.go index fc08fd0d6a0..1d36e8fed7c 100644 --- a/pkg/apis/eventing/v1alpha1/trigger_types_test.go +++ b/pkg/apis/eventing/v1alpha1/trigger_types_test.go @@ -24,30 +24,32 @@ import ( corev1 "k8s.io/api/core/v1" ) -var triggerConditionReady = duckv1alpha1.Condition{ - Type: TriggerConditionReady, - Status: corev1.ConditionTrue, -} +var ( + triggerConditionReady = duckv1alpha1.Condition{ + Type: TriggerConditionReady, + Status: corev1.ConditionTrue, + } -var triggerConditionBrokerExists = duckv1alpha1.Condition{ - Type: TriggerConditionBrokerExists, - Status: corev1.ConditionTrue, -} + triggerConditionBrokerExists = duckv1alpha1.Condition{ + Type: TriggerConditionBrokerExists, + Status: corev1.ConditionTrue, + } -var triggerConditionKubernetesService = duckv1alpha1.Condition{ - Type: TriggerConditionKubernetesService, - Status: corev1.ConditionTrue, -} + triggerConditionKubernetesService = duckv1alpha1.Condition{ + Type: TriggerConditionKubernetesService, + Status: corev1.ConditionTrue, + } -var triggerConditionVirtualService = duckv1alpha1.Condition{ - Type: TriggerConditionVirtualService, - Status: corev1.ConditionTrue, -} + triggerConditionVirtualService = duckv1alpha1.Condition{ + Type: TriggerConditionVirtualService, + Status: corev1.ConditionTrue, + } -var triggerConditionSubscribed = duckv1alpha1.Condition{ - Type: TriggerConditionSubscribed, - Status: corev1.ConditionFalse, -} + triggerConditionSubscribed = duckv1alpha1.Condition{ + Type: TriggerConditionSubscribed, + Status: corev1.ConditionFalse, + } +) func TestTriggerGetCondition(t *testing.T) { tests := []struct { diff --git a/pkg/apis/eventing/v1alpha1/trigger_validation.go b/pkg/apis/eventing/v1alpha1/trigger_validation.go index 073784972b1..e58150fdf48 100644 --- a/pkg/apis/eventing/v1alpha1/trigger_validation.go +++ b/pkg/apis/eventing/v1alpha1/trigger_validation.go @@ -37,7 +37,7 @@ func (ts *TriggerSpec) Validate() *apis.FieldError { errs = errs.Also(fe) } - if ts.Filter.SourceAndType == nil { + if ts.Filter != nil && ts.Filter.SourceAndType == nil { fe := apis.ErrMissingField("filter.sourceAndType") errs = errs.Also(fe) } diff --git a/pkg/apis/eventing/v1alpha1/trigger_validation_test.go b/pkg/apis/eventing/v1alpha1/trigger_validation_test.go new file mode 100644 index 00000000000..8afa183faca --- /dev/null +++ b/pkg/apis/eventing/v1alpha1/trigger_validation_test.go @@ -0,0 +1,245 @@ +/* +Copyright 2019 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "testing" + + "github.com/knative/pkg/apis" + + "github.com/google/go-cmp/cmp" + corev1 "k8s.io/api/core/v1" +) + +var ( + validTriggerFilter = &TriggerFilter{ + SourceAndType: &TriggerFilterSourceAndType{ + Type: "other_type", + Source: "other_source"}, + } + validSubscriber = &SubscriberSpec{ + Ref: &corev1.ObjectReference{ + Name: "subscriber_test", + Kind: "Service", + APIVersion: "serving.knative.dev/v1alpha1", + }, + } + invalidSubscriber = &SubscriberSpec{ + Ref: &corev1.ObjectReference{ + Kind: "Service", + APIVersion: "serving.knative.dev/v1alpha1", + }, + } +) + +func TestTriggerValidation(t *testing.T) { + name := "invalid trigger spec" + trigger := &Trigger{Spec: TriggerSpec{}} + + want := &apis.FieldError{ + Paths: []string{"spec.broker", "spec.filter", "spec.subscriber"}, + Message: "missing field(s)", + } + + t.Run(name, func(t *testing.T) { + got := trigger.Validate() + if diff := cmp.Diff(want.Error(), got.Error()); diff != "" { + t.Errorf("Trigger.Validate (-want, +got) = %v", diff) + } + }) +} + +func TestTriggerSpecValidation(t *testing.T) { + tests := []struct { + name string + ts *TriggerSpec + want *apis.FieldError + }{{ + name: "invalid trigger spec", + ts: &TriggerSpec{}, + want: func() *apis.FieldError { + fe := apis.ErrMissingField("broker", "filter", "subscriber") + return fe + }(), + }, { + name: "missing broker", + ts: &TriggerSpec{ + Broker: "", + Filter: validTriggerFilter, + Subscriber: validSubscriber, + }, + want: func() *apis.FieldError { + fe := apis.ErrMissingField("broker") + return fe + }(), + }, { + name: "missing filter", + ts: &TriggerSpec{ + Broker: "test_broker", + Subscriber: validSubscriber, + }, + want: func() *apis.FieldError { + fe := apis.ErrMissingField("filter") + return fe + }(), + }, { + name: "missing filter.sourceAndType", + ts: &TriggerSpec{ + Broker: "test_broker", + Filter: &TriggerFilter{}, + Subscriber: validSubscriber, + }, + want: func() *apis.FieldError { + fe := apis.ErrMissingField("filter.sourceAndType") + return fe + }(), + }, { + name: "missing subscriber", + ts: &TriggerSpec{ + Broker: "test_broker", + Filter: validTriggerFilter, + }, + want: func() *apis.FieldError { + fe := apis.ErrMissingField("subscriber") + return fe + }(), + }, { + name: "missing subscriber.ref.name", + ts: &TriggerSpec{ + Broker: "test_broker", + Filter: validTriggerFilter, + Subscriber: invalidSubscriber, + }, + want: func() *apis.FieldError { + fe := apis.ErrMissingField("subscriber.ref.name") + return fe + }(), + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got := test.ts.Validate() + if diff := cmp.Diff(test.want.Error(), got.Error()); diff != "" { + t.Errorf("%s: Validate TriggerSpec (-want, +got) = %v", test.name, diff) + } + }) + } +} + +func TestTriggerImmutableFields(t *testing.T) { + tests := []struct { + name string + current *Trigger + original *Trigger + want *apis.FieldError + }{{ + name: "good (no change)", + current: &Trigger{ + Spec: TriggerSpec{ + Broker: "broker", + }, + }, + original: &Trigger{ + Spec: TriggerSpec{ + Broker: "broker", + }, + }, + want: nil, + }, { + name: "new nil is ok", + current: &Trigger{ + Spec: TriggerSpec{ + Broker: "broker", + }, + }, + original: nil, + want: nil, + }, { + name: "good (filter change)", + current: &Trigger{ + Spec: TriggerSpec{ + Broker: "broker", + }, + }, + original: &Trigger{ + Spec: TriggerSpec{ + Broker: "broker", + Filter: validTriggerFilter, + }, + }, + want: nil, + }, { + name: "bad (broker change)", + current: &Trigger{ + Spec: TriggerSpec{ + Broker: "broker", + }, + }, + original: &Trigger{ + Spec: TriggerSpec{ + Broker: "original_broker", + }, + }, + want: &apis.FieldError{ + Message: "Immutable fields changed (-old +new)", + Paths: []string{"spec", "broker"}, + Details: `{string}: + -: "original_broker" + +: "broker" +`, + }, + }} + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got := test.current.CheckImmutableFields(test.original) + if diff := cmp.Diff(test.want.Error(), got.Error()); diff != "" { + t.Errorf("CheckImmutableFields (-want, +got) = %v", diff) + } + }) + } +} + +func TestTriggerInvalidImmutableType(t *testing.T) { + tests := []struct { + name string + current apis.Immutable + original apis.Immutable + want *apis.FieldError + }{{ + name: "invalid type", + current: &Trigger{ + Spec: TriggerSpec{ + Broker: "broker", + }, + }, + original: nil, + want: &apis.FieldError{ + Message: "The provided original was not a Trigger", + }, + }} + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got := test.current.CheckImmutableFields(test.original) + if diff := cmp.Diff(test.want.Error(), got.Error()); diff != "" { + t.Errorf("CheckImmutableType (-want, +got) = %v", diff) + } + }) + } +} From 11a1734e1706b290322c875d880a36dc488ce1d5 Mon Sep 17 00:00:00 2001 From: nachocano Date: Tue, 19 Feb 2019 11:44:04 -0800 Subject: [PATCH 065/128] Cleaner trigger validation --- .../v1alpha1/broker_validation_test.go | 17 +++++++ .../eventing/v1alpha1/trigger_validation.go | 7 +-- .../v1alpha1/trigger_validation_test.go | 44 ++++++------------- 3 files changed, 34 insertions(+), 34 deletions(-) create mode 100644 pkg/apis/eventing/v1alpha1/broker_validation_test.go diff --git a/pkg/apis/eventing/v1alpha1/broker_validation_test.go b/pkg/apis/eventing/v1alpha1/broker_validation_test.go new file mode 100644 index 00000000000..833cf12f577 --- /dev/null +++ b/pkg/apis/eventing/v1alpha1/broker_validation_test.go @@ -0,0 +1,17 @@ +/* +Copyright 2019 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 diff --git a/pkg/apis/eventing/v1alpha1/trigger_validation.go b/pkg/apis/eventing/v1alpha1/trigger_validation.go index e58150fdf48..ae57a5bf4e6 100644 --- a/pkg/apis/eventing/v1alpha1/trigger_validation.go +++ b/pkg/apis/eventing/v1alpha1/trigger_validation.go @@ -53,13 +53,14 @@ func (ts *TriggerSpec) Validate() *apis.FieldError { } func (t *Trigger) CheckImmutableFields(og apis.Immutable) *apis.FieldError { + if og == nil { + return nil + } + original, ok := og.(*Trigger) if !ok { return &apis.FieldError{Message: "The provided original was not a Trigger"} } - if original == nil { - return nil - } if diff := cmp.Diff(original.Spec.Broker, t.Spec.Broker); diff != "" { return &apis.FieldError{ diff --git a/pkg/apis/eventing/v1alpha1/trigger_validation_test.go b/pkg/apis/eventing/v1alpha1/trigger_validation_test.go index 8afa183faca..313f8132708 100644 --- a/pkg/apis/eventing/v1alpha1/trigger_validation_test.go +++ b/pkg/apis/eventing/v1alpha1/trigger_validation_test.go @@ -144,8 +144,8 @@ func TestTriggerSpecValidation(t *testing.T) { func TestTriggerImmutableFields(t *testing.T) { tests := []struct { name string - current *Trigger - original *Trigger + current apis.Immutable + original apis.Immutable want *apis.FieldError }{{ name: "good (no change)", @@ -169,6 +169,17 @@ func TestTriggerImmutableFields(t *testing.T) { }, original: nil, want: nil, + }, { + name: "invalid type", + current: &Trigger{ + Spec: TriggerSpec{ + Broker: "broker", + }, + }, + original: &Broker{}, + want: &apis.FieldError{ + Message: "The provided original was not a Trigger", + }, }, { name: "good (filter change)", current: &Trigger{ @@ -214,32 +225,3 @@ func TestTriggerImmutableFields(t *testing.T) { }) } } - -func TestTriggerInvalidImmutableType(t *testing.T) { - tests := []struct { - name string - current apis.Immutable - original apis.Immutable - want *apis.FieldError - }{{ - name: "invalid type", - current: &Trigger{ - Spec: TriggerSpec{ - Broker: "broker", - }, - }, - original: nil, - want: &apis.FieldError{ - Message: "The provided original was not a Trigger", - }, - }} - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - got := test.current.CheckImmutableFields(test.original) - if diff := cmp.Diff(test.want.Error(), got.Error()); diff != "" { - t.Errorf("CheckImmutableType (-want, +got) = %v", diff) - } - }) - } -} From d713eeb3dd17f10acc2e5813245b8112bf48db86 Mon Sep 17 00:00:00 2001 From: nachocano Date: Tue, 19 Feb 2019 11:52:03 -0800 Subject: [PATCH 066/128] Adding dummy tests for broker validation... Should be implemented --- .../eventing/v1alpha1/broker_validation.go | 2 ++ .../v1alpha1/broker_validation_test.go | 23 +++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/pkg/apis/eventing/v1alpha1/broker_validation.go b/pkg/apis/eventing/v1alpha1/broker_validation.go index 028b8db11f6..3dbe58255b5 100644 --- a/pkg/apis/eventing/v1alpha1/broker_validation.go +++ b/pkg/apis/eventing/v1alpha1/broker_validation.go @@ -25,9 +25,11 @@ func (b *Broker) Validate() *apis.FieldError { } func (bs *BrokerSpec) Validate() *apis.FieldError { + // TODO implement return nil } func (b *Broker) CheckImmutableFields(og apis.Immutable) *apis.FieldError { + // TODO implement return nil } diff --git a/pkg/apis/eventing/v1alpha1/broker_validation_test.go b/pkg/apis/eventing/v1alpha1/broker_validation_test.go index 833cf12f577..b7842e869bb 100644 --- a/pkg/apis/eventing/v1alpha1/broker_validation_test.go +++ b/pkg/apis/eventing/v1alpha1/broker_validation_test.go @@ -15,3 +15,26 @@ limitations under the License. */ package v1alpha1 + +import ( + "testing" +) + +// No-op test because method does nothing. +func TestBrokerValidation(t *testing.T) { + b := Broker{} + _ = b.Validate() +} + +// No-op test because method does nothing. +func TestBrokerSpecValidation(t *testing.T) { + bs := BrokerSpec{} + _ = bs.Validate() +} + +// No-op test because method does nothing. +func TestBrokerImmutableFields(t *testing.T) { + original := &Broker{} + current := &Broker{} + _ = current.CheckImmutableFields(original) +} From 49fd1a0ac57ba5589535639132d8d0786011a7a4 Mon Sep 17 00:00:00 2001 From: nachocano Date: Tue, 19 Feb 2019 15:06:16 -0800 Subject: [PATCH 067/128] Compiling and moving things around --- ..._filter_test.go => broker_trigger_test.go} | 90 +++++++------------ test/e2e/builder/broker.go | 48 ---------- test/e2e/{builder/trigger.go => builders.go} | 55 ++++++++++-- test/e2e/e2e.go | 40 +++++++++ test/e2e/fixtures.go | 13 +-- test/e2e/single_event_test.go | 45 +--------- test/states.go | 4 +- 7 files changed, 128 insertions(+), 167 deletions(-) rename test/e2e/{broker_filter_test.go => broker_trigger_test.go} (70%) delete mode 100644 test/e2e/builder/broker.go rename test/e2e/{builder/trigger.go => builders.go} (60%) diff --git a/test/e2e/broker_filter_test.go b/test/e2e/broker_trigger_test.go similarity index 70% rename from test/e2e/broker_filter_test.go rename to test/e2e/broker_trigger_test.go index f42436dae2b..905c1d6083b 100644 --- a/test/e2e/broker_filter_test.go +++ b/test/e2e/broker_trigger_test.go @@ -18,15 +18,14 @@ limitations under the License. package e2e import ( + "context" "fmt" "testing" - "time" "github.com/knative/eventing/test" + "github.com/knative/eventing/test/e2e/broker_trigger/builder" pkgTest "github.com/knative/pkg/test" "github.com/knative/pkg/test/logging" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/uuid" ) @@ -34,62 +33,24 @@ import ( const ( defaultBrokerName = "default" altBrokerName = "alternate" - anyEvent = "Any" untypedEvent = "test.untyped" typedEvent = "test.typed" + unsourcedEvent = "test.unsourced" + sourcedEvent = "test.sourced" ) func triggerName(broker, eventType string) string { fmt.Sprintf("%s-dump-%s", broker, eventType) } -func namespaceExists(t *testing.T, clients *test.Clients) (string, func()) { - logger := logging.GetContextLogger("TestBrokerFilter") - shutdown := func() {} - ns := pkgTest.Flags.Namespace - logger.Infof("Namespace: %s", ns) - - nsSpec, err := clients.Kube.Kube.CoreV1().Namespaces().Get(ns, metav1.GetOptions{}) - - if err != nil && errors.IsNotFound(err) { - nsSpec = &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: ns}} - logger.Infof("Creating Namespace: %s", ns) - nsSpec, err = clients.Kube.Kube.CoreV1().Namespaces().Create(nsSpec) - if err != nil { - t.Fatalf("Failed to create Namespace: %s; %v", ns, err) - } else { - shutdown = func() { - clients.Kube.Kube.CoreV1().Namespaces().Delete(nsSpec.Name, nil) - // TODO: this is a bit hacky but in order for the tests to work - // correctly for a clean namespace to be created we need to also - // wait for it to be removed. - // To fix this we could generate namespace names. - // This only happens when the namespace provided does not exist. - // - // wait up to 120 seconds for the namespace to be removed. - logger.Infof("Deleting Namespace: %s", ns) - for i := 0; i < 120; i++ { - time.Sleep(1 * time.Second) - if _, err := clients.Kube.Kube.CoreV1().Namespaces().Get(ns, metav1.GetOptions{}); err != nil && errors.IsNotFound(err) { - logger.Info("Namespace has been deleted") - // the namespace is gone. - break - } - } - } - } - } - return ns, shutdown -} - -func TestBrokerFilter(t *testing.T) { - logger := logging.GetContextLogger("TestBrokerFilter") +func TestBrokerTrigger(t *testing.T) { + logger := logging.GetContextLogger("TestBrokerTrigger") clients, cleaner := Setup(t, logger) defer TearDown(clients, cleaner, logger) // verify namespace - ns, cleanupNS := namespaceExists(t, clients) + ns, cleanupNS := NamespaceExists(t, clients, logger) defer cleanupNS() // Fixtures @@ -99,16 +60,21 @@ func TestBrokerFilter(t *testing.T) { fixtures := []Fixture{ // Default Any Trigger &KnativeFixture{ - Object: builder.Trigger("default-dump-any"), pkgTest.Flags.Namespace). - Type(anyEvent). + Object: builder.Trigger("default-dump-any", pkgTest.Flags.Namespace). SubscriberSvc("default-any-dumper"), }, // Default Typed Trigger &KnativeFixture{ - Object: builder.Trigger("default-dump-typed"), pkgTest.Flags.Namespace). + Object: builder.Trigger("default-dump-typed", pkgTest.Flags.Namespace). Type(typedEvent). SubscriberSvc("default-typed-dumper"), }, + // Default Sourced Trigger + &KnativeFixture{ + Object: builder.Trigger("default-dump-sourced", pkgTest.Flags.Namespace). + Type(sourcedEvent). + SubscriberSvc("default-sourced-dumper"), + }, // Alternate Broker &KnativeFixture{ @@ -116,18 +82,24 @@ func TestBrokerFilter(t *testing.T) { }, // Alternate Any Trigger &KnativeFixture{ - Object: builder.Trigger("alt-dump-any"), pkgTest.Flags.Namespace). + Object: builder.Trigger("alt-dump-any", pkgTest.Flags.Namespace). Broker(altBrokerName). - Type(anyEvent). - SubscriberSvc(fmt.Sprintf("alt-any-dumper"), + SubscriberSvc(fmt.Sprintf("alt-any-dumper")), }, // Alternate Typed Trigger &KnativeFixture{ - Object: builder.Trigger("alt-dump-typed"), pkgTest.Flags.Namespace). + Object: builder.Trigger("alt-dump-typed", pkgTest.Flags.Namespace). Broker(altBrokerName). Type(typedEvent). SubscriberSvc("alt-typed-dumper"), }, + // Alternate Sourced Trigger + &KnativeFixture{ + Object: builder.Trigger("alt-dump-sourced", pkgTest.Flags.Namespace). + Broker(altBrokerName). + Type(sourcedEvent). + SubscriberSvc("alt-sourced-dumper"), + }, } ctx := context.Background() @@ -145,13 +117,13 @@ func TestBrokerFilter(t *testing.T) { // Create message dumper services // Create: pod // Verify: pod status is ready - // + // - pods := []Fixture{ - &PodSuccess{ - - } - } + //pods := []Fixture{ + // &PodSuccess{ + // + // } + //} // create alt Broker diff --git a/test/e2e/builder/broker.go b/test/e2e/builder/broker.go deleted file mode 100644 index dc00c6245d3..00000000000 --- a/test/e2e/builder/broker.go +++ /dev/null @@ -1,48 +0,0 @@ -/* -Copyright 2019 The Knative Authors -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package builder - -import ( - eventingv1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" -) - -type BrokerBuilder struct { - *eventingv1alpha1.Broker -} - -func Broker(name, namespace string) *BrokerBuilder { - broker := &eventingv1alpha1.Broker{ - TypeMeta: metav1.TypeMeta{ - APIVersion: eventingv1alpha1.SchemeGroupVersion.String(), - Kind: "Broker", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - Spec: eventingv1alpha1.BrokerSpec{}, - } - - return &BrokerBuilder{ - Broker: broker, - } -} - -func (b *BrokerBuilder) Build() runtime.Object { - return b.Broker.DeepCopy() -} \ No newline at end of file diff --git a/test/e2e/builder/trigger.go b/test/e2e/builders.go similarity index 60% rename from test/e2e/builder/trigger.go rename to test/e2e/builders.go index 745a71a34ad..ae1853fbbb1 100644 --- a/test/e2e/builder/trigger.go +++ b/test/e2e/builders.go @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package builder +package e2e import ( eventingv1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" @@ -22,6 +22,34 @@ import ( "k8s.io/apimachinery/pkg/runtime" ) +// Broker builder. +type BrokerBuilder struct { + *eventingv1alpha1.Broker +} + +func Broker(name, namespace string) *BrokerBuilder { + broker := &eventingv1alpha1.Broker{ + TypeMeta: metav1.TypeMeta{ + APIVersion: eventingv1alpha1.SchemeGroupVersion.String(), + Kind: "Broker", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: eventingv1alpha1.BrokerSpec{}, + } + + return &BrokerBuilder{ + Broker: broker, + } +} + +func (b *BrokerBuilder) Build() runtime.Object { + return b.Broker.DeepCopy() +} + +// Trigger builder. type TriggerBuilder struct { *eventingv1alpha1.Trigger } @@ -36,7 +64,15 @@ func Trigger(name, namespace string) *TriggerBuilder { Name: name, Namespace: namespace, }, - Spec: eventingv1alpha1.TriggerSpec{}, + Spec: eventingv1alpha1.TriggerSpec{ + Filter: &eventingv1alpha1.TriggerFilter{ + // Create a Any filter by default. + SourceAndType: &eventingv1alpha1.TriggerFilterSourceAndType{ + Source: eventingv1alpha1.TriggerAnyFilter, + Type: eventingv1alpha1.TriggerAnyFilter, + }, + }, + }, } return &TriggerBuilder{ @@ -49,7 +85,12 @@ func (b *TriggerBuilder) Build() runtime.Object { } func (b *TriggerBuilder) Type(eventType string) *TriggerBuilder { - b.Trigger.Spec.Type = eventType + b.Trigger.Spec.Filter.SourceAndType.Type = eventType + return b +} + +func (b *TriggerBuilder) Source(eventSource string) *TriggerBuilder { + b.Trigger.Spec.Filter.SourceAndType.Source = eventSource return b } @@ -66,9 +107,9 @@ func (b *TriggerBuilder) Subscriber(ref *corev1.ObjectReference) *TriggerBuilder func (b *TriggerBuilder) SubscriberSvc(svcName string) *TriggerBuilder { b.Trigger.Spec.Subscriber.Ref = &corev1.ObjectReference{ APIVersion: corev1.SchemeGroupVersion.String(), - Kind: "Service", - Name: svcName, - Namespace: b.Trigger.GetNamespace(), + Kind: "Service", + Name: svcName, + Namespace: b.Trigger.GetNamespace(), } return b -} \ No newline at end of file +} diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go index 453c0c69ea6..5fd3fffd4c3 100644 --- a/test/e2e/e2e.go +++ b/test/e2e/e2e.go @@ -21,6 +21,8 @@ import ( "testing" "time" + "k8s.io/apimachinery/pkg/api/errors" + "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" "github.com/knative/eventing/test" pkgTest "github.com/knative/pkg/test" @@ -305,3 +307,41 @@ func WaitForAllPodsRunning(clients *test.Clients, logger *logging.BaseLogger, na } return nil } + +func NamespaceExists(t *testing.T, clients *test.Clients, logger *logging.BaseLogger) (string, func()) { + shutdown := func() {} + ns := pkgTest.Flags.Namespace + logger.Infof("Namespace: %s", ns) + + nsSpec, err := clients.Kube.Kube.CoreV1().Namespaces().Get(ns, metav1.GetOptions{}) + + if err != nil && errors.IsNotFound(err) { + nsSpec = &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: ns}} + logger.Infof("Creating Namespace: %s", ns) + nsSpec, err = clients.Kube.Kube.CoreV1().Namespaces().Create(nsSpec) + if err != nil { + t.Fatalf("Failed to create Namespace: %s; %v", ns, err) + } else { + shutdown = func() { + clients.Kube.Kube.CoreV1().Namespaces().Delete(nsSpec.Name, nil) + // TODO: this is a bit hacky but in order for the tests to work + // correctly for a clean namespace to be created we need to also + // wait for it to be removed. + // To fix this we could generate namespace names. + // This only happens when the namespace provided does not exist. + // + // wait up to 120 seconds for the namespace to be removed. + logger.Infof("Deleting Namespace: %s", ns) + for i := 0; i < 120; i++ { + time.Sleep(1 * time.Second) + if _, err := clients.Kube.Kube.CoreV1().Namespaces().Get(ns, metav1.GetOptions{}); err != nil && errors.IsNotFound(err) { + logger.Info("Namespace has been deleted") + // the namespace is gone. + break + } + } + } + } + } + return ns, shutdown +} diff --git a/test/e2e/fixtures.go b/test/e2e/fixtures.go index f2174c58048..3bfa969bd7d 100644 --- a/test/e2e/fixtures.go +++ b/test/e2e/fixtures.go @@ -16,6 +16,7 @@ package e2e import ( "context" + "github.com/knative/eventing/pkg/reconciler/testing" "github.com/knative/pkg/apis/duck" duckv1alpha1 "github.com/knative/pkg/apis/duck/v1alpha1" @@ -41,7 +42,7 @@ type Fixture interface { } type KnativeFixture struct { - Object testing.Buildable + Object testing.Buildable } type CreateFunc func(context.Context, client.Client) error @@ -50,9 +51,9 @@ type DebugFunc func(context.Context, client.Client) type TeardownFunc func(context.Context, client.Client) error type FixtureFuncs struct { - Create CreateFunc - Verify VerifyFunc - Debug DebugFunc + Create CreateFunc + Verify VerifyFunc + Debug DebugFunc Teardown TeardownFunc } @@ -102,7 +103,7 @@ func (f *KnativeFixture) Debug(ctx context.Context, cl client.Client) { } // TODO if Teardown returns an error, log it and give up (maybe retry for a while?) -func (f *KnativeFixture) Teardownctx context.Context, cl client.Client) error { +func (f *KnativeFixture) Teardown(ctx context.Context, cl client.Client) error { obj := f.Object.Build() return cl.Delete(ctx, obj) -} \ No newline at end of file +} diff --git a/test/e2e/single_event_test.go b/test/e2e/single_event_test.go index f62f2bab46d..a1d9455c71d 100644 --- a/test/e2e/single_event_test.go +++ b/test/e2e/single_event_test.go @@ -20,13 +20,9 @@ package e2e import ( "fmt" "testing" - "time" "github.com/knative/eventing/test" - pkgTest "github.com/knative/pkg/test" "github.com/knative/pkg/test/logging" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/uuid" ) @@ -39,45 +35,6 @@ const ( routeName = "e2e-singleevent-route" ) -func namespaceExists(t *testing.T, clients *test.Clients) (string, func()) { - logger := logging.GetContextLogger("TestSingleEvent") - shutdown := func() {} - ns := pkgTest.Flags.Namespace - logger.Infof("Namespace: %s", ns) - - nsSpec, err := clients.Kube.Kube.CoreV1().Namespaces().Get(ns, metav1.GetOptions{}) - - if err != nil && errors.IsNotFound(err) { - nsSpec = &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: ns}} - logger.Infof("Creating Namespace: %s", ns) - nsSpec, err = clients.Kube.Kube.CoreV1().Namespaces().Create(nsSpec) - if err != nil { - t.Fatalf("Failed to create Namespace: %s; %v", ns, err) - } else { - shutdown = func() { - clients.Kube.Kube.CoreV1().Namespaces().Delete(nsSpec.Name, nil) - // TODO: this is a bit hacky but in order for the tests to work - // correctly for a clean namespace to be created we need to also - // wait for it to be removed. - // To fix this we could generate namespace names. - // This only happens when the namespace provided does not exist. - // - // wait up to 120 seconds for the namespace to be removed. - logger.Infof("Deleting Namespace: %s", ns) - for i := 0; i < 120; i++ { - time.Sleep(1 * time.Second) - if _, err := clients.Kube.Kube.CoreV1().Namespaces().Get(ns, metav1.GetOptions{}); err != nil && errors.IsNotFound(err) { - logger.Info("Namespace has been deleted") - // the namespace is gone. - break - } - } - } - } - } - return ns, shutdown -} - func TestSingleBinaryEvent(t *testing.T) { SingleEvent(t, test.CloudEventEncodingBinary) } @@ -94,7 +51,7 @@ func SingleEvent(t *testing.T, encoding string) { // verify namespace - ns, cleanupNS := namespaceExists(t, clients) + ns, cleanupNS := NamespaceExists(t, clients, logger) defer cleanupNS() // create logger pod diff --git a/test/states.go b/test/states.go index e5a8dd0cdcf..26db82aee3d 100644 --- a/test/states.go +++ b/test/states.go @@ -54,7 +54,7 @@ func IsSubscriptionReady(s *eventingv1alpha1.Subscription) (bool, error) { return s.Status.IsReady(), nil } -/ IsBrokerReady will check the status conditions of the Broker and return true +// IsBrokerReady will check the status conditions of the Broker and return true // if the Broker is ready. func IsBrokerReady(b *eventingv1alpha1.Broker) (bool, error) { return b.Status.IsReady(), nil @@ -66,8 +66,6 @@ func IsTriggerReady(t *eventingv1alpha1.Trigger) (bool, error) { return t.Status.IsReady(), nil } - - // PodsRunning will check the status conditions of the pod list and return true // if all pods are Running. func PodsRunning(podList *corev1.PodList) (bool, error) { From 4cdfd5888b86042b406d347d4f5d7f7aea8e0a43 Mon Sep 17 00:00:00 2001 From: nachocano Date: Tue, 19 Feb 2019 18:04:18 -0800 Subject: [PATCH 068/128] Updating test --- test/crd.go | 19 ++- test/e2e/broker_trigger_test.go | 203 +++++++++++--------------------- test/e2e/builders.go | 45 +++---- test/e2e/e2e.go | 20 ++++ 4 files changed, 128 insertions(+), 159 deletions(-) diff --git a/test/crd.go b/test/crd.go index 0806b221a70..f9f350e5939 100644 --- a/test/crd.go +++ b/test/crd.go @@ -19,6 +19,7 @@ package test import ( "fmt" + "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" servingv1alpha1 "github.com/knative/serving/pkg/apis/serving/v1alpha1" corev1 "k8s.io/api/core/v1" @@ -167,6 +168,7 @@ func Subscription(name string, namespace string, channel *corev1.ObjectReference } } +// Broker returns a Broker. func Broker(name string, namespace string) *v1alpha1.Broker { return &v1alpha1.Broker{ ObjectMeta: metav1.ObjectMeta{ @@ -177,7 +179,8 @@ func Broker(name string, namespace string) *v1alpha1.Broker { } } -func Trigger(name string, namespace string, eventType string, subscriberRef *corev1.ObjectReference, brokerName string) *v1alpha1.Trigger { +// Trigger returns a Trigger. +func Trigger(name, namespace, eventType, eventSource, brokerName, svcName string) *v1alpha1.Trigger { return &v1alpha1.Trigger{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -185,9 +188,19 @@ func Trigger(name string, namespace string, eventType string, subscriberRef *cor }, Spec: v1alpha1.TriggerSpec{ Broker: brokerName, - Type: fmt.Sprintf("%q", eventType), + Filter: &v1alpha1.TriggerFilter{ + SourceAndType: &v1alpha1.TriggerFilterSourceAndType{ + Type: fmt.Sprintf("%q", eventType), + Source: fmt.Sprintf("%q", eventSource), + }, + }, Subscriber: &v1alpha1.SubscriberSpec{ - Ref: subscriberRef, + Ref: &corev1.ObjectReference{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "Service", + Name: svcName, + Namespace: namespace, + }, }, }, } diff --git a/test/e2e/broker_trigger_test.go b/test/e2e/broker_trigger_test.go index 905c1d6083b..cfbb94408ee 100644 --- a/test/e2e/broker_trigger_test.go +++ b/test/e2e/broker_trigger_test.go @@ -18,27 +18,35 @@ limitations under the License. package e2e import ( - "context" "fmt" "testing" + "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" + "github.com/knative/eventing/test" - "github.com/knative/eventing/test/e2e/broker_trigger/builder" - pkgTest "github.com/knative/pkg/test" "github.com/knative/pkg/test/logging" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/uuid" ) const ( defaultBrokerName = "default" - altBrokerName = "alternate" - untypedEvent = "test.untyped" - typedEvent = "test.typed" - unsourcedEvent = "test.unsourced" - sourcedEvent = "test.sourced" + + eventAny = v1alpha1.TriggerAnyFilter + eventType1 = "test.type1" + eventType2 = "test.type2" + eventSource1 = "test.source1" + eventSource2 = "test.source2" + + defaultAnyDumper = "default-any-dumper" + defaultType1Dumper = "default-type1-dumper" + defaultSource1Dumper = "default-source1-dumper" + defaultType1Source1Dumper = "default-type1-source1-dumper" ) +type TypeAndSource struct { + Type, Source string +} + func triggerName(broker, eventType string) string { fmt.Sprintf("%s-dump-%s", broker, eventType) } @@ -53,151 +61,78 @@ func TestBrokerTrigger(t *testing.T) { ns, cleanupNS := NamespaceExists(t, clients, logger) defer cleanupNS() - // Fixtures - // TODO label namespace to get default Broker - fixtures := []Fixture{ - // Default Any Trigger - &KnativeFixture{ - Object: builder.Trigger("default-dump-any", pkgTest.Flags.Namespace). - SubscriberSvc("default-any-dumper"), - }, - // Default Typed Trigger - &KnativeFixture{ - Object: builder.Trigger("default-dump-typed", pkgTest.Flags.Namespace). - Type(typedEvent). - SubscriberSvc("default-typed-dumper"), - }, - // Default Sourced Trigger - &KnativeFixture{ - Object: builder.Trigger("default-dump-sourced", pkgTest.Flags.Namespace). - Type(sourcedEvent). - SubscriberSvc("default-sourced-dumper"), - }, - - // Alternate Broker - &KnativeFixture{ - Object: builder.Broker(altBrokerName, pkgTest.Flags.Namespace), - }, - // Alternate Any Trigger - &KnativeFixture{ - Object: builder.Trigger("alt-dump-any", pkgTest.Flags.Namespace). - Broker(altBrokerName). - SubscriberSvc(fmt.Sprintf("alt-any-dumper")), - }, - // Alternate Typed Trigger - &KnativeFixture{ - Object: builder.Trigger("alt-dump-typed", pkgTest.Flags.Namespace). - Broker(altBrokerName). - Type(typedEvent). - SubscriberSvc("alt-typed-dumper"), - }, - // Alternate Sourced Trigger - &KnativeFixture{ - Object: builder.Trigger("alt-dump-sourced", pkgTest.Flags.Namespace). - Broker(altBrokerName). - Type(sourcedEvent). - SubscriberSvc("alt-sourced-dumper"), - }, - } + logger.Info("Creating Subscriber pods") - ctx := context.Background() + eventLoggerSelectors := []map[string]string{{"svcName": defaultAnyDumper}, {"svcName": defaultType1Dumper}, {"svcName": defaultSource1Dumper}, {"svcName": defaultType1Source1Dumper}} - //TODO move this to runners + for eventLoggerSelector := range eventLoggerSelectors { + subscriberPod := test.EventLoggerPod(eventLoggerSelector["svcName"], ns, eventLoggerSelector) - for _, f := range fixtures { - f.Create(ctx, client) + if err := CreatePod(clients, subscriberPod, logger, cleaner); err != nil { + t.Fatalf("Failed to create event logger pod: %v", err) + } } - for _, f := range fixtures { - f.Verify(ctx, client) + if err := WaitForAllPodsRunning(clients, logger, ns); err != nil { + t.Fatalf("Error waiting for event logger pod to become running: %v", err) } - // Create message dumper services - // Create: pod - // Verify: pod status is ready - // - - //pods := []Fixture{ - // &PodSuccess{ - // - // } - //} - - // create alt Broker - - // For each broker, create a typed and untyped trigger. - // For each trigger, create a logevents pod. - // Wait for Broker, triggers, and pods to become ready. - - // Take Action + logger.Info("Subscriber pods running") - // For each tuple of typed and untyped, default and alt broker: - // create a sendevents pod to send an event of the type to the broker address - - // Verify - - // For each logevents pod: - // check logs to ensure the correct message(s) got there + for eventLoggerSelector := range eventLoggerSelectors { + subscriberSvc := test.Service(eventLoggerSelector["svcName"], ns, eventLoggerSelector) + if err := CreateService(clients, subscriberSvc, logger, cleaner); err != nil { + t.Fatalf("Failed to create event logger service: %v", err) + } + } - // create logger pod + logger.Info("Creating Triggers") - logger.Infof("creating subscriber pod") - selector := map[string]string{"e2etest": string(uuid.NewUUID())} - subscriberPod := test.EventLoggerPod(routeName, ns, selector) - if err := CreatePod(clients, subscriberPod, logger, cleaner); err != nil { - t.Fatalf("Failed to create event logger pod: %v", err) + defaultTriggers := []*v1alpha1.Trigger{ + test.Trigger("default-dump-any", ns, eventAny, eventAny, defaultBrokerName, defaultAnyDumper), + test.Trigger("default-dump-type1", ns, eventType1, eventAny, defaultBrokerName, defaultType1Dumper), + test.Trigger("default-dump-source1", ns, eventAny, eventSource1, defaultBrokerName, defaultSource1Dumper), + test.Trigger("default-dump-type1-source1", ns, eventType1, eventSource1, defaultBrokerName, defaultType1Source1Dumper), } - if err := WaitForAllPodsRunning(clients, logger, ns); err != nil { - t.Fatalf("Error waiting for logger pod to become running: %v", err) - } - logger.Infof("subscriber pod running") - - subscriberSvc := test.Service(routeName, ns, selector) - if err := CreateService(clients, subscriberSvc, logger, cleaner); err != nil { - t.Fatalf("Failed to create event logger service: %v", err) + for defaultTrigger := range defaultTriggers { + err := WithTriggerReady(clients, defaultTrigger, logger, cleaner) + if err != nil { + t.Fatalf("Error waiting for default trigger to become ready: %v", err) + } } - // Reload subscriberPod to get IP - subscriberPod, err := clients.Kube.Kube.CoreV1().Pods(subscriberPod.Namespace).Get(subscriberPod.Name, metav1.GetOptions{}) - if err != nil { - t.Fatalf("Failed to get subscriber pod: %v", err) - } + logger.Info("Triggers ready") - // create channel + logger.Infof("Creating event sender pods") - logger.Infof("Creating Channel and Subscription") - if test.EventingFlags.Provisioner == "" { - t.Fatal("ClusterChannelProvisioner must be set to a non-empty string. Either do not specify --clusterChannelProvisioner or set to something other than the empty string") + events = []TypeAndSource{ + {eventType1, eventSource1}, + {eventType1, eventSource2}, + {eventType2, eventSource1}, + {eventType2, eventSource2}, } - channel := test.Channel(channelName, ns, test.ClusterChannelProvisioner(test.EventingFlags.Provisioner)) - logger.Infof("channel: %#v", channel) - sub := test.Subscription(subscriptionName, ns, test.ChannelRef(channelName), test.SubscriberSpecForService(routeName), nil) - logger.Infof("sub: %#v", sub) - if err := WithChannelAndSubscriptionReady(clients, channel, sub, logger, cleaner); err != nil { - t.Fatalf("The Channel or Subscription were not marked as Ready: %v", err) + for event := range events { + body := fmt.Sprintf("Testing Broker-Trigger %s", uuid.NewUUID()) + cloudEvent := test.CloudEvent{ + Source: event.Source, + Type: event.Type, + Data: fmt.Sprintf(`{"msg":%q}`, body), + Encoding: encoding, + } + url := fmt.Sprintf("http://%s", channel.Status.Address.Hostname) + senderPod := test.EventSenderPod(event.Source, ns, url, cloudEvent) + logger.Infof("Sender pod: %#v", senderPod) + if err := CreatePod(clients, senderPod, logger, cleaner); err != nil { + t.Fatalf("Failed to create event sender pod: %v", err) + } } - // create sender pod - - logger.Infof("Creating event sender") - body := fmt.Sprintf("TestSingleEvent %s", uuid.NewUUID()) - event := test.CloudEvent{ - Source: senderName, - Type: "test.eventing.knative.dev", - Data: fmt.Sprintf(`{"msg":%q}`, body), - Encoding: encoding, - } - url := fmt.Sprintf("http://%s", channel.Status.Address.Hostname) - pod := test.EventSenderPod(senderName, ns, url, event) - logger.Infof("sender pod: %#v", pod) - if err := CreatePod(clients, pod, logger, cleaner); err != nil { - t.Fatalf("Failed to create event sender pod: %v", err) - } + // Verify that each arrived to its own place. + //if err := WaitForLogContent(clients, logger, routeName, subscriberPod.Spec.Containers[0].Name, ns, body); err != nil { + // t.Fatalf("String %q not found in logs of subscriber pod %q: %v", body, routeName, err) + //} - if err := WaitForLogContent(clients, logger, routeName, subscriberPod.Spec.Containers[0].Name, ns, body); err != nil { - t.Fatalf("String %q not found in logs of subscriber pod %q: %v", body, routeName, err) - } } diff --git a/test/e2e/builders.go b/test/e2e/builders.go index ae1853fbbb1..e89bb26f0a2 100644 --- a/test/e2e/builders.go +++ b/test/e2e/builders.go @@ -19,7 +19,6 @@ import ( eventingv1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" ) // Broker builder. @@ -27,7 +26,7 @@ type BrokerBuilder struct { *eventingv1alpha1.Broker } -func Broker(name, namespace string) *BrokerBuilder { +func NewBrokerBuilder(name, namespace string) *BrokerBuilder { broker := &eventingv1alpha1.Broker{ TypeMeta: metav1.TypeMeta{ APIVersion: eventingv1alpha1.SchemeGroupVersion.String(), @@ -45,7 +44,7 @@ func Broker(name, namespace string) *BrokerBuilder { } } -func (b *BrokerBuilder) Build() runtime.Object { +func (b *BrokerBuilder) Build() *eventingv1alpha1.Broker { return b.Broker.DeepCopy() } @@ -54,7 +53,7 @@ type TriggerBuilder struct { *eventingv1alpha1.Trigger } -func Trigger(name, namespace string) *TriggerBuilder { +func NewTriggerBuilder(name, namespace string) *TriggerBuilder { trigger := &eventingv1alpha1.Trigger{ TypeMeta: metav1.TypeMeta{ APIVersion: eventingv1alpha1.SchemeGroupVersion.String(), @@ -65,6 +64,8 @@ func Trigger(name, namespace string) *TriggerBuilder { Namespace: namespace, }, Spec: eventingv1alpha1.TriggerSpec{ + // Bind to 'default' Broker by default + Broker: "default", Filter: &eventingv1alpha1.TriggerFilter{ // Create a Any filter by default. SourceAndType: &eventingv1alpha1.TriggerFilterSourceAndType{ @@ -80,36 +81,36 @@ func Trigger(name, namespace string) *TriggerBuilder { } } -func (b *TriggerBuilder) Build() runtime.Object { - return b.Trigger.DeepCopy() +func (t *TriggerBuilder) Build() *eventingv1alpha1.Trigger { + return t.Trigger.DeepCopy() } -func (b *TriggerBuilder) Type(eventType string) *TriggerBuilder { - b.Trigger.Spec.Filter.SourceAndType.Type = eventType - return b +func (t *TriggerBuilder) Type(eventType string) *TriggerBuilder { + t.Trigger.Spec.Filter.SourceAndType.Type = eventType + return t } -func (b *TriggerBuilder) Source(eventSource string) *TriggerBuilder { - b.Trigger.Spec.Filter.SourceAndType.Source = eventSource - return b +func (t *TriggerBuilder) Source(eventSource string) *TriggerBuilder { + t.Trigger.Spec.Filter.SourceAndType.Source = eventSource + return t } -func (b *TriggerBuilder) Broker(brokerName string) *TriggerBuilder { - b.Trigger.Spec.Broker = brokerName - return b +func (t *TriggerBuilder) Broker(brokerName string) *TriggerBuilder { + t.Trigger.Spec.Broker = brokerName + return t } -func (b *TriggerBuilder) Subscriber(ref *corev1.ObjectReference) *TriggerBuilder { - b.Trigger.Spec.Subscriber.Ref = ref - return b +func (t *TriggerBuilder) Subscriber(ref *corev1.ObjectReference) *TriggerBuilder { + t.Trigger.Spec.Subscriber.Ref = ref + return t } -func (b *TriggerBuilder) SubscriberSvc(svcName string) *TriggerBuilder { - b.Trigger.Spec.Subscriber.Ref = &corev1.ObjectReference{ +func (t *TriggerBuilder) SubscriberSvc(svcName string) *TriggerBuilder { + t.Trigger.Spec.Subscriber.Ref = &corev1.ObjectReference{ APIVersion: corev1.SchemeGroupVersion.String(), Kind: "Service", Name: svcName, - Namespace: b.Trigger.GetNamespace(), + Namespace: t.Trigger.GetNamespace(), } - return b + return t } diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go index 5fd3fffd4c3..eedceab5ac1 100644 --- a/test/e2e/e2e.go +++ b/test/e2e/e2e.go @@ -183,6 +183,26 @@ func CreateTrigger(clients *test.Clients, trigger *v1alpha1.Trigger, logger *log return nil } +// WithTriggerReady creates a Trigger and waits until it is Ready. +func WithTriggerReady(clients *test.Clients, trigger *v1alpha1.Trigger, logger *logging.BaseLogger, cleaner *test.Cleaner) error { + if err := CreateTrigger(clients, trigger, logger, cleaner); err != nil { + return err + } + + triggers := clients.Eventing.EventingV1alpha1().Triggers(pkgTest.Flags.Namespace) + if err := test.WaitForTriggerState(triggers, trigger.Name, test.IsTriggerReady, "TriggerIsReady"); err != nil { + return err + } + // Update the given object so they'll reflect the ready state + updatedTrigger, err := triggers.Get(trigger.Name, metav1.GetOptions{}) + if err != nil { + return err + } + updatedTrigger.DeepCopyInto(trigger) + + return nil +} + // CreateServiceAccount will create a service account func CreateServiceAccount(clients *test.Clients, sa *corev1.ServiceAccount, logger *logging.BaseLogger, cleaner *test.Cleaner) error { sas := clients.Kube.Kube.CoreV1().ServiceAccounts(pkgTest.Flags.Namespace) From b42d359bbde96b0f184eb30a217790470ce2a025 Mon Sep 17 00:00:00 2001 From: Nacho Cano Date: Tue, 19 Feb 2019 23:58:33 -0800 Subject: [PATCH 069/128] More updates --- test/crd_checks.go | 22 ++++++++ test/e2e/broker_trigger_test.go | 91 ++++++++++++++++++--------------- test/e2e/e2e.go | 49 +++++++++++++++++- test/e2e/single_event_test.go | 2 +- 4 files changed, 120 insertions(+), 44 deletions(-) diff --git a/test/crd_checks.go b/test/crd_checks.go index f996b8e5fd0..6c66fc03ab9 100644 --- a/test/crd_checks.go +++ b/test/crd_checks.go @@ -23,6 +23,10 @@ import ( "fmt" "time" + v1 "k8s.io/client-go/kubernetes/typed/core/v1" + + corev1 "k8s.io/api/core/v1" + eventingv1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" eventingclient "github.com/knative/eventing/pkg/client/clientset/versioned/typed/eventing/v1alpha1" servingv1alpha1 "github.com/knative/serving/pkg/apis/serving/v1alpha1" @@ -127,3 +131,21 @@ func WaitForTriggerState(client eventingclient.TriggerInterface, name string, in return inState(r) }) } + +// WaitForServiceState polls the status of the Service called name from client +// every interval until inState returns `true` indicating it is done, returns an +// error or timeout. desc will be used to name the metric that is emitted to +// track how long it took for name to get into the state checked by inState. +func WaitForServiceState(client v1.ServiceInterface, name string, inState func(r *corev1.Service) (bool, error), desc string) error { + metricName := fmt.Sprintf("WaitForServiceState/%s/%s", name, desc) + _, span := trace.StartSpan(context.Background(), metricName) + defer span.End() + + return wait.PollImmediate(interval, timeout, func() (bool, error) { + r, err := client.Get(name, metav1.GetOptions{}) + if err != nil { + return true, err + } + return inState(r) + }) +} diff --git a/test/e2e/broker_trigger_test.go b/test/e2e/broker_trigger_test.go index cfbb94408ee..c5c8433c5f0 100644 --- a/test/e2e/broker_trigger_test.go +++ b/test/e2e/broker_trigger_test.go @@ -19,6 +19,7 @@ package e2e import ( "fmt" + "strings" "testing" "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" @@ -30,25 +31,21 @@ import ( const ( defaultBrokerName = "default" + selectorKey = "end2end-test-broker-trigger" - eventAny = v1alpha1.TriggerAnyFilter - eventType1 = "test.type1" - eventType2 = "test.type2" - eventSource1 = "test.source1" - eventSource2 = "test.source2" - - defaultAnyDumper = "default-any-dumper" - defaultType1Dumper = "default-type1-dumper" - defaultSource1Dumper = "default-source1-dumper" - defaultType1Source1Dumper = "default-type1-source1-dumper" + any = v1alpha1.TriggerAnyFilter + eventType1 = "type1" + eventType2 = "type2" + eventSource1 = "source1" + eventSource2 = "source2" ) type TypeAndSource struct { Type, Source string } -func triggerName(broker, eventType string) string { - fmt.Sprintf("%s-dump-%s", broker, eventType) +func name(brokerName, eventType, eventSource string) string { + return fmt.Sprintf("%s-%s-%s", brokerName, eventType, eventSource) } func TestBrokerTrigger(t *testing.T) { @@ -57,21 +54,30 @@ func TestBrokerTrigger(t *testing.T) { clients, cleaner := Setup(t, logger) defer TearDown(clients, cleaner, logger) - // verify namespace - ns, cleanupNS := NamespaceExists(t, clients, logger) + // verify namespace and annotate to create default broker + ns, cleanupNS := NamespaceExists(t, clients, logger, true) defer cleanupNS() - // TODO label namespace to get default Broker - - logger.Info("Creating Subscriber pods") + defaultBroker := test.Broker(defaultBrokerName, ns) + err := WaitForBrokerReady(clients, defaultBroker) + if err != nil { + t.Fatalf("Error waiting for default broker to become ready: %v", err) + } - eventLoggerSelectors := []map[string]string{{"svcName": defaultAnyDumper}, {"svcName": defaultType1Dumper}, {"svcName": defaultSource1Dumper}, {"svcName": defaultType1Source1Dumper}} + // name -> selector + eventLoggers := []map[string]map[string]string{ + {name(defaultBroker, any, any): {selectorKey: string(uuid.NewUUID())}}, + {name(defaultBroker, eventType1, any): {selectorKey: string(uuid.NewUUID())}}, + {name(defaultBroker, any, eventSource1): {selectorKey: string(uuid.NewUUID())}}, + {name(defaultBroker, eventType1, eventSource1): {selectorKey: string(uuid.NewUUID())}}, + } - for eventLoggerSelector := range eventLoggerSelectors { - subscriberPod := test.EventLoggerPod(eventLoggerSelector["svcName"], ns, eventLoggerSelector) + logger.Info("Creating Subscriber pods") + for name, selector := range eventLoggers { + subscriberPod := test.EventLoggerPod(name, ns, selector) if err := CreatePod(clients, subscriberPod, logger, cleaner); err != nil { - t.Fatalf("Failed to create event logger pod: %v", err) + t.Fatalf("Failed to create subscriber pod: %v", err) } } @@ -81,40 +87,41 @@ func TestBrokerTrigger(t *testing.T) { logger.Info("Subscriber pods running") - for eventLoggerSelector := range eventLoggerSelectors { - subscriberSvc := test.Service(eventLoggerSelector["svcName"], ns, eventLoggerSelector) - if err := CreateService(clients, subscriberSvc, logger, cleaner); err != nil { - t.Fatalf("Failed to create event logger service: %v", err) + logger.Info("Creating Subscriber services") + + for name, selector := range eventLoggers { + subscriberSvc := test.Service(name, ns, selector) + if err := WithServiceReady(clients, subscriberSvc, logger, cleaner); err != nil { + t.Fatalf("Error waiting for subscriber service to become ready: %v", err) } } + logger.Info("Subscriber services ready") + logger.Info("Creating Triggers") - defaultTriggers := []*v1alpha1.Trigger{ - test.Trigger("default-dump-any", ns, eventAny, eventAny, defaultBrokerName, defaultAnyDumper), - test.Trigger("default-dump-type1", ns, eventType1, eventAny, defaultBrokerName, defaultType1Dumper), - test.Trigger("default-dump-source1", ns, eventAny, eventSource1, defaultBrokerName, defaultSource1Dumper), - test.Trigger("default-dump-type1-source1", ns, eventType1, eventSource1, defaultBrokerName, defaultType1Source1Dumper), - } - for defaultTrigger := range defaultTriggers { + for name, selector := range eventLoggers { + strs := strings.Split(name, "-") + _, brokerName, eventType, eventSource := strs[0], strs[1], strs[2], strs[3] + trigger := test.Trigger(name, ns, eventType, eventSource, name) err := WithTriggerReady(clients, defaultTrigger, logger, cleaner) if err != nil { - t.Fatalf("Error waiting for default trigger to become ready: %v", err) + t.Fatalf("Error waiting for trigger to become ready: %v", err) } } logger.Info("Triggers ready") - logger.Infof("Creating event sender pods") - - events = []TypeAndSource{ + typesAndSources = []TypeAndSource{ {eventType1, eventSource1}, {eventType1, eventSource2}, {eventType2, eventSource1}, {eventType2, eventSource2}, } - for event := range events { + logger.Infof("Creating event sender pods") + // TODO should create a single pod that can send multiple events + for event := range typesAndSources { body := fmt.Sprintf("Testing Broker-Trigger %s", uuid.NewUUID()) cloudEvent := test.CloudEvent{ Source: event.Source, @@ -122,7 +129,7 @@ func TestBrokerTrigger(t *testing.T) { Data: fmt.Sprintf(`{"msg":%q}`, body), Encoding: encoding, } - url := fmt.Sprintf("http://%s", channel.Status.Address.Hostname) + url := fmt.Sprintf("http://%s", defaultBroker.Status.Address.Hostname) senderPod := test.EventSenderPod(event.Source, ns, url, cloudEvent) logger.Infof("Sender pod: %#v", senderPod) if err := CreatePod(clients, senderPod, logger, cleaner); err != nil { @@ -130,9 +137,9 @@ func TestBrokerTrigger(t *testing.T) { } } - // Verify that each arrived to its own place. - //if err := WaitForLogContent(clients, logger, routeName, subscriberPod.Spec.Containers[0].Name, ns, body); err != nil { - // t.Fatalf("String %q not found in logs of subscriber pod %q: %v", body, routeName, err) - //} + // Verify that each event arrived to its own place. + if err := WaitForLogContent(clients, logger, routeName, subscriberPod.Spec.Containers[0].Name, ns, body); err != nil { + t.Fatalf("String %q not found in logs of subscriber pod %q: %v", body, routeName, err) + } } diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go index eedceab5ac1..756c161ccdd 100644 --- a/test/e2e/e2e.go +++ b/test/e2e/e2e.go @@ -172,6 +172,29 @@ func CreateBroker(clients *test.Clients, broker *v1alpha1.Broker, logger *loggin return nil } +// WithBrokerReady creates a Broker and waits until it is Ready. +func WithBrokerReady(clients *test.Clients, broker *v1alpha1.Broker, logger *logging.BaseLogger, cleaner *test.Cleaner) error { + if err := CreateBroker(clients, broker, logger, cleaner); err != nil { + return err + } + return WaitForBrokerReady(clients, broker) +} + +// WaitForBrokerReady waits until the broker is Ready. +func WaitForBrokerReady(clients *test.Clients, broker *v1alpha1.Broker) error { + brokers := clients.Eventing.EventingV1alpha1().Brokers(pkgTest.Flags.Namespace) + if err := test.WaitForBrokerState(brokers, broker.Name, test.IsBrokerReady, "BrokerIsReady"); err != nil { + return err + } + // Update the given object so they'll reflect the ready state + updatedBroker, err := brokers.Get(broker.Name, metav1.GetOptions{}) + if err != nil { + return err + } + updatedBroker.DeepCopyInto(broker) + return nil +} + // CreateTrigger will create a Trigger func CreateTrigger(clients *test.Clients, trigger *v1alpha1.Trigger, logger *logging.BaseLogger, cleaner *test.Cleaner) error { triggers := clients.Eventing.EventingV1alpha1().Triggers(pkgTest.Flags.Namespace) @@ -273,6 +296,26 @@ func CreateService(clients *test.Clients, svc *corev1.Service, logger *logging.B return nil } +// WithServiceReady creates a Service and waits until it is Ready. +func WithServiceReady(clients *test.Clients, svc *corev1.Service, logger *logging.BaseLogger, cleaner *test.Cleaner) error { + if err := CreateService(clients, svc, logger, cleaner); err != nil { + return err + } + + svcs := clients.Kube.Kube.CoreV1().Services(pkgTest.Flags.Namespace) + if err := test.WaitForServiceState(svcs, svc.Name, test.IsServiceReady, "ServiceIsReady"); err != nil { + return err + } + // Update the given object so they'll reflect the ready state + updatedSvc, err := svcs.Get(svc.Name, metav1.GetOptions{}) + if err != nil { + return err + } + updatedSvc.DeepCopyInto(svc) + + return nil +} + // CreatePod will create a Pod func CreatePod(clients *test.Clients, pod *corev1.Pod, logger *logging.BaseLogger, cleaner *test.Cleaner) error { pods := clients.Kube.Kube.CoreV1().Pods(pod.GetNamespace()) @@ -328,7 +371,7 @@ func WaitForAllPodsRunning(clients *test.Clients, logger *logging.BaseLogger, na return nil } -func NamespaceExists(t *testing.T, clients *test.Clients, logger *logging.BaseLogger) (string, func()) { +func NamespaceExists(t *testing.T, clients *test.Clients, logger *logging.BaseLogger, annotate bool) (string, func()) { shutdown := func() {} ns := pkgTest.Flags.Namespace logger.Infof("Namespace: %s", ns) @@ -337,8 +380,12 @@ func NamespaceExists(t *testing.T, clients *test.Clients, logger *logging.BaseLo if err != nil && errors.IsNotFound(err) { nsSpec = &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: ns}} + if annotate { + nsSpec.Annotations = map[string]string{"eventing.knative.dev/inject": "true"} + } logger.Infof("Creating Namespace: %s", ns) nsSpec, err = clients.Kube.Kube.CoreV1().Namespaces().Create(nsSpec) + if err != nil { t.Fatalf("Failed to create Namespace: %s; %v", ns, err) } else { diff --git a/test/e2e/single_event_test.go b/test/e2e/single_event_test.go index a1d9455c71d..8ccf04037fe 100644 --- a/test/e2e/single_event_test.go +++ b/test/e2e/single_event_test.go @@ -51,7 +51,7 @@ func SingleEvent(t *testing.T, encoding string) { // verify namespace - ns, cleanupNS := NamespaceExists(t, clients, logger) + ns, cleanupNS := NamespaceExists(t, clients, logger, false) defer cleanupNS() // create logger pod From 29be515f1974b26ca6d64f51fe505214889905f3 Mon Sep 17 00:00:00 2001 From: nachocano Date: Wed, 20 Feb 2019 11:04:28 -0800 Subject: [PATCH 070/128] Waiting for potentially multiple contents. Removing check for corev1.Service ready. Removing grant's great design. Just making it simpler for now. --- test/e2e/broker_trigger_test.go | 146 +++++++++++++++++++++++--------- test/e2e/builders.go | 116 ------------------------- test/e2e/e2e.go | 40 ++++----- test/e2e/fixtures.go | 109 ------------------------ 4 files changed, 120 insertions(+), 291 deletions(-) delete mode 100644 test/e2e/builders.go delete mode 100644 test/e2e/fixtures.go diff --git a/test/e2e/broker_trigger_test.go b/test/e2e/broker_trigger_test.go index c5c8433c5f0..bb7821d9187 100644 --- a/test/e2e/broker_trigger_test.go +++ b/test/e2e/broker_trigger_test.go @@ -19,13 +19,13 @@ package e2e import ( "fmt" - "strings" "testing" "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" "github.com/knative/eventing/test" "github.com/knative/pkg/test/logging" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/uuid" ) @@ -40,12 +40,29 @@ const ( eventSource2 = "source2" ) -type TypeAndSource struct { - Type, Source string +// Helper function to create names for different objects (e.g., triggers, services, etc.) +func name(obj, brokerName, eventType, eventSource string) string { + return fmt.Sprintf("%s-%s-%s-%s", obj, brokerName, eventType, eventSource) } -func name(brokerName, eventType, eventSource string) string { - return fmt.Sprintf("%s-%s-%s", brokerName, eventType, eventSource) +// Helper object to easily create subscriber pods, services and triggers. +// We also use this to verify the expected events that should be received +// by the particular subscriber pods. +type DumperInfo struct { + Namespace string + Broker string + EventType string + EventSource string + Selector map[string]string + ExpectedBodies []string +} + +// Helper object to easily create event sender pods. +type SenderInfo struct { + Namespace string + Url string + EventType string + EventSource string } func TestBrokerTrigger(t *testing.T) { @@ -54,33 +71,49 @@ func TestBrokerTrigger(t *testing.T) { clients, cleaner := Setup(t, logger) defer TearDown(clients, cleaner, logger) - // verify namespace and annotate to create default broker + // Verify namespace and annotate to create default broker. ns, cleanupNS := NamespaceExists(t, clients, logger, true) defer cleanupNS() + // Wait for default broker ready. defaultBroker := test.Broker(defaultBrokerName, ns) err := WaitForBrokerReady(clients, defaultBroker) if err != nil { t.Fatalf("Error waiting for default broker to become ready: %v", err) } - // name -> selector - eventLoggers := []map[string]map[string]string{ - {name(defaultBroker, any, any): {selectorKey: string(uuid.NewUUID())}}, - {name(defaultBroker, eventType1, any): {selectorKey: string(uuid.NewUUID())}}, - {name(defaultBroker, any, eventSource1): {selectorKey: string(uuid.NewUUID())}}, - {name(defaultBroker, eventType1, eventSource1): {selectorKey: string(uuid.NewUUID())}}, + defaultBrokerUrl := fmt.Sprintf("http://%s", defaultBroker.Status.Address.Hostname) + + // Create sender helpers. + senders := []SenderInfo{ + {ns, defaultBrokerUrl, eventType1, eventSource1}, + {ns, defaultBrokerUrl, eventType1, eventSource2}, + {ns, defaultBrokerUrl, eventType2, eventSource1}, + {ns, defaultBrokerUrl, eventType2, eventSource2}, + } + + // Create DumperInfo helpers. + dumpers := []DumperInfo{ + {ns, defaultBrokerName, any, any, {selectorKey: string(uuid.NewUUID())}, make([]string, 0)}, + {ns, defaultBrokerName, eventType1, any, {selectorKey: string(uuid.NewUUID())}, make([]string, 0)}, + {ns, defaultBrokerName, any, eventSource1, {selectorKey: string(uuid.NewUUID())}, make([]string, 0)}, + {ns, defaultBrokerName, eventType1, eventSource1, {selectorKey: string(uuid.NewUUID())}, make([]string, 0)}, } logger.Info("Creating Subscriber pods") - for name, selector := range eventLoggers { - subscriberPod := test.EventLoggerPod(name, ns, selector) + // Save the references in this map for later use. + subscriberPods := make(map[string]*corev1.Pod, 0) + for _, dumper := range dumpers { + subscriberPodName := name("dumper", dumper.Broker, dumper.EventType, dumper.EventSource) + subscriberPod := test.EventLoggerPod(subscriberPodName, dumper.Namespace, dumper.Selector) if err := CreatePod(clients, subscriberPod, logger, cleaner); err != nil { t.Fatalf("Failed to create subscriber pod: %v", err) } + subscriberPods[subscriberPodName] = subscriberPod } + // Wait for all of them to be running. if err := WaitForAllPodsRunning(clients, logger, ns); err != nil { t.Fatalf("Error waiting for event logger pod to become running: %v", err) } @@ -89,22 +122,22 @@ func TestBrokerTrigger(t *testing.T) { logger.Info("Creating Subscriber services") - for name, selector := range eventLoggers { - subscriberSvc := test.Service(name, ns, selector) - if err := WithServiceReady(clients, subscriberSvc, logger, cleaner); err != nil { - t.Fatalf("Error waiting for subscriber service to become ready: %v", err) - } + for _, dumper := range dumpers { + subscriberSvcName := name("svc", dumper.Broker, dumper.EventType, dumper.EventSource) + subscriberSvc := test.Service(subscriberSvcName, dumper.Namespace, dumper.Selector) } - logger.Info("Subscriber services ready") + logger.Info("Subscriber services created") logger.Info("Creating Triggers") - for name, selector := range eventLoggers { - strs := strings.Split(name, "-") - _, brokerName, eventType, eventSource := strs[0], strs[1], strs[2], strs[3] - trigger := test.Trigger(name, ns, eventType, eventSource, name) - err := WithTriggerReady(clients, defaultTrigger, logger, cleaner) + for _, dumper := range dumpers { + triggerName := name("trigger", dumper.Broker, dumper.EventType, dumper.EventSource) + // subscriberName should be the same as the subscriberSvc from before. + subscriberName := name("svc", dumper.Broker, dumper.EventType, dumper.EventSource) + trigger := test.Trigger(triggerName, dumper.Namespace, dumper.EventType, dumper.EventSource, dumper.Broker, subscriberName) + // Wait for the triggers to be ready + err := WithTriggerReady(clients, trigger, logger, cleaner) if err != nil { t.Fatalf("Error waiting for trigger to become ready: %v", err) } @@ -112,34 +145,63 @@ func TestBrokerTrigger(t *testing.T) { logger.Info("Triggers ready") - typesAndSources = []TypeAndSource{ - {eventType1, eventSource1}, - {eventType1, eventSource2}, - {eventType2, eventSource1}, - {eventType2, eventSource2}, - } + logger.Info("Creating event sender pods") - logger.Infof("Creating event sender pods") - // TODO should create a single pod that can send multiple events - for event := range typesAndSources { + for _, sender := range senders { + // Create cloud event. body := fmt.Sprintf("Testing Broker-Trigger %s", uuid.NewUUID()) cloudEvent := test.CloudEvent{ - Source: event.Source, - Type: event.Type, + Source: sender.EventSource, + Type: sender.EventType, Data: fmt.Sprintf(`{"msg":%q}`, body), - Encoding: encoding, + Encoding: test.CloudEventEncodingStructured, } - url := fmt.Sprintf("http://%s", defaultBroker.Status.Address.Hostname) - senderPod := test.EventSenderPod(event.Source, ns, url, cloudEvent) + // Create sender pod. + senderPodName := fmt.Sprintf("sender-%s-%s", sender.EventType, sender.EventSource) + senderPod := test.EventSenderPod(senderPodName, sender.Namespace, url, cloudEvent) logger.Infof("Sender pod: %#v", senderPod) if err := CreatePod(clients, senderPod, logger, cleaner); err != nil { t.Fatalf("Failed to create event sender pod: %v", err) } + + // Check on every dumper whether we should expect this event or not, and add its body if so. + for _, dumper := range dumpers { + if shouldExpectEvent(dumper, sender) { + dumper.ExpectedBodies = append(dumper.ExpectedBodies, body) + } + } } - // Verify that each event arrived to its own place. - if err := WaitForLogContent(clients, logger, routeName, subscriberPod.Spec.Containers[0].Name, ns, body); err != nil { - t.Fatalf("String %q not found in logs of subscriber pod %q: %v", body, routeName, err) + logger.Info("Created event sender pods") + + // Wait for all of them to be running. + if err := WaitForAllPodsRunning(clients, logger, ns); err != nil { + t.Fatalf("Error waiting for event sender pod to become running: %v", err) } + logger.Info("Verifying events arrived to appropriate dumpers") + + for _, dumper := range dumpers { + subscriberPodName := name("dumper", dumper.Broker, dumper.EventType, dumper.EventSource) + subscriberPod := subscriberPods[subscriberPodName] + if err := WaitForLogContents(clients, logger, routeName, subscriberPod.Spec.Containers[0].Name, dumper.Namespace, dumper.ExpectedBodies); err != nil { + t.Fatalf("String(s) not found in logs of subscriber pod %q: %v", subscriberPodName, err) + } + } + + logger.Info("Successfully completed!") + +} + +func shouldExpectEvent(dumper *DumperInfo, sender *SenderInfo) bool { + if dumper.Namespace != sender.Namespace { + return false + } + if dumper.EventType != any && dumper.EventType != sender.EventType { + return false + } + if dumper.EventSource != any && dumper != sender.EventSource { + return false + } + return true } diff --git a/test/e2e/builders.go b/test/e2e/builders.go deleted file mode 100644 index e89bb26f0a2..00000000000 --- a/test/e2e/builders.go +++ /dev/null @@ -1,116 +0,0 @@ -/* -Copyright 2019 The Knative Authors -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package e2e - -import ( - eventingv1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// Broker builder. -type BrokerBuilder struct { - *eventingv1alpha1.Broker -} - -func NewBrokerBuilder(name, namespace string) *BrokerBuilder { - broker := &eventingv1alpha1.Broker{ - TypeMeta: metav1.TypeMeta{ - APIVersion: eventingv1alpha1.SchemeGroupVersion.String(), - Kind: "Broker", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - Spec: eventingv1alpha1.BrokerSpec{}, - } - - return &BrokerBuilder{ - Broker: broker, - } -} - -func (b *BrokerBuilder) Build() *eventingv1alpha1.Broker { - return b.Broker.DeepCopy() -} - -// Trigger builder. -type TriggerBuilder struct { - *eventingv1alpha1.Trigger -} - -func NewTriggerBuilder(name, namespace string) *TriggerBuilder { - trigger := &eventingv1alpha1.Trigger{ - TypeMeta: metav1.TypeMeta{ - APIVersion: eventingv1alpha1.SchemeGroupVersion.String(), - Kind: "Trigger", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - Spec: eventingv1alpha1.TriggerSpec{ - // Bind to 'default' Broker by default - Broker: "default", - Filter: &eventingv1alpha1.TriggerFilter{ - // Create a Any filter by default. - SourceAndType: &eventingv1alpha1.TriggerFilterSourceAndType{ - Source: eventingv1alpha1.TriggerAnyFilter, - Type: eventingv1alpha1.TriggerAnyFilter, - }, - }, - }, - } - - return &TriggerBuilder{ - Trigger: trigger, - } -} - -func (t *TriggerBuilder) Build() *eventingv1alpha1.Trigger { - return t.Trigger.DeepCopy() -} - -func (t *TriggerBuilder) Type(eventType string) *TriggerBuilder { - t.Trigger.Spec.Filter.SourceAndType.Type = eventType - return t -} - -func (t *TriggerBuilder) Source(eventSource string) *TriggerBuilder { - t.Trigger.Spec.Filter.SourceAndType.Source = eventSource - return t -} - -func (t *TriggerBuilder) Broker(brokerName string) *TriggerBuilder { - t.Trigger.Spec.Broker = brokerName - return t -} - -func (t *TriggerBuilder) Subscriber(ref *corev1.ObjectReference) *TriggerBuilder { - t.Trigger.Spec.Subscriber.Ref = ref - return t -} - -func (t *TriggerBuilder) SubscriberSvc(svcName string) *TriggerBuilder { - t.Trigger.Spec.Subscriber.Ref = &corev1.ObjectReference{ - APIVersion: corev1.SchemeGroupVersion.String(), - Kind: "Service", - Name: svcName, - Namespace: t.Trigger.GetNamespace(), - } - return t -} diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go index 756c161ccdd..7a4976621b9 100644 --- a/test/e2e/e2e.go +++ b/test/e2e/e2e.go @@ -296,26 +296,6 @@ func CreateService(clients *test.Clients, svc *corev1.Service, logger *logging.B return nil } -// WithServiceReady creates a Service and waits until it is Ready. -func WithServiceReady(clients *test.Clients, svc *corev1.Service, logger *logging.BaseLogger, cleaner *test.Cleaner) error { - if err := CreateService(clients, svc, logger, cleaner); err != nil { - return err - } - - svcs := clients.Kube.Kube.CoreV1().Services(pkgTest.Flags.Namespace) - if err := test.WaitForServiceState(svcs, svc.Name, test.IsServiceReady, "ServiceIsReady"); err != nil { - return err - } - // Update the given object so they'll reflect the ready state - updatedSvc, err := svcs.Get(svc.Name, metav1.GetOptions{}) - if err != nil { - return err - } - updatedSvc.DeepCopyInto(svc) - - return nil -} - // CreatePod will create a Pod func CreatePod(clients *test.Clients, pod *corev1.Pod, logger *logging.BaseLogger, cleaner *test.Cleaner) error { pods := clients.Kube.Kube.CoreV1().Pods(pod.GetNamespace()) @@ -351,18 +331,30 @@ func PodLogs(clients *test.Clients, podName string, containerName string, namesp return nil, fmt.Errorf("Could not find logs for %s/%s", podName, containerName) } -// WaitForLogContent waits until logs for given Pod/Container include the given content. -// If the content is not present within timeout it returns error. -func WaitForLogContent(clients *test.Clients, logger *logging.BaseLogger, podName string, containerName string, namespace string, content string) error { +// WaitForLogContents waits until logs for given Pod/Container include the given contents. +// If the contents are not present within timeout it returns error. +func WaitForLogContents(clients *test.Clients, logger *logging.BaseLogger, podName string, containerName string, namespace string, contents []string) error { return wait.PollImmediate(interval, timeout, func() (bool, error) { logs, err := PodLogs(clients, podName, containerName, namespace, logger) if err != nil { return true, err } - return strings.Contains(string(logs), content), nil + for _, content := range contents { + if !strings.Contains(string(logs), content) { + logger.Infof("Could not find content %s for %s/%s", content, podName, containerName) + return false, nil + } + } + return true, nil }) } +// WaitForLogContent waits until logs for given Pod/Container include the given content. +// If the content is not present within timeout it returns error. +func WaitForLogContent(clients *test.Clients, logger *logging.BaseLogger, podName string, containerName string, namespace string, content string) error { + return WaitForLogContents(clients, logger, podName, containerName, namespace, []string{content}) +} + // WaitForAllPodsRunning will wait until all pods in the given namespace are running func WaitForAllPodsRunning(clients *test.Clients, logger *logging.BaseLogger, namespace string) error { if err := pkgTest.WaitForPodListState(clients.Kube, test.PodsRunning, "PodsAreRunning", namespace); err != nil { diff --git a/test/e2e/fixtures.go b/test/e2e/fixtures.go deleted file mode 100644 index 3bfa969bd7d..00000000000 --- a/test/e2e/fixtures.go +++ /dev/null @@ -1,109 +0,0 @@ -/* -Copyright 2019 The Knative Authors -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -package e2e - -import ( - "context" - - "github.com/knative/eventing/pkg/reconciler/testing" - "github.com/knative/pkg/apis/duck" - duckv1alpha1 "github.com/knative/pkg/apis/duck/v1alpha1" - "k8s.io/apimachinery/pkg/api/meta" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - - "sigs.k8s.io/controller-runtime/pkg/client" -) - -type Fixture interface { - // Create attempts to instantiate the fixture. It may be called multiple times if earlier attempts return an error. - Create(context.Context, client.Client) error - - // Verify determines if the fixture was successfully instantiated once. It may be called multiple times if earlier - // attempts return false or an error. - Verify(context.Context, client.Client) (bool, error) - - // Debug surfaces context about a failure during Create or Verify. - Debug(context.Context, client.Client) - - // Teardown removes any state instantiated by Create or Verify. - Teardown(context.Context, client.Client) error -} - -type KnativeFixture struct { - Object testing.Buildable -} - -type CreateFunc func(context.Context, client.Client) error -type VerifyFunc func(context.Context, client.Client) (bool, error) -type DebugFunc func(context.Context, client.Client) -type TeardownFunc func(context.Context, client.Client) error - -type FixtureFuncs struct { - Create CreateFunc - Verify VerifyFunc - Debug DebugFunc - Teardown TeardownFunc -} - -//Fixtures: -// - Broker, Trigger Verify checks status ready -// - Sendevent Verify checks that pod completed successfully -// - Logevent Verify checks that the expected message was logged - -func (f *KnativeFixture) Create(ctx context.Context, cl client.Client) error { - obj := f.Object.Build() - err := cl.Create(ctx, obj) - if err != nil { - return err - } - return nil -} - -var standardCondSet = duckv1alpha1.NewLivingConditionSet() - -// TODO the runner should do this with backoff and retry. -// This method only verifies Knative objects -func (f *KnativeFixture) Verify(ctx context.Context, cl client.Client) (bool, error) { - obj := f.Object.Build() - u := &unstructured.Unstructured{} - acc, err := meta.Accessor(obj) - if err != nil { - return false, err - } - if err := cl.Get(ctx, client.ObjectKey{Name: acc.GetName(), Namespace: acc.GetNamespace()}, u); err != nil { - return false, err - } - - kr := &duckv1alpha1.KResource{} - if err := duck.FromUnstructured(u, kr); err != nil { - return false, err - } - - if !standardCondSet.Manage(kr.Status).IsHappy() { - return false, nil - } - - return true, nil -} - -// TODO If Create or Verify return an error, run this method, then Cleanup -func (f *KnativeFixture) Debug(ctx context.Context, cl client.Client) { -} - -// TODO if Teardown returns an error, log it and give up (maybe retry for a while?) -func (f *KnativeFixture) Teardown(ctx context.Context, cl client.Client) error { - obj := f.Object.Build() - return cl.Delete(ctx, obj) -} From 1edbf0a0a9485d7402c1e9d4b9fc83f3f04d53dd Mon Sep 17 00:00:00 2001 From: nachocano Date: Wed, 20 Feb 2019 11:18:30 -0800 Subject: [PATCH 071/128] Compiling --- test/e2e/broker_trigger_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/broker_trigger_test.go b/test/e2e/broker_trigger_test.go index bb7821d9187..9c7d523c390 100644 --- a/test/e2e/broker_trigger_test.go +++ b/test/e2e/broker_trigger_test.go @@ -200,7 +200,7 @@ func shouldExpectEvent(dumper *DumperInfo, sender *SenderInfo) bool { if dumper.EventType != any && dumper.EventType != sender.EventType { return false } - if dumper.EventSource != any && dumper != sender.EventSource { + if dumper.EventSource != any && dumper.EventType != sender.EventSource { return false } return true From 9c415e67674f210431013f762fef915aeb88cee5 Mon Sep 17 00:00:00 2001 From: nachocano Date: Wed, 20 Feb 2019 11:23:43 -0800 Subject: [PATCH 072/128] Fixing compilation --- test/e2e/broker_trigger_test.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/test/e2e/broker_trigger_test.go b/test/e2e/broker_trigger_test.go index 9c7d523c390..34e1c67ae9f 100644 --- a/test/e2e/broker_trigger_test.go +++ b/test/e2e/broker_trigger_test.go @@ -86,18 +86,18 @@ func TestBrokerTrigger(t *testing.T) { // Create sender helpers. senders := []SenderInfo{ - {ns, defaultBrokerUrl, eventType1, eventSource1}, - {ns, defaultBrokerUrl, eventType1, eventSource2}, - {ns, defaultBrokerUrl, eventType2, eventSource1}, - {ns, defaultBrokerUrl, eventType2, eventSource2}, + SenderInfo{ns, defaultBrokerUrl, eventType1, eventSource1}, + SenderInfo{ns, defaultBrokerUrl, eventType1, eventSource2}, + SenderInfo{ns, defaultBrokerUrl, eventType2, eventSource1}, + SenderInfo{ns, defaultBrokerUrl, eventType2, eventSource2}, } // Create DumperInfo helpers. dumpers := []DumperInfo{ - {ns, defaultBrokerName, any, any, {selectorKey: string(uuid.NewUUID())}, make([]string, 0)}, - {ns, defaultBrokerName, eventType1, any, {selectorKey: string(uuid.NewUUID())}, make([]string, 0)}, - {ns, defaultBrokerName, any, eventSource1, {selectorKey: string(uuid.NewUUID())}, make([]string, 0)}, - {ns, defaultBrokerName, eventType1, eventSource1, {selectorKey: string(uuid.NewUUID())}, make([]string, 0)}, + DumperInfo{ns, defaultBrokerName, any, any, {selectorKey: string(uuid.NewUUID())}, make([]string, 0)}, + DumperInfo{ns, defaultBrokerName, eventType1, any, {selectorKey: string(uuid.NewUUID())}, make([]string, 0)}, + DumperInfo{ns, defaultBrokerName, any, eventSource1, {selectorKey: string(uuid.NewUUID())}, make([]string, 0)}, + DumperInfo{ns, defaultBrokerName, eventType1, eventSource1, {selectorKey: string(uuid.NewUUID())}, make([]string, 0)}, } logger.Info("Creating Subscriber pods") @@ -158,7 +158,7 @@ func TestBrokerTrigger(t *testing.T) { } // Create sender pod. senderPodName := fmt.Sprintf("sender-%s-%s", sender.EventType, sender.EventSource) - senderPod := test.EventSenderPod(senderPodName, sender.Namespace, url, cloudEvent) + senderPod := test.EventSenderPod(senderPodName, sender.Namespace, sender.Url, cloudEvent) logger.Infof("Sender pod: %#v", senderPod) if err := CreatePod(clients, senderPod, logger, cleaner); err != nil { t.Fatalf("Failed to create event sender pod: %v", err) From ac9b485c525ba17ffbbae78bb0f4e29e24e40d2a Mon Sep 17 00:00:00 2001 From: nachocano Date: Wed, 20 Feb 2019 12:50:00 -0800 Subject: [PATCH 073/128] Fixing compilation errors. Adding AnnotateNamespace function. --- test/e2e/broker_trigger_test.go | 62 +++++++++++++-------------------- test/e2e/e2e.go | 17 ++++++--- test/e2e/single_event_test.go | 2 +- test/helpers.go | 36 +++++++++++++++++++ 4 files changed, 74 insertions(+), 43 deletions(-) create mode 100644 test/helpers.go diff --git a/test/e2e/broker_trigger_test.go b/test/e2e/broker_trigger_test.go index 34e1c67ae9f..9ee7c6b2084 100644 --- a/test/e2e/broker_trigger_test.go +++ b/test/e2e/broker_trigger_test.go @@ -45,39 +45,25 @@ func name(obj, brokerName, eventType, eventSource string) string { return fmt.Sprintf("%s-%s-%s-%s", obj, brokerName, eventType, eventSource) } -// Helper object to easily create subscriber pods, services and triggers. -// We also use this to verify the expected events that should be received -// by the particular subscriber pods. -type DumperInfo struct { - Namespace string - Broker string - EventType string - EventSource string - Selector map[string]string - ExpectedBodies []string -} - -// Helper object to easily create event sender pods. -type SenderInfo struct { - Namespace string - Url string - EventType string - EventSource string -} - func TestBrokerTrigger(t *testing.T) { logger := logging.GetContextLogger("TestBrokerTrigger") clients, cleaner := Setup(t, logger) defer TearDown(clients, cleaner, logger) - // Verify namespace and annotate to create default broker. - ns, cleanupNS := NamespaceExists(t, clients, logger, true) + // Verify namespace exists. + ns, cleanupNS := NamespaceExists(t, clients, logger) defer cleanupNS() + // Annotate namespace so that it creates the default broker. + err := AnnotateNamespace(clients, logger, map[string]string{"eventing.knative.dev/inject": "true"}) + if err != nil { + t.Fatalf("Error annotating namespace: %v", err) + } + // Wait for default broker ready. defaultBroker := test.Broker(defaultBrokerName, ns) - err := WaitForBrokerReady(clients, defaultBroker) + err = WaitForBrokerReady(clients, defaultBroker) if err != nil { t.Fatalf("Error waiting for default broker to become ready: %v", err) } @@ -85,19 +71,19 @@ func TestBrokerTrigger(t *testing.T) { defaultBrokerUrl := fmt.Sprintf("http://%s", defaultBroker.Status.Address.Hostname) // Create sender helpers. - senders := []SenderInfo{ - SenderInfo{ns, defaultBrokerUrl, eventType1, eventSource1}, - SenderInfo{ns, defaultBrokerUrl, eventType1, eventSource2}, - SenderInfo{ns, defaultBrokerUrl, eventType2, eventSource1}, - SenderInfo{ns, defaultBrokerUrl, eventType2, eventSource2}, + senders := []test.SenderInfo{ + {ns, defaultBrokerUrl, eventType1, eventSource1}, + {ns, defaultBrokerUrl, eventType1, eventSource2}, + {ns, defaultBrokerUrl, eventType2, eventSource1}, + {ns, defaultBrokerUrl, eventType2, eventSource2}, } - // Create DumperInfo helpers. - dumpers := []DumperInfo{ - DumperInfo{ns, defaultBrokerName, any, any, {selectorKey: string(uuid.NewUUID())}, make([]string, 0)}, - DumperInfo{ns, defaultBrokerName, eventType1, any, {selectorKey: string(uuid.NewUUID())}, make([]string, 0)}, - DumperInfo{ns, defaultBrokerName, any, eventSource1, {selectorKey: string(uuid.NewUUID())}, make([]string, 0)}, - DumperInfo{ns, defaultBrokerName, eventType1, eventSource1, {selectorKey: string(uuid.NewUUID())}, make([]string, 0)}, + // Create dumper helpers. + dumpers := []test.DumperInfo{ + {ns, defaultBrokerName, any, any, map[string]string{selectorKey: string(uuid.NewUUID())}, make([]string, 0)}, + {ns, defaultBrokerName, eventType1, any, map[string]string{selectorKey: string(uuid.NewUUID())}, make([]string, 0)}, + {ns, defaultBrokerName, any, eventSource1, map[string]string{selectorKey: string(uuid.NewUUID())}, make([]string, 0)}, + {ns, defaultBrokerName, eventType1, eventSource1, map[string]string{selectorKey: string(uuid.NewUUID())}, make([]string, 0)}, } logger.Info("Creating Subscriber pods") @@ -124,7 +110,7 @@ func TestBrokerTrigger(t *testing.T) { for _, dumper := range dumpers { subscriberSvcName := name("svc", dumper.Broker, dumper.EventType, dumper.EventSource) - subscriberSvc := test.Service(subscriberSvcName, dumper.Namespace, dumper.Selector) + test.Service(subscriberSvcName, dumper.Namespace, dumper.Selector) } logger.Info("Subscriber services created") @@ -166,7 +152,7 @@ func TestBrokerTrigger(t *testing.T) { // Check on every dumper whether we should expect this event or not, and add its body if so. for _, dumper := range dumpers { - if shouldExpectEvent(dumper, sender) { + if shouldExpectEvent(&dumper, &sender) { dumper.ExpectedBodies = append(dumper.ExpectedBodies, body) } } @@ -184,7 +170,7 @@ func TestBrokerTrigger(t *testing.T) { for _, dumper := range dumpers { subscriberPodName := name("dumper", dumper.Broker, dumper.EventType, dumper.EventSource) subscriberPod := subscriberPods[subscriberPodName] - if err := WaitForLogContents(clients, logger, routeName, subscriberPod.Spec.Containers[0].Name, dumper.Namespace, dumper.ExpectedBodies); err != nil { + if err := WaitForLogContents(clients, logger, subscriberPodName, subscriberPod.Spec.Containers[0].Name, dumper.Namespace, dumper.ExpectedBodies); err != nil { t.Fatalf("String(s) not found in logs of subscriber pod %q: %v", subscriberPodName, err) } } @@ -193,7 +179,7 @@ func TestBrokerTrigger(t *testing.T) { } -func shouldExpectEvent(dumper *DumperInfo, sender *SenderInfo) bool { +func shouldExpectEvent(dumper *test.DumperInfo, sender *test.SenderInfo) bool { if dumper.Namespace != sender.Namespace { return false } diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go index 7a4976621b9..3c45ded21ef 100644 --- a/test/e2e/e2e.go +++ b/test/e2e/e2e.go @@ -363,7 +363,19 @@ func WaitForAllPodsRunning(clients *test.Clients, logger *logging.BaseLogger, na return nil } -func NamespaceExists(t *testing.T, clients *test.Clients, logger *logging.BaseLogger, annotate bool) (string, func()) { +// AnnotateNamespace annotates the test namespace with the annotations map +func AnnotateNamespace(clients *test.Clients, logger *logging.BaseLogger, annotations map[string]string) error { + ns := pkgTest.Flags.Namespace + nsSpec, err := clients.Kube.Kube.CoreV1().Namespaces().Get(ns, metav1.GetOptions{}) + if err != nil && errors.IsNotFound(err) { + return err + } + nsSpec.Annotations = annotations + _, err = clients.Kube.Kube.CoreV1().Namespaces().Update(nsSpec) + return err +} + +func NamespaceExists(t *testing.T, clients *test.Clients, logger *logging.BaseLogger) (string, func()) { shutdown := func() {} ns := pkgTest.Flags.Namespace logger.Infof("Namespace: %s", ns) @@ -372,9 +384,6 @@ func NamespaceExists(t *testing.T, clients *test.Clients, logger *logging.BaseLo if err != nil && errors.IsNotFound(err) { nsSpec = &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: ns}} - if annotate { - nsSpec.Annotations = map[string]string{"eventing.knative.dev/inject": "true"} - } logger.Infof("Creating Namespace: %s", ns) nsSpec, err = clients.Kube.Kube.CoreV1().Namespaces().Create(nsSpec) diff --git a/test/e2e/single_event_test.go b/test/e2e/single_event_test.go index 8ccf04037fe..a1d9455c71d 100644 --- a/test/e2e/single_event_test.go +++ b/test/e2e/single_event_test.go @@ -51,7 +51,7 @@ func SingleEvent(t *testing.T, encoding string) { // verify namespace - ns, cleanupNS := NamespaceExists(t, clients, logger, false) + ns, cleanupNS := NamespaceExists(t, clients, logger) defer cleanupNS() // create logger pod diff --git a/test/helpers.go b/test/helpers.go new file mode 100644 index 00000000000..3dec1061c90 --- /dev/null +++ b/test/helpers.go @@ -0,0 +1,36 @@ +/* +Copyright 2019 The Knative Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package test + +// Helper struct to easily create subscriber pods, services and triggers. +// We also use this to verify the expected events that should be received +// by the particular subscriber pods. +type DumperInfo struct { + Namespace string + Broker string + EventType string + EventSource string + Selector map[string]string + ExpectedBodies []string +} + +// Helper struct to easily create event sender pods. +type SenderInfo struct { + Namespace string + Url string + EventType string + EventSource string +} From 2b3916808ff2e9bb23e8702ec8fa7e9c139f0468 Mon Sep 17 00:00:00 2001 From: nachocano Date: Wed, 20 Feb 2019 13:11:26 -0800 Subject: [PATCH 074/128] Adding ns --- test/e2e/e2e.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go index 3c45ded21ef..929a6e93944 100644 --- a/test/e2e/e2e.go +++ b/test/e2e/e2e.go @@ -163,12 +163,12 @@ func WithChannelAndSubscriptionReady(clients *test.Clients, channel *v1alpha1.Ch // CreateBroker will create a Broker func CreateBroker(clients *test.Clients, broker *v1alpha1.Broker, logger *logging.BaseLogger, cleaner *test.Cleaner) error { - brokers := clients.Eventing.EventingV1alpha1().Brokers(pkgTest.Flags.Namespace) + brokers := clients.Eventing.EventingV1alpha1().Brokers(broker.Namespace) res, err := brokers.Create(broker) if err != nil { return err } - cleaner.Add(v1alpha1.SchemeGroupVersion.Group, v1alpha1.SchemeGroupVersion.Version, "brokers", pkgTest.Flags.Namespace, res.ObjectMeta.Name) + cleaner.Add(v1alpha1.SchemeGroupVersion.Group, v1alpha1.SchemeGroupVersion.Version, "brokers", broker.Namespace, res.ObjectMeta.Name) return nil } @@ -182,7 +182,7 @@ func WithBrokerReady(clients *test.Clients, broker *v1alpha1.Broker, logger *log // WaitForBrokerReady waits until the broker is Ready. func WaitForBrokerReady(clients *test.Clients, broker *v1alpha1.Broker) error { - brokers := clients.Eventing.EventingV1alpha1().Brokers(pkgTest.Flags.Namespace) + brokers := clients.Eventing.EventingV1alpha1().Brokers(broker.Namespace) if err := test.WaitForBrokerState(brokers, broker.Name, test.IsBrokerReady, "BrokerIsReady"); err != nil { return err } @@ -197,12 +197,12 @@ func WaitForBrokerReady(clients *test.Clients, broker *v1alpha1.Broker) error { // CreateTrigger will create a Trigger func CreateTrigger(clients *test.Clients, trigger *v1alpha1.Trigger, logger *logging.BaseLogger, cleaner *test.Cleaner) error { - triggers := clients.Eventing.EventingV1alpha1().Triggers(pkgTest.Flags.Namespace) + triggers := clients.Eventing.EventingV1alpha1().Triggers(trigger.Namespace) res, err := triggers.Create(trigger) if err != nil { return err } - cleaner.Add(v1alpha1.SchemeGroupVersion.Group, v1alpha1.SchemeGroupVersion.Version, "triggers", pkgTest.Flags.Namespace, res.ObjectMeta.Name) + cleaner.Add(v1alpha1.SchemeGroupVersion.Group, v1alpha1.SchemeGroupVersion.Version, "triggers", trigger.Namespace, res.ObjectMeta.Name) return nil } @@ -212,7 +212,7 @@ func WithTriggerReady(clients *test.Clients, trigger *v1alpha1.Trigger, logger * return err } - triggers := clients.Eventing.EventingV1alpha1().Triggers(pkgTest.Flags.Namespace) + triggers := clients.Eventing.EventingV1alpha1().Triggers(trigger.Namespace) if err := test.WaitForTriggerState(triggers, trigger.Name, test.IsTriggerReady, "TriggerIsReady"); err != nil { return err } From ed9fcbe46dd1286240e406104c00ca06e8301f52 Mon Sep 17 00:00:00 2001 From: nachocano Date: Wed, 20 Feb 2019 13:19:45 -0800 Subject: [PATCH 075/128] Adding logs. Changing to lowercase any otherwise the pod name is invalid --- test/e2e/broker_trigger_test.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/e2e/broker_trigger_test.go b/test/e2e/broker_trigger_test.go index 9ee7c6b2084..71972138e35 100644 --- a/test/e2e/broker_trigger_test.go +++ b/test/e2e/broker_trigger_test.go @@ -21,8 +21,6 @@ import ( "fmt" "testing" - "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" - "github.com/knative/eventing/test" "github.com/knative/pkg/test/logging" corev1 "k8s.io/api/core/v1" @@ -33,7 +31,7 @@ const ( defaultBrokerName = "default" selectorKey = "end2end-test-broker-trigger" - any = v1alpha1.TriggerAnyFilter + any = "any" eventType1 = "type1" eventType2 = "type2" eventSource1 = "source1" @@ -55,12 +53,16 @@ func TestBrokerTrigger(t *testing.T) { ns, cleanupNS := NamespaceExists(t, clients, logger) defer cleanupNS() + logger.Infof("Annotating namespace %s", ns) + // Annotate namespace so that it creates the default broker. err := AnnotateNamespace(clients, logger, map[string]string{"eventing.knative.dev/inject": "true"}) if err != nil { t.Fatalf("Error annotating namespace: %v", err) } + logger.Infof("Namespace %s annotated", ns) + // Wait for default broker ready. defaultBroker := test.Broker(defaultBrokerName, ns) err = WaitForBrokerReady(clients, defaultBroker) From 484895291eb8258f66760d6f89d82014012be15a Mon Sep 17 00:00:00 2001 From: nachocano Date: Wed, 20 Feb 2019 13:38:58 -0800 Subject: [PATCH 076/128] Removing namespace when creating trigger subscriber spec. Adding wait time constant for default broker creation. --- test/crd.go | 1 - test/e2e/broker_trigger_test.go | 13 +++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/test/crd.go b/test/crd.go index f9f350e5939..198ce679d0b 100644 --- a/test/crd.go +++ b/test/crd.go @@ -199,7 +199,6 @@ func Trigger(name, namespace, eventType, eventSource, brokerName, svcName string APIVersion: corev1.SchemeGroupVersion.String(), Kind: "Service", Name: svcName, - Namespace: namespace, }, }, }, diff --git a/test/e2e/broker_trigger_test.go b/test/e2e/broker_trigger_test.go index 71972138e35..e65d94ad87f 100644 --- a/test/e2e/broker_trigger_test.go +++ b/test/e2e/broker_trigger_test.go @@ -20,6 +20,7 @@ package e2e import ( "fmt" "testing" + "time" "github.com/knative/eventing/test" "github.com/knative/pkg/test/logging" @@ -28,8 +29,9 @@ import ( ) const ( - defaultBrokerName = "default" - selectorKey = "end2end-test-broker-trigger" + defaultBrokerName = "default" + waitForDefaultBrokerCreation = 3 * time.Second + selectorKey = "end2end-test-broker-trigger" any = "any" eventType1 = "type1" @@ -63,6 +65,13 @@ func TestBrokerTrigger(t *testing.T) { logger.Infof("Namespace %s annotated", ns) + // As we are not creating the default broker, + // we wait for a few seconds for the broker to get created. + // Otherwise, if we try to wait for its Ready status and the namespace controller + // didn't create it yet, it fails. + logger.Info("Waiting for default broker creation") + time.Sleep(waitForDefaultBrokerCreation) + // Wait for default broker ready. defaultBroker := test.Broker(defaultBrokerName, ns) err = WaitForBrokerReady(clients, defaultBroker) From 311e174a64e4359b1ee7f6c74f33c2b2053bc363 Mon Sep 17 00:00:00 2001 From: nachocano Date: Wed, 20 Feb 2019 14:24:56 -0800 Subject: [PATCH 077/128] Checking if all triggers are ready --- test/crd.go | 6 +++--- test/crd_checks.go | 18 ++++++++++++++++++ test/e2e/broker_trigger_test.go | 28 +++++++++++++++++++--------- test/e2e/e2e.go | 9 +++++++++ test/states.go | 16 ++++++++++++++++ 5 files changed, 65 insertions(+), 12 deletions(-) diff --git a/test/crd.go b/test/crd.go index 198ce679d0b..5121cd925a1 100644 --- a/test/crd.go +++ b/test/crd.go @@ -180,14 +180,14 @@ func Broker(name string, namespace string) *v1alpha1.Broker { } // Trigger returns a Trigger. -func Trigger(name, namespace, eventType, eventSource, brokerName, svcName string) *v1alpha1.Trigger { +func Trigger(name, namespace, eventType, eventSource, broker, svcName string) *v1alpha1.Trigger { return &v1alpha1.Trigger{ ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: namespace, }, Spec: v1alpha1.TriggerSpec{ - Broker: brokerName, + Broker: broker, Filter: &v1alpha1.TriggerFilter{ SourceAndType: &v1alpha1.TriggerFilterSourceAndType{ Type: fmt.Sprintf("%q", eventType), @@ -196,7 +196,7 @@ func Trigger(name, namespace, eventType, eventSource, brokerName, svcName string }, Subscriber: &v1alpha1.SubscriberSpec{ Ref: &corev1.ObjectReference{ - APIVersion: corev1.SchemeGroupVersion.String(), + APIVersion: "v1", Kind: "Service", Name: svcName, }, diff --git a/test/crd_checks.go b/test/crd_checks.go index 6c66fc03ab9..93648af5707 100644 --- a/test/crd_checks.go +++ b/test/crd_checks.go @@ -132,6 +132,24 @@ func WaitForTriggerState(client eventingclient.TriggerInterface, name string, in }) } +// WaitForTriggersListState polls the status of the TriggerList +// from client every interval until inState returns `true` indicating it +// is done, returns an error or timeout. desc will be used to name the metric +// that is emitted to track how long it took to get into the state checked by inState. +func WaitForTriggersListState(clients eventingclient.TriggerInterface, inState func(t *eventingv1alpha1.TriggerList) (bool, error), desc string) error { + metricName := fmt.Sprintf("WaitForTriggerListState/%s", desc) + _, span := trace.StartSpan(context.Background(), metricName) + defer span.End() + + return wait.PollImmediate(interval, timeout, func() (bool, error) { + t, err := clients.List(metav1.ListOptions{}) + if err != nil { + return true, err + } + return inState(t) + }) +} + // WaitForServiceState polls the status of the Service called name from client // every interval until inState returns `true` indicating it is done, returns an // error or timeout. desc will be used to name the metric that is emitted to diff --git a/test/e2e/broker_trigger_test.go b/test/e2e/broker_trigger_test.go index e65d94ad87f..55611971497 100644 --- a/test/e2e/broker_trigger_test.go +++ b/test/e2e/broker_trigger_test.go @@ -19,9 +19,12 @@ package e2e import ( "fmt" + "strings" "testing" "time" + "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" + "github.com/knative/eventing/test" "github.com/knative/pkg/test/logging" corev1 "k8s.io/api/core/v1" @@ -33,7 +36,7 @@ const ( waitForDefaultBrokerCreation = 3 * time.Second selectorKey = "end2end-test-broker-trigger" - any = "any" + any = v1alpha1.TriggerAnyFilter eventType1 = "type1" eventType2 = "type2" eventSource1 = "source1" @@ -42,7 +45,8 @@ const ( // Helper function to create names for different objects (e.g., triggers, services, etc.) func name(obj, brokerName, eventType, eventSource string) string { - return fmt.Sprintf("%s-%s-%s-%s", obj, brokerName, eventType, eventSource) + // pod names need to be lowercase. We might have an eventType as Any, that is why we lowercase them. + return strings.ToLower(fmt.Sprintf("%s-%s-%s-%s", obj, brokerName, eventType, eventSource)) } func TestBrokerTrigger(t *testing.T) { @@ -105,7 +109,7 @@ func TestBrokerTrigger(t *testing.T) { subscriberPodName := name("dumper", dumper.Broker, dumper.EventType, dumper.EventSource) subscriberPod := test.EventLoggerPod(subscriberPodName, dumper.Namespace, dumper.Selector) if err := CreatePod(clients, subscriberPod, logger, cleaner); err != nil { - t.Fatalf("Failed to create subscriber pod: %v", err) + t.Fatalf("Error creating subscriber pod: %v", err) } subscriberPods[subscriberPodName] = subscriberPod } @@ -133,13 +137,19 @@ func TestBrokerTrigger(t *testing.T) { // subscriberName should be the same as the subscriberSvc from before. subscriberName := name("svc", dumper.Broker, dumper.EventType, dumper.EventSource) trigger := test.Trigger(triggerName, dumper.Namespace, dumper.EventType, dumper.EventSource, dumper.Broker, subscriberName) - // Wait for the triggers to be ready - err := WithTriggerReady(clients, trigger, logger, cleaner) + err := CreateTrigger(clients, trigger, logger, cleaner) if err != nil { - t.Fatalf("Error waiting for trigger to become ready: %v", err) + t.Fatalf("Error creating trigger: %v", err) } } + logger.Info("Triggers created") + + // Wait for all of them to be ready. + if err := WaitForAllTriggersReady(clients, logger, ns); err != nil { + t.Fatalf("Error waiting for triggers to become ready: %v", err) + } + logger.Info("Triggers ready") logger.Info("Creating event sender pods") @@ -154,11 +164,11 @@ func TestBrokerTrigger(t *testing.T) { Encoding: test.CloudEventEncodingStructured, } // Create sender pod. - senderPodName := fmt.Sprintf("sender-%s-%s", sender.EventType, sender.EventSource) + senderPodName := name("sender", "", sender.EventType, sender.EventSource) senderPod := test.EventSenderPod(senderPodName, sender.Namespace, sender.Url, cloudEvent) logger.Infof("Sender pod: %#v", senderPod) if err := CreatePod(clients, senderPod, logger, cleaner); err != nil { - t.Fatalf("Failed to create event sender pod: %v", err) + t.Fatalf("Error creating event sender pod: %v", err) } // Check on every dumper whether we should expect this event or not, and add its body if so. @@ -186,7 +196,7 @@ func TestBrokerTrigger(t *testing.T) { } } - logger.Info("Successfully completed!") + logger.Info("Verification successful!") } diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go index 929a6e93944..5ff93f132b5 100644 --- a/test/e2e/e2e.go +++ b/test/e2e/e2e.go @@ -363,6 +363,15 @@ func WaitForAllPodsRunning(clients *test.Clients, logger *logging.BaseLogger, na return nil } +// WaitForAllTriggersReady will wait until all triggers in the given namespace are ready +func WaitForAllTriggersReady(clients *test.Clients, logger *logging.BaseLogger, namespace string) error { + triggers := clients.Eventing.EventingV1alpha1().Triggers(namespace) + if err := test.WaitForTriggersListState(triggers, test.TriggersReady, "TriggerIsReady"); err != nil { + return err + } + return nil +} + // AnnotateNamespace annotates the test namespace with the annotations map func AnnotateNamespace(clients *test.Clients, logger *logging.BaseLogger, annotations map[string]string) error { ns := pkgTest.Flags.Namespace diff --git a/test/states.go b/test/states.go index 26db82aee3d..a05e08728d0 100644 --- a/test/states.go +++ b/test/states.go @@ -66,6 +66,22 @@ func IsTriggerReady(t *eventingv1alpha1.Trigger) (bool, error) { return t.Status.IsReady(), nil } +// TriggersReady will check the status conditions of the trigger list and return true +// if all triggers are Ready. +func TriggersReady(triggerList *eventingv1alpha1.TriggerList) (bool, error) { + var names []string + for _, t := range triggerList.Items { + names = append(names, t.Name) + } + log.Printf("Checking triggers: %v", names) + for _, trigger := range triggerList.Items { + if !trigger.Status.IsReady() { + return false, nil + } + } + return true, nil +} + // PodsRunning will check the status conditions of the pod list and return true // if all pods are Running. func PodsRunning(podList *corev1.PodList) (bool, error) { From d361afb271079f0dd201f2ba32eb6ab4edb51fcc Mon Sep 17 00:00:00 2001 From: nachocano Date: Wed, 20 Feb 2019 14:47:45 -0800 Subject: [PATCH 078/128] Updated logs --- test/e2e/broker_trigger_test.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/e2e/broker_trigger_test.go b/test/e2e/broker_trigger_test.go index 55611971497..7a0056e3df1 100644 --- a/test/e2e/broker_trigger_test.go +++ b/test/e2e/broker_trigger_test.go @@ -72,7 +72,7 @@ func TestBrokerTrigger(t *testing.T) { // As we are not creating the default broker, // we wait for a few seconds for the broker to get created. // Otherwise, if we try to wait for its Ready status and the namespace controller - // didn't create it yet, it fails. + // didn't actually create it yet, the test will fail. logger.Info("Waiting for default broker creation") time.Sleep(waitForDefaultBrokerCreation) @@ -114,6 +114,8 @@ func TestBrokerTrigger(t *testing.T) { subscriberPods[subscriberPodName] = subscriberPod } + logger.Info("Subscriber pods created") + // Wait for all of them to be running. if err := WaitForAllPodsRunning(clients, logger, ns); err != nil { t.Fatalf("Error waiting for event logger pod to become running: %v", err) @@ -179,14 +181,14 @@ func TestBrokerTrigger(t *testing.T) { } } - logger.Info("Created event sender pods") + logger.Info("Event sender pods created") // Wait for all of them to be running. if err := WaitForAllPodsRunning(clients, logger, ns); err != nil { t.Fatalf("Error waiting for event sender pod to become running: %v", err) } - logger.Info("Verifying events arrived to appropriate dumpers") + logger.Info("Verifying events delivered to appropriate dumpers") for _, dumper := range dumpers { subscriberPodName := name("dumper", dumper.Broker, dumper.EventType, dumper.EventSource) From d1198a96e1e05d6d59705ca9fd2f801a21b21ebf Mon Sep 17 00:00:00 2001 From: nachocano Date: Wed, 20 Feb 2019 15:53:07 -0800 Subject: [PATCH 079/128] Working --- test/e2e/broker_trigger_test.go | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/test/e2e/broker_trigger_test.go b/test/e2e/broker_trigger_test.go index 7a0056e3df1..80ff3043315 100644 --- a/test/e2e/broker_trigger_test.go +++ b/test/e2e/broker_trigger_test.go @@ -77,12 +77,15 @@ func TestBrokerTrigger(t *testing.T) { time.Sleep(waitForDefaultBrokerCreation) // Wait for default broker ready. + logger.Info("Waiting for default broker to be ready") defaultBroker := test.Broker(defaultBrokerName, ns) err = WaitForBrokerReady(clients, defaultBroker) if err != nil { t.Fatalf("Error waiting for default broker to become ready: %v", err) } + logger.Info("Default broker ready") + defaultBrokerUrl := fmt.Sprintf("http://%s", defaultBroker.Status.Address.Hostname) // Create sender helpers. @@ -116,6 +119,8 @@ func TestBrokerTrigger(t *testing.T) { logger.Info("Subscriber pods created") + logger.Info("Waiting for subscriber pods to become running") + // Wait for all of them to be running. if err := WaitForAllPodsRunning(clients, logger, ns); err != nil { t.Fatalf("Error waiting for event logger pod to become running: %v", err) @@ -127,7 +132,10 @@ func TestBrokerTrigger(t *testing.T) { for _, dumper := range dumpers { subscriberSvcName := name("svc", dumper.Broker, dumper.EventType, dumper.EventSource) - test.Service(subscriberSvcName, dumper.Namespace, dumper.Selector) + service := test.Service(subscriberSvcName, dumper.Namespace, dumper.Selector) + if err := CreateService(clients, service, logger, cleaner); err != nil { + t.Fatalf("Error creating subscriber service: %v", err) + } } logger.Info("Subscriber services created") @@ -147,6 +155,8 @@ func TestBrokerTrigger(t *testing.T) { logger.Info("Triggers created") + logger.Info("Waiting for triggers to become ready") + // Wait for all of them to be ready. if err := WaitForAllTriggersReady(clients, logger, ns); err != nil { t.Fatalf("Error waiting for triggers to become ready: %v", err) @@ -168,7 +178,6 @@ func TestBrokerTrigger(t *testing.T) { // Create sender pod. senderPodName := name("sender", "", sender.EventType, sender.EventSource) senderPod := test.EventSenderPod(senderPodName, sender.Namespace, sender.Url, cloudEvent) - logger.Infof("Sender pod: %#v", senderPod) if err := CreatePod(clients, senderPod, logger, cleaner); err != nil { t.Fatalf("Error creating event sender pod: %v", err) } @@ -183,11 +192,15 @@ func TestBrokerTrigger(t *testing.T) { logger.Info("Event sender pods created") + logger.Info("Waiting for event sender pods to be running") + // Wait for all of them to be running. if err := WaitForAllPodsRunning(clients, logger, ns); err != nil { t.Fatalf("Error waiting for event sender pod to become running: %v", err) } + logger.Info("Event sender pods running") + logger.Info("Verifying events delivered to appropriate dumpers") for _, dumper := range dumpers { From 6bd659c4e940cd0031d66ecd7a0efda8a033b3a2 Mon Sep 17 00:00:00 2001 From: nachocano Date: Wed, 20 Feb 2019 16:45:10 -0800 Subject: [PATCH 080/128] Adding logs... Still not receiving the events. --- test/e2e/broker_trigger_test.go | 23 +++++++++++++---------- test/e2e/e2e.go | 4 ++-- test/helpers.go | 12 ++++++------ 3 files changed, 21 insertions(+), 18 deletions(-) diff --git a/test/e2e/broker_trigger_test.go b/test/e2e/broker_trigger_test.go index 80ff3043315..cad5b1d6887 100644 --- a/test/e2e/broker_trigger_test.go +++ b/test/e2e/broker_trigger_test.go @@ -1,5 +1,3 @@ -// +build e2e - /* Copyright 2019 The Knative Authors Licensed under the Apache License, Version 2.0 (the "License"); @@ -89,7 +87,7 @@ func TestBrokerTrigger(t *testing.T) { defaultBrokerUrl := fmt.Sprintf("http://%s", defaultBroker.Status.Address.Hostname) // Create sender helpers. - senders := []test.SenderInfo{ + senders := []*test.SenderInfo{ {ns, defaultBrokerUrl, eventType1, eventSource1}, {ns, defaultBrokerUrl, eventType1, eventSource2}, {ns, defaultBrokerUrl, eventType2, eventSource1}, @@ -97,7 +95,7 @@ func TestBrokerTrigger(t *testing.T) { } // Create dumper helpers. - dumpers := []test.DumperInfo{ + dumpers := []*test.DumperInfo{ {ns, defaultBrokerName, any, any, map[string]string{selectorKey: string(uuid.NewUUID())}, make([]string, 0)}, {ns, defaultBrokerName, eventType1, any, map[string]string{selectorKey: string(uuid.NewUUID())}, make([]string, 0)}, {ns, defaultBrokerName, any, eventSource1, map[string]string{selectorKey: string(uuid.NewUUID())}, make([]string, 0)}, @@ -168,7 +166,8 @@ func TestBrokerTrigger(t *testing.T) { for _, sender := range senders { // Create cloud event. - body := fmt.Sprintf("Testing Broker-Trigger %s", uuid.NewUUID()) + // Using event type and source as part of the body for easier debugging. + body := fmt.Sprintf("Body-%s-%s", sender.EventSource, sender.EventType) cloudEvent := test.CloudEvent{ Source: sender.EventSource, Type: sender.EventType, @@ -184,8 +183,8 @@ func TestBrokerTrigger(t *testing.T) { // Check on every dumper whether we should expect this event or not, and add its body if so. for _, dumper := range dumpers { - if shouldExpectEvent(&dumper, &sender) { - dumper.ExpectedBodies = append(dumper.ExpectedBodies, body) + if shouldExpectEvent(dumper, sender, logger) { + dumper.Expect = append(dumper.Expect, body) } } } @@ -206,7 +205,8 @@ func TestBrokerTrigger(t *testing.T) { for _, dumper := range dumpers { subscriberPodName := name("dumper", dumper.Broker, dumper.EventType, dumper.EventSource) subscriberPod := subscriberPods[subscriberPodName] - if err := WaitForLogContents(clients, logger, subscriberPodName, subscriberPod.Spec.Containers[0].Name, dumper.Namespace, dumper.ExpectedBodies); err != nil { + logger.Infof("Dumper %q expecting %q", subscriberPodName, strings.Join(dumper.Expect, ",")) + if err := WaitForLogContents(clients, logger, subscriberPodName, subscriberPod.Spec.Containers[0].Name, dumper.Namespace, dumper.Expect); err != nil { t.Fatalf("String(s) not found in logs of subscriber pod %q: %v", subscriberPodName, err) } } @@ -215,14 +215,17 @@ func TestBrokerTrigger(t *testing.T) { } -func shouldExpectEvent(dumper *test.DumperInfo, sender *test.SenderInfo) bool { +func shouldExpectEvent(dumper *test.DumperInfo, sender *test.SenderInfo, logger *logging.BaseLogger) bool { if dumper.Namespace != sender.Namespace { + logger.Debugf("Namespaces mismatch, dumper %s, sender %s", dumper.Namespace, sender.Namespace) return false } if dumper.EventType != any && dumper.EventType != sender.EventType { + logger.Debugf("Event types mismatch, dumper %s, sender %s", dumper.EventType, sender.EventType) return false } - if dumper.EventSource != any && dumper.EventType != sender.EventSource { + if dumper.EventSource != any && dumper.EventSource != sender.EventSource { + logger.Debugf("Event sources mismatch, dumper %s, sender %s", dumper.EventSource, sender.EventSource) return false } return true diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go index 5ff93f132b5..4972f02eb75 100644 --- a/test/e2e/e2e.go +++ b/test/e2e/e2e.go @@ -321,7 +321,7 @@ func PodLogs(clients *test.Clients, podName string, containerName string, namesp }).Do() raw, err := result.Raw() if err == nil { - logger.Infof("%s logs request result: %#v", podName, string(raw)) + logger.Debugf("%s logs request result: %#v", podName, string(raw)) } else { logger.Infof("%s logs request result: %#v", podName, err) } @@ -341,7 +341,7 @@ func WaitForLogContents(clients *test.Clients, logger *logging.BaseLogger, podNa } for _, content := range contents { if !strings.Contains(string(logs), content) { - logger.Infof("Could not find content %s for %s/%s", content, podName, containerName) + logger.Infof("Could not find content %q for %s/%s. Found %q instead", content, podName, containerName, string(logs)) return false, nil } } diff --git a/test/helpers.go b/test/helpers.go index 3dec1061c90..d6e283cb87e 100644 --- a/test/helpers.go +++ b/test/helpers.go @@ -19,12 +19,12 @@ package test // We also use this to verify the expected events that should be received // by the particular subscriber pods. type DumperInfo struct { - Namespace string - Broker string - EventType string - EventSource string - Selector map[string]string - ExpectedBodies []string + Namespace string + Broker string + EventType string + EventSource string + Selector map[string]string + Expect []string } // Helper struct to easily create event sender pods. From 0d80f9ca1f8b060eae5966a6c68fcc4ba1fac1f4 Mon Sep 17 00:00:00 2001 From: nachocano Date: Wed, 20 Feb 2019 17:04:31 -0800 Subject: [PATCH 081/128] More logs --- test/e2e/broker_trigger_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e/broker_trigger_test.go b/test/e2e/broker_trigger_test.go index cad5b1d6887..c70f040a51f 100644 --- a/test/e2e/broker_trigger_test.go +++ b/test/e2e/broker_trigger_test.go @@ -82,10 +82,10 @@ func TestBrokerTrigger(t *testing.T) { t.Fatalf("Error waiting for default broker to become ready: %v", err) } - logger.Info("Default broker ready") - defaultBrokerUrl := fmt.Sprintf("http://%s", defaultBroker.Status.Address.Hostname) + logger.Infof("Default broker ready: %q", defaultBrokerUrl) + // Create sender helpers. senders := []*test.SenderInfo{ {ns, defaultBrokerUrl, eventType1, eventSource1}, From cd862ffdbbc127c124488c3ab751ba85cb1e25e5 Mon Sep 17 00:00:00 2001 From: Nacho Cano Date: Wed, 20 Feb 2019 23:07:32 -0800 Subject: [PATCH 082/128] Adding build constraint --- test/e2e/broker_trigger_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/e2e/broker_trigger_test.go b/test/e2e/broker_trigger_test.go index c70f040a51f..b876156be2d 100644 --- a/test/e2e/broker_trigger_test.go +++ b/test/e2e/broker_trigger_test.go @@ -1,3 +1,5 @@ +// +build e2e + /* Copyright 2019 The Knative Authors Licensed under the Apache License, Version 2.0 (the "License"); @@ -71,6 +73,7 @@ func TestBrokerTrigger(t *testing.T) { // we wait for a few seconds for the broker to get created. // Otherwise, if we try to wait for its Ready status and the namespace controller // didn't actually create it yet, the test will fail. + // TODO improve logger.Info("Waiting for default broker creation") time.Sleep(waitForDefaultBrokerCreation) From 2ea7aa3ff4e31a2ff4a6697bf6f5c9efcc210543 Mon Sep 17 00:00:00 2001 From: Nacho Cano Date: Wed, 20 Feb 2019 23:14:02 -0800 Subject: [PATCH 083/128] Removing unnecessary stuff --- test/crd_checks.go | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/test/crd_checks.go b/test/crd_checks.go index 93648af5707..faebe200ed4 100644 --- a/test/crd_checks.go +++ b/test/crd_checks.go @@ -23,10 +23,6 @@ import ( "fmt" "time" - v1 "k8s.io/client-go/kubernetes/typed/core/v1" - - corev1 "k8s.io/api/core/v1" - eventingv1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" eventingclient "github.com/knative/eventing/pkg/client/clientset/versioned/typed/eventing/v1alpha1" servingv1alpha1 "github.com/knative/serving/pkg/apis/serving/v1alpha1" @@ -149,21 +145,3 @@ func WaitForTriggersListState(clients eventingclient.TriggerInterface, inState f return inState(t) }) } - -// WaitForServiceState polls the status of the Service called name from client -// every interval until inState returns `true` indicating it is done, returns an -// error or timeout. desc will be used to name the metric that is emitted to -// track how long it took for name to get into the state checked by inState. -func WaitForServiceState(client v1.ServiceInterface, name string, inState func(r *corev1.Service) (bool, error), desc string) error { - metricName := fmt.Sprintf("WaitForServiceState/%s/%s", name, desc) - _, span := trace.StartSpan(context.Background(), metricName) - defer span.End() - - return wait.PollImmediate(interval, timeout, func() (bool, error) { - r, err := client.Get(name, metav1.GetOptions{}) - if err != nil { - return true, err - } - return inState(r) - }) -} From aa60a8a938a94219732b8cc096af5fbf22ce0d7b Mon Sep 17 00:00:00 2001 From: Nacho Cano Date: Wed, 20 Feb 2019 23:59:52 -0800 Subject: [PATCH 084/128] Removing ugly structs --- test/crd.go | 6 ++ test/e2e/broker_trigger_test.go | 111 +++++++++++++++++--------------- test/helpers.go | 36 ----------- 3 files changed, 65 insertions(+), 88 deletions(-) delete mode 100644 test/helpers.go diff --git a/test/crd.go b/test/crd.go index 5121cd925a1..af3ac3971c8 100644 --- a/test/crd.go +++ b/test/crd.go @@ -215,6 +215,12 @@ type CloudEvent struct { Encoding string // binary or structured } +// TypeAndSource specifies the type and source of an Event. +type TypeAndSource struct { + Type string + Source string +} + const ( CloudEventEncodingBinary = "binary" CloudEventEncodingStructured = "structured" diff --git a/test/e2e/broker_trigger_test.go b/test/e2e/broker_trigger_test.go index b876156be2d..4ef7188a537 100644 --- a/test/e2e/broker_trigger_test.go +++ b/test/e2e/broker_trigger_test.go @@ -39,14 +39,16 @@ const ( any = v1alpha1.TriggerAnyFilter eventType1 = "type1" eventType2 = "type2" + eventType3 = "type3" eventSource1 = "source1" eventSource2 = "source2" + eventSource3 = "source3" ) // Helper function to create names for different objects (e.g., triggers, services, etc.) -func name(obj, brokerName, eventType, eventSource string) string { +func name(obj, eventType, eventSource string) string { // pod names need to be lowercase. We might have an eventType as Any, that is why we lowercase them. - return strings.ToLower(fmt.Sprintf("%s-%s-%s-%s", obj, brokerName, eventType, eventSource)) + return strings.ToLower(fmt.Sprintf("%s-%s-%s", obj, eventType, eventSource)) } func TestBrokerTrigger(t *testing.T) { @@ -73,7 +75,7 @@ func TestBrokerTrigger(t *testing.T) { // we wait for a few seconds for the broker to get created. // Otherwise, if we try to wait for its Ready status and the namespace controller // didn't actually create it yet, the test will fail. - // TODO improve + // TODO improve this logger.Info("Waiting for default broker creation") time.Sleep(waitForDefaultBrokerCreation) @@ -89,29 +91,22 @@ func TestBrokerTrigger(t *testing.T) { logger.Infof("Default broker ready: %q", defaultBrokerUrl) - // Create sender helpers. - senders := []*test.SenderInfo{ - {ns, defaultBrokerUrl, eventType1, eventSource1}, - {ns, defaultBrokerUrl, eventType1, eventSource2}, - {ns, defaultBrokerUrl, eventType2, eventSource1}, - {ns, defaultBrokerUrl, eventType2, eventSource2}, + eventsToReceive := []test.TypeAndSource{ + {any, any}, + {eventType1, any}, + {any, eventSource1}, + {eventType1, eventSource1}, } - // Create dumper helpers. - dumpers := []*test.DumperInfo{ - {ns, defaultBrokerName, any, any, map[string]string{selectorKey: string(uuid.NewUUID())}, make([]string, 0)}, - {ns, defaultBrokerName, eventType1, any, map[string]string{selectorKey: string(uuid.NewUUID())}, make([]string, 0)}, - {ns, defaultBrokerName, any, eventSource1, map[string]string{selectorKey: string(uuid.NewUUID())}, make([]string, 0)}, - {ns, defaultBrokerName, eventType1, eventSource1, map[string]string{selectorKey: string(uuid.NewUUID())}, make([]string, 0)}, - } + selector := map[string]string{selectorKey: string(uuid.NewUUID())} logger.Info("Creating Subscriber pods") - // Save the references in this map for later use. + // Save the pods references in this map for later use. subscriberPods := make(map[string]*corev1.Pod, 0) - for _, dumper := range dumpers { - subscriberPodName := name("dumper", dumper.Broker, dumper.EventType, dumper.EventSource) - subscriberPod := test.EventLoggerPod(subscriberPodName, dumper.Namespace, dumper.Selector) + for _, event := range eventsToReceive { + subscriberPodName := name("dumper", event.Type, event.Source) + subscriberPod := test.EventLoggerPod(subscriberPodName, ns, selector) if err := CreatePod(clients, subscriberPod, logger, cleaner); err != nil { t.Fatalf("Error creating subscriber pod: %v", err) } @@ -122,7 +117,7 @@ func TestBrokerTrigger(t *testing.T) { logger.Info("Waiting for subscriber pods to become running") - // Wait for all of them to be running. + // Wait for all of the pods in the namespace to become running. if err := WaitForAllPodsRunning(clients, logger, ns); err != nil { t.Fatalf("Error waiting for event logger pod to become running: %v", err) } @@ -131,9 +126,9 @@ func TestBrokerTrigger(t *testing.T) { logger.Info("Creating Subscriber services") - for _, dumper := range dumpers { - subscriberSvcName := name("svc", dumper.Broker, dumper.EventType, dumper.EventSource) - service := test.Service(subscriberSvcName, dumper.Namespace, dumper.Selector) + for _, event := range eventsToReceive { + subscriberSvcName := name("svc", event.Type, event.Source) + service := test.Service(subscriberSvcName, ns, selector) if err := CreateService(clients, service, logger, cleaner); err != nil { t.Fatalf("Error creating subscriber service: %v", err) } @@ -143,11 +138,11 @@ func TestBrokerTrigger(t *testing.T) { logger.Info("Creating Triggers") - for _, dumper := range dumpers { - triggerName := name("trigger", dumper.Broker, dumper.EventType, dumper.EventSource) + for _, event := range eventsToReceive { + triggerName := name("trigger", event.Type, event.Source) // subscriberName should be the same as the subscriberSvc from before. - subscriberName := name("svc", dumper.Broker, dumper.EventType, dumper.EventSource) - trigger := test.Trigger(triggerName, dumper.Namespace, dumper.EventType, dumper.EventSource, dumper.Broker, subscriberName) + subscriberName := name("svc", event.Type, event.Source) + trigger := test.Trigger(triggerName, ns, event.Type, event.Source, defaultBrokerName, subscriberName) err := CreateTrigger(clients, trigger, logger, cleaner) if err != nil { t.Fatalf("Error creating trigger: %v", err) @@ -158,36 +153,52 @@ func TestBrokerTrigger(t *testing.T) { logger.Info("Waiting for triggers to become ready") - // Wait for all of them to be ready. + // Wait for all of the triggers in the namespace to be ready. if err := WaitForAllTriggersReady(clients, logger, ns); err != nil { t.Fatalf("Error waiting for triggers to become ready: %v", err) } logger.Info("Triggers ready") + eventsToSend := []test.TypeAndSource{ + {eventType1, eventSource1}, + {eventType1, eventSource2}, + {eventType1, eventSource3}, + {eventType2, eventSource1}, + {eventType2, eventSource2}, + {eventType2, eventSource3}, + {eventType3, eventSource1}, + {eventType3, eventSource2}, + {eventType3, eventSource3}, + } + logger.Info("Creating event sender pods") - for _, sender := range senders { + // Map to save the expected events per dumper so that we can verify the delivery. + expectedEvents := make(map[string][]string) + for _, eventToSend := range eventsToSend { // Create cloud event. // Using event type and source as part of the body for easier debugging. - body := fmt.Sprintf("Body-%s-%s", sender.EventSource, sender.EventType) + body := fmt.Sprintf("Body-%s-%s", eventToSend.Type, eventToSend.Source) cloudEvent := test.CloudEvent{ - Source: sender.EventSource, - Type: sender.EventType, + Source: eventToSend.Source, + Type: eventToSend.Type, Data: fmt.Sprintf(`{"msg":%q}`, body), Encoding: test.CloudEventEncodingStructured, } // Create sender pod. - senderPodName := name("sender", "", sender.EventType, sender.EventSource) - senderPod := test.EventSenderPod(senderPodName, sender.Namespace, sender.Url, cloudEvent) + senderPodName := name("sender", eventToSend.Type, eventToSend.Source) + senderPod := test.EventSenderPod(senderPodName, ns, defaultBrokerUrl, cloudEvent) if err := CreatePod(clients, senderPod, logger, cleaner); err != nil { t.Fatalf("Error creating event sender pod: %v", err) } - // Check on every dumper whether we should expect this event or not, and add its body if so. - for _, dumper := range dumpers { - if shouldExpectEvent(dumper, sender, logger) { - dumper.Expect = append(dumper.Expect, body) + // Check on every dumper whether we should expect this event or not, and add its body + // to the expectedEvents map if so. + for _, eventToReceive := range eventsToReceive { + if shouldExpectEvent(&eventToSend, &eventToReceive, logger) { + subscriberPodName := name("dumper", eventToReceive.Type, eventToReceive.Source) + expectedEvents[subscriberPodName] = append(expectedEvents[subscriberPodName], body) } } } @@ -205,11 +216,11 @@ func TestBrokerTrigger(t *testing.T) { logger.Info("Verifying events delivered to appropriate dumpers") - for _, dumper := range dumpers { - subscriberPodName := name("dumper", dumper.Broker, dumper.EventType, dumper.EventSource) + for _, event := range eventsToReceive { + subscriberPodName := name("dumper", event.Type, event.Source) subscriberPod := subscriberPods[subscriberPodName] - logger.Infof("Dumper %q expecting %q", subscriberPodName, strings.Join(dumper.Expect, ",")) - if err := WaitForLogContents(clients, logger, subscriberPodName, subscriberPod.Spec.Containers[0].Name, dumper.Namespace, dumper.Expect); err != nil { + logger.Infof("Dumper %q expecting %q", subscriberPodName, strings.Join(expectedEvents[subscriberPodName], ",")) + if err := WaitForLogContents(clients, logger, subscriberPodName, subscriberPod.Spec.Containers[0].Name, ns, expectedEvents[subscriberPodName]); err != nil { t.Fatalf("String(s) not found in logs of subscriber pod %q: %v", subscriberPodName, err) } } @@ -218,17 +229,13 @@ func TestBrokerTrigger(t *testing.T) { } -func shouldExpectEvent(dumper *test.DumperInfo, sender *test.SenderInfo, logger *logging.BaseLogger) bool { - if dumper.Namespace != sender.Namespace { - logger.Debugf("Namespaces mismatch, dumper %s, sender %s", dumper.Namespace, sender.Namespace) - return false - } - if dumper.EventType != any && dumper.EventType != sender.EventType { - logger.Debugf("Event types mismatch, dumper %s, sender %s", dumper.EventType, sender.EventType) +func shouldExpectEvent(eventToSend *test.TypeAndSource, eventToReceive *test.TypeAndSource, logger *logging.BaseLogger) bool { + if eventToReceive.Type != any && eventToReceive.Type != eventToSend.Type { + logger.Debugf("Event types mismatch, receive %s, send %s", eventToReceive.Type, eventToSend.Type) return false } - if dumper.EventSource != any && dumper.EventSource != sender.EventSource { - logger.Debugf("Event sources mismatch, dumper %s, sender %s", dumper.EventSource, sender.EventSource) + if eventToReceive.Source != any && eventToReceive.Source != eventToSend.Source { + logger.Debugf("Event sources mismatch, receive %s, send %s", eventToReceive.Source, eventToSend.Source) return false } return true diff --git a/test/helpers.go b/test/helpers.go deleted file mode 100644 index d6e283cb87e..00000000000 --- a/test/helpers.go +++ /dev/null @@ -1,36 +0,0 @@ -/* -Copyright 2019 The Knative Authors -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package test - -// Helper struct to easily create subscriber pods, services and triggers. -// We also use this to verify the expected events that should be received -// by the particular subscriber pods. -type DumperInfo struct { - Namespace string - Broker string - EventType string - EventSource string - Selector map[string]string - Expect []string -} - -// Helper struct to easily create event sender pods. -type SenderInfo struct { - Namespace string - Url string - EventType string - EventSource string -} From 6471b4f2153a048352af8cec23b50735ccfa67c7 Mon Sep 17 00:00:00 2001 From: nachocano Date: Thu, 21 Feb 2019 10:32:42 -0800 Subject: [PATCH 085/128] More logs --- test/e2e/broker_trigger_test.go | 6 ++++-- test/e2e/e2e.go | 3 +++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/test/e2e/broker_trigger_test.go b/test/e2e/broker_trigger_test.go index 4ef7188a537..15244cba36d 100644 --- a/test/e2e/broker_trigger_test.go +++ b/test/e2e/broker_trigger_test.go @@ -51,8 +51,8 @@ func name(obj, eventType, eventSource string) string { return strings.ToLower(fmt.Sprintf("%s-%s-%s", obj, eventType, eventSource)) } -func TestBrokerTrigger(t *testing.T) { - logger := logging.GetContextLogger("TestBrokerTrigger") +func TestDefaultBrokerWithManyTriggers(t *testing.T) { + logger := logging.GetContextLogger("TestDefaultBrokerWithManyTriggers") clients, cleaner := Setup(t, logger) defer TearDown(clients, cleaner, logger) @@ -91,6 +91,7 @@ func TestBrokerTrigger(t *testing.T) { logger.Infof("Default broker ready: %q", defaultBrokerUrl) + // These are the event types and sources that triggers will listen to. eventsToReceive := []test.TypeAndSource{ {any, any}, {eventType1, any}, @@ -160,6 +161,7 @@ func TestBrokerTrigger(t *testing.T) { logger.Info("Triggers ready") + // These are the event types and sources that will be send. eventsToSend := []test.TypeAndSource{ {eventType1, eventSource1}, {eventType1, eventSource2}, diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go index 4972f02eb75..8aac5228087 100644 --- a/test/e2e/e2e.go +++ b/test/e2e/e2e.go @@ -343,6 +343,9 @@ func WaitForLogContents(clients *test.Clients, logger *logging.BaseLogger, podNa if !strings.Contains(string(logs), content) { logger.Infof("Could not find content %q for %s/%s. Found %q instead", content, podName, containerName, string(logs)) return false, nil + } else { + logger.Infof("Found content %q for %s/%s in logs %q", content, podName, containerName, string(logs)) + // do not return as we will keep on looking for the other contents in the slice } } return true, nil From c4959e213aa0ee481eedfe8ddbed6304e5eb21e9 Mon Sep 17 00:00:00 2001 From: nachocano Date: Thu, 21 Feb 2019 14:22:27 -0800 Subject: [PATCH 086/128] Removing quotes --- pkg/broker/receiver.go | 5 +- .../v1alpha1/namespace/namespace.go | 51 +++++++++---------- test/crd.go | 6 +-- 3 files changed, 30 insertions(+), 32 deletions(-) diff --git a/pkg/broker/receiver.go b/pkg/broker/receiver.go index 9dd264337ee..93ada7417d3 100644 --- a/pkg/broker/receiver.go +++ b/pkg/broker/receiver.go @@ -98,18 +98,19 @@ func (r *Receiver) getTrigger(ctx context.Context, ref provisioners.ChannelRefer } func (r *Receiver) shouldSendMessage(ts *eventingv1alpha1.TriggerSpec, m *provisioners.Message) bool { + r.logger.Debug("Message headers", zap.Any("headers", m.Headers)) if ts.Filter == nil || ts.Filter.SourceAndType == nil { r.logger.Error("No filter specified") return false } filterType := ts.Filter.SourceAndType.Type if filterType != eventingv1alpha1.TriggerAnyFilter && filterType != m.Headers["Ce-Eventtype"] { - r.logger.Debug("Wrong type", zap.String("trigger.spec.filter.exactMatch.type", filterType), zap.String("message.type", m.Headers["Ce-Eventtype"])) + r.logger.Debug("Wrong type", zap.String("trigger.spec.filter.sourceAndType.type", filterType), zap.String("message.type", m.Headers["Ce-Eventtype"])) return false } filterSource := ts.Filter.SourceAndType.Source if filterSource != eventingv1alpha1.TriggerAnyFilter && filterSource != m.Headers["Ce-Source"] { - r.logger.Debug("Wrong source", zap.String("trigger.spec.filter.exactMatch.source", filterSource), zap.String("message.source", m.Headers["Ce-Source"])) + r.logger.Debug("Wrong source", zap.String("trigger.spec.filter.sourceAndType.source", filterSource), zap.String("message.source", m.Headers["Ce-Source"])) return false } return true diff --git a/pkg/reconciler/v1alpha1/namespace/namespace.go b/pkg/reconciler/v1alpha1/namespace/namespace.go index 483d659604d..e0c874b80b6 100644 --- a/pkg/reconciler/v1alpha1/namespace/namespace.go +++ b/pkg/reconciler/v1alpha1/namespace/namespace.go @@ -19,11 +19,12 @@ package namespace import ( "context" "fmt" + "github.com/knative/eventing/contrib/gcppubsub/pkg/util/logging" "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" "go.uber.org/zap" - "k8s.io/api/core/v1" corev1 "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/api/errors" k8serrors "k8s.io/apimachinery/pkg/api/errors" @@ -46,16 +47,16 @@ const ( // itself when creating events. controllerAgentName = "knative-eventing-namespace-controller" - defaultBroker = "default" - brokerFilterSA = "eventing-broker-filter" - brokerFilterRB = "eventing-broker-filter" + defaultBroker = "default" + brokerFilterSA = "eventing-broker-filter" + brokerFilterRB = "eventing-broker-filter" brokerFilterClusterRole = "eventing-broker-filter" knativeEventingAnnotation = "eventing.knative.dev/inject" // Name of the corev1.Events emitted from the reconciliation process. - brokerCreated = "BrokerCreated" - serviceAccountCreated = "BrokerFilterServiceAccountCreated" + brokerCreated = "BrokerCreated" + serviceAccountCreated = "BrokerFilterServiceAccountCreated" serviceAccountRBACCreated = "BrokerFilterServiceAccountRBACCreated" ) @@ -92,8 +93,8 @@ func ProvideController(logger *zap.Logger) func(manager.Manager) (controller.Con } // Watch all the resources that this reconciler reconciles. - for _, t := range []runtime.Object{ &corev1.ServiceAccount{}, &rbacv1.RoleBinding{}, &v1alpha1.Broker{} } { - err = c.Watch(&source.Kind{Type: t}, &handler.EnqueueRequestsFromMapFunc{ToRequests: &namespaceMapper{}}); + for _, t := range []runtime.Object{&corev1.ServiceAccount{}, &rbacv1.RoleBinding{}, &v1alpha1.Broker{}} { + err = c.Watch(&source.Kind{Type: t}, &handler.EnqueueRequestsFromMapFunc{ToRequests: &namespaceMapper{}}) if err != nil { return nil, err } @@ -228,8 +229,8 @@ func newBrokerFilterServiceAccount(ns *corev1.Namespace) *corev1.ServiceAccount return &corev1.ServiceAccount{ ObjectMeta: metav1.ObjectMeta{ Namespace: ns.Name, - Name: brokerFilterSA, - Labels: injectedLabels(), + Name: brokerFilterSA, + Labels: injectedLabels(), }, } } @@ -240,7 +241,7 @@ func injectedLabels() map[string]string { } } -func (r *reconciler) reconcileBrokerFilterRBAC(ctx context.Context, ns *corev1.Namespace, sa *corev1.ServiceAccount) (*rbacv1.RoleBinding, error) { +func (r *reconciler) reconcileBrokerFilterRBAC(ctx context.Context, ns *corev1.Namespace, sa *corev1.ServiceAccount) (*rbacv1.ClusterRoleBinding, error) { current, err := r.getBrokerFilterRBAC(ctx, ns) // If the resource doesn't exist, we'll create it. @@ -262,33 +263,31 @@ func (r *reconciler) reconcileBrokerFilterRBAC(ctx context.Context, ns *corev1.N return current, nil } -func (r *reconciler) getBrokerFilterRBAC(ctx context.Context, ns *corev1.Namespace) (*rbacv1.RoleBinding, error) { - rb := &rbacv1.RoleBinding{} +func (r *reconciler) getBrokerFilterRBAC(ctx context.Context, ns *corev1.Namespace) (*rbacv1.ClusterRoleBinding, error) { + rb := &rbacv1.ClusterRoleBinding{} name := types.NamespacedName{ - Namespace: ns.Name, - Name: brokerFilterRB, + Name: brokerFilterRB, } err := r.client.Get(ctx, name, rb) return rb, err } -func newBrokerFilterRBAC(ns *corev1.Namespace, sa *corev1.ServiceAccount) *rbacv1.RoleBinding { - return &rbacv1.RoleBinding{ +func newBrokerFilterRBAC(ns *corev1.Namespace, sa *corev1.ServiceAccount) *rbacv1.ClusterRoleBinding { + return &rbacv1.ClusterRoleBinding{ ObjectMeta: metav1.ObjectMeta{ - Namespace: ns.Name, - Name: brokerFilterRB, + Name: brokerFilterRB, Labels: injectedLabels(), }, - RoleRef:rbacv1.RoleRef{ - APIGroup:"rbac.authorization.k8s.io", - Kind: "ClusterRole", - Name: brokerFilterClusterRole, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "ClusterRole", + Name: brokerFilterClusterRole, }, - Subjects:[]rbacv1.Subject{ + Subjects: []rbacv1.Subject{ { - Kind: "ServiceAccount", + Kind: "ServiceAccount", Namespace: ns.Name, - Name: sa.Name, + Name: sa.Name, }, }, } diff --git a/test/crd.go b/test/crd.go index af3ac3971c8..1e7ff0760c6 100644 --- a/test/crd.go +++ b/test/crd.go @@ -18,8 +18,6 @@ package test // crd contains functions that construct boilerplate CRD definitions. import ( - "fmt" - "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" servingv1alpha1 "github.com/knative/serving/pkg/apis/serving/v1alpha1" corev1 "k8s.io/api/core/v1" @@ -190,8 +188,8 @@ func Trigger(name, namespace, eventType, eventSource, broker, svcName string) *v Broker: broker, Filter: &v1alpha1.TriggerFilter{ SourceAndType: &v1alpha1.TriggerFilterSourceAndType{ - Type: fmt.Sprintf("%q", eventType), - Source: fmt.Sprintf("%q", eventSource), + Type: eventType, + Source: eventSource, }, }, Subscriber: &v1alpha1.SubscriberSpec{ From 69ebf1857071dd43a193a2e62fa8f4df9b5ef8f8 Mon Sep 17 00:00:00 2001 From: nachocano Date: Thu, 21 Feb 2019 14:51:27 -0800 Subject: [PATCH 087/128] More logs --- pkg/broker/receiver.go | 1 - test/e2e/broker_trigger_test.go | 14 +++----------- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/pkg/broker/receiver.go b/pkg/broker/receiver.go index 93ada7417d3..14ca97b0d5a 100644 --- a/pkg/broker/receiver.go +++ b/pkg/broker/receiver.go @@ -98,7 +98,6 @@ func (r *Receiver) getTrigger(ctx context.Context, ref provisioners.ChannelRefer } func (r *Receiver) shouldSendMessage(ts *eventingv1alpha1.TriggerSpec, m *provisioners.Message) bool { - r.logger.Debug("Message headers", zap.Any("headers", m.Headers)) if ts.Filter == nil || ts.Filter.SourceAndType == nil { r.logger.Error("No filter specified") return false diff --git a/test/e2e/broker_trigger_test.go b/test/e2e/broker_trigger_test.go index 15244cba36d..49358512eac 100644 --- a/test/e2e/broker_trigger_test.go +++ b/test/e2e/broker_trigger_test.go @@ -39,10 +39,8 @@ const ( any = v1alpha1.TriggerAnyFilter eventType1 = "type1" eventType2 = "type2" - eventType3 = "type3" eventSource1 = "source1" eventSource2 = "source2" - eventSource3 = "source3" ) // Helper function to create names for different objects (e.g., triggers, services, etc.) @@ -165,13 +163,8 @@ func TestDefaultBrokerWithManyTriggers(t *testing.T) { eventsToSend := []test.TypeAndSource{ {eventType1, eventSource1}, {eventType1, eventSource2}, - {eventType1, eventSource3}, {eventType2, eventSource1}, {eventType2, eventSource2}, - {eventType2, eventSource3}, - {eventType3, eventSource1}, - {eventType3, eventSource2}, - {eventType3, eventSource3}, } logger.Info("Creating event sender pods") @@ -183,10 +176,9 @@ func TestDefaultBrokerWithManyTriggers(t *testing.T) { // Using event type and source as part of the body for easier debugging. body := fmt.Sprintf("Body-%s-%s", eventToSend.Type, eventToSend.Source) cloudEvent := test.CloudEvent{ - Source: eventToSend.Source, - Type: eventToSend.Type, - Data: fmt.Sprintf(`{"msg":%q}`, body), - Encoding: test.CloudEventEncodingStructured, + Source: eventToSend.Source, + Type: eventToSend.Type, + Data: fmt.Sprintf(`{"msg":%q}`, body), } // Create sender pod. senderPodName := name("sender", eventToSend.Type, eventToSend.Source) From 1dccad8dc007f143c26c94e4f3101285214a333f Mon Sep 17 00:00:00 2001 From: nachocano Date: Thu, 21 Feb 2019 15:35:57 -0800 Subject: [PATCH 088/128] Adding delay --- test/crd.go | 9 +++++++++ test/e2e/broker_trigger_test.go | 3 ++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/test/crd.go b/test/crd.go index 1e7ff0760c6..e68d26933d8 100644 --- a/test/crd.go +++ b/test/crd.go @@ -226,6 +226,13 @@ const ( // EventSenderPod creates a Pod that sends a single event to the given address. func EventSenderPod(name string, namespace string, sink string, event CloudEvent) *corev1.Pod { + // By setting delay to an invalid integer (i.e., default), + // the event sender pod will end up using its default one. + return EventSenderPodWithDelay(name, namespace, sink, "default", event) +} + +// EventSenderPod creates a Pod that sends a single event to the given address with a given delay. +func EventSenderPodWithDelay(name string, namespace string, sink string, delay string, event CloudEvent) *corev1.Pod { if event.Encoding == "" { event.Encoding = CloudEventEncodingBinary } @@ -254,6 +261,8 @@ func EventSenderPod(name string, namespace string, sink string, event CloudEvent event.Encoding, "-sink", sink, + "-delay", + delay, }, }}, //TODO restart on failure? diff --git a/test/e2e/broker_trigger_test.go b/test/e2e/broker_trigger_test.go index 49358512eac..99f704ee722 100644 --- a/test/e2e/broker_trigger_test.go +++ b/test/e2e/broker_trigger_test.go @@ -34,6 +34,7 @@ import ( const ( defaultBrokerName = "default" waitForDefaultBrokerCreation = 3 * time.Second + sendEventDelay = "15" selectorKey = "end2end-test-broker-trigger" any = v1alpha1.TriggerAnyFilter @@ -182,7 +183,7 @@ func TestDefaultBrokerWithManyTriggers(t *testing.T) { } // Create sender pod. senderPodName := name("sender", eventToSend.Type, eventToSend.Source) - senderPod := test.EventSenderPod(senderPodName, ns, defaultBrokerUrl, cloudEvent) + senderPod := test.EventSenderPodWithDelay(senderPodName, ns, defaultBrokerUrl, sendEventDelay, cloudEvent) if err := CreatePod(clients, senderPod, logger, cleaner); err != nil { t.Fatalf("Error creating event sender pod: %v", err) } From 2b67eb8b2e8df3e33d980741313c1b46fc65957e Mon Sep 17 00:00:00 2001 From: nachocano Date: Thu, 21 Feb 2019 17:30:57 -0800 Subject: [PATCH 089/128] Listing triggers in receiver when we create it, so that we don't miss any message because the client couldn't find the existing trigger. This is a problem in the in-memory-channel as it doesn't do retries. Maybe the right solution is to add that functionality there. --- pkg/broker/receiver.go | 32 ++++++++++++++++++++++++++++++++ test/e2e/broker_trigger_test.go | 29 ++++++++++++++++++----------- 2 files changed, 50 insertions(+), 11 deletions(-) diff --git a/pkg/broker/receiver.go b/pkg/broker/receiver.go index 14ca97b0d5a..5998b63bd40 100644 --- a/pkg/broker/receiver.go +++ b/pkg/broker/receiver.go @@ -23,6 +23,7 @@ import ( eventingv1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" "github.com/knative/eventing/pkg/provisioners" "go.uber.org/zap" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" @@ -48,6 +49,10 @@ func New(logger *zap.Logger, client client.Client) (*Receiver, manager.Runnable) } func (r *Receiver) newMessageReceiver() *provisioners.MessageReceiver { + err := r.initClient() + if err != nil { + r.logger.Warn("Failed to initialize client", zap.Error(err)) + } return provisioners.NewMessageReceiver(r.sendEvent, r.logger.Sugar()) } @@ -82,6 +87,33 @@ func (r *Receiver) sendEvent(channel provisioners.ChannelReference, message *pro return nil } +// Initialize the client. Mainly intended to load stuff in its cache. +func (r *Receiver) initClient() error { + // We list triggers so that we can load the client's cache. + // Otherwise, on receiving an event, it may not find the trigger + // and would return an error. + opts := &client.ListOptions{ + Raw: &metav1.ListOptions{ + TypeMeta: metav1.TypeMeta{ + APIVersion: eventingv1alpha1.SchemeGroupVersion.String(), + Kind: "Trigger", + }, + }, + } + for { + tl := &eventingv1alpha1.TriggerList{} + if err := r.client.List(context.TODO(), opts, tl); err != nil { + return err + } + if tl.Continue != "" { + opts.Raw.Continue = tl.Continue + } else { + break + } + } + return nil +} + func (r *Receiver) getTrigger(ctx context.Context, ref provisioners.ChannelReference) (*eventingv1alpha1.Trigger, error) { // Sadly this doesn't work well because we do not yet have // https://github.com/kubernetes-sigs/controller-runtime/pull/136, so controller runtime watches diff --git a/test/e2e/broker_trigger_test.go b/test/e2e/broker_trigger_test.go index 99f704ee722..695663ae72e 100644 --- a/test/e2e/broker_trigger_test.go +++ b/test/e2e/broker_trigger_test.go @@ -44,12 +44,6 @@ const ( eventSource2 = "source2" ) -// Helper function to create names for different objects (e.g., triggers, services, etc.) -func name(obj, eventType, eventSource string) string { - // pod names need to be lowercase. We might have an eventType as Any, that is why we lowercase them. - return strings.ToLower(fmt.Sprintf("%s-%s-%s", obj, eventType, eventSource)) -} - func TestDefaultBrokerWithManyTriggers(t *testing.T) { logger := logging.GetContextLogger("TestDefaultBrokerWithManyTriggers") @@ -98,15 +92,16 @@ func TestDefaultBrokerWithManyTriggers(t *testing.T) { {eventType1, eventSource1}, } - selector := map[string]string{selectorKey: string(uuid.NewUUID())} + // Create one selector for each subscriberPod and service + selectors := []map[string]string{newSelector(), newSelector(), newSelector(), newSelector()} logger.Info("Creating Subscriber pods") // Save the pods references in this map for later use. subscriberPods := make(map[string]*corev1.Pod, 0) - for _, event := range eventsToReceive { + for i, event := range eventsToReceive { subscriberPodName := name("dumper", event.Type, event.Source) - subscriberPod := test.EventLoggerPod(subscriberPodName, ns, selector) + subscriberPod := test.EventLoggerPod(subscriberPodName, ns, selectors[i]) if err := CreatePod(clients, subscriberPod, logger, cleaner); err != nil { t.Fatalf("Error creating subscriber pod: %v", err) } @@ -126,9 +121,9 @@ func TestDefaultBrokerWithManyTriggers(t *testing.T) { logger.Info("Creating Subscriber services") - for _, event := range eventsToReceive { + for i, event := range eventsToReceive { subscriberSvcName := name("svc", event.Type, event.Source) - service := test.Service(subscriberSvcName, ns, selector) + service := test.Service(subscriberSvcName, ns, selectors[i]) if err := CreateService(clients, service, logger, cleaner); err != nil { t.Fatalf("Error creating subscriber service: %v", err) } @@ -224,6 +219,18 @@ func TestDefaultBrokerWithManyTriggers(t *testing.T) { } +// Helper function to create names for different objects (e.g., triggers, services, etc.) +func name(obj, eventType, eventSource string) string { + // pod names need to be lowercase. We might have an eventType as Any, that is why we lowercase them. + return strings.ToLower(fmt.Sprintf("%s-%s-%s", obj, eventType, eventSource)) +} + +// Returns a new selector with a random uuid. +func newSelector() map[string]string { + return map[string]string{selectorKey: string(uuid.NewUUID())} +} + +// Checks whether we should expect to receive 'eventToSend' based on 'eventToReceive' filter pattern. func shouldExpectEvent(eventToSend *test.TypeAndSource, eventToReceive *test.TypeAndSource, logger *logging.BaseLogger) bool { if eventToReceive.Type != any && eventToReceive.Type != eventToSend.Type { logger.Debugf("Event types mismatch, receive %s, send %s", eventToReceive.Type, eventToSend.Type) From a44d6d28df16cbb110db786fdf975c654fa08eb4 Mon Sep 17 00:00:00 2001 From: nachocano Date: Thu, 21 Feb 2019 17:51:02 -0800 Subject: [PATCH 090/128] Adding delay to sender pod --- test/e2e/broker_trigger_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/e2e/broker_trigger_test.go b/test/e2e/broker_trigger_test.go index 695663ae72e..6fba925fe9a 100644 --- a/test/e2e/broker_trigger_test.go +++ b/test/e2e/broker_trigger_test.go @@ -33,8 +33,8 @@ import ( const ( defaultBrokerName = "default" - waitForDefaultBrokerCreation = 3 * time.Second - sendEventDelay = "15" + waitForDefaultBrokerCreation = 5 * time.Second + senderPodDelay = 15 selectorKey = "end2end-test-broker-trigger" any = v1alpha1.TriggerAnyFilter @@ -178,7 +178,7 @@ func TestDefaultBrokerWithManyTriggers(t *testing.T) { } // Create sender pod. senderPodName := name("sender", eventToSend.Type, eventToSend.Source) - senderPod := test.EventSenderPodWithDelay(senderPodName, ns, defaultBrokerUrl, sendEventDelay, cloudEvent) + senderPod := test.EventSenderPodWithDelay(senderPodName, ns, defaultBrokerUrl, string(senderPodDelay), cloudEvent) if err := CreatePod(clients, senderPod, logger, cleaner); err != nil { t.Fatalf("Error creating event sender pod: %v", err) } From 41ab0a82370117ceac049087a03f1198504edd3a Mon Sep 17 00:00:00 2001 From: nachocano Date: Thu, 21 Feb 2019 18:07:22 -0800 Subject: [PATCH 091/128] Removing withDelay method and just sleep for a while --- test/crd.go | 9 --------- test/e2e/broker_trigger_test.go | 10 ++++++++-- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/test/crd.go b/test/crd.go index e68d26933d8..1e7ff0760c6 100644 --- a/test/crd.go +++ b/test/crd.go @@ -226,13 +226,6 @@ const ( // EventSenderPod creates a Pod that sends a single event to the given address. func EventSenderPod(name string, namespace string, sink string, event CloudEvent) *corev1.Pod { - // By setting delay to an invalid integer (i.e., default), - // the event sender pod will end up using its default one. - return EventSenderPodWithDelay(name, namespace, sink, "default", event) -} - -// EventSenderPod creates a Pod that sends a single event to the given address with a given delay. -func EventSenderPodWithDelay(name string, namespace string, sink string, delay string, event CloudEvent) *corev1.Pod { if event.Encoding == "" { event.Encoding = CloudEventEncodingBinary } @@ -261,8 +254,6 @@ func EventSenderPodWithDelay(name string, namespace string, sink string, delay s event.Encoding, "-sink", sink, - "-delay", - delay, }, }}, //TODO restart on failure? diff --git a/test/e2e/broker_trigger_test.go b/test/e2e/broker_trigger_test.go index 6fba925fe9a..651733defbf 100644 --- a/test/e2e/broker_trigger_test.go +++ b/test/e2e/broker_trigger_test.go @@ -34,7 +34,7 @@ import ( const ( defaultBrokerName = "default" waitForDefaultBrokerCreation = 5 * time.Second - senderPodDelay = 15 + waitForFilterPodRunning = 30 * time.Second selectorKey = "end2end-test-broker-trigger" any = v1alpha1.TriggerAnyFilter @@ -163,6 +163,12 @@ func TestDefaultBrokerWithManyTriggers(t *testing.T) { {eventType2, eventSource2}, } + // We notice few crashLoopBacks in the filter pod creation. + // We then delay the creation of the sender pods to not miss events. + // TODO improve this + logger.Info("Waiting for filter pod up and running") + time.Sleep(waitForFilterPodRunning) + logger.Info("Creating event sender pods") // Map to save the expected events per dumper so that we can verify the delivery. @@ -178,7 +184,7 @@ func TestDefaultBrokerWithManyTriggers(t *testing.T) { } // Create sender pod. senderPodName := name("sender", eventToSend.Type, eventToSend.Source) - senderPod := test.EventSenderPodWithDelay(senderPodName, ns, defaultBrokerUrl, string(senderPodDelay), cloudEvent) + senderPod := test.EventSenderPod(senderPodName, ns, defaultBrokerUrl, cloudEvent) if err := CreatePod(clients, senderPod, logger, cleaner); err != nil { t.Fatalf("Error creating event sender pod: %v", err) } From 6a832eaa28a4d136953399e85abd7d4d8be32705 Mon Sep 17 00:00:00 2001 From: nachocano Date: Thu, 21 Feb 2019 18:13:07 -0800 Subject: [PATCH 092/128] Improve log... --- test/e2e/broker_trigger_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/e2e/broker_trigger_test.go b/test/e2e/broker_trigger_test.go index 651733defbf..d472dd293d3 100644 --- a/test/e2e/broker_trigger_test.go +++ b/test/e2e/broker_trigger_test.go @@ -163,10 +163,10 @@ func TestDefaultBrokerWithManyTriggers(t *testing.T) { {eventType2, eventSource2}, } - // We notice few crashLoopBacks in the filter pod creation. - // We then delay the creation of the sender pods to not miss events. + // We notice some crashLoopBacks in the filter and ingress pod creation. + // We then delay the creation of the sender pods in order not to miss events. // TODO improve this - logger.Info("Waiting for filter pod up and running") + logger.Info("Waiting for filter and ingress pods to become running") time.Sleep(waitForFilterPodRunning) logger.Info("Creating event sender pods") From 446022aa784cd22d69ac910a90aa3ac9d32bb091 Mon Sep 17 00:00:00 2001 From: nachocano Date: Fri, 22 Feb 2019 17:02:52 -0800 Subject: [PATCH 093/128] Updates after code review. --- test/builders.go | 79 +++++++++++++++++++++++ test/crd.go | 26 -------- test/crd_checks.go | 8 ++- test/e2e/broker_trigger_test.go | 108 +++++++++++++++++++------------- test/e2e/e2e.go | 26 ++++++-- 5 files changed, 170 insertions(+), 77 deletions(-) create mode 100644 test/builders.go diff --git a/test/builders.go b/test/builders.go new file mode 100644 index 00000000000..3de022ad65e --- /dev/null +++ b/test/builders.go @@ -0,0 +1,79 @@ +/* +Copyright 2019 The Knative Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package test + +import ( + eventingv1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type TriggerBuilder struct { + *eventingv1alpha1.Trigger +} + +func NewTriggerBuilder(name, namespace string) *TriggerBuilder { + trigger := &eventingv1alpha1.Trigger{ + TypeMeta: metav1.TypeMeta{ + APIVersion: eventingv1alpha1.SchemeGroupVersion.String(), + Kind: "Trigger", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: eventingv1alpha1.TriggerSpec{ + Broker: "default", + Filter: &eventingv1alpha1.TriggerFilter{ + SourceAndType: &eventingv1alpha1.TriggerFilterSourceAndType{ + Type: eventingv1alpha1.TriggerAnyFilter, + Source: eventingv1alpha1.TriggerAnyFilter, + }, + }, + Subscriber: &eventingv1alpha1.SubscriberSpec{}, + }, + } + + return &TriggerBuilder{ + Trigger: trigger, + } +} + +func (b *TriggerBuilder) Build() *eventingv1alpha1.Trigger { + return b.Trigger.DeepCopy() +} + +func (b *TriggerBuilder) EventType(eventType string) *TriggerBuilder { + b.Trigger.Spec.Filter.SourceAndType.Type = eventType + return b +} + +func (b *TriggerBuilder) EventSource(eventType string) *TriggerBuilder { + b.Trigger.Spec.Filter.SourceAndType.Type = eventType + return b +} + +func (b *TriggerBuilder) Broker(brokerName string) *TriggerBuilder { + b.Trigger.Spec.Broker = brokerName + return b +} + +func (b *TriggerBuilder) SubscriberSvc(svcName string) *TriggerBuilder { + b.Trigger.Spec.Subscriber.Ref = &corev1.ObjectReference{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "Service", + Name: svcName, + } + return b +} diff --git a/test/crd.go b/test/crd.go index 1e7ff0760c6..37016def59b 100644 --- a/test/crd.go +++ b/test/crd.go @@ -177,32 +177,6 @@ func Broker(name string, namespace string) *v1alpha1.Broker { } } -// Trigger returns a Trigger. -func Trigger(name, namespace, eventType, eventSource, broker, svcName string) *v1alpha1.Trigger { - return &v1alpha1.Trigger{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - Spec: v1alpha1.TriggerSpec{ - Broker: broker, - Filter: &v1alpha1.TriggerFilter{ - SourceAndType: &v1alpha1.TriggerFilterSourceAndType{ - Type: eventType, - Source: eventSource, - }, - }, - Subscriber: &v1alpha1.SubscriberSpec{ - Ref: &corev1.ObjectReference{ - APIVersion: "v1", - Kind: "Service", - Name: svcName, - }, - }, - }, - } -} - // CloudEvent specifies the arguments for a CloudEvent sent by the sendevent // binary. type CloudEvent struct { diff --git a/test/crd_checks.go b/test/crd_checks.go index faebe200ed4..ab933bd394b 100644 --- a/test/crd_checks.go +++ b/test/crd_checks.go @@ -28,6 +28,7 @@ import ( servingv1alpha1 "github.com/knative/serving/pkg/apis/serving/v1alpha1" servingclient "github.com/knative/serving/pkg/client/clientset/versioned/typed/serving/v1alpha1" "go.opencensus.io/trace" + k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" ) @@ -103,7 +104,12 @@ func WaitForBrokerState(client eventingclient.BrokerInterface, name string, inSt return wait.PollImmediate(interval, timeout, func() (bool, error) { r, err := client.Get(name, metav1.GetOptions{}) - if err != nil { + if k8serrors.IsNotFound(err) { + // Return false as we are not done yet. + // We swallow the error to keep on polling + return false, nil + } else if err != nil { + // Return true to stop and return the error. return true, err } return inState(r) diff --git a/test/e2e/broker_trigger_test.go b/test/e2e/broker_trigger_test.go index d472dd293d3..aa7b674d104 100644 --- a/test/e2e/broker_trigger_test.go +++ b/test/e2e/broker_trigger_test.go @@ -32,10 +32,9 @@ import ( ) const ( - defaultBrokerName = "default" - waitForDefaultBrokerCreation = 5 * time.Second - waitForFilterPodRunning = 30 * time.Second - selectorKey = "end2end-test-broker-trigger" + defaultBrokerName = "default" + waitForFilterPodRunning = 30 * time.Second + selectorKey = "end2end-test-broker-trigger" any = v1alpha1.TriggerAnyFilter eventType1 = "type1" @@ -44,15 +43,27 @@ const ( eventSource2 = "source2" ) +// Helper struct to tie the type and sources of the events we expect to receive +// in subscribers with the selectors we use when creating their pods. +type eventReceiver struct { + typeAndSource test.TypeAndSource + selector map[string]string +} + +// This test annotates the testing namespace so that a default broker is created. +// It then binds many triggers with different filtering patterns to that default broker, +// and sends different events to the broker's address. Finally, it verifies that only +// the appropriate events are routed to the subscribers. func TestDefaultBrokerWithManyTriggers(t *testing.T) { logger := logging.GetContextLogger("TestDefaultBrokerWithManyTriggers") clients, cleaner := Setup(t, logger) - defer TearDown(clients, cleaner, logger) // Verify namespace exists. ns, cleanupNS := NamespaceExists(t, clients, logger) + defer cleanupNS() + defer TearDown(clients, cleaner, logger) logger.Infof("Annotating namespace %s", ns) @@ -64,14 +75,6 @@ func TestDefaultBrokerWithManyTriggers(t *testing.T) { logger.Infof("Namespace %s annotated", ns) - // As we are not creating the default broker, - // we wait for a few seconds for the broker to get created. - // Otherwise, if we try to wait for its Ready status and the namespace controller - // didn't actually create it yet, the test will fail. - // TODO improve this - logger.Info("Waiting for default broker creation") - time.Sleep(waitForDefaultBrokerCreation) - // Wait for default broker ready. logger.Info("Waiting for default broker to be ready") defaultBroker := test.Broker(defaultBrokerName, ns) @@ -84,24 +87,22 @@ func TestDefaultBrokerWithManyTriggers(t *testing.T) { logger.Infof("Default broker ready: %q", defaultBrokerUrl) - // These are the event types and sources that triggers will listen to. - eventsToReceive := []test.TypeAndSource{ - {any, any}, - {eventType1, any}, - {any, eventSource1}, - {eventType1, eventSource1}, + // These are the event types and sources that triggers will listen to, as well as the selectors + // to set in the subscriber and services pods. + eventsToReceive := []eventReceiver{ + {test.TypeAndSource{Type: any, Source: any}, newSelector()}, + {test.TypeAndSource{Type: eventType1, Source: any}, newSelector()}, + {test.TypeAndSource{Type: any, Source: eventSource1}, newSelector()}, + {test.TypeAndSource{Type: eventType1, Source: eventSource1}, newSelector()}, } - // Create one selector for each subscriberPod and service - selectors := []map[string]string{newSelector(), newSelector(), newSelector(), newSelector()} - logger.Info("Creating Subscriber pods") // Save the pods references in this map for later use. - subscriberPods := make(map[string]*corev1.Pod, 0) - for i, event := range eventsToReceive { - subscriberPodName := name("dumper", event.Type, event.Source) - subscriberPod := test.EventLoggerPod(subscriberPodName, ns, selectors[i]) + subscriberPods := make(map[string]*corev1.Pod, len(eventsToReceive)) + for _, event := range eventsToReceive { + subscriberPodName := name("dumper", event.typeAndSource.Type, event.typeAndSource.Source) + subscriberPod := test.EventLoggerPod(subscriberPodName, ns, event.selector) if err := CreatePod(clients, subscriberPod, logger, cleaner); err != nil { t.Fatalf("Error creating subscriber pod: %v", err) } @@ -121,9 +122,9 @@ func TestDefaultBrokerWithManyTriggers(t *testing.T) { logger.Info("Creating Subscriber services") - for i, event := range eventsToReceive { - subscriberSvcName := name("svc", event.Type, event.Source) - service := test.Service(subscriberSvcName, ns, selectors[i]) + for _, event := range eventsToReceive { + subscriberSvcName := name("svc", event.typeAndSource.Type, event.typeAndSource.Source) + service := test.Service(subscriberSvcName, ns, event.selector) if err := CreateService(clients, service, logger, cleaner); err != nil { t.Fatalf("Error creating subscriber service: %v", err) } @@ -134,10 +135,17 @@ func TestDefaultBrokerWithManyTriggers(t *testing.T) { logger.Info("Creating Triggers") for _, event := range eventsToReceive { - triggerName := name("trigger", event.Type, event.Source) + triggerName := name("trigger", event.typeAndSource.Type, event.typeAndSource.Source) // subscriberName should be the same as the subscriberSvc from before. - subscriberName := name("svc", event.Type, event.Source) - trigger := test.Trigger(triggerName, ns, event.Type, event.Source, defaultBrokerName, subscriberName) + subscriberName := name("svc", event.typeAndSource.Type, event.typeAndSource.Source) + trigger := test.NewTriggerBuilder(triggerName, ns). + EventType(event.typeAndSource.Type). + EventSource(event.typeAndSource.Source). + // Don't need to set the broker as we use the default one + // but wanted to be more explicit. + Broker(defaultBrokerName). + SubscriberSvc(subscriberName). + Build() err := CreateTrigger(clients, trigger, logger, cleaner) if err != nil { t.Fatalf("Error creating trigger: %v", err) @@ -173,6 +181,8 @@ func TestDefaultBrokerWithManyTriggers(t *testing.T) { // Map to save the expected events per dumper so that we can verify the delivery. expectedEvents := make(map[string][]string) + // Map to save the unexpected events per dumper so that we can verify that they weren't delivered. + unexpectedEvents := make(map[string][]string) for _, eventToSend := range eventsToSend { // Create cloud event. // Using event type and source as part of the body for easier debugging. @@ -190,11 +200,13 @@ func TestDefaultBrokerWithManyTriggers(t *testing.T) { } // Check on every dumper whether we should expect this event or not, and add its body - // to the expectedEvents map if so. + // to the expectedEvents/unexpectedEvents maps. for _, eventToReceive := range eventsToReceive { + subscriberPodName := name("dumper", eventToReceive.typeAndSource.Type, eventToReceive.typeAndSource.Source) if shouldExpectEvent(&eventToSend, &eventToReceive, logger) { - subscriberPodName := name("dumper", eventToReceive.Type, eventToReceive.Source) expectedEvents[subscriberPodName] = append(expectedEvents[subscriberPodName], body) + } else { + unexpectedEvents[subscriberPodName] = append(unexpectedEvents[subscriberPodName], body) } } } @@ -213,16 +225,22 @@ func TestDefaultBrokerWithManyTriggers(t *testing.T) { logger.Info("Verifying events delivered to appropriate dumpers") for _, event := range eventsToReceive { - subscriberPodName := name("dumper", event.Type, event.Source) + subscriberPodName := name("dumper", event.typeAndSource.Type, event.typeAndSource.Source) subscriberPod := subscriberPods[subscriberPodName] logger.Infof("Dumper %q expecting %q", subscriberPodName, strings.Join(expectedEvents[subscriberPodName], ",")) if err := WaitForLogContents(clients, logger, subscriberPodName, subscriberPod.Spec.Containers[0].Name, ns, expectedEvents[subscriberPodName]); err != nil { - t.Fatalf("String(s) not found in logs of subscriber pod %q: %v", subscriberPodName, err) + t.Fatalf("Event(s) not found in logs of subscriber pod %q: %v", subscriberPodName, err) + } + // At this point all the events should have been received in the pod. + // We check whether we find them unexpected events. If so, then we fail. + found, err := FindAnyLogContents(clients, logger, subscriberPodName, subscriberPod.Spec.Containers[0].Name, ns, unexpectedEvents[subscriberPodName]) + if err != nil { + t.Fatalf("Failed querying to find log contents in pod %q: %v", subscriberPodName, err) + } + if found { + t.Fatalf("Unexpected event(s) found in logs of subscriber pod %q", subscriberPodName) } } - - logger.Info("Verification successful!") - } // Helper function to create names for different objects (e.g., triggers, services, etc.) @@ -236,14 +254,14 @@ func newSelector() map[string]string { return map[string]string{selectorKey: string(uuid.NewUUID())} } -// Checks whether we should expect to receive 'eventToSend' based on 'eventToReceive' filter pattern. -func shouldExpectEvent(eventToSend *test.TypeAndSource, eventToReceive *test.TypeAndSource, logger *logging.BaseLogger) bool { - if eventToReceive.Type != any && eventToReceive.Type != eventToSend.Type { - logger.Debugf("Event types mismatch, receive %s, send %s", eventToReceive.Type, eventToSend.Type) +// Checks whether we should expect to receive 'eventToSend' in 'eventReceiver' based on its type and source pattern. +func shouldExpectEvent(eventToSend *test.TypeAndSource, receiver *eventReceiver, logger *logging.BaseLogger) bool { + if receiver.typeAndSource.Type != any && receiver.typeAndSource.Type != eventToSend.Type { + logger.Debugf("Event types mismatch, receive %s, send %s", receiver.typeAndSource.Type, eventToSend.Type) return false } - if eventToReceive.Source != any && eventToReceive.Source != eventToSend.Source { - logger.Debugf("Event sources mismatch, receive %s, send %s", eventToReceive.Source, eventToSend.Source) + if receiver.typeAndSource.Source != any && receiver.typeAndSource.Source != eventToSend.Source { + logger.Debugf("Event sources mismatch, receive %s, send %s", receiver.typeAndSource.Source, eventToSend.Source) return false } return true diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go index 8aac5228087..f81d7452812 100644 --- a/test/e2e/e2e.go +++ b/test/e2e/e2e.go @@ -161,7 +161,7 @@ func WithChannelAndSubscriptionReady(clients *test.Clients, channel *v1alpha1.Ch return nil } -// CreateBroker will create a Broker +// CreateBroker will create a Broker. func CreateBroker(clients *test.Clients, broker *v1alpha1.Broker, logger *logging.BaseLogger, cleaner *test.Cleaner) error { brokers := clients.Eventing.EventingV1alpha1().Brokers(broker.Namespace) res, err := brokers.Create(broker) @@ -186,7 +186,7 @@ func WaitForBrokerReady(clients *test.Clients, broker *v1alpha1.Broker) error { if err := test.WaitForBrokerState(brokers, broker.Name, test.IsBrokerReady, "BrokerIsReady"); err != nil { return err } - // Update the given object so they'll reflect the ready state + // Update the given object so they'll reflect the ready state. updatedBroker, err := brokers.Get(broker.Name, metav1.GetOptions{}) if err != nil { return err @@ -195,7 +195,7 @@ func WaitForBrokerReady(clients *test.Clients, broker *v1alpha1.Broker) error { return nil } -// CreateTrigger will create a Trigger +// CreateTrigger will create a Trigger. func CreateTrigger(clients *test.Clients, trigger *v1alpha1.Trigger, logger *logging.BaseLogger, cleaner *test.Cleaner) error { triggers := clients.Eventing.EventingV1alpha1().Triggers(trigger.Namespace) res, err := triggers.Create(trigger) @@ -216,7 +216,7 @@ func WithTriggerReady(clients *test.Clients, trigger *v1alpha1.Trigger, logger * if err := test.WaitForTriggerState(triggers, trigger.Name, test.IsTriggerReady, "TriggerIsReady"); err != nil { return err } - // Update the given object so they'll reflect the ready state + // Update the given object so they'll reflect the ready state. updatedTrigger, err := triggers.Get(trigger.Name, metav1.GetOptions{}) if err != nil { return err @@ -321,7 +321,7 @@ func PodLogs(clients *test.Clients, podName string, containerName string, namesp }).Do() raw, err := result.Raw() if err == nil { - logger.Debugf("%s logs request result: %#v", podName, string(raw)) + logger.Infof("%s logs request result: %#v", podName, string(raw)) } else { logger.Infof("%s logs request result: %#v", podName, err) } @@ -352,6 +352,22 @@ func WaitForLogContents(clients *test.Clients, logger *logging.BaseLogger, podNa }) } +// FindAnyLogContents attempts to find logs for given Pod/Container that has 'any' of the given contents. +// It returns an error if it couldn't retrieve the logs. In case 'any' of the contents are there, it returns true. +func FindAnyLogContents(clients *test.Clients, logger *logging.BaseLogger, podName string, containerName string, namespace string, contents []string) (bool, error) { + logs, err := PodLogs(clients, podName, containerName, namespace, logger) + if err != nil { + return false, err + } + for _, content := range contents { + if strings.Contains(string(logs), content) { + logger.Infof("Found content %q for %s/%s.", content, podName, containerName) + return true, nil + } + } + return false, nil +} + // WaitForLogContent waits until logs for given Pod/Container include the given content. // If the content is not present within timeout it returns error. func WaitForLogContent(clients *test.Clients, logger *logging.BaseLogger, podName string, containerName string, namespace string, content string) error { From 7f24f14770e27c3d90e5a179971fde1f7f2a996a Mon Sep 17 00:00:00 2001 From: Nacho Cano Date: Fri, 22 Feb 2019 20:14:11 -0800 Subject: [PATCH 094/128] Adding some more logs and trailing dots. --- test/builders.go | 1 + test/e2e/broker_trigger_test.go | 6 +++--- test/e2e/e2e.go | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/test/builders.go b/test/builders.go index 3de022ad65e..c73822bad4d 100644 --- a/test/builders.go +++ b/test/builders.go @@ -19,6 +19,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +// Builder for trigger objects. type TriggerBuilder struct { *eventingv1alpha1.Trigger } diff --git a/test/e2e/broker_trigger_test.go b/test/e2e/broker_trigger_test.go index aa7b674d104..b478c673381 100644 --- a/test/e2e/broker_trigger_test.go +++ b/test/e2e/broker_trigger_test.go @@ -232,7 +232,7 @@ func TestDefaultBrokerWithManyTriggers(t *testing.T) { t.Fatalf("Event(s) not found in logs of subscriber pod %q: %v", subscriberPodName, err) } // At this point all the events should have been received in the pod. - // We check whether we find them unexpected events. If so, then we fail. + // We check whether we find unexpected events. If so, then we fail. found, err := FindAnyLogContents(clients, logger, subscriberPodName, subscriberPod.Spec.Containers[0].Name, ns, unexpectedEvents[subscriberPodName]) if err != nil { t.Fatalf("Failed querying to find log contents in pod %q: %v", subscriberPodName, err) @@ -243,9 +243,9 @@ func TestDefaultBrokerWithManyTriggers(t *testing.T) { } } -// Helper function to create names for different objects (e.g., triggers, services, etc.) +// Helper function to create names for different objects (e.g., triggers, services, etc.). func name(obj, eventType, eventSource string) string { - // pod names need to be lowercase. We might have an eventType as Any, that is why we lowercase them. + // Pod names need to be lowercase. We might have an eventType as Any, that is why we lowercase them. return strings.ToLower(fmt.Sprintf("%s-%s-%s", obj, eventType, eventSource)) } diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go index f81d7452812..98079dfe015 100644 --- a/test/e2e/e2e.go +++ b/test/e2e/e2e.go @@ -382,7 +382,7 @@ func WaitForAllPodsRunning(clients *test.Clients, logger *logging.BaseLogger, na return nil } -// WaitForAllTriggersReady will wait until all triggers in the given namespace are ready +// WaitForAllTriggersReady will wait until all triggers in the given namespace are ready. func WaitForAllTriggersReady(clients *test.Clients, logger *logging.BaseLogger, namespace string) error { triggers := clients.Eventing.EventingV1alpha1().Triggers(namespace) if err := test.WaitForTriggersListState(triggers, test.TriggersReady, "TriggerIsReady"); err != nil { @@ -391,7 +391,7 @@ func WaitForAllTriggersReady(clients *test.Clients, logger *logging.BaseLogger, return nil } -// AnnotateNamespace annotates the test namespace with the annotations map +// AnnotateNamespace annotates the test namespace with the annotations map. func AnnotateNamespace(clients *test.Clients, logger *logging.BaseLogger, annotations map[string]string) error { ns := pkgTest.Flags.Namespace nsSpec, err := clients.Kube.Kube.CoreV1().Namespaces().Get(ns, metav1.GetOptions{}) From 9debaf250b6c488b6ed49e1fd5b38ebeeb96bb79 Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Mon, 25 Feb 2019 11:42:32 -0800 Subject: [PATCH 095/128] Switch import order. --- cmd/controller/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/controller/main.go b/cmd/controller/main.go index 2619ec145aa..382e79d38a7 100644 --- a/cmd/controller/main.go +++ b/cmd/controller/main.go @@ -24,8 +24,8 @@ import ( "os" "time" - "github.com/knative/eventing/pkg/reconciler/v1alpha1/namespace" "github.com/knative/eventing/pkg/reconciler/v1alpha1/broker" + "github.com/knative/eventing/pkg/reconciler/v1alpha1/namespace" "github.com/knative/eventing/pkg/reconciler/v1alpha1/subscription" "github.com/knative/eventing/pkg/reconciler/v1alpha1/trigger" "k8s.io/apimachinery/pkg/runtime" From a83ca8223dca0a36b42ba9bb4190b9f5d92a6c7b Mon Sep 17 00:00:00 2001 From: nachocano Date: Mon, 25 Feb 2019 12:06:45 -0800 Subject: [PATCH 096/128] Updating comments. --- cmd/broker/filter/main.go | 6 +- cmd/broker/ingress/main.go | 10 +-- pkg/broker/receiver.go | 6 +- pkg/reconciler/v1alpha1/broker/broker.go | 26 +++++--- .../v1alpha1/namespace/namespace.go | 62 ++++++++++++------- pkg/reconciler/v1alpha1/trigger/trigger.go | 24 +++++-- 6 files changed, 87 insertions(+), 47 deletions(-) diff --git a/cmd/broker/filter/main.go b/cmd/broker/filter/main.go index 0c39dec0550..9e3b2d5fdb5 100644 --- a/cmd/broker/filter/main.go +++ b/cmd/broker/filter/main.go @@ -47,10 +47,8 @@ func main() { // Add custom types to this array to get them into the manager's scheme. eventingv1alpha1.AddToScheme(mgr.GetScheme()) - // We are running both the receiver (takes messages in from the cluster and writes them to - // PubSub) and the dispatcher (takes messages in PubSub and sends them in cluster) in this - // binary. - + // We are running both the receiver (takes messages in from the cluster) and the dispatcher (send the messages + // to the triggers' subscribers) in this binary. _, runnable := broker.New(logger, mgr.GetClient()) err = mgr.Add(runnable) if err != nil { diff --git a/cmd/broker/ingress/main.go b/cmd/broker/ingress/main.go index 1be24c382bb..3db11df68a9 100644 --- a/cmd/broker/ingress/main.go +++ b/cmd/broker/ingress/main.go @@ -74,7 +74,7 @@ func main() { // Start both the manager (which notices ConfigMap changes) and the HTTP server. var g errgroup.Group g.Go(func() error { - // set up signals so we handle the first shutdown signal gracefully + // Set up signals so we handle the first shutdown signal gracefully. stopCh := signals.SetupSignalHandler() // Start blocks forever, so run it in a goroutine. return mgr.Start(stopCh) @@ -99,7 +99,7 @@ func getRequiredEnv(envKey string) string { return val } -// http.Handler that takes a single request in and fans it out to N other servers. +// http.Handler that takes a single request in and sends it out to a single destination. type Handler struct { receiver *provisioners.MessageReceiver dispatcher *provisioners.MessageDispatcher @@ -108,7 +108,7 @@ type Handler struct { logger *zap.Logger } -// NewHandler creates a new fanout.Handler. +// NewHandler creates a new ingress.Handler. func NewHandler(logger *zap.Logger, destination string) *Handler { handler := &Handler{ logger: logger, @@ -133,8 +133,8 @@ func (f *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { f.receiver.HandleRequest(w, r) } -// dispatch takes the request, fans it out to each subscription in f.config. If all the fanned out -// requests return successfully, then return nil. Else, return an error. +// dispatch takes the request, and sends it out the f.destination. If the dispatched +// request returns successfully, then return nil. Else, return an error. func (f *Handler) dispatch(msg *provisioners.Message) error { err := f.dispatcher.DispatchMessage(msg, f.destination, "", provisioners.DispatchDefaults{}) if err != nil { diff --git a/pkg/broker/receiver.go b/pkg/broker/receiver.go index 9dd264337ee..1f294d7c942 100644 --- a/pkg/broker/receiver.go +++ b/pkg/broker/receiver.go @@ -28,7 +28,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" ) -// Receiver parses Cloud Events and sends them to the channel. +// Receiver parses Cloud Events and sends them to the subscriber(s). type Receiver struct { logger *zap.Logger client client.Client @@ -51,7 +51,7 @@ func (r *Receiver) newMessageReceiver() *provisioners.MessageReceiver { return provisioners.NewMessageReceiver(r.sendEvent, r.logger.Sugar()) } -// sendEvent sends a message to the Channel. +// sendEvent sends an event to a subscriber if the trigger filter passes. func (r *Receiver) sendEvent(channel provisioners.ChannelReference, message *provisioners.Message) error { r.logger.Debug("received message") ctx := context.Background() @@ -97,6 +97,8 @@ func (r *Receiver) getTrigger(ctx context.Context, ref provisioners.ChannelRefer return t, err } +// shouldSendMessage determines whether message 'm' should be sent based on the triggerSpec 'ts'. +// Currently it supports exact matching on type and/or source of events. func (r *Receiver) shouldSendMessage(ts *eventingv1alpha1.TriggerSpec, m *provisioners.Message) bool { if ts.Filter == nil || ts.Filter.SourceAndType == nil { r.logger.Error("No filter specified") diff --git a/pkg/reconciler/v1alpha1/broker/broker.go b/pkg/reconciler/v1alpha1/broker/broker.go index 1e4c52b2735..f970acd9dc4 100644 --- a/pkg/reconciler/v1alpha1/broker/broker.go +++ b/pkg/reconciler/v1alpha1/broker/broker.go @@ -19,9 +19,10 @@ package broker import ( "context" "fmt" - "k8s.io/apimachinery/pkg/runtime" "time" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/dynamic" "k8s.io/client-go/rest" "k8s.io/client-go/tools/record" @@ -60,7 +61,7 @@ const ( // itself when creating events. controllerAgentName = "broker-controller" - // Name of the corev1.Events emitted from the reconciliation process + // Name of the corev1.Events emitted from the reconciliation process. brokerReconciled = "BrokerReconciled" brokerUpdateStatusFailed = "BrokerUpdateStatusFailed" ) @@ -79,7 +80,7 @@ type reconciler struct { filterServiceAccountName string } -// Verify the struct implements reconcile.Reconciler +// Verify the struct implements reconcile.Reconciler. var _ reconcile.Reconciler = &reconciler{} // ProvideController returns a function that returns a Broker controller. @@ -155,7 +156,7 @@ func (r *reconciler) Reconcile(request reconcile.Request) (reconcile.Result, err result, reconcileErr := r.reconcile(ctx, broker) if reconcileErr != nil { logging.FromContext(ctx).Error("Error reconciling Broker", zap.Error(reconcileErr)) - } else if result.Requeue || result.RequeueAfter > 0 { + } else if result.Requeue || result.RequeueAfter > 0 { logging.FromContext(ctx).Debug("Broker reconcile requeuing") } else { logging.FromContext(ctx).Debug("Broker reconciled") @@ -178,7 +179,7 @@ func (r *reconciler) reconcile(ctx context.Context, b *v1alpha1.Broker) (reconci // 1. Channel is created for all events. // 2. Filter Deployment. // 3. Ingress Deployment. - // 4. K8s Service that points at the Deployment. + // 4. K8s Services that point at the Deployments. if b.DeletionTimestamp != nil { // Everything is cleaned up by the garbage collector. @@ -217,7 +218,7 @@ func (r *reconciler) reconcile(ctx context.Context, b *v1alpha1.Broker) (reconci svc, err := r.reconcileIngressService(ctx, b) if err != nil { - logging.FromContext(ctx).Error("Problem reconciling ingress Service", zap.Error(err)) + logging.FromContext(ctx).Error("Problem reconciling ingress service", zap.Error(err)) return reconcile.Result{}, err } b.Status.MarkIngressReady() @@ -226,7 +227,7 @@ func (r *reconciler) reconcile(ctx context.Context, b *v1alpha1.Broker) (reconci return reconcile.Result{}, nil } -// updateStatus may in fact update the broker's finalizers in addition to the status +// updateStatus may in fact update the broker's finalizers in addition to the status. func (r *reconciler) updateStatus(broker *v1alpha1.Broker) (*v1alpha1.Broker, error) { objectKey := client.ObjectKey{Namespace: broker.Namespace, Name: broker.Name} latestBroker := &v1alpha1.Broker{} @@ -250,7 +251,7 @@ func (r *reconciler) updateStatus(broker *v1alpha1.Broker) (*v1alpha1.Broker, er } if brokerChanged { - // Refetch + // Re-fetch. latestBroker = &v1alpha1.Broker{} if err := r.client.Get(context.TODO(), objectKey, latestBroker); err != nil { return nil, err @@ -265,6 +266,7 @@ func (r *reconciler) updateStatus(broker *v1alpha1.Broker) (*v1alpha1.Broker, er return latestBroker, nil } +// reconcileFilterDeployment reconciles Broker's 'b' filter deployment. func (r *reconciler) reconcileFilterDeployment(ctx context.Context, b *v1alpha1.Broker) (*v1.Deployment, error) { expected := resources.MakeFilterDeployment(&resources.FilterArgs{ Broker: b, @@ -274,11 +276,13 @@ func (r *reconciler) reconcileFilterDeployment(ctx context.Context, b *v1alpha1. return r.reconcileDeployment(ctx, expected) } +// reconcileFilterService reconciles Broker's 'b' filter service. func (r *reconciler) reconcileFilterService(ctx context.Context, b *v1alpha1.Broker) (*corev1.Service, error) { expected := resources.MakeFilterService(b) return r.reconcileService(ctx, expected) } +// reconcileChannel reconciles Broker's 'b' underlying channel. func (r *reconciler) reconcileChannel(ctx context.Context, b *v1alpha1.Broker) (*v1alpha1.Channel, error) { c, err := r.getChannel(ctx, b) // If the resource doesn't exist, we'll create it @@ -307,6 +311,7 @@ func (r *reconciler) reconcileChannel(ctx context.Context, b *v1alpha1.Broker) ( return c, nil } +// getChannel returns the Channel object for Broker 'b' if exists, otherwise it returns an error. func (r *reconciler) getChannel(ctx context.Context, b *v1alpha1.Broker) (*v1alpha1.Channel, error) { list := &v1alpha1.ChannelList{} opts := &runtimeclient.ListOptions{ @@ -335,6 +340,7 @@ func (r *reconciler) getChannel(ctx context.Context, b *v1alpha1.Broker) (*v1alp return nil, k8serrors.NewNotFound(schema.GroupResource{}, "") } +// newChannel creates a new placeholder Channel object for Broker 'b'. func newChannel(b *v1alpha1.Broker) *v1alpha1.Channel { var spec v1alpha1.ChannelSpec if b.Spec.ChannelTemplate != nil { @@ -365,6 +371,7 @@ func ChannelLabels(b *v1alpha1.Broker) map[string]string { } } +// reconcileDeployment reconciles the K8s Deployment 'd'. func (r *reconciler) reconcileDeployment(ctx context.Context, d *v1.Deployment) (*v1.Deployment, error) { name := types.NamespacedName{ Namespace: d.Namespace, @@ -392,6 +399,7 @@ func (r *reconciler) reconcileDeployment(ctx context.Context, d *v1.Deployment) return current, nil } +// reconcileService reconciles the K8s Service 'svc'. func (r *reconciler) reconcileService(ctx context.Context, svc *corev1.Service) (*corev1.Service, error) { name := types.NamespacedName{ Namespace: svc.Namespace, @@ -422,6 +430,7 @@ func (r *reconciler) reconcileService(ctx context.Context, svc *corev1.Service) return current, nil } +// reconcileIngressDeployment reconciles the Ingress Deployment. func (r *reconciler) reconcileIngressDeployment(ctx context.Context, b *v1alpha1.Broker, c *v1alpha1.Channel) (*v1.Deployment, error) { expected := resources.MakeIngress(&resources.IngressArgs{ Broker: b, @@ -432,6 +441,7 @@ func (r *reconciler) reconcileIngressDeployment(ctx context.Context, b *v1alpha1 return r.reconcileDeployment(ctx, expected) } +// reconcileIngressService reconciles the Ingress Service. func (r *reconciler) reconcileIngressService(ctx context.Context, b *v1alpha1.Broker) (*corev1.Service, error) { expected := resources.MakeIngressService(b) return r.reconcileService(ctx, expected) diff --git a/pkg/reconciler/v1alpha1/namespace/namespace.go b/pkg/reconciler/v1alpha1/namespace/namespace.go index 483d659604d..58b1bcd045e 100644 --- a/pkg/reconciler/v1alpha1/namespace/namespace.go +++ b/pkg/reconciler/v1alpha1/namespace/namespace.go @@ -19,11 +19,12 @@ package namespace import ( "context" "fmt" + "github.com/knative/eventing/contrib/gcppubsub/pkg/util/logging" "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" "go.uber.org/zap" - "k8s.io/api/core/v1" corev1 "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/api/errors" k8serrors "k8s.io/apimachinery/pkg/api/errors" @@ -46,16 +47,17 @@ const ( // itself when creating events. controllerAgentName = "knative-eventing-namespace-controller" - defaultBroker = "default" - brokerFilterSA = "eventing-broker-filter" - brokerFilterRB = "eventing-broker-filter" + defaultBroker = "default" + brokerFilterSA = "eventing-broker-filter" + brokerFilterRB = "eventing-broker-filter" brokerFilterClusterRole = "eventing-broker-filter" + // Annotation to enable knative-eventing in a namespace. knativeEventingAnnotation = "eventing.knative.dev/inject" // Name of the corev1.Events emitted from the reconciliation process. - brokerCreated = "BrokerCreated" - serviceAccountCreated = "BrokerFilterServiceAccountCreated" + brokerCreated = "BrokerCreated" + serviceAccountCreated = "BrokerFilterServiceAccountCreated" serviceAccountRBACCreated = "BrokerFilterServiceAccountRBACCreated" ) @@ -71,10 +73,10 @@ type reconciler struct { // Verify the struct implements reconcile.Reconciler var _ reconcile.Reconciler = &reconciler{} -// ProvideController returns a function that returns a Broker controller. +// ProvideController returns a function that returns a Namespace controller. func ProvideController(logger *zap.Logger) func(manager.Manager) (controller.Controller, error) { return func(mgr manager.Manager) (controller.Controller, error) { - // Setup a new controller to Reconcile Brokers. + // Setup a new controller to Reconcile Namespaces. r := &reconciler{ recorder: mgr.GetRecorder(controllerAgentName), logger: logger, @@ -92,8 +94,8 @@ func ProvideController(logger *zap.Logger) func(manager.Manager) (controller.Con } // Watch all the resources that this reconciler reconciles. - for _, t := range []runtime.Object{ &corev1.ServiceAccount{}, &rbacv1.RoleBinding{}, &v1alpha1.Broker{} } { - err = c.Watch(&source.Kind{Type: t}, &handler.EnqueueRequestsFromMapFunc{ToRequests: &namespaceMapper{}}); + for _, t := range []runtime.Object{&corev1.ServiceAccount{}, &rbacv1.RoleBinding{}, &v1alpha1.Broker{}} { + err = c.Watch(&source.Kind{Type: t}, &handler.EnqueueRequestsFromMapFunc{ToRequests: &namespaceMapper{}}) if err != nil { return nil, err } @@ -131,7 +133,7 @@ func (r *reconciler) InjectConfig(c *rest.Config) error { } // Reconcile compares the actual state with the desired, and attempts to -// converge the two. It then updates the Status block of the Trigger resource +// converge the two. It then updates the Status block of the Namespace resource // with the current status of the resource. func (r *reconciler) Reconcile(request reconcile.Request) (reconcile.Result, error) { ctx := context.TODO() @@ -155,7 +157,7 @@ func (r *reconciler) Reconcile(request reconcile.Request) (reconcile.Result, err return reconcile.Result{}, nil } - // Reconcile this copy of the Trigger and then write back any status updates regardless of + // Reconcile this copy of the Namespace and then write back any status updates regardless of // whether the reconcile error out. reconcileErr := r.reconcile(ctx, ns) if reconcileErr != nil { @@ -185,13 +187,14 @@ func (r *reconciler) reconcile(ctx context.Context, ns *corev1.Namespace) error } _, err = r.reconcileBroker(ctx, ns) if err != nil { - logging.FromContext(ctx).Error("Unable to reconcile broker for the namespace", zap.Error(err)) + logging.FromContext(ctx).Error("Unable to reconcile Broker for the namespace", zap.Error(err)) return err } return nil } +// reconcileBrokerFilterServiceAccount reconciles the Broker's filter service account for Namespace 'ns'. func (r *reconciler) reconcileBrokerFilterServiceAccount(ctx context.Context, ns *corev1.Namespace) (*corev1.ServiceAccount, error) { current, err := r.getBrokerFilterServiceAccount(ctx, ns) @@ -214,6 +217,8 @@ func (r *reconciler) reconcileBrokerFilterServiceAccount(ctx context.Context, ns return current, nil } +// getBrokerFilterServiceAccount returns the Broker's filter service account for Namespace 'ns' if exists, +// otherwise it returns an error. func (r *reconciler) getBrokerFilterServiceAccount(ctx context.Context, ns *corev1.Namespace) (*corev1.ServiceAccount, error) { sa := &corev1.ServiceAccount{} name := types.NamespacedName{ @@ -224,12 +229,13 @@ func (r *reconciler) getBrokerFilterServiceAccount(ctx context.Context, ns *core return sa, err } +// newBrokerFilterServiceAccount creates a ServiceAccount object for the Namespace 'ns'. func newBrokerFilterServiceAccount(ns *corev1.Namespace) *corev1.ServiceAccount { return &corev1.ServiceAccount{ ObjectMeta: metav1.ObjectMeta{ Namespace: ns.Name, - Name: brokerFilterSA, - Labels: injectedLabels(), + Name: brokerFilterSA, + Labels: injectedLabels(), }, } } @@ -240,6 +246,7 @@ func injectedLabels() map[string]string { } } +// reconcileBrokerFilterRBAC reconciles the Broker's filter service account RBAC for the Namespace 'ns'. func (r *reconciler) reconcileBrokerFilterRBAC(ctx context.Context, ns *corev1.Namespace, sa *corev1.ServiceAccount) (*rbacv1.RoleBinding, error) { current, err := r.getBrokerFilterRBAC(ctx, ns) @@ -262,6 +269,8 @@ func (r *reconciler) reconcileBrokerFilterRBAC(ctx context.Context, ns *corev1.N return current, nil } +// getBrokerFilterRBAC returns the Broker's filter role binding for Namespace 'ns' if exists, +// otherwise it returns an error. func (r *reconciler) getBrokerFilterRBAC(ctx context.Context, ns *corev1.Namespace) (*rbacv1.RoleBinding, error) { rb := &rbacv1.RoleBinding{} name := types.NamespacedName{ @@ -272,28 +281,31 @@ func (r *reconciler) getBrokerFilterRBAC(ctx context.Context, ns *corev1.Namespa return rb, err } +// newBrokerFilterRBAC creates a RpleBinding object for the Broker's filter service account 'sa' in the Namespace 'ns'. func newBrokerFilterRBAC(ns *corev1.Namespace, sa *corev1.ServiceAccount) *rbacv1.RoleBinding { return &rbacv1.RoleBinding{ ObjectMeta: metav1.ObjectMeta{ Namespace: ns.Name, - Name: brokerFilterRB, - Labels: injectedLabels(), + Name: brokerFilterRB, + Labels: injectedLabels(), }, - RoleRef:rbacv1.RoleRef{ - APIGroup:"rbac.authorization.k8s.io", - Kind: "ClusterRole", - Name: brokerFilterClusterRole, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "ClusterRole", + Name: brokerFilterClusterRole, }, - Subjects:[]rbacv1.Subject{ + Subjects: []rbacv1.Subject{ { - Kind: "ServiceAccount", + Kind: "ServiceAccount", Namespace: ns.Name, - Name: sa.Name, + Name: sa.Name, }, }, } } +// getBroker returns the default broker for Namespace 'ns' if exists, +// otherwise it returns an error. func (r *reconciler) getBroker(ctx context.Context, ns *corev1.Namespace) (*v1alpha1.Broker, error) { b := &v1alpha1.Broker{} name := types.NamespacedName{ @@ -304,6 +316,7 @@ func (r *reconciler) getBroker(ctx context.Context, ns *corev1.Namespace) (*v1al return b, err } +// reconcileBroker reconciles the default Broker for the Namespace 'ns'. func (r *reconciler) reconcileBroker(ctx context.Context, ns *corev1.Namespace) (*v1alpha1.Broker, error) { current, err := r.getBroker(ctx, ns) @@ -323,6 +336,7 @@ func (r *reconciler) reconcileBroker(ctx context.Context, ns *corev1.Namespace) return current, nil } +// newBroker creates a placeholder default Broker object for Namespace 'ns'. func newBroker(ns *corev1.Namespace) *v1alpha1.Broker { return &v1alpha1.Broker{ ObjectMeta: metav1.ObjectMeta{ diff --git a/pkg/reconciler/v1alpha1/trigger/trigger.go b/pkg/reconciler/v1alpha1/trigger/trigger.go index 42404fed7fa..b9e6e8b2341 100644 --- a/pkg/reconciler/v1alpha1/trigger/trigger.go +++ b/pkg/reconciler/v1alpha1/trigger/trigger.go @@ -62,7 +62,7 @@ const ( finalizerName = controllerAgentName - // Name of the corev1.Events emitted from the reconciliation process + // Name of the corev1.Events emitted from the reconciliation process. triggerReconciled = "TriggerReconciled" triggerReconcileFailed = "TriggerReconcileFailed" triggerUpdateStatusFailed = "TriggerUpdateStatusFailed" @@ -87,7 +87,7 @@ type reconciler struct { logger *zap.Logger } -// Verify the struct implements reconcile.Reconciler +// Verify the struct implements reconcile.Reconciler. var _ reconcile.Reconciler = &reconciler{} // ProvideController returns a function that returns a Trigger controller. @@ -369,6 +369,8 @@ func (r *reconciler) updateStatus(trigger *v1alpha1.Trigger) (*v1alpha1.Trigger, return latestTrigger, nil } +// getBroker returns the Broker for Trigger 't' if exists, +// otherwise it returns an error. func (r *reconciler) getBroker(ctx context.Context, t *v1alpha1.Trigger) (*v1alpha1.Broker, error) { b := &v1alpha1.Broker{} name := types.NamespacedName{ @@ -379,6 +381,8 @@ func (r *reconciler) getBroker(ctx context.Context, t *v1alpha1.Trigger) (*v1alp return b, err } +// getBrokerChannel returns the Broker's channel if exists, +// otherwise it returns an error. func (r *reconciler) getBrokerChannel(ctx context.Context, b *v1alpha1.Broker) (*v1alpha1.Channel, error) { list := &v1alpha1.ChannelList{} opts := &runtimeclient.ListOptions{ @@ -407,6 +411,8 @@ func (r *reconciler) getBrokerChannel(ctx context.Context, b *v1alpha1.Broker) ( return nil, k8serrors.NewNotFound(schema.GroupResource{}, "") } +// getK8sService returns the K8s service for trigger 't' if exists, +// otherwise it returns an error. func (r *reconciler) getK8sService(ctx context.Context, t *v1alpha1.Trigger) (*corev1.Service, error) { list := &corev1.ServiceList{} opts := &runtimeclient.ListOptions{ @@ -435,6 +441,7 @@ func (r *reconciler) getK8sService(ctx context.Context, t *v1alpha1.Trigger) (*c return nil, k8serrors.NewNotFound(schema.GroupResource{}, "") } +// reconcileK8sService reconciles the K8s service for trigger 't'. func (r *reconciler) reconcileK8sService(ctx context.Context, t *v1alpha1.Trigger) (*corev1.Service, error) { current, err := r.getK8sService(ctx, t) @@ -464,6 +471,7 @@ func (r *reconciler) reconcileK8sService(ctx context.Context, t *v1alpha1.Trigge return current, nil } +// newK8sService returns a K8s placeholder service for trigger 't'. func newK8sService(t *v1alpha1.Trigger) *corev1.Service { return &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -495,6 +503,8 @@ func k8sServiceLabels(t *v1alpha1.Trigger) map[string]string { } } +// getVirtualService returns the virtual service for trigger 't' if exists, +// otherwise it returns an error. func (r *reconciler) getVirtualService(ctx context.Context, t *v1alpha1.Trigger) (*istiov1alpha3.VirtualService, error) { list := &istiov1alpha3.VirtualServiceList{} opts := &runtimeclient.ListOptions{ @@ -523,6 +533,7 @@ func (r *reconciler) getVirtualService(ctx context.Context, t *v1alpha1.Trigger) return nil, k8serrors.NewNotFound(schema.GroupResource{}, "") } +// reconcileVirtualService reconciles the virtual service for trigger 't' and service 'svc'. func (r *reconciler) reconcileVirtualService(ctx context.Context, t *v1alpha1.Trigger, svc *corev1.Service) (*istiov1alpha3.VirtualService, error) { virtualService, err := r.getVirtualService(ctx, t) @@ -549,6 +560,7 @@ func (r *reconciler) reconcileVirtualService(ctx context.Context, t *v1alpha1.Tr return virtualService, nil } +// newVirtualService returns a placeholder virtual service object for trigger 't' and service 'svc'. func newVirtualService(t *v1alpha1.Trigger, svc *corev1.Service) *istiov1alpha3.VirtualService { // TODO Make this work with endings other than cluster.local destinationHost := fmt.Sprintf("%s-broker-filter.%s.svc.cluster.local", t.Spec.Broker, t.Namespace) @@ -593,10 +605,11 @@ func virtualServiceLabels(t *v1alpha1.Trigger) map[string]string { } } +// subscribeToBrokerChannel subscribes service 'svc' to Broker's channel 'c'. func (r *reconciler) subscribeToBrokerChannel(ctx context.Context, t *v1alpha1.Trigger, c *v1alpha1.Channel, svc *corev1.Service) (*v1alpha1.Subscription, error) { expected := makeSubscription(t, c, svc) - sub, err := r.getSubscription(ctx, t, c) + sub, err := r.getSubscription(ctx, t) // If the resource doesn't exist, we'll create it if k8serrors.IsNotFound(err) { sub = expected @@ -631,7 +644,9 @@ func (r *reconciler) subscribeToBrokerChannel(ctx context.Context, t *v1alpha1.T return sub, nil } -func (r *reconciler) getSubscription(ctx context.Context, t *v1alpha1.Trigger, c *v1alpha1.Channel) (*v1alpha1.Subscription, error) { +// getSubscription returns the subscription of trigger 't' if exists, +// otherwise it returns an error. +func (r *reconciler) getSubscription(ctx context.Context, t *v1alpha1.Trigger) (*v1alpha1.Subscription, error) { list := &v1alpha1.SubscriptionList{} opts := &runtimeclient.ListOptions{ Namespace: t.Namespace, @@ -659,6 +674,7 @@ func (r *reconciler) getSubscription(ctx context.Context, t *v1alpha1.Trigger, c return nil, k8serrors.NewNotFound(schema.GroupResource{}, "") } +// makeSubscription returns a placeholder subscription for trigger 't', channel 'c', and service 'svc'. func makeSubscription(t *v1alpha1.Trigger, c *v1alpha1.Channel, svc *corev1.Service) *v1alpha1.Subscription { return &v1alpha1.Subscription{ ObjectMeta: metav1.ObjectMeta{ From fda6550eb16ceb2ef305c6afcc9059a0a3f0b40c Mon Sep 17 00:00:00 2001 From: nachocano Date: Mon, 25 Feb 2019 12:08:48 -0800 Subject: [PATCH 097/128] Updating comments. --- pkg/broker/receiver.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/broker/receiver.go b/pkg/broker/receiver.go index 1f294d7c942..8034b495799 100644 --- a/pkg/broker/receiver.go +++ b/pkg/broker/receiver.go @@ -28,7 +28,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" ) -// Receiver parses Cloud Events and sends them to the subscriber(s). +// Receiver parses Cloud Events and sends them to a subscriber. type Receiver struct { logger *zap.Logger client client.Client From 2aae7eba80de2e578596525831896beb55931efd Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Mon, 25 Feb 2019 12:16:43 -0800 Subject: [PATCH 098/128] Replace the bad errgroup usage with the runnableServer. --- cmd/broker/ingress/main.go | 41 +++++++++++++++++++++++++------------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/cmd/broker/ingress/main.go b/cmd/broker/ingress/main.go index 1be24c382bb..2c116bdd18a 100644 --- a/cmd/broker/ingress/main.go +++ b/cmd/broker/ingress/main.go @@ -25,8 +25,6 @@ import ( "os" "time" - "golang.org/x/sync/errgroup" - eventingv1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" "github.com/knative/eventing/pkg/provisioners" "github.com/knative/pkg/signals" @@ -71,24 +69,27 @@ func main() { WriteTimeout: writeTimeout, } - // Start both the manager (which notices ConfigMap changes) and the HTTP server. - var g errgroup.Group - g.Go(func() error { - // set up signals so we handle the first shutdown signal gracefully - stopCh := signals.SetupSignalHandler() - // Start blocks forever, so run it in a goroutine. - return mgr.Start(stopCh) + err = mgr.Add(&runnableServer{ + logger: logger, + s: s, }) - logger.Info("Ingress Listening...", zap.String("Address", s.Addr)) - g.Go(s.ListenAndServe) - err = g.Wait() if err != nil { - logger.Error("HTTP server failed.", zap.Error(err)) + logger.Fatal("Unable to add ListenAndServe", zap.Error(err)) + } + + // Set up signals so we handle the first shutdown signal gracefully. + stopCh := signals.SetupSignalHandler() + // Start blocks forever. + if err = mgr.Start(stopCh); err != nil { + logger.Error("manager.Start() returned an error", zap.Error(err)) } + logger.Info("Exiting...") ctx, cancel := context.WithTimeout(context.Background(), writeTimeout) defer cancel() - s.Shutdown(ctx) + if err = s.Shutdown(ctx); err != nil { + logger.Error("Shutdown returned an error", zap.Error(err)) + } } func getRequiredEnv(envKey string) string { @@ -142,3 +143,15 @@ func (f *Handler) dispatch(msg *provisioners.Message) error { } return err } + +// runnableServer is a small wrapper around http.Server so that it matches the manager.Runnable +// interface. +type runnableServer struct { + logger *zap.Logger + s *http.Server +} + +func (r *runnableServer) Start(<-chan struct{}) error { + r.logger.Info("Ingress Listening...", zap.String("Address", r.s.Addr)) + return r.s.ListenAndServe() +} From 5f201fa4afc44652856111e27359860f3bbd7e19 Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Mon, 25 Feb 2019 12:36:51 -0800 Subject: [PATCH 099/128] Namespace scoped the Broker Filter's client. --- cmd/broker/filter/main.go | 22 ++++++++++- pkg/broker/receiver.go | 4 -- .../v1alpha1/broker/resources/filter.go | 10 +++-- .../v1alpha1/namespace/namespace.go | 39 ++++++++++--------- 4 files changed, 47 insertions(+), 28 deletions(-) diff --git a/cmd/broker/filter/main.go b/cmd/broker/filter/main.go index 0c39dec0550..d38919b1766 100644 --- a/cmd/broker/filter/main.go +++ b/cmd/broker/filter/main.go @@ -18,6 +18,8 @@ package main import ( "flag" + "log" + "os" eventingv1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" "github.com/knative/eventing/pkg/broker" @@ -29,6 +31,10 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" ) +const ( + NAMESPACE = "NAMESPACE" +) + func main() { logConfig := provisioners.NewLoggingConfig() logConfig.LoggingLevel["provisioner"] = zapcore.DebugLevel @@ -39,13 +45,17 @@ func main() { logger.Info("Starting...") - mgr, err := manager.New(config.GetConfigOrDie(), manager.Options{}) + mgr, err := manager.New(config.GetConfigOrDie(), manager.Options{ + Namespace: getRequiredEnv(NAMESPACE), + }) if err != nil { logger.Fatal("Error starting up.", zap.Error(err)) } // Add custom types to this array to get them into the manager's scheme. - eventingv1alpha1.AddToScheme(mgr.GetScheme()) + if err = eventingv1alpha1.AddToScheme(mgr.GetScheme()); err != nil { + logger.Fatal("Unable to add eventingv1alpha1 scheme", zap.Error(err)) + } // We are running both the receiver (takes messages in from the cluster and writes them to // PubSub) and the dispatcher (takes messages in PubSub and sends them in cluster) in this @@ -67,3 +77,11 @@ func main() { logger.Fatal("Manager.Start() returned an error", zap.Error(err)) } } + +func getRequiredEnv(envKey string) string { + val, defined := os.LookupEnv(envKey) + if !defined { + log.Fatalf("required environment variable not defined '%s'", envKey) + } + return val +} diff --git a/pkg/broker/receiver.go b/pkg/broker/receiver.go index 9dd264337ee..d51c3d6c755 100644 --- a/pkg/broker/receiver.go +++ b/pkg/broker/receiver.go @@ -83,10 +83,6 @@ func (r *Receiver) sendEvent(channel provisioners.ChannelReference, message *pro } func (r *Receiver) getTrigger(ctx context.Context, ref provisioners.ChannelReference) (*eventingv1alpha1.Trigger, error) { - // Sadly this doesn't work well because we do not yet have - // https://github.com/kubernetes-sigs/controller-runtime/pull/136, so controller runtime watches - // all Triggers, not just those in this namespace. And it doesn't have the RBAC (by default) for - // that to work. t := &eventingv1alpha1.Trigger{} err := r.client.Get(ctx, types.NamespacedName{ diff --git a/pkg/reconciler/v1alpha1/broker/resources/filter.go b/pkg/reconciler/v1alpha1/broker/resources/filter.go index 428d0e4a38e..21c404fae41 100644 --- a/pkg/reconciler/v1alpha1/broker/resources/filter.go +++ b/pkg/reconciler/v1alpha1/broker/resources/filter.go @@ -63,12 +63,16 @@ func MakeFilterDeployment(args *FilterArgs) *appsv1.Deployment { ServiceAccountName: args.ServiceAccountName, Containers: []corev1.Container{ { - Image: args.Image, Name: "filter", + Image: args.Image, Env: []corev1.EnvVar{ { - Name: "BROKER", - Value: args.Broker.Name, + Name: "NAMESPACE", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.namespace", + }, + }, }, }, }, diff --git a/pkg/reconciler/v1alpha1/namespace/namespace.go b/pkg/reconciler/v1alpha1/namespace/namespace.go index 483d659604d..82a72750767 100644 --- a/pkg/reconciler/v1alpha1/namespace/namespace.go +++ b/pkg/reconciler/v1alpha1/namespace/namespace.go @@ -19,11 +19,12 @@ package namespace import ( "context" "fmt" + "github.com/knative/eventing/contrib/gcppubsub/pkg/util/logging" "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" "go.uber.org/zap" - "k8s.io/api/core/v1" corev1 "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/api/errors" k8serrors "k8s.io/apimachinery/pkg/api/errors" @@ -46,16 +47,16 @@ const ( // itself when creating events. controllerAgentName = "knative-eventing-namespace-controller" - defaultBroker = "default" - brokerFilterSA = "eventing-broker-filter" - brokerFilterRB = "eventing-broker-filter" + defaultBroker = "default" + brokerFilterSA = "eventing-broker-filter" + brokerFilterRB = "eventing-broker-filter" brokerFilterClusterRole = "eventing-broker-filter" knativeEventingAnnotation = "eventing.knative.dev/inject" // Name of the corev1.Events emitted from the reconciliation process. - brokerCreated = "BrokerCreated" - serviceAccountCreated = "BrokerFilterServiceAccountCreated" + brokerCreated = "BrokerCreated" + serviceAccountCreated = "BrokerFilterServiceAccountCreated" serviceAccountRBACCreated = "BrokerFilterServiceAccountRBACCreated" ) @@ -92,8 +93,8 @@ func ProvideController(logger *zap.Logger) func(manager.Manager) (controller.Con } // Watch all the resources that this reconciler reconciles. - for _, t := range []runtime.Object{ &corev1.ServiceAccount{}, &rbacv1.RoleBinding{}, &v1alpha1.Broker{} } { - err = c.Watch(&source.Kind{Type: t}, &handler.EnqueueRequestsFromMapFunc{ToRequests: &namespaceMapper{}}); + for _, t := range []runtime.Object{&corev1.ServiceAccount{}, &rbacv1.RoleBinding{}, &v1alpha1.Broker{}} { + err = c.Watch(&source.Kind{Type: t}, &handler.EnqueueRequestsFromMapFunc{ToRequests: &namespaceMapper{}}) if err != nil { return nil, err } @@ -228,8 +229,8 @@ func newBrokerFilterServiceAccount(ns *corev1.Namespace) *corev1.ServiceAccount return &corev1.ServiceAccount{ ObjectMeta: metav1.ObjectMeta{ Namespace: ns.Name, - Name: brokerFilterSA, - Labels: injectedLabels(), + Name: brokerFilterSA, + Labels: injectedLabels(), }, } } @@ -276,19 +277,19 @@ func newBrokerFilterRBAC(ns *corev1.Namespace, sa *corev1.ServiceAccount) *rbacv return &rbacv1.RoleBinding{ ObjectMeta: metav1.ObjectMeta{ Namespace: ns.Name, - Name: brokerFilterRB, - Labels: injectedLabels(), + Name: brokerFilterRB, + Labels: injectedLabels(), }, - RoleRef:rbacv1.RoleRef{ - APIGroup:"rbac.authorization.k8s.io", - Kind: "ClusterRole", - Name: brokerFilterClusterRole, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "ClusterRole", + Name: brokerFilterClusterRole, }, - Subjects:[]rbacv1.Subject{ + Subjects: []rbacv1.Subject{ { - Kind: "ServiceAccount", + Kind: "ServiceAccount", Namespace: ns.Name, - Name: sa.Name, + Name: sa.Name, }, }, } From ba726b3a26301ca49cceea8728d07e1b0f507e48 Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Mon, 25 Feb 2019 12:41:51 -0800 Subject: [PATCH 100/128] Fix unit tests. --- .../v1alpha1/namespace/namespace_test.go | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/pkg/reconciler/v1alpha1/namespace/namespace_test.go b/pkg/reconciler/v1alpha1/namespace/namespace_test.go index 9f3364caff1..a413c87a435 100644 --- a/pkg/reconciler/v1alpha1/namespace/namespace_test.go +++ b/pkg/reconciler/v1alpha1/namespace/namespace_test.go @@ -20,6 +20,8 @@ import ( "context" "errors" "fmt" + "testing" + "github.com/google/go-cmp/cmp" "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" controllertesting "github.com/knative/eventing/pkg/reconciler/testing" @@ -35,7 +37,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "testing" ) const ( @@ -50,6 +51,13 @@ var ( // deletionTime is used when objects are marked as deleted. Rfc3339Copy() // truncates to seconds to match the loss of precision during serialization. deletionTime = metav1.Now().Rfc3339Copy() + + // map of events to set test cases' expectations easier + events = map[string]corev1.Event{ + brokerCreated: {Reason: brokerCreated, Type: corev1.EventTypeNormal}, + serviceAccountCreated: {Reason: serviceAccountCreated, Type: corev1.EventTypeNormal}, + serviceAccountRBACCreated: {Reason: serviceAccountRBACCreated, Type: corev1.EventTypeNormal}, + } ) func init() { @@ -214,6 +222,7 @@ func TestReconcile(t *testing.T) { WantAbsent: []runtime.Object{ makeBroker(), }, + WantEvent: []corev1.Event{events[serviceAccountCreated], events[serviceAccountRBACCreated]}, }, { Name: "Broker Found", @@ -222,6 +231,7 @@ func TestReconcile(t *testing.T) { makeNamespace(&trueString), makeBroker(), }, + WantEvent: []corev1.Event{events[serviceAccountCreated], events[serviceAccountRBACCreated]}, }, { Name: "Broker.Create fails", @@ -240,6 +250,7 @@ func TestReconcile(t *testing.T) { }, }, WantErrMsg: "test error creating the Broker", + WantEvent: []corev1.Event{events[serviceAccountCreated], events[serviceAccountRBACCreated]}, }, { Name: "Broker created", @@ -251,10 +262,9 @@ func TestReconcile(t *testing.T) { makeBroker(), }, WantEvent: []corev1.Event{ - { - Reason: brokerCreated, Type: corev1.EventTypeNormal, - }, - }, + events[serviceAccountCreated], + events[serviceAccountRBACCreated], + events[brokerCreated]}, }, } for _, tc := range testCases { From e7ff5b225d486230d5230e69787d1cd3bab2cc61 Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Mon, 25 Feb 2019 12:46:46 -0800 Subject: [PATCH 101/128] Fix yaml --- config/500-controller.yaml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/config/500-controller.yaml b/config/500-controller.yaml index 48ee083917c..7dd95d66b36 100644 --- a/config/500-controller.yaml +++ b/config/500-controller.yaml @@ -35,6 +35,10 @@ spec: "-stderrthreshold", "INFO" ] env: + - name: SYSTEM_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace - name: INGRESS_IMAGE value: github.com/knative/eventing/cmd/broker/ingress - name: INGRESS_SERVICE_ACCOUNT @@ -46,11 +50,6 @@ spec: volumeMounts: - name: config-logging mountPath: /etc/config-logging - env: - - name: SYSTEM_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace volumes: - name: config-logging configMap: From dcb8774e28f48cb3280948abc4e1ca783e703ffe Mon Sep 17 00:00:00 2001 From: nachocano Date: Mon, 25 Feb 2019 14:36:39 -0800 Subject: [PATCH 102/128] Setting source to source not type. Updating comment. --- pkg/broker/receiver.go | 9 +++------ test/builders.go | 4 ++-- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/pkg/broker/receiver.go b/pkg/broker/receiver.go index 06b9b0e0f06..1644b48f442 100644 --- a/pkg/broker/receiver.go +++ b/pkg/broker/receiver.go @@ -93,12 +93,9 @@ func (r *Receiver) initClient() error { // Otherwise, on receiving an event, it may not find the trigger // and would return an error. opts := &client.ListOptions{ - Raw: &metav1.ListOptions{ - TypeMeta: metav1.TypeMeta{ - APIVersion: eventingv1alpha1.SchemeGroupVersion.String(), - Kind: "Trigger", - }, - }, + // Set Raw because if we need to get more than one page, then we will put the continue token + // into opts.Raw.Continue. + Raw: &metav1.ListOptions{}, } for { tl := &eventingv1alpha1.TriggerList{} diff --git a/test/builders.go b/test/builders.go index c73822bad4d..23d7c6538e3 100644 --- a/test/builders.go +++ b/test/builders.go @@ -60,8 +60,8 @@ func (b *TriggerBuilder) EventType(eventType string) *TriggerBuilder { return b } -func (b *TriggerBuilder) EventSource(eventType string) *TriggerBuilder { - b.Trigger.Spec.Filter.SourceAndType.Type = eventType +func (b *TriggerBuilder) EventSource(eventSource string) *TriggerBuilder { + b.Trigger.Spec.Filter.SourceAndType.Source = eventSource return b } From 71c52503aef56ab9ae2bb087f812977c359b536c Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Tue, 26 Feb 2019 08:45:41 -0800 Subject: [PATCH 103/128] Switch from annotating the namespace to labeling it, to match Istio. --- .../v1alpha1/namespace/namespace.go | 7 ++-- .../v1alpha1/namespace/namespace_test.go | 32 +++++++++---------- test/e2e/broker_trigger_test.go | 6 ++-- test/e2e/e2e.go | 11 +++++-- 4 files changed, 31 insertions(+), 25 deletions(-) diff --git a/pkg/reconciler/v1alpha1/namespace/namespace.go b/pkg/reconciler/v1alpha1/namespace/namespace.go index 58b1bcd045e..a48cef05c19 100644 --- a/pkg/reconciler/v1alpha1/namespace/namespace.go +++ b/pkg/reconciler/v1alpha1/namespace/namespace.go @@ -52,8 +52,9 @@ const ( brokerFilterRB = "eventing-broker-filter" brokerFilterClusterRole = "eventing-broker-filter" - // Annotation to enable knative-eventing in a namespace. - knativeEventingAnnotation = "eventing.knative.dev/inject" + // Label to enable knative-eventing in a namespace. + knativeEventingLabelKey = "knative-eventing-injection" + knativeEventingLabelValue = "enabled" // Name of the corev1.Events emitted from the reconciliation process. brokerCreated = "BrokerCreated" @@ -152,7 +153,7 @@ func (r *reconciler) Reconcile(request reconcile.Request) (reconcile.Result, err return reconcile.Result{}, err } - if ns.Annotations[knativeEventingAnnotation] != "true" { + if ns.Labels[knativeEventingLabelKey] != knativeEventingLabelValue { logging.FromContext(ctx).Debug("Not reconciling Namespace") return reconcile.Result{}, nil } diff --git a/pkg/reconciler/v1alpha1/namespace/namespace_test.go b/pkg/reconciler/v1alpha1/namespace/namespace_test.go index a413c87a435..f65c8d85cae 100644 --- a/pkg/reconciler/v1alpha1/namespace/namespace_test.go +++ b/pkg/reconciler/v1alpha1/namespace/namespace_test.go @@ -45,8 +45,8 @@ const ( ) var ( - falseString = "false" - trueString = "true" + disabled = "disabled" + enabled = "enabled" // deletionTime is used when objects are marked as deleted. Rfc3339Copy() // truncates to seconds to match the loss of precision during serialization. @@ -173,7 +173,7 @@ func TestReconcile(t *testing.T) { WantErrMsg: "test error getting the NS", }, { - Name: "Namespace is not annotated", + Name: "Namespace is not labeled", Scheme: scheme.Scheme, InitialState: []runtime.Object{ makeNamespace(nil), @@ -183,10 +183,10 @@ func TestReconcile(t *testing.T) { }, }, { - Name: "Namespace is annotated off", + Name: "Namespace is labeled disabled", Scheme: scheme.Scheme, InitialState: []runtime.Object{ - makeNamespace(&falseString), + makeNamespace(&disabled), }, WantAbsent: []runtime.Object{ makeBroker(), @@ -206,7 +206,7 @@ func TestReconcile(t *testing.T) { Name: "Broker.Get fails", Scheme: scheme.Scheme, InitialState: []runtime.Object{ - makeNamespace(&trueString), + makeNamespace(&enabled), }, Mocks: controllertesting.Mocks{ MockGets: []controllertesting.MockGet{ @@ -228,7 +228,7 @@ func TestReconcile(t *testing.T) { Name: "Broker Found", Scheme: scheme.Scheme, InitialState: []runtime.Object{ - makeNamespace(&trueString), + makeNamespace(&enabled), makeBroker(), }, WantEvent: []corev1.Event{events[serviceAccountCreated], events[serviceAccountRBACCreated]}, @@ -237,7 +237,7 @@ func TestReconcile(t *testing.T) { Name: "Broker.Create fails", Scheme: scheme.Scheme, InitialState: []runtime.Object{ - makeNamespace(&trueString), + makeNamespace(&enabled), }, Mocks: controllertesting.Mocks{ MockCreates: []controllertesting.MockCreate{ @@ -256,7 +256,7 @@ func TestReconcile(t *testing.T) { Name: "Broker created", Scheme: scheme.Scheme, InitialState: []runtime.Object{ - makeNamespace(&trueString), + makeNamespace(&enabled), }, WantPresent: []runtime.Object{ makeBroker(), @@ -285,10 +285,10 @@ func TestReconcile(t *testing.T) { } } -func makeNamespace(annotationValue *string) *corev1.Namespace { - annotations := map[string]string{} - if annotationValue != nil { - annotations["eventing.knative.dev/inject"] = *annotationValue +func makeNamespace(labelValue *string) *corev1.Namespace { + labels := map[string]string{} + if labelValue != nil { + labels["knative-eventing-injection"] = *labelValue } return &corev1.Namespace{ @@ -297,14 +297,14 @@ func makeNamespace(annotationValue *string) *corev1.Namespace { Kind: "Namespace", }, ObjectMeta: metav1.ObjectMeta{ - Name: testNS, - Annotations: annotations, + Name: testNS, + Labels: labels, }, } } func makeDeletingNamespace() *corev1.Namespace { - ns := makeNamespace(&trueString) + ns := makeNamespace(&enabled) ns.DeletionTimestamp = &deletionTime return ns } diff --git a/test/e2e/broker_trigger_test.go b/test/e2e/broker_trigger_test.go index b478c673381..6deacb0e8ab 100644 --- a/test/e2e/broker_trigger_test.go +++ b/test/e2e/broker_trigger_test.go @@ -65,10 +65,10 @@ func TestDefaultBrokerWithManyTriggers(t *testing.T) { defer cleanupNS() defer TearDown(clients, cleaner, logger) - logger.Infof("Annotating namespace %s", ns) + logger.Infof("Labeling namespace %s", ns) - // Annotate namespace so that it creates the default broker. - err := AnnotateNamespace(clients, logger, map[string]string{"eventing.knative.dev/inject": "true"}) + // Label namespace so that it creates the default broker. + err := LabelNamespace(clients, logger, map[string]string{"knative-eventing-injection": "enabled"}) if err != nil { t.Fatalf("Error annotating namespace: %v", err) } diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go index 98079dfe015..d4b364e4573 100644 --- a/test/e2e/e2e.go +++ b/test/e2e/e2e.go @@ -391,14 +391,19 @@ func WaitForAllTriggersReady(clients *test.Clients, logger *logging.BaseLogger, return nil } -// AnnotateNamespace annotates the test namespace with the annotations map. -func AnnotateNamespace(clients *test.Clients, logger *logging.BaseLogger, annotations map[string]string) error { +// LabelNamespace labels the test namespace with the labels map. +func LabelNamespace(clients *test.Clients, logger *logging.BaseLogger, labels map[string]string) error { ns := pkgTest.Flags.Namespace nsSpec, err := clients.Kube.Kube.CoreV1().Namespaces().Get(ns, metav1.GetOptions{}) if err != nil && errors.IsNotFound(err) { return err } - nsSpec.Annotations = annotations + if nsSpec.Labels == nil { + nsSpec.Labels = map[string]string{} + } + for k, v := range labels { + nsSpec.Labels[k] = v + } _, err = clients.Kube.Kube.CoreV1().Namespaces().Update(nsSpec) return err } From 8d6df67cd7ed5dc977257d5a8d08ce8c867c1ac6 Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Wed, 27 Feb 2019 13:34:51 -0800 Subject: [PATCH 104/128] General clean up. --- cmd/broker/filter/main.go | 10 +-- cmd/broker/ingress/main.go | 15 ++-- cmd/controller/main.go | 8 +- cmd/webhook/main.go | 14 ++-- config/500-controller.yaml | 8 +- pkg/apis/eventing/v1alpha1/broker_types.go | 15 ++-- .../eventing/v1alpha1/broker_types_test.go | 2 +- .../eventing/v1alpha1/broker_validation.go | 7 +- .../eventing/v1alpha1/trigger_defaults.go | 14 +++- pkg/apis/eventing/v1alpha1/trigger_types.go | 14 +++- .../eventing/v1alpha1/trigger_types_test.go | 2 +- pkg/broker/receiver.go | 26 +++--- pkg/broker/receiver_test.go | 11 ++- pkg/reconciler/v1alpha1/broker/broker.go | 56 ++++++------- pkg/reconciler/v1alpha1/broker/broker_test.go | 68 +++++---------- .../v1alpha1/broker/resources/filter.go | 2 +- .../v1alpha1/broker/resources/ingress.go | 2 +- .../v1alpha1/namespace/namespace.go | 32 +++---- .../v1alpha1/namespace/namespace_test.go | 41 +-------- pkg/reconciler/v1alpha1/trigger/trigger.go | 83 +++++++------------ .../v1alpha1/trigger/trigger_test.go | 9 +- 21 files changed, 188 insertions(+), 251 deletions(-) diff --git a/cmd/broker/filter/main.go b/cmd/broker/filter/main.go index 8382060256c..f60b33340e0 100644 --- a/cmd/broker/filter/main.go +++ b/cmd/broker/filter/main.go @@ -1,5 +1,5 @@ /* - * Copyright 2018 The Knative Authors + * Copyright 2019 The Knative Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -52,20 +52,19 @@ func main() { logger.Fatal("Error starting up.", zap.Error(err)) } - // Add custom types to this array to get them into the manager's scheme. if err = eventingv1alpha1.AddToScheme(mgr.GetScheme()); err != nil { logger.Fatal("Unable to add eventingv1alpha1 scheme", zap.Error(err)) } - // We are running both the receiver (takes messages in from the cluster) and the dispatcher (send the messages - // to the triggers' subscribers) in this binary. + // We are running both the receiver (takes messages in from the Broker) and the dispatcher (send + // the messages to the triggers' subscribers) in this binary. _, runnable := broker.New(logger, mgr.GetClient()) err = mgr.Add(runnable) if err != nil { logger.Fatal("Unable to start the receivers runnable", zap.Error(err), zap.Any("runnable", runnable)) } - // set up signals so we handle the first shutdown signal gracefully + // Set up signals so we handle the first shutdown signal gracefully. stopCh := signals.SetupSignalHandler() // Start blocks forever. @@ -74,6 +73,7 @@ func main() { if err != nil { logger.Fatal("Manager.Start() returned an error", zap.Error(err)) } + logger.Info("Exiting...") } func getRequiredEnv(envKey string) string { diff --git a/cmd/broker/ingress/main.go b/cmd/broker/ingress/main.go index 946277c3854..6027e82d5d9 100644 --- a/cmd/broker/ingress/main.go +++ b/cmd/broker/ingress/main.go @@ -1,5 +1,5 @@ /* - * Copyright 2018 The Knative Authors + * Copyright 2019 The Knative Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -34,6 +34,8 @@ import ( ) var ( + port = 8080 + readTimeout = 1 * time.Minute writeTimeout = 1 * time.Minute ) @@ -51,10 +53,8 @@ func main() { logger.Fatal("Error starting up.", zap.Error(err)) } - // Add custom types to this array to get them into the manager's scheme. - err = eventingv1alpha1.AddToScheme(mgr.GetScheme()) - if err != nil { - logger.Fatal("Unable to add scheme", zap.Error(err)) + if err = eventingv1alpha1.AddToScheme(mgr.GetScheme()); err != nil { + logger.Fatal("Unable to add eventingv1alpha1 scheme", zap.Error(err)) } c := getRequiredEnv("CHANNEL") @@ -62,7 +62,7 @@ func main() { h := NewHandler(logger, c) s := &http.Server{ - Addr: ":8080", + Addr: fmt.Sprintf(":%d", port), Handler: h, ErrorLog: zap.NewStdLog(logger), ReadTimeout: readTimeout, @@ -74,7 +74,7 @@ func main() { s: s, }) if err != nil { - logger.Fatal("Unable to add ListenAndServe", zap.Error(err)) + logger.Fatal("Unable to add runnableServer", zap.Error(err)) } // Set up signals so we handle the first shutdown signal gracefully. @@ -130,6 +130,7 @@ func createReceiverFunction(f *Handler) func(provisioners.ChannelReference, *pro } } +// http.Handler interface. func (f *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { f.receiver.HandleRequest(w, r) } diff --git a/cmd/controller/main.go b/cmd/controller/main.go index 382e79d38a7..60c46d4d967 100644 --- a/cmd/controller/main.go +++ b/cmd/controller/main.go @@ -123,7 +123,13 @@ func main() { // manager run it. providers := []ProvideFunc{ subscription.ProvideController, - broker.ProvideController(logger.Desugar(), getRequiredEnv("INGRESS_IMAGE"), getRequiredEnv("INGRESS_SERVICE_ACCOUNT"), getRequiredEnv("FILTER_IMAGE"), getRequiredEnv("FILTER_SERVICE_ACCOUNT")), + broker.ProvideController(logger.Desugar(), + broker.ReconcilerArgs{ + IngressImage: getRequiredEnv("BROKER_INGRESS_IMAGE"), + IngressServiceAccountName: getRequiredEnv("BROKER_INGRESS_SERVICE_ACCOUNT"), + FilterImage: getRequiredEnv("BROKER_FILTER_IMAGE"), + FilterServiceAccountName: getRequiredEnv("BROKER_FILTER_SERVICE_ACCOUNT"), + }), trigger.ProvideController(logger.Desugar()), namespace.ProvideController(logger.Desugar()), } diff --git a/cmd/webhook/main.go b/cmd/webhook/main.go index c82ab87a6dd..a8e0732084a 100644 --- a/cmd/webhook/main.go +++ b/cmd/webhook/main.go @@ -53,19 +53,19 @@ func main() { defer logger.Sync() logger = logger.With(zap.String(logkey.ControllerType, logconfig.Webhook)) - logger.Info("Starting the Eventing Webhook") + logger.Infow("Starting the Eventing Webhook") // set up signals so we handle the first shutdown signal gracefully stopCh := signals.SetupSignalHandler() clusterConfig, err := rest.InClusterConfig() if err != nil { - logger.Fatal("Failed to get in cluster config", zap.Error(err)) + logger.Fatalw("Failed to get in cluster config", zap.Error(err)) } kubeClient, err := kubernetes.NewForConfig(clusterConfig) if err != nil { - logger.Fatal("Failed to get the client set", zap.Error(err)) + logger.Fatalw("Failed to get the client set", zap.Error(err)) } // Watch the logging config map and dynamically update logging levels. @@ -105,8 +105,10 @@ func main() { Logger: logger, } if err != nil { - logger.Fatal("Failed to create the admission controller", zap.Error(err)) + logger.Fatalw("Failed to create the admission controller", zap.Error(err)) } - err = controller.Run(stopCh) - logger.Errorw("Webhook stopping", zap.Error(err)) + if err = controller.Run(stopCh); err != nil { + logger.Errorw("controller.Run() failed", zap.Error(err)) + } + logger.Infow("Webhook stopping") } diff --git a/config/500-controller.yaml b/config/500-controller.yaml index 7dd95d66b36..b09ead19958 100644 --- a/config/500-controller.yaml +++ b/config/500-controller.yaml @@ -39,13 +39,13 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - - name: INGRESS_IMAGE + - name: BROKER_INGRESS_IMAGE value: github.com/knative/eventing/cmd/broker/ingress - - name: INGRESS_SERVICE_ACCOUNT + - name: BROKER_INGRESS_SERVICE_ACCOUNT value: default - - name: FILTER_IMAGE + - name: BROKER_FILTER_IMAGE value: github.com/knative/eventing/cmd/broker/filter - - name: FILTER_SERVICE_ACCOUNT + - name: BROKER_FILTER_SERVICE_ACCOUNT value: eventing-broker-filter volumeMounts: - name: config-logging diff --git a/pkg/apis/eventing/v1alpha1/broker_types.go b/pkg/apis/eventing/v1alpha1/broker_types.go index 6cad4d3efbd..7e0c5a1c9be 100644 --- a/pkg/apis/eventing/v1alpha1/broker_types.go +++ b/pkg/apis/eventing/v1alpha1/broker_types.go @@ -58,6 +58,11 @@ type BrokerSpec struct { // +optional DeprecatedGeneration int64 `json:"generation,omitempty"` + // ChannelTemplate, if specified will be used to create all the Channels used internally by the + // Broker. Only Provisioner and Arguments may be specified. If left unspecified, the default + // Channel for the namespace will be used. + // + // +optional ChannelTemplate *ChannelSpec `json:"channelTemplate,omitempty"` } @@ -90,11 +95,11 @@ type BrokerStatus struct { const ( BrokerConditionReady = duckv1alpha1.ConditionReady - BrokerConditionIngress duckv1alpha1.ConditionType = "Ingress" + BrokerConditionIngress duckv1alpha1.ConditionType = "IngressReady" - BrokerConditionChannel duckv1alpha1.ConditionType = "Channel" + BrokerConditionChannel duckv1alpha1.ConditionType = "ChannelReady" - BrokerConditionFilter duckv1alpha1.ConditionType = "Filter" + BrokerConditionFilter duckv1alpha1.ConditionType = "FilterReady" BrokerConditionAddressable duckv1alpha1.ConditionType = "Addressable" ) @@ -134,8 +139,8 @@ func (bs *BrokerStatus) MarkFilterReady() { brokerCondSet.Manage(bs).MarkTrue(BrokerConditionFilter) } -// SetAddress makes this Channel addressable by setting the hostname. It also -// sets the ChannelConditionAddressable to true. +// SetAddress makes this Broker addressable by setting the hostname. It also +// sets the BrokerConditionAddressable to true. func (bs *BrokerStatus) SetAddress(hostname string) { bs.Address.Hostname = hostname if hostname != "" { diff --git a/pkg/apis/eventing/v1alpha1/broker_types_test.go b/pkg/apis/eventing/v1alpha1/broker_types_test.go index 623a002b642..3970fbcf56a 100644 --- a/pkg/apis/eventing/v1alpha1/broker_types_test.go +++ b/pkg/apis/eventing/v1alpha1/broker_types_test.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Knative Authors +Copyright 2019 The Knative Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/apis/eventing/v1alpha1/broker_validation.go b/pkg/apis/eventing/v1alpha1/broker_validation.go index 3dbe58255b5..39b495ef31a 100644 --- a/pkg/apis/eventing/v1alpha1/broker_validation.go +++ b/pkg/apis/eventing/v1alpha1/broker_validation.go @@ -25,11 +25,14 @@ func (b *Broker) Validate() *apis.FieldError { } func (bs *BrokerSpec) Validate() *apis.FieldError { - // TODO implement + // TODO validate that the channelTemplate only specifies the provisioner and arguments. return nil } func (b *Broker) CheckImmutableFields(og apis.Immutable) *apis.FieldError { - // TODO implement + // Currently there are no immutable fields. We could make spec.channelTemplate immutable, as + // changing it will normally not have the desired effect of changing the Channel inside the + // Broker. It would have an effect if the existing Channel was then deleted, the newly created + // Channel would use the new spec.channelTemplate. return nil } diff --git a/pkg/apis/eventing/v1alpha1/trigger_defaults.go b/pkg/apis/eventing/v1alpha1/trigger_defaults.go index b95ce930497..2e14b5f28ca 100644 --- a/pkg/apis/eventing/v1alpha1/trigger_defaults.go +++ b/pkg/apis/eventing/v1alpha1/trigger_defaults.go @@ -26,6 +26,18 @@ func (ts *TriggerSpec) SetDefaults() { } // Make a default filter that allows anything. if ts.Filter == nil { - ts.Filter = &TriggerFilter{&TriggerFilterSourceAndType{Type: TriggerAnyFilter, Source: TriggerAnyFilter}} + ts.Filter = &TriggerFilter{} + } + + // Note that this logic will need to change once there are other filtering options, as it should + // only apply if no other filter is applied. + if ts.Filter.SourceAndType == nil { + ts.Filter.SourceAndType = &TriggerFilterSourceAndType{} + } + if ts.Filter.SourceAndType.Type == "" { + ts.Filter.SourceAndType.Type = TriggerAnyFilter + } + if ts.Filter.SourceAndType.Source == "" { + ts.Filter.SourceAndType.Source = TriggerAnyFilter } } diff --git a/pkg/apis/eventing/v1alpha1/trigger_types.go b/pkg/apis/eventing/v1alpha1/trigger_types.go index 794a91b4dc3..89e5b909078 100644 --- a/pkg/apis/eventing/v1alpha1/trigger_types.go +++ b/pkg/apis/eventing/v1alpha1/trigger_types.go @@ -58,11 +58,18 @@ type TriggerSpec struct { // +optional DeprecatedGeneration int64 `json:"generation,omitempty"` + // Broker is the broker that this trigger receives events from. If not specified, will default + // to 'default'. Broker string `json:"broker,omitempty"` + // Filter is the filter to apply against all events from the Broker. Only events that pass this + // filter will be sent to the Subscriber. If not specified, will default to allowing all events. + // // +optional Filter *TriggerFilter `json:"filter,omitempty"` + // Subscriber is the addressable that receives events from the Broker that pass the Filter. It + // is required. Subscriber *SubscriberSpec `json:"subscriber,omitempty"` } @@ -70,6 +77,9 @@ type TriggerFilter struct { SourceAndType *TriggerFilterSourceAndType `json:"sourceAndType,omitempty"` } +// TriggerFilterSourceAndType filters events based on exact matches on the cloud event's type and +// source attributes. Only exact matches will pass the filter. Either or both type and source can +// use the value 'Any' to indicate all strings match. type TriggerFilterSourceAndType struct { Type string `json:"type,omitempty"` Source string `json:"source,omitempty"` @@ -101,9 +111,9 @@ const ( TriggerConditionBrokerExists duckv1alpha1.ConditionType = "BrokerExists" - TriggerConditionKubernetesService duckv1alpha1.ConditionType = "KubernetesService" + TriggerConditionKubernetesService duckv1alpha1.ConditionType = "KubernetesServiceReady" - TriggerConditionVirtualService duckv1alpha1.ConditionType = "VirtualService" + TriggerConditionVirtualService duckv1alpha1.ConditionType = "VirtualServiceReady" TriggerConditionSubscribed duckv1alpha1.ConditionType = "Subscribed" diff --git a/pkg/apis/eventing/v1alpha1/trigger_types_test.go b/pkg/apis/eventing/v1alpha1/trigger_types_test.go index 1d36e8fed7c..3c3c37525d5 100644 --- a/pkg/apis/eventing/v1alpha1/trigger_types_test.go +++ b/pkg/apis/eventing/v1alpha1/trigger_types_test.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Knative Authors +Copyright 2019 The Knative Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/broker/receiver.go b/pkg/broker/receiver.go index c48c7527056..b591a70ced2 100644 --- a/pkg/broker/receiver.go +++ b/pkg/broker/receiver.go @@ -1,5 +1,5 @@ /* - * Copyright 2018 The Knative Authors + * Copyright 2019 The Knative Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -29,7 +29,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" ) -// Receiver parses Cloud Events and sends them to a subscriber. +// Receiver parses Cloud Events, determines if they pass a filter, and sends them to a subscriber. type Receiver struct { logger *zap.Logger client client.Client @@ -49,21 +49,20 @@ func New(logger *zap.Logger, client client.Client) (*Receiver, manager.Runnable) } func (r *Receiver) newMessageReceiver() *provisioners.MessageReceiver { - err := r.initClient() - if err != nil { + if err := r.initClient(); err != nil { r.logger.Warn("Failed to initialize client", zap.Error(err)) } return provisioners.NewMessageReceiver(r.sendEvent, r.logger.Sugar()) } // sendEvent sends an event to a subscriber if the trigger filter passes. -func (r *Receiver) sendEvent(channel provisioners.ChannelReference, message *provisioners.Message) error { - r.logger.Debug("received message") +func (r *Receiver) sendEvent(trigger provisioners.ChannelReference, message *provisioners.Message) error { + r.logger.Debug("Received message", zap.Any("triggerRef", trigger)) ctx := context.Background() - t, err := r.getTrigger(ctx, channel) + t, err := r.getTrigger(ctx, trigger) if err != nil { - r.logger.Info("Unable to get the Trigger", zap.Error(err), zap.Any("channelRef", channel)) + r.logger.Info("Unable to get the Trigger", zap.Error(err), zap.Any("triggerRef", trigger)) return err } @@ -74,24 +73,23 @@ func (r *Receiver) sendEvent(channel provisioners.ChannelReference, message *pro } if !r.shouldSendMessage(&t.Spec, message) { - r.logger.Debug("Message did not pass filter") + r.logger.Debug("Message did not pass filter", zap.Any("triggerRef", trigger)) return nil } err = r.dispatcher.DispatchMessage(message, subscriberURI, "", provisioners.DispatchDefaults{}) if err != nil { - r.logger.Info("Failed to dispatch message", zap.Error(err)) + r.logger.Info("Failed to dispatch message", zap.Error(err), zap.Any("triggerRef", trigger)) return err } - r.logger.Debug("Successfully sent message") + r.logger.Debug("Successfully sent message", zap.Any("triggerRef", trigger)) return nil } // Initialize the client. Mainly intended to load stuff in its cache. func (r *Receiver) initClient() error { - // We list triggers so that we can load the client's cache. - // Otherwise, on receiving an event, it may not find the trigger - // and would return an error. + // We list triggers so that we can load the client's cache. Otherwise, on receiving an event, it + // may not find the trigger and would return an error. opts := &client.ListOptions{ // Set Raw because if we need to get more than one page, then we will put the continue token // into opts.Raw.Continue. diff --git a/pkg/broker/receiver_test.go b/pkg/broker/receiver_test.go index d841d895f6b..02a8518a351 100644 --- a/pkg/broker/receiver_test.go +++ b/pkg/broker/receiver_test.go @@ -1,5 +1,5 @@ /* - * Copyright 2018 The Knative Authors + * Copyright 2019 The Knative Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,18 +19,21 @@ package broker import ( "errors" "fmt" - "github.com/knative/eventing/pkg/provisioners" "net/http" "net/http/httptest" "strings" "testing" + "github.com/knative/eventing/pkg/utils" + + "github.com/knative/eventing/pkg/provisioners" + eventingv1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" "k8s.io/client-go/kubernetes/scheme" "go.uber.org/zap" - "k8s.io/apimachinery/pkg/apis/meta/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -177,7 +180,7 @@ func makeTriggerWithoutSubscriberURI() *eventingv1alpha1.Trigger { func makeRequest() *http.Request { req := httptest.NewRequest("POST", "/", strings.NewReader(``)) - req.Host = fmt.Sprintf("%s.%s.triggers.cluster.local", triggerName, testNS) + req.Host = fmt.Sprintf("%s.%s.triggers.%s", triggerName, testNS, utils.GetClusterDomainName()) eventAttributes := map[string]string{ "CE-CloudEventsVersion": `"0.1"`, diff --git a/pkg/reconciler/v1alpha1/broker/broker.go b/pkg/reconciler/v1alpha1/broker/broker.go index d72c63cff34..3ad3854589b 100644 --- a/pkg/reconciler/v1alpha1/broker/broker.go +++ b/pkg/reconciler/v1alpha1/broker/broker.go @@ -23,7 +23,6 @@ import ( "k8s.io/apimachinery/pkg/runtime" - "k8s.io/client-go/dynamic" "k8s.io/client-go/rest" "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/controller" @@ -67,10 +66,9 @@ const ( ) type reconciler struct { - client client.Client - restConfig *rest.Config - dynamicClient dynamic.Interface - recorder record.EventRecorder + client client.Client + restConfig *rest.Config + recorder record.EventRecorder logger *zap.Logger @@ -83,8 +81,15 @@ type reconciler struct { // Verify the struct implements reconcile.Reconciler. var _ reconcile.Reconciler = &reconciler{} +type ReconcilerArgs struct { + IngressImage string + IngressServiceAccountName string + FilterImage string + FilterServiceAccountName string +} + // ProvideController returns a function that returns a Broker controller. -func ProvideController(logger *zap.Logger, ingressImage, ingressServiceAccount, filterImage, filterServiceAccount string) func(manager.Manager) (controller.Controller, error) { +func ProvideController(logger *zap.Logger, args ReconcilerArgs) func(manager.Manager) (controller.Controller, error) { return func(mgr manager.Manager) (controller.Controller, error) { // Setup a new controller to Reconcile Brokers. c, err := controller.New(controllerAgentName, mgr, controller.Options{ @@ -92,10 +97,10 @@ func ProvideController(logger *zap.Logger, ingressImage, ingressServiceAccount, recorder: mgr.GetRecorder(controllerAgentName), logger: logger, - ingressImage: ingressImage, - ingressServiceAccountName: ingressServiceAccount, - filterImage: filterImage, - filterServiceAccountName: filterServiceAccount, + ingressImage: args.IngressImage, + ingressServiceAccountName: args.IngressServiceAccountName, + filterImage: args.FilterImage, + filterServiceAccountName: args.FilterServiceAccountName, }, }) if err != nil { @@ -124,13 +129,6 @@ func (r *reconciler) InjectClient(c client.Client) error { return nil } -func (r *reconciler) InjectConfig(c *rest.Config) error { - r.restConfig = c - var err error - r.dynamicClient, err = dynamic.NewForConfig(c) - return err -} - // Reconcile compares the actual state with the desired, and attempts to // converge the two. It then updates the Status block of the Broker resource // with the current status of the resource. @@ -139,7 +137,7 @@ func (r *reconciler) Reconcile(request reconcile.Request) (reconcile.Result, err ctx = logging.WithLogger(ctx, r.logger.With(zap.Any("request", request))) broker := &v1alpha1.Broker{} - err := r.client.Get(context.TODO(), request.NamespacedName, broker) + err := r.client.Get(ctx, request.NamespacedName, broker) if errors.IsNotFound(err) { logging.FromContext(ctx).Info("Could not find Broker") @@ -229,10 +227,11 @@ func (r *reconciler) reconcile(ctx context.Context, b *v1alpha1.Broker) (reconci // updateStatus may in fact update the broker's finalizers in addition to the status. func (r *reconciler) updateStatus(broker *v1alpha1.Broker) (*v1alpha1.Broker, error) { + ctx := context.TODO() objectKey := client.ObjectKey{Namespace: broker.Namespace, Name: broker.Name} latestBroker := &v1alpha1.Broker{} - if err := r.client.Get(context.TODO(), objectKey, latestBroker); err != nil { + if err := r.client.Get(ctx, objectKey, latestBroker); err != nil { return nil, err } @@ -240,7 +239,7 @@ func (r *reconciler) updateStatus(broker *v1alpha1.Broker) (*v1alpha1.Broker, er if !equality.Semantic.DeepEqual(latestBroker.Finalizers, broker.Finalizers) { latestBroker.SetFinalizers(broker.ObjectMeta.Finalizers) - if err := r.client.Update(context.TODO(), latestBroker); err != nil { + if err := r.client.Update(ctx, latestBroker); err != nil { return nil, err } brokerChanged = true @@ -253,13 +252,13 @@ func (r *reconciler) updateStatus(broker *v1alpha1.Broker) (*v1alpha1.Broker, er if brokerChanged { // Re-fetch. latestBroker = &v1alpha1.Broker{} - if err := r.client.Get(context.TODO(), objectKey, latestBroker); err != nil { + if err := r.client.Get(ctx, objectKey, latestBroker); err != nil { return nil, err } } latestBroker.Status = broker.Status - if err := r.client.Status().Update(context.TODO(), latestBroker); err != nil { + if err := r.client.Status().Update(ctx, latestBroker); err != nil { return nil, err } @@ -317,14 +316,9 @@ func (r *reconciler) getChannel(ctx context.Context, b *v1alpha1.Broker) (*v1alp opts := &runtimeclient.ListOptions{ Namespace: b.Namespace, LabelSelector: labels.SelectorFromSet(ChannelLabels(b)), - // TODO this is here because the fake client needs it. Remove this when it's no longer - // needed. - Raw: &metav1.ListOptions{ - TypeMeta: metav1.TypeMeta{ - APIVersion: v1alpha1.SchemeGroupVersion.String(), - Kind: "Channel", - }, - }, + // Set Raw because if we need to get more than one page, then we will put the continue token + // into opts.Raw.Continue. + Raw: &metav1.ListOptions{}, } err := r.client.List(ctx, opts, list) @@ -340,7 +334,7 @@ func (r *reconciler) getChannel(ctx context.Context, b *v1alpha1.Broker) (*v1alp return nil, k8serrors.NewNotFound(schema.GroupResource{}, "") } -// newChannel creates a new placeholder Channel object for Broker 'b'. +// newChannel creates a new Channel for Broker 'b'. func newChannel(b *v1alpha1.Broker) *v1alpha1.Channel { var spec v1alpha1.ChannelSpec if b.Spec.ChannelTemplate != nil { diff --git a/pkg/reconciler/v1alpha1/broker/broker_test.go b/pkg/reconciler/v1alpha1/broker/broker_test.go index 992bbe5708e..72de1f15252 100644 --- a/pkg/reconciler/v1alpha1/broker/broker_test.go +++ b/pkg/reconciler/v1alpha1/broker/broker_test.go @@ -20,7 +20,12 @@ import ( "context" "errors" "fmt" - "github.com/google/go-cmp/cmp" + "strings" + "testing" + "time" + + "github.com/knative/eventing/pkg/utils" + "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" controllertesting "github.com/knative/eventing/pkg/reconciler/testing" "github.com/knative/eventing/pkg/reconciler/v1alpha1/broker/resources" @@ -30,21 +35,16 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "strings" - "testing" - "time" ) const ( - testNS = "test-namespace" - brokerName = "test-broker" - channelHostname = "foo.bar.svc.cluster.local" + testNS = "test-namespace" + brokerName = "test-broker" filterImage = "filter-image" filterSA = "filter-SA" @@ -61,6 +61,8 @@ var ( Name: "my-provisioner", } + channelHostname = fmt.Sprintf("foo.bar.svc.%s", utils.GetClusterDomainName()) + // deletionTime is used when objects are marked as deleted. Rfc3339Copy() // truncates to seconds to match the loss of precision during serialization. deletionTime = metav1.Now().Rfc3339Copy() @@ -107,36 +109,6 @@ func TestInjectClient(t *testing.T) { } } -func TestInjectConfig(t *testing.T) { - r := &reconciler{} - wantCfg := &rest.Config{ - Host: "http://foo", - } - - err := r.InjectConfig(wantCfg) - if err != nil { - t.Fatalf("Unexpected error injecting the config: %v", err) - } - - gotCfg := r.restConfig - if diff := cmp.Diff(wantCfg, gotCfg); diff != "" { - t.Errorf("Unexpected config (-want, +got): %v", diff) - } - - wantDynClient, err := dynamic.NewForConfig(wantCfg) - if err != nil { - t.Fatalf("Unexpected error generating dynamic client: %v", err) - } - - // Since dynamicClient doesn't export any fields, we can only test its type. - switch r.dynamicClient.(type) { - case dynamic.Interface: - // ok - default: - t.Errorf("Unexpected dynamicClient type. Expected: %T, Got: %T", wantDynClient, r.dynamicClient) - } -} - func TestReconcile(t *testing.T) { testCases := []controllertesting.TestCase{ { @@ -578,15 +550,13 @@ func TestReconcile(t *testing.T) { } for _, tc := range testCases { c := tc.GetClient() - dc := tc.GetDynamicClient() recorder := tc.GetEventRecorder() r := &reconciler{ - client: c, - dynamicClient: dc, - restConfig: &rest.Config{}, - recorder: recorder, - logger: zap.NewNop(), + client: c, + restConfig: &rest.Config{}, + recorder: recorder, + logger: zap.NewNop(), filterImage: filterImage, filterServiceAccountName: filterSA, @@ -621,7 +591,7 @@ func makeReadyBroker() *v1alpha1.Broker { b := makeBroker() b.Status.InitializeConditions() b.Status.MarkChannelReady() - b.Status.SetAddress(fmt.Sprintf("%s-broker.%s.svc.cluster.local", brokerName, testNS)) + b.Status.SetAddress(fmt.Sprintf("%s-broker.%s.svc.%s", brokerName, testNS, utils.GetClusterDomainName())) b.Status.MarkFilterReady() b.Status.MarkIngressReady() return b @@ -677,7 +647,7 @@ func makeFilterDeployment() *appsv1.Deployment { }) d.TypeMeta = metav1.TypeMeta{ APIVersion: "apps/v1", - Kind: "Deployment", + Kind: "Deployment", } return d } @@ -692,7 +662,7 @@ func makeFilterService() *corev1.Service { svc := resources.MakeFilterService(makeBroker()) svc.TypeMeta = metav1.TypeMeta{ APIVersion: "v1", - Kind: "Service", + Kind: "Service", } return svc } @@ -712,7 +682,7 @@ func makeIngressDeployment() *appsv1.Deployment { }) d.TypeMeta = metav1.TypeMeta{ APIVersion: "apps/v1", - Kind: "Deployment", + Kind: "Deployment", } return d } @@ -727,7 +697,7 @@ func makeIngressService() *corev1.Service { svc := resources.MakeIngressService(makeBroker()) svc.TypeMeta = metav1.TypeMeta{ APIVersion: "v1", - Kind: "Service", + Kind: "Service", } return svc } diff --git a/pkg/reconciler/v1alpha1/broker/resources/filter.go b/pkg/reconciler/v1alpha1/broker/resources/filter.go index 21c404fae41..f51847c90a8 100644 --- a/pkg/reconciler/v1alpha1/broker/resources/filter.go +++ b/pkg/reconciler/v1alpha1/broker/resources/filter.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Knative Authors +Copyright 2019 The Knative Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/reconciler/v1alpha1/broker/resources/ingress.go b/pkg/reconciler/v1alpha1/broker/resources/ingress.go index 406f1b85daa..1ebf8957cec 100644 --- a/pkg/reconciler/v1alpha1/broker/resources/ingress.go +++ b/pkg/reconciler/v1alpha1/broker/resources/ingress.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Knative Authors +Copyright 2019 The Knative Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/reconciler/v1alpha1/namespace/namespace.go b/pkg/reconciler/v1alpha1/namespace/namespace.go index a48cef05c19..a8fd331b4bf 100644 --- a/pkg/reconciler/v1alpha1/namespace/namespace.go +++ b/pkg/reconciler/v1alpha1/namespace/namespace.go @@ -31,7 +31,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/dynamic" "k8s.io/client-go/rest" "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" @@ -47,15 +46,15 @@ const ( // itself when creating events. controllerAgentName = "knative-eventing-namespace-controller" + // Label to enable knative-eventing in a namespace. + knativeEventingLabelKey = "knative-eventing-injection" + knativeEventingLabelValue = "enabled" + defaultBroker = "default" brokerFilterSA = "eventing-broker-filter" brokerFilterRB = "eventing-broker-filter" brokerFilterClusterRole = "eventing-broker-filter" - // Label to enable knative-eventing in a namespace. - knativeEventingLabelKey = "knative-eventing-injection" - knativeEventingLabelValue = "enabled" - // Name of the corev1.Events emitted from the reconciliation process. brokerCreated = "BrokerCreated" serviceAccountCreated = "BrokerFilterServiceAccountCreated" @@ -63,10 +62,9 @@ const ( ) type reconciler struct { - client client.Client - restConfig *rest.Config - dynamicClient dynamic.Interface - recorder record.EventRecorder + client client.Client + restConfig *rest.Config + recorder record.EventRecorder logger *zap.Logger } @@ -126,13 +124,6 @@ func (r *reconciler) InjectClient(c client.Client) error { return nil } -func (r *reconciler) InjectConfig(c *rest.Config) error { - r.restConfig = c - var err error - r.dynamicClient, err = dynamic.NewForConfig(c) - return err -} - // Reconcile compares the actual state with the desired, and attempts to // converge the two. It then updates the Status block of the Namespace resource // with the current status of the resource. @@ -141,7 +132,7 @@ func (r *reconciler) Reconcile(request reconcile.Request) (reconcile.Result, err ctx = logging.WithLogger(ctx, r.logger.With(zap.Any("request", request))) ns := &corev1.Namespace{} - err := r.client.Get(context.TODO(), request.NamespacedName, ns) + err := r.client.Get(ctx, request.NamespacedName, ns) if errors.IsNotFound(err) { logging.FromContext(ctx).Info("Could not find Namespace") @@ -172,6 +163,9 @@ func (r *reconciler) Reconcile(request reconcile.Request) (reconcile.Result, err } func (r *reconciler) reconcile(ctx context.Context, ns *corev1.Namespace) error { + // No need for a finalizer, because everything reconciled is created inside the Namespace. If + // the Namespace is being deleted, then all the reconciled objects will be too. + if ns.DeletionTimestamp != nil { return nil } @@ -305,8 +299,8 @@ func newBrokerFilterRBAC(ns *corev1.Namespace, sa *corev1.ServiceAccount) *rbacv } } -// getBroker returns the default broker for Namespace 'ns' if exists, -// otherwise it returns an error. +// getBroker returns the default broker for Namespace 'ns' if it exists, otherwise it returns an +// error. func (r *reconciler) getBroker(ctx context.Context, ns *corev1.Namespace) (*v1alpha1.Broker, error) { b := &v1alpha1.Broker{} name := types.NamespacedName{ diff --git a/pkg/reconciler/v1alpha1/namespace/namespace_test.go b/pkg/reconciler/v1alpha1/namespace/namespace_test.go index f65c8d85cae..7c1b9e2769c 100644 --- a/pkg/reconciler/v1alpha1/namespace/namespace_test.go +++ b/pkg/reconciler/v1alpha1/namespace/namespace_test.go @@ -30,7 +30,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" @@ -101,36 +100,6 @@ func TestInjectClient(t *testing.T) { } } -func TestInjectConfig(t *testing.T) { - r := &reconciler{} - wantCfg := &rest.Config{ - Host: "http://foo", - } - - err := r.InjectConfig(wantCfg) - if err != nil { - t.Fatalf("Unexpected error injecting the config: %v", err) - } - - gotCfg := r.restConfig - if diff := cmp.Diff(wantCfg, gotCfg); diff != "" { - t.Errorf("Unexpected config (-want, +got): %v", diff) - } - - wantDynClient, err := dynamic.NewForConfig(wantCfg) - if err != nil { - t.Fatalf("Unexpected error generating dynamic client: %v", err) - } - - // Since dynamicClient doesn't export any fields, we can only test its type. - switch r.dynamicClient.(type) { - case dynamic.Interface: - // ok - default: - t.Errorf("Unexpected dynamicClient type. Expected: %T, Got: %T", wantDynClient, r.dynamicClient) - } -} - func TestNamespaceMapper_Map(t *testing.T) { m := &namespaceMapper{} @@ -269,15 +238,13 @@ func TestReconcile(t *testing.T) { } for _, tc := range testCases { c := tc.GetClient() - dc := tc.GetDynamicClient() recorder := tc.GetEventRecorder() r := &reconciler{ - client: c, - dynamicClient: dc, - restConfig: &rest.Config{}, - recorder: recorder, - logger: zap.NewNop(), + client: c, + restConfig: &rest.Config{}, + recorder: recorder, + logger: zap.NewNop(), } tc.ReconcileKey = fmt.Sprintf("%s/%s", "", testNS) tc.IgnoreTimes = true diff --git a/pkg/reconciler/v1alpha1/trigger/trigger.go b/pkg/reconciler/v1alpha1/trigger/trigger.go index b9e6e8b2341..70529f16740 100644 --- a/pkg/reconciler/v1alpha1/trigger/trigger.go +++ b/pkg/reconciler/v1alpha1/trigger/trigger.go @@ -21,6 +21,8 @@ import ( "fmt" "sync" + "github.com/knative/eventing/pkg/utils" + "github.com/knative/eventing/pkg/provisioners" "k8s.io/apimachinery/pkg/runtime" @@ -131,6 +133,7 @@ func ProvideController(logger *zap.Logger) func(manager.Manager) (controller.Con } } +// mapAllTriggers maps Broker change notifications to Trigger reconcileRequests. type mapAllTriggers struct { r *reconciler } @@ -170,7 +173,7 @@ func (r *reconciler) Reconcile(request reconcile.Request) (reconcile.Result, err ctx = logging.WithLogger(ctx, r.logger.With(zap.Any("request", request))) trigger := &v1alpha1.Trigger{} - err := r.client.Get(context.TODO(), request.NamespacedName, trigger) + err := r.client.Get(ctx, request.NamespacedName, trigger) if errors.IsNotFound(err) { logging.FromContext(ctx).Info("Could not find Trigger") @@ -286,22 +289,15 @@ func (r *reconciler) AddToTriggers(t *v1alpha1.Trigger) { // We will be reconciling an already existing Trigger far more often than adding a new one, so // check with a read lock before using the write lock. r.triggersLock.RLock() - triggersInBrokerNamespacedName := r.triggers[brokerNamespacedName] - var present bool - if triggersInBrokerNamespacedName != nil { - _, present = triggersInBrokerNamespacedName[name] - } else { - present = false - } + _, present := r.triggers[brokerNamespacedName][name] r.triggersLock.RUnlock() - if present { // Already present in the map. return } r.triggersLock.Lock() - triggersInBrokerNamespacedName = r.triggers[brokerNamespacedName] + triggersInBrokerNamespacedName := r.triggers[brokerNamespacedName] if triggersInBrokerNamespacedName == nil { r.triggers[brokerNamespacedName] = make(map[reconcile.Request]struct{}) triggersInBrokerNamespacedName = r.triggers[brokerNamespacedName] @@ -330,12 +326,13 @@ func (r *reconciler) removeFromTriggers(t *v1alpha1.Trigger) { r.triggersLock.Unlock() } -// updateStatus may in fact update the trigger's finalizers in addition to the status +// updateStatus may in fact update the trigger's finalizers in addition to the status. func (r *reconciler) updateStatus(trigger *v1alpha1.Trigger) (*v1alpha1.Trigger, error) { + ctx := context.TODO() objectKey := client.ObjectKey{Namespace: trigger.Namespace, Name: trigger.Name} latestTrigger := &v1alpha1.Trigger{} - if err := r.client.Get(context.TODO(), objectKey, latestTrigger); err != nil { + if err := r.client.Get(ctx, objectKey, latestTrigger); err != nil { return nil, err } @@ -343,7 +340,7 @@ func (r *reconciler) updateStatus(trigger *v1alpha1.Trigger) (*v1alpha1.Trigger, if !equality.Semantic.DeepEqual(latestTrigger.Finalizers, trigger.Finalizers) { latestTrigger.SetFinalizers(trigger.ObjectMeta.Finalizers) - if err := r.client.Update(context.TODO(), latestTrigger); err != nil { + if err := r.client.Update(ctx, latestTrigger); err != nil { return nil, err } triggerChanged = true @@ -356,21 +353,20 @@ func (r *reconciler) updateStatus(trigger *v1alpha1.Trigger) (*v1alpha1.Trigger, if triggerChanged { // Refetch latestTrigger = &v1alpha1.Trigger{} - if err := r.client.Get(context.TODO(), objectKey, latestTrigger); err != nil { + if err := r.client.Get(ctx, objectKey, latestTrigger); err != nil { return nil, err } } latestTrigger.Status = trigger.Status - if err := r.client.Status().Update(context.TODO(), latestTrigger); err != nil { + if err := r.client.Status().Update(ctx, latestTrigger); err != nil { return nil, err } return latestTrigger, nil } -// getBroker returns the Broker for Trigger 't' if exists, -// otherwise it returns an error. +// getBroker returns the Broker for Trigger 't' if exists, otherwise it returns an error. func (r *reconciler) getBroker(ctx context.Context, t *v1alpha1.Trigger) (*v1alpha1.Broker, error) { b := &v1alpha1.Broker{} name := types.NamespacedName{ @@ -381,21 +377,15 @@ func (r *reconciler) getBroker(ctx context.Context, t *v1alpha1.Trigger) (*v1alp return b, err } -// getBrokerChannel returns the Broker's channel if exists, -// otherwise it returns an error. +// getBrokerChannel returns the Broker's channel if exists, otherwise it returns an error. func (r *reconciler) getBrokerChannel(ctx context.Context, b *v1alpha1.Broker) (*v1alpha1.Channel, error) { list := &v1alpha1.ChannelList{} opts := &runtimeclient.ListOptions{ Namespace: b.Namespace, LabelSelector: labels.SelectorFromSet(broker.ChannelLabels(b)), - // TODO this is here because the fake client needs it. Remove this when it's no longer - // needed. - Raw: &metav1.ListOptions{ - TypeMeta: metav1.TypeMeta{ - APIVersion: v1alpha1.SchemeGroupVersion.String(), - Kind: "Channel", - }, - }, + // Set Raw because if we need to get more than one page, then we will put the continue token + // into opts.Raw.Continue. + Raw: &metav1.ListOptions{}, } err := r.client.List(ctx, opts, list) @@ -418,14 +408,9 @@ func (r *reconciler) getK8sService(ctx context.Context, t *v1alpha1.Trigger) (*c opts := &runtimeclient.ListOptions{ Namespace: t.Namespace, LabelSelector: labels.SelectorFromSet(k8sServiceLabels(t)), - // TODO this is here because the fake client needs it. Remove this when it's no longer - // needed. - Raw: &metav1.ListOptions{ - TypeMeta: metav1.TypeMeta{ - APIVersion: corev1.SchemeGroupVersion.String(), - Kind: "Service", - }, - }, + // Set Raw because if we need to get more than one page, then we will put the continue token + // into opts.Raw.Continue. + Raw: &metav1.ListOptions{}, } err := r.client.List(ctx, opts, list) @@ -510,14 +495,9 @@ func (r *reconciler) getVirtualService(ctx context.Context, t *v1alpha1.Trigger) opts := &runtimeclient.ListOptions{ Namespace: t.Namespace, LabelSelector: labels.SelectorFromSet(virtualServiceLabels(t)), - // TODO this is here because the fake client needs it. Remove this when it's no longer - // needed. - Raw: &metav1.ListOptions{ - TypeMeta: metav1.TypeMeta{ - APIVersion: istiov1alpha3.SchemeGroupVersion.String(), - Kind: "VirtualService", - }, - }, + // Set Raw because if we need to get more than one page, then we will put the continue token + // into opts.Raw.Continue. + Raw: &metav1.ListOptions{}, } err := r.client.List(ctx, opts, list) @@ -562,8 +542,7 @@ func (r *reconciler) reconcileVirtualService(ctx context.Context, t *v1alpha1.Tr // newVirtualService returns a placeholder virtual service object for trigger 't' and service 'svc'. func newVirtualService(t *v1alpha1.Trigger, svc *corev1.Service) *istiov1alpha3.VirtualService { - // TODO Make this work with endings other than cluster.local - destinationHost := fmt.Sprintf("%s-broker-filter.%s.svc.cluster.local", t.Spec.Broker, t.Namespace) + destinationHost := fmt.Sprintf("%s-broker-filter.%s.svc.%s", t.Spec.Broker, t.Namespace, utils.GetClusterDomainName()) return &istiov1alpha3.VirtualService{ ObjectMeta: metav1.ObjectMeta{ GenerateName: fmt.Sprintf("%s-", t.Name), @@ -583,8 +562,7 @@ func newVirtualService(t *v1alpha1.Trigger, svc *corev1.Service) *istiov1alpha3. }, Http: []istiov1alpha3.HTTPRoute{{ Rewrite: &istiov1alpha3.HTTPRewrite{ - // Never really used, so cluster.local should be a good enough ending everywhere. - Authority: fmt.Sprintf("%s.%s.triggers.cluster.local", t.Name, t.Namespace), + Authority: fmt.Sprintf("%s.%s.triggers.%s", t.Name, t.Namespace, utils.GetClusterDomainName()), }, Route: []istiov1alpha3.DestinationWeight{{ Destination: istiov1alpha3.Destination{ @@ -651,14 +629,9 @@ func (r *reconciler) getSubscription(ctx context.Context, t *v1alpha1.Trigger) ( opts := &runtimeclient.ListOptions{ Namespace: t.Namespace, LabelSelector: labels.SelectorFromSet(subscriptionLabels(t)), - // TODO this is here because the fake client needs it. Remove this when it's no longer - // needed. - Raw: &metav1.ListOptions{ - TypeMeta: metav1.TypeMeta{ - APIVersion: v1alpha1.SchemeGroupVersion.String(), - Kind: "Subscription", - }, - }, + // Set Raw because if we need to get more than one page, then we will put the continue token + // into opts.Raw.Continue. + Raw: &metav1.ListOptions{}, } err := r.client.List(ctx, opts, list) diff --git a/pkg/reconciler/v1alpha1/trigger/trigger_test.go b/pkg/reconciler/v1alpha1/trigger/trigger_test.go index bb1b8e0336e..229546a684b 100644 --- a/pkg/reconciler/v1alpha1/trigger/trigger_test.go +++ b/pkg/reconciler/v1alpha1/trigger/trigger_test.go @@ -22,6 +22,8 @@ import ( "fmt" "testing" + "github.com/knative/eventing/pkg/utils" + "github.com/knative/eventing/pkg/provisioners" "github.com/knative/eventing/pkg/reconciler/names" @@ -55,9 +57,6 @@ const ( subscriberAPIVersion = "v1" subscriberKind = "Service" subscriberName = "subscriberName" - - channelHostname = "foo.bar.svc.cluster.local" - channelProvisioner = "my-channel-provisioner" ) var ( @@ -498,7 +497,7 @@ func makeReadyTrigger() *v1alpha1.Trigger { provisioners.AddFinalizer(t, finalizerName) t.Status.InitializeConditions() t.Status.MarkBrokerExists() - t.Status.SubscriberURI = fmt.Sprintf("http://%s.%s.svc.cluster.local/", subscriberName, testNS) + t.Status.SubscriberURI = fmt.Sprintf("http://%s.%s.svc.%s/", subscriberName, testNS, utils.GetClusterDomainName()) t.Status.MarkKubernetesServiceExists() t.Status.MarkVirtualServiceExists() t.Status.MarkSubscribed() @@ -555,7 +554,7 @@ func newChannel(name string) *v1alpha1.Channel { }, Status: v1alpha1.ChannelStatus{ Address: duckv1alpha1.Addressable{ - Hostname: channelHostname, + Hostname: "any-non-empty-string", }, }, } From fef762824bd4a6af6bb223e4c83468c6428bbb41 Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Thu, 28 Feb 2019 15:36:31 -0800 Subject: [PATCH 105/128] Create an ingress channel for Brokers. --- pkg/apis/eventing/v1alpha1/broker_types.go | 38 +++- pkg/reconciler/v1alpha1/broker/broker.go | 192 +++++++++++++++++++-- pkg/reconciler/v1alpha1/trigger/trigger.go | 40 +++-- 3 files changed, 233 insertions(+), 37 deletions(-) diff --git a/pkg/apis/eventing/v1alpha1/broker_types.go b/pkg/apis/eventing/v1alpha1/broker_types.go index 7e0c5a1c9be..8accfa8f510 100644 --- a/pkg/apis/eventing/v1alpha1/broker_types.go +++ b/pkg/apis/eventing/v1alpha1/broker_types.go @@ -66,7 +66,13 @@ type BrokerSpec struct { ChannelTemplate *ChannelSpec `json:"channelTemplate,omitempty"` } -var brokerCondSet = duckv1alpha1.NewLivingConditionSet(BrokerConditionIngress, BrokerConditionChannel, BrokerConditionFilter, BrokerConditionAddressable) +var brokerCondSet = duckv1alpha1.NewLivingConditionSet( + BrokerConditionIngress, + BrokerConditionTriggerChannel, + BrokerConditionIngressChannel, + BrokerConditionFilter, + BrokerConditionAddressable, + BrokerConditionIngressSubscription) // BrokerStatus represents the current state of a Broker. type BrokerStatus struct { @@ -97,7 +103,11 @@ const ( BrokerConditionIngress duckv1alpha1.ConditionType = "IngressReady" - BrokerConditionChannel duckv1alpha1.ConditionType = "ChannelReady" + BrokerConditionTriggerChannel duckv1alpha1.ConditionType = "TriggerChannelReady" + + BrokerConditionIngressChannel duckv1alpha1.ConditionType = "IngressChannelReady" + + BrokerConditionIngressSubscription duckv1alpha1.ConditionType = "IngressSubscriptionReady" BrokerConditionFilter duckv1alpha1.ConditionType = "FilterReady" @@ -127,12 +137,28 @@ func (bs *BrokerStatus) MarkIngressFailed(err error) { brokerCondSet.Manage(bs).MarkFalse(BrokerConditionIngress, "failed", "%v", err) } -func (bs *BrokerStatus) MarkChannelReady() { - brokerCondSet.Manage(bs).MarkTrue(BrokerConditionChannel) +func (bs *BrokerStatus) MarkTriggerChannelReady() { + brokerCondSet.Manage(bs).MarkTrue(BrokerConditionTriggerChannel) +} + +func (bs *BrokerStatus) MarkTriggerChannelFailed(err error) { + brokerCondSet.Manage(bs).MarkFalse(BrokerConditionTriggerChannel, "failed", "%v", err) +} + +func (bs *BrokerStatus) MarkIngressChannelReady() { + brokerCondSet.Manage(bs).MarkTrue(BrokerConditionIngressChannel) +} + +func (bs *BrokerStatus) MarkIngressChannelFailed(err error) { + brokerCondSet.Manage(bs).MarkFalse(BrokerConditionIngressChannel, "failed", "%v", err) +} + +func (bs *BrokerStatus) MarkIngressSubscriptionReady() { + brokerCondSet.Manage(bs).MarkTrue(BrokerConditionIngressSubscription) } -func (bs *BrokerStatus) MarkChannelFailed(err error) { - brokerCondSet.Manage(bs).MarkFalse(BrokerConditionChannel, "failed", "%v", err) +func (bs *BrokerStatus) MarkIngressSubscriptionFailed(err error) { + brokerCondSet.Manage(bs).MarkFalse(BrokerConditionIngressSubscription, "failed", "%v", err) } func (bs *BrokerStatus) MarkFilterReady() { diff --git a/pkg/reconciler/v1alpha1/broker/broker.go b/pkg/reconciler/v1alpha1/broker/broker.go index 3ad3854589b..f9918cbff8e 100644 --- a/pkg/reconciler/v1alpha1/broker/broker.go +++ b/pkg/reconciler/v1alpha1/broker/broker.go @@ -61,8 +61,10 @@ const ( controllerAgentName = "broker-controller" // Name of the corev1.Events emitted from the reconciliation process. - brokerReconciled = "BrokerReconciled" - brokerUpdateStatusFailed = "BrokerUpdateStatusFailed" + brokerReconciled = "BrokerReconciled" + brokerUpdateStatusFailed = "BrokerUpdateStatusFailed" + ingressSubscriptionDeleteFailed = "IngressSubscriptionDeleteFailed" + ingressSubscriptionCreateFailed = "IngressSubscriptionCreateFailed" ) type reconciler struct { @@ -174,27 +176,30 @@ func (r *reconciler) Reconcile(request reconcile.Request) (reconcile.Result, err func (r *reconciler) reconcile(ctx context.Context, b *v1alpha1.Broker) (reconcile.Result, error) { b.Status.InitializeConditions() - // 1. Channel is created for all events. + // 1. Trigger Channel is created for all events. Triggers will Subscribe to this Channel. // 2. Filter Deployment. // 3. Ingress Deployment. // 4. K8s Services that point at the Deployments. + // 5. Ingress Channel is created to get events from Triggers back into this Broker via the + // Ingress Deployment. + // 6. Subscription from the Ingress Channel to the Ingress Service. if b.DeletionTimestamp != nil { // Everything is cleaned up by the garbage collector. return reconcile.Result{}, nil } - c, err := r.reconcileChannel(ctx, b) + triggerChan, err := r.reconcileTriggerChannel(ctx, b) if err != nil { - logging.FromContext(ctx).Error("Problem reconciling the channel", zap.Error(err)) - b.Status.MarkChannelFailed(err) + logging.FromContext(ctx).Error("Problem reconciling the trigger channel", zap.Error(err)) + b.Status.MarkTriggerChannelFailed(err) return reconcile.Result{}, err - } else if c.Status.Address.Hostname == "" { - logging.FromContext(ctx).Info("Channel is not yet ready", zap.Any("c", c)) + } else if triggerChan.Status.Address.Hostname == "" { + logging.FromContext(ctx).Info("Trigger Channel is not yet ready", zap.Any("triggerChan", triggerChan)) // Give the Channel some time to get its address. One second was chosen arbitrarily. return reconcile.Result{RequeueAfter: time.Second}, nil } - b.Status.MarkChannelReady() + b.Status.MarkTriggerChannelReady() _, err = r.reconcileFilterDeployment(ctx, b) if err != nil { @@ -208,7 +213,7 @@ func (r *reconciler) reconcile(ctx context.Context, b *v1alpha1.Broker) (reconci } b.Status.MarkFilterReady() - _, err = r.reconcileIngressDeployment(ctx, b, c) + _, err = r.reconcileIngressDeployment(ctx, b, triggerChan) if err != nil { logging.FromContext(ctx).Error("Problem reconciling ingress Deployment", zap.Error(err)) return reconcile.Result{}, err @@ -222,6 +227,26 @@ func (r *reconciler) reconcile(ctx context.Context, b *v1alpha1.Broker) (reconci b.Status.MarkIngressReady() b.Status.SetAddress(names.ServiceHostName(svc.Name, svc.Namespace)) + ingressChan, err := r.reconcileIngressChannel(ctx, b) + if err != nil { + logging.FromContext(ctx).Error("Problem reconciling the ingress channel", zap.Error(err)) + b.Status.MarkIngressChannelFailed(err) + return reconcile.Result{}, err + } else if ingressChan.Status.Address.Hostname == "" { + logging.FromContext(ctx).Info("Ingress Channel is not yet ready", zap.Any("ingressChan", ingressChan)) + // Give the Channel some time to get its address. One second was chosen arbitrarily. + return reconcile.Result{RequeueAfter: time.Second}, nil + } + b.Status.MarkIngressChannelReady() + + _, err = r.reconcileIngressSubscription(ctx, b, ingressChan, svc) + if err != nil { + logging.FromContext(ctx).Error("Problem reconciling the ingress subscription", zap.Error(err)) + b.Status.MarkIngressSubscriptionFailed(err) + return reconcile.Result{}, err + } + b.Status.MarkIngressSubscriptionReady() + return reconcile.Result{}, nil } @@ -281,12 +306,26 @@ func (r *reconciler) reconcileFilterService(ctx context.Context, b *v1alpha1.Bro return r.reconcileService(ctx, expected) } +func (r *reconciler) reconcileTriggerChannel(ctx context.Context, b *v1alpha1.Broker) (*v1alpha1.Channel, error) { + get := func() (*v1alpha1.Channel, error) { + return r.getChannel(ctx, b, labels.SelectorFromSet(TriggerChannelLabels(b))) + } + return r.reconcileChannel(ctx, get, newTriggerChannel(b)) +} + +func (r *reconciler) reconcileIngressChannel(ctx context.Context, b *v1alpha1.Broker) (*v1alpha1.Channel, error) { + get := func() (*v1alpha1.Channel, error) { + return r.getChannel(ctx, b, labels.SelectorFromSet(IngressChannelLabels(b))) + } + return r.reconcileChannel(ctx, get, newIngressChannel(b)) +} + // reconcileChannel reconciles Broker's 'b' underlying channel. -func (r *reconciler) reconcileChannel(ctx context.Context, b *v1alpha1.Broker) (*v1alpha1.Channel, error) { - c, err := r.getChannel(ctx, b) +func (r *reconciler) reconcileChannel(ctx context.Context, get func() (*v1alpha1.Channel, error), newChan *v1alpha1.Channel) (*v1alpha1.Channel, error) { + c, err := get() // If the resource doesn't exist, we'll create it if k8serrors.IsNotFound(err) { - c = newChannel(b) + c = newChan err = r.client.Create(ctx, c) if err != nil { return nil, err @@ -311,11 +350,11 @@ func (r *reconciler) reconcileChannel(ctx context.Context, b *v1alpha1.Broker) ( } // getChannel returns the Channel object for Broker 'b' if exists, otherwise it returns an error. -func (r *reconciler) getChannel(ctx context.Context, b *v1alpha1.Broker) (*v1alpha1.Channel, error) { +func (r *reconciler) getChannel(ctx context.Context, b *v1alpha1.Broker, ls labels.Selector) (*v1alpha1.Channel, error) { list := &v1alpha1.ChannelList{} opts := &runtimeclient.ListOptions{ Namespace: b.Namespace, - LabelSelector: labels.SelectorFromSet(ChannelLabels(b)), + LabelSelector: ls, // Set Raw because if we need to get more than one page, then we will put the continue token // into opts.Raw.Continue. Raw: &metav1.ListOptions{}, @@ -334,8 +373,16 @@ func (r *reconciler) getChannel(ctx context.Context, b *v1alpha1.Broker) (*v1alp return nil, k8serrors.NewNotFound(schema.GroupResource{}, "") } +func newTriggerChannel(b *v1alpha1.Broker) *v1alpha1.Channel { + return newChannel(b, TriggerChannelLabels(b)) +} + +func newIngressChannel(b *v1alpha1.Broker) *v1alpha1.Channel { + return newChannel(b, IngressChannelLabels(b)) +} + // newChannel creates a new Channel for Broker 'b'. -func newChannel(b *v1alpha1.Broker) *v1alpha1.Channel { +func newChannel(b *v1alpha1.Broker, l map[string]string) *v1alpha1.Channel { var spec v1alpha1.ChannelSpec if b.Spec.ChannelTemplate != nil { spec = *b.Spec.ChannelTemplate @@ -345,7 +392,7 @@ func newChannel(b *v1alpha1.Broker) *v1alpha1.Channel { ObjectMeta: metav1.ObjectMeta{ Namespace: b.Namespace, GenerateName: fmt.Sprintf("%s-broker-", b.Name), - Labels: ChannelLabels(b), + Labels: l, OwnerReferences: []metav1.OwnerReference{ *metav1.NewControllerRef(b, schema.GroupVersionKind{ Group: v1alpha1.SchemeGroupVersion.Group, @@ -358,13 +405,20 @@ func newChannel(b *v1alpha1.Broker) *v1alpha1.Channel { } } -func ChannelLabels(b *v1alpha1.Broker) map[string]string { +func TriggerChannelLabels(b *v1alpha1.Broker) map[string]string { return map[string]string{ "eventing.knative.dev/broker": b.Name, "eventing.knative.dev/brokerEverything": "true", } } +func IngressChannelLabels(b *v1alpha1.Broker) map[string]string { + return map[string]string{ + "eventing.knative.dev/broker": b.Name, + "eventing.knative.dev/brokerIngress": "true", + } +} + // reconcileDeployment reconciles the K8s Deployment 'd'. func (r *reconciler) reconcileDeployment(ctx context.Context, d *v1.Deployment) (*v1.Deployment, error) { name := types.NamespacedName{ @@ -440,3 +494,105 @@ func (r *reconciler) reconcileIngressService(ctx context.Context, b *v1alpha1.Br expected := resources.MakeIngressService(b) return r.reconcileService(ctx, expected) } + +func (r *reconciler) reconcileIngressSubscription(ctx context.Context, b *v1alpha1.Broker, c *v1alpha1.Channel, svc *corev1.Service) (*v1alpha1.Subscription, error) { + expected := makeSubscription(b, c, svc) + + sub, err := r.getIngressSubscription(ctx, b) + // If the resource doesn't exist, we'll create it + if k8serrors.IsNotFound(err) { + sub = expected + err = r.client.Create(ctx, sub) + if err != nil { + return nil, err + } + return sub, nil + } else if err != nil { + return nil, err + } + + // Update Subscription if it has changed. Ignore the generation. + expected.Spec.DeprecatedGeneration = sub.Spec.DeprecatedGeneration + if !equality.Semantic.DeepDerivative(expected.Spec, sub.Spec) { + // Given that spec.channel is immutable, we cannot just update the subscription. We delete + // it instead, and re-create it. + err = r.client.Delete(ctx, sub) + if err != nil { + logging.FromContext(ctx).Info("Cannot delete subscription", zap.Error(err)) + r.recorder.Eventf(b, corev1.EventTypeWarning, ingressSubscriptionDeleteFailed, "Delete Broker Ingress' subscription failed: %v", err) + return nil, err + } + sub = expected + err = r.client.Create(ctx, sub) + if err != nil { + logging.FromContext(ctx).Info("Cannot create subscription", zap.Error(err)) + r.recorder.Eventf(b, corev1.EventTypeWarning, ingressSubscriptionCreateFailed, "Create Broker Ingress' subscription failed: %v", err) + return nil, err + } + } + return sub, nil +} + +// getSubscription returns the subscription of trigger 't' if exists, +// otherwise it returns an error. +func (r *reconciler) getIngressSubscription(ctx context.Context, b *v1alpha1.Broker) (*v1alpha1.Subscription, error) { + list := &v1alpha1.SubscriptionList{} + opts := &runtimeclient.ListOptions{ + Namespace: b.Namespace, + LabelSelector: labels.SelectorFromSet(ingressSubscriptionLabels(b)), + // Set Raw because if we need to get more than one page, then we will put the continue token + // into opts.Raw.Continue. + Raw: &metav1.ListOptions{}, + } + + err := r.client.List(ctx, opts, list) + if err != nil { + return nil, err + } + for _, s := range list.Items { + if metav1.IsControlledBy(&s, b) { + return &s, nil + } + } + + return nil, k8serrors.NewNotFound(schema.GroupResource{}, "") +} + +// makeSubscription returns a placeholder subscription for trigger 't', channel 'c', and service 'svc'. +func makeSubscription(b *v1alpha1.Broker, c *v1alpha1.Channel, svc *corev1.Service) *v1alpha1.Subscription { + return &v1alpha1.Subscription{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: b.Namespace, + GenerateName: fmt.Sprintf("internal-ingress-%s-", b.Name), + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(b, schema.GroupVersionKind{ + Group: v1alpha1.SchemeGroupVersion.Group, + Version: v1alpha1.SchemeGroupVersion.Version, + Kind: "Broker", + }), + }, + Labels: ingressSubscriptionLabels(b), + }, + Spec: v1alpha1.SubscriptionSpec{ + Channel: corev1.ObjectReference{ + APIVersion: v1alpha1.SchemeGroupVersion.String(), + Kind: "Channel", + Name: c.Name, + }, + Subscriber: &v1alpha1.SubscriberSpec{ + Ref: &corev1.ObjectReference{ + APIVersion: "v1", + Kind: "Service", + Name: svc.Name, + }, + }, + }, + } +} + +func ingressSubscriptionLabels(b *v1alpha1.Broker) map[string]string { + return map[string]string{ + "eventing.knative.dev/broker": b.Name, + "eventing.knative.dev/brokerIngress": "true", + } +} diff --git a/pkg/reconciler/v1alpha1/trigger/trigger.go b/pkg/reconciler/v1alpha1/trigger/trigger.go index 70529f16740..9922b65234c 100644 --- a/pkg/reconciler/v1alpha1/trigger/trigger.go +++ b/pkg/reconciler/v1alpha1/trigger/trigger.go @@ -236,9 +236,14 @@ func (r *reconciler) reconcile(ctx context.Context, t *v1alpha1.Trigger) error { } t.Status.MarkBrokerExists() - c, err := r.getBrokerChannel(ctx, b) + brokerTrigger, err := r.getBrokerTriggerChannel(ctx, b) if err != nil { - logging.FromContext(ctx).Error("Unable to get the Broker's Channel", zap.Error(err)) + logging.FromContext(ctx).Error("Unable to get the Broker's Trigger Channel", zap.Error(err)) + return err + } + brokerIngress, err := r.getBrokerIngressChannel(ctx, b) + if err != nil { + logging.FromContext(ctx).Error("Unable to get the Broker's Ingress Channel", zap.Error(err)) return err } @@ -263,7 +268,7 @@ func (r *reconciler) reconcile(ctx context.Context, t *v1alpha1.Trigger) error { } t.Status.MarkVirtualServiceExists() - _, err = r.subscribeToBrokerChannel(ctx, t, c, svc) + _, err = r.subscribeToBrokerChannel(ctx, t, brokerTrigger, brokerIngress, svc) if err != nil { logging.FromContext(ctx).Error("Unable to Subscribe", zap.Error(err)) t.Status.MarkNotSubscribed("notSubscribed", "%v", err) @@ -378,11 +383,21 @@ func (r *reconciler) getBroker(ctx context.Context, t *v1alpha1.Trigger) (*v1alp } // getBrokerChannel returns the Broker's channel if exists, otherwise it returns an error. -func (r *reconciler) getBrokerChannel(ctx context.Context, b *v1alpha1.Broker) (*v1alpha1.Channel, error) { +func (r *reconciler) getBrokerTriggerChannel(ctx context.Context, b *v1alpha1.Broker) (*v1alpha1.Channel, error) { + return r.getChannel(ctx, b, labels.SelectorFromSet(broker.TriggerChannelLabels(b))) +} + +// getBrokerChannel returns the Broker's channel if exists, otherwise it returns an error. +func (r *reconciler) getBrokerIngressChannel(ctx context.Context, b *v1alpha1.Broker) (*v1alpha1.Channel, error) { + return r.getChannel(ctx, b, labels.SelectorFromSet(broker.IngressChannelLabels(b))) +} + +// getBrokerChannel returns the Broker's channel if exists, otherwise it returns an error. +func (r *reconciler) getChannel(ctx context.Context, b *v1alpha1.Broker, ls labels.Selector) (*v1alpha1.Channel, error) { list := &v1alpha1.ChannelList{} opts := &runtimeclient.ListOptions{ Namespace: b.Namespace, - LabelSelector: labels.SelectorFromSet(broker.ChannelLabels(b)), + LabelSelector: ls, // Set Raw because if we need to get more than one page, then we will put the continue token // into opts.Raw.Continue. Raw: &metav1.ListOptions{}, @@ -584,8 +599,8 @@ func virtualServiceLabels(t *v1alpha1.Trigger) map[string]string { } // subscribeToBrokerChannel subscribes service 'svc' to Broker's channel 'c'. -func (r *reconciler) subscribeToBrokerChannel(ctx context.Context, t *v1alpha1.Trigger, c *v1alpha1.Channel, svc *corev1.Service) (*v1alpha1.Subscription, error) { - expected := makeSubscription(t, c, svc) +func (r *reconciler) subscribeToBrokerChannel(ctx context.Context, t *v1alpha1.Trigger, brokerTrigger, brokerIngress *v1alpha1.Channel, svc *corev1.Service) (*v1alpha1.Subscription, error) { + expected := makeSubscription(t, brokerTrigger, brokerIngress, svc) sub, err := r.getSubscription(ctx, t) // If the resource doesn't exist, we'll create it @@ -603,8 +618,8 @@ func (r *reconciler) subscribeToBrokerChannel(ctx context.Context, t *v1alpha1.T // Update Subscription if it has changed. Ignore the generation. expected.Spec.DeprecatedGeneration = sub.Spec.DeprecatedGeneration if !equality.Semantic.DeepDerivative(expected.Spec, sub.Spec) { - // Given that the backing channel spec is immutable, we cannot just update the subscription. - // We delete it instead, and re-create it. + // Given that spec.channel is immutable, we cannot just update the subscription. We delete + // it instead, and re-create it. err = r.client.Delete(ctx, sub) if err != nil { logging.FromContext(ctx).Info("Cannot delete subscription", zap.Error(err)) @@ -648,7 +663,7 @@ func (r *reconciler) getSubscription(ctx context.Context, t *v1alpha1.Trigger) ( } // makeSubscription returns a placeholder subscription for trigger 't', channel 'c', and service 'svc'. -func makeSubscription(t *v1alpha1.Trigger, c *v1alpha1.Channel, svc *corev1.Service) *v1alpha1.Subscription { +func makeSubscription(t *v1alpha1.Trigger, brokerTrigger, brokerIngress *v1alpha1.Channel, svc *corev1.Service) *v1alpha1.Subscription { return &v1alpha1.Subscription{ ObjectMeta: metav1.ObjectMeta{ Namespace: t.Namespace, @@ -666,7 +681,7 @@ func makeSubscription(t *v1alpha1.Trigger, c *v1alpha1.Channel, svc *corev1.Serv Channel: corev1.ObjectReference{ APIVersion: v1alpha1.SchemeGroupVersion.String(), Kind: "Channel", - Name: c.Name, + Name: brokerTrigger.Name, }, Subscriber: &v1alpha1.SubscriberSpec{ Ref: &corev1.ObjectReference{ @@ -675,12 +690,11 @@ func makeSubscription(t *v1alpha1.Trigger, c *v1alpha1.Channel, svc *corev1.Serv Name: svc.Name, }, }, - // TODO This pushes directly into the Channel, it should probably point at the Broker ingress instead. Reply: &v1alpha1.ReplyStrategy{ Channel: &corev1.ObjectReference{ APIVersion: v1alpha1.SchemeGroupVersion.String(), Kind: "Channel", - Name: c.Name, + Name: brokerIngress.Name, }, }, }, From 6dd814f5f9fd181a6b0c2beb53e1033aa28e3676 Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Fri, 1 Mar 2019 17:02:01 -0800 Subject: [PATCH 106/128] adding cloudevents/sdk-go --- Gopkg.lock | 19 ++ Gopkg.toml | 6 + cmd/broker/filter/main.go | 6 +- pkg/broker/ce_receiver.go | 322 ++++++++++++++++++ pkg/broker/{ => old}/receiver.go | 2 +- pkg/broker/{ => old}/receiver_test.go | 2 +- third_party/VENDOR-LICENSE | 207 +++++++++++ vendor/github.com/cloudevents/sdk-go/LICENSE | 201 +++++++++++ .../sdk-go/pkg/cloudevents/client/client.go | 76 +++++ .../pkg/cloudevents/client/client_http.go | 54 +++ .../pkg/cloudevents/client/client_nats.go | 52 +++ .../pkg/cloudevents/client/defaulters.go | 62 ++++ .../sdk-go/pkg/cloudevents/client/options.go | 177 ++++++++++ .../sdk-go/pkg/cloudevents/codec/jsoncodec.go | 160 +++++++++ .../sdk-go/pkg/cloudevents/content_type.go | 34 ++ .../sdk-go/pkg/cloudevents/datacodec/codec.go | 48 +++ .../pkg/cloudevents/datacodec/json/data.go | 66 ++++ .../pkg/cloudevents/datacodec/xml/data.go | 59 ++++ .../cloudevents/sdk-go/pkg/cloudevents/doc.go | 86 +++++ .../sdk-go/pkg/cloudevents/event.go | 73 ++++ .../sdk-go/pkg/cloudevents/eventcontext.go | 34 ++ .../pkg/cloudevents/eventcontext_v01.go | 218 ++++++++++++ .../pkg/cloudevents/eventcontext_v02.go | 203 +++++++++++ .../pkg/cloudevents/eventcontext_v03.go | 181 ++++++++++ .../sdk-go/pkg/cloudevents/transport/codec.go | 10 + .../pkg/cloudevents/transport/http/codec.go | 214 ++++++++++++ .../cloudevents/transport/http/codec_v01.go | 219 ++++++++++++ .../cloudevents/transport/http/codec_v02.go | 272 +++++++++++++++ .../cloudevents/transport/http/codec_v03.go | 280 +++++++++++++++ .../cloudevents/transport/http/encoding.go | 76 +++++ .../pkg/cloudevents/transport/http/message.go | 69 ++++ .../cloudevents/transport/http/transport.go | 164 +++++++++ .../pkg/cloudevents/transport/message.go | 8 + .../pkg/cloudevents/transport/nats/codec.go | 54 +++ .../cloudevents/transport/nats/codec_v02.go | 48 +++ .../cloudevents/transport/nats/codec_v03.go | 48 +++ .../cloudevents/transport/nats/encoding.go | 45 +++ .../pkg/cloudevents/transport/nats/message.go | 31 ++ .../cloudevents/transport/nats/transport.go | 91 +++++ .../pkg/cloudevents/transport/transport.go | 17 + .../sdk-go/pkg/cloudevents/types/allocate.go | 36 ++ .../sdk-go/pkg/cloudevents/types/timestamp.go | 51 +++ .../sdk-go/pkg/cloudevents/types/urlref.go | 44 +++ 43 files changed, 4120 insertions(+), 5 deletions(-) create mode 100644 pkg/broker/ce_receiver.go rename pkg/broker/{ => old}/receiver.go (99%) rename pkg/broker/{ => old}/receiver_test.go (99%) create mode 100644 vendor/github.com/cloudevents/sdk-go/LICENSE create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/client.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/client_http.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/client_nats.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/defaulters.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/options.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/codec/jsoncodec.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/content_type.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/codec.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/json/data.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/xml/data.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/doc.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/event.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/eventcontext.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/eventcontext_v01.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/eventcontext_v02.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/eventcontext_v03.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/codec.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/codec.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/codec_v01.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/codec_v02.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/codec_v03.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/encoding.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/message.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/transport.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/message.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/nats/codec.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/nats/codec_v02.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/nats/codec_v03.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/nats/encoding.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/nats/message.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/nats/transport.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/transport.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/types/allocate.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/types/timestamp.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/types/urlref.go diff --git a/Gopkg.lock b/Gopkg.lock index bb6fd7165f4..0f2894ceb91 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -57,6 +57,24 @@ revision = "c618e605e15c0d7535f6c96ff8efbb0dba4fd66c" version = "v2.1.15" +[[projects]] + digest = "1:d9cc48e9d4caa22c2af5b18f404d3e4a34b05e6723b002c0c18a39762a14777d" + name = "github.com/cloudevents/sdk-go" + packages = [ + "pkg/cloudevents", + "pkg/cloudevents/client", + "pkg/cloudevents/codec", + "pkg/cloudevents/datacodec", + "pkg/cloudevents/datacodec/json", + "pkg/cloudevents/datacodec/xml", + "pkg/cloudevents/transport", + "pkg/cloudevents/transport/http", + "pkg/cloudevents/transport/nats", + "pkg/cloudevents/types", + ] + pruneopts = "NUT" + revision = "d50361a5655081514f406b4e672d72e9886c17ad" + [[projects]] digest = "1:ffe9824d294da03b391f44e1ae8281281b4afc1bdaa9588c9097785e3af10cec" name = "github.com/davecgh/go-spew" @@ -1235,6 +1253,7 @@ "cloud.google.com/go/pubsub", "github.com/Shopify/sarama", "github.com/bsm/sarama-cluster", + "github.com/cloudevents/sdk-go/pkg/cloudevents/client", "github.com/fsnotify/fsnotify", "github.com/golang/glog", "github.com/google/go-cmp/cmp", diff --git a/Gopkg.toml b/Gopkg.toml index 5056356a75c..c06ab55e95b 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -129,3 +129,9 @@ required = [ [[override]] name = "github.com/nats-io/nats-streaming-server" version = "0.11.0" + +# Use CloudEvents, master as of Feb 28, 2019 🎂 +# Will use releases as soon as they get releases going. +[[override]] + name = "github.com/cloudevents/sdk-go" + revision = "d50361a5655081514f406b4e672d72e9886c17ad" diff --git a/cmd/broker/filter/main.go b/cmd/broker/filter/main.go index f60b33340e0..2fecd632f92 100644 --- a/cmd/broker/filter/main.go +++ b/cmd/broker/filter/main.go @@ -58,10 +58,10 @@ func main() { // We are running both the receiver (takes messages in from the Broker) and the dispatcher (send // the messages to the triggers' subscribers) in this binary. - _, runnable := broker.New(logger, mgr.GetClient()) - err = mgr.Add(runnable) + receiver := broker.New(logger, mgr.GetClient()) + err = mgr.Add(receiver) if err != nil { - logger.Fatal("Unable to start the receivers runnable", zap.Error(err), zap.Any("runnable", runnable)) + logger.Fatal("Unable to start the receivers runnable", zap.Error(err), zap.Any("receiver", receiver)) } // Set up signals so we handle the first shutdown signal gracefully. diff --git a/pkg/broker/ce_receiver.go b/pkg/broker/ce_receiver.go new file mode 100644 index 00000000000..538c5636cc1 --- /dev/null +++ b/pkg/broker/ce_receiver.go @@ -0,0 +1,322 @@ +/* + * Copyright 2019 The Knative Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package broker + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/url" + "reflect" + + "github.com/cloudevents/sdk-go/pkg/cloudevents" + + cehttp "github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http" + + eventingv1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" + "github.com/knative/eventing/pkg/provisioners" + "go.uber.org/zap" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + defaultPort = 8080 +) + +// Receiver parses Cloud Events, determines if they pass a filter, and sends them to a subscriber. +type Receiver struct { + logger *zap.Logger + client client.Client + + port int + + httpClient *http.Client + codec cehttp.Codec +} + +// New creates a new Receiver and its associated MessageReceiver. The caller is responsible for +// Start()ing the returned MessageReceiver. +func New(logger *zap.Logger, client client.Client) *Receiver { + r := &Receiver{ + logger: logger, + client: client, + + port: defaultPort, + + httpClient: &http.Client{}, + codec: cehttp.Codec{ + Encoding: cehttp.BinaryV01, + }, + } + return r +} + +var _ http.Handler = &Receiver{} + +// Start begins to receive messages for the receiver. +// +// Only HTTP POST requests to the root path (/) are accepted. If other paths or +// methods are needed, use the HandleRequest method directly with another HTTP +// server. +// +// This method will block until a message is received on the stop channel. +func (r *Receiver) Start(stopCh <-chan struct{}) error { + svr := r.start() + defer r.stop(svr) + + <-stopCh + return nil +} + +func (r *Receiver) start() *http.Server { + srv := &http.Server{ + Addr: fmt.Sprintf(":%d", r.port), + Handler: r, + } + r.logger.Info("Starting web server", zap.String("addr", srv.Addr)) + go func() { + if err := srv.ListenAndServe(); err != http.ErrServerClosed { + r.logger.Error("HttpServer: ListenAndServe() error", zap.Error(err)) + } + }() + return srv +} + +func (r *Receiver) stop(srv *http.Server) { + r.logger.Info("Shutdown web server") + if err := srv.Shutdown(nil); err != nil { + r.logger.Error("Error shutting down the HTTP Server", zap.Error(err)) + } +} + +func (r *Receiver) ServeHTTP(w http.ResponseWriter, req *http.Request) { + if req.URL.Path != "/" { + w.WriteHeader(http.StatusNotFound) + return + } + if req.Method != http.MethodPost { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + + triggerRef, err := provisioners.ParseChannel(req.Host) + if err != nil { + r.logger.Error("Unable to parse host as a trigger", zap.Error(err), zap.String("host", req.Host)) + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte(`{"error":"Bad Host"}`)) + return + } + + event, err := r.decodeHTTPRequest(req) + if err != nil { + r.logger.Error("Error decoding HTTP Request", zap.Error(err)) + w.WriteHeader(http.StatusBadRequest) + return + } + + responseEvent, err := r.sendEvent(triggerRef, event) + if err != nil { + r.logger.Error("Error sending the event", zap.Error(err)) + w.WriteHeader(http.StatusBadRequest) + return + } + if responseEvent == nil { + w.WriteHeader(http.StatusAccepted) + return + } + + encodedEvent, err := r.codec.Encode(*event) + if err != nil { + r.logger.Error("Error encoding the response event", zap.Error(err)) + w.WriteHeader(http.StatusInternalServerError) + return + } + msg, ok := encodedEvent.(*cehttp.Message) + if !ok { + r.logger.Error("Error casting the encoded response event", zap.Error(err), zap.Any("encodedEvent", reflect.TypeOf(encodedEvent))) + w.WriteHeader(http.StatusInternalServerError) + return + } + for n, v := range msg.Header { + w.Header().Del(n) + for _, s := range v { + w.Header().Add(n, s) + } + } + w.WriteHeader(http.StatusAccepted) + _, err = w.Write(msg.Body) + if err != nil { + r.logger.Error("Error writing the response body", zap.Error(err)) + w.WriteHeader(http.StatusInternalServerError) + return + } +} + +func (r *Receiver) decodeHTTPRequest(req *http.Request) (*cloudevents.Event, error) { + return r.decodeHTTP(req.Header, req.Body) +} + +func (r *Receiver) decodeHTTPResponse(resp *http.Response) (*cloudevents.Event, error) { + return r.decodeHTTP(resp.Header, resp.Body) +} + +func (r *Receiver) decodeHTTP(headers http.Header, bodyReadCloser io.ReadCloser) (*cloudevents.Event, error) { + body, err := ioutil.ReadAll(bodyReadCloser) + if err != nil { + return nil, err + } + msg := &cehttp.Message{ + Header: headers, + Body: body, + } + + return r.codec.Decode(msg) +} + +// sendEvent sends an event to a subscriber if the trigger filter passes. +func (r *Receiver) sendEvent(trigger provisioners.ChannelReference, event *cloudevents.Event) (*cloudevents.Event, error) { + r.logger.Debug("Received message", zap.Any("triggerRef", trigger)) + ctx := context.Background() + + t, err := r.getTrigger(ctx, trigger) + if err != nil { + r.logger.Info("Unable to get the Trigger", zap.Error(err), zap.Any("triggerRef", trigger)) + return nil, err + } + + subscriberURIString := t.Status.SubscriberURI + if subscriberURIString == "" { + r.logger.Error("Unable to read subscriberURI") + return nil, errors.New("unable to read subscriberURI") + } + subscriberURI, err := url.Parse(subscriberURIString) + if err != nil { + // sojemthing + return nil, err + } + + if !r.shouldSendMessage(&t.Spec, event) { + r.logger.Debug("Message did not pass filter", zap.Any("triggerRef", trigger)) + return nil, nil + } + + return r.dispatch(event, subscriberURI) +} + +func (r *Receiver) dispatch(event *cloudevents.Event, uri *url.URL) (*cloudevents.Event, error) { + encodedEvent, err := r.codec.Encode(*event) + if err != nil { + return nil, err + } + msg, ok := encodedEvent.(*cehttp.Message) + if !ok { + return nil, errors.New("msg was not a cehttp.Message") + } + + req, err := http.NewRequest(http.MethodPost, uri.String(), bytes.NewReader(msg.Body)) + if err != nil { + return nil, err + } + + req.Header = msg.Header + res, err := r.httpClient.Do(req) + if err != nil { + return nil, err + } + if res == nil { + // I don't think this is actually reachable with http.Client.Do(), but just to be sure we + // check anyway. + return nil, errors.New("non-error nil result from http.Client.Do()") + } + + defer res.Body.Close() + if isFailure(res.StatusCode) { + // reject non-successful responses + return nil, fmt.Errorf("unexpected HTTP response, expected 2xx, got %d", res.StatusCode) + } + + return r.decodeHTTPResponse(res) +} + +// isFailure returns true if the status code is not a successful HTTP status. +func isFailure(statusCode int) bool { + return statusCode < http.StatusOK /* 200 */ || + statusCode >= http.StatusMultipleChoices /* 300 */ +} + +// Initialize the client. Mainly intended to load stuff in its cache. +func (r *Receiver) initClient() error { + // We list triggers so that we can load the client's cache. Otherwise, on receiving an event, it + // may not find the trigger and would return an error. + opts := &client.ListOptions{ + // Set Raw because if we need to get more than one page, then we will put the continue token + // into opts.Raw.Continue. + Raw: &metav1.ListOptions{}, + } + for { + tl := &eventingv1alpha1.TriggerList{} + if err := r.client.List(context.TODO(), opts, tl); err != nil { + return err + } + if tl.Continue != "" { + opts.Raw.Continue = tl.Continue + } else { + break + } + } + return nil +} + +func (r *Receiver) getTrigger(ctx context.Context, ref provisioners.ChannelReference) (*eventingv1alpha1.Trigger, error) { + t := &eventingv1alpha1.Trigger{} + err := r.client.Get(ctx, + types.NamespacedName{ + Namespace: ref.Namespace, + Name: ref.Name, + }, + t) + return t, err +} + +// shouldSendMessage determines whether message 'm' should be sent based on the triggerSpec 'ts'. +// Currently it supports exact matching on type and/or source of events. +func (r *Receiver) shouldSendMessage(ts *eventingv1alpha1.TriggerSpec, event *cloudevents.Event) bool { + if ts.Filter == nil || ts.Filter.SourceAndType == nil { + r.logger.Error("No filter specified") + return false + } + filterType := ts.Filter.SourceAndType.Type + if filterType != eventingv1alpha1.TriggerAnyFilter && filterType != event.Type() { + r.logger.Debug("Wrong type", zap.String("trigger.spec.filter.sourceAndType.type", filterType), zap.String("event.Type()", event.Type())) + return false + } + filterSource := ts.Filter.SourceAndType.Source + s := event.Context.AsV01().Source + actualSource := s.String() + //actualSource := event.Context.AsV01().Source.String() + if filterSource != eventingv1alpha1.TriggerAnyFilter && filterSource != actualSource { + r.logger.Debug("Wrong source", zap.String("trigger.spec.filter.sourceAndType.source", filterSource), zap.String("message.source", actualSource)) + return false + } + return true +} diff --git a/pkg/broker/receiver.go b/pkg/broker/old/receiver.go similarity index 99% rename from pkg/broker/receiver.go rename to pkg/broker/old/receiver.go index b591a70ced2..7bbcdea8a18 100644 --- a/pkg/broker/receiver.go +++ b/pkg/broker/old/receiver.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package broker +package old import ( "context" diff --git a/pkg/broker/receiver_test.go b/pkg/broker/old/receiver_test.go similarity index 99% rename from pkg/broker/receiver_test.go rename to pkg/broker/old/receiver_test.go index 02a8518a351..377fd26414b 100644 --- a/pkg/broker/receiver_test.go +++ b/pkg/broker/old/receiver_test.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package broker +package old import ( "errors" diff --git a/third_party/VENDOR-LICENSE b/third_party/VENDOR-LICENSE index e7c5fe8e690..bdfec23c2a0 100644 --- a/third_party/VENDOR-LICENSE +++ b/third_party/VENDOR-LICENSE @@ -287,6 +287,213 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +=========================================================== +Import: github.com/knative/eventing/vendor/github.com/cloudevents/sdk-go + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + + =========================================================== Import: github.com/knative/eventing/vendor/github.com/davecgh/go-spew diff --git a/vendor/github.com/cloudevents/sdk-go/LICENSE b/vendor/github.com/cloudevents/sdk-go/LICENSE new file mode 100644 index 00000000000..261eeb9e9f8 --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/client.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/client.go new file mode 100644 index 00000000000..da5f67a4254 --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/client.go @@ -0,0 +1,76 @@ +package client + +import ( + "context" + "fmt" + "github.com/cloudevents/sdk-go/pkg/cloudevents" + "github.com/cloudevents/sdk-go/pkg/cloudevents/transport" + "github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http" + "github.com/cloudevents/sdk-go/pkg/cloudevents/transport/nats" +) + +type Receiver func(event cloudevents.Event) + +type Client interface { + Send(ctx context.Context, event cloudevents.Event) error + StartReceiver(ctx context.Context, fn Receiver) error + + Receive(event cloudevents.Event) +} + +type ceClient struct { + transport transport.Sender + receiver Receiver + + eventDefaulterFns []EventDefaulter +} + +func (c *ceClient) Send(ctx context.Context, event cloudevents.Event) error { + // Confirm we have a transport set. + if c.transport == nil { + return fmt.Errorf("client not ready, transport not initalized") + } + // Apply the defaulter chain to the incoming event. + if len(c.eventDefaulterFns) > 0 { + for _, fn := range c.eventDefaulterFns { + event = fn(event) + } + } + // Validate the event conforms to the CloudEvents Spec. + if err := event.Validate(); err != nil { + return err + } + // Send the event over the transport. + return c.transport.Send(ctx, event) +} + +func (c *ceClient) Receive(event cloudevents.Event) { + if c.receiver != nil { + c.receiver(event) + } +} + +func (c *ceClient) StartReceiver(ctx context.Context, fn Receiver) error { + if c.transport == nil { + return fmt.Errorf("client not ready, transport not initalized") + } + + if t, ok := c.transport.(*http.Transport); ok { + return c.startHTTPReceiver(ctx, t, fn) + } + + if t, ok := c.transport.(*nats.Transport); ok { + return c.startNATSReceiver(ctx, t, fn) + } + + return fmt.Errorf("unknown transport type: %T", c.transport) +} + +func (c *ceClient) applyOptions(opts ...Option) error { + for _, fn := range opts { + if err := fn(c); err != nil { + return err + } + } + return nil +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/client_http.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/client_http.go new file mode 100644 index 00000000000..83135945afa --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/client_http.go @@ -0,0 +1,54 @@ +package client + +import ( + "context" + "fmt" + cloudeventshttp "github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http" + "log" + "net/http" +) + +func NewHTTPClient(opts ...Option) (Client, error) { + c := &ceClient{ + transport: &cloudeventshttp.Transport{ + // Default the request method. + Req: &http.Request{ + Method: http.MethodPost, + }, + }, + } + + if err := c.applyOptions(opts...); err != nil { + return nil, err + } + return c, nil +} + +func StartHTTPReceiver(ctx context.Context, fn Receiver, opts ...Option) (Client, error) { + c, err := NewHTTPClient(opts...) + if err != nil { + return nil, err + } + + if err := c.StartReceiver(ctx, fn); err != nil { + return nil, err + } + return c, nil +} + +func (c *ceClient) startHTTPReceiver(ctx context.Context, t *cloudeventshttp.Transport, fn Receiver) error { + if c.receiver != nil { + return fmt.Errorf("client already has a receiver") + } + if t.Receiver != nil { + return fmt.Errorf("transport already has a receiver") + } + c.receiver = fn + t.Receiver = c + + go func() { + log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", t.GetPort()), t)) + }() + + return nil +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/client_nats.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/client_nats.go new file mode 100644 index 00000000000..1033e116092 --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/client_nats.go @@ -0,0 +1,52 @@ +package client + +import ( + "context" + "fmt" + cloudeventsnats "github.com/cloudevents/sdk-go/pkg/cloudevents/transport/nats" + "github.com/nats-io/go-nats" + "log" +) + +func NewNATSClient(natsServer, subject string, opts ...Option) (Client, error) { + conn, err := nats.Connect(natsServer) + if err != nil { + return nil, err + } + transport := cloudeventsnats.Transport{ + Conn: conn, + Subject: subject, + } + c := &ceClient{ + transport: &transport, + } + + if err := c.applyOptions(opts...); err != nil { + return nil, err + } + + return c, nil +} + +func (c *ceClient) startNATSReceiver(ctx context.Context, t *cloudeventsnats.Transport, fn Receiver) error { + if t.Conn == nil { + return fmt.Errorf("nats connection is required to be set") + } + if c.receiver != nil { + return fmt.Errorf("client already has a receiver") + } + if t.Receiver != nil { + return fmt.Errorf("transport already has a receiver") + } + + c.receiver = fn + t.Receiver = c + + go func() { + if err := t.Listen(ctx); err != nil { + log.Fatalf("failed to listen, %s", err.Error()) + } + }() + + return nil +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/defaulters.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/defaulters.go new file mode 100644 index 00000000000..ee87e0467a1 --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/defaulters.go @@ -0,0 +1,62 @@ +package client + +import ( + "github.com/cloudevents/sdk-go/pkg/cloudevents" + "github.com/cloudevents/sdk-go/pkg/cloudevents/types" + "github.com/google/uuid" + "time" +) + +type EventDefaulter func(event cloudevents.Event) cloudevents.Event + +func DefaultIDToUUIDIfNotSet(event cloudevents.Event) cloudevents.Event { + if event.Context != nil { + switch event.Context.GetSpecVersion() { + case cloudevents.CloudEventsVersionV01: + ec := event.Context.AsV01() + if ec.EventID == "" { + ec.EventID = uuid.New().String() + event.Context = ec + } + case cloudevents.CloudEventsVersionV02: + ec := event.Context.AsV02() + if ec.ID == "" { + ec.ID = uuid.New().String() + event.Context = ec + } + case cloudevents.CloudEventsVersionV03: + ec := event.Context.AsV03() + if ec.ID == "" { + ec.ID = uuid.New().String() + event.Context = ec + } + } + } + return event +} + +func DefaultTimeToNowIfNotSet(event cloudevents.Event) cloudevents.Event { + if event.Context != nil { + switch event.Context.GetSpecVersion() { + case cloudevents.CloudEventsVersionV01: + ec := event.Context.AsV01() + if ec.EventTime == nil || ec.EventTime.IsZero() { + ec.EventTime = &types.Timestamp{Time: time.Now()} + event.Context = ec + } + case cloudevents.CloudEventsVersionV02: + ec := event.Context.AsV02() + if ec.Time == nil || ec.Time.IsZero() { + ec.Time = &types.Timestamp{Time: time.Now()} + event.Context = ec + } + case cloudevents.CloudEventsVersionV03: + ec := event.Context.AsV03() + if ec.Time == nil || ec.Time.IsZero() { + ec.Time = &types.Timestamp{Time: time.Now()} + event.Context = ec + } + } + } + return event +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/options.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/options.go new file mode 100644 index 00000000000..f534f48fc77 --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/options.go @@ -0,0 +1,177 @@ +package client + +import ( + "fmt" + "github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http" + "github.com/cloudevents/sdk-go/pkg/cloudevents/transport/nats" + nethttp "net/http" + "net/url" + "strings" +) + +type Option func(*ceClient) error + +// WithTarget sets the outbound recipient of cloudevents when using an HTTP request. +func WithTarget(targetUrl string) Option { + return func(c *ceClient) error { + if t, ok := c.transport.(*http.Transport); ok { + targetUrl = strings.TrimSpace(targetUrl) + if targetUrl != "" { + var err error + var target *url.URL + target, err = url.Parse(targetUrl) + if err != nil { + return fmt.Errorf("client option failed to parse target url: %s", err.Error()) + } + + if t.Req == nil { + t.Req = &nethttp.Request{ + Method: nethttp.MethodPost, + } + } + t.Req.URL = target + return nil + } + return fmt.Errorf("target option was empty string") + } + return fmt.Errorf("invalid target client option received for transport type") + } +} + +// WithHTTPMethod sets the outbound recipient of cloudevents when using an HTTP request. +func WithHTTPMethod(method string) Option { + return func(c *ceClient) error { + if t, ok := c.transport.(*http.Transport); ok { + method = strings.TrimSpace(method) + if method != "" { + if t.Req == nil { + t.Req = &nethttp.Request{} + } + t.Req.Method = method + return nil + } + return fmt.Errorf("method option was empty string") + } + return fmt.Errorf("invalid HTTP method client option received for transport type") + } +} + +// WithHTTPEncoding sets the encoding for clients with HTTP transports. +func WithHTTPEncoding(encoding http.Encoding) Option { + return func(c *ceClient) error { + if t, ok := c.transport.(*http.Transport); ok { + t.Encoding = encoding + return nil + } + return fmt.Errorf("invalid HTTP encoding client option received for transport type") + } +} + +// WithHTTPDefaultEncodingSelector sets the encoding selection strategy for +// default encoding selections based on Event. +func WithHTTPDefaultEncodingSelector(fn http.EncodingSelector) Option { + return func(c *ceClient) error { + if t, ok := c.transport.(*http.Transport); ok { + if fn != nil { + t.DefaultEncodingSelectionFn = fn + return nil + } + return fmt.Errorf("fn for DefaultEncodingSelector was nil") + } + return fmt.Errorf("invalid HTTP default encoding selector client option received for transport type") + } +} + +// WithHTTPBinaryEncodingSelector sets the encoding selection strategy for +// default encoding selections based on Event, the encoded event will be the +// given version in Binary form. +func WithHTTPBinaryEncoding() Option { + return func(c *ceClient) error { + if t, ok := c.transport.(*http.Transport); ok { + t.DefaultEncodingSelectionFn = http.DefaultBinaryEncodingSelectionStrategy + return nil + } + return fmt.Errorf("invalid HTTP binary encoding client option received for transport type") + } +} + +// WithHTTPStructuredEncodingSelector sets the encoding selection strategy for +// default encoding selections based on Event, the encoded event will be the +//// given version in Structured form. +func WithHTTPStructuredEncoding() Option { + return func(c *ceClient) error { + if t, ok := c.transport.(*http.Transport); ok { + t.DefaultEncodingSelectionFn = http.DefaultStructuredEncodingSelectionStrategy + return nil + } + return fmt.Errorf("invalid HTTP structured encoding client option received for transport type") + } +} + +// WithHTTPPort sets the port for for clients with HTTP transports. +func WithHTTPPort(port int) Option { + return func(c *ceClient) error { + if t, ok := c.transport.(*http.Transport); ok { + if port == 0 { + return fmt.Errorf("client option was given an invalid port: %d", port) + } + t.Port = port + return nil + } + return fmt.Errorf("invalid HTTP port client option received for transport type") + } +} + +// WithHTTPClient sets the internal HTTP client for cloudevent clients with HTTP transports. +func WithHTTPClient(netclient *nethttp.Client) Option { + return func(c *ceClient) error { + if t, ok := c.transport.(*http.Transport); ok { + if netclient == nil { + return fmt.Errorf("client option was given an nil HTTP client") + } + t.Client = netclient + return nil + } + return fmt.Errorf("invalid HTTP client client option received for transport type") + } +} + +// WithNATSEncoding sets the encoding for clients with NATS transport. +func WithNATSEncoding(encoding nats.Encoding) Option { + return func(c *ceClient) error { + if t, ok := c.transport.(*nats.Transport); ok { + t.Encoding = encoding + return nil + } + return fmt.Errorf("invalid NATS encoding client option received for transport type") + } +} + +// WithEventDefaulter adds an event defaulter to the end of the defaulter chain. +func WithEventDefaulter(fn EventDefaulter) Option { + return func(c *ceClient) error { + if fn == nil { + return fmt.Errorf("client option was given an nil event defaulter") + } + c.eventDefaulterFns = append(c.eventDefaulterFns, fn) + return nil + } +} + +// WithUUIDs adds DefaultIDToUUIDIfNotSet event defaulter to the end of the +// defaulter chain. +func WithUUIDs() Option { + return func(c *ceClient) error { + c.eventDefaulterFns = append(c.eventDefaulterFns, DefaultIDToUUIDIfNotSet) + return nil + } +} + +// WithTimeNow adds DefaultTimeToNowIfNotSet event defaulter to the end of the +// defaulter chain. +func WithTimeNow() Option { + return func(c *ceClient) error { + c.eventDefaulterFns = append(c.eventDefaulterFns, DefaultTimeToNowIfNotSet) + return nil + } +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/codec/jsoncodec.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/codec/jsoncodec.go new file mode 100644 index 00000000000..21f10c68375 --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/codec/jsoncodec.go @@ -0,0 +1,160 @@ +package codec + +import ( + "encoding/json" + "github.com/cloudevents/sdk-go/pkg/cloudevents" + "github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec" + "log" + "strconv" +) + +func JsonEncodeV01(e cloudevents.Event) ([]byte, error) { + ctx := e.Context.AsV01() + if ctx.ContentType == nil { + ctx.ContentType = cloudevents.StringOfApplicationJSON() + } + return jsonEncode(ctx, e.Data) +} + +func JsonEncodeV02(e cloudevents.Event) ([]byte, error) { + ctx := e.Context.AsV02() + if ctx.ContentType == nil { + ctx.ContentType = cloudevents.StringOfApplicationJSON() + } + return jsonEncode(ctx, e.Data) +} + +func JsonEncodeV03(e cloudevents.Event) ([]byte, error) { + ctx := e.Context.AsV03() + if ctx.DataContentType == nil { + ctx.DataContentType = cloudevents.StringOfApplicationJSON() + } + return jsonEncode(ctx, e.Data) +} + +func jsonEncode(ctx cloudevents.EventContext, data interface{}) ([]byte, error) { + ctxb, err := marshalEvent(ctx) + if err != nil { + return nil, err + } + + var body []byte + + b := map[string]json.RawMessage{} + if err := json.Unmarshal(ctxb, &b); err != nil { + return nil, err + } + + mediaType := ctx.GetDataMediaType() + datab, err := marshalEventData(mediaType, data) + if err != nil { + return nil, err + } + if data != nil { + if mediaType == "" || mediaType == cloudevents.ApplicationJSON { + b["data"] = datab + } else if datab[0] != byte('"') { + b["data"] = []byte(strconv.QuoteToASCII(string(datab))) + } else { + // already quoted + b["data"] = datab + } + } + + body, err = json.Marshal(b) + if err != nil { + return nil, err + } + + return body, nil +} + +func JsonDecodeV01(body []byte) (*cloudevents.Event, error) { + ec := cloudevents.EventContextV01{} + if err := json.Unmarshal(body, &ec); err != nil { + return nil, err + } + + raw := make(map[string]json.RawMessage) + + if err := json.Unmarshal(body, &raw); err != nil { + return nil, err + } + var data interface{} + if d, ok := raw["data"]; ok { + data = []byte(d) + } + + return &cloudevents.Event{ + Context: ec, + Data: data, + }, nil +} + +func JsonDecodeV02(body []byte) (*cloudevents.Event, error) { + ec := cloudevents.EventContextV02{} + if err := json.Unmarshal(body, &ec); err != nil { + return nil, err + } + + raw := make(map[string]json.RawMessage) + + if err := json.Unmarshal(body, &raw); err != nil { + return nil, err + } + var data interface{} + if d, ok := raw["data"]; ok { + data = []byte(d) + } + + return &cloudevents.Event{ + Context: ec, + Data: data, + }, nil +} + +func JsonDecodeV03(body []byte) (*cloudevents.Event, error) { + ec := cloudevents.EventContextV03{} + if err := json.Unmarshal(body, &ec); err != nil { + return nil, err + } + + raw := make(map[string]json.RawMessage) + + if err := json.Unmarshal(body, &raw); err != nil { + return nil, err + } + var data interface{} + if d, ok := raw["data"]; ok { + data = []byte(d) + } + + return &cloudevents.Event{ + Context: ec, + Data: data, + }, nil +} + +func marshalEvent(event interface{}) ([]byte, error) { + if b, ok := event.([]byte); ok { + log.Printf("json.marshalEvent asked to encode bytes... wrong? %s", string(b)) + } + + b, err := json.Marshal(event) + if err != nil { + return nil, err + } + return b, nil +} + +// TODO: not sure about this location for eventdata. +func marshalEventData(encoding string, data interface{}) ([]byte, error) { + if data == nil { + return []byte(nil), nil + } + // already encoded? + if b, ok := data.([]byte); ok { + return b, nil + } + return datacodec.Encode(encoding, data) +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/content_type.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/content_type.go new file mode 100644 index 00000000000..077ff2bd970 --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/content_type.go @@ -0,0 +1,34 @@ +package cloudevents + +const ( + ApplicationJSON = "application/json" + ApplicationXML = "application/xml" + ApplicationCloudEventsJSON = "application/cloudevents+json" + ApplicationCloudEventsBatchJSON = "application/cloudevents-batch+json" +) + +// StringOfApplicationJSON returns a string pointer to "application/json" +func StringOfApplicationJSON() *string { + a := ApplicationJSON + return &a +} + +// StringOfApplicationXML returns a string pointer to "application/xml" +func StringOfApplicationXML() *string { + a := ApplicationXML + return &a +} + +// StringOfApplicationCloudEventsJSON returns a string pointer to +// "application/cloudevents+json" +func StringOfApplicationCloudEventsJSON() *string { + a := ApplicationCloudEventsJSON + return &a +} + +// StringOfApplicationCloudEventsBatchJSON returns a string pointer to +// "application/cloudevents-batch+json" +func StringOfApplicationCloudEventsBatchJSON() *string { + a := ApplicationCloudEventsBatchJSON + return &a +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/codec.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/codec.go new file mode 100644 index 00000000000..1f6ce63d365 --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/codec.go @@ -0,0 +1,48 @@ +package datacodec + +import ( + "fmt" + "github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/json" + "github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/xml" +) + +type Decoder func(in, out interface{}) error +type Encoder func(in interface{}) ([]byte, error) + +var decoder map[string]Decoder +var encoder map[string]Encoder + +func init() { + decoder = make(map[string]Decoder, 10) + encoder = make(map[string]Encoder, 10) + + AddDecoder("", json.Decode) + AddDecoder("application/json", json.Decode) + AddDecoder("application/xml", xml.Decode) + + AddEncoder("", json.Encode) + AddEncoder("application/json", json.Encode) + AddEncoder("application/xml", xml.Encode) +} + +func AddDecoder(contentType string, fn Decoder) { + decoder[contentType] = fn +} + +func AddEncoder(contentType string, fn Encoder) { + encoder[contentType] = fn +} + +func Decode(contentType string, in, out interface{}) error { + if fn, ok := decoder[contentType]; ok { + return fn(in, out) + } + return fmt.Errorf("[decode] unsupported content type: %q", contentType) +} + +func Encode(contentType string, in interface{}) ([]byte, error) { + if fn, ok := encoder[contentType]; ok { + return fn(in) + } + return nil, fmt.Errorf("[encode] unsupported content type: %q", contentType) +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/json/data.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/json/data.go new file mode 100644 index 00000000000..38d75566713 --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/json/data.go @@ -0,0 +1,66 @@ +package json + +import ( + "encoding/json" + "fmt" + "reflect" + "strconv" +) + +func Decode(in, out interface{}) error { + if in == nil { + return nil + } + if out == nil { + return fmt.Errorf("out is nil") + } + + b, ok := in.([]byte) // TODO: I think there is fancy marshaling happening here. Fix with reflection? + if !ok { + var err error + b, err = json.Marshal(in) + if err != nil { + return fmt.Errorf("[json] failed to marshal in: %s", err.Error()) + } + } + + // TODO: the spec says json could be just data... At the moment we expect wrapped. + if len(b) > 1 && (b[0] == byte('"') || (b[0] == byte('\\') && b[1] == byte('"'))) { + s, err := strconv.Unquote(string(b)) + if err != nil { + return fmt.Errorf("[json] failed to unquote in: %s", err.Error()) + } + if len(s) > 0 && (s[0] == '{' || s[0] == '[') { + // looks like json, use it + b = []byte(s) + } + } + + if err := json.Unmarshal(b, out); err != nil { + return fmt.Errorf("[json] found bytes \"%s\", but failed to unmarshal: %s", string(b), err.Error()) + } + return nil +} + +func Encode(in interface{}) ([]byte, error) { + if in == nil { + return nil, nil + } + + it := reflect.TypeOf(in) + switch it.Kind() { + case reflect.Slice: + if it.Elem().Kind() == reflect.Uint8 { + + if b, ok := in.([]byte); ok && len(b) > 0 { + // check to see if it is a pre-encoded byte string. + if b[0] == byte('"') || b[0] == byte('{') || b[0] == byte('[') { + return b, nil + } + } + + } + } + + return json.Marshal(in) +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/xml/data.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/xml/data.go new file mode 100644 index 00000000000..fb36f2eb285 --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/xml/data.go @@ -0,0 +1,59 @@ +package xml + +import ( + "encoding/base64" + "encoding/xml" + "fmt" + "strconv" +) + +func Decode(in, out interface{}) error { + if in == nil { + return nil + } + + b, ok := in.([]byte) + if !ok { + var err error + b, err = xml.Marshal(in) + if err != nil { + return fmt.Errorf("[xml] failed to marshal in: %s", err.Error()) + } + } + + // If the message is encoded as a base64 block as a string, we need to + // decode that first before trying to unmarshal the bytes + if len(b) > 1 && (b[0] == byte('"') || (b[0] == byte('\\') && b[1] == byte('"'))) { + s, err := strconv.Unquote(string(b)) + if err != nil { + return err + } + if len(s) > 0 && s[0] == '<' { + // looks like xml, use it + b = []byte(s) + } else if len(s) > 0 { + // looks like base64, decode + bs, err := base64.StdEncoding.DecodeString(s) + if err != nil { + return err + } + b = bs + } + } + + if err := xml.Unmarshal(b, out); err != nil { + return fmt.Errorf("[xml] found bytes, but failed to unmarshal: %s %s", err.Error(), string(b)) + } + return nil +} + +func Encode(in interface{}) ([]byte, error) { + if b, ok := in.([]byte); ok { + // check to see if it is a pre-encoded byte string. + if len(b) > 0 && b[0] == byte('"') { + return b, nil + } + } + + return xml.Marshal(in) +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/doc.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/doc.go new file mode 100644 index 00000000000..cfb6eac590c --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/doc.go @@ -0,0 +1,86 @@ +/* +Package cloudevents provides primitives to work with CloudEvents specification: https://github.com/cloudevents/spec. + + +Parsing Event from HTTP Request: + // req is *http.Request + event, err := cloudEvents.FromHTTPRequest(req) + if err != nil { + panic("Unable to parse event from http Request: " + err.String()) + } + + +Creating a minimal CloudEvent in version 0.1: + import "github.com/cloudevents/sdk-go/pkg/cloudevents/v01" + event := v01.Event{ + EventType: "com.example.file.created", + Source: "/providers/Example.COM/storage/account#fileServices/default/{new-file}", + EventID: "ea35b24ede421", + } + + +The goal of this package is to provide support for all released versions of CloudEvents, ideally while maintaining +the same API. It will use semantic versioning with following rules: +* MAJOR version increments when backwards incompatible changes is introduced. +* MINOR version increments when backwards compatible feature is introduced INCLUDING support for new CloudEvents version. +* PATCH version increments when a backwards compatible bug fix is introduced. +*/ +package cloudevents + +/* + +New plan. + +Everything gets converted into the Canonical form of the event, this +then can select a transport, the transport provides encodings. + +At the moment we have cloudevents.[v01, v02] + +Canonical form holds an encoded data packet that takes in a provided Codec + +Canonical form has two members: Context, and Data + + +Sending: +cloudevents.[v01, v02] -> { Codec.Encode -> HttpMessage -> Transport[Http] } + +Receiving: +{ Transport[Http] -> HttpMessage -> Codec.Decode } -> cloudevents.[v01, v02] + +Note: Transport and Codec are grouped. + +Transport Codecs supported: +[Binary, Structured, StructuredMirrorHeaders] + + +## Working with inner data: +cloudevents.[v01, v02].Decode(DataCodec) -> custom data + +Working with inner data, + +Sending: +cloudevents.[v01, v02].data -> DataCodec.Decode -> custom data + +Receiving: +custom data -> DataCodec.Encode -> cloudevents.[v01, v02].data,contentType + +Data Codecs supported: +[json, xml, base64, text] + + +This imples that there is only one canonical form and it evolves and is marked +deprecated as the model evolves. + +Spec says: + +Event[Context, Data] -> Message + +Http Message should have: + +Headers +Body +ContentType +ContentLength + + +*/ diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/event.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/event.go new file mode 100644 index 00000000000..5964d612484 --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/event.go @@ -0,0 +1,73 @@ +package cloudevents + +import ( + "fmt" + "github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec" + "strings" +) + +// Event represents the canonical representation of a CloudEvent. +type Event struct { + Context EventContext + Data interface{} +} + +func (e Event) DataAs(data interface{}) error { + return datacodec.Decode(e.Context.GetDataMediaType(), e.Data, data) +} + +func (e Event) SpecVersion() string { + return e.Context.GetSpecVersion() +} + +func (e Event) Type() string { + return e.Context.GetType() +} + +func (e Event) DataContentType() string { + return e.Context.GetDataContentType() +} + +func (e Event) Validate() error { + if e.Context == nil { + return fmt.Errorf("every event conforming to the CloudEvents specification MUST include a context") + } + + if err := e.Context.Validate(); err != nil { + return err + } + + // TODO: validate data. + + return nil +} + +func (e Event) String() string { + sb := strings.Builder{} + + if s := e.SpecVersion(); s != "" { + if sb.Len() > 0 { + sb.WriteString("\n") + } + sb.WriteString("SpecVersion: ") + sb.WriteString(s) + } + + if s := e.Type(); s != "" { + if sb.Len() > 0 { + sb.WriteString("\n") + } + sb.WriteString("Type: ") + sb.WriteString(s) + } + + if s := e.DataContentType(); s != "" { + if sb.Len() > 0 { + sb.WriteString("\n") + } + sb.WriteString("DataContentType: ") + sb.WriteString(s) + } + + return sb.String() +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/eventcontext.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/eventcontext.go new file mode 100644 index 00000000000..1f5b4d943c8 --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/eventcontext.go @@ -0,0 +1,34 @@ +package cloudevents + +type EventContext interface { + // AsV01 provides a translation from whatever the "native" encoding of the + // CloudEvent was to the equivalent in v0.1 field names, moving fields to or + // from extensions as necessary. + AsV01() EventContextV01 + + // AsV02 provides a translation from whatever the "native" encoding of the + // CloudEvent was to the equivalent in v0.2 field names, moving fields to or + // from extensions as necessary. + AsV02() EventContextV02 + + // AsV03 provides a translation from whatever the "native" encoding of the + // CloudEvent was to the equivalent in v0.3 field names, moving fields to or + // from extensions as necessary. + AsV03() EventContextV03 + + // GetDataContentType returns content type on the context. + GetDataContentType() string + + // GetDataMediaType returns the MIME media type for encoded data, which is + // needed by both encoding and decoding. + GetDataMediaType() string + + // GetSpecVersion returns the native CloudEvents Spec version of the event + // context. + GetSpecVersion() string + + // GetType returns the CloudEvents type from the context. + GetType() string + + Validate() error +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/eventcontext_v01.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/eventcontext_v01.go new file mode 100644 index 00000000000..318df337ac2 --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/eventcontext_v01.go @@ -0,0 +1,218 @@ +package cloudevents + +import ( + "fmt" + "github.com/cloudevents/sdk-go/pkg/cloudevents/types" + "log" + "mime" + "strings" +) + +const ( + // CloudEventsVersionV01 represents the version 0.1 of the CloudEvents spec. + CloudEventsVersionV01 = "0.1" +) + +// EventContextV01 holds standard metadata about an event. See +// https://github.com/cloudevents/spec/blob/v0.1/spec.md#context-attributes for +// details on these fields. +type EventContextV01 struct { + // The version of the CloudEvents specification used by the event. + CloudEventsVersion string `json:"cloudEventsVersion,omitempty"` + // ID of the event; must be non-empty and unique within the scope of the producer. + EventID string `json:"eventID"` + // Timestamp when the event happened. + EventTime *types.Timestamp `json:"eventTime,omitempty"` + // Type of occurrence which has happened. + EventType string `json:"eventType"` + // The version of the `eventType`; this is producer-specific. + EventTypeVersion *string `json:"eventTypeVersion,omitempty"` + // A link to the schema that the `data` attribute adheres to. + SchemaURL *types.URLRef `json:"schemaURL,omitempty"` + // A MIME (RFC 2046) string describing the media type of `data`. + // TODO: Should an empty string assume `application/json`, or auto-detect the content? + ContentType *string `json:"contentType,omitempty"` + // A URI describing the event producer. + Source types.URLRef `json:"source"` + // Additional metadata without a well-defined structure. + Extensions map[string]interface{} `json:"extensions,omitempty"` +} + +var _ EventContext = (*EventContextV01)(nil) + +func (ec EventContextV01) GetSpecVersion() string { + if ec.CloudEventsVersion != "" { + return ec.CloudEventsVersion + } + return CloudEventsVersionV01 +} + +func (ec EventContextV01) GetDataContentType() string { + if ec.ContentType != nil { + return *ec.ContentType + } + return "" +} + +func (ec EventContextV01) GetDataMediaType() string { + if ec.ContentType != nil { + mediaType, _, err := mime.ParseMediaType(*ec.ContentType) + if err != nil { + log.Printf("failed to parse media type from ContentType: %s", err) + return "" + } + return mediaType + } + return "" +} + +func (ec EventContextV01) GetType() string { + return ec.EventType +} + +func (ec EventContextV01) AsV01() EventContextV01 { + ec.CloudEventsVersion = CloudEventsVersionV01 + return ec +} + +func (ec EventContextV01) AsV02() EventContextV02 { + ret := EventContextV02{ + SpecVersion: CloudEventsVersionV02, + Type: ec.EventType, + Source: ec.Source, + ID: ec.EventID, + Time: ec.EventTime, + SchemaURL: ec.SchemaURL, + ContentType: ec.ContentType, + Extensions: make(map[string]interface{}), + } + + // eventTypeVersion was retired in v0.2, so put it in an extension. + if ec.EventTypeVersion != nil { + ret.Extensions["eventTypeVersion"] = *ec.EventTypeVersion + } + if ec.Extensions != nil { + for k, v := range ec.Extensions { + ret.Extensions[k] = v + } + } + if len(ret.Extensions) == 0 { + ret.Extensions = nil + } + return ret +} + +func (ec EventContextV01) AsV03() EventContextV03 { + ecv2 := ec.AsV02() + return ecv2.AsV03() +} + +// Validate returns errors based on requirements from the CloudEvents spec. +// For more details, see https://github.com/cloudevents/spec/blob/v0.1/spec.md +func (ec EventContextV01) Validate() error { + errors := []string(nil) + + // eventType + // Type: String + // Constraints: + // REQUIRED + // MUST be a non-empty string + // SHOULD be prefixed with a reverse-DNS name. The prefixed domain dictates the organization which defines the semantics of this event type. + eventType := strings.TrimSpace(ec.EventType) + if eventType == "" { + errors = append(errors, "eventType: MUST be a non-empty string") + } + + // eventTypeVersion + // Type: String + // Constraints: + // OPTIONAL + // If present, MUST be a non-empty string + if ec.EventTypeVersion != nil { + eventTypeVersion := strings.TrimSpace(*ec.EventTypeVersion) + if eventTypeVersion == "" { + errors = append(errors, "eventTypeVersion: if present, MUST be a non-empty string") + } + } + + // cloudEventsVersion + // Type: String + // Constraints: + // REQUIRED + // MUST be a non-empty string + cloudEventsVersion := strings.TrimSpace(ec.CloudEventsVersion) + if cloudEventsVersion == "" { + errors = append(errors, "cloudEventsVersion: MUST be a non-empty string") + } + + // source + // Type: URI + // Constraints: + // REQUIRED + source := strings.TrimSpace(ec.Source.String()) + if source == "" { + errors = append(errors, "source: REQUIRED") + } + + // eventID + // Type: String + // Constraints: + // REQUIRED + // MUST be a non-empty string + // MUST be unique within the scope of the producer + eventID := strings.TrimSpace(ec.EventID) + if eventID == "" { + errors = append(errors, "eventID: MUST be a non-empty string") + + // no way to test "MUST be unique within the scope of the producer" + } + + // eventTime + // Type: Timestamp + // Constraints: + // OPTIONAL + // If present, MUST adhere to the format specified in RFC 3339 + // --> no need to test this, no way to set the eventTime without it being valid. + + // schemaURL + // Type: URI + // Constraints: + // OPTIONAL + // If present, MUST adhere to the format specified in RFC 3986 + if ec.SchemaURL != nil { + schemaURL := strings.TrimSpace(ec.SchemaURL.String()) + // empty string is not RFC 3986 compatible. + if schemaURL == "" { + errors = append(errors, "schemaURL: if present, MUST adhere to the format specified in RFC 3986") + } + } + + // contentType + // Type: String per RFC 2046 + // Constraints: + // OPTIONAL + // If present, MUST adhere to the format specified in RFC 2046 + if ec.ContentType != nil { + contentType := strings.TrimSpace(*ec.ContentType) + if contentType == "" { + // TODO: need to test for RFC 2046 + errors = append(errors, "contentType: if present, MUST adhere to the format specified in RFC 2046") + } + } + + // extensions + // Type: Map + // Constraints: + // OPTIONAL + // If present, MUST contain at least one entry + if ec.Extensions != nil { + if len(ec.Extensions) == 0 { + errors = append(errors, "extensions: if present, MUST contain at least one entry") + } + } + + if len(errors) > 0 { + return fmt.Errorf(strings.Join(errors, "\n")) + } + return nil +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/eventcontext_v02.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/eventcontext_v02.go new file mode 100644 index 00000000000..c2bdba2d246 --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/eventcontext_v02.go @@ -0,0 +1,203 @@ +package cloudevents + +import ( + "fmt" + "github.com/cloudevents/sdk-go/pkg/cloudevents/types" + "log" + "mime" + "strings" +) + +const ( + // CloudEventsVersionV02 represents the version 0.2 of the CloudEvents spec. + CloudEventsVersionV02 = "0.2" +) + +// EventContextV02 represents the non-data attributes of a CloudEvents v0.2 +// event. +type EventContextV02 struct { + // The version of the CloudEvents specification used by the event. + SpecVersion string `json:"specversion"` + // The type of the occurrence which has happened. + Type string `json:"type"` + // A URI describing the event producer. + Source types.URLRef `json:"source"` + // ID of the event; must be non-empty and unique within the scope of the producer. + ID string `json:"id"` + // Timestamp when the event happened. + Time *types.Timestamp `json:"time,omitempty"` + // A link to the schema that the `data` attribute adheres to. + SchemaURL *types.URLRef `json:"schemaurl,omitempty"` + // A MIME (RFC2046) string describing the media type of `data`. + // TODO: Should an empty string assume `application/json`, `application/octet-stream`, or auto-detect the content? + ContentType *string `json:"contenttype,omitempty"` + // Additional extension metadata beyond the base spec. + Extensions map[string]interface{} `json:"-,omitempty"` // TODO: decide how we want extensions to be inserted +} + +var _ EventContext = (*EventContextV02)(nil) + +func (ec EventContextV02) GetSpecVersion() string { + if ec.SpecVersion != "" { + return ec.SpecVersion + } + return CloudEventsVersionV02 +} + +func (ec EventContextV02) GetDataContentType() string { + if ec.ContentType != nil { + return *ec.ContentType + } + return "" +} + +func (ec EventContextV02) GetDataMediaType() string { + if ec.ContentType != nil { + mediaType, _, err := mime.ParseMediaType(*ec.ContentType) + if err != nil { + log.Printf("failed to parse media type from ContentType: %s", err) + return "" + } + return mediaType + } + return "" +} + +func (ec EventContextV02) GetType() string { + return ec.Type +} + +func (ec EventContextV02) AsV01() EventContextV01 { + ret := EventContextV01{ + CloudEventsVersion: CloudEventsVersionV01, + EventID: ec.ID, + EventTime: ec.Time, + EventType: ec.Type, + SchemaURL: ec.SchemaURL, + Source: ec.Source, + ContentType: ec.ContentType, + Extensions: make(map[string]interface{}), + } + + for k, v := range ec.Extensions { + // eventTypeVersion was retired in v0.2 + if strings.EqualFold(k, "eventTypeVersion") { + etv, ok := v.(string) + if ok && etv != "" { + ret.EventTypeVersion = &etv + } + continue + } + ret.Extensions[k] = v + } + if len(ret.Extensions) == 0 { + ret.Extensions = nil + } + return ret +} + +func (ec EventContextV02) AsV02() EventContextV02 { + ec.SpecVersion = CloudEventsVersionV02 + return ec +} + +func (ec EventContextV02) AsV03() EventContextV03 { + ret := EventContextV03{ + SpecVersion: CloudEventsVersionV03, + ID: ec.ID, + Time: ec.Time, + Type: ec.Type, + SchemaURL: ec.SchemaURL, + DataContentType: ec.ContentType, + Source: ec.Source, + Extensions: ec.Extensions, + } + return ret +} + +// Validate returns errors based on requirements from the CloudEvents spec. +// For more details, see https://github.com/cloudevents/spec/blob/v0.2/spec.md +func (ec EventContextV02) Validate() error { + errors := []string(nil) + + // type + // Type: String + // Constraints: + // REQUIRED + // MUST be a non-empty string + // SHOULD be prefixed with a reverse-DNS name. The prefixed domain dictates the organization which defines the semantics of this event type. + eventType := strings.TrimSpace(ec.Type) + if eventType == "" { + errors = append(errors, "type: MUST be a non-empty string") + } + + // specversion + // Type: String + // Constraints: + // REQUIRED + // MUST be a non-empty string + specVersion := strings.TrimSpace(ec.SpecVersion) + if specVersion == "" { + errors = append(errors, "specversion: MUST be a non-empty string") + } + + // source + // Type: URI-reference + // Constraints: + // REQUIRED + source := strings.TrimSpace(ec.Source.String()) + if source == "" { + errors = append(errors, "source: REQUIRED") + } + + // id + // Type: String + // Constraints: + // REQUIRED + // MUST be a non-empty string + // MUST be unique within the scope of the producer + id := strings.TrimSpace(ec.ID) + if id == "" { + errors = append(errors, "id: MUST be a non-empty string") + + // no way to test "MUST be unique within the scope of the producer" + } + + // time + // Type: Timestamp + // Constraints: + // OPTIONAL + // If present, MUST adhere to the format specified in RFC 3339 + // --> no need to test this, no way to set the time without it being valid. + + // schemaurl + // Type: URI + // Constraints: + // OPTIONAL + // If present, MUST adhere to the format specified in RFC 3986 + if ec.SchemaURL != nil { + schemaURL := strings.TrimSpace(ec.SchemaURL.String()) + // empty string is not RFC 3986 compatible. + if schemaURL == "" { + errors = append(errors, "schemaurl: if present, MUST adhere to the format specified in RFC 3986") + } + } + + // contenttype + // Type: String per RFC 2046 + // Constraints: + // OPTIONAL + // If present, MUST adhere to the format specified in RFC 2046 + if ec.ContentType != nil { + contentType := strings.TrimSpace(*ec.ContentType) + if contentType == "" { + // TODO: need to test for RFC 2046 + errors = append(errors, "contenttype: if present, MUST adhere to the format specified in RFC 2046") + } + } + + if len(errors) > 0 { + return fmt.Errorf(strings.Join(errors, "\n")) + } + return nil +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/eventcontext_v03.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/eventcontext_v03.go new file mode 100644 index 00000000000..465a815d1c9 --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/eventcontext_v03.go @@ -0,0 +1,181 @@ +package cloudevents + +import ( + "fmt" + "github.com/cloudevents/sdk-go/pkg/cloudevents/types" + "log" + "mime" + "strings" +) + +// WIP: AS OF FEB 19, 2019 + +const ( + // CloudEventsVersionV03 represents the version 0.3 of the CloudEvents spec. + CloudEventsVersionV03 = "0.3" +) + +// EventContextV03 represents the non-data attributes of a CloudEvents v0.3 +// event. +type EventContextV03 struct { + // SpecVersion - The version of the CloudEvents specification used by the event. + SpecVersion string `json:"specversion"` + // Type - The type of the occurrence which has happened. + Type string `json:"type"` + // Source - A URI describing the event producer. + Source types.URLRef `json:"source"` + // ID of the event; must be non-empty and unique within the scope of the producer. + ID string `json:"id"` + // Time - A Timestamp when the event happened. + Time *types.Timestamp `json:"time,omitempty"` + // SchemaURL - A link to the schema that the `data` attribute adheres to. + SchemaURL *types.URLRef `json:"schemaurl,omitempty"` + // GetDataMediaType - A MIME (RFC2046) string describing the media type of `data`. + // TODO: Should an empty string assume `application/json`, `application/octet-stream`, or auto-detect the content? + DataContentType *string `json:"datacontenttype,omitempty"` + // Extensions - Additional extension metadata beyond the base spec. + Extensions map[string]interface{} `json:"-,omitempty"` // TODO: decide how we want extensions to be inserted +} + +var _ EventContext = (*EventContextV03)(nil) + +func (ec EventContextV03) GetSpecVersion() string { + if ec.SpecVersion != "" { + return ec.SpecVersion + } + return CloudEventsVersionV03 +} + +func (ec EventContextV03) GetDataContentType() string { + if ec.DataContentType != nil { + return *ec.DataContentType + } + return "" +} + +func (ec EventContextV03) GetDataMediaType() string { + if ec.DataContentType != nil { + mediaType, _, err := mime.ParseMediaType(*ec.DataContentType) + if err != nil { + log.Printf("failed to parse media type from DataContentType: %s", err) + return "" + } + return mediaType + } + return "" +} +func (ec EventContextV03) GetType() string { + return ec.Type +} + +func (ec EventContextV03) AsV01() EventContextV01 { + ecv2 := ec.AsV02() + return ecv2.AsV01() +} + +func (ec EventContextV03) AsV02() EventContextV02 { + ret := EventContextV02{ + SpecVersion: CloudEventsVersionV02, + ID: ec.ID, + Time: ec.Time, + Type: ec.Type, + SchemaURL: ec.SchemaURL, + ContentType: ec.DataContentType, + Source: ec.Source, + Extensions: ec.Extensions, + } + return ret +} + +func (ec EventContextV03) AsV03() EventContextV03 { + ec.SpecVersion = CloudEventsVersionV03 + return ec +} + +// Validate returns errors based on requirements from the CloudEvents spec. +// For more details, see https://github.com/cloudevents/spec/blob/master/spec.md +// As of Feb 26, 2019, commit 17c32ea26baf7714ad027d9917d03d2fff79fc7e +func (ec EventContextV03) Validate() error { + errors := []string(nil) + + // type + // Type: String + // Constraints: + // REQUIRED + // MUST be a non-empty string + // SHOULD be prefixed with a reverse-DNS name. The prefixed domain dictates the organization which defines the semantics of this event type. + eventType := strings.TrimSpace(ec.Type) + if eventType == "" { + errors = append(errors, "type: MUST be a non-empty string") + } + + // specversion + // Type: String + // Constraints: + // REQUIRED + // MUST be a non-empty string + specVersion := strings.TrimSpace(ec.SpecVersion) + if specVersion == "" { + errors = append(errors, "specversion: MUST be a non-empty string") + } + + // source + // Type: URI-reference + // Constraints: + // REQUIRED + source := strings.TrimSpace(ec.Source.String()) + if source == "" { + errors = append(errors, "source: REQUIRED") + } + + // id + // Type: String + // Constraints: + // REQUIRED + // MUST be a non-empty string + // MUST be unique within the scope of the producer + id := strings.TrimSpace(ec.ID) + if id == "" { + errors = append(errors, "id: MUST be a non-empty string") + + // no way to test "MUST be unique within the scope of the producer" + } + + // time + // Type: Timestamp + // Constraints: + // OPTIONAL + // If present, MUST adhere to the format specified in RFC 3339 + // --> no need to test this, no way to set the time without it being valid. + + // schemaurl + // Type: URI + // Constraints: + // OPTIONAL + // If present, MUST adhere to the format specified in RFC 3986 + if ec.SchemaURL != nil { + schemaURL := strings.TrimSpace(ec.SchemaURL.String()) + // empty string is not RFC 3986 compatible. + if schemaURL == "" { + errors = append(errors, "schemaurl: if present, MUST adhere to the format specified in RFC 3986") + } + } + + // datacontenttype + // Type: String per RFC 2046 + // Constraints: + // OPTIONAL + // If present, MUST adhere to the format specified in RFC 2046 + if ec.DataContentType != nil { + dataContentType := strings.TrimSpace(*ec.DataContentType) + if dataContentType == "" { + // TODO: need to test for RFC 2046 + errors = append(errors, "datacontenttype: if present, MUST adhere to the format specified in RFC 2046") + } + } + + if len(errors) > 0 { + return fmt.Errorf(strings.Join(errors, "\n")) + } + return nil +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/codec.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/codec.go new file mode 100644 index 00000000000..6564dc9e502 --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/codec.go @@ -0,0 +1,10 @@ +package transport + +import "github.com/cloudevents/sdk-go/pkg/cloudevents" + +// Codec is the interface for transport codecs to convert between transport +// specific payloads and the Message interface. +type Codec interface { + Encode(cloudevents.Event) (Message, error) + Decode(Message) (*cloudevents.Event, error) +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/codec.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/codec.go new file mode 100644 index 00000000000..5493a1ecb35 --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/codec.go @@ -0,0 +1,214 @@ +package http + +import ( + "fmt" + "github.com/cloudevents/sdk-go/pkg/cloudevents" + "github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec" + "github.com/cloudevents/sdk-go/pkg/cloudevents/transport" +) + +type Codec struct { + Encoding Encoding + + DefaultEncodingSelectionFn EncodingSelector + + v01 *CodecV01 + v02 *CodecV02 + v03 *CodecV03 +} + +var _ transport.Codec = (*Codec)(nil) + +func DefaultBinaryEncodingSelectionStrategy(e cloudevents.Event) Encoding { + switch e.SpecVersion() { + case cloudevents.CloudEventsVersionV01: + return BinaryV01 + case cloudevents.CloudEventsVersionV02: + return BinaryV02 + case cloudevents.CloudEventsVersionV03: + return BinaryV03 + } + // Unknown version, return Default. + return Default +} + +func DefaultStructuredEncodingSelectionStrategy(e cloudevents.Event) Encoding { + switch e.SpecVersion() { + case cloudevents.CloudEventsVersionV01: + return StructuredV01 + case cloudevents.CloudEventsVersionV02: + return StructuredV02 + case cloudevents.CloudEventsVersionV03: + return StructuredV03 + } + // Unknown version, return Default. + return Default +} + +func (c *Codec) Encode(e cloudevents.Event) (transport.Message, error) { + encoding := c.Encoding + + if encoding == Default && c.DefaultEncodingSelectionFn != nil { + encoding = c.DefaultEncodingSelectionFn(e) + } + + switch encoding { + case Default: + fallthrough + case BinaryV01: + fallthrough + case StructuredV01: + if c.v01 == nil { + c.v01 = &CodecV01{Encoding: encoding} + } + return c.v01.Encode(e) + case BinaryV02: + fallthrough + case StructuredV02: + if c.v02 == nil { + c.v02 = &CodecV02{Encoding: encoding} + } + return c.v02.Encode(e) + case BinaryV03: + fallthrough + case StructuredV03: + if c.v03 == nil { + c.v03 = &CodecV03{Encoding: encoding} + } + return c.v03.Encode(e) + default: + return nil, fmt.Errorf("unknown encoding: %s", encoding) + } +} + +func (c *Codec) Decode(msg transport.Message) (*cloudevents.Event, error) { + switch c.inspectEncoding(msg) { + case BinaryV01: + fallthrough + case StructuredV01: + if c.v01 == nil { + c.v01 = &CodecV01{Encoding: c.Encoding} + } + if event, err := c.v01.Decode(msg); err != nil { + return nil, err + } else { + return c.convertEvent(event), nil + } + case BinaryV02: + fallthrough + case StructuredV02: + if c.v02 == nil { + c.v02 = &CodecV02{Encoding: c.Encoding} + } + if event, err := c.v02.Decode(msg); err != nil { + return nil, err + } else { + return c.convertEvent(event), nil + } + case BinaryV03: + fallthrough + case StructuredV03: + fallthrough + case BatchedV03: + if c.v03 == nil { + c.v03 = &CodecV03{Encoding: c.Encoding} + } + if event, err := c.v03.Decode(msg); err != nil { + return nil, err + } else { + return c.convertEvent(event), nil + } + default: + return nil, fmt.Errorf("unknown encoding for message %v", msg) + } +} + +// Give the context back as the user expects +func (c *Codec) convertEvent(event *cloudevents.Event) *cloudevents.Event { + if event == nil { + return nil + } + switch c.Encoding { + case Default: + return event + case BinaryV01: + fallthrough + case StructuredV01: + if c.v01 == nil { + c.v01 = &CodecV01{Encoding: c.Encoding} + } + ctx := event.Context.AsV01() + event.Context = ctx + return event + case BinaryV02: + fallthrough + case StructuredV02: + if c.v02 == nil { + c.v02 = &CodecV02{Encoding: c.Encoding} + } + ctx := event.Context.AsV02() + event.Context = ctx + return event + case BinaryV03: + fallthrough + case StructuredV03: + fallthrough + case BatchedV03: + if c.v03 == nil { + c.v03 = &CodecV03{Encoding: c.Encoding} + } + ctx := event.Context.AsV03() + event.Context = ctx + return event + default: + return nil + } +} + +func (c *Codec) inspectEncoding(msg transport.Message) Encoding { + // TODO: there should be a better way to make the version codecs on demand. + if c.v01 == nil { + c.v01 = &CodecV01{Encoding: c.Encoding} + } + // Try v0.1 first. + encoding := c.v01.inspectEncoding(msg) + if encoding != Unknown { + return encoding + } + + if c.v02 == nil { + c.v02 = &CodecV02{Encoding: c.Encoding} + } + // Try v0.2. + encoding = c.v02.inspectEncoding(msg) + if encoding != Unknown { + return encoding + } + + if c.v03 == nil { + c.v03 = &CodecV03{Encoding: c.Encoding} + } + // Try v0.3. + encoding = c.v03.inspectEncoding(msg) + if encoding != Unknown { + return encoding + } + + // We do not understand the message encoding. + return Unknown +} + +// --------- +// TODO: Should move these somewhere else. the methods are shared for all versions. + +func marshalEventData(encoding string, data interface{}) ([]byte, error) { + if data == nil { + return []byte(nil), nil + } + // already encoded? + if b, ok := data.([]byte); ok { + return b, nil + } + + return datacodec.Encode(encoding, data) +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/codec_v01.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/codec_v01.go new file mode 100644 index 00000000000..13dd6e0b483 --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/codec_v01.go @@ -0,0 +1,219 @@ +package http + +import ( + "encoding/json" + "reflect" + "fmt" + "github.com/cloudevents/sdk-go/pkg/cloudevents" + "github.com/cloudevents/sdk-go/pkg/cloudevents/codec" + "github.com/cloudevents/sdk-go/pkg/cloudevents/transport" + "github.com/cloudevents/sdk-go/pkg/cloudevents/types" + "log" + "net/http" + "net/textproto" + "strings" +) + +type CodecV01 struct { + Encoding Encoding +} + +var _ transport.Codec = (*CodecV01)(nil) + +func (v CodecV01) Encode(e cloudevents.Event) (transport.Message, error) { + switch v.Encoding { + case Default: + fallthrough + case BinaryV01: + return v.encodeBinary(e) + case StructuredV01: + return v.encodeStructured(e) + default: + return nil, fmt.Errorf("unknown encoding: %d", v.Encoding) + } +} + +func (v CodecV01) Decode(msg transport.Message) (*cloudevents.Event, error) { + switch v.inspectEncoding(msg) { + case BinaryV01: + return v.decodeBinary(msg) + case StructuredV01: + return v.decodeStructured(msg) + default: + return nil, fmt.Errorf("unknown encoding for message %v", msg) + } +} + +func (v CodecV01) encodeBinary(e cloudevents.Event) (transport.Message, error) { + header, err := v.toHeaders(e.Context.AsV01()) + if err != nil { + return nil, err + } + + body, err := marshalEventData(e.Context.GetDataMediaType(), e.Data) + if err != nil { + return nil, err + } + + msg := &Message{ + Header: header, + Body: body, + } + + return msg, nil +} + +func (v CodecV01) toHeaders(ec cloudevents.EventContextV01) (http.Header, error) { + // Preserve case in v0.1, even though HTTP headers are case-insensitive. + h := http.Header{} + h["CE-CloudEventsVersion"] = []string{ec.CloudEventsVersion} + h["CE-EventID"] = []string{ec.EventID} + h["CE-EventType"] = []string{ec.EventType} + h["CE-Source"] = []string{ec.Source.String()} + if ec.EventTime != nil && !ec.EventTime.IsZero() { + h["CE-EventTime"] = []string{ec.EventTime.String()} + } + if ec.EventTypeVersion != nil { + h["CE-EventTypeVersion"] = []string{*ec.EventTypeVersion} + } + if ec.SchemaURL != nil { + h["CE-SchemaURL"] = []string{ec.SchemaURL.String()} + } + if ec.ContentType != nil { + h.Set("Content-Type", *ec.ContentType) + } else if v.Encoding == Default || v.Encoding == BinaryV01 { + // in binary v0.1, the Content-Type header is tied to ec.ContentType + // This was later found to be an issue with the spec, but yolo. + // TODO: not sure what the default should be? + h.Set("Content-Type", cloudevents.ApplicationJSON) + } + + // Regarding Extensions, v0.1 Spec says the following: + // * Each map entry name MUST be prefixed with "CE-X-" + // * Each map entry name's first character MUST be capitalized + for k, v := range ec.Extensions { + encoded, err := json.Marshal(v) + if err != nil { + return nil, err + } + h["CE-X-"+strings.Title(k)] = []string{string(encoded)} + } + return h, nil +} + +func (v CodecV01) encodeStructured(e cloudevents.Event) (transport.Message, error) { + header := http.Header{} + header.Set("Content-Type", cloudevents.ApplicationCloudEventsJSON) + + body, err := codec.JsonEncodeV01(e) + if err != nil { + return nil, err + } + + msg := &Message{ + Header: header, + Body: body, + } + + return msg, nil +} + +func (v CodecV01) decodeBinary(msg transport.Message) (*cloudevents.Event, error) { + m, ok := msg.(*Message) + if !ok { + return nil, fmt.Errorf("failed to convert transport.Message to http.Message") + } + ctx, err := v.fromHeaders(m.Header) + if err != nil { + return nil, err + } + var body interface{} + if len(m.Body) > 0 { + body = m.Body + } + return &cloudevents.Event{ + Context: ctx, + Data: body, + }, nil +} + +func (v CodecV01) fromHeaders(h http.Header) (cloudevents.EventContextV01, error) { + // Normalize headers. + for k, v := range h { + ck := textproto.CanonicalMIMEHeaderKey(k) + if k != ck { + log.Printf("[warn] received header with non-canonical form; canonical: %q, got %q", ck, k) + h[ck] = v + } + } + + ec := cloudevents.EventContextV01{} + ec.CloudEventsVersion = h.Get("CE-CloudEventsVersion") + ec.EventID = h.Get("CE-EventID") + ec.EventType = h.Get("CE-EventType") + source := types.ParseURLRef(h.Get("CE-Source")) + if source != nil { + ec.Source = *source + } + ec.EventTime = types.ParseTimestamp(h.Get("CE-EventTime")) + etv := h.Get("CE-EventTypeVersion") + if etv != "" { + ec.EventTypeVersion = &etv + } + ec.SchemaURL = types.ParseURLRef(h.Get("CE-SchemaURL")) + et := h.Get("Content-Type") + ec.ContentType = &et + + extensions := make(map[string]interface{}) + for k, v := range h { + if strings.EqualFold(k[:len("CE-X-")], "CE-X-") { + key := k[len("CE-X-"):] + var tmp interface{} + if err := json.Unmarshal([]byte(v[0]), &tmp); err == nil { + extensions[key] = tmp + } else { + // If we can't unmarshal the data, treat it as a string. + extensions[key] = v[0] + } + } + } + if len(extensions) > 0 { + ec.Extensions = extensions + } + return ec, nil +} + +func (v CodecV01) decodeStructured(msg transport.Message) (*cloudevents.Event, error) { + m, ok := msg.(*Message) + if !ok { + return nil, fmt.Errorf("failed to convert transport.Message to http.Message") + } + return codec.JsonDecodeV01(m.Body) +} + +func (v CodecV01) inspectEncoding(msg transport.Message) Encoding { + log.Println("Inside v01.inspectEncoding") + version := msg.CloudEventsVersion() + log.Printf("v01.inspectEncoding version %v", version) + if version != cloudevents.CloudEventsVersionV01 { + log.Printf("v01.insepectEncoding wrong version") + return Unknown + } + m, ok := msg.(*Message) + if !ok { + log.Printf("v01.insepectEncoding wrong type: %v", reflect.TypeOf(msg)) + return Unknown + } + contentType := m.Header.Get("Content-Type") + log.Printf("v01.inspectEncoding content-type %v", contentType) + if contentType == cloudevents.ApplicationJSON { + return BinaryV01 + } + if contentType == cloudevents.ApplicationXML { + return BinaryV01 + } + if contentType == cloudevents.ApplicationCloudEventsJSON { + return StructuredV01 + } + return Unknown +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/codec_v02.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/codec_v02.go new file mode 100644 index 00000000000..51201e87e74 --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/codec_v02.go @@ -0,0 +1,272 @@ +package http + +import ( + "encoding/json" + "fmt" + "github.com/cloudevents/sdk-go/pkg/cloudevents" + "github.com/cloudevents/sdk-go/pkg/cloudevents/codec" + "github.com/cloudevents/sdk-go/pkg/cloudevents/transport" + "github.com/cloudevents/sdk-go/pkg/cloudevents/types" + "log" + "net/http" + "net/textproto" + "strings" +) + +type CodecV02 struct { + Encoding Encoding +} + +var _ transport.Codec = (*CodecV02)(nil) + +func (v CodecV02) Encode(e cloudevents.Event) (transport.Message, error) { + switch v.Encoding { + case Default: + fallthrough + case BinaryV02: + return v.encodeBinary(e) + case StructuredV02: + return v.encodeStructured(e) + default: + return nil, fmt.Errorf("unknown encoding: %d", v.Encoding) + } +} + +func (v CodecV02) Decode(msg transport.Message) (*cloudevents.Event, error) { + switch v.inspectEncoding(msg) { + case BinaryV02: + return v.decodeBinary(msg) + case StructuredV02: + return v.decodeStructured(msg) + default: + return nil, fmt.Errorf("unknown encoding for message %v", msg) + } +} + +func (v CodecV02) encodeBinary(e cloudevents.Event) (transport.Message, error) { + header, err := v.toHeaders(e.Context.AsV02()) + if err != nil { + return nil, err + } + + body, err := marshalEventData(e.Context.GetDataMediaType(), e.Data) + if err != nil { + return nil, err + } + + msg := &Message{ + Header: header, + Body: body, + } + + return msg, nil +} + +func (v CodecV02) toHeaders(ec cloudevents.EventContextV02) (http.Header, error) { + h := http.Header{} + h.Set("ce-specversion", ec.SpecVersion) + h.Set("ce-type", ec.Type) + h.Set("ce-source", ec.Source.String()) + h.Set("ce-id", ec.ID) + if ec.Time != nil && !ec.Time.IsZero() { + h.Set("ce-time", ec.Time.String()) + } + if ec.SchemaURL != nil { + h.Set("ce-schemaurl", ec.SchemaURL.String()) + } + if ec.ContentType != nil { + h.Set("Content-Type", *ec.ContentType) + } else if v.Encoding == Default || v.Encoding == BinaryV02 { + // in binary v0.2, the Content-Type header is tied to ec.ContentType + // This was later found to be an issue with the spec, but yolo. + // TODO: not sure what the default should be? + h.Set("Content-Type", cloudevents.ApplicationJSON) + } + for k, v := range ec.Extensions { + // Per spec, map-valued extensions are converted to a list of headers as: + // CE-attrib-key + if mapVal, ok := v.(map[string]interface{}); ok { + for subkey, subval := range mapVal { + encoded, err := json.Marshal(subval) + if err != nil { + return nil, err + } + h.Set("ce-"+k+"-"+subkey, string(encoded)) + } + continue + } + encoded, err := json.Marshal(v) + if err != nil { + return nil, err + } + h.Set("ce-"+k, string(encoded)) + } + + return h, nil +} + +func (v CodecV02) encodeStructured(e cloudevents.Event) (transport.Message, error) { + header := http.Header{} + header.Set("Content-Type", "application/cloudevents+json") + + body, err := codec.JsonEncodeV02(e) + if err != nil { + return nil, err + } + + msg := &Message{ + Header: header, + Body: body, + } + + return msg, nil +} + +func (v CodecV02) decodeBinary(msg transport.Message) (*cloudevents.Event, error) { + m, ok := msg.(*Message) + if !ok { + return nil, fmt.Errorf("failed to convert transport.Message to http.Message") + } + ctx, err := v.fromHeaders(m.Header) + if err != nil { + return nil, err + } + var body interface{} + if len(m.Body) > 0 { + body = m.Body + } + return &cloudevents.Event{ + Context: ctx, + Data: body, + }, nil +} + +func (v CodecV02) fromHeaders(h http.Header) (cloudevents.EventContextV02, error) { + // Normalize headers. + for k, v := range h { + ck := textproto.CanonicalMIMEHeaderKey(k) + if k != ck { + log.Printf("[warn] received header with non-canonical form; canonical: %q, got %q", ck, k) + delete(h, k) + h[ck] = v + } + } + + ec := cloudevents.EventContextV02{} + + ec.SpecVersion = h.Get("ce-specversion") + h.Del("ce-specversion") + + ec.ID = h.Get("ce-id") + h.Del("ce-id") + + ec.Type = h.Get("ce-type") + h.Del("ce-type") + + source := types.ParseURLRef(h.Get("ce-source")) + if source != nil { + ec.Source = *source + } + h.Del("ce-source") + + ec.Time = types.ParseTimestamp(h.Get("ce-time")) + h.Del("ce-time") + + ec.SchemaURL = types.ParseURLRef(h.Get("ce-schemaurl")) + h.Del("ce-schemaurl") + + contentType := h.Get("Content-Type") + if contentType != "" { + ec.ContentType = &contentType + } + h.Del("Content-Type") + + // At this point, we have deleted all the known headers. + // Everything left is assumed to be an extension. + + extensions := make(map[string]interface{}) + for k, v := range h { + if strings.EqualFold(k[:len("ce-")], "ce-") { + ak := strings.ToLower(k[len("ce-"):]) + if i := strings.Index(ak, "-"); i > 0 { + // attrib-key + attrib := ak[:i] + key := ak[(i + 1):] + if xv, ok := extensions[attrib]; ok { + if m, ok := xv.(map[string]interface{}); ok { + m[key] = v + continue + } + // TODO: revisit how we want to bubble errors up. + return ec, fmt.Errorf("failed to process map type extension") + } else { + m := make(map[string]interface{}) + m[key] = v + extensions[attrib] = m + } + } else { + // key + var tmp interface{} + if err := json.Unmarshal([]byte(v[0]), &tmp); err == nil { + extensions[ak] = tmp + } else { + // If we can't unmarshal the data, treat it as a string. + extensions[ak] = v[0] + } + } + } + } + if len(extensions) > 0 { + ec.Extensions = extensions + } + return ec, nil +} + +func (v CodecV02) decodeStructured(msg transport.Message) (*cloudevents.Event, error) { + m, ok := msg.(*Message) + if !ok { + return nil, fmt.Errorf("failed to convert transport.Message to http.Message") + } + + ec := cloudevents.EventContextV02{} + if err := json.Unmarshal(m.Body, &ec); err != nil { + return nil, err + } + + raw := make(map[string]json.RawMessage) + + if err := json.Unmarshal(m.Body, &raw); err != nil { + return nil, err + } + var data interface{} + if d, ok := raw["data"]; ok { + data = []byte(d) + } + + return &cloudevents.Event{ + Context: ec, + Data: data, + }, nil +} + +func (v CodecV02) inspectEncoding(msg transport.Message) Encoding { + version := msg.CloudEventsVersion() + if version != cloudevents.CloudEventsVersionV02 { + return Unknown + } + m, ok := msg.(*Message) + if !ok { + return Unknown + } + contentType := m.Header.Get("Content-Type") + if contentType == cloudevents.ApplicationJSON { + return BinaryV02 + } + if contentType == cloudevents.ApplicationXML { + return BinaryV02 + } + if contentType == cloudevents.ApplicationCloudEventsJSON { + return StructuredV02 + } + return Unknown +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/codec_v03.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/codec_v03.go new file mode 100644 index 00000000000..8393111d137 --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/codec_v03.go @@ -0,0 +1,280 @@ +package http + +import ( + "encoding/json" + "fmt" + "github.com/cloudevents/sdk-go/pkg/cloudevents" + "github.com/cloudevents/sdk-go/pkg/cloudevents/codec" + "github.com/cloudevents/sdk-go/pkg/cloudevents/transport" + "github.com/cloudevents/sdk-go/pkg/cloudevents/types" + "log" + "net/http" + "net/textproto" + "strings" +) + +type CodecV03 struct { + Encoding Encoding +} + +var _ transport.Codec = (*CodecV03)(nil) + +func (v CodecV03) Encode(e cloudevents.Event) (transport.Message, error) { + switch v.Encoding { + case Default: + fallthrough + case BinaryV03: + return v.encodeBinary(e) + case StructuredV03: + return v.encodeStructured(e) + case BatchedV03: + return nil, fmt.Errorf("not implemented") + default: + return nil, fmt.Errorf("unknown encoding: %d", v.Encoding) + } +} + +func (v CodecV03) Decode(msg transport.Message) (*cloudevents.Event, error) { + switch v.inspectEncoding(msg) { + case BinaryV03: + return v.decodeBinary(msg) + case StructuredV03: + return v.decodeStructured(msg) + case BatchedV03: + return nil, fmt.Errorf("not implemented") + default: + return nil, fmt.Errorf("unknown encoding for message %v", msg) + } +} + +func (v CodecV03) encodeBinary(e cloudevents.Event) (transport.Message, error) { + header, err := v.toHeaders(e.Context.AsV03()) + if err != nil { + return nil, err + } + + body, err := marshalEventData(e.Context.GetDataMediaType(), e.Data) + if err != nil { + return nil, err + } + + msg := &Message{ + Header: header, + Body: body, + } + + return msg, nil +} + +func (v CodecV03) toHeaders(ec cloudevents.EventContextV03) (http.Header, error) { + h := http.Header{} + h.Set("ce-specversion", ec.SpecVersion) + h.Set("ce-type", ec.Type) + h.Set("ce-source", ec.Source.String()) + h.Set("ce-id", ec.ID) + if ec.Time != nil && !ec.Time.IsZero() { + h.Set("ce-time", ec.Time.String()) + } + if ec.SchemaURL != nil { + h.Set("ce-schemaurl", ec.SchemaURL.String()) + } + + if ec.DataContentType != nil { + h.Set("Content-Type", *ec.DataContentType) + } else if v.Encoding == Default || v.Encoding == BinaryV03 { + // in binary v0.2, the Content-Type header is tied to ec.ContentType + // This was later found to be an issue with the spec, but yolo. + // TODO: not sure what the default should be? + h.Set("Content-Type", cloudevents.ApplicationJSON) + } + for k, v := range ec.Extensions { + // Per spec, map-valued extensions are converted to a list of headers as: + // CE-attrib-key + if mapVal, ok := v.(map[string]interface{}); ok { + for subkey, subval := range mapVal { + encoded, err := json.Marshal(subval) + if err != nil { + return nil, err + } + h.Set("ce-"+k+"-"+subkey, string(encoded)) + } + continue + } + encoded, err := json.Marshal(v) + if err != nil { + return nil, err + } + h.Set("ce-"+k, string(encoded)) + } + + return h, nil +} + +func (v CodecV03) encodeStructured(e cloudevents.Event) (transport.Message, error) { + header := http.Header{} + header.Set("Content-Type", "application/cloudevents+json") + + body, err := codec.JsonEncodeV03(e) + if err != nil { + return nil, err + } + + msg := &Message{ + Header: header, + Body: body, + } + + return msg, nil +} + +func (v CodecV03) decodeBinary(msg transport.Message) (*cloudevents.Event, error) { + m, ok := msg.(*Message) + if !ok { + return nil, fmt.Errorf("failed to convert transport.Message to http.Message") + } + ctx, err := v.fromHeaders(m.Header) + if err != nil { + return nil, err + } + var body interface{} + if len(m.Body) > 0 { + body = m.Body + } + return &cloudevents.Event{ + Context: ctx, + Data: body, + }, nil +} + +func (v CodecV03) fromHeaders(h http.Header) (cloudevents.EventContextV03, error) { + // Normalize headers. + for k, v := range h { + ck := textproto.CanonicalMIMEHeaderKey(k) + if k != ck { + log.Printf("[warn] received header with non-canonical form; canonical: %q, got %q", ck, k) + delete(h, k) + h[ck] = v + } + } + + ec := cloudevents.EventContextV03{} + + ec.SpecVersion = h.Get("ce-specversion") + h.Del("ce-specversion") + + ec.ID = h.Get("ce-id") + h.Del("ce-id") + + ec.Type = h.Get("ce-type") + h.Del("ce-type") + + source := types.ParseURLRef(h.Get("ce-source")) + if source != nil { + ec.Source = *source + } + h.Del("ce-source") + + ec.Time = types.ParseTimestamp(h.Get("ce-time")) + h.Del("ce-time") + + ec.SchemaURL = types.ParseURLRef(h.Get("ce-schemaurl")) + h.Del("ce-schemaurl") + + contentType := h.Get("Content-Type") + if contentType != "" { + ec.DataContentType = &contentType + } + h.Del("Content-Type") + + // At this point, we have deleted all the known headers. + // Everything left is assumed to be an extension. + + extensions := make(map[string]interface{}) + for k, v := range h { + if strings.EqualFold(k[:len("ce-")], "ce-") { + ak := strings.ToLower(k[len("ce-"):]) + if i := strings.Index(ak, "-"); i > 0 { + // attrib-key + attrib := ak[:i] + key := ak[(i + 1):] + if xv, ok := extensions[attrib]; ok { + if m, ok := xv.(map[string]interface{}); ok { + m[key] = v + continue + } + // TODO: revisit how we want to bubble errors up. + return ec, fmt.Errorf("failed to process map type extension") + } else { + m := make(map[string]interface{}) + m[key] = v + extensions[attrib] = m + } + } else { + // key + var tmp interface{} + if err := json.Unmarshal([]byte(v[0]), &tmp); err == nil { + extensions[ak] = tmp + } else { + // If we can't unmarshal the data, treat it as a string. + extensions[ak] = v[0] + } + } + } + } + if len(extensions) > 0 { + ec.Extensions = extensions + } + return ec, nil +} + +func (v CodecV03) decodeStructured(msg transport.Message) (*cloudevents.Event, error) { + m, ok := msg.(*Message) + if !ok { + return nil, fmt.Errorf("failed to convert transport.Message to http.Message") + } + + ec := cloudevents.EventContextV03{} + if err := json.Unmarshal(m.Body, &ec); err != nil { + return nil, err + } + + raw := make(map[string]json.RawMessage) + + if err := json.Unmarshal(m.Body, &raw); err != nil { + return nil, err + } + var data interface{} + if d, ok := raw["data"]; ok { + data = []byte(d) + } + + return &cloudevents.Event{ + Context: ec, + Data: data, + }, nil +} + +func (v CodecV03) inspectEncoding(msg transport.Message) Encoding { + version := msg.CloudEventsVersion() + if version != cloudevents.CloudEventsVersionV03 { + return Unknown + } + m, ok := msg.(*Message) + if !ok { + return Unknown + } + contentType := m.Header.Get("Content-Type") + if contentType == cloudevents.ApplicationJSON { + return BinaryV03 + } + if contentType == cloudevents.ApplicationXML { + return BinaryV03 + } + if contentType == cloudevents.ApplicationCloudEventsJSON { + return StructuredV03 + } + if contentType == cloudevents.ApplicationCloudEventsBatchJSON { + return BatchedV03 + } + return Unknown +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/encoding.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/encoding.go new file mode 100644 index 00000000000..e16483f9a15 --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/encoding.go @@ -0,0 +1,76 @@ +package http + +type Encoding int32 + +const ( + Default Encoding = iota + BinaryV01 + StructuredV01 + BinaryV02 + StructuredV02 + BinaryV03 + StructuredV03 + BatchedV03 + Unknown +) + +func (e Encoding) String() string { + switch e { + case Default: + return "Default Encoding " + e.Version() + + // Binary + case BinaryV01: + fallthrough + case BinaryV02: + fallthrough + case BinaryV03: + return "Binary Encoding " + e.Version() + + // Structured + case StructuredV01: + fallthrough + case StructuredV02: + fallthrough + case StructuredV03: + return "Structured Encoding " + e.Version() + + // Batched + case BatchedV03: + return "Batched Encoding " + e.Version() + + default: + return "Unknown Encoding" + } +} + +func (e Encoding) Version() string { + switch e { + case Default: + return "Default" + + // Version 0.1 + case BinaryV01: + fallthrough + case StructuredV01: + return "v0.1" + + // Version 0.2 + case BinaryV02: + fallthrough + case StructuredV02: + return "v0.2" + + // Version 0.3 + case BinaryV03: + fallthrough + case StructuredV03: + fallthrough + case BatchedV03: + return "v0.3" + + // Unknown + default: + return "Unknown" + } +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/message.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/message.go new file mode 100644 index 00000000000..c7eb1813624 --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/message.go @@ -0,0 +1,69 @@ +package http + +import ( + "encoding/json" + "github.com/cloudevents/sdk-go/pkg/cloudevents/transport" + "net/http" +) + +// type check that this transport message impl matches the contract +var _ transport.Message = (*Message)(nil) + +type Message struct { + Header http.Header + Body []byte +} + +func (m Message) CloudEventsVersion() string { + + // TODO: the impl of this method needs to move into the codec. + + if m.Header != nil { + // Try headers first. + // v0.1, cased from the spec + if v := m.Header["CE-CloudEventsVersion"]; len(v) == 1 { + return v[0] + } + // v0.2, canonical casing + if ver := m.Header.Get("CE-CloudEventsVersion"); ver != "" { + return ver + } + + // v0.2, cased from the spec + if v := m.Header["ce-specversion"]; len(v) == 1 { + return v[0] + } + // v0.2, canonical casing + if ver := m.Header.Get("ce-specversion"); ver != "" { + return ver + } + } + + // Then try the data body. + // TODO: we need to use the correct decoding based on content type. + + raw := make(map[string]json.RawMessage) + if err := json.Unmarshal(m.Body, &raw); err != nil { + return "" + } + + // v0.1 + if v, ok := raw["cloudEventsVersion"]; ok { + var version string + if err := json.Unmarshal(v, &version); err != nil { + return "" + } + return version + } + + // v0.2 + if v, ok := raw["specversion"]; ok { + var version string + if err := json.Unmarshal(v, &version); err != nil { + return "" + } + return version + } + + return "" +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/transport.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/transport.go new file mode 100644 index 00000000000..674bad9fc25 --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/transport.go @@ -0,0 +1,164 @@ +package http + +import ( + "bytes" + "context" + "fmt" + "github.com/cloudevents/sdk-go/pkg/cloudevents" + "github.com/cloudevents/sdk-go/pkg/cloudevents/transport" + "io/ioutil" + "log" + "net/http" +) + +type EncodingSelector func(e cloudevents.Event) Encoding + +// type check that this transport message impl matches the contract +var _ transport.Sender = (*Transport)(nil) + +// Transport acts as both a http client and a http handler. +type Transport struct { + Encoding Encoding + DefaultEncodingSelectionFn EncodingSelector + + // Sending + Client *http.Client + Req *http.Request + + // Receiving + Port int + Receiver transport.Receiver + + codec transport.Codec +} + +func (t *Transport) loadCodec() bool { + if t.codec == nil { + if t.DefaultEncodingSelectionFn != nil && t.Encoding != Default { + log.Printf("[warn] Transport has a DefaultEncodingSelectionFn set but Encoding is not Default. DefaultEncodingSelectionFn will be ignored.") + } + t.codec = &Codec{ + Encoding: t.Encoding, + DefaultEncodingSelectionFn: t.DefaultEncodingSelectionFn, + } + } + return true +} + +func (t *Transport) Send(ctx context.Context, event cloudevents.Event) error { + if t.Client == nil { + t.Client = &http.Client{} + } + + var req http.Request + if t.Req != nil { + req.Method = t.Req.Method + req.URL = t.Req.URL + } + + if ok := t.loadCodec(); !ok { + return fmt.Errorf("unknown encoding set on transport: %d", t.Encoding) + } + + msg, err := t.codec.Encode(event) + if err != nil { + return err + } + + // TODO: merge the incoming request with msg, for now just replace. + if m, ok := msg.(*Message); ok { + req.Header = m.Header + req.Body = ioutil.NopCloser(bytes.NewBuffer(m.Body)) + req.ContentLength = int64(len(m.Body)) + return httpDo(ctx, &req, func(resp *http.Response, err error) error { + if err != nil { + return err + } + defer resp.Body.Close() + + if accepted(resp) { + return nil + } + return fmt.Errorf("error sending cloudevent: %s", status(resp)) + }) + } + + return fmt.Errorf("failed to encode Event into a Message") +} + +func httpDo(ctx context.Context, req *http.Request, f func(*http.Response, error) error) error { + // Run the HTTP request in a goroutine and pass the response to f. + c := make(chan error, 1) + req = req.WithContext(ctx) + go func() { c <- f(http.DefaultClient.Do(req)) }() + select { + case <-ctx.Done(): + <-c // Wait for f to return. + return ctx.Err() + case err := <-c: + return err + } +} + +// accepted is a helper method to understand if the response from the target +// accepted the CloudEvent. +func accepted(resp *http.Response) bool { + if resp.StatusCode >= 200 && resp.StatusCode < 300 { + return true + } + return false +} + +// status is a helper method to read the response of the target. +func status(resp *http.Response) string { + status := resp.Status + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return fmt.Sprintf("Status[%s] error reading response body: %v", status, err) + } + return fmt.Sprintf("Status[%s] %s", status, body) +} + +// ServeHTTP implements http.Handler +func (t *Transport) ServeHTTP(w http.ResponseWriter, r *http.Request) { + body, err := ioutil.ReadAll(r.Body) + if err != nil { + log.Printf("failed to handle request: %s %v", err, r) + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte(`{"error":"Invalid request"}`)) + return + } + msg := &Message{ + Header: r.Header, + Body: body, + } + + if ok := t.loadCodec(); !ok { + err := fmt.Errorf("unknown encoding set on transport: %d", t.Encoding) + log.Printf("failed to load codec: %s", err) + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte(fmt.Sprintf(`{"error":%q}`, err.Error()))) + return + } + event, err := t.codec.Decode(msg) + if err != nil { + log.Printf("failed to decode message: %s %v", err, r) + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte(`{"error":"Decoding Error"}`)) + return + } + + if t.Receiver != nil { + go t.Receiver.Receive(*event) + } + + // TODO: respond correctly based on decode. + w.WriteHeader(http.StatusNoContent) +} + +func (t *Transport) GetPort() int { + if t.Port > 0 { + return t.Port + } + return 8080 // default +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/message.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/message.go new file mode 100644 index 00000000000..77e4bee8aaa --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/message.go @@ -0,0 +1,8 @@ +package transport + +type Message interface { + // CloudEventsVersion returns the version of the CloudEvent. + CloudEventsVersion() string + + // TODO maybe get encoding +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/nats/codec.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/nats/codec.go new file mode 100644 index 00000000000..56132f0464d --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/nats/codec.go @@ -0,0 +1,54 @@ +package nats + +import ( + "fmt" + "github.com/cloudevents/sdk-go/pkg/cloudevents" + "github.com/cloudevents/sdk-go/pkg/cloudevents/transport" +) + +type Codec struct { + Encoding Encoding + + v02 *CodecV02 + v03 *CodecV03 +} + +var _ transport.Codec = (*Codec)(nil) + +func (c *Codec) Encode(e cloudevents.Event) (transport.Message, error) { + switch c.Encoding { + case Default: + fallthrough + case StructuredV02: + if c.v02 == nil { + c.v02 = &CodecV02{Encoding: c.Encoding} + } + return c.v02.Encode(e) + case StructuredV03: + if c.v03 == nil { + c.v03 = &CodecV03{Encoding: c.Encoding} + } + return c.v03.Encode(e) + default: + return nil, fmt.Errorf("unknown encoding: %d", c.Encoding) + } +} + +func (c *Codec) Decode(msg transport.Message) (*cloudevents.Event, error) { + switch c.Encoding { + case Default: + fallthrough + case StructuredV02: + if c.v02 == nil { + c.v02 = &CodecV02{Encoding: c.Encoding} + } + return c.v02.Decode(msg) + case StructuredV03: + if c.v03 == nil { + c.v03 = &CodecV03{Encoding: c.Encoding} + } + return c.v03.Decode(msg) + default: + return nil, fmt.Errorf("unknown encoding: %d", c.Encoding) + } +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/nats/codec_v02.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/nats/codec_v02.go new file mode 100644 index 00000000000..2b98f96366e --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/nats/codec_v02.go @@ -0,0 +1,48 @@ +package nats + +import ( + "fmt" + "github.com/cloudevents/sdk-go/pkg/cloudevents" + "github.com/cloudevents/sdk-go/pkg/cloudevents/codec" + "github.com/cloudevents/sdk-go/pkg/cloudevents/transport" +) + +type CodecV02 struct { + Encoding Encoding +} + +var _ transport.Codec = (*CodecV02)(nil) + +func (v CodecV02) Encode(e cloudevents.Event) (transport.Message, error) { + switch v.Encoding { + case Default: + fallthrough + case StructuredV02: + return v.encodeStructured(e) + default: + return nil, fmt.Errorf("unknown encoding: %d", v.Encoding) + } +} + +func (v CodecV02) Decode(msg transport.Message) (*cloudevents.Event, error) { + // only structured is supported as of v0.2 + return v.decodeStructured(msg) +} + +func (v CodecV02) encodeStructured(e cloudevents.Event) (transport.Message, error) { + body, err := codec.JsonEncodeV02(e) + if err != nil { + return nil, err + } + return &Message{ + Body: body, + }, nil +} + +func (v CodecV02) decodeStructured(msg transport.Message) (*cloudevents.Event, error) { + m, ok := msg.(*Message) + if !ok { + return nil, fmt.Errorf("failed to convert transport.Message to http.Message") + } + return codec.JsonDecodeV02(m.Body) +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/nats/codec_v03.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/nats/codec_v03.go new file mode 100644 index 00000000000..38793583caf --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/nats/codec_v03.go @@ -0,0 +1,48 @@ +package nats + +import ( + "fmt" + "github.com/cloudevents/sdk-go/pkg/cloudevents" + "github.com/cloudevents/sdk-go/pkg/cloudevents/codec" + "github.com/cloudevents/sdk-go/pkg/cloudevents/transport" +) + +type CodecV03 struct { + Encoding Encoding +} + +var _ transport.Codec = (*CodecV03)(nil) + +func (v CodecV03) Encode(e cloudevents.Event) (transport.Message, error) { + switch v.Encoding { + case Default: + fallthrough + case StructuredV03: + return v.encodeStructured(e) + default: + return nil, fmt.Errorf("unknown encoding: %d", v.Encoding) + } +} + +func (v CodecV03) Decode(msg transport.Message) (*cloudevents.Event, error) { + // only structured is supported as of v0.3 + return v.decodeStructured(msg) +} + +func (v CodecV03) encodeStructured(e cloudevents.Event) (transport.Message, error) { + body, err := codec.JsonEncodeV03(e) + if err != nil { + return nil, err + } + return &Message{ + Body: body, + }, nil +} + +func (v CodecV03) decodeStructured(msg transport.Message) (*cloudevents.Event, error) { + m, ok := msg.(*Message) + if !ok { + return nil, fmt.Errorf("failed to convert transport.Message to http.Message") + } + return codec.JsonDecodeV03(m.Body) +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/nats/encoding.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/nats/encoding.go new file mode 100644 index 00000000000..38bcfbdeeab --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/nats/encoding.go @@ -0,0 +1,45 @@ +package nats + +type Encoding int32 + +const ( + Default Encoding = iota + StructuredV02 + StructuredV03 + Unknown +) + +func (e Encoding) String() string { + switch e { + case Default: + return "Default Encoding " + e.Version() + + // Structured + case StructuredV02: + fallthrough + case StructuredV03: + return "Structured Encoding " + e.Version() + + default: + return "Unknown Encoding" + } +} + +func (e Encoding) Version() string { + switch e { + + // Version 0.2 + case Default: // <-- Move when a new default is wanted. + fallthrough + case StructuredV02: + return "v0.2" + + // Version 0.3 + case StructuredV03: + return "v0.3" + + // Unknown + default: + return "Unknown" + } +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/nats/message.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/nats/message.go new file mode 100644 index 00000000000..cc60ebca438 --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/nats/message.go @@ -0,0 +1,31 @@ +package nats + +import ( + "encoding/json" + "github.com/cloudevents/sdk-go/pkg/cloudevents/transport" +) + +// type check that this transport message impl matches the contract +var _ transport.Message = (*Message)(nil) + +type Message struct { + Body []byte +} + +func (m Message) CloudEventsVersion() string { + raw := make(map[string]json.RawMessage) + if err := json.Unmarshal(m.Body, &raw); err != nil { + return "" + } + + // v0.2 + if v, ok := raw["specversion"]; ok { + var version string + if err := json.Unmarshal(v, &version); err != nil { + return "" + } + return version + } + + return "" +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/nats/transport.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/nats/transport.go new file mode 100644 index 00000000000..6c0a0265791 --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/nats/transport.go @@ -0,0 +1,91 @@ +package nats + +import ( + "context" + "fmt" + "github.com/cloudevents/sdk-go/pkg/cloudevents" + "github.com/cloudevents/sdk-go/pkg/cloudevents/transport" + "github.com/nats-io/go-nats" + "log" +) + +// type check that this transport message impl matches the contract +var _ transport.Sender = (*Transport)(nil) + +// Transport acts as both a http client and a http handler. +type Transport struct { + Encoding Encoding + Conn *nats.Conn + Subject string + + sub *nats.Subscription + + Receiver transport.Receiver + + codec transport.Codec +} + +func (t *Transport) loadCodec() bool { + if t.codec == nil { + switch t.Encoding { + case Default: + t.codec = &Codec{} + case StructuredV02: + t.codec = &CodecV02{Encoding: t.Encoding} + case StructuredV03: + t.codec = &CodecV03{Encoding: t.Encoding} + default: + return false + } + } + return true +} + +func (t *Transport) Send(ctx context.Context, event cloudevents.Event) error { + if ok := t.loadCodec(); !ok { + return fmt.Errorf("unknown encoding set on transport: %d", t.Encoding) + } + + msg, err := t.codec.Encode(event) + if err != nil { + return err + } + + if m, ok := msg.(*Message); ok { + return t.Conn.Publish(t.Subject, m.Body) + } + + return fmt.Errorf("failed to encode Event into a Message") +} + +func (t *Transport) Listen(ctx context.Context) error { + if t.sub != nil { + return fmt.Errorf("already subscribed") + } + + if ok := t.loadCodec(); !ok { + return fmt.Errorf("unknown encoding set on transport: %d", t.Encoding) + } + + // TODO: there could be more than one subscription. Might have to do a map + // of subject to subscription. + + if t.Subject == "" { + return fmt.Errorf("subject required for nats listen") + } + + var err error + // Simple Async Subscriber + t.sub, err = t.Conn.Subscribe(t.Subject, func(m *nats.Msg) { + msg := &Message{ + Body: m.Data, + } + event, err := t.codec.Decode(msg) + if err != nil { + log.Printf("failed to decode message: %s", err) + return + } + t.Receiver.Receive(*event) + }) + return err +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/transport.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/transport.go new file mode 100644 index 00000000000..d6f727bc54a --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/transport.go @@ -0,0 +1,17 @@ +package transport + +import ( + "context" + "github.com/cloudevents/sdk-go/pkg/cloudevents" +) + +// Transport is the interface for transport sender to send the converted Message +// over the underlying transport. +type Sender interface { + Send(context.Context, cloudevents.Event) error +} + +// Receiver TODO not sure yet. +type Receiver interface { + Receive(cloudevents.Event) +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/types/allocate.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/types/allocate.go new file mode 100644 index 00000000000..00a4870f900 --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/types/allocate.go @@ -0,0 +1,36 @@ +package types + +import "reflect" + +// Allocates allocates a new instance of type t and returns: +// asPtr is of type t if t is a pointer type and of type &t otherwise +// asValue is a Value of type t pointing to the same data as asPtr +func Allocate(obj interface{}) (asPtr interface{}, asValue reflect.Value) { + if obj == nil { + return nil, reflect.Value{} + } + + switch t := reflect.TypeOf(obj); t.Kind() { + case reflect.Ptr: + reflectPtr := reflect.New(t.Elem()) + asPtr = reflectPtr.Interface() + asValue = reflectPtr + case reflect.Map: + reflectPtr := reflect.MakeMap(t) + asPtr = reflectPtr.Interface() + asValue = reflectPtr + case reflect.String: + reflectPtr := reflect.New(t) + asPtr = "" + asValue = reflectPtr.Elem() + case reflect.Slice: + reflectPtr := reflect.MakeSlice(t, 0, 0) + asPtr = reflectPtr.Interface() + asValue = reflectPtr + default: + reflectPtr := reflect.New(t) + asPtr = reflectPtr.Interface() + asValue = reflectPtr.Elem() + } + return +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/types/timestamp.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/types/timestamp.go new file mode 100644 index 00000000000..b535e5dad6d --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/types/timestamp.go @@ -0,0 +1,51 @@ +package types + +import ( + "encoding/json" + "fmt" + "time" +) + +type Timestamp struct { + time.Time +} + +func ParseTimestamp(t string) *Timestamp { + if t == "" { + return nil + } + timestamp, err := time.Parse(time.RFC3339Nano, t) + if err != nil { + return nil + } + return &Timestamp{Time: timestamp} +} + +// This allows json marshaling to always be in RFC3339Nano format. +func (t *Timestamp) MarshalJSON() ([]byte, error) { + if t == nil || t.IsZero() { + return []byte(`""`), nil + } + rfc3339 := fmt.Sprintf("%q", t.Format(time.RFC3339Nano)) + return []byte(rfc3339), nil +} + +func (t *Timestamp) UnmarshalJSON(b []byte) error { + var timestamp string + if err := json.Unmarshal(b, ×tamp); err != nil { + return err + } + pt := ParseTimestamp(timestamp) + if pt != nil { + *t = *pt + } + return nil +} + +func (t *Timestamp) String() string { + if t == nil { + return time.Time{}.Format(time.RFC3339Nano) + } + + return t.Format(time.RFC3339Nano) +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/types/urlref.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/types/urlref.go new file mode 100644 index 00000000000..0898e4ece8e --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/types/urlref.go @@ -0,0 +1,44 @@ +package types + +import ( + "encoding/json" + "fmt" + "net/url" +) + +type URLRef struct { + url.URL +} + +func ParseURLRef(u string) *URLRef { + if u == "" { + return nil + } + pu, err := url.Parse(u) + if err != nil { + return nil + } + return &URLRef{URL: *pu} +} + +// This allows json marshaling to always be in RFC3339Nano format. +func (u URLRef) MarshalJSON() ([]byte, error) { + b := fmt.Sprintf("%q", u.String()) + return []byte(b), nil +} + +func (u *URLRef) UnmarshalJSON(b []byte) error { + var ref string + if err := json.Unmarshal(b, &ref); err != nil { + return err + } + *u = *ParseURLRef(ref) + return nil +} + +func (u *URLRef) String() string { + if u == nil { + return "" + } + return u.URL.String() +} From f8d606d50d2001b71095c57b3a000687fca71529 Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Mon, 4 Mar 2019 14:48:27 -0800 Subject: [PATCH 107/128] Fix unit tests. --- .../eventing/v1alpha1/broker_types_test.go | 176 +++++++++------ pkg/broker/ce_receiver.go | 15 +- pkg/broker/ce_receiver_test.go | 200 ++++++++++++++++++ pkg/reconciler/v1alpha1/broker/broker_test.go | 8 +- .../v1alpha1/trigger/trigger_test.go | 4 +- 5 files changed, 335 insertions(+), 68 deletions(-) create mode 100644 pkg/broker/ce_receiver_test.go diff --git a/pkg/apis/eventing/v1alpha1/broker_types_test.go b/pkg/apis/eventing/v1alpha1/broker_types_test.go index 3970fbcf56a..05764b8777c 100644 --- a/pkg/apis/eventing/v1alpha1/broker_types_test.go +++ b/pkg/apis/eventing/v1alpha1/broker_types_test.go @@ -35,8 +35,8 @@ var ( Status: corev1.ConditionTrue, } - brokerConditionChannel = duckv1alpha1.Condition{ - Type: BrokerConditionChannel, + brokerConditionTriggerChannel = duckv1alpha1.Condition{ + Type: BrokerConditionTriggerChannel, Status: corev1.ConditionTrue, } @@ -71,7 +71,7 @@ func TestBrokerGetCondition(t *testing.T) { bs: &BrokerStatus{ Conditions: []duckv1alpha1.Condition{ brokerConditionIngress, - brokerConditionChannel, + brokerConditionTriggerChannel, brokerConditionFilter, }, }, @@ -81,7 +81,7 @@ func TestBrokerGetCondition(t *testing.T) { name: "multiple conditions, condition false", bs: &BrokerStatus{ Conditions: []duckv1alpha1.Condition{ - brokerConditionChannel, + brokerConditionTriggerChannel, brokerConditionFilter, brokerConditionAddressable, }, @@ -123,24 +123,30 @@ func TestBrokerInitializeConditions(t *testing.T) { Type: BrokerConditionAddressable, Status: corev1.ConditionUnknown, }, { - Type: BrokerConditionChannel, + Type: BrokerConditionFilter, Status: corev1.ConditionUnknown, }, { - Type: BrokerConditionFilter, + Type: BrokerConditionIngressChannel, Status: corev1.ConditionUnknown, }, { Type: BrokerConditionIngress, Status: corev1.ConditionUnknown, + }, { + Type: BrokerConditionIngressSubscription, + Status: corev1.ConditionUnknown, }, { Type: BrokerConditionReady, Status: corev1.ConditionUnknown, + }, { + Type: BrokerConditionTriggerChannel, + Status: corev1.ConditionUnknown, }}, }, }, { name: "one false", bs: &BrokerStatus{ Conditions: []duckv1alpha1.Condition{{ - Type: BrokerConditionChannel, + Type: BrokerConditionTriggerChannel, Status: corev1.ConditionFalse, }}, }, @@ -148,18 +154,24 @@ func TestBrokerInitializeConditions(t *testing.T) { Conditions: []duckv1alpha1.Condition{{ Type: BrokerConditionAddressable, Status: corev1.ConditionUnknown, - }, { - Type: BrokerConditionChannel, - Status: corev1.ConditionFalse, }, { Type: BrokerConditionFilter, Status: corev1.ConditionUnknown, + }, { + Type: BrokerConditionIngressChannel, + Status: corev1.ConditionUnknown, }, { Type: BrokerConditionIngress, Status: corev1.ConditionUnknown, + }, { + Type: BrokerConditionIngressSubscription, + Status: corev1.ConditionUnknown, }, { Type: BrokerConditionReady, Status: corev1.ConditionUnknown, + }, { + Type: BrokerConditionTriggerChannel, + Status: corev1.ConditionFalse, }}, }, }, { @@ -174,18 +186,24 @@ func TestBrokerInitializeConditions(t *testing.T) { Conditions: []duckv1alpha1.Condition{{ Type: BrokerConditionAddressable, Status: corev1.ConditionUnknown, - }, { - Type: BrokerConditionChannel, - Status: corev1.ConditionUnknown, }, { Type: BrokerConditionFilter, Status: corev1.ConditionTrue, + }, { + Type: BrokerConditionIngressChannel, + Status: corev1.ConditionUnknown, }, { Type: BrokerConditionIngress, Status: corev1.ConditionUnknown, + }, { + Type: BrokerConditionIngressSubscription, + Status: corev1.ConditionUnknown, }, { Type: BrokerConditionReady, Status: corev1.ConditionUnknown, + }, { + Type: BrokerConditionTriggerChannel, + Status: corev1.ConditionUnknown, }}, }, }} @@ -202,68 +220,106 @@ func TestBrokerInitializeConditions(t *testing.T) { func TestBrokerIsReady(t *testing.T) { tests := []struct { - name string - markChannelReady bool - markFilterReady bool - markIngressReady bool - address string - wantReady bool + name string + markIngressReady bool + markTriggerChannelReady bool + markIngressChannelReady bool + markFilterReady bool + address string + markIngressSubscriptionReady bool + wantReady bool }{{ - name: "all happy", - markChannelReady: true, - markFilterReady: true, - markIngressReady: true, - address: "hostname", - wantReady: true, + name: "all happy", + markIngressReady: true, + markTriggerChannelReady: true, + markIngressChannelReady: true, + markFilterReady: true, + address: "hostname", + markIngressSubscriptionReady: true, + wantReady: true, + }, { + name: "ingress sad", + markIngressReady: false, + markTriggerChannelReady: true, + markIngressChannelReady: true, + markFilterReady: true, + address: "hostname", + markIngressSubscriptionReady: true, + wantReady: false, + }, { + name: "trigger channel sad", + markIngressReady: true, + markTriggerChannelReady: false, + markIngressChannelReady: true, + markFilterReady: true, + address: "hostname", + markIngressSubscriptionReady: true, + wantReady: false, }, { - name: "channel sad", - markChannelReady: false, - markFilterReady: true, - markIngressReady: true, - address: "hostname", - wantReady: false, + name: "ingress channel sad", + markIngressReady: true, + markTriggerChannelReady: true, + markIngressChannelReady: false, + markFilterReady: true, + address: "hostname", + markIngressSubscriptionReady: true, + wantReady: false, }, { - name: "filter sad", - markChannelReady: true, - markFilterReady: false, - markIngressReady: true, - address: "hostname", - wantReady: false, + name: "filter sad", + markIngressReady: true, + markTriggerChannelReady: true, + markIngressChannelReady: true, + markFilterReady: false, + address: "hostname", + markIngressSubscriptionReady: true, + wantReady: false, }, { - name: "ingress sad", - markChannelReady: true, - markFilterReady: true, - markIngressReady: false, - address: "hostname", - wantReady: false, + name: "addressable sad", + markIngressReady: true, + markTriggerChannelReady: true, + markIngressChannelReady: true, + markFilterReady: true, + address: "", + markIngressSubscriptionReady: true, + wantReady: false, }, { - name: "addressable sad", - markChannelReady: true, - markFilterReady: true, - markIngressReady: true, - address: "", - wantReady: false, + name: "ingress subscription sad", + markIngressReady: true, + markTriggerChannelReady: true, + markIngressChannelReady: true, + markFilterReady: true, + address: "hostname", + markIngressSubscriptionReady: false, + wantReady: false, }, { - name: "all sad", - markChannelReady: false, - markFilterReady: false, - markIngressReady: false, - address: "", - wantReady: false, + name: "all sad", + markIngressReady: false, + markTriggerChannelReady: false, + markIngressChannelReady: true, + markFilterReady: false, + address: "", + markIngressSubscriptionReady: true, + wantReady: false, }} for _, test := range tests { t.Run(test.name, func(t *testing.T) { ts := &BrokerStatus{} - if test.markChannelReady { - ts.MarkChannelReady() + if test.markIngressReady { + ts.MarkIngressReady() + } + if test.markTriggerChannelReady { + ts.MarkTriggerChannelReady() + } + if test.markIngressChannelReady { + ts.MarkIngressChannelReady() } if test.markFilterReady { ts.MarkFilterReady() } - if test.markIngressReady { - ts.MarkIngressReady() - } ts.SetAddress(test.address) + if test.markIngressSubscriptionReady { + ts.MarkIngressSubscriptionReady() + } got := ts.IsReady() if test.wantReady != got { t.Errorf("unexpected readiness: want %v, got %v", test.wantReady, got) diff --git a/pkg/broker/ce_receiver.go b/pkg/broker/ce_receiver.go index 538c5636cc1..9f3331187e7 100644 --- a/pkg/broker/ce_receiver.go +++ b/pkg/broker/ce_receiver.go @@ -50,10 +50,16 @@ type Receiver struct { port int - httpClient *http.Client + httpClient HTTPDoer codec cehttp.Codec } +type HTTPDoer interface { + Do(*http.Request) (*http.Response, error) +} + +var _ HTTPDoer = &http.Client{} + // New creates a new Receiver and its associated MessageReceiver. The caller is responsible for // Start()ing the returned MessageReceiver. func New(logger *zap.Logger, client client.Client) *Receiver { @@ -185,11 +191,14 @@ func (r *Receiver) decodeHTTP(headers http.Header, bodyReadCloser io.ReadCloser) if err != nil { return nil, err } + if len(body) == 0 { + return nil, nil + } + msg := &cehttp.Message{ Header: headers, Body: body, } - return r.codec.Decode(msg) } @@ -211,7 +220,7 @@ func (r *Receiver) sendEvent(trigger provisioners.ChannelReference, event *cloud } subscriberURI, err := url.Parse(subscriberURIString) if err != nil { - // sojemthing + r.logger.Error("Unable to parse subscriberURI", zap.Error(err), zap.String("subscriberURIString", subscriberURIString)) return nil, err } diff --git a/pkg/broker/ce_receiver_test.go b/pkg/broker/ce_receiver_test.go new file mode 100644 index 00000000000..07f79bed903 --- /dev/null +++ b/pkg/broker/ce_receiver_test.go @@ -0,0 +1,200 @@ +/* + * Copyright 2019 The Knative Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package broker + +import ( + "bytes" + "fmt" + "io/ioutil" + "net/http" + "net/http/httptest" + "strings" + "testing" + + eventingv1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" + "github.com/knative/eventing/pkg/utils" + "go.uber.org/zap" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +const ( + testNS = "test-namespace" + triggerName = "test-trigger" + eventType = `com.example.someevent` + eventSource = `/mycontext` +) + +func init() { + // Add types to scheme. + eventingv1alpha1.AddToScheme(scheme.Scheme) +} + +func TestReceiver(t *testing.T) { + testCases := map[string]struct { + initialState []runtime.Object + requestFails bool + expectedErr bool + expectedDispatch bool + }{ + "Trigger.Get fails": { + // No trigger exists, so the Get will fail. + expectedErr: true, + }, + "Trigger doesn't have SubscriberURI": { + initialState: []runtime.Object{ + makeTriggerWithoutSubscriberURI(), + }, + expectedErr: true, + }, + "Trigger without a Filter": { + initialState: []runtime.Object{ + makeTriggerWithoutFilter(), + }, + }, + "Wrong type": { + initialState: []runtime.Object{ + makeTrigger("some-other-type", "Any"), + }, + }, + "Wrong source": { + initialState: []runtime.Object{ + makeTrigger("Any", "some-other-source"), + }, + }, + "Dispatch failed": { + initialState: []runtime.Object{ + makeTrigger("Any", "Any"), + }, + requestFails: true, + expectedErr: true, + expectedDispatch: true, + }, + "Dispatch succeeded - Any": { + initialState: []runtime.Object{ + makeTrigger("Any", "Any"), + }, + expectedDispatch: true, + }, + "Dispatch succeeded - Specific": { + initialState: []runtime.Object{ + makeTrigger(eventType, eventSource), + }, + expectedDispatch: true, + }, + } + for n, tc := range testCases { + t.Run(n, func(t *testing.T) { + mr := New( + zap.NewNop(), + fake.NewFakeClient(tc.initialState...)) + fh := &fakeHTTPDoer{ + failRequest: tc.requestFails, + } + mr.httpClient = fh + + resp := httptest.NewRecorder() + mr.ServeHTTP(resp, makeRequest()) + if tc.expectedErr { + if resp.Result().StatusCode >= 200 && resp.Result().StatusCode < 300 { + t.Errorf("Expected an error. Actual: %v", resp.Result()) + } + } else { + if resp.Result().StatusCode < 200 || resp.Result().StatusCode >= 300 { + t.Errorf("Expected success. Actual: %v", resp.Result()) + } + } + if tc.expectedDispatch != fh.requestReceived { + t.Errorf("Incorrect dispatch. Expected %v, Actual %v", tc.expectedDispatch, fh.requestReceived) + } + }) + } +} + +type fakeHTTPDoer struct { + failRequest bool + requestReceived bool +} + +func (h *fakeHTTPDoer) Do(_ *http.Request) (*http.Response, error) { + h.requestReceived = true + sc := http.StatusOK + if h.failRequest { + sc = http.StatusBadRequest + } + return &http.Response{ + StatusCode: sc, + Body: ioutil.NopCloser(bytes.NewBufferString("")), + }, nil +} + +func makeTrigger(t, s string) *eventingv1alpha1.Trigger { + return &eventingv1alpha1.Trigger{ + TypeMeta: v1.TypeMeta{ + APIVersion: "eventing.knative.dev/v1alpha1", + Kind: "Trigger", + }, + ObjectMeta: v1.ObjectMeta{ + Namespace: testNS, + Name: triggerName, + }, + Spec: eventingv1alpha1.TriggerSpec{ + Filter: &eventingv1alpha1.TriggerFilter{ + SourceAndType: &eventingv1alpha1.TriggerFilterSourceAndType{ + Type: t, + Source: s, + }, + }, + }, + Status: eventingv1alpha1.TriggerStatus{ + SubscriberURI: "subscriberURI", + }, + } +} + +func makeTriggerWithoutFilter() *eventingv1alpha1.Trigger { + t := makeTrigger("Any", "Any") + t.Spec.Filter = nil + return t +} + +func makeTriggerWithoutSubscriberURI() *eventingv1alpha1.Trigger { + t := makeTrigger("Any", "Any") + t.Status = eventingv1alpha1.TriggerStatus{} + return t +} + +func makeRequest() *http.Request { + req := httptest.NewRequest("POST", "/", strings.NewReader(``)) + req.Host = fmt.Sprintf("%s.%s.triggers.%s", triggerName, testNS, utils.GetClusterDomainName()) + + eventAttributes := map[string]string{ + "CE-CloudEventsVersion": `0.1`, + "CE-EventType": eventType, + "CE-EventTypeVersion": `1.0`, + "CE-Source": eventSource, + "CE-EventID": `A234-1234-1234`, + "CE-EventTime": `2018-04-05T17:31:00Z`, + "Content-Type": "application/xml", + } + for k, v := range eventAttributes { + req.Header.Set(k, v) + } + return req +} diff --git a/pkg/reconciler/v1alpha1/broker/broker_test.go b/pkg/reconciler/v1alpha1/broker/broker_test.go index 72de1f15252..9e8776a317c 100644 --- a/pkg/reconciler/v1alpha1/broker/broker_test.go +++ b/pkg/reconciler/v1alpha1/broker/broker_test.go @@ -590,10 +590,12 @@ func makeBroker() *v1alpha1.Broker { func makeReadyBroker() *v1alpha1.Broker { b := makeBroker() b.Status.InitializeConditions() - b.Status.MarkChannelReady() - b.Status.SetAddress(fmt.Sprintf("%s-broker.%s.svc.%s", brokerName, testNS, utils.GetClusterDomainName())) - b.Status.MarkFilterReady() b.Status.MarkIngressReady() + b.Status.MarkTriggerChannelReady() + b.Status.MarkIngressChannelReady() + b.Status.MarkFilterReady() + b.Status.SetAddress(fmt.Sprintf("%s-broker.%s.svc.%s", brokerName, testNS, utils.GetClusterDomainName())) + b.Status.MarkIngressSubscriptionReady() return b } diff --git a/pkg/reconciler/v1alpha1/trigger/trigger_test.go b/pkg/reconciler/v1alpha1/trigger/trigger_test.go index 229546a684b..4b0a617a616 100644 --- a/pkg/reconciler/v1alpha1/trigger/trigger_test.go +++ b/pkg/reconciler/v1alpha1/trigger/trigger_test.go @@ -614,11 +614,11 @@ func makeDifferentVirtualService() *istiov1alpha3.VirtualService { } func makeSameSubscription() *v1alpha1.Subscription { - return makeSubscription(makeTrigger(), makeChannel(), makeK8sService()) + return makeSubscription(makeTrigger(), makeChannel(), makeChannel(), makeK8sService()) } func makeDifferentSubscription() *v1alpha1.Subscription { - return makeSubscription(makeTrigger(), makeDifferentChannel(), makeK8sService()) + return makeSubscription(makeTrigger(), makeChannel(), makeDifferentChannel(), makeK8sService()) } func getOwnerReference() metav1.OwnerReference { From a3b77289c288c6fb90878d6376b61982002d753d Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Mon, 4 Mar 2019 14:50:41 -0800 Subject: [PATCH 108/128] Delete old recevier. --- pkg/broker/old/receiver.go | 141 ----------------------- pkg/broker/old/receiver_test.go | 198 -------------------------------- 2 files changed, 339 deletions(-) delete mode 100644 pkg/broker/old/receiver.go delete mode 100644 pkg/broker/old/receiver_test.go diff --git a/pkg/broker/old/receiver.go b/pkg/broker/old/receiver.go deleted file mode 100644 index 7bbcdea8a18..00000000000 --- a/pkg/broker/old/receiver.go +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Copyright 2019 The Knative Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package old - -import ( - "context" - "errors" - - eventingv1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" - "github.com/knative/eventing/pkg/provisioners" - "go.uber.org/zap" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/manager" -) - -// Receiver parses Cloud Events, determines if they pass a filter, and sends them to a subscriber. -type Receiver struct { - logger *zap.Logger - client client.Client - - dispatcher provisioners.Dispatcher -} - -// New creates a new Receiver and its associated MessageReceiver. The caller is responsible for -// Start()ing the returned MessageReceiver. -func New(logger *zap.Logger, client client.Client) (*Receiver, manager.Runnable) { - r := &Receiver{ - logger: logger, - client: client, - dispatcher: provisioners.NewMessageDispatcher(logger.Sugar()), - } - return r, r.newMessageReceiver() -} - -func (r *Receiver) newMessageReceiver() *provisioners.MessageReceiver { - if err := r.initClient(); err != nil { - r.logger.Warn("Failed to initialize client", zap.Error(err)) - } - return provisioners.NewMessageReceiver(r.sendEvent, r.logger.Sugar()) -} - -// sendEvent sends an event to a subscriber if the trigger filter passes. -func (r *Receiver) sendEvent(trigger provisioners.ChannelReference, message *provisioners.Message) error { - r.logger.Debug("Received message", zap.Any("triggerRef", trigger)) - ctx := context.Background() - - t, err := r.getTrigger(ctx, trigger) - if err != nil { - r.logger.Info("Unable to get the Trigger", zap.Error(err), zap.Any("triggerRef", trigger)) - return err - } - - subscriberURI := t.Status.SubscriberURI - if subscriberURI == "" { - r.logger.Error("Unable to read subscriberURI") - return errors.New("unable to read subscriberURI") - } - - if !r.shouldSendMessage(&t.Spec, message) { - r.logger.Debug("Message did not pass filter", zap.Any("triggerRef", trigger)) - return nil - } - - err = r.dispatcher.DispatchMessage(message, subscriberURI, "", provisioners.DispatchDefaults{}) - if err != nil { - r.logger.Info("Failed to dispatch message", zap.Error(err), zap.Any("triggerRef", trigger)) - return err - } - r.logger.Debug("Successfully sent message", zap.Any("triggerRef", trigger)) - return nil -} - -// Initialize the client. Mainly intended to load stuff in its cache. -func (r *Receiver) initClient() error { - // We list triggers so that we can load the client's cache. Otherwise, on receiving an event, it - // may not find the trigger and would return an error. - opts := &client.ListOptions{ - // Set Raw because if we need to get more than one page, then we will put the continue token - // into opts.Raw.Continue. - Raw: &metav1.ListOptions{}, - } - for { - tl := &eventingv1alpha1.TriggerList{} - if err := r.client.List(context.TODO(), opts, tl); err != nil { - return err - } - if tl.Continue != "" { - opts.Raw.Continue = tl.Continue - } else { - break - } - } - return nil -} - -func (r *Receiver) getTrigger(ctx context.Context, ref provisioners.ChannelReference) (*eventingv1alpha1.Trigger, error) { - t := &eventingv1alpha1.Trigger{} - err := r.client.Get(ctx, - types.NamespacedName{ - Namespace: ref.Namespace, - Name: ref.Name, - }, - t) - return t, err -} - -// shouldSendMessage determines whether message 'm' should be sent based on the triggerSpec 'ts'. -// Currently it supports exact matching on type and/or source of events. -func (r *Receiver) shouldSendMessage(ts *eventingv1alpha1.TriggerSpec, m *provisioners.Message) bool { - if ts.Filter == nil || ts.Filter.SourceAndType == nil { - r.logger.Error("No filter specified") - return false - } - filterType := ts.Filter.SourceAndType.Type - if filterType != eventingv1alpha1.TriggerAnyFilter && filterType != m.Headers["Ce-Eventtype"] { - r.logger.Debug("Wrong type", zap.String("trigger.spec.filter.sourceAndType.type", filterType), zap.String("message.type", m.Headers["Ce-Eventtype"])) - return false - } - filterSource := ts.Filter.SourceAndType.Source - if filterSource != eventingv1alpha1.TriggerAnyFilter && filterSource != m.Headers["Ce-Source"] { - r.logger.Debug("Wrong source", zap.String("trigger.spec.filter.sourceAndType.source", filterSource), zap.String("message.source", m.Headers["Ce-Source"])) - return false - } - return true -} diff --git a/pkg/broker/old/receiver_test.go b/pkg/broker/old/receiver_test.go deleted file mode 100644 index 377fd26414b..00000000000 --- a/pkg/broker/old/receiver_test.go +++ /dev/null @@ -1,198 +0,0 @@ -/* - * Copyright 2019 The Knative Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package old - -import ( - "errors" - "fmt" - "net/http" - "net/http/httptest" - "strings" - "testing" - - "github.com/knative/eventing/pkg/utils" - - "github.com/knative/eventing/pkg/provisioners" - - eventingv1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" - "k8s.io/client-go/kubernetes/scheme" - - "go.uber.org/zap" - - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - - "sigs.k8s.io/controller-runtime/pkg/client/fake" -) - -const ( - testNS = "test-namespace" - triggerName = "test-trigger" - eventType = `"com.example.someevent"` - eventSource = `"/mycontext"` -) - -func init() { - // Add types to scheme. - eventingv1alpha1.AddToScheme(scheme.Scheme) -} - -func TestReceiver(t *testing.T) { - testCases := map[string]struct { - initialState []runtime.Object - dispatchErr error - expectedErr bool - expectedDispatch bool - }{ - "Trigger.Get fails": { - // No trigger exists, so the Get will fail. - expectedErr: true, - }, - "Trigger doesn't have SubscriberURI": { - initialState: []runtime.Object{ - makeTriggerWithoutSubscriberURI(), - }, - expectedErr: true, - }, - "Trigger without a Filter": { - initialState: []runtime.Object{ - makeTriggerWithoutFilter(), - }, - }, - "Wrong type": { - initialState: []runtime.Object{ - makeTrigger("some-other-type", "Any"), - }, - }, - "Wrong source": { - initialState: []runtime.Object{ - makeTrigger("Any", "some-other-source"), - }, - }, - "Dispatch failed": { - initialState: []runtime.Object{ - makeTrigger("Any", "Any"), - }, - dispatchErr: errors.New("test error dispatching"), - expectedErr: true, - expectedDispatch: true, - }, - "Dispatch succeeded - Any": { - initialState: []runtime.Object{ - makeTrigger("Any", "Any"), - }, - expectedDispatch: true, - }, - "Dispatch succeeded - Specific": { - initialState: []runtime.Object{ - makeTrigger(eventType, eventSource), - }, - expectedDispatch: true, - }, - } - for n, tc := range testCases { - t.Run(n, func(t *testing.T) { - mr, _ := New( - zap.NewNop(), - fake.NewFakeClient(tc.initialState...)) - fd := &fakeDispatcher{ - err: tc.dispatchErr, - } - mr.dispatcher = fd - - resp := httptest.NewRecorder() - mr.newMessageReceiver().HandleRequest(resp, makeRequest()) - if tc.expectedErr { - if resp.Result().StatusCode >= 200 && resp.Result().StatusCode < 300 { - t.Errorf("Expected an error. Actual: %v", resp.Result()) - } - } else { - if resp.Result().StatusCode < 200 || resp.Result().StatusCode >= 300 { - t.Errorf("Expected success. Actual: %v", resp.Result()) - } - } - if tc.expectedDispatch != fd.requestReceived { - t.Errorf("Incorrect dispatch. Expected %v, Actual %v", tc.expectedDispatch, fd.requestReceived) - } - }) - } -} - -type fakeDispatcher struct { - err error - requestReceived bool -} - -func (d *fakeDispatcher) DispatchMessage(_ *provisioners.Message, _, _ string, _ provisioners.DispatchDefaults) error { - d.requestReceived = true - return d.err -} - -func makeTrigger(t, s string) *eventingv1alpha1.Trigger { - return &eventingv1alpha1.Trigger{ - TypeMeta: v1.TypeMeta{ - APIVersion: "eventing.knative.dev/v1alpha1", - Kind: "Trigger", - }, - ObjectMeta: v1.ObjectMeta{ - Namespace: testNS, - Name: triggerName, - }, - Spec: eventingv1alpha1.TriggerSpec{ - Filter: &eventingv1alpha1.TriggerFilter{ - SourceAndType: &eventingv1alpha1.TriggerFilterSourceAndType{ - Type: t, - Source: s, - }, - }, - }, - Status: eventingv1alpha1.TriggerStatus{ - SubscriberURI: "subscriberURI", - }, - } -} - -func makeTriggerWithoutFilter() *eventingv1alpha1.Trigger { - t := makeTrigger("Any", "Any") - t.Spec.Filter = nil - return t -} - -func makeTriggerWithoutSubscriberURI() *eventingv1alpha1.Trigger { - t := makeTrigger("Any", "Any") - t.Status = eventingv1alpha1.TriggerStatus{} - return t -} - -func makeRequest() *http.Request { - req := httptest.NewRequest("POST", "/", strings.NewReader(``)) - req.Host = fmt.Sprintf("%s.%s.triggers.%s", triggerName, testNS, utils.GetClusterDomainName()) - - eventAttributes := map[string]string{ - "CE-CloudEventsVersion": `"0.1"`, - "CE-EventType": eventType, - "CE-EventTypeVersion": `"1.0"`, - "CE-Source": eventSource, - "CE-EventID": `"A234-1234-1234"`, - "CE-EventTime": `"2018-04-05T17:31:00Z"`, - "contentType": "text/xml", - } - for k, v := range eventAttributes { - req.Header.Set(k, v) - } - return req -} From 80005e85e414de0864c5007880e63c9a9bffe26a Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Mon, 4 Mar 2019 14:56:47 -0800 Subject: [PATCH 109/128] Change assumption about response to only being a CloudEvent if it can be parsed as such. --- pkg/broker/ce_receiver.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/broker/ce_receiver.go b/pkg/broker/ce_receiver.go index 9f3331187e7..2f3eea9619a 100644 --- a/pkg/broker/ce_receiver.go +++ b/pkg/broker/ce_receiver.go @@ -183,7 +183,10 @@ func (r *Receiver) decodeHTTPRequest(req *http.Request) (*cloudevents.Event, err } func (r *Receiver) decodeHTTPResponse(resp *http.Response) (*cloudevents.Event, error) { - return r.decodeHTTP(resp.Header, resp.Body) + // The HTTP Response could be anything, so just assume that if it does not parse as a + // CloudEvent, then it isn't a CloudEvent. + e, _ := r.decodeHTTP(resp.Header, resp.Body) + return e, nil } func (r *Receiver) decodeHTTP(headers http.Header, bodyReadCloser io.ReadCloser) (*cloudevents.Event, error) { @@ -191,14 +194,11 @@ func (r *Receiver) decodeHTTP(headers http.Header, bodyReadCloser io.ReadCloser) if err != nil { return nil, err } - if len(body) == 0 { - return nil, nil - } - msg := &cehttp.Message{ Header: headers, Body: body, } + return r.codec.Decode(msg) } From a1065533908e4fc8a161c102c81c1a0ba49ab0b4 Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Mon, 18 Mar 2019 13:45:33 -0700 Subject: [PATCH 110/128] Clean up from the merge. --- b.yaml | 11 -- b2.yaml | 5 - pkg/broker/ce_receiver.go | 331 --------------------------------- pkg/broker/ce_receiver_test.go | 200 -------------------- pkg/broker/receiver.go | 284 +++++++++++++++++++++++----- pkg/broker/receiver_test.go | 102 +++++----- t.yaml | 11 -- t2.yaml | 16 -- t3.yaml | 16 -- t4.yaml | 11 -- 10 files changed, 282 insertions(+), 705 deletions(-) delete mode 100644 b.yaml delete mode 100644 b2.yaml delete mode 100644 pkg/broker/ce_receiver.go delete mode 100644 pkg/broker/ce_receiver_test.go delete mode 100644 t.yaml delete mode 100644 t2.yaml delete mode 100644 t3.yaml delete mode 100644 t4.yaml diff --git a/b.yaml b/b.yaml deleted file mode 100644 index 595b6eb20ac..00000000000 --- a/b.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: eventing.knative.dev/v1alpha1 -kind: Broker -metadata: - name: default -spec: - channelTemplate: - provisioner: - apiVersion: eventing.knative.dev/v1alpha1 - kind: ClusterChannelProvisioner - name: gcp-pubsub - diff --git a/b2.yaml b/b2.yaml deleted file mode 100644 index 1c0f7b43a82..00000000000 --- a/b2.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: eventing.knative.dev/v1alpha1 -kind: Broker -metadata: - name: default -spec: {} diff --git a/pkg/broker/ce_receiver.go b/pkg/broker/ce_receiver.go deleted file mode 100644 index 2f3eea9619a..00000000000 --- a/pkg/broker/ce_receiver.go +++ /dev/null @@ -1,331 +0,0 @@ -/* - * Copyright 2019 The Knative Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package broker - -import ( - "bytes" - "context" - "errors" - "fmt" - "io" - "io/ioutil" - "net/http" - "net/url" - "reflect" - - "github.com/cloudevents/sdk-go/pkg/cloudevents" - - cehttp "github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http" - - eventingv1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" - "github.com/knative/eventing/pkg/provisioners" - "go.uber.org/zap" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -const ( - defaultPort = 8080 -) - -// Receiver parses Cloud Events, determines if they pass a filter, and sends them to a subscriber. -type Receiver struct { - logger *zap.Logger - client client.Client - - port int - - httpClient HTTPDoer - codec cehttp.Codec -} - -type HTTPDoer interface { - Do(*http.Request) (*http.Response, error) -} - -var _ HTTPDoer = &http.Client{} - -// New creates a new Receiver and its associated MessageReceiver. The caller is responsible for -// Start()ing the returned MessageReceiver. -func New(logger *zap.Logger, client client.Client) *Receiver { - r := &Receiver{ - logger: logger, - client: client, - - port: defaultPort, - - httpClient: &http.Client{}, - codec: cehttp.Codec{ - Encoding: cehttp.BinaryV01, - }, - } - return r -} - -var _ http.Handler = &Receiver{} - -// Start begins to receive messages for the receiver. -// -// Only HTTP POST requests to the root path (/) are accepted. If other paths or -// methods are needed, use the HandleRequest method directly with another HTTP -// server. -// -// This method will block until a message is received on the stop channel. -func (r *Receiver) Start(stopCh <-chan struct{}) error { - svr := r.start() - defer r.stop(svr) - - <-stopCh - return nil -} - -func (r *Receiver) start() *http.Server { - srv := &http.Server{ - Addr: fmt.Sprintf(":%d", r.port), - Handler: r, - } - r.logger.Info("Starting web server", zap.String("addr", srv.Addr)) - go func() { - if err := srv.ListenAndServe(); err != http.ErrServerClosed { - r.logger.Error("HttpServer: ListenAndServe() error", zap.Error(err)) - } - }() - return srv -} - -func (r *Receiver) stop(srv *http.Server) { - r.logger.Info("Shutdown web server") - if err := srv.Shutdown(nil); err != nil { - r.logger.Error("Error shutting down the HTTP Server", zap.Error(err)) - } -} - -func (r *Receiver) ServeHTTP(w http.ResponseWriter, req *http.Request) { - if req.URL.Path != "/" { - w.WriteHeader(http.StatusNotFound) - return - } - if req.Method != http.MethodPost { - w.WriteHeader(http.StatusMethodNotAllowed) - return - } - - triggerRef, err := provisioners.ParseChannel(req.Host) - if err != nil { - r.logger.Error("Unable to parse host as a trigger", zap.Error(err), zap.String("host", req.Host)) - w.WriteHeader(http.StatusBadRequest) - w.Write([]byte(`{"error":"Bad Host"}`)) - return - } - - event, err := r.decodeHTTPRequest(req) - if err != nil { - r.logger.Error("Error decoding HTTP Request", zap.Error(err)) - w.WriteHeader(http.StatusBadRequest) - return - } - - responseEvent, err := r.sendEvent(triggerRef, event) - if err != nil { - r.logger.Error("Error sending the event", zap.Error(err)) - w.WriteHeader(http.StatusBadRequest) - return - } - if responseEvent == nil { - w.WriteHeader(http.StatusAccepted) - return - } - - encodedEvent, err := r.codec.Encode(*event) - if err != nil { - r.logger.Error("Error encoding the response event", zap.Error(err)) - w.WriteHeader(http.StatusInternalServerError) - return - } - msg, ok := encodedEvent.(*cehttp.Message) - if !ok { - r.logger.Error("Error casting the encoded response event", zap.Error(err), zap.Any("encodedEvent", reflect.TypeOf(encodedEvent))) - w.WriteHeader(http.StatusInternalServerError) - return - } - for n, v := range msg.Header { - w.Header().Del(n) - for _, s := range v { - w.Header().Add(n, s) - } - } - w.WriteHeader(http.StatusAccepted) - _, err = w.Write(msg.Body) - if err != nil { - r.logger.Error("Error writing the response body", zap.Error(err)) - w.WriteHeader(http.StatusInternalServerError) - return - } -} - -func (r *Receiver) decodeHTTPRequest(req *http.Request) (*cloudevents.Event, error) { - return r.decodeHTTP(req.Header, req.Body) -} - -func (r *Receiver) decodeHTTPResponse(resp *http.Response) (*cloudevents.Event, error) { - // The HTTP Response could be anything, so just assume that if it does not parse as a - // CloudEvent, then it isn't a CloudEvent. - e, _ := r.decodeHTTP(resp.Header, resp.Body) - return e, nil -} - -func (r *Receiver) decodeHTTP(headers http.Header, bodyReadCloser io.ReadCloser) (*cloudevents.Event, error) { - body, err := ioutil.ReadAll(bodyReadCloser) - if err != nil { - return nil, err - } - msg := &cehttp.Message{ - Header: headers, - Body: body, - } - - return r.codec.Decode(msg) -} - -// sendEvent sends an event to a subscriber if the trigger filter passes. -func (r *Receiver) sendEvent(trigger provisioners.ChannelReference, event *cloudevents.Event) (*cloudevents.Event, error) { - r.logger.Debug("Received message", zap.Any("triggerRef", trigger)) - ctx := context.Background() - - t, err := r.getTrigger(ctx, trigger) - if err != nil { - r.logger.Info("Unable to get the Trigger", zap.Error(err), zap.Any("triggerRef", trigger)) - return nil, err - } - - subscriberURIString := t.Status.SubscriberURI - if subscriberURIString == "" { - r.logger.Error("Unable to read subscriberURI") - return nil, errors.New("unable to read subscriberURI") - } - subscriberURI, err := url.Parse(subscriberURIString) - if err != nil { - r.logger.Error("Unable to parse subscriberURI", zap.Error(err), zap.String("subscriberURIString", subscriberURIString)) - return nil, err - } - - if !r.shouldSendMessage(&t.Spec, event) { - r.logger.Debug("Message did not pass filter", zap.Any("triggerRef", trigger)) - return nil, nil - } - - return r.dispatch(event, subscriberURI) -} - -func (r *Receiver) dispatch(event *cloudevents.Event, uri *url.URL) (*cloudevents.Event, error) { - encodedEvent, err := r.codec.Encode(*event) - if err != nil { - return nil, err - } - msg, ok := encodedEvent.(*cehttp.Message) - if !ok { - return nil, errors.New("msg was not a cehttp.Message") - } - - req, err := http.NewRequest(http.MethodPost, uri.String(), bytes.NewReader(msg.Body)) - if err != nil { - return nil, err - } - - req.Header = msg.Header - res, err := r.httpClient.Do(req) - if err != nil { - return nil, err - } - if res == nil { - // I don't think this is actually reachable with http.Client.Do(), but just to be sure we - // check anyway. - return nil, errors.New("non-error nil result from http.Client.Do()") - } - - defer res.Body.Close() - if isFailure(res.StatusCode) { - // reject non-successful responses - return nil, fmt.Errorf("unexpected HTTP response, expected 2xx, got %d", res.StatusCode) - } - - return r.decodeHTTPResponse(res) -} - -// isFailure returns true if the status code is not a successful HTTP status. -func isFailure(statusCode int) bool { - return statusCode < http.StatusOK /* 200 */ || - statusCode >= http.StatusMultipleChoices /* 300 */ -} - -// Initialize the client. Mainly intended to load stuff in its cache. -func (r *Receiver) initClient() error { - // We list triggers so that we can load the client's cache. Otherwise, on receiving an event, it - // may not find the trigger and would return an error. - opts := &client.ListOptions{ - // Set Raw because if we need to get more than one page, then we will put the continue token - // into opts.Raw.Continue. - Raw: &metav1.ListOptions{}, - } - for { - tl := &eventingv1alpha1.TriggerList{} - if err := r.client.List(context.TODO(), opts, tl); err != nil { - return err - } - if tl.Continue != "" { - opts.Raw.Continue = tl.Continue - } else { - break - } - } - return nil -} - -func (r *Receiver) getTrigger(ctx context.Context, ref provisioners.ChannelReference) (*eventingv1alpha1.Trigger, error) { - t := &eventingv1alpha1.Trigger{} - err := r.client.Get(ctx, - types.NamespacedName{ - Namespace: ref.Namespace, - Name: ref.Name, - }, - t) - return t, err -} - -// shouldSendMessage determines whether message 'm' should be sent based on the triggerSpec 'ts'. -// Currently it supports exact matching on type and/or source of events. -func (r *Receiver) shouldSendMessage(ts *eventingv1alpha1.TriggerSpec, event *cloudevents.Event) bool { - if ts.Filter == nil || ts.Filter.SourceAndType == nil { - r.logger.Error("No filter specified") - return false - } - filterType := ts.Filter.SourceAndType.Type - if filterType != eventingv1alpha1.TriggerAnyFilter && filterType != event.Type() { - r.logger.Debug("Wrong type", zap.String("trigger.spec.filter.sourceAndType.type", filterType), zap.String("event.Type()", event.Type())) - return false - } - filterSource := ts.Filter.SourceAndType.Source - s := event.Context.AsV01().Source - actualSource := s.String() - //actualSource := event.Context.AsV01().Source.String() - if filterSource != eventingv1alpha1.TriggerAnyFilter && filterSource != actualSource { - r.logger.Debug("Wrong source", zap.String("trigger.spec.filter.sourceAndType.source", filterSource), zap.String("message.source", actualSource)) - return false - } - return true -} diff --git a/pkg/broker/ce_receiver_test.go b/pkg/broker/ce_receiver_test.go deleted file mode 100644 index 07f79bed903..00000000000 --- a/pkg/broker/ce_receiver_test.go +++ /dev/null @@ -1,200 +0,0 @@ -/* - * Copyright 2019 The Knative Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package broker - -import ( - "bytes" - "fmt" - "io/ioutil" - "net/http" - "net/http/httptest" - "strings" - "testing" - - eventingv1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" - "github.com/knative/eventing/pkg/utils" - "go.uber.org/zap" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/client-go/kubernetes/scheme" - "sigs.k8s.io/controller-runtime/pkg/client/fake" -) - -const ( - testNS = "test-namespace" - triggerName = "test-trigger" - eventType = `com.example.someevent` - eventSource = `/mycontext` -) - -func init() { - // Add types to scheme. - eventingv1alpha1.AddToScheme(scheme.Scheme) -} - -func TestReceiver(t *testing.T) { - testCases := map[string]struct { - initialState []runtime.Object - requestFails bool - expectedErr bool - expectedDispatch bool - }{ - "Trigger.Get fails": { - // No trigger exists, so the Get will fail. - expectedErr: true, - }, - "Trigger doesn't have SubscriberURI": { - initialState: []runtime.Object{ - makeTriggerWithoutSubscriberURI(), - }, - expectedErr: true, - }, - "Trigger without a Filter": { - initialState: []runtime.Object{ - makeTriggerWithoutFilter(), - }, - }, - "Wrong type": { - initialState: []runtime.Object{ - makeTrigger("some-other-type", "Any"), - }, - }, - "Wrong source": { - initialState: []runtime.Object{ - makeTrigger("Any", "some-other-source"), - }, - }, - "Dispatch failed": { - initialState: []runtime.Object{ - makeTrigger("Any", "Any"), - }, - requestFails: true, - expectedErr: true, - expectedDispatch: true, - }, - "Dispatch succeeded - Any": { - initialState: []runtime.Object{ - makeTrigger("Any", "Any"), - }, - expectedDispatch: true, - }, - "Dispatch succeeded - Specific": { - initialState: []runtime.Object{ - makeTrigger(eventType, eventSource), - }, - expectedDispatch: true, - }, - } - for n, tc := range testCases { - t.Run(n, func(t *testing.T) { - mr := New( - zap.NewNop(), - fake.NewFakeClient(tc.initialState...)) - fh := &fakeHTTPDoer{ - failRequest: tc.requestFails, - } - mr.httpClient = fh - - resp := httptest.NewRecorder() - mr.ServeHTTP(resp, makeRequest()) - if tc.expectedErr { - if resp.Result().StatusCode >= 200 && resp.Result().StatusCode < 300 { - t.Errorf("Expected an error. Actual: %v", resp.Result()) - } - } else { - if resp.Result().StatusCode < 200 || resp.Result().StatusCode >= 300 { - t.Errorf("Expected success. Actual: %v", resp.Result()) - } - } - if tc.expectedDispatch != fh.requestReceived { - t.Errorf("Incorrect dispatch. Expected %v, Actual %v", tc.expectedDispatch, fh.requestReceived) - } - }) - } -} - -type fakeHTTPDoer struct { - failRequest bool - requestReceived bool -} - -func (h *fakeHTTPDoer) Do(_ *http.Request) (*http.Response, error) { - h.requestReceived = true - sc := http.StatusOK - if h.failRequest { - sc = http.StatusBadRequest - } - return &http.Response{ - StatusCode: sc, - Body: ioutil.NopCloser(bytes.NewBufferString("")), - }, nil -} - -func makeTrigger(t, s string) *eventingv1alpha1.Trigger { - return &eventingv1alpha1.Trigger{ - TypeMeta: v1.TypeMeta{ - APIVersion: "eventing.knative.dev/v1alpha1", - Kind: "Trigger", - }, - ObjectMeta: v1.ObjectMeta{ - Namespace: testNS, - Name: triggerName, - }, - Spec: eventingv1alpha1.TriggerSpec{ - Filter: &eventingv1alpha1.TriggerFilter{ - SourceAndType: &eventingv1alpha1.TriggerFilterSourceAndType{ - Type: t, - Source: s, - }, - }, - }, - Status: eventingv1alpha1.TriggerStatus{ - SubscriberURI: "subscriberURI", - }, - } -} - -func makeTriggerWithoutFilter() *eventingv1alpha1.Trigger { - t := makeTrigger("Any", "Any") - t.Spec.Filter = nil - return t -} - -func makeTriggerWithoutSubscriberURI() *eventingv1alpha1.Trigger { - t := makeTrigger("Any", "Any") - t.Status = eventingv1alpha1.TriggerStatus{} - return t -} - -func makeRequest() *http.Request { - req := httptest.NewRequest("POST", "/", strings.NewReader(``)) - req.Host = fmt.Sprintf("%s.%s.triggers.%s", triggerName, testNS, utils.GetClusterDomainName()) - - eventAttributes := map[string]string{ - "CE-CloudEventsVersion": `0.1`, - "CE-EventType": eventType, - "CE-EventTypeVersion": `1.0`, - "CE-Source": eventSource, - "CE-EventID": `A234-1234-1234`, - "CE-EventTime": `2018-04-05T17:31:00Z`, - "Content-Type": "application/xml", - } - for k, v := range eventAttributes { - req.Header.Set(k, v) - } - return req -} diff --git a/pkg/broker/receiver.go b/pkg/broker/receiver.go index c1d4b22578b..2f3eea9619a 100644 --- a/pkg/broker/receiver.go +++ b/pkg/broker/receiver.go @@ -17,15 +17,30 @@ package broker import ( + "bytes" "context" "errors" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/url" + "reflect" + + "github.com/cloudevents/sdk-go/pkg/cloudevents" + + cehttp "github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http" eventingv1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" "github.com/knative/eventing/pkg/provisioners" "go.uber.org/zap" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/manager" +) + +const ( + defaultPort = 8080 ) // Receiver parses Cloud Events, determines if they pass a filter, and sends them to a subscriber. @@ -33,66 +48,250 @@ type Receiver struct { logger *zap.Logger client client.Client - dispatcher provisioners.Dispatcher + port int + + httpClient HTTPDoer + codec cehttp.Codec } +type HTTPDoer interface { + Do(*http.Request) (*http.Response, error) +} + +var _ HTTPDoer = &http.Client{} + // New creates a new Receiver and its associated MessageReceiver. The caller is responsible for // Start()ing the returned MessageReceiver. -func New(logger *zap.Logger, client client.Client) (*Receiver, manager.Runnable) { +func New(logger *zap.Logger, client client.Client) *Receiver { r := &Receiver{ - logger: logger, - client: client, - dispatcher: provisioners.NewMessageDispatcher(logger.Sugar()), + logger: logger, + client: client, + + port: defaultPort, + + httpClient: &http.Client{}, + codec: cehttp.Codec{ + Encoding: cehttp.BinaryV01, + }, + } + return r +} + +var _ http.Handler = &Receiver{} + +// Start begins to receive messages for the receiver. +// +// Only HTTP POST requests to the root path (/) are accepted. If other paths or +// methods are needed, use the HandleRequest method directly with another HTTP +// server. +// +// This method will block until a message is received on the stop channel. +func (r *Receiver) Start(stopCh <-chan struct{}) error { + svr := r.start() + defer r.stop(svr) + + <-stopCh + return nil +} + +func (r *Receiver) start() *http.Server { + srv := &http.Server{ + Addr: fmt.Sprintf(":%d", r.port), + Handler: r, + } + r.logger.Info("Starting web server", zap.String("addr", srv.Addr)) + go func() { + if err := srv.ListenAndServe(); err != http.ErrServerClosed { + r.logger.Error("HttpServer: ListenAndServe() error", zap.Error(err)) + } + }() + return srv +} + +func (r *Receiver) stop(srv *http.Server) { + r.logger.Info("Shutdown web server") + if err := srv.Shutdown(nil); err != nil { + r.logger.Error("Error shutting down the HTTP Server", zap.Error(err)) + } +} + +func (r *Receiver) ServeHTTP(w http.ResponseWriter, req *http.Request) { + if req.URL.Path != "/" { + w.WriteHeader(http.StatusNotFound) + return + } + if req.Method != http.MethodPost { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + + triggerRef, err := provisioners.ParseChannel(req.Host) + if err != nil { + r.logger.Error("Unable to parse host as a trigger", zap.Error(err), zap.String("host", req.Host)) + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte(`{"error":"Bad Host"}`)) + return + } + + event, err := r.decodeHTTPRequest(req) + if err != nil { + r.logger.Error("Error decoding HTTP Request", zap.Error(err)) + w.WriteHeader(http.StatusBadRequest) + return + } + + responseEvent, err := r.sendEvent(triggerRef, event) + if err != nil { + r.logger.Error("Error sending the event", zap.Error(err)) + w.WriteHeader(http.StatusBadRequest) + return } - return r, r.newMessageReceiver() + if responseEvent == nil { + w.WriteHeader(http.StatusAccepted) + return + } + + encodedEvent, err := r.codec.Encode(*event) + if err != nil { + r.logger.Error("Error encoding the response event", zap.Error(err)) + w.WriteHeader(http.StatusInternalServerError) + return + } + msg, ok := encodedEvent.(*cehttp.Message) + if !ok { + r.logger.Error("Error casting the encoded response event", zap.Error(err), zap.Any("encodedEvent", reflect.TypeOf(encodedEvent))) + w.WriteHeader(http.StatusInternalServerError) + return + } + for n, v := range msg.Header { + w.Header().Del(n) + for _, s := range v { + w.Header().Add(n, s) + } + } + w.WriteHeader(http.StatusAccepted) + _, err = w.Write(msg.Body) + if err != nil { + r.logger.Error("Error writing the response body", zap.Error(err)) + w.WriteHeader(http.StatusInternalServerError) + return + } +} + +func (r *Receiver) decodeHTTPRequest(req *http.Request) (*cloudevents.Event, error) { + return r.decodeHTTP(req.Header, req.Body) +} + +func (r *Receiver) decodeHTTPResponse(resp *http.Response) (*cloudevents.Event, error) { + // The HTTP Response could be anything, so just assume that if it does not parse as a + // CloudEvent, then it isn't a CloudEvent. + e, _ := r.decodeHTTP(resp.Header, resp.Body) + return e, nil } -func (r *Receiver) newMessageReceiver() *provisioners.MessageReceiver { - if err := r.initClient(); err != nil { - r.logger.Warn("Failed to initialize client", zap.Error(err)) +func (r *Receiver) decodeHTTP(headers http.Header, bodyReadCloser io.ReadCloser) (*cloudevents.Event, error) { + body, err := ioutil.ReadAll(bodyReadCloser) + if err != nil { + return nil, err + } + msg := &cehttp.Message{ + Header: headers, + Body: body, } - return provisioners.NewMessageReceiver(r.sendEvent, r.logger.Sugar()) + + return r.codec.Decode(msg) } // sendEvent sends an event to a subscriber if the trigger filter passes. -func (r *Receiver) sendEvent(trigger provisioners.ChannelReference, message *provisioners.Message) error { +func (r *Receiver) sendEvent(trigger provisioners.ChannelReference, event *cloudevents.Event) (*cloudevents.Event, error) { r.logger.Debug("Received message", zap.Any("triggerRef", trigger)) ctx := context.Background() t, err := r.getTrigger(ctx, trigger) if err != nil { r.logger.Info("Unable to get the Trigger", zap.Error(err), zap.Any("triggerRef", trigger)) - return err + return nil, err } - subscriberURI := t.Status.SubscriberURI - if subscriberURI == "" { + subscriberURIString := t.Status.SubscriberURI + if subscriberURIString == "" { r.logger.Error("Unable to read subscriberURI") - return errors.New("unable to read subscriberURI") + return nil, errors.New("unable to read subscriberURI") + } + subscriberURI, err := url.Parse(subscriberURIString) + if err != nil { + r.logger.Error("Unable to parse subscriberURI", zap.Error(err), zap.String("subscriberURIString", subscriberURIString)) + return nil, err } - if !r.shouldSendMessage(&t.Spec, message) { + if !r.shouldSendMessage(&t.Spec, event) { r.logger.Debug("Message did not pass filter", zap.Any("triggerRef", trigger)) - return nil + return nil, nil } - err = r.dispatcher.DispatchMessage(message, subscriberURI, "", provisioners.DispatchDefaults{}) + return r.dispatch(event, subscriberURI) +} + +func (r *Receiver) dispatch(event *cloudevents.Event, uri *url.URL) (*cloudevents.Event, error) { + encodedEvent, err := r.codec.Encode(*event) if err != nil { - r.logger.Info("Failed to dispatch message", zap.Error(err), zap.Any("triggerRef", trigger)) - return err + return nil, err } - r.logger.Debug("Successfully sent message", zap.Any("triggerRef", trigger)) - return nil + msg, ok := encodedEvent.(*cehttp.Message) + if !ok { + return nil, errors.New("msg was not a cehttp.Message") + } + + req, err := http.NewRequest(http.MethodPost, uri.String(), bytes.NewReader(msg.Body)) + if err != nil { + return nil, err + } + + req.Header = msg.Header + res, err := r.httpClient.Do(req) + if err != nil { + return nil, err + } + if res == nil { + // I don't think this is actually reachable with http.Client.Do(), but just to be sure we + // check anyway. + return nil, errors.New("non-error nil result from http.Client.Do()") + } + + defer res.Body.Close() + if isFailure(res.StatusCode) { + // reject non-successful responses + return nil, fmt.Errorf("unexpected HTTP response, expected 2xx, got %d", res.StatusCode) + } + + return r.decodeHTTPResponse(res) } -// Initialize the client. Mainly intended to create the informer/indexer in order not to drop messages. +// isFailure returns true if the status code is not a successful HTTP status. +func isFailure(statusCode int) bool { + return statusCode < http.StatusOK /* 200 */ || + statusCode >= http.StatusMultipleChoices /* 300 */ +} + +// Initialize the client. Mainly intended to load stuff in its cache. func (r *Receiver) initClient() error { - // We list triggers so that we do not drop messages. Otherwise, on receiving an event, it + // We list triggers so that we can load the client's cache. Otherwise, on receiving an event, it // may not find the trigger and would return an error. - opts := &client.ListOptions{} - tl := &eventingv1alpha1.TriggerList{} - if err := r.client.List(context.TODO(), opts, tl); err != nil { - return err + opts := &client.ListOptions{ + // Set Raw because if we need to get more than one page, then we will put the continue token + // into opts.Raw.Continue. + Raw: &metav1.ListOptions{}, + } + for { + tl := &eventingv1alpha1.TriggerList{} + if err := r.client.List(context.TODO(), opts, tl); err != nil { + return err + } + if tl.Continue != "" { + opts.Raw.Continue = tl.Continue + } else { + break + } } return nil } @@ -110,33 +309,22 @@ func (r *Receiver) getTrigger(ctx context.Context, ref provisioners.ChannelRefer // shouldSendMessage determines whether message 'm' should be sent based on the triggerSpec 'ts'. // Currently it supports exact matching on type and/or source of events. -func (r *Receiver) shouldSendMessage(ts *eventingv1alpha1.TriggerSpec, m *provisioners.Message) bool { +func (r *Receiver) shouldSendMessage(ts *eventingv1alpha1.TriggerSpec, event *cloudevents.Event) bool { if ts.Filter == nil || ts.Filter.SourceAndType == nil { r.logger.Error("No filter specified") return false } filterType := ts.Filter.SourceAndType.Type - // TODO the inspection of Headers should be removed once we start using the cloud events SDK. - cloudEventType := "" - if et, ok := m.Headers["Ce-Eventtype"]; ok { - // cloud event spec v0.1. - cloudEventType = et - } else if et, ok := m.Headers["Ce-Type"]; ok { - // cloud event spec v0.2. - cloudEventType = et - } - if filterType != eventingv1alpha1.TriggerAnyFilter && filterType != cloudEventType { - r.logger.Debug("Wrong type", zap.String("trigger.spec.filter.sourceAndType.type", filterType), zap.String("message.type", cloudEventType)) + if filterType != eventingv1alpha1.TriggerAnyFilter && filterType != event.Type() { + r.logger.Debug("Wrong type", zap.String("trigger.spec.filter.sourceAndType.type", filterType), zap.String("event.Type()", event.Type())) return false } filterSource := ts.Filter.SourceAndType.Source - cloudEventSource := "" - // cloud event spec v0.1 and v0.2. - if es, ok := m.Headers["Ce-Source"]; ok { - cloudEventSource = es - } - if filterSource != eventingv1alpha1.TriggerAnyFilter && filterSource != cloudEventSource { - r.logger.Debug("Wrong source", zap.String("trigger.spec.filter.sourceAndType.source", filterSource), zap.String("message.source", cloudEventSource)) + s := event.Context.AsV01().Source + actualSource := s.String() + //actualSource := event.Context.AsV01().Source.String() + if filterSource != eventingv1alpha1.TriggerAnyFilter && filterSource != actualSource { + r.logger.Debug("Wrong source", zap.String("trigger.spec.filter.sourceAndType.source", filterSource), zap.String("message.source", actualSource)) return false } return true diff --git a/pkg/broker/receiver_test.go b/pkg/broker/receiver_test.go index d98edb6be45..07f79bed903 100644 --- a/pkg/broker/receiver_test.go +++ b/pkg/broker/receiver_test.go @@ -17,33 +17,28 @@ package broker import ( - "errors" + "bytes" "fmt" + "io/ioutil" "net/http" "net/http/httptest" "strings" "testing" - "github.com/knative/eventing/pkg/utils" - - "github.com/knative/eventing/pkg/provisioners" - eventingv1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" - "k8s.io/client-go/kubernetes/scheme" - + "github.com/knative/eventing/pkg/utils" "go.uber.org/zap" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - + "k8s.io/client-go/kubernetes/scheme" "sigs.k8s.io/controller-runtime/pkg/client/fake" ) const ( testNS = "test-namespace" triggerName = "test-trigger" - eventType = `"com.example.someevent"` - eventSource = `"/mycontext"` + eventType = `com.example.someevent` + eventSource = `/mycontext` ) func init() { @@ -54,7 +49,7 @@ func init() { func TestReceiver(t *testing.T) { testCases := map[string]struct { initialState []runtime.Object - dispatchErr error + requestFails bool expectedErr bool expectedDispatch bool }{ @@ -87,7 +82,7 @@ func TestReceiver(t *testing.T) { initialState: []runtime.Object{ makeTrigger("Any", "Any"), }, - dispatchErr: errors.New("test error dispatching"), + requestFails: true, expectedErr: true, expectedDispatch: true, }, @@ -106,44 +101,47 @@ func TestReceiver(t *testing.T) { } for n, tc := range testCases { t.Run(n, func(t *testing.T) { - // Support cloud spec v0.1 and v0.2 requests. - requests := [2]*http.Request{makeV01Request(), makeV02Request()} - for _, request := range requests { - mr, _ := New( - zap.NewNop(), - fake.NewFakeClient(tc.initialState...)) - fd := &fakeDispatcher{ - err: tc.dispatchErr, - } - mr.dispatcher = fd + mr := New( + zap.NewNop(), + fake.NewFakeClient(tc.initialState...)) + fh := &fakeHTTPDoer{ + failRequest: tc.requestFails, + } + mr.httpClient = fh - resp := httptest.NewRecorder() - mr.newMessageReceiver().HandleRequest(resp, request) - if tc.expectedErr { - if resp.Result().StatusCode >= 200 && resp.Result().StatusCode < 300 { - t.Errorf("Expected an error. Actual: %v", resp.Result()) - } - } else { - if resp.Result().StatusCode < 200 || resp.Result().StatusCode >= 300 { - t.Errorf("Expected success. Actual: %v", resp.Result()) - } + resp := httptest.NewRecorder() + mr.ServeHTTP(resp, makeRequest()) + if tc.expectedErr { + if resp.Result().StatusCode >= 200 && resp.Result().StatusCode < 300 { + t.Errorf("Expected an error. Actual: %v", resp.Result()) } - if tc.expectedDispatch != fd.requestReceived { - t.Errorf("Incorrect dispatch. Expected %v, Actual %v", tc.expectedDispatch, fd.requestReceived) + } else { + if resp.Result().StatusCode < 200 || resp.Result().StatusCode >= 300 { + t.Errorf("Expected success. Actual: %v", resp.Result()) } } + if tc.expectedDispatch != fh.requestReceived { + t.Errorf("Incorrect dispatch. Expected %v, Actual %v", tc.expectedDispatch, fh.requestReceived) + } }) } } -type fakeDispatcher struct { - err error +type fakeHTTPDoer struct { + failRequest bool requestReceived bool } -func (d *fakeDispatcher) DispatchMessage(_ *provisioners.Message, _, _ string, _ provisioners.DispatchDefaults) error { - d.requestReceived = true - return d.err +func (h *fakeHTTPDoer) Do(_ *http.Request) (*http.Response, error) { + h.requestReceived = true + sc := http.StatusOK + if h.failRequest { + sc = http.StatusBadRequest + } + return &http.Response{ + StatusCode: sc, + Body: ioutil.NopCloser(bytes.NewBufferString("")), + }, nil } func makeTrigger(t, s string) *eventingv1alpha1.Trigger { @@ -182,29 +180,21 @@ func makeTriggerWithoutSubscriberURI() *eventingv1alpha1.Trigger { return t } -func makeRequest(cloudEventVersionValue, eventTypeVersionValue, eventTypeKey, eventSourceKey string) *http.Request { +func makeRequest() *http.Request { req := httptest.NewRequest("POST", "/", strings.NewReader(``)) req.Host = fmt.Sprintf("%s.%s.triggers.%s", triggerName, testNS, utils.GetClusterDomainName()) eventAttributes := map[string]string{ - "CE-CloudEventsVersion": cloudEventVersionValue, - eventTypeKey: eventType, - "CE-EventTypeVersion": eventTypeVersionValue, - eventSourceKey: eventSource, - "CE-EventID": `"A234-1234-1234"`, - "CE-EventTime": `"2018-04-05T17:31:00Z"`, - "contentType": "text/xml", + "CE-CloudEventsVersion": `0.1`, + "CE-EventType": eventType, + "CE-EventTypeVersion": `1.0`, + "CE-Source": eventSource, + "CE-EventID": `A234-1234-1234`, + "CE-EventTime": `2018-04-05T17:31:00Z`, + "Content-Type": "application/xml", } for k, v := range eventAttributes { req.Header.Set(k, v) } return req } - -func makeV01Request() *http.Request { - return makeRequest(`"0.1"`, `"1.0"`, "CE-EventType", "CE-Source") -} - -func makeV02Request() *http.Request { - return makeRequest(`"0.2"`, `"2.0"`, "ce-type", "ce-source") -} diff --git a/t.yaml b/t.yaml deleted file mode 100644 index 0e2604dd595..00000000000 --- a/t.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: eventing.knative.dev/v1alpha1 -kind: Trigger -metadata: - name: t -spec: - subscriber: - ref: - apiVersion: serving.knative.dev/v1alpha1 - kind: Service - name: message-dumper - diff --git a/t2.yaml b/t2.yaml deleted file mode 100644 index 105b912796e..00000000000 --- a/t2.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: eventing.knative.dev/v1alpha1 -kind: Trigger -metadata: - name: t -spec: - # Our Cloud Event parsing library seems to have a bug and forces to put type and source in double - # quotes (as it thinks the actual value is `"foo"`, not `foo`). - filter: - sourceAndType: - type: '"com.example.someevent"' - source: '"/mycontext/subcontext"' - subscriber: - ref: - apiVersion: serving.knative.dev/v1alpha1 - kind: Service - name: message-dumper diff --git a/t3.yaml b/t3.yaml deleted file mode 100644 index 0ea783d5675..00000000000 --- a/t3.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: eventing.knative.dev/v1alpha1 -kind: Trigger -metadata: - name: t -spec: - # Our Cloud Event parsing library seems to have a bug and forces to put type and source in double - # quotes (as it thinks the actual value is `"foo"`, not `foo`). - filter: - sourceAndType: - type: '"com.example.someevent"' - source: '"/mycontext/subcontext"' - subscriber: - ref: - apiVersion: v1 - kind: Service - name: svc diff --git a/t4.yaml b/t4.yaml deleted file mode 100644 index a6a981bfc0b..00000000000 --- a/t4.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: eventing.knative.dev/v1alpha1 -kind: Trigger -metadata: - name: t -spec: - subscriber: - ref: - apiVersion: v1 - kind: Service - name: svc - From bc05fdf71f976987493bbe2fe20f8a391ff6482f Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Mon, 18 Mar 2019 13:58:11 -0700 Subject: [PATCH 111/128] More merge clean up. --- cmd/broker/filter/main.go | 6 +-- pkg/broker/receiver.go | 26 +++-------- pkg/reconciler/v1alpha1/broker/broker.go | 46 ++++++++----------- pkg/reconciler/v1alpha1/broker/broker_test.go | 16 +++---- 4 files changed, 36 insertions(+), 58 deletions(-) diff --git a/cmd/broker/filter/main.go b/cmd/broker/filter/main.go index f60b33340e0..23f680c11e0 100644 --- a/cmd/broker/filter/main.go +++ b/cmd/broker/filter/main.go @@ -58,10 +58,10 @@ func main() { // We are running both the receiver (takes messages in from the Broker) and the dispatcher (send // the messages to the triggers' subscribers) in this binary. - _, runnable := broker.New(logger, mgr.GetClient()) - err = mgr.Add(runnable) + receiver := broker.New(logger, mgr.GetClient()) + err = mgr.Add(receiver) if err != nil { - logger.Fatal("Unable to start the receivers runnable", zap.Error(err), zap.Any("runnable", runnable)) + logger.Fatal("Unable to start the receiver", zap.Error(err), zap.Any("receiver", receiver)) } // Set up signals so we handle the first shutdown signal gracefully. diff --git a/pkg/broker/receiver.go b/pkg/broker/receiver.go index 2f3eea9619a..a7e5688afbd 100644 --- a/pkg/broker/receiver.go +++ b/pkg/broker/receiver.go @@ -28,13 +28,10 @@ import ( "reflect" "github.com/cloudevents/sdk-go/pkg/cloudevents" - cehttp "github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http" - eventingv1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" "github.com/knative/eventing/pkg/provisioners" "go.uber.org/zap" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -275,23 +272,12 @@ func isFailure(statusCode int) bool { // Initialize the client. Mainly intended to load stuff in its cache. func (r *Receiver) initClient() error { - // We list triggers so that we can load the client's cache. Otherwise, on receiving an event, it - // may not find the trigger and would return an error. - opts := &client.ListOptions{ - // Set Raw because if we need to get more than one page, then we will put the continue token - // into opts.Raw.Continue. - Raw: &metav1.ListOptions{}, - } - for { - tl := &eventingv1alpha1.TriggerList{} - if err := r.client.List(context.TODO(), opts, tl); err != nil { - return err - } - if tl.Continue != "" { - opts.Raw.Continue = tl.Continue - } else { - break - } + // We list triggers so that we do not drop messages. Otherwise, on receiving an event, it + // may not find the Trigger and would return an error. + opts := &client.ListOptions{} + tl := &eventingv1alpha1.TriggerList{} + if err := r.client.List(context.TODO(), opts, tl); err != nil { + return err } return nil } diff --git a/pkg/reconciler/v1alpha1/broker/broker.go b/pkg/reconciler/v1alpha1/broker/broker.go index f9918cbff8e..4a55490f89c 100644 --- a/pkg/reconciler/v1alpha1/broker/broker.go +++ b/pkg/reconciler/v1alpha1/broker/broker.go @@ -21,38 +21,29 @@ import ( "fmt" "time" - "k8s.io/apimachinery/pkg/runtime" - - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/record" - "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/handler" - "sigs.k8s.io/controller-runtime/pkg/manager" - "sigs.k8s.io/controller-runtime/pkg/source" - + "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" + "github.com/knative/eventing/pkg/logging" "github.com/knative/eventing/pkg/reconciler/names" - - "github.com/knative/eventing/contrib/gcppubsub/pkg/util/logging" - v1 "k8s.io/api/apps/v1" - - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/runtime/schema" - "github.com/knative/eventing/pkg/reconciler/v1alpha1/broker/resources" - "go.uber.org/zap" - - "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" + v1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/errors" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" runtimeclient "sigs.k8s.io/controller-runtime/pkg/client" - + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" ) const ( @@ -68,9 +59,8 @@ const ( ) type reconciler struct { - client client.Client - restConfig *rest.Config - recorder record.EventRecorder + client client.Client + recorder record.EventRecorder logger *zap.Logger @@ -91,8 +81,8 @@ type ReconcilerArgs struct { } // ProvideController returns a function that returns a Broker controller. -func ProvideController(logger *zap.Logger, args ReconcilerArgs) func(manager.Manager) (controller.Controller, error) { - return func(mgr manager.Manager) (controller.Controller, error) { +func ProvideController(args ReconcilerArgs) func(manager.Manager, *zap.Logger) (controller.Controller, error) { + return func(mgr manager.Manager, logger *zap.Logger) (controller.Controller, error) { // Setup a new controller to Reconcile Brokers. c, err := controller.New(controllerAgentName, mgr, controller.Options{ Reconciler: &reconciler{ @@ -163,7 +153,7 @@ func (r *reconciler) Reconcile(request reconcile.Request) (reconcile.Result, err r.recorder.Event(broker, corev1.EventTypeNormal, brokerReconciled, "Broker reconciled") } - if _, err = r.updateStatus(broker.DeepCopy()); err != nil { + if _, err = r.updateStatus(broker); err != nil { logging.FromContext(ctx).Error("Failed to update Broker status", zap.Error(err)) r.recorder.Eventf(broker, corev1.EventTypeWarning, brokerUpdateStatusFailed, "Failed to update Broker's status: %v", err) return reconcile.Result{}, err @@ -204,11 +194,13 @@ func (r *reconciler) reconcile(ctx context.Context, b *v1alpha1.Broker) (reconci _, err = r.reconcileFilterDeployment(ctx, b) if err != nil { logging.FromContext(ctx).Error("Problem reconciling filter Deployment", zap.Error(err)) + b.Status.MarkFilterFailed(err) return reconcile.Result{}, err } _, err = r.reconcileFilterService(ctx, b) if err != nil { logging.FromContext(ctx).Error("Problem reconciling filter Service", zap.Error(err)) + b.Status.MarkFilterFailed(err) return reconcile.Result{}, err } b.Status.MarkFilterReady() @@ -216,12 +208,14 @@ func (r *reconciler) reconcile(ctx context.Context, b *v1alpha1.Broker) (reconci _, err = r.reconcileIngressDeployment(ctx, b, triggerChan) if err != nil { logging.FromContext(ctx).Error("Problem reconciling ingress Deployment", zap.Error(err)) + b.Status.MarkIngressFailed(err) return reconcile.Result{}, err } svc, err := r.reconcileIngressService(ctx, b) if err != nil { logging.FromContext(ctx).Error("Problem reconciling ingress Service", zap.Error(err)) + b.Status.MarkIngressFailed(err) return reconcile.Result{}, err } b.Status.MarkIngressReady() diff --git a/pkg/reconciler/v1alpha1/broker/broker_test.go b/pkg/reconciler/v1alpha1/broker/broker_test.go index 9e8776a317c..3960a46b897 100644 --- a/pkg/reconciler/v1alpha1/broker/broker_test.go +++ b/pkg/reconciler/v1alpha1/broker/broker_test.go @@ -36,7 +36,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes/scheme" - "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -70,11 +69,11 @@ var ( func init() { // Add types to scheme - v1alpha1.AddToScheme(scheme.Scheme) + _ = v1alpha1.AddToScheme(scheme.Scheme) } func TestProvideController(t *testing.T) { - //TODO(grantr) This needs a mock of manager.Manager. Creating a manager + // TODO(grantr) This needs a mock of manager.Manager. Creating a manager // with a fake Config fails because the Manager tries to contact the // apiserver. @@ -189,7 +188,7 @@ func TestReconcile(t *testing.T) { // differ from expected. // TODO uncomment the following line once our test framework supports searching for // GenerateName. - //makeDifferentChannel(), + // makeDifferentChannel(), }, WantEvent: []corev1.Event{ { @@ -535,7 +534,7 @@ func TestReconcile(t *testing.T) { WantPresent: []runtime.Object{ makeReadyBroker(), // TODO Uncomment makeChannel() when our test framework handles generateName. - //makeChannel(), + // makeChannel(), makeFilterDeployment(), makeFilterService(), makeIngressDeployment(), @@ -553,10 +552,9 @@ func TestReconcile(t *testing.T) { recorder := tc.GetEventRecorder() r := &reconciler{ - client: c, - restConfig: &rest.Config{}, - recorder: recorder, - logger: zap.NewNop(), + client: c, + recorder: recorder, + logger: zap.NewNop(), filterImage: filterImage, filterServiceAccountName: filterSA, From 134198d29d6d16f1517d981044f0395f74040ccf Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Mon, 18 Mar 2019 14:06:13 -0700 Subject: [PATCH 112/128] Update cloudevents/sdk-go to 0.4.0. --- Gopkg.lock | 11 +- Gopkg.toml | 5 +- .../sdk-go/pkg/cloudevents/client/client.go | 76 ----- .../pkg/cloudevents/client/client_http.go | 54 ---- .../pkg/cloudevents/client/client_nats.go | 52 ---- .../pkg/cloudevents/client/defaulters.go | 62 ----- .../sdk-go/pkg/cloudevents/client/options.go | 177 ------------ .../sdk-go/pkg/cloudevents/context/context.go | 27 ++ .../pkg/cloudevents/datacodec/xml/data.go | 4 +- .../sdk-go/pkg/cloudevents/event_response.go | 29 ++ .../cloudevents/transport/http/codec_v01.go | 16 +- .../cloudevents/transport/http/codec_v02.go | 10 +- .../cloudevents/transport/http/codec_v03.go | 10 +- .../pkg/cloudevents/transport/http/context.go | 34 +++ .../pkg/cloudevents/transport/http/message.go | 6 + .../pkg/cloudevents/transport/http/options.go | 140 ++++++++++ .../cloudevents/transport/http/transport.go | 260 +++++++++++++++--- .../pkg/cloudevents/transport/message.go | 5 + .../pkg/cloudevents/transport/nats/codec.go | 54 ---- .../cloudevents/transport/nats/codec_v02.go | 48 ---- .../cloudevents/transport/nats/codec_v03.go | 48 ---- .../cloudevents/transport/nats/encoding.go | 45 --- .../pkg/cloudevents/transport/nats/message.go | 31 --- .../cloudevents/transport/nats/transport.go | 91 ------ .../pkg/cloudevents/transport/transport.go | 10 +- .../sdk-go/pkg/cloudevents/types/timestamp.go | 29 +- .../sdk-go/pkg/cloudevents/types/urlref.go | 23 +- 27 files changed, 537 insertions(+), 820 deletions(-) delete mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/client.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/client_http.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/client_nats.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/defaulters.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/options.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/context/context.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/event_response.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/context.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/options.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/nats/codec.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/nats/codec_v02.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/nats/codec_v03.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/nats/encoding.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/nats/message.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/nats/transport.go diff --git a/Gopkg.lock b/Gopkg.lock index 92ae2960442..6296dbdec50 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -58,22 +58,22 @@ version = "v2.1.15" [[projects]] - digest = "1:d9cc48e9d4caa22c2af5b18f404d3e4a34b05e6723b002c0c18a39762a14777d" + digest = "1:6a76ae82f291e0188bd9471cdb4a8cd1eb5c8a8e7be46a248d31d6b4a45dc973" name = "github.com/cloudevents/sdk-go" packages = [ "pkg/cloudevents", - "pkg/cloudevents/client", "pkg/cloudevents/codec", + "pkg/cloudevents/context", "pkg/cloudevents/datacodec", "pkg/cloudevents/datacodec/json", "pkg/cloudevents/datacodec/xml", "pkg/cloudevents/transport", "pkg/cloudevents/transport/http", - "pkg/cloudevents/transport/nats", "pkg/cloudevents/types", ] pruneopts = "NUT" - revision = "d50361a5655081514f406b4e672d72e9886c17ad" + revision = "c1a6fb0cc8226884014fb5063ab406ed9504a663" + version = "0.4.0" [[projects]] digest = "1:ffe9824d294da03b391f44e1ae8281281b4afc1bdaa9588c9097785e3af10cec" @@ -1254,7 +1254,8 @@ "cloud.google.com/go/pubsub", "github.com/Shopify/sarama", "github.com/bsm/sarama-cluster", - "github.com/cloudevents/sdk-go/pkg/cloudevents/client", + "github.com/cloudevents/sdk-go/pkg/cloudevents", + "github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http", "github.com/fsnotify/fsnotify", "github.com/google/go-cmp/cmp", "github.com/google/go-cmp/cmp/cmpopts", diff --git a/Gopkg.toml b/Gopkg.toml index cc785521253..9cdd54063f6 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -130,8 +130,7 @@ required = [ name = "github.com/nats-io/nats-streaming-server" version = "0.11.0" -# Use CloudEvents, master as of Feb 28, 2019 🎂 -# Will use releases as soon as they get releases going. +# The latest release as of March 12, 2019. [[override]] name = "github.com/cloudevents/sdk-go" - revision = "d50361a5655081514f406b4e672d72e9886c17ad" + version = "=0.4.0" diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/client.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/client.go deleted file mode 100644 index da5f67a4254..00000000000 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/client.go +++ /dev/null @@ -1,76 +0,0 @@ -package client - -import ( - "context" - "fmt" - "github.com/cloudevents/sdk-go/pkg/cloudevents" - "github.com/cloudevents/sdk-go/pkg/cloudevents/transport" - "github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http" - "github.com/cloudevents/sdk-go/pkg/cloudevents/transport/nats" -) - -type Receiver func(event cloudevents.Event) - -type Client interface { - Send(ctx context.Context, event cloudevents.Event) error - StartReceiver(ctx context.Context, fn Receiver) error - - Receive(event cloudevents.Event) -} - -type ceClient struct { - transport transport.Sender - receiver Receiver - - eventDefaulterFns []EventDefaulter -} - -func (c *ceClient) Send(ctx context.Context, event cloudevents.Event) error { - // Confirm we have a transport set. - if c.transport == nil { - return fmt.Errorf("client not ready, transport not initalized") - } - // Apply the defaulter chain to the incoming event. - if len(c.eventDefaulterFns) > 0 { - for _, fn := range c.eventDefaulterFns { - event = fn(event) - } - } - // Validate the event conforms to the CloudEvents Spec. - if err := event.Validate(); err != nil { - return err - } - // Send the event over the transport. - return c.transport.Send(ctx, event) -} - -func (c *ceClient) Receive(event cloudevents.Event) { - if c.receiver != nil { - c.receiver(event) - } -} - -func (c *ceClient) StartReceiver(ctx context.Context, fn Receiver) error { - if c.transport == nil { - return fmt.Errorf("client not ready, transport not initalized") - } - - if t, ok := c.transport.(*http.Transport); ok { - return c.startHTTPReceiver(ctx, t, fn) - } - - if t, ok := c.transport.(*nats.Transport); ok { - return c.startNATSReceiver(ctx, t, fn) - } - - return fmt.Errorf("unknown transport type: %T", c.transport) -} - -func (c *ceClient) applyOptions(opts ...Option) error { - for _, fn := range opts { - if err := fn(c); err != nil { - return err - } - } - return nil -} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/client_http.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/client_http.go deleted file mode 100644 index 83135945afa..00000000000 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/client_http.go +++ /dev/null @@ -1,54 +0,0 @@ -package client - -import ( - "context" - "fmt" - cloudeventshttp "github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http" - "log" - "net/http" -) - -func NewHTTPClient(opts ...Option) (Client, error) { - c := &ceClient{ - transport: &cloudeventshttp.Transport{ - // Default the request method. - Req: &http.Request{ - Method: http.MethodPost, - }, - }, - } - - if err := c.applyOptions(opts...); err != nil { - return nil, err - } - return c, nil -} - -func StartHTTPReceiver(ctx context.Context, fn Receiver, opts ...Option) (Client, error) { - c, err := NewHTTPClient(opts...) - if err != nil { - return nil, err - } - - if err := c.StartReceiver(ctx, fn); err != nil { - return nil, err - } - return c, nil -} - -func (c *ceClient) startHTTPReceiver(ctx context.Context, t *cloudeventshttp.Transport, fn Receiver) error { - if c.receiver != nil { - return fmt.Errorf("client already has a receiver") - } - if t.Receiver != nil { - return fmt.Errorf("transport already has a receiver") - } - c.receiver = fn - t.Receiver = c - - go func() { - log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", t.GetPort()), t)) - }() - - return nil -} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/client_nats.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/client_nats.go deleted file mode 100644 index 1033e116092..00000000000 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/client_nats.go +++ /dev/null @@ -1,52 +0,0 @@ -package client - -import ( - "context" - "fmt" - cloudeventsnats "github.com/cloudevents/sdk-go/pkg/cloudevents/transport/nats" - "github.com/nats-io/go-nats" - "log" -) - -func NewNATSClient(natsServer, subject string, opts ...Option) (Client, error) { - conn, err := nats.Connect(natsServer) - if err != nil { - return nil, err - } - transport := cloudeventsnats.Transport{ - Conn: conn, - Subject: subject, - } - c := &ceClient{ - transport: &transport, - } - - if err := c.applyOptions(opts...); err != nil { - return nil, err - } - - return c, nil -} - -func (c *ceClient) startNATSReceiver(ctx context.Context, t *cloudeventsnats.Transport, fn Receiver) error { - if t.Conn == nil { - return fmt.Errorf("nats connection is required to be set") - } - if c.receiver != nil { - return fmt.Errorf("client already has a receiver") - } - if t.Receiver != nil { - return fmt.Errorf("transport already has a receiver") - } - - c.receiver = fn - t.Receiver = c - - go func() { - if err := t.Listen(ctx); err != nil { - log.Fatalf("failed to listen, %s", err.Error()) - } - }() - - return nil -} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/defaulters.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/defaulters.go deleted file mode 100644 index ee87e0467a1..00000000000 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/defaulters.go +++ /dev/null @@ -1,62 +0,0 @@ -package client - -import ( - "github.com/cloudevents/sdk-go/pkg/cloudevents" - "github.com/cloudevents/sdk-go/pkg/cloudevents/types" - "github.com/google/uuid" - "time" -) - -type EventDefaulter func(event cloudevents.Event) cloudevents.Event - -func DefaultIDToUUIDIfNotSet(event cloudevents.Event) cloudevents.Event { - if event.Context != nil { - switch event.Context.GetSpecVersion() { - case cloudevents.CloudEventsVersionV01: - ec := event.Context.AsV01() - if ec.EventID == "" { - ec.EventID = uuid.New().String() - event.Context = ec - } - case cloudevents.CloudEventsVersionV02: - ec := event.Context.AsV02() - if ec.ID == "" { - ec.ID = uuid.New().String() - event.Context = ec - } - case cloudevents.CloudEventsVersionV03: - ec := event.Context.AsV03() - if ec.ID == "" { - ec.ID = uuid.New().String() - event.Context = ec - } - } - } - return event -} - -func DefaultTimeToNowIfNotSet(event cloudevents.Event) cloudevents.Event { - if event.Context != nil { - switch event.Context.GetSpecVersion() { - case cloudevents.CloudEventsVersionV01: - ec := event.Context.AsV01() - if ec.EventTime == nil || ec.EventTime.IsZero() { - ec.EventTime = &types.Timestamp{Time: time.Now()} - event.Context = ec - } - case cloudevents.CloudEventsVersionV02: - ec := event.Context.AsV02() - if ec.Time == nil || ec.Time.IsZero() { - ec.Time = &types.Timestamp{Time: time.Now()} - event.Context = ec - } - case cloudevents.CloudEventsVersionV03: - ec := event.Context.AsV03() - if ec.Time == nil || ec.Time.IsZero() { - ec.Time = &types.Timestamp{Time: time.Now()} - event.Context = ec - } - } - } - return event -} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/options.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/options.go deleted file mode 100644 index f534f48fc77..00000000000 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/options.go +++ /dev/null @@ -1,177 +0,0 @@ -package client - -import ( - "fmt" - "github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http" - "github.com/cloudevents/sdk-go/pkg/cloudevents/transport/nats" - nethttp "net/http" - "net/url" - "strings" -) - -type Option func(*ceClient) error - -// WithTarget sets the outbound recipient of cloudevents when using an HTTP request. -func WithTarget(targetUrl string) Option { - return func(c *ceClient) error { - if t, ok := c.transport.(*http.Transport); ok { - targetUrl = strings.TrimSpace(targetUrl) - if targetUrl != "" { - var err error - var target *url.URL - target, err = url.Parse(targetUrl) - if err != nil { - return fmt.Errorf("client option failed to parse target url: %s", err.Error()) - } - - if t.Req == nil { - t.Req = &nethttp.Request{ - Method: nethttp.MethodPost, - } - } - t.Req.URL = target - return nil - } - return fmt.Errorf("target option was empty string") - } - return fmt.Errorf("invalid target client option received for transport type") - } -} - -// WithHTTPMethod sets the outbound recipient of cloudevents when using an HTTP request. -func WithHTTPMethod(method string) Option { - return func(c *ceClient) error { - if t, ok := c.transport.(*http.Transport); ok { - method = strings.TrimSpace(method) - if method != "" { - if t.Req == nil { - t.Req = &nethttp.Request{} - } - t.Req.Method = method - return nil - } - return fmt.Errorf("method option was empty string") - } - return fmt.Errorf("invalid HTTP method client option received for transport type") - } -} - -// WithHTTPEncoding sets the encoding for clients with HTTP transports. -func WithHTTPEncoding(encoding http.Encoding) Option { - return func(c *ceClient) error { - if t, ok := c.transport.(*http.Transport); ok { - t.Encoding = encoding - return nil - } - return fmt.Errorf("invalid HTTP encoding client option received for transport type") - } -} - -// WithHTTPDefaultEncodingSelector sets the encoding selection strategy for -// default encoding selections based on Event. -func WithHTTPDefaultEncodingSelector(fn http.EncodingSelector) Option { - return func(c *ceClient) error { - if t, ok := c.transport.(*http.Transport); ok { - if fn != nil { - t.DefaultEncodingSelectionFn = fn - return nil - } - return fmt.Errorf("fn for DefaultEncodingSelector was nil") - } - return fmt.Errorf("invalid HTTP default encoding selector client option received for transport type") - } -} - -// WithHTTPBinaryEncodingSelector sets the encoding selection strategy for -// default encoding selections based on Event, the encoded event will be the -// given version in Binary form. -func WithHTTPBinaryEncoding() Option { - return func(c *ceClient) error { - if t, ok := c.transport.(*http.Transport); ok { - t.DefaultEncodingSelectionFn = http.DefaultBinaryEncodingSelectionStrategy - return nil - } - return fmt.Errorf("invalid HTTP binary encoding client option received for transport type") - } -} - -// WithHTTPStructuredEncodingSelector sets the encoding selection strategy for -// default encoding selections based on Event, the encoded event will be the -//// given version in Structured form. -func WithHTTPStructuredEncoding() Option { - return func(c *ceClient) error { - if t, ok := c.transport.(*http.Transport); ok { - t.DefaultEncodingSelectionFn = http.DefaultStructuredEncodingSelectionStrategy - return nil - } - return fmt.Errorf("invalid HTTP structured encoding client option received for transport type") - } -} - -// WithHTTPPort sets the port for for clients with HTTP transports. -func WithHTTPPort(port int) Option { - return func(c *ceClient) error { - if t, ok := c.transport.(*http.Transport); ok { - if port == 0 { - return fmt.Errorf("client option was given an invalid port: %d", port) - } - t.Port = port - return nil - } - return fmt.Errorf("invalid HTTP port client option received for transport type") - } -} - -// WithHTTPClient sets the internal HTTP client for cloudevent clients with HTTP transports. -func WithHTTPClient(netclient *nethttp.Client) Option { - return func(c *ceClient) error { - if t, ok := c.transport.(*http.Transport); ok { - if netclient == nil { - return fmt.Errorf("client option was given an nil HTTP client") - } - t.Client = netclient - return nil - } - return fmt.Errorf("invalid HTTP client client option received for transport type") - } -} - -// WithNATSEncoding sets the encoding for clients with NATS transport. -func WithNATSEncoding(encoding nats.Encoding) Option { - return func(c *ceClient) error { - if t, ok := c.transport.(*nats.Transport); ok { - t.Encoding = encoding - return nil - } - return fmt.Errorf("invalid NATS encoding client option received for transport type") - } -} - -// WithEventDefaulter adds an event defaulter to the end of the defaulter chain. -func WithEventDefaulter(fn EventDefaulter) Option { - return func(c *ceClient) error { - if fn == nil { - return fmt.Errorf("client option was given an nil event defaulter") - } - c.eventDefaulterFns = append(c.eventDefaulterFns, fn) - return nil - } -} - -// WithUUIDs adds DefaultIDToUUIDIfNotSet event defaulter to the end of the -// defaulter chain. -func WithUUIDs() Option { - return func(c *ceClient) error { - c.eventDefaulterFns = append(c.eventDefaulterFns, DefaultIDToUUIDIfNotSet) - return nil - } -} - -// WithTimeNow adds DefaultTimeToNowIfNotSet event defaulter to the end of the -// defaulter chain. -func WithTimeNow() Option { - return func(c *ceClient) error { - c.eventDefaulterFns = append(c.eventDefaulterFns, DefaultTimeToNowIfNotSet) - return nil - } -} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/context/context.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/context/context.go new file mode 100644 index 00000000000..2c4526b1035 --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/context/context.go @@ -0,0 +1,27 @@ +package context + +import ( + "context" + "net/url" +) + +// Opaque key type used to store target +type targetKeyType struct{} + +var targetKey = targetKeyType{} + +func WithTarget(ctx context.Context, target string) context.Context { + return context.WithValue(ctx, targetKey, target) +} + +func TargetFrom(ctx context.Context) *url.URL { + c := ctx.Value(targetKey) + if c != nil { + if s, ok := c.(string); ok && s != "" { + if target, err := url.Parse(s); err == nil { + return target + } + } + } + return nil +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/xml/data.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/xml/data.go index fb36f2eb285..a904a250c07 100644 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/xml/data.go +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/xml/data.go @@ -26,7 +26,7 @@ func Decode(in, out interface{}) error { if len(b) > 1 && (b[0] == byte('"') || (b[0] == byte('\\') && b[1] == byte('"'))) { s, err := strconv.Unquote(string(b)) if err != nil { - return err + return fmt.Errorf("[xml] failed to unquote quoted data: %s", err.Error()) } if len(s) > 0 && s[0] == '<' { // looks like xml, use it @@ -35,7 +35,7 @@ func Decode(in, out interface{}) error { // looks like base64, decode bs, err := base64.StdEncoding.DecodeString(s) if err != nil { - return err + return fmt.Errorf("[xml] failed to decode base64 encoded string: %s", err.Error()) } b = bs } diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/event_response.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/event_response.go new file mode 100644 index 00000000000..b8e09746204 --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/event_response.go @@ -0,0 +1,29 @@ +package cloudevents + +// EventResponse represents the canonical representation of a Response to a +// CloudEvent from a receiver. +type EventResponse struct { + Status int + Event *Event + Reason string +} + +func (e *EventResponse) RespondWith(status int, event *Event) { + if e == nil { + // if nil, response not supported + return + } + e.Status = status + if event != nil { + e.Event = event + } +} + +func (e *EventResponse) Error(status int, reason string) { + if e == nil { + // if nil, response not supported + return + } + e.Status = status + e.Reason = reason +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/codec_v01.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/codec_v01.go index 13dd6e0b483..9be36a4b2d4 100644 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/codec_v01.go +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/codec_v01.go @@ -2,7 +2,6 @@ package http import ( "encoding/json" - "reflect" "fmt" "github.com/cloudevents/sdk-go/pkg/cloudevents" "github.com/cloudevents/sdk-go/pkg/cloudevents/codec" @@ -166,7 +165,7 @@ func (v CodecV01) fromHeaders(h http.Header) (cloudevents.EventContextV01, error extensions := make(map[string]interface{}) for k, v := range h { - if strings.EqualFold(k[:len("CE-X-")], "CE-X-") { + if len(k) > len("CE-X-") && strings.EqualFold(k[:len("CE-X-")], "CE-X-") { key := k[len("CE-X-"):] var tmp interface{} if err := json.Unmarshal([]byte(v[0]), &tmp); err == nil { @@ -192,28 +191,17 @@ func (v CodecV01) decodeStructured(msg transport.Message) (*cloudevents.Event, e } func (v CodecV01) inspectEncoding(msg transport.Message) Encoding { - log.Println("Inside v01.inspectEncoding") version := msg.CloudEventsVersion() - log.Printf("v01.inspectEncoding version %v", version) if version != cloudevents.CloudEventsVersionV01 { - log.Printf("v01.insepectEncoding wrong version") return Unknown } m, ok := msg.(*Message) if !ok { - log.Printf("v01.insepectEncoding wrong type: %v", reflect.TypeOf(msg)) return Unknown } contentType := m.Header.Get("Content-Type") - log.Printf("v01.inspectEncoding content-type %v", contentType) - if contentType == cloudevents.ApplicationJSON { - return BinaryV01 - } - if contentType == cloudevents.ApplicationXML { - return BinaryV01 - } if contentType == cloudevents.ApplicationCloudEventsJSON { return StructuredV01 } - return Unknown + return BinaryV01 } diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/codec_v02.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/codec_v02.go index 51201e87e74..4c10ac9868c 100644 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/codec_v02.go +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/codec_v02.go @@ -186,7 +186,7 @@ func (v CodecV02) fromHeaders(h http.Header) (cloudevents.EventContextV02, error extensions := make(map[string]interface{}) for k, v := range h { - if strings.EqualFold(k[:len("ce-")], "ce-") { + if len(k) > len("ce-") && strings.EqualFold(k[:len("ce-")], "ce-") { ak := strings.ToLower(k[len("ce-"):]) if i := strings.Index(ak, "-"); i > 0 { // attrib-key @@ -259,14 +259,8 @@ func (v CodecV02) inspectEncoding(msg transport.Message) Encoding { return Unknown } contentType := m.Header.Get("Content-Type") - if contentType == cloudevents.ApplicationJSON { - return BinaryV02 - } - if contentType == cloudevents.ApplicationXML { - return BinaryV02 - } if contentType == cloudevents.ApplicationCloudEventsJSON { return StructuredV02 } - return Unknown + return BinaryV02 } diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/codec_v03.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/codec_v03.go index 8393111d137..78cfd2011d5 100644 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/codec_v03.go +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/codec_v03.go @@ -191,7 +191,7 @@ func (v CodecV03) fromHeaders(h http.Header) (cloudevents.EventContextV03, error extensions := make(map[string]interface{}) for k, v := range h { - if strings.EqualFold(k[:len("ce-")], "ce-") { + if len(k) > len("ce-") && strings.EqualFold(k[:len("ce-")], "ce-") { ak := strings.ToLower(k[len("ce-"):]) if i := strings.Index(ak, "-"); i > 0 { // attrib-key @@ -264,17 +264,11 @@ func (v CodecV03) inspectEncoding(msg transport.Message) Encoding { return Unknown } contentType := m.Header.Get("Content-Type") - if contentType == cloudevents.ApplicationJSON { - return BinaryV03 - } - if contentType == cloudevents.ApplicationXML { - return BinaryV03 - } if contentType == cloudevents.ApplicationCloudEventsJSON { return StructuredV03 } if contentType == cloudevents.ApplicationCloudEventsBatchJSON { return BatchedV03 } - return Unknown + return BinaryV03 } diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/context.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/context.go new file mode 100644 index 00000000000..70ac8de8ec0 --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/context.go @@ -0,0 +1,34 @@ +package http + +import ( + "context" +) + +// TransportContext allows a Receiver to understand the context of a request. +type TransportContext struct { + URI string + Host string + Method string +} + +// Opaque key type used to store TransportContext +type transportContextKeyType struct{} + +var transportContextKey = transportContextKeyType{} + +func WithTransportContext(ctx context.Context, tcxt TransportContext) context.Context { + return context.WithValue(ctx, transportContextKey, tcxt) +} + +func TransportContextFrom(ctx context.Context) TransportContext { + tctx := ctx.Value(transportContextKey) + if tctx != nil { + if tx, ok := tctx.(TransportContext); ok { + return tx + } + if tx, ok := tctx.(*TransportContext); ok { + return *tx + } + } + return TransportContext{} +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/message.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/message.go index c7eb1813624..f9c55265039 100644 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/message.go +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/message.go @@ -14,6 +14,12 @@ type Message struct { Body []byte } +type Response struct { + StatusCode int + Header http.Header + Body []byte +} + func (m Message) CloudEventsVersion() string { // TODO: the impl of this method needs to move into the codec. diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/options.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/options.go new file mode 100644 index 00000000000..ea54869c746 --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/options.go @@ -0,0 +1,140 @@ +package http + +import ( + "fmt" + nethttp "net/http" + "net/url" + "strings" +) + +type Option func(*Transport) error + +// WithTarget sets the outbound recipient of cloudevents when using an HTTP +// request. +func WithTarget(targetUrl string) Option { + return func(t *Transport) error { + if t == nil { + return fmt.Errorf("http target option can not set nil transport") + } + targetUrl = strings.TrimSpace(targetUrl) + if targetUrl != "" { + var err error + var target *url.URL + target, err = url.Parse(targetUrl) + if err != nil { + return fmt.Errorf("http target option failed to parse target url: %s", err.Error()) + } + + if t.Req == nil { + t.Req = &nethttp.Request{ + Method: nethttp.MethodPost, + } + } + t.Req.URL = target + return nil + } + return fmt.Errorf("http target option was empty string") + } +} + +// WithMethod sets the outbound recipient of cloudevents when using an HTTP +// request. +func WithMethod(method string) Option { + return func(t *Transport) error { + if t == nil { + return fmt.Errorf("http method option can not set nil transport") + } + method = strings.TrimSpace(method) + if method != "" { + if t.Req == nil { + t.Req = &nethttp.Request{} + } + t.Req.Method = method + return nil + } + return fmt.Errorf("http method option was empty string") + } +} + +// WithEncoding sets the encoding for clients with HTTP transports. +func WithEncoding(encoding Encoding) Option { + return func(t *Transport) error { + if t == nil { + return fmt.Errorf("http encoding option can not set nil transport") + } + t.Encoding = encoding + return nil + } +} + +// WithDefaultEncodingSelector sets the encoding selection strategy for +// default encoding selections based on Event. +func WithDefaultEncodingSelector(fn EncodingSelector) Option { + return func(t *Transport) error { + if t == nil { + return fmt.Errorf("http default encoding selector option can not set nil transport") + } + if fn != nil { + t.DefaultEncodingSelectionFn = fn + return nil + } + return fmt.Errorf("http fn for DefaultEncodingSelector was nil") + } +} + +// WithBinaryEncoding sets the encoding selection strategy for +// default encoding selections based on Event, the encoded event will be the +// given version in Binary form. +func WithBinaryEncoding() Option { + return func(t *Transport) error { + if t == nil { + return fmt.Errorf("http binary encoding option can not set nil transport") + } + + t.DefaultEncodingSelectionFn = DefaultBinaryEncodingSelectionStrategy + return nil + } +} + +// WithStructuredEncoding sets the encoding selection strategy for +// default encoding selections based on Event, the encoded event will be the +// given version in Structured form. +func WithStructuredEncoding() Option { + return func(t *Transport) error { + if t == nil { + return fmt.Errorf("http structured encoding option can not set nil transport") + } + + t.DefaultEncodingSelectionFn = DefaultStructuredEncodingSelectionStrategy + return nil + } +} + +// WithPort sets the port for for clients with HTTP transports. +func WithPort(port int) Option { + return func(t *Transport) error { + if t == nil { + return fmt.Errorf("http port option can not set nil transport") + } + if port < 0 { + return fmt.Errorf("http port option was given an invalid port: %d", port) + } + t.Port = &port + return nil + } +} + +// WithPath sets the path to receive cloudevents on for HTTP transports. +func WithPath(path string) Option { + return func(t *Transport) error { + if t == nil { + return fmt.Errorf("http path option can not set nil transport") + } + path = strings.TrimSpace(path) + if len(path) == 0 { + return fmt.Errorf("http path option was given an invalid path: %q", path) + } + t.Path = path + return nil + } +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/transport.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/transport.go index 674bad9fc25..8f0f8d5dcf7 100644 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/transport.go +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/transport.go @@ -4,17 +4,22 @@ import ( "bytes" "context" "fmt" - "github.com/cloudevents/sdk-go/pkg/cloudevents" - "github.com/cloudevents/sdk-go/pkg/cloudevents/transport" "io/ioutil" "log" + "net" "net/http" + "strings" + "sync" + + "github.com/cloudevents/sdk-go/pkg/cloudevents" + cecontext "github.com/cloudevents/sdk-go/pkg/cloudevents/context" + "github.com/cloudevents/sdk-go/pkg/cloudevents/transport" ) type EncodingSelector func(e cloudevents.Event) Encoding // type check that this transport message impl matches the contract -var _ transport.Sender = (*Transport)(nil) +var _ transport.Transport = (*Transport)(nil) // Transport acts as both a http client and a http handler. type Transport struct { @@ -26,14 +31,45 @@ type Transport struct { Req *http.Request // Receiving - Port int Receiver transport.Receiver + Port *int // if nil, default 8080 + Path string // if "", default "/" + Handler *http.ServeMux - codec transport.Codec + realPort int + server *http.Server + handlerRegistered bool + codec transport.Codec + // Create Mutex + crMu sync.Mutex + // Receive Mutex + reMu sync.Mutex +} + +func New(opts ...Option) (*Transport, error) { + t := &Transport{ + Req: &http.Request{ + Method: http.MethodPost, + }, + } + if err := t.applyOptions(opts...); err != nil { + return nil, err + } + return t, nil +} + +func (t *Transport) applyOptions(opts ...Option) error { + for _, fn := range opts { + if err := fn(t); err != nil { + return err + } + } + return nil } func (t *Transport) loadCodec() bool { if t.codec == nil { + t.crMu.Lock() if t.DefaultEncodingSelectionFn != nil && t.Encoding != Default { log.Printf("[warn] Transport has a DefaultEncodingSelectionFn set but Encoding is not Default. DefaultEncodingSelectionFn will be ignored.") } @@ -41,13 +77,16 @@ func (t *Transport) loadCodec() bool { Encoding: t.Encoding, DefaultEncodingSelectionFn: t.DefaultEncodingSelectionFn, } + t.crMu.Unlock() } return true } -func (t *Transport) Send(ctx context.Context, event cloudevents.Event) error { +func (t *Transport) Send(ctx context.Context, event cloudevents.Event) (*cloudevents.Event, error) { if t.Client == nil { + t.crMu.Lock() t.Client = &http.Client{} + t.crMu.Unlock() } var req http.Request @@ -56,13 +95,18 @@ func (t *Transport) Send(ctx context.Context, event cloudevents.Event) error { req.URL = t.Req.URL } + // Override the default request with target from context. + if target := cecontext.TargetFrom(ctx); target != nil { + req.URL = target + } + if ok := t.loadCodec(); !ok { - return fmt.Errorf("unknown encoding set on transport: %d", t.Encoding) + return nil, fmt.Errorf("unknown encoding set on transport: %d", t.Encoding) } msg, err := t.codec.Encode(event) if err != nil { - return err + return nil, err } // TODO: merge the incoming request with msg, for now just replace. @@ -70,36 +114,110 @@ func (t *Transport) Send(ctx context.Context, event cloudevents.Event) error { req.Header = m.Header req.Body = ioutil.NopCloser(bytes.NewBuffer(m.Body)) req.ContentLength = int64(len(m.Body)) - return httpDo(ctx, &req, func(resp *http.Response, err error) error { + return httpDo(ctx, &req, func(resp *http.Response, err error) (*cloudevents.Event, error) { if err != nil { - return err + return nil, err } defer resp.Body.Close() + body, _ := ioutil.ReadAll(resp.Body) + msg := &Message{ + Header: resp.Header, + Body: body, + } + + var respEvent *cloudevents.Event + if msg.CloudEventsVersion() != "" { + if ok := t.loadCodec(); !ok { + err := fmt.Errorf("unknown encoding set on transport: %d", t.Encoding) + log.Printf("failed to load codec: %s", err) + } + if respEvent, err = t.codec.Decode(msg); err != nil { + log.Printf("failed to decode message: %s %v", err, resp) + } + } + if accepted(resp) { - return nil + return respEvent, nil } - return fmt.Errorf("error sending cloudevent: %s", status(resp)) + return respEvent, fmt.Errorf("error sending cloudevent: %s", status(resp)) }) } + return nil, fmt.Errorf("failed to encode Event into a Message") +} - return fmt.Errorf("failed to encode Event into a Message") +func (t *Transport) SetReceiver(r transport.Receiver) { + t.Receiver = r } -func httpDo(ctx context.Context, req *http.Request, f func(*http.Response, error) error) error { - // Run the HTTP request in a goroutine and pass the response to f. - c := make(chan error, 1) - req = req.WithContext(ctx) - go func() { c <- f(http.DefaultClient.Do(req)) }() +func (t *Transport) StartReceiver(ctx context.Context) error { + t.reMu.Lock() + defer t.reMu.Unlock() + + if t.Handler == nil { + t.Handler = http.NewServeMux() + } + if !t.handlerRegistered { + // handler.Handle might panic if the user tries to use the same path as the sdk. + t.Handler.Handle(t.GetPath(), t) + t.handlerRegistered = true + } + + addr := fmt.Sprintf(":%d", t.GetPort()) + t.server = &http.Server{ + Addr: addr, + Handler: t.Handler, + } + + listener, err := net.Listen("tcp", addr) + if err != nil { + return err + } + t.realPort = listener.Addr().(*net.TCPAddr).Port + + // Shutdown + defer func() { + t.realPort = 0 + t.server.Close() + t.server = nil + }() + + errChan := make(chan error, 1) + go func() { + errChan <- t.server.Serve(listener) + }() + + // wait for the server to return or ctx.Done(). select { case <-ctx.Done(): - <-c // Wait for f to return. - return ctx.Err() - case err := <-c: + // Try a gracefully shutdown. + return t.server.Shutdown(context.Background()) + case err := <-errChan: return err } } +type eventError struct { + event *cloudevents.Event + err error +} + +func httpDo(ctx context.Context, req *http.Request, fn func(*http.Response, error) (*cloudevents.Event, error)) (*cloudevents.Event, error) { + // Run the HTTP request in a goroutine and pass the response to fn. + c := make(chan eventError, 1) + req = req.WithContext(ctx) + go func() { + event, err := fn(http.DefaultClient.Do(req)) + c <- eventError{event: event, err: err} + }() + select { + case <-ctx.Done(): + return nil, ctx.Err() + case ee := <-c: + return ee.event, ee.err + } +} + // accepted is a helper method to understand if the response from the target // accepted the CloudEvent. func accepted(resp *http.Response) bool { @@ -119,17 +237,55 @@ func status(resp *http.Response) string { return fmt.Sprintf("Status[%s] %s", status, body) } +func (t *Transport) invokeReceiver(ctx context.Context, event cloudevents.Event) (*Response, error) { + if t.Receiver != nil { + // Note: http does not use eventResp.Reason + eventResp := cloudevents.EventResponse{} + resp := Response{} + + err := t.Receiver.Receive(ctx, event, &eventResp) + if err != nil { + log.Printf("got an error from receiver fn: %s", err.Error()) + resp.StatusCode = http.StatusInternalServerError + return &resp, err + } + + if eventResp.Event != nil { + if t.loadCodec() { + if m, err := t.codec.Encode(*eventResp.Event); err != nil { + log.Printf("failed to encode response from receiver fn: %s", err.Error()) + } else if msg, ok := m.(*Message); ok { + resp.Header = msg.Header + resp.Body = msg.Body + } + } else { + log.Printf("failed to load codec") + resp.StatusCode = http.StatusInternalServerError + return &resp, err + } + } + + if eventResp.Status != 0 { + resp.StatusCode = eventResp.Status + } else { + resp.StatusCode = http.StatusAccepted // default is 202 - Accepted + } + return &resp, err + } + return nil, nil +} + // ServeHTTP implements http.Handler -func (t *Transport) ServeHTTP(w http.ResponseWriter, r *http.Request) { - body, err := ioutil.ReadAll(r.Body) +func (t *Transport) ServeHTTP(w http.ResponseWriter, req *http.Request) { + body, err := ioutil.ReadAll(req.Body) if err != nil { - log.Printf("failed to handle request: %s %v", err, r) + log.Printf("failed to handle request: %s %v", err, req) w.WriteHeader(http.StatusBadRequest) w.Write([]byte(`{"error":"Invalid request"}`)) return } msg := &Message{ - Header: r.Header, + Header: req.Header, Body: body, } @@ -142,23 +298,63 @@ func (t *Transport) ServeHTTP(w http.ResponseWriter, r *http.Request) { } event, err := t.codec.Decode(msg) if err != nil { - log.Printf("failed to decode message: %s %v", err, r) + log.Printf("failed to decode message: %s %v", err, req) w.WriteHeader(http.StatusBadRequest) - w.Write([]byte(`{"error":"Decoding Error"}`)) + w.Write([]byte(fmt.Sprintf(`{"error":%q}`, err.Error()))) return } - if t.Receiver != nil { - go t.Receiver.Receive(*event) + ctx := req.Context() + + if req != nil { + ctx = WithTransportContext(ctx, TransportContext{ + URI: req.RequestURI, + Host: req.Host, + Method: req.Method, + }) + } + + resp, err := t.invokeReceiver(ctx, *event) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte(fmt.Sprintf(`{"error":%q}`, err.Error()))) + return + } + if resp != nil { + if len(resp.Header) > 0 { + for k, vs := range resp.Header { + for _, v := range vs { + w.Header().Set(k, v) + } + } + } + if resp.StatusCode >= 200 && resp.StatusCode < 600 { + w.WriteHeader(resp.StatusCode) + } + if len(resp.Body) > 0 { + w.Write(resp.Body) + } + return } - // TODO: respond correctly based on decode. w.WriteHeader(http.StatusNoContent) } func (t *Transport) GetPort() int { - if t.Port > 0 { - return t.Port + if t.Port != nil && *t.Port == 0 && t.realPort != 0 { + return t.realPort + } + + if t.Port != nil && *t.Port >= 0 { // 0 means next open port + return *t.Port } return 8080 // default } + +func (t *Transport) GetPath() string { + path := strings.TrimSpace(t.Path) + if len(path) > 0 { + return path + } + return "/" // default +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/message.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/message.go index 77e4bee8aaa..bef74aa663c 100644 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/message.go +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/message.go @@ -6,3 +6,8 @@ type Message interface { // TODO maybe get encoding } + +type Response struct { + ResponseCode int + Body []byte +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/nats/codec.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/nats/codec.go deleted file mode 100644 index 56132f0464d..00000000000 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/nats/codec.go +++ /dev/null @@ -1,54 +0,0 @@ -package nats - -import ( - "fmt" - "github.com/cloudevents/sdk-go/pkg/cloudevents" - "github.com/cloudevents/sdk-go/pkg/cloudevents/transport" -) - -type Codec struct { - Encoding Encoding - - v02 *CodecV02 - v03 *CodecV03 -} - -var _ transport.Codec = (*Codec)(nil) - -func (c *Codec) Encode(e cloudevents.Event) (transport.Message, error) { - switch c.Encoding { - case Default: - fallthrough - case StructuredV02: - if c.v02 == nil { - c.v02 = &CodecV02{Encoding: c.Encoding} - } - return c.v02.Encode(e) - case StructuredV03: - if c.v03 == nil { - c.v03 = &CodecV03{Encoding: c.Encoding} - } - return c.v03.Encode(e) - default: - return nil, fmt.Errorf("unknown encoding: %d", c.Encoding) - } -} - -func (c *Codec) Decode(msg transport.Message) (*cloudevents.Event, error) { - switch c.Encoding { - case Default: - fallthrough - case StructuredV02: - if c.v02 == nil { - c.v02 = &CodecV02{Encoding: c.Encoding} - } - return c.v02.Decode(msg) - case StructuredV03: - if c.v03 == nil { - c.v03 = &CodecV03{Encoding: c.Encoding} - } - return c.v03.Decode(msg) - default: - return nil, fmt.Errorf("unknown encoding: %d", c.Encoding) - } -} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/nats/codec_v02.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/nats/codec_v02.go deleted file mode 100644 index 2b98f96366e..00000000000 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/nats/codec_v02.go +++ /dev/null @@ -1,48 +0,0 @@ -package nats - -import ( - "fmt" - "github.com/cloudevents/sdk-go/pkg/cloudevents" - "github.com/cloudevents/sdk-go/pkg/cloudevents/codec" - "github.com/cloudevents/sdk-go/pkg/cloudevents/transport" -) - -type CodecV02 struct { - Encoding Encoding -} - -var _ transport.Codec = (*CodecV02)(nil) - -func (v CodecV02) Encode(e cloudevents.Event) (transport.Message, error) { - switch v.Encoding { - case Default: - fallthrough - case StructuredV02: - return v.encodeStructured(e) - default: - return nil, fmt.Errorf("unknown encoding: %d", v.Encoding) - } -} - -func (v CodecV02) Decode(msg transport.Message) (*cloudevents.Event, error) { - // only structured is supported as of v0.2 - return v.decodeStructured(msg) -} - -func (v CodecV02) encodeStructured(e cloudevents.Event) (transport.Message, error) { - body, err := codec.JsonEncodeV02(e) - if err != nil { - return nil, err - } - return &Message{ - Body: body, - }, nil -} - -func (v CodecV02) decodeStructured(msg transport.Message) (*cloudevents.Event, error) { - m, ok := msg.(*Message) - if !ok { - return nil, fmt.Errorf("failed to convert transport.Message to http.Message") - } - return codec.JsonDecodeV02(m.Body) -} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/nats/codec_v03.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/nats/codec_v03.go deleted file mode 100644 index 38793583caf..00000000000 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/nats/codec_v03.go +++ /dev/null @@ -1,48 +0,0 @@ -package nats - -import ( - "fmt" - "github.com/cloudevents/sdk-go/pkg/cloudevents" - "github.com/cloudevents/sdk-go/pkg/cloudevents/codec" - "github.com/cloudevents/sdk-go/pkg/cloudevents/transport" -) - -type CodecV03 struct { - Encoding Encoding -} - -var _ transport.Codec = (*CodecV03)(nil) - -func (v CodecV03) Encode(e cloudevents.Event) (transport.Message, error) { - switch v.Encoding { - case Default: - fallthrough - case StructuredV03: - return v.encodeStructured(e) - default: - return nil, fmt.Errorf("unknown encoding: %d", v.Encoding) - } -} - -func (v CodecV03) Decode(msg transport.Message) (*cloudevents.Event, error) { - // only structured is supported as of v0.3 - return v.decodeStructured(msg) -} - -func (v CodecV03) encodeStructured(e cloudevents.Event) (transport.Message, error) { - body, err := codec.JsonEncodeV03(e) - if err != nil { - return nil, err - } - return &Message{ - Body: body, - }, nil -} - -func (v CodecV03) decodeStructured(msg transport.Message) (*cloudevents.Event, error) { - m, ok := msg.(*Message) - if !ok { - return nil, fmt.Errorf("failed to convert transport.Message to http.Message") - } - return codec.JsonDecodeV03(m.Body) -} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/nats/encoding.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/nats/encoding.go deleted file mode 100644 index 38bcfbdeeab..00000000000 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/nats/encoding.go +++ /dev/null @@ -1,45 +0,0 @@ -package nats - -type Encoding int32 - -const ( - Default Encoding = iota - StructuredV02 - StructuredV03 - Unknown -) - -func (e Encoding) String() string { - switch e { - case Default: - return "Default Encoding " + e.Version() - - // Structured - case StructuredV02: - fallthrough - case StructuredV03: - return "Structured Encoding " + e.Version() - - default: - return "Unknown Encoding" - } -} - -func (e Encoding) Version() string { - switch e { - - // Version 0.2 - case Default: // <-- Move when a new default is wanted. - fallthrough - case StructuredV02: - return "v0.2" - - // Version 0.3 - case StructuredV03: - return "v0.3" - - // Unknown - default: - return "Unknown" - } -} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/nats/message.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/nats/message.go deleted file mode 100644 index cc60ebca438..00000000000 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/nats/message.go +++ /dev/null @@ -1,31 +0,0 @@ -package nats - -import ( - "encoding/json" - "github.com/cloudevents/sdk-go/pkg/cloudevents/transport" -) - -// type check that this transport message impl matches the contract -var _ transport.Message = (*Message)(nil) - -type Message struct { - Body []byte -} - -func (m Message) CloudEventsVersion() string { - raw := make(map[string]json.RawMessage) - if err := json.Unmarshal(m.Body, &raw); err != nil { - return "" - } - - // v0.2 - if v, ok := raw["specversion"]; ok { - var version string - if err := json.Unmarshal(v, &version); err != nil { - return "" - } - return version - } - - return "" -} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/nats/transport.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/nats/transport.go deleted file mode 100644 index 6c0a0265791..00000000000 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/nats/transport.go +++ /dev/null @@ -1,91 +0,0 @@ -package nats - -import ( - "context" - "fmt" - "github.com/cloudevents/sdk-go/pkg/cloudevents" - "github.com/cloudevents/sdk-go/pkg/cloudevents/transport" - "github.com/nats-io/go-nats" - "log" -) - -// type check that this transport message impl matches the contract -var _ transport.Sender = (*Transport)(nil) - -// Transport acts as both a http client and a http handler. -type Transport struct { - Encoding Encoding - Conn *nats.Conn - Subject string - - sub *nats.Subscription - - Receiver transport.Receiver - - codec transport.Codec -} - -func (t *Transport) loadCodec() bool { - if t.codec == nil { - switch t.Encoding { - case Default: - t.codec = &Codec{} - case StructuredV02: - t.codec = &CodecV02{Encoding: t.Encoding} - case StructuredV03: - t.codec = &CodecV03{Encoding: t.Encoding} - default: - return false - } - } - return true -} - -func (t *Transport) Send(ctx context.Context, event cloudevents.Event) error { - if ok := t.loadCodec(); !ok { - return fmt.Errorf("unknown encoding set on transport: %d", t.Encoding) - } - - msg, err := t.codec.Encode(event) - if err != nil { - return err - } - - if m, ok := msg.(*Message); ok { - return t.Conn.Publish(t.Subject, m.Body) - } - - return fmt.Errorf("failed to encode Event into a Message") -} - -func (t *Transport) Listen(ctx context.Context) error { - if t.sub != nil { - return fmt.Errorf("already subscribed") - } - - if ok := t.loadCodec(); !ok { - return fmt.Errorf("unknown encoding set on transport: %d", t.Encoding) - } - - // TODO: there could be more than one subscription. Might have to do a map - // of subject to subscription. - - if t.Subject == "" { - return fmt.Errorf("subject required for nats listen") - } - - var err error - // Simple Async Subscriber - t.sub, err = t.Conn.Subscribe(t.Subject, func(m *nats.Msg) { - msg := &Message{ - Body: m.Data, - } - event, err := t.codec.Decode(msg) - if err != nil { - log.Printf("failed to decode message: %s", err) - return - } - t.Receiver.Receive(*event) - }) - return err -} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/transport.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/transport.go index d6f727bc54a..313e4d07b20 100644 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/transport.go +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/transport.go @@ -7,11 +7,13 @@ import ( // Transport is the interface for transport sender to send the converted Message // over the underlying transport. -type Sender interface { - Send(context.Context, cloudevents.Event) error +type Transport interface { + Send(context.Context, cloudevents.Event) (*cloudevents.Event, error) + + SetReceiver(Receiver) + StartReceiver(context.Context) error } -// Receiver TODO not sure yet. type Receiver interface { - Receive(cloudevents.Event) + Receive(context.Context, cloudevents.Event, *cloudevents.EventResponse) error } diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/types/timestamp.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/types/timestamp.go index b535e5dad6d..c87e24804d3 100644 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/types/timestamp.go +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/types/timestamp.go @@ -2,6 +2,7 @@ package types import ( "encoding/json" + "encoding/xml" "fmt" "time" ) @@ -26,7 +27,7 @@ func (t *Timestamp) MarshalJSON() ([]byte, error) { if t == nil || t.IsZero() { return []byte(`""`), nil } - rfc3339 := fmt.Sprintf("%q", t.Format(time.RFC3339Nano)) + rfc3339 := fmt.Sprintf("%q", t.UTC().Format(time.RFC3339Nano)) return []byte(rfc3339), nil } @@ -35,8 +36,26 @@ func (t *Timestamp) UnmarshalJSON(b []byte) error { if err := json.Unmarshal(b, ×tamp); err != nil { return err } - pt := ParseTimestamp(timestamp) - if pt != nil { + if pt := ParseTimestamp(timestamp); pt != nil { + *t = *pt + } + return nil +} + +func (t *Timestamp) MarshalXML(e *xml.Encoder, start xml.StartElement) error { + if t == nil || t.IsZero() { + return e.EncodeElement(nil, start) + } + v := t.UTC().Format(time.RFC3339Nano) + return e.EncodeElement(v, start) +} + +func (t *Timestamp) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { + var timestamp string + if err := d.DecodeElement(×tamp, &start); err != nil { + return err + } + if pt := ParseTimestamp(timestamp); pt != nil { *t = *pt } return nil @@ -44,8 +63,8 @@ func (t *Timestamp) UnmarshalJSON(b []byte) error { func (t *Timestamp) String() string { if t == nil { - return time.Time{}.Format(time.RFC3339Nano) + return time.Time{}.UTC().Format(time.RFC3339Nano) } - return t.Format(time.RFC3339Nano) + return t.UTC().Format(time.RFC3339Nano) } diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/types/urlref.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/types/urlref.go index 0898e4ece8e..f79fafa0860 100644 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/types/urlref.go +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/types/urlref.go @@ -2,6 +2,7 @@ package types import ( "encoding/json" + "encoding/xml" "fmt" "net/url" ) @@ -32,7 +33,27 @@ func (u *URLRef) UnmarshalJSON(b []byte) error { if err := json.Unmarshal(b, &ref); err != nil { return err } - *u = *ParseURLRef(ref) + r := ParseURLRef(ref) + if r != nil { + *u = *r + } + return nil +} + +func (u URLRef) MarshalXML(e *xml.Encoder, start xml.StartElement) error { + v := fmt.Sprintf("%s", u.String()) + return e.EncodeElement(v, start) +} + +func (u *URLRef) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { + var ref string + if err := d.DecodeElement(&ref, &start); err != nil { + return err + } + r := ParseURLRef(ref) + if r != nil { + *u = *r + } return nil } From 0bdac3501a4fffba18388201d0784d06aa26c055 Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Mon, 18 Mar 2019 15:11:11 -0700 Subject: [PATCH 113/128] Move to the newer, idiomatic CloudEvents library useage. --- Gopkg.lock | 4 +- cmd/broker/filter/main.go | 5 +- pkg/broker/receiver.go | 251 +++++------------- .../sdk-go/pkg/cloudevents/client/client.go | 141 ++++++++++ .../pkg/cloudevents/client/defaulters.go | 62 +++++ .../sdk-go/pkg/cloudevents/client/options.go | 36 +++ .../sdk-go/pkg/cloudevents/client/receiver.go | 187 +++++++++++++ 7 files changed, 503 insertions(+), 183 deletions(-) create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/client.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/defaulters.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/options.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/receiver.go diff --git a/Gopkg.lock b/Gopkg.lock index 6296dbdec50..2d676f35470 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -58,10 +58,11 @@ version = "v2.1.15" [[projects]] - digest = "1:6a76ae82f291e0188bd9471cdb4a8cd1eb5c8a8e7be46a248d31d6b4a45dc973" + digest = "1:70a8e5f09e19aba14064648751abc86f20484e25476866b03ce711007e96f339" name = "github.com/cloudevents/sdk-go" packages = [ "pkg/cloudevents", + "pkg/cloudevents/client", "pkg/cloudevents/codec", "pkg/cloudevents/context", "pkg/cloudevents/datacodec", @@ -1255,6 +1256,7 @@ "github.com/Shopify/sarama", "github.com/bsm/sarama-cluster", "github.com/cloudevents/sdk-go/pkg/cloudevents", + "github.com/cloudevents/sdk-go/pkg/cloudevents/client", "github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http", "github.com/fsnotify/fsnotify", "github.com/google/go-cmp/cmp", diff --git a/cmd/broker/filter/main.go b/cmd/broker/filter/main.go index 23f680c11e0..9f03cc9367a 100644 --- a/cmd/broker/filter/main.go +++ b/cmd/broker/filter/main.go @@ -58,7 +58,10 @@ func main() { // We are running both the receiver (takes messages in from the Broker) and the dispatcher (send // the messages to the triggers' subscribers) in this binary. - receiver := broker.New(logger, mgr.GetClient()) + receiver, err := broker.New(logger, mgr.GetClient()) + if err != nil { + logger.Fatal("Error creating Receiver", zap.Error(err)) + } err = mgr.Add(receiver) if err != nil { logger.Fatal("Unable to start the receiver", zap.Error(err), zap.Any("receiver", receiver)) diff --git a/pkg/broker/receiver.go b/pkg/broker/receiver.go index a7e5688afbd..9381b35d869 100644 --- a/pkg/broker/receiver.go +++ b/pkg/broker/receiver.go @@ -17,17 +17,15 @@ package broker import ( - "bytes" "context" "errors" "fmt" - "io" - "io/ioutil" "net/http" "net/url" - "reflect" "github.com/cloudevents/sdk-go/pkg/cloudevents" + ceclient "github.com/cloudevents/sdk-go/pkg/cloudevents/client" + cecontext "github.com/cloudevents/sdk-go/pkg/cloudevents/context" cehttp "github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http" eventingv1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" "github.com/knative/eventing/pkg/provisioners" @@ -42,40 +40,50 @@ const ( // Receiver parses Cloud Events, determines if they pass a filter, and sends them to a subscriber. type Receiver struct { - logger *zap.Logger - client client.Client - - port int - - httpClient HTTPDoer - codec cehttp.Codec -} - -type HTTPDoer interface { - Do(*http.Request) (*http.Response, error) + logger *zap.Logger + client client.Client + ceClient ceclient.Client + ceHttp *cehttp.Transport } -var _ HTTPDoer = &http.Client{} - // New creates a new Receiver and its associated MessageReceiver. The caller is responsible for // Start()ing the returned MessageReceiver. -func New(logger *zap.Logger, client client.Client) *Receiver { +func New(logger *zap.Logger, client client.Client) (*Receiver, error) { + ceHttp, err := cehttp.New(cehttp.WithBinaryEncoding(), cehttp.WithPort(defaultPort)) + if err != nil { + return nil, err + } + ceClient, err := ceclient.New(ceHttp) + if err != nil { + return nil, err + } + r := &Receiver{ - logger: logger, - client: client, + logger: logger, + client: client, + ceClient: ceClient, + ceHttp: ceHttp, + } + err = r.initClient() + if err != nil { + return nil, err + } - port: defaultPort, + return r, nil +} - httpClient: &http.Client{}, - codec: cehttp.Codec{ - Encoding: cehttp.BinaryV01, - }, +// Initialize the client. Mainly intended to load stuff in its cache. +func (r *Receiver) initClient() error { + // We list triggers so that we do not drop messages. Otherwise, on receiving an event, it + // may not find the Trigger and would return an error. + opts := &client.ListOptions{} + tl := &eventingv1alpha1.TriggerList{} + if err := r.client.List(context.TODO(), opts, tl); err != nil { + return err } - return r + return nil } -var _ http.Handler = &Receiver{} - // Start begins to receive messages for the receiver. // // Only HTTP POST requests to the root path (/) are accepted. If other paths or @@ -84,126 +92,58 @@ var _ http.Handler = &Receiver{} // // This method will block until a message is received on the stop channel. func (r *Receiver) Start(stopCh <-chan struct{}) error { - svr := r.start() - defer r.stop(svr) - - <-stopCh - return nil -} + ctx := context.Background() + defer ctx.Done() -func (r *Receiver) start() *http.Server { - srv := &http.Server{ - Addr: fmt.Sprintf(":%d", r.port), - Handler: r, - } - r.logger.Info("Starting web server", zap.String("addr", srv.Addr)) + errCh := make(chan error, 1) go func() { - if err := srv.ListenAndServe(); err != http.ErrServerClosed { - r.logger.Error("HttpServer: ListenAndServe() error", zap.Error(err)) - } + errCh <- r.ceClient.StartReceiver(ctx, r.serveHTTP) }() - return srv -} -func (r *Receiver) stop(srv *http.Server) { - r.logger.Info("Shutdown web server") - if err := srv.Shutdown(nil); err != nil { - r.logger.Error("Error shutting down the HTTP Server", zap.Error(err)) + select { + case err := <-errCh: + return err + case <-stopCh: + return nil } } -func (r *Receiver) ServeHTTP(w http.ResponseWriter, req *http.Request) { - if req.URL.Path != "/" { - w.WriteHeader(http.StatusNotFound) - return - } - if req.Method != http.MethodPost { - w.WriteHeader(http.StatusMethodNotAllowed) - return +func (r *Receiver) serveHTTP(ctx context.Context, event cloudevents.Event, resp *cloudevents.EventResponse) error { + tctx := cehttp.TransportContextFrom(ctx) + if tctx.Method != http.MethodPost { + resp.Status = http.StatusMethodNotAllowed + return nil } - triggerRef, err := provisioners.ParseChannel(req.Host) + uri, err := url.Parse(tctx.URI) if err != nil { - r.logger.Error("Unable to parse host as a trigger", zap.Error(err), zap.String("host", req.Host)) - w.WriteHeader(http.StatusBadRequest) - w.Write([]byte(`{"error":"Bad Host"}`)) - return + return fmt.Errorf("unable to parse URI: %v", err) } - - event, err := r.decodeHTTPRequest(req) - if err != nil { - r.logger.Error("Error decoding HTTP Request", zap.Error(err)) - w.WriteHeader(http.StatusBadRequest) - return + if uri.Path != "/" { + resp.Status = http.StatusNotFound + return nil } - responseEvent, err := r.sendEvent(triggerRef, event) + triggerRef, err := provisioners.ParseChannel(uri.Host) if err != nil { - r.logger.Error("Error sending the event", zap.Error(err)) - w.WriteHeader(http.StatusBadRequest) - return - } - if responseEvent == nil { - w.WriteHeader(http.StatusAccepted) - return + r.logger.Error("Unable to parse host as a trigger", zap.Error(err), zap.String("host", uri.Host)) + return errors.New("unable to parse host as a Trigger") } - encodedEvent, err := r.codec.Encode(*event) - if err != nil { - r.logger.Error("Error encoding the response event", zap.Error(err)) - w.WriteHeader(http.StatusInternalServerError) - return - } - msg, ok := encodedEvent.(*cehttp.Message) - if !ok { - r.logger.Error("Error casting the encoded response event", zap.Error(err), zap.Any("encodedEvent", reflect.TypeOf(encodedEvent))) - w.WriteHeader(http.StatusInternalServerError) - return - } - for n, v := range msg.Header { - w.Header().Del(n) - for _, s := range v { - w.Header().Add(n, s) - } - } - w.WriteHeader(http.StatusAccepted) - _, err = w.Write(msg.Body) - if err != nil { - r.logger.Error("Error writing the response body", zap.Error(err)) - w.WriteHeader(http.StatusInternalServerError) - return - } -} - -func (r *Receiver) decodeHTTPRequest(req *http.Request) (*cloudevents.Event, error) { - return r.decodeHTTP(req.Header, req.Body) -} + r.logger.Debug("Received message", zap.Any("triggerRef", triggerRef)) -func (r *Receiver) decodeHTTPResponse(resp *http.Response) (*cloudevents.Event, error) { - // The HTTP Response could be anything, so just assume that if it does not parse as a - // CloudEvent, then it isn't a CloudEvent. - e, _ := r.decodeHTTP(resp.Header, resp.Body) - return e, nil -} - -func (r *Receiver) decodeHTTP(headers http.Header, bodyReadCloser io.ReadCloser) (*cloudevents.Event, error) { - body, err := ioutil.ReadAll(bodyReadCloser) + responseEvent, err := r.sendEvent(ctx, triggerRef, &event) if err != nil { - return nil, err - } - msg := &cehttp.Message{ - Header: headers, - Body: body, + r.logger.Error("Error sending the event", zap.Error(err)) + return err } - - return r.codec.Decode(msg) + resp.Status = http.StatusAccepted + resp.Event = responseEvent + return nil } // sendEvent sends an event to a subscriber if the trigger filter passes. -func (r *Receiver) sendEvent(trigger provisioners.ChannelReference, event *cloudevents.Event) (*cloudevents.Event, error) { - r.logger.Debug("Received message", zap.Any("triggerRef", trigger)) - ctx := context.Background() - +func (r *Receiver) sendEvent(ctx context.Context, trigger provisioners.ChannelReference, event *cloudevents.Event) (*cloudevents.Event, error) { t, err := r.getTrigger(ctx, trigger) if err != nil { r.logger.Info("Unable to get the Trigger", zap.Error(err), zap.Any("triggerRef", trigger)) @@ -215,6 +155,8 @@ func (r *Receiver) sendEvent(trigger provisioners.ChannelReference, event *cloud r.logger.Error("Unable to read subscriberURI") return nil, errors.New("unable to read subscriberURI") } + // We could just send the request to this URI regardless, but let's just check to see if it well + // formed first, that way we can generate better error message if it isn't. subscriberURI, err := url.Parse(subscriberURIString) if err != nil { r.logger.Error("Unable to parse subscriberURI", zap.Error(err), zap.String("subscriberURIString", subscriberURIString)) @@ -226,60 +168,8 @@ func (r *Receiver) sendEvent(trigger provisioners.ChannelReference, event *cloud return nil, nil } - return r.dispatch(event, subscriberURI) -} - -func (r *Receiver) dispatch(event *cloudevents.Event, uri *url.URL) (*cloudevents.Event, error) { - encodedEvent, err := r.codec.Encode(*event) - if err != nil { - return nil, err - } - msg, ok := encodedEvent.(*cehttp.Message) - if !ok { - return nil, errors.New("msg was not a cehttp.Message") - } - - req, err := http.NewRequest(http.MethodPost, uri.String(), bytes.NewReader(msg.Body)) - if err != nil { - return nil, err - } - - req.Header = msg.Header - res, err := r.httpClient.Do(req) - if err != nil { - return nil, err - } - if res == nil { - // I don't think this is actually reachable with http.Client.Do(), but just to be sure we - // check anyway. - return nil, errors.New("non-error nil result from http.Client.Do()") - } - - defer res.Body.Close() - if isFailure(res.StatusCode) { - // reject non-successful responses - return nil, fmt.Errorf("unexpected HTTP response, expected 2xx, got %d", res.StatusCode) - } - - return r.decodeHTTPResponse(res) -} - -// isFailure returns true if the status code is not a successful HTTP status. -func isFailure(statusCode int) bool { - return statusCode < http.StatusOK /* 200 */ || - statusCode >= http.StatusMultipleChoices /* 300 */ -} - -// Initialize the client. Mainly intended to load stuff in its cache. -func (r *Receiver) initClient() error { - // We list triggers so that we do not drop messages. Otherwise, on receiving an event, it - // may not find the Trigger and would return an error. - opts := &client.ListOptions{} - tl := &eventingv1alpha1.TriggerList{} - if err := r.client.List(context.TODO(), opts, tl); err != nil { - return err - } - return nil + sendingCtx := cecontext.WithTarget(ctx, subscriberURI.String()) + return r.ceHttp.Send(sendingCtx, *event) } func (r *Receiver) getTrigger(ctx context.Context, ref provisioners.ChannelReference) (*eventingv1alpha1.Trigger, error) { @@ -308,7 +198,6 @@ func (r *Receiver) shouldSendMessage(ts *eventingv1alpha1.TriggerSpec, event *cl filterSource := ts.Filter.SourceAndType.Source s := event.Context.AsV01().Source actualSource := s.String() - //actualSource := event.Context.AsV01().Source.String() if filterSource != eventingv1alpha1.TriggerAnyFilter && filterSource != actualSource { r.logger.Debug("Wrong source", zap.String("trigger.spec.filter.sourceAndType.source", filterSource), zap.String("message.source", actualSource)) return false diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/client.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/client.go new file mode 100644 index 00000000000..d9c948cd7a4 --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/client.go @@ -0,0 +1,141 @@ +package client + +import ( + "context" + "fmt" + "github.com/cloudevents/sdk-go/pkg/cloudevents" + "github.com/cloudevents/sdk-go/pkg/cloudevents/transport" + "github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http" + "sync" +) + +type Client interface { + // Send will transmit the given event over the client's configured transport. + Send(ctx context.Context, event cloudevents.Event) (*cloudevents.Event, error) + + // StartReceiver will register the provided function for callback on receipt + // of a cloudevent. It will also start the underlying transport as it has + // been configured. + // This call is blocking. + // Valid fn signatures are: + // * func() + // * func() error + // * func(context.Context) + // * func(context.Context) error + // * func(cloudevents.Event) + // * func(cloudevents.Event) error + // * func(context.Context, cloudevents.Event) + // * func(context.Context, cloudevents.Event) error + // * func(cloudevents.Event, *cloudevents.EventResponse) + // * func(cloudevents.Event, *cloudevents.EventResponse) error + // * func(context.Context, cloudevents.Event, *cloudevents.EventResponse) + // * func(context.Context, cloudevents.Event, *cloudevents.EventResponse) error + // Note: if fn returns an error, it is treated as a critical and + // EventResponse will not be processed. + StartReceiver(ctx context.Context, fn interface{}) error +} + +func New(t transport.Transport, opts ...Option) (Client, error) { + c := &ceClient{ + transport: t, + } + if err := c.applyOptions(opts...); err != nil { + return nil, err + } + t.SetReceiver(c) + return c, nil +} + +// NewDefault provides the good defaults for the common case using an HTTP +// Transport client. +func NewDefault() (Client, error) { + t, err := http.New(http.WithBinaryEncoding()) + if err != nil { + return nil, err + } + c, err := New(t, WithTimeNow(), WithUUIDs()) + if err != nil { + return nil, err + } + return c, nil +} + +type ceClient struct { + transport transport.Transport + fn *receiverFn + + receiverMu sync.Mutex + eventDefaulterFns []EventDefaulter +} + +func (c *ceClient) Send(ctx context.Context, event cloudevents.Event) (*cloudevents.Event, error) { + // Confirm we have a transport set. + if c.transport == nil { + return nil, fmt.Errorf("client not ready, transport not initialized") + } + // Apply the defaulter chain to the incoming event. + if len(c.eventDefaulterFns) > 0 { + for _, fn := range c.eventDefaulterFns { + event = fn(event) + } + } + // Validate the event conforms to the CloudEvents Spec. + if err := event.Validate(); err != nil { + return nil, err + } + // Send the event over the transport. + return c.transport.Send(ctx, event) +} + +// Receive is called from from the transport on event delivery. +func (c *ceClient) Receive(ctx context.Context, event cloudevents.Event, resp *cloudevents.EventResponse) error { + if c.fn != nil { + err := c.fn.invoke(ctx, event, resp) + // Apply the defaulter chain to the outgoing event. + if err == nil && resp != nil && resp.Event != nil && len(c.eventDefaulterFns) > 0 { + for _, fn := range c.eventDefaulterFns { + *resp.Event = fn(*resp.Event) + } + // Validate the event conforms to the CloudEvents Spec. + if err := resp.Event.Validate(); err != nil { + return fmt.Errorf("cloudevent validation failed on response event: %v", err) + } + } + return err + } + return nil +} + +// Blocking Call +func (c *ceClient) StartReceiver(ctx context.Context, fn interface{}) error { + c.receiverMu.Lock() + defer c.receiverMu.Unlock() + + if c.transport == nil { + return fmt.Errorf("client not ready, transport not initialized") + } + if c.fn != nil { + return fmt.Errorf("client already has a receiver") + } + + if fn, err := receiver(fn); err != nil { + return err + } else { + c.fn = fn + } + + defer func() { + c.fn = nil + }() + + return c.transport.StartReceiver(ctx) +} + +func (c *ceClient) applyOptions(opts ...Option) error { + for _, fn := range opts { + if err := fn(c); err != nil { + return err + } + } + return nil +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/defaulters.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/defaulters.go new file mode 100644 index 00000000000..ee87e0467a1 --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/defaulters.go @@ -0,0 +1,62 @@ +package client + +import ( + "github.com/cloudevents/sdk-go/pkg/cloudevents" + "github.com/cloudevents/sdk-go/pkg/cloudevents/types" + "github.com/google/uuid" + "time" +) + +type EventDefaulter func(event cloudevents.Event) cloudevents.Event + +func DefaultIDToUUIDIfNotSet(event cloudevents.Event) cloudevents.Event { + if event.Context != nil { + switch event.Context.GetSpecVersion() { + case cloudevents.CloudEventsVersionV01: + ec := event.Context.AsV01() + if ec.EventID == "" { + ec.EventID = uuid.New().String() + event.Context = ec + } + case cloudevents.CloudEventsVersionV02: + ec := event.Context.AsV02() + if ec.ID == "" { + ec.ID = uuid.New().String() + event.Context = ec + } + case cloudevents.CloudEventsVersionV03: + ec := event.Context.AsV03() + if ec.ID == "" { + ec.ID = uuid.New().String() + event.Context = ec + } + } + } + return event +} + +func DefaultTimeToNowIfNotSet(event cloudevents.Event) cloudevents.Event { + if event.Context != nil { + switch event.Context.GetSpecVersion() { + case cloudevents.CloudEventsVersionV01: + ec := event.Context.AsV01() + if ec.EventTime == nil || ec.EventTime.IsZero() { + ec.EventTime = &types.Timestamp{Time: time.Now()} + event.Context = ec + } + case cloudevents.CloudEventsVersionV02: + ec := event.Context.AsV02() + if ec.Time == nil || ec.Time.IsZero() { + ec.Time = &types.Timestamp{Time: time.Now()} + event.Context = ec + } + case cloudevents.CloudEventsVersionV03: + ec := event.Context.AsV03() + if ec.Time == nil || ec.Time.IsZero() { + ec.Time = &types.Timestamp{Time: time.Now()} + event.Context = ec + } + } + } + return event +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/options.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/options.go new file mode 100644 index 00000000000..09631d0835a --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/options.go @@ -0,0 +1,36 @@ +package client + +import ( + "fmt" +) + +type Option func(*ceClient) error + +// WithEventDefaulter adds an event defaulter to the end of the defaulter chain. +func WithEventDefaulter(fn EventDefaulter) Option { + return func(c *ceClient) error { + if fn == nil { + return fmt.Errorf("client option was given an nil event defaulter") + } + c.eventDefaulterFns = append(c.eventDefaulterFns, fn) + return nil + } +} + +// WithUUIDs adds DefaultIDToUUIDIfNotSet event defaulter to the end of the +// defaulter chain. +func WithUUIDs() Option { + return func(c *ceClient) error { + c.eventDefaulterFns = append(c.eventDefaulterFns, DefaultIDToUUIDIfNotSet) + return nil + } +} + +// WithTimeNow adds DefaultTimeToNowIfNotSet event defaulter to the end of the +// defaulter chain. +func WithTimeNow() Option { + return func(c *ceClient) error { + c.eventDefaulterFns = append(c.eventDefaulterFns, DefaultTimeToNowIfNotSet) + return nil + } +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/receiver.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/receiver.go new file mode 100644 index 00000000000..6264a3f79d0 --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/receiver.go @@ -0,0 +1,187 @@ +package client + +import ( + "context" + "errors" + "fmt" + "github.com/cloudevents/sdk-go/pkg/cloudevents" + "reflect" +) + +// Receive is the signature of a fn to be invoked for incoming cloudevents. +// If fn returns an error, EventResponse will not be considered by the client or +// or transport. +// This is just an FYI: +type ReceiveFull func(context.Context, cloudevents.Event, *cloudevents.EventResponse) error + +type receiverFn struct { + numIn int + fnValue reflect.Value + + hasContextIn bool + hasEventIn bool + hasEventResponseIn bool + + hasErrorOut bool +} + +const ( + inParamUsage = "expected a function taking either no parameters, one or more of (context.Context, cloudevents.Event, *cloudevents.EventResponse) ordered" + outParamUsage = "expected a function returning either nothing or an error" +) + +var ( + contextType = reflect.TypeOf((*context.Context)(nil)).Elem() + eventType = reflect.TypeOf((*cloudevents.Event)(nil)).Elem() + eventResponseType = reflect.TypeOf((*cloudevents.EventResponse)(nil)) // want the ptr type + errorType = reflect.TypeOf((*error)(nil)).Elem() +) + +// receiver creates a receiverFn wrapper class that is used by the client to +// validate and invoke the provided function. +// Valid fn signatures are: +// * func() +// * func() error +// * func(context.Context) +// * func(context.Context) error +// * func(cloudevents.Event) +// * func(cloudevents.Event) error +// * func(context.Context, cloudevents.Event) +// * func(context.Context, cloudevents.Event) error +// * func(cloudevents.Event, *cloudevents.EventResponse) +// * func(cloudevents.Event, *cloudevents.EventResponse) error +// * func(context.Context, cloudevents.Event, *cloudevents.EventResponse) +// * func(context.Context, cloudevents.Event, *cloudevents.EventResponse) error +// +func receiver(fn interface{}) (*receiverFn, error) { + fnType := reflect.TypeOf(fn) + if fnType.Kind() != reflect.Func { + return nil, errors.New("must pass a function to handle events") + } + + r := &receiverFn{ + fnValue: reflect.ValueOf(fn), + numIn: fnType.NumIn(), + } + if err := r.validate(fnType); err != nil { + return nil, err + } + + return r, nil +} + +func (r *receiverFn) invoke(ctx context.Context, event cloudevents.Event, resp *cloudevents.EventResponse) error { + args := make([]reflect.Value, 0, r.numIn) + + if r.numIn > 0 { + if r.hasContextIn { + args = append(args, reflect.ValueOf(ctx)) + } + if r.hasEventIn { + args = append(args, reflect.ValueOf(event)) + } + if r.hasEventResponseIn { + args = append(args, reflect.ValueOf(resp)) + } + } + v := r.fnValue.Call(args) + if r.hasErrorOut && len(v) >= 1 { + if err, ok := v[0].Interface().(error); ok { + return err + } + } + return nil +} + +// Verifies that the inputs to a function have a valid signature +// Valid input is to be [0, all] of +// context.Context, cloudevents.Event, *cloudevents.EventResponse in this order. +func (r *receiverFn) validateInParamSignature(fnType reflect.Type) error { + r.hasContextIn = false + r.hasEventIn = false + r.hasEventResponseIn = false + + switch fnType.NumIn() { + case 3: + // has to be cloudevents.Event, *cloudevents.EventResponse + if !fnType.In(2).ConvertibleTo(eventResponseType) { + return fmt.Errorf("%s; cannot convert parameter 2 from %s to *cloudevents.EventResponse", inParamUsage, fnType.In(2)) + } else { + r.hasEventResponseIn = true + } + fallthrough + case 2: + // can be cloudevents.Event or *cloudevents.EventResponse + if !fnType.In(1).ConvertibleTo(eventResponseType) { + if !fnType.In(1).ConvertibleTo(eventType) { + return fmt.Errorf("%s; cannot convert parameter 1 from %s to cloudevents.Event or *cloudevents.EventResponse", inParamUsage, fnType.In(1)) + } else { + r.hasEventIn = true + } + } else if r.hasEventResponseIn { + return fmt.Errorf("%s; duplicate parameter of type *cloudevents.EventResponse", inParamUsage) + } else { + r.hasEventResponseIn = true + } + fallthrough + case 1: + if !fnType.In(0).ConvertibleTo(contextType) { + if !fnType.In(0).ConvertibleTo(eventResponseType) { + if !fnType.In(0).ConvertibleTo(eventType) { + return fmt.Errorf("%s; cannot convert parameter 0 from %s to context.Context, cloudevents.Event or *cloudevents.EventResponse", inParamUsage, fnType.In(0)) + } else if r.hasEventIn { + return fmt.Errorf("%s; duplicate parameter of type cloudevents.Event", inParamUsage) + } else { + r.hasEventIn = true + } + } else if r.hasEventResponseIn { + return fmt.Errorf("%s; duplicate parameter of type *cloudevents.EventResponse", inParamUsage) + } else if r.hasEventIn { + return fmt.Errorf("%s; out of order parameter 0 for %s", inParamUsage, fnType.In(1)) + } else { + r.hasEventResponseIn = true + } + } else { + r.hasContextIn = true + } + fallthrough + case 0: + return nil + default: + return fmt.Errorf("%s; function has too many parameters (%d)", inParamUsage, fnType.NumIn()) + } +} + +// Verifies that the outputs of a function have a valid signature +// Valid output signatures: +// (), (error) +func (r *receiverFn) validateOutParamSignature(fnType reflect.Type) error { + r.hasErrorOut = false + switch fnType.NumOut() { + case 1: + paramNo := fnType.NumOut() - 1 + paramType := fnType.Out(paramNo) + if !paramType.ConvertibleTo(errorType) { + return fmt.Errorf("%s; cannot convert return type %d from %s to error", outParamUsage, paramNo, paramType) + } else { + r.hasErrorOut = true + } + fallthrough + case 0: + return nil + default: + return fmt.Errorf("%s; function has too many return types (%d)", outParamUsage, fnType.NumOut()) + } +} + +// validateReceiverFn validates that a function has the right number of in and +// out params and that they are of allowed types. +func (r *receiverFn) validate(fnType reflect.Type) error { + if err := r.validateInParamSignature(fnType); err != nil { + return err + } + if err := r.validateOutParamSignature(fnType); err != nil { + return err + } + return nil +} From 2e500603fe3efcc87847aff4e47413e9fde36061 Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Mon, 18 Mar 2019 15:26:04 -0700 Subject: [PATCH 114/128] tctx.URI is actually just the path... --- pkg/broker/receiver.go | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/pkg/broker/receiver.go b/pkg/broker/receiver.go index 9381b35d869..d6ee5b528c0 100644 --- a/pkg/broker/receiver.go +++ b/pkg/broker/receiver.go @@ -19,7 +19,6 @@ package broker import ( "context" "errors" - "fmt" "net/http" "net/url" @@ -115,18 +114,15 @@ func (r *Receiver) serveHTTP(ctx context.Context, event cloudevents.Event, resp return nil } - uri, err := url.Parse(tctx.URI) - if err != nil { - return fmt.Errorf("unable to parse URI: %v", err) - } - if uri.Path != "/" { + // tctx.URI is actually the path... + if tctx.URI != "/" { resp.Status = http.StatusNotFound return nil } - triggerRef, err := provisioners.ParseChannel(uri.Host) + triggerRef, err := provisioners.ParseChannel(tctx.Host) if err != nil { - r.logger.Error("Unable to parse host as a trigger", zap.Error(err), zap.String("host", uri.Host)) + r.logger.Error("Unable to parse host as a trigger", zap.Error(err), zap.String("host", tctx.Host)) return errors.New("unable to parse host as a Trigger") } From f27c63a99c3caff33446fb11576a5984edabc98c Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Mon, 18 Mar 2019 15:59:52 -0700 Subject: [PATCH 115/128] Partial update to the unit tests. --- pkg/broker/receiver_test.go | 115 +++++++++++++++++++++++++----------- 1 file changed, 82 insertions(+), 33 deletions(-) diff --git a/pkg/broker/receiver_test.go b/pkg/broker/receiver_test.go index 07f79bed903..c77bff44780 100644 --- a/pkg/broker/receiver_test.go +++ b/pkg/broker/receiver_test.go @@ -18,13 +18,19 @@ package broker import ( "bytes" + "context" "fmt" "io/ioutil" "net/http" - "net/http/httptest" - "strings" + "net/url" "testing" + "k8s.io/apimachinery/pkg/api/equality" + + "github.com/cloudevents/sdk-go/pkg/cloudevents" + "github.com/cloudevents/sdk-go/pkg/cloudevents/types" + + cehttp "github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http" eventingv1alpha1 "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" "github.com/knative/eventing/pkg/utils" "go.uber.org/zap" @@ -41,18 +47,41 @@ const ( eventSource = `/mycontext` ) +var ( + host = fmt.Sprintf("%s.%s.triggers.%s", triggerName, testNS, utils.GetClusterDomainName()) +) + func init() { // Add types to scheme. - eventingv1alpha1.AddToScheme(scheme.Scheme) + _ = eventingv1alpha1.AddToScheme(scheme.Scheme) } func TestReceiver(t *testing.T) { testCases := map[string]struct { initialState []runtime.Object + tctx cehttp.TransportContext requestFails bool + returnedEvent cloudevents.Event expectedErr bool expectedDispatch bool + expectedStatus int }{ + "Not POST": { + tctx: cehttp.TransportContext{ + Method: "GET", + Host: host, + URI: "/", + }, + expectedStatus: http.StatusMethodNotAllowed, + }, + "Other path": { + tctx: cehttp.TransportContext{ + Method: "POST", + Host: host, + URI: "/someotherEndpoint", + }, + expectedStatus: http.StatusNotFound, + }, "Trigger.Get fails": { // No trigger exists, so the Get will fail. expectedErr: true, @@ -98,28 +127,41 @@ func TestReceiver(t *testing.T) { }, expectedDispatch: true, }, + "Returned Cloud Event": { + returnedEvent: makeDifferentEvent(), + }, } for n, tc := range testCases { t.Run(n, func(t *testing.T) { - mr := New( + r, err := New( zap.NewNop(), fake.NewFakeClient(tc.initialState...)) - fh := &fakeHTTPDoer{ - failRequest: tc.requestFails, + if err != nil { + t.Fatalf("Unable to create receiver: %v", err) } - mr.httpClient = fh - - resp := httptest.NewRecorder() - mr.ServeHTTP(resp, makeRequest()) - if tc.expectedErr { - if resp.Result().StatusCode >= 200 && resp.Result().StatusCode < 300 { - t.Errorf("Expected an error. Actual: %v", resp.Result()) - } - } else { - if resp.Result().StatusCode < 200 || resp.Result().StatusCode >= 300 { - t.Errorf("Expected success. Actual: %v", resp.Result()) - } + + // TODO either use a fake client here or make a fake HTTP server for the test. Either + // way, update the tc.expectedDispatch test to ensure the request was sent or not. + + r.ceHttp.Client = fakeClient{} + + ctx := context.Background() + resp := &cloudevents.EventResponse{} + err = r.serveHTTP(ctx, makeEvent(), resp) + + if tc.expectedErr && err == nil { + t.Errorf("Expected an error, received nil") + } else if !tc.expectedErr && err != nil { + t.Errorf("Expected no error, received %v", err) } + + if tc.expectedStatus != 0 && tc.expectedStatus != resp.Status { + t.Errorf("Unexpected status. Expected %v. Actual %v.", tc.expectedStatus, resp.Status) + } + if !equality.Semantic.DeepEqual(tc.returnedEvent, resp.Event) { + t.Errorf("Unexpected response event. Expected '%v'. Actual '%v'", tc.returnedEvent, resp.Event) + } + if tc.expectedDispatch != fh.requestReceived { t.Errorf("Incorrect dispatch. Expected %v, Actual %v", tc.expectedDispatch, fh.requestReceived) } @@ -180,21 +222,28 @@ func makeTriggerWithoutSubscriberURI() *eventingv1alpha1.Trigger { return t } -func makeRequest() *http.Request { - req := httptest.NewRequest("POST", "/", strings.NewReader(``)) - req.Host = fmt.Sprintf("%s.%s.triggers.%s", triggerName, testNS, utils.GetClusterDomainName()) - - eventAttributes := map[string]string{ - "CE-CloudEventsVersion": `0.1`, - "CE-EventType": eventType, - "CE-EventTypeVersion": `1.0`, - "CE-Source": eventSource, - "CE-EventID": `A234-1234-1234`, - "CE-EventTime": `2018-04-05T17:31:00Z`, - "Content-Type": "application/xml", +func makeEvent() cloudevents.Event { + return cloudevents.Event{ + Context: cloudevents.EventContextV02{ + Type: eventType, + Source: types.URLRef{ + URL: url.URL{ + Path: eventSource, + }, + }, + }, } - for k, v := range eventAttributes { - req.Header.Set(k, v) +} + +func makeDifferentEvent() cloudevents.Event { + return cloudevents.Event{ + Context: cloudevents.EventContextV02{ + Type: "some-other-type", + Source: types.URLRef{ + URL: url.URL{ + Path: eventSource, + }, + }, + }, } - return req } From 747e290f14af129ded270e3c24066a64421121ec Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Tue, 19 Mar 2019 12:20:56 -0700 Subject: [PATCH 116/128] Unit tests working. --- pkg/broker/receiver_test.go | 128 +++++++++++++++++++++++++----------- 1 file changed, 91 insertions(+), 37 deletions(-) diff --git a/pkg/broker/receiver_test.go b/pkg/broker/receiver_test.go index c77bff44780..ac4efafba84 100644 --- a/pkg/broker/receiver_test.go +++ b/pkg/broker/receiver_test.go @@ -17,15 +17,14 @@ package broker import ( - "bytes" "context" "fmt" - "io/ioutil" "net/http" + "net/http/httptest" "net/url" "testing" - "k8s.io/apimachinery/pkg/api/equality" + "github.com/google/go-cmp/cmp" "github.com/cloudevents/sdk-go/pkg/cloudevents" "github.com/cloudevents/sdk-go/pkg/cloudevents/types" @@ -58,16 +57,16 @@ func init() { func TestReceiver(t *testing.T) { testCases := map[string]struct { - initialState []runtime.Object - tctx cehttp.TransportContext + triggers []*eventingv1alpha1.Trigger + tctx *cehttp.TransportContext requestFails bool - returnedEvent cloudevents.Event + returnedEvent *cloudevents.Event expectedErr bool expectedDispatch bool expectedStatus int }{ "Not POST": { - tctx: cehttp.TransportContext{ + tctx: &cehttp.TransportContext{ Method: "GET", Host: host, URI: "/", @@ -75,7 +74,7 @@ func TestReceiver(t *testing.T) { expectedStatus: http.StatusMethodNotAllowed, }, "Other path": { - tctx: cehttp.TransportContext{ + tctx: &cehttp.TransportContext{ Method: "POST", Host: host, URI: "/someotherEndpoint", @@ -87,28 +86,28 @@ func TestReceiver(t *testing.T) { expectedErr: true, }, "Trigger doesn't have SubscriberURI": { - initialState: []runtime.Object{ + triggers: []*eventingv1alpha1.Trigger{ makeTriggerWithoutSubscriberURI(), }, expectedErr: true, }, "Trigger without a Filter": { - initialState: []runtime.Object{ + triggers: []*eventingv1alpha1.Trigger{ makeTriggerWithoutFilter(), }, }, "Wrong type": { - initialState: []runtime.Object{ + triggers: []*eventingv1alpha1.Trigger{ makeTrigger("some-other-type", "Any"), }, }, "Wrong source": { - initialState: []runtime.Object{ + triggers: []*eventingv1alpha1.Trigger{ makeTrigger("Any", "some-other-source"), }, }, "Dispatch failed": { - initialState: []runtime.Object{ + triggers: []*eventingv1alpha1.Trigger{ makeTrigger("Any", "Any"), }, requestFails: true, @@ -116,36 +115,60 @@ func TestReceiver(t *testing.T) { expectedDispatch: true, }, "Dispatch succeeded - Any": { - initialState: []runtime.Object{ + triggers: []*eventingv1alpha1.Trigger{ makeTrigger("Any", "Any"), }, expectedDispatch: true, }, "Dispatch succeeded - Specific": { - initialState: []runtime.Object{ + triggers: []*eventingv1alpha1.Trigger{ makeTrigger(eventType, eventSource), }, expectedDispatch: true, }, "Returned Cloud Event": { - returnedEvent: makeDifferentEvent(), + triggers: []*eventingv1alpha1.Trigger{ + makeTrigger("Any", "Any"), + }, + expectedDispatch: true, + returnedEvent: makeDifferentEvent(), }, } for n, tc := range testCases { t.Run(n, func(t *testing.T) { + + fh := fakeHandler{ + failRequest: tc.requestFails, + returnedEvent: tc.returnedEvent, + } + s := httptest.NewServer(&fh) + defer s.Client() + + // Replace the SubscriberURI to point at our fake server. + correctURI := make([]runtime.Object, 0, len(tc.triggers)) + for _, trig := range tc.triggers { + if trig.Status.SubscriberURI != "" { + trig.Status.SubscriberURI = s.URL + } + correctURI = append(correctURI, trig) + } + r, err := New( zap.NewNop(), - fake.NewFakeClient(tc.initialState...)) + fake.NewFakeClient(correctURI...)) if err != nil { t.Fatalf("Unable to create receiver: %v", err) } - // TODO either use a fake client here or make a fake HTTP server for the test. Either - // way, update the tc.expectedDispatch test to ensure the request was sent or not. - - r.ceHttp.Client = fakeClient{} - - ctx := context.Background() + tctx := tc.tctx + if tctx == nil { + tctx = &cehttp.TransportContext{ + Method: http.MethodPost, + Host: host, + URI: "/", + } + } + ctx := cehttp.WithTransportContext(context.Background(), *tctx) resp := &cloudevents.EventResponse{} err = r.serveHTTP(ctx, makeEvent(), resp) @@ -158,32 +181,61 @@ func TestReceiver(t *testing.T) { if tc.expectedStatus != 0 && tc.expectedStatus != resp.Status { t.Errorf("Unexpected status. Expected %v. Actual %v.", tc.expectedStatus, resp.Status) } - if !equality.Semantic.DeepEqual(tc.returnedEvent, resp.Event) { - t.Errorf("Unexpected response event. Expected '%v'. Actual '%v'", tc.returnedEvent, resp.Event) - } - if tc.expectedDispatch != fh.requestReceived { t.Errorf("Incorrect dispatch. Expected %v, Actual %v", tc.expectedDispatch, fh.requestReceived) } + + // Compare the returned event. + if tc.returnedEvent == nil { + if resp.Event != nil { + t.Errorf("Unexpected response event: %v", resp.Event) + } + return + } + if diff := cmp.Diff(tc.returnedEvent.Context.AsV02(), resp.Event.Context.AsV02()); diff != "" { + t.Errorf("Incorrect response event context (-want +got): %s", diff) + } + if diff := cmp.Diff(tc.returnedEvent.Data, resp.Event.Data); diff != "" { + t.Errorf("Incorrect response event data (-want +got): %s", diff) + } }) } } -type fakeHTTPDoer struct { +type fakeHandler struct { failRequest bool requestReceived bool + returnedEvent *cloudevents.Event + t testing.T } -func (h *fakeHTTPDoer) Do(_ *http.Request) (*http.Response, error) { +func (h *fakeHandler) ServeHTTP(resp http.ResponseWriter, _ *http.Request) { h.requestReceived = true - sc := http.StatusOK if h.failRequest { - sc = http.StatusBadRequest + resp.WriteHeader(http.StatusBadRequest) + return + } + if h.returnedEvent == nil { + resp.WriteHeader(http.StatusAccepted) + return + } + + c := &cehttp.CodecV02{} + m, err := c.Encode(*h.returnedEvent) + if err != nil { + h.t.Fatalf("Could not encode message: %v", err) + } + msg := m.(*cehttp.Message) + for k, vs := range msg.Header { + resp.Header().Del(k) + for _, v := range vs { + resp.Header().Set(k, v) + } + } + _, err = resp.Write(msg.Body) + if err != nil { + h.t.Fatalf("Unable to write body: %v", err) } - return &http.Response{ - StatusCode: sc, - Body: ioutil.NopCloser(bytes.NewBufferString("")), - }, nil } func makeTrigger(t, s string) *eventingv1alpha1.Trigger { @@ -231,12 +283,13 @@ func makeEvent() cloudevents.Event { Path: eventSource, }, }, + ContentType: cloudevents.StringOfApplicationJSON(), }, } } -func makeDifferentEvent() cloudevents.Event { - return cloudevents.Event{ +func makeDifferentEvent() *cloudevents.Event { + return &cloudevents.Event{ Context: cloudevents.EventContextV02{ Type: "some-other-type", Source: types.URLRef{ @@ -244,6 +297,7 @@ func makeDifferentEvent() cloudevents.Event { Path: eventSource, }, }, + ContentType: cloudevents.StringOfApplicationJSON(), }, } } From f3121ae528b1fc39363c11daf258fc8af12d946c Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Tue, 19 Mar 2019 12:44:56 -0700 Subject: [PATCH 117/128] Increase broker_types.go code coverage. --- .../eventing/v1alpha1/broker_types_test.go | 145 +++++++++++------- 1 file changed, 87 insertions(+), 58 deletions(-) diff --git a/pkg/apis/eventing/v1alpha1/broker_types_test.go b/pkg/apis/eventing/v1alpha1/broker_types_test.go index 319614c350d..73fa5663e24 100644 --- a/pkg/apis/eventing/v1alpha1/broker_types_test.go +++ b/pkg/apis/eventing/v1alpha1/broker_types_test.go @@ -17,6 +17,7 @@ limitations under the License. package v1alpha1 import ( + "errors" "testing" "github.com/google/go-cmp/cmp" @@ -24,6 +25,13 @@ import ( corev1 "k8s.io/api/core/v1" ) +var ( + trueVal = true + falseVal = false + + err = errors.New("foobar") +) + var ( brokerConditionReady = duckv1alpha1.Condition{ Type: BrokerConditionReady, @@ -239,106 +247,127 @@ func TestBrokerInitializeConditions(t *testing.T) { func TestBrokerIsReady(t *testing.T) { tests := []struct { name string - markIngressReady bool - markTriggerChannelReady bool - markIngressChannelReady bool - markFilterReady bool + markIngressReady *bool + markTriggerChannelReady *bool + markIngressChannelReady *bool + markFilterReady *bool address string - markIngressSubscriptionReady bool + markIngressSubscriptionReady *bool wantReady bool }{{ name: "all happy", - markIngressReady: true, - markTriggerChannelReady: true, - markIngressChannelReady: true, - markFilterReady: true, + markIngressReady: &trueVal, + markTriggerChannelReady: &trueVal, + markIngressChannelReady: &trueVal, + markFilterReady: &trueVal, address: "hostname", - markIngressSubscriptionReady: true, + markIngressSubscriptionReady: &trueVal, wantReady: true, }, { name: "ingress sad", - markIngressReady: false, - markTriggerChannelReady: true, - markIngressChannelReady: true, - markFilterReady: true, + markIngressReady: &falseVal, + markTriggerChannelReady: &trueVal, + markIngressChannelReady: &trueVal, + markFilterReady: &trueVal, address: "hostname", - markIngressSubscriptionReady: true, + markIngressSubscriptionReady: &trueVal, wantReady: false, }, { name: "trigger channel sad", - markIngressReady: true, - markTriggerChannelReady: false, - markIngressChannelReady: true, - markFilterReady: true, + markIngressReady: &trueVal, + markTriggerChannelReady: &falseVal, + markIngressChannelReady: &trueVal, + markFilterReady: &trueVal, address: "hostname", - markIngressSubscriptionReady: true, + markIngressSubscriptionReady: &trueVal, wantReady: false, }, { name: "ingress channel sad", - markIngressReady: true, - markTriggerChannelReady: true, - markIngressChannelReady: false, - markFilterReady: true, + markIngressReady: &trueVal, + markTriggerChannelReady: &trueVal, + markIngressChannelReady: &falseVal, + markFilterReady: &trueVal, address: "hostname", - markIngressSubscriptionReady: true, + markIngressSubscriptionReady: &trueVal, wantReady: false, }, { name: "filter sad", - markIngressReady: true, - markTriggerChannelReady: true, - markIngressChannelReady: true, - markFilterReady: false, + markIngressReady: &trueVal, + markTriggerChannelReady: &trueVal, + markIngressChannelReady: &trueVal, + markFilterReady: &falseVal, address: "hostname", - markIngressSubscriptionReady: true, + markIngressSubscriptionReady: &trueVal, wantReady: false, }, { name: "addressable sad", - markIngressReady: true, - markTriggerChannelReady: true, - markIngressChannelReady: true, - markFilterReady: true, + markIngressReady: &trueVal, + markTriggerChannelReady: &trueVal, + markIngressChannelReady: &trueVal, + markFilterReady: &trueVal, address: "", - markIngressSubscriptionReady: true, + markIngressSubscriptionReady: &trueVal, wantReady: false, }, { name: "ingress subscription sad", - markIngressReady: true, - markTriggerChannelReady: true, - markIngressChannelReady: true, - markFilterReady: true, + markIngressReady: &trueVal, + markTriggerChannelReady: &trueVal, + markIngressChannelReady: &trueVal, + markFilterReady: &trueVal, address: "hostname", - markIngressSubscriptionReady: false, + markIngressSubscriptionReady: &falseVal, wantReady: false, }, { name: "all sad", - markIngressReady: false, - markTriggerChannelReady: false, - markIngressChannelReady: true, - markFilterReady: false, + markIngressReady: &falseVal, + markTriggerChannelReady: &falseVal, + markIngressChannelReady: &trueVal, + markFilterReady: &falseVal, address: "", - markIngressSubscriptionReady: true, + markIngressSubscriptionReady: &trueVal, wantReady: false, }} for _, test := range tests { t.Run(test.name, func(t *testing.T) { - ts := &BrokerStatus{} - if test.markIngressReady { - ts.MarkIngressReady() + bs := &BrokerStatus{} + if test.markIngressReady != nil { + if *test.markIngressReady { + bs.MarkIngressReady() + } else { + bs.MarkIngressFailed(err) + } } - if test.markTriggerChannelReady { - ts.MarkTriggerChannelReady() + if test.markTriggerChannelReady != nil { + if *test.markTriggerChannelReady { + bs.MarkTriggerChannelReady() + } else { + bs.MarkTriggerChannelFailed(err) + } } - if test.markIngressChannelReady { - ts.MarkIngressChannelReady() + if test.markIngressChannelReady != nil { + if *test.markIngressChannelReady { + bs.MarkIngressChannelReady() + } else { + bs.MarkIngressChannelFailed(err) + } } - if test.markFilterReady { - ts.MarkFilterReady() + if test.markIngressSubscriptionReady != nil { + if *test.markIngressSubscriptionReady { + bs.MarkIngressSubscriptionReady() + } else { + bs.MarkIngressSubscriptionFailed(err) + } } - ts.SetAddress(test.address) - if test.markIngressSubscriptionReady { - ts.MarkIngressSubscriptionReady() + if test.markFilterReady != nil { + if *test.markFilterReady { + bs.MarkFilterReady() + } else { + bs.MarkFilterFailed(err) + } } - got := ts.IsReady() + bs.SetAddress(test.address) + + got := bs.IsReady() if test.wantReady != got { t.Errorf("unexpected readiness: want %v, got %v", test.wantReady, got) } From 50aa925f209d812a8052199d0a626ac534971d3f Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Tue, 19 Mar 2019 12:45:48 -0700 Subject: [PATCH 118/128] hack/update-deps.sh --- Gopkg.lock | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Gopkg.lock b/Gopkg.lock index b55c4f03fb6..8ebbb9faa65 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -1259,7 +1259,9 @@ "github.com/bsm/sarama-cluster", "github.com/cloudevents/sdk-go/pkg/cloudevents", "github.com/cloudevents/sdk-go/pkg/cloudevents/client", + "github.com/cloudevents/sdk-go/pkg/cloudevents/context", "github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http", + "github.com/cloudevents/sdk-go/pkg/cloudevents/types", "github.com/fsnotify/fsnotify", "github.com/google/go-cmp/cmp", "github.com/google/go-cmp/cmp/cmpopts", From c63f23e43b94b557cc7848115807f34600c60c10 Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Tue, 19 Mar 2019 13:04:38 -0700 Subject: [PATCH 119/128] Increase receiver.go code coverage. --- pkg/broker/receiver_test.go | 59 ++++++++++++++++++++++++++++++++++--- 1 file changed, 55 insertions(+), 4 deletions(-) diff --git a/pkg/broker/receiver_test.go b/pkg/broker/receiver_test.go index ac4efafba84..56ce6b17685 100644 --- a/pkg/broker/receiver_test.go +++ b/pkg/broker/receiver_test.go @@ -18,12 +18,17 @@ package broker import ( "context" + "errors" "fmt" "net/http" "net/http/httptest" "net/url" "testing" + "sigs.k8s.io/controller-runtime/pkg/client" + + controllertesting "github.com/knative/eventing/pkg/reconciler/testing" + "github.com/google/go-cmp/cmp" "github.com/cloudevents/sdk-go/pkg/cloudevents" @@ -44,6 +49,8 @@ const ( triggerName = "test-trigger" eventType = `com.example.someevent` eventSource = `/mycontext` + + toBeReplaced = "toBeReplaced" ) var ( @@ -58,13 +65,25 @@ func init() { func TestReceiver(t *testing.T) { testCases := map[string]struct { triggers []*eventingv1alpha1.Trigger + mocks controllertesting.Mocks tctx *cehttp.TransportContext requestFails bool returnedEvent *cloudevents.Event + expectNewToFail bool expectedErr bool expectedDispatch bool expectedStatus int }{ + "Cannot init": { + mocks: controllertesting.Mocks{ + MockLists: []controllertesting.MockList{ + func(_ client.Client, _ context.Context, _ *client.ListOptions, _ runtime.Object) (controllertesting.MockHandled, error) { + return controllertesting.Handled, errors.New("test induced error") + }, + }, + }, + expectNewToFail: true, + }, "Not POST": { tctx: &cehttp.TransportContext{ Method: "GET", @@ -81,6 +100,14 @@ func TestReceiver(t *testing.T) { }, expectedStatus: http.StatusNotFound, }, + "Bad host": { + tctx: &cehttp.TransportContext{ + Method: "POST", + Host: "badhost-cant-be-parsed-as-a-trigger-name-plus-namespace", + URI: "/", + }, + expectedErr: true, + }, "Trigger.Get fails": { // No trigger exists, so the Get will fail. expectedErr: true, @@ -91,6 +118,12 @@ func TestReceiver(t *testing.T) { }, expectedErr: true, }, + "Trigger with bad SubscriberURI": { + triggers: []*eventingv1alpha1.Trigger{ + makeTriggerWithBadSubscriberURI(), + }, + expectedErr: true, + }, "Trigger without a Filter": { triggers: []*eventingv1alpha1.Trigger{ makeTriggerWithoutFilter(), @@ -147,7 +180,7 @@ func TestReceiver(t *testing.T) { // Replace the SubscriberURI to point at our fake server. correctURI := make([]runtime.Object, 0, len(tc.triggers)) for _, trig := range tc.triggers { - if trig.Status.SubscriberURI != "" { + if trig.Status.SubscriberURI == toBeReplaced { trig.Status.SubscriberURI = s.URL } correctURI = append(correctURI, trig) @@ -155,8 +188,13 @@ func TestReceiver(t *testing.T) { r, err := New( zap.NewNop(), - fake.NewFakeClient(correctURI...)) - if err != nil { + getClient(correctURI, tc.mocks)) + if tc.expectNewToFail { + if err == nil { + t.Fatal("Expected New to fail, it didn't") + } + return + } else if err != nil { t.Fatalf("Unable to create receiver: %v", err) } @@ -238,6 +276,11 @@ func (h *fakeHandler) ServeHTTP(resp http.ResponseWriter, _ *http.Request) { } } +func getClient(initial []runtime.Object, mocks controllertesting.Mocks) *controllertesting.MockClient { + innerClient := fake.NewFakeClient(initial...) + return controllertesting.NewMockClient(innerClient, mocks) +} + func makeTrigger(t, s string) *eventingv1alpha1.Trigger { return &eventingv1alpha1.Trigger{ TypeMeta: v1.TypeMeta{ @@ -257,7 +300,7 @@ func makeTrigger(t, s string) *eventingv1alpha1.Trigger { }, }, Status: eventingv1alpha1.TriggerStatus{ - SubscriberURI: "subscriberURI", + SubscriberURI: "toBeReplaced", }, } } @@ -274,6 +317,14 @@ func makeTriggerWithoutSubscriberURI() *eventingv1alpha1.Trigger { return t } +func makeTriggerWithBadSubscriberURI() *eventingv1alpha1.Trigger { + t := makeTrigger("Any", "Any") + // This should fail url.Parse(). It was taken from the unit tests for url.Parse(), it violates + // rfc3986 3.2.3, namely that the port must be digits. + t.Status.SubscriberURI = "http://[::1]:namedport" + return t +} + func makeEvent() cloudevents.Event { return cloudevents.Event{ Context: cloudevents.EventContextV02{ From ad9bf7c7f64547ac106390c850a3137a543e3033 Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Tue, 19 Mar 2019 13:29:11 -0700 Subject: [PATCH 120/128] Increase trigger.go code coverage. --- .../v1alpha1/trigger/trigger_test.go | 67 ++++++++++++++----- 1 file changed, 49 insertions(+), 18 deletions(-) diff --git a/pkg/reconciler/v1alpha1/trigger/trigger_test.go b/pkg/reconciler/v1alpha1/trigger/trigger_test.go index 9b3655937c2..a861e924486 100644 --- a/pkg/reconciler/v1alpha1/trigger/trigger_test.go +++ b/pkg/reconciler/v1alpha1/trigger/trigger_test.go @@ -25,6 +25,7 @@ import ( "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" "github.com/knative/eventing/pkg/reconciler/names" controllertesting "github.com/knative/eventing/pkg/reconciler/testing" + "github.com/knative/eventing/pkg/reconciler/v1alpha1/broker" "github.com/knative/eventing/pkg/utils" duckv1alpha1 "github.com/knative/pkg/apis/duck/v1alpha1" istiov1alpha3 "github.com/knative/pkg/apis/istio/v1alpha3" @@ -32,6 +33,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes/scheme" @@ -181,7 +183,7 @@ func TestReconcile(t *testing.T) { WantEvent: []corev1.Event{events[triggerReconcileFailed]}, }, { - Name: "Get Broker channel error", + Name: "Get Broker Trigger channel error", Scheme: scheme.Scheme, InitialState: []runtime.Object{ makeTrigger(), @@ -189,15 +191,44 @@ func TestReconcile(t *testing.T) { }, Mocks: controllertesting.Mocks{ MockLists: []controllertesting.MockList{ - func(_ client.Client, _ context.Context, _ *client.ListOptions, list runtime.Object) (controllertesting.MockHandled, error) { - if _, ok := list.(*v1alpha1.ChannelList); ok { - return controllertesting.Handled, errors.New("test error getting broker's channel") + func(_ client.Client, _ context.Context, opts *client.ListOptions, list runtime.Object) (controllertesting.MockHandled, error) { + // Only match the Trigger Channel labels. + ls := labels.FormatLabels(broker.TriggerChannelLabels(makeBroker())) + l, _ := labels.ConvertSelectorToLabelsMap(ls) + + if _, ok := list.(*v1alpha1.ChannelList); ok && opts.LabelSelector.Matches(l) { + return controllertesting.Handled, errors.New("test error getting broker's Trigger channel") + } + return controllertesting.Unhandled, nil + }, + }, + }, + WantErrMsg: "test error getting broker's Trigger channel", + WantEvent: []corev1.Event{events[triggerReconcileFailed]}, + }, + { + Name: "Get Broker Ingress channel error", + Scheme: scheme.Scheme, + InitialState: []runtime.Object{ + makeTrigger(), + makeBroker(), + makeTriggerChannel(), + }, + Mocks: controllertesting.Mocks{ + MockLists: []controllertesting.MockList{ + func(_ client.Client, _ context.Context, opts *client.ListOptions, list runtime.Object) (handled controllertesting.MockHandled, e error) { + // Only match the Ingress Channel labels. + ls := labels.FormatLabels(broker.IngressChannelLabels(makeBroker())) + l, _ := labels.ConvertSelectorToLabelsMap(ls) + + if _, ok := list.(*v1alpha1.ChannelList); ok && opts.LabelSelector.Matches(l) { + return controllertesting.Handled, errors.New("test error getting broker's Ingress channel") } return controllertesting.Unhandled, nil }, }, }, - WantErrMsg: "test error getting broker's channel", + WantErrMsg: "test error getting broker's Ingress channel", WantEvent: []corev1.Event{events[triggerReconcileFailed]}, }, { @@ -206,7 +237,7 @@ func TestReconcile(t *testing.T) { InitialState: []runtime.Object{ makeTrigger(), makeBroker(), - makeChannel(), + makeTriggerChannel(), }, DynamicMocks: controllertesting.DynamicMocks{ MockGets: []controllertesting.MockDynamicGet{ @@ -228,7 +259,7 @@ func TestReconcile(t *testing.T) { InitialState: []runtime.Object{ makeTrigger(), makeBroker(), - makeChannel(), + makeTriggerChannel(), }, Objects: []runtime.Object{ makeSubscriberServiceAsUnstructured(), @@ -252,7 +283,7 @@ func TestReconcile(t *testing.T) { InitialState: []runtime.Object{ makeTrigger(), makeBroker(), - makeChannel(), + makeTriggerChannel(), makeDifferentK8sService(), }, Objects: []runtime.Object{ @@ -277,7 +308,7 @@ func TestReconcile(t *testing.T) { InitialState: []runtime.Object{ makeTrigger(), makeBroker(), - makeChannel(), + makeTriggerChannel(), makeK8sService(), }, Objects: []runtime.Object{ @@ -302,7 +333,7 @@ func TestReconcile(t *testing.T) { InitialState: []runtime.Object{ makeTrigger(), makeBroker(), - makeChannel(), + makeTriggerChannel(), makeK8sService(), makeDifferentVirtualService(), }, @@ -328,7 +359,7 @@ func TestReconcile(t *testing.T) { InitialState: []runtime.Object{ makeTrigger(), makeBroker(), - makeChannel(), + makeTriggerChannel(), makeK8sService(), makeVirtualService(), }, @@ -354,7 +385,7 @@ func TestReconcile(t *testing.T) { InitialState: []runtime.Object{ makeTrigger(), makeBroker(), - makeChannel(), + makeTriggerChannel(), makeK8sService(), makeVirtualService(), makeDifferentSubscription(), @@ -381,7 +412,7 @@ func TestReconcile(t *testing.T) { InitialState: []runtime.Object{ makeTrigger(), makeBroker(), - makeChannel(), + makeTriggerChannel(), makeK8sService(), makeVirtualService(), makeDifferentSubscription(), @@ -408,7 +439,7 @@ func TestReconcile(t *testing.T) { InitialState: []runtime.Object{ makeTrigger(), makeBroker(), - makeChannel(), + makeTriggerChannel(), makeK8sService(), makeVirtualService(), makeSameSubscription(), @@ -435,7 +466,7 @@ func TestReconcile(t *testing.T) { InitialState: []runtime.Object{ makeTrigger(), makeBroker(), - makeChannel(), + makeTriggerChannel(), makeK8sService(), makeVirtualService(), makeSameSubscription(), @@ -562,7 +593,7 @@ func newChannel(name string) *v1alpha1.Channel { } } -func makeChannel() *v1alpha1.Channel { +func makeTriggerChannel() *v1alpha1.Channel { return newChannel(fmt.Sprintf("%s-broker", brokerName)) } @@ -611,11 +642,11 @@ func makeDifferentVirtualService() *istiov1alpha3.VirtualService { } func makeSameSubscription() *v1alpha1.Subscription { - return makeSubscription(makeTrigger(), makeChannel(), makeChannel(), makeK8sService()) + return makeSubscription(makeTrigger(), makeTriggerChannel(), makeTriggerChannel(), makeK8sService()) } func makeDifferentSubscription() *v1alpha1.Subscription { - return makeSubscription(makeTrigger(), makeChannel(), makeDifferentChannel(), makeK8sService()) + return makeSubscription(makeTrigger(), makeTriggerChannel(), makeDifferentChannel(), makeK8sService()) } func getOwnerReference() metav1.OwnerReference { From b314abdf6b5fa842dbca2757a3a9b89f707ef23a Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Tue, 19 Mar 2019 14:07:32 -0700 Subject: [PATCH 121/128] Constraint, not override. --- Gopkg.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gopkg.toml b/Gopkg.toml index 7e0dba1523b..4f616e4b7fc 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -132,6 +132,6 @@ required = [ version = "0.11.0" # The latest release as of March 12, 2019. -[[override]] +[[constraint]] name = "github.com/cloudevents/sdk-go" version = "=0.4.0" From a9d7111029ff18318ca1e871ef1919cc3ea217a1 Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Tue, 19 Mar 2019 16:01:17 -0700 Subject: [PATCH 122/128] broker.go unit tests. --- pkg/reconciler/v1alpha1/broker/broker.go | 3 + pkg/reconciler/v1alpha1/broker/broker_test.go | 441 ++++++++++++++++-- 2 files changed, 403 insertions(+), 41 deletions(-) diff --git a/pkg/reconciler/v1alpha1/broker/broker.go b/pkg/reconciler/v1alpha1/broker/broker.go index 4a55490f89c..4f0842d0143 100644 --- a/pkg/reconciler/v1alpha1/broker/broker.go +++ b/pkg/reconciler/v1alpha1/broker/broker.go @@ -172,6 +172,9 @@ func (r *reconciler) reconcile(ctx context.Context, b *v1alpha1.Broker) (reconci // 4. K8s Services that point at the Deployments. // 5. Ingress Channel is created to get events from Triggers back into this Broker via the // Ingress Deployment. + // - Ideally this wouldn't exist and we would point the Trigger's reply directly to the K8s + // Service. However, Subscriptions only allow us to send replies to Channels, so we need + // this as an intermediary. // 6. Subscription from the Ingress Channel to the Ingress Service. if b.DeletionTimestamp != nil { diff --git a/pkg/reconciler/v1alpha1/broker/broker_test.go b/pkg/reconciler/v1alpha1/broker/broker_test.go index 3960a46b897..c6e7a9799b5 100644 --- a/pkg/reconciler/v1alpha1/broker/broker_test.go +++ b/pkg/reconciler/v1alpha1/broker/broker_test.go @@ -24,16 +24,17 @@ import ( "testing" "time" - "github.com/knative/eventing/pkg/utils" - + "github.com/google/go-cmp/cmp" "github.com/knative/eventing/pkg/apis/eventing/v1alpha1" controllertesting "github.com/knative/eventing/pkg/reconciler/testing" "github.com/knative/eventing/pkg/reconciler/v1alpha1/broker/resources" + "github.com/knative/eventing/pkg/utils" duckv1alpha1 "github.com/knative/pkg/apis/duck/v1alpha1" "go.uber.org/zap" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes/scheme" "sigs.k8s.io/controller-runtime/pkg/client" @@ -60,11 +61,22 @@ var ( Name: "my-provisioner", } - channelHostname = fmt.Sprintf("foo.bar.svc.%s", utils.GetClusterDomainName()) + triggerChannelHostname = fmt.Sprintf("foo.bar.svc.%s", utils.GetClusterDomainName()) + ingressChannelHostname = fmt.Sprintf("baz.qux.svc.%s", utils.GetClusterDomainName()) + + ingressChannelName = "ingress-channel" // deletionTime is used when objects are marked as deleted. Rfc3339Copy() // truncates to seconds to match the loss of precision during serialization. deletionTime = metav1.Now().Rfc3339Copy() + + // Map of events to set test cases' expectations easier. + events = map[string]corev1.Event{ + brokerReconciled: {Reason: brokerReconciled, Type: corev1.EventTypeNormal}, + brokerUpdateStatusFailed: {Reason: brokerUpdateStatusFailed, Type: corev1.EventTypeWarning}, + ingressSubscriptionDeleteFailed: {Reason: ingressSubscriptionDeleteFailed, Type: corev1.EventTypeWarning}, + ingressSubscriptionCreateFailed: {Reason: ingressSubscriptionCreateFailed, Type: corev1.EventTypeWarning}, + } ) func init() { @@ -141,25 +153,29 @@ func TestReconcile(t *testing.T) { }, }, { - Name: "Channel.List error", + Name: "Trigger Channel.List error", Scheme: scheme.Scheme, InitialState: []runtime.Object{ makeBroker(), }, Mocks: controllertesting.Mocks{ MockLists: []controllertesting.MockList{ - func(_ client.Client, _ context.Context, _ *client.ListOptions, list runtime.Object) (controllertesting.MockHandled, error) { - if _, ok := list.(*v1alpha1.ChannelList); ok { - return controllertesting.Handled, errors.New("test error listing channels") + func(_ client.Client, _ context.Context, opts *client.ListOptions, list runtime.Object) (controllertesting.MockHandled, error) { + // Only match the Trigger Channel labels. + ls := labels.FormatLabels(TriggerChannelLabels(makeBroker())) + l, _ := labels.ConvertSelectorToLabelsMap(ls) + + if _, ok := list.(*v1alpha1.ChannelList); ok && opts.LabelSelector.Matches(l) { + return controllertesting.Handled, errors.New("test error getting Trigger Channel") } return controllertesting.Unhandled, nil }, }, }, - WantErrMsg: "test error listing channels", + WantErrMsg: "test error getting Trigger Channel", }, { - Name: "Channel.Create error", + Name: "Trigger Channel.Create error", Scheme: scheme.Scheme, InitialState: []runtime.Object{ makeBroker(), @@ -167,28 +183,30 @@ func TestReconcile(t *testing.T) { Mocks: controllertesting.Mocks{ MockCreates: []controllertesting.MockCreate{ func(_ client.Client, _ context.Context, obj runtime.Object) (controllertesting.MockHandled, error) { - if _, ok := obj.(*v1alpha1.Channel); ok { - return controllertesting.Handled, errors.New("test error creating Channel") + if c, ok := obj.(*v1alpha1.Channel); ok { + if cmp.Equal(c.Labels, TriggerChannelLabels(makeBroker())) { + return controllertesting.Handled, errors.New("test error creating Trigger Channel") + } } return controllertesting.Unhandled, nil }, }, }, - WantErrMsg: "test error creating Channel", + WantErrMsg: "test error creating Trigger Channel", }, { - Name: "Channel is different than expected", + Name: "Trigger Channel is different than expected", Scheme: scheme.Scheme, InitialState: []runtime.Object{ makeBroker(), - makeDifferentChannel(), + makeDifferentTriggerChannel(), }, WantPresent: []runtime.Object{ // This is special because the Channel is not updated, unlike most things that // differ from expected. // TODO uncomment the following line once our test framework supports searching for // GenerateName. - // makeDifferentChannel(), + // makeDifferentTriggerChannel(), }, WantEvent: []corev1.Event{ { @@ -197,11 +215,11 @@ func TestReconcile(t *testing.T) { }, }, { - Name: "Channel is not yet Addressable", + Name: "Trigger Channel is not yet Addressable", Scheme: scheme.Scheme, InitialState: []runtime.Object{ makeBroker(), - makeNonAddressableChannel(), + makeNonAddressableTriggerChannel(), }, WantResult: reconcile.Result{RequeueAfter: time.Second}, }, @@ -210,7 +228,7 @@ func TestReconcile(t *testing.T) { Scheme: scheme.Scheme, InitialState: []runtime.Object{ makeBroker(), - makeChannel(), + makeTriggerChannel(), }, Mocks: controllertesting.Mocks{ MockGets: []controllertesting.MockGet{ @@ -231,7 +249,7 @@ func TestReconcile(t *testing.T) { Scheme: scheme.Scheme, InitialState: []runtime.Object{ makeBroker(), - makeChannel(), + makeTriggerChannel(), }, Mocks: controllertesting.Mocks{ MockCreates: []controllertesting.MockCreate{ @@ -252,7 +270,7 @@ func TestReconcile(t *testing.T) { Scheme: scheme.Scheme, InitialState: []runtime.Object{ makeBroker(), - makeChannel(), + makeTriggerChannel(), makeDifferentFilterDeployment(), }, Mocks: controllertesting.Mocks{ @@ -274,7 +292,7 @@ func TestReconcile(t *testing.T) { Scheme: scheme.Scheme, InitialState: []runtime.Object{ makeBroker(), - makeChannel(), + makeTriggerChannel(), }, Mocks: controllertesting.Mocks{ MockGets: []controllertesting.MockGet{ @@ -295,7 +313,7 @@ func TestReconcile(t *testing.T) { Scheme: scheme.Scheme, InitialState: []runtime.Object{ makeBroker(), - makeChannel(), + makeTriggerChannel(), }, Mocks: controllertesting.Mocks{ MockCreates: []controllertesting.MockCreate{ @@ -316,7 +334,7 @@ func TestReconcile(t *testing.T) { Scheme: scheme.Scheme, InitialState: []runtime.Object{ makeBroker(), - makeChannel(), + makeTriggerChannel(), makeDifferentFilterService(), }, Mocks: controllertesting.Mocks{ @@ -338,7 +356,7 @@ func TestReconcile(t *testing.T) { Scheme: scheme.Scheme, InitialState: []runtime.Object{ makeBroker(), - makeChannel(), + makeTriggerChannel(), }, Mocks: controllertesting.Mocks{ MockGets: []controllertesting.MockGet{ @@ -359,7 +377,7 @@ func TestReconcile(t *testing.T) { Scheme: scheme.Scheme, InitialState: []runtime.Object{ makeBroker(), - makeChannel(), + makeTriggerChannel(), }, Mocks: controllertesting.Mocks{ MockCreates: []controllertesting.MockCreate{ @@ -380,7 +398,7 @@ func TestReconcile(t *testing.T) { Scheme: scheme.Scheme, InitialState: []runtime.Object{ makeBroker(), - makeChannel(), + makeTriggerChannel(), makeDifferentIngressDeployment(), }, Mocks: controllertesting.Mocks{ @@ -402,7 +420,7 @@ func TestReconcile(t *testing.T) { Scheme: scheme.Scheme, InitialState: []runtime.Object{ makeBroker(), - makeChannel(), + makeTriggerChannel(), }, Mocks: controllertesting.Mocks{ MockGets: []controllertesting.MockGet{ @@ -423,7 +441,7 @@ func TestReconcile(t *testing.T) { Scheme: scheme.Scheme, InitialState: []runtime.Object{ makeBroker(), - makeChannel(), + makeTriggerChannel(), }, Mocks: controllertesting.Mocks{ MockCreates: []controllertesting.MockCreate{ @@ -444,7 +462,7 @@ func TestReconcile(t *testing.T) { Scheme: scheme.Scheme, InitialState: []runtime.Object{ makeBroker(), - makeChannel(), + makeTriggerChannel(), makeDifferentIngressService(), }, Mocks: controllertesting.Mocks{ @@ -461,12 +479,247 @@ func TestReconcile(t *testing.T) { }, WantErrMsg: "test error updating ingress Service", }, + { + Name: "Ingress Channel.List error", + Scheme: scheme.Scheme, + InitialState: []runtime.Object{ + makeBroker(), + makeTriggerChannel(), + }, + Mocks: controllertesting.Mocks{ + MockLists: []controllertesting.MockList{ + func(_ client.Client, _ context.Context, opts *client.ListOptions, list runtime.Object) (controllertesting.MockHandled, error) { + // Only match the Ingress Channel labels. + ls := labels.FormatLabels(IngressChannelLabels(makeBroker())) + l, _ := labels.ConvertSelectorToLabelsMap(ls) + + if _, ok := list.(*v1alpha1.ChannelList); ok && opts.LabelSelector.Matches(l) { + return controllertesting.Handled, errors.New("test error getting Ingress Channel") + } + return controllertesting.Unhandled, nil + }, + }, + }, + WantErrMsg: "test error getting Ingress Channel", + }, + { + Name: "Ingress Channel.Create error", + Scheme: scheme.Scheme, + InitialState: []runtime.Object{ + makeBroker(), + makeTriggerChannel(), + }, + Mocks: controllertesting.Mocks{ + MockLists: []controllertesting.MockList{ + // Controller Runtime's fake client totally ignores the opts.LabelSelector, so + // picks up the Trigger Channel while listing the Ingress Channel. Use a mock to + // force the correct behavior. + func(innerClient client.Client, ctx context.Context, opts *client.ListOptions, list runtime.Object) (handled controllertesting.MockHandled, e error) { + if _, ok := list.(*v1alpha1.ChannelList); ok { + // Only match the Ingress Channel labels. + ls := labels.FormatLabels(IngressChannelLabels(makeBroker())) + l, _ := labels.ConvertSelectorToLabelsMap(ls) + if opts.LabelSelector.Matches(l) { + return controllertesting.Handled, nil + } + } + return controllertesting.Unhandled, nil + }, + }, + MockCreates: []controllertesting.MockCreate{ + func(_ client.Client, _ context.Context, obj runtime.Object) (controllertesting.MockHandled, error) { + if c, ok := obj.(*v1alpha1.Channel); ok { + if cmp.Equal(c.Labels, IngressChannelLabels(makeBroker())) { + return controllertesting.Handled, errors.New("test error creating Ingress Channel") + } + } + return controllertesting.Unhandled, nil + }, + }, + }, + WantErrMsg: "test error creating Ingress Channel", + }, + { + Name: "Ingress Channel is different than expected", + Scheme: scheme.Scheme, + InitialState: []runtime.Object{ + makeBroker(), + makeTriggerChannel(), + makeDifferentIngressChannel(), + }, + Mocks: controllertesting.Mocks{ + MockLists: []controllertesting.MockList{ + // Controller Runtime's fake client totally ignores the opts.LabelSelector, so + // picks up the Trigger Channel while listing the Ingress Channel. Use a mock to + // force the correct behavior. + func(innerClient client.Client, ctx context.Context, opts *client.ListOptions, list runtime.Object) (handled controllertesting.MockHandled, e error) { + if cl, ok := list.(*v1alpha1.ChannelList); ok { + // Only match the Ingress Channel labels. + ls := labels.FormatLabels(IngressChannelLabels(makeBroker())) + l, _ := labels.ConvertSelectorToLabelsMap(ls) + if opts.LabelSelector.Matches(l) { + cl.Items = append(cl.Items, *makeDifferentIngressChannel()) + return controllertesting.Handled, nil + } + } + return controllertesting.Unhandled, nil + }, + }, + }, + WantPresent: []runtime.Object{ + // This is special because the Channel is not updated, unlike most things that + // differ from expected. + // TODO uncomment the following line once our test framework supports searching for + // GenerateName. + // makeDifferentIngressChannel(), + }, + WantEvent: []corev1.Event{ + { + Reason: brokerReconciled, Type: corev1.EventTypeNormal, + }, + }, + }, + { + Name: "Ingress Channel is not yet Addressable", + Scheme: scheme.Scheme, + InitialState: []runtime.Object{ + makeBroker(), + makeTriggerChannel(), + makeNonAddressableIngressChannel(), + }, + Mocks: controllertesting.Mocks{ + MockLists: []controllertesting.MockList{ + // Controller Runtime's fake client totally ignores the opts.LabelSelector, so + // picks up the Trigger Channel while listing the Ingress Channel. Use a mock to + // force the correct behavior. + func(innerClient client.Client, ctx context.Context, opts *client.ListOptions, list runtime.Object) (handled controllertesting.MockHandled, e error) { + if cl, ok := list.(*v1alpha1.ChannelList); ok { + // Only match the Ingress Channel labels. + ls := labels.FormatLabels(IngressChannelLabels(makeBroker())) + l, _ := labels.ConvertSelectorToLabelsMap(ls) + if opts.LabelSelector.Matches(l) { + cl.Items = append(cl.Items, *makeNonAddressableIngressChannel()) + return controllertesting.Handled, nil + } + } + return controllertesting.Unhandled, nil + }, + }, + }, + WantResult: reconcile.Result{RequeueAfter: time.Second}, + }, + { + Name: "Subscription.List error", + Scheme: scheme.Scheme, + InitialState: []runtime.Object{ + makeBroker(), + makeTriggerChannel(), + makeIngressChannel(), + }, + Mocks: controllertesting.Mocks{ + MockLists: []controllertesting.MockList{ + func(_ client.Client, _ context.Context, opts *client.ListOptions, list runtime.Object) (controllertesting.MockHandled, error) { + if _, ok := list.(*v1alpha1.SubscriptionList); ok { + return controllertesting.Handled, errors.New("test error getting Subscription") + } + return controllertesting.Unhandled, nil + }, + }, + }, + WantErrMsg: "test error getting Subscription", + }, + { + Name: "Subscription.Create error", + Scheme: scheme.Scheme, + InitialState: []runtime.Object{ + makeBroker(), + makeTriggerChannel(), + makeIngressChannel(), + }, + Mocks: controllertesting.Mocks{ + MockCreates: []controllertesting.MockCreate{ + func(_ client.Client, _ context.Context, obj runtime.Object) (controllertesting.MockHandled, error) { + if _, ok := obj.(*v1alpha1.Subscription); ok { + return controllertesting.Handled, errors.New("test error creating Subscription") + } + return controllertesting.Unhandled, nil + }, + }, + }, + WantErrMsg: "test error creating Subscription", + }, + { + Name: "Subscription is different than expected", + Scheme: scheme.Scheme, + InitialState: []runtime.Object{ + makeBroker(), + makeTriggerChannel(), + makeIngressChannel(), + }, + WantPresent: []runtime.Object{ + // This is special because the Channel is not updated, unlike most things that + // differ from expected. + // TODO uncomment the following line once our test framework supports searching for + // GenerateName. + // makeDifferentSubscription(), + }, + WantEvent: []corev1.Event{ + { + Reason: brokerReconciled, Type: corev1.EventTypeNormal, + }, + }, + }, + { + Name: "Subscription.Delete error", + Scheme: scheme.Scheme, + InitialState: []runtime.Object{ + makeBroker(), + makeTriggerChannel(), + makeIngressChannel(), + makeDifferentSubscription(), + }, + Mocks: controllertesting.Mocks{ + MockDeletes: []controllertesting.MockDelete{ + func(_ client.Client, _ context.Context, obj runtime.Object) (controllertesting.MockHandled, error) { + if _, ok := obj.(*v1alpha1.Subscription); ok { + return controllertesting.Handled, errors.New("test error deleting Subscription") + } + return controllertesting.Unhandled, nil + }, + }, + }, + WantEvent: []corev1.Event{events[ingressSubscriptionDeleteFailed]}, + WantErrMsg: "test error deleting Subscription", + }, + { + Name: "Subscription.Create error when recreating", + Scheme: scheme.Scheme, + InitialState: []runtime.Object{ + makeBroker(), + makeTriggerChannel(), + makeIngressChannel(), + makeDifferentSubscription(), + }, + Mocks: controllertesting.Mocks{ + MockCreates: []controllertesting.MockCreate{ + func(_ client.Client, _ context.Context, obj runtime.Object) (controllertesting.MockHandled, error) { + if _, ok := obj.(*v1alpha1.Subscription); ok { + return controllertesting.Handled, errors.New("test error creating Subscription") + } + return controllertesting.Unhandled, nil + }, + }, + }, + WantEvent: []corev1.Event{events[ingressSubscriptionCreateFailed]}, + WantErrMsg: "test error creating Subscription", + }, { Name: "Broker.Get for status update fails", Scheme: scheme.Scheme, InitialState: []runtime.Object{ makeBroker(), - makeChannel(), + makeTriggerChannel(), + makeIngressChannel(), }, Mocks: controllertesting.Mocks{ MockGets: []controllertesting.MockGet{ @@ -501,7 +754,8 @@ func TestReconcile(t *testing.T) { Scheme: scheme.Scheme, InitialState: []runtime.Object{ makeBroker(), - makeChannel(), + makeTriggerChannel(), + makeIngressChannel(), }, Mocks: controllertesting.Mocks{ MockStatusUpdates: []controllertesting.MockStatusUpdate{ @@ -529,16 +783,40 @@ func TestReconcile(t *testing.T) { InitialState: []runtime.Object{ makeBroker(), // The Channel needs to be addressable for the reconcile to succeed. - makeChannel(), + makeTriggerChannel(), + makeIngressChannel(), + }, + Mocks: controllertesting.Mocks{ + MockLists: []controllertesting.MockList{ + // Controller Runtime's fake client totally ignores the opts.LabelSelector, so + // picks up the Trigger Channel while listing the Ingress Channel. Use a mock to + // force the correct behavior. + func(innerClient client.Client, ctx context.Context, opts *client.ListOptions, list runtime.Object) (handled controllertesting.MockHandled, e error) { + if cl, ok := list.(*v1alpha1.ChannelList); ok { + // Only match the Ingress Channel labels. + ls := labels.FormatLabels(IngressChannelLabels(makeBroker())) + l, _ := labels.ConvertSelectorToLabelsMap(ls) + if opts.LabelSelector.Matches(l) { + cl.Items = append(cl.Items, *makeIngressChannel()) + return controllertesting.Handled, nil + } + } + return controllertesting.Unhandled, nil + }, + }, }, WantPresent: []runtime.Object{ makeReadyBroker(), - // TODO Uncomment makeChannel() when our test framework handles generateName. - // makeChannel(), + // TODO Uncomment makeTriggerChannel() when our test framework handles generateName. + // makeTriggerChannel(), makeFilterDeployment(), makeFilterService(), makeIngressDeployment(), makeIngressService(), + // TODO Uncomment makeIngressChannel() when our test framework handles generateName. + // makeIngressChannel(), + // Because the + makeTestSubscription(), }, WantEvent: []corev1.Event{ { @@ -603,7 +881,7 @@ func makeDeletingBroker() *v1alpha1.Broker { return b } -func makeChannel() *v1alpha1.Channel { +func makeTriggerChannel() *v1alpha1.Channel { return &v1alpha1.Channel{ ObjectMeta: metav1.ObjectMeta{ Namespace: testNS, @@ -621,20 +899,59 @@ func makeChannel() *v1alpha1.Channel { }, Status: v1alpha1.ChannelStatus{ Address: duckv1alpha1.Addressable{ - Hostname: channelHostname, + Hostname: triggerChannelHostname, }, }, } } -func makeNonAddressableChannel() *v1alpha1.Channel { - c := makeChannel() +func makeNonAddressableTriggerChannel() *v1alpha1.Channel { + c := makeTriggerChannel() c.Status.Address = duckv1alpha1.Addressable{} return c } -func makeDifferentChannel() *v1alpha1.Channel { - c := makeChannel() +func makeDifferentTriggerChannel() *v1alpha1.Channel { + c := makeTriggerChannel() + c.Spec.Provisioner.Name = "some-other-provisioner" + return c +} + +func makeIngressChannel() *v1alpha1.Channel { + return &v1alpha1.Channel{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testNS, + GenerateName: fmt.Sprintf("%s-broker-ingress-", brokerName), + // The Fake library doesn't understand GenerateName, so give this a name so it doesn't + // collide with the Trigger Channel. + Name: ingressChannelName, + Labels: map[string]string{ + "eventing.knative.dev/broker": brokerName, + "eventing.knative.dev/brokerIngress": "true", + }, + OwnerReferences: []metav1.OwnerReference{ + getOwnerReference(), + }, + }, + Spec: v1alpha1.ChannelSpec{ + Provisioner: channelProvisioner, + }, + Status: v1alpha1.ChannelStatus{ + Address: duckv1alpha1.Addressable{ + Hostname: ingressChannelHostname, + }, + }, + } +} + +func makeNonAddressableIngressChannel() *v1alpha1.Channel { + c := makeIngressChannel() + c.Status.Address = duckv1alpha1.Addressable{} + return c +} + +func makeDifferentIngressChannel() *v1alpha1.Channel { + c := makeIngressChannel() c.Spec.Provisioner.Name = "some-other-provisioner" return c } @@ -678,7 +995,7 @@ func makeIngressDeployment() *appsv1.Deployment { Broker: makeBroker(), Image: ingressImage, ServiceAccountName: ingressSA, - ChannelAddress: channelHostname, + ChannelAddress: triggerChannelHostname, }) d.TypeMeta = metav1.TypeMeta{ APIVersion: "apps/v1", @@ -708,6 +1025,48 @@ func makeDifferentIngressService() *corev1.Service { return s } +func makeTestSubscription() *v1alpha1.Subscription { + return &v1alpha1.Subscription{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "eventing.knative.dev/v1alpha1", + Kind: "Subscription", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: testNS, + GenerateName: fmt.Sprintf("internal-ingress-%s-", brokerName), + Labels: map[string]string{ + "eventing.knative.dev/broker": brokerName, + "eventing.knative.dev/brokerIngress": "true", + }, + OwnerReferences: []metav1.OwnerReference{ + getOwnerReference(), + }, + }, + Spec: v1alpha1.SubscriptionSpec{ + Channel: corev1.ObjectReference{ + APIVersion: v1alpha1.SchemeGroupVersion.String(), + Kind: "Channel", + Name: ingressChannelName, + }, + Subscriber: &v1alpha1.SubscriberSpec{ + Ref: &corev1.ObjectReference{ + APIVersion: "v1", + Kind: "Service", + Name: makeIngressService().Name, + }, + }, + }, + } +} + +func makeDifferentSubscription() *v1alpha1.Subscription { + s := makeTestSubscription() + s.Spec.Subscriber.Ref = nil + url := "http://example.com/" + s.Spec.Subscriber.DNSName = &url + return s +} + func getOwnerReference() metav1.OwnerReference { return metav1.OwnerReference{ APIVersion: v1alpha1.SchemeGroupVersion.String(), From 36ef2a4023dbedae2b586d7e5dd37c6b33c5ffac Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Wed, 20 Mar 2019 08:46:13 -0700 Subject: [PATCH 123/128] Better receiver shutdown logic. --- pkg/broker/receiver.go | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/pkg/broker/receiver.go b/pkg/broker/receiver.go index d6ee5b528c0..8d06df8c9d9 100644 --- a/pkg/broker/receiver.go +++ b/pkg/broker/receiver.go @@ -21,6 +21,7 @@ import ( "errors" "net/http" "net/url" + "time" "github.com/cloudevents/sdk-go/pkg/cloudevents" ceclient "github.com/cloudevents/sdk-go/pkg/cloudevents/client" @@ -35,6 +36,8 @@ import ( const ( defaultPort = 8080 + + writeTimeout = 1 * time.Minute ) // Receiver parses Cloud Events, determines if they pass a filter, and sends them to a subscriber. @@ -91,7 +94,7 @@ func (r *Receiver) initClient() error { // // This method will block until a message is received on the stop channel. func (r *Receiver) Start(stopCh <-chan struct{}) error { - ctx := context.Background() + ctx, cancel := context.WithCancel(context.Background()) defer ctx.Done() errCh := make(chan error, 1) @@ -99,11 +102,22 @@ func (r *Receiver) Start(stopCh <-chan struct{}) error { errCh <- r.ceClient.StartReceiver(ctx, r.serveHTTP) }() + // Stop either if the receiver stops (sending to errCh) or if stopCh is closed. select { case err := <-errCh: return err case <-stopCh: - return nil + break + } + + // stopCh has been closed, we need to gracefully shutdown h.ceClient. cancel() will start its + // shutdown, if it hasn't finished in a reasonable amount of time, just return an error. + cancel() + select { + case err := <-errCh: + return err + case <-time.After(writeTimeout): + return errors.New("timeout shutting down ceClient") } } From c66f21e52725f525e8e6d9a96f9d107d11d219a0 Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Wed, 20 Mar 2019 12:19:49 -0700 Subject: [PATCH 124/128] cloudevents/sdk-go 0.4.1 --- Gopkg.lock | 7 +- Gopkg.toml | 2 +- .../sdk-go/pkg/cloudevents/client/client.go | 30 +++++ .../sdk-go/pkg/cloudevents/client/doc.go | 6 + .../pkg/cloudevents/client/observability.go | 59 +++++++++ .../sdk-go/pkg/cloudevents/codec/doc.go | 5 + .../sdk-go/pkg/cloudevents/codec/jsoncodec.go | 80 ++++++++++++ .../pkg/cloudevents/codec/observability.go | 75 +++++++++++ .../sdk-go/pkg/cloudevents/context/context.go | 3 + .../sdk-go/pkg/cloudevents/context/doc.go | 5 + .../sdk-go/pkg/cloudevents/datacodec/codec.go | 42 +++++++ .../sdk-go/pkg/cloudevents/datacodec/doc.go | 5 + .../pkg/cloudevents/datacodec/json/data.go | 32 +++++ .../pkg/cloudevents/datacodec/json/doc.go | 4 + .../datacodec/json/observability.go | 54 ++++++++ .../cloudevents/datacodec/observability.go | 54 ++++++++ .../pkg/cloudevents/datacodec/xml/data.go | 32 +++++ .../pkg/cloudevents/datacodec/xml/doc.go | 4 + .../datacodec/xml/observability.go | 54 ++++++++ .../cloudevents/sdk-go/pkg/cloudevents/doc.go | 82 ------------ .../sdk-go/pkg/cloudevents/event.go | 119 +++++++++++++++--- .../pkg/cloudevents/observability/keys.go | 15 +++ .../pkg/cloudevents/observability/observer.go | 88 +++++++++++++ .../sdk-go/pkg/cloudevents/transport/doc.go | 5 + .../cloudevents/transport/http/codec_v01.go | 26 ++++ .../cloudevents/transport/http/codec_v02.go | 26 ++++ .../cloudevents/transport/http/codec_v03.go | 26 ++++ .../pkg/cloudevents/transport/http/context.go | 114 +++++++++++++++++ .../pkg/cloudevents/transport/http/doc.go | 4 + .../cloudevents/transport/http/encoding.go | 31 +++++ .../transport/http/observability.go | 93 ++++++++++++++ .../pkg/cloudevents/transport/http/options.go | 34 +++++ .../cloudevents/transport/http/transport.go | 87 +++++++++++-- .../sdk-go/pkg/cloudevents/types/doc.go | 4 + .../sdk-go/pkg/cloudevents/types/timestamp.go | 15 ++- .../sdk-go/pkg/cloudevents/types/urlref.go | 15 ++- 36 files changed, 1218 insertions(+), 119 deletions(-) create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/doc.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/observability.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/codec/doc.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/codec/observability.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/context/doc.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/doc.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/json/doc.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/json/observability.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/observability.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/xml/doc.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/xml/observability.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/observability/keys.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/observability/observer.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/doc.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/doc.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/observability.go create mode 100644 vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/types/doc.go diff --git a/Gopkg.lock b/Gopkg.lock index 5f095636bc3..06530b6736f 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -58,7 +58,7 @@ version = "v2.1.15" [[projects]] - digest = "1:70a8e5f09e19aba14064648751abc86f20484e25476866b03ce711007e96f339" + digest = "1:7ded9717607b3eea18859dca3323fac420fa142ff4aa592cb07d8c7eb4a67d5e" name = "github.com/cloudevents/sdk-go" packages = [ "pkg/cloudevents", @@ -68,13 +68,14 @@ "pkg/cloudevents/datacodec", "pkg/cloudevents/datacodec/json", "pkg/cloudevents/datacodec/xml", + "pkg/cloudevents/observability", "pkg/cloudevents/transport", "pkg/cloudevents/transport/http", "pkg/cloudevents/types", ] pruneopts = "NUT" - revision = "c1a6fb0cc8226884014fb5063ab406ed9504a663" - version = "0.4.0" + revision = "a4c06e590662f2aa30ac8f5974347c9832889765" + version = "0.4.1" [[projects]] digest = "1:ffe9824d294da03b391f44e1ae8281281b4afc1bdaa9588c9097785e3af10cec" diff --git a/Gopkg.toml b/Gopkg.toml index 4f616e4b7fc..a10eeb4ef60 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -134,4 +134,4 @@ required = [ # The latest release as of March 12, 2019. [[constraint]] name = "github.com/cloudevents/sdk-go" - version = "=0.4.0" + version = "=0.4.1" diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/client.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/client.go index d9c948cd7a4..41544d929b4 100644 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/client.go +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/client.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "github.com/cloudevents/sdk-go/pkg/cloudevents" + "github.com/cloudevents/sdk-go/pkg/cloudevents/observability" "github.com/cloudevents/sdk-go/pkg/cloudevents/transport" "github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http" "sync" @@ -69,6 +70,17 @@ type ceClient struct { } func (c *ceClient) Send(ctx context.Context, event cloudevents.Event) (*cloudevents.Event, error) { + ctx, r := observability.NewReporter(ctx, ReportSend) + resp, err := c.obsSend(ctx, event) + if err != nil { + r.Error() + } else { + r.OK() + } + return resp, err +} + +func (c *ceClient) obsSend(ctx context.Context, event cloudevents.Event) (*cloudevents.Event, error) { // Confirm we have a transport set. if c.transport == nil { return nil, fmt.Errorf("client not ready, transport not initialized") @@ -89,8 +101,26 @@ func (c *ceClient) Send(ctx context.Context, event cloudevents.Event) (*cloudeve // Receive is called from from the transport on event delivery. func (c *ceClient) Receive(ctx context.Context, event cloudevents.Event, resp *cloudevents.EventResponse) error { + ctx, r := observability.NewReporter(ctx, ReportReceive) + err := c.obsReceive(ctx, event, resp) + if err != nil { + r.Error() + } else { + r.OK() + } + return err +} + +func (c *ceClient) obsReceive(ctx context.Context, event cloudevents.Event, resp *cloudevents.EventResponse) error { if c.fn != nil { + ctx, rFn := observability.NewReporter(ctx, ReportReceiveFn) err := c.fn.invoke(ctx, event, resp) + if err != nil { + rFn.Error() + } else { + rFn.OK() + } + // Apply the defaulter chain to the outgoing event. if err == nil && resp != nil && resp.Event != nil && len(c.eventDefaulterFns) > 0 { for _, fn := range c.eventDefaulterFns { diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/doc.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/doc.go new file mode 100644 index 00000000000..a6a602bb410 --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/doc.go @@ -0,0 +1,6 @@ +/* +Package client holds the recommended entry points for interacting with the CloudEvents Golang SDK. The client wraps +a selected transport. The client adds validation and defaulting for sending events, and flexible receiver method +registration. For full details, read the `client.Client` documentation. +*/ +package client diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/observability.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/observability.go new file mode 100644 index 00000000000..69f0df2dfd6 --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/client/observability.go @@ -0,0 +1,59 @@ +package client + +import ( + "github.com/cloudevents/sdk-go/pkg/cloudevents/observability" + "go.opencensus.io/stats" + "go.opencensus.io/stats/view" +) + +var ( + LatencyMs = stats.Float64("client/latency", "The latency in milliseconds for the CloudEvents client methods.", "ms") +) + +var ( + LatencyView = &view.View{ + Name: "client/latency", + Measure: LatencyMs, + Description: "The distribution of latency inside of client for CloudEvents.", + Aggregation: view.Distribution(0, .01, .1, 1, 10, 100, 1000, 10000), + TagKeys: observability.LatencyTags(), + } +) + +type Observed int32 + +const ( + ReportSend Observed = iota + ReportReceive + ReportReceiveFn +) + +func (o Observed) TraceName() string { + switch o { + case ReportSend: + return "client/send" + case ReportReceive: + return "client/receive" + case ReportReceiveFn: + return "client/receive/fn" + default: + return "client/unknown" + } +} + +func (o Observed) MethodName() string { + switch o { + case ReportSend: + return "send" + case ReportReceive: + return "receive" + case ReportReceiveFn: + return "receive/fn" + default: + return "unknown" + } +} + +func (o Observed) LatencyMs() *stats.Float64Measure { + return LatencyMs +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/codec/doc.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/codec/doc.go new file mode 100644 index 00000000000..f6028398f5a --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/codec/doc.go @@ -0,0 +1,5 @@ +/* +Package codec holds the encoder/decoder implementation for structured encodings using `application/json` of the +whole CloudEvent. +*/ +package codec diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/codec/jsoncodec.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/codec/jsoncodec.go index 21f10c68375..42847d88c0d 100644 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/codec/jsoncodec.go +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/codec/jsoncodec.go @@ -1,14 +1,29 @@ package codec import ( + "context" "encoding/json" "github.com/cloudevents/sdk-go/pkg/cloudevents" "github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec" + "github.com/cloudevents/sdk-go/pkg/cloudevents/observability" "log" "strconv" ) +// JsonEncodeV01 takes in a cloudevent.Event and outputs the byte representation of that event using CloudEvents +// version 0.1 structured json formatting rules. func JsonEncodeV01(e cloudevents.Event) ([]byte, error) { + _, r := observability.NewReporter(context.Background(), CodecObserved{o: ReportEncode, v: "v0.1"}) + b, err := obsJsonEncodeV01(e) + if err != nil { + r.Error() + } else { + r.OK() + } + return b, err +} + +func obsJsonEncodeV01(e cloudevents.Event) ([]byte, error) { ctx := e.Context.AsV01() if ctx.ContentType == nil { ctx.ContentType = cloudevents.StringOfApplicationJSON() @@ -16,7 +31,20 @@ func JsonEncodeV01(e cloudevents.Event) ([]byte, error) { return jsonEncode(ctx, e.Data) } +// JsonEncodeV02 takes in a cloudevent.Event and outputs the byte representation of that event using CloudEvents +// version 0.2 structured json formatting rules. func JsonEncodeV02(e cloudevents.Event) ([]byte, error) { + _, r := observability.NewReporter(context.Background(), CodecObserved{o: ReportEncode, v: "v0.2"}) + b, err := obsJsonEncodeV02(e) + if err != nil { + r.Error() + } else { + r.OK() + } + return b, err +} + +func obsJsonEncodeV02(e cloudevents.Event) ([]byte, error) { ctx := e.Context.AsV02() if ctx.ContentType == nil { ctx.ContentType = cloudevents.StringOfApplicationJSON() @@ -24,7 +52,20 @@ func JsonEncodeV02(e cloudevents.Event) ([]byte, error) { return jsonEncode(ctx, e.Data) } +// JsonEncodeV03 takes in a cloudevent.Event and outputs the byte representation of that event using CloudEvents +// version 0.3 structured json formatting rules. func JsonEncodeV03(e cloudevents.Event) ([]byte, error) { + _, r := observability.NewReporter(context.Background(), CodecObserved{o: ReportEncode, v: "v0.3"}) + b, err := obsJsonEncodeV03(e) + if err != nil { + r.Error() + } else { + r.OK() + } + return b, err +} + +func obsJsonEncodeV03(e cloudevents.Event) ([]byte, error) { ctx := e.Context.AsV03() if ctx.DataContentType == nil { ctx.DataContentType = cloudevents.StringOfApplicationJSON() @@ -69,7 +110,20 @@ func jsonEncode(ctx cloudevents.EventContext, data interface{}) ([]byte, error) return body, nil } +// JsonDecodeV01 takes in the byte representation of a version 0.1 structured json CloudEvent and returns a +// cloudevent.Event or an error if there are parsing errors. func JsonDecodeV01(body []byte) (*cloudevents.Event, error) { + _, r := observability.NewReporter(context.Background(), CodecObserved{o: ReportDecode, v: "v0.1"}) + e, err := obsJsonDecodeV01(body) + if err != nil { + r.Error() + } else { + r.OK() + } + return e, err +} + +func obsJsonDecodeV01(body []byte) (*cloudevents.Event, error) { ec := cloudevents.EventContextV01{} if err := json.Unmarshal(body, &ec); err != nil { return nil, err @@ -91,7 +145,20 @@ func JsonDecodeV01(body []byte) (*cloudevents.Event, error) { }, nil } +// JsonDecodeV02 takes in the byte representation of a version 0.2 structured json CloudEvent and returns a +// cloudevent.Event or an error if there are parsing errors. func JsonDecodeV02(body []byte) (*cloudevents.Event, error) { + _, r := observability.NewReporter(context.Background(), CodecObserved{o: ReportDecode, v: "v0.2"}) + e, err := obsJsonDecodeV02(body) + if err != nil { + r.Error() + } else { + r.OK() + } + return e, err +} + +func obsJsonDecodeV02(body []byte) (*cloudevents.Event, error) { ec := cloudevents.EventContextV02{} if err := json.Unmarshal(body, &ec); err != nil { return nil, err @@ -113,7 +180,20 @@ func JsonDecodeV02(body []byte) (*cloudevents.Event, error) { }, nil } +// JsonDecodeV03 takes in the byte representation of a version 0.3 structured json CloudEvent and returns a +// cloudevent.Event or an error if there are parsing errors. func JsonDecodeV03(body []byte) (*cloudevents.Event, error) { + _, r := observability.NewReporter(context.Background(), CodecObserved{o: ReportDecode, v: "v0.3"}) + e, err := obsJsonDecodeV03(body) + if err != nil { + r.Error() + } else { + r.OK() + } + return e, err +} + +func obsJsonDecodeV03(body []byte) (*cloudevents.Event, error) { ec := cloudevents.EventContextV03{} if err := json.Unmarshal(body, &ec); err != nil { return nil, err diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/codec/observability.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/codec/observability.go new file mode 100644 index 00000000000..c710ea191f0 --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/codec/observability.go @@ -0,0 +1,75 @@ +package codec + +import ( + "fmt" + "github.com/cloudevents/sdk-go/pkg/cloudevents/observability" + "go.opencensus.io/stats" + "go.opencensus.io/stats/view" +) + +var ( + LatencyMs = stats.Float64("codec/json/latency", "The latency in milliseconds for the CloudEvents json codec methods.", "ms") +) + +var ( + LatencyView = &view.View{ + Name: "codec/json/latency", + Measure: LatencyMs, + Description: "The distribution of latency inside of the json codec for CloudEvents.", + Aggregation: view.Distribution(0, .01, .1, 1, 10, 100, 1000, 10000), + TagKeys: observability.LatencyTags(), + } +) + +type Observed int32 + +const ( + ReportEncode Observed = iota + ReportDecode +) + +func (o Observed) TraceName() string { + switch o { + case ReportEncode: + return "codec/json/encode" + case ReportDecode: + return "codec/json/decode" + default: + return "codec/unknown" + } +} + +func (o Observed) MethodName() string { + switch o { + case ReportEncode: + return "encode" + case ReportDecode: + return "decode" + default: + return "unknown" + } +} + +func (o Observed) LatencyMs() *stats.Float64Measure { + return LatencyMs +} + +// CodecObserved is a wrapper to append version to Observed. +type CodecObserved struct { + // Method + o Observed + // Version + v string +} + +func (c CodecObserved) TraceName() string { + return fmt.Sprintf("%s/%s", c.o.TraceName(), c.v) +} + +func (c CodecObserved) MethodName() string { + return fmt.Sprintf("%s/%s", c.o.MethodName(), c.v) +} + +func (c CodecObserved) LatencyMs() *stats.Float64Measure { + return c.o.LatencyMs() +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/context/context.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/context/context.go index 2c4526b1035..18afb6282b6 100644 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/context/context.go +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/context/context.go @@ -10,10 +10,13 @@ type targetKeyType struct{} var targetKey = targetKeyType{} +// WithTarget returns back a new context with the given target. Target is intended to be transport dependent. +// For http transport, `target` should be a full URL and will be injected into the outbound http request. func WithTarget(ctx context.Context, target string) context.Context { return context.WithValue(ctx, targetKey, target) } +// TargetFrom looks in the given context and returns `target` as a parsed url if found and valid, otherwise nil. func TargetFrom(ctx context.Context) *url.URL { c := ctx.Value(targetKey) if c != nil { diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/context/doc.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/context/doc.go new file mode 100644 index 00000000000..377cab850fc --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/context/doc.go @@ -0,0 +1,5 @@ +/* +Package context holds the last resort overrides and fyi objects that can be passed to clients and transports added to +context.Context objects. +*/ +package context diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/codec.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/codec.go index 1f6ce63d365..929a8754bb9 100644 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/codec.go +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/codec.go @@ -1,12 +1,20 @@ package datacodec import ( + "context" "fmt" "github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/json" "github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/xml" + "github.com/cloudevents/sdk-go/pkg/cloudevents/observability" ) +// Decoder is the expected function signature for decoding `in` to `out`. What +// `in` is could be decoder dependent. For example, `in` could be bytes, or a +// base64 string. type Decoder func(in, out interface{}) error + +// Encoder is the expected function signature for encoding `in` to bytes. +// Returns an error if the encoder has an issue encoding `in`. type Encoder func(in interface{}) ([]byte, error) var decoder map[string]Decoder @@ -25,22 +33,56 @@ func init() { AddEncoder("application/xml", xml.Encode) } +// AddDecoder registers a decoder for a given content type. The codecs will use +// these to decode the data payload from a cloudevent.Event object. func AddDecoder(contentType string, fn Decoder) { decoder[contentType] = fn } +// AddEncoder registers an encoder for a given content type. The codecs will +// use these to encode the data payload for a cloudevent.Event object. func AddEncoder(contentType string, fn Encoder) { encoder[contentType] = fn } +// Decode looks up and invokes the decoder registered for the given content +// type. An error is returned if no decoder is registered for the given +// content type. func Decode(contentType string, in, out interface{}) error { + // TODO: wire in context. + _, r := observability.NewReporter(context.Background(), ReportDecode) + err := obsDecode(contentType, in, out) + if err != nil { + r.Error() + } else { + r.OK() + } + return err +} + +func obsDecode(contentType string, in, out interface{}) error { if fn, ok := decoder[contentType]; ok { return fn(in, out) } return fmt.Errorf("[decode] unsupported content type: %q", contentType) } +// Encode looks up and invokes the encoder registered for the given content +// type. An error is returned if no encoder is registered for the given +// content type. func Encode(contentType string, in interface{}) ([]byte, error) { + // TODO: wire in context. + _, r := observability.NewReporter(context.Background(), ReportEncode) + b, err := obsEncode(contentType, in) + if err != nil { + r.Error() + } else { + r.OK() + } + return b, err +} + +func obsEncode(contentType string, in interface{}) ([]byte, error) { if fn, ok := encoder[contentType]; ok { return fn(in) } diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/doc.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/doc.go new file mode 100644 index 00000000000..9e401534e27 --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/doc.go @@ -0,0 +1,5 @@ +/* +Package datacodec holds the data codec registry and adds known encoders and decoders supporting media types such as +`application/json` and `application/xml`. +*/ +package datacodec diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/json/data.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/json/data.go index 38d75566713..9b362d305f2 100644 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/json/data.go +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/json/data.go @@ -1,13 +1,30 @@ package json import ( + "context" "encoding/json" "fmt" + "github.com/cloudevents/sdk-go/pkg/cloudevents/observability" "reflect" "strconv" ) +// Decode takes `in` as []byte, or base64 string, normalizes in to unquoted and +// base64 decoded []byte if required, and then attempts to use json.Unmarshal +// to convert those bytes to `out`. Returns and error if this process fails. func Decode(in, out interface{}) error { + // TODO: wire in context. + _, r := observability.NewReporter(context.Background(), ReportDecode) + err := obsDecode(in, out) + if err != nil { + r.Error() + } else { + r.OK() + } + return err +} + +func obsDecode(in, out interface{}) error { if in == nil { return nil } @@ -42,7 +59,22 @@ func Decode(in, out interface{}) error { return nil } +// Encode attempts to json.Marshal `in` into bytes. Encode will inspect `in` +// and returns `in` unmodified if it is detected that `in` is already a []byte; +// Or json.Marshal errors. func Encode(in interface{}) ([]byte, error) { + // TODO: wire in context. + _, r := observability.NewReporter(context.Background(), ReportEncode) + b, err := obsEncode(in) + if err != nil { + r.Error() + } else { + r.OK() + } + return b, err +} + +func obsEncode(in interface{}) ([]byte, error) { if in == nil { return nil, nil } diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/json/doc.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/json/doc.go new file mode 100644 index 00000000000..2f3d2982d25 --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/json/doc.go @@ -0,0 +1,4 @@ +/* +Package datacodec/json holds the encoder/decoder implementation for `application/json`. +*/ +package json diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/json/observability.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/json/observability.go new file mode 100644 index 00000000000..5be43645486 --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/json/observability.go @@ -0,0 +1,54 @@ +package json + +import ( + "github.com/cloudevents/sdk-go/pkg/cloudevents/observability" + "go.opencensus.io/stats" + "go.opencensus.io/stats/view" +) + +var ( + LatencyMs = stats.Float64("datacodec/json/latency", "The latency in milliseconds for the CloudEvents json data codec methods.", "ms") +) + +var ( + LatencyView = &view.View{ + Name: "datacodec/json/latency", + Measure: LatencyMs, + Description: "The distribution of latency inside of the json data codec for CloudEvents.", + Aggregation: view.Distribution(0, .01, .1, 1, 10, 100, 1000, 10000), + TagKeys: observability.LatencyTags(), + } +) + +type Observed int32 + +const ( + ReportEncode Observed = iota + ReportDecode +) + +func (o Observed) TraceName() string { + switch o { + case ReportEncode: + return "datacodec/json/encode" + case ReportDecode: + return "datacodec/json/decode" + default: + return "datacodec/json/unknown" + } +} + +func (o Observed) MethodName() string { + switch o { + case ReportEncode: + return "encode" + case ReportDecode: + return "decode" + default: + return "unknown" + } +} + +func (o Observed) LatencyMs() *stats.Float64Measure { + return LatencyMs +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/observability.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/observability.go new file mode 100644 index 00000000000..0a87278f811 --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/observability.go @@ -0,0 +1,54 @@ +package datacodec + +import ( + "github.com/cloudevents/sdk-go/pkg/cloudevents/observability" + "go.opencensus.io/stats" + "go.opencensus.io/stats/view" +) + +var ( + LatencyMs = stats.Float64("datacodec/latency", "The latency in milliseconds for the CloudEvents generic data codec methods.", "ms") +) + +var ( + LatencyView = &view.View{ + Name: "datacodec/latency", + Measure: LatencyMs, + Description: "The distribution of latency inside of the generic data codec for CloudEvents.", + Aggregation: view.Distribution(0, .01, .1, 1, 10, 100, 1000, 10000), + TagKeys: observability.LatencyTags(), + } +) + +type Observed int32 + +const ( + ReportEncode Observed = iota + ReportDecode +) + +func (o Observed) TraceName() string { + switch o { + case ReportEncode: + return "datacodec/encode" + case ReportDecode: + return "datacodec/decode" + default: + return "datacodec/unknown" + } +} + +func (o Observed) MethodName() string { + switch o { + case ReportEncode: + return "encode" + case ReportDecode: + return "decode" + default: + return "unknown" + } +} + +func (o Observed) LatencyMs() *stats.Float64Measure { + return LatencyMs +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/xml/data.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/xml/data.go index a904a250c07..8a644932f2a 100644 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/xml/data.go +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/xml/data.go @@ -1,13 +1,30 @@ package xml import ( + "context" "encoding/base64" "encoding/xml" "fmt" + "github.com/cloudevents/sdk-go/pkg/cloudevents/observability" "strconv" ) +// Decode takes `in` as []byte, or base64 string, normalizes in to unquoted and +// base64 decoded []byte if required, and then attempts to use xml.Unmarshal +// to convert those bytes to `out`. Returns and error if this process fails. func Decode(in, out interface{}) error { + // TODO: wire in context. + _, r := observability.NewReporter(context.Background(), ReportDecode) + err := obsDecode(in, out) + if err != nil { + r.Error() + } else { + r.OK() + } + return err +} + +func obsDecode(in, out interface{}) error { if in == nil { return nil } @@ -47,7 +64,22 @@ func Decode(in, out interface{}) error { return nil } +// Encode attempts to xml.Marshal `in` into bytes. Encode will inspect `in` +// and returns `in` unmodified if it is detected that `in` is already a []byte; +// Or xml.Marshal errors. func Encode(in interface{}) ([]byte, error) { + // TODO: wire in context. + _, r := observability.NewReporter(context.Background(), ReportEncode) + b, err := obsEncode(in) + if err != nil { + r.Error() + } else { + r.OK() + } + return b, err +} + +func obsEncode(in interface{}) ([]byte, error) { if b, ok := in.([]byte); ok { // check to see if it is a pre-encoded byte string. if len(b) > 0 && b[0] == byte('"') { diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/xml/doc.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/xml/doc.go new file mode 100644 index 00000000000..e7e013d3f02 --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/xml/doc.go @@ -0,0 +1,4 @@ +/* +Package datacodec/xml holds the encoder/decoder implementation for `application/xml`. +*/ +package xml diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/xml/observability.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/xml/observability.go new file mode 100644 index 00000000000..4948c9be477 --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec/xml/observability.go @@ -0,0 +1,54 @@ +package xml + +import ( + "github.com/cloudevents/sdk-go/pkg/cloudevents/observability" + "go.opencensus.io/stats" + "go.opencensus.io/stats/view" +) + +var ( + LatencyMs = stats.Float64("datacodec/xml/latency", "The latency in milliseconds for the CloudEvents xml data codec methods.", "ms") +) + +var ( + LatencyView = &view.View{ + Name: "datacodec/xml/latency", + Measure: LatencyMs, + Description: "The distribution of latency inside of the xml data codec for CloudEvents.", + Aggregation: view.Distribution(0, .01, .1, 1, 10, 100, 1000, 10000), + TagKeys: observability.LatencyTags(), + } +) + +type Observed int32 + +const ( + ReportEncode Observed = iota + ReportDecode +) + +func (o Observed) TraceName() string { + switch o { + case ReportEncode: + return "datacodec/xml/encode" + case ReportDecode: + return "datacodec/xml/decode" + default: + return "datacodec/xml/unknown" + } +} + +func (o Observed) MethodName() string { + switch o { + case ReportEncode: + return "encode" + case ReportDecode: + return "decode" + default: + return "unknown" + } +} + +func (o Observed) LatencyMs() *stats.Float64Measure { + return LatencyMs +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/doc.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/doc.go index cfb6eac590c..cc2201da915 100644 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/doc.go +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/doc.go @@ -1,86 +1,4 @@ /* Package cloudevents provides primitives to work with CloudEvents specification: https://github.com/cloudevents/spec. - - -Parsing Event from HTTP Request: - // req is *http.Request - event, err := cloudEvents.FromHTTPRequest(req) - if err != nil { - panic("Unable to parse event from http Request: " + err.String()) - } - - -Creating a minimal CloudEvent in version 0.1: - import "github.com/cloudevents/sdk-go/pkg/cloudevents/v01" - event := v01.Event{ - EventType: "com.example.file.created", - Source: "/providers/Example.COM/storage/account#fileServices/default/{new-file}", - EventID: "ea35b24ede421", - } - - -The goal of this package is to provide support for all released versions of CloudEvents, ideally while maintaining -the same API. It will use semantic versioning with following rules: -* MAJOR version increments when backwards incompatible changes is introduced. -* MINOR version increments when backwards compatible feature is introduced INCLUDING support for new CloudEvents version. -* PATCH version increments when a backwards compatible bug fix is introduced. */ package cloudevents - -/* - -New plan. - -Everything gets converted into the Canonical form of the event, this -then can select a transport, the transport provides encodings. - -At the moment we have cloudevents.[v01, v02] - -Canonical form holds an encoded data packet that takes in a provided Codec - -Canonical form has two members: Context, and Data - - -Sending: -cloudevents.[v01, v02] -> { Codec.Encode -> HttpMessage -> Transport[Http] } - -Receiving: -{ Transport[Http] -> HttpMessage -> Codec.Decode } -> cloudevents.[v01, v02] - -Note: Transport and Codec are grouped. - -Transport Codecs supported: -[Binary, Structured, StructuredMirrorHeaders] - - -## Working with inner data: -cloudevents.[v01, v02].Decode(DataCodec) -> custom data - -Working with inner data, - -Sending: -cloudevents.[v01, v02].data -> DataCodec.Decode -> custom data - -Receiving: -custom data -> DataCodec.Encode -> cloudevents.[v01, v02].data,contentType - -Data Codecs supported: -[json, xml, base64, text] - - -This imples that there is only one canonical form and it evolves and is marked -deprecated as the model evolves. - -Spec says: - -Event[Context, Data] -> Message - -Http Message should have: - -Headers -Body -ContentType -ContentLength - - -*/ diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/event.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/event.go index 5964d612484..4389a5de9b9 100644 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/event.go +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/event.go @@ -1,8 +1,11 @@ package cloudevents import ( + "bytes" + "encoding/json" "fmt" "github.com/cloudevents/sdk-go/pkg/cloudevents/datacodec" + "sort" "strings" ) @@ -43,31 +46,111 @@ func (e Event) Validate() error { } func (e Event) String() string { - sb := strings.Builder{} + b := strings.Builder{} - if s := e.SpecVersion(); s != "" { - if sb.Len() > 0 { - sb.WriteString("\n") - } - sb.WriteString("SpecVersion: ") - sb.WriteString(s) + b.WriteString("Validation: ") + + valid := e.Validate() + if valid == nil { + b.WriteString("valid\n") + } else { + b.WriteString("invalid\n") + } + if valid != nil { + b.WriteString(fmt.Sprintf("Validation Error: \n%s\n", valid.Error())) } - if s := e.Type(); s != "" { - if sb.Len() > 0 { - sb.WriteString("\n") + b.WriteString("Context Attributes,\n") + + var extensions map[string]interface{} + + switch e.SpecVersion() { + case CloudEventsVersionV01: + if ec, ok := e.Context.(EventContextV01); ok { + b.WriteString(" cloudEventsVersion: " + ec.CloudEventsVersion + "\n") + b.WriteString(" eventType: " + ec.EventType + "\n") + if ec.EventTypeVersion != nil { + b.WriteString(" eventTypeVersion: " + *ec.EventTypeVersion + "\n") + } + b.WriteString(" source: " + ec.Source.String() + "\n") + b.WriteString(" eventID: " + ec.EventID + "\n") + if ec.EventTime != nil { + b.WriteString(" eventTime: " + ec.EventTime.String() + "\n") + } + if ec.SchemaURL != nil { + b.WriteString(" schemaURL: " + ec.SchemaURL.String() + "\n") + } + if ec.ContentType != nil { + b.WriteString(" contentType: " + *ec.ContentType + "\n") + } + extensions = ec.Extensions + } + + case CloudEventsVersionV02: + if ec, ok := e.Context.(EventContextV02); ok { + b.WriteString(" specversion: " + ec.SpecVersion + "\n") + b.WriteString(" type: " + ec.Type + "\n") + b.WriteString(" source: " + ec.Source.String() + "\n") + b.WriteString(" id: " + ec.ID + "\n") + if ec.Time != nil { + b.WriteString(" time: " + ec.Time.String() + "\n") + } + if ec.SchemaURL != nil { + b.WriteString(" schemaurl: " + ec.SchemaURL.String() + "\n") + } + if ec.ContentType != nil { + b.WriteString(" contenttype: " + *ec.ContentType + "\n") + } + extensions = ec.Extensions + } + + case CloudEventsVersionV03: + if ec, ok := e.Context.(EventContextV03); ok { + b.WriteString(" specversion: " + ec.SpecVersion + "\n") + b.WriteString(" type: " + ec.Type + "\n") + b.WriteString(" source: " + ec.Source.String() + "\n") + b.WriteString(" id: " + ec.ID + "\n") + if ec.Time != nil { + b.WriteString(" time: " + ec.Time.String() + "\n") + } + if ec.SchemaURL != nil { + b.WriteString(" schemaurl: " + ec.SchemaURL.String() + "\n") + } + if ec.DataContentType != nil { + b.WriteString(" datacontenttype: " + *ec.DataContentType + "\n") + } + extensions = ec.Extensions } - sb.WriteString("Type: ") - sb.WriteString(s) + default: + b.WriteString(e.String() + "\n") } - if s := e.DataContentType(); s != "" { - if sb.Len() > 0 { - sb.WriteString("\n") + if extensions != nil && len(extensions) > 0 { + b.WriteString("Extensions,\n") + keys := make([]string, 0, len(extensions)) + for k := range extensions { + keys = append(keys, k) + } + sort.Strings(keys) + for _, key := range keys { + b.WriteString(fmt.Sprintf(" %s: %v\n", key, extensions[key])) } - sb.WriteString("DataContentType: ") - sb.WriteString(s) } - return sb.String() + if e.Data != nil { + b.WriteString("Data,\n ") + if strings.HasPrefix(e.DataContentType(), "application/json") { + var prettyJSON bytes.Buffer + err := json.Indent(&prettyJSON, e.Data.([]byte), " ", " ") + if err != nil { + b.Write(e.Data.([]byte)) + } else { + b.Write(prettyJSON.Bytes()) + } + } else { + b.Write(e.Data.([]byte)) + } + b.WriteString("\n") + } + return b.String() } diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/observability/keys.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/observability/keys.go new file mode 100644 index 00000000000..d9f961e86c7 --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/observability/keys.go @@ -0,0 +1,15 @@ +package observability + +import ( + "go.opencensus.io/tag" +) + +var ( + KeyMethod, _ = tag.NewKey("method") + KeyResult, _ = tag.NewKey("result") +) + +const ( + ResultError = "error" + ResultOK = "success" +) diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/observability/observer.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/observability/observer.go new file mode 100644 index 00000000000..944059b5c37 --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/observability/observer.go @@ -0,0 +1,88 @@ +package observability + +import ( + "context" + "sync" + "time" + + "go.opencensus.io/stats" + "go.opencensus.io/tag" + "go.opencensus.io/trace" +) + +// Observable represents the the customization used by the Reporter for a given +// measurement and trace for a single method. +type Observable interface { + TraceName() string + MethodName() string + LatencyMs() *stats.Float64Measure +} + +// Reporter represents a running latency counter and trace span. When Error or +// OK are called, the latency is calculated and the trace space is ended. Error +// or OK are only allowed to be called once. +type Reporter interface { + Error() + OK() +} + +type reporter struct { + ctx context.Context + span *trace.Span + on Observable + start time.Time + measure stats.Measure + once sync.Once +} + +// All tags used for Latency measurements. +func LatencyTags() []tag.Key { + return []tag.Key{KeyMethod, KeyResult} +} + +func NewReporter(ctx context.Context, on Observable) (context.Context, Reporter) { + ctx, span := trace.StartSpan(ctx, on.TraceName()) + r := &reporter{ + ctx: ctx, + on: on, + span: span, + start: time.Now(), + } + r.tagMethod() + return ctx, r +} + +func (r *reporter) tagMethod() { + var err error + r.ctx, err = tag.New(r.ctx, tag.Insert(KeyMethod, r.on.MethodName())) + if err != nil { + panic(err) // or ignore? + } +} + +func (r *reporter) record() { + ms := float64(time.Since(r.start) / time.Millisecond) + stats.Record(r.ctx, r.on.LatencyMs().M(ms)) + r.span.End() +} + +func (r *reporter) Error() { + r.once.Do(func() { + r.result(ResultError) + }) +} + +func (r *reporter) OK() { + r.once.Do(func() { + r.result(ResultOK) + }) +} + +func (r *reporter) result(v string) { + var err error + r.ctx, err = tag.New(r.ctx, tag.Insert(KeyResult, v)) + if err != nil { + panic(err) // or ignore? + } + r.record() +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/doc.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/doc.go new file mode 100644 index 00000000000..b93cd60a8cf --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/doc.go @@ -0,0 +1,5 @@ +/* +Package transport is the toplevel package to define interfaces that the client and codec packages use to decouple from +the transport implementations. +*/ +package transport diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/codec_v01.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/codec_v01.go index 9be36a4b2d4..7a23d707d6e 100644 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/codec_v01.go +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/codec_v01.go @@ -1,10 +1,12 @@ package http import ( + "context" "encoding/json" "fmt" "github.com/cloudevents/sdk-go/pkg/cloudevents" "github.com/cloudevents/sdk-go/pkg/cloudevents/codec" + "github.com/cloudevents/sdk-go/pkg/cloudevents/observability" "github.com/cloudevents/sdk-go/pkg/cloudevents/transport" "github.com/cloudevents/sdk-go/pkg/cloudevents/types" "log" @@ -20,6 +22,18 @@ type CodecV01 struct { var _ transport.Codec = (*CodecV01)(nil) func (v CodecV01) Encode(e cloudevents.Event) (transport.Message, error) { + // TODO: wire context + _, r := observability.NewReporter(context.Background(), CodecObserved{o: ReportEncode, c: v.Encoding.Codec()}) + m, err := v.obsEncode(e) + if err != nil { + r.Error() + } else { + r.OK() + } + return m, err +} + +func (v CodecV01) obsEncode(e cloudevents.Event) (transport.Message, error) { switch v.Encoding { case Default: fallthrough @@ -33,6 +47,18 @@ func (v CodecV01) Encode(e cloudevents.Event) (transport.Message, error) { } func (v CodecV01) Decode(msg transport.Message) (*cloudevents.Event, error) { + // TODO: wire context + _, r := observability.NewReporter(context.Background(), CodecObserved{o: ReportDecode, c: v.inspectEncoding(msg).Codec()}) // TODO: inspectEncoding is not free. + e, err := v.obsDecode(msg) + if err != nil { + r.Error() + } else { + r.OK() + } + return e, err +} + +func (v CodecV01) obsDecode(msg transport.Message) (*cloudevents.Event, error) { switch v.inspectEncoding(msg) { case BinaryV01: return v.decodeBinary(msg) diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/codec_v02.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/codec_v02.go index 4c10ac9868c..5377a973494 100644 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/codec_v02.go +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/codec_v02.go @@ -1,10 +1,12 @@ package http import ( + "context" "encoding/json" "fmt" "github.com/cloudevents/sdk-go/pkg/cloudevents" "github.com/cloudevents/sdk-go/pkg/cloudevents/codec" + "github.com/cloudevents/sdk-go/pkg/cloudevents/observability" "github.com/cloudevents/sdk-go/pkg/cloudevents/transport" "github.com/cloudevents/sdk-go/pkg/cloudevents/types" "log" @@ -20,6 +22,18 @@ type CodecV02 struct { var _ transport.Codec = (*CodecV02)(nil) func (v CodecV02) Encode(e cloudevents.Event) (transport.Message, error) { + // TODO: wire context + _, r := observability.NewReporter(context.Background(), CodecObserved{o: ReportEncode, c: v.Encoding.Codec()}) + m, err := v.obsEncode(e) + if err != nil { + r.Error() + } else { + r.OK() + } + return m, err +} + +func (v CodecV02) obsEncode(e cloudevents.Event) (transport.Message, error) { switch v.Encoding { case Default: fallthrough @@ -33,6 +47,18 @@ func (v CodecV02) Encode(e cloudevents.Event) (transport.Message, error) { } func (v CodecV02) Decode(msg transport.Message) (*cloudevents.Event, error) { + // TODO: wire context + _, r := observability.NewReporter(context.Background(), CodecObserved{o: ReportDecode, c: v.inspectEncoding(msg).Codec()}) // TODO: inspectEncoding is not free. + e, err := v.obsDecode(msg) + if err != nil { + r.Error() + } else { + r.OK() + } + return e, err +} + +func (v CodecV02) obsDecode(msg transport.Message) (*cloudevents.Event, error) { switch v.inspectEncoding(msg) { case BinaryV02: return v.decodeBinary(msg) diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/codec_v03.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/codec_v03.go index 78cfd2011d5..0ceee6aa90f 100644 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/codec_v03.go +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/codec_v03.go @@ -1,10 +1,12 @@ package http import ( + "context" "encoding/json" "fmt" "github.com/cloudevents/sdk-go/pkg/cloudevents" "github.com/cloudevents/sdk-go/pkg/cloudevents/codec" + "github.com/cloudevents/sdk-go/pkg/cloudevents/observability" "github.com/cloudevents/sdk-go/pkg/cloudevents/transport" "github.com/cloudevents/sdk-go/pkg/cloudevents/types" "log" @@ -20,6 +22,18 @@ type CodecV03 struct { var _ transport.Codec = (*CodecV03)(nil) func (v CodecV03) Encode(e cloudevents.Event) (transport.Message, error) { + // TODO: wire context + _, r := observability.NewReporter(context.Background(), CodecObserved{o: ReportEncode, c: v.Encoding.Codec()}) + m, err := v.obsEncode(e) + if err != nil { + r.Error() + } else { + r.OK() + } + return m, err +} + +func (v CodecV03) obsEncode(e cloudevents.Event) (transport.Message, error) { switch v.Encoding { case Default: fallthrough @@ -35,6 +49,18 @@ func (v CodecV03) Encode(e cloudevents.Event) (transport.Message, error) { } func (v CodecV03) Decode(msg transport.Message) (*cloudevents.Event, error) { + // TODO: wire context + _, r := observability.NewReporter(context.Background(), CodecObserved{o: ReportDecode, c: v.inspectEncoding(msg).Codec()}) // TODO: inspectEncoding is not free. + e, err := v.obsDecode(msg) + if err != nil { + r.Error() + } else { + r.OK() + } + return e, err +} + +func (v CodecV03) obsDecode(msg transport.Message) (*cloudevents.Event, error) { switch v.inspectEncoding(msg) { case BinaryV03: return v.decodeBinary(msg) diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/context.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/context.go index 70ac8de8ec0..b0a397e210e 100644 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/context.go +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/context.go @@ -2,6 +2,9 @@ package http import ( "context" + "fmt" + "net/http" + "strings" ) // TransportContext allows a Receiver to understand the context of a request. @@ -9,6 +12,92 @@ type TransportContext struct { URI string Host string Method string + Header http.Header + + // IgnoreHeaderPrefixes controls what comes back from AttendToHeaders. + // AttendToHeaders controls what is output for .String() + IgnoreHeaderPrefixes []string +} + +func NewTransportContext(req *http.Request) TransportContext { + var tx *TransportContext + if req != nil { + tx = &TransportContext{ + URI: req.RequestURI, + Host: req.Host, + Method: req.Method, + Header: req.Header, + } + } else { + tx = &TransportContext{} + } + tx.AddIgnoreHeaderPrefix("accept-encoding", "user-agent", "connection", "content-type") + return *tx +} + +// AttendToHeaders returns the list of headers that exist in the TransportContext that are not currently in +// tx.IgnoreHeaderPrefix. +func (tx TransportContext) AttendToHeaders() []string { + a := []string(nil) + if tx.Header != nil && len(tx.Header) > 0 { + for k := range tx.Header { + if tx.shouldIgnoreHeader(k) { + continue + } + a = append(a, k) + } + } + return a +} + +func (tx TransportContext) shouldIgnoreHeader(h string) bool { + for _, v := range tx.IgnoreHeaderPrefixes { + if strings.HasPrefix(strings.ToLower(h), strings.ToLower(v)) { + return true + } + } + return false +} + +// String generates a pretty-printed version of the resource as a string. +func (tx TransportContext) String() string { + b := strings.Builder{} + + b.WriteString("Transport Context,\n") + + empty := b.Len() + + if tx.URI != "" { + b.WriteString(" URI: " + tx.URI + "\n") + } + if tx.Host != "" { + b.WriteString(" Host: " + tx.Host + "\n") + } + + if tx.Method != "" { + b.WriteString(" Method: " + tx.Method + "\n") + } + + if tx.Header != nil && len(tx.Header) > 0 { + b.WriteString(" Header:\n") + for _, k := range tx.AttendToHeaders() { + b.WriteString(fmt.Sprintf(" %s: %s\n", k, tx.Header.Get(k))) + } + } + + if b.Len() == empty { + b.WriteString(" nil\n") + } + + return b.String() +} + +// AddIgnoreHeaderPrefix controls what header key is to be attended to and/or printed. +func (tx *TransportContext) AddIgnoreHeaderPrefix(prefix ...string) { + if tx.IgnoreHeaderPrefixes == nil { + tx.IgnoreHeaderPrefixes = []string(nil) + } + tx.IgnoreHeaderPrefixes = append(tx.IgnoreHeaderPrefixes, prefix...) } // Opaque key type used to store TransportContext @@ -16,6 +105,7 @@ type transportContextKeyType struct{} var transportContextKey = transportContextKeyType{} +// WithTransportContext return a context with the given TransportContext into the provided context object. func WithTransportContext(ctx context.Context, tcxt TransportContext) context.Context { return context.WithValue(ctx, transportContextKey, tcxt) } @@ -32,3 +122,27 @@ func TransportContextFrom(ctx context.Context) TransportContext { } return TransportContext{} } + +// Opaque key type used to store Headers +type headerKeyType struct{} + +var headerKey = headerKeyType{} + +// ContextWithHeader returns a context with a header added to the given context. +// Can be called multiple times to set multiple header key/value pairs. +func ContextWithHeader(ctx context.Context, key, value string) context.Context { + header := HeaderFrom(ctx) + header.Add(key, value) + return context.WithValue(ctx, headerKey, header) +} + +// HeaderFrom extracts the header oject in the given context. Always returns a non-nil Header. +func HeaderFrom(ctx context.Context) http.Header { + header := ctx.Value(headerKey) + if header != nil { + if h, ok := header.(http.Header); ok { + return h + } + } + return http.Header{} +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/doc.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/doc.go new file mode 100644 index 00000000000..7a8df0e88a2 --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/doc.go @@ -0,0 +1,4 @@ +/* +Package transport/http implements the CloudEvent transport implementation using HTTP. +*/ +package http diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/encoding.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/encoding.go index e16483f9a15..92017cbf372 100644 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/encoding.go +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/encoding.go @@ -74,3 +74,34 @@ func (e Encoding) Version() string { return "Unknown" } } + +func (e Encoding) Codec() string { + switch e { + case Default: + return "default" + + // Version 0.1 + case BinaryV01: + return "binary/v0.1" + case StructuredV01: + return "structured/v0.1" + + // Version 0.2 + case BinaryV02: + return "binary/v0.3" + case StructuredV02: + return "structured/v0.2" + + // Version 0.3 + case BinaryV03: + return "binary/v0.3" + case StructuredV03: + return "structured/v0.3" + case BatchedV03: + return "batched/v0.3" + + // Unknown + default: + return "unknown" + } +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/observability.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/observability.go new file mode 100644 index 00000000000..b87ca8dd998 --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/observability.go @@ -0,0 +1,93 @@ +package http + +import ( + "fmt" + "github.com/cloudevents/sdk-go/pkg/cloudevents/observability" + "go.opencensus.io/stats" + "go.opencensus.io/stats/view" +) + +var ( + LatencyMs = stats.Float64( + "transport/http/latency", + "The latency in milliseconds for the http transport methods for CloudEvents.", + "ms") +) + +var ( + LatencyView = &view.View{ + Name: "transport/http/latency", + Measure: LatencyMs, + Description: "The distribution of latency inside of http transport for CloudEvents.", + Aggregation: view.Distribution(0, .01, .1, 1, 10, 100, 1000, 10000), + TagKeys: observability.LatencyTags(), + } +) + +type Observed int32 + +const ( + ReportSend Observed = iota + ReportReceive + ReportServeHTTP + ReportEncode + ReportDecode +) + +func (o Observed) TraceName() string { + switch o { + case ReportSend: + return "transport/http/send" + case ReportReceive: + return "transport/http/receive" + case ReportServeHTTP: + return "transport/http/servehttp" + case ReportEncode: + return "transport/http/encode" + case ReportDecode: + return "transport/http/decode" + default: + return "transport/http/unknown" + } +} + +func (o Observed) MethodName() string { + switch o { + case ReportSend: + return "send" + case ReportReceive: + return "receive" + case ReportServeHTTP: + return "servehttp" + case ReportEncode: + return "encode" + case ReportDecode: + return "decode" + default: + return "unknown" + } +} + +func (o Observed) LatencyMs() *stats.Float64Measure { + return LatencyMs +} + +// CodecObserved is a wrapper to append version to Observed. +type CodecObserved struct { + // Method + o Observed + // Codec + c string +} + +func (c CodecObserved) TraceName() string { + return fmt.Sprintf("%s/%s", c.o.TraceName(), c.c) +} + +func (c CodecObserved) MethodName() string { + return fmt.Sprintf("%s/%s", c.o.MethodName(), c.c) +} + +func (c CodecObserved) LatencyMs() *stats.Float64Measure { + return c.o.LatencyMs() +} diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/options.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/options.go index ea54869c746..0f8e70901b2 100644 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/options.go +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/options.go @@ -5,6 +5,7 @@ import ( nethttp "net/http" "net/url" "strings" + "time" ) type Option func(*Transport) error @@ -56,6 +57,39 @@ func WithMethod(method string) Option { } } +// WithHeader sets an additional default outbound header for all cloudevents +// when using an HTTP request. +func WithHeader(key, value string) Option { + return func(t *Transport) error { + if t == nil { + return fmt.Errorf("http header option can not set nil transport") + } + key = strings.TrimSpace(key) + if key != "" { + if t.Req == nil { + t.Req = &nethttp.Request{} + } + if t.Req.Header == nil { + t.Req.Header = nethttp.Header{} + } + t.Req.Header.Add(key, value) + return nil + } + return fmt.Errorf("http header option was empty string") + } +} + +// WithShutdownTimeout sets the shutdown timeout when the http server is being shutdown. +func WithShutdownTimeout(timeout time.Duration) Option { + return func(t *Transport) error { + if t == nil { + return fmt.Errorf("http shutdown timeout option can not set nil transport") + } + t.ShutdownTimeout = &timeout + return nil + } +} + // WithEncoding sets the encoding for clients with HTTP transports. func WithEncoding(encoding Encoding) Option { return func(t *Transport) error { diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/transport.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/transport.go index 8f0f8d5dcf7..90b1c2ec8ef 100644 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/transport.go +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/transport/http/transport.go @@ -4,12 +4,14 @@ import ( "bytes" "context" "fmt" + "github.com/cloudevents/sdk-go/pkg/cloudevents/observability" "io/ioutil" "log" "net" "net/http" "strings" "sync" + "time" "github.com/cloudevents/sdk-go/pkg/cloudevents" cecontext "github.com/cloudevents/sdk-go/pkg/cloudevents/context" @@ -21,11 +23,20 @@ type EncodingSelector func(e cloudevents.Event) Encoding // type check that this transport message impl matches the contract var _ transport.Transport = (*Transport)(nil) +const ( + // DefaultShutdownTimeout defines the default timeout given to the http.Server when calling Shutdown. + DefaultShutdownTimeout = time.Minute * 1 +) + // Transport acts as both a http client and a http handler. type Transport struct { Encoding Encoding DefaultEncodingSelectionFn EncodingSelector + // ShutdownTimeout defines the timeout given to the http.Server when calling Shutdown. + // If nil, DefaultShutdownTimeout is used. + ShutdownTimeout *time.Duration + // Sending Client *http.Client Req *http.Request @@ -82,17 +93,42 @@ func (t *Transport) loadCodec() bool { return true } +func copyHeaders(from, to http.Header) { + if from == nil || to == nil { + return + } + for header, values := range from { + for _, value := range values { + to.Add(header, value) + } + } +} + func (t *Transport) Send(ctx context.Context, event cloudevents.Event) (*cloudevents.Event, error) { + ctx, r := observability.NewReporter(ctx, ReportSend) + resp, err := t.obsSend(ctx, event) + if err != nil { + r.Error() + } else { + r.OK() + } + return resp, err +} + +func (t *Transport) obsSend(ctx context.Context, event cloudevents.Event) (*cloudevents.Event, error) { if t.Client == nil { t.crMu.Lock() t.Client = &http.Client{} t.crMu.Unlock() } - var req http.Request + req := http.Request{ + Header: HeaderFrom(ctx), + } if t.Req != nil { req.Method = t.Req.Method req.URL = t.Req.URL + copyHeaders(t.Req.Header, req.Header) } // Override the default request with target from context. @@ -109,11 +145,11 @@ func (t *Transport) Send(ctx context.Context, event cloudevents.Event) (*cloudev return nil, err } - // TODO: merge the incoming request with msg, for now just replace. if m, ok := msg.(*Message); ok { - req.Header = m.Header + copyHeaders(m.Header, req.Header) req.Body = ioutil.NopCloser(bytes.NewBuffer(m.Body)) req.ContentLength = int64(len(m.Body)) + req.Close = true return httpDo(ctx, &req, func(resp *http.Response, err error) (*cloudevents.Event, error) { if err != nil { return nil, err @@ -191,7 +227,13 @@ func (t *Transport) StartReceiver(ctx context.Context) error { select { case <-ctx.Done(): // Try a gracefully shutdown. - return t.server.Shutdown(context.Background()) + timeout := DefaultShutdownTimeout + if t.ShutdownTimeout != nil { + timeout = *t.ShutdownTimeout + } + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + return t.server.Shutdown(ctx) case err := <-errChan: return err } @@ -238,6 +280,17 @@ func status(resp *http.Response) string { } func (t *Transport) invokeReceiver(ctx context.Context, event cloudevents.Event) (*Response, error) { + ctx, r := observability.NewReporter(ctx, ReportReceive) + resp, err := t.obsInvokeReceiver(ctx, event) + if err != nil { + r.Error() + } else { + r.OK() + } + return resp, err +} + +func (t *Transport) obsInvokeReceiver(ctx context.Context, event cloudevents.Event) (*Response, error) { if t.Receiver != nil { // Note: http does not use eventResp.Reason eventResp := cloudevents.EventResponse{} @@ -277,11 +330,14 @@ func (t *Transport) invokeReceiver(ctx context.Context, event cloudevents.Event) // ServeHTTP implements http.Handler func (t *Transport) ServeHTTP(w http.ResponseWriter, req *http.Request) { + ctx, r := observability.NewReporter(req.Context(), ReportServeHTTP) + body, err := ioutil.ReadAll(req.Body) if err != nil { log.Printf("failed to handle request: %s %v", err, req) w.WriteHeader(http.StatusBadRequest) w.Write([]byte(`{"error":"Invalid request"}`)) + r.Error() return } msg := &Message{ @@ -294,6 +350,7 @@ func (t *Transport) ServeHTTP(w http.ResponseWriter, req *http.Request) { log.Printf("failed to load codec: %s", err) w.WriteHeader(http.StatusBadRequest) w.Write([]byte(fmt.Sprintf(`{"error":%q}`, err.Error()))) + r.Error() return } event, err := t.codec.Decode(msg) @@ -301,23 +358,19 @@ func (t *Transport) ServeHTTP(w http.ResponseWriter, req *http.Request) { log.Printf("failed to decode message: %s %v", err, req) w.WriteHeader(http.StatusBadRequest) w.Write([]byte(fmt.Sprintf(`{"error":%q}`, err.Error()))) + r.Error() return } - ctx := req.Context() - if req != nil { - ctx = WithTransportContext(ctx, TransportContext{ - URI: req.RequestURI, - Host: req.Host, - Method: req.Method, - }) + ctx = WithTransportContext(ctx, NewTransportContext(req)) } resp, err := t.invokeReceiver(ctx, *event) if err != nil { w.WriteHeader(http.StatusBadRequest) w.Write([]byte(fmt.Sprintf(`{"error":%q}`, err.Error()))) + r.Error() return } if resp != nil { @@ -328,16 +381,24 @@ func (t *Transport) ServeHTTP(w http.ResponseWriter, req *http.Request) { } } } + status := http.StatusAccepted if resp.StatusCode >= 200 && resp.StatusCode < 600 { - w.WriteHeader(resp.StatusCode) + status = resp.StatusCode } + w.WriteHeader(status) if len(resp.Body) > 0 { - w.Write(resp.Body) + if _, err := w.Write(resp.Body); err != nil { + r.Error() + return + } } + + r.OK() return } w.WriteHeader(http.StatusNoContent) + r.OK() } func (t *Transport) GetPort() int { diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/types/doc.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/types/doc.go new file mode 100644 index 00000000000..1019b4a2dd2 --- /dev/null +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/types/doc.go @@ -0,0 +1,4 @@ +/* +Package types provides custom types to support CloudEvents. +*/ +package types diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/types/timestamp.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/types/timestamp.go index c87e24804d3..c11e4e15a6d 100644 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/types/timestamp.go +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/types/timestamp.go @@ -7,10 +7,15 @@ import ( "time" ) +// Timestamp wraps time.Time to normalize the time layout to RFC3339. It is +// intended to enforce compliance with the CloudEvents spec for their +// definition of Timestamp. Custom marshal methods are implemented to ensure +// the outbound Timestamp is a string in the RFC3339 layout. type Timestamp struct { time.Time } +// ParseTimestamp attempts to parse the given time assuming RFC3339 layout func ParseTimestamp(t string) *Timestamp { if t == "" { return nil @@ -22,7 +27,8 @@ func ParseTimestamp(t string) *Timestamp { return &Timestamp{Time: timestamp} } -// This allows json marshaling to always be in RFC3339Nano format. +// MarshalJSON implements a custom json marshal method used when this type is +// marshaled using json.Marshal. func (t *Timestamp) MarshalJSON() ([]byte, error) { if t == nil || t.IsZero() { return []byte(`""`), nil @@ -31,6 +37,8 @@ func (t *Timestamp) MarshalJSON() ([]byte, error) { return []byte(rfc3339), nil } +// UnmarshalJSON implements the json unmarshal method used when this type is +// unmarshed using json.Unmarshal. func (t *Timestamp) UnmarshalJSON(b []byte) error { var timestamp string if err := json.Unmarshal(b, ×tamp); err != nil { @@ -42,6 +50,8 @@ func (t *Timestamp) UnmarshalJSON(b []byte) error { return nil } +// MarshalXML implements a custom xml marshal method used when this type is +// marshaled using xml.Marshal. func (t *Timestamp) MarshalXML(e *xml.Encoder, start xml.StartElement) error { if t == nil || t.IsZero() { return e.EncodeElement(nil, start) @@ -50,6 +60,8 @@ func (t *Timestamp) MarshalXML(e *xml.Encoder, start xml.StartElement) error { return e.EncodeElement(v, start) } +// UnmarshalXML implements the xml unmarshal method used when this type is +// unmarshed using xml.Unmarshal. func (t *Timestamp) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { var timestamp string if err := d.DecodeElement(×tamp, &start); err != nil { @@ -61,6 +73,7 @@ func (t *Timestamp) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { return nil } +// String outputs the time using layout RFC3339. func (t *Timestamp) String() string { if t == nil { return time.Time{}.UTC().Format(time.RFC3339Nano) diff --git a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/types/urlref.go b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/types/urlref.go index f79fafa0860..d1cc2703645 100644 --- a/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/types/urlref.go +++ b/vendor/github.com/cloudevents/sdk-go/pkg/cloudevents/types/urlref.go @@ -7,10 +7,15 @@ import ( "net/url" ) +// URLRef is a wrapper to url.URL. It is intended to enforce compliance with +// the CloudEvents spec for their definition of URI-Reference. Custom +// marshal methods are implemented to ensure the outbound URLRef object is +// is a flat string. type URLRef struct { url.URL } +// ParseURLRef attempts to parse the given string as a URI-Reference. func ParseURLRef(u string) *URLRef { if u == "" { return nil @@ -22,12 +27,15 @@ func ParseURLRef(u string) *URLRef { return &URLRef{URL: *pu} } -// This allows json marshaling to always be in RFC3339Nano format. +// MarshalJSON implements a custom json marshal method used when this type is +// marshaled using json.Marshal. func (u URLRef) MarshalJSON() ([]byte, error) { b := fmt.Sprintf("%q", u.String()) return []byte(b), nil } +// UnmarshalJSON implements the json unmarshal method used when this type is +// unmarshed using json.Unmarshal. func (u *URLRef) UnmarshalJSON(b []byte) error { var ref string if err := json.Unmarshal(b, &ref); err != nil { @@ -40,11 +48,15 @@ func (u *URLRef) UnmarshalJSON(b []byte) error { return nil } +// MarshalXML implements a custom xml marshal method used when this type is +// marshaled using xml.Marshal. func (u URLRef) MarshalXML(e *xml.Encoder, start xml.StartElement) error { v := fmt.Sprintf("%s", u.String()) return e.EncodeElement(v, start) } +// UnmarshalXML implements the xml unmarshal method used when this type is +// unmarshed using xml.Unmarshal. func (u *URLRef) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { var ref string if err := d.DecodeElement(&ref, &start); err != nil { @@ -57,6 +69,7 @@ func (u *URLRef) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { return nil } +// String returns the full string representation of the URI-Reference. func (u *URLRef) String() string { if u == nil { return "" From f94d5a80e506d165ebb326f6964f40b356a27834 Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Wed, 20 Mar 2019 12:44:03 -0700 Subject: [PATCH 125/128] Forward tracing headers. --- pkg/broker/receiver.go | 48 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/pkg/broker/receiver.go b/pkg/broker/receiver.go index 8d06df8c9d9..469fcbd765f 100644 --- a/pkg/broker/receiver.go +++ b/pkg/broker/receiver.go @@ -21,6 +21,7 @@ import ( "errors" "net/http" "net/url" + "strings" "time" "github.com/cloudevents/sdk-go/pkg/cloudevents" @@ -40,6 +41,22 @@ const ( writeTimeout = 1 * time.Minute ) +var ( + // These MUST be lowercase strings, as they will be compared against lowercase strings. + forwardHeaders = []string{ + // tracing + "x-request-id", + } + // These MUST be lowercase strings, as they will be compared against lowercase strings. + forwardPrefixes = []string{ + // knative + "knative-", + // tracing + "x-b3-", + "x-ot-", + } +) + // Receiver parses Cloud Events, determines if they pass a filter, and sends them to a subscriber. type Receiver struct { logger *zap.Logger @@ -142,7 +159,7 @@ func (r *Receiver) serveHTTP(ctx context.Context, event cloudevents.Event, resp r.logger.Debug("Received message", zap.Any("triggerRef", triggerRef)) - responseEvent, err := r.sendEvent(ctx, triggerRef, &event) + responseEvent, err := r.sendEvent(ctx, tctx, triggerRef, &event) if err != nil { r.logger.Error("Error sending the event", zap.Error(err)) return err @@ -153,7 +170,7 @@ func (r *Receiver) serveHTTP(ctx context.Context, event cloudevents.Event, resp } // sendEvent sends an event to a subscriber if the trigger filter passes. -func (r *Receiver) sendEvent(ctx context.Context, trigger provisioners.ChannelReference, event *cloudevents.Event) (*cloudevents.Event, error) { +func (r *Receiver) sendEvent(ctx context.Context, tctx cehttp.TransportContext, trigger provisioners.ChannelReference, event *cloudevents.Event) (*cloudevents.Event, error) { t, err := r.getTrigger(ctx, trigger) if err != nil { r.logger.Info("Unable to get the Trigger", zap.Error(err), zap.Any("triggerRef", trigger)) @@ -179,6 +196,7 @@ func (r *Receiver) sendEvent(ctx context.Context, trigger provisioners.ChannelRe } sendingCtx := cecontext.WithTarget(ctx, subscriberURI.String()) + sendingCtx = addFilteredHeaders(sendingCtx, tctx) return r.ceHttp.Send(sendingCtx, *event) } @@ -214,3 +232,29 @@ func (r *Receiver) shouldSendMessage(ts *eventingv1alpha1.TriggerSpec, event *cl } return true } + +func addFilteredHeaders(ctx context.Context, tctx cehttp.TransportContext) context.Context { + // Helper function that adds the header name and all its values. + addHeader := func(ctx context.Context, n string, v []string) context.Context { + for _, iv := range v { + ctx = cehttp.ContextWithHeader(ctx, n, iv) + } + return ctx + } + + for n, v := range tctx.Header { + lower := strings.ToLower(n) + for _, header := range forwardHeaders { + if lower == header { + ctx = addHeader(ctx, n, v) + continue + } + } + for _, prefix := range forwardPrefixes { + if strings.HasPrefix(lower, prefix) { + ctx = addHeader(ctx, n, v) + } + } + } + return ctx +} From a6f6bd44f489e11c751d98d20ea7c3311bd55995 Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Wed, 20 Mar 2019 12:46:40 -0700 Subject: [PATCH 126/128] Update comment. --- Gopkg.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gopkg.toml b/Gopkg.toml index a10eeb4ef60..1007ae38aea 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -131,7 +131,7 @@ required = [ name = "github.com/nats-io/nats-streaming-server" version = "0.11.0" -# The latest release as of March 12, 2019. +# The latest release as of March 20, 2019. [[constraint]] name = "github.com/cloudevents/sdk-go" version = "=0.4.1" From 591fd60f218752c648807dfce4ed19d4d5a6288b Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Wed, 20 Mar 2019 13:42:49 -0700 Subject: [PATCH 127/128] Relearning why message_dispatcher was written the way it was. --- pkg/broker/receiver.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/broker/receiver.go b/pkg/broker/receiver.go index 469fcbd765f..f639dd6c472 100644 --- a/pkg/broker/receiver.go +++ b/pkg/broker/receiver.go @@ -32,6 +32,7 @@ import ( "github.com/knative/eventing/pkg/provisioners" "go.uber.org/zap" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -43,10 +44,10 @@ const ( var ( // These MUST be lowercase strings, as they will be compared against lowercase strings. - forwardHeaders = []string{ + forwardHeaders = sets.NewString( // tracing "x-request-id", - } + ) // These MUST be lowercase strings, as they will be compared against lowercase strings. forwardPrefixes = []string{ // knative @@ -244,15 +245,14 @@ func addFilteredHeaders(ctx context.Context, tctx cehttp.TransportContext) conte for n, v := range tctx.Header { lower := strings.ToLower(n) - for _, header := range forwardHeaders { - if lower == header { - ctx = addHeader(ctx, n, v) - continue - } + if forwardHeaders.Has(lower) { + ctx = addHeader(ctx, n, v) + continue } for _, prefix := range forwardPrefixes { if strings.HasPrefix(lower, prefix) { ctx = addHeader(ctx, n, v) + break } } } From f6ea5ac04a55c463cc23ddb5dcf63c629b2ae433 Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Wed, 20 Mar 2019 14:05:59 -0700 Subject: [PATCH 128/128] Add a TODO for where to make the change for response tracing too. --- pkg/broker/receiver.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/broker/receiver.go b/pkg/broker/receiver.go index f639dd6c472..ca590b6fe34 100644 --- a/pkg/broker/receiver.go +++ b/pkg/broker/receiver.go @@ -167,6 +167,10 @@ func (r *Receiver) serveHTTP(ctx context.Context, event cloudevents.Event, resp } resp.Status = http.StatusAccepted resp.Event = responseEvent + + // TODO Add filtered headers (mostly tracing) to the response. We are waiting for CloudEvents + // SDK to allow this. + return nil }