From 1663eeb6780b78eb338335ce11128bc3af387db8 Mon Sep 17 00:00:00 2001 From: Scott Nichols Date: Fri, 10 Aug 2018 10:21:55 -0700 Subject: [PATCH 1/3] Convert github source to cloud events. --- pkg/sources/github/README.md | 11 ++++ pkg/sources/github/events.go | 22 +++++++ pkg/sources/github/eventsource.yaml | 3 +- pkg/sources/github/eventtype.yaml | 2 +- pkg/sources/github/{ => feedlet}/feedlet.go | 5 +- .../github/receive_adapter/receive_adapter.go | 62 +++++++++++++------ sample/github/legit.go | 22 ++----- 7 files changed, 86 insertions(+), 41 deletions(-) create mode 100644 pkg/sources/github/README.md create mode 100644 pkg/sources/github/events.go rename pkg/sources/github/{ => feedlet}/feedlet.go (99%) diff --git a/pkg/sources/github/README.md b/pkg/sources/github/README.md new file mode 100644 index 00000000000..25756cbd4f0 --- /dev/null +++ b/pkg/sources/github/README.md @@ -0,0 +1,11 @@ +GitHub +====== + +Produces the following eventtypes: + +- dev.knative.github.pullrequest + +Uses two images, the feedlet and the receive adapter. + +### Feedlet + diff --git a/pkg/sources/github/events.go b/pkg/sources/github/events.go new file mode 100644 index 00000000000..19ab61df30d --- /dev/null +++ b/pkg/sources/github/events.go @@ -0,0 +1,22 @@ +/* +Copyright 2018 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package github + +const ( + // Event Names + PullrequestEvent = "dev.knative.github.pullrequest" +) diff --git a/pkg/sources/github/eventsource.yaml b/pkg/sources/github/eventsource.yaml index ee8e685bfb2..a86410f0a4b 100644 --- a/pkg/sources/github/eventsource.yaml +++ b/pkg/sources/github/eventsource.yaml @@ -18,8 +18,7 @@ metadata: name: github namespace: default spec: - type: github source: github - image: github.com/knative/eventing/pkg/sources/github + image: github.com/knative/eventing/pkg/sources/github/feedlet parameters: image: github.com/knative/eventing/pkg/sources/github/receive_adapter diff --git a/pkg/sources/github/eventtype.yaml b/pkg/sources/github/eventtype.yaml index 049018a74a3..32069634693 100644 --- a/pkg/sources/github/eventtype.yaml +++ b/pkg/sources/github/eventtype.yaml @@ -15,7 +15,7 @@ apiVersion: feeds.knative.dev/v1alpha1 kind: EventType metadata: - name: pullrequest + name: dev.knative.github.pullrequest namespace: default spec: eventSource: github diff --git a/pkg/sources/github/feedlet.go b/pkg/sources/github/feedlet/feedlet.go similarity index 99% rename from pkg/sources/github/feedlet.go rename to pkg/sources/github/feedlet/feedlet.go index f4cbcfa83be..e1ee960ccea 100644 --- a/pkg/sources/github/feedlet.go +++ b/pkg/sources/github/feedlet/feedlet.go @@ -66,6 +66,9 @@ const ( secretNameKey = "secretName" // secretKeyKey is the name of key inside the secret that contains the GitHub credentials. secretKeyKey = "secretKey" + + // Event Names + pullrequestEvent = "dev.knative.github.pullrequest" ) type githubEventSource struct { @@ -452,7 +455,7 @@ func parseEventsFrom(eventType string) ([]string, error) { return []string(nil), fmt.Errorf("event type is empty") } switch eventType { - case "pullrequest": + case pullrequestEvent: return []string{"pull_request"}, nil // TODO: Add more supported event types. default: diff --git a/pkg/sources/github/receive_adapter/receive_adapter.go b/pkg/sources/github/receive_adapter/receive_adapter.go index 7ff51036574..018157b37cf 100644 --- a/pkg/sources/github/receive_adapter/receive_adapter.go +++ b/pkg/sources/github/receive_adapter/receive_adapter.go @@ -22,15 +22,17 @@ import ( "fmt" "os" - webhooks "gopkg.in/go-playground/webhooks.v3" - "gopkg.in/go-playground/webhooks.v3/github" + "gopkg.in/go-playground/webhooks.v3" + gh "gopkg.in/go-playground/webhooks.v3/github" - "bytes" "io/ioutil" "net/http" ghclient "github.com/google/go-github/github" + "github.com/knative/eventing/pkg/event" + "github.com/knative/eventing/pkg/sources/github" "log" + "time" ) const ( @@ -54,8 +56,27 @@ type GithubSecrets struct { // HandlePullRequest is invoked whenever a PullRequest is modified (created, updated, etc.) func (h *GithubHandler) HandlePullRequest(payload interface{}, header webhooks.Header) { log.Print("Handling Pull Request") - pl := payload.(github.PullRequestPayload) - postMessage(h.target, &pl) + pl := payload.(gh.PullRequestPayload) + + source := pl.PullRequest.HTMLURL + + var eventType string + if len(header["X-GitHub-Event"]) == 1 { + switch header["X-GitHub-Event"][0] { + case "pull_request": + eventType = github.PullrequestEvent + default: + eventType = "unsupported" + } + + } + + eventID := "unknown" + if len(header["X-GitHub-Delivery"]) == 1 { + eventID = header["X-GitHub-Delivery"][0] + } + + postMessage(h.target, &pl, source, eventType, eventID) } func main() { @@ -86,10 +107,10 @@ func main() { target: target, } - hook := github.New(&github.Config{Secret: credentials.SecretToken}) + hook := gh.New(&gh.Config{Secret: credentials.SecretToken}) // TODO: GitHub has more than just Pull Request Events. This needs to // handle them all? - hook.RegisterEvents(h.HandlePullRequest, github.PullRequestEvent) + hook.RegisterEvents(h.HandlePullRequest, gh.PullRequestEvent) // TODO(n3wscott): Do we need to configure the PORT? err = webhooks.Run(hook, ":8080", "/") @@ -98,30 +119,31 @@ func main() { } } -func postMessage(target string, m *github.PullRequestPayload) error { - jsonStr, err := json.Marshal(m) - if err != nil { - log.Printf("Error: Failed to marshal the message: %+v : %v", m, err) - return err - } - +func postMessage(target string, m *gh.PullRequestPayload, source, eventType, eventID string) error { URL := fmt.Sprintf("http://%s/", target) - req, err := http.NewRequest("POST", URL, bytes.NewBuffer(jsonStr)) + + ctx := event.EventContext{ + CloudEventsVersion: event.CloudEventsVersion, + EventType: eventType, + EventID: eventID, + EventTime: time.Now(), + Source: source, + } + req, err := event.Binary.NewRequest(URL, m, ctx) if err != nil { - log.Printf("Error: Failed to create http request: %v", err) + log.Printf("Failed to marshal the message: %+v : %s", m, err) return err } - req.Header.Set("Content-Type", "application/json") + log.Printf("Posting to %q", URL) client := &http.Client{} resp, err := client.Do(req) if err != nil { - log.Printf("Error: Failed to do POST: %v", err) return err } defer resp.Body.Close() - log.Printf("Error: response Status: %s", resp.Status) + log.Printf("response Status: %s", resp.Status) body, _ := ioutil.ReadAll(resp.Body) - log.Printf("Error: response Body: %s", string(body)) + log.Printf("response Body: %s", string(body)) return nil } diff --git a/sample/github/legit.go b/sample/github/legit.go index 766b2ef62f5..bde49f65c79 100644 --- a/sample/github/legit.go +++ b/sample/github/legit.go @@ -22,9 +22,9 @@ import ( "flag" "fmt" ghclient "github.com/google/go-github/github" + "github.com/knative/eventing/pkg/event" "golang.org/x/oauth2" "gopkg.in/go-playground/webhooks.v3/github" - "io/ioutil" "log" "net/http" "os" @@ -49,20 +49,8 @@ type GithubSecrets struct { SecretToken string `json:"secretToken"` } -func (h *GithubHandler) handlePost(rw http.ResponseWriter, req *http.Request) { - body, err := ioutil.ReadAll(req.Body) - if err != nil { - panic(err) - } - rw.WriteHeader(200) - - var pl github.PullRequestPayload - if err := json.Unmarshal(body, &pl); err != nil { - log.Printf("error: could not unmarshal payload %v", err) - return - } +func (h *GithubHandler) newPullRequestPayload(ctx context.Context, pl *github.PullRequestPayload) { - // Do whatever you want from here... title := pl.PullRequest.Title log.Printf("GOT PR with Title: %q", title) @@ -76,7 +64,8 @@ func (h *GithubHandler) handlePost(rw http.ResponseWriter, req *http.Request) { updatedPR := ghclient.PullRequest{ Title: &newTitle, } - newPR, response, err := h.client.PullRequests.Edit(h.ctx, pl.Repository.Owner.Login, pl.Repository.Name, int(pl.Number), &updatedPR) + newPR, response, err := h.client.PullRequests.Edit(h.ctx, + pl.Repository.Owner.Login, pl.Repository.Name, int(pl.Number), &updatedPR) if err != nil { log.Printf("Failed to update PR: %s\n%s", err, response) return @@ -115,6 +104,5 @@ func main() { ctx: ctx, } - http.HandleFunc("/", h.handlePost) - log.Fatal(http.ListenAndServe(":8080", nil)) + log.Fatal(http.ListenAndServe(":8080", event.Handler(h.newPullRequestPayload))) } From bb5822c825ea37fe0f943d471b55cba626f40956 Mon Sep 17 00:00:00 2001 From: Scott Nichols Date: Fri, 10 Aug 2018 10:35:45 -0700 Subject: [PATCH 2/3] Add more words to the readme. --- pkg/sources/github/README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pkg/sources/github/README.md b/pkg/sources/github/README.md index 25756cbd4f0..763eaca84ee 100644 --- a/pkg/sources/github/README.md +++ b/pkg/sources/github/README.md @@ -9,3 +9,15 @@ Uses two images, the feedlet and the receive adapter. ### Feedlet +A Feedlet is a container to create or destroy event source bindings. + +The Feedlet is run as a Job by the Feed controller. + +This Feedlet will create the Receive Adapter and register a webhook in GitHub +for the account provided. + +### Receive Adapter + +The Receive Adapter is a Service.serving.knative.dev resource that is registered +to receive GitHub webhook requests. The source will wrap this request in a +CloudEvent and send it to the action provided. \ No newline at end of file From 54767b0460dcb8bfe13b6f7fbc7c8c2eadbab71d Mon Sep 17 00:00:00 2001 From: Scott Nichols Date: Fri, 10 Aug 2018 13:34:31 -0700 Subject: [PATCH 3/3] Use cloud event map and updat the readme. --- pkg/sources/github/README.md | 84 ++++++++++++++++++- pkg/sources/github/events.go | 10 +++ pkg/sources/github/feedlet/feedlet.go | 12 +-- .../github/receive_adapter/receive_adapter.go | 42 +++++----- 4 files changed, 117 insertions(+), 31 deletions(-) diff --git a/pkg/sources/github/README.md b/pkg/sources/github/README.md index 763eaca84ee..c71e1061496 100644 --- a/pkg/sources/github/README.md +++ b/pkg/sources/github/README.md @@ -1,10 +1,14 @@ -GitHub -====== +Knative GitHub Source +===================== -Produces the following eventtypes: +This source will enable receiving of GitHub events from within Knative Eventing. + +## Events - dev.knative.github.pullrequest +## Images + Uses two images, the feedlet and the receive adapter. ### Feedlet @@ -20,4 +24,76 @@ for the account provided. The Receive Adapter is a Service.serving.knative.dev resource that is registered to receive GitHub webhook requests. The source will wrap this request in a -CloudEvent and send it to the action provided. \ No newline at end of file +CloudEvent and send it to the action provided. + +## Usage + +The GitHub Source expects there to be a secret in the following form: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: githubsecret +type: Opaque +stringData: + githubCredentials: > + { + "accessToken": "", + "secretToken": "" + } + +``` + +The Feedlet requires a ServiceAccount to run as a cluster admin for the targeted namespace: + +```yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: feed-sa + namespace: default +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: feed-admin +subjects: + - kind: ServiceAccount + name: feed-sa + namespace: default +roleRef: + kind: ClusterRole + name: cluster-admin + apiGroup: rbac.authorization.k8s.io + +``` + +To create a Receive Adapter (via the Feedlet), a Flow needs to exist: + +```yaml +apiVersion: flows.knative.dev/v1alpha1 +kind: Flow +metadata: + name: my-github-flow + namespace: default +spec: + serviceAccountName: feed-sa + trigger: + eventType: pullrequest + resource: / # TODO: Fill this out + service: github + parameters: + secretName: githubsecret + secretKey: githubCredentials + parametersFrom: + - secretKeyRef: + name: githubsecret + key: githubCredentials + action: + target: + kind: Service + apiVersion: serving.knative.dev/v1alpha1 + name: my-service + +``` \ No newline at end of file diff --git a/pkg/sources/github/events.go b/pkg/sources/github/events.go index 19ab61df30d..310fd55287e 100644 --- a/pkg/sources/github/events.go +++ b/pkg/sources/github/events.go @@ -19,4 +19,14 @@ package github const ( // Event Names PullrequestEvent = "dev.knative.github.pullrequest" + UnsupportedEvent = "dev.knative.github.unsupported" ) + +var CloudEventType = map[string]string{ + "pull_request": PullrequestEvent, + "": UnsupportedEvent, +} + +var GithubEventType = map[string]string{ + PullrequestEvent: "pull_request", +} diff --git a/pkg/sources/github/feedlet/feedlet.go b/pkg/sources/github/feedlet/feedlet.go index e1ee960ccea..5f479d33f20 100644 --- a/pkg/sources/github/feedlet/feedlet.go +++ b/pkg/sources/github/feedlet/feedlet.go @@ -35,6 +35,7 @@ import ( "os" ghclient "github.com/google/go-github/github" + "github.com/knative/eventing/pkg/sources/github" "github.com/knative/eventing/pkg/sources/github/resources" "github.com/knative/serving/pkg/apis/serving/v1alpha1" "k8s.io/apimachinery/pkg/watch" @@ -66,9 +67,6 @@ const ( secretNameKey = "secretName" // secretKeyKey is the name of key inside the secret that contains the GitHub credentials. secretKeyKey = "secretKey" - - // Event Names - pullrequestEvent = "dev.knative.github.pullrequest" ) type githubEventSource struct { @@ -454,11 +452,9 @@ func parseEventsFrom(eventType string) ([]string, error) { if len(eventType) == 0 { return []string(nil), fmt.Errorf("event type is empty") } - switch eventType { - case pullrequestEvent: - return []string{"pull_request"}, nil - // TODO: Add more supported event types. - default: + event, ok := github.GithubEventType[eventType] + if !ok { return []string(nil), fmt.Errorf("event type is unknown: %s", eventType) } + return []string{event}, nil } diff --git a/pkg/sources/github/receive_adapter/receive_adapter.go b/pkg/sources/github/receive_adapter/receive_adapter.go index 018157b37cf..78238a695ec 100644 --- a/pkg/sources/github/receive_adapter/receive_adapter.go +++ b/pkg/sources/github/receive_adapter/receive_adapter.go @@ -29,6 +29,7 @@ import ( "net/http" ghclient "github.com/google/go-github/github" + "github.com/google/uuid" "github.com/knative/eventing/pkg/event" "github.com/knative/eventing/pkg/sources/github" "log" @@ -56,27 +57,26 @@ type GithubSecrets struct { // HandlePullRequest is invoked whenever a PullRequest is modified (created, updated, etc.) func (h *GithubHandler) HandlePullRequest(payload interface{}, header webhooks.Header) { log.Print("Handling Pull Request") + + hdr := http.Header(header) + pl := payload.(gh.PullRequestPayload) source := pl.PullRequest.HTMLURL - var eventType string - if len(header["X-GitHub-Event"]) == 1 { - switch header["X-GitHub-Event"][0] { - case "pull_request": - eventType = github.PullrequestEvent - default: - eventType = "unsupported" - } - + eventType, ok := github.CloudEventType[hdr.Get("X-GitHub-Event")] + if !ok { + eventType = github.UnsupportedEvent } - eventID := "unknown" - if len(header["X-GitHub-Delivery"]) == 1 { - eventID = header["X-GitHub-Delivery"][0] + eventID := hdr.Get("X-GitHub-Delivery") + if len(eventID) == 0 { + if uuid, err := uuid.NewRandom(); err != nil { + eventID = uuid.String() + } } - postMessage(h.target, &pl, source, eventType, eventID) + postMessage(h.target, payload, source, eventType, eventID) } func main() { @@ -119,7 +119,7 @@ func main() { } } -func postMessage(target string, m *gh.PullRequestPayload, source, eventType, eventID string) error { +func postMessage(target string, payload interface{}, source, eventType, eventID string) error { URL := fmt.Sprintf("http://%s/", target) ctx := event.EventContext{ @@ -129,9 +129,9 @@ func postMessage(target string, m *gh.PullRequestPayload, source, eventType, eve EventTime: time.Now(), Source: source, } - req, err := event.Binary.NewRequest(URL, m, ctx) + req, err := event.Binary.NewRequest(URL, payload, ctx) if err != nil { - log.Printf("Failed to marshal the message: %+v : %s", m, err) + log.Printf("Failed to marshal the message: %+v : %s", payload, err) return err } @@ -142,8 +142,12 @@ func postMessage(target string, m *gh.PullRequestPayload, source, eventType, eve return err } defer resp.Body.Close() - log.Printf("response Status: %s", resp.Status) - body, _ := ioutil.ReadAll(resp.Body) - log.Printf("response Body: %s", string(body)) + + if resp.StatusCode != http.StatusOK { + // TODO: in general, receive adapters may have to be able to retry for error cases. + log.Printf("response Status: %s", resp.Status) + body, _ := ioutil.ReadAll(resp.Body) + log.Printf("response Body: %s", string(body)) + } return nil }