From a31ada575da1467a32189857fecec82b8c5ed0e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Brod=C3=A9n?= Date: Tue, 23 Apr 2024 13:48:42 +0200 Subject: [PATCH 1/3] Add retry when creating webhook to handle busy backend --- api/webhook.go | 105 +++++++++++++++++++++++++++++++++---------------- 1 file changed, 72 insertions(+), 33 deletions(-) diff --git a/api/webhook.go b/api/webhook.go index 5e4ddff..978b559 100644 --- a/api/webhook.go +++ b/api/webhook.go @@ -4,66 +4,102 @@ import ( "fmt" "log" "strconv" + "strings" + "time" ) // CreateWebhook - create a webhook for a vhost and a specific qeueu -func (api *API) CreateWebhook(instanceID int, params map[string]interface{}) (map[string]interface{}, error) { - data := make(map[string]interface{}) - failed := make(map[string]interface{}) - log.Printf("[DEBUG] go-api::webhook::create params: %v", params) - path := fmt.Sprintf("/api/instances/%d/webhooks", instanceID) +func (api *API) CreateWebhook(instanceID int, params map[string]interface{}, + sleep, timeout int) (map[string]interface{}, error) { + + return api.createWebhookWithRetry(instanceID, params, 1, sleep, timeout) +} + +// createWebhookWithRetry: create webhook with retry if backend is busy. +func (api *API) createWebhookWithRetry(instanceID int, params map[string]interface{}, + attempt, sleep, timeout int) (map[string]interface{}, error) { + + var ( + data = make(map[string]interface{}) + failed map[string]interface{} + path = fmt.Sprintf("/api/instances/%d/webhooks", instanceID) + ) + + log.Printf("[DEBUG] go-api::webhook#create path: %s, params: %v", path, params) response, err := api.sling.New().Post(path).BodyJSON(params).Receive(&data, &failed) - log.Printf("[DEBUG] go-api::webhook::create response data: %v", data) + log.Printf("[DEBUG] go-api::webhook#create response data: %v", data) if err != nil { return nil, err - } - if response.StatusCode != 201 { - return nil, fmt.Errorf(fmt.Sprintf("CreateWebhook failed, status: %v, message: %s", response.StatusCode, failed)) + } else if attempt*sleep > timeout { + return nil, fmt.Errorf("create webhook reached timeout of %d seconds", timeout) } - if v, ok := data["id"]; ok { - data["id"] = strconv.FormatFloat(v.(float64), 'f', 0, 64) - } else { - msg := fmt.Sprintf("go-api::webhook::create Invalid webhook identifier: %v", data["id"]) - log.Printf("[ERROR] %s", msg) - return nil, fmt.Errorf(msg) + switch response.StatusCode { + case 201: + if v, ok := data["id"]; ok { + data["id"] = strconv.FormatFloat(v.(float64), 'f', 0, 64) + } else { + msg := fmt.Sprintf("go-api::webhook#create Invalid webhook identifier: %v", data["id"]) + log.Printf("[ERROR] %s", msg) + return nil, fmt.Errorf(msg) + } + return data, nil + case 400: + if strings.Compare(failed["error"].(string), "Timeout talking to backend") == 0 { + log.Printf("[INFO] go-api::webhook#read Timeout talking to backend "+ + "attempt: %d, until timeout: %d", attempt, (timeout - (attempt * sleep))) + attempt++ + time.Sleep(time.Duration(sleep) * time.Second) + return api.createWebhookWithRetry(instanceID, params, attempt, sleep, timeout) + } + return nil, fmt.Errorf("create webhook failed, status: %v, message: %s", 400, failed) + default: + return nil, + fmt.Errorf("create webhook with retry failed, status: %v, message: %s", + response.StatusCode, failed) } - - return data, err } // ReadWebhook - retrieves a specific webhook for an instance func (api *API) ReadWebhook(instanceID int, webhookID string) (map[string]interface{}, error) { - data := make(map[string]interface{}) - failed := make(map[string]interface{}) - log.Printf("[DEBUG] go-api::webhook::read instance ID: %d, webhookID: %s", instanceID, webhookID) - path := fmt.Sprintf("/api/instances/%d/webhooks/%s", instanceID, webhookID) + var ( + data map[string]interface{} + failed map[string]interface{} + path = fmt.Sprintf("/api/instances/%d/webhooks/%s", instanceID, webhookID) + ) + + log.Printf("[DEBUG] go-api::webhook#read path: %s", path) response, err := api.sling.New().Path(path).Receive(&data, &failed) if err != nil { return nil, err } if response.StatusCode != 200 { - return nil, fmt.Errorf("ReadWebhook failed, status: %v, message: %s", response.StatusCode, failed) + return nil, fmt.Errorf("read webhook failed, status: %v, message: %s", + response.StatusCode, failed) } return data, err } -// ReadWebhooks - retrieves all webhooks for an instance. -func (api *API) ReadWebhooks(instanceID int) (map[string]interface{}, error) { - data := make(map[string]interface{}) - failed := make(map[string]interface{}) - log.Printf("[DEBUG] go-api::webhook::read instance ID: %d", instanceID) - path := fmt.Sprintf("/api/instances/%d/webhooks", instanceID) +// ListWebhooks - list all webhooks for an instance. +func (api *API) ListWebhooks(instanceID int) (map[string]interface{}, error) { + var ( + data map[string]interface{} + failed map[string]interface{} + path = fmt.Sprintf("/api/instances/%d/webhooks", instanceID) + ) + + log.Printf("[DEBUG] go-api::webhook#list path: %s", path) response, err := api.sling.New().Path(path).Receive(&data, &failed) if err != nil { return nil, err } if response.StatusCode != 200 { - return nil, fmt.Errorf("ReadWebhooks failed, status: %v, message: %s", response.StatusCode, failed) + return nil, fmt.Errorf("list webhooks failed, status: %v, message: %s", + response.StatusCode, failed) } return data, err @@ -71,13 +107,16 @@ func (api *API) ReadWebhooks(instanceID int) (map[string]interface{}, error) { // DeleteWebhook - removes a specific webhook for an instance func (api *API) DeleteWebhook(instanceID int, webhookID string) error { - failed := make(map[string]interface{}) - log.Printf("[DEBUG] go-api::webhook::delete instance ID: %d, webhookID: %s", instanceID, webhookID) - path := fmt.Sprintf("/api/instances/%d/webhooks/%s", instanceID, webhookID) + var ( + failed map[string]interface{} + path = fmt.Sprintf("/api/instances/%d/webhooks/%s", instanceID, webhookID) + ) + + log.Printf("[DEBUG] go-api::webhook#delete path: %s", path) response, err := api.sling.New().Delete(path).Receive(nil, &failed) if response.StatusCode != 204 { - return fmt.Errorf("DeleteWebhook failed, status: %v, message: %s", response.StatusCode, failed) + return fmt.Errorf("delete webhook failed, status: %v, message: %s", response.StatusCode, failed) } return err From 47995fabbf2625008284e0ed80be6190dd108dff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Brod=C3=A9n?= Date: Tue, 23 Apr 2024 14:05:00 +0200 Subject: [PATCH 2/3] Add update function with retry --- api/webhook.go | 47 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/api/webhook.go b/api/webhook.go index 978b559..496518a 100644 --- a/api/webhook.go +++ b/api/webhook.go @@ -47,7 +47,7 @@ func (api *API) createWebhookWithRetry(instanceID int, params map[string]interfa return data, nil case 400: if strings.Compare(failed["error"].(string), "Timeout talking to backend") == 0 { - log.Printf("[INFO] go-api::webhook#read Timeout talking to backend "+ + log.Printf("[INFO] go-api::webhook#create Timeout talking to backend "+ "attempt: %d, until timeout: %d", attempt, (timeout - (attempt * sleep))) attempt++ time.Sleep(time.Duration(sleep) * time.Second) @@ -105,6 +105,51 @@ func (api *API) ListWebhooks(instanceID int) (map[string]interface{}, error) { return data, err } +// UpdateWebhook - updates a specific webhook for an instance +func (api *API) UpdateWebhook(instanceID int, webhookID string, params map[string]interface{}, + sleep, timeout int) error { + + path := fmt.Sprintf("/api/instances/%d/webhooks/%s", instanceID, webhookID) + return api.updateWebhookWithRetry(path, params, 1, sleep, timeout) +} + +// updateWebhookWithRetry: update webhook with retry if backend is busy. +func (api *API) updateWebhookWithRetry(path string, params map[string]interface{}, + attempt, sleep, timeout int) error { + + var ( + data = make(map[string]interface{}) + failed map[string]interface{} + ) + + log.Printf("[DEBUG] go-api::webhook#update path: %s, params: %v", path, params) + response, err := api.sling.New().Put(path).BodyJSON(params).Receive(&data, &failed) + log.Printf("[DEBUG] go-api::webhook#update response data: %v", data) + + if err != nil { + return err + } else if attempt*sleep > timeout { + return fmt.Errorf("update webhook reached timeout of %d seconds", timeout) + } + + switch response.StatusCode { + case 201: + return nil + case 400: + if strings.Compare(failed["error"].(string), "Timeout talking to backend") == 0 { + log.Printf("[INFO] go-api::webhook#update Timeout talking to backend "+ + "attempt: %d, until timeout: %d", attempt, (timeout - (attempt * sleep))) + attempt++ + time.Sleep(time.Duration(sleep) * time.Second) + return api.updateWebhookWithRetry(path, params, attempt, sleep, timeout) + } + return fmt.Errorf("update webhook failed, status: %v, message: %s", 400, failed) + default: + return fmt.Errorf("update webhook with retry failed, status: %v, message: %s", + response.StatusCode, failed) + } +} + // DeleteWebhook - removes a specific webhook for an instance func (api *API) DeleteWebhook(instanceID int, webhookID string) error { var ( From 2cfbcad81e61533a03a957605e40fb25144ea9fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Brod=C3=A9n?= Date: Wed, 24 Apr 2024 11:08:43 +0200 Subject: [PATCH 3/3] Add retries to read and delete webhook requests --- api/webhook.go | 69 +++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 57 insertions(+), 12 deletions(-) diff --git a/api/webhook.go b/api/webhook.go index 496518a..70a4de5 100644 --- a/api/webhook.go +++ b/api/webhook.go @@ -62,25 +62,48 @@ func (api *API) createWebhookWithRetry(instanceID int, params map[string]interfa } // ReadWebhook - retrieves a specific webhook for an instance -func (api *API) ReadWebhook(instanceID int, webhookID string) (map[string]interface{}, error) { +func (api *API) ReadWebhook(instanceID int, webhookID string, sleep, timeout int) ( + map[string]interface{}, error) { + + path := fmt.Sprintf("/api/instances/%d/webhooks/%s", instanceID, webhookID) + return api.readWebhookWithRetry(path, 1, sleep, timeout) +} + +// readWebhookWithRetry: read webhook with retry if backend is busy. +func (api *API) readWebhookWithRetry(path string, attempt, sleep, timeout int) ( + map[string]interface{}, error) { + var ( data map[string]interface{} failed map[string]interface{} - path = fmt.Sprintf("/api/instances/%d/webhooks/%s", instanceID, webhookID) ) log.Printf("[DEBUG] go-api::webhook#read path: %s", path) - response, err := api.sling.New().Path(path).Receive(&data, &failed) + response, err := api.sling.New().Get(path).Receive(&data, &failed) + log.Printf("[DEBUG] go-api::webhook#read response data: %v", data) if err != nil { return nil, err + } else if attempt*sleep > timeout { + return nil, fmt.Errorf("read webhook reached timeout of %d seconds", timeout) } - if response.StatusCode != 200 { - return nil, fmt.Errorf("read webhook failed, status: %v, message: %s", + + switch response.StatusCode { + case 200: + return data, nil + case 400: + if strings.Compare(failed["error"].(string), "Timeout talking to backend") == 0 { + log.Printf("[INFO] go-api::webhook#read Timeout talking to backend "+ + "attempt: %d, until timeout: %d", attempt, (timeout - (attempt * sleep))) + attempt++ + time.Sleep(time.Duration(sleep) * time.Second) + return api.readWebhookWithRetry(path, attempt, sleep, timeout) + } + return nil, fmt.Errorf("read webhook failed, status: %v, message: %s", 400, failed) + default: + return nil, fmt.Errorf("read webhook with retry failed, status: %v, message: %s", response.StatusCode, failed) } - - return data, err } // ListWebhooks - list all webhooks for an instance. @@ -151,18 +174,40 @@ func (api *API) updateWebhookWithRetry(path string, params map[string]interface{ } // DeleteWebhook - removes a specific webhook for an instance -func (api *API) DeleteWebhook(instanceID int, webhookID string) error { +func (api *API) DeleteWebhook(instanceID int, webhookID string, sleep, timeout int) error { + path := fmt.Sprintf("/api/instances/%d/webhooks/%s", instanceID, webhookID) + return api.deleteWebhookWithRetry(path, 1, sleep, timeout) +} + +// deleteWebhookWithRetry: delete webhook with retry if backend is busy. +func (api *API) deleteWebhookWithRetry(path string, attempt, sleep, timeout int) error { var ( failed map[string]interface{} - path = fmt.Sprintf("/api/instances/%d/webhooks/%s", instanceID, webhookID) ) log.Printf("[DEBUG] go-api::webhook#delete path: %s", path) response, err := api.sling.New().Delete(path).Receive(nil, &failed) - if response.StatusCode != 204 { - return fmt.Errorf("delete webhook failed, status: %v, message: %s", response.StatusCode, failed) + if err != nil { + return err + } else if attempt*sleep > timeout { + return fmt.Errorf("delete webhook reached timeout of %d seconds", timeout) } - return err + switch response.StatusCode { + case 204: + return nil + case 400: + if strings.Compare(failed["error"].(string), "Timeout talking to backend") == 0 { + log.Printf("[INFO] go-api::webhook#delete Timeout talking to backend "+ + "attempt: %d, until timeout: %d", attempt, (timeout - (attempt * sleep))) + attempt++ + time.Sleep(time.Duration(sleep) * time.Second) + return api.deleteWebhookWithRetry(path, attempt, sleep, timeout) + } + return fmt.Errorf("delete webhook failed, status: %v, message: %s", 400, failed) + default: + return fmt.Errorf("delete webhook with retry failed, status: %v, message: %s", + response.StatusCode, failed) + } }