diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b79d817 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +/tmp +/go.sum diff --git a/README.md b/README.md index 098711b..1cbac30 100644 --- a/README.md +++ b/README.md @@ -1,239 +1,313 @@ - +# Requests [![license](http://dmlc.github.io/img/apache2.svg)](https://raw.githubusercontent.com/asmcos/requests/master/LICENSE) -# requests +# Requests +Requests is an HTTP library, it is easy to use. Similar to Python requests. -Requests is an HTTP library , it is easy to use. Similar to Python requests. +Warning: Session is not safe in multiple goroutines. You can not do as following: -# Installation + // Bad! Do not call session in in multiple goroutines!!!!! + session := requests.Sessions() -``` -go get -u github.com/asmcos/requests -``` + // goroutine 1 + go func(){ + session.Post(url1) + }() -# Start + // goroutine 2 + go func(){ + session.Post(url2) + }() -``` go -package main -import "github.com/asmcos/requests" - -func main (){ +# Installation - resp,err := requests.Get("http://www.zhanluejia.net.cn") - if err != nil{ - return - } - println(resp.Text()) -} +``` +go get -u github.com/asmcos/requests ``` -## Post - -``` go -package main +# Examples +> For more examples, refer to https://github.com/asmcos/requests/tree/master/examples + +## Get + package main + import ( + "github.com/asmcos/requests" + "fmt" + ) + + func main(){ + var json map[string]interface{} + params := requests.Params{"name": "asmcos", "page":"1"} + resp, err := requests.Get("https://httpbin.org/json", params) + if err != nil { + panic(err) + }else{ + resp.Json(&json) + for k, v := range json { + fmt.Println(k, v) + } + } + } -import "github.com/asmcos/requests" +## Post -func main (){ +### Post params + // Post params + func TestPostParams(t *testing.T) { + println("Test POST: post params") + data := requests.Params{ + "name": "asmcos", + } + resp, err := requests.Post("https://www.httpbin.org/post", data) + if err == nil { + fmt.Println(resp.Text()) + } + } +#### Post query string + + // Post QueryString: application/x-www-form-urlencoded + func TestPostQueryString(t *testing.T) { + queryString := "name=Alex&age=29" + resp, err := requests.Post("https://www.httpbin.org/post", queryString) + if err != nil { + t.Fatal(err) + } + var data = struct { + Form struct{ + Name string + Age string + } + }{} + err = resp.Json(&data) + if data.Form.Age != "29"{ + t.Error("invalid response body:", resp.Text()) + } + } + + +### Post Form Data + // Post Form Data + func TestPostForm(t *testing.T) { + println("Test POST: post form data") data := requests.Datas{ - "name":"requests_post_test", + "comments": "ew", } - resp,_ := requests.Post("https://www.httpbin.org/post",data) - println(resp.Text()) -} + resp, err := requests.Post("https://www.httpbin.org/post", data) + if err == nil { + fmt.Println(resp.Text()) + } + } -``` - Server return data... - -``` json -{ - "args": {}, - "data": "", - "files": {}, - "form": { - "name": "requests_post_test" - }, - "headers": { - "Accept-Encoding": "gzip", - "Connection": "close", - "Content-Length": "23", - "Content-Type": "application/x-www-form-urlencoded", - "Host": "www.httpbin.org", - "User-Agent": "Go-Requests 0.5" - }, - "json": null, - "origin": "114.242.34.110", - "url": "https://www.httpbin.org/post" -} +### Post Json (default) + func TestPostJson(t *testing.T) { + println("Test POST: post json data") + json := requests.Json{ + "key": "value", + } + /* + //it still works! + json = map[string]interface{}{ + "key": "value", + } + */ + resp, err := requests.Post("https://www.httpbin.org/post", json) + if err == nil { + fmt.Println(resp.Text()) + } + } + +### Post Raw text/plain + func TestRawString(t *testing.T) { + println("Test POST: post data and json") + rawText := "raw data: Hi, Jack!" + resp, err := requests.Post( + "https://www.httpbin.org/post", + rawText, + requests.Header{"Content-Type": "text/plain"}, + ) + if err == nil { + fmt.Println(resp.Text()) + } + } + +### PostFiles + + path, _ := os.Getwd() + session := requests.Sessions() + + resp, err := session.SetDebug(true).Post( + "https://www.httpbin.org/post", + requests.Files{ + "file1": path + "/README.md", + "file2": path + "/version", + }, + ) + if err == nil { + fmt.Println(resp.Text()) + } + +## Session Support + // 0. Make a session + session := r.Sessions() + + // 1. First, set cookies: count=100 + var data struct { + Cookies struct { + Count string `json:"count"` + } + } + session.Get("https://httpbin.org/cookies/set?count=100") + + // 2. Second, get cookies + resp, err := session.Get("https://httpbin.org/cookies") + if err == nil { + resp.Json(&data) + if data.Cookies.Count!="100"{ + t.Fatal("Failed to get valid cookies: "+resp.Text()) + } + } -``` +Warning: Session is not safe in multi goroutine. You can not do as following: + // Bad! Do not call session in in multi goroutine!!!!! + session := requests.Sessions() -## PostJson + // goroutine 1 + go func(){ + session.Post(url1) + }() -``` go -package main + // goroutine 2 + go func(){ + session.Post(url2) + }() -import "github.com/asmcos/requests" +## Request Options +### SetTimeout -func main (){ + session := Requests.Sessions() + session.SetTimeout(20) - jsonStr := "{\"name\":\"requests_post_test\"}" - resp,_ := requests.PostJson("https://www.httpbin.org/post",jsonStr) - println(resp.Text()) -} +### Debug Mode + + session.Debug = 1 -``` +### Set Authentication + session := requests.Sessions() + resp,_ := session.Get("https://api.github.com/user",requests.Auth{"asmcos","password...."}) - Server return data... - -``` json -{ - "args": {}, - "data": "", - "files": {}, - "form": { - "name": "requests_post_test" - }, - "headers": { - "Accept-Encoding": "gzip", - "Connection": "close", - "Content-Length": "23", - "Content-Type": "application/x-www-form-urlencoded", - "Host": "www.httpbin.org", - "User-Agent": "Go-Requests 0.5" - }, - "json": null, - "origin": "114.242.34.110", - "url": "https://www.httpbin.org/post" -} +### Set Cookie + cookie1 := http.Cookie{Name: "cookie_name", Value: "cookie_value"} + session.SetCookie(&cookie1) -``` +### Set header -# Feature Support - - Set headers - - Set params - - Multipart File Uploads - - Sessions with Cookie Persistence - - Proxy - - Authentication - - JSON - - Chunked Requests - - Debug - - SetTimeout - - -# Set header - -### example 1 - -``` go -req := requests.Requests() - -resp,err := req.Get("http://www.zhanluejia.net.cn",requests.Header{"Referer":"http://www.jeapedu.com"}) -if (err == nil){ - println(resp.Text()) -} -``` + func TestGetParamsHeaders(t *testing.T) { + println("Test Get: custom header and params") + requests.Get("http://www.zhanluejia.net.cn", + requests.Header{"Referer": "http://www.jeapedu.com"}, + requests.Params{"page": "1", "size": "20"}, + requests.Params{"name": "ahuio"}, + ) + } -### example 2 + func TestGetParamsHeaders2(t *testing.T) { + session := requests.Sessions() + session.SetHeader("accept-encoding", "gzip, deflate, br") + session.Run("http://www.zhanluejia.net.cn", + requests.Params{"page": "1", "size": "20"}, + requests.Params{"name": "ahuio"}, + ) + } -``` go -req := requests.Requests() -req.Header.Set("accept-encoding", "gzip, deflate, br") -resp,_ := req.Get("http://www.zhanluejia.net.cn",requests.Header{"Referer":"http://www.jeapedu.com"}) -println(resp.Text()) + func TestResponseHeader(t *testing.T) { + resp, _ := requests.Get("https://www.baidu.com/") + println(resp.Text()) + println(resp.R.Header.Get("location")) + println(resp.R.Header.Get("Location")) + } -``` +two or more headers ... -### example 3 - -``` go -h := requests.Header{ - "Referer": "http://www.jeapedu.com", - "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8", -} -resp,_ := req.Get("http://wwww.zhanluejia.net.cn",h) - -h2 := requests.Header{ - ... - ... -} -h3,h4 .... -// two or more headers ... -resp,_ = req.Get("http://www.zhanluejia.net.cn",h,h2,h3,h4) -``` + headers1 := requests.Header{"Referer": "http://www.jeapedu.com"}, + .... + resp,_ = session.Get( + "http://www.zhanluejia.net.cn", + headers1, + headers2, + headers3, + ) -# Set params +## Response +### Fetch Response Body +https://github.com/asmcos/requests/blob/master/examples/resp_test.go -``` go -p := requests.Params{ - "title": "The blog", - "name": "file", - "id": "12345", -} -resp,_ := req.Get("http://www.cpython.org", p) + fmt.Println(resp.Text()) + fmt.Println(resp.Content()) -``` +### Fetch Response Cookies +https://github.com/asmcos/requests/blob/master/examples/cookie_test.go + resp,_ = session.Get("https://www.httpbin.org") + coo := resp.Cookies() + for _, c:= range coo{ + fmt.Println(c.Name,c.Value) + } -# Auth +## Custom -Test with the `correct` user information. +### Custom User Agent -``` go -req := requests.Requests() -resp,_ := req.Get("https://api.github.com/user",requests.Auth{"asmcos","password...."}) -println(resp.Text()) -``` + headerK := "User-Agent" + headerV := "Custom-Test-Go-User-Agent" + requests.SetHeader(headerK, headerV) -github return +# Utils -``` -{"login":"asmcos","id":xxxxx,"node_id":"Mxxxxxxxxx=="..... -``` +## Build Request -# JSON + req, err := r.BuildRequest("post", "http://baidu.com/a/b/c", r.Json{ + "age": 1, + }) + if err != nil { + t.Fatal(err) + } + body, _ := ioutil.ReadAll(req.Body) + expectedBody := `{"age":1}` + if string(body) != expectedBody { + t.Fatal("Failed to build request") + } -``` go -req := requests.Requests() -req.Header.Set("Content-Type","application/json") -resp,_ = req.Get("https://httpbin.org/json") +## Generate curl shell command -var json map[string]interface{} -resp.Json(&json) + req, _ := requests.BuildRequest("post", "https://baidu.com/path?q=curl&v=1", requests.Json{ + "age": 1, + }) + curl := requests.BuildCurlRequest(req) + if !regexp.MustCompile(`^curl -X POST .+ 'https://baidu.com/path\?q=curl&v=1'`).MatchString(curl) { + t.Fatal(`bad curl cmd: ` + curl) + } -for k,v := range json{ - fmt.Println(k,v) -} -``` +# Feature Support + - Set headers + - Set params + - Multipart File Uploads + - Sessions with Cookie Persistence + - Proxy + - Authentication + - JSON + - Chunked Requests + - Debug + - SetTimeout -# SetTimeout - -``` -req := Requests() -req.Debug = 1 - -// 20 Second -req.SetTimeout(20) -req.Get("http://golang.org") -``` - -# Get Cookies +# Thanks +This project is inspired by [github.com/asmcos/requests](http://github.com/asmcos/requests). -``` go -resp,_ = req.Get("https://www.httpbin.org") -coo := resp.Cookies() -// coo is [] *http.Cookies -println("********cookies*******") -for _, c:= range coo{ - fmt.Println(c.Name,c.Value) -} -``` +Great thanks to it :). diff --git a/docs/index.md b/docs/index.md deleted file mode 100644 index bd21bd7..0000000 --- a/docs/index.md +++ /dev/null @@ -1,21 +0,0 @@ -# Installation - - -# example 1 - -``` -package main - -import "github.com/asmcos/requests" - - -func main (){ - - resp,err := requests.Get("http://go.xiulian.net.cn") - if err != nil { - return - } - println(resp.Text()) -} - -``` diff --git a/examples/auth_test.go b/examples/auth_test.go new file mode 100644 index 0000000..360e18e --- /dev/null +++ b/examples/auth_test.go @@ -0,0 +1,24 @@ +package examples + +import ( + "fmt" + "testing" + + "github.com/asmcos/requests" + _ "github.com/asmcos/requests/init" +) + +func TestAuth(t *testing.T) { + println("3. Get: Set Auth") + // test authentication usernae,password + //documentation https://www.httpwatch.com/httpgallery/authentication/#showExample10 + resp, err := requests.Get( + "https://www.httpwatch.com/httpgallery/authentication/authenticatedimage/default.aspx?0.45874470316137206", + requests.Auth{"httpwatch", "foo"}, + ) + if err == nil { + fmt.Println(resp.R) + } + // this save file test PASS + // resp.SaveFile("auth.jpeg") +} diff --git a/examples/cookie_test.go b/examples/cookie_test.go new file mode 100644 index 0000000..0a6954d --- /dev/null +++ b/examples/cookie_test.go @@ -0,0 +1,59 @@ +package examples + +import ( + "fmt" + "net/http" + "testing" + + "github.com/asmcos/requests" + _ "github.com/asmcos/requests/init" +) + +func TestSendCookie(t *testing.T) { + resp, err := requests.Get("https://www.httpbin.org/cookies", + requests.Header{"Cookie": "id_token=1234"}, + requests.Json{"workflow_id": "wfid1234"}, + ) + if err != nil { + panic(err) + } + data := map[string]interface{}{} + resp.Json(&data) + fmt.Println(data) + +} + +// Test session Cookie +func TestSessionCookie(t *testing.T) { + println("Test: send and get cookie") + req := requests.Sessions().SetDebug(true) + cookie := &http.Cookie{ + Name: "name1", + Value: "value1", + Path: "/", + } + req.SetCookie(cookie).Get("https://www.httpbin.org") + resp, err := req.Get("https://www.httpbin.org", + &http.Cookie{ + Name: "name2", + Value: "value2", + }, + ) + // resp, err := req.SetCookie(cookie).Get("http://127.0.0.1:8088") + if err != nil { + panic(err) + } + cookies := map[string]string{} + // cookies's type is `[]*http.Cookies` + println("********session cookies*******") + for _, c := range resp.Cookies() { + if _, exists := cookies[c.Name]; exists { + t.Fatal("duplicated cookie:", c.Name, c.Value) + } + cookies[c.Name] = c.Value + } + if cookies["name1"] != "value1" || cookies["name2"] != "value2" { + t.Fatal("Failed to send valid cookie") + } + +} diff --git a/examples/debug_test.go b/examples/debug_test.go new file mode 100644 index 0000000..6c6a507 --- /dev/null +++ b/examples/debug_test.go @@ -0,0 +1,27 @@ +package examples + +import ( + "fmt" + "net/http" + "testing" + + "github.com/asmcos/requests" + _ "github.com/asmcos/requests/init" +) + +func TestGetDebug(t *testing.T) { + println("4. Get: SetDebug") + session := requests.Sessions().SetDebug(true) + resp, err := session.Post("https://httpbin.org/post", + requests.Json{ + "name": "asmcos", + }, + &http.Cookie{ + Name: "count", + Value: "1", + }, + ) + if err == nil { + fmt.Println("response text:", resp.Text()) + } +} diff --git a/examples/delete_test.go b/examples/delete_test.go new file mode 100644 index 0000000..17247c4 --- /dev/null +++ b/examples/delete_test.go @@ -0,0 +1,22 @@ +package examples + +import ( + "fmt" + "testing" + + "github.com/asmcos/requests" + _ "github.com/asmcos/requests/init" +) + +// Delete Form Request +func TestDeleteForm(t *testing.T) { + println("Test DELETE method: form data(x-wwww-form-urlencoded)") + data := requests.Datas{ + "comments": "ew", + } + session := requests.Sessions() //.SetDebug(true) + resp, err := session.Delete("https://www.httpbin.org/delete", data) + if err == nil { + fmt.Println(resp.Text()) + } +} diff --git a/examples/get1.go b/examples/get1.go deleted file mode 100644 index 12da14f..0000000 --- a/examples/get1.go +++ /dev/null @@ -1,10 +0,0 @@ -package main - -import "github.com/asmcos/requests" - - -func main (){ - - resp,_ := requests.Get("http://go.xiulian.net.cn") - println(resp.Text()) -} diff --git a/examples/get_auth.go b/examples/get_auth.go deleted file mode 100644 index f6b7e1d..0000000 --- a/examples/get_auth.go +++ /dev/null @@ -1,16 +0,0 @@ -package main - -import ( - "github.com/asmcos/requests" - "fmt" -) - -func main (){ - - req := requests.Requests() - - resp,_ := req.Get("https://api.github.com/user",requests.Auth{"asmcos","password...."}) - println(resp.Text()) - fmt.Println(resp.R.StatusCode) - fmt.Println(resp.R.Header["Content-Type"]) -} diff --git a/examples/get_set_header.go b/examples/get_set_header.go deleted file mode 100644 index 08eb543..0000000 --- a/examples/get_set_header.go +++ /dev/null @@ -1,14 +0,0 @@ -package main - -import "github.com/asmcos/requests" - - -func main (){ - - req := requests.Requests() - - resp,_ := req.Get("http://go.xiulian.net.cn",requests.Header{"Referer":"http://www.jeapedu.com"}) - - println(resp.Text()) - -} diff --git a/examples/get_test.go b/examples/get_test.go new file mode 100644 index 0000000..c6c3b77 --- /dev/null +++ b/examples/get_test.go @@ -0,0 +1,48 @@ +package examples + +import ( + "fmt" + "testing" + + "github.com/asmcos/requests" + _ "github.com/asmcos/requests/init" +) + +// Get Json Response +func TestGetJson(t *testing.T) { + println("Test Get: fetch json response") + resp, err := requests.Get("https://httpbin.org/json") + if err == nil { + var json map[string]interface{} + err = resp.Json(&json) + for k, v := range json { + fmt.Println(k, v) + } + } + if err != nil { + t.Fatal(err) + } +} + +type HbResponse struct { + Args map[string]string `json:"args"` +} + +// Get with params +func TestGetParams(t *testing.T) { + params := requests.Params{"name": "asmcos", "page": "1"} + resp, err := requests.Get("https://httpbin.org/get", params) + + if err == nil { + json := &HbResponse{} + if err := resp.Json(&json); err != nil { + t.Fatal(err) + } + if json.Args["name"] != "asmcos" { + t.Fatal("Invalid response: " + resp.Text()) + } + } + if err != nil { + t.Fatal(err) + } +} diff --git a/examples/post1.go b/examples/post1.go deleted file mode 100644 index 5e45208..0000000 --- a/examples/post1.go +++ /dev/null @@ -1,13 +0,0 @@ -package main - -import "github.com/asmcos/requests" - - -func main (){ - - data := requests.Datas{ - "name":"requests_post_test", - } - resp,_ := requests.Post("https://www.httpbin.org/post",data) - println(resp.Text()) -} diff --git a/examples/post_file_test.go b/examples/post_file_test.go new file mode 100644 index 0000000..c6781e8 --- /dev/null +++ b/examples/post_file_test.go @@ -0,0 +1,34 @@ +package examples + +import ( + "os" + "testing" + + "github.com/asmcos/requests" + _ "github.com/asmcos/requests/init" +) + +func TestPostFile(t *testing.T) { + path, _ := os.Getwd() + + resp, err := requests.Post( + "https://www.httpbin.org/post", + requests.Files{ + "file1": path + "/README.md", + "file2": path + "/version", + }, + ) + if err !=nil { + t.Error(err) + } + var data = struct { + Files struct{ + File2 string + } + }{} + err = resp.Json(&data) + if data.Files.File2== ""{ + t.Error("invalid response body:", resp.Text()) + } + +} diff --git a/examples/post_test.go b/examples/post_test.go new file mode 100644 index 0000000..0c2cc67 --- /dev/null +++ b/examples/post_test.go @@ -0,0 +1,142 @@ +package examples + +import ( + ejson "encoding/json" + "testing" + + "github.com/asmcos/requests" + _ "github.com/asmcos/requests/init" +) + +// Post params +func TestPostParams(t *testing.T) { + println("Test POST: post params") + resp, err := requests.Post( + "https://www.httpbin.org/post", + requests.Params{ + "name": "asmcos", + }, + ) + if err != nil { + t.Error(err) + } + var data = struct { + Args struct { + Name string + } + }{} + _ = resp.Json(&data) + if data.Args.Name != "asmcos" { + t.Error("invalid response body:", resp.Text()) + } +} + +// Post Form Data +func TestPostForm(t *testing.T) { + println("Test POST: post form data(x-wwww-form-urlencoded)") + resp, err := requests.Post( + "https://www.httpbin.org/post", + requests.Datas{ + "name": "asmcos", + }, + ) + if err != nil { + t.Error(err) + } + var data = struct { + Form struct { + Name string + } + }{} + err = resp.Json(&data) + if data.Form.Name != "asmcos" { + t.Error("invalid response body:", resp.Text()) + } +} + +// Post Json Request +func TestPostJson(t *testing.T) { + println("Test POST: post json data") + json := requests.Json{ + "name": "Alex", + } + /* + //it still works! + json = map[string]interface{}{ + "key": "value", + } + */ + resp, err := requests.Post("https://www.httpbin.org/post", json) + if err != nil { + t.Error(err) + } + + // parse data + var data = struct { + Data string + }{} + resp.Json(&data) + + // is expected results + jsonData, _ := ejson.Marshal(json) // if data.Data!= "{\"name\":\"Alex\"}"{ + if data.Data != string(jsonData) { + t.Error("invalid response body:", resp.Text()) + } +} + +// Post QueryString: application/x-www-form-urlencoded +func TestPostQueryString(t *testing.T) { + println("Test POST: raw post data ") + queryString := "name=Alex&age=29" + resp, err := requests.Post("https://www.httpbin.org/post", queryString) + if err != nil { + t.Fatal(err) + } + var data = struct { + Form struct { + Name string + Age string + } + }{} + err = resp.Json(&data) + if data.Form.Age != "29" { + t.Error("invalid response body:", resp.Text()) + } +} + +// Post Raw text/plain +func TestRawString(t *testing.T) { + println("Test POST: raw post data ") + rawText := "raw data: Hi, Jack!" + resp, err := requests.Post("https://www.httpbin.org/post", rawText, + requests.Header{"Content-Type": "text/plain"}, + ) + if err != nil { + t.Fatal(err) + } + var data interface{} + err = resp.Json(&data) + if data.(map[string]interface{})["data"].(string) != rawText { + t.Error("invalid response body:", resp.Text()) + } +} + +// Post Raw text/plain (with bytes) +func TestRawBytes(t *testing.T) { + println("Test POST: post bytes data") + rawText := "raw data: Hi, Jack!" + resp, err := requests.Post("https://www.httpbin.org/post", []byte(rawText), + requests.Header{"Content-Type": "text/plain"}, + ) + if err != nil { + t.Error(err) + } + var data = struct { + Data string + }{} + err = resp.Json(&data) + if data.Data != rawText { + t.Error("invalid response body:", resp.Text()) + } + +} diff --git a/examples/proxy_test.go b/examples/proxy_test.go new file mode 100644 index 0000000..9a43124 --- /dev/null +++ b/examples/proxy_test.go @@ -0,0 +1,16 @@ +package examples + +import ( + "testing" + + "github.com/asmcos/requests" + _ "github.com/asmcos/requests/init" +) + +func TestProxy(t *testing.T) { + println("5. Get: with proxy") + session := requests.Sessions() + // session.Proxy("http://192.168.1.190:8888") + session.Get("https://www.httpbin.org/cookies/set?freeform=1234") + session.Get("https://www.httpbin.org") +} diff --git a/examples/req_build_test.go b/examples/req_build_test.go new file mode 100644 index 0000000..14c4de2 --- /dev/null +++ b/examples/req_build_test.go @@ -0,0 +1,35 @@ +package examples + +import ( + "io/ioutil" + "regexp" + "testing" + + "github.com/asmcos/requests" +) + +// TestBuildRequest +func TestBuildRequest(t *testing.T) { + req, err := requests.BuildRequest("post", "http://baidu.com/a/b/c", requests.Json{ + "age": 1, + }) + if err != nil { + t.Fatal(err) + } + body, _ := ioutil.ReadAll(req.Body) + expectedBody := `{"age":1}` + if string(body) != expectedBody { + t.Fatal("Failed to build request") + } +} + +func TestBuildCurlRequest(t *testing.T) { + req, _ := requests.BuildRequest("post", "https://baidu.com/path?q=curl&v=1", requests.Json{ + "age": 1, + }) + curl := requests.BuildCurlRequest(req) + if !regexp.MustCompile(`^curl -X POST .+ 'https://baidu.com/path\?q=curl&v=1'`).MatchString(curl) { + t.Fatal(`bad curl cmd: ` + curl) + } + t.Log(curl) +} diff --git a/examples/req_header_test.go b/examples/req_header_test.go new file mode 100644 index 0000000..f2b949c --- /dev/null +++ b/examples/req_header_test.go @@ -0,0 +1,37 @@ +package examples + +import ( + "testing" + + "github.com/asmcos/requests" +) + +// Send headers +func TestSendHeaders(t *testing.T) { + println("Test Get: send header") + requests.Get( + "http://www.zhanluejia.net.cn", + requests.Header{"Referer": "http://www.jeapedu.com"}, + ) +} + +// Set session headers +func TestSendSessionHeader(t *testing.T) { + session := requests.Sessions() + session.SetHeader("accept-encoding", "gzip, deflate, br") + session.Get("http://www.zhanluejia.net.cn") +} + +// Set global header(user-agent) +func TestSetGlobalHeader(t *testing.T) { + headerK := "User-Agent" + headerV := "Custom-Test-Go-User-Agent" + requests.SetHeader(headerK, headerV) + req, err := requests.BuildRequest("post", "http://baidu.com/a/b/c") + if err != nil { + t.Fatal(err) + } + if req.Header.Get(headerK) != headerV { + t.Fatalf("Expected header %s is %s", headerK, headerV) + } +} diff --git a/examples/resp_handler_test.go b/examples/resp_handler_test.go new file mode 100644 index 0000000..036d5d7 --- /dev/null +++ b/examples/resp_handler_test.go @@ -0,0 +1,15 @@ +package examples + +import ( + "testing" + + "github.com/asmcos/requests" + _ "github.com/asmcos/requests/init" +) + +// Test response headers +func TestResponseHeader1(t *testing.T) { + resp, _ := requests.Get("https://httpbin.org/get") + println("content-type:", resp.R.Header.Get("content-type")) + //println(resp.Text()) +} diff --git a/examples/resp_test.go b/examples/resp_test.go new file mode 100644 index 0000000..a0bc9d6 --- /dev/null +++ b/examples/resp_test.go @@ -0,0 +1,21 @@ +package examples + +import ( + "testing" + + "github.com/asmcos/requests" + _ "github.com/asmcos/requests/init" +) + +// Test response headers +func TestResponseHeader(t *testing.T) { + resp, _ := requests.Get("https://httpbin.org/get") + println("content-type:", resp.R.Header.Get("content-type")) + //println(resp.Text()) +} + +// Test response body +func TestResponseBody(t *testing.T) { + resp, _ := requests.Get("https://httpbin.org/get") + println(resp.Text()) +} diff --git a/examples/session_test.go b/examples/session_test.go new file mode 100644 index 0000000..f5b7f13 --- /dev/null +++ b/examples/session_test.go @@ -0,0 +1,30 @@ +package examples + +import ( + "testing" + + r "github.com/asmcos/requests" +) + +// Test Session with cookies +func TestSessionWithCookie(t *testing.T) { + var data struct { + Cookies struct { + Count string `json:"count"` + } + } + session := r.Sessions() + // set cookies: count=100 + session.Get("https://httpbin.org/cookies/set?count=100") + // get cookies + resp, err := session.Get("https://httpbin.org/cookies") + if err == nil { + resp.Json(&data) + if data.Cookies.Count != "100" { + t.Fatal("Failed to get valid cookies: " + resp.Text()) + } + } + if err != nil { + t.Fatal(err) + } +} diff --git a/examples/timeout_test.go b/examples/timeout_test.go new file mode 100644 index 0000000..efb3592 --- /dev/null +++ b/examples/timeout_test.go @@ -0,0 +1,30 @@ +package examples + +import ( + "fmt" + "testing" + + "github.com/asmcos/requests" + _ "github.com/asmcos/requests/init" + "github.com/davecgh/go-spew/spew" +) + +func TestClose(t *testing.T) { + fmt.Println("Test Close") + req := requests.Sessions() + for i := 0; i < 10; i++ { + _, err := req.Post( + "http://localhost:1337/requests", + requests.Datas{"SrcIp": "4312"}) + fmt.Printf("\r%d %v", i, err) + req.Close() + } + + spew.Dump(req) + fmt.Println("10 times get test end.") +} +func TestTimeout(t *testing.T) { + println("Test Timeout") + req := requests.Sessions().SetTimeout(20) + req.Get("http://golang.org") +} diff --git a/go.mod b/go.mod index 863d801..c6856fe 100644 --- a/go.mod +++ b/go.mod @@ -2,4 +2,7 @@ module github.com/asmcos/requests go 1.13 -require github.com/davecgh/go-spew v1.1.1 +require ( + github.com/alessio/shellescape v1.4.1 + github.com/davecgh/go-spew v1.1.1 +) diff --git a/helpers.go b/helpers.go new file mode 100644 index 0000000..593e513 --- /dev/null +++ b/helpers.go @@ -0,0 +1,52 @@ +package requests + +import ( + "io/ioutil" + "net/url" + "os" + "path" + "runtime" +) + +var VERSION string = "v0.0.0" + +func getVersion() string{ + _, filename, _, _ := runtime.Caller(0) + versionFile := path.Dir(filename) + "/version" + version, _ := ioutil.ReadFile(versionFile) + VERSION = string(version) + return VERSION + +} + +func init() { + getVersion() +} + +// open file for post upload files +func openFile(filename string) *os.File { + r, err := os.Open(filename) + if err != nil { + panic(err) + } + return r +} + +// handle URL params +func buildURLParams(userURL string, params ...map[string]string) (string, error) { + parsedURL, err := url.Parse(userURL) + + if err != nil { + return "", err + } + + values := parsedURL.Query() + + for _, param := range params { + for key, value := range param { + values.Set(key, value) + } + } + parsedURL.RawQuery = values.Encode() + return parsedURL.String(), nil +} diff --git a/init/init-test.go b/init/init-test.go new file mode 100644 index 0000000..a7bfcb0 --- /dev/null +++ b/init/init-test.go @@ -0,0 +1,16 @@ +package init + +import ( + "os" + "path" + "runtime" +) + +func init() { + _, filename, _, _ := runtime.Caller(0) + dir := path.Join(path.Dir(filename), "..") + err := os.Chdir(dir) + if err != nil { + panic(err) + } +} diff --git a/makefile b/makefile new file mode 100644 index 0000000..06106fb --- /dev/null +++ b/makefile @@ -0,0 +1,14 @@ +msg?= +test: + go test -v ./examples + +.ONESHELL: +gitcheck: + if [[ "$(msg)" = "" ]] ; then echo "Usage: make pkg msg='commit msg'";exit 20; fi + +.ONESHELL: +pkg: gitcheck test + hash newversion.py && newversion.py version + git commit -am "$(msg)" + jfrog "rt" "go-publish" "go-pl" $$(cat version) "--url=$$GOPROXY_API" --user=$$GOPROXY_USER --apikey=$$GOPROXY_PASS + v=`cat version` && git tag "$$v" && git push origin "$$v" diff --git a/methods.go b/methods.go new file mode 100644 index 0000000..1a83926 --- /dev/null +++ b/methods.go @@ -0,0 +1,70 @@ +/* Copyright(2) 2018 by asmcos and asmcos . +Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package requests + +/**************post/get/delete/patch*************************/ +func Get(origurl string, args ...interface{}) (resp *Response, err error) { + // call request Get + resp, err = Sessions().Get(origurl, args...) + return resp, err +} + +func Post(origurl string, args ...interface{}) (resp *Response, err error) { + resp, err = Sessions().Post(origurl, args...) + return +} + +// Put +func Put(origurl string, args ...interface{}) (resp *Response, err error) { + resp, err = Sessions().Put(origurl, args...) + return +} + +// Delete +func Delete(origurl string, args ...interface{}) (resp *Response, err error) { + resp, err = Sessions().Delete(origurl, args...) + return +} + +// Patch +func Patch(origurl string, args ...interface{}) (resp *Response, err error) { + resp, err = Sessions().Patch(origurl, args...) + return +} + +func (session *Session) Get(origurl string, args ...interface{}) (resp *Response, err error) { + session.httpreq.Method = "GET" + resp, err = session.Run(origurl, args...) + return resp, err +} +func (session *Session) Post(origurl string, args ...interface{}) (resp *Response, err error) { + session.httpreq.Method = "POST" + resp, err = session.Run(origurl, args...) + return resp, err +} +func (session *Session) Delete(origurl string, args ...interface{}) (resp *Response, err error) { + session.httpreq.Method = "DELETE" + resp, err = session.Run(origurl, args...) + return resp, err +} +func (session *Session) Put(origurl string, args ...interface{}) (resp *Response, err error) { + session.httpreq.Method = "PUT" + resp, err = session.Run(origurl, args...) + return resp, err +} + +func (session *Session) Patch(origurl string, args ...interface{}) (resp *Response, err error) { + session.httpreq.Method = "PATCH" + resp, err = session.Run(origurl, args...) + return resp, err +} diff --git a/requests.go b/requests.go deleted file mode 100644 index 5d91b08..0000000 --- a/requests.go +++ /dev/null @@ -1,582 +0,0 @@ -/* Copyright(2) 2018 by asmcos . -Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package requests - -import ( - "bytes" - "compress/gzip" - "crypto/tls" - "encoding/json" - "fmt" - "io" - "io/ioutil" - "mime/multipart" - "net/http" - "net/http/cookiejar" - "net/http/httputil" - "net/url" - "os" - "strings" - "time" -) - -var VERSION string = "0.8" - -type Request struct { - httpreq *http.Request - Header *http.Header - Client *http.Client - Debug int - Cookies []*http.Cookie -} - -type Response struct { - R *http.Response - content []byte - text string - req *Request -} - -type Header map[string]string -type Params map[string]string -type Datas map[string]string // for post form -type Files map[string]string // name ,filename - -// {username,password} -type Auth []string - -func Requests() *Request { - - req := new(Request) - - req.httpreq = &http.Request{ - Method: "GET", - Header: make(http.Header), - Proto: "HTTP/1.1", - ProtoMajor: 1, - ProtoMinor: 1, - } - req.Header = &req.httpreq.Header - req.httpreq.Header.Set("User-Agent", "Go-Requests "+VERSION) - - req.Client = &http.Client{} - - // auto with Cookies - // cookiejar.New source code return jar, nil - jar, _ := cookiejar.New(nil) - - req.Client.Jar = jar - - return req -} - -// Get ,req.Get - -func Get(origurl string, args ...interface{}) (resp *Response, err error) { - req := Requests() - - // call request Get - resp, err = req.Get(origurl, args...) - return resp, err -} - -func (req *Request) Get(origurl string, args ...interface{}) (resp *Response, err error) { - - req.httpreq.Method = "GET" - - // set params ?a=b&b=c - //set Header - params := []map[string]string{} - - //reset Cookies, - //Client.Do can copy cookie from client.Jar to req.Header - delete(req.httpreq.Header, "Cookie") - - for _, arg := range args { - switch a := arg.(type) { - // arg is Header , set to request header - case Header: - - for k, v := range a { - req.Header.Set(k, v) - } - // arg is "GET" params - // ?title=website&id=1860&from=login - case Params: - params = append(params, a) - case Auth: - // a{username,password} - req.httpreq.SetBasicAuth(a[0], a[1]) - } - } - - disturl, _ := buildURLParams(origurl, params...) - - //prepare to Do - URL, err := url.Parse(disturl) - if err != nil { - return nil, err - } - req.httpreq.URL = URL - - req.ClientSetCookies() - - req.RequestDebug() - - res, err := req.Client.Do(req.httpreq) - - if err != nil { - fmt.Println(err) - return nil, err - } - - - resp = &Response{} - resp.R = res - resp.req = req - - resp.Content() - defer res.Body.Close() - - resp.ResponseDebug() - return resp, nil -} - -// handle URL params -func buildURLParams(userURL string, params ...map[string]string) (string, error) { - parsedURL, err := url.Parse(userURL) - - if err != nil { - return "", err - } - - parsedQuery, err := url.ParseQuery(parsedURL.RawQuery) - - if err != nil { - return "", nil - } - - for _, param := range params { - for key, value := range param { - parsedQuery.Add(key, value) - } - } - return addQueryParams(parsedURL, parsedQuery), nil -} - -func addQueryParams(parsedURL *url.URL, parsedQuery url.Values) string { - if len(parsedQuery) > 0 { - return strings.Join([]string{strings.Replace(parsedURL.String(), "?"+parsedURL.RawQuery, "", -1), parsedQuery.Encode()}, "?") - } - return strings.Replace(parsedURL.String(), "?"+parsedURL.RawQuery, "", -1) -} - -func (req *Request) RequestDebug() { - - if req.Debug != 1 { - return - } - - fmt.Println("===========Go RequestDebug ============") - - message, err := httputil.DumpRequestOut(req.httpreq, false) - if err != nil { - return - } - fmt.Println(string(message)) - - if len(req.Client.Jar.Cookies(req.httpreq.URL)) > 0 { - fmt.Println("Cookies:") - for _, cookie := range req.Client.Jar.Cookies(req.httpreq.URL) { - fmt.Println(cookie) - } - } -} - -// cookies -// cookies only save to Client.Jar -// req.Cookies is temporary -func (req *Request) SetCookie(cookie *http.Cookie) { - req.Cookies = append(req.Cookies, cookie) -} - -func (req *Request) ClearCookies() { - req.Cookies = req.Cookies[0:0] -} - -func (req *Request) ClientSetCookies() { - - if len(req.Cookies) > 0 { - // 1. Cookies have content, Copy Cookies to Client.jar - // 2. Clear Cookies - req.Client.Jar.SetCookies(req.httpreq.URL, req.Cookies) - req.ClearCookies() - } - -} - -// set timeout s = second -func (req *Request) SetTimeout(n time.Duration) { - req.Client.Timeout = time.Duration(n * time.Second) -} - - -func (req *Request) Close( ) { - req.httpreq.Close = true -} - -func (req *Request) Proxy(proxyurl string) { - - urli := url.URL{} - urlproxy, err := urli.Parse(proxyurl) - if err != nil { - fmt.Println("Set proxy failed") - return - } - req.Client.Transport = &http.Transport{ - Proxy: http.ProxyURL(urlproxy), - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - } - -} - -/**************/ -func (resp *Response) ResponseDebug() { - - if resp.req.Debug != 1 { - return - } - - fmt.Println("===========Go ResponseDebug ============") - - message, err := httputil.DumpResponse(resp.R, false) - if err != nil { - return - } - - fmt.Println(string(message)) - -} - -func (resp *Response) Content() []byte { - - var err error - - if len(resp.content) > 0{ - return resp.content - } - - var Body = resp.R.Body - if resp.R.Header.Get("Content-Encoding") == "gzip" && resp.req.Header.Get("Accept-Encoding") != "" { - // fmt.Println("gzip") - reader, err := gzip.NewReader(Body) - if err != nil { - return nil - } - Body = reader - } - - resp.content, err = ioutil.ReadAll(Body) - if err != nil { - return nil - } - - return resp.content -} - -func (resp *Response) Text() string { - if resp.content == nil { - resp.Content() - } - resp.text = string(resp.content) - return resp.text -} - -func (resp *Response) SaveFile(filename string) error { - if resp.content == nil { - resp.Content() - } - f, err := os.Create(filename) - if err != nil { - return err - } - defer f.Close() - - _, err = f.Write(resp.content) - f.Sync() - - return err -} - -func (resp *Response) Json(v interface{}) error { - if resp.content == nil { - resp.Content() - } - return json.Unmarshal(resp.content, v) -} - -func (resp *Response) Cookies() (cookies []*http.Cookie) { - httpreq := resp.req.httpreq - client := resp.req.Client - - cookies = client.Jar.Cookies(httpreq.URL) - - return cookies - -} - -/**************post*************************/ -// call req.Post ,only for easy -func Post(origurl string, args ...interface{}) (resp *Response, err error) { - req := Requests() - - // call request Get - resp, err = req.Post(origurl, args...) - return resp, err -} - -func PostJson(origurl string, args ...interface{}) (resp *Response, err error) { - req := Requests() - - // call request Get - resp, err = req.PostJson(origurl, args...) - return resp, err -} - -// POST requests - -func (req *Request) PostJson(origurl string, args ...interface{}) (resp *Response, err error) { - - req.httpreq.Method = "POST" - - req.Header.Set("Content-Type", "application/json") - - //reset Cookies, - //Client.Do can copy cookie from client.Jar to req.Header - delete(req.httpreq.Header, "Cookie") - - for _, arg := range args { - switch a := arg.(type) { - // arg is Header , set to request header - case Header: - - for k, v := range a { - req.Header.Set(k, v) - } - case string: - req.setBodyRawBytes(ioutil.NopCloser(strings.NewReader(arg.(string)))) - case Auth: - // a{username,password} - req.httpreq.SetBasicAuth(a[0], a[1]) - default: - b := new(bytes.Buffer) - err = json.NewEncoder(b).Encode(a) - if err != nil { - return nil, err - } - req.setBodyRawBytes(ioutil.NopCloser(b)) - } - } - - //prepare to Do - URL, err := url.Parse(origurl) - if err != nil { - return nil, err - } - req.httpreq.URL = URL - - req.ClientSetCookies() - - req.RequestDebug() - - res, err := req.Client.Do(req.httpreq) - - // clear post request information - req.httpreq.Body = nil - req.httpreq.GetBody = nil - req.httpreq.ContentLength = 0 - - if err != nil { - fmt.Println(err) - return nil, err - } - - - resp = &Response{} - resp.R = res - resp.req = req - - resp.Content() - defer res.Body.Close() - resp.ResponseDebug() - return resp, nil -} - -func (req *Request) Post(origurl string, args ...interface{}) (resp *Response, err error) { - - req.httpreq.Method = "POST" - - //set default - req.Header.Set("Content-Type", "application/x-www-form-urlencoded") - - // set params ?a=b&b=c - //set Header - params := []map[string]string{} - datas := []map[string]string{} // POST - files := []map[string]string{} //post file - - //reset Cookies, - //Client.Do can copy cookie from client.Jar to req.Header - delete(req.httpreq.Header, "Cookie") - - for _, arg := range args { - switch a := arg.(type) { - // arg is Header , set to request header - case Header: - - for k, v := range a { - req.Header.Set(k, v) - } - // arg is "GET" params - // ?title=website&id=1860&from=login - case Params: - params = append(params, a) - - case Datas: //Post form data,packaged in body. - datas = append(datas, a) - case Files: - files = append(files, a) - case Auth: - // a{username,password} - req.httpreq.SetBasicAuth(a[0], a[1]) - } - } - - disturl, _ := buildURLParams(origurl, params...) - - if len(files) > 0 { - req.buildFilesAndForms(files, datas) - - } else { - Forms := req.buildForms(datas...) - req.setBodyBytes(Forms) // set forms to body - } - //prepare to Do - URL, err := url.Parse(disturl) - if err != nil { - return nil, err - } - req.httpreq.URL = URL - - req.ClientSetCookies() - - req.RequestDebug() - - res, err := req.Client.Do(req.httpreq) - - // clear post param - req.httpreq.Body = nil - req.httpreq.GetBody = nil - req.httpreq.ContentLength = 0 - - if err != nil { - fmt.Println(err) - return nil, err - } - - - resp = &Response{} - resp.R = res - resp.req = req - - resp.Content() - defer res.Body.Close() - - resp.ResponseDebug() - return resp, nil -} - -// only set forms -func (req *Request) setBodyBytes(Forms url.Values) { - - // maybe - data := Forms.Encode() - req.httpreq.Body = ioutil.NopCloser(strings.NewReader(data)) - req.httpreq.ContentLength = int64(len(data)) -} - -// only set forms -func (req *Request) setBodyRawBytes(read io.ReadCloser) { - req.httpreq.Body = read -} - -// upload file and form -// build to body format -func (req *Request) buildFilesAndForms(files []map[string]string, datas []map[string]string) { - - //handle file multipart - - var b bytes.Buffer - w := multipart.NewWriter(&b) - - for _, file := range files { - for k, v := range file { - part, err := w.CreateFormFile(k, v) - if err != nil { - fmt.Printf("Upload %s failed!", v) - panic(err) - } - file := openFile(v) - _, err = io.Copy(part, file) - if err != nil { - panic(err) - } - } - } - - for _, data := range datas { - for k, v := range data { - w.WriteField(k, v) - } - } - - w.Close() - // set file header example: - // "Content-Type": "multipart/form-data; boundary=------------------------7d87eceb5520850c", - req.httpreq.Body = ioutil.NopCloser(bytes.NewReader(b.Bytes())) - req.httpreq.ContentLength = int64(b.Len()) - req.Header.Set("Content-Type", w.FormDataContentType()) -} - -// build post Form data -func (req *Request) buildForms(datas ...map[string]string) (Forms url.Values) { - Forms = url.Values{} - for _, data := range datas { - for key, value := range data { - Forms.Add(key, value) - } - } - return Forms -} - -// open file for post upload files - -func openFile(filename string) *os.File { - r, err := os.Open(filename) - if err != nil { - panic(err) - } - return r -} diff --git a/requests_test.go b/requests_test.go deleted file mode 100644 index 4c55468..0000000 --- a/requests_test.go +++ /dev/null @@ -1,270 +0,0 @@ -package requests - -import ( - "fmt" - "github.com/davecgh/go-spew/spew" - "net/http" - "os" - "testing" -) - -func TestGet(t *testing.T) { - // example 1 - println("Get example1") - req := Requests() - - req.Header.Set("accept-encoding", "gzip, deflate, br") - req.Get("http://www.zhanluejia.net.cn", Header{"Referer": "http://www.jeapedu.com"}, Params{"c": "d", "e": "f"}, Params{"c": "a"}) - - // example 2 - println("Get example2") - h := Header{ - "Referer": "http://www.zhanluejia.net.cn", - "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8", - } - - Get("http://www.zhanluejia.net.cn", h, Header{"accept-encoding": "gzip, deflate, br"}) - - // example 3 - println("Get example3") - p := Params{ - "title": "The blog", - "name": "file", - "id": "12345", - } - resp, err := Requests().Get("http://www.cpython.org", p) - - if err == nil { - resp.Text() - fmt.Println(resp.Text()) - } - - // example 4 - println("Get example4") - // test authentication usernae,password - //documentation https://www.httpwatch.com/httpgallery/authentication/#showExample10 - req = Requests() - resp, err = req.Get("https://www.httpwatch.com/httpgallery/authentication/authenticatedimage/default.aspx?0.45874470316137206", Auth{"httpwatch", "foo"}) - if err == nil { - fmt.Println(resp.R) - } - // this save file test PASS - // resp.SaveFile("auth.jpeg") - - //example 5 test Json - println("Get example5") - req = Requests() - req.Header.Set("Content-Type", "application/json") - resp, err = req.Get("https://httpbin.org/json") - - if err == nil { - var json map[string]interface{} - resp.Json(&json) - - for k, v := range json { - fmt.Println(k, v) - } - } - - // example 6 test gzip - println("Get example6") - req = Requests() - req.Debug = 1 - resp, err = req.Get("https://httpbin.org/gzip") - if err == nil { - fmt.Println(resp.Text()) - } - // example 7 proxy and debug - println("Get example7") - req = Requests() - req.Debug = 1 - - // You need open the line - //req.Proxy("http://192.168.1.190:8888") - - req.Get("https://www.sina.com.cn") - - //example 8 test auto Cookies - println("Get example8") - req = Requests() - req.Debug = 1 - // req.Proxy("http://192.168.1.190:8888") - req.Get("https://www.httpbin.org/cookies/set?freeform=1234") - req.Get("https://www.httpbin.org") - req.Get("https://www.httpbin.org/cookies/set?a=33d") - req.Get("https://www.httpbin.org") - - // example 9 test AddCookie - println("Get example9") - req = Requests() - req.Debug = 1 - - cookie := &http.Cookie{} - cookie.Name = "anewcookie" - cookie.Value = "20180825" - cookie.Path = "/" - - req.SetCookie(cookie) - - fmt.Println(req.Cookies) - // req.Proxy("http://127.0.0.1:8888") - req.Get("https://www.httpbin.org/cookies/set?freeform=1234") - req.Get("https://www.httpbin.org") - req.Get("https://www.httpbin.org/cookies/set?a=33d") - resp, err = req.Get("https://www.httpbin.org") - - if err == nil { - coo := resp.Cookies() - // coo is [] *http.Cookies - println("********cookies*******") - for _, c := range coo { - fmt.Println(c.Name, c.Value) - } - } - -} - -func TestClose(t *testing.T) { - - req := Requests() - fmt.Println("Start 1000 times get test...") - for i:=0;i<1000;i++{ - _,err := req.Post("http://localhost:1337/requests",Datas{"SrcIp":"4312"}) - fmt.Printf("\r%d %v",i,err) - req.Close() - } - - fmt.Println("1000 times get test end.") -} -func TestPost(t *testing.T) { - - // example 1 - // set post formdata - println("Post example1") - req := Requests() - req.Debug = 1 - - data := Datas{ - "comments": "ew", - "custemail": "a@231.com", - "custname": "1", - "custtel": "2", - "delivery": "12:45", - "size": "small", - "topping": "bacon", - } - - resp, err := req.Post("https://www.httpbin.org/post", data) - if err == nil { - fmt.Println(resp.Text()) - } - - //example 2 upload files - println("Post example2") - req = Requests() - req.Debug = 1 - path, _ := os.Getwd() - path1 := path + "/README.md" - path2 := path + "/docs/index.md" - - resp, err = req.Post("https://www.httpbin.org/post", data, Files{"a": path1, "b": path2}) - if err == nil { - fmt.Println(resp.Text()) - } - - req = Requests() - cookie := &http.Cookie{} - cookie.Name = "postcookie" - cookie.Value = "20200725" - cookie.Path = "/" - - req.SetCookie(cookie) - - //test post cookies - resp, err = req.Post("https://www.httpbin.org/post", data) - if err == nil { - coo := resp.Cookies() - // coo is [] *http.Cookies - println("********Post cookies*******") - for _, c := range coo { - fmt.Println(c.Name, c.Value) - } - } - -} - -func TestTimeout(t *testing.T) { - println("Timeout example1") - req := Requests() - req.Debug = 1 - - // 20 Second - req.SetTimeout(20) - req.Get("http://golang.org") - -} - -func TestPostGet(t *testing.T) { - - println("Test Post and Get") - - client := Requests() - client.Debug = 1 - - resp, err := client.Post("https://www.httpbin.org/post", Datas{"abc": "123", "ddd": "789"}) - - spew.Dump(client) - - resp, err = client.Get("https://www.httpbin.org/get") - if err != nil { - fmt.Println(err) - } - fmt.Println(resp.Text()) - -} - -func TestPostJson(t *testing.T) { - - type StructReq struct { - ContainerId string `json:"id"` - Worker string `json:"worker"` - Force bool `json:"force"` - } - - dataStruct := StructReq{ - ContainerId: "123456", - Worker: "worker1", - Force: true, - } - - dataMap := map[string]interface{}{ - "id": "123456", - "worker": "worker1", - "force": true, - } - - dataJsonStr := "{\"id\":\"123456\",\"worker\":\"worker1\",\"force\": true}" - - println("Test PostJson") - - client := Requests() - client.Debug = 1 - - resp, err := client.PostJson("https://www.httpbin.org/post", dataStruct) - if err != nil { - t.Fatalf("post struct json error: %v", err) - } - fmt.Println(resp.Text()) - - resp, err = client.PostJson("https://www.httpbin.org/post", dataMap) - if err != nil { - t.Fatalf("post struct json error: %v", err) - } - fmt.Println(resp.Text()) - - resp, err = client.PostJson("https://www.httpbin.org/post", dataJsonStr) - if err != nil { - t.Fatalf("post struct json error: %v", err) - } - fmt.Println(resp.Text()) -} diff --git a/response.go b/response.go new file mode 100644 index 0000000..fd72769 --- /dev/null +++ b/response.go @@ -0,0 +1,109 @@ +/* Copyright(2) 2018 by asmcos and asmcos . +Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package requests + +import ( + "compress/gzip" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "net/http/httputil" + "os" +) + +type Response struct { + R *http.Response + content []byte + text string + session *Session +} + +func (resp *Response) ResponseDebug() { + if !resp.session.debug { + return + } + fmt.Println("===========Go ResponseDebug ============") + + message, err := httputil.DumpResponse(resp.R, false) + if err != nil { + return + } + + fmt.Println(string(message)) +} + +func (resp *Response) Content() []byte { + var err error + if resp.content != nil { + return resp.content + } + defer resp.R.Body.Close() + + var Body = resp.R.Body + if resp.R.Header.Get("Content-Encoding") == "gzip" && resp.session.Header.Get("Accept-Encoding") != "" { + reader, err := gzip.NewReader(Body) + if err != nil { + return nil + } + Body = reader + } + + resp.content, err = ioutil.ReadAll(Body) + if err != nil { + return nil + } + + return resp.content +} + +func (resp *Response) Text() string { + if resp.content == nil { + resp.Content() + } + resp.text = string(resp.content) + return resp.text +} + +func (resp *Response) SaveFile(filename string) error { + if resp.content == nil { + resp.Content() + } + f, err := os.Create(filename) + if err != nil { + return err + } + defer f.Close() + + _, err = f.Write(resp.content) + f.Sync() + + return err +} + +func (resp *Response) Json(v interface{}) error { + if resp.content == nil { + resp.Content() + } + return json.Unmarshal(resp.content, v) +} + +func (resp *Response) Cookies() (cookies []*http.Cookie) { + httpreq := resp.session.httpreq + client := resp.session.Client + + cookies = client.Jar.Cookies(httpreq.URL) + + return cookies + +} diff --git a/sessions.go b/sessions.go new file mode 100644 index 0000000..3167607 --- /dev/null +++ b/sessions.go @@ -0,0 +1,357 @@ +/* Copyright(2) 2018 by asmcos and asmcos . +Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package requests + +import ( + "bytes" + "crypto/tls" + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "mime/multipart" + "net/http" + "net/http/cookiejar" + "net/http/httputil" + "net/url" + "strings" + "time" +) + +var respHandler func(*Response) +var gHeader = map[string]string{ + "User-Agent": "Go-requests-" + getVersion(), +} + +// SetRespHandler +func SetRespHandler(fn func(*Response)) { + respHandler = fn +} + +type Session struct { + httpreq *http.Request + Client *http.Client + debug bool + respHandler func(*Response) + // global header + Header *http.Header + Cookies []*http.Cookie +} + +type Header map[string]string +type Params map[string]string +type Datas map[string]string // for post form +type Json map[string]interface{} // for Json +type Files map[string]string // name ,filename +// type AnyData interface{} // for AnyData + +// Auth - {username,password} +type Auth []string +type Method string + +// Sessions +// @params method GET|POST|PUT|DELETE|PATCH +func Sessions() *Session { + + session := new(Session) + session.reset() + + session.Client = &http.Client{} + + // cookiejar.New source code return jar, nil + jar, _ := cookiejar.New(nil) + + session.Client.Jar = jar + + return session +} + +// Set global header +func SetHeader(key, value string) { + if value == "" { + delete(gHeader, key) + return + } + gHeader[key] = value +} + +func (session *Session) reset() { + session.httpreq = &http.Request{ + Method: "GET", + Header: make(http.Header), + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + } + session.Header = &session.httpreq.Header + for key, value := range gHeader { + session.httpreq.Header.Set(key, value) + } +} + +func (session *Session) RequestDebug() { + if !session.debug { + return + } + fmt.Println("===========Go RequestDebug !============") + curl := BuildCurlRequest(session.httpreq) + fmt.Println(curl) + message, err := httputil.DumpRequestOut(session.httpreq, false) + if err != nil { + return + } + fmt.Println(string(message)) + + if len(session.Client.Jar.Cookies(session.httpreq.URL)) > 0 { + fmt.Println("Cookies:") + for _, cookie := range session.Client.Jar.Cookies(session.httpreq.URL) { + fmt.Println(cookie) + } + } +} + +// cookies +// cookies only save to Client.Jar +// session.Cookies is temporary +func (session *Session) SetCookie(cookie *http.Cookie) *Session { + session.Cookies = append(session.Cookies, cookie) + return session +} + +func (session *Session) ClearCookies() { + session.Cookies = session.Cookies[0:0] +} + +// ClientSetCookies - +func (session *Session) ClientSetCookies() { + if len(session.Cookies) > 0 { + // 1. Cookies have content, Copy Cookies to Client.jar + // for _, cookie := range session.Cookies { + // session.httpreq.AddCookie(cookie) + // } + session.Client.Jar.SetCookies(session.httpreq.URL, session.Cookies) + // 2. Clear Cookies + session.ClearCookies() + } + +} + +// set timeout s = second +func (session *Session) SetTimeout(n time.Duration) *Session { + session.Client.Timeout = time.Duration(n * time.Second) + return session +} + +func (session *Session) Close() { + session.httpreq.Close = true +} + +func (session *Session) Proxy(proxyurl string) { + urli := url.URL{} + urlproxy, err := urli.Parse(proxyurl) + if err != nil { + fmt.Println("Set proxy failed") + return + } + session.Client.Transport = &http.Transport{ + Proxy: http.ProxyURL(urlproxy), + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } +} + +// SetRespHandler - +func (session *Session) SetRespHandler(fn func(*Response)) *Session { + session.respHandler = fn + return session +} + +// SetMethod +func (session *Session) SetMethod(method string) *Session { + session.httpreq.Method = strings.ToUpper(method) + return session +} + +// SetHeader +func (session *Session) SetHeader(key, value string) *Session { + session.Header.Set(key, value) + return session +} + +// BuildRequest +func (session *Session) BuildRequest(origurl string, args ...interface{}) (*http.Request, error) { + contentType := "application/x-www-form-urlencoded" + params := []map[string]string{} + datas := []map[string]string{} // form data + files := []map[string]string{} //file data + bodyBytes := []byte{} + + for _, arg := range args { + switch a := arg.(type) { + // arg is Header , set to request header + case Method: + session.httpreq.Method = strings.ToUpper(string(a)) + case Header: + for k, v := range a { + session.httpreq.Header.Set(k, v) + } + case Params: + params = append(params, a) + case Datas: //Post form data,packaged in body. + datas = append(datas, a) + case Files: + files = append(files, a) + case Auth: + session.httpreq.SetBasicAuth(a[0], a[1]) + case string: + bodyBytes = []byte(a) + case []byte: + bodyBytes = a + case *http.Cookie: + session.SetCookie(a) + case Json: + contentType = "application/json" + bodyBytes = session.buildJSON(a) + default: + contentType = "application/json" + bodyBytes = session.buildJSON(a) + } + } + if session.httpreq.Header.Get("Content-Type") == "" { + session.httpreq.Header.Set("Content-Type", contentType) + } + + disturl, _ := buildURLParams(origurl, params...) + + if len(files) > 0 { + session.buildFilesAndForms(files, datas) + } else if len(bodyBytes) > 0 { + // fmt.Printf("jsonBytes=%#v\n", string(jsonBytes)) + session.setBodyBytes(bodyBytes) // set forms to body + } else { + Forms := session.buildForms(datas...) + session.setBodyForms(Forms) // set forms to body + } + //prepare to Do + URL, err := url.Parse(disturl) + if err != nil { + return nil, err + } + session.httpreq.URL = URL + + session.ClientSetCookies() + // fmt.Printf("session:%#v\n", session.httpreq) + // fmt.Printf("session-url:%#v\n", session.httpreq.URL.String()) + return session.httpreq, nil + +} + +// Post - +func (session *Session) Run(origurl string, args ...interface{}) (resp *Response, err error) { + session.BuildRequest(origurl, args...) + session.RequestDebug() + res, err := session.Client.Do(session.httpreq) + + if err != nil { + return nil, errors.New(session.httpreq.Method + " " + origurl + " " + err.Error()) + } + + resp = &Response{} + resp.R = res + req_dup := *session + resp.session = &req_dup + resp.ResponseDebug() + resp.Content() + session.reset() + if respHandler != nil { + respHandler(resp) + } + if session.respHandler != nil { + session.respHandler(resp) + } + return resp, nil +} + +// only set forms +func (session *Session) setBodyForms(Forms url.Values) { + data := Forms.Encode() + session.httpreq.Body = ioutil.NopCloser(strings.NewReader(data)) + session.httpreq.ContentLength = int64(len(data)) +} + +// only set forms +func (session *Session) setBodyBytes(data []byte) { + session.httpreq.Body = ioutil.NopCloser(bytes.NewReader(data)) + session.httpreq.ContentLength = int64(len(data)) +} + +// upload file and form +// build to body format +func (session *Session) buildFilesAndForms(files []map[string]string, datas []map[string]string) { + + //handle file multipart + + var b bytes.Buffer + w := multipart.NewWriter(&b) + + for _, file := range files { + for k, v := range file { + part, err := w.CreateFormFile(k, v) + if err != nil { + fmt.Printf("Upload %s failed!", v) + panic(err) + } + file := openFile(v) + _, err = io.Copy(part, file) + if err != nil { + panic(err) + } + } + } + + for _, data := range datas { + for k, v := range data { + w.WriteField(k, v) + } + } + + w.Close() + // set file header example: + // "Content-Type": "multipart/form-data; boundary=------------------------7d87eceb5520850c", + session.httpreq.Body = ioutil.NopCloser(bytes.NewReader(b.Bytes())) + session.httpreq.ContentLength = int64(b.Len()) + session.Header.Set("Content-Type", w.FormDataContentType()) +} + +// build post Form data +func (session *Session) buildForms(datas ...map[string]string) (Forms url.Values) { + Forms = url.Values{} + for _, data := range datas { + for key, value := range data { + Forms.Add(key, value) + } + } + return Forms +} + +func (session *Session) buildJSON(data interface{}) []byte { + jsonBytes, _ := json.Marshal(data) + + // fmt.Printf("a1=%#v,jsons=%#v\nahui\n", data, string(jsonBytes)) + return jsonBytes +} + +func (session *Session) SetDebug(debug bool) *Session { + session.debug = debug + return session +} diff --git a/utils.go b/utils.go new file mode 100644 index 0000000..4132417 --- /dev/null +++ b/utils.go @@ -0,0 +1,62 @@ +package requests + +import ( + "bytes" + "io/ioutil" + "net/http" + + "github.com/alessio/shellescape" +) + +// BuildRequest - +func BuildRequest(method string, origurl string, args ...interface{}) (req *http.Request, err error) { + // call request Get + args = append(args, Method(method)) + req, err = Sessions().BuildRequest(origurl, args...) + return +} + +func BuildCurlRequest(req *http.Request) (curl string) { + curl = "curl -X " + req.Method + " " + // req.Host + req.URL.Path + "?" + req.URL.RawQuery + " " + req.Proto + " " + headers := getHeaders(req) + for _, kv := range *headers { + curl += `-H ` + shellescape.Quote(kv[0]+": "+kv[1]) + ` ` + } + // // cookies + // for _, cookie := range req.Cookies() { + // fmt.Printf("cookie:%#v\n", cookie) + // } + // body + buf, _ := ioutil.ReadAll(req.Body) + req.Body = ioutil.NopCloser(bytes.NewBuffer(buf)) // important!! + if len(buf) > 0 { + curl += `-d ` + shellescape.Quote(string(buf)) + } + + curl += " " + shellescape.Quote(req.URL.String()) + return curl +} + +// getHeaders +func getHeaders(req *http.Request) *[][2]string { + headers := [][2]string{} + for k, vs := range req.Header { + for _, v := range vs { + headers = append(headers, [2]string{k, v}) + } + } + n := len(headers) + // fmt.Printf("%#v\n", headers) + // sort headers + for i := 0; i < n; i++ { + for j := n - 1; j > i; j-- { + jj := j - 1 + h1, h2 := headers[j], headers[jj] + if h1[0] < h2[0] { + headers[jj], headers[j] = headers[j], headers[jj] + } + } + } + return &headers +} diff --git a/version b/version new file mode 100644 index 0000000..0f8d34d --- /dev/null +++ b/version @@ -0,0 +1 @@ +v0.1.24 \ No newline at end of file