From a911e5f82adf50daaa8a9b0daf1984450121ce56 Mon Sep 17 00:00:00 2001 From: bxd <2216403312@qq.com> Date: Tue, 17 Oct 2023 11:33:20 +0800 Subject: [PATCH] optinmize doc info and print error --- README.md | 201 +++++++++++++++++++++++++++++++++++++++++------- client.go | 44 +++++------ option.go | 52 ++++++------- roundTripper.go | 4 +- 4 files changed, 218 insertions(+), 83 deletions(-) diff --git a/README.md b/README.md index 74fff60..3efee4e 100644 --- a/README.md +++ b/README.md @@ -20,49 +20,194 @@ go get github.com/gospider007/requests - Powerful and convenient request callback, very convenient to obtain every request detail # quick start -## send request,support :"GET","HEAD","POST","PUT","PATCH" +## Quickly send requests ```go -resp,error:=request.Get(nil,"http://httpbin.org/get") -``` -## use proxy -```go -resp, error := request.Get(nil, "http://httpbin.org/get", requests.RequestOption{Proxy: "http://127.0.0.1:8888"}) -``` -## use ja3 fingerprint -```go -resp, _ := request.Get(nil, "http://httpbin.org/get", requests.RequestOption{Ja3: true}) +package main + +import ( + "log" + "time" + + "github.com/gospider007/requests" +) + +func main() { + href := "http://httpbin.org/anything" + resp, err := requests.Post(nil, href, requests.RequestOption{ + Ja3: true, //enable ja3 fingerprint + Cookies: "a=1&b=2", //set cookies + // Proxy: "http://127.0.0.1:8888", //set proxy + Params: map[string]any{"query": "cat"}, //set query params + Headers: map[string]any{"token": 12345}, //set headers + Json: map[string]any{"age": 60}, //send json data + Timeout: time.Second * 10, //set timeout + }) + if err != nil { + log.Panic(err) + } + log.Print(resp.Text()) // Get content and parse as string + log.Print(resp.Content()) // Get content as bytes + log.Print(resp.Json()) // Get JSON and parse with gjson + log.Print(resp.Html()) // Get content and parse as DOM + log.Print(resp.Cookies()) // Get cookies +} ``` ## use session ```go -session, _ := requests.NewClient(nil) -resp, _ := session.Get(nil, "http://httpbin.org/get") +package main + +import ( + "log" + + "github.com/gospider007/requests" +) + +func main() { + href := "http://httpbin.org/anything" + session, err := requests.NewClient(nil) //use session + if err != nil { + log.Panic(err) + } + resp, err := session.Get(nil, href) + if err != nil { + log.Panic(err) + } + log.Print(resp.StatusCode()) //return status code +} ``` -## close body +## send websocket ```go -resp,_:=request.Get(nil,"http://httpbin.org/get") -resp.Close() +package main + +import ( + "log" + + "github.com/gospider007/requests" + "github.com/gospider007/websocket" +) + +func main() { + response, err := requests.Get(nil, "ws://82.157.123.54:9010/ajaxchattest", requests.RequestOption{Headers: map[string]string{ + "Origin": "http://coolaf.com", + }}) // Send WebSocket request + if err != nil { + log.Panic(err) + } + defer response.Close() + wsCli := response.WebSocket() + if err = wsCli.Send(nil, websocket.MessageText, "test"); err != nil { // Send text message + log.Panic(err) + } + msgType, con, err := wsCli.Recv(nil) // Receive message + if err != nil { + log.Panic(err) + } + log.Print(msgType) // Message type + log.Print(string(con)) // Message content +} ``` -## safe delete net.conn +## IPv4, IPv6 Address Control Parsing ```go -resp,_:=request.Get(nil,"http://httpbin.org/get") -resp.Delete() -``` -## force delete net.conn +package main + +import ( + "log" + + "github.com/gospider007/requests" +) + +func main() { + session, _ := requests.NewClient(nil, requests.ClientOption{ + AddrType: requests.Ipv4, // Prioritize parsing IPv4 addresses + // AddrType: requests.Ipv6, // Prioritize parsing IPv6 addresses + }) + resp, err := session.Get(nil, "https://test.ipw.cn") + if err != nil { + log.Panic(err) + } + log.Print(resp.Text()) + log.Print(resp.StatusCode()) +} +``` +## Generate Fingerprint from String ```go -resp,_:=request.Get(nil,"http://httpbin.org/get") -resp.ForceDelete() +package main + +import ( + "log" + + "github.com/gospider007/ja3" + "github.com/gospider007/requests" +) + +func main() { + ja3Str := "772,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,0-23-65281-10-11-35-16-5-13-18-51-45-43-27-17513,29-23-24,0" + Ja3Spec, _ := ja3.CreateSpecWithStr(ja3Str) // Generate fingerprint from string + resp, err := requests.Get(nil, "https://tools.scrapfly.io/api/fp/ja3?extended=1", requests.RequestOption{Ja3Spec: Ja3Spec}) + if err != nil { + log.Panic(err) + } + jsonData, _ := resp.Json() + log.Print(jsonData.Get("ja3").String()) + log.Print(jsonData.Get("ja3").String() == ja3Str) +} ``` -## websocket +## Generate Fingerprint from ID ```go -response, _ := request.Get(nil, "ws://82.157.123.54:9010/ajaxchattest", requests.RequestOption{Headers: map[string]string{"Origin": "http://coolaf.com"}}) -defer response.Close() -wsCli := response.WebSocket() +package main + +import ( + "log" + + "github.com/gospider007/ja3" + "github.com/gospider007/requests" +) + +func main() { + Ja3Spec, _ := ja3.CreateSpecWithId(ja3.HelloChrome_Auto) // Generate fingerprint from ID + resp, err := requests.Get(nil, "https://tools.scrapfly.io/api/fp/ja3?extended=1", requests.RequestOption{Ja3Spec: Ja3Spec}) + if err != nil { + log.Panic(err) + } + jsonData, _ := resp.Json() + log.Print(jsonData.Get("ja3").String()) +} ``` +## Modify H2 Fingerprint +```go +package main +import ( + "log" + "github.com/gospider007/ja3" + "github.com/gospider007/requests" +) - - +func main() { + h2ja3Spec := ja3.H2Ja3Spec{ + InitialSetting: []ja3.Setting{ + {Id: 1, Val: 65555}, + {Id: 2, Val: 1}, + {Id: 3, Val: 2000}, + {Id: 4, Val: 6291457}, + {Id: 6, Val: 262145}, + }, + ConnFlow: 15663106, + OrderHeaders: []string{ + ":method", + ":path", + ":scheme", + ":authority", + }, + } + resp, err := requests.Get(nil, "https://tools.scrapfly.io/api/fp/anything", requests.RequestOption{H2Ja3Spec: h2ja3Spec}) + if err != nil { + log.Panic(err) + } + log.Print(resp.Text()) +} +``` # Contributing If you have a bug report or feature request, you can [open an issue](../../issues/new) diff --git a/client.go b/client.go index 4b5c093..72237fb 100644 --- a/client.go +++ b/client.go @@ -13,39 +13,33 @@ import ( ) type ClientOption struct { - GetProxy func(ctx context.Context, url *url.URL) (string, error) //proxy callback:support https,http,socks5 proxy - Proxy string //proxy,support https,http,socks5 - - DisCookie bool //disable cookies - LocalAddr *net.TCPAddr //network card ip - - DialTimeout time.Duration //dial tcp timeout,default:15 - TlsHandshakeTimeout time.Duration //tls timeout,default:15 - KeepAlive time.Duration //keepalive,default:30 - ResponseHeaderTimeout time.Duration //ResponseHeaderTimeout ,default:30 - - AddrType AddrType //dns parse addr type - GetAddrType func(string) AddrType - Dns net.IP //dns - - Ja3 bool //enable ja3 fingerprint - Ja3Spec ja3.Ja3Spec //custom ja3Spec,use ja3.CreateSpecWithStr or ja3.CreateSpecWithId create - H2Ja3Spec ja3.H2Ja3Spec //h2 fingerprint - - RedirectNum int //redirect num ,<0 no redirect,==0 no limit - + Ja3 bool //enable ja3 fingerprint + Ja3Spec ja3.Ja3Spec //custom ja3Spec,use ja3.CreateSpecWithStr or ja3.CreateSpecWithId create + H2Ja3Spec ja3.H2Ja3Spec //h2 fingerprint + Proxy string //proxy,support https,http,socks5 + DisCookie bool //disable cookies DisDecode bool //disable auto decode DisUnZip bool //disable auto zip decode DisAlive bool //disable keepalive - TryNum int //try num + Bar bool ////enable bar display + Timeout time.Duration //request timeout OptionCallBack func(context.Context, *Client, *RequestOption) error //option callback,if error is returnd, break request ResultCallBack func(context.Context, *Client, *Response) error //result callback,if error is returnd,next errCallback ErrCallBack func(context.Context, *Client, error) error //error callback,if error is returnd,break request RequestCallBack func(context.Context, *http.Request, *http.Response) error //request and response callback,if error is returnd,reponse is error + TryNum int //try num + RedirectNum int //redirect num ,<0 no redirect,==0 no limit + Headers any //default headers - Timeout time.Duration //request timeout - Headers any //default headers - Bar bool ////enable bar display + GetProxy func(ctx context.Context, url *url.URL) (string, error) //proxy callback:support https,http,socks5 proxy + LocalAddr *net.TCPAddr //network card ip + DialTimeout time.Duration //dial tcp timeout,default:15 + TlsHandshakeTimeout time.Duration //tls timeout,default:15 + KeepAlive time.Duration //keepalive,default:30 + ResponseHeaderTimeout time.Duration //ResponseHeaderTimeout ,default:30 + AddrType AddrType //dns parse addr type + GetAddrType func(string) AddrType + Dns net.IP //dns } type Client struct { jar *Jar diff --git a/option.go b/option.go index ee8f481..7f6a9c8 100644 --- a/option.go +++ b/option.go @@ -17,48 +17,46 @@ import ( ) type RequestOption struct { - Ja3 bool //enable ja3 fingerprint - Ja3Spec ja3.Ja3Spec //custom ja3Spec,use ja3.CreateSpecWithStr or ja3.CreateSpecWithId create - H2Ja3Spec ja3.H2Ja3Spec //custom h2 fingerprint + Ja3 bool //enable ja3 fingerprint + Ja3Spec ja3.Ja3Spec //custom ja3Spec,use ja3.CreateSpecWithStr or ja3.CreateSpecWithId create + H2Ja3Spec ja3.H2Ja3Spec //custom h2 fingerprint + Proxy string //proxy,support http,https,socks5,example：http://127.0.0.1:7005 + DisCookie bool //disable cookies,not use cookies + DisDecode bool //disable auto decode + DisUnZip bool //disable auto zip decode + DisAlive bool //disable keepalive + Bar bool //enable bar display + Timeout time.Duration //request timeout + OptionCallBack func(context.Context, *Client, *RequestOption) error //option callback,if error is returnd, break request + ResultCallBack func(context.Context, *Client, *Response) error //result callback,if error is returnd,next errCallback + ErrCallBack func(context.Context, *Client, error) error //error callback,if error is returnd,break request + RequestCallBack func(context.Context, *http.Request, *http.Response) error //request and response callback,if error is returnd,reponse is error + TryNum int //try num + RedirectNum int //redirect num ,<0 no redirect,==0 no limit + Headers any //request headers：json,map，header + DisRead bool //disable auto read Referer string //set headers referer value Method string //method Url *url.URL Host string - Proxy string //proxy,support http,https,socks5,example：http://127.0.0.1:7005 - Timeout time.Duration //request timeout - Headers any //request headers：json,map，header - Cookies any // cookies,support : json,map,str，http.Header - Files []File //send multipart/form-data, file upload - Params any //url params，join url query,support json,map - Form any //send multipart/form-data,file upload,support json,map - Data any //send application/x-www-form-urlencoded, support string,[]bytes,json,map + Cookies any // cookies,support : json,map,str，http.Header + Files []File //send multipart/form-data, file upload + Params any //url params，join url query,support json,map + Form any //send multipart/form-data,file upload,support json,map + Data any //send application/x-www-form-urlencoded, support string,[]bytes,json,map Body io.Reader Json any //send application/json,support：string,[]bytes,json,map Text any //send text/xml,support: string,[]bytes,json,map ContentType string //headers Content-Type value Raw any //not setting context-type,support string,[]bytes,json,map - DisCookie bool //disable cookies,not use cookies - DisDecode bool //disable auto decode - DisRead bool //disable auto read - DisAlive bool //disable keepalive - - Bar bool //enable bar display DisProxy bool //force disable proxy - TryNum int //try num - - OptionCallBack func(context.Context, *Client, *RequestOption) error //option callback,if error is returnd, break request - ResultCallBack func(context.Context, *Client, *Response) error //result callback,if error is returnd,next errCallback - ErrCallBack func(context.Context, *Client, error) error //error callback,if error is returnd,break request - RequestCallBack func(context.Context, *http.Request, *http.Response) error //request and response callback,if error is returnd,reponse is error Jar *Jar //custom cookies - RedirectNum int //redirect num ,<0 no redirect,==0 no limit - DisUnZip bool //disable auto zip decode - WsOption websocket.Option //websocket option - converUrl string + WsOption websocket.Option //websocket option + converUrl string } func (obj *RequestOption) initBody() (err error) { diff --git a/roundTripper.go b/roundTripper.go index 2493325..2fa75af 100644 --- a/roundTripper.go +++ b/roundTripper.go @@ -7,7 +7,6 @@ import ( "errors" "fmt" "io" - "log" "net" "net/url" "sync" @@ -150,7 +149,6 @@ func (obj *RoundTripper) dial(ctxData *reqCtxData, addr string, key string, req if err != nil { return conne, tools.WrapError(err, "add tls error") } - log.Print(tlsConn.ConnectionState().NegotiatedProtocol) conne.h2 = tlsConn.ConnectionState().NegotiatedProtocol == "h2" netConn = tlsConn } else { @@ -165,7 +163,7 @@ func (obj *RoundTripper) dial(ctxData *reqCtxData, addr string, key string, req conne.rawConn = netConn if conne.h2 { if conne.h2RawConn, err = http2.NewClientConn(func() { - conne.deleteCnl() + conne.closeCnl() }, netConn, ctxData.h2Ja3Spec); err != nil { return conne, err }