diff --git a/README.md b/README.md deleted file mode 100644 index 7eeb5e6..0000000 --- a/README.md +++ /dev/null @@ -1,368 +0,0 @@ -# 功能概述 -- cookies 开关,连接池,http2,ja3 -- 自实现socks五,http代理,https代理 -- 自动解压缩,解码 -- dns缓存 -- 类型自动转化 -- 尝试重试,请求回调 -- websocket 协议 -- sse协议 -# 设置代理 -## 代理设置的优先级 -``` -全局代理方法 < 全局代理字符串 < 局部代理字符串 -``` -## 设置并修改全局代理方法 (只会在新建连接的时候调用获取代理的方法,复用连接的时候不会调用) -```golang -package main - -import ( - "log" - - "gitee.com/baixudong/requests" -) - -func main() { - //创建请求客户端 - reqCli, err := requests.NewClient(nil, requests.ClientOption{ - GetProxy: func(ctx context.Context, url *url.URL) (string, error) { //设置全局代理方法 - return "http://127.0.0.1:7005", nil - }}) - if err != nil { - log.Panic(err) - } - response, err := reqCli.Request(nil, "get", "http://myip.top") //发送get请求 - if err != nil { - log.Panic(err) - } - reqCli.SetGetProxy(func(ctx context.Context, url *url.URL) (string, error) { //修改全局代理方法 - return "http://127.0.0.1:7006", nil - }) - log.Print(response.Text()) //获取内容,解析为字符串 -} -``` -## 设置并修改全局代理 -```golang -package main - -import ( - "log" - - "gitee.com/baixudong/requests" -) - -func main() { - //创建请求客户端 - reqCli, err := requests.NewClient(nil, requests.ClientOption{ - Proxy: "http://127.0.0.1:7005", //设置全局代理 - }) - if err != nil { - log.Panic(err) - } - //发送get请求 - response, err := reqCli.Request(nil, "get", "http://myip.top") - if err != nil { - log.Panic(err) - } - err = reqCli.SetProxy("http://127.0.0.1:7006") //修改全局代理 - if err != nil { - log.Panic(err) - } - log.Print(response.Text()) //获取内容,解析为字符串 -} -``` -## 设置局部代理 -```golang -package main - -import ( - "log" - - "gitee.com/baixudong/requests" -) - -func main() { - //创建请求客户端 - reqCli, err := requests.NewClient(nil) - if err != nil { - log.Panic(err) - } - //发送get请求 - response, err := reqCli.Request(nil, "get", "http://myip.top", requests.RequestOption{ - Proxy: "http://127.0.0.1:7005", - }) - if err != nil { - log.Panic(err) - } - log.Print(response.Text()) //获取内容,解析为字符串 -} -``` -## 强制关闭代理,走本地网络 -```golang -package main - -import ( - "log" - - "gitee.com/baixudong/requests" -) - -func main() { - //创建请求客户端 - reqCli, err := requests.NewClient(nil) - if err != nil { - log.Panic(err) - } - //发送get请求 - response, err := reqCli.Request(nil, "get", "http://myip.top", requests.RequestOption{ - DisProxy: true, //强制走本地代理 - }) - if err != nil { - log.Panic(err) - } - log.Print(response.Text()) //获取内容,解析为字符串 -} -``` - -# 发送http请求 - -```golang -package main - -import ( - "log" - - "gitee.com/baixudong/requests" -) - -func main() { - reqCli, err := requests.NewClient(nil) //创建请求客户端 - if err != nil { - log.Panic(err) - } - response, err := reqCli.Request(nil, "get", "http://myip.top") //发送get请求 - if err != nil { - log.Panic(err) - } - log.Print(response.Text()) //获取内容,解析为字符串 - log.Print(response.Content()) //获取内容,解析为字节 - log.Print(response.Json()) //获取json,解析为gjson - log.Print(response.Html()) //获取内容,解析为dom - log.Print(response.Cookies()) //获取cookies -} - -``` - -# 发送websocket 请求 - -```golang -package main - -import ( - "context" - "log" - - "gitee.com/baixudong/requests" - "gitee.com/baixudong/websocket" -) - -func main() { - reqCli, err := requests.NewClient(nil) //创建请求客户端 - if err != nil { - log.Panic(err) - } - response, err := reqCli.Request(nil, "get", "ws://82.157.123.54:9010/ajaxchattest", requests.RequestOption{Headers: map[string]string{ - "Origin": "http://coolaf.com", - }}) //发送websocket请求 - if err != nil { - log.Panic(err) - } - defer response.Close() - wsCli := response.WebSocket() - if err = wsCli.Send(context.TODO(), websocket.MessageText, "测试"); err != nil { //发送txt 消息 - log.Panic(err) - } - msgType, con, err := wsCli.Recv(context.TODO()) //接收消息 - if err != nil { - log.Panic(err) - } - log.Print(msgType) //消息类型 - log.Print(string(con)) //消息内容 -} -``` -# ipv4,ipv6 地址控制解析 -```go -func main() { - reqCli, err := requests.NewClient(nil, requests.ClientOption{ - AddrType: requests.Ipv4, //优先解析ipv4地址 - // AddrType: requests.Ipv6,//优先解析ipv6地址 - }) - if err != nil { - log.Panic(err) - } - href := "https://test.ipw.cn" - resp, err := reqCli.Request(nil, "get", href) - if err != nil { - log.Panic(err) - } - log.Print(resp.Text()) - log.Print(resp.StatusCode()) -} -``` -# ja3 伪造指纹 -## 根据字符串生成指纹 -```go -func main() { - ja3Str := "772,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,0-23-65281-10-11-35-16-5-13-18-51-45-43-27-17513,29-23-24,0" - Ja3Spec, err := ja3.CreateSpecWithStr(ja3Str)//根据字符串生成指纹 - if err != nil { - log.Panic(err) - } - reqCli, err := requests.NewClient(nil, requests.ClientOption{Ja3Spec: Ja3Spec}) - if err != nil { - log.Panic(err) - } - response, err := reqCli.Request(nil, "get", "https://tools.scrapfly.io/api/fp/ja3?extended=1") - if err != nil { - log.Panic(err) - } - jsonData,_:=response.Json() - log.Print(jsonData.Get("ja3").String()) - log.Print(jsonData.Get("ja3").String() == ja3Str) -} -``` -## 根据id 生成指纹 -```go -func main() { - Ja3Spec, err := ja3.CreateSpecWithId(ja3.HelloChrome_Auto) //根据id 生成指纹 - if err != nil { - log.Panic(err) - } - reqCli, err := requests.NewClient(nil, requests.ClientOption{Ja3Spec: Ja3Spec}) - if err != nil { - log.Panic(err) - } - response, err := reqCli.Request(nil, "get", "https://tools.scrapfly.io/api/fp/ja3?extended=1") - if err != nil { - log.Panic(err) - } - jsonData,_:=response.Json() - log.Print(jsonData.Get("ja3").String()) -} -``` -## ja3 开关 -```go -func main() { - reqCli, err := requests.NewClient(nil) - if err != nil { - log.Panic(err) - } - response, err := reqCli.Request(nil, "get", "https://tools.scrapfly.io/api/fp/ja3?extended=1", requests.RequestOption{Ja3: true})//使用最新chrome 指纹 - if err != nil { - log.Panic(err) - } - jsonData,_:=response.Json() - log.Print(jsonData.Get("ja3").String()) -} -``` -## h2 指纹开关 -```go -func main() { - reqCli, err := requests.NewClient(nil, requests.ClientOption{ - H2Ja3: true, - }) - if err != nil { - log.Panic(err) - } - href := "https://tools.scrapfly.io/api/fp/anything" - resp, err := reqCli.Request(nil, "get", href) - if err != nil { - log.Panic(err) - } - log.Print(resp.Text()) -} -``` -## 修改h2指纹 -```go -func main() { - reqCli, err := requests.NewClient(nil, requests.ClientOption{ - H2Ja3Spec: ja3.H2Ja3Spec{ - InitialSetting: []ja3.Setting{ - {Id: 1, Val: 65555}, - {Id: 2, Val: 1}, - {Id: 3, Val: 2000}, - {Id: 4, Val: 6291457}, - {Id: 6, Val: 262145}, - }, - ConnFlow: 15663106, - OrderHeaders: []string{ - ":method", - ":path", - ":scheme", - ":authority", - }, - }, - }) - if err != nil { - log.Panic(err) - } - href := "https://tools.scrapfly.io/api/fp/anything" - resp, err := reqCli.Request(nil, "get", href) - if err != nil { - log.Panic(err) - } - log.Print(resp.Text()) -} -``` -# 采集全国公共资源网和中国政府采购网的列表页的标题 -```go -package main -import ( - "log" - "gitee.com/baixudong/requests" -) -func main() { - reqCli, err := requests.NewClient(nil) - if err != nil { - log.Panic(err) - } - resp, err := reqCli.Request(nil, "get", "http://www.ccgp.gov.cn/cggg/zygg/") - if err != nil { - log.Panic(err) - } - html := resp.Html() - lis := html.Finds("ul.c_list_bid li") - for _, li := range lis { - title := li.Find("a").Get("title") - log.Print(title) - } - resp, err = reqCli.Request(nil, "post", "http://deal.ggzy.gov.cn/ds/deal/dealList_find.jsp", requests.RequestOption{ - Data: map[string]string{ - "TIMEBEGIN_SHOW": "2023-04-26", - "TIMEEND_SHOW": "2023-05-05", - "TIMEBEGIN": "2023-04-26", - "TIMEEND": "2023-05-05", - "SOURCE_TYPE": "1", - "DEAL_TIME": "02", - "DEAL_CLASSIFY": "01", - "DEAL_STAGE": "0100", - "DEAL_PROVINCE": "0", - "DEAL_CITY": "0", - "DEAL_PLATFORM": "0", - "BID_PLATFORM": "0", - "DEAL_TRADE": "0", - "isShowAll": "1", - "PAGENUMBER": "2", - "FINDTXT": "", - }, - }) - if err != nil { - log.Panic(err) - } - jsonData,_ := resp.Json() - lls := jsonData.Get("data").Array() - for _, ll := range lls { - log.Print(ll.Get("title")) - } -} -``` \ No newline at end of file diff --git a/body.go b/body.go index 257d364..925b6d1 100644 --- a/body.go +++ b/body.go @@ -6,8 +6,8 @@ import ( "fmt" "net/url" - "gitee.com/baixudong/gson" - "gitee.com/baixudong/tools" + "github.com/gospider007/gson" + "github.com/gospider007/tools" ) // 构造一个文件 diff --git a/client.go b/client.go index 105b392..b73e0df 100644 --- a/client.go +++ b/client.go @@ -8,7 +8,8 @@ import ( "net/http" - "gitee.com/baixudong/ja3" + "github.com/gospider007/gtls" + "github.com/gospider007/ja3" ) type ClientOption struct { @@ -43,8 +44,7 @@ type ClientOption struct { Headers any //请求头 Bar bool //是否开启请求进度条 - RequestCallBack func(context.Context, *http.Request) error - ResponseCallBack func(context.Context, *http.Request, *http.Response) error + RequestCallBack func(context.Context, *http.Request, *http.Response) error } type Client struct { jar *Jar @@ -53,8 +53,7 @@ type Client struct { disUnZip bool //变比自动解压 tryNum int //重试次数 - requestCallBack func(context.Context, *http.Request) error - responseCallBack func(context.Context, *http.Request, *http.Response) error + requestCallBack func(context.Context, *http.Request, *http.Response) error optionCallBack func(context.Context, *Client, *RequestOption) error //请求参数回调,用于对请求参数进行修改。返回error,中断重试请求,返回nil继续 resultCallBack func(context.Context, *Client, *Response) error //结果回调,用于对结果进行校验。返回nil,直接返回,返回err的话,如果有errCallBack 走errCallBack,没有继续try @@ -151,14 +150,13 @@ func NewClient(preCtx context.Context, options ...ClientOption) (*Client, error) } } result := &Client{ - jar: jar, - ctx: ctx, - cnl: cnl, - client: client, - transport: transport, - noJarClient: noJarClient, - requestCallBack: option.RequestCallBack, - responseCallBack: option.ResponseCallBack, + jar: jar, + ctx: ctx, + cnl: cnl, + client: client, + transport: transport, + noJarClient: noJarClient, + requestCallBack: option.RequestCallBack, disCookie: option.DisCookie, redirectNum: option.RedirectNum, @@ -174,7 +172,7 @@ func NewClient(preCtx context.Context, options ...ClientOption) (*Client, error) } var err error if option.Proxy != "" { - result.proxy, err = VerifyProxy(option.Proxy) + result.proxy, err = gtls.VerifyProxy(option.Proxy) } if option.Ja3Spec.IsSet() { @@ -189,7 +187,7 @@ func (obj *Client) HttpClient() *http.Client { return obj.client } func (obj *Client) SetProxy(proxyUrl string) (err error) { - obj.proxy, err = VerifyProxy(proxyUrl) + obj.proxy, err = gtls.VerifyProxy(proxyUrl) return } func (obj *Client) SetGetProxy(getProxy func(ctx context.Context, url *url.URL) (string, error)) { diff --git a/cookies.go b/cookies.go index ae4161b..e676de3 100644 --- a/cookies.go +++ b/cookies.go @@ -6,7 +6,7 @@ import ( "net/http" - "gitee.com/baixudong/gson" + "github.com/gospider007/gson" ) //go:linkname readCookies net/http.readCookies diff --git a/dial.go b/dial.go index fc815ad..b688328 100644 --- a/dial.go +++ b/dial.go @@ -14,8 +14,9 @@ import ( "net/http" - "gitee.com/baixudong/ja3" - "gitee.com/baixudong/tools" + "github.com/gospider007/gtls" + "github.com/gospider007/ja3" + "github.com/gospider007/tools" utls "github.com/refraction-networking/utls" ) @@ -95,7 +96,7 @@ func (obj *DialClient) AddrToIp(ctx context.Context, addr string) (string, error if err != nil { return addr, tools.WrapError(err, "addrToIp 错误,SplitHostPort") } - _, ipInt := ParseHost(host) + _, ipInt := gtls.ParseHost(host) if ipInt == 4 || ipInt == 6 { return addr, nil } @@ -164,11 +165,11 @@ func (obj *DialClient) clientVerifySocks5(ctx context.Context, proxyUrl *url.URL } var host string var port int - if host, port, err = SplitHostPort(addr); err != nil { + if host, port, err = gtls.SplitHostPort(addr); err != nil { return } writeCon := []byte{5, 1, 0} - ip, ipInt := ParseHost(host) + ip, ipInt := gtls.ParseHost(host) switch ipInt { case 4: writeCon = append(writeCon, 1) @@ -245,7 +246,7 @@ func (obj *DialClient) lookupIPAddr(ctx context.Context, host string) (net.IP, e } for _, ipAddr := range ips { ip := ipAddr.IP - if ipType := ParseIp(ip); ipType == 4 || ipType == 6 { + if ipType := gtls.ParseIp(ip); ipType == 4 || ipType == 6 { if addrType == 0 || addrType == ipType { return ip, nil } @@ -253,7 +254,7 @@ func (obj *DialClient) lookupIPAddr(ctx context.Context, host string) (net.IP, e } for _, ipAddr := range ips { ip := ipAddr.IP - if ipType := ParseIp(ip); ipType == 4 || ipType == 6 { + if ipType := gtls.ParseIp(ip); ipType == 4 || ipType == 6 { return ip, nil } } @@ -277,7 +278,7 @@ func (obj *DialClient) Dialer() *net.Dialer { } func (obj *DialClient) AddTls(ctx context.Context, conn net.Conn, host string, disHttp2 bool, tlsConfig *tls.Config) (*tls.Conn, error) { var tlsConn *tls.Conn - tlsConfig.ServerName = GetServerName(host) + tlsConfig.ServerName = gtls.GetServerName(host) if disHttp2 { tlsConfig.NextProtos = []string{"http/1.1"} } else { @@ -287,7 +288,7 @@ func (obj *DialClient) AddTls(ctx context.Context, conn net.Conn, host string, d return tlsConn, tlsConn.HandshakeContext(ctx) } func (obj *DialClient) AddJa3Tls(ctx context.Context, conn net.Conn, host string, disHttp2 bool, ja3Spec ja3.Ja3Spec, tlsConfig *utls.Config) (*utls.UConn, error) { - tlsConfig.ServerName = GetServerName(host) + tlsConfig.ServerName = gtls.GetServerName(host) if disHttp2 { tlsConfig.NextProtos = []string{"http/1.1"} } else { diff --git a/go.mod b/go.mod index bb0d6cb..00a5fe7 100644 --- a/go.mod +++ b/go.mod @@ -1,23 +1,22 @@ -module gitee.com/baixudong/requests +module github.com/gospider007/requests go 1.21.1 require ( - gitee.com/baixudong/bar v0.0.0-20230923032414-c19cc384edeb - gitee.com/baixudong/bs4 v0.0.0-20230927091031-00dd545f5784 - gitee.com/baixudong/gson v0.0.0-20230927090937-556b58fb6899 - gitee.com/baixudong/ja3 v0.0.0-20231010095622-1f26d89ef419 - gitee.com/baixudong/net v0.0.0-20230927083058-4a6a7a20f917 - gitee.com/baixudong/re v0.0.0-20230809033040-360c1d945e59 - gitee.com/baixudong/tools v0.0.0-20230926010219-77f29cbf5935 - gitee.com/baixudong/websocket v0.0.0-20230927082325-5b4eb178c3ec + github.com/gospider007/bar v0.0.0-20231015040703-7d4e67e1741a + github.com/gospider007/bs4 v0.0.0-20231015041001-064efde5bd08 + github.com/gospider007/gson v0.0.0-20231015042048-4fae1418931b + github.com/gospider007/gtls v0.0.0-20231015025737-e18d927ad339 + github.com/gospider007/ja3 v0.0.0-20231015030157-fdfe60187c35 + github.com/gospider007/net v0.0.0-20231015031122-61ca42e0006f + github.com/gospider007/re v0.0.0-20231015023348-717c984874af + github.com/gospider007/tools v0.0.0-20231015025637-9b1d99c470f4 + github.com/gospider007/websocket v0.0.0-20231015042221-9176c1ebbdfc github.com/refraction-networking/utls v1.5.4 - golang.org/x/net v0.16.0 + golang.org/x/net v0.17.0 ) require ( - gitee.com/baixudong/blog v0.0.0-20230923032533-d83e7cb672f7 // indirect - gitee.com/baixudong/kinds v0.0.0-20230809033013-c3d6d3479f8c // indirect github.com/PuerkitoBio/goquery v1.8.1 // indirect github.com/andybalholm/brotli v1.0.5 // indirect github.com/andybalholm/cascadia v1.3.2 // indirect @@ -29,7 +28,8 @@ require ( github.com/gobwas/httphead v0.1.0 // indirect github.com/gobwas/pool v0.2.1 // indirect github.com/goccy/go-json v0.10.2 // indirect - github.com/google/uuid v1.3.1 // indirect + github.com/gospider007/blog v0.0.0-20231015040505-4b08187c11ee // indirect + github.com/gospider007/kinds v0.0.0-20231015025159-69ab7d8d9b94 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.17.0 // indirect github.com/leodido/go-urn v1.2.4 // indirect diff --git a/go.sum b/go.sum index 06fa15d..c9a2b0b 100644 --- a/go.sum +++ b/go.sum @@ -1,25 +1,3 @@ -gitee.com/baixudong/bar v0.0.0-20230923032414-c19cc384edeb h1:mqsOumc9CnMEYOefPzVxedRWaXP8XmpuN3loom1zLno= -gitee.com/baixudong/bar v0.0.0-20230923032414-c19cc384edeb/go.mod h1:vrjSAS99QLVH3l/S7ZJ/flF+4e+9ayHZ3h9SC1mDVv4= -gitee.com/baixudong/blog v0.0.0-20230923032533-d83e7cb672f7 h1:QDS4QkH6BHy95RnoGJVSlOdpwpSjZrdne0jqIuIEEd4= -gitee.com/baixudong/blog v0.0.0-20230923032533-d83e7cb672f7/go.mod h1:CRRC/3rq5rR/0+3MUuuxP3BX2gj8aR0SM0gumsd0pUk= -gitee.com/baixudong/bs4 v0.0.0-20230927091031-00dd545f5784 h1:mPmPbhSf61sSa8BlztLqu904ceay0qAwyzvoagNBRik= -gitee.com/baixudong/bs4 v0.0.0-20230927091031-00dd545f5784/go.mod h1:lVGpwtBVvx5LuWTtWjoNxNw7XQDl9lvkDdFKX4VumVI= -gitee.com/baixudong/gson v0.0.0-20230927090937-556b58fb6899 h1:JFV4d4J07fYDmp5hyZirQNSv4FmE2JQliZVSYoa4ieM= -gitee.com/baixudong/gson v0.0.0-20230927090937-556b58fb6899/go.mod h1:1DBHleeT45+7ogNCRkJYKhq28Krc3oC5Hr4pHmd+GM0= -gitee.com/baixudong/ja3 v0.0.0-20230927070510-152bc4a3675a h1:IONxzaxrHVCEINNw6oMcguyYqAu5DD9DjmYq2DoOuhY= -gitee.com/baixudong/ja3 v0.0.0-20230927070510-152bc4a3675a/go.mod h1:Y3tcqTP6YapuS8pWDT7zkHgoKV14Sx652+OuJJyssk4= -gitee.com/baixudong/ja3 v0.0.0-20231010095622-1f26d89ef419 h1:EUeBpoYVKI/rK0FQ8I8g50wx510HrQNju1orZ9RF25M= -gitee.com/baixudong/ja3 v0.0.0-20231010095622-1f26d89ef419/go.mod h1:SiOf/JBnmWJ/aTQKVUHFKSDLF+gnlV2rIZ73Ecy/RD8= -gitee.com/baixudong/kinds v0.0.0-20230809033013-c3d6d3479f8c h1:MZewpjU0+82TcA+nHrcAbbgZhTCqVH2I5zwYBWJ4v54= -gitee.com/baixudong/kinds v0.0.0-20230809033013-c3d6d3479f8c/go.mod h1:J3r4FWnZOMgk7/1EWEEDaPQnx5uH4OLCRzpTOyKg5cA= -gitee.com/baixudong/net v0.0.0-20230927083058-4a6a7a20f917 h1:/eAVC7BfyvZCMNMhYXtAw50RezEREx5baecMQZHGFAs= -gitee.com/baixudong/net v0.0.0-20230927083058-4a6a7a20f917/go.mod h1:q4sczw+EvXxYspUBQESB+Kr1lwkyOZ2Kc0bPItIgoTo= -gitee.com/baixudong/re v0.0.0-20230809033040-360c1d945e59 h1:HtdoUzBIqepeVTeeZ0QcPKoQSDqgWQn/fs84fsByJv0= -gitee.com/baixudong/re v0.0.0-20230809033040-360c1d945e59/go.mod h1:i5L3z/q42lHZvd3JSk3390XdujEY2QHXFMEcaO/uZD0= -gitee.com/baixudong/tools v0.0.0-20230926010219-77f29cbf5935 h1:+RccqXXfx7yPpwc7Q13z8YZlXFTtqh+A8tlsfd00jS4= -gitee.com/baixudong/tools v0.0.0-20230926010219-77f29cbf5935/go.mod h1:nGjhspfGKu9Z6knMz0i0riNzk8Q7kmv46qRl+6Wx+rY= -gitee.com/baixudong/websocket v0.0.0-20230927082325-5b4eb178c3ec h1:MAHSoPaI3qKYVXvd7eSSo4McWH2tgaZ5qDS9zG+vR0c= -gitee.com/baixudong/websocket v0.0.0-20230927082325-5b4eb178c3ec/go.mod h1:oKErVWktOk6crTjCHDDioz08/9oY8H3EvZQ7roNmIJ4= github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM= github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ= github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= @@ -29,8 +7,8 @@ github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsVi github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= github.com/bytedance/sonic v1.10.0-rc/go.mod h1:ElCzW+ufi8qKqNW0FY314xriJhyJhuoJ3gFZdAHF7NM= -github.com/bytedance/sonic v1.10.1 h1:7a1wuFXL1cMy7a3f7/VFcEtriuXQnUBhtoVfOZiaysc= -github.com/bytedance/sonic v1.10.1/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4= +github.com/bytedance/sonic v1.10.2 h1:GQebETVBxYB7JGWJtLBi07OVzWwt+8dWA00gEVW2ZFE= +github.com/bytedance/sonic v1.10.2/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4= github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d h1:77cEq6EriyTZ0g/qfRdp61a3Uu/AWrgIq2s0ClJV1g0= @@ -61,8 +39,8 @@ github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= -github.com/go-playground/validator/v10 v10.15.4 h1:zMXza4EpOdooxPel5xDqXEdXG5r+WggpvnAKMsalBjs= -github.com/go-playground/validator/v10 v10.15.4/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= +github.com/go-playground/validator/v10 v10.15.5 h1:LEBecTWb/1j5TNY1YYG2RcOUN3R7NLylN+x8TTueE24= +github.com/go-playground/validator/v10 v10.15.5/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= @@ -88,11 +66,31 @@ github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98 h1:pUa4ghanp6q4IJHwE9RwLgmVFfReJN+KbQ8ExNEUUoQ= github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= -github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= -github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gospider007/bar v0.0.0-20231015040703-7d4e67e1741a h1:CK3MzqDX3TaT+WCoVTALF7ssj8fufZxtrzICK2/Wpu8= +github.com/gospider007/bar v0.0.0-20231015040703-7d4e67e1741a/go.mod h1:Yv0zw5KpG9gFwXVUs2dIauORHbUgzX+sR9UTVdgdWzo= +github.com/gospider007/blog v0.0.0-20231015040505-4b08187c11ee h1:F+MircZJ9ysDFsq4Mea43+MbvbDcnHQFq/lOHLSO0IE= +github.com/gospider007/blog v0.0.0-20231015040505-4b08187c11ee/go.mod h1:y4zOnf7HvNA5TJst6YEQP3kJllg5PBjl6yMogqAMGq0= +github.com/gospider007/bs4 v0.0.0-20231015041001-064efde5bd08 h1:1xP5y0hLlvRQJ1TW58hEx+oPlh2tKm88wMKtFHaQeHo= +github.com/gospider007/bs4 v0.0.0-20231015041001-064efde5bd08/go.mod h1:R4MKplqr25t6ewFY6LqzH4Oggur81IegmRNJEzuhwfw= +github.com/gospider007/gson v0.0.0-20231015042048-4fae1418931b h1:Gd5tOuEAlrs+5IWE/Q7WmJFmNmWA+tarN1MZe1G4oPw= +github.com/gospider007/gson v0.0.0-20231015042048-4fae1418931b/go.mod h1:frJXgZBXIUJlfsIB7LrEGEIU2SeQu7xP/1xs/Tphlwg= +github.com/gospider007/gtls v0.0.0-20231015025737-e18d927ad339 h1:YmZe17AF0QZQp0K0QlArKa0UPoULgQepI5BXeHGs8/8= +github.com/gospider007/gtls v0.0.0-20231015025737-e18d927ad339/go.mod h1:HNrQHMSTsG73mweP5p010QAgXVaZ6RgcfxfJOdyieUY= +github.com/gospider007/ja3 v0.0.0-20231015030157-fdfe60187c35 h1:Eyfpv7ZG9+l//KIKeEs0CJRTGPIeP5sfZ0kJCspfakU= +github.com/gospider007/ja3 v0.0.0-20231015030157-fdfe60187c35/go.mod h1:BNtOUDtO8NF8aR7rtl3CAvE/LG9+RWtUiLssswjGLTI= +github.com/gospider007/kinds v0.0.0-20231015025159-69ab7d8d9b94 h1:K2XYy6B6M62o35KMabuzl9DmA4WQftanUgKomb2YYU4= +github.com/gospider007/kinds v0.0.0-20231015025159-69ab7d8d9b94/go.mod h1:5e4rIpZmhwqOLVVRZa0uZFAuMPH15dO0qqzKf1s3IgI= +github.com/gospider007/net v0.0.0-20231015031122-61ca42e0006f h1:Cj8eyFdQnOxTIt8VbyYwAwUZRg/sTX+9hL8jvg4ksqs= +github.com/gospider007/net v0.0.0-20231015031122-61ca42e0006f/go.mod h1:hadIeF78Sy2/gCvOAr/ABCKe7cmudBh9w2a5YhCaRXQ= +github.com/gospider007/re v0.0.0-20231015023348-717c984874af h1:bYip5he4jUPslsEY3GtBndUUJppR84Ml9areZ/wG0Hk= +github.com/gospider007/re v0.0.0-20231015023348-717c984874af/go.mod h1:fOcqR9yJHQjUmpLerdxJSoMJ4pFyMfRirM12rTMEH38= +github.com/gospider007/tools v0.0.0-20231015025637-9b1d99c470f4 h1:XeDsy/Qy/uH+MhRPeBK44Owl5V/zotyklIgP9rS2ONc= +github.com/gospider007/tools v0.0.0-20231015025637-9b1d99c470f4/go.mod h1:UAIsaldUoH2zZqg4i0es5Q6x/GS0BkgCeeQ/sLnnZJk= +github.com/gospider007/websocket v0.0.0-20231015042221-9176c1ebbdfc h1:eLhD3+H6ads14/yDR8L6UQ/7ZCAjO5QC5fHLGTpqCiw= +github.com/gospider007/websocket v0.0.0-20231015042221-9176c1ebbdfc/go.mod h1:xfgcqdRvRpjIyPAhO5/o+ubuTmJAey/wGnY8DyiNHRc= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= @@ -127,8 +125,6 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/quic-go/quic-go v0.39.0 h1:AgP40iThFMY0bj8jGxROhw3S0FMGa8ryqsmi9tBH3So= github.com/quic-go/quic-go v0.39.0/go.mod h1:T09QsDQWjLiQ74ZmacDfqZmhY/NLnw5BC40MANNNZ1Q= -github.com/refraction-networking/utls v1.5.3 h1:Ds5Ocg1+MC1ahNx5iBEcHe0jHeLaA/fLey61EENm7ro= -github.com/refraction-networking/utls v1.5.3/go.mod h1:SPuDbBmgLGp8s+HLNc83FuavwZCFoMmExj+ltUHiHUw= github.com/refraction-networking/utls v1.5.4 h1:9k6EO2b8TaOGsQ7Pl7p9w6PUhx18/ZCeT0WNTZ7Uw4o= github.com/refraction-networking/utls v1.5.4/go.mod h1:SPuDbBmgLGp8s+HLNc83FuavwZCFoMmExj+ltUHiHUw= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -175,16 +171,10 @@ golang.org/x/arch v0.5.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= -golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= -golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= -golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= -golang.org/x/image v0.12.0 h1:w13vZbU4o5rKOFFR8y7M+c4A5jXDC0uXTdHYRP8X2DQ= -golang.org/x/image v0.12.0/go.mod h1:Lu90jvHG7GfemOIcldsh9A2hS01ocl6oNO7ype5mEnk= golang.org/x/image v0.13.0 h1:3cge/F/QTkNLauhf2QoE9zp+7sr+ZcL4HnoZmdwg9sg= golang.org/x/image v0.13.0/go.mod h1:6mmbMOeV28HuMTgA6OSRkdXKYw/t5W9Uwn2Yv1r3Yxk= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= @@ -197,10 +187,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= -golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= -golang.org/x/net v0.16.0 h1:7eBu7KsSvFDtSXUIDbh3aqlK4DPsZ1rByC8PFfBThos= -golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -214,8 +202,6 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= -golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -237,8 +223,8 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ= -golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= +golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= diff --git a/headers.go b/headers.go index f0c3643..5b47042 100644 --- a/headers.go +++ b/headers.go @@ -5,7 +5,7 @@ import ( "net/http" - "gitee.com/baixudong/gson" + "github.com/gospider007/gson" ) var UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36 Edg/117.0.2045.31" diff --git a/option.go b/option.go index 6fb5362..043cfac 100644 --- a/option.go +++ b/option.go @@ -12,8 +12,8 @@ import ( "strings" "time" - "gitee.com/baixudong/ja3" - "gitee.com/baixudong/websocket" + "github.com/gospider007/ja3" + "github.com/gospider007/websocket" ) // 请求参数选项 @@ -48,12 +48,10 @@ type RequestOption struct { DisProxy bool //是否关闭代理,强制关闭代理 TryNum int //重试次数 - OptionCallBack func(context.Context, *Client, *RequestOption) error //请求参数回调,用于对请求参数进行修改。返回error,中断重试请求,返回nil继续 - ResultCallBack func(context.Context, *Client, *Response) error //结果回调,用于对结果进行校验。返回nil,直接返回,返回err的话,如果有errCallBack 走errCallBack,没有继续try - ErrCallBack func(context.Context, *Client, error) error //错误回调,返回error,中断重试请求,返回nil继续 - - RequestCallBack func(context.Context, *http.Request) error - ResponseCallBack func(context.Context, *http.Request, *http.Response) error + OptionCallBack func(context.Context, *Client, *RequestOption) error //请求参数回调,用于对请求参数进行修改。返回error,中断重试请求,返回nil继续 + ResultCallBack func(context.Context, *Client, *Response) error //结果回调,用于对结果进行校验。返回nil,直接返回,返回err的话,如果有errCallBack 走errCallBack,没有继续try + ErrCallBack func(context.Context, *Client, error) error //错误回调,返回error,中断重试请求,返回nil继续 + RequestCallBack func(context.Context, *http.Request, *http.Response) error Jar *Jar //自定义临时cookies 管理 @@ -239,8 +237,5 @@ func (obj *Client) newRequestOption(option RequestOption) RequestOption { if option.RequestCallBack == nil { option.RequestCallBack = obj.requestCallBack } - if option.ResponseCallBack == nil { - option.ResponseCallBack = obj.responseCallBack - } return option } diff --git a/requests.go b/requests.go index 770712c..0e65e1f 100644 --- a/requests.go +++ b/requests.go @@ -14,10 +14,11 @@ import ( "net/http" - "gitee.com/baixudong/ja3" - "gitee.com/baixudong/re" - "gitee.com/baixudong/tools" - "gitee.com/baixudong/websocket" + "github.com/gospider007/gtls" + "github.com/gospider007/ja3" + "github.com/gospider007/re" + "github.com/gospider007/tools" + "github.com/gospider007/websocket" ) //go:linkname ReadRequest net/http.readRequest @@ -139,8 +140,7 @@ type reqCtxData struct { proxy *url.URL disProxy bool - requestCallBack func(context.Context, *http.Request) error - responseCallBack func(context.Context, *http.Request, *http.Response) error + requestCallBack func(context.Context, *http.Request, *http.Response) error h2Ja3Spec ja3.H2Ja3Spec ja3Spec ja3.Ja3Spec @@ -148,93 +148,83 @@ type reqCtxData struct { func Get(preCtx context.Context, href string, options ...RequestOption) (resp *Response, err error) { client, _ := NewClient(preCtx) - defer func() { - if err != nil || (resp.webSocket == nil && resp.sseClient == nil) { - client.Close() - } - }() - return client.Request(preCtx, http.MethodGet, href, options...) + resp, err = client.Request(preCtx, http.MethodGet, href, options...) + if err != nil || resp == nil || !resp.oneceAlive() { + client.Close() + } + return } func Head(preCtx context.Context, href string, options ...RequestOption) (resp *Response, err error) { client, _ := NewClient(preCtx) - defer func() { - if err != nil || (resp.webSocket == nil && resp.sseClient == nil) { - client.Close() - } - }() - return client.Request(preCtx, http.MethodHead, href, options...) + resp, err = client.Request(preCtx, http.MethodHead, href, options...) + if err != nil || resp == nil || !resp.oneceAlive() { + client.Close() + } + return } func Post(preCtx context.Context, href string, options ...RequestOption) (resp *Response, err error) { client, _ := NewClient(preCtx) - defer func() { - if err != nil || (resp.webSocket == nil && resp.sseClient == nil) { - client.Close() - } - }() - return client.Request(preCtx, http.MethodPost, href, options...) + resp, err = client.Request(preCtx, http.MethodPost, href, options...) + if err != nil || resp == nil || !resp.oneceAlive() { + client.Close() + } + return } func Put(preCtx context.Context, href string, options ...RequestOption) (resp *Response, err error) { client, _ := NewClient(preCtx) - defer func() { - if err != nil || (resp.webSocket == nil && resp.sseClient == nil) { - client.Close() - } - }() - return client.Request(preCtx, http.MethodPut, href, options...) + resp, err = client.Request(preCtx, http.MethodPut, href, options...) + if err != nil || resp == nil || !resp.oneceAlive() { + client.Close() + } + return } func Patch(preCtx context.Context, href string, options ...RequestOption) (resp *Response, err error) { client, _ := NewClient(preCtx) - defer func() { - if err != nil || (resp.webSocket == nil && resp.sseClient == nil) { - client.Close() - } - }() - return client.Request(preCtx, http.MethodPatch, href, options...) + resp, err = client.Request(preCtx, http.MethodPatch, href, options...) + if err != nil || resp == nil || !resp.oneceAlive() { + client.Close() + } + return } func Delete(preCtx context.Context, href string, options ...RequestOption) (resp *Response, err error) { client, _ := NewClient(preCtx) - defer func() { - if err != nil || (resp.webSocket == nil && resp.sseClient == nil) { - client.Close() - } - }() - return client.Request(preCtx, http.MethodDelete, href, options...) + resp, err = client.Request(preCtx, http.MethodDelete, href, options...) + if err != nil || resp == nil || !resp.oneceAlive() { + client.Close() + } + return } func Connect(preCtx context.Context, href string, options ...RequestOption) (resp *Response, err error) { client, _ := NewClient(preCtx) - defer func() { - if err != nil || (resp.webSocket == nil && resp.sseClient == nil) { - client.Close() - } - }() - return client.Request(preCtx, http.MethodConnect, href, options...) + resp, err = client.Request(preCtx, http.MethodConnect, href, options...) + if err != nil || resp == nil || !resp.oneceAlive() { + client.Close() + } + return } func Options(preCtx context.Context, href string, options ...RequestOption) (resp *Response, err error) { client, _ := NewClient(preCtx) - defer func() { - if err != nil || (resp.webSocket == nil && resp.sseClient == nil) { - client.Close() - } - }() - return client.Request(preCtx, http.MethodOptions, href, options...) + resp, err = client.Request(preCtx, http.MethodOptions, href, options...) + if err != nil || resp == nil || !resp.oneceAlive() { + client.Close() + } + return } func Trace(preCtx context.Context, href string, options ...RequestOption) (resp *Response, err error) { client, _ := NewClient(preCtx) - defer func() { - if err != nil || (resp.webSocket == nil && resp.sseClient == nil) { - client.Close() - } - }() - return client.Request(preCtx, http.MethodTrace, href, options...) + resp, err = client.Request(preCtx, http.MethodTrace, href, options...) + if err != nil || resp == nil || !resp.oneceAlive() { + client.Close() + } + return } func Request(preCtx context.Context, method string, href string, options ...RequestOption) (resp *Response, err error) { client, _ := NewClient(preCtx) - defer func() { - if err != nil || (resp.webSocket == nil && resp.sseClient == nil) { - client.Close() - } - }() - return client.Request(preCtx, method, href, options...) + resp, err = client.Request(preCtx, method, href, options...) + if err != nil || resp == nil || !resp.oneceAlive() { + client.Close() + } + return } func (obj *Client) Get(preCtx context.Context, href string, options ...RequestOption) (*Response, error) { return obj.Request(preCtx, http.MethodGet, href, options...) @@ -313,7 +303,7 @@ func (obj *Client) Request(preCtx context.Context, method string, href string, o func (obj *Client) request(preCtx context.Context, option RequestOption) (response *Response, err error) { response = new(Response) defer func() { - if err == nil && response.webSocket == nil && response.sseClient == nil && !option.DisRead { //判断是否读取body,和对body的处理 + if err == nil && !response.oneceAlive() && !option.DisRead { //判断是否读取body,和对body的处理 err = response.ReadBody() defer response.Close() } @@ -349,12 +339,11 @@ func (obj *Client) request(preCtx context.Context, option RequestOption) (respon ctxData := new(reqCtxData) ctxData.requestCallBack = option.RequestCallBack - ctxData.responseCallBack = option.ResponseCallBack //构造代理 ctxData.disProxy = option.DisProxy if !ctxData.disProxy { if option.Proxy != "" { //代理相关构造 - tempProxy, err := VerifyProxy(option.Proxy) + tempProxy, err := gtls.VerifyProxy(option.Proxy) if err != nil { return response, tools.WrapError(errFatal, errors.New("tempRequest 构造代理失败"), err) } @@ -386,24 +375,6 @@ func (obj *Client) request(preCtx context.Context, option RequestOption) (respon if err != nil { return response, tools.WrapError(errFatal, errors.New("tempRequest 构造request失败"), err) } - - //解析Scheme - var isWebSocket bool - switch reqs.URL.Scheme { - case "ws", "wss": - isWebSocket = true - case "file": - response.filePath = re.Sub(`^/+`, "", reqs.URL.Path) - response.content, err = os.ReadFile(response.filePath) - if err != nil { - err = tools.WrapError(errFatal, errors.New("read filePath data error"), err) - } - return - case "http", "https": - default: - err = tools.WrapError(errFatal, fmt.Errorf("url scheme error: %s", reqs.URL.Scheme)) - return - } //添加headers var headOk bool if reqs.Header, headOk = option.Headers.(http.Header); !headOk { @@ -418,6 +389,24 @@ func (obj *Client) request(preCtx context.Context, option RequestOption) (respon if reqs.Header.Get("Content-Type") == "" && reqs.Header.Get("content-type") == "" && option.ContentType != "" { reqs.Header.Set("Content-Type", option.ContentType) } + + //解析Scheme + switch reqs.URL.Scheme { + case "ws", "wss": + websocket.SetClientHeadersOption(reqs.Header, option.WsOption) + case "file": + response.filePath = re.Sub(`^/+`, "", reqs.URL.Path) + response.content, err = os.ReadFile(response.filePath) + if err != nil { + err = tools.WrapError(errFatal, errors.New("read filePath data error"), err) + } + return + case "http", "https": + default: + err = tools.WrapError(errFatal, fmt.Errorf("url scheme error: %s", reqs.URL.Scheme)) + return + } + //host构造 if option.Host != "" { reqs.Host = option.Host @@ -434,10 +423,7 @@ func (obj *Client) request(preCtx context.Context, option RequestOption) (respon reqs.AddCookie(vv) } } - //开始发送请求 - if isWebSocket { //设置websocket headers - websocket.SetClientHeadersOption(reqs.Header, option.WsOption) - } + if response.response, err = obj.getClient(option).Do(reqs); err != nil { return } else if response.response == nil { @@ -447,13 +433,8 @@ func (obj *Client) request(preCtx context.Context, option RequestOption) (respon if !response.disUnzip { response.disUnzip = response.response.Uncompressed } - - if isWebSocket { //判断ws 的状态码是否正确 - if response.response.StatusCode == 101 { - response.webSocket, err = websocket.NewClientConn(response.response) - } else { - err = errors.New("statusCode not 101, url为websocket链接,但是对方服务器没有将请求升级到websocket") - } + if response.response.StatusCode == 101 { + response.webSocket, err = websocket.NewClientConn(response.response) } else if response.response.Header.Get("Content-Type") == "text/event-stream" { //如果为sse协议就关闭读取 response.sseClient = newSseClient(response) } diff --git a/response.go b/response.go index 14e9e06..6d972af 100644 --- a/response.go +++ b/response.go @@ -14,11 +14,12 @@ import ( "net/http" - "gitee.com/baixudong/bar" - "gitee.com/baixudong/bs4" - "gitee.com/baixudong/gson" - "gitee.com/baixudong/tools" - "gitee.com/baixudong/websocket" + "github.com/gospider007/bar" + "github.com/gospider007/bs4" + "github.com/gospider007/gson" + "github.com/gospider007/gtls" + "github.com/gospider007/tools" + "github.com/gospider007/websocket" ) type Response struct { @@ -230,11 +231,11 @@ func (obj *Response) Html() *bs4.Client { // 获取headers 的Content-Type func (obj *Response) ContentType() string { if obj.filePath != "" { - return GetContentTypeWithBytes(obj.content) + return gtls.GetContentTypeWithBytes(obj.content) } contentType := obj.response.Header.Get("Content-Type") if contentType == "" { - contentType = GetContentTypeWithBytes(obj.content) + contentType = gtls.GetContentTypeWithBytes(obj.content) } return contentType } @@ -303,7 +304,14 @@ func (obj *Response) Read(con []byte) (i int, err error) { //读取body } } +func (obj *Response) oneceAlive() bool { //读取body,对body 解压,解码操作 + return obj.webSocket != nil || obj.sseClient != nil +} + func (obj *Response) ReadBody() error { //读取body,对body 解压,解码操作 + if obj.oneceAlive() { + return errors.New("ws 或 sse 不能读取") + } var bBody *bytes.Buffer var err error if obj.bar && obj.ContentLength() > 0 { //是否打印进度条,读取内容 diff --git a/roundTripper.go b/roundTripper.go index 026d18f..5bfee04 100644 --- a/roundTripper.go +++ b/roundTripper.go @@ -15,8 +15,9 @@ import ( "net/http" - "gitee.com/baixudong/net/http2" - "gitee.com/baixudong/tools" + "github.com/gospider007/gtls" + "github.com/gospider007/net/http2" + "github.com/gospider007/tools" utls "github.com/refraction-networking/utls" ) @@ -491,7 +492,7 @@ func (obj *RoundTripper) GetProxy(ctx context.Context, proxyUrl *url.URL) (*url. if err != nil { return nil, err } - return VerifyProxy(proxy) + return gtls.VerifyProxy(proxy) } func (obj *RoundTripper) poolRoundTrip(task *reqTask, key string) (bool, error) { @@ -527,7 +528,7 @@ func (obj *RoundTripper) CloseIdleConnections() { func (obj *RoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { ctxData := req.Context().Value(keyPrincipalID).(*reqCtxData) if ctxData.requestCallBack != nil { - if err := ctxData.requestCallBack(req.Context(), req); err != nil { + if err := ctxData.requestCallBack(req.Context(), req, nil); err != nil { return nil, err } } @@ -539,8 +540,8 @@ func (obj *RoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { if ok, err := obj.poolRoundTrip(task, key); err != nil { return nil, err } else if ok { //is conn multi - if ctxData.responseCallBack != nil { - if err = ctxData.responseCallBack(task.req.Context(), req, task.res); err != nil { + if ctxData.requestCallBack != nil { + if err = ctxData.requestCallBack(task.req.Context(), req, task.res); err != nil { task.err = err } } @@ -561,8 +562,8 @@ newConn: if task.isPool() { obj.putConnPool(key, conn) } - if ctxData.responseCallBack != nil { - if err = ctxData.responseCallBack(task.req.Context(), req, task.res); err != nil { + if ctxData.requestCallBack != nil { + if err = ctxData.requestCallBack(task.req.Context(), req, task.res); err != nil { task.err = err conn.Close() } diff --git a/ssl/gospider.crt b/ssl/gospider.crt deleted file mode 100644 index 0a73f50..0000000 --- a/ssl/gospider.crt +++ /dev/null @@ -1,14 +0,0 @@ ------BEGIN CERTIFICATE----- -MIICKzCCAdGgAwIBAgIEZCFPpzAKBggqhkjOPQQDAjB5MQswCQYDVQQGEwJDTjER -MA8GA1UECBMIU2hhbmdoYWkxETAPBgNVBAcTCFNoYW5naGFpMREwDwYDVQQKEwhH -b1NwaWRlcjEWMBQGA1UECxMNR29TcGlkZXJQcm94eTEZMBcGA1UEAxMQR29zcGlk -ZXIgUm9vdCBDQTAgFw0yMzAzMTkxNjAwMDBaGA8zMDIzMDMxOTE2MDAwMFoweTEL -MAkGA1UEBhMCQ04xETAPBgNVBAgTCFNoYW5naGFpMREwDwYDVQQHEwhTaGFuZ2hh -aTERMA8GA1UEChMIR29TcGlkZXIxFjAUBgNVBAsTDUdvU3BpZGVyUHJveHkxGTAX -BgNVBAMTEEdvc3BpZGVyIFJvb3QgQ0EwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNC -AASvquXLARToWOkMCtyujW+yu9Amq69hYuO0wsEMfDy35LRGcKDKXBYQAlzI4oK/ -QG/jHTe/aq1sS4ee5tHtJsAdo0UwQzAOBgNVHQ8BAf8EBAMCAQYwEgYDVR0TAQH/ -BAgwBgEB/wIBATAdBgNVHQ4EFgQUFgJ50m8ArIS37WgP6lL7NqmsPMcwCgYIKoZI -zj0EAwIDSAAwRQIgWzRUfg2baj/hFyCJJIbnrwk9H4XfzVEJqXJHMb0dbD4CIQC6 -JA5hlVq6PW49+/R90EZLn51NhSuRHgW9bmosKUMkZw== ------END CERTIFICATE----- diff --git a/ssl/gospider.key b/ssl/gospider.key deleted file mode 100644 index 5c630e5..0000000 --- a/ssl/gospider.key +++ /dev/null @@ -1,5 +0,0 @@ ------BEGIN EC PRIVATE KEY----- -MHcCAQEEIEIygwBaoy/fcxW08s9EJMAR2sXaFVNZ2uLrJO53zegvoAoGCCqGSM49 -AwEHoUQDQgAEr6rlywEU6FjpDArcro1vsrvQJquvYWLjtMLBDHw8t+S0RnCgylwW -EAJcyOKCv0Bv4x03v2qtbEuHnubR7SbAHQ== ------END EC PRIVATE KEY----- diff --git a/tools.go b/tools.go deleted file mode 100644 index 8797e8a..0000000 --- a/tools.go +++ /dev/null @@ -1,268 +0,0 @@ -package requests - -import ( - "crypto/ecdsa" - "crypto/elliptic" - "crypto/rand" - "crypto/tls" - "crypto/x509" - "crypto/x509/pkix" - _ "embed" - "encoding/pem" - "errors" - "math/big" - "net" - "net/http" - "net/url" - "strconv" - "time" -) - -//go:embed ssl/gospider.crt -var CrtFile []byte - -//go:embed ssl/gospider.key -var KeyFile []byte - -func ParseIp(ip net.IP) int { - if ip != nil { - if ip4 := ip.To4(); ip4 != nil { - return 4 - } else if ip6 := ip.To16(); ip6 != nil { - return 6 - } - } - return 0 -} -func SplitHostPort(address string) (string, int, error) { - host, port, err := net.SplitHostPort(address) - if err != nil { - return "", 0, err - } - portnum, err := strconv.Atoi(port) - if err != nil { - return "", 0, err - } - if 1 > portnum || portnum > 0xffff { - return "", 0, errors.New("port number out of range " + port) - } - return host, portnum, nil -} -func ParseHost(host string) (net.IP, int) { - if ip := net.ParseIP(host); ip != nil { - if ip4 := ip.To4(); ip4 != nil { - return ip4, 4 - } else if ip6 := ip.To16(); ip6 != nil { - return ip6, 6 - } - } - return nil, 0 -} - -func GetHost(addrTypes ...int) net.IP { - hosts := GetHosts(addrTypes...) - if len(hosts) == 0 { - return nil - } else { - return hosts[0] - } -} -func GetHosts(addrTypes ...int) []net.IP { - var addrType int - if len(addrTypes) > 0 { - addrType = addrTypes[0] - } - result := []net.IP{} - lls, err := net.InterfaceAddrs() - if err != nil { - return result - } - for _, ll := range lls { - mm, ok := ll.(*net.IPNet) - if ok && mm.IP.IsPrivate() { - if addrType == 0 || ParseIp(mm.IP) == addrType { - result = append(result, mm.IP) - } - } - } - return result -} -func VerifyProxy(proxyUrl string) (*url.URL, error) { - proxy, err := url.Parse(proxyUrl) - if err != nil { - return nil, err - } - switch proxy.Scheme { - case "http", "socks5", "https": - return proxy, nil - default: - return nil, err - } -} - -func GetContentTypeWithBytes(content []byte) string { - return http.DetectContentType(content) -} -func GetServerName(addr string) string { - host, _, err := net.SplitHostPort(addr) - if err != nil { - return addr - } - return host -} - -// 生成根证书 -func CreateRootCert(key *ecdsa.PrivateKey) (*x509.Certificate, error) { - beforDate, err := time.ParseInLocation(time.DateOnly, "2023-03-20", time.Local) - if err != nil { - return nil, err - } - afterDate, err := time.ParseInLocation(time.DateOnly, "3023-03-20", time.Local) - if err != nil { - return nil, err - } - rootCsr := &x509.Certificate{ - Version: 3, - SerialNumber: big.NewInt(time.Now().Unix()), - Subject: pkix.Name{ - Country: []string{"CN"}, - Province: []string{"Shanghai"}, - Locality: []string{"Shanghai"}, - Organization: []string{"GoSpider"}, - OrganizationalUnit: []string{"GoSpiderProxy"}, - CommonName: "Gospider Root CA", - }, - NotBefore: beforDate, - NotAfter: afterDate, - BasicConstraintsValid: true, - IsCA: true, - MaxPathLen: 1, - MaxPathLenZero: false, - KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign, - } - rootDer, err := x509.CreateCertificate(rand.Reader, rootCsr, rootCsr, key.Public(), key) - if err != nil { - return nil, err - } - return x509.ParseCertificate(rootDer) -} - -// 生成私钥 -func CreateCertKey() (*ecdsa.PrivateKey, error) { - return ecdsa.GenerateKey(elliptic.P256(), rand.Reader) -} -func GetCertWithCN(rootCert *x509.Certificate, key *ecdsa.PrivateKey, commonName string) (*x509.Certificate, error) { - csr := &x509.Certificate{ - Version: 3, - SerialNumber: big.NewInt(time.Now().Unix()), - Subject: rootCert.Subject, - IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1)}, - NotBefore: time.Now(), - NotAfter: time.Now().AddDate(1000, 0, 0), - BasicConstraintsValid: true, - IsCA: false, - KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, - } - csr.IPAddresses = []net.IP{} - if commonName != "" { - if ip, ipType := ParseHost(commonName); ipType == 0 { - csr.Subject.CommonName = commonName - csr.DNSNames = []string{commonName} - } else { - csr.IPAddresses = append(csr.IPAddresses, ip) - } - } - der, err := x509.CreateCertificate(rand.Reader, csr, rootCert, key.Public(), key) - if err != nil { - return nil, err - } - return x509.ParseCertificate(der) -} - -func CreateCertWithCert(rootCert *x509.Certificate, key *ecdsa.PrivateKey, preCert *x509.Certificate) (*x509.Certificate, error) { - if preCert.DNSNames == nil && preCert.Subject.CommonName != "" { - preCert.DNSNames = []string{preCert.Subject.CommonName} - } - rootCert.Subject.CommonName = preCert.Subject.CommonName - csr := &x509.Certificate{ - Version: 3, - SerialNumber: big.NewInt(time.Now().Unix()), - Subject: rootCert.Subject, - DNSNames: preCert.DNSNames, - IPAddresses: preCert.IPAddresses, - NotBefore: time.Now(), - NotAfter: time.Now().AddDate(1000, 0, 0), - BasicConstraintsValid: true, - IsCA: false, - KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, - } - if len(preCert.DNSNames) > 0 { - csr.Subject.CommonName = preCert.DNSNames[0] - } - der, err := x509.CreateCertificate(rand.Reader, csr, rootCert, key.Public(), key) - if err != nil { - return nil, err - } - return x509.ParseCertificate(der) -} -func CreateProxyCertWithName(serverName string) (tlsCert tls.Certificate, err error) { - crt, err := LoadCert(CrtFile) - if err != nil { - return tlsCert, err - } - key, err := LoadCertKey(KeyFile) - if err != nil { - return tlsCert, err - } - cert, err := GetCertWithCN(crt, key, serverName) - if err != nil { - return tlsCert, err - } - return GetTlsCert(cert, key) -} -func CreateProxyCertWithCert(crt *x509.Certificate, key *ecdsa.PrivateKey, preCert *x509.Certificate) (tlsCert tls.Certificate, err error) { - if crt == nil { - crt, err = LoadCert(CrtFile) - if err != nil { - return tlsCert, err - } - } - if key == nil { - key, err = LoadCertKey(KeyFile) - if err != nil { - return tlsCert, err - } - } - cert, err := CreateCertWithCert(crt, key, preCert) - if err != nil { - return tlsCert, err - } - return GetTlsCert(cert, key) -} -func GetTlsCert(cert *x509.Certificate, key *ecdsa.PrivateKey) (tls.Certificate, error) { - keyFile, err := GetCertKeyData(key) - if err != nil { - return tls.Certificate{}, err - } - return tls.X509KeyPair(GetCertData(cert), keyFile) -} -func GetCertData(cert *x509.Certificate) []byte { - return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: cert.Raw}) -} -func GetCertKeyData(key *ecdsa.PrivateKey) ([]byte, error) { - keyDer, err := x509.MarshalECPrivateKey(key) - if err != nil { - return nil, err - } - return pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDer}), nil -} -func LoadCertKey(data []byte) (*ecdsa.PrivateKey, error) { - block, _ := pem.Decode(data) - return x509.ParseECPrivateKey(block.Bytes) -} -func LoadCert(data []byte) (*x509.Certificate, error) { - block, _ := pem.Decode(data) - return x509.ParseCertificate(block.Bytes) -}