package/requests
1
0
Fork 0
mirror of https://github.com/gospider007/requests.git synced 2025-12-24 13:57:52 +08:00
Code Issues Packages Projects Releases Wiki Activity

new

Browse Source
This commit is contained in:
bxd
2023-07-26 15:05:11 +08:00
commit 70d9c0cbc3
10 changed files with 2598 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

368
README.md Normal file
View File

@@ -0,0 +1,368 @@
# 功能概述
- cookies 开关，连接池，http2，ja3
- 自实现socks五,http代理,https代理
- 自动解压缩,解码
- dns缓存
- 类型自动转化
- 尝试重试，请求回调
- websocket 协议
- sse协议
# 设置代理
## 代理设置的优先级
```
全局代理方法 < 全局代理字符串 < 局部代理字符串
```
## 设置并修改全局代理方法 (只会在新建连接的时候调用获取代理的方法,复用连接的时候不会调用)
```golang
package main
import (
"log"
"gitee.com/baixudong/requests"
)
func main() {
//创建请求客户端
reqCli, err := requests.NewClient(nil, requests.ClientOption{
GetProxy: func(ctx context.Context, url *url.URL) (string, error) { //设置全局代理方法
return "http://127.0.0.1:7005", nil
}})
if err != nil {
log.Panic(err)
}
response, err := reqCli.Request(nil, "get", "http://myip.top") //发送get请求
if err != nil {
log.Panic(err)
}
reqCli.SetGetProxy(func(ctx context.Context, url *url.URL) (string, error) { //修改全局代理方法
return "http://127.0.0.1:7006", nil
})
log.Print(response.Text()) //获取内容,解析为字符串
}
```
## 设置并修改全局代理
```golang
package main
import (
"log"
"gitee.com/baixudong/requests"
)
func main() {
//创建请求客户端
reqCli, err := requests.NewClient(nil, requests.ClientOption{
Proxy: "http://127.0.0.1:7005", //设置全局代理
})
if err != nil {
log.Panic(err)
}
//发送get请求
response, err := reqCli.Request(nil, "get", "http://myip.top")
if err != nil {
log.Panic(err)
}
err = reqCli.SetProxy("http://127.0.0.1:7006") //修改全局代理
if err != nil {
log.Panic(err)
}
log.Print(response.Text()) //获取内容,解析为字符串
}
```
## 设置局部代理
```golang
package main
import (
"log"
"gitee.com/baixudong/requests"
)
func main() {
//创建请求客户端
reqCli, err := requests.NewClient(nil)
if err != nil {
log.Panic(err)
}
//发送get请求
response, err := reqCli.Request(nil, "get", "http://myip.top", requests.RequestOption{
Proxy: "http://127.0.0.1:7005",
})
if err != nil {
log.Panic(err)
}
log.Print(response.Text()) //获取内容,解析为字符串
}
```
## 强制关闭代理，走本地网络
```golang
package main
import (
"log"
"gitee.com/baixudong/requests"
)
func main() {
//创建请求客户端
reqCli, err := requests.NewClient(nil)
if err != nil {
log.Panic(err)
}
//发送get请求
response, err := reqCli.Request(nil, "get", "http://myip.top", requests.RequestOption{
DisProxy: true, //强制走本地代理
})
if err != nil {
log.Panic(err)
}
log.Print(response.Text()) //获取内容,解析为字符串
}
```
# 发送http请求
```golang
package main
import (
"log"
"gitee.com/baixudong/requests"
)
func main() {
reqCli, err := requests.NewClient(nil) //创建请求客户端
if err != nil {
log.Panic(err)
}
response, err := reqCli.Request(nil, "get", "http://myip.top") //发送get请求
if err != nil {
log.Panic(err)
}
log.Print(response.Text()) //获取内容,解析为字符串
log.Print(response.Content()) //获取内容,解析为字节
log.Print(response.Json()) //获取json,解析为gjson
log.Print(response.Html()) //获取内容,解析为dom
log.Print(response.Cookies()) //获取cookies
}
```
# 发送websocket 请求
```golang
package main
import (
"context"
"log"
"gitee.com/baixudong/requests"
"gitee.com/baixudong/websocket"
)
func main() {
reqCli, err := requests.NewClient(nil) //创建请求客户端
if err != nil {
log.Panic(err)
}
response, err := reqCli.Request(nil, "get", "ws://82.157.123.54:9010/ajaxchattest", requests.RequestOption{Headers: map[string]string{
"Origin": "http://coolaf.com",
}}) //发送websocket请求
if err != nil {
log.Panic(err)
}
defer response.Close()
wsCli := response.WebSocket()
if err = wsCli.Send(context.TODO(), websocket.MessageText, "测试"); err != nil { //发送txt 消息
log.Panic(err)
}
msgType, con, err := wsCli.Recv(context.TODO()) //接收消息
if err != nil {
log.Panic(err)
}
log.Print(msgType) //消息类型
log.Print(string(con)) //消息内容
}
```
# ipv4,ipv6 地址控制解析
```go
func main() {
reqCli, err := requests.NewClient(nil, requests.ClientOption{
AddrType: requests.Ipv4, //优先解析ipv4地址
// AddrType: requests.Ipv6,//优先解析ipv6地址
})
if err != nil {
log.Panic(err)
}
href := "https://test.ipw.cn"
resp, err := reqCli.Request(nil, "get", href)
if err != nil {
log.Panic(err)
}
log.Print(resp.Text())
log.Print(resp.StatusCode())
}
```
# ja3 伪造指纹
## 根据字符串生成指纹
```go
func main() {
ja3Str := "772,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,0-23-65281-10-11-35-16-5-13-18-51-45-43-27-17513,29-23-24,0"
Ja3Spec, err := ja3.CreateSpecWithStr(ja3Str)//根据字符串生成指纹
if err != nil {
log.Panic(err)
}
reqCli, err := requests.NewClient(nil, requests.ClientOption{Ja3Spec: Ja3Spec})
if err != nil {
log.Panic(err)
}
response, err := reqCli.Request(nil, "get", "https://tools.scrapfly.io/api/fp/ja3?extended=1")
if err != nil {
log.Panic(err)
}
jsonData,_:=response.Json()
log.Print(jsonData.Get("ja3").String())
log.Print(jsonData.Get("ja3").String() == ja3Str)
}
```
## 根据id 生成指纹
```go
func main() {
Ja3Spec, err := ja3.CreateSpecWithId(ja3.HelloChrome_Auto) //根据id 生成指纹
if err != nil {
log.Panic(err)
}
reqCli, err := requests.NewClient(nil, requests.ClientOption{Ja3Spec: Ja3Spec})
if err != nil {
log.Panic(err)
}
response, err := reqCli.Request(nil, "get", "https://tools.scrapfly.io/api/fp/ja3?extended=1")
if err != nil {
log.Panic(err)
}
jsonData,_:=response.Json()
log.Print(jsonData.Get("ja3").String())
}
```
## ja3 开关
```go
func main() {
reqCli, err := requests.NewClient(nil)
if err != nil {
log.Panic(err)
}
response, err := reqCli.Request(nil, "get", "https://tools.scrapfly.io/api/fp/ja3?extended=1", requests.RequestOption{Ja3: true})//使用最新chrome 指纹
if err != nil {
log.Panic(err)
}
jsonData,_:=response.Json()
log.Print(jsonData.Get("ja3").String())
}
```
## h2 指纹开关
```go
func main() {
reqCli, err := requests.NewClient(nil, requests.ClientOption{
H2Ja3: true,
})
if err != nil {
log.Panic(err)
}
href := "https://tools.scrapfly.io/api/fp/anything"
resp, err := reqCli.Request(nil, "get", href)
if err != nil {
log.Panic(err)
}
log.Print(resp.Text())
}
```
## 修改h2指纹
```go
func main() {
reqCli, err := requests.NewClient(nil, requests.ClientOption{
H2Ja3Spec: ja3.H2Ja3Spec{
InitialSetting: []ja3.Setting{
{Id: 1, Val: 65555},
{Id: 2, Val: 1},
{Id: 3, Val: 2000},
{Id: 4, Val: 6291457},
{Id: 6, Val: 262145},
},
ConnFlow: 15663106,
OrderHeaders: []string{
":method",
":path",
":scheme",
":authority",
},
},
})
if err != nil {
log.Panic(err)
}
href := "https://tools.scrapfly.io/api/fp/anything"
resp, err := reqCli.Request(nil, "get", href)
if err != nil {
log.Panic(err)
}
log.Print(resp.Text())
}
```
# 采集全国公共资源网和中国政府采购网的列表页的标题
```go
package main
import (
"log"
"gitee.com/baixudong/requests"
)
func main() {
reqCli, err := requests.NewClient(nil)
if err != nil {
log.Panic(err)
}
resp, err := reqCli.Request(nil, "get", "http://www.ccgp.gov.cn/cggg/zygg/")
if err != nil {
log.Panic(err)
}
html := resp.Html()
lis := html.Finds("ul.c_list_bid li")
for _, li := range lis {
title := li.Find("a").Get("title")
log.Print(title)
}
resp, err = reqCli.Request(nil, "post", "http://deal.ggzy.gov.cn/ds/deal/dealList_find.jsp", requests.RequestOption{
Data: map[string]string{
"TIMEBEGIN_SHOW": "2023-04-26",
"TIMEEND_SHOW": "2023-05-05",
"TIMEBEGIN": "2023-04-26",
"TIMEEND": "2023-05-05",
"SOURCE_TYPE": "1",
"DEAL_TIME": "02",
"DEAL_CLASSIFY": "01",
"DEAL_STAGE": "0100",
"DEAL_PROVINCE": "0",
"DEAL_CITY": "0",
"DEAL_PLATFORM": "0",
"BID_PLATFORM": "0",
"DEAL_TRADE": "0",
"isShowAll": "1",
"PAGENUMBER": "2",
"FINDTXT": "",
},
})
if err != nil {
log.Panic(err)
}
jsonData,_ := resp.Json()
lls := jsonData.Get("data").Array()
for _, ll := range lls {
log.Print(ll.Get("title"))
}
}
```

89
body.go Normal file
View File

@@ -0,0 +1,89 @@
package requests
import (
"bytes"
"errors"
"fmt"
"net/url"
"gitee.com/baixudong/tools"
"github.com/tidwall/gjson"
)
// 构造一个文件
type File struct {
Key string //字段的key
Name string //文件名
Content []byte //文件的内容
Type string //文件类型
}
type bodyType = int
const (
jsonType = iota
textType
rawType
dataType
formType
paramsType
)
func newBody(val any, valType bodyType, dataMap map[string][]string) (*bytes.Reader, error) {
switch value := val.(type) {
case gjson.Result:
if !value.IsObject() {
return nil, errors.New("body-type错误")
}
switch valType {
case jsonType, textType, rawType:
return bytes.NewReader(tools.StringToBytes(value.Raw)), nil
case dataType:
tempVal := url.Values{}
for kk, vv := range value.Map() {
if vv.IsArray() {
for _, v := range vv.Array() {
tempVal.Add(kk, v.String())
}
} else {
tempVal.Add(kk, vv.String())
}
}
return bytes.NewReader(tools.StringToBytes(tempVal.Encode())), nil
case formType, paramsType:
for kk, vv := range value.Map() {
kkvv := []string{}
if vv.IsArray() {
for _, v := range vv.Array() {
kkvv = append(kkvv, v.String())
}
} else {
kkvv = append(kkvv, vv.String())
}
dataMap[kk] = kkvv
}
return nil, nil
default:
return nil, fmt.Errorf("未知的content-type：%d", valType)
}
case string:
switch valType {
case jsonType, textType, dataType, rawType:
return bytes.NewReader(tools.StringToBytes(value)), nil
default:
return nil, fmt.Errorf("未知的content-type：%d", valType)
}
case []byte:
switch valType {
case jsonType, textType, dataType, rawType:
return bytes.NewReader(value), nil
default:
return nil, fmt.Errorf("未知的content-type：%d", valType)
}
default:
result, err := tools.Any2json(value)
if err != nil {
return nil, err
}
return newBody(result, valType, dataMap)
}
}

326
client.go Normal file
View File

@@ -0,0 +1,326 @@
package requests
import (
"context"
"crypto/tls"
"net"
"net/http"
"net/http/cookiejar"
"net/url"
"time"
"gitee.com/baixudong/http2"
"gitee.com/baixudong/ja3"
utls "github.com/refraction-networking/utls"
)
type ClientOption struct {
GetProxy func(ctx context.Context, url *url.URL) (string, error) //根据url 返回代理，支持https,http,socks5 代理协议
Proxy string //设置代理,支持https,http,socks5 代理协议
TLSHandshakeTimeout time.Duration //tls 超时时间,default:15
ResponseHeaderTimeout time.Duration //第一个response headers 接收超时时间,default:30
DisCookie bool //关闭cookies管理
DisCompression bool //关闭请求头中的压缩功能
LocalAddr *net.TCPAddr //本地网卡出口ip
IdleConnTimeout time.Duration //空闲连接在连接池中的超时时间,default:90
DialTimeout time.Duration //dial tcp 超时时间,default:15
KeepAlive time.Duration //keepalive保活检测定时,default:30
AddrType AddrType //优先使用的addr 类型
GetAddrType func(string) AddrType
Dns net.IP //dns
Ja3 bool //开启ja3指纹
Ja3Spec ja3.ClientHelloSpec //指定ja3Spec,使用ja3.CreateSpecWithStr 或者ja3.CreateSpecWithId 生成
H2Ja3 bool //开启h2指纹
H2Ja3Spec ja3.H2Ja3Spec //h2指纹
RedirectNum int //重定向次数,小于0为禁用,0:不限制
DisDecode bool //关闭自动编码
DisRead bool //关闭默认读取请求体
DisUnZip bool //关闭自动解压
TryNum int64 //重试次数
OptionCallBack func(context.Context, *Client, *RequestOption) error //请求参数回调,用于对请求参数进行修改。返回error,中断重试请求,返回nil继续
ResultCallBack func(context.Context, *Client, *Response) error //结果回调,用于对结果进行校验。返回nil，直接返回,返回err的话，如果有errCallBack 走errCallBack，没有继续try
ErrCallBack func(context.Context, *Client, error) error //错误回调,返回error,中断重试请求,返回nil继续
Timeout time.Duration //请求超时时间
Headers any //请求头
Bar bool //是否开启请求进度条
}
type Client struct {
http2Upg *http2.Upg
redirectNum int //重定向次数
disDecode bool //关闭自动编码
disRead bool //关闭默认读取请求体
disUnZip bool //变比自动解压
tryNum int64 //重试次数
optionCallBack func(context.Context, *Client, *RequestOption) error //请求参数回调,用于对请求参数进行修改。返回error,中断重试请求,返回nil继续
resultCallBack func(context.Context, *Client, *Response) error //结果回调,用于对结果进行校验。返回nil，直接返回,返回err的话，如果有errCallBack 走errCallBack，没有继续try
errCallBack func(context.Context, *Client, error) error //错误回调,返回error,中断重试请求,返回nil继续
timeout time.Duration //请求超时时间
headers any //请求头
bar bool //是否开启bar
dialer *DialClient //dialer
disCookie bool
client *http.Client
noJarClient *http.Client
ctx context.Context
cnl context.CancelFunc
}
// 新建一个请求客户端,发送请求必须创建哈
func NewClient(preCtx context.Context, options ...ClientOption) (*Client, error) {
if preCtx == nil {
preCtx = context.TODO()
}
ctx, cnl := context.WithCancel(preCtx)
var option ClientOption
//初始化参数
if len(options) > 0 {
option = options[0]
}
if option.IdleConnTimeout == 0 {
option.IdleConnTimeout = time.Second * 90
}
if option.KeepAlive == 0 {
option.KeepAlive = time.Second * 30
}
if option.DialTimeout == 0 {
option.DialTimeout = time.Second * 15
}
if option.TLSHandshakeTimeout == 0 {
option.TLSHandshakeTimeout = time.Second * 15
}
if option.ResponseHeaderTimeout == 0 {
option.ResponseHeaderTimeout = time.Second * 30
}
if option.Ja3Spec.IsSet() {
option.Ja3 = true
}
tlsConfig := &tls.Config{
InsecureSkipVerify: true,
SessionTicketKey: [32]byte{},
ClientSessionCache: tls.NewLRUClientSessionCache(0),
}
utlsConfig := &utls.Config{
InsecureSkipVerify: true,
InsecureSkipTimeVerify: true,
SessionTicketKey: [32]byte{},
ClientSessionCache: utls.NewLRUClientSessionCache(0),
}
dialClient, err := NewDail(ctx, DialOption{
Ja3: option.Ja3,
Ja3Spec: option.Ja3Spec,
DialTimeout: option.DialTimeout,
TLSHandshakeTimeout: option.TLSHandshakeTimeout,
Dns: option.Dns,
GetProxy: option.GetProxy,
Proxy: option.Proxy,
KeepAlive: option.KeepAlive,
LocalAddr: option.LocalAddr,
AddrType: option.AddrType,
GetAddrType: option.GetAddrType,
TlsConfig: tlsConfig,
UtlsConfig: utlsConfig,
})
if err != nil {
cnl()
return nil, err
}
//创建cookiesjar
var jar *cookiejar.Jar
if !option.DisCookie {
jar = newJar()
}
transport := &http.Transport{
MaxIdleConns: 655350,
MaxConnsPerHost: 655350,
MaxIdleConnsPerHost: 655350,
ProxyConnectHeader: http.Header{
"User-Agent": []string{UserAgent},
},
TLSHandshakeTimeout: option.TLSHandshakeTimeout,
ResponseHeaderTimeout: option.ResponseHeaderTimeout,
DisableCompression: option.DisCompression,
TLSClientConfig: tlsConfig,
IdleConnTimeout: option.IdleConnTimeout, //空闲连接在连接池中的超时时间
DialContext: dialClient.requestHttpDialContext,
DialTLSContext: dialClient.requestHttpDialTlsContext,
ForceAttemptHTTP2: true,
Proxy: func(r *http.Request) (*url.URL, error) {
ctxData := r.Context().Value(keyPrincipalID).(*reqCtxData)
ctxData.url, ctxData.host = r.URL, r.Host
if ctxData.host == "" {
ctxData.host = ctxData.url.Host
}
if ctxData.requestCallBack != nil {
req, err := cloneRequest(r, ctxData.disBody)
if err != nil {
return nil, err
}
if err = ctxData.requestCallBack(r.Context(), req); err != nil {
return nil, err
}
}
if option.Ja3 || ctxData.disProxy { //ja3 或 关闭代理，走自实现代理
return nil, nil
}
if ctxData.proxy != nil { //走官方代理
ctxData.isRawConn = true
}
return ctxData.proxy, nil
},
}
var http2Upg *http2.Upg
// h3Round := &http3.RoundTripper{
// TLSClientConfig: tlsConfig,
// }
if option.H2Ja3 || option.H2Ja3Spec.IsSet() {
http2Upg = http2.NewUpg(transport, http2.UpgOption{
H2Ja3Spec: option.H2Ja3Spec,
DialTLSContext: dialClient.requestHttp2DialTlsContext,
IdleConnTimeout: option.TLSHandshakeTimeout,
TLSHandshakeTimeout: option.TLSHandshakeTimeout,
ResponseHeaderTimeout: option.ResponseHeaderTimeout,
TlsConfig: tlsConfig,
})
transport.TLSNextProto = map[string]func(authority string, c *tls.Conn) http.RoundTripper{
"h2": func(authority string, c *tls.Conn) http.RoundTripper {
return http2Upg.UpgradeFn(authority, c)
},
// "h3": func(authority string, c *tls.Conn) http.RoundTripper {
// return h3Round
// },
}
}
client := &http.Client{
Transport: transport,
Jar: jar,
CheckRedirect: func(req *http.Request, via []*http.Request) error {
ctxData := req.Context().Value(keyPrincipalID).(*reqCtxData)
if ctxData.responseCallBack != nil {
resp, err := cloneResponse(req.Response, false)
if err != nil {
return err
}
if err = ctxData.responseCallBack(req.Context(), resp); err != nil {
return err
}
}
if ctxData.redirectNum == 0 || ctxData.redirectNum >= len(via) {
return nil
}
return http.ErrUseLastResponse
},
}
var noJarClient *http.Client
if client.Jar != nil {
noJarClient = &http.Client{
Transport: client.Transport,
CheckRedirect: client.CheckRedirect,
}
}
result := &Client{
ctx: ctx,
cnl: cnl,
dialer: dialClient,
client: client,
noJarClient: noJarClient,
http2Upg: http2Upg,
disCookie: option.DisCookie,
redirectNum: option.RedirectNum,
disDecode: option.DisDecode,
disRead: option.DisRead,
disUnZip: option.DisUnZip,
tryNum: option.TryNum,
optionCallBack: option.OptionCallBack,
resultCallBack: option.ResultCallBack,
errCallBack: option.ErrCallBack,
timeout: option.Timeout,
headers: option.Headers,
bar: option.Bar,
}
return result, nil
}
func (obj *Client) SetProxy(proxy string) error {
return obj.dialer.SetProxy(proxy)
}
func (obj *Client) SetGetProxy(getProxy func(ctx context.Context, url *url.URL) (string, error)) {
obj.dialer.SetGetProxy(getProxy)
}
// 关闭客户端
func (obj *Client) Close() {
obj.CloseIdleConnections()
obj.cnl()
}
// 关闭客户端中的空闲连接
func (obj *Client) CloseIdleConnections() {
obj.client.CloseIdleConnections()
if obj.http2Upg != nil {
obj.http2Upg.CloseIdleConnections()
}
}
// 返回url 的cookies,也可以设置url 的cookies
func (obj *Client) Cookies(href string, cookies ...any) (Cookies, error) {
return cookie(obj.client.Jar, href, cookies...)
}
type Jar struct {
jar *cookiejar.Jar
}
func newJar() *cookiejar.Jar {
jar, _ := cookiejar.New(nil)
return jar
}
func NewJar() *Jar {
return &Jar{
jar: newJar(),
}
}
func (obj *Jar) Cookies(href string, cookies ...any) (Cookies, error) {
return cookie(obj.jar, href, cookies...)
}
func (obj *Jar) ClearCookies() {
*obj.jar = *newJar()
}
func cookie(jar http.CookieJar, href string, cookies ...any) (Cookies, error) {
if jar == nil {
return nil, nil
}
u, err := url.Parse(href)
if err != nil {
return nil, err
}
for _, cookie := range cookies {
cooks, err := ReadCookies(cookie)
if err != nil {
return nil, err
}
jar.SetCookies(u, cooks)
}
return jar.Cookies(u), nil
}
// 清除cookies
func (obj *Client) ClearCookies() {
if obj.client.Jar != nil {
*obj.client.Jar.(*cookiejar.Jar) = *newJar()
}
}
func (obj *Client) getClient(option RequestOption) *http.Client {
if option.DisCookie && obj.noJarClient != nil {
return obj.noJarClient
}
return obj.client
}

133
cookies.go Normal file
View File

@@ -0,0 +1,133 @@
package requests
import (
"errors"
"net/http"
_ "unsafe"
"gitee.com/baixudong/tools"
"github.com/tidwall/gjson"
)
//go:linkname readCookies net/http.readCookies
func readCookies(h http.Header, filter string) []*http.Cookie
//go:linkname readSetCookies net/http.readSetCookies
func readSetCookies(h http.Header) []*http.Cookie
// 支持json,map,[]string,http.Header,string
func ReadCookies(val any) (Cookies, error) {
switch cook := val.(type) {
case *http.Cookie:
return Cookies{
cook,
}, nil
case http.Cookie:
return Cookies{
&cook,
}, nil
case Cookies:
return cook, nil
case []*http.Cookie:
return Cookies(cook), nil
case string:
return readCookies(http.Header{"Cookie": []string{cook}}, ""), nil
case http.Header:
return readCookies(cook, ""), nil
case []string:
return readCookies(http.Header{"Cookie": cook}, ""), nil
default:
return any2Cookies(cook)
}
}
func ReadSetCookies(val any) (Cookies, error) {
switch cook := val.(type) {
case Cookies:
return cook, nil
case []*http.Cookie:
return Cookies(cook), nil
case string:
return readSetCookies(http.Header{"Set-Cookie": []string{cook}}), nil
case http.Header:
return readSetCookies(cook), nil
case []string:
return readSetCookies(http.Header{"Set-Cookie": cook}), nil
default:
return any2Cookies(cook)
}
}
func any2Cookies(val any) (Cookies, error) {
switch cooks := val.(type) {
case map[string]string:
cookies := Cookies{}
for kk, vv := range cooks {
cookies = append(cookies, &http.Cookie{
Name: kk,
Value: vv,
})
}
return cookies, nil
case map[string][]string:
cookies := Cookies{}
for kk, vvs := range cooks {
for _, vv := range vvs {
cookies = append(cookies, &http.Cookie{
Name: kk,
Value: vv,
})
}
}
return cookies, nil
case gjson.Result:
if !cooks.IsObject() {
return nil, errors.New("cookies不支持的类型")
}
cookies := Cookies{}
for kk, vvs := range cooks.Map() {
if vvs.IsArray() {
for _, vv := range vvs.Array() {
cookies = append(cookies, &http.Cookie{
Name: kk,
Value: vv.String(),
})
}
} else {
cookies = append(cookies, &http.Cookie{
Name: kk,
Value: vvs.String(),
})
}
}
return cookies, nil
default:
jsonData, err := tools.Any2json(cooks)
if err != nil {
return nil, err
}
cookies := Cookies{}
for kk, vvs := range jsonData.Map() {
if vvs.IsArray() {
for _, vv := range vvs.Array() {
cookies = append(cookies, &http.Cookie{
Name: kk,
Value: vv.String(),
})
}
} else {
cookies = append(cookies, &http.Cookie{
Name: kk,
Value: vvs.String(),
})
}
}
return cookies, nil
}
}
func (obj *RequestOption) initCookies() (err error) {
if obj.Cookies == nil {
return nil
}
obj.Cookies, err = ReadCookies(obj.Cookies)
return err
}

591
dial.go Normal file
View File

@@ -0,0 +1,591 @@
package requests
import (
"bufio"
"context"
"crypto/tls"
"errors"
"io"
"net"
"net/http"
"net/url"
"strings"
"sync"
"time"
"gitee.com/baixudong/ja3"
"gitee.com/baixudong/tools"
utls "github.com/refraction-networking/utls"
)
type DialClient struct {
getProxy func(ctx context.Context, url *url.URL) (string, error)
proxy *url.URL
dns *net.UDPAddr
dialer *net.Dialer
dnsIpData sync.Map
addrType AddrType //使用ipv4,ipv6 ,或自动选项
getAddrType func(string) AddrType
proxyJa3 bool //是否启用ja3
proxyJa3Spec ja3.ClientHelloSpec
ja3 bool //是否启用ja3
ja3Spec ja3.ClientHelloSpec
ctx context.Context
tlsHandshakeTimeout time.Duration
tlsConfig *tls.Config
utlsConfig *utls.Config
}
type msgClient struct {
time time.Time
host string
}
type AddrType int
const (
Auto AddrType = 0
Ipv4 AddrType = 4
Ipv6 AddrType = 6
)
type DialOption struct {
DialTimeout time.Duration
TLSHandshakeTimeout time.Duration
KeepAlive time.Duration
GetProxy func(ctx context.Context, url *url.URL) (string, error)
Proxy string //代理
LocalAddr *net.TCPAddr //使用本地网卡
AddrType AddrType //优先使用的地址类型,ipv4,ipv6 ,或自动选项
GetAddrType func(string) AddrType
Dns net.IP
Ja3 bool //是否启用ja3
Ja3Spec ja3.ClientHelloSpec //指定ja3Spec,使用ja3.CreateSpecWithStr 或者ja3.CreateSpecWithId 生成
ProxyJa3 bool //代理是否启用ja3
ProxyJa3Spec ja3.ClientHelloSpec //指定代理ja3Spec,使用ja3.CreateSpecWithStr 或者ja3.CreateSpecWithId 生成
TlsConfig *tls.Config
UtlsConfig *utls.Config
}
func NewDail(ctx context.Context, option DialOption) (*DialClient, error) {
if ctx == nil {
ctx = context.TODO()
}
if option.KeepAlive == 0 {
option.KeepAlive = time.Second * 30
}
if option.TLSHandshakeTimeout == 0 {
option.TLSHandshakeTimeout = time.Second * 15
}
if option.DialTimeout == 0 {
option.DialTimeout = time.Second * 15
}
if option.Ja3Spec.IsSet() {
option.Ja3 = true
}
if option.ProxyJa3Spec.IsSet() {
option.ProxyJa3 = true
}
if option.TlsConfig == nil {
option.TlsConfig = &tls.Config{
InsecureSkipVerify: true,
SessionTicketKey: [32]byte{},
ClientSessionCache: tls.NewLRUClientSessionCache(0),
}
}
if option.UtlsConfig == nil {
option.UtlsConfig = &utls.Config{
InsecureSkipVerify: true,
InsecureSkipTimeVerify: true,
SessionTicketKey: [32]byte{},
ClientSessionCache: utls.NewLRUClientSessionCache(0),
}
}
var err error
dialCli := &DialClient{
ctx: ctx,
tlsHandshakeTimeout: option.TLSHandshakeTimeout,
dialer: &net.Dialer{
Timeout: option.DialTimeout,
KeepAlive: option.KeepAlive,
},
getProxy: option.GetProxy,
addrType: option.AddrType,
getAddrType: option.GetAddrType,
proxyJa3: option.ProxyJa3,
proxyJa3Spec: option.ProxyJa3Spec,
ja3: option.Ja3,
ja3Spec: option.Ja3Spec,
tlsConfig: option.TlsConfig,
utlsConfig: option.UtlsConfig,
}
if option.Proxy != "" {
if dialCli.proxy, err = verifyProxy(option.Proxy); err != nil {
return dialCli, err
}
}
if option.LocalAddr != nil {
dialCli.dialer.LocalAddr = option.LocalAddr
}
if option.Dns != nil {
dialCli.dns = &net.UDPAddr{IP: option.Dns, Port: 53}
dialCli.dialer.Resolver = &net.Resolver{
PreferGo: true,
Dial: dialCli.DnsDialContext,
}
}
// dialCli.dialer.SetMultipathTCP(true)
return dialCli, err
}
func (obj *DialClient) UtlsConfig() *utls.Config {
return obj.utlsConfig.Clone()
}
func (obj *DialClient) TlsConfig() *tls.Config {
return obj.tlsConfig.Clone()
}
func (obj *DialClient) GetProxy(ctx context.Context, href *url.URL) (*url.URL, error) {
if obj.proxy != nil {
return obj.proxy, nil
}
if obj.getProxy != nil {
proxy, err := obj.getProxy(ctx, href)
if proxy == "" || err != nil {
return nil, err
}
return url.Parse(proxy)
}
return nil, nil
}
func (obj *DialClient) Proxy() *url.URL {
if obj.proxy == nil {
return nil
}
return cloneUrl(obj.proxy)
}
func (obj *DialClient) SetProxy(proxy string) error {
if proxy == "" {
obj.proxy = nil
return nil
}
tmpProxy, err := verifyProxy(proxy)
if err != nil {
return err
}
if obj.proxy == nil {
obj.proxy = tmpProxy
} else {
*obj.proxy = *tmpProxy
}
return nil
}
func (obj *DialClient) SetGetProxy(getProxy func(ctx context.Context, url *url.URL) (string, error)) {
if getProxy == nil {
obj.getProxy = func(ctx context.Context, url *url.URL) (string, error) {
return "", nil
}
} else {
obj.getProxy = getProxy
}
}
func (obj *DialClient) Dialer() *net.Dialer {
return obj.dialer
}
func (obj *DialClient) loadHost(host string) (string, bool) {
msgDataAny, ok := obj.dnsIpData.Load(host)
if ok {
msgdata := msgDataAny.(msgClient)
if time.Since(msgdata.time) < time.Second*60*5 {
return msgdata.host, true
}
}
return host, false
}
func (obj *DialClient) AddrToIp(ctx context.Context, addr string) (string, error) {
host, port, err := net.SplitHostPort(addr)
if err != nil {
return addr, tools.WrapError(err, "addrToIp 错误,SplitHostPort")
}
_, ipInt := tools.ParseHost(host)
if ipInt == 4 || ipInt == 6 {
return addr, nil
}
host, ok := obj.loadHost(host)
if !ok {
ip, err := obj.lookupIPAddr(ctx, host)
if err != nil {
return addr, tools.WrapError(err, "addrToIp 错误,lookupIPAddr")
}
host = ip.String()
obj.dnsIpData.Store(addr, msgClient{time: time.Now(), host: host})
}
return net.JoinHostPort(host, port), nil
}
func (obj *DialClient) clientVerifySocks5(ctx context.Context, proxyUrl *url.URL, addr string, conn net.Conn) (err error) {
if _, err = conn.Write([]byte{5, 2, 0, 2}); err != nil {
return
}
readCon := make([]byte, 4)
if _, err = io.ReadFull(conn, readCon[:2]); err != nil {
return
}
switch readCon[1] {
case 2:
if proxyUrl.User == nil {
err = errors.New("需要验证")
return
}
pwd, pwdOk := proxyUrl.User.Password()
if !pwdOk {
err = errors.New("密码格式不对")
return
}
usr := proxyUrl.User.Username()
if usr == "" {
err = errors.New("用户名格式不对")
return
}
if _, err = conn.Write(append(
append(
[]byte{1, byte(len(usr))},
tools.StringToBytes(usr)...,
),
append(
[]byte{byte(len(pwd))},
tools.StringToBytes(pwd)...,
)...,
)); err != nil {
return
}
if _, err = io.ReadFull(conn, readCon[:2]); err != nil {
return
}
switch readCon[1] {
case 0:
default:
err = errors.New("验证失败")
return
}
case 0:
default:
err = errors.New("不支持的验证方式")
return
}
var host string
var port int
if host, port, err = tools.SplitHostPort(addr); err != nil {
return
}
writeCon := []byte{5, 1, 0}
ip, ipInt := tools.ParseHost(host)
switch ipInt {
case 4:
writeCon = append(writeCon, 1)
writeCon = append(writeCon, ip...)
case 6:
writeCon = append(writeCon, 4)
writeCon = append(writeCon, ip...)
case 0:
if len(host) > 255 {
err = errors.New("FQDN too long")
return
}
writeCon = append(writeCon, 3)
writeCon = append(writeCon, byte(len(host)))
writeCon = append(writeCon, host...)
}
writeCon = append(writeCon, byte(port>>8), byte(port))
if _, err = conn.Write(writeCon); err != nil {
return
}
if _, err = io.ReadFull(conn, readCon); err != nil {
return
}
if readCon[0] != 5 {
err = errors.New("版本不对")
return
}
if readCon[1] != 0 {
err = errors.New("连接失败")
return
}
if readCon[3] != 1 {
err = errors.New("连接类型不一致")
return
}
switch readCon[3] {
case 1: //ipv4地址
if _, err = io.ReadFull(conn, readCon); err != nil {
return
}
case 3: //域名
if _, err = io.ReadFull(conn, readCon[:1]); err != nil { //域名的长度
return
}
if _, err = io.ReadFull(conn, make([]byte, readCon[0])); err != nil {
return
}
case 4: //IPv6地址
if _, err = io.ReadFull(conn, make([]byte, 16)); err != nil {
return
}
default:
err = errors.New("invalid atyp")
return
}
_, err = io.ReadFull(conn, readCon[:2])
return
}
func cloneUrl(u *url.URL) *url.URL {
r := *u
return &r
}
func (obj *DialClient) lookupIPAddr(ctx context.Context, host string) (net.IP, error) {
var addrType int
if obj.addrType != 0 {
addrType = int(obj.addrType)
} else if obj.getAddrType != nil {
addrType = int(obj.getAddrType(host))
}
ips, err := obj.dialer.Resolver.LookupIPAddr(ctx, host)
if err != nil {
return nil, err
}
for _, ipAddr := range ips {
ip := ipAddr.IP
if ipType := tools.ParseIp(ip); ipType == 4 || ipType == 6 {
if addrType == 0 || addrType == ipType {
return ip, nil
}
}
}
for _, ipAddr := range ips {
ip := ipAddr.IP
if ipType := tools.ParseIp(ip); ipType == 4 || ipType == 6 {
return ip, nil
}
}
return nil, errors.New("dns 解析host 失败")
}
func (obj *DialClient) DnsDialContext(ctx context.Context, netword string, addr string) (net.Conn, error) {
if obj.dns != nil {
return obj.dialer.DialContext(ctx, netword, obj.dns.String())
}
return obj.dialer.DialContext(ctx, netword, addr)
}
func (obj *DialClient) DialContext(ctx context.Context, netword string, addr string) (net.Conn, error) {
revHost, err := obj.AddrToIp(ctx, addr)
if err != nil {
return nil, err
}
return obj.dialer.DialContext(ctx, netword, revHost)
}
func (obj *DialClient) AddProxyTls(ctx context.Context, conn net.Conn, host string) (net.Conn, error) {
if obj.proxyJa3 {
utlsConfig := obj.UtlsConfig()
utlsConfig.NextProtos = []string{"http/1.1"}
utlsConfig.ServerName = tools.GetServerName(host)
return ja3.NewClient(ctx, conn, obj.proxyJa3Spec, true, utlsConfig)
}
tlsConfig := obj.TlsConfig()
tlsConfig.ServerName = tools.GetServerName(host)
tlsConfig.NextProtos = []string{"http/1.1"}
tlsConn := tls.Client(conn, tlsConfig)
return tlsConn, tlsConn.HandshakeContext(ctx)
}
func (obj *DialClient) AddTls(ctx context.Context, conn net.Conn, host string, disHttp bool) (tconn net.Conn, err error) {
if obj.ja3 {
var utlsConn *utls.UConn
utlsConfig := obj.UtlsConfig()
utlsConfig.ServerName = tools.GetServerName(host)
if disHttp {
utlsConfig.NextProtos = []string{"http/1.1"}
} else {
utlsConfig.NextProtos = []string{"h2", "http/1.1"}
}
utlsConn, err = ja3.NewClient(ctx, conn, obj.ja3Spec, disHttp, utlsConfig)
if err != nil {
err = tools.WrapError(err, "dialClient AddTls ja3.NewClient 错误")
return nil, err
}
if utlsConn.ConnectionState().NegotiatedProtocol != "h2" {
return utlsConn, err
}
tlsConfig := obj.TlsConfig()
tlsConfig.ServerName = tools.GetServerName(host)
if tconn, err = ja3.Utls2Tls(obj.ctx, ctx, utlsConn, tlsConfig); err != nil {
err = tools.WrapError(err, "dialClient AddTls Utls2Tls 错误")
}
return
}
var tlsConn *tls.Conn
tlsConfig := obj.TlsConfig()
tlsConfig.ServerName = tools.GetServerName(host)
if disHttp {
tlsConfig.NextProtos = []string{"http/1.1"}
tlsConn = tls.Client(conn, tlsConfig)
} else {
tlsConfig.NextProtos = []string{"h2", "http/1.1"}
tlsConn = tls.Client(conn, tlsConfig)
}
if err = tlsConn.HandshakeContext(ctx); err != nil {
err = tools.WrapError(err, "dialClient AddTls tls HandshakeContext 错误")
}
return tlsConn, err
}
func (obj *DialClient) Socks5Proxy(ctx context.Context, network string, addr string, proxyUrl *url.URL) (conn net.Conn, err error) {
defer func() {
if err != nil && conn != nil {
conn.Close()
}
}()
if conn, err = obj.DialContext(ctx, network, net.JoinHostPort(proxyUrl.Hostname(), proxyUrl.Port())); err != nil {
return
}
didVerify := make(chan struct{})
go func() {
defer close(didVerify)
err = obj.clientVerifySocks5(ctx, proxyUrl, addr, conn)
}()
select {
case <-ctx.Done():
return conn, ctx.Err()
case <-didVerify:
return
}
}
func (obj *DialClient) clientVerifyHttps(ctx context.Context, scheme string, proxyUrl *url.URL, addr string, host string, conn net.Conn) (err error) {
hdr := make(http.Header)
hdr.Set("User-Agent", UserAgent)
if proxyUrl.User != nil {
if password, ok := proxyUrl.User.Password(); ok {
hdr.Set("Proxy-Authorization", "Basic "+tools.Base64Encode(proxyUrl.User.Username()+":"+password))
}
}
didReadResponse := make(chan struct{}) // closed after CONNECT write+read is done or fails
cHost := host
_, hport, _ := net.SplitHostPort(host)
if hport == "" {
_, aport, _ := net.SplitHostPort(addr)
if aport != "" {
cHost = net.JoinHostPort(cHost, aport)
} else if scheme == "http" {
cHost = net.JoinHostPort(cHost, "80")
} else if scheme == "https" {
cHost = net.JoinHostPort(cHost, "443")
} else {
return errors.New("clientVerifyHttps 连接代理没有解析出端口")
}
}
var resp *http.Response
go func() {
defer close(didReadResponse)
connectReq := &http.Request{
Method: http.MethodConnect,
URL: &url.URL{Opaque: addr},
Host: cHost,
Header: hdr,
}
if err = connectReq.Write(conn); err != nil {
return
}
resp, err = http.ReadResponse(bufio.NewReader(conn), connectReq)
}()
select {
case <-ctx.Done():
return ctx.Err()
case <-didReadResponse:
}
if err != nil {
return
}
if resp.StatusCode != 200 {
_, text, ok := strings.Cut(resp.Status, " ")
if !ok {
return errors.New("unknown status code")
}
return errors.New(text)
}
return
}
func (obj *DialClient) DialContextWithProxy(ctx context.Context, netword string, scheme string, addr string, host string, proxyUrl *url.URL) (net.Conn, error) {
if proxyUrl == nil {
return obj.DialContext(ctx, netword, addr)
}
if proxyUrl.Port() == "" {
if proxyUrl.Scheme == "http" {
proxyUrl.Host = net.JoinHostPort(proxyUrl.Hostname(), "80")
} else if proxyUrl.Scheme == "https" {
proxyUrl.Host = net.JoinHostPort(proxyUrl.Hostname(), "443")
}
}
switch proxyUrl.Scheme {
case "http":
conn, err := obj.DialContext(ctx, netword, net.JoinHostPort(proxyUrl.Hostname(), proxyUrl.Port()))
if err != nil {
return conn, err
}
return conn, obj.clientVerifyHttps(ctx, scheme, proxyUrl, addr, host, conn)
case "https":
conn, err := obj.DialContext(ctx, netword, net.JoinHostPort(proxyUrl.Hostname(), proxyUrl.Port()))
if err != nil {
return conn, err
}
tlsConn, err := obj.AddProxyTls(ctx, conn, proxyUrl.Host)
if err == nil {
return tlsConn, obj.clientVerifyHttps(ctx, scheme, proxyUrl, addr, host, tlsConn)
}
return tlsConn, err
case "socks5":
return obj.Socks5Proxy(ctx, netword, addr, proxyUrl)
default:
return nil, errors.New("proxyUrl Scheme error")
}
}
func (obj *DialClient) requestHttpDialContext(ctx context.Context, network string, addr string) (conn net.Conn, err error) {
reqData := ctx.Value(keyPrincipalID).(*reqCtxData)
if reqData.url == nil {
return nil, tools.WrapError(ErrFatal, "not found reqData.url")
}
var nowProxy *url.URL
if reqData.disProxy || reqData.isRawConn { //如果强制关闭代理，或强制走官方代理
if conn, err = obj.DialContext(ctx, network, addr); err != nil {
err = tools.WrapError(err, "requestHttpDialContext DialContext 错误")
}
return
} else if reqData.proxy != nil { //单独代理设置优先级最高
nowProxy = reqData.proxy
} else if nowProxy, err = obj.GetProxy(ctx, reqData.url); err != nil {
err = tools.WrapError(err, "requestHttpDialContext GetProxy 错误")
return nil, err
}
if nowProxy != nil { //走自实现代理
if conn, err = obj.DialContextWithProxy(ctx, network, reqData.url.Scheme, addr, reqData.host, nowProxy); err != nil {
err = tools.WrapError(err, "requestHttpDialContext DialContextWithProxy 错误")
}
return
}
if conn, err = obj.DialContext(ctx, network, addr); err != nil {
err = tools.WrapError(err, "requestHttpDialContext DialContext2 错误")
}
return
}
func (obj *DialClient) requestHttpDialTlsContext(preCtx context.Context, network string, addr string) (conn net.Conn, err error) {
if conn, err = obj.requestHttpDialContext(preCtx, network, addr); err != nil {
return conn, err
}
ctx, cnl := context.WithTimeout(preCtx, obj.tlsHandshakeTimeout)
defer cnl()
reqData := ctx.Value(keyPrincipalID).(*reqCtxData)
return obj.AddTls(ctx, conn, reqData.host, reqData.ws)
}
func (obj *DialClient) requestHttp2DialTlsContext(ctx context.Context, network string, addr string, cfg *tls.Config) (net.Conn, error) { //验证tls 是否可以直接用
if cfg.ServerName != "" {
ctx.Value(keyPrincipalID).(*reqCtxData).host = cfg.ServerName
}
return obj.requestHttpDialTlsContext(ctx, network, addr)
}

3
go.mod Normal file
View File

@@ -0,0 +1,3 @@
module gitee.com/baixudong/requests
go 1.20

56
headers.go Normal file
View File

@@ -0,0 +1,56 @@
package requests
import (
"errors"
"net/http"
"gitee.com/baixudong/tools"
"github.com/tidwall/gjson"
)
var UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"
var AcceptLanguage = "zh-CN,zh;q=0.9"
// 请求操作========================================================================= start
func DefaultHeaders() http.Header {
return http.Header{
"Accept-Encoding": []string{"gzip, deflate, br"},
"Accept": []string{"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7"},
"Accept-Language": []string{AcceptLanguage},
"User-Agent": []string{UserAgent},
}
}
func (obj *RequestOption) initHeaders() error {
if obj.Headers == nil {
obj.Headers = DefaultHeaders()
return nil
}
switch headers := obj.Headers.(type) {
case http.Header:
obj.Headers = headers.Clone()
return nil
case gjson.Result:
if !headers.IsObject() {
return errors.New("new headers error")
}
head := http.Header{}
for kk, vv := range headers.Map() {
if vv.IsArray() {
for _, v := range vv.Array() {
head.Add(kk, v.String())
}
} else {
head.Add(kk, vv.String())
}
}
obj.Headers = head
return nil
default:
jsonData, err := tools.Any2json(headers)
if err != nil {
return err
}
obj.Headers = jsonData
return obj.initHeaders()
}
}

224
option.go Normal file
View File

@@ -0,0 +1,224 @@
package requests
import (
"bytes"
"context"
"fmt"
"io"
"mime/multipart"
"net/textproto"
"net/url"
"strings"
"time"
"gitee.com/baixudong/websocket"
)
// 请求参数选项
type RequestOption struct {
Method string //method
Url *url.URL //请求的url
Host string //网站的host
Proxy string //代理,支持http,https,socks5协议代理,例如：http://127.0.0.1:7005
Timeout time.Duration //请求超时时间
Headers any //请求头,支持：json,map，header
Cookies any // cookies,支持json,map,str，http.Header
Files []File //发送multipart/form-data,文件上传
Params any //url 中的参数，用以拼接url,支持json,map
Form any //发送multipart/form-data,适用于文件上传,支持json,map
Data any //发送application/x-www-form-urlencoded,适用于key,val,支持string,[]bytes,json,map
body io.Reader
Body io.Reader
Json any //发送application/json,支持：string,[]bytes,json,map
Text any //发送text/xml,支持string,[]bytes,json,map
ContentType string //headers 中Content-Type 的值
Raw any //不设置context-type,支持string,[]bytes,json,map
TempData map[string]any //临时变量，用于回调存储或自由度更高的用法
DisCookie bool //关闭cookies管理,这个请求不用cookies池
DisDecode bool //关闭自动解码
Bar bool //是否开启bar
DisProxy bool //是否关闭代理,强制关闭代理
TryNum int64 //重试次数
OptionCallBack func(context.Context, *Client, *RequestOption) error //请求参数回调,用于对请求参数进行修改。返回error,中断重试请求,返回nil继续
ResultCallBack func(context.Context, *Client, *Response) error //结果回调,用于对结果进行校验。返回nil，直接返回,返回err的话，如果有errCallBack 走errCallBack，没有继续try
ErrCallBack func(context.Context, *Client, error) error //错误回调,返回error,中断重试请求,返回nil继续
RequestCallBack func(context.Context, *RequestDebug) error //request回调,用于定位bug,正常请求不要设置这个函数
ResponseCallBack func(context.Context, *ResponseDebug) error //response回调,用于定位bug,正常请求不要设置这个函数
Jar *Jar //自定义临时cookies 管理
RedirectNum int //重定向次数,小于零 关闭重定向
DisRead bool //关闭默认读取请求体,不会主动读取body里面的内容，需用你自己读取
DisUnZip bool //关闭自动解压
WsOption websocket.Option //websocket option,使用websocket 请求的option
converUrl string
}
func (obj *RequestOption) initBody() (err error) {
if obj.Body != nil {
obj.body = obj.Body
} else if obj.Raw != nil {
if obj.body, err = newBody(obj.Raw, rawType, nil); err != nil {
return err
}
} else if obj.Form != nil {
dataMap := map[string][]string{}
if obj.body, err = newBody(obj.Form, formType, dataMap); err != nil {
return err
}
tempBody := bytes.NewBuffer(nil)
writer := multipart.NewWriter(tempBody)
for key, vals := range dataMap {
for _, val := range vals {
if err = writer.WriteField(key, val); err != nil {
return err
}
}
}
escapeQuotes := strings.NewReplacer("\\", "\\\\", `"`, "\\\"")
for _, file := range obj.Files {
h := make(textproto.MIMEHeader)
h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="%s"; filename="%s"`, escapeQuotes.Replace(file.Key), escapeQuotes.Replace(file.Name)))
if file.Type == "" {
h.Set("Content-Type", "application/octet-stream")
} else {
h.Set("Content-Type", file.Type)
}
if wp, err := writer.CreatePart(h); err != nil {
return err
} else if _, err = wp.Write(file.Content); err != nil {
return err
}
}
if err = writer.Close(); err != nil {
return err
}
if obj.ContentType == "" {
obj.ContentType = writer.FormDataContentType()
}
obj.body = tempBody
} else if obj.Files != nil {
tempBody := bytes.NewBuffer(nil)
writer := multipart.NewWriter(tempBody)
escapeQuotes := strings.NewReplacer("\\", "\\\\", `"`, "\\\"")
for _, file := range obj.Files {
h := make(textproto.MIMEHeader)
h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="%s"; filename="%s"`, escapeQuotes.Replace(file.Key), escapeQuotes.Replace(file.Name)))
if file.Type == "" {
h.Set("Content-Type", "application/octet-stream")
} else {
h.Set("Content-Type", file.Type)
}
if wp, err := writer.CreatePart(h); err != nil {
return err
} else if _, err = wp.Write(file.Content); err != nil {
return err
}
}
if err = writer.Close(); err != nil {
return err
}
if obj.ContentType == "" {
obj.ContentType = writer.FormDataContentType()
}
obj.body = tempBody
} else if obj.Data != nil {
if obj.body, err = newBody(obj.Data, dataType, nil); err != nil {
return err
}
if obj.ContentType == "" {
obj.ContentType = "application/x-www-form-urlencoded"
}
} else if obj.Json != nil {
if obj.body, err = newBody(obj.Json, jsonType, nil); err != nil {
return err
}
if obj.ContentType == "" {
obj.ContentType = "application/json"
}
} else if obj.Text != nil {
if obj.body, err = newBody(obj.Text, textType, nil); err != nil {
return err
}
if obj.ContentType == "" {
obj.ContentType = "text/plain"
}
}
return nil
}
func (obj *RequestOption) optionInit() error {
obj.converUrl = obj.Url.String()
var err error
//构造body
if err = obj.initBody(); err != nil {
return err
}
//构造params
if obj.Params != nil {
dataMap := map[string][]string{}
if _, err = newBody(obj.Params, paramsType, dataMap); err != nil {
return err
}
pu := cloneUrl(obj.Url)
puValues := pu.Query()
for kk, vvs := range dataMap {
for _, vv := range vvs {
puValues.Add(kk, vv)
}
}
pu.RawQuery = puValues.Encode()
obj.converUrl = pu.String()
}
//构造headers
if err = obj.initHeaders(); err != nil {
return err
}
//构造cookies
return obj.initCookies()
}
func (obj *Client) newRequestOption(option RequestOption) RequestOption {
if option.TryNum == 0 {
option.TryNum = obj.tryNum
}
if option.OptionCallBack == nil {
option.OptionCallBack = obj.optionCallBack
}
if option.ResultCallBack == nil {
option.ResultCallBack = obj.resultCallBack
}
if option.ErrCallBack == nil {
option.ErrCallBack = obj.errCallBack
}
if option.Headers == nil {
if obj.headers == nil {
option.Headers = DefaultHeaders()
} else {
option.Headers = obj.headers
}
}
if !option.Bar {
option.Bar = obj.bar
}
if option.RedirectNum == 0 {
option.RedirectNum = obj.redirectNum
}
if option.Timeout == 0 {
option.Timeout = obj.timeout
}
if !option.DisCookie {
option.DisCookie = obj.disCookie
}
if !option.DisDecode {
option.DisDecode = obj.disDecode
}
if !option.DisRead {
option.DisRead = obj.disRead
}
if !option.DisUnZip {
option.DisUnZip = obj.disUnZip
}
return option
}

432
requests.go Normal file
View File

@@ -0,0 +1,432 @@
package requests
import (
"bufio"
"bytes"
"context"
"errors"
"fmt"
"net/http"
"net/url"
"os"
"strings"
_ "unsafe"
"gitee.com/baixudong/re"
"gitee.com/baixudong/tools"
"gitee.com/baixudong/websocket"
)
//go:linkname ReadRequest net/http.readRequest
func ReadRequest(b *bufio.Reader) (*http.Request, error)
type RequestDebug struct {
Proto string
Url *url.URL
Method string
Header http.Header
con *bytes.Buffer
}
type ResponseDebug struct {
Proto string
Url *url.URL
Method string
Header http.Header
con *bytes.Buffer
request *http.Request
Status string
}
func (obj *RequestDebug) Request() (*http.Request, error) {
return ReadRequest(bufio.NewReader(bytes.NewBuffer(obj.con.Bytes())))
}
func (obj *RequestDebug) String() string {
return obj.con.String()
}
func (obj *RequestDebug) HeadBuffer() *bytes.Buffer {
con := bytes.NewBuffer(nil)
con.WriteString(fmt.Sprintf("%s %s %s\r\n", obj.Method, obj.Url, obj.Proto))
obj.Header.Write(con)
con.WriteString("\r\n")
return con
}
func cloneRequest(r *http.Request, disBody bool) (*RequestDebug, error) {
request := new(RequestDebug)
request.Proto = r.Proto
request.Method = r.Method
request.Url = r.URL
request.Header = r.Header
var err error
if !disBody {
request.con = bytes.NewBuffer(nil)
if err = r.Write(request.con); err != nil {
return request, err
}
req, err := request.Request()
if err != nil {
return nil, err
}
r.Body = req.Body
} else {
request.con = request.HeadBuffer()
}
return request, err
}
func (obj *ResponseDebug) Response() (*http.Response, error) {
return http.ReadResponse(bufio.NewReader(bytes.NewBuffer(obj.con.Bytes())), obj.request)
}
func (obj *ResponseDebug) String() string {
return obj.con.String()
}
func (obj *ResponseDebug) HeadBuffer() *bytes.Buffer {
con := bytes.NewBuffer(nil)
con.WriteString(fmt.Sprintf("%s %s\r\n", obj.Proto, obj.Status))
obj.Header.Write(con)
con.WriteString("\r\n")
return con
}
func cloneResponse(r *http.Response, disBody bool) (*ResponseDebug, error) {
response := new(ResponseDebug)
response.con = bytes.NewBuffer(nil)
response.Url = r.Request.URL
response.Method = r.Request.Method
response.Proto = r.Proto
response.Status = r.Status
response.Header = r.Header
response.request = r.Request
var err error
if !disBody {
response.con = bytes.NewBuffer(nil)
if err = r.Write(response.con); err != nil {
return response, err
}
rsp, err := response.Response()
if err != nil {
return nil, err
}
r.Body = rsp.Body
} else {
response.con = response.HeadBuffer()
}
return response, err
}
type keyPrincipal string
const keyPrincipalID keyPrincipal = "gospiderContextData"
var (
ErrFatal = errors.New("致命错误")
)
type reqCtxData struct {
isRawConn bool
proxy *url.URL
url *url.URL
host string
redirectNum int
disProxy bool
ws bool
requestCallBack func(context.Context, *RequestDebug) error
disBody bool
responseCallBack func(context.Context, *ResponseDebug) error
}
func Get(preCtx context.Context, href string, options ...RequestOption) (*Response, error) {
client, _ := NewClient(preCtx)
defer client.Close()
return client.Request(preCtx, http.MethodGet, href, options...)
}
func Head(preCtx context.Context, href string, options ...RequestOption) (*Response, error) {
client, _ := NewClient(preCtx)
defer client.Close()
return client.Request(preCtx, http.MethodHead, href, options...)
}
func Post(preCtx context.Context, href string, options ...RequestOption) (*Response, error) {
client, _ := NewClient(preCtx)
defer client.Close()
return client.Request(preCtx, http.MethodPost, href, options...)
}
func Put(preCtx context.Context, href string, options ...RequestOption) (*Response, error) {
client, _ := NewClient(preCtx)
defer client.Close()
return client.Request(preCtx, http.MethodPut, href, options...)
}
func Patch(preCtx context.Context, href string, options ...RequestOption) (*Response, error) {
client, _ := NewClient(preCtx)
defer client.Close()
return client.Request(preCtx, http.MethodPatch, href, options...)
}
func Delete(preCtx context.Context, href string, options ...RequestOption) (*Response, error) {
client, _ := NewClient(preCtx)
defer client.Close()
return client.Request(preCtx, http.MethodDelete, href, options...)
}
func Connect(preCtx context.Context, href string, options ...RequestOption) (*Response, error) {
client, _ := NewClient(preCtx)
defer client.Close()
return client.Request(preCtx, http.MethodConnect, href, options...)
}
func Options(preCtx context.Context, href string, options ...RequestOption) (*Response, error) {
client, _ := NewClient(preCtx)
defer client.Close()
return client.Request(preCtx, http.MethodOptions, href, options...)
}
func Trace(preCtx context.Context, href string, options ...RequestOption) (*Response, error) {
client, _ := NewClient(preCtx)
defer client.Close()
return client.Request(preCtx, http.MethodTrace, href, options...)
}
func Request(preCtx context.Context, method string, href string, options ...RequestOption) (*Response, error) {
client, _ := NewClient(preCtx)
defer client.Close()
return client.Request(preCtx, method, href, options...)
}
func (obj *Client) Get(preCtx context.Context, href string, options ...RequestOption) (*Response, error) {
return obj.Request(preCtx, http.MethodGet, href, options...)
}
func (obj *Client) Head(preCtx context.Context, href string, options ...RequestOption) (*Response, error) {
return obj.Request(preCtx, http.MethodHead, href, options...)
}
func (obj *Client) Post(preCtx context.Context, href string, options ...RequestOption) (*Response, error) {
return obj.Request(preCtx, http.MethodPost, href, options...)
}
func (obj *Client) Put(preCtx context.Context, href string, options ...RequestOption) (*Response, error) {
return obj.Request(preCtx, http.MethodPut, href, options...)
}
func (obj *Client) Patch(preCtx context.Context, href string, options ...RequestOption) (*Response, error) {
return obj.Request(preCtx, http.MethodPatch, href, options...)
}
func (obj *Client) Delete(preCtx context.Context, href string, options ...RequestOption) (*Response, error) {
return obj.Request(preCtx, http.MethodDelete, href, options...)
}
func (obj *Client) Connect(preCtx context.Context, href string, options ...RequestOption) (*Response, error) {
return obj.Request(preCtx, http.MethodConnect, href, options...)
}
func (obj *Client) Options(preCtx context.Context, href string, options ...RequestOption) (*Response, error) {
return obj.Request(preCtx, http.MethodOptions, href, options...)
}
func (obj *Client) Trace(preCtx context.Context, href string, options ...RequestOption) (*Response, error) {
return obj.Request(preCtx, http.MethodTrace, href, options...)
}
// 发送请求
func (obj *Client) Request(preCtx context.Context, method string, href string, options ...RequestOption) (resp *Response, err error) {
if obj == nil {
return nil, errors.New("client is nil")
}
if preCtx == nil {
preCtx = obj.ctx
}
var rawOption RequestOption
if len(options) > 0 {
rawOption = options[0]
}
optionBak := obj.newRequestOption(rawOption)
if rawOption.Body != nil && optionBak.TryNum > 0 {
optionBak.TryNum = 0
}
//开始请求
var tryNum int64
for tryNum = 0; tryNum <= optionBak.TryNum; tryNum++ {
select {
case <-obj.ctx.Done():
obj.Close()
return nil, tools.WrapError(obj.ctx.Err(), "client ctx 错误")
case <-preCtx.Done():
return nil, tools.WrapError(preCtx.Err(), "request ctx 错误")
default:
option := optionBak
if option.Method == "" {
option.Method = method
}
if option.Url == nil {
if option.Url, err = url.Parse(href); err != nil {
err = tools.WrapError(err, "url 解析错误")
return
}
}
if option.OptionCallBack != nil {
if err = option.OptionCallBack(preCtx, obj, &option); err != nil {
return
}
}
resp, err = obj.request(preCtx, option)
if err != nil { //有错误
if errors.Is(err, ErrFatal) { //致命错误直接返回
return
} else if option.ErrCallBack != nil && option.ErrCallBack(preCtx, obj, err) != nil { //不是致命错误，有错误回调,有错误,直接返回
return
}
} else if option.ResultCallBack == nil { //没有错误，且没有回调，直接返回
return
} else if err = option.ResultCallBack(preCtx, obj, resp); err != nil { //没有错误，有回调，回调错误
if option.ErrCallBack != nil && option.ErrCallBack(preCtx, obj, err) != nil { //有错误回调,有错误直接返回
return
}
} else { //没有错误，有回调，没有回调错误，直接返回
return
}
}
}
if err != nil { //有错误直接返回错误
return
}
return resp, errors.New("max try num")
}
func verifyProxy(proxyUrl string) (*url.URL, error) {
proxy, err := url.Parse(proxyUrl)
if err != nil {
return nil, err
}
switch proxy.Scheme {
case "http", "socks5", "https":
return proxy, nil
default:
return nil, tools.WrapError(ErrFatal, "不支持的代理协议")
}
}
func (obj *Client) request(preCtx context.Context, option RequestOption) (response *Response, err error) {
if err = option.optionInit(); err != nil {
err = tools.WrapError(err, "option 初始化错误")
return
}
method := strings.ToUpper(option.Method)
href := option.converUrl
var reqs *http.Request
//构造ctxData
ctxData := new(reqCtxData)
ctxData.requestCallBack = option.RequestCallBack
ctxData.responseCallBack = option.ResponseCallBack
if option.Body != nil {
ctxData.disBody = true
}
ctxData.disProxy = option.DisProxy
if option.Proxy != "" { //代理相关构造
tempProxy, err := verifyProxy(option.Proxy)
if err != nil {
return response, tools.WrapError(ErrFatal, errors.New("tempRequest 构造代理失败"), err)
}
ctxData.proxy = tempProxy
} else if tempProxy := obj.dialer.Proxy(); tempProxy != nil {
ctxData.proxy = tempProxy
}
if option.RedirectNum != 0 { //重定向次数
ctxData.redirectNum = option.RedirectNum
}
//构造ctx,cnl
var cancel context.CancelFunc
var reqCtx context.Context
if option.Timeout > 0 { //超时
reqCtx, cancel = context.WithTimeout(context.WithValue(preCtx, keyPrincipalID, ctxData), option.Timeout)
} else {
reqCtx, cancel = context.WithCancel(context.WithValue(preCtx, keyPrincipalID, ctxData))
}
defer func() {
if err != nil {
cancel()
if response != nil {
response.Close()
}
}
}()
//创建request
if option.body != nil {
reqs, err = http.NewRequestWithContext(reqCtx, method, href, option.body)
} else {
reqs, err = http.NewRequestWithContext(reqCtx, method, href, nil)
}
if err != nil {
return response, tools.WrapError(ErrFatal, errors.New("tempRequest 构造request失败"), err)
}
ctxData.url = reqs.URL
ctxData.host = reqs.Host
if reqs.URL.Scheme == "file" {
filePath := re.Sub(`^/+`, "", reqs.URL.Path)
fileContent, err := os.ReadFile(filePath)
if err != nil {
return response, tools.WrapError(ErrFatal, errors.New("read filePath data error"), err)
}
cancel()
return &Response{
content: fileContent,
filePath: filePath,
}, nil
}
//判断ws
switch reqs.URL.Scheme {
case "ws":
ctxData.ws = true
reqs.URL.Scheme = "http"
case "wss":
ctxData.ws = true
reqs.URL.Scheme = "https"
}
//添加headers
var headOk bool
if reqs.Header, headOk = option.Headers.(http.Header); !headOk {
return response, tools.WrapError(ErrFatal, "request headers 转换错误")
}
if reqs.Header.Get("Content-Type") == "" && reqs.Header.Get("content-type") == "" && option.ContentType != "" {
reqs.Header.Set("Content-Type", option.ContentType)
}
//host构造
if option.Host != "" {
reqs.Host = option.Host
} else if reqs.Header.Get("Host") != "" {
reqs.Host = reqs.Header.Get("Host")
}
//添加cookies
if option.Cookies != nil {
cooks, cookOk := option.Cookies.(Cookies)
if !cookOk {
return response, tools.WrapError(ErrFatal, "request cookies 转换错误")
}
for _, vv := range cooks {
reqs.AddCookie(vv)
}
}
//开始发送请求
var r *http.Response
var err2 error
if ctxData.ws {
websocket.SetClientHeaders(reqs.Header, option.WsOption)
}
r, err = obj.getClient(option).Do(reqs)
if r != nil {
isSse := r.Header.Get("Content-Type") == "text/event-stream"
if ctxData.responseCallBack != nil {
var resp *ResponseDebug
if resp, err = cloneResponse(r, isSse || ctxData.ws); err != nil {
return
}
if err = ctxData.responseCallBack(reqCtx, resp); err != nil {
return response, tools.WrapError(ErrFatal, "request requestCallBack 回调错误", err)
}
}
if ctxData.ws {
if r.StatusCode == 101 {
option.DisRead = true
} else if err == nil {
err = errors.New("statusCode not 101, url为websocket链接，但是对方服务器没有将请求升级到websocket")
}
} else if isSse {
option.DisRead = true
}
if response, err2 = obj.newResponse(reqCtx, cancel, r, option); err2 != nil { //创建 response
return response, err2
}
if ctxData.ws && r.StatusCode == 101 {
if response.webSocket, err2 = websocket.NewClientConn(r); err2 != nil { //创建 websocket
return response, err2
}
}
}
return response, err
}

376
response.go Normal file
View File

@@ -0,0 +1,376 @@
package requests
import (
"bufio"
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"strconv"
"strings"
"gitee.com/baixudong/bar"
"gitee.com/baixudong/bs4"
"gitee.com/baixudong/tools"
"gitee.com/baixudong/websocket"
"github.com/tidwall/gjson"
)
type Response struct {
response *http.Response
webSocket *websocket.Conn
ctx context.Context
cnl context.CancelFunc
content []byte
encoding string
disDecode bool
disUnzip bool
filePath string
bar bool
}
type SseClient struct {
reader *bufio.Reader
}
type Event struct {
Data string
Event string
Id string
Retry int
Comment string
}
func newSseClient(rd io.Reader) *SseClient {
return &SseClient{reader: bufio.NewReader(rd)}
}
func (obj *SseClient) Recv() (Event, error) {
var event Event
for {
readStr, err := obj.reader.ReadString('\n')
if err != nil || readStr == "\n" {
return event, err
}
if strings.HasPrefix(readStr, "data: ") {
event.Data += readStr[6 : len(readStr)-1]
} else if strings.HasPrefix(readStr, "event: ") {
event.Event = readStr[7 : len(readStr)-1]
} else if strings.HasPrefix(readStr, "id: ") {
event.Id = readStr[4 : len(readStr)-1]
} else if strings.HasPrefix(readStr, "retry: ") {
if event.Retry, err = strconv.Atoi(readStr[7 : len(readStr)-1]); err != nil {
return event, err
}
} else if strings.HasPrefix(readStr, ": ") {
event.Comment = readStr[2 : len(readStr)-1]
} else {
return event, errors.New("内容解析错误：" + readStr)
}
}
}
func (obj *Client) newResponse(ctx context.Context, cnl context.CancelFunc, r *http.Response, request_option RequestOption) (*Response, error) {
response := &Response{response: r, ctx: ctx, cnl: cnl, bar: request_option.Bar}
if request_option.DisRead { //是否预读
return response, nil
}
if request_option.DisUnZip || r.Uncompressed { //是否解压
response.disUnzip = true
}
response.disDecode = request_option.DisDecode //是否解码
return response, response.read() //读取内容
}
type Cookies []*http.Cookie
// 返回cookies 的字符串形式
func (obj Cookies) String() string {
cooks := []string{}
for _, cook := range obj {
cooks = append(cooks, fmt.Sprintf("%s=%s", cook.Name, cook.Value))
}
return strings.Join(cooks, "; ")
}
// 获取符合key 条件的所有cookies
func (obj Cookies) Gets(name string) Cookies {
var result Cookies
for _, cook := range obj {
if cook.Name == name {
result = append(result, cook)
}
}
return result
}
// 获取符合key 条件的cookies
func (obj Cookies) Get(name string) *http.Cookie {
vals := obj.Gets(name)
if i := len(vals); i == 0 {
return nil
} else {
return vals[i-1]
}
}
// 获取符合key 条件的所有cookies的值
func (obj Cookies) GetVals(name string) []string {
var result []string
for _, cook := range obj {
if cook.Name == name {
result = append(result, cook.Value)
}
}
return result
}
// 获取符合key 条件的cookies的值
func (obj Cookies) GetVal(name string) string {
vals := obj.GetVals(name)
if i := len(vals); i == 0 {
return ""
} else {
return vals[i-1]
}
}
// 返回原始http.Response
func (obj *Response) Response() *http.Response {
return obj.response
}
// 返回websocket 对象,当发送websocket 请求时使用
func (obj *Response) WebSocket() *websocket.Conn {
return obj.webSocket
}
func (obj *Response) SseClient() *SseClient {
select {
case <-obj.ctx.Done():
return newSseClient(bytes.NewBuffer(obj.Content()))
default:
return newSseClient(obj)
}
}
// 返回当前的Location
func (obj *Response) Location() (*url.URL, error) {
return obj.response.Location()
}
// 返回这个请求的setCookies
func (obj *Response) Cookies() Cookies {
if obj.filePath != "" {
return nil
}
return obj.response.Cookies()
}
// 返回这个请求的状态码
func (obj *Response) StatusCode() int {
if obj.filePath != "" {
return 200
}
return obj.response.StatusCode
}
// 返回这个请求的状态
func (obj *Response) Status() string {
if obj.filePath != "" {
return "200 OK"
}
return obj.response.Status
}
// 返回这个请求的url
func (obj *Response) Url() *url.URL {
if obj.filePath != "" {
return nil
}
return obj.response.Request.URL
}
// 返回response 的请求头
func (obj *Response) Headers() http.Header {
if obj.filePath != "" {
return http.Header{
"Content-Type": []string{obj.ContentType()},
}
}
return obj.response.Header
}
// 对内容进行解码
func (obj *Response) Decode(encoding string) {
if obj.encoding != encoding {
obj.encoding = encoding
obj.SetContent(tools.Decode(obj.Content(), encoding))
}
}
// 尝试将内容解析成map
func (obj *Response) Map() map[string]any {
var data map[string]any
if err := json.Unmarshal(obj.Content(), &data); err != nil {
return nil
}
return data
}
// 尝试将请求解析成gjson, 如果传值将会解析到val中返回的gjson为空struct
func (obj *Response) Json(vals ...any) (gjson.Result, error) {
if len(vals) > 0 {
return gjson.Result{}, tools.JsonUnMarshal(obj.Content(), vals[0])
}
return tools.Any2json(obj.Content())
}
// 返回内容的字符串形式
func (obj *Response) Text() string {
return tools.BytesToString(obj.Content())
}
// 返回内容的二进制，也可设置内容
func (obj *Response) SetContent(val []byte) {
obj.content = val
}
func (obj *Response) Content() []byte {
if obj.webSocket != nil {
return obj.content
}
select {
case <-obj.ctx.Done():
default:
defer obj.Close()
bytesWrite := bytes.NewBuffer(nil)
tools.CopyWitchContext(obj.ctx, bytesWrite, obj.response.Body)
obj.content = bytesWrite.Bytes()
}
return obj.content
}
// 尝试解析成dom 对象
func (obj *Response) Html() *bs4.Client {
return bs4.NewClient(obj.Text(), obj.Url().String())
}
// 获取headers 的Content-Type
func (obj *Response) ContentType() string {
if obj.filePath != "" {
return tools.GetContentTypeWithBytes(obj.content)
}
contentType := obj.response.Header.Get("Content-Type")
if contentType == "" {
contentType = tools.GetContentTypeWithBytes(obj.content)
}
return contentType
}
// 获取headers 的Content-Encoding
func (obj *Response) ContentEncoding() string {
if obj.filePath != "" {
return ""
}
return obj.response.Header.Get("Content-Encoding")
}
// 获取response 的内容长度
func (obj *Response) ContentLength() int64 {
if obj.filePath != "" {
return int64(len(obj.content))
}
if obj.response.ContentLength >= 0 {
return obj.response.ContentLength
}
return int64(len(obj.content))
}
type barBody struct {
body *bytes.Buffer
bar *bar.Client
}
func (obj *barBody) Write(con []byte) (int, error) {
l, err := obj.body.Write(con)
obj.bar.Print(int64(l))
return l, err
}
func (obj *Response) barRead() (*bytes.Buffer, error) {
barData := &barBody{
bar: bar.NewClient(obj.response.ContentLength),
body: bytes.NewBuffer(nil),
}
err := tools.CopyWitchContext(obj.response.Request.Context(), barData, obj.response.Body)
if err != nil {
return nil, err
}
return barData.body, nil
}
func (obj *Response) defaultDecode() bool {
return strings.Contains(obj.ContentType(), "html")
}
func (obj *Response) Read(con []byte) (i int, err error) { //读取body
done := make(chan struct{})
go func() {
defer close(done)
defer func() {
if recErr := recover(); recErr != nil && err == nil {
err, _ = recErr.(error)
}
}()
i, err = obj.response.Body.Read(con)
}()
select {
case <-obj.ctx.Done():
obj.response.Body.Close()
return 0, obj.ctx.Err()
case <-done:
return
}
}
func (obj *Response) read() error { //读取body,对body 解压，解码操作
defer obj.Close()
var bBody *bytes.Buffer
var err error
if obj.bar && obj.ContentLength() > 0 { //是否打印进度条,读取内容
bBody, err = obj.barRead()
} else {
bBody = bytes.NewBuffer(nil)
err = tools.CopyWitchContext(obj.response.Request.Context(), bBody, obj.response.Body)
}
if err != nil {
return errors.New("response 读取内容 错误: " + err.Error())
}
if !obj.disUnzip {
if bBody, err = tools.CompressionDecode(obj.ctx, bBody, obj.ContentEncoding()); err != nil {
return errors.New("response 解压缩错误: " + err.Error())
}
}
if !obj.disDecode && obj.defaultDecode() {
if content, encoding, err := tools.Charset(bBody.Bytes(), obj.ContentType()); err == nil {
obj.content, obj.encoding = content, encoding
} else {
obj.content = bBody.Bytes()
}
} else {
obj.content = bBody.Bytes()
}
return nil
}
// 关闭response ,当disRead 为true 请一定要手动关闭
func (obj *Response) Close() error {
defer obj.cnl()
if obj.webSocket != nil {
obj.webSocket.Close("close")
}
if obj.response != nil && obj.response.Body != nil {
tools.CopyWitchContext(obj.ctx, io.Discard, obj.response.Body)
return obj.response.Body.Close()
}
return nil
}