package/requests
1
0
Fork 0
mirror of https://github.com/gospider007/requests.git synced 2025-12-24 13:57:52 +08:00
Code Issues Packages Projects Releases Wiki Activity

更新依赖

Browse Source
This commit is contained in:
bxd
2023-09-05 10:30:58 +08:00
parent 375f84b1df
commit 3d409e7fa0
6 changed files with 70 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
client.go
View File

@@ -98,7 +98,7 @@ func NewClient(preCtx context.Context, options ...ClientOption) (*Client, error)
option.DialTimeout = time.Second * 15
}
if option.TlsHandshakeTimeout == 0 {
option.TlsHandshakeTimeout = time.Second * 15
option.TlsHandshakeTimeout = time.Second * 5
}
//创建cookiesjar
var jar *cookiejar.Jar

12
go.mod
View File

@@ -5,12 +5,12 @@ go 1.21.0
require (
gitee.com/baixudong/bar v0.0.0-20230814072341-e111c30a1386
gitee.com/baixudong/bs4 v0.0.0-20230814072447-945ab1fd97ed
gitee.com/baixudong/ja3 v0.0.0-20230831080840-7a02183909a0
gitee.com/baixudong/net v0.0.0-20230821075807-0a284fde7633
gitee.com/baixudong/ja3 v0.0.0-20230905022841-a30a5742bdb7
gitee.com/baixudong/net v0.0.0-20230905022950-d3a73f7bf4cc
gitee.com/baixudong/re v0.0.0-20230809033040-360c1d945e59
gitee.com/baixudong/tools v0.0.0-20230821075140-7422c51fd918
gitee.com/baixudong/websocket v0.0.0-20230821075043-d36a43fd76a3
github.com/refraction-networking/utls v1.5.2
github.com/refraction-networking/utls v1.5.3
github.com/tidwall/gjson v1.16.0
)
@@ -42,9 +42,9 @@ require (
go.uber.org/zap v1.25.0 // indirect
golang.org/x/crypto v0.12.0 // indirect
golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 // indirect
golang.org/x/image v0.11.0 // indirect
golang.org/x/image v0.12.0 // indirect
golang.org/x/net v0.14.0 // indirect
golang.org/x/sys v0.11.0 // indirect
golang.org/x/text v0.12.0 // indirect
golang.org/x/sys v0.12.0 // indirect
golang.org/x/text v0.13.0 // indirect
nhooyr.io/websocket v1.8.7 // indirect
)

12
go.sum
View File

@@ -12,10 +12,14 @@ gitee.com/baixudong/ja3 v0.0.0-20230829015831-02d9ec0c9f91 h1:0I5G0ifMf0FrI8dEcK
gitee.com/baixudong/ja3 v0.0.0-20230829015831-02d9ec0c9f91/go.mod h1:0/mVl1zo674mlVEMKpWGsDtN11MEMkg/e2JbhLZ+F6U=
gitee.com/baixudong/ja3 v0.0.0-20230831080840-7a02183909a0 h1:8x9h3K+fFPbL+EwjQ3AdQX9NrYezB1ZIJUfyBUdwZcY=
gitee.com/baixudong/ja3 v0.0.0-20230831080840-7a02183909a0/go.mod h1:0/mVl1zo674mlVEMKpWGsDtN11MEMkg/e2JbhLZ+F6U=
gitee.com/baixudong/ja3 v0.0.0-20230905022841-a30a5742bdb7 h1:Tz7PmFK9l7aLQ7y3zARNIQ0bo8slPpGjCzxLVnNO2Hk=
gitee.com/baixudong/ja3 v0.0.0-20230905022841-a30a5742bdb7/go.mod h1:jebK1zBD90X4lMLQm8sBU0rdtLEcd6DLbjG+Xqj46kI=
gitee.com/baixudong/kinds v0.0.0-20230809033013-c3d6d3479f8c h1:MZewpjU0+82TcA+nHrcAbbgZhTCqVH2I5zwYBWJ4v54=
gitee.com/baixudong/kinds v0.0.0-20230809033013-c3d6d3479f8c/go.mod h1:J3r4FWnZOMgk7/1EWEEDaPQnx5uH4OLCRzpTOyKg5cA=
gitee.com/baixudong/net v0.0.0-20230821075807-0a284fde7633 h1:GB/Bzofv5A5Mygezvz/kcXNfdog9llcqg4bKwjg2HF8=
gitee.com/baixudong/net v0.0.0-20230821075807-0a284fde7633/go.mod h1:uEqn2cfzNchgA61Tdf10Pih19y6vYldtyWvlPe1krGk=
gitee.com/baixudong/net v0.0.0-20230905022950-d3a73f7bf4cc h1:Nsjpx5ivraXr9Vcg6XuuRq015RyXXD30CAaDOVmHfow=
gitee.com/baixudong/net v0.0.0-20230905022950-d3a73f7bf4cc/go.mod h1:IJKFl20rRznKF8ymot1B8E4B0miz7MZISyBca2fgnYM=
gitee.com/baixudong/re v0.0.0-20230809033040-360c1d945e59 h1:HtdoUzBIqepeVTeeZ0QcPKoQSDqgWQn/fs84fsByJv0=
gitee.com/baixudong/re v0.0.0-20230809033040-360c1d945e59/go.mod h1:i5L3z/q42lHZvd3JSk3390XdujEY2QHXFMEcaO/uZD0=
gitee.com/baixudong/tools v0.0.0-20230821075140-7422c51fd918 h1:2X5AmuHDIPCZ1HGnBOxJoBMpFMi9rcWi7uzNtKP23jg=
@@ -124,6 +128,8 @@ github.com/refraction-networking/utls v1.5.0 h1:Ucbojt9Wc4yFc3oyQyCmmuvPUFmTdb3A
github.com/refraction-networking/utls v1.5.0/go.mod h1:SPuDbBmgLGp8s+HLNc83FuavwZCFoMmExj+ltUHiHUw=
github.com/refraction-networking/utls v1.5.2 h1:l6diiLbEoRqdQ+/osPDO0z0lTc8O8VZV+p82N+Hi+ws=
github.com/refraction-networking/utls v1.5.2/go.mod h1:SPuDbBmgLGp8s+HLNc83FuavwZCFoMmExj+ltUHiHUw=
github.com/refraction-networking/utls v1.5.3 h1:Ds5Ocg1+MC1ahNx5iBEcHe0jHeLaA/fLey61EENm7ro=
github.com/refraction-networking/utls v1.5.3/go.mod h1:SPuDbBmgLGp8s+HLNc83FuavwZCFoMmExj+ltUHiHUw=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
@@ -165,6 +171,8 @@ golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 h1:m64FZMko/V45gv0bNmrNYoDEq
golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63/go.mod h1:0v4NqG35kSWCMzLaMeX+IQrlSnVE/bqGSyC2cz/9Le8=
golang.org/x/image v0.11.0 h1:ds2RoQvBvYTiJkwpSFDwCcDFNX7DqjL2WsUgTNk0Ooo=
golang.org/x/image v0.11.0/go.mod h1:bglhjqbqVuEb9e9+eNR45Jfu7D+T4Qan+NhQk8Ck2P8=
golang.org/x/image v0.12.0 h1:w13vZbU4o5rKOFFR8y7M+c4A5jXDC0uXTdHYRP8X2DQ=
golang.org/x/image v0.12.0/go.mod h1:Lu90jvHG7GfemOIcldsh9A2hS01ocl6oNO7ype5mEnk=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -191,6 +199,8 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM=
golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o=
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
@@ -204,6 +214,8 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc=
golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=

1
option.go
View File

@@ -22,6 +22,7 @@ type RequestOption struct {
Ja3Spec ja3.Ja3Spec //指定ja3Spec,使用ja3.CreateSpecWithStr 或者ja3.CreateSpecWithId 生成
H2Ja3Spec ja3.H2Ja3Spec //h2指纹
Referer string
Method string //method
Url *url.URL //请求的url
Host string //网站的host

19
requests.go
View File

@@ -320,16 +320,12 @@ func (obj *Client) Request(preCtx context.Context, method string, href string, o
func (obj *Client) request(preCtx context.Context, option RequestOption) (response *Response, err error) {
response = new(Response)
defer func() {
if err == nil { //判断是否读取body,和对body的处理
if response.webSocket == nil && response.sseClient == nil && !option.DisRead {
err = response.ReadBody()
defer response.Close()
}
if err == nil && response.webSocket == nil && response.sseClient == nil && !option.DisRead { //判断是否读取body,和对body的处理
err = response.ReadBody()
defer response.Close()
}
if err == nil { //result 回调处理
if option.ResultCallBack != nil {
err = option.ResultCallBack(preCtx, obj, response)
}
if err == nil && option.ResultCallBack != nil { //result 回调处理
err = option.ResultCallBack(preCtx, obj, response)
}
if err != nil { //err 回调处理
response.Close()
@@ -418,6 +414,11 @@ func (obj *Client) request(preCtx context.Context, option RequestOption) (respon
if reqs.Header, headOk = option.Headers.(http.Header); !headOk {
return response, tools.WrapError(ErrFatal, "request headers 转换错误")
}
//添加Referer
if option.Referer != "" && reqs.Header.Get("Referer") == "" {
reqs.Header.Set("Referer", option.Referer)
}
//设置 ContentType 值
if reqs.Header.Get("Content-Type") == "" && reqs.Header.Get("content-type") == "" && option.ContentType != "" {
reqs.Header.Set("Content-Type", option.ContentType)

72
roundTripper.go
View File

@@ -79,7 +79,7 @@ func getHost(req *http.Request) string {
func getKey(ctxData *reqCtxData, req *http.Request) string {
return fmt.Sprintf("%s@%s", getAddr(ctxData.proxy), getAddr(req.URL))
}
func (obj *RoundTripper) newConnPool(key string, conn *Connecotr) *connPool {
func (obj *RoundTripper) newConnPool(key string, conn *Connecotr) *connPool { //新建连接池
pool := new(connPool)
pool.ctx, pool.cnl = context.WithCancel(obj.ctx)
pool.total.Add(1)
@@ -89,12 +89,6 @@ func (obj *RoundTripper) newConnPool(key string, conn *Connecotr) *connPool {
go pool.rwMain(conn)
return pool
}
func (obj *RoundTripper) delConnPool(key string, pool *connPool) {
obj.connsLock.Lock()
defer obj.connsLock.Unlock()
pool.Close()
delete(obj.connPools, key)
}
func (obj *RoundTripper) getConnPool(key string) *connPool {
obj.connsLock.Lock()
defer obj.connsLock.Unlock()
@@ -109,6 +103,7 @@ func (obj *RoundTripper) putConnPool(key string, conn *Connecotr) {
}
pool, ok := obj.connPools[key]
if ok {
pool.total.Add(1)
go pool.rwMain(conn)
} else {
obj.connPools[key] = obj.newConnPool(key, conn)
@@ -188,6 +183,7 @@ func (obj *RoundTripper) dial(ctxData *reqCtxData, addr string, key string, req
}
type Connecotr struct {
err error
ctx context.Context
ctx2 context.Context
cnl context.CancelFunc
@@ -202,7 +198,7 @@ type Connecotr struct {
}
func (obj *Connecotr) Close() error {
obj.cnl()
defer obj.cnl()
if obj.h2RawConn != nil {
obj.h2RawConn.Close()
}
@@ -212,39 +208,50 @@ func (obj *Connecotr) read() {
defer obj.Close()
obj.isRead = true
con := make([]byte, 1024)
var i int
for {
i, err := obj.rawConn.Read(con)
if err != nil {
if i, obj.err = obj.rawConn.Read(con); obj.err != nil && i == 0 {
return
}
b := con[:i]
for once := true; once || len(b) > 0; once = false {
select {
case obj.rc <- b:
nw := <-obj.rn
b = b[nw:]
select {
case nw := <-obj.rn:
b = b[nw:]
case <-obj.ctx.Done():
return
}
case <-obj.ctx.Done():
return
}
}
}
}
func (obj *Connecotr) Read(b []byte) (int, error) {
func (obj *Connecotr) Read(b []byte) (i int, err error) {
if !obj.isRead {
return obj.rawConn.Read(b)
}
select {
case con := <-obj.rc:
i := copy(b, con)
i, err = copy(b, con), obj.err
select {
case obj.rn <- i:
return i, nil
if i < len(con) {
err = nil
}
case <-obj.ctx.Done():
return i, obj.ctx.Err()
if err = obj.err; err == nil {
err = tools.WrapError(obj.ctx.Err(), "connecotr close")
}
}
case <-obj.ctx.Done():
return 0, obj.ctx.Err()
if err = obj.err; err == nil {
err = tools.WrapError(obj.ctx.Err(), "connecotr close")
}
}
return
}
func (obj *Connecotr) Write(b []byte) (int, error) {
return obj.rawConn.Write(b)
@@ -271,7 +278,6 @@ func (obj *Connecotr) h2Closed() bool {
}
type ReadWriteCloser struct {
ctx context.Context
cnl context.CancelFunc
cnl2 context.CancelFunc
body io.ReadCloser
@@ -301,7 +307,7 @@ func (obj *ReadWriteCloser) Delete() (err error) {
func wrapBody(conn *Connecotr, task *reqTask) {
body := new(ReadWriteCloser)
conn.ctx2, body.cnl = context.WithCancel(conn.ctx)
body.cnl2 = conn.cnl
body.cnl2 = conn.cnl //使body 具备关闭连接的能力
body.body = task.res.Body
body.conn = conn
task.res.Body = body
@@ -311,6 +317,8 @@ func http1Req(conn *Connecotr, task *reqTask) {
defer func() {
if task.res == nil || task.err != nil {
conn.Close()
} else if task.res != nil {
wrapBody(conn, task)
}
}()
err := task.req.Write(conn)
@@ -318,9 +326,6 @@ func http1Req(conn *Connecotr, task *reqTask) {
task.err = err
} else {
task.res, task.err = http.ReadResponse(conn.r, task.req)
if task.res != nil {
wrapBody(conn, task)
}
}
}
@@ -329,24 +334,24 @@ func http2Req(conn *Connecotr, task *reqTask) {
defer func() {
if task.res == nil || task.err != nil {
conn.Close()
} else if task.res != nil {
wrapBody(conn, task)
}
}()
task.res, task.err = conn.h2RawConn.RoundTrip(task.req)
if task.res != nil {
wrapBody(conn, task)
}
}
func (obj *connPool) rwMain(conn *Connecotr) {
conn.ctx, conn.cnl = context.WithCancel(obj.ctx)
defer func() {
if obj.total.Load() == 0 {
obj.rt.delConnPool(obj.key, obj)
obj.rt.connsLock.Lock()
defer obj.rt.connsLock.Unlock()
conn.Close()
obj.total.Add(-1)
if obj.total.Load() <= 0 {
obj.Close()
delete(obj.rt.connPools, obj.key)
}
}()
defer obj.total.Add(-1)
defer conn.Close()
defer conn.cnl()
for {
select {
case <-conn.ctx.Done(): //连接池通知关闭，等待连接被释放掉
@@ -485,8 +490,11 @@ func (obj *RoundTripper) poolRoundTrip(task *reqTask, key string) (bool, error)
}
func (obj *RoundTripper) CloseIdleConnections() {
obj.connsLock.Lock()
defer obj.connsLock.Unlock()
for key, pool := range obj.connPools {
obj.delConnPool(key, pool)
pool.Close()
delete(obj.connPools, key)
}
}
func (obj *RoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {