mirror of
https://github.com/gospider007/requests.git
synced 2025-12-24 13:57:52 +08:00
sync
This commit is contained in:
9
conn.go
9
conn.go
@@ -11,14 +11,11 @@ import (
|
||||
"github.com/gospider007/tools"
|
||||
)
|
||||
|
||||
var maxRetryCount = 5
|
||||
|
||||
func taskMain(conn http1.Conn, task *reqTask) (err error) {
|
||||
defer func() {
|
||||
if err != nil && task.reqCtx.option.ErrCallBack != nil {
|
||||
task.reqCtx.err = err
|
||||
if err2 := task.reqCtx.option.ErrCallBack(task.reqCtx); err2 != nil {
|
||||
task.isNotice = false
|
||||
task.disRetry = true
|
||||
err = err2
|
||||
}
|
||||
@@ -50,8 +47,6 @@ func taskMain(conn http1.Conn, task *reqTask) (err error) {
|
||||
select {
|
||||
case <-conn.Context().Done(): //force conn close
|
||||
err = context.Cause(conn.Context())
|
||||
task.enableRetry = true
|
||||
task.isNotice = true
|
||||
return
|
||||
default:
|
||||
}
|
||||
@@ -65,7 +60,7 @@ func taskMain(conn http1.Conn, task *reqTask) (err error) {
|
||||
if task.reqCtx.option.ResponseHeaderTimeout > 0 {
|
||||
select {
|
||||
case <-conn.Context().Done(): //force conn close
|
||||
err = tools.WrapError(context.Cause(conn.Context()), "taskMain delete ctx error: ")
|
||||
err = tools.WrapError(context.Cause(conn.Context()), "taskMain delete ctx error")
|
||||
case <-time.After(task.reqCtx.option.ResponseHeaderTimeout):
|
||||
err = errors.New("ResponseHeaderTimeout error: ")
|
||||
case <-task.ctx.Done():
|
||||
@@ -75,7 +70,7 @@ func taskMain(conn http1.Conn, task *reqTask) (err error) {
|
||||
} else {
|
||||
select {
|
||||
case <-conn.Context().Done(): //force conn close
|
||||
err = tools.WrapError(context.Cause(conn.Context()), "taskMain delete ctx error: ")
|
||||
err = tools.WrapError(context.Cause(conn.Context()), "taskMain delete ctx error")
|
||||
case <-task.ctx.Done():
|
||||
err = context.Cause(task.ctx)
|
||||
case <-done:
|
||||
|
||||
4
go.mod
4
go.mod
@@ -9,8 +9,8 @@ require (
|
||||
github.com/gospider007/bs4 v0.0.0-20250413121342-fed910fb00c9
|
||||
github.com/gospider007/gson v0.0.0-20250718004537-ff15820964bd
|
||||
github.com/gospider007/gtls v0.0.0-20250718003831-90cdeb97a23f
|
||||
github.com/gospider007/http1 v0.0.0-20250729013541-5b61c808855d
|
||||
github.com/gospider007/http2 v0.0.0-20250729013742-fe541491a17b
|
||||
github.com/gospider007/http1 v0.0.0-20250729080302-0419175456ec
|
||||
github.com/gospider007/http2 v0.0.0-20250729081659-a0f2ab33dc98
|
||||
github.com/gospider007/http3 v0.0.0-20250729013804-e1c3cbcf998f
|
||||
github.com/gospider007/ja3 v0.0.0-20250724085013-aa7e0a527d6c
|
||||
github.com/gospider007/re v0.0.0-20250217075352-bcb79f285d6c
|
||||
|
||||
8
go.sum
8
go.sum
@@ -114,10 +114,10 @@ github.com/gospider007/gson v0.0.0-20250718004537-ff15820964bd h1:aby4HnAGVJt5pI
|
||||
github.com/gospider007/gson v0.0.0-20250718004537-ff15820964bd/go.mod h1:GxCATDh+u/TLHTAI9p1kXfaGUkVNjHuY+Mhxdo4l5k8=
|
||||
github.com/gospider007/gtls v0.0.0-20250718003831-90cdeb97a23f h1:W/ug9EHRcduL40RNtsKR9Ob0VnMQF/pps6iLTmorhck=
|
||||
github.com/gospider007/gtls v0.0.0-20250718003831-90cdeb97a23f/go.mod h1:iTnRK0DU3YH7MlZZ9VdT5gQAFPKzHRtFi3EXpnosOAM=
|
||||
github.com/gospider007/http1 v0.0.0-20250729013541-5b61c808855d h1:X/52LHwn9Be8boPdrinxYtWubANz4Mr6wYly0GKXdZY=
|
||||
github.com/gospider007/http1 v0.0.0-20250729013541-5b61c808855d/go.mod h1:UwjRyGpzSwrWgNU/QVEX4VUSzo3oWTV0Oxr1nLxxAI0=
|
||||
github.com/gospider007/http2 v0.0.0-20250729013742-fe541491a17b h1:hN9Ubw1AQFu/UpTz4hXebdOcm5YkzDw2lwzVxAr29So=
|
||||
github.com/gospider007/http2 v0.0.0-20250729013742-fe541491a17b/go.mod h1:peh0PrDWg5e8mlpvsGmKx96qqXEdHukjgFY9fmpNlyI=
|
||||
github.com/gospider007/http1 v0.0.0-20250729080302-0419175456ec h1:+4KGSfU0/w5EaSi97l5HvQBW1sezxlbKK/Z0wjEf/IE=
|
||||
github.com/gospider007/http1 v0.0.0-20250729080302-0419175456ec/go.mod h1:UwjRyGpzSwrWgNU/QVEX4VUSzo3oWTV0Oxr1nLxxAI0=
|
||||
github.com/gospider007/http2 v0.0.0-20250729081659-a0f2ab33dc98 h1:jmCxprkgCDHf6vSMz8y2kKXkW4Dl6eSEY6yA3Ojia+o=
|
||||
github.com/gospider007/http2 v0.0.0-20250729081659-a0f2ab33dc98/go.mod h1:9l+aUQCtn/1AFEPZ5TQsei6g5GxR8PrrZG5xIOQNpzU=
|
||||
github.com/gospider007/http3 v0.0.0-20250729013804-e1c3cbcf998f h1:SpBKsy/k5JEHNvurZx7Gj8SvbMrbtzXGIZNt9i3mvHk=
|
||||
github.com/gospider007/http3 v0.0.0-20250729013804-e1c3cbcf998f/go.mod h1:i9nANnbPcIhoui751Ds2ptft3UGgfsL4T0UKCAT3bfk=
|
||||
github.com/gospider007/ja3 v0.0.0-20250724085013-aa7e0a527d6c h1:Ksw8S50iGK0dqYmScPRn/oOuKegHJ2j9DupSURZZmoI=
|
||||
|
||||
86
option.go
86
option.go
@@ -48,19 +48,22 @@ type ClientOption struct {
|
||||
UtlsConfig *utls.Config
|
||||
QuicConfig *quic.Config
|
||||
UquicConfig *uquic.Config
|
||||
USpec any //support ja3.USpec,uquic.QUICID,bool
|
||||
UserAgent string //headers User-Agent value
|
||||
Proxy any //strong or []string, ,support https,http,socks5
|
||||
MaxRetries int //try num
|
||||
MaxRedirect int //redirect num ,<0 no redirect,==0 no limit
|
||||
Timeout time.Duration //request timeout
|
||||
ResponseHeaderTimeout time.Duration //ResponseHeaderTimeout
|
||||
TlsHandshakeTimeout time.Duration //tls timeout
|
||||
ForceHttp3 bool //force use http3 send requests
|
||||
ForceHttp1 bool //force use http1 send requests
|
||||
DisCookie bool //disable cookies
|
||||
DisDecode bool //disable auto decode
|
||||
Bar bool ////enable bar display
|
||||
USpec any //support ja3.USpec,uquic.QUICID,bool
|
||||
UserAgent string //headers User-Agent value
|
||||
Proxy any //strong or []string, ,support https,http,socks5
|
||||
MaxRetries int //try num
|
||||
MaxRedirect int //redirect num ,<0 no redirect,==0 no limit
|
||||
Timeout time.Duration //request timeout
|
||||
ResponseHeaderTimeout time.Duration //ResponseHeaderTimeout
|
||||
TlsHandshakeTimeout time.Duration //tls timeout
|
||||
ForceHttp3 bool //force use http3 send requests
|
||||
ForceHttp1 bool //force use http1 send requests
|
||||
DisCookie bool //disable cookies
|
||||
DisDecode bool //disable auto decode
|
||||
Bar bool ////enable bar display
|
||||
Headers any //default headers
|
||||
WsOption *websocket.Option //websocket option
|
||||
Stream bool //disable auto read
|
||||
}
|
||||
|
||||
// Options for sending requests
|
||||
@@ -79,37 +82,37 @@ type RequestOption struct {
|
||||
UtlsConfig *utls.Config
|
||||
QuicConfig *quic.Config
|
||||
UquicConfig *uquic.Config
|
||||
USpec any //support ja3.USpec,uquic.QUICID,bool
|
||||
UserAgent string //headers User-Agent value
|
||||
Proxy any //strong or []string, ,support https,http,socks5
|
||||
MaxRetries int //try num
|
||||
MaxRedirect int //redirect num ,<0 no redirect,==0 no limit
|
||||
Timeout time.Duration //request timeout
|
||||
ResponseHeaderTimeout time.Duration //ResponseHeaderTimeout
|
||||
TlsHandshakeTimeout time.Duration //tls timeout
|
||||
ForceHttp3 bool //force use http3 send requests
|
||||
ForceHttp1 bool //force use http1 send requests
|
||||
DisCookie bool //disable cookies
|
||||
DisDecode bool //disable auto decode
|
||||
Bar bool ////enable bar display
|
||||
USpec any //support ja3.USpec,uquic.QUICID,bool
|
||||
UserAgent string //headers User-Agent value
|
||||
Proxy any //strong or []string, ,support https,http,socks5
|
||||
MaxRetries int //try num
|
||||
MaxRedirect int //redirect num ,<0 no redirect,==0 no limit
|
||||
Timeout time.Duration //request timeout
|
||||
ResponseHeaderTimeout time.Duration //ResponseHeaderTimeout
|
||||
TlsHandshakeTimeout time.Duration //tls timeout
|
||||
ForceHttp3 bool //force use http3 send requests
|
||||
ForceHttp1 bool //force use http1 send requests
|
||||
DisCookie bool //disable cookies
|
||||
DisDecode bool //disable auto decode
|
||||
Bar bool ////enable bar display
|
||||
Headers any //default headers
|
||||
WsOption *websocket.Option //websocket option
|
||||
Stream bool //disable auto read
|
||||
//分割线
|
||||
//以下为请求参数
|
||||
Headers any //default headers
|
||||
WsOption websocket.Option //websocket option
|
||||
Cookies any // cookies,support :json,map,str,http.Header
|
||||
Params any //url params,join url query,json,map
|
||||
Json any //send application/json,support io.Reader,:string,[]bytes,json,map
|
||||
Data any //send application/x-www-form-urlencoded, support io.Reader, string,[]bytes,json,map
|
||||
Form any //send multipart/form-data,file upload,support io.Reader, json,map
|
||||
Text any //send text/xml,support: io.Reader, string,[]bytes,json,map
|
||||
Body any //not setting context-type,support io.Reader, string,[]bytes,json,map
|
||||
Url *url.URL
|
||||
Cookies any // cookies,support :json,map,str,http.Header
|
||||
Params any //url params,join url query,json,map
|
||||
Json any //send application/json,support io.Reader,:string,[]bytes,json,map
|
||||
Data any //send application/x-www-form-urlencoded, support io.Reader, string,[]bytes,json,map
|
||||
Form any //send multipart/form-data,file upload,support io.Reader, json,map
|
||||
Text any //send text/xml,support: io.Reader, string,[]bytes,json,map
|
||||
Body any //not setting context-type,support io.Reader, string,[]bytes,json,map
|
||||
Url *url.URL
|
||||
// other option
|
||||
Method string //method
|
||||
Host string
|
||||
Referer string //set headers referer value
|
||||
ContentType string //headers Content-Type value
|
||||
Stream bool //disable auto read
|
||||
DisProxy bool //force disable proxy
|
||||
|
||||
readOne bool
|
||||
@@ -122,6 +125,12 @@ func merge(option *RequestOption, clientOption *ClientOption) {
|
||||
if option.Spec == "" {
|
||||
option.Spec = clientOption.Spec
|
||||
}
|
||||
if option.WsOption == nil {
|
||||
option.WsOption = clientOption.WsOption
|
||||
}
|
||||
if option.Headers == nil {
|
||||
option.Headers = clientOption.Headers
|
||||
}
|
||||
if option.OrderHeaders == nil {
|
||||
option.OrderHeaders = clientOption.OrderHeaders
|
||||
}
|
||||
@@ -200,6 +209,9 @@ func merge(option *RequestOption, clientOption *ClientOption) {
|
||||
if !option.Bar {
|
||||
option.Bar = clientOption.Bar
|
||||
}
|
||||
if !option.Stream {
|
||||
option.Stream = clientOption.Stream
|
||||
}
|
||||
}
|
||||
|
||||
func (obj *Client) newRequestOption(option RequestOption) (RequestOption, error) {
|
||||
|
||||
@@ -22,22 +22,17 @@ import (
|
||||
)
|
||||
|
||||
type reqTask struct {
|
||||
ctx context.Context
|
||||
cnl context.CancelCauseFunc
|
||||
reqCtx *Response
|
||||
enableRetry bool
|
||||
disRetry bool
|
||||
isNotice bool
|
||||
key string
|
||||
ctx context.Context
|
||||
cnl context.CancelCauseFunc
|
||||
reqCtx *Response
|
||||
disRetry bool
|
||||
key string
|
||||
}
|
||||
|
||||
func (obj *reqTask) suppertRetry() bool {
|
||||
if obj.disRetry {
|
||||
return false
|
||||
}
|
||||
if obj.enableRetry {
|
||||
return true
|
||||
}
|
||||
if obj.reqCtx.request.Body == nil {
|
||||
return true
|
||||
} else if body, ok := obj.reqCtx.request.Body.(io.Seeker); ok {
|
||||
@@ -348,15 +343,6 @@ func (obj *roundTripper) waitTask(task *reqTask) error {
|
||||
}
|
||||
return err
|
||||
}
|
||||
func (obj *roundTripper) poolRoundTrip(task *reqTask) error {
|
||||
task.ctx, task.cnl = context.WithCancelCause(task.reqCtx.Context())
|
||||
select {
|
||||
case obj.getConnPool(task) <- task:
|
||||
return obj.waitTask(task)
|
||||
default:
|
||||
return obj.newRoundTrip(task)
|
||||
}
|
||||
}
|
||||
|
||||
func (obj *roundTripper) newRoundTrip(task *reqTask) error {
|
||||
task.reqCtx.isNewConn = true
|
||||
@@ -372,7 +358,6 @@ func (obj *roundTripper) newRoundTrip(task *reqTask) error {
|
||||
err = err2
|
||||
}
|
||||
}
|
||||
task.enableRetry = true
|
||||
}
|
||||
if err == nil {
|
||||
go rwMain(conn, task, obj.getConnPool(task))
|
||||
@@ -405,9 +390,8 @@ func (obj *roundTripper) RoundTrip(ctx *Response) (err error) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
currentRetry := 0
|
||||
var task *reqTask
|
||||
for ; currentRetry <= maxRetryCount; currentRetry++ {
|
||||
for send := true; send; {
|
||||
select {
|
||||
case <-ctx.Context().Done():
|
||||
return context.Cause(ctx.Context())
|
||||
@@ -417,13 +401,18 @@ func (obj *roundTripper) RoundTrip(ctx *Response) (err error) {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = obj.poolRoundTrip(task)
|
||||
if err == nil || !task.suppertRetry() {
|
||||
task.ctx, task.cnl = context.WithCancelCause(task.reqCtx.Context())
|
||||
select {
|
||||
case obj.getConnPool(task) <- task:
|
||||
err = obj.waitTask(task)
|
||||
default:
|
||||
err = obj.newRoundTrip(task)
|
||||
task.disRetry = true
|
||||
}
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
if task.isNotice {
|
||||
currentRetry--
|
||||
}
|
||||
send = task.suppertRetry()
|
||||
}
|
||||
if err == nil && ctx.option.RequestCallBack != nil {
|
||||
if err2 := ctx.option.RequestCallBack(ctx); err2 != nil {
|
||||
|
||||
@@ -12,7 +12,7 @@ func TestLocalAddr(t *testing.T) {
|
||||
|
||||
DialOption: &requests.DialOption{
|
||||
LocalAddr: &net.TCPAddr{ //set dns server
|
||||
IP: net.ParseIP("192.168.1.239"),
|
||||
// IP: net.ParseIP("192.168.1.239"),
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user