mirror of
https://github.com/gospider007/requests.git
synced 2025-12-24 13:57:52 +08:00
sync
This commit is contained in:
24
conn.go
24
conn.go
@@ -73,6 +73,7 @@ func (obj *conn) DoRequest(req *http.Request, orderHeaders []string) (*http.Resp
|
||||
}
|
||||
func (obj *conn) run() (err error) {
|
||||
_, err = io.Copy(obj.pw, obj.conn)
|
||||
|
||||
return obj.CloseWithError(err)
|
||||
}
|
||||
func (obj *conn) Read(b []byte) (i int, err error) {
|
||||
@@ -107,9 +108,9 @@ type connecotr struct {
|
||||
bodyCtx context.Context //body close
|
||||
bodyCnl context.CancelCauseFunc
|
||||
|
||||
rawConn Conn
|
||||
proxys []*url.URL
|
||||
// inPool bool
|
||||
Conn Conn
|
||||
c net.Conn
|
||||
proxys []*url.URL
|
||||
}
|
||||
|
||||
func (obj *connecotr) withCancel(forceCtx context.Context, safeCtx context.Context) {
|
||||
@@ -118,7 +119,14 @@ func (obj *connecotr) withCancel(forceCtx context.Context, safeCtx context.Conte
|
||||
obj.safeCtx, obj.safeCnl = context.WithCancelCause(safeCtx)
|
||||
}
|
||||
func (obj *connecotr) Close() error {
|
||||
return obj.rawConn.CloseWithError(errors.New("connecotr Close close"))
|
||||
return obj.CloseWithError(errors.New("connecotr Close close"))
|
||||
}
|
||||
func (obj *connecotr) CloseWithError(err error) error {
|
||||
err = obj.Conn.CloseWithError(err)
|
||||
if obj.c != nil {
|
||||
return obj.c.Close()
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (obj *connecotr) wrapBody(task *reqTask) {
|
||||
@@ -129,7 +137,7 @@ func (obj *connecotr) wrapBody(task *reqTask) {
|
||||
task.res.Body = body
|
||||
}
|
||||
func (obj *connecotr) httpReq(task *reqTask, done chan struct{}) {
|
||||
if task.res, task.err = obj.rawConn.DoRequest(task.req, task.option.OrderHeaders); task.res != nil && task.err == nil {
|
||||
if task.res, task.err = obj.Conn.DoRequest(task.req, task.option.OrderHeaders); task.res != nil && task.err == nil {
|
||||
obj.wrapBody(task)
|
||||
} else if task.err != nil {
|
||||
task.err = tools.WrapError(task.err, "http1 roundTrip error")
|
||||
@@ -147,7 +155,7 @@ func (obj *connecotr) taskMain(task *reqTask) (retry bool) {
|
||||
}
|
||||
if retry {
|
||||
task.err = nil
|
||||
obj.rawConn.CloseWithError(errors.New("taskMain retry close"))
|
||||
obj.CloseWithError(errors.New("taskMain retry close"))
|
||||
if task.res != nil && task.res.Body != nil {
|
||||
task.res.Body.Close()
|
||||
}
|
||||
@@ -166,7 +174,7 @@ func (obj *connecotr) taskMain(task *reqTask) (retry bool) {
|
||||
}
|
||||
}
|
||||
if task.err != nil {
|
||||
obj.rawConn.CloseWithError(task.err)
|
||||
obj.CloseWithError(task.err)
|
||||
if task.res != nil && task.res.Body != nil {
|
||||
task.res.Body.Close()
|
||||
}
|
||||
@@ -275,7 +283,7 @@ func (obj *connPool) notice(task *reqTask) {
|
||||
func (obj *connPool) rwMain(conn *connecotr) {
|
||||
conn.withCancel(obj.forceCtx, obj.safeCtx)
|
||||
defer func() {
|
||||
conn.rawConn.CloseWithError(errors.New("connPool rwMain close"))
|
||||
conn.CloseWithError(errors.New("connPool rwMain close"))
|
||||
obj.total.Add(-1)
|
||||
if obj.total.Load() <= 0 {
|
||||
obj.safeClose()
|
||||
|
||||
67
dial.go
67
dial.go
@@ -225,29 +225,38 @@ func (obj *DialClient) verifyProxyToRemote(ctx context.Context, option *RequestO
|
||||
})
|
||||
}
|
||||
}
|
||||
switch proxyUrl.Scheme {
|
||||
case "http", "https":
|
||||
err = obj.clientVerifyHttps(ctx, conn, proxyUrl, remoteUrl)
|
||||
if option.Logger != nil {
|
||||
option.Logger(Log{
|
||||
Id: option.requestId,
|
||||
Time: time.Now(),
|
||||
Type: LogType_ProxyConnectRemote,
|
||||
Msg: remoteUrl.String(),
|
||||
})
|
||||
}
|
||||
case "socks5":
|
||||
err = obj.socks5ProxyWithConn(ctx, conn, proxyUrl, remoteUrl)
|
||||
if option.Logger != nil {
|
||||
option.Logger(Log{
|
||||
Id: option.requestId,
|
||||
Time: time.Now(),
|
||||
Type: LogType_ProxyTCPConnect,
|
||||
Msg: remoteUrl.String(),
|
||||
})
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
switch proxyUrl.Scheme {
|
||||
case "http", "https":
|
||||
err = obj.clientVerifyHttps(ctx, conn, proxyUrl, remoteUrl)
|
||||
if option.Logger != nil {
|
||||
option.Logger(Log{
|
||||
Id: option.requestId,
|
||||
Time: time.Now(),
|
||||
Type: LogType_ProxyConnectRemote,
|
||||
Msg: remoteUrl.String(),
|
||||
})
|
||||
}
|
||||
case "socks5":
|
||||
err = obj.clientVerifySocks5(conn, proxyUrl, remoteUrl)
|
||||
if option.Logger != nil {
|
||||
option.Logger(Log{
|
||||
Id: option.requestId,
|
||||
Time: time.Now(),
|
||||
Type: LogType_ProxyTCPConnect,
|
||||
Msg: remoteUrl.String(),
|
||||
})
|
||||
}
|
||||
}
|
||||
close(done)
|
||||
}()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return conn, context.Cause(ctx)
|
||||
case <-done:
|
||||
return conn, err
|
||||
}
|
||||
return conn, err
|
||||
}
|
||||
func (obj *DialClient) loadHost(host string) (string, bool) {
|
||||
msgDataAny, ok := obj.dnsIpData.Load(host)
|
||||
@@ -500,22 +509,6 @@ func (obj *DialClient) Socks5Proxy(ctx context.Context, ctxData *RequestOption,
|
||||
return
|
||||
}
|
||||
}
|
||||
func (obj *DialClient) socks5ProxyWithConn(ctx context.Context, conn net.Conn, proxyUrl *url.URL, remoteUrl *url.URL) (err error) {
|
||||
didVerify := make(chan struct{})
|
||||
go func() {
|
||||
if err = obj.clientVerifySocks5(conn, proxyUrl, remoteUrl); err != nil {
|
||||
conn.Close()
|
||||
}
|
||||
close(didVerify)
|
||||
}()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
conn.Close()
|
||||
return context.Cause(ctx)
|
||||
case <-didVerify:
|
||||
return
|
||||
}
|
||||
}
|
||||
func (obj *DialClient) clientVerifyHttps(ctx context.Context, conn net.Conn, proxyUrl *url.URL, remoteUrl *url.URL) (err error) {
|
||||
hdr := make(http.Header)
|
||||
hdr.Set("User-Agent", tools.UserAgent)
|
||||
|
||||
17
go.mod
17
go.mod
@@ -4,17 +4,17 @@ go 1.23.0
|
||||
|
||||
require (
|
||||
github.com/gospider007/bar v0.0.0-20241205091858-06f3c7e16dd9
|
||||
github.com/gospider007/bs4 v0.0.0-20241205092056-32204f68d82e
|
||||
github.com/gospider007/gson v0.0.0-20240912023741-2238f9748e4a
|
||||
github.com/gospider007/gtls v0.0.0-20240527084326-e580531eb89e
|
||||
github.com/gospider007/http2 v0.0.0-20241215124211-46ad3fc8e608
|
||||
github.com/gospider007/bs4 v0.0.0-20241216122612-aa3f29179e6f
|
||||
github.com/gospider007/gson v0.0.0-20241216122450-fb66d0ba2a07
|
||||
github.com/gospider007/gtls v0.0.0-20241216053748-dc1a232bb017
|
||||
github.com/gospider007/http2 v0.0.0-20241216122734-aa1047df028c
|
||||
github.com/gospider007/http3 v0.0.0-20241215120136-980caa047c47
|
||||
github.com/gospider007/ja3 v0.0.0-20240620005139-f0602f169903
|
||||
github.com/gospider007/ja3 v0.0.0-20241216123149-83352be79439
|
||||
github.com/gospider007/re v0.0.0-20240227100911-e27255e48eff
|
||||
github.com/gospider007/tools v0.0.0-20241120013952-ff42051bfc9f
|
||||
github.com/gospider007/websocket v0.0.0-20241205004859-e11571a81def
|
||||
github.com/gospider007/websocket v0.0.0-20241216130619-89829336d9a6
|
||||
github.com/refraction-networking/utls v1.6.7
|
||||
golang.org/x/exp v0.0.0-20241210194714-1829a127f884
|
||||
golang.org/x/exp v0.0.0-20241215155358-4a5509556b9e
|
||||
golang.org/x/net v0.32.0
|
||||
)
|
||||
|
||||
@@ -46,7 +46,7 @@ require (
|
||||
github.com/klauspost/pgzip v1.2.6 // indirect
|
||||
github.com/libdns/libdns v0.2.2 // indirect
|
||||
github.com/mholt/acmez/v2 v2.0.3 // indirect
|
||||
github.com/mholt/archives v0.0.0-20241207175349-5e373c52f8aa // indirect
|
||||
github.com/mholt/archives v0.0.0-20241216060121-23e0af8fe73d // indirect
|
||||
github.com/miekg/dns v1.1.62 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
@@ -61,6 +61,7 @@ require (
|
||||
github.com/tidwall/gjson v1.18.0 // indirect
|
||||
github.com/tidwall/match v1.1.1 // indirect
|
||||
github.com/tidwall/pretty v1.2.1 // indirect
|
||||
github.com/tidwall/sjson v1.2.5 // indirect
|
||||
github.com/ulikunitz/xz v0.5.12 // indirect
|
||||
github.com/zeebo/blake3 v0.2.4 // indirect
|
||||
go.mongodb.org/mongo-driver v1.17.1 // indirect
|
||||
|
||||
47
go.sum
47
go.sum
@@ -58,7 +58,6 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
|
||||
github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
|
||||
github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=
|
||||
github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
||||
@@ -100,34 +99,26 @@ github.com/gospider007/bar v0.0.0-20241205091858-06f3c7e16dd9 h1:mKDRxvZNftY8msm
|
||||
github.com/gospider007/bar v0.0.0-20241205091858-06f3c7e16dd9/go.mod h1:dZ50Fil3qVmYCGGR9hne3RwgZRfZj+NZQE7NaKBNdPI=
|
||||
github.com/gospider007/blog v0.0.0-20241205091827-6bcaf48620d4 h1:MA0vcRGiy5JPlQYPU9wg7reG4dZkW3v5Q/DVtAUJ6yQ=
|
||||
github.com/gospider007/blog v0.0.0-20241205091827-6bcaf48620d4/go.mod h1:y7eLYgD+hFH2v8qrlrO5KtFTGwYxQaQuWyGAuZh7QHU=
|
||||
github.com/gospider007/bs4 v0.0.0-20241205092056-32204f68d82e h1:9qj2iMbPkuY2VeXplp3mF3dfDJdhZMWZPVLK9BNGJW8=
|
||||
github.com/gospider007/bs4 v0.0.0-20241205092056-32204f68d82e/go.mod h1:5QNzcd53FL191TUMzWq502w+dEICQGgrRuc9GHzeCCo=
|
||||
github.com/gospider007/gson v0.0.0-20240912023741-2238f9748e4a h1:Gu0mPyLzS27Ij6Chr/HDFUs36yTegGUt1kNsqbXligY=
|
||||
github.com/gospider007/gson v0.0.0-20240912023741-2238f9748e4a/go.mod h1:nz5cp3n/fquuXqhHdpamPnOJz/yBEKuOGsFfezpr3f8=
|
||||
github.com/gospider007/gtls v0.0.0-20240527084326-e580531eb89e h1:3bTAtZx+JstbWbTVRd4DivK7QDf6cJwBpWUJpVAgyjY=
|
||||
github.com/gospider007/gtls v0.0.0-20240527084326-e580531eb89e/go.mod h1:pUD3WKesQAdqD1W8O3v03qRYyq760iPNE8IPEILf52Y=
|
||||
github.com/gospider007/http2 v0.0.0-20241214105312-2cf2a243a9eb h1:BqfS6+kMHYPoRM8CDUtE9awVqWABvcvzVoGLWyObulA=
|
||||
github.com/gospider007/http2 v0.0.0-20241214105312-2cf2a243a9eb/go.mod h1:0A43UUydE6EB5QlgYiRyPlidwN3e0faIGEEaI8mxYeU=
|
||||
github.com/gospider007/http2 v0.0.0-20241215075940-632d27d19c93 h1:GpHEydQZfXxH+uHbP87JtfNKe6MVZ5ygwsmqmE1E1aQ=
|
||||
github.com/gospider007/http2 v0.0.0-20241215075940-632d27d19c93/go.mod h1:0A43UUydE6EB5QlgYiRyPlidwN3e0faIGEEaI8mxYeU=
|
||||
github.com/gospider007/http2 v0.0.0-20241215120203-9ee063748247 h1:OaW1iVBa0GdYE5CILbO/40nF/Rtktv1TCXh9muCw08s=
|
||||
github.com/gospider007/http2 v0.0.0-20241215120203-9ee063748247/go.mod h1:0A43UUydE6EB5QlgYiRyPlidwN3e0faIGEEaI8mxYeU=
|
||||
github.com/gospider007/http2 v0.0.0-20241215124211-46ad3fc8e608 h1:/icO1CnWE7r6CCmWs6QHURUL+6bqp+43ITXDS0tAuWM=
|
||||
github.com/gospider007/http2 v0.0.0-20241215124211-46ad3fc8e608/go.mod h1:0A43UUydE6EB5QlgYiRyPlidwN3e0faIGEEaI8mxYeU=
|
||||
github.com/gospider007/http3 v0.0.0-20241214104411-f9a6d924a254 h1:qur79Bvjm07E11mcdXFrvtGW9HKo0ZUB7qn5TJmx4v8=
|
||||
github.com/gospider007/http3 v0.0.0-20241214104411-f9a6d924a254/go.mod h1:dCygjc5CZHjrAOYZkdGcwHTZV8w4DbTQkbuTEIl7jMg=
|
||||
github.com/gospider007/bs4 v0.0.0-20241216122612-aa3f29179e6f h1:/pbT94pif600rZwdOKPi5VaZ6D2pdnQuBSmiqOzRkSA=
|
||||
github.com/gospider007/bs4 v0.0.0-20241216122612-aa3f29179e6f/go.mod h1:M3XPdOCrwkB1UJiaF506o8KCDpRKnkzZBQ5PWitVc90=
|
||||
github.com/gospider007/gson v0.0.0-20241216122450-fb66d0ba2a07 h1:cqsuxxIKxbRG/xRPNwgFUfvGKc992snPjn5WlXkCtnA=
|
||||
github.com/gospider007/gson v0.0.0-20241216122450-fb66d0ba2a07/go.mod h1:XcFiXq3t2+b1x9zezQV/FbuB5Y1rdUrtvpJRbxZXef4=
|
||||
github.com/gospider007/gtls v0.0.0-20241216053748-dc1a232bb017 h1:4lCqtxoQS+PO6YlcoYO8w7p1bi5LzPkzIaIuveouoYk=
|
||||
github.com/gospider007/gtls v0.0.0-20241216053748-dc1a232bb017/go.mod h1:jtwtOHFTKJPLANLr06DtUmkCuoEoQjfKuMrB6sIhIh8=
|
||||
github.com/gospider007/http2 v0.0.0-20241216122734-aa1047df028c h1:xymNUHM1ozqBhptmxpugyfb4+Bl7LoCXt7/wycvpNV4=
|
||||
github.com/gospider007/http2 v0.0.0-20241216122734-aa1047df028c/go.mod h1:b8VbmpEUTyGR8B8h4MGY2E/5klXqs/WziEgTZx5kU3I=
|
||||
github.com/gospider007/http3 v0.0.0-20241215120136-980caa047c47 h1:HYYKcKHPFqpjFjEjwceE8hi2lxQ+EZtMLFJG4KEr9NY=
|
||||
github.com/gospider007/http3 v0.0.0-20241215120136-980caa047c47/go.mod h1:dCygjc5CZHjrAOYZkdGcwHTZV8w4DbTQkbuTEIl7jMg=
|
||||
github.com/gospider007/ja3 v0.0.0-20240620005139-f0602f169903 h1:elWGt/rRpoPLc7dsnC8axzXnzAAyNdrtOzOgJ3OxisY=
|
||||
github.com/gospider007/ja3 v0.0.0-20240620005139-f0602f169903/go.mod h1:coutudbhlLqA/xpGB+IFKUGKkzivbYO/Ghl4e0N6hmI=
|
||||
github.com/gospider007/ja3 v0.0.0-20241216123149-83352be79439 h1:zDuskJWFKk7JMs7XkX22XSrcbEDLe0rHs1baHSYCH3c=
|
||||
github.com/gospider007/ja3 v0.0.0-20241216123149-83352be79439/go.mod h1:AZIKhClKQolSI8o6ixFKIVGOC4lWOdsrF4fshR5axO4=
|
||||
github.com/gospider007/kinds v0.0.0-20240929092451-8f867acde255 h1:X+AM/mgmh/EfyQUjKZp1VFc9TSlrhkwS0eSYeo5fMs4=
|
||||
github.com/gospider007/kinds v0.0.0-20240929092451-8f867acde255/go.mod h1:yZx7Zfp1I4P6CO3TcDyDY5SuXQYr0bZjzT9zG0XrJAI=
|
||||
github.com/gospider007/re v0.0.0-20240227100911-e27255e48eff h1:tiTX2nKAnHZ45JacB/cOf1p4pv5+gw4s1/iGX0ikF0E=
|
||||
github.com/gospider007/re v0.0.0-20240227100911-e27255e48eff/go.mod h1:X58uk0/F3mVskuQOZng0ZKJiAt3ETn0wxuLN//rVZrE=
|
||||
github.com/gospider007/tools v0.0.0-20241120013952-ff42051bfc9f h1:nIUWpSjqtNcDu75oGA2E6NtIIHNOAMhY0qg+U9NJDrE=
|
||||
github.com/gospider007/tools v0.0.0-20241120013952-ff42051bfc9f/go.mod h1:mAvRYlxXzlp7LU2iyhhbB+OSXiq6Nu7VdxV4m1pM20A=
|
||||
github.com/gospider007/websocket v0.0.0-20241205004859-e11571a81def h1:VJaQ0MKX+4lIJ7HW+CGiHm2usMYRMDYM/YT81zCLcLo=
|
||||
github.com/gospider007/websocket v0.0.0-20241205004859-e11571a81def/go.mod h1:rM5q3nQCL+s8VPl8fCAJGq+y6n8wQPvK4vwZ2ZlMlHI=
|
||||
github.com/gospider007/websocket v0.0.0-20241216130619-89829336d9a6 h1:GlEpnVq3W5SbDUGydFnQUavyZeNEXGWz08XsMRokNjs=
|
||||
github.com/gospider007/websocket v0.0.0-20241216130619-89829336d9a6/go.mod h1:7Q66Huzk5NiXzJXSlUG41uWSSjZMDGXljjWoSVDz3kc=
|
||||
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
|
||||
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
@@ -156,12 +147,10 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/libdns/libdns v0.2.2 h1:O6ws7bAfRPaBsgAYt8MDe2HcNBGC29hkZ9MX2eUSX3s=
|
||||
github.com/libdns/libdns v0.2.2/go.mod h1:4Bj9+5CQiNMVGf87wjX4CY3HQJypUHRuLvlsfsZqLWQ=
|
||||
github.com/mholt/acmez v1.2.0 h1:1hhLxSgY5FvH5HCnGUuwbKY2VQVo8IU7rxXKSnZ7F30=
|
||||
github.com/mholt/acmez v1.2.0/go.mod h1:VT9YwH1xgNX1kmYY89gY8xPJC84BFAisjo8Egigt4kE=
|
||||
github.com/mholt/acmez/v2 v2.0.3 h1:CgDBlEwg3QBp6s45tPQmFIBrkRIkBT4rW4orMM6p4sw=
|
||||
github.com/mholt/acmez/v2 v2.0.3/go.mod h1:pQ1ysaDeGrIMvJ9dfJMk5kJNkn7L2sb3UhyrX6Q91cw=
|
||||
github.com/mholt/archives v0.0.0-20241207175349-5e373c52f8aa h1:SzCiDuxF6489FegHHZe/ktxpRIDhu5IemNnWlW1Fquo=
|
||||
github.com/mholt/archives v0.0.0-20241207175349-5e373c52f8aa/go.mod h1:j/Ire/jm42GN7h90F5kzj6hf6ZFzEH66de+hmjEKu+I=
|
||||
github.com/mholt/archives v0.0.0-20241216060121-23e0af8fe73d h1:Vw3f39TqFSQLA+OyW+8SouppHTYzX8/fDv6Ao8uj3Ho=
|
||||
github.com/mholt/archives v0.0.0-20241216060121-23e0af8fe73d/go.mod h1:j/Ire/jm42GN7h90F5kzj6hf6ZFzEH66de+hmjEKu+I=
|
||||
github.com/miekg/dns v1.1.62 h1:cN8OuEF1/x5Rq6Np+h1epln8OiyPWV+lROx9LxcGgIQ=
|
||||
github.com/miekg/dns v1.1.62/go.mod h1:mvDlcItzm+br7MToIKqkglaGhlFMHJ9DTNNWONWXbNQ=
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
@@ -175,7 +164,6 @@ github.com/onsi/ginkgo/v2 v2.22.0 h1:Yed107/8DjTr0lKCNt7Dn8yQ6ybuDRQoMGrNFKzMfHg
|
||||
github.com/onsi/ginkgo/v2 v2.22.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo=
|
||||
github.com/onsi/gomega v1.34.2 h1:pNCwDkzrsv7MS9kpaQvVb1aVLahQXyJ/Tv5oAZMI3i8=
|
||||
github.com/onsi/gomega v1.34.2/go.mod h1:v1xfxRgk0KIsG+QOdm7p8UosrOzPYRo60fd3B/1Dukc=
|
||||
github.com/pierrec/lz4/v4 v4.0.0/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
|
||||
github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU=
|
||||
github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
@@ -205,6 +193,7 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT
|
||||
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw=
|
||||
github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY=
|
||||
github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
|
||||
github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY=
|
||||
github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
|
||||
github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
|
||||
@@ -212,6 +201,8 @@ github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JT
|
||||
github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
|
||||
github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
|
||||
github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
|
||||
github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
|
||||
github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
|
||||
github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
|
||||
github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc=
|
||||
github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
|
||||
@@ -255,8 +246,8 @@ golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE
|
||||
golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
|
||||
golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
|
||||
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
|
||||
golang.org/x/exp v0.0.0-20241210194714-1829a127f884 h1:Y/Mj/94zIQQGHVSv1tTtQBDaQaJe62U9bkDZKKyhPCU=
|
||||
golang.org/x/exp v0.0.0-20241210194714-1829a127f884/go.mod h1:qj5a5QZpwLU2NLQudwIN5koi3beDhSAlJwa67PuM98c=
|
||||
golang.org/x/exp v0.0.0-20241215155358-4a5509556b9e h1:4qufH0hlUYs6AO6XmZC3GqfDPGSXHVXUFR6OND+iJX4=
|
||||
golang.org/x/exp v0.0.0-20241215155358-4a5509556b9e/go.mod h1:qj5a5QZpwLU2NLQudwIN5koi3beDhSAlJwa67PuM98c=
|
||||
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
|
||||
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
|
||||
golang.org/x/image v0.23.0 h1:HseQ7c2OpPKTPVzNjG5fwJsOTCiiwS4QdsYi5XU6H68=
|
||||
|
||||
179
roundTripper.go
179
roundTripper.go
@@ -108,7 +108,7 @@ func (obj *roundTripper) http3Dial(option *RequestOption, req *http.Request) (co
|
||||
return
|
||||
}
|
||||
conn = obj.newConnecotr()
|
||||
conn.rawConn = http3.NewClient(netConn, func() {
|
||||
conn.Conn = http3.NewClient(netConn, func() {
|
||||
conn.forceCnl(errors.New("http3 client close"))
|
||||
})
|
||||
return
|
||||
@@ -123,19 +123,13 @@ func (obj *roundTripper) ghttp3Dial(option *RequestOption, req *http.Request) (c
|
||||
return
|
||||
}
|
||||
conn = obj.newConnecotr()
|
||||
conn.rawConn = http3.NewUClient(netConn, func() {
|
||||
conn.Conn = http3.NewUClient(netConn, func() {
|
||||
conn.forceCnl(errors.New("http3 client close"))
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
func (obj *roundTripper) dial(option *RequestOption, req *http.Request) (conn *connecotr, err error) {
|
||||
var netConn net.Conn
|
||||
defer func() {
|
||||
if err != nil && netConn != nil {
|
||||
netConn.Close()
|
||||
}
|
||||
}()
|
||||
if option.H3 {
|
||||
if option.Ja3Spec.IsSet() {
|
||||
return obj.ghttp3Dial(option, req)
|
||||
@@ -143,100 +137,126 @@ func (obj *roundTripper) dial(option *RequestOption, req *http.Request) (conn *c
|
||||
return obj.http3Dial(option, req)
|
||||
}
|
||||
}
|
||||
var proxys []*url.URL
|
||||
if !option.DisProxy {
|
||||
if option.proxy != nil {
|
||||
proxys = []*url.URL{cloneUrl(option.proxy)}
|
||||
}
|
||||
if len(proxys) == 0 && len(option.proxys) > 0 {
|
||||
proxys = make([]*url.URL, len(option.proxys))
|
||||
for i, proxy := range option.proxys {
|
||||
proxys[i] = cloneUrl(proxy)
|
||||
}
|
||||
}
|
||||
if len(proxys) == 0 && obj.getProxy != nil {
|
||||
proxyStr, err := obj.getProxy(req.Context(), req.URL)
|
||||
if err != nil {
|
||||
return conn, err
|
||||
}
|
||||
proxy, err := gtls.VerifyProxy(proxyStr)
|
||||
if err != nil {
|
||||
return conn, err
|
||||
}
|
||||
proxys = []*url.URL{proxy}
|
||||
}
|
||||
if len(proxys) == 0 && obj.getProxys != nil {
|
||||
proxyStrs, err := obj.getProxys(req.Context(), req.URL)
|
||||
if err != nil {
|
||||
return conn, err
|
||||
}
|
||||
if l := len(proxyStrs); l > 0 {
|
||||
proxys = make([]*url.URL, l)
|
||||
for i, proxyStr := range proxyStrs {
|
||||
proxy, err := gtls.VerifyProxy(proxyStr)
|
||||
if err != nil {
|
||||
return conn, err
|
||||
}
|
||||
proxys[i] = proxy
|
||||
}
|
||||
}
|
||||
}
|
||||
proxys, err := obj.initProxys(option, req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
host := getHost(req)
|
||||
var netConn net.Conn
|
||||
if len(proxys) > 0 {
|
||||
netConn, err = obj.dialer.DialProxyContext(req.Context(), option, "tcp", option.TlsConfig.Clone(), append(proxys, cloneUrl(req.URL))...)
|
||||
} else {
|
||||
netConn, err = obj.dialer.DialContext(req.Context(), option, "tcp", getAddr(req.URL))
|
||||
}
|
||||
defer func() {
|
||||
if err != nil && netConn != nil {
|
||||
netConn.Close()
|
||||
}
|
||||
}()
|
||||
if err != nil {
|
||||
return conn, err
|
||||
return nil, err
|
||||
}
|
||||
var h2 bool
|
||||
if req.URL.Scheme == "https" {
|
||||
ctx, cnl := context.WithTimeout(req.Context(), option.TlsHandshakeTimeout)
|
||||
defer cnl()
|
||||
if option.Ja3Spec.IsSet() {
|
||||
if tlsConn, err := obj.dialer.addJa3Tls(ctx, netConn, host, option.ForceHttp1, option.Ja3Spec, option.UtlsConfig.Clone()); err != nil {
|
||||
return conn, tools.WrapError(err, "add ja3 tls error")
|
||||
} else {
|
||||
h2 = tlsConn.ConnectionState().NegotiatedProtocol == "h2"
|
||||
netConn = tlsConn
|
||||
}
|
||||
} else {
|
||||
if tlsConn, err := obj.dialer.addTls(ctx, netConn, host, option.ForceHttp1, option.TlsConfig.Clone()); err != nil {
|
||||
return conn, tools.WrapError(err, "add tls error")
|
||||
} else {
|
||||
h2 = tlsConn.ConnectionState().NegotiatedProtocol == "h2"
|
||||
netConn = tlsConn
|
||||
}
|
||||
}
|
||||
netConn, h2, err = obj.dialAddTls(option, req, netConn)
|
||||
if option.Logger != nil {
|
||||
option.Logger(Log{
|
||||
Id: option.requestId,
|
||||
Time: time.Now(),
|
||||
Type: LogType_TLSHandshake,
|
||||
Msg: host,
|
||||
Msg: getHost(req),
|
||||
})
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
conne := obj.newConnecotr()
|
||||
conne.proxys = proxys
|
||||
conne.c = netConn
|
||||
err = obj.dialConnecotr(option, req, conne, h2)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return conne, err
|
||||
}
|
||||
func (obj *roundTripper) dialConnecotr(option *RequestOption, req *http.Request, conne *connecotr, h2 bool) (err error) {
|
||||
if h2 {
|
||||
if option.H2Ja3Spec.OrderHeaders != nil {
|
||||
option.OrderHeaders = option.H2Ja3Spec.OrderHeaders
|
||||
}
|
||||
if conne.rawConn, err = http2.NewClientConn(func() {
|
||||
if conne.Conn, err = http2.NewClientConn(req.Context(), conne.c, option.H2Ja3Spec, func() {
|
||||
conne.forceCnl(errors.New("http2 client close"))
|
||||
}, netConn, option.H2Ja3Spec); err != nil {
|
||||
return conne, err
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
conne.rawConn = newConn(conne.forceCtx, netConn, func() {
|
||||
conne.Conn = newConn(conne.forceCtx, conne.c, func() {
|
||||
conne.forceCnl(errors.New("http1 client close"))
|
||||
})
|
||||
}
|
||||
return conne, err
|
||||
return err
|
||||
}
|
||||
func (obj *roundTripper) dialAddTls(option *RequestOption, req *http.Request, netConn net.Conn) (net.Conn, bool, error) {
|
||||
ctx, cnl := context.WithTimeout(req.Context(), option.TlsHandshakeTimeout)
|
||||
defer cnl()
|
||||
if option.Ja3Spec.IsSet() {
|
||||
if tlsConn, err := obj.dialer.addJa3Tls(ctx, netConn, getHost(req), option.ForceHttp1, option.Ja3Spec, option.UtlsConfig.Clone()); err != nil {
|
||||
return tlsConn, false, tools.WrapError(err, "add ja3 tls error")
|
||||
} else {
|
||||
return tlsConn, tlsConn.ConnectionState().NegotiatedProtocol == "h2", nil
|
||||
}
|
||||
} else {
|
||||
if tlsConn, err := obj.dialer.addTls(ctx, netConn, getHost(req), option.ForceHttp1, option.TlsConfig.Clone()); err != nil {
|
||||
return tlsConn, false, tools.WrapError(err, "add tls error")
|
||||
} else {
|
||||
return tlsConn, tlsConn.ConnectionState().NegotiatedProtocol == "h2", nil
|
||||
}
|
||||
}
|
||||
}
|
||||
func (obj *roundTripper) initProxys(option *RequestOption, req *http.Request) ([]*url.URL, error) {
|
||||
var proxys []*url.URL
|
||||
if option.DisProxy {
|
||||
return nil, nil
|
||||
}
|
||||
if option.proxy != nil {
|
||||
proxys = []*url.URL{cloneUrl(option.proxy)}
|
||||
}
|
||||
if len(proxys) == 0 && len(option.proxys) > 0 {
|
||||
proxys = make([]*url.URL, len(option.proxys))
|
||||
for i, proxy := range option.proxys {
|
||||
proxys[i] = cloneUrl(proxy)
|
||||
}
|
||||
}
|
||||
if len(proxys) == 0 && obj.getProxy != nil {
|
||||
proxyStr, err := obj.getProxy(req.Context(), req.URL)
|
||||
if err != nil {
|
||||
return proxys, err
|
||||
}
|
||||
proxy, err := gtls.VerifyProxy(proxyStr)
|
||||
if err != nil {
|
||||
return proxys, err
|
||||
}
|
||||
proxys = []*url.URL{proxy}
|
||||
}
|
||||
if len(proxys) == 0 && obj.getProxys != nil {
|
||||
proxyStrs, err := obj.getProxys(req.Context(), req.URL)
|
||||
if err != nil {
|
||||
return proxys, err
|
||||
}
|
||||
if l := len(proxyStrs); l > 0 {
|
||||
proxys = make([]*url.URL, l)
|
||||
for i, proxyStr := range proxyStrs {
|
||||
proxy, err := gtls.VerifyProxy(proxyStr)
|
||||
if err != nil {
|
||||
return proxys, err
|
||||
}
|
||||
proxys[i] = proxy
|
||||
}
|
||||
}
|
||||
}
|
||||
return proxys, nil
|
||||
}
|
||||
|
||||
func (obj *roundTripper) setGetProxy(getProxy func(ctx context.Context, url *url.URL) (string, error)) {
|
||||
obj.getProxy = getProxy
|
||||
}
|
||||
@@ -264,17 +284,7 @@ func (obj *roundTripper) poolRoundTrip(option *RequestOption, pool *connPool, ta
|
||||
|
||||
func (obj *roundTripper) createPool(option *RequestOption, task *reqTask, key string) (isOk bool, err error) {
|
||||
option.isNewConn = true
|
||||
var conn *connecotr
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
conn, err = obj.dial(option, task.req)
|
||||
close(done)
|
||||
}()
|
||||
select {
|
||||
case <-obj.ctx.Done():
|
||||
return true, obj.ctx.Err()
|
||||
case <-done:
|
||||
}
|
||||
conn, err := obj.dial(option, task.req)
|
||||
if err != nil {
|
||||
if task.option.ErrCallBack != nil {
|
||||
if err2 := task.option.ErrCallBack(task.req.Context(), task.option, nil, err); err2 != nil {
|
||||
@@ -329,6 +339,11 @@ func (obj *roundTripper) RoundTrip(req *http.Request) (response *http.Response,
|
||||
var errNum int
|
||||
var isOk bool
|
||||
for {
|
||||
select {
|
||||
case <-req.Context().Done():
|
||||
return nil, context.Cause(req.Context())
|
||||
default:
|
||||
}
|
||||
if errNum >= maxRetry {
|
||||
task.err = fmt.Errorf("roundTrip retry %d times", maxRetry)
|
||||
break
|
||||
|
||||
4
rw.go
4
rw.go
@@ -13,7 +13,7 @@ type readWriteCloser struct {
|
||||
}
|
||||
|
||||
func (obj *readWriteCloser) Conn() net.Conn {
|
||||
return obj.conn.rawConn.(net.Conn)
|
||||
return obj.conn.Conn.(net.Conn)
|
||||
}
|
||||
func (obj *readWriteCloser) Read(p []byte) (n int, err error) {
|
||||
i, err := obj.body.Read(p)
|
||||
@@ -47,5 +47,5 @@ func (obj *readWriteCloser) CloseConn() {
|
||||
|
||||
// force close conn
|
||||
func (obj *readWriteCloser) ForceCloseConn() {
|
||||
obj.conn.rawConn.CloseWithError(errConnectionForceClosed)
|
||||
obj.conn.CloseWithError(errConnectionForceClosed)
|
||||
}
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"testing"
|
||||
|
||||
"github.com/gospider007/requests"
|
||||
@@ -9,6 +11,13 @@ import (
|
||||
func TestHttp1(t *testing.T) {
|
||||
resp, err := requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{
|
||||
ForceHttp1: true,
|
||||
Logger: func(l requests.Log) {
|
||||
log.Print(l)
|
||||
},
|
||||
ErrCallBack: func(ctx context.Context, option *requests.RequestOption, response *requests.Response, err error) error {
|
||||
log.Print(err)
|
||||
return nil
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
|
||||
Reference in New Issue
Block a user