mirror of
https://github.com/gospider007/requests.git
synced 2025-12-24 13:57:52 +08:00
sync
This commit is contained in:
gospider
@@ -50,6 +50,7 @@ Requests is a fully featured HTTP client library for Golang. Network requests ca
|
||||
* [IPv4, IPv6 Address Control Parsing](https://github.com/gospider007/requests/blob/master/test/addType_test.go)
|
||||
* [DNS Settings](https://github.com/gospider007/requests/blob/master/test/dns_test.go)
|
||||
* [Proxy](https://github.com/gospider007/requests/blob/master/test/proxy_test.go)
|
||||
* [Logger](https://github.com/gospider007/requests/blob/master/test/logger/logger_test.go)
|
||||
* [Well tested client library](https://github.com/gospider007/requests/tree/master/test)
|
||||
## [Benchmark](https://github.com/gospider007/benchmark)
|
||||
[gospider007/requests](https://github.com/gospider007/requests) > [imroc/req](github.com/imroc/req) > [go-resty](github.com/go-resty/resty) > [wangluozhe/requests](github.com/wangluozhe/requests) > [curl_cffi](https://github.com/yifeikong/curl_cffi) > [httpx](https://github.com/encode/httpx) > [psf/requests](https://github.com/psf/requests)
|
||||
|
||||
25
conn.go
25
conn.go
@@ -12,8 +12,8 @@ import (
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/gospider007/http2"
|
||||
"github.com/gospider007/http3"
|
||||
"github.com/gospider007/net/http2"
|
||||
"github.com/gospider007/tools"
|
||||
)
|
||||
|
||||
@@ -28,7 +28,7 @@ type connecotr struct {
|
||||
bodyCnl context.CancelCauseFunc
|
||||
|
||||
rawConn net.Conn
|
||||
h2RawConn *http2.ClientConn
|
||||
h2RawConn *http2.Http2ClientConn
|
||||
h3RawConn http3.RoundTripper
|
||||
proxy *url.URL
|
||||
r *bufio.Reader
|
||||
@@ -101,13 +101,6 @@ func (obj *connecotr) SetWriteDeadline(t time.Time) error {
|
||||
return obj.rawConn.SetWriteDeadline(t)
|
||||
}
|
||||
|
||||
func (obj *connecotr) h2Closed() bool {
|
||||
if obj.h2RawConn == nil {
|
||||
return false
|
||||
}
|
||||
state := obj.h2RawConn.State()
|
||||
return state.Closed || state.Closing
|
||||
}
|
||||
func (obj *connecotr) wrapBody(task *reqTask) {
|
||||
body := new(readWriteCloser)
|
||||
obj.bodyCtx, obj.bodyCnl = context.WithCancelCause(task.req.Context())
|
||||
@@ -130,7 +123,7 @@ func (obj *connecotr) http1Req(task *reqTask, done chan struct{}) {
|
||||
}
|
||||
|
||||
func (obj *connecotr) http2Req(task *reqTask, done chan struct{}) {
|
||||
if task.res, task.err = obj.h2RawConn.RoundTripWithOrderHeaders(task.req, task.orderHeaders2); task.res != nil && task.err == nil {
|
||||
if task.res, task.err = obj.h2RawConn.DoRequest(task.req, task.orderHeaders2); task.res != nil && task.err == nil {
|
||||
obj.wrapBody(task)
|
||||
} else if task.err != nil {
|
||||
task.err = tools.WrapError(task.err, "http2 roundTrip error")
|
||||
@@ -160,7 +153,6 @@ func (obj *connecotr) waitBodyClose() error {
|
||||
|
||||
func (obj *connecotr) taskMain(task *reqTask, waitBody bool) (retry bool) {
|
||||
defer func() {
|
||||
// log.Print("taskMain", retry, task.err)
|
||||
if retry {
|
||||
task.err = nil
|
||||
obj.closeWithError(errors.New("taskMain retry close"))
|
||||
@@ -175,9 +167,6 @@ func (obj *connecotr) taskMain(task *reqTask, waitBody bool) (retry bool) {
|
||||
}
|
||||
}
|
||||
}()
|
||||
if obj.h2Closed() {
|
||||
return true
|
||||
}
|
||||
select {
|
||||
case <-obj.safeCtx.Done():
|
||||
return true
|
||||
@@ -208,6 +197,14 @@ func (obj *connecotr) taskMain(task *reqTask, waitBody bool) (retry bool) {
|
||||
}
|
||||
return task.suppertRetry()
|
||||
}
|
||||
if task.ctxData.logger != nil {
|
||||
task.ctxData.logger(Log{
|
||||
Id: task.ctxData.requestId,
|
||||
Time: time.Now(),
|
||||
Type: LogType_ResponseHeader,
|
||||
Msg: "response header",
|
||||
})
|
||||
}
|
||||
return false
|
||||
case <-obj.forceCtx.Done(): //force conn close
|
||||
err := context.Cause(obj.forceCtx)
|
||||
|
||||
77
dial.go
77
dial.go
@@ -82,7 +82,7 @@ func NewDail(option DialOption) *DialClient {
|
||||
getAddrType: option.GetAddrType,
|
||||
}
|
||||
}
|
||||
func (obj *DialClient) DialContext(ctx context.Context, ctxData *reqCtxData, network string, addr string) (net.Conn, error) {
|
||||
func (obj *DialClient) dialContext(ctx context.Context, ctxData *reqCtxData, network string, addr string, isProxy bool) (net.Conn, error) {
|
||||
if ctxData == nil {
|
||||
ctxData = &reqCtxData{}
|
||||
}
|
||||
@@ -108,13 +108,54 @@ func (obj *DialClient) DialContext(ctx context.Context, ctxData *reqCtxData, net
|
||||
if host, err = obj.addrToIp(host, ips, addrType); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if ctxData.logger != nil {
|
||||
if isProxy {
|
||||
ctxData.logger(Log{
|
||||
Id: ctxData.requestId,
|
||||
Time: time.Now(),
|
||||
Type: LogType_ProxyDNSLookup,
|
||||
Msg: host,
|
||||
})
|
||||
} else {
|
||||
ctxData.logger(Log{
|
||||
Id: ctxData.requestId,
|
||||
Time: time.Now(),
|
||||
Type: LogType_DNSLookup,
|
||||
Msg: host,
|
||||
})
|
||||
}
|
||||
}
|
||||
addr = net.JoinHostPort(host, port)
|
||||
}
|
||||
}
|
||||
if dialer == nil {
|
||||
dialer = obj.getDialer(ctxData, false)
|
||||
}
|
||||
return dialer.DialContext(ctx, network, addr)
|
||||
con, err := dialer.DialContext(ctx, network, addr)
|
||||
if ctxData.logger != nil {
|
||||
if isProxy {
|
||||
ctxData.logger(Log{
|
||||
Id: ctxData.requestId,
|
||||
Time: time.Now(),
|
||||
Type: LogType_ProxyTCPConnect,
|
||||
Msg: addr,
|
||||
})
|
||||
} else {
|
||||
ctxData.logger(Log{
|
||||
Id: ctxData.requestId,
|
||||
Time: time.Now(),
|
||||
Type: LogType_TCPConnect,
|
||||
Msg: addr,
|
||||
})
|
||||
}
|
||||
}
|
||||
return con, err
|
||||
}
|
||||
func (obj *DialClient) DialContext(ctx context.Context, ctxData *reqCtxData, network string, addr string) (net.Conn, error) {
|
||||
return obj.dialContext(ctx, ctxData, network, addr, false)
|
||||
}
|
||||
func (obj *DialClient) ProxyDialContext(ctx context.Context, ctxData *reqCtxData, network string, addr string) (net.Conn, error) {
|
||||
return obj.dialContext(ctx, ctxData, network, addr, true)
|
||||
}
|
||||
func (obj *DialClient) DialContextWithProxy(ctx context.Context, ctxData *reqCtxData, network string, scheme string, addr string, host string, proxyUrl *url.URL, tlsConfig *tls.Config) (net.Conn, error) {
|
||||
if ctxData == nil {
|
||||
@@ -132,17 +173,43 @@ func (obj *DialClient) DialContextWithProxy(ctx context.Context, ctxData *reqCtx
|
||||
}
|
||||
switch proxyUrl.Scheme {
|
||||
case "http", "https":
|
||||
conn, err := obj.DialContext(ctx, ctxData, network, net.JoinHostPort(proxyUrl.Hostname(), proxyUrl.Port()))
|
||||
conn, err := obj.ProxyDialContext(ctx, ctxData, network, net.JoinHostPort(proxyUrl.Hostname(), proxyUrl.Port()))
|
||||
if err != nil {
|
||||
return conn, err
|
||||
} else if proxyUrl.Scheme == "https" {
|
||||
if conn, err = obj.addTls(ctx, conn, proxyUrl.Host, true, tlsConfig); err != nil {
|
||||
return conn, err
|
||||
}
|
||||
if ctxData.logger != nil {
|
||||
ctxData.logger(Log{
|
||||
Id: ctxData.requestId,
|
||||
Time: time.Now(),
|
||||
Type: LogType_ProxyTLSHandshake,
|
||||
Msg: addr,
|
||||
})
|
||||
}
|
||||
}
|
||||
return conn, obj.clientVerifyHttps(ctx, scheme, proxyUrl, addr, host, conn)
|
||||
err = obj.clientVerifyHttps(ctx, scheme, proxyUrl, addr, host, conn)
|
||||
if ctxData.logger != nil {
|
||||
ctxData.logger(Log{
|
||||
Id: ctxData.requestId,
|
||||
Time: time.Now(),
|
||||
Type: LogType_ProxyConnectRemote,
|
||||
Msg: addr,
|
||||
})
|
||||
}
|
||||
return conn, err
|
||||
case "socks5":
|
||||
return obj.Socks5Proxy(ctx, ctxData, network, addr, proxyUrl)
|
||||
conn, err := obj.Socks5Proxy(ctx, ctxData, network, addr, proxyUrl)
|
||||
if ctxData.logger != nil {
|
||||
ctxData.logger(Log{
|
||||
Id: ctxData.requestId,
|
||||
Time: time.Now(),
|
||||
Type: LogType_ProxyTCPConnect,
|
||||
Msg: addr,
|
||||
})
|
||||
}
|
||||
return conn, err
|
||||
default:
|
||||
return nil, errors.New("proxyUrl Scheme error")
|
||||
}
|
||||
|
||||
12
go.mod
12
go.mod
@@ -4,12 +4,12 @@ go 1.23.0
|
||||
|
||||
require (
|
||||
github.com/gospider007/bar v0.0.0-20231215084215-956cfa59ce61
|
||||
github.com/gospider007/bs4 v0.0.0-20240531060354-fe6c0582dfd9
|
||||
github.com/gospider007/bs4 v0.0.0-20241120020109-42eda1e01a18
|
||||
github.com/gospider007/gson v0.0.0-20240912023741-2238f9748e4a
|
||||
github.com/gospider007/gtls v0.0.0-20240527084326-e580531eb89e
|
||||
github.com/gospider007/http2 v0.0.0-20241124010135-caaf1749745b
|
||||
github.com/gospider007/http3 v0.0.0-20240925091819-2e960e23b259
|
||||
github.com/gospider007/ja3 v0.0.0-20240620005139-f0602f169903
|
||||
github.com/gospider007/net v0.0.0-20240620005014-93bab3eb6b6c
|
||||
github.com/gospider007/re v0.0.0-20240227100911-e27255e48eff
|
||||
github.com/gospider007/tools v0.0.0-20241120013952-ff42051bfc9f
|
||||
github.com/gospider007/websocket v0.0.0-20240331132617-8217ca7a8439
|
||||
@@ -32,7 +32,7 @@ require (
|
||||
github.com/cloudflare/circl v1.5.0 // indirect
|
||||
github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect
|
||||
github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
|
||||
github.com/google/pprof v0.0.0-20241101162523-b92577c0c142 // indirect
|
||||
github.com/google/pprof v0.0.0-20241122213907-cbe949e5a41b // indirect
|
||||
github.com/gospider007/blog v0.0.0-20231121084103-59a004dafccf // indirect
|
||||
github.com/gospider007/kinds v0.0.0-20240929092451-8f867acde255 // indirect
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
@@ -44,12 +44,12 @@ require (
|
||||
github.com/klauspost/pgzip v1.2.6 // indirect
|
||||
github.com/libdns/libdns v0.2.2 // indirect
|
||||
github.com/mholt/acmez/v2 v2.0.3 // indirect
|
||||
github.com/mholt/archives v0.0.0-20241119192853-c1ebc8406725 // indirect
|
||||
github.com/mholt/archives v0.0.0-20241123153623-b85dd98f4298 // indirect
|
||||
github.com/miekg/dns v1.1.62 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/nwaples/rardecode/v2 v2.0.0-beta.4.0.20241112120701-034e449c6e78 // indirect
|
||||
github.com/onsi/ginkgo/v2 v2.21.0 // indirect
|
||||
github.com/nwaples/rardecode/v2 v2.0.0 // indirect
|
||||
github.com/onsi/ginkgo/v2 v2.22.0 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.21 // indirect
|
||||
github.com/quic-go/qpack v0.5.1 // indirect
|
||||
github.com/sorairolake/lzip-go v0.3.5 // indirect
|
||||
|
||||
24
go.sum
24
go.sum
@@ -82,8 +82,8 @@ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXi
|
||||
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
||||
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
||||
github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
|
||||
github.com/google/pprof v0.0.0-20241101162523-b92577c0c142 h1:sAGdeJj0bnMgUNVeUpp6AYlVdCt3/GdI3pGRqsNSQLs=
|
||||
github.com/google/pprof v0.0.0-20241101162523-b92577c0c142/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144=
|
||||
github.com/google/pprof v0.0.0-20241122213907-cbe949e5a41b h1:SXO0REt4iu865upYCk8aKBBJQ4BqoE0ReP23ClMu60s=
|
||||
github.com/google/pprof v0.0.0-20241122213907-cbe949e5a41b/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144=
|
||||
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
|
||||
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
|
||||
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
|
||||
@@ -93,20 +93,20 @@ github.com/gospider007/bar v0.0.0-20231215084215-956cfa59ce61 h1:XS7tOVjv207CKZT
|
||||
github.com/gospider007/bar v0.0.0-20231215084215-956cfa59ce61/go.mod h1:aYPgmG9340i9x9VQZhf34/XtIj7PHDTq0wSO+7zU/8s=
|
||||
github.com/gospider007/blog v0.0.0-20231121084103-59a004dafccf h1:1laTsuH/wl5pZ5QlHzacX09QzvwQw0DFENoRMpGBK8Y=
|
||||
github.com/gospider007/blog v0.0.0-20231121084103-59a004dafccf/go.mod h1:CCJ+hvQ0kxL+qB/Wfr1xt7xspsG4XiczhnAPVxG2m3M=
|
||||
github.com/gospider007/bs4 v0.0.0-20240531060354-fe6c0582dfd9 h1:SiePZGEq9yUj97mSzNAGO9wuKnGrAui9VI4jgUUZEts=
|
||||
github.com/gospider007/bs4 v0.0.0-20240531060354-fe6c0582dfd9/go.mod h1:kFlkn2u1ulhiybnhC9I840oS98tRSLJVwoBI5qNfZ2U=
|
||||
github.com/gospider007/bs4 v0.0.0-20241120020109-42eda1e01a18 h1:X0VOdziaqSlQzJLYrbPoYAZW4I1ub3rWTtQYzHIc6xE=
|
||||
github.com/gospider007/bs4 v0.0.0-20241120020109-42eda1e01a18/go.mod h1:4f04luriR/6gaBqfyUFC7jgSlrlu6YJj6/ya9llb5M0=
|
||||
github.com/gospider007/gson v0.0.0-20240912023741-2238f9748e4a h1:Gu0mPyLzS27Ij6Chr/HDFUs36yTegGUt1kNsqbXligY=
|
||||
github.com/gospider007/gson v0.0.0-20240912023741-2238f9748e4a/go.mod h1:nz5cp3n/fquuXqhHdpamPnOJz/yBEKuOGsFfezpr3f8=
|
||||
github.com/gospider007/gtls v0.0.0-20240527084326-e580531eb89e h1:3bTAtZx+JstbWbTVRd4DivK7QDf6cJwBpWUJpVAgyjY=
|
||||
github.com/gospider007/gtls v0.0.0-20240527084326-e580531eb89e/go.mod h1:pUD3WKesQAdqD1W8O3v03qRYyq760iPNE8IPEILf52Y=
|
||||
github.com/gospider007/http2 v0.0.0-20241124010135-caaf1749745b h1:qRpUCl+MzzjPxkZQ99wt8UY60bSckJ624j0GN8wq/QI=
|
||||
github.com/gospider007/http2 v0.0.0-20241124010135-caaf1749745b/go.mod h1:pyzs3hnbrKDH/IcQN+kVxqEXJHNzEWs6Londu7MXGoE=
|
||||
github.com/gospider007/http3 v0.0.0-20240925091819-2e960e23b259 h1:2UBbFJLn+B6sjHXnQxSwjOwvLQr40x45tCHdOegS0JM=
|
||||
github.com/gospider007/http3 v0.0.0-20240925091819-2e960e23b259/go.mod h1:ypu2W38mKOS757LzsBHbs/qqaBFnEdOUwqK30rtPCgE=
|
||||
github.com/gospider007/ja3 v0.0.0-20240620005139-f0602f169903 h1:elWGt/rRpoPLc7dsnC8axzXnzAAyNdrtOzOgJ3OxisY=
|
||||
github.com/gospider007/ja3 v0.0.0-20240620005139-f0602f169903/go.mod h1:coutudbhlLqA/xpGB+IFKUGKkzivbYO/Ghl4e0N6hmI=
|
||||
github.com/gospider007/kinds v0.0.0-20240929092451-8f867acde255 h1:X+AM/mgmh/EfyQUjKZp1VFc9TSlrhkwS0eSYeo5fMs4=
|
||||
github.com/gospider007/kinds v0.0.0-20240929092451-8f867acde255/go.mod h1:yZx7Zfp1I4P6CO3TcDyDY5SuXQYr0bZjzT9zG0XrJAI=
|
||||
github.com/gospider007/net v0.0.0-20240620005014-93bab3eb6b6c h1:5g2/mRpUxNcGRB4DDUepbeo9qBYOqdPCdMFgDADiwr4=
|
||||
github.com/gospider007/net v0.0.0-20240620005014-93bab3eb6b6c/go.mod h1:cM4DcAwxPcg4TaBkCoksZ2TZGTA7YCmvzzYOdDbjS3c=
|
||||
github.com/gospider007/re v0.0.0-20240227100911-e27255e48eff h1:tiTX2nKAnHZ45JacB/cOf1p4pv5+gw4s1/iGX0ikF0E=
|
||||
github.com/gospider007/re v0.0.0-20240227100911-e27255e48eff/go.mod h1:X58uk0/F3mVskuQOZng0ZKJiAt3ETn0wxuLN//rVZrE=
|
||||
github.com/gospider007/tools v0.0.0-20241120013952-ff42051bfc9f h1:nIUWpSjqtNcDu75oGA2E6NtIIHNOAMhY0qg+U9NJDrE=
|
||||
@@ -143,8 +143,8 @@ github.com/libdns/libdns v0.2.2 h1:O6ws7bAfRPaBsgAYt8MDe2HcNBGC29hkZ9MX2eUSX3s=
|
||||
github.com/libdns/libdns v0.2.2/go.mod h1:4Bj9+5CQiNMVGf87wjX4CY3HQJypUHRuLvlsfsZqLWQ=
|
||||
github.com/mholt/acmez/v2 v2.0.3 h1:CgDBlEwg3QBp6s45tPQmFIBrkRIkBT4rW4orMM6p4sw=
|
||||
github.com/mholt/acmez/v2 v2.0.3/go.mod h1:pQ1ysaDeGrIMvJ9dfJMk5kJNkn7L2sb3UhyrX6Q91cw=
|
||||
github.com/mholt/archives v0.0.0-20241119192853-c1ebc8406725 h1:1RnJdkUevC/SFdFMKbSiZnlyZb36s02bmJnmAfL2SE4=
|
||||
github.com/mholt/archives v0.0.0-20241119192853-c1ebc8406725/go.mod h1:j/Ire/jm42GN7h90F5kzj6hf6ZFzEH66de+hmjEKu+I=
|
||||
github.com/mholt/archives v0.0.0-20241123153623-b85dd98f4298 h1:KyCSnt0fdY6Bi4vxM0On3zlonJwnDtTYSpMg7Y1ZNhs=
|
||||
github.com/mholt/archives v0.0.0-20241123153623-b85dd98f4298/go.mod h1:j/Ire/jm42GN7h90F5kzj6hf6ZFzEH66de+hmjEKu+I=
|
||||
github.com/miekg/dns v1.1.62 h1:cN8OuEF1/x5Rq6Np+h1epln8OiyPWV+lROx9LxcGgIQ=
|
||||
github.com/miekg/dns v1.1.62/go.mod h1:mvDlcItzm+br7MToIKqkglaGhlFMHJ9DTNNWONWXbNQ=
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
@@ -152,10 +152,10 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
|
||||
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
|
||||
github.com/nwaples/rardecode/v2 v2.0.0-beta.4.0.20241112120701-034e449c6e78 h1:MYzLheyVx1tJVDqfu3YnN4jtnyALNzLvwl+f58TcvQY=
|
||||
github.com/nwaples/rardecode/v2 v2.0.0-beta.4.0.20241112120701-034e449c6e78/go.mod h1:yntwv/HfMc/Hbvtq9I19D1n58te3h6KsqCf3GxyfBGY=
|
||||
github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM=
|
||||
github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo=
|
||||
github.com/nwaples/rardecode/v2 v2.0.0 h1:C+58s2TPUdvgVFioQLpUNmIjTchtegHGS2xTNiKBYXA=
|
||||
github.com/nwaples/rardecode/v2 v2.0.0/go.mod h1:yntwv/HfMc/Hbvtq9I19D1n58te3h6KsqCf3GxyfBGY=
|
||||
github.com/onsi/ginkgo/v2 v2.22.0 h1:Yed107/8DjTr0lKCNt7Dn8yQ6ybuDRQoMGrNFKzMfHg=
|
||||
github.com/onsi/ginkgo/v2 v2.22.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo=
|
||||
github.com/onsi/gomega v1.34.2 h1:pNCwDkzrsv7MS9kpaQvVb1aVLahQXyJ/Tv5oAZMI3i8=
|
||||
github.com/onsi/gomega v1.34.2/go.mod h1:v1xfxRgk0KIsG+QOdm7p8UosrOzPYRo60fd3B/1Dukc=
|
||||
github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=
|
||||
|
||||
55
option.go
55
option.go
@@ -15,8 +15,38 @@ import (
|
||||
utls "github.com/refraction-networking/utls"
|
||||
)
|
||||
|
||||
type LogType string
|
||||
|
||||
const (
|
||||
LogType_DNSLookup LogType = "DNSLookup"
|
||||
|
||||
LogType_TCPConnect LogType = "TCPConnect"
|
||||
|
||||
LogType_TLSHandshake LogType = "TLSHandshake"
|
||||
|
||||
LogType_ProxyDNSLookup LogType = "ProxyDNSLookup"
|
||||
|
||||
LogType_ProxyTCPConnect LogType = "ProxyTCPConnect"
|
||||
|
||||
LogType_ProxyTLSHandshake LogType = "ProxyTLSHandshake"
|
||||
|
||||
LogType_ProxyConnectRemote LogType = "ProxyConnectRemote"
|
||||
|
||||
LogType_ResponseHeader LogType = "ResponseHeader"
|
||||
|
||||
LogType_ResponseBody LogType = "ResponseBody"
|
||||
)
|
||||
|
||||
type Log struct {
|
||||
Id string `json:"id"`
|
||||
Type LogType `json:"type"`
|
||||
Time time.Time
|
||||
Msg any `json:"msg"`
|
||||
}
|
||||
|
||||
// Connection Management Options
|
||||
type ClientOption struct {
|
||||
Logger func(Log) //debuggable
|
||||
H3 bool //开启http3
|
||||
OrderHeaders []string //order headers
|
||||
Ja3Spec ja3.Ja3Spec //custom ja3Spec,use ja3.CreateSpecWithStr or ja3.CreateSpecWithId create
|
||||
@@ -57,6 +87,7 @@ type ClientOption struct {
|
||||
|
||||
// Options for sending requests
|
||||
type RequestOption struct {
|
||||
Logger func(Log) //debuggable
|
||||
H3 bool //开启http3
|
||||
OrderHeaders []string //order headers
|
||||
Ja3Spec ja3.Ja3Spec //custom ja3Spec,use ja3.CreateSpecWithStr or ja3.CreateSpecWithId create
|
||||
@@ -255,6 +286,30 @@ func (obj *Client) newRequestOption(option RequestOption) RequestOption {
|
||||
if !option.H3 {
|
||||
option.H3 = obj.option.H3
|
||||
}
|
||||
if option.Logger == nil {
|
||||
option.Logger = obj.option.Logger
|
||||
}
|
||||
if option.Headers == nil {
|
||||
option.Headers = obj.option.Headers
|
||||
}
|
||||
if option.Timeout == 0 {
|
||||
option.Timeout = obj.option.Timeout
|
||||
}
|
||||
if option.ResponseHeaderTimeout == 0 {
|
||||
option.ResponseHeaderTimeout = obj.option.ResponseHeaderTimeout
|
||||
}
|
||||
if option.TlsHandshakeTimeout == 0 {
|
||||
option.TlsHandshakeTimeout = obj.option.TlsHandshakeTimeout
|
||||
}
|
||||
if option.DialTimeout == 0 {
|
||||
option.DialTimeout = obj.option.DialTimeout
|
||||
}
|
||||
if option.KeepAlive == 0 {
|
||||
option.KeepAlive = obj.option.KeepAlive
|
||||
}
|
||||
if option.LocalAddr == nil {
|
||||
option.LocalAddr = obj.option.LocalAddr
|
||||
}
|
||||
if option.TlsConfig == nil {
|
||||
option.TlsConfig = obj.option.TlsConfig
|
||||
}
|
||||
|
||||
29
requests.go
29
requests.go
@@ -54,11 +54,15 @@ type reqCtxData struct {
|
||||
addrType gtls.AddrType //first ip type
|
||||
dns *net.UDPAddr
|
||||
isNewConn bool
|
||||
logger func(Log)
|
||||
requestId string
|
||||
}
|
||||
|
||||
func NewReqCtxData(ctx context.Context, option *RequestOption) (*reqCtxData, error) {
|
||||
//init ctxData
|
||||
ctxData := new(reqCtxData)
|
||||
ctxData.requestId = tools.NaoId()
|
||||
ctxData.logger = option.Logger
|
||||
ctxData.h3 = option.H3
|
||||
ctxData.tlsConfig = option.TlsConfig
|
||||
ctxData.utlsConfig = option.UtlsConfig
|
||||
@@ -261,11 +265,7 @@ func (obj *Client) request(ctx context.Context, option *RequestOption) (response
|
||||
return
|
||||
}
|
||||
}
|
||||
response.bar = option.Bar
|
||||
response.disUnzip = option.DisUnZip
|
||||
response.disDecode = option.DisDecode
|
||||
response.stream = option.Stream
|
||||
|
||||
response.requestOption = option
|
||||
//init headers and orderheaders,befor init ctxData
|
||||
headers, orderHeaders, err := option.initHeaders()
|
||||
if err != nil {
|
||||
@@ -284,7 +284,7 @@ func (obj *Client) request(ctx context.Context, option *RequestOption) (response
|
||||
}
|
||||
}
|
||||
//init ctxData
|
||||
ctxData, err := NewReqCtxData(ctx, option)
|
||||
response.reqCtxData, err = NewReqCtxData(ctx, option)
|
||||
if err != nil {
|
||||
return response, tools.WrapError(err, " reqCtxData init error")
|
||||
}
|
||||
@@ -293,15 +293,15 @@ func (obj *Client) request(ctx context.Context, option *RequestOption) (response
|
||||
}
|
||||
//设置 h1 请求头顺序
|
||||
if orderHeaders != nil {
|
||||
ctxData.orderHeaders = orderHeaders
|
||||
response.reqCtxData.orderHeaders = orderHeaders
|
||||
} else {
|
||||
ctxData.orderHeaders = ja3.DefaultOrderHeaders()
|
||||
response.reqCtxData.orderHeaders = ja3.DefaultOrderHeaders()
|
||||
}
|
||||
//init ctx,cnl
|
||||
if option.Timeout > 0 { //超时
|
||||
response.ctx, response.cnl = context.WithTimeout(CreateReqCtx(ctx, ctxData), option.Timeout)
|
||||
response.ctx, response.cnl = context.WithTimeout(CreateReqCtx(ctx, response.reqCtxData), option.Timeout)
|
||||
} else {
|
||||
response.ctx, response.cnl = context.WithCancel(CreateReqCtx(ctx, ctxData))
|
||||
response.ctx, response.cnl = context.WithCancel(CreateReqCtx(ctx, response.reqCtxData))
|
||||
}
|
||||
//init url
|
||||
href, err := option.initParams()
|
||||
@@ -338,7 +338,7 @@ func (obj *Client) request(ctx context.Context, option *RequestOption) (response
|
||||
}
|
||||
|
||||
//init ws
|
||||
if ctxData.isWs {
|
||||
if response.reqCtxData.isWs {
|
||||
websocket.SetClientHeadersWithOption(reqs.Header, option.WsOption)
|
||||
}
|
||||
|
||||
@@ -369,7 +369,6 @@ func (obj *Client) request(ctx context.Context, option *RequestOption) (response
|
||||
}
|
||||
//send req
|
||||
response.response, err = obj.do(reqs, option)
|
||||
response.isNewConn = ctxData.isNewConn
|
||||
if err != nil {
|
||||
err = tools.WrapError(err, "client do error")
|
||||
return
|
||||
@@ -381,14 +380,14 @@ func (obj *Client) request(ctx context.Context, option *RequestOption) (response
|
||||
if response.Body() != nil {
|
||||
response.rawConn = response.Body().(*readWriteCloser)
|
||||
}
|
||||
if !response.disUnzip {
|
||||
response.disUnzip = response.response.Uncompressed
|
||||
if !response.requestOption.DisUnZip {
|
||||
response.requestOption.DisUnZip = response.response.Uncompressed
|
||||
}
|
||||
if response.response.StatusCode == 101 {
|
||||
response.webSocket = websocket.NewClientConn(response.rawConn.Conn(), websocket.GetResponseHeaderOption(response.response.Header))
|
||||
} else if strings.Contains(response.response.Header.Get("Content-Type"), "text/event-stream") {
|
||||
response.sse = newSse(response.Body())
|
||||
} else if !response.disUnzip {
|
||||
} else if !response.requestOption.DisUnZip {
|
||||
var unCompressionBody io.ReadCloser
|
||||
unCompressionBody, err = tools.CompressionDecode(response.Body(), response.ContentEncoding())
|
||||
if err != nil {
|
||||
|
||||
50
response.go
50
response.go
@@ -9,6 +9,7 @@ import (
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"net/http"
|
||||
|
||||
@@ -21,21 +22,24 @@ import (
|
||||
)
|
||||
|
||||
type Response struct {
|
||||
rawConn *readWriteCloser
|
||||
response *http.Response
|
||||
webSocket *websocket.Conn
|
||||
sse *Sse
|
||||
ctx context.Context
|
||||
cnl context.CancelFunc
|
||||
content []byte
|
||||
encoding string
|
||||
stream bool
|
||||
disDecode bool
|
||||
disUnzip bool
|
||||
filePath string
|
||||
bar bool
|
||||
isNewConn bool
|
||||
readBody bool
|
||||
rawConn *readWriteCloser
|
||||
response *http.Response
|
||||
webSocket *websocket.Conn
|
||||
sse *Sse
|
||||
ctx context.Context
|
||||
cnl context.CancelFunc
|
||||
reqCtxData *reqCtxData
|
||||
requestOption *RequestOption
|
||||
|
||||
content []byte
|
||||
encoding string
|
||||
// stream bool
|
||||
// disDecode bool
|
||||
// disUnzip bool
|
||||
filePath string
|
||||
// bar bool
|
||||
// isNewConn bool
|
||||
readBody bool
|
||||
}
|
||||
|
||||
type Sse struct {
|
||||
@@ -273,7 +277,7 @@ func (obj *Response) Body() io.ReadCloser {
|
||||
|
||||
// return true if response is stream
|
||||
func (obj *Response) IsStream() bool {
|
||||
return obj.stream
|
||||
return obj.requestOption.Stream
|
||||
}
|
||||
|
||||
// return true if response is other stream
|
||||
@@ -294,7 +298,7 @@ func (obj *Response) ReadBody() (err error) {
|
||||
}
|
||||
obj.readBody = true
|
||||
bBody := bytes.NewBuffer(nil)
|
||||
if obj.bar && obj.ContentLength() > 0 {
|
||||
if obj.requestOption.Bar && obj.ContentLength() > 0 {
|
||||
_, err = io.Copy(&barBody{
|
||||
bar: bar.NewClient(obj.response.ContentLength),
|
||||
body: bBody,
|
||||
@@ -302,11 +306,19 @@ func (obj *Response) ReadBody() (err error) {
|
||||
} else {
|
||||
_, err = io.Copy(bBody, obj.Body())
|
||||
}
|
||||
if obj.reqCtxData.logger != nil {
|
||||
obj.reqCtxData.logger(Log{
|
||||
Id: obj.reqCtxData.requestId,
|
||||
Time: time.Now(),
|
||||
Type: LogType_ResponseBody,
|
||||
Msg: "response body",
|
||||
})
|
||||
}
|
||||
if err != nil {
|
||||
obj.ForceCloseConn()
|
||||
return errors.New("response read content error: " + err.Error())
|
||||
}
|
||||
if !obj.disDecode && obj.defaultDecode() {
|
||||
if !obj.requestOption.DisDecode && obj.defaultDecode() {
|
||||
obj.content, obj.encoding, _ = tools.Charset(bBody.Bytes(), obj.ContentType())
|
||||
} else {
|
||||
obj.content = bBody.Bytes()
|
||||
@@ -317,7 +329,7 @@ func (obj *Response) ReadBody() (err error) {
|
||||
|
||||
// conn is new conn
|
||||
func (obj *Response) IsNewConn() bool {
|
||||
return obj.isNewConn
|
||||
return obj.reqCtxData.isNewConn
|
||||
}
|
||||
|
||||
// conn proxy
|
||||
|
||||
@@ -15,14 +15,15 @@ import (
|
||||
"net/http"
|
||||
|
||||
"github.com/gospider007/gtls"
|
||||
"github.com/gospider007/http2"
|
||||
"github.com/gospider007/http3"
|
||||
"github.com/gospider007/net/http2"
|
||||
"github.com/gospider007/tools"
|
||||
"github.com/quic-go/quic-go"
|
||||
utls "github.com/refraction-networking/utls"
|
||||
)
|
||||
|
||||
type reqTask struct {
|
||||
ctxData *reqCtxData
|
||||
ctx context.Context
|
||||
cnl context.CancelFunc
|
||||
req *http.Request
|
||||
@@ -106,6 +107,7 @@ func (obj *roundTripper) newConnPool(conn *connecotr, key string) *connPool {
|
||||
pool.forceCtx, pool.forceCnl = context.WithCancelCause(obj.ctx)
|
||||
pool.safeCtx, pool.safeCnl = context.WithCancelCause(pool.forceCtx)
|
||||
pool.tasks = make(chan *reqTask)
|
||||
|
||||
pool.connPools = obj.connPools
|
||||
pool.total.Add(1)
|
||||
go pool.rwMain(conn)
|
||||
@@ -172,7 +174,8 @@ func (obj *roundTripper) dial(ctxData *reqCtxData, req *http.Request) (conn *con
|
||||
}
|
||||
}
|
||||
}
|
||||
netConn, err := obj.dialer.DialContextWithProxy(req.Context(), ctxData, "tcp", req.URL.Scheme, getAddr(req.URL), getHost(req), proxy, obj.tlsConfigClone(ctxData))
|
||||
host := getHost(req)
|
||||
netConn, err := obj.dialer.DialContextWithProxy(req.Context(), ctxData, "tcp", req.URL.Scheme, getAddr(req.URL), host, proxy, obj.tlsConfigClone(ctxData))
|
||||
if err != nil {
|
||||
return conn, err
|
||||
}
|
||||
@@ -180,24 +183,29 @@ func (obj *roundTripper) dial(ctxData *reqCtxData, req *http.Request) (conn *con
|
||||
if req.URL.Scheme == "https" {
|
||||
ctx, cnl := context.WithTimeout(req.Context(), ctxData.tlsHandshakeTimeout)
|
||||
defer cnl()
|
||||
disHttp2 := ctxData.isWs || ctxData.forceHttp1
|
||||
if ctxData.ja3Spec.IsSet() {
|
||||
tlsConfig := obj.utlsConfigClone(ctxData)
|
||||
if ctxData.forceHttp1 {
|
||||
tlsConfig.NextProtos = []string{"http/1.1"}
|
||||
}
|
||||
tlsConn, err := obj.dialer.addJa3Tls(ctx, netConn, getHost(req), ctxData.isWs || ctxData.forceHttp1, ctxData.ja3Spec, tlsConfig)
|
||||
if err != nil {
|
||||
if tlsConn, err := obj.dialer.addJa3Tls(ctx, netConn, host, disHttp2, ctxData.ja3Spec, obj.utlsConfigClone(ctxData)); err != nil {
|
||||
return conn, tools.WrapError(err, "add ja3 tls error")
|
||||
} else {
|
||||
h2 = tlsConn.ConnectionState().NegotiatedProtocol == "h2"
|
||||
netConn = tlsConn
|
||||
}
|
||||
h2 = tlsConn.ConnectionState().NegotiatedProtocol == "h2"
|
||||
netConn = tlsConn
|
||||
} else {
|
||||
tlsConn, err := obj.dialer.addTls(ctx, netConn, getHost(req), ctxData.isWs || ctxData.forceHttp1, obj.tlsConfigClone(ctxData))
|
||||
if err != nil {
|
||||
if tlsConn, err := obj.dialer.addTls(ctx, netConn, host, disHttp2, obj.tlsConfigClone(ctxData)); err != nil {
|
||||
return conn, tools.WrapError(err, "add tls error")
|
||||
} else {
|
||||
h2 = tlsConn.ConnectionState().NegotiatedProtocol == "h2"
|
||||
netConn = tlsConn
|
||||
}
|
||||
h2 = tlsConn.ConnectionState().NegotiatedProtocol == "h2"
|
||||
netConn = tlsConn
|
||||
}
|
||||
if ctxData.logger != nil {
|
||||
ctxData.logger(Log{
|
||||
Id: ctxData.requestId,
|
||||
Time: time.Now(),
|
||||
Type: LogType_TLSHandshake,
|
||||
Msg: host,
|
||||
})
|
||||
}
|
||||
}
|
||||
conne := obj.newConnecotr(netConn)
|
||||
@@ -206,7 +214,7 @@ func (obj *roundTripper) dial(ctxData *reqCtxData, req *http.Request) (conn *con
|
||||
}
|
||||
if h2 {
|
||||
if conne.h2RawConn, err = http2.NewClientConn(func() {
|
||||
conne.safeCnl(errors.New("http2 client close"))
|
||||
conne.forceCnl(errors.New("http2 client close"))
|
||||
}, netConn, ctxData.h2Ja3Spec); err != nil {
|
||||
return conne, err
|
||||
}
|
||||
@@ -283,6 +291,7 @@ func (obj *roundTripper) newReqTask(req *http.Request, ctxData *reqCtxData) *req
|
||||
}
|
||||
task := new(reqTask)
|
||||
task.req = req
|
||||
task.ctxData = ctxData
|
||||
task.emptyPool = make(chan struct{})
|
||||
task.orderHeaders = ctxData.orderHeaders
|
||||
if ctxData.h2Ja3Spec.OrderHeaders != nil {
|
||||
|
||||
8
rw.go
8
rw.go
@@ -21,8 +21,10 @@ func (obj *readWriteCloser) InPool() bool {
|
||||
return obj.conn.inPool
|
||||
}
|
||||
func (obj *readWriteCloser) Proxy() *url.URL {
|
||||
p := *obj.conn.proxy
|
||||
return &p
|
||||
if obj.conn.proxy == nil {
|
||||
return nil
|
||||
}
|
||||
return cloneUrl(obj.conn.proxy)
|
||||
}
|
||||
|
||||
var ErrgospiderBodyClose = errors.New("gospider body close error")
|
||||
@@ -31,8 +33,8 @@ func (obj *readWriteCloser) Close() (err error) {
|
||||
if !obj.InPool() {
|
||||
obj.ForceCloseConn()
|
||||
} else {
|
||||
err = obj.body.Close() //reuse conn
|
||||
obj.conn.bodyCnl(ErrgospiderBodyClose)
|
||||
err = obj.body.Close() //reuse conn
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
@@ -44,6 +44,7 @@ func TestJa3Psk(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
log.Print(ja3Spec.String())
|
||||
session, _ := requests.NewClient(nil)
|
||||
for i := 0; i < 2; i++ {
|
||||
resp, err := session.Get(nil, "https://tools.scrapfly.io/api/fp/anything", requests.RequestOption{
|
||||
|
||||
25
test/logger/logger_test.go
Normal file
25
test/logger/logger_test.go
Normal file
@@ -0,0 +1,25 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
"testing"
|
||||
|
||||
"github.com/gospider007/requests"
|
||||
)
|
||||
|
||||
func TestLogger(t *testing.T) {
|
||||
for i := 0; i < 2; i++ {
|
||||
response, err := requests.Get(nil, "https://www.httpbin.org", requests.RequestOption{
|
||||
Ja3: true,
|
||||
Logger: func(l requests.Log) {
|
||||
log.Print(l)
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
log.Print(len(response.Text()))
|
||||
log.Print(response.Status())
|
||||
log.Print(response.Proto())
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user