mirror of
https://github.com/gospider007/requests.git
synced 2025-12-24 13:57:52 +08:00
sync
This commit is contained in:
17
client.go
17
client.go
@@ -62,6 +62,23 @@ func (obj *Client) SetGetProxy(getProxy func(ctx context.Context, url *url.URL)
|
||||
obj.transport.setGetProxy(getProxy)
|
||||
}
|
||||
|
||||
// Modifying the client's proxy
|
||||
func (obj *Client) SetProxys(proxyUrls []string) (err error) {
|
||||
for _, proxy := range proxyUrls {
|
||||
_, err = gtls.VerifyProxy(proxy)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
obj.option.Proxys = proxyUrls
|
||||
return
|
||||
}
|
||||
|
||||
// Modify the proxy method of the client
|
||||
func (obj *Client) SetGetProxys(getProxys func(ctx context.Context, url *url.URL) ([]string, error)) {
|
||||
obj.transport.setGetProxys(getProxys)
|
||||
}
|
||||
|
||||
// Close idle connections. If the connection is in use, wait until it ends before closing
|
||||
func (obj *Client) CloseConns() {
|
||||
obj.transport.closeConns()
|
||||
|
||||
2
conn.go
2
conn.go
@@ -30,7 +30,7 @@ type connecotr struct {
|
||||
rawConn net.Conn
|
||||
h2RawConn *http2.Http2ClientConn
|
||||
h3RawConn http3.RoundTripper
|
||||
proxy *url.URL
|
||||
proxys []*url.URL
|
||||
r *bufio.Reader
|
||||
w *bufio.Writer
|
||||
pr *pipCon
|
||||
|
||||
163
dial.go
163
dial.go
@@ -8,6 +8,7 @@ import (
|
||||
"io"
|
||||
"net"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -157,62 +158,96 @@ func (obj *DialClient) DialContext(ctx context.Context, ctxData *reqCtxData, net
|
||||
func (obj *DialClient) ProxyDialContext(ctx context.Context, ctxData *reqCtxData, network string, addr string) (net.Conn, error) {
|
||||
return obj.dialContext(ctx, ctxData, network, addr, true)
|
||||
}
|
||||
func (obj *DialClient) DialContextWithProxy(ctx context.Context, ctxData *reqCtxData, network string, scheme string, addr string, host string, proxyUrl *url.URL, tlsConfig *tls.Config) (net.Conn, error) {
|
||||
|
||||
func (obj *DialClient) DialProxyContext(ctx context.Context, ctxData *reqCtxData, network string, proxyTlsConfig *tls.Config, proxyUrls ...*url.URL) (net.Conn, error) {
|
||||
proxyLen := len(proxyUrls)
|
||||
if proxyLen < 2 {
|
||||
return nil, errors.New("proxyUrls is nil")
|
||||
}
|
||||
var conn net.Conn
|
||||
var err error
|
||||
for index := range proxyLen - 1 {
|
||||
oneProxy := proxyUrls[index]
|
||||
remoteUrl := proxyUrls[index+1]
|
||||
if index == 0 {
|
||||
conn, err = obj.dialProxyContext(ctx, ctxData, network, oneProxy)
|
||||
if err != nil {
|
||||
return conn, err
|
||||
}
|
||||
}
|
||||
conn, err = obj.verifyProxyToRemote(ctx, ctxData, conn, proxyTlsConfig, oneProxy, remoteUrl)
|
||||
}
|
||||
return conn, err
|
||||
}
|
||||
func getProxyAddr(proxyUrl *url.URL) (addr string, err error) {
|
||||
if proxyUrl.Port() == "" {
|
||||
switch proxyUrl.Scheme {
|
||||
case "http":
|
||||
addr = net.JoinHostPort(proxyUrl.Hostname(), "80")
|
||||
case "https":
|
||||
addr = net.JoinHostPort(proxyUrl.Hostname(), "443")
|
||||
case "socks5":
|
||||
addr = net.JoinHostPort(proxyUrl.Hostname(), "1080")
|
||||
default:
|
||||
return "", errors.New("not support scheme")
|
||||
}
|
||||
} else {
|
||||
addr = net.JoinHostPort(proxyUrl.Hostname(), proxyUrl.Port())
|
||||
}
|
||||
return
|
||||
}
|
||||
func (obj *DialClient) dialProxyContext(ctx context.Context, ctxData *reqCtxData, network string, proxyUrl *url.URL) (net.Conn, error) {
|
||||
if proxyUrl == nil {
|
||||
return nil, errors.New("proxyUrl is nil")
|
||||
}
|
||||
if ctxData == nil {
|
||||
ctxData = &reqCtxData{}
|
||||
}
|
||||
if proxyUrl == nil {
|
||||
return obj.DialContext(ctx, ctxData, network, addr)
|
||||
addr, err := getProxyAddr(proxyUrl)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if proxyUrl.Port() == "" {
|
||||
if proxyUrl.Scheme == "http" {
|
||||
proxyUrl.Host = net.JoinHostPort(proxyUrl.Hostname(), "80")
|
||||
} else if proxyUrl.Scheme == "https" {
|
||||
proxyUrl.Host = net.JoinHostPort(proxyUrl.Hostname(), "443")
|
||||
return obj.ProxyDialContext(ctx, ctxData, network, addr)
|
||||
}
|
||||
|
||||
func (obj *DialClient) verifyProxyToRemote(ctx context.Context, ctxData *reqCtxData, conn net.Conn, proxyTlsConfig *tls.Config, proxyUrl *url.URL, remoteUrl *url.URL) (net.Conn, error) {
|
||||
var err error
|
||||
if proxyUrl.Scheme == "https" {
|
||||
if conn, err = obj.addTls(ctx, conn, proxyUrl.Host, true, proxyTlsConfig); err != nil {
|
||||
return conn, err
|
||||
}
|
||||
if ctxData.logger != nil {
|
||||
ctxData.logger(Log{
|
||||
Id: ctxData.requestId,
|
||||
Time: time.Now(),
|
||||
Type: LogType_ProxyTLSHandshake,
|
||||
Msg: proxyUrl.String(),
|
||||
})
|
||||
}
|
||||
}
|
||||
switch proxyUrl.Scheme {
|
||||
case "http", "https":
|
||||
conn, err := obj.ProxyDialContext(ctx, ctxData, network, net.JoinHostPort(proxyUrl.Hostname(), proxyUrl.Port()))
|
||||
if err != nil {
|
||||
return conn, err
|
||||
} else if proxyUrl.Scheme == "https" {
|
||||
if conn, err = obj.addTls(ctx, conn, proxyUrl.Host, true, tlsConfig); err != nil {
|
||||
return conn, err
|
||||
}
|
||||
if ctxData.logger != nil {
|
||||
ctxData.logger(Log{
|
||||
Id: ctxData.requestId,
|
||||
Time: time.Now(),
|
||||
Type: LogType_ProxyTLSHandshake,
|
||||
Msg: addr,
|
||||
})
|
||||
}
|
||||
}
|
||||
err = obj.clientVerifyHttps(ctx, scheme, proxyUrl, addr, host, conn)
|
||||
err = obj.clientVerifyHttps(ctx, conn, proxyUrl, remoteUrl)
|
||||
if ctxData.logger != nil {
|
||||
ctxData.logger(Log{
|
||||
Id: ctxData.requestId,
|
||||
Time: time.Now(),
|
||||
Type: LogType_ProxyConnectRemote,
|
||||
Msg: addr,
|
||||
Msg: remoteUrl.String(),
|
||||
})
|
||||
}
|
||||
return conn, err
|
||||
case "socks5":
|
||||
conn, err := obj.Socks5Proxy(ctx, ctxData, network, addr, proxyUrl)
|
||||
err = obj.socks5ProxyWithConn(ctx, conn, proxyUrl, remoteUrl)
|
||||
if ctxData.logger != nil {
|
||||
ctxData.logger(Log{
|
||||
Id: ctxData.requestId,
|
||||
Time: time.Now(),
|
||||
Type: LogType_ProxyTCPConnect,
|
||||
Msg: addr,
|
||||
Msg: remoteUrl.String(),
|
||||
})
|
||||
}
|
||||
return conn, err
|
||||
default:
|
||||
return nil, errors.New("proxyUrl Scheme error")
|
||||
}
|
||||
return conn, err
|
||||
}
|
||||
func (obj *DialClient) loadHost(host string) (string, bool) {
|
||||
msgDataAny, ok := obj.dnsIpData.Load(host)
|
||||
@@ -232,7 +267,7 @@ func (obj *DialClient) addrToIp(host string, ips []net.IPAddr, addrType gtls.Add
|
||||
obj.dnsIpData.Store(host, msgClient{time: time.Now(), host: ip.String()})
|
||||
return ip.String(), nil
|
||||
}
|
||||
func (obj *DialClient) clientVerifySocks5(proxyUrl *url.URL, addr string, conn net.Conn) (err error) {
|
||||
func (obj *DialClient) clientVerifySocks5(conn net.Conn, proxyUrl *url.URL, remoteUrl *url.URL) (err error) {
|
||||
if _, err = conn.Write([]byte{5, 2, 0, 2}); err != nil {
|
||||
return
|
||||
}
|
||||
@@ -283,10 +318,26 @@ func (obj *DialClient) clientVerifySocks5(proxyUrl *url.URL, addr string, conn n
|
||||
err = errors.New("not support auth format")
|
||||
return
|
||||
}
|
||||
var host string
|
||||
|
||||
host := remoteUrl.Hostname()
|
||||
var port int
|
||||
if host, port, err = gtls.SplitHostPort(addr); err != nil {
|
||||
return
|
||||
if cport := remoteUrl.Port(); cport == "" {
|
||||
switch remoteUrl.Scheme {
|
||||
case "http":
|
||||
port = 80
|
||||
case "https":
|
||||
port = 443
|
||||
case "socks5":
|
||||
port = 1080
|
||||
default:
|
||||
err = errors.New("not support scheme")
|
||||
return
|
||||
}
|
||||
} else {
|
||||
port, err = strconv.Atoi(cport)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
writeCon := []byte{5, 1, 0}
|
||||
ip, ipInt := gtls.ParseHost(host)
|
||||
@@ -431,13 +482,13 @@ func (obj *DialClient) addJa3Tls(ctx context.Context, conn net.Conn, host string
|
||||
}
|
||||
return ja3.NewClient(ctx, conn, ja3Spec, disHttp2, tlsConfig)
|
||||
}
|
||||
func (obj *DialClient) Socks5Proxy(ctx context.Context, ctxData *reqCtxData, network string, addr string, proxyUrl *url.URL) (conn net.Conn, err error) {
|
||||
func (obj *DialClient) Socks5Proxy(ctx context.Context, ctxData *reqCtxData, network string, proxyUrl *url.URL, remoteUrl *url.URL) (conn net.Conn, err error) {
|
||||
if conn, err = obj.DialContext(ctx, ctxData, network, net.JoinHostPort(proxyUrl.Hostname(), proxyUrl.Port())); err != nil {
|
||||
return
|
||||
}
|
||||
didVerify := make(chan struct{})
|
||||
go func() {
|
||||
if err = obj.clientVerifySocks5(proxyUrl, addr, conn); err != nil {
|
||||
if err = obj.clientVerifySocks5(conn, proxyUrl, remoteUrl); err != nil {
|
||||
conn.Close()
|
||||
}
|
||||
close(didVerify)
|
||||
@@ -449,7 +500,23 @@ func (obj *DialClient) Socks5Proxy(ctx context.Context, ctxData *reqCtxData, net
|
||||
return
|
||||
}
|
||||
}
|
||||
func (obj *DialClient) clientVerifyHttps(ctx context.Context, scheme string, proxyUrl *url.URL, addr string, host string, conn net.Conn) (err error) {
|
||||
func (obj *DialClient) socks5ProxyWithConn(ctx context.Context, conn net.Conn, proxyUrl *url.URL, remoteUrl *url.URL) (err error) {
|
||||
didVerify := make(chan struct{})
|
||||
go func() {
|
||||
if err = obj.clientVerifySocks5(conn, proxyUrl, remoteUrl); err != nil {
|
||||
conn.Close()
|
||||
}
|
||||
close(didVerify)
|
||||
}()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
conn.Close()
|
||||
return context.Cause(ctx)
|
||||
case <-didVerify:
|
||||
return
|
||||
}
|
||||
}
|
||||
func (obj *DialClient) clientVerifyHttps(ctx context.Context, conn net.Conn, proxyUrl *url.URL, remoteUrl *url.URL) (err error) {
|
||||
hdr := make(http.Header)
|
||||
hdr.Set("User-Agent", tools.UserAgent)
|
||||
if proxyUrl.User != nil {
|
||||
@@ -457,26 +524,16 @@ func (obj *DialClient) clientVerifyHttps(ctx context.Context, scheme string, pro
|
||||
hdr.Set("Proxy-Authorization", "Basic "+tools.Base64Encode(proxyUrl.User.Username()+":"+password))
|
||||
}
|
||||
}
|
||||
cHost := host
|
||||
_, hport, _ := net.SplitHostPort(host)
|
||||
if hport == "" {
|
||||
_, aport, _ := net.SplitHostPort(addr)
|
||||
if aport != "" {
|
||||
cHost = net.JoinHostPort(cHost, aport)
|
||||
} else if scheme == "http" {
|
||||
cHost = net.JoinHostPort(cHost, "80")
|
||||
} else if scheme == "https" {
|
||||
cHost = net.JoinHostPort(cHost, "443")
|
||||
} else {
|
||||
return errors.New("clientVerifyHttps not found port")
|
||||
}
|
||||
addr, err := getProxyAddr(remoteUrl)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
connectReq, err := NewRequestWithContext(ctx, http.MethodConnect, &url.URL{Opaque: addr}, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
connectReq.Header = hdr
|
||||
connectReq.Host = cHost
|
||||
connectReq.Host = remoteUrl.Host
|
||||
if err = connectReq.Write(conn); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
4
go.mod
4
go.mod
@@ -34,7 +34,7 @@ require (
|
||||
github.com/gaukas/godicttls v0.0.4 // indirect
|
||||
github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
|
||||
github.com/google/gopacket v1.1.19 // indirect
|
||||
github.com/google/pprof v0.0.0-20241122213907-cbe949e5a41b // indirect
|
||||
github.com/google/pprof v0.0.0-20241203143554-1e3fdc7de467 // indirect
|
||||
github.com/gospider007/blog v0.0.0-20231121084103-59a004dafccf // indirect
|
||||
github.com/gospider007/kinds v0.0.0-20240929092451-8f867acde255 // indirect
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
@@ -50,7 +50,7 @@ require (
|
||||
github.com/miekg/dns v1.1.62 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/nwaples/rardecode/v2 v2.0.0 // indirect
|
||||
github.com/nwaples/rardecode/v2 v2.0.1 // indirect
|
||||
github.com/onsi/ginkgo/v2 v2.22.0 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.21 // indirect
|
||||
github.com/quic-go/qpack v0.5.1 // indirect
|
||||
|
||||
6
go.sum
6
go.sum
@@ -58,6 +58,7 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
|
||||
github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
|
||||
github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=
|
||||
github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
||||
@@ -90,6 +91,8 @@ github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OI
|
||||
github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
|
||||
github.com/google/pprof v0.0.0-20241122213907-cbe949e5a41b h1:SXO0REt4iu865upYCk8aKBBJQ4BqoE0ReP23ClMu60s=
|
||||
github.com/google/pprof v0.0.0-20241122213907-cbe949e5a41b/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144=
|
||||
github.com/google/pprof v0.0.0-20241203143554-1e3fdc7de467 h1:keEZFtbLJugfE0qHn+Ge1JCE71spzkchQobDf3mzS/4=
|
||||
github.com/google/pprof v0.0.0-20241203143554-1e3fdc7de467/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144=
|
||||
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
|
||||
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
|
||||
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
|
||||
@@ -147,6 +150,7 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/libdns/libdns v0.2.2 h1:O6ws7bAfRPaBsgAYt8MDe2HcNBGC29hkZ9MX2eUSX3s=
|
||||
github.com/libdns/libdns v0.2.2/go.mod h1:4Bj9+5CQiNMVGf87wjX4CY3HQJypUHRuLvlsfsZqLWQ=
|
||||
github.com/mholt/acmez v1.2.0 h1:1hhLxSgY5FvH5HCnGUuwbKY2VQVo8IU7rxXKSnZ7F30=
|
||||
github.com/mholt/acmez/v2 v2.0.3 h1:CgDBlEwg3QBp6s45tPQmFIBrkRIkBT4rW4orMM6p4sw=
|
||||
github.com/mholt/acmez/v2 v2.0.3/go.mod h1:pQ1ysaDeGrIMvJ9dfJMk5kJNkn7L2sb3UhyrX6Q91cw=
|
||||
github.com/mholt/archives v0.0.0-20241123153623-b85dd98f4298 h1:KyCSnt0fdY6Bi4vxM0On3zlonJwnDtTYSpMg7Y1ZNhs=
|
||||
@@ -160,6 +164,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
|
||||
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
|
||||
github.com/nwaples/rardecode/v2 v2.0.0 h1:C+58s2TPUdvgVFioQLpUNmIjTchtegHGS2xTNiKBYXA=
|
||||
github.com/nwaples/rardecode/v2 v2.0.0/go.mod h1:yntwv/HfMc/Hbvtq9I19D1n58te3h6KsqCf3GxyfBGY=
|
||||
github.com/nwaples/rardecode/v2 v2.0.1 h1:3MN6/R+Y4c7e+21U3yhWuUcf72sYmcmr6jtiuAVSH1A=
|
||||
github.com/nwaples/rardecode/v2 v2.0.1/go.mod h1:yntwv/HfMc/Hbvtq9I19D1n58te3h6KsqCf3GxyfBGY=
|
||||
github.com/onsi/ginkgo/v2 v2.22.0 h1:Yed107/8DjTr0lKCNt7Dn8yQ6ybuDRQoMGrNFKzMfHg=
|
||||
github.com/onsi/ginkgo/v2 v2.22.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo=
|
||||
github.com/onsi/gomega v1.34.2 h1:pNCwDkzrsv7MS9kpaQvVb1aVLahQXyJ/Tv5oAZMI3i8=
|
||||
|
||||
13
option.go
13
option.go
@@ -52,6 +52,7 @@ type ClientOption struct {
|
||||
Ja3Spec ja3.Ja3Spec //custom ja3Spec,use ja3.CreateSpecWithStr or ja3.CreateSpecWithId create
|
||||
H2Ja3Spec ja3.H2Ja3Spec //h2 fingerprint
|
||||
Proxy string //proxy,support https,http,socks5
|
||||
Proxys []string //proxy list,support https,http,socks5
|
||||
ForceHttp1 bool //force use http1 send requests
|
||||
Ja3 bool //enable ja3 fingerprint
|
||||
DisCookie bool //disable cookies
|
||||
@@ -81,8 +82,9 @@ type ClientOption struct {
|
||||
UtlsConfig *utls.Config
|
||||
|
||||
//other option
|
||||
UserAgent string //headers User-Agent value
|
||||
GetProxy func(ctx context.Context, url *url.URL) (string, error) //proxy callback:support https,http,socks5 proxy
|
||||
UserAgent string //headers User-Agent value
|
||||
GetProxy func(ctx context.Context, url *url.URL) (string, error) //proxy callback:support https,http,socks5 proxy
|
||||
GetProxys func(ctx context.Context, url *url.URL) ([]string, error) //proxys callback:support https,http,socks5 proxy
|
||||
GetAddrType func(host string) gtls.AddrType
|
||||
}
|
||||
|
||||
@@ -94,6 +96,7 @@ type RequestOption struct {
|
||||
Ja3Spec ja3.Ja3Spec //custom ja3Spec,use ja3.CreateSpecWithStr or ja3.CreateSpecWithId create
|
||||
H2Ja3Spec ja3.H2Ja3Spec //custom h2 fingerprint
|
||||
Proxy string //proxy,support http,https,socks5,example:http://127.0.0.1:7005
|
||||
Proxys []string //proxy list,support http,https,socks5,example:http://127.0.0.1:7005
|
||||
ForceHttp1 bool //force use http1 send requests
|
||||
Ja3 bool //enable ja3 fingerprint
|
||||
DisCookie bool //disable cookies,not use cookies
|
||||
@@ -264,6 +267,9 @@ func (obj *Client) newRequestOption(option RequestOption) RequestOption {
|
||||
if option.Proxy == "" {
|
||||
option.Proxy = obj.option.Proxy
|
||||
}
|
||||
if len(option.Proxys) == 0 {
|
||||
option.Proxys = obj.option.Proxys
|
||||
}
|
||||
if !option.ForceHttp1 {
|
||||
option.ForceHttp1 = obj.option.ForceHttp1
|
||||
}
|
||||
@@ -376,6 +382,9 @@ func (obj *Client) newRequestOption(option RequestOption) RequestOption {
|
||||
if !option.Ja3Spec.IsSet() && option.Ja3 {
|
||||
option.Ja3Spec = ja3.DefaultJa3Spec()
|
||||
}
|
||||
if option.UserAgent == "" {
|
||||
option.UserAgent = obj.option.UserAgent
|
||||
}
|
||||
if option.DisCookie {
|
||||
option.Jar = nil
|
||||
}
|
||||
|
||||
14
requests.go
14
requests.go
@@ -35,6 +35,7 @@ type reqCtxData struct {
|
||||
forceHttp1 bool
|
||||
maxRedirect int
|
||||
proxy *url.URL
|
||||
proxys []*url.URL
|
||||
disProxy bool
|
||||
disAlive bool
|
||||
orderHeaders []string
|
||||
@@ -102,6 +103,16 @@ func NewReqCtxData(ctx context.Context, option *RequestOption) (*reqCtxData, err
|
||||
}
|
||||
ctxData.proxy = tempProxy
|
||||
}
|
||||
if l := len(option.Proxys); l > 0 {
|
||||
ctxData.proxys = make([]*url.URL, l)
|
||||
for i, proxy := range option.Proxys {
|
||||
tempProxy, err := gtls.VerifyProxy(proxy)
|
||||
if err != nil {
|
||||
return ctxData, tools.WrapError(errFatal, errors.New("tempRequest init proxy error"), err)
|
||||
}
|
||||
ctxData.proxys[i] = tempProxy
|
||||
}
|
||||
}
|
||||
return ctxData, nil
|
||||
}
|
||||
func CreateReqCtx(ctx context.Context, ctxData *reqCtxData) context.Context {
|
||||
@@ -271,6 +282,9 @@ func (obj *Client) request(ctx context.Context, option *RequestOption) (response
|
||||
if err != nil {
|
||||
return response, tools.WrapError(err, errors.New("tempRequest init headers error"), err)
|
||||
}
|
||||
if headers != nil && option.UserAgent != "" {
|
||||
headers.Set("User-Agent", option.UserAgent)
|
||||
}
|
||||
if orderHeaders == nil {
|
||||
orderHeaders = option.OrderHeaders
|
||||
}
|
||||
|
||||
@@ -333,9 +333,9 @@ func (obj *Response) IsNewConn() bool {
|
||||
}
|
||||
|
||||
// conn proxy
|
||||
func (obj *Response) Proxy() *url.URL {
|
||||
func (obj *Response) Proxys() []*url.URL {
|
||||
if obj.rawConn != nil {
|
||||
return obj.rawConn.Proxy()
|
||||
return obj.rawConn.Proxys()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -62,6 +62,7 @@ type roundTripper struct {
|
||||
tlsConfig *tls.Config
|
||||
utlsConfig *utls.Config
|
||||
getProxy func(ctx context.Context, url *url.URL) (string, error)
|
||||
getProxys func(ctx context.Context, url *url.URL) ([]string, error)
|
||||
}
|
||||
|
||||
func newRoundTripper(preCtx context.Context, option ClientOption) *roundTripper {
|
||||
@@ -97,6 +98,7 @@ func newRoundTripper(preCtx context.Context, option ClientOption) *roundTripper
|
||||
cnl: cnl,
|
||||
dialer: dialClient,
|
||||
getProxy: option.GetProxy,
|
||||
getProxys: option.GetProxys,
|
||||
connPools: newConnPools(),
|
||||
}
|
||||
}
|
||||
@@ -178,20 +180,52 @@ func (obj *roundTripper) dial(ctxData *reqCtxData, req *http.Request) (conn *con
|
||||
return obj.http3Dial(ctxData, req)
|
||||
}
|
||||
}
|
||||
var proxy *url.URL
|
||||
var proxys []*url.URL
|
||||
if !ctxData.disProxy {
|
||||
if proxy = cloneUrl(ctxData.proxy); proxy == nil && obj.getProxy != nil {
|
||||
proxyStr, err := obj.getProxy(req.Context(), proxy)
|
||||
if ctxData.proxy != nil {
|
||||
proxys = []*url.URL{cloneUrl(ctxData.proxy)}
|
||||
}
|
||||
if len(proxys) == 0 && len(ctxData.proxys) > 0 {
|
||||
proxys = make([]*url.URL, len(ctxData.proxys))
|
||||
for i, proxy := range ctxData.proxys {
|
||||
proxys[i] = cloneUrl(proxy)
|
||||
}
|
||||
}
|
||||
if len(proxys) == 0 && obj.getProxy != nil {
|
||||
proxyStr, err := obj.getProxy(req.Context(), req.URL)
|
||||
if err != nil {
|
||||
return conn, err
|
||||
}
|
||||
if proxy, err = gtls.VerifyProxy(proxyStr); err != nil {
|
||||
proxy, err := gtls.VerifyProxy(proxyStr)
|
||||
if err != nil {
|
||||
return conn, err
|
||||
}
|
||||
proxys = []*url.URL{proxy}
|
||||
}
|
||||
if len(proxys) == 0 && obj.getProxys != nil {
|
||||
proxyStrs, err := obj.getProxys(req.Context(), req.URL)
|
||||
if err != nil {
|
||||
return conn, err
|
||||
}
|
||||
if l := len(proxyStrs); l > 0 {
|
||||
proxys = make([]*url.URL, l)
|
||||
for i, proxyStr := range proxyStrs {
|
||||
proxy, err := gtls.VerifyProxy(proxyStr)
|
||||
if err != nil {
|
||||
return conn, err
|
||||
}
|
||||
proxys[i] = proxy
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
host := getHost(req)
|
||||
netConn, err := obj.dialer.DialContextWithProxy(req.Context(), ctxData, "tcp", req.URL.Scheme, getAddr(req.URL), host, proxy, obj.tlsConfigClone(ctxData))
|
||||
var netConn net.Conn
|
||||
if len(proxys) > 0 {
|
||||
netConn, err = obj.dialer.DialProxyContext(req.Context(), ctxData, "tcp", obj.tlsConfigClone(ctxData), append(proxys, cloneUrl(req.URL))...)
|
||||
} else {
|
||||
netConn, err = obj.dialer.DialContext(req.Context(), ctxData, "tcp", getAddr(req.URL))
|
||||
}
|
||||
if err != nil {
|
||||
return conn, err
|
||||
}
|
||||
@@ -225,9 +259,7 @@ func (obj *roundTripper) dial(ctxData *reqCtxData, req *http.Request) (conn *con
|
||||
}
|
||||
}
|
||||
conne := obj.newConnecotr(netConn)
|
||||
if proxy != nil {
|
||||
conne.proxy = proxy
|
||||
}
|
||||
conne.proxys = proxys
|
||||
if h2 {
|
||||
if conne.h2RawConn, err = http2.NewClientConn(func() {
|
||||
conne.forceCnl(errors.New("http2 client close"))
|
||||
@@ -242,6 +274,9 @@ func (obj *roundTripper) dial(ctxData *reqCtxData, req *http.Request) (conn *con
|
||||
func (obj *roundTripper) setGetProxy(getProxy func(ctx context.Context, url *url.URL) (string, error)) {
|
||||
obj.getProxy = getProxy
|
||||
}
|
||||
func (obj *roundTripper) setGetProxys(getProxys func(ctx context.Context, url *url.URL) ([]string, error)) {
|
||||
obj.getProxys = getProxys
|
||||
}
|
||||
|
||||
func (obj *roundTripper) poolRoundTrip(ctxData *reqCtxData, pool *connPool, task *reqTask, key string) (isTry bool) {
|
||||
task.ctx, task.cnl = context.WithTimeout(task.req.Context(), ctxData.responseHeaderTimeout)
|
||||
|
||||
11
rw.go
11
rw.go
@@ -20,11 +20,14 @@ func (obj *readWriteCloser) Read(p []byte) (n int, err error) {
|
||||
func (obj *readWriteCloser) InPool() bool {
|
||||
return obj.conn.inPool
|
||||
}
|
||||
func (obj *readWriteCloser) Proxy() *url.URL {
|
||||
if obj.conn.proxy == nil {
|
||||
return nil
|
||||
func (obj *readWriteCloser) Proxys() []*url.URL {
|
||||
if l := len(obj.conn.proxys); l > 0 {
|
||||
proxys := make([]*url.URL, l)
|
||||
for i, proxy := range obj.conn.proxys {
|
||||
proxys[i] = cloneUrl(proxy)
|
||||
}
|
||||
}
|
||||
return cloneUrl(obj.conn.proxy)
|
||||
return obj.conn.proxys
|
||||
}
|
||||
|
||||
var ErrgospiderBodyClose = errors.New("gospider body close error")
|
||||
|
||||
@@ -11,7 +11,8 @@ func TestUseProxy(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
if resp.Proxy() != nil {
|
||||
|
||||
if resp.Proxys() != nil {
|
||||
t.Error("proxy error")
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user