This commit is contained in:
gospider
2024-12-04 16:29:02 +08:00
parent 179dd44b56
commit 1e2ba7c4a6
11 changed files with 215 additions and 73 deletions

View File

@@ -62,6 +62,23 @@ func (obj *Client) SetGetProxy(getProxy func(ctx context.Context, url *url.URL)
obj.transport.setGetProxy(getProxy)
}
// Modifying the client's proxy
func (obj *Client) SetProxys(proxyUrls []string) (err error) {
for _, proxy := range proxyUrls {
_, err = gtls.VerifyProxy(proxy)
if err != nil {
return
}
}
obj.option.Proxys = proxyUrls
return
}
// Modify the proxy method of the client
func (obj *Client) SetGetProxys(getProxys func(ctx context.Context, url *url.URL) ([]string, error)) {
obj.transport.setGetProxys(getProxys)
}
// Close idle connections. If the connection is in use, wait until it ends before closing
func (obj *Client) CloseConns() {
obj.transport.closeConns()

View File

@@ -30,7 +30,7 @@ type connecotr struct {
rawConn net.Conn
h2RawConn *http2.Http2ClientConn
h3RawConn http3.RoundTripper
proxy *url.URL
proxys []*url.URL
r *bufio.Reader
w *bufio.Writer
pr *pipCon

163
dial.go
View File

@@ -8,6 +8,7 @@ import (
"io"
"net"
"net/url"
"strconv"
"sync"
"time"
@@ -157,62 +158,96 @@ func (obj *DialClient) DialContext(ctx context.Context, ctxData *reqCtxData, net
func (obj *DialClient) ProxyDialContext(ctx context.Context, ctxData *reqCtxData, network string, addr string) (net.Conn, error) {
return obj.dialContext(ctx, ctxData, network, addr, true)
}
func (obj *DialClient) DialContextWithProxy(ctx context.Context, ctxData *reqCtxData, network string, scheme string, addr string, host string, proxyUrl *url.URL, tlsConfig *tls.Config) (net.Conn, error) {
func (obj *DialClient) DialProxyContext(ctx context.Context, ctxData *reqCtxData, network string, proxyTlsConfig *tls.Config, proxyUrls ...*url.URL) (net.Conn, error) {
proxyLen := len(proxyUrls)
if proxyLen < 2 {
return nil, errors.New("proxyUrls is nil")
}
var conn net.Conn
var err error
for index := range proxyLen - 1 {
oneProxy := proxyUrls[index]
remoteUrl := proxyUrls[index+1]
if index == 0 {
conn, err = obj.dialProxyContext(ctx, ctxData, network, oneProxy)
if err != nil {
return conn, err
}
}
conn, err = obj.verifyProxyToRemote(ctx, ctxData, conn, proxyTlsConfig, oneProxy, remoteUrl)
}
return conn, err
}
func getProxyAddr(proxyUrl *url.URL) (addr string, err error) {
if proxyUrl.Port() == "" {
switch proxyUrl.Scheme {
case "http":
addr = net.JoinHostPort(proxyUrl.Hostname(), "80")
case "https":
addr = net.JoinHostPort(proxyUrl.Hostname(), "443")
case "socks5":
addr = net.JoinHostPort(proxyUrl.Hostname(), "1080")
default:
return "", errors.New("not support scheme")
}
} else {
addr = net.JoinHostPort(proxyUrl.Hostname(), proxyUrl.Port())
}
return
}
func (obj *DialClient) dialProxyContext(ctx context.Context, ctxData *reqCtxData, network string, proxyUrl *url.URL) (net.Conn, error) {
if proxyUrl == nil {
return nil, errors.New("proxyUrl is nil")
}
if ctxData == nil {
ctxData = &reqCtxData{}
}
if proxyUrl == nil {
return obj.DialContext(ctx, ctxData, network, addr)
addr, err := getProxyAddr(proxyUrl)
if err != nil {
return nil, err
}
if proxyUrl.Port() == "" {
if proxyUrl.Scheme == "http" {
proxyUrl.Host = net.JoinHostPort(proxyUrl.Hostname(), "80")
} else if proxyUrl.Scheme == "https" {
proxyUrl.Host = net.JoinHostPort(proxyUrl.Hostname(), "443")
return obj.ProxyDialContext(ctx, ctxData, network, addr)
}
func (obj *DialClient) verifyProxyToRemote(ctx context.Context, ctxData *reqCtxData, conn net.Conn, proxyTlsConfig *tls.Config, proxyUrl *url.URL, remoteUrl *url.URL) (net.Conn, error) {
var err error
if proxyUrl.Scheme == "https" {
if conn, err = obj.addTls(ctx, conn, proxyUrl.Host, true, proxyTlsConfig); err != nil {
return conn, err
}
if ctxData.logger != nil {
ctxData.logger(Log{
Id: ctxData.requestId,
Time: time.Now(),
Type: LogType_ProxyTLSHandshake,
Msg: proxyUrl.String(),
})
}
}
switch proxyUrl.Scheme {
case "http", "https":
conn, err := obj.ProxyDialContext(ctx, ctxData, network, net.JoinHostPort(proxyUrl.Hostname(), proxyUrl.Port()))
if err != nil {
return conn, err
} else if proxyUrl.Scheme == "https" {
if conn, err = obj.addTls(ctx, conn, proxyUrl.Host, true, tlsConfig); err != nil {
return conn, err
}
if ctxData.logger != nil {
ctxData.logger(Log{
Id: ctxData.requestId,
Time: time.Now(),
Type: LogType_ProxyTLSHandshake,
Msg: addr,
})
}
}
err = obj.clientVerifyHttps(ctx, scheme, proxyUrl, addr, host, conn)
err = obj.clientVerifyHttps(ctx, conn, proxyUrl, remoteUrl)
if ctxData.logger != nil {
ctxData.logger(Log{
Id: ctxData.requestId,
Time: time.Now(),
Type: LogType_ProxyConnectRemote,
Msg: addr,
Msg: remoteUrl.String(),
})
}
return conn, err
case "socks5":
conn, err := obj.Socks5Proxy(ctx, ctxData, network, addr, proxyUrl)
err = obj.socks5ProxyWithConn(ctx, conn, proxyUrl, remoteUrl)
if ctxData.logger != nil {
ctxData.logger(Log{
Id: ctxData.requestId,
Time: time.Now(),
Type: LogType_ProxyTCPConnect,
Msg: addr,
Msg: remoteUrl.String(),
})
}
return conn, err
default:
return nil, errors.New("proxyUrl Scheme error")
}
return conn, err
}
func (obj *DialClient) loadHost(host string) (string, bool) {
msgDataAny, ok := obj.dnsIpData.Load(host)
@@ -232,7 +267,7 @@ func (obj *DialClient) addrToIp(host string, ips []net.IPAddr, addrType gtls.Add
obj.dnsIpData.Store(host, msgClient{time: time.Now(), host: ip.String()})
return ip.String(), nil
}
func (obj *DialClient) clientVerifySocks5(proxyUrl *url.URL, addr string, conn net.Conn) (err error) {
func (obj *DialClient) clientVerifySocks5(conn net.Conn, proxyUrl *url.URL, remoteUrl *url.URL) (err error) {
if _, err = conn.Write([]byte{5, 2, 0, 2}); err != nil {
return
}
@@ -283,10 +318,26 @@ func (obj *DialClient) clientVerifySocks5(proxyUrl *url.URL, addr string, conn n
err = errors.New("not support auth format")
return
}
var host string
host := remoteUrl.Hostname()
var port int
if host, port, err = gtls.SplitHostPort(addr); err != nil {
return
if cport := remoteUrl.Port(); cport == "" {
switch remoteUrl.Scheme {
case "http":
port = 80
case "https":
port = 443
case "socks5":
port = 1080
default:
err = errors.New("not support scheme")
return
}
} else {
port, err = strconv.Atoi(cport)
if err != nil {
return
}
}
writeCon := []byte{5, 1, 0}
ip, ipInt := gtls.ParseHost(host)
@@ -431,13 +482,13 @@ func (obj *DialClient) addJa3Tls(ctx context.Context, conn net.Conn, host string
}
return ja3.NewClient(ctx, conn, ja3Spec, disHttp2, tlsConfig)
}
func (obj *DialClient) Socks5Proxy(ctx context.Context, ctxData *reqCtxData, network string, addr string, proxyUrl *url.URL) (conn net.Conn, err error) {
func (obj *DialClient) Socks5Proxy(ctx context.Context, ctxData *reqCtxData, network string, proxyUrl *url.URL, remoteUrl *url.URL) (conn net.Conn, err error) {
if conn, err = obj.DialContext(ctx, ctxData, network, net.JoinHostPort(proxyUrl.Hostname(), proxyUrl.Port())); err != nil {
return
}
didVerify := make(chan struct{})
go func() {
if err = obj.clientVerifySocks5(proxyUrl, addr, conn); err != nil {
if err = obj.clientVerifySocks5(conn, proxyUrl, remoteUrl); err != nil {
conn.Close()
}
close(didVerify)
@@ -449,7 +500,23 @@ func (obj *DialClient) Socks5Proxy(ctx context.Context, ctxData *reqCtxData, net
return
}
}
func (obj *DialClient) clientVerifyHttps(ctx context.Context, scheme string, proxyUrl *url.URL, addr string, host string, conn net.Conn) (err error) {
func (obj *DialClient) socks5ProxyWithConn(ctx context.Context, conn net.Conn, proxyUrl *url.URL, remoteUrl *url.URL) (err error) {
didVerify := make(chan struct{})
go func() {
if err = obj.clientVerifySocks5(conn, proxyUrl, remoteUrl); err != nil {
conn.Close()
}
close(didVerify)
}()
select {
case <-ctx.Done():
conn.Close()
return context.Cause(ctx)
case <-didVerify:
return
}
}
func (obj *DialClient) clientVerifyHttps(ctx context.Context, conn net.Conn, proxyUrl *url.URL, remoteUrl *url.URL) (err error) {
hdr := make(http.Header)
hdr.Set("User-Agent", tools.UserAgent)
if proxyUrl.User != nil {
@@ -457,26 +524,16 @@ func (obj *DialClient) clientVerifyHttps(ctx context.Context, scheme string, pro
hdr.Set("Proxy-Authorization", "Basic "+tools.Base64Encode(proxyUrl.User.Username()+":"+password))
}
}
cHost := host
_, hport, _ := net.SplitHostPort(host)
if hport == "" {
_, aport, _ := net.SplitHostPort(addr)
if aport != "" {
cHost = net.JoinHostPort(cHost, aport)
} else if scheme == "http" {
cHost = net.JoinHostPort(cHost, "80")
} else if scheme == "https" {
cHost = net.JoinHostPort(cHost, "443")
} else {
return errors.New("clientVerifyHttps not found port")
}
addr, err := getProxyAddr(remoteUrl)
if err != nil {
return err
}
connectReq, err := NewRequestWithContext(ctx, http.MethodConnect, &url.URL{Opaque: addr}, nil)
if err != nil {
return err
}
connectReq.Header = hdr
connectReq.Host = cHost
connectReq.Host = remoteUrl.Host
if err = connectReq.Write(conn); err != nil {
return err
}

4
go.mod
View File

@@ -34,7 +34,7 @@ require (
github.com/gaukas/godicttls v0.0.4 // indirect
github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
github.com/google/gopacket v1.1.19 // indirect
github.com/google/pprof v0.0.0-20241122213907-cbe949e5a41b // indirect
github.com/google/pprof v0.0.0-20241203143554-1e3fdc7de467 // indirect
github.com/gospider007/blog v0.0.0-20231121084103-59a004dafccf // indirect
github.com/gospider007/kinds v0.0.0-20240929092451-8f867acde255 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
@@ -50,7 +50,7 @@ require (
github.com/miekg/dns v1.1.62 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/nwaples/rardecode/v2 v2.0.0 // indirect
github.com/nwaples/rardecode/v2 v2.0.1 // indirect
github.com/onsi/ginkgo/v2 v2.22.0 // indirect
github.com/pierrec/lz4/v4 v4.1.21 // indirect
github.com/quic-go/qpack v0.5.1 // indirect

6
go.sum
View File

@@ -58,6 +58,7 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=
github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
@@ -90,6 +91,8 @@ github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OI
github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
github.com/google/pprof v0.0.0-20241122213907-cbe949e5a41b h1:SXO0REt4iu865upYCk8aKBBJQ4BqoE0ReP23ClMu60s=
github.com/google/pprof v0.0.0-20241122213907-cbe949e5a41b/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144=
github.com/google/pprof v0.0.0-20241203143554-1e3fdc7de467 h1:keEZFtbLJugfE0qHn+Ge1JCE71spzkchQobDf3mzS/4=
github.com/google/pprof v0.0.0-20241203143554-1e3fdc7de467/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
@@ -147,6 +150,7 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/libdns/libdns v0.2.2 h1:O6ws7bAfRPaBsgAYt8MDe2HcNBGC29hkZ9MX2eUSX3s=
github.com/libdns/libdns v0.2.2/go.mod h1:4Bj9+5CQiNMVGf87wjX4CY3HQJypUHRuLvlsfsZqLWQ=
github.com/mholt/acmez v1.2.0 h1:1hhLxSgY5FvH5HCnGUuwbKY2VQVo8IU7rxXKSnZ7F30=
github.com/mholt/acmez/v2 v2.0.3 h1:CgDBlEwg3QBp6s45tPQmFIBrkRIkBT4rW4orMM6p4sw=
github.com/mholt/acmez/v2 v2.0.3/go.mod h1:pQ1ysaDeGrIMvJ9dfJMk5kJNkn7L2sb3UhyrX6Q91cw=
github.com/mholt/archives v0.0.0-20241123153623-b85dd98f4298 h1:KyCSnt0fdY6Bi4vxM0On3zlonJwnDtTYSpMg7Y1ZNhs=
@@ -160,6 +164,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/nwaples/rardecode/v2 v2.0.0 h1:C+58s2TPUdvgVFioQLpUNmIjTchtegHGS2xTNiKBYXA=
github.com/nwaples/rardecode/v2 v2.0.0/go.mod h1:yntwv/HfMc/Hbvtq9I19D1n58te3h6KsqCf3GxyfBGY=
github.com/nwaples/rardecode/v2 v2.0.1 h1:3MN6/R+Y4c7e+21U3yhWuUcf72sYmcmr6jtiuAVSH1A=
github.com/nwaples/rardecode/v2 v2.0.1/go.mod h1:yntwv/HfMc/Hbvtq9I19D1n58te3h6KsqCf3GxyfBGY=
github.com/onsi/ginkgo/v2 v2.22.0 h1:Yed107/8DjTr0lKCNt7Dn8yQ6ybuDRQoMGrNFKzMfHg=
github.com/onsi/ginkgo/v2 v2.22.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo=
github.com/onsi/gomega v1.34.2 h1:pNCwDkzrsv7MS9kpaQvVb1aVLahQXyJ/Tv5oAZMI3i8=

View File

@@ -52,6 +52,7 @@ type ClientOption struct {
Ja3Spec ja3.Ja3Spec //custom ja3Spec,use ja3.CreateSpecWithStr or ja3.CreateSpecWithId create
H2Ja3Spec ja3.H2Ja3Spec //h2 fingerprint
Proxy string //proxy,support https,http,socks5
Proxys []string //proxy list,support https,http,socks5
ForceHttp1 bool //force use http1 send requests
Ja3 bool //enable ja3 fingerprint
DisCookie bool //disable cookies
@@ -81,8 +82,9 @@ type ClientOption struct {
UtlsConfig *utls.Config
//other option
UserAgent string //headers User-Agent value
GetProxy func(ctx context.Context, url *url.URL) (string, error) //proxy callback:support https,http,socks5 proxy
UserAgent string //headers User-Agent value
GetProxy func(ctx context.Context, url *url.URL) (string, error) //proxy callback:support https,http,socks5 proxy
GetProxys func(ctx context.Context, url *url.URL) ([]string, error) //proxys callback:support https,http,socks5 proxy
GetAddrType func(host string) gtls.AddrType
}
@@ -94,6 +96,7 @@ type RequestOption struct {
Ja3Spec ja3.Ja3Spec //custom ja3Spec,use ja3.CreateSpecWithStr or ja3.CreateSpecWithId create
H2Ja3Spec ja3.H2Ja3Spec //custom h2 fingerprint
Proxy string //proxy,support http,https,socks5,examplehttp://127.0.0.1:7005
Proxys []string //proxy list,support http,https,socks5,examplehttp://127.0.0.1:7005
ForceHttp1 bool //force use http1 send requests
Ja3 bool //enable ja3 fingerprint
DisCookie bool //disable cookies,not use cookies
@@ -264,6 +267,9 @@ func (obj *Client) newRequestOption(option RequestOption) RequestOption {
if option.Proxy == "" {
option.Proxy = obj.option.Proxy
}
if len(option.Proxys) == 0 {
option.Proxys = obj.option.Proxys
}
if !option.ForceHttp1 {
option.ForceHttp1 = obj.option.ForceHttp1
}
@@ -376,6 +382,9 @@ func (obj *Client) newRequestOption(option RequestOption) RequestOption {
if !option.Ja3Spec.IsSet() && option.Ja3 {
option.Ja3Spec = ja3.DefaultJa3Spec()
}
if option.UserAgent == "" {
option.UserAgent = obj.option.UserAgent
}
if option.DisCookie {
option.Jar = nil
}

View File

@@ -35,6 +35,7 @@ type reqCtxData struct {
forceHttp1 bool
maxRedirect int
proxy *url.URL
proxys []*url.URL
disProxy bool
disAlive bool
orderHeaders []string
@@ -102,6 +103,16 @@ func NewReqCtxData(ctx context.Context, option *RequestOption) (*reqCtxData, err
}
ctxData.proxy = tempProxy
}
if l := len(option.Proxys); l > 0 {
ctxData.proxys = make([]*url.URL, l)
for i, proxy := range option.Proxys {
tempProxy, err := gtls.VerifyProxy(proxy)
if err != nil {
return ctxData, tools.WrapError(errFatal, errors.New("tempRequest init proxy error"), err)
}
ctxData.proxys[i] = tempProxy
}
}
return ctxData, nil
}
func CreateReqCtx(ctx context.Context, ctxData *reqCtxData) context.Context {
@@ -271,6 +282,9 @@ func (obj *Client) request(ctx context.Context, option *RequestOption) (response
if err != nil {
return response, tools.WrapError(err, errors.New("tempRequest init headers error"), err)
}
if headers != nil && option.UserAgent != "" {
headers.Set("User-Agent", option.UserAgent)
}
if orderHeaders == nil {
orderHeaders = option.OrderHeaders
}

View File

@@ -333,9 +333,9 @@ func (obj *Response) IsNewConn() bool {
}
// conn proxy
func (obj *Response) Proxy() *url.URL {
func (obj *Response) Proxys() []*url.URL {
if obj.rawConn != nil {
return obj.rawConn.Proxy()
return obj.rawConn.Proxys()
}
return nil
}

View File

@@ -62,6 +62,7 @@ type roundTripper struct {
tlsConfig *tls.Config
utlsConfig *utls.Config
getProxy func(ctx context.Context, url *url.URL) (string, error)
getProxys func(ctx context.Context, url *url.URL) ([]string, error)
}
func newRoundTripper(preCtx context.Context, option ClientOption) *roundTripper {
@@ -97,6 +98,7 @@ func newRoundTripper(preCtx context.Context, option ClientOption) *roundTripper
cnl: cnl,
dialer: dialClient,
getProxy: option.GetProxy,
getProxys: option.GetProxys,
connPools: newConnPools(),
}
}
@@ -178,20 +180,52 @@ func (obj *roundTripper) dial(ctxData *reqCtxData, req *http.Request) (conn *con
return obj.http3Dial(ctxData, req)
}
}
var proxy *url.URL
var proxys []*url.URL
if !ctxData.disProxy {
if proxy = cloneUrl(ctxData.proxy); proxy == nil && obj.getProxy != nil {
proxyStr, err := obj.getProxy(req.Context(), proxy)
if ctxData.proxy != nil {
proxys = []*url.URL{cloneUrl(ctxData.proxy)}
}
if len(proxys) == 0 && len(ctxData.proxys) > 0 {
proxys = make([]*url.URL, len(ctxData.proxys))
for i, proxy := range ctxData.proxys {
proxys[i] = cloneUrl(proxy)
}
}
if len(proxys) == 0 && obj.getProxy != nil {
proxyStr, err := obj.getProxy(req.Context(), req.URL)
if err != nil {
return conn, err
}
if proxy, err = gtls.VerifyProxy(proxyStr); err != nil {
proxy, err := gtls.VerifyProxy(proxyStr)
if err != nil {
return conn, err
}
proxys = []*url.URL{proxy}
}
if len(proxys) == 0 && obj.getProxys != nil {
proxyStrs, err := obj.getProxys(req.Context(), req.URL)
if err != nil {
return conn, err
}
if l := len(proxyStrs); l > 0 {
proxys = make([]*url.URL, l)
for i, proxyStr := range proxyStrs {
proxy, err := gtls.VerifyProxy(proxyStr)
if err != nil {
return conn, err
}
proxys[i] = proxy
}
}
}
}
host := getHost(req)
netConn, err := obj.dialer.DialContextWithProxy(req.Context(), ctxData, "tcp", req.URL.Scheme, getAddr(req.URL), host, proxy, obj.tlsConfigClone(ctxData))
var netConn net.Conn
if len(proxys) > 0 {
netConn, err = obj.dialer.DialProxyContext(req.Context(), ctxData, "tcp", obj.tlsConfigClone(ctxData), append(proxys, cloneUrl(req.URL))...)
} else {
netConn, err = obj.dialer.DialContext(req.Context(), ctxData, "tcp", getAddr(req.URL))
}
if err != nil {
return conn, err
}
@@ -225,9 +259,7 @@ func (obj *roundTripper) dial(ctxData *reqCtxData, req *http.Request) (conn *con
}
}
conne := obj.newConnecotr(netConn)
if proxy != nil {
conne.proxy = proxy
}
conne.proxys = proxys
if h2 {
if conne.h2RawConn, err = http2.NewClientConn(func() {
conne.forceCnl(errors.New("http2 client close"))
@@ -242,6 +274,9 @@ func (obj *roundTripper) dial(ctxData *reqCtxData, req *http.Request) (conn *con
func (obj *roundTripper) setGetProxy(getProxy func(ctx context.Context, url *url.URL) (string, error)) {
obj.getProxy = getProxy
}
func (obj *roundTripper) setGetProxys(getProxys func(ctx context.Context, url *url.URL) ([]string, error)) {
obj.getProxys = getProxys
}
func (obj *roundTripper) poolRoundTrip(ctxData *reqCtxData, pool *connPool, task *reqTask, key string) (isTry bool) {
task.ctx, task.cnl = context.WithTimeout(task.req.Context(), ctxData.responseHeaderTimeout)

11
rw.go
View File

@@ -20,11 +20,14 @@ func (obj *readWriteCloser) Read(p []byte) (n int, err error) {
func (obj *readWriteCloser) InPool() bool {
return obj.conn.inPool
}
func (obj *readWriteCloser) Proxy() *url.URL {
if obj.conn.proxy == nil {
return nil
func (obj *readWriteCloser) Proxys() []*url.URL {
if l := len(obj.conn.proxys); l > 0 {
proxys := make([]*url.URL, l)
for i, proxy := range obj.conn.proxys {
proxys[i] = cloneUrl(proxy)
}
}
return cloneUrl(obj.conn.proxy)
return obj.conn.proxys
}
var ErrgospiderBodyClose = errors.New("gospider body close error")

View File

@@ -11,7 +11,8 @@ func TestUseProxy(t *testing.T) {
if err != nil {
t.Error(err)
}
if resp.Proxy() != nil {
if resp.Proxys() != nil {
t.Error("proxy error")
}
}