mirror of
https://github.com/gospider007/requests.git
synced 2025-12-24 13:57:52 +08:00
329 lines
9.0 KiB
Go
329 lines
9.0 KiB
Go
package requests
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"net/http"
|
|
|
|
"github.com/gospider007/http1"
|
|
"github.com/gospider007/http2"
|
|
"github.com/gospider007/http3"
|
|
"github.com/gospider007/ja3"
|
|
"github.com/gospider007/tools"
|
|
"github.com/quic-go/quic-go"
|
|
uquic "github.com/refraction-networking/uquic"
|
|
)
|
|
|
|
func (obj *Response) suppertRetry() bool {
|
|
if obj.isNewConn {
|
|
return false
|
|
}
|
|
if obj.request.Body == nil {
|
|
return true
|
|
} else if body, ok := obj.request.Body.(io.Seeker); ok {
|
|
if i, err := body.Seek(0, io.SeekStart); i == 0 && err == nil {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
func getKey(ctx *Response) (string, error) {
|
|
adds := []string{}
|
|
for _, p := range ctx.proxys {
|
|
pd, err := GetAddressWithUrl(p)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
adds = append(adds, pd.String())
|
|
}
|
|
pd, err := GetAddressWithUrl(ctx.Request().URL)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
adds = append(adds, pd.String())
|
|
return strings.Join(adds, "@"), nil
|
|
}
|
|
|
|
type roundTripper struct {
|
|
ctx context.Context
|
|
connPools map[string]chan http1.Conn
|
|
dialer *Dialer
|
|
lock sync.Mutex
|
|
}
|
|
|
|
var specClient = ja3.NewClient()
|
|
|
|
func newRoundTripper(preCtx context.Context) *roundTripper {
|
|
if preCtx == nil {
|
|
preCtx = context.TODO()
|
|
}
|
|
return &roundTripper{
|
|
ctx: preCtx,
|
|
connPools: make(map[string]chan http1.Conn),
|
|
dialer: new(Dialer),
|
|
}
|
|
}
|
|
|
|
func (obj *roundTripper) getConnPool(key string) chan http1.Conn {
|
|
obj.lock.Lock()
|
|
defer obj.lock.Unlock()
|
|
val, ok := obj.connPools[key]
|
|
if ok {
|
|
return val
|
|
}
|
|
val = make(chan http1.Conn)
|
|
obj.connPools[key] = val
|
|
return val
|
|
}
|
|
func (obj *roundTripper) putConnPool(key string, conn http1.Conn) {
|
|
if conn != nil {
|
|
go func() {
|
|
select {
|
|
case obj.getConnPool(key) <- conn:
|
|
case <-conn.Context().Done():
|
|
case <-obj.ctx.Done():
|
|
conn.CloseWithError(context.Cause(obj.ctx))
|
|
}
|
|
}()
|
|
}
|
|
}
|
|
|
|
func (obj *roundTripper) http3Dial(ctx *Response, remoteAddress Address, proxyAddress ...Address) (http1.Conn, error) {
|
|
var udpConn net.PacketConn
|
|
var err error
|
|
if len(proxyAddress) > 0 {
|
|
if proxyAddress[len(proxyAddress)-1].Scheme != "socks5" {
|
|
return nil, errors.New("http3 last proxy must socks5 proxy")
|
|
}
|
|
udpConn, _, err = obj.dialer.DialProxyContext(ctx, "tcp", ctx.option.TlsConfig.Clone(), append(proxyAddress, remoteAddress)...)
|
|
} else {
|
|
udpConn, err = net.ListenUDP("udp", nil)
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if remoteAddress.IP == nil && len(proxyAddress) == 0 {
|
|
remoteAddress.IP, err = obj.dialer.loadHost(ctx.Context(), remoteAddress.Host, ctx.option.DialOption)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
var netConn any
|
|
if ctx.option.USpec != nil {
|
|
netConn, err = obj.newUhttp3Conn(ctx, remoteAddress, udpConn)
|
|
} else {
|
|
netConn, err = obj.newHttp3Conn(ctx, remoteAddress, udpConn)
|
|
}
|
|
if err != nil {
|
|
udpConn.Close()
|
|
return nil, err
|
|
}
|
|
return http3.NewConn(obj.ctx, netConn, udpConn), nil
|
|
}
|
|
|
|
func (obj *roundTripper) newHttp3Conn(ctx *Response, remoteAddress Address, udpConn net.PacketConn) (any, error) {
|
|
tlsConfig := ctx.option.TlsConfig.Clone()
|
|
tlsConfig.NextProtos = []string{http3.NextProtoH3}
|
|
tlsConfig.ServerName = remoteAddress.Host
|
|
var quicConfig *quic.Config
|
|
if ctx.option.UquicConfig != nil {
|
|
quicConfig = ctx.option.QuicConfig.Clone()
|
|
}
|
|
return quic.DialEarly(ctx.Context(), udpConn, &net.UDPAddr{IP: remoteAddress.IP, Port: remoteAddress.Port}, tlsConfig, quicConfig)
|
|
}
|
|
func (obj *roundTripper) newUhttp3Conn(ctx *Response, remoteAddress Address, udpConn net.PacketConn) (any, error) {
|
|
spec, err := ja3.CreateUSpec(ctx.option.USpec)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
tlsConfig := ctx.option.UtlsConfig.Clone()
|
|
tlsConfig.NextProtos = []string{http3.NextProtoH3}
|
|
tlsConfig.ServerName = remoteAddress.Host
|
|
var quicConfig *uquic.Config
|
|
if ctx.option.UquicConfig != nil {
|
|
quicConfig = ctx.option.UquicConfig.Clone()
|
|
}
|
|
return (&uquic.UTransport{
|
|
Transport: &uquic.Transport{
|
|
Conn: udpConn,
|
|
},
|
|
QUICSpec: &spec,
|
|
}).DialEarly(ctx.Context(), &net.UDPAddr{IP: remoteAddress.IP, Port: remoteAddress.Port}, tlsConfig, quicConfig)
|
|
}
|
|
|
|
func (obj *roundTripper) httpDial(ctx *Response, remoteAddress Address, proxyAddress ...Address) (http1.Conn, error) {
|
|
var rawNetConn net.Conn
|
|
var arch string
|
|
var err error
|
|
if len(proxyAddress) > 0 {
|
|
arch = proxyAddress[len(proxyAddress)-1].Compression
|
|
_, rawNetConn, err = obj.dialer.DialProxyContext(ctx, "tcp", ctx.option.TlsConfig.Clone(), append(proxyAddress, remoteAddress)...)
|
|
} else {
|
|
_, rawNetConn, err = obj.dialer.DialProxyContext(ctx, "tcp", nil, remoteAddress)
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var h2 bool
|
|
var rawConn net.Conn
|
|
if ctx.request.URL.Scheme == "https" {
|
|
rawConn, h2, err = obj.dialAddTlsWithResponse(ctx, rawNetConn)
|
|
} else {
|
|
rawConn = rawNetConn
|
|
}
|
|
if err == nil && arch != "" {
|
|
rawConn, err = tools.NewCompressionConn(rawConn, arch)
|
|
}
|
|
if err != nil {
|
|
rawNetConn.Close()
|
|
return nil, err
|
|
}
|
|
if !h2 {
|
|
return http1.NewConn(obj.ctx, rawConn), nil
|
|
}
|
|
var spec *http2.Spec
|
|
if ctx.option.gospiderSpec != nil {
|
|
spec = ctx.option.gospiderSpec.H2Spec
|
|
}
|
|
h2conn, err := http2.NewConn(obj.ctx, ctx.Context(), rawConn, spec)
|
|
if err != nil {
|
|
rawConn.Close()
|
|
rawNetConn.Close()
|
|
return nil, err
|
|
}
|
|
return h2conn, nil
|
|
}
|
|
func (obj *roundTripper) newConn(ctx *Response) (conn http1.Conn, err error) {
|
|
defer func() {
|
|
if err != nil && conn != nil {
|
|
conn.CloseWithError(err)
|
|
}
|
|
}()
|
|
proxys, err := obj.initProxys(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
remoteAddress, err := GetAddressWithUrl(ctx.request.URL)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if ctx.option.ForceHttp3 {
|
|
return obj.http3Dial(ctx, remoteAddress, proxys...)
|
|
}
|
|
return obj.httpDial(ctx, remoteAddress, proxys...)
|
|
}
|
|
|
|
func (obj *roundTripper) dialAddTlsWithResponse(ctx *Response, rawNetConn net.Conn) (tlsConn net.Conn, h2 bool, err error) {
|
|
if ctx.option.TlsHandshakeTimeout > 0 {
|
|
tlsCtx, tlsCnl := context.WithTimeout(ctx.Context(), ctx.option.TlsHandshakeTimeout)
|
|
tlsConn, h2, err = obj.dialAddTls(tlsCtx, ctx.option, ctx.request, rawNetConn)
|
|
tlsCnl()
|
|
} else {
|
|
tlsConn, h2, err = obj.dialAddTls(ctx.Context(), ctx.option, ctx.request, rawNetConn)
|
|
}
|
|
if ctx.option.Logger != nil {
|
|
ctx.option.Logger(Log{
|
|
Id: ctx.requestId,
|
|
Time: time.Now(),
|
|
Type: LogType_TLSHandshake,
|
|
Msg: fmt.Sprintf("host:%s, h2:%t", getHost(ctx.request), h2),
|
|
})
|
|
}
|
|
if err != nil && tlsConn != nil {
|
|
tlsConn.Close()
|
|
}
|
|
return tlsConn, h2, err
|
|
}
|
|
func (obj *roundTripper) dialAddTls(ctx context.Context, option *RequestOption, req *http.Request, netConn net.Conn) (net.Conn, bool, error) {
|
|
if option.gospiderSpec != nil && option.gospiderSpec.TLSSpec != nil {
|
|
if tlsConn, err := obj.dialer.addJa3Tls(ctx, netConn, getHost(req), option.gospiderSpec.TLSSpec, option.UtlsConfig.Clone(), option.ForceHttp1); err != nil {
|
|
return tlsConn, false, tools.WrapError(err, "add ja3 tls error")
|
|
} else {
|
|
return tlsConn, tlsConn.ConnectionState().NegotiatedProtocol == "h2", nil
|
|
}
|
|
}
|
|
if tlsConn, err := obj.dialer.addTls(ctx, netConn, getHost(req), option.TlsConfig.Clone(), option.ForceHttp1); err != nil {
|
|
return tlsConn, false, tools.WrapError(err, "add tls error")
|
|
} else {
|
|
return tlsConn, tlsConn.ConnectionState().NegotiatedProtocol == "h2", nil
|
|
}
|
|
}
|
|
func (obj *roundTripper) initProxys(ctx *Response) ([]Address, error) {
|
|
if ctx.option.DisProxy {
|
|
return nil, nil
|
|
}
|
|
pps := ctx.proxys
|
|
if len(pps) == 0 {
|
|
if ctx.option.GetProxy == nil {
|
|
return nil, nil
|
|
}
|
|
proxyA, err := ctx.option.GetProxy(ctx)
|
|
if err != nil || proxyA == nil {
|
|
return nil, err
|
|
}
|
|
pps, err = parseProxy(proxyA)
|
|
if err != nil || len(pps) == 0 {
|
|
return nil, err
|
|
}
|
|
}
|
|
proxys := make([]Address, len(pps))
|
|
for i, proxy := range pps {
|
|
proxyAddress, err := GetAddressWithUrl(proxy)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
proxys[i] = proxyAddress
|
|
}
|
|
return proxys, nil
|
|
}
|
|
|
|
func (obj *roundTripper) RoundTrip(ctx *Response) (err error) {
|
|
if ctx.option.RequestCallBack != nil {
|
|
if err = ctx.option.RequestCallBack(ctx); err != nil {
|
|
if err == http.ErrUseLastResponse {
|
|
if ctx.lastResponse == nil {
|
|
return errors.New("errUseLastResponse response is nil")
|
|
} else {
|
|
ctx.response = ctx.lastResponse
|
|
}
|
|
}
|
|
return err
|
|
}
|
|
}
|
|
ctx.connKey, err = getKey(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var conn http1.Conn
|
|
for send := true; send; {
|
|
select {
|
|
case <-ctx.Context().Done():
|
|
return context.Cause(ctx.Context())
|
|
case conn = <-obj.getConnPool(ctx.connKey):
|
|
ctx.isNewConn = false
|
|
default:
|
|
ctx.isNewConn = true
|
|
conn, err = obj.newConn(ctx)
|
|
}
|
|
if err == nil {
|
|
if err = ctx.doRequest(conn); err == nil {
|
|
break
|
|
}
|
|
}
|
|
send = ctx.suppertRetry()
|
|
}
|
|
if err == nil && ctx.option.RequestCallBack != nil {
|
|
if err2 := ctx.option.RequestCallBack(ctx); err2 != nil {
|
|
err = err2
|
|
}
|
|
}
|
|
return err
|
|
}
|