From e060caef3c0b635db48e597eb1681fcfbf2e425d Mon Sep 17 00:00:00 2001 From: gospider <2216403312@qq.com> Date: Fri, 22 Aug 2025 16:20:50 +0800 Subject: [PATCH] sync --- conn.go | 1 + go.mod | 2 +- go.sum | 4 +- requests.go | 10 +-- response.go | 11 ++- roundTripper.go | 186 +++++++++++++++++++++--------------------------- 6 files changed, 99 insertions(+), 115 deletions(-) diff --git a/conn.go b/conn.go index 7143ac9..e5ea08d 100644 --- a/conn.go +++ b/conn.go @@ -24,6 +24,7 @@ func (obj *Response) doRequest(conn http1.Conn) (err error) { } else { obj.response = response obj.response.Request = obj.request + obj.rawBody = response.Body.(*http1.Body) } if obj.option.Logger != nil { obj.option.Logger(Log{ diff --git a/go.mod b/go.mod index 049cb27..9a2ce80 100644 --- a/go.mod +++ b/go.mod @@ -62,7 +62,7 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/nwaples/rardecode/v2 v2.1.1 // indirect - github.com/onsi/ginkgo/v2 v2.25.0 // indirect + github.com/onsi/ginkgo/v2 v2.25.1 // indirect github.com/patrickmn/go-cache v2.1.0+incompatible // indirect github.com/pierrec/lz4/v4 v4.1.22 // indirect github.com/quic-go/qpack v0.5.1 // indirect diff --git a/go.sum b/go.sum index 4a5795f..766c9a9 100644 --- a/go.sum +++ b/go.sum @@ -177,8 +177,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/nwaples/rardecode/v2 v2.1.1 h1:OJaYalXdliBUXPmC8CZGQ7oZDxzX1/5mQmgn0/GASew= github.com/nwaples/rardecode/v2 v2.1.1/go.mod h1:7uz379lSxPe6j9nvzxUZ+n7mnJNgjsRNb6IbvGVHRmw= -github.com/onsi/ginkgo/v2 v2.25.0 h1:LJu94oDZUdgnw+GD0Sk/iwG9c5Fnr1vLgMb4FUUwWxE= -github.com/onsi/ginkgo/v2 v2.25.0/go.mod h1:ppTWQ1dh9KM/F1XgpeRqelR+zHVwV81DGRSDnFxK7Sk= +github.com/onsi/ginkgo/v2 v2.25.1 h1:Fwp6crTREKM+oA6Cz4MsO8RhKQzs2/gOIVOUscMAfZY= +github.com/onsi/ginkgo/v2 v2.25.1/go.mod h1:ppTWQ1dh9KM/F1XgpeRqelR+zHVwV81DGRSDnFxK7Sk= github.com/onsi/gomega v1.37.0 h1:CdEG8g0S133B4OswTDC/5XPSzE1OeP29QOioj2PID2Y= github.com/onsi/gomega v1.37.0/go.mod h1:8D9+Txp43QWKhM24yyOBEdpkzN8FvJyAwecBgsU4KU0= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= diff --git a/requests.go b/requests.go index fb5d537..55c7457 100644 --- a/requests.go +++ b/requests.go @@ -11,7 +11,6 @@ import ( "net/http" - "github.com/gospider007/http1" "github.com/gospider007/re" "github.com/gospider007/tools" "github.com/gospider007/websocket" @@ -161,7 +160,9 @@ func (obj *Client) retryRequest(ctx context.Context, option RequestOption, uhref option.Headers = map[string]any{"Authorization": Authorization} } } + lastResponse := response.response response = obj.newResponse(ctx, option, loc, requestId) + response.lastResponse = lastResponse } } } @@ -341,15 +342,16 @@ func (obj *Client) request(ctx *Response) (err error) { } } - if err != nil && err != ErrUseLastResponse { - err = tools.WrapError(err, "client do error") + if err != nil { + if err != ErrUseLastResponse { + err = tools.WrapError(err, "client do error") + } return } if ctx.response == nil { err = errors.New("send req response is nil") return } - ctx.rawBody = ctx.response.Body.(*http1.Body) if encoding := ctx.ContentEncoding(); encoding != "" && ctx.response.Body != nil { arch, cerr := tools.NewRawCompression(encoding) if cerr != nil { diff --git a/response.go b/response.go index cda857b..99ee539 100644 --- a/response.go +++ b/response.go @@ -58,6 +58,7 @@ func (obj *Response) Client() *Client { type Response struct { err error + lastResponse *http.Response ctx context.Context request *http.Request rawBody *http1.Body @@ -380,11 +381,14 @@ func (obj *Response) ReadBody() (err error) { if obj.bodyErr != nil { return nil } + body := obj.Body() + if body == nil { + return errors.New("not found body") + } + defer body.close(false) bBody := bytes.NewBuffer(nil) done := make(chan struct{}) var readErr error - body := obj.Body() - defer body.close(false) go func() { defer close(done) if obj.option.Bar && obj.ContentLength() > 0 { @@ -458,6 +462,9 @@ func (obj *body) closeWithError(i bool, err error) error { } func (obj *Response) Body() *body { + if obj.response == nil || obj.response.Body == nil { + return nil + } return &body{ctx: obj} } diff --git a/roundTripper.go b/roundTripper.go index b2a814f..d5586dd 100644 --- a/roundTripper.go +++ b/roundTripper.go @@ -82,104 +82,124 @@ func (obj *roundTripper) getConnPool(key string) chan http1.Conn { obj.connPools[key] = val return val } -func runConn(ctx context.Context, pool chan http1.Conn, conn http1.Conn) { - select { - case pool <- conn: - case <-conn.Context().Done(): - case <-ctx.Done(): - conn.CloseWithError(context.Cause(ctx)) +func (obj *roundTripper) putConnPool(key string, conn http1.Conn) { + if conn != nil { + go func() { + select { + case obj.getConnPool(key) <- conn: + case <-conn.Context().Done(): + case <-obj.ctx.Done(): + conn.CloseWithError(context.Cause(obj.ctx)) + } + }() } } -func (obj *roundTripper) putConnPool(key string, con http1.Conn) { - go runConn(obj.ctx, obj.getConnPool(key), con) -} -func (obj *roundTripper) http3Dial(ctx *Response, remtoeAddress Address, proxyAddress ...Address) (udpConn net.PacketConn, err error) { +func (obj *roundTripper) http3Dial(ctx *Response, remoteAddress Address, proxyAddress ...Address) (http1.Conn, error) { + var udpConn net.PacketConn + var err error if len(proxyAddress) > 0 { if proxyAddress[len(proxyAddress)-1].Scheme != "socks5" { - err = errors.New("http3 last proxy must socks5 proxy") - return + return nil, errors.New("http3 last proxy must socks5 proxy") } - udpConn, _, err = obj.dialer.DialProxyContext(ctx, "tcp", ctx.option.TlsConfig.Clone(), append(proxyAddress, remtoeAddress)...) + udpConn, _, err = obj.dialer.DialProxyContext(ctx, "tcp", ctx.option.TlsConfig.Clone(), append(proxyAddress, remoteAddress)...) } else { udpConn, err = net.ListenUDP("udp", nil) } - if err != nil && udpConn != nil { - udpConn.Close() - } - return -} -func (obj *roundTripper) ghttp3Dial(ctx *Response, remoteAddress Address, proxyAddress ...Address) (conn http1.Conn, err error) { - udpConn, err := obj.http3Dial(ctx, remoteAddress, proxyAddress...) if err != nil { return nil, err } - defer func() { - if err != nil { - udpConn.Close() - } - }() - tlsConfig := ctx.option.TlsConfig.Clone() - tlsConfig.NextProtos = []string{http3.NextProtoH3} - tlsConfig.ServerName = remoteAddress.Host if remoteAddress.IP == nil && len(proxyAddress) == 0 { remoteAddress.IP, err = obj.dialer.loadHost(ctx.Context(), remoteAddress.Host, ctx.option.DialOption) if err != nil { return nil, err } } + var netConn any + if ctx.option.USpec != nil { + netConn, err = obj.newUhttp3Conn(ctx, remoteAddress, udpConn) + } else { + netConn, err = obj.newHttp3Conn(ctx, remoteAddress, udpConn) + } + if err != nil { + udpConn.Close() + return nil, err + } + return http3.NewConn(obj.ctx, netConn, udpConn), nil +} + +func (obj *roundTripper) newHttp3Conn(ctx *Response, remoteAddress Address, udpConn net.PacketConn) (any, error) { + tlsConfig := ctx.option.TlsConfig.Clone() + tlsConfig.NextProtos = []string{http3.NextProtoH3} + tlsConfig.ServerName = remoteAddress.Host var quicConfig *quic.Config if ctx.option.UquicConfig != nil { quicConfig = ctx.option.QuicConfig.Clone() } - netConn, err := quic.DialEarly(ctx.Context(), udpConn, &net.UDPAddr{IP: remoteAddress.IP, Port: remoteAddress.Port}, tlsConfig, quicConfig) - if err != nil { - return nil, err - } - conn = http3.NewConn(obj.ctx, netConn, udpConn) - return + return quic.DialEarly(ctx.Context(), udpConn, &net.UDPAddr{IP: remoteAddress.IP, Port: remoteAddress.Port}, tlsConfig, quicConfig) } - -func (obj *roundTripper) uhttp3Dial(ctx *Response, remoteAddress Address, proxyAddress ...Address) (conn http1.Conn, err error) { +func (obj *roundTripper) newUhttp3Conn(ctx *Response, remoteAddress Address, udpConn net.PacketConn) (any, error) { spec, err := ja3.CreateUSpec(ctx.option.USpec) if err != nil { return nil, err } - udpConn, err := obj.http3Dial(ctx, remoteAddress, proxyAddress...) - if err != nil { - return nil, err - } - defer func() { - if err != nil { - udpConn.Close() - } - }() tlsConfig := ctx.option.UtlsConfig.Clone() tlsConfig.NextProtos = []string{http3.NextProtoH3} tlsConfig.ServerName = remoteAddress.Host - if remoteAddress.IP == nil && len(proxyAddress) == 0 { - remoteAddress.IP, err = obj.dialer.loadHost(ctx.Context(), remoteAddress.Host, ctx.option.DialOption) - if err != nil { - return nil, err - } - } var quicConfig *uquic.Config if ctx.option.UquicConfig != nil { quicConfig = ctx.option.UquicConfig.Clone() } - netConn, err := (&uquic.UTransport{ + return (&uquic.UTransport{ Transport: &uquic.Transport{ Conn: udpConn, }, QUICSpec: &spec, }).DialEarly(ctx.Context(), &net.UDPAddr{IP: remoteAddress.IP, Port: remoteAddress.Port}, tlsConfig, quicConfig) +} + +func (obj *roundTripper) httpDial(ctx *Response, remoteAddress Address, proxyAddress ...Address) (http1.Conn, error) { + var rawNetConn net.Conn + var arch string + var err error + if len(proxyAddress) > 0 { + arch = proxyAddress[len(proxyAddress)-1].Compression + _, rawNetConn, err = obj.dialer.DialProxyContext(ctx, "tcp", ctx.option.TlsConfig.Clone(), append(proxyAddress, remoteAddress)...) + } else { + _, rawNetConn, err = obj.dialer.DialProxyContext(ctx, "tcp", nil, remoteAddress) + } if err != nil { return nil, err } - conn = http3.NewConn(obj.ctx, netConn, udpConn) - return + var h2 bool + var rawConn net.Conn + if ctx.request.URL.Scheme == "https" { + rawConn, h2, err = obj.dialAddTlsWithResponse(ctx, rawNetConn) + } else { + rawConn = rawNetConn + } + if err == nil && arch != "" { + rawConn, err = tools.NewCompressionConn(rawConn, arch) + } + if err != nil { + rawNetConn.Close() + return nil, err + } + if !h2 { + return http1.NewConn(obj.ctx, rawConn), nil + } + var spec *http2.Spec + if ctx.option.gospiderSpec != nil { + spec = ctx.option.gospiderSpec.H2Spec + } + h2conn, err := http2.NewConn(obj.ctx, ctx.Context(), rawConn, spec) + if err != nil { + rawConn.Close() + rawNetConn.Close() + return nil, err + } + return h2conn, nil } - func (obj *roundTripper) newConn(ctx *Response) (conn http1.Conn, err error) { defer func() { if err != nil && conn != nil { @@ -195,57 +215,11 @@ func (obj *roundTripper) newConn(ctx *Response) (conn http1.Conn, err error) { return nil, err } if ctx.option.ForceHttp3 { - if ctx.option.USpec != nil { - return obj.uhttp3Dial(ctx, remoteAddress, proxys...) - } else { - return obj.ghttp3Dial(ctx, remoteAddress, proxys...) - } - } - var rawNetConn net.Conn - var arch string - if len(proxys) > 0 { - arch = proxys[len(proxys)-1].Compression - _, rawNetConn, err = obj.dialer.DialProxyContext(ctx, "tcp", ctx.option.TlsConfig.Clone(), append(proxys, remoteAddress)...) - } else { - _, rawNetConn, err = obj.dialer.DialProxyContext(ctx, "tcp", nil, remoteAddress) + return obj.http3Dial(ctx, remoteAddress, proxys...) } + return obj.httpDial(ctx, remoteAddress, proxys...) +} - if err != nil { - if rawNetConn != nil { - rawNetConn.Close() - } - return nil, err - } - var h2 bool - var rawConn net.Conn - if ctx.request.URL.Scheme == "https" { - rawConn, h2, err = obj.dialAddTlsWithResponse(ctx, rawNetConn) - } else { - rawConn = rawNetConn - } - if arch != "" { - rawConn, err = tools.NewCompressionConn(rawConn, arch) - } - if err != nil { - if rawConn != nil { - rawConn.Close() - } - return nil, err - } - return obj.dialConnecotr(ctx, rawConn, h2) -} -func (obj *roundTripper) dialConnecotr(ctx *Response, rawCon net.Conn, h2 bool) (conn http1.Conn, err error) { - if h2 { - var spec *http2.Spec - if ctx.option.gospiderSpec != nil { - spec = ctx.option.gospiderSpec.H2Spec - } - conn, err = http2.NewConn(obj.ctx, ctx.Context(), rawCon, spec) - } else { - conn = http1.NewConn(obj.ctx, rawCon) - } - return -} func (obj *roundTripper) dialAddTlsWithResponse(ctx *Response, rawNetConn net.Conn) (tlsConn net.Conn, h2 bool, err error) { if ctx.option.TlsHandshakeTimeout > 0 { tlsCtx, tlsCnl := context.WithTimeout(ctx.Context(), ctx.option.TlsHandshakeTimeout) @@ -314,10 +288,10 @@ func (obj *roundTripper) RoundTrip(ctx *Response) (err error) { if ctx.option.RequestCallBack != nil { if err = ctx.option.RequestCallBack(ctx); err != nil { if err == http.ErrUseLastResponse { - if ctx.response == nil { + if ctx.lastResponse == nil { return errors.New("errUseLastResponse response is nil") } else { - return nil + ctx.response = ctx.lastResponse } } return err