diff --git a/README.md b/README.md index 40ad8a9..91799b9 100644 --- a/README.md +++ b/README.md @@ -20,13 +20,11 @@

Requests is a fully featured HTTP client library for Golang. Network requests can be completed with just a few lines of code. Unified support for http1, http2, http3, websocket, sse, utls, uquic

## Innovative Features -| **gospider007/requests** | **Other Request Libraries** | -|---------------------------|----------------------------| -| Unlimited chained proxy | Not supported | -| HTTP/3 fingerprint spoofing protection | Not supported | -| Arbitrary closure of underlying connections | Not supported | -| Genuine request-level proxy settings | Not supported | -| Unique transport layer management mechanism, fully unifying HTTP/1, HTTP/2, HTTP/3, WebSocket, SSE, UTLS, QUIC protocol handling | Not supported | +* Unlimited chained proxy +* HTTP/3 fingerprint spoofing protection +* Arbitrary closure of underlying connections +* Genuine request-level proxy settings +* Unique transport layer management mechanism, fully unifying HTTP/1, HTTP/2, HTTP/3, WebSocket, SSE, UTLS, QUIC protocol handling ## Features * [Simple for settings and Request](https://github.com/gospider007/requests#quickly-send-requests) @@ -69,7 +67,7 @@ * [Well tested client library](https://github.com/gospider007/requests/tree/master/test) ## Supported Go Versions -Recommended to use `go1.23.0` and above. +Recommended to use `go1.24.0` and above. Initially Requests started supporting `go modules` ## Installation diff --git a/compressConn.go b/compressConn.go index 764cbeb..1639a42 100644 --- a/compressConn.go +++ b/compressConn.go @@ -220,11 +220,11 @@ func NewReaderCompression(conn io.Reader, arch Compression) (*ReaderCompression, func NewCompressionConn(conn net.Conn, arch Compression) (net.Conn, error) { w, err := NewWriterCompression(conn, arch) if err != nil { - return nil, err + return conn, err } r, err := NewReaderCompression(conn, arch) if err != nil { - return nil, err + return conn, err } ccon := &CompressionConn{ conn: conn, diff --git a/conn.go b/conn.go index bd6ff00..40ec846 100644 --- a/conn.go +++ b/conn.go @@ -62,29 +62,42 @@ func taskMain(conn http1.Conn, task *reqTask) (err error) { defer close(done) response, derr = conn.DoRequest(task.reqCtx.request, &http1.Option{OrderHeaders: task.reqCtx.option.orderHeaders.Data()}) }() - select { - case <-conn.Context().Done(): //force conn close - err = tools.WrapError(context.Cause(conn.Context()), "taskMain delete ctx error: ") - case <-time.After(task.reqCtx.option.ResponseHeaderTimeout): - err = errors.New("ResponseHeaderTimeout error: ") - case <-task.ctx.Done(): - err = context.Cause(task.ctx) - case <-done: - if derr != nil { - err = tools.WrapError(derr, "roundTrip error") - } else { - task.reqCtx.response = response - task.reqCtx.response.Request = task.reqCtx.request + if task.reqCtx.option.ResponseHeaderTimeout > 0 { + select { + case <-conn.Context().Done(): //force conn close + err = tools.WrapError(context.Cause(conn.Context()), "taskMain delete ctx error: ") + case <-time.After(task.reqCtx.option.ResponseHeaderTimeout): + err = errors.New("ResponseHeaderTimeout error: ") + case <-task.ctx.Done(): + err = context.Cause(task.ctx) + case <-done: } - if task.reqCtx.option.Logger != nil { - task.reqCtx.option.Logger(Log{ - Id: task.reqCtx.requestId, - Time: time.Now(), - Type: LogType_ResponseHeader, - Msg: "response header", - }) + } else { + select { + case <-conn.Context().Done(): //force conn close + err = tools.WrapError(context.Cause(conn.Context()), "taskMain delete ctx error: ") + case <-task.ctx.Done(): + err = context.Cause(task.ctx) + case <-done: } } + + if err != nil { + err = tools.WrapError(err, "roundTrip error") + } else if derr != nil { + err = tools.WrapError(derr, "roundTrip error") + } else { + task.reqCtx.response = response + task.reqCtx.response.Request = task.reqCtx.request + } + if task.reqCtx.option.Logger != nil { + task.reqCtx.option.Logger(Log{ + Id: task.reqCtx.requestId, + Time: time.Now(), + Type: LogType_ResponseHeader, + Msg: "response header", + }) + } return } diff --git a/option.go b/option.go index becf6ab..f82fb78 100644 --- a/option.go +++ b/option.go @@ -54,8 +54,8 @@ type ClientOption struct { MaxRetries int //try num MaxRedirect int //redirect num ,<0 no redirect,==0 no limit Timeout time.Duration //request timeout - ResponseHeaderTimeout time.Duration //ResponseHeaderTimeout ,default:300 - TlsHandshakeTimeout time.Duration //tls timeout,default:15 + ResponseHeaderTimeout time.Duration //ResponseHeaderTimeout + TlsHandshakeTimeout time.Duration //tls timeout ForceHttp3 bool //force use http3 send requests ForceHttp1 bool //force use http1 send requests DisCookie bool //disable cookies @@ -85,8 +85,8 @@ type RequestOption struct { MaxRetries int //try num MaxRedirect int //redirect num ,<0 no redirect,==0 no limit Timeout time.Duration //request timeout - ResponseHeaderTimeout time.Duration //ResponseHeaderTimeout ,default:300 - TlsHandshakeTimeout time.Duration //tls timeout,default:15 + ResponseHeaderTimeout time.Duration //ResponseHeaderTimeout + TlsHandshakeTimeout time.Duration //tls timeout ForceHttp3 bool //force use http3 send requests ForceHttp1 bool //force use http1 send requests DisCookie bool //disable cookies @@ -201,10 +201,9 @@ func merge(option *RequestOption, clientOption *ClientOption) { option.Bar = clientOption.Bar } } + func (obj *Client) newRequestOption(option RequestOption) (RequestOption, error) { - // err := tools.Merge(&option, obj.ClientOption) merge(&option, obj.ClientOption) - //end if option.MaxRetries < 0 { option.MaxRetries = 0 } diff --git a/requests.go b/requests.go index 3bfd6f4..7b4a145 100644 --- a/requests.go +++ b/requests.go @@ -6,7 +6,6 @@ import ( "io" "os" "strings" - "time" "net/url" @@ -239,10 +238,6 @@ func (obj *Client) request(ctx *Response) (err error) { return } } - //init tls timeout - if ctx.option.TlsHandshakeTimeout == 0 { - ctx.option.TlsHandshakeTimeout = time.Second * 15 - } //init proxy if ctx.option.Proxy != nil { ctx.proxys, err = parseProxy(ctx.option.Proxy) diff --git a/roundTripper.go b/roundTripper.go index 7e5c876..f7e26ac 100644 --- a/roundTripper.go +++ b/roundTripper.go @@ -115,6 +115,11 @@ func (obj *roundTripper) ghttp3Dial(ctx *Response, remoteAddress Address, proxyA if err != nil { return nil, err } + defer func() { + if err != nil { + udpConn.Close() + } + }() tlsConfig := ctx.option.TlsConfig.Clone() tlsConfig.NextProtos = []string{http3.NextProtoH3} tlsConfig.ServerName = remoteAddress.Host @@ -133,7 +138,6 @@ func (obj *roundTripper) ghttp3Dial(ctx *Response, remoteAddress Address, proxyA return nil, err } cctx, ccnl := context.WithCancelCause(obj.ctx) - // conn = obj.newConnecotr() conn = http3.NewClient(cctx, netConn, udpConn, func() { ccnl(errors.New("http3 client close")) }) @@ -156,6 +160,11 @@ func (obj *roundTripper) uhttp3Dial(ctx *Response, remoteAddress Address, proxyA if err != nil { return nil, err } + defer func() { + if err != nil { + udpConn.Close() + } + }() tlsConfig := ctx.option.UtlsConfig.Clone() tlsConfig.NextProtos = []string{http3.NextProtoH3} tlsConfig.ServerName = remoteAddress.Host @@ -227,18 +236,22 @@ func (obj *roundTripper) dial(ctx *Response) (conn http1.Conn, err error) { } rawNetConn, err = obj.dialer.DialContext(ctx, "tcp", remoteAddress) } - defer func() { - if err != nil && rawNetConn != nil { + if err != nil { + if rawNetConn != nil { rawNetConn.Close() } - }() - if err != nil { return nil, err } var h2 bool var rawConn net.Conn if ctx.request.URL.Scheme == "https" { - rawConn, h2, err = obj.dialAddTls(ctx.option, ctx.request, rawNetConn) + if ctx.option.TlsHandshakeTimeout > 0 { + tlsCtx, tlsCnl := context.WithTimeout(ctx.Context(), ctx.option.TlsHandshakeTimeout) + rawConn, h2, err = obj.dialAddTls(tlsCtx, ctx.option, ctx.request, rawNetConn) + tlsCnl() + } else { + rawConn, h2, err = obj.dialAddTls(ctx.Context(), ctx.option, ctx.request, rawNetConn) + } if ctx.option.Logger != nil { ctx.option.Logger(Log{ Id: ctx.requestId, @@ -247,17 +260,23 @@ func (obj *roundTripper) dial(ctx *Response) (conn http1.Conn, err error) { Msg: fmt.Sprintf("host:%s, h2:%t", getHost(ctx.request), h2), }) } - if err != nil { - return nil, err - } } else { rawConn = rawNetConn } + if err != nil { + if rawConn != nil { + rawConn.Close() + } + return nil, err + } if arch != nil { rawConn, err = NewCompressionConn(rawConn, arch) - if err != nil { - return nil, err + } + if err != nil { + if rawConn != nil { + rawConn.Close() } + return nil, err } return obj.dialConnecotr(ctx, rawConn, h2) } @@ -278,9 +297,7 @@ func (obj *roundTripper) dialConnecotr(ctx *Response, rawCon net.Conn, h2 bool) } return } -func (obj *roundTripper) dialAddTls(option *RequestOption, req *http.Request, netConn net.Conn) (net.Conn, bool, error) { - ctx, cnl := context.WithTimeout(req.Context(), option.TlsHandshakeTimeout) - defer cnl() +func (obj *roundTripper) dialAddTls(ctx context.Context, option *RequestOption, req *http.Request, netConn net.Conn) (net.Conn, bool, error) { if option.gospiderSpec != nil && option.gospiderSpec.TLSSpec != nil { if tlsConn, err := obj.dialer.addJa3Tls(ctx, netConn, getHost(req), option.gospiderSpec.TLSSpec, option.UtlsConfig.Clone(), option.ForceHttp1); err != nil { return tlsConn, false, tools.WrapError(err, "add ja3 tls error") @@ -345,6 +362,9 @@ func (obj *roundTripper) newRoundTrip(task *reqTask) error { task.reqCtx.isNewConn = true conn, err := obj.dial(task.reqCtx) if err != nil { + if conn != nil { + conn.CloseWithError(err) + } err = tools.WrapError(err, "newRoudTrip dial error") if task.reqCtx.option.ErrCallBack != nil { task.reqCtx.err = err @@ -362,9 +382,6 @@ func (obj *roundTripper) newRoundTrip(task *reqTask) error { } func (obj *roundTripper) newReqTask(ctx *Response) (*reqTask, error) { - if ctx.option.ResponseHeaderTimeout == 0 { - ctx.option.ResponseHeaderTimeout = time.Second * 300 - } task := new(reqTask) task.reqCtx = ctx task.reqCtx.response = nil diff --git a/test/request/stream_test.go b/test/request/stream_test.go index 9171905..2ed86df 100644 --- a/test/request/stream_test.go +++ b/test/request/stream_test.go @@ -27,8 +27,9 @@ func TestStream(t *testing.T) { } } func TestStreamWithConn(t *testing.T) { + session, _ := requests.NewClient(nil) for i := 0; i < 2; i++ { - resp, err := requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{Stream: true}) + resp, err := session.Get(nil, "https://httpbin.org/anything", requests.RequestOption{Stream: true}) if err != nil { t.Fatal(err) } @@ -52,8 +53,9 @@ func TestStreamWithConn(t *testing.T) { } func TestStreamWithConn2(t *testing.T) { + session, _ := requests.NewClient(nil) for i := 0; i < 2; i++ { - resp, err := requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{Stream: true}) + resp, err := session.Get(nil, "https://httpbin.org/anything", requests.RequestOption{Stream: true}) if err != nil { t.Fatal(err) } @@ -77,8 +79,9 @@ func TestStreamWithConn2(t *testing.T) { } func TestStreamWithConn3(t *testing.T) { + session, _ := requests.NewClient(nil) for i := 0; i < 2; i++ { - resp, err := requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{Stream: true}) + resp, err := session.Get(nil, "https://httpbin.org/anything", requests.RequestOption{Stream: true}) if err != nil { t.Fatal(err) } @@ -102,8 +105,9 @@ func TestStreamWithConn3(t *testing.T) { } func TestStreamWithConn4(t *testing.T) { + session, _ := requests.NewClient(nil) for i := 0; i < 2; i++ { - resp, err := requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{Stream: true}) + resp, err := session.Get(nil, "https://httpbin.org/anything", requests.RequestOption{Stream: true}) if err != nil { t.Fatal(err) } @@ -129,8 +133,9 @@ func TestStreamWithConn4(t *testing.T) { } } func TestStreamWithConn5(t *testing.T) { + session, _ := requests.NewClient(nil) for i := 0; i < 2; i++ { - resp, err := requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{Stream: true}) + resp, err := session.Get(nil, "https://httpbin.org/anything", requests.RequestOption{Stream: true}) if err != nil { t.Fatal(err) }