diff --git a/compressConn.go b/compressConn.go index f27fde9..2dd83ca 100644 --- a/compressConn.go +++ b/compressConn.go @@ -116,7 +116,6 @@ func NewCompressionConn(conn net.Conn, arch Compression) (net.Conn, error) { r: r, w: w, oneFunc: sync.OnceFunc(func() { - defer recover() if snW, ok := w.(*snappy.Writer); ok { putSnappyWriter(snW) } diff --git a/go.mod b/go.mod index c86c10b..3c7c033 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/gospider007/bs4 v0.0.0-20250413121342-fed910fb00c9 github.com/gospider007/gson v0.0.0-20250611163241-fa021e9c5531 github.com/gospider007/gtls v0.0.0-20250610060422-446e017b9858 - github.com/gospider007/http2 v0.0.0-20250611163252-ee5823d0f04a + github.com/gospider007/http2 v0.0.0-20250616085600-a49f075f841e github.com/gospider007/http3 v0.0.0-20250611163115-77b77de7a3f9 github.com/gospider007/ja3 v0.0.0-20250615121404-6d7f30dfa700 github.com/gospider007/re v0.0.0-20250217075352-bcb79f285d6c @@ -80,7 +80,7 @@ require ( go.uber.org/zap v1.27.0 // indirect go.uber.org/zap/exp v0.3.0 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect - golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476 // indirect + golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b // indirect golang.org/x/image v0.28.0 // indirect golang.org/x/mod v0.25.0 // indirect golang.org/x/sync v0.15.0 // indirect diff --git a/go.sum b/go.sum index 9514991..0d3baff 100644 --- a/go.sum +++ b/go.sum @@ -114,8 +114,8 @@ github.com/gospider007/gson v0.0.0-20250611163241-fa021e9c5531 h1:dzUU4p3u9eaY1v github.com/gospider007/gson v0.0.0-20250611163241-fa021e9c5531/go.mod h1:YO6h5iR/nAFAaZ34JJbkJPuMG2nq2mShKWI7llEWRQA= github.com/gospider007/gtls v0.0.0-20250610060422-446e017b9858 h1:Tb+T5YfysjR7Mp22AgrJhMPF28KBgjWkbKj2zi8AyBU= github.com/gospider007/gtls v0.0.0-20250610060422-446e017b9858/go.mod h1:SxhMpxq4EYHlUre2EdQ4dKKGZyvj7+1PYZ2LQ9RO0y4= -github.com/gospider007/http2 v0.0.0-20250611163252-ee5823d0f04a h1:OtgLN7N2rFHxF1HXutXeY33bzNAZ/npzQQ4JlaDZk+8= -github.com/gospider007/http2 v0.0.0-20250611163252-ee5823d0f04a/go.mod h1:r0XfXyaHynGBLi0o7SN6KxPL0V7q5sX2JVvk2ISDyH0= +github.com/gospider007/http2 v0.0.0-20250616085600-a49f075f841e h1:fO2FFvKS/BAX7saBk6YKuJD28wD/UZ078fkfBetcB2s= +github.com/gospider007/http2 v0.0.0-20250616085600-a49f075f841e/go.mod h1:Uz/rGu9oVkQmroHDeDSS1yhWqaLGHj5UbJdvAXu645E= github.com/gospider007/http3 v0.0.0-20250611163115-77b77de7a3f9 h1:6OiYSTsyrabSDV0dj5SpAGenpPm8VA20fD/FVzrx/TM= github.com/gospider007/http3 v0.0.0-20250611163115-77b77de7a3f9/go.mod h1:FObAlfJRtytX4nI47Ra+j2mKoJlUmxfraqpeOBX2Bi0= github.com/gospider007/ja3 v0.0.0-20250615121404-6d7f30dfa700 h1:UsY1LwHsnvZXdXjoExpwKK/bAiqvfowpwUoLUCG25WU= @@ -283,8 +283,8 @@ golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476 h1:bsqhLWFR6G6xiQcb+JoGqdKdRU6WzPWmK8E0jxTjzo4= -golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8= +golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b h1:M2rDM6z3Fhozi9O7NWsxAkg/yqS/lQJ6PmkyIV3YP+o= +golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.28.0 h1:gdem5JW1OLS4FbkWgLO+7ZeFzYtL3xClb97GaUzYMFE= diff --git a/http.go b/http.go index c247878..68c09e9 100644 --- a/http.go +++ b/http.go @@ -68,13 +68,14 @@ func (obj *clientConn) send(req *http.Request, orderHeaders []interface { return } rawBody := res.Body + isStream := res.StatusCode == 101 || strings.Contains(res.Header.Get("Content-Type"), "text/event-stream") pr, pw := io.Pipe() go func() { var readErr error defer func() { obj.readWriteCtx.readCnl(readErr) }() - if res.Body != nil { + if rawBody != nil { _, readErr = tools.Copy(pw, rawBody) } if readErr != nil && readErr != io.EOF && readErr != io.ErrUnexpectedEOF { @@ -88,7 +89,7 @@ func (obj *clientConn) send(req *http.Request, orderHeaders []interface { } else { select { case <-obj.readWriteCtx.writeCtx.Done(): - if res.StatusCode == 101 || strings.Contains(res.Header.Get("Content-Type"), "text/event-stream") { + if isStream { <-obj.ctx.Done() return } diff --git a/option.go b/option.go index 23daa43..60aa2a6 100644 --- a/option.go +++ b/option.go @@ -42,6 +42,7 @@ type Log struct { // Connection Management Options type ClientOption struct { Spec string //goSpiderSpec origin : https://github.com/gospider007/fp + OrderHeaders []string DialOption DialOption Headers any //default headers Jar Jar //custom cookies diff --git a/requests.go b/requests.go index 551f37f..2b484e2 100644 --- a/requests.go +++ b/requests.go @@ -201,6 +201,9 @@ func (obj *Client) Request(ctx context.Context, method string, href string, opti } func (obj *Client) request(ctx *Response) (err error) { defer func() { + if ctx.filePath != "" { + return + } //read body if err == nil && ctx.sse == nil && !ctx.option.Stream { err = ctx.ReadBody() diff --git a/response.go b/response.go index 74917b7..9726351 100644 --- a/response.go +++ b/response.go @@ -161,6 +161,9 @@ func (obj *Response) SSE() *SSE { // return URL redirected address func (obj *Response) Location() (*url.URL, error) { + if obj.filePath != "" { + return nil, nil + } u, err := obj.response.Location() if err == http.ErrNoLocation { err = nil @@ -246,7 +249,7 @@ func (obj *Response) SetContent(val []byte) { // return content with []byte func (obj *Response) Content() []byte { - if obj.webSocket == nil && obj.sse == nil { + if obj.webSocket == nil && obj.sse == nil && obj.filePath == "" { obj.ReadBody() } return obj.content diff --git a/sepc.go b/sepc.go index d0c988e..4606891 100644 --- a/sepc.go +++ b/sepc.go @@ -112,31 +112,46 @@ func (obj *RequestOption) initSpec() error { return err } obj.gospiderSpec = gospiderSpec - if gospiderSpec.H1Spec != nil { - if obj.orderHeaders == nil { - orderData := NewOrderData() + if obj.orderHeaders == nil { + if len(obj.OrderHeaders) > 0 { + obj.orderHeaders = NewOrderData() + var ods [][2]string + if gospiderSpec.H1Spec != nil { + ods = gospiderSpec.H1Spec.OrderHeaders + } else if gospiderSpec.H2Spec != nil { + ods = gospiderSpec.H2Spec.OrderHeaders + } + for _, key := range obj.OrderHeaders { + key = textproto.CanonicalMIMEHeaderKey(key) + var val any + for _, kv := range ods { + if key == kv[0] && slices.Contains(tools.DefaultHeaderKeys, kv[0]) { + val = kv[1] + break + } + } + obj.orderHeaders.Add(key, val) + } + } else if gospiderSpec.H1Spec != nil { + obj.orderHeaders = NewOrderData() for _, kv := range gospiderSpec.H1Spec.OrderHeaders { if slices.Contains(tools.DefaultHeaderKeys, kv[0]) { - orderData.Add(kv[0], kv[1]) + obj.orderHeaders.Add(kv[0], kv[1]) } else { - orderData.Add(kv[0], nil) + obj.orderHeaders.Add(kv[0], nil) } } - obj.orderHeaders = orderData - } - } - if gospiderSpec.H2Spec != nil { - if obj.orderHeaders == nil { - orderData := NewOrderData() + } else if gospiderSpec.H2Spec != nil { + obj.orderHeaders = NewOrderData() for _, kv := range gospiderSpec.H2Spec.OrderHeaders { key := textproto.CanonicalMIMEHeaderKey(kv[0]) if slices.Contains(tools.DefaultHeaderKeys, key) { - orderData.Add(key, kv[1]) + obj.orderHeaders.Add(key, kv[1]) } else { - orderData.Add(key, nil) + obj.orderHeaders.Add(key, nil) } } - obj.orderHeaders = orderData + } } return nil diff --git a/snappy.go b/snappy.go index c4f18e0..4312eb3 100644 --- a/snappy.go +++ b/snappy.go @@ -32,6 +32,8 @@ func getSnappyWriter(w io.Writer) *snappy.Writer { // 释放 snappy.Writer func putSnappyWriter(sw *snappy.Writer) { + defer recover() + // sw.Reset(nil) snappyWriterPool.Put(sw) } @@ -44,6 +46,7 @@ func getSnappyReader(r io.Reader) *snappy.Reader { // 释放 snappy.Reader func putSnappyReader(sr *snappy.Reader) { - // snappy.Reader 没有 Close 方法，直接放回池 + defer recover() + // sr.Reset(nil) snappyReaderPool.Put(sr) }