diff --git a/go.mod b/go.mod index cd39e91..66cc985 100644 --- a/go.mod +++ b/go.mod @@ -9,11 +9,12 @@ require ( github.com/gospider007/bs4 v0.0.0-20250413121342-fed910fb00c9 github.com/gospider007/gson v0.0.0-20250630120534-cce6e3c6756d github.com/gospider007/gtls v0.0.0-20250630120509-4e99c91661ee + github.com/gospider007/http1 v0.0.0-20250717080213-6dc69bcb2a33 github.com/gospider007/http2 v0.0.0-20250711035043-daabc8e205b3 - github.com/gospider007/http3 v0.0.0-20250630120526-1066890881e5 + github.com/gospider007/http3 v0.0.0-20250711035127-be7539ebc56a github.com/gospider007/ja3 v0.0.0-20250627013834-1d2966014638 github.com/gospider007/re v0.0.0-20250217075352-bcb79f285d6c - github.com/gospider007/tools v0.0.0-20250630120304-b22c2ddf35b5 + github.com/gospider007/tools v0.0.0-20250711035142-241d6041da43 github.com/gospider007/websocket v0.0.0-20250630120328-1ec26253d082 github.com/klauspost/compress v1.18.0 github.com/minio/minlz v1.0.1 @@ -50,7 +51,7 @@ require ( github.com/gospider007/kinds v0.0.0-20250217075226-10f199f7215d // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/cpuid/v2 v2.2.11 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/klauspost/pgzip v1.2.6 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/libdns/libdns v1.1.0 // indirect @@ -81,11 +82,11 @@ require ( go.uber.org/zap v1.27.0 // indirect go.uber.org/zap/exp v0.3.0 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect - golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b // indirect + golang.org/x/exp v0.0.0-20250717185816-542afb5b7346 // indirect golang.org/x/image v0.29.0 // indirect golang.org/x/mod v0.26.0 // indirect golang.org/x/sync v0.16.0 // indirect golang.org/x/sys v0.34.0 // indirect golang.org/x/text v0.27.0 // indirect - golang.org/x/tools v0.34.0 // indirect + golang.org/x/tools v0.35.0 // indirect ) diff --git a/go.sum b/go.sum index da78413..e0983e1 100644 --- a/go.sum +++ b/go.sum @@ -116,18 +116,20 @@ github.com/gospider007/gson v0.0.0-20250630120534-cce6e3c6756d h1:/FaI2cPp/DgiEJ github.com/gospider007/gson v0.0.0-20250630120534-cce6e3c6756d/go.mod h1:Vb5Ehy5UGQYVUW2XvzqLW1rc7/WPljQspyF3YZ/u164= github.com/gospider007/gtls v0.0.0-20250630120509-4e99c91661ee h1:rcQAnk5Szv/Vi9dVieU1uwKUSVPNB+lqRPggI6oZgFM= github.com/gospider007/gtls v0.0.0-20250630120509-4e99c91661ee/go.mod h1:ee+Fo1W5Abt9mdLOOiTrb76r0qbO6ubGmRDklCMAPIQ= +github.com/gospider007/http1 v0.0.0-20250717080213-6dc69bcb2a33 h1:VNzJJZ/Yw72/NffOeYk61+OoMaRsrvC68aRGoT4u0QQ= +github.com/gospider007/http1 v0.0.0-20250717080213-6dc69bcb2a33/go.mod h1:HFJILuzw8YEWk3SGHwHhymbUeF4LpuGAuzOFUHaSV58= github.com/gospider007/http2 v0.0.0-20250711035043-daabc8e205b3 h1:HqEs4ZAU1U7hGKwEQwE/0YYwldByzaD/YfXpSrhgdVA= github.com/gospider007/http2 v0.0.0-20250711035043-daabc8e205b3/go.mod h1:rji8qgw9KJ3ddOqT7xvgd0GKV7Z4+l1uSbFrgDX3blg= -github.com/gospider007/http3 v0.0.0-20250630120526-1066890881e5 h1:+v1s/f9AYApykZ/alJIOX4V7xj8YTiydfV8izakjaY4= -github.com/gospider007/http3 v0.0.0-20250630120526-1066890881e5/go.mod h1:dyg0PmV4MW4mxUG6VzJxhAOTBboMbrpC3/XVVMDvV3Q= +github.com/gospider007/http3 v0.0.0-20250711035127-be7539ebc56a h1:qeKaaZlrWddcvuWWv0pbT77L5oxlurYGoR96wiyS9lc= +github.com/gospider007/http3 v0.0.0-20250711035127-be7539ebc56a/go.mod h1:4Og1NRXDGkmcoMwKfeG9vihSNkU/uNnQ3QxsxWRuWJ8= github.com/gospider007/ja3 v0.0.0-20250627013834-1d2966014638 h1:qQmKi3FsCK0WJcDOq4wpOzwVZ2UWto1Da05SGRVBKPQ= github.com/gospider007/ja3 v0.0.0-20250627013834-1d2966014638/go.mod h1:m62r83RcU8Dx7yyXNDnOfa8AfjPKAkOSzPZzgQRfP1k= github.com/gospider007/kinds v0.0.0-20250217075226-10f199f7215d h1:+Fih572EdNmYCK1L82MyWOj9tAwIgZ2JqepmasAclg8= github.com/gospider007/kinds v0.0.0-20250217075226-10f199f7215d/go.mod h1:3u6J+nbdWhCNJGiQjq/hKSukn2k6ebjd98rd2+fzKNo= github.com/gospider007/re v0.0.0-20250217075352-bcb79f285d6c h1:8/Cf+c2680tkWJ+ueZ9RLLK5R5R8nhE8pNBUPHjkvkM= github.com/gospider007/re v0.0.0-20250217075352-bcb79f285d6c/go.mod h1:dd8aDIUG1vDPP5r+vHBtiUK0Zn6uk3SsWt1ZvmCCHLs= -github.com/gospider007/tools v0.0.0-20250630120304-b22c2ddf35b5 h1:zUWAbun8/xSrLZeMaQZNbSrQ7qmkrxAiv8ee4Ya09R8= -github.com/gospider007/tools v0.0.0-20250630120304-b22c2ddf35b5/go.mod h1:xfS5GwtnDblCI/vbvA5epqjwWb+olFWygUVDSYV01f4= +github.com/gospider007/tools v0.0.0-20250711035142-241d6041da43 h1:dZnCK7rDn8MBU2vqN7ELfG9rGAzdxuPlPVBZuEnzw2o= +github.com/gospider007/tools v0.0.0-20250711035142-241d6041da43/go.mod h1:i0V5k/zJW/ghTLLWY40pwTVuwa/Gxm2Q1IP1vQYHBX8= github.com/gospider007/websocket v0.0.0-20250630120328-1ec26253d082 h1:0UAc9x1HqsNn+XDTZEcGS/d93dRO0AHryh9KMCtq9LQ= github.com/gospider007/websocket v0.0.0-20250630120328-1ec26253d082/go.mod h1:UiboIV1XnPhEMEd4yXPJAGO/ykHh/fG5nsJRDqpuaEw= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -144,8 +146,8 @@ github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0 github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU= -github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -285,8 +287,8 @@ golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b h1:M2rDM6z3Fhozi9O7NWsxAkg/yqS/lQJ6PmkyIV3YP+o= -golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8= +golang.org/x/exp v0.0.0-20250717185816-542afb5b7346 h1:vuCObX8mQzik1tfEcYxWZBuVsmQtD1IjxCyPKM18Bh4= +golang.org/x/exp v0.0.0-20250717185816-542afb5b7346/go.mod h1:A+z0yzpGtvnG90cToK5n2tu8UJVP2XUATh+r+sfOOOc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.29.0 h1:HcdsyR4Gsuys/Axh0rDEmlBmB68rW1U9BUdB3UVHsas= @@ -445,8 +447,8 @@ golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= -golang.org/x/tools v0.34.0 h1:qIpSLOxeCYGg9TrcJokLBG4KFA6d795g0xkBkiESGlo= -golang.org/x/tools v0.34.0/go.mod h1:pAP9OwEaY1CAW3HOmg3hLZC5Z0CCmzjAF2UQMSqNARg= +golang.org/x/tools v0.35.0 h1:mBffYraMEf7aa0sB+NuKnuCy8qI/9Bughn8dC2Gu5r0= +golang.org/x/tools v0.35.0/go.mod h1:NKdj5HkL/73byiZSJjqJgKn3ep7KjFkBOkR/Hps3VPw= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/response.go b/response.go index 42f2e60..2ff9a48 100644 --- a/response.go +++ b/response.go @@ -347,11 +347,6 @@ func (obj *Response) ReadBody() (err error) { return nil } obj.readBody = true - defer func() { - if err == nil && obj.response.StatusCode == 101 && obj.webSocket == nil { - obj.webSocket = websocket.NewConn(newFakeConn(obj.body.connStream()), func() { obj.CloseConn() }, true, obj.Headers().Get("Sec-WebSocket-Extensions")) - } - }() bBody := bytes.NewBuffer(nil) done := make(chan struct{}) var readErr error @@ -408,26 +403,35 @@ func (obj *body) Read(p []byte) (n int, err error) { } n, err = obj.ctx.response.Body.Read(p) if err != nil { - obj.closed = true if err != io.EOF && err != io.ErrUnexpectedEOF { - obj.err = err - obj.ctx.Close(err) - obj.ctx.CloseConn() + obj.CloseWithError(err) } else { - obj.ctx.Close(nil) + obj.CloseWithError(nil) } } return } func (obj *body) Close() (err error) { - obj.closed = true - if !obj.closed { - obj.err = errors.New("response body force closed") - obj.ctx.Close(errors.New("response body force closed")) - obj.ctx.CloseConn() + return obj.CloseWithError(obj.err) +} +func (obj *body) CloseWithError(err error) error { + if obj.closed { + return obj.err + } + obj.closed = true + if err == nil { + if obj.ctx.StatusCode() == 101 && obj.ctx.webSocket == nil { + obj.ctx.webSocket = websocket.NewConn(newFakeConn(obj.ctx.body.connStream()), func() { obj.ctx.CloseConn() }, true, obj.ctx.Headers().Get("Sec-WebSocket-Extensions")) + } + obj.ctx.Close(nil) + return nil + } else { + obj.err = err + obj.ctx.Close(err) + obj.ctx.CloseConn() + return err } - return nil } func (obj *Response) Body() *body { diff --git a/roundTripper.go b/roundTripper.go index de2b97a..91ea290 100644 --- a/roundTripper.go +++ b/roundTripper.go @@ -11,6 +11,7 @@ import ( "net/http" + "github.com/gospider007/http1" "github.com/gospider007/http2" "github.com/gospider007/http3" "github.com/gospider007/ja3" @@ -292,7 +293,7 @@ func (obj *roundTripper) dialConnecotr(ctx *Response, conne *connecotr, h2 bool) return err } } else { - conne.Conn = NewClientConn(conne.c, func(err error) { + conne.Conn = http1.NewClientConn(conne.c, func(err error) { conne.forceCnl(tools.WrapError(err, "http1 client close")) }) } diff --git a/test/fingerprint/ja4_test.go b/test/fingerprint/ja4_test.go index c325dc4..585ca96 100644 --- a/test/fingerprint/ja4_test.go +++ b/test/fingerprint/ja4_test.go @@ -1,7 +1,6 @@ package main import ( - "log" "net/textproto" "slices" "testing" @@ -49,55 +48,3 @@ func TestOrderHeaders(t *testing.T) { } } } -func TestOrderHeaders2(t *testing.T) { - - headers := map[string]any{ - "Accept-Encoding": "gzip, deflate, br", - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7", - "Sec-Ch-Ua-Mobile": "?0", - "Sec-Ch-Ua-Platform": `"Windows"`, - } - orderHeaders := []string{ - "Accept-Encoding", - "Accept", - "User-Agent", - "Accept-Language", - "Sec-Ch-Ua", - "Sec-Ch-Ua-Mobile", - "Sec-Ch-Ua-Platform", - } - resp, err := requests.Get(nil, "https://tools.scrapfly.io/api/fp/anything", requests.RequestOption{ - ClientOption: requests.ClientOption{ - - Headers: headers, - OrderHeaders: orderHeaders, - }, - // ForceHttp1: true, - }) - if err != nil { - t.Fatal(err) - } - jsonData, err := resp.Json() - header_order := jsonData.Find("ordered_headers_key") - if !header_order.Exists() { - t.Fatal("not found akamai") - } - i := -1 - log.Print(header_order) - // log.Print(headers.Keys()) - kks := []string{} - for _, kk := range orderHeaders { - kks = append(kks, textproto.CanonicalMIMEHeaderKey(kk)) - } - for _, key := range header_order.Array() { - kk := textproto.CanonicalMIMEHeaderKey(key.String()) - if slices.Contains(kks, kk) { - i2 := slices.Index(kks, textproto.CanonicalMIMEHeaderKey(kk)) - if i2 < i { - log.Print(header_order) - t.Fatal("not equal") - } - i = i2 - } - } -} diff --git a/tools.go b/tools.go index 4d54944..044367f 100644 --- a/tools.go +++ b/tools.go @@ -185,6 +185,7 @@ func NewRequestWithContext(ctx context.Context, method string, u *url.URL, body req.Proto = "HTTP/1.1" req.ProtoMajor = 1 req.ProtoMinor = 1 + req.ContentLength = -1 if body != nil { if v, ok := body.(interface{ Len() int }); ok {