package/requests
1
0
Fork 0
mirror of https://github.com/gospider007/requests.git synced 2025-12-24 13:57:52 +08:00
Code Issues Packages Projects Releases Wiki Activity

sync

Browse Source
This commit is contained in:
gospider
2025-07-18 08:52:19 +08:00
parent 45c7a3262f
commit dd76b7ca0b
6 changed files with 41 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
go.mod
View File

@@ -9,11 +9,12 @@ require (
github.com/gospider007/bs4 v0.0.0-20250413121342-fed910fb00c9
github.com/gospider007/gson v0.0.0-20250630120534-cce6e3c6756d
github.com/gospider007/gtls v0.0.0-20250630120509-4e99c91661ee
github.com/gospider007/http1 v0.0.0-20250717080213-6dc69bcb2a33
github.com/gospider007/http2 v0.0.0-20250711035043-daabc8e205b3
github.com/gospider007/http3 v0.0.0-20250630120526-1066890881e5
github.com/gospider007/http3 v0.0.0-20250711035127-be7539ebc56a
github.com/gospider007/ja3 v0.0.0-20250627013834-1d2966014638
github.com/gospider007/re v0.0.0-20250217075352-bcb79f285d6c
github.com/gospider007/tools v0.0.0-20250630120304-b22c2ddf35b5
github.com/gospider007/tools v0.0.0-20250711035142-241d6041da43
github.com/gospider007/websocket v0.0.0-20250630120328-1ec26253d082
github.com/klauspost/compress v1.18.0
github.com/minio/minlz v1.0.1
@@ -50,7 +51,7 @@ require (
github.com/gospider007/kinds v0.0.0-20250217075226-10f199f7215d // indirect
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/cpuid/v2 v2.2.11 // indirect
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
github.com/klauspost/pgzip v1.2.6 // indirect
github.com/kr/pretty v0.3.1 // indirect
github.com/libdns/libdns v1.1.0 // indirect
@@ -81,11 +82,11 @@ require (
go.uber.org/zap v1.27.0 // indirect
go.uber.org/zap/exp v0.3.0 // indirect
go4.org v0.0.0-20230225012048-214862532bf5 // indirect
golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b // indirect
golang.org/x/exp v0.0.0-20250717185816-542afb5b7346 // indirect
golang.org/x/image v0.29.0 // indirect
golang.org/x/mod v0.26.0 // indirect
golang.org/x/sync v0.16.0 // indirect
golang.org/x/sys v0.34.0 // indirect
golang.org/x/text v0.27.0 // indirect
golang.org/x/tools v0.34.0 // indirect
golang.org/x/tools v0.35.0 // indirect
)

22
go.sum
View File

@@ -116,18 +116,20 @@ github.com/gospider007/gson v0.0.0-20250630120534-cce6e3c6756d h1:/FaI2cPp/DgiEJ
github.com/gospider007/gson v0.0.0-20250630120534-cce6e3c6756d/go.mod h1:Vb5Ehy5UGQYVUW2XvzqLW1rc7/WPljQspyF3YZ/u164=
github.com/gospider007/gtls v0.0.0-20250630120509-4e99c91661ee h1:rcQAnk5Szv/Vi9dVieU1uwKUSVPNB+lqRPggI6oZgFM=
github.com/gospider007/gtls v0.0.0-20250630120509-4e99c91661ee/go.mod h1:ee+Fo1W5Abt9mdLOOiTrb76r0qbO6ubGmRDklCMAPIQ=
github.com/gospider007/http1 v0.0.0-20250717080213-6dc69bcb2a33 h1:VNzJJZ/Yw72/NffOeYk61+OoMaRsrvC68aRGoT4u0QQ=
github.com/gospider007/http1 v0.0.0-20250717080213-6dc69bcb2a33/go.mod h1:HFJILuzw8YEWk3SGHwHhymbUeF4LpuGAuzOFUHaSV58=
github.com/gospider007/http2 v0.0.0-20250711035043-daabc8e205b3 h1:HqEs4ZAU1U7hGKwEQwE/0YYwldByzaD/YfXpSrhgdVA=
github.com/gospider007/http2 v0.0.0-20250711035043-daabc8e205b3/go.mod h1:rji8qgw9KJ3ddOqT7xvgd0GKV7Z4+l1uSbFrgDX3blg=
github.com/gospider007/http3 v0.0.0-20250630120526-1066890881e5 h1:+v1s/f9AYApykZ/alJIOX4V7xj8YTiydfV8izakjaY4=
github.com/gospider007/http3 v0.0.0-20250630120526-1066890881e5/go.mod h1:dyg0PmV4MW4mxUG6VzJxhAOTBboMbrpC3/XVVMDvV3Q=
github.com/gospider007/http3 v0.0.0-20250711035127-be7539ebc56a h1:qeKaaZlrWddcvuWWv0pbT77L5oxlurYGoR96wiyS9lc=
github.com/gospider007/http3 v0.0.0-20250711035127-be7539ebc56a/go.mod h1:4Og1NRXDGkmcoMwKfeG9vihSNkU/uNnQ3QxsxWRuWJ8=
github.com/gospider007/ja3 v0.0.0-20250627013834-1d2966014638 h1:qQmKi3FsCK0WJcDOq4wpOzwVZ2UWto1Da05SGRVBKPQ=
github.com/gospider007/ja3 v0.0.0-20250627013834-1d2966014638/go.mod h1:m62r83RcU8Dx7yyXNDnOfa8AfjPKAkOSzPZzgQRfP1k=
github.com/gospider007/kinds v0.0.0-20250217075226-10f199f7215d h1:+Fih572EdNmYCK1L82MyWOj9tAwIgZ2JqepmasAclg8=
github.com/gospider007/kinds v0.0.0-20250217075226-10f199f7215d/go.mod h1:3u6J+nbdWhCNJGiQjq/hKSukn2k6ebjd98rd2+fzKNo=
github.com/gospider007/re v0.0.0-20250217075352-bcb79f285d6c h1:8/Cf+c2680tkWJ+ueZ9RLLK5R5R8nhE8pNBUPHjkvkM=
github.com/gospider007/re v0.0.0-20250217075352-bcb79f285d6c/go.mod h1:dd8aDIUG1vDPP5r+vHBtiUK0Zn6uk3SsWt1ZvmCCHLs=
github.com/gospider007/tools v0.0.0-20250630120304-b22c2ddf35b5 h1:zUWAbun8/xSrLZeMaQZNbSrQ7qmkrxAiv8ee4Ya09R8=
github.com/gospider007/tools v0.0.0-20250630120304-b22c2ddf35b5/go.mod h1:xfS5GwtnDblCI/vbvA5epqjwWb+olFWygUVDSYV01f4=
github.com/gospider007/tools v0.0.0-20250711035142-241d6041da43 h1:dZnCK7rDn8MBU2vqN7ELfG9rGAzdxuPlPVBZuEnzw2o=
github.com/gospider007/tools v0.0.0-20250711035142-241d6041da43/go.mod h1:i0V5k/zJW/ghTLLWY40pwTVuwa/Gxm2Q1IP1vQYHBX8=
github.com/gospider007/websocket v0.0.0-20250630120328-1ec26253d082 h1:0UAc9x1HqsNn+XDTZEcGS/d93dRO0AHryh9KMCtq9LQ=
github.com/gospider007/websocket v0.0.0-20250630120328-1ec26253d082/go.mod h1:UiboIV1XnPhEMEd4yXPJAGO/ykHh/fG5nsJRDqpuaEw=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
@@ -144,8 +146,8 @@ github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU=
github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU=
github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
@@ -285,8 +287,8 @@ golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE
golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b h1:M2rDM6z3Fhozi9O7NWsxAkg/yqS/lQJ6PmkyIV3YP+o=
golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8=
golang.org/x/exp v0.0.0-20250717185816-542afb5b7346 h1:vuCObX8mQzik1tfEcYxWZBuVsmQtD1IjxCyPKM18Bh4=
golang.org/x/exp v0.0.0-20250717185816-542afb5b7346/go.mod h1:A+z0yzpGtvnG90cToK5n2tu8UJVP2XUATh+r+sfOOOc=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/image v0.29.0 h1:HcdsyR4Gsuys/Axh0rDEmlBmB68rW1U9BUdB3UVHsas=
@@ -445,8 +447,8 @@ golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
golang.org/x/tools v0.34.0 h1:qIpSLOxeCYGg9TrcJokLBG4KFA6d795g0xkBkiESGlo=
golang.org/x/tools v0.34.0/go.mod h1:pAP9OwEaY1CAW3HOmg3hLZC5Z0CCmzjAF2UQMSqNARg=
golang.org/x/tools v0.35.0 h1:mBffYraMEf7aa0sB+NuKnuCy8qI/9Bughn8dC2Gu5r0=
golang.org/x/tools v0.35.0/go.mod h1:NKdj5HkL/73byiZSJjqJgKn3ep7KjFkBOkR/Hps3VPw=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

36
response.go
View File

@@ -347,11 +347,6 @@ func (obj *Response) ReadBody() (err error) {
return nil
}
obj.readBody = true
defer func() {
if err == nil && obj.response.StatusCode == 101 && obj.webSocket == nil {
obj.webSocket = websocket.NewConn(newFakeConn(obj.body.connStream()), func() { obj.CloseConn() }, true, obj.Headers().Get("Sec-WebSocket-Extensions"))
}
}()
bBody := bytes.NewBuffer(nil)
done := make(chan struct{})
var readErr error
@@ -408,26 +403,35 @@ func (obj *body) Read(p []byte) (n int, err error) {
}
n, err = obj.ctx.response.Body.Read(p)
if err != nil {
obj.closed = true
if err != io.EOF && err != io.ErrUnexpectedEOF {
obj.err = err
obj.ctx.Close(err)
obj.ctx.CloseConn()
obj.CloseWithError(err)
} else {
obj.ctx.Close(nil)
obj.CloseWithError(nil)
}
}
return
}
func (obj *body) Close() (err error) {
obj.closed = true
if !obj.closed {
obj.err = errors.New("response body force closed")
obj.ctx.Close(errors.New("response body force closed"))
obj.ctx.CloseConn()
return obj.CloseWithError(obj.err)
}
func (obj *body) CloseWithError(err error) error {
if obj.closed {
return obj.err
}
obj.closed = true
if err == nil {
if obj.ctx.StatusCode() == 101 && obj.ctx.webSocket == nil {
obj.ctx.webSocket = websocket.NewConn(newFakeConn(obj.ctx.body.connStream()), func() { obj.ctx.CloseConn() }, true, obj.ctx.Headers().Get("Sec-WebSocket-Extensions"))
}
obj.ctx.Close(nil)
return nil
} else {
obj.err = err
obj.ctx.Close(err)
obj.ctx.CloseConn()
return err
}
return nil
}
func (obj *Response) Body() *body {

3
roundTripper.go
View File

@@ -11,6 +11,7 @@ import (
"net/http"
"github.com/gospider007/http1"
"github.com/gospider007/http2"
"github.com/gospider007/http3"
"github.com/gospider007/ja3"
@@ -292,7 +293,7 @@ func (obj *roundTripper) dialConnecotr(ctx *Response, conne *connecotr, h2 bool)
return err
}
} else {
conne.Conn = NewClientConn(conne.c, func(err error) {
conne.Conn = http1.NewClientConn(conne.c, func(err error) {
conne.forceCnl(tools.WrapError(err, "http1 client close"))
})
}

53
test/fingerprint/ja4_test.go
View File

@@ -1,7 +1,6 @@
package main
import (
"log"
"net/textproto"
"slices"
"testing"
@@ -49,55 +48,3 @@ func TestOrderHeaders(t *testing.T) {
}
}
}
func TestOrderHeaders2(t *testing.T) {
headers := map[string]any{
"Accept-Encoding": "gzip, deflate, br",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
"Sec-Ch-Ua-Mobile": "?0",
"Sec-Ch-Ua-Platform": `"Windows"`,
}
orderHeaders := []string{
"Accept-Encoding",
"Accept",
"User-Agent",
"Accept-Language",
"Sec-Ch-Ua",
"Sec-Ch-Ua-Mobile",
"Sec-Ch-Ua-Platform",
}
resp, err := requests.Get(nil, "https://tools.scrapfly.io/api/fp/anything", requests.RequestOption{
ClientOption: requests.ClientOption{
Headers: headers,
OrderHeaders: orderHeaders,
},
// ForceHttp1: true,
})
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json()
header_order := jsonData.Find("ordered_headers_key")
if !header_order.Exists() {
t.Fatal("not found akamai")
}
i := -1
log.Print(header_order)
// log.Print(headers.Keys())
kks := []string{}
for _, kk := range orderHeaders {
kks = append(kks, textproto.CanonicalMIMEHeaderKey(kk))
}
for _, key := range header_order.Array() {
kk := textproto.CanonicalMIMEHeaderKey(key.String())
if slices.Contains(kks, kk) {
i2 := slices.Index(kks, textproto.CanonicalMIMEHeaderKey(kk))
if i2 < i {
log.Print(header_order)
t.Fatal("not equal")
}
i = i2
}
}
}

1
tools.go
View File

@@ -185,6 +185,7 @@ func NewRequestWithContext(ctx context.Context, method string, u *url.URL, body
req.Proto = "HTTP/1.1"
req.ProtoMajor = 1
req.ProtoMinor = 1
req.ContentLength = -1
if body != nil {
if v, ok := body.(interface{ Len() int }); ok {