mirror of
https://github.com/gospider007/requests.git
synced 2025-12-24 13:57:52 +08:00
修复恐慌
This commit is contained in:
gospider
6
body.go
6
body.go
@@ -100,6 +100,9 @@ func formWrite(writer *multipart.Writer, key string, val any) (err error) {
|
||||
return err
|
||||
}
|
||||
_, err = wp.Write(con)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
case []byte:
|
||||
err = writer.WriteField(key, tools.BytesToString(value))
|
||||
@@ -111,6 +114,9 @@ func formWrite(writer *multipart.Writer, key string, val any) (err error) {
|
||||
return err
|
||||
}
|
||||
err = writer.WriteField(key, tools.BytesToString(con))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
16
client.go
16
client.go
@@ -20,7 +20,7 @@ type Client struct {
|
||||
closed bool
|
||||
}
|
||||
|
||||
var defaultClient, _ = NewClient(nil)
|
||||
var defaultClient, _ = NewClient(context.TODO())
|
||||
|
||||
// New Connection Management
|
||||
func NewClient(preCtx context.Context, options ...ClientOption) (*Client, error) {
|
||||
@@ -79,13 +79,13 @@ func (obj *Client) Close() {
|
||||
obj.cnl()
|
||||
}
|
||||
|
||||
func checkRedirect(req *http.Request, via []*http.Request) error {
|
||||
ctxData := GetReqCtxData(req.Context())
|
||||
if ctxData.maxRedirect == 0 || ctxData.maxRedirect >= len(via) {
|
||||
return nil
|
||||
}
|
||||
return http.ErrUseLastResponse
|
||||
}
|
||||
// func checkRedirect(req *http.Request, via []*http.Request) error {
|
||||
// ctxData := GetReqCtxData(req.Context())
|
||||
// if ctxData.maxRedirect == 0 || ctxData.maxRedirect >= len(via) {
|
||||
// return nil
|
||||
// }
|
||||
// return http.ErrUseLastResponse
|
||||
// }
|
||||
|
||||
func (obj *Client) do(req *http.Request, option *RequestOption) (resp *http.Response, err error) {
|
||||
var redirectNum int
|
||||
|
||||
2
conn.go
2
conn.go
@@ -108,7 +108,7 @@ func (obj *connecotr) http2Req(task *reqTask) {
|
||||
func (obj *connecotr) waitBodyClose() error {
|
||||
select {
|
||||
case <-obj.bodyCtx.Done(): //wait body close
|
||||
if err := context.Cause(obj.bodyCtx); errors.Is(err, gospiderBodyCloseErr) {
|
||||
if err := context.Cause(obj.bodyCtx); errors.Is(err, ErrgospiderBodyClose) {
|
||||
return nil
|
||||
} else {
|
||||
return err
|
||||
|
||||
18
go.mod
18
go.mod
@@ -7,19 +7,19 @@ require (
|
||||
github.com/gospider007/bs4 v0.0.0-20240101025928-00848c76b4c5
|
||||
github.com/gospider007/gson v0.0.0-20231215083934-e7762897c4db
|
||||
github.com/gospider007/gtls v0.0.0-20240104034426-5073ca328e04
|
||||
github.com/gospider007/ja3 v0.0.0-20240104034350-073c71b66be1
|
||||
github.com/gospider007/ja3 v0.0.0-20240114072950-a58a93a771b9
|
||||
github.com/gospider007/net v0.0.0-20240104034506-5448a8a979e3
|
||||
github.com/gospider007/re v0.0.0-20231024115818-adfd03636256
|
||||
github.com/gospider007/tools v0.0.0-20231202084937-8b2bc66f8198
|
||||
github.com/gospider007/websocket v0.0.0-20240104034652-af938aeed446
|
||||
github.com/refraction-networking/utls v1.6.0
|
||||
golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc
|
||||
golang.org/x/net v0.19.0
|
||||
github.com/refraction-networking/utls v1.6.1
|
||||
golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3
|
||||
golang.org/x/net v0.20.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/PuerkitoBio/goquery v1.8.1 // indirect
|
||||
github.com/andybalholm/brotli v1.0.6 // indirect
|
||||
github.com/andybalholm/brotli v1.1.0 // indirect
|
||||
github.com/andybalholm/cascadia v1.3.2 // indirect
|
||||
github.com/caddyserver/certmagic v0.20.0 // indirect
|
||||
github.com/cloudflare/circl v1.3.7 // indirect
|
||||
@@ -41,11 +41,11 @@ require (
|
||||
go.mongodb.org/mongo-driver v1.13.1 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
go.uber.org/zap v1.26.0 // indirect
|
||||
golang.org/x/crypto v0.17.0 // indirect
|
||||
golang.org/x/image v0.14.0 // indirect
|
||||
golang.org/x/crypto v0.18.0 // indirect
|
||||
golang.org/x/image v0.15.0 // indirect
|
||||
golang.org/x/mod v0.14.0 // indirect
|
||||
golang.org/x/sys v0.15.0 // indirect
|
||||
golang.org/x/sys v0.16.0 // indirect
|
||||
golang.org/x/text v0.14.0 // indirect
|
||||
golang.org/x/tools v0.16.1 // indirect
|
||||
golang.org/x/tools v0.17.0 // indirect
|
||||
nhooyr.io/websocket v1.8.10 // indirect
|
||||
)
|
||||
|
||||
40
go.sum
40
go.sum
@@ -1,7 +1,7 @@
|
||||
github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM=
|
||||
github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ=
|
||||
github.com/andybalholm/brotli v1.0.6 h1:Yf9fFpf49Zrxb9NlQaluyE92/+X7UVHlhMNJN2sxfOI=
|
||||
github.com/andybalholm/brotli v1.0.6/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
|
||||
github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
|
||||
github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
|
||||
github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
|
||||
github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss=
|
||||
github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
|
||||
@@ -33,8 +33,8 @@ github.com/gospider007/gson v0.0.0-20231215083934-e7762897c4db h1:EXDXNnZ1jizcq3
|
||||
github.com/gospider007/gson v0.0.0-20231215083934-e7762897c4db/go.mod h1:tL+3YduR/6ofo4cdv4lpBw5+nph0OJ11x1uOKLyo9Ro=
|
||||
github.com/gospider007/gtls v0.0.0-20240104034426-5073ca328e04 h1:iAmk4YVKoOllDO9LryTAMWUgjEAALG1KYWLiRim+LfI=
|
||||
github.com/gospider007/gtls v0.0.0-20240104034426-5073ca328e04/go.mod h1:3yfOTfRE03+Itp1M+paIy+AJVVpoTFyRyqJboTNLrSg=
|
||||
github.com/gospider007/ja3 v0.0.0-20240104034350-073c71b66be1 h1:NU1aqbSjMkUt6h8Zi0djuOwsMBsc/9uwiCF3mNE3uMw=
|
||||
github.com/gospider007/ja3 v0.0.0-20240104034350-073c71b66be1/go.mod h1:HOb9iaH7Dp3Yfv0Pxo9v2xL6juIyXWSblsKPHIAp44s=
|
||||
github.com/gospider007/ja3 v0.0.0-20240114072950-a58a93a771b9 h1:7CQuIQU3or1WILd3Ds4KnJBr06q2qxUXVQzoZMYoK9Y=
|
||||
github.com/gospider007/ja3 v0.0.0-20240114072950-a58a93a771b9/go.mod h1:L20PT+hDmt5GF6Bfz5SxUKKxPked2qFRHYzOuk65xi4=
|
||||
github.com/gospider007/kinds v0.0.0-20231024093643-7a4424f2d30e h1:lmX6IQKkrNDbXfHsvrv1Uz0MoG2v5+4VC6Gdh9irUNY=
|
||||
github.com/gospider007/kinds v0.0.0-20231024093643-7a4424f2d30e/go.mod h1:nB4OMmd8Ji92yEmgjbHcqLcBHTAhSSmlGNb2JpTYK9A=
|
||||
github.com/gospider007/net v0.0.0-20240104034506-5448a8a979e3 h1:eY2J1hf9k6djbyOA381SiQRFN2/m8IYFtzsMzIFvOlg=
|
||||
@@ -73,8 +73,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/quic-go/quic-go v0.40.1 h1:X3AGzUNFs0jVuO3esAGnTfvdgvL4fq655WaOi1snv1Q=
|
||||
github.com/quic-go/quic-go v0.40.1/go.mod h1:PeN7kuVJ4xZbxSv/4OX6S1USOX8MJvydwpTx31vx60c=
|
||||
github.com/refraction-networking/utls v1.6.0 h1:X5vQMqVx7dY7ehxxqkFER/W6DSjy8TMqSItXm8hRDYQ=
|
||||
github.com/refraction-networking/utls v1.6.0/go.mod h1:kHJ6R9DFFA0WsRgBM35iiDku4O7AqPR6y79iuzW7b10=
|
||||
github.com/refraction-networking/utls v1.6.1 h1:n1JG5karzdGWsI6iZmGrOv3SNzR4c+4M8J6KWGsk3lA=
|
||||
github.com/refraction-networking/utls v1.6.1/go.mod h1:+EbcQOvQvXoFV9AEKbuGlljt1doLRKAVY1jJHe9EtDo=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
|
||||
@@ -108,12 +108,12 @@ go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||
golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
|
||||
golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
|
||||
golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc h1:ao2WRsKSzW6KuUY9IWPwWahcHCgR0s52IfwutMfEbdM=
|
||||
golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI=
|
||||
golang.org/x/image v0.14.0 h1:tNgSxAFe3jC4uYqvZdTr84SZoM1KfwdC9SKIFrLjFn4=
|
||||
golang.org/x/image v0.14.0/go.mod h1:HUYqC05R2ZcZ3ejNQsIHQDQiwWM4JBqmm6MKANTp4LE=
|
||||
golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc=
|
||||
golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg=
|
||||
golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3 h1:hNQpMuAJe5CtcUqCXaWga3FHu+kQvCqcsoVaQgSV60o=
|
||||
golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08=
|
||||
golang.org/x/image v0.15.0 h1:kOELfmgrmJlw4Cdb7g/QGuB3CvDrXbqEIww/pNtNBm8=
|
||||
golang.org/x/image v0.15.0/go.mod h1:HUYqC05R2ZcZ3ejNQsIHQDQiwWM4JBqmm6MKANTp4LE=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0=
|
||||
@@ -126,13 +126,13 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug
|
||||
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
|
||||
golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c=
|
||||
golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U=
|
||||
golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo=
|
||||
golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE=
|
||||
golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
|
||||
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
@@ -141,8 +141,8 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
|
||||
golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
|
||||
golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||
@@ -160,8 +160,8 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/tools v0.16.1 h1:TLyB3WofjdOEepBHAU20JdNC1Zbg87elYofWYAY5oZA=
|
||||
golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0=
|
||||
golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc=
|
||||
golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
|
||||
18
option.go
18
option.go
@@ -124,7 +124,7 @@ func (obj *RequestOption) initBody(ctx context.Context) (io.Reader, error) {
|
||||
return body, err
|
||||
} else if obj.Form != nil {
|
||||
var orderMap *orderMap
|
||||
body, orderMap, _, err := obj.newBody(obj.Form, mapType)
|
||||
_, orderMap, _, err := obj.newBody(obj.Form, mapType)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -132,6 +132,9 @@ func (obj *RequestOption) initBody(ctx context.Context) (io.Reader, error) {
|
||||
return nil, nil
|
||||
}
|
||||
body, contentType, once, err := orderMap.parseForm(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
obj.once = once
|
||||
if obj.ContentType == "" {
|
||||
obj.ContentType = contentType
|
||||
@@ -192,8 +195,9 @@ func (obj *RequestOption) initBody(ctx context.Context) (io.Reader, error) {
|
||||
}
|
||||
}
|
||||
func (obj *RequestOption) initParams() (*url.URL, error) {
|
||||
baseUrl := cloneUrl(obj.Url)
|
||||
if obj.Params == nil {
|
||||
return obj.Url, nil
|
||||
return baseUrl, nil
|
||||
}
|
||||
_, dataMap, _, err := obj.newBody(obj.Params, mapType)
|
||||
if err != nil {
|
||||
@@ -201,15 +205,15 @@ func (obj *RequestOption) initParams() (*url.URL, error) {
|
||||
}
|
||||
query := dataMap.parseParams().String()
|
||||
if query == "" {
|
||||
return obj.Url, nil
|
||||
return baseUrl, nil
|
||||
}
|
||||
pquery := obj.Url.Query().Encode()
|
||||
pquery := baseUrl.Query().Encode()
|
||||
if pquery == "" {
|
||||
obj.Url.RawQuery = query
|
||||
baseUrl.RawQuery = query
|
||||
} else {
|
||||
obj.Url.RawQuery = pquery + "&" + query
|
||||
baseUrl.RawQuery = pquery + "&" + query
|
||||
}
|
||||
return obj.Url, nil
|
||||
return baseUrl, nil
|
||||
}
|
||||
func (obj *Client) newRequestOption(option RequestOption) RequestOption {
|
||||
// start
|
||||
|
||||
@@ -26,7 +26,7 @@ type contextKey string
|
||||
|
||||
const gospiderContextKey contextKey = "GospiderContextKey"
|
||||
|
||||
var errFatal = errors.New("Fatal error")
|
||||
var errFatal = errors.New("ErrFatal")
|
||||
|
||||
type reqCtxData struct {
|
||||
isWs bool
|
||||
@@ -326,7 +326,7 @@ func (obj *Client) request(ctx context.Context, option *RequestOption) (response
|
||||
if option.Referer != "" {
|
||||
reqs.Header.Set("Referer", option.Referer)
|
||||
} else if reqs.URL.Scheme != "" && reqs.URL.Host != "" {
|
||||
referBuild := builderPool.Get().(strings.Builder)
|
||||
referBuild := builderPool.Get().(*strings.Builder)
|
||||
referBuild.WriteString(reqs.URL.Scheme)
|
||||
referBuild.WriteString("://")
|
||||
referBuild.WriteString(reqs.URL.Host)
|
||||
|
||||
@@ -33,13 +33,8 @@ func (obj *reqTask) inPool() bool {
|
||||
return obj.err == nil && obj.res != nil && obj.res.StatusCode != 101 && obj.res.Header.Get("Content-Type") != "text/event-stream"
|
||||
}
|
||||
|
||||
type connKey struct {
|
||||
proxy string
|
||||
addr string
|
||||
}
|
||||
|
||||
func getKey(ctxData *reqCtxData, req *http.Request) (key string) {
|
||||
b := builderPool.Get().(strings.Builder)
|
||||
b := builderPool.Get().(*strings.Builder)
|
||||
b.WriteString(getAddr(ctxData.proxy))
|
||||
b.WriteString("@")
|
||||
b.WriteString(getAddr(req.URL))
|
||||
@@ -59,16 +54,6 @@ type roundTripper struct {
|
||||
getProxy func(ctx context.Context, url *url.URL) (string, error)
|
||||
}
|
||||
|
||||
type roundTripperOption struct {
|
||||
DialTimeout time.Duration
|
||||
KeepAlive time.Duration
|
||||
LocalAddr *net.TCPAddr //network card ip
|
||||
AddrType gtls.AddrType //first ip type
|
||||
GetAddrType func(host string) gtls.AddrType
|
||||
Dns *net.UDPAddr
|
||||
GetProxy func(ctx context.Context, url *url.URL) (string, error)
|
||||
}
|
||||
|
||||
func newRoundTripper(preCtx context.Context, option ClientOption) *roundTripper {
|
||||
if preCtx == nil {
|
||||
preCtx = context.TODO()
|
||||
@@ -213,8 +198,8 @@ func (obj *roundTripper) poolRoundTrip(ctxData *reqCtxData, task *reqTask, key s
|
||||
case <-task.emptyPool:
|
||||
return true
|
||||
case <-task.ctx.Done():
|
||||
if task.err==nil && task.res==nil {
|
||||
task.err=task.ctx.Err()
|
||||
if task.err == nil && task.res == nil {
|
||||
task.err = task.ctx.Err()
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
4
rw.go
4
rw.go
@@ -23,14 +23,14 @@ func (obj *readWriteCloser) Proxy() string {
|
||||
return obj.conn.proxy
|
||||
}
|
||||
|
||||
var gospiderBodyCloseErr = errors.New("gospider body close error")
|
||||
var ErrgospiderBodyClose = errors.New("gospider body close error")
|
||||
|
||||
func (obj *readWriteCloser) Close() (err error) {
|
||||
if !obj.InPool() {
|
||||
obj.ForceCloseConn()
|
||||
} else {
|
||||
err = obj.body.Close() //reuse conn
|
||||
obj.conn.bodyCnl(gospiderBodyCloseErr)
|
||||
obj.conn.bodyCnl(ErrgospiderBodyClose)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
@@ -10,22 +10,25 @@ import (
|
||||
|
||||
func TestH2(t *testing.T) {
|
||||
j := "1:65536,2:0,4:6291456,6:262144|15663105|0|m,a,s,p"
|
||||
j2 := "1:65536;2:0;4:6291456;6:262144|15663105|1:true:0:255|a,m,p,s"
|
||||
h2Spec, err := ja3.CreateH2SpecWithStr(j) //create h2 spec with string
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// log.Print(h2Spec)
|
||||
resp, err := requests.Get(nil, "https://tools.scrapfly.io/api/fp/anything", requests.RequestOption{
|
||||
H2Ja3Spec: h2Spec, //set h2 spec
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
jsonData, err := resp.Json()
|
||||
ja3 := jsonData.Get("akamai.fingerprint")
|
||||
if ja3 == nil {
|
||||
t.Fatal("not found akamai")
|
||||
ja3 := jsonData.Get("http2.fingerprint")
|
||||
if !ja3.Exists() {
|
||||
t.Fatal("not found http2")
|
||||
}
|
||||
if j != ja3.String() {
|
||||
if j2 != ja3.String() {
|
||||
log.Print(j)
|
||||
log.Print(ja3)
|
||||
t.Fatal("not equal")
|
||||
|
||||
@@ -21,7 +21,7 @@ func TestJa3(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
jsonData, err := resp.Json() //parse json
|
||||
ja3 := jsonData.Get("ja3.ja3") //get ja3 value
|
||||
ja3 := jsonData.Get("tls.ja3") //get ja3 value
|
||||
if ja3 == nil {
|
||||
t.Fatal("not found ja3")
|
||||
}
|
||||
@@ -32,7 +32,8 @@ func TestJa3(t *testing.T) {
|
||||
}
|
||||
}
|
||||
func TestJa3Psk(t *testing.T) {
|
||||
j := "772,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,5-27-13-35-16-18-43-17513-65281-51-45-11-0-10-23-41,12092-29-23-24,0"
|
||||
// j := "772,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,5-27-13-35-16-18-43-17513-65281-51-45-11-0-10-23-41,12092-29-23-24,0"
|
||||
j := "772,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,5-27-13-35-16-18-43-17513-65281-51-45-11-0-10-23,12092-29-23-24,0"
|
||||
j2 := "772,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,5-27-13-35-16-18-43-17513-65281-51-45-11-0-10-23,12092-29-23-24,0"
|
||||
ja3Spec, err := ja3.CreateSpecWithStr(j) //create ja3 spec with string
|
||||
if err != nil {
|
||||
@@ -47,7 +48,7 @@ func TestJa3Psk(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
jsonData, err := resp.Json()
|
||||
ja3 := jsonData.Get("ja3.ja3")
|
||||
ja3 := jsonData.Get("tls.ja3")
|
||||
if ja3 == nil {
|
||||
t.Fatal("not found ja3")
|
||||
}
|
||||
@@ -78,7 +79,7 @@ func TestJa3ClientHello(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
jsonData, err := resp.Json() //parse json
|
||||
ja3 := jsonData.Get("ja3.ja3") //get ja3 value
|
||||
ja3 := jsonData.Get("tls.ja3") //get ja3 value
|
||||
if ja3 == nil {
|
||||
t.Fatal("not found ja3")
|
||||
}
|
||||
|
||||
@@ -19,14 +19,14 @@ func TestOrderHeaders(t *testing.T) {
|
||||
headers.Set("Sec-Ch-Ua-Mobile", "?0")
|
||||
headers.Set("Sec-Ch-Ua-Platform", `"Windows"`)
|
||||
resp, err := requests.Get(nil, "https://tools.scrapfly.io/api/fp/anything", requests.RequestOption{
|
||||
Headers: headers,
|
||||
ForceHttp1: true,
|
||||
Headers: headers,
|
||||
// ForceHttp1: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
jsonData, err := resp.Json()
|
||||
header_order := jsonData.Get("http.header_order")
|
||||
header_order := jsonData.Get("headers.headers")
|
||||
if header_order == nil {
|
||||
t.Fatal("not found akamai")
|
||||
}
|
||||
|
||||
6
tools.go
6
tools.go
@@ -42,7 +42,7 @@ func getAddr(uurl *url.URL) (addr string) {
|
||||
}
|
||||
_, port, _ := net.SplitHostPort(uurl.Host)
|
||||
if port == "" {
|
||||
bs := builderPool.Get().(strings.Builder)
|
||||
bs := builderPool.Get().(*strings.Builder)
|
||||
bs.WriteString(uurl.Host)
|
||||
bs.WriteString(":")
|
||||
if uurl.Scheme == "https" {
|
||||
@@ -125,7 +125,7 @@ func httpWrite(r *http.Request, w *bufio.Writer, orderHeaders []string) (err err
|
||||
if r.Header.Get("Content-Length") == "" && shouldSendContentLength(r) {
|
||||
r.Header.Set("Content-Length", fmt.Sprint(r.ContentLength))
|
||||
}
|
||||
bs := builderPool.Get().(strings.Builder)
|
||||
bs := builderPool.Get().(*strings.Builder)
|
||||
defer func() {
|
||||
bs.Reset()
|
||||
builderPool.Put(bs)
|
||||
@@ -192,7 +192,7 @@ func init() {
|
||||
return bytes.NewBuffer(nil)
|
||||
}
|
||||
builderPool.New = func() interface{} {
|
||||
return strings.Builder{}
|
||||
return &strings.Builder{}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user