package/requests
1
0
Fork 0
mirror of https://github.com/gospider007/requests.git synced 2025-12-24 13:57:52 +08:00
Code Issues Packages Projects Releases Wiki Activity

add ja3 with client hello

Browse Source
This commit is contained in:
bxd
2023-12-15 16:26:41 +08:00
parent 6830a982e0
commit 355c5cea52
12 changed files with 140 additions and 117 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
body.go
View File

@@ -23,29 +23,32 @@ const (
)
type orderMap struct {
data map[string]any
order []string
data map[string]any
keys []string
}
func NewOrderMap() *orderMap {
return &orderMap{
data: make(map[string]any),
order: []string{},
data: make(map[string]any),
keys: []string{},
}
}
func (obj *orderMap) Set(key string, val any) {
obj.Del(key)
obj.data[key] = val
obj.order = append(obj.order, key)
obj.keys = append(obj.keys, key)
}
func (obj *orderMap) Del(key string) {
delete(obj.data, key)
obj.order = tools.DelSliceVals(obj.order, key)
obj.keys = tools.DelSliceVals(obj.keys, key)
}
func (obj *orderMap) Keys() []string {
return obj.keys
}
func (obj *orderMap) parseHeaders() (map[string][]string, []string) {
head := make(http.Header)
for _, kk := range obj.order {
for _, kk := range obj.keys {
if vvs, ok := obj.data[kk].([]any); ok {
for _, vv := range vvs {
head.Add(kk, fmt.Sprint(vv))
@@ -54,7 +57,7 @@ func (obj *orderMap) parseHeaders() (map[string][]string, []string) {
head.Add(kk, fmt.Sprint(obj.data[kk]))
}
}
return head, obj.order
return head, obj.keys
}
func formWrite(writer *multipart.Writer, key string, val any) (err error) {
@@ -112,7 +115,7 @@ func formWrite(writer *multipart.Writer, key string, val any) (err error) {
return
}
func (obj *orderMap) parseForm(ctx context.Context) (io.Reader, string, bool, error) {
if len(obj.order) == 0 || len(obj.data) == 0 {
if len(obj.keys) == 0 || len(obj.data) == 0 {
return nil, "", false, nil
}
if obj.isformPip() {
@@ -137,10 +140,10 @@ func (obj *orderMap) parseForm(ctx context.Context) (io.Reader, string, bool, er
return body, writer.FormDataContentType(), false, err
}
func (obj *orderMap) isformPip() bool {
if len(obj.order) == 0 || len(obj.data) == 0 {
if len(obj.keys) == 0 || len(obj.data) == 0 {
return false
}
for _, key := range obj.order {
for _, key := range obj.keys {
if vals, ok := obj.data[key].([]any); ok {
for _, val := range vals {
if file, ok := val.(File); ok {
@@ -160,7 +163,7 @@ func (obj *orderMap) isformPip() bool {
return false
}
func (obj *orderMap) formWriteMain(writer *multipart.Writer) (err error) {
for _, key := range obj.order {
for _, key := range obj.keys {
if vals, ok := obj.data[key].([]any); ok {
for _, val := range vals {
if err = formWrite(writer, key, val); err != nil {
@@ -186,7 +189,7 @@ func paramsWrite(buf *bytes.Buffer, key string, val any) {
}
func (obj *orderMap) parseParams() *bytes.Buffer {
buf := bytes.NewBuffer(nil)
for _, k := range obj.order {
for _, k := range obj.keys {
if vals, ok := obj.data[k].([]any); ok {
for _, v := range vals {
paramsWrite(buf, k, v)
@@ -210,7 +213,7 @@ func (obj *orderMap) MarshalJSON() ([]byte, error) {
if err != nil {
return nil, err
}
for i, k := range obj.order {
for i, k := range obj.keys {
if i > 0 {
if err = buf.WriteByte(','); err != nil {
return nil, err
@@ -310,7 +313,7 @@ mapL:
case map[string]any:
orderMap := NewOrderMap()
orderMap.data = value
orderMap.order = maps.Keys(value)
orderMap.keys = maps.Keys(value)
return nil, orderMap, nil, nil
}
if val, err = gson.Decode(val); err != nil {

14
go.mod
View File

@@ -7,13 +7,13 @@ require (
github.com/gospider007/bs4 v0.0.0-20231123090151-001db0b91941
github.com/gospider007/gson v0.0.0-20231119141525-66095080057d
github.com/gospider007/gtls v0.0.0-20231120122450-e763299259db
github.com/gospider007/ja3 v0.0.0-20231208073248-8fd919857f6f
github.com/gospider007/net v0.0.0-20231028084010-313c148cf0a1
github.com/gospider007/ja3 v0.0.0-20231215081451-4f0f2494ca9f
github.com/gospider007/net v0.0.0-20231215081728-064725ce355a
github.com/gospider007/re v0.0.0-20231024115818-adfd03636256
github.com/gospider007/tools v0.0.0-20231202084937-8b2bc66f8198
github.com/gospider007/websocket v0.0.0-20231128065110-6296f87425c4
github.com/refraction-networking/utls v1.5.4
golang.org/x/exp v0.0.0-20231206192017-f3f8817b8deb
golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611
golang.org/x/net v0.19.0
)
@@ -24,6 +24,8 @@ require (
github.com/caddyserver/certmagic v0.20.0 // indirect
github.com/cloudflare/circl v1.3.6 // indirect
github.com/gaukas/godicttls v0.0.4 // indirect
github.com/google/go-cmp v0.6.0 // indirect
github.com/google/pprof v0.0.0-20231205033806-a5a03c77bf08 // indirect
github.com/gospider007/blog v0.0.0-20231121084103-59a004dafccf // indirect
github.com/gospider007/kinds v0.0.0-20231024093643-7a4424f2d30e // indirect
github.com/json-iterator/go v1.1.12 // indirect
@@ -34,7 +36,9 @@ require (
github.com/miekg/dns v1.1.57 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/quic-go/quic-go v0.40.0 // indirect
github.com/onsi/ginkgo/v2 v2.13.2 // indirect
github.com/onsi/gomega v1.30.0 // indirect
github.com/quic-go/quic-go v0.40.1 // indirect
github.com/tidwall/gjson v1.17.0 // indirect
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.1 // indirect
@@ -47,6 +51,6 @@ require (
golang.org/x/mod v0.14.0 // indirect
golang.org/x/sys v0.15.0 // indirect
golang.org/x/text v0.14.0 // indirect
golang.org/x/tools v0.16.0 // indirect
golang.org/x/tools v0.16.1 // indirect
nhooyr.io/websocket v1.8.10 // indirect
)

47
go.sum
View File

@@ -5,8 +5,6 @@ github.com/andybalholm/brotli v1.0.6/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHG
github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss=
github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
github.com/caddyserver/certmagic v0.19.2 h1:HZd1AKLx4592MalEGQS39DKs2ZOAJCEM/xYPMQ2/ui0=
github.com/caddyserver/certmagic v0.19.2/go.mod h1:fsL01NomQ6N+kE2j37ZCnig2MFosG+MIO4ztnmG/zz8=
github.com/caddyserver/certmagic v0.20.0 h1:bTw7LcEZAh9ucYCRXyCpIrSAGplplI0vGYJ4BpCQ/Fc=
github.com/caddyserver/certmagic v0.20.0/go.mod h1:N4sXgpICQUskEWpj7zVzvWD41p3NYacrNoZYiRM2jTg=
github.com/cloudflare/circl v1.3.6 h1:/xbKIqSHbZXHwkhbrhrt2YOHIwYJlXH94E3tI/gDlUg=
@@ -16,19 +14,16 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/gaukas/godicttls v0.0.4 h1:NlRaXb3J6hAnTmWdsEKb9bcSBD6BvcIjdGdeb0zfXbk=
github.com/gaukas/godicttls v0.0.4/go.mod h1:l6EenT4TLWgTdwslVb4sEMOCf7Bv0JAK67deKr9/NCI=
github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY=
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98 h1:pUa4ghanp6q4IJHwE9RwLgmVFfReJN+KbQ8ExNEUUoQ=
github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik=
github.com/gospider007/bar v0.0.0-20231121084140-33c7b6797626 h1:zDK4PcXQmAX37JdGUp45gFDMolnBGzWVXgemm5ekG1Y=
github.com/gospider007/bar v0.0.0-20231121084140-33c7b6797626/go.mod h1:aYPgmG9340i9x9VQZhf34/XtIj7PHDTq0wSO+7zU/8s=
github.com/google/pprof v0.0.0-20231205033806-a5a03c77bf08 h1:PxlBVtIFHR/mtWk2i0gTEdCz+jBnqiuHNSki0epDbVs=
github.com/google/pprof v0.0.0-20231205033806-a5a03c77bf08/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik=
github.com/gospider007/bar v0.0.0-20231201075546-252b6e7b6a54 h1:3rtF5ZK6b9L8pIsn9AUtSbRBiSpcWW/3Os9XYNyDvKI=
github.com/gospider007/bar v0.0.0-20231201075546-252b6e7b6a54/go.mod h1:aYPgmG9340i9x9VQZhf34/XtIj7PHDTq0wSO+7zU/8s=
github.com/gospider007/blog v0.0.0-20231121084103-59a004dafccf h1:1laTsuH/wl5pZ5QlHzacX09QzvwQw0DFENoRMpGBK8Y=
@@ -39,22 +34,18 @@ github.com/gospider007/gson v0.0.0-20231119141525-66095080057d h1:K9IbMHY2dZfi1d
github.com/gospider007/gson v0.0.0-20231119141525-66095080057d/go.mod h1:r8uL+sPUwP8+6n8GZFa2iz2bJe0qNS6EKwkqmiZhHqI=
github.com/gospider007/gtls v0.0.0-20231120122450-e763299259db h1:8ghU2J0g7BKC1t61TCA3kCkcUsgCn36YwEtTWXsqe+o=
github.com/gospider007/gtls v0.0.0-20231120122450-e763299259db/go.mod h1:yE9d9KMYxJEQawkOOHBea03dN89uo3hWQxYiv9tnz0A=
github.com/gospider007/ja3 v0.0.0-20231029025157-38fc2f8f2d91 h1:qQokihfTAX+/U8GIMvZauRtE4G+/1Jq8XIJx8xLr04A=
github.com/gospider007/ja3 v0.0.0-20231029025157-38fc2f8f2d91/go.mod h1:ur78/uhYDDULSy1ldA/pPpGhjk973Q1VsPnbktXGU/g=
github.com/gospider007/ja3 v0.0.0-20231202085054-c1b92675187e h1:6Brr8+E6fht8IVFBJBkUoFMr8HQ9yzsrVxt8m/wVtQ8=
github.com/gospider007/ja3 v0.0.0-20231202085054-c1b92675187e/go.mod h1:kWf9x0hQS+pgOpu1lRiVdE5nozLj71j740cWvjzKqok=
github.com/gospider007/ja3 v0.0.0-20231208073248-8fd919857f6f h1:nCKHMwa4riH9DUgDq3BuGeBc/DC/VZJ85eWqUIYc58A=
github.com/gospider007/ja3 v0.0.0-20231208073248-8fd919857f6f/go.mod h1:DfT58VcUEQtuAfRaoU8qE6BbnGAO+kRnFJbrZRmjP/E=
github.com/gospider007/ja3 v0.0.0-20231215081451-4f0f2494ca9f h1:6ti0B70R4S6eTyd06JyT4Yf1PpJsGxBeRcpiBvEu5uA=
github.com/gospider007/ja3 v0.0.0-20231215081451-4f0f2494ca9f/go.mod h1:xGzVELcwt5tiprBlyk/X4XhnxD0aCYzSfdj06s3VjfI=
github.com/gospider007/kinds v0.0.0-20231024093643-7a4424f2d30e h1:lmX6IQKkrNDbXfHsvrv1Uz0MoG2v5+4VC6Gdh9irUNY=
github.com/gospider007/kinds v0.0.0-20231024093643-7a4424f2d30e/go.mod h1:nB4OMmd8Ji92yEmgjbHcqLcBHTAhSSmlGNb2JpTYK9A=
github.com/gospider007/net v0.0.0-20231028084010-313c148cf0a1 h1:tYOQEvELrV+USjKGsAroC1cvsLMHgGlUPQY1TKS/PDM=
github.com/gospider007/net v0.0.0-20231028084010-313c148cf0a1/go.mod h1:3ggAwYdh0NB0OvtiX0l5AfHdBjgsIt9MGsXCQ3iCzQc=
github.com/gospider007/net v0.0.0-20231215081728-064725ce355a h1:ZQFHnC3rVqKCVud2nhWLbwrz81PGQAiqnYH8h0a0QMY=
github.com/gospider007/net v0.0.0-20231215081728-064725ce355a/go.mod h1:SpSZEVVotfx4jMj0gv2NsmW1UhAg7RvhC01jbmYrEWg=
github.com/gospider007/re v0.0.0-20231024115818-adfd03636256 h1:Z6kHRANoWB+/4rDzq51vBts0rIXilDrF8pdRNmbMJi4=
github.com/gospider007/re v0.0.0-20231024115818-adfd03636256/go.mod h1:X58uk0/F3mVskuQOZng0ZKJiAt3ETn0wxuLN//rVZrE=
github.com/gospider007/tools v0.0.0-20231128142841-23217c299fc2 h1:io2bXntt5LSwWIBPztZUQrGnXlqrphVi5zdGBgHuyb8=
github.com/gospider007/tools v0.0.0-20231128142841-23217c299fc2/go.mod h1:wiILK6EotceHz/Rnb6ux8PzY3sr5OV+mYuIcbtxpkYI=
github.com/gospider007/tools v0.0.0-20231201075443-f0a4bc8cd616 h1:Ix8hbbbaIX9REGs0qqU1b48L3BlleDyNkCPdv297LF8=
github.com/gospider007/tools v0.0.0-20231201075443-f0a4bc8cd616/go.mod h1:wiILK6EotceHz/Rnb6ux8PzY3sr5OV+mYuIcbtxpkYI=
github.com/gospider007/tools v0.0.0-20231202084937-8b2bc66f8198 h1:phk1GNobIIQWL5/G5dtgs35hotucdYv2FScMjlHHZ+Q=
github.com/gospider007/tools v0.0.0-20231202084937-8b2bc66f8198/go.mod h1:wiILK6EotceHz/Rnb6ux8PzY3sr5OV+mYuIcbtxpkYI=
github.com/gospider007/websocket v0.0.0-20231128065110-6296f87425c4 h1:h+74nkhhTDN2tiaDjHwR4CjqBTHgh+t1pqE2IAWHN3k=
@@ -62,8 +53,6 @@ github.com/gospider007/websocket v0.0.0-20231128065110-6296f87425c4/go.mod h1:On
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
github.com/klauspost/compress v1.17.3 h1:qkRjuerhUU1EmXLYGkSH6EZL+vPSxIrYjLNAK4slzwA=
github.com/klauspost/compress v1.17.3/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4=
github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
@@ -81,14 +70,16 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
github.com/onsi/ginkgo/v2 v2.12.1 h1:uHNEO1RP2SpuZApSkel9nEh1/Mu+hmQe7Q+Pepg5OYA=
github.com/onsi/ginkgo/v2 v2.12.1/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o=
github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI=
github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M=
github.com/onsi/ginkgo/v2 v2.13.2 h1:Bi2gGVkfn6gQcjNjZJVO8Gf0FHzMPf2phUei9tejVMs=
github.com/onsi/ginkgo/v2 v2.13.2/go.mod h1:XStQ8QcGwLyF4HdfcZB8SFOS/MWCgDuXMSBe6zrvLgM=
github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8=
github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/quic-go/quic-go v0.40.0 h1:GYd1iznlKm7dpHD7pOVpUvItgMPo/jrMgDWZhMCecqw=
github.com/quic-go/quic-go v0.40.0/go.mod h1:PeN7kuVJ4xZbxSv/4OX6S1USOX8MJvydwpTx31vx60c=
github.com/quic-go/quic-go v0.40.1 h1:X3AGzUNFs0jVuO3esAGnTfvdgvL4fq655WaOi1snv1Q=
github.com/quic-go/quic-go v0.40.1/go.mod h1:PeN7kuVJ4xZbxSv/4OX6S1USOX8MJvydwpTx31vx60c=
github.com/refraction-networking/utls v1.5.4 h1:9k6EO2b8TaOGsQ7Pl7p9w6PUhx18/ZCeT0WNTZ7Uw4o=
github.com/refraction-networking/utls v1.5.4/go.mod h1:SPuDbBmgLGp8s+HLNc83FuavwZCFoMmExj+ltUHiHUw=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -113,8 +104,6 @@ github.com/zeebo/blake3 v0.2.3 h1:TFoLXsjeXqRNFxSbk35Dk4YtszE/MQQGK10BH4ptoTg=
github.com/zeebo/blake3 v0.2.3/go.mod h1:mjJjZpnsyIVtVgTOSpJ9vmRE4wgDeyt2HU3qXvvKCaQ=
github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo=
github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4=
go.mongodb.org/mongo-driver v1.13.0 h1:67DgFFjYOCMWdtTEmKFpV3ffWlFnh+CYZ8ZS/tXWUfY=
go.mongodb.org/mongo-driver v1.13.0/go.mod h1:/rGBTebI3XYboVmgz+Wv3Bcbl3aD0QF9zl6kDDw18rQ=
go.mongodb.org/mongo-driver v1.13.1 h1:YIc7HTYsKndGK4RFzJ3covLz1byri52x0IoMB0Pt/vk=
go.mongodb.org/mongo-driver v1.13.1/go.mod h1:wcDf1JBCXy2mOW0bWHwO/IOYqdca1MPCwDtFu/Z9+eo=
go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk=
@@ -128,10 +117,10 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY=
golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
golang.org/x/exp v0.0.0-20231127185646-65229373498e h1:Gvh4YaCaXNs6dKTlfgismwWZKyjVZXwOPfIyUaqU3No=
golang.org/x/exp v0.0.0-20231127185646-65229373498e/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI=
golang.org/x/exp v0.0.0-20231206192017-f3f8817b8deb h1:c0vyKkb6yr3KR7jEfJaOSv4lG7xPkbN6r52aJz1d8a8=
golang.org/x/exp v0.0.0-20231206192017-f3f8817b8deb/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI=
golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611 h1:qCEDpW1G+vcj3Y7Fy52pEM1AWm3abj8WimGYejI3SC4=
golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI=
golang.org/x/image v0.14.0 h1:tNgSxAFe3jC4uYqvZdTr84SZoM1KfwdC9SKIFrLjFn4=
golang.org/x/image v0.14.0/go.mod h1:HUYqC05R2ZcZ3ejNQsIHQDQiwWM4JBqmm6MKANTp4LE=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
@@ -182,6 +171,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.16.0 h1:GO788SKMRunPIBCXiQyo2AaexLstOrVhuAL5YwsckQM=
golang.org/x/tools v0.16.0/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0=
golang.org/x/tools v0.16.1 h1:TLyB3WofjdOEepBHAU20JdNC1Zbg87elYofWYAY5oZA=
golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

13
headers.go
View File

@@ -24,23 +24,22 @@ func defaultHeaders() http.Header {
}
}
func (obj *RequestOption) initHeaders() (http.Header, error) {
func (obj *RequestOption) initHeaders() (http.Header, []string, error) {
if obj.Headers == nil {
return nil, nil
return nil, nil, nil
}
switch headers := obj.Headers.(type) {
case http.Header:
return headers.Clone(), nil
return headers.Clone(), nil, nil
default:
_, dataMap, _, err := obj.newBody(headers, mapType)
if err != nil {
return nil, err
return nil, nil, err
}
if dataMap == nil {
return nil, nil
return nil, nil, nil
}
head, order := dataMap.parseHeaders()
obj.OrderHeaders = order
return head, err
return head, order, err
}
}

81
option.go
View File

@@ -16,27 +16,26 @@ import (
// Connection Management Options
type ClientOption struct {
OrderHeaders []string //order headers with http1
Ja3Spec ja3.Ja3Spec //custom ja3Spec,use ja3.CreateSpecWithStr or ja3.CreateSpecWithId create
H2Ja3Spec ja3.H2Ja3Spec //h2 fingerprint
Proxy string //proxy,support https,http,socks5
ForceHttp1 bool //force use http1 send requests
Ja3 bool //enable ja3 fingerprint
DisCookie bool //disable cookies
DisDecode bool //disable auto decode
DisUnZip bool //disable auto zip decode
DisAlive bool //disable keepalive
Bar bool ////enable bar display
OptionCallBack func(ctx context.Context, client *Client, option *RequestOption) error //option callback,if error is returnd, break request
ResultCallBack func(ctx context.Context, client *Client, response *Response) error //result callback,if error is returnd,next errCallback
ErrCallBack func(ctx context.Context, client *Client, response *Response, err error) error //error callback,if error is returnd,break request
RequestCallBack func(ctx context.Context, request *http.Request, response *http.Response) error //request and response callback,if error is returnd,reponse is error
MaxRetries int //try num
MaxRedirect int //redirect num ,<0 no redirect,==0 no limit
Headers any //default headers
Timeout time.Duration //request timeout
ResponseHeaderTimeout time.Duration //ResponseHeaderTimeout ,default:30
TlsHandshakeTimeout time.Duration //tls timeout,default:15
Ja3Spec ja3.Ja3Spec //custom ja3Spec,use ja3.CreateSpecWithStr or ja3.CreateSpecWithId create
H2Ja3Spec ja3.H2Ja3Spec //h2 fingerprint
Proxy string //proxy,support https,http,socks5
ForceHttp1 bool //force use http1 send requests
Ja3 bool //enable ja3 fingerprint
DisCookie bool //disable cookies
DisDecode bool //disable auto decode
DisUnZip bool //disable auto zip decode
DisAlive bool //disable keepalive
Bar bool ////enable bar display
OptionCallBack func(ctx context.Context, option *RequestOption) error //option callback,if error is returnd, break request
ResultCallBack func(ctx context.Context, option *RequestOption, response *Response) error //result callback,if error is returnd,next errCallback
ErrCallBack func(ctx context.Context, option *RequestOption, response *Response, err error) error //error callback,if error is returnd,break request
RequestCallBack func(ctx context.Context, request *http.Request, response *http.Response) error //request and response callback,if error is returnd,reponse is error
MaxRetries int //try num
MaxRedirect int //redirect num ,<0 no redirect,==0 no limit
Headers any //default headers
Timeout time.Duration //request timeout
ResponseHeaderTimeout time.Duration //ResponseHeaderTimeout ,default:30
TlsHandshakeTimeout time.Duration //tls timeout,default:15
//network card ip
DialTimeout time.Duration //dial tcp timeout,default:15
@@ -53,21 +52,20 @@ type ClientOption struct {
// Options for sending requests
type RequestOption struct {
OrderHeaders []string //order headers with http1
Ja3Spec ja3.Ja3Spec //custom ja3Spec,use ja3.CreateSpecWithStr or ja3.CreateSpecWithId create
H2Ja3Spec ja3.H2Ja3Spec //custom h2 fingerprint
Proxy string //proxy,support http,https,socks5,example：http://127.0.0.1:7005
ForceHttp1 bool //force use http1 send requests
Ja3 bool //enable ja3 fingerprint
DisCookie bool //disable cookies,not use cookies
DisDecode bool //disable auto decode
DisUnZip bool //disable auto zip decode
DisAlive bool //disable keepalive
Bar bool //enable bar display
OptionCallBack func(ctx context.Context, client *Client, option *RequestOption) error //option callback,if error is returnd, break request
ResultCallBack func(ctx context.Context, client *Client, response *Response) error //result callback,if error is returnd,next errCallback
ErrCallBack func(ctx context.Context, client *Client, response *Response, err error) error //error callback,if error is returnd,break request
RequestCallBack func(ctx context.Context, request *http.Request, response *http.Response) error //request and response callback,if error is returnd,reponse is error
Ja3Spec ja3.Ja3Spec //custom ja3Spec,use ja3.CreateSpecWithStr or ja3.CreateSpecWithId create
H2Ja3Spec ja3.H2Ja3Spec //custom h2 fingerprint
Proxy string //proxy,support http,https,socks5,example：http://127.0.0.1:7005
ForceHttp1 bool //force use http1 send requests
Ja3 bool //enable ja3 fingerprint
DisCookie bool //disable cookies,not use cookies
DisDecode bool //disable auto decode
DisUnZip bool //disable auto zip decode
DisAlive bool //disable keepalive
Bar bool //enable bar display
OptionCallBack func(ctx context.Context, option *RequestOption) error //option callback,if error is returnd, break request
ResultCallBack func(ctx context.Context, option *RequestOption, response *Response) error //result callback,if error is returnd,next errCallback
ErrCallBack func(ctx context.Context, option *RequestOption, response *Response, err error) error //error callback,if error is returnd,break request
RequestCallBack func(ctx context.Context, request *http.Request, response *http.Response) error //request and response callback,if error is returnd,reponse is error
MaxRetries int //try num
MaxRedirect int //redirect num ,<0 no redirect,==0 no limit
@@ -103,6 +101,11 @@ type RequestOption struct {
WsOption websocket.Option //websocket option
DisProxy bool //force disable proxy
once bool
client *Client
}
func (obj *RequestOption) Client() *Client {
return obj.client
}
// Upload files with form-data,
@@ -210,9 +213,6 @@ func (obj *RequestOption) initParams() (*url.URL, error) {
}
func (obj *Client) newRequestOption(option RequestOption) RequestOption {
// start
if option.OrderHeaders == nil {
option.OrderHeaders = obj.option.OrderHeaders
}
if !option.Ja3Spec.IsSet() {
option.Ja3Spec = obj.option.Ja3Spec
}
@@ -301,5 +301,8 @@ func (obj *Client) newRequestOption(option RequestOption) RequestOption {
if option.DisCookie {
option.Jar = nil
}
if option.DisProxy {
option.Proxy = ""
}
return option
}

12
requests.go
View File

@@ -69,8 +69,6 @@ func NewReqCtxData(ctx context.Context, option *RequestOption) (*reqCtxData, err
ctxData.dns = option.Dns
ctxData.disProxy = option.DisProxy
ctxData.tlsHandshakeTimeout = option.TlsHandshakeTimeout
ctxData.orderHeaders = option.OrderHeaders
//init scheme
if option.Url != nil {
if option.Url.Scheme == "ws" {
@@ -242,6 +240,7 @@ func (obj *Client) Request(ctx context.Context, method string, href string, opti
for maxRetries := 0; maxRetries <= optionBak.MaxRetries; maxRetries++ {
option := optionBak
option.Url = cloneUrl(uhref)
option.client = obj
response, err = obj.request(ctx, &option)
if err == nil || errors.Is(err, errFatal) || option.once {
return
@@ -258,13 +257,13 @@ func (obj *Client) request(ctx context.Context, option *RequestOption) (response
}
//result callback
if err == nil && option.ResultCallBack != nil {
err = option.ResultCallBack(ctx, obj, response)
err = option.ResultCallBack(ctx, option, response)
}
if err != nil { //err callback, must close body
response.CloseBody()
if option.ErrCallBack != nil {
if err2 := option.ErrCallBack(ctx, obj, response, err); err2 != nil {
if err2 := option.ErrCallBack(ctx, option, response, err); err2 != nil {
err = tools.WrapError(errFatal, err2)
}
}
@@ -273,7 +272,7 @@ func (obj *Client) request(ctx context.Context, option *RequestOption) (response
}
}()
if option.OptionCallBack != nil {
if err = option.OptionCallBack(ctx, obj, option); err != nil {
if err = option.OptionCallBack(ctx, option); err != nil {
return
}
}
@@ -283,7 +282,7 @@ func (obj *Client) request(ctx context.Context, option *RequestOption) (response
response.stream = option.Stream
//init headers and orderheaders,befor init ctxData
headers, err := option.initHeaders()
headers, orderHeaders, err := option.initHeaders()
if err != nil {
return response, tools.WrapError(err, errors.New("tempRequest init headers error"), err)
}
@@ -295,6 +294,7 @@ func (obj *Client) request(ctx context.Context, option *RequestOption) (response
if err != nil {
return response, tools.WrapError(err, " reqCtxData init error")
}
ctxData.orderHeaders = orderHeaders
//init ctx,cnl
if option.Timeout > 0 { //超时
response.ctx, response.cnl = context.WithTimeout(CreateReqCtx(ctx, ctxData), option.Timeout)

23
test/fingerprint/ja3_test.go
View File

@@ -64,3 +64,26 @@ func TestJa3Psk(t *testing.T) {
}
}
}
func TestJa3ClientHello(t *testing.T) {
hexStream := "16030107da010007d60303f6e7130bee8e1362fb26e166e87da2e00d2ac2212370464b6dbb04a615b39eff20037c899a1ee9bec636722374c44308d7d40d8953994cec5dd44eb06786898f1e00206a6a130113021303c02bc02fc02cc030cca9cca8c013c014009c009d002f00350100076dcaca0000ff0100010000170000000a000c000acaca2f3c001d0017001800050005010000000000230000fe0d00fa00000100016e0020ab82cd7ab430aae210d15b45225c1be0b861d67b15428565458023468a48680c00d0427920f67aa56fc94e41a1b93be0d5d1596977a8113bd5eb9b32c93c7a75a47f93294a8b37e228fc64d4959224351efaacba5ddb7b3a19f8704501d8df49af7d7d03ee3c59f6a52cd8734d5317c8cc107411816aacaede9d5fc000e326ef05a8676fe8bc41fe8137aab39b9006d13ce768e731dc1e783862cecdf7d6ae8b2494f827f7e4cb4b22b5b2b8c98a8cb002620f0cf0a89e0303fe296392c10aa724a90c95cf48e9cfee49b0efbb7ac8ad1ac001c8e84d7c2ba68a1498b07594b8c547447b04d1c8b6c8717d91020f49c543c5001b0003020002000d0012001004030804040105030805050108060601446900050003026832000b00020100002b0007064a4a03040303001200000010000e000c02683208687474702f312e3100330530052ecaca0001002f3c050104e734eef6ce9d4e7dc128b8e01bd5602713f452286397fe21373a0a34a2cd04686c2434fa4bde624d401cf47ab703233cc47728b8efbce586c728103260ccf4c4ed5feb9c18a8318a6542766e318b8e61a109b68a242f205dd1759913332e14b2713b890b5c47e6e27c4c1021686827fa606554773a2b87060972241845a085c634536ca1a59a6e9ca40bcc543348a7745d581d4612545005068829bc37a41ec874177137368ac67c7df353d2674d62a1ab5c92ca5b489d8842844194aac5c853ba0b7b76139333a072e041419f9cc2f182cf1312519cc8192c129da8a5641924a5e2f06f19d6b9a5c9a0d3d65451d82383704579237d48159bb66214c5559738c95eebe595cbf81893c378c3f5a202660c8c0946012b42ed93bf17b7a43f44ae843735de7bbcf6643b0a03a7a3c0c71a068cf1f9b9d3265d016a3b5e94cf7173b698baa7f1a256224c36a71263768c6078c74bf6b8ba65550bb43cad4a04a407588aaef76a799cc4290c0044b893f4b5aa34176e648c515d11330d7b7a599922f8834efba00b4b7573c8c9b0a3367c237875f9f46e4298a26d02286b8c16ab4249847985fac03647bb686d132d9153186d50608dd19888161c899098b174037d6985401356ea92092eb91e6e6c30d3e05e1e5068d55a0b39d692675bcbb59b3c40a8733b59483e403922544bc8d63960ac35e3c7430f26235a159df153747887a259d81ab1811a236b57a2f4227d009fa9a988f9e45acae5128e4a4d5c1b31860454fd3012806568df489cfb389da087bae8e3bc68da62d86b77e7115acf761be3384bb6bc9f12fa62cb4810b2336d11c4b99e7458e95a32918364d846408acb779414663f19b6dcd6a58d65afdd899925141bf4692ee654690a3c6b906c6d4d892356175f8526cf183b6721e034d4a69a52f0148ff82b80babae073908bc60efa51c85d77cf183c2f75b1c675d79465474a5518c41c97511265c1dc67ad41c5320fa8c12b5cca15061a5292453fa18f35e42d7e6a4ce960c8bca6102a592a78c75a7ac41a95830a257441832c12c034b9cb95394078a6abe971262b000b341838eb748ce1cad878325a534da0016a0a924b306b872819235e182281c6a3c9a604e0ac262dfa30a4e4664b2c00f526cecb14125ff21f49c727db781f791573d39876956889ea990002bb74f180c26933b1428a998dd2bb1a8b40fb906b5680c08393cfc934520ad81ac2c388af06ad0a93125226a157d605e1f955c643113d387a7becbba50b5626351036ec329621ce4f9a9e39eab80ff6341151213deb66bd4256be415fc14093eda75f660b78997155fa6079afe3c10fd25a956a6e6c449f4275bf1ee40f3fb930b4130073718d06123b96f3c3f6227e3ff81d86f53b1025546df22f4303907fa07f344a086b1a34e6645f3e05b9b4242606f5c83935624ff5c6e7c49401c7725404bf2792709e0390ca082806189ca88799e71b2f1943761df228320c4c9b245b12b5565574967eb31c0993a059f5c6ccd1820ef2a994958085a866351333c17a676d5b0808d00dc1fa9912833eb1745f5b461f5407692053a912d43315aab1f4b69abd1986a9621f3c3722c633c36115460a47325a33335c68410f80a4e0fb4a79d75458032d285ba7d9f167340952fed75cce99a80a74661db32949072082978e7586598d4c2c82500a205760f821728587c1339105203711a2c55267e734569c915be2937409436dca2b098ac216492b57708d41986962e105957c6ea05ef0aac995722671961507ea6b21e8bcd687db34d24a8786ed001d00207e52cc0bfb373444c54aeb19720f42b2379d7db8910e2451321e440c8da6df360000001d001b000018676f737069646572322e676f737069646572622e61736961002d000201016a6a00010000290094006f0069b4f2a27d7236f2b6e8d4b6db52a2450dd48a8d833b2479a9f8be35157c9ab0b971f349d1eaf577a7046540a4f1ded29d88b9f3aef7ec4225de9e622981351e0e8eadc7ab0318f74704f63d3056f9fa399fb1fc757bb7c73f5f66e652db9b9f613fe30288436979337443cfd0fb002120f06d72218279775978c6be6fb2d36dc9e362768f4bb353b71141cf95e3ae7750"
ja3Spec, err := ja3.CreateSpecWithClientHello(hexStream)
if err != nil {
t.Fatal(err)
}
log.Print(ja3Spec)
resp, err := requests.Get(nil, "https://tools.scrapfly.io/api/fp/anything", requests.RequestOption{
Ja3Spec: ja3Spec,
})
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json() //parse json
ja3 := jsonData.Get("ja3.ja3") //get ja3 value
if ja3 == nil {
t.Fatal("not found ja3")
}
if ja3Spec.String() != ja3.String() {
log.Print(ja3)
t.Fatal("not equal")
}
}

23
test/fingerprint/ja4_test.go
View File

@@ -10,18 +10,17 @@ import (
)
func TestOrderHeaders(t *testing.T) {
orderHeaders := []string{
"Accept-Encoding",
"Accept",
"Sec-Ch-Ua",
"Sec-Ch-Ua-Platform",
"Sec-Ch-Ua-Mobile",
"Accept-Language",
"User-Agent",
}
headers := requests.NewOrderMap()
headers.Set("Accept-Encoding", "gzip, deflate, br")
headers.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7")
headers.Set("User-Agent", requests.UserAgent)
headers.Set("Accept-Language", requests.AcceptLanguage)
headers.Set("Sec-Ch-Ua", requests.SecChUa)
headers.Set("Sec-Ch-Ua-Mobile", "?0")
headers.Set("Sec-Ch-Ua-Platform", `"Windows"`)
resp, err := requests.Get(nil, "https://tools.scrapfly.io/api/fp/anything", requests.RequestOption{
OrderHeaders: orderHeaders, //set http1.1 order headers
ForceHttp1: true,
Headers: headers,
ForceHttp1: true,
})
if err != nil {
t.Fatal(err)
@@ -33,7 +32,7 @@ func TestOrderHeaders(t *testing.T) {
}
i := -1
for _, key := range header_order.Array() {
i2 := slices.Index(orderHeaders, textproto.CanonicalMIMEHeaderKey(key.String()))
i2 := slices.Index(headers.Keys(), textproto.CanonicalMIMEHeaderKey(key.String()))
if i2 < i {
log.Print(header_order)
t.Fatal("not equal")

4
test/middleware/errCallBack_test.go
View File

@@ -12,10 +12,10 @@ func TestErrCallBack(t *testing.T) {
n := 0
_, err := requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{
MaxRetries: 3,
ResultCallBack: func(ctx context.Context, client *requests.Client, response *requests.Response) error {
ResultCallBack: func(ctx context.Context, option *requests.RequestOption, response *requests.Response) error {
return errors.New("try")
},
ErrCallBack: func(ctx context.Context, client *requests.Client, response *requests.Response, err error) error {
ErrCallBack: func(ctx context.Context, option *requests.RequestOption, response *requests.Response, err error) error {
if n == 0 {
n++
return nil

2
test/middleware/optionltCallBack_test.go
View File

@@ -9,7 +9,7 @@ import (
func TestOptionCallBack(t *testing.T) {
resp, err := requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{
OptionCallBack: func(ctx context.Context, client *requests.Client, option *requests.RequestOption) error {
OptionCallBack: func(ctx context.Context, option *requests.RequestOption) error {
option.Params = map[string]string{"name": "test"}
return nil
},

2
test/middleware/resultCallBack_test.go
View File

@@ -11,7 +11,7 @@ import (
func TestResultCallBack(t *testing.T) {
var code int
_, err := requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{
ResultCallBack: func(ctx context.Context, client *requests.Client, response *requests.Response) error {
ResultCallBack: func(ctx context.Context, option *requests.RequestOption, response *requests.Response) error {
if response.StatusCode() != 200 {
return errors.New("resp.StatusCode!= 200")
}

3
test/middleware/try_test.go
View File

@@ -12,7 +12,8 @@ func TestMaxRetries(t *testing.T) {
n := 0
resp, err := requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{
MaxRetries: 3,
ResultCallBack: func(ctx context.Context, client *requests.Client, response *requests.Response) error {
ResultCallBack: func(ctx context.Context, option *requests.RequestOption, response *requests.Response) error {
if n == 0 {
n++
return errors.New("try")