package/requests
1
0
Fork 0
mirror of https://github.com/gospider007/requests.git synced 2025-12-24 13:57:52 +08:00
Code Issues Packages Projects Releases Wiki Activity

sync

Browse Source
This commit is contained in:
gospider
2025-02-13 08:38:19 +08:00
parent 5c275b161c
commit 99f9906eb2
9 changed files with 55 additions and 142 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
go.mod
View File

@@ -9,8 +9,8 @@ require (
github.com/gospider007/gtls v0.0.0-20250107100054-2a14fa6fc9c5
github.com/gospider007/http2 v0.0.0-20250211075152-9b93ea5934a7
github.com/gospider007/http3 v0.0.0-20250211075910-8bad8da7d8cc
github.com/gospider007/ja3 v0.0.0-20250210150438-a0b8d7729724
github.com/gospider007/proxy v0.0.0-20250114011401-cd5ee0138e0e
github.com/gospider007/ja3 v0.0.0-20250213002809-fed6ef545daf
github.com/gospider007/proxy v0.0.0-20250212051747-ecb05461e726
github.com/gospider007/re v0.0.0-20241216142712-efbef8d55ea2
github.com/gospider007/tools v0.0.0-20250208030917-26051a107d8c
github.com/gospider007/websocket v0.0.0-20250107091829-c78035a103b2

14
go.sum
View File

@@ -59,7 +59,6 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=
github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
@@ -107,22 +106,18 @@ github.com/gospider007/gson v0.0.0-20250110063619-6fd4001287da h1:54D7y2SYeIsUVH
github.com/gospider007/gson v0.0.0-20250110063619-6fd4001287da/go.mod h1:4oRt+sgROKHOLdxw3DiK69VUsdhBS0RsPix88PqGCjM=
github.com/gospider007/gtls v0.0.0-20250107100054-2a14fa6fc9c5 h1:c0payCC+agt1c+SK66VYXcs8eFktGlxWhK+t6V/6PPc=
github.com/gospider007/gtls v0.0.0-20250107100054-2a14fa6fc9c5/go.mod h1:qIlB6X0WLv9QVqo/LLqkml0JDKSjeL3x4tI1QCINU94=
github.com/gospider007/http2 v0.0.0-20250207072457-e5aee546e6ef h1:uit8QvhFLCA26ZQHV8akzHu4BHOZ7nSuph097sQHB/8=
github.com/gospider007/http2 v0.0.0-20250207072457-e5aee546e6ef/go.mod h1:3uzGktvy4KnYZm+DwWNui4RsvPUQSSphJ1VTtfO1tE0=
github.com/gospider007/http2 v0.0.0-20250211075152-9b93ea5934a7 h1:eEqGJ+D+WhsQyD0i1NOapkpSxhb2CS+S2JRnJGm9kn0=
github.com/gospider007/http2 v0.0.0-20250211075152-9b93ea5934a7/go.mod h1:tyoQk56/HeqWXUL5dXpmCWoH+jVIaxo3//HAYeMPWkM=
github.com/gospider007/http3 v0.0.0-20250207072522-fbe0ea54a292 h1:CpWOjsI8+sFP2uDHXCyN+pcifwLQ3vz9SlRuJFcb0mk=
github.com/gospider007/http3 v0.0.0-20250207072522-fbe0ea54a292/go.mod h1:CCgRzDZa7vZZEx+RJK/tGOFAF0L1MX7wKKw+h5ntIA8=
github.com/gospider007/http3 v0.0.0-20250211075910-8bad8da7d8cc h1:vZDb6z5iPkAnCkfS77cIiR8gbcWbL9yRD5HAiYovXFA=
github.com/gospider007/http3 v0.0.0-20250211075910-8bad8da7d8cc/go.mod h1:B/9ytNa5gyQwktT9klskrWVZliYgNXRpXQtZCu7XGVU=
github.com/gospider007/ja3 v0.0.0-20250210150438-a0b8d7729724 h1:RH16GaKyQftA5SFdGCZ7qH0V17T4T9MGcqF4AMXE2UY=
github.com/gospider007/ja3 v0.0.0-20250210150438-a0b8d7729724/go.mod h1:LyhufzO6wrBlxeilbUJElfEdDNdD+1v9fB5p30pDAjE=
github.com/gospider007/ja3 v0.0.0-20250213002809-fed6ef545daf h1:IwmovyoSdIKvyOf1kiDa9FWQ0oM8jZSLrHdLwoCCfkU=
github.com/gospider007/ja3 v0.0.0-20250213002809-fed6ef545daf/go.mod h1:UnXMh9FDg/5QKfAt/wCxxByQDmLh0AKQetkRYITDcGM=
github.com/gospider007/kinds v0.0.0-20240929092451-8f867acde255 h1:X+AM/mgmh/EfyQUjKZp1VFc9TSlrhkwS0eSYeo5fMs4=
github.com/gospider007/kinds v0.0.0-20240929092451-8f867acde255/go.mod h1:yZx7Zfp1I4P6CO3TcDyDY5SuXQYr0bZjzT9zG0XrJAI=
github.com/gospider007/net v0.0.0-20250207031315-05571927d356 h1:RRft8tp12L3VCrdwWWCFmhChTmQtvESs/o58zdOhnV8=
github.com/gospider007/net v0.0.0-20250207031315-05571927d356/go.mod h1:xsH1vN8Vo1+JsHC/voUXhX3tRWD93S9700t3ynQXET8=
github.com/gospider007/proxy v0.0.0-20250114011401-cd5ee0138e0e h1:5lvt+1JK8fLooCr9RSAf22bsOj2qmQueaT+yr698OLY=
github.com/gospider007/proxy v0.0.0-20250114011401-cd5ee0138e0e/go.mod h1:058+64s2TFP/IoAjsLMpjNa8tduvhDfUP68W6xH82XU=
github.com/gospider007/proxy v0.0.0-20250212051747-ecb05461e726 h1:x2y6r+D7mECj5/TZ9/ppr/YUoZ+oaa2AOnHu7f5Dpbc=
github.com/gospider007/proxy v0.0.0-20250212051747-ecb05461e726/go.mod h1:x2sBfbvWDjWzcsn/PRKzoqXt/JB1kJ6dJcnpFNrLRLo=
github.com/gospider007/re v0.0.0-20241216142712-efbef8d55ea2 h1:ixXFS1DqP0NnHna+b0JKaPqMRYRmahzUADZn7PawQq0=
github.com/gospider007/re v0.0.0-20241216142712-efbef8d55ea2/go.mod h1:kr9bUaC42FS019Ak23fSctbTRB2JpfPPg/pSVjQmsws=
github.com/gospider007/tools v0.0.0-20250208030917-26051a107d8c h1:5ea20FyPjhgvsfqo6RyC9fHzWYE73Un/Yypgh2mWTaU=
@@ -157,7 +152,6 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/libdns/libdns v0.2.2 h1:O6ws7bAfRPaBsgAYt8MDe2HcNBGC29hkZ9MX2eUSX3s=
github.com/libdns/libdns v0.2.2/go.mod h1:4Bj9+5CQiNMVGf87wjX4CY3HQJypUHRuLvlsfsZqLWQ=
github.com/mholt/acmez v1.2.0 h1:1hhLxSgY5FvH5HCnGUuwbKY2VQVo8IU7rxXKSnZ7F30=
github.com/mholt/acmez/v3 v3.0.1 h1:4PcjKjaySlgXK857aTfDuRbmnM5gb3Ruz3tvoSJAUp8=
github.com/mholt/acmez/v3 v3.0.1/go.mod h1:L1wOU06KKvq7tswuMDwKdcHeKpFFgkppZy/y0DFxagQ=
github.com/mholt/archives v0.1.0 h1:FacgJyrjiuyomTuNA92X5GyRBRZjE43Y/lrzKIlF35Q=

14
option.go
View File

@@ -38,7 +38,7 @@ type Log struct {
// Connection Management Options
type ClientOption struct {
Ja3Spec any //support utls.ClientHelloID, clienthello with hex string, clienthello with byte array
Spec any //support []bytes,hex string,bool
DialOption DialOption
Headers any //default headers
Jar Jar //custom cookies
@@ -53,12 +53,12 @@ type ClientOption struct {
UtlsConfig *utls.Config
QuicConfig *quic.Config
UquicConfig *uquic.Config
UJa3Spec uquic.QUICID
USpec any //support ja3.USpec,uquic.QUICID,bool
Proxy string //proxy,support https,http,socks5
UserAgent string //headers User-Agent value
OrderHeaders []string //order headers
Proxys []string //proxy list,support https,http,socks5
H2Ja3Spec ja3.H2Spec //h2 fingerprint
HSpec ja3.HSpec //h2 fingerprint
MaxRetries int //try num
MaxRedirect int //redirect num ,<0 no redirect,==0 no limit
Timeout time.Duration //request timeout
@@ -66,11 +66,9 @@ type ClientOption struct {
TlsHandshakeTimeout time.Duration //tls timeout,default:15
H3 bool //开启http3
ForceHttp1 bool //force use http1 send requests
Ja3 bool //enable ja3 fingerprint
UJa3 bool
DisCookie bool //disable cookies
DisDecode bool //disable auto decode
Bar bool ////enable bar display
DisCookie bool //disable cookies
DisDecode bool //disable auto decode
Bar bool ////enable bar display
}
// Options for sending requests

10
requests.go
View File

@@ -200,11 +200,11 @@ func (obj *Client) request(ctx *Response) (err error) {
}
//设置 h2 请求头顺序
if ctx.option.OrderHeaders != nil {
if !ctx.option.H2Ja3Spec.IsSet() {
ctx.option.H2Ja3Spec = ja3.DefaultH2Spec()
ctx.option.H2Ja3Spec.OrderHeaders = ctx.option.OrderHeaders
} else if ctx.option.H2Ja3Spec.OrderHeaders == nil {
ctx.option.H2Ja3Spec.OrderHeaders = ctx.option.OrderHeaders
if !ctx.option.HSpec.IsSet() {
ctx.option.HSpec = ja3.DefaultHSpec()
ctx.option.HSpec.OrderHeaders = ctx.option.OrderHeaders
} else if ctx.option.HSpec.OrderHeaders == nil {
ctx.option.HSpec.OrderHeaders = ctx.option.OrderHeaders
}
}
//init tls timeout

62
roundTripper.go
View File

@@ -18,7 +18,6 @@ import (
"github.com/gospider007/tools"
"github.com/quic-go/quic-go"
uquic "github.com/refraction-networking/uquic"
utls "github.com/refraction-networking/utls"
)
type reqTask struct {
@@ -137,7 +136,7 @@ func (obj *roundTripper) ghttp3Dial(ctx *Response, remoteAddress Address, proxyA
return
}
func (obj *roundTripper) uhttp3Dial(ctx *Response, remoteAddress Address, proxyAddress ...Address) (conn *connecotr, err error) {
func (obj *roundTripper) uhttp3Dial(ctx *Response, spec uquic.QUICSpec, remoteAddress Address, proxyAddress ...Address) (conn *connecotr, err error) {
udpConn, err := obj.http3Dial(ctx, remoteAddress, proxyAddress...)
if err != nil {
return nil, err
@@ -155,14 +154,6 @@ func (obj *roundTripper) uhttp3Dial(ctx *Response, remoteAddress Address, proxyA
if ctx.option.UquicConfig != nil {
quicConfig = ctx.option.UquicConfig.Clone()
}
var spec uquic.QUICSpec
if ctx.option.UJa3Spec.Client != "" {
if spec, err = uquic.QUICID2Spec(ctx.option.UJa3Spec); err != nil {
return nil, err
}
} else {
spec = ja3.DefaultUSpec()
}
netConn, err := (&uquic.UTransport{
Transport: &uquic.Transport{
Conn: udpConn,
@@ -186,11 +177,16 @@ func (obj *roundTripper) dial(ctx *Response) (conn *connecotr, err error) {
return nil, err
}
if ctx.option.H3 {
if ctx.option.UJa3Spec.Client != "" || ctx.option.UJa3 {
return obj.uhttp3Dial(ctx, remoteAddress, proxys...)
} else {
return obj.ghttp3Dial(ctx, remoteAddress, proxys...)
if ctx.option.USpec != nil {
spec, err := ja3.CreateUSpec(ctx.option.USpec)
if err != nil {
return nil, err
}
if spec.ClientHelloSpec != nil {
return obj.uhttp3Dial(ctx, spec, remoteAddress, proxys...)
}
}
return obj.ghttp3Dial(ctx, remoteAddress, proxys...)
}
var rawNetConn net.Conn
if len(proxys) > 0 {
@@ -238,10 +234,10 @@ func (obj *roundTripper) dial(ctx *Response) (conn *connecotr, err error) {
}
func (obj *roundTripper) dialConnecotr(option *RequestOption, req *http.Request, conne *connecotr, h2 bool) (err error) {
if h2 {
if option.H2Ja3Spec.OrderHeaders != nil {
option.OrderHeaders = option.H2Ja3Spec.OrderHeaders
if option.HSpec.OrderHeaders != nil {
option.OrderHeaders = option.HSpec.OrderHeaders
}
if conne.Conn, err = http2.NewClientConn(req.Context(), conne.c, option.H2Ja3Spec, func() {
if conne.Conn, err = http2.NewClientConn(req.Context(), conne.c, option.HSpec, func() {
conne.forceCnl(errors.New("http2 client close"))
}); err != nil {
return err
@@ -256,27 +252,23 @@ func (obj *roundTripper) dialConnecotr(option *RequestOption, req *http.Request,
func (obj *roundTripper) dialAddTls(option *RequestOption, req *http.Request, netConn net.Conn) (net.Conn, bool, error) {
ctx, cnl := context.WithTimeout(req.Context(), option.TlsHandshakeTimeout)
defer cnl()
if option.Ja3Spec != nil || option.Ja3 {
var spec utls.ClientHelloSpec
var err error
if option.Ja3Spec != nil {
if spec, err = ja3.CreateSpecWithClientHello(option.Ja3Spec); err != nil {
return nil, false, err
if option.Spec != nil {
spec, err := ja3.CreateSpec(option.Spec)
if err != nil {
return nil, false, err
}
if len(spec.Extensions) > 0 {
if tlsConn, err := obj.dialer.addJa3Tls(ctx, netConn, getHost(req), !option.ForceHttp1, spec, option.UtlsConfig.Clone()); err != nil {
return tlsConn, false, tools.WrapError(err, "add ja3 tls error")
} else {
return tlsConn, tlsConn.ConnectionState().NegotiatedProtocol == "h2", nil
}
} else {
spec = ja3.DefaultSpec()
}
if tlsConn, err := obj.dialer.addJa3Tls(ctx, netConn, getHost(req), !option.ForceHttp1, spec, option.UtlsConfig.Clone()); err != nil {
return tlsConn, false, tools.WrapError(err, "add ja3 tls error")
} else {
return tlsConn, tlsConn.ConnectionState().NegotiatedProtocol == "h2", nil
}
}
if tlsConn, err := obj.dialer.addTls(ctx, netConn, getHost(req), !option.ForceHttp1, option.TlsConfig.Clone()); err != nil {
return tlsConn, false, tools.WrapError(err, "add tls error")
} else {
if tlsConn, err := obj.dialer.addTls(ctx, netConn, getHost(req), !option.ForceHttp1, option.TlsConfig.Clone()); err != nil {
return tlsConn, false, tools.WrapError(err, "add tls error")
} else {
return tlsConn, tlsConn.ConnectionState().NegotiatedProtocol == "h2", nil
}
return tlsConn, tlsConn.ConnectionState().NegotiatedProtocol == "h2", nil
}
}
func (obj *roundTripper) initProxys(ctx *Response) ([]Address, error) {

4
test/fingerprint/http2_test.go
View File

@@ -11,14 +11,14 @@ import (
func TestH2(t *testing.T) {
j := "1:65536,2:0,4:6291456,6:262144|15663105|0|m,a,s,p"
j2 := "1:65536;2:0;4:6291456;6:262144|15663105|0|m,a,s,p"
h2Spec, err := ja3.CreateH2SpecWithStr(j) //create h2 spec with string
h2Spec, err := ja3.CreateHSpec(j) //create h2 spec with string
if err != nil {
t.Fatal(err)
}
// log.Print(h2Spec)
resp, err := requests.Get(nil, "https://tools.scrapfly.io/api/fp/anything", requests.RequestOption{
ClientOption: requests.ClientOption{
H2Ja3Spec: h2Spec, //set h2 spec
HSpec: h2Spec, //set h2 spec
},
})
// log.Print(resp.Text())

78
test/fingerprint/ja3_test.go
View File

File diff suppressed because one or more lines are too long

8
test/fingerprint/quic_test.go
View File

@@ -11,8 +11,8 @@ import (
func TestHttp3(t *testing.T) {
resp, err := requests.Get(context.TODO(), "https://cloudflare-quic.com/", requests.RequestOption{
ClientOption: requests.ClientOption{
H3: true,
Ja3: true,
H3: true,
Spec: true,
},
},
)
@@ -31,8 +31,8 @@ func TestHttp3(t *testing.T) {
func TestHttp32(t *testing.T) {
resp, err := requests.Get(context.TODO(), "https://cloudflare-quic.com/", requests.RequestOption{
ClientOption: requests.ClientOption{
UJa3: true,
H3: true,
USpec: true,
H3: true,
},
},
)

3
test/logger/logger_test.go
View File

@@ -11,8 +11,7 @@ func TestLogger(t *testing.T) {
for i := 0; i < 2; i++ {
response, err := requests.Get(nil, "https://www.httpbin.org", requests.RequestOption{
ClientOption: requests.ClientOption{
Ja3: true,
Spec: true,
Logger: func(l requests.Log) {
log.Print(l)
},