package/requests
1
0
Fork 0
mirror of https://github.com/gospider007/requests.git synced 2025-12-24 13:57:52 +08:00
Code Issues Packages Projects Releases Wiki Activity

sync

Browse Source
This commit is contained in:
gospider
2025-07-29 09:38:55 +08:00
parent 40526d9faa
commit 4c8f3b794c
33 changed files with 459 additions and 479 deletions
Download Patch File Download Diff File Expand all files Collapse all files

122
body.go
View File

@@ -3,6 +3,7 @@ package requests
import (
"bytes"
"context"
"crypto/rand"
"errors"
"fmt"
"io"
@@ -11,6 +12,7 @@ import (
"net/textproto"
"net/url"
"sort"
"strings"
"github.com/gospider007/gson"
"github.com/gospider007/tools"
@@ -320,3 +322,123 @@ func (obj *OrderData) parseText() (io.Reader, error) {
}
return bytes.NewReader(con), nil
}
// Upload files with form-data,
type File struct {
Content any
FileName string
ContentType string
}
func randomBoundary() (string, string) {
var buf [30]byte
io.ReadFull(rand.Reader, buf[:])
boundary := fmt.Sprintf("%x", buf[:])
if strings.ContainsAny(boundary, `()<>@,;:\"/[]?= `) {
boundary = `"` + boundary + `"`
}
return "multipart/form-data; boundary=" + boundary, boundary
}
func (obj *RequestOption) initBody(ctx context.Context) (io.Reader, error) {
if obj.Body != nil {
body, orderData, err := obj.newBody(obj.Body)
if err != nil {
return nil, err
}
if body != nil {
return body, nil
}
con, err := orderData.MarshalJSON()
if err != nil {
return nil, err
}
return bytes.NewReader(con), nil
} else if obj.Form != nil {
var boundary string
if obj.ContentType == "" {
obj.ContentType, boundary = randomBoundary()
}
body, orderData, err := obj.newBody(obj.Form)
if err != nil {
return nil, err
}
if body != nil {
return body, nil
}
body, once, err := orderData.parseForm(ctx, boundary)
if err != nil {
return nil, err
}
obj.readOne = once
return body, err
} else if obj.Data != nil {
if obj.ContentType == "" {
obj.ContentType = "application/x-www-form-urlencoded"
}
body, orderData, err := obj.newBody(obj.Data)
if err != nil {
return nil, err
}
if body != nil {
return body, nil
}
return orderData.parseData(), nil
} else if obj.Json != nil {
if obj.ContentType == "" {
obj.ContentType = "application/json"
}
body, orderData, err := obj.newBody(obj.Json)
if err != nil {
return nil, err
}
if body != nil {
return body, nil
}
return orderData.parseJson()
} else if obj.Text != nil {
if obj.ContentType == "" {
obj.ContentType = "text/plain"
}
body, orderData, err := obj.newBody(obj.Text)
if err != nil {
return nil, err
}
if body != nil {
return body, nil
}
return orderData.parseText()
} else {
return nil, nil
}
}
func (obj *RequestOption) initParams() (*url.URL, error) {
baseUrl := cloneUrl(obj.Url)
if obj.Params == nil {
return baseUrl, nil
}
body, dataData, err := obj.newBody(obj.Params)
if err != nil {
return nil, err
}
var query string
if body != nil {
paramsBytes, err := io.ReadAll(body)
if err != nil {
return nil, err
}
query = tools.BytesToString(paramsBytes)
} else {
query = dataData.parseParams().String()
}
if query == "" {
return baseUrl, nil
}
pquery := baseUrl.Query().Encode()
if pquery == "" {
baseUrl.RawQuery = query
} else {
baseUrl.RawQuery = pquery + "&" + query
}
return baseUrl, nil
}

4
client.go
View File

@@ -12,7 +12,7 @@ type Client struct {
ctx context.Context
transport *roundTripper
cnl context.CancelFunc
ClientOption ClientOption
ClientOption *ClientOption
closed bool
}
@@ -30,7 +30,7 @@ func NewClient(preCtx context.Context, options ...ClientOption) (*Client, error)
result := new(Client)
result.ctx, result.cnl = context.WithCancel(preCtx)
result.transport = newRoundTripper(result.ctx)
result.ClientOption = option
result.ClientOption = &option
if result.ClientOption.TlsConfig == nil {
result.ClientOption.TlsConfig = &tls.Config{
InsecureSkipVerify: true,

128
conn.go
View File

@@ -4,6 +4,7 @@ import (
"context"
"errors"
"net"
"net/http"
"time"
"github.com/gospider007/http1"
@@ -12,50 +13,7 @@ import (
var maxRetryCount = 5
type connecotr struct {
forceCtx context.Context //force close
forceCnl context.CancelCauseFunc
Conn http1.Conn
c net.Conn
proxys []Address
}
func (obj *connecotr) withCancel(forceCtx context.Context) {
obj.forceCtx, obj.forceCnl = context.WithCancelCause(forceCtx)
}
func (obj *connecotr) Close() error {
return obj.CloseWithError(errors.New("connecotr Close close"))
}
func (obj *connecotr) CloseWithError(err error) error {
err = obj.Conn.CloseWithError(err)
if obj.c != nil {
return obj.c.Close()
}
return err
}
func (obj *connecotr) wrapBody(task *reqTask) {
body := new(wrapBody)
body.rawBody = task.reqCtx.response.Body.(*http1.ClientBody)
body.conn = obj
task.reqCtx.response.Body = body
task.reqCtx.response.Request = task.reqCtx.request
}
func (obj *connecotr) httpReq(task *reqTask, done chan struct{}) (err error) {
defer close(done)
response, bodyCtx, derr := obj.Conn.DoRequest(task.reqCtx.request, &http1.Option{OrderHeaders: task.reqCtx.option.orderHeaders.Data()})
if derr != nil {
err = tools.WrapError(derr, "roundTrip error")
return
}
task.reqCtx.response = response
task.bodyCtx = bodyCtx
obj.wrapBody(task)
return
}
func (obj *connecotr) taskMain(task *reqTask) (err error) {
func taskMain(conn http1.Conn, task *reqTask) (err error) {
defer func() {
if err != nil && task.reqCtx.option.ErrCallBack != nil {
task.reqCtx.err = err
@@ -70,59 +28,53 @@ func (obj *connecotr) taskMain(task *reqTask) (err error) {
} else {
task.cnl(err)
}
if err == nil && task.reqCtx.response != nil && task.reqCtx.response.Body != nil && task.bodyCtx != nil {
select {
case <-obj.forceCtx.Done():
err = context.Cause(obj.forceCtx)
case <-task.reqCtx.Context().Done():
if context.Cause(task.reqCtx.Context()) != tools.ErrNoErr {
err = context.Cause(task.reqCtx.Context())
}
if err == nil && task.reqCtx.response.StatusCode == 101 {
select {
case <-obj.forceCtx.Done():
err = context.Cause(obj.forceCtx)
case <-task.bodyCtx.Done():
if context.Cause(task.bodyCtx) != tools.ErrNoErr {
err = context.Cause(task.bodyCtx)
}
if err == nil && task.reqCtx.response != nil && task.reqCtx.response.Body != nil {
if bodyCtx := task.reqCtx.response.Body.(*http1.Body).Context(); bodyCtx != nil {
select {
case <-task.reqCtx.Context().Done():
if context.Cause(task.reqCtx.Context()) != tools.ErrNoErr {
err = context.Cause(task.reqCtx.Context())
}
case <-bodyCtx.Done():
if context.Cause(bodyCtx) != tools.ErrNoErr {
err = context.Cause(bodyCtx)
}
}
case <-task.bodyCtx.Done():
if context.Cause(task.bodyCtx) != tools.ErrNoErr {
err = context.Cause(task.bodyCtx)
}
}
}
if err != nil {
obj.CloseWithError(tools.WrapError(err, "taskMain close with error"))
conn.CloseWithError(tools.WrapError(err, "taskMain close with error"))
}
}()
select {
case <-obj.forceCtx.Done(): //force conn close
err = context.Cause(obj.forceCtx)
case <-conn.Context().Done(): //force conn close
err = context.Cause(conn.Context())
task.enableRetry = true
task.isNotice = true
return
default:
}
done := make(chan struct{})
var derr error
var response *http.Response
go func() {
err = obj.httpReq(task, done)
defer close(done)
response, derr = conn.DoRequest(task.reqCtx.request, &http1.Option{OrderHeaders: task.reqCtx.option.orderHeaders.Data()})
}()
select {
case <-obj.forceCtx.Done(): //force conn close
err = tools.WrapError(context.Cause(obj.forceCtx), "taskMain delete ctx error: ")
case <-conn.Context().Done(): //force conn close
err = tools.WrapError(context.Cause(conn.Context()), "taskMain delete ctx error: ")
case <-time.After(task.reqCtx.option.ResponseHeaderTimeout):
err = errors.New("ResponseHeaderTimeout error: ")
case <-task.ctx.Done():
err = context.Cause(task.ctx)
case <-done:
if err == nil && task.reqCtx.response == nil {
err = context.Cause(task.ctx)
if err == nil {
err = errors.New("body done response is nil")
}
if derr != nil {
err = tools.WrapError(derr, "roundTrip error")
} else {
task.reqCtx.response = response
task.reqCtx.response.Request = task.reqCtx.request
}
if task.reqCtx.option.Logger != nil {
task.reqCtx.option.Logger(Log{
@@ -136,29 +88,37 @@ func (obj *connecotr) taskMain(task *reqTask) (err error) {
return
}
func (obj *connecotr) rwMain(ctx context.Context, done chan struct{}, tasks chan *reqTask) (err error) {
obj.withCancel(ctx)
func taskM(conn http1.Conn, task *reqTask) error {
err := taskMain(conn, task)
if err != nil {
return err
}
if task.reqCtx.response != nil && task.reqCtx.response.StatusCode == 101 {
return tools.ErrNoErr
}
return err
}
func rwMain(conn http1.Conn, task *reqTask, tasks chan *reqTask) (err error) {
defer func() {
if err != nil && err != tools.ErrNoErr {
obj.CloseWithError(tools.WrapError(err, "rwMain close with error"))
conn.CloseWithError(tools.WrapError(err, "rwMain close with error"))
}
}()
close(done)
if err = taskM(conn, task); err != nil {
return
}
for {
select {
case <-obj.forceCtx.Done(): //force close conn
case <-conn.Context().Done(): //force close conn
return errors.New("connecotr force close")
case task := <-tasks: //recv task
if task == nil {
return errors.New("task is nil")
}
err = obj.taskMain(task)
err = taskM(conn, task)
if err != nil {
return
}
if task.reqCtx.response != nil && task.reqCtx.response.StatusCode == 101 {
return tools.ErrNoErr
}
}
}
}

20
dial.go
View File

@@ -53,7 +53,11 @@ func (d *myDialer) LookupIPAddr(ctx context.Context, host string) ([]net.IPAddr,
return d.dialer.Resolver.LookupIPAddr(ctx, host)
}
func newDialer(option DialOption) dialer {
func newDialer(dialOption *DialOption) dialer {
var option DialOption
if dialOption != nil {
option = *dialOption
}
if option.KeepAlive == 0 {
option.KeepAlive = time.Second * 5
}
@@ -267,7 +271,7 @@ func (obj *Dialer) verifyProxyToRemote(ctx *Response, conn net.Conn, proxyTlsCon
}
}
func (obj *Dialer) loadHost(ctx context.Context, host string, option DialOption) (net.IP, error) {
func (obj *Dialer) loadHost(ctx context.Context, host string, option *DialOption) (net.IP, error) {
msgDataAny, ok := obj.dnsIpData.Load(host)
if ok {
msgdata := msgDataAny.(msgClient)
@@ -280,10 +284,12 @@ func (obj *Dialer) loadHost(ctx context.Context, host string, option DialOption)
return ip, nil
}
var addrType gtls.AddrType
if option.AddrType != 0 {
addrType = option.AddrType
} else if option.GetAddrType != nil {
addrType = option.GetAddrType(host)
if option != nil {
if option.AddrType != 0 {
addrType = option.AddrType
} else if option.GetAddrType != nil {
addrType = option.GetAddrType(host)
}
}
ips, err := newDialer(option).LookupIPAddr(ctx, host)
if err != nil {
@@ -334,7 +340,7 @@ func readUdpAddr(r io.Reader) (Address, error) {
UdpAddress.Port = int(binary.BigEndian.Uint16(port[:]))
return UdpAddress, nil
}
func (obj *Dialer) ReadUdpAddr(ctx context.Context, r io.Reader, option DialOption) (Address, error) {
func (obj *Dialer) ReadUdpAddr(ctx context.Context, r io.Reader, option *DialOption) (Address, error) {
udpAddress, err := readUdpAddr(r)
if err != nil {
return udpAddress, err

244
option.go
View File

@@ -1,17 +1,10 @@
package requests
import (
"bytes"
"context"
"crypto/rand"
"crypto/tls"
"fmt"
"io"
"net/url"
"strings"
"time"
"github.com/gospider007/tools"
"github.com/gospider007/websocket"
"github.com/quic-go/quic-go"
uquic "github.com/refraction-networking/uquic"
@@ -43,8 +36,7 @@ type Log struct {
type ClientOption struct {
Spec string //goSpiderSpec origin : https://github.com/gospider007/fp
OrderHeaders []string
DialOption DialOption
Headers any //default headers
DialOption *DialOption
Jar Jar //custom cookies
Logger func(Log) //debuggable
OptionCallBack func(ctx *Response) error //option callback,if error is returnd, break request
@@ -73,6 +65,36 @@ type ClientOption struct {
// Options for sending requests
type RequestOption struct {
Spec string //goSpiderSpec origin : https://github.com/gospider007/fp
OrderHeaders []string
DialOption *DialOption
Jar Jar //custom cookies
Logger func(Log) //debuggable
OptionCallBack func(ctx *Response) error //option callback,if error is returnd, break request
ResultCallBack func(ctx *Response) error //result callback,if error is returnd,next errCallback
ErrCallBack func(ctx *Response) error //error callback,if error is returnd,break request
RequestCallBack func(ctx *Response) error //request and response callback,if error is returnd,reponse is error
GetProxy func(ctx *Response) (any, error) //proxy callback:support https,http,socks5 proxy
TlsConfig *tls.Config
UtlsConfig *utls.Config
QuicConfig *quic.Config
UquicConfig *uquic.Config
USpec any //support ja3.USpec,uquic.QUICID,bool
UserAgent string //headers User-Agent value
Proxy any //strong or []string, ,support https,http,socks5
MaxRetries int //try num
MaxRedirect int //redirect num ,<0 no redirect,==0 no limit
Timeout time.Duration //request timeout
ResponseHeaderTimeout time.Duration //ResponseHeaderTimeout ,default:300
TlsHandshakeTimeout time.Duration //tls timeout,default:15
ForceHttp3 bool //force use http3 send requests
ForceHttp1 bool //force use http1 send requests
DisCookie bool //disable cookies
DisDecode bool //disable auto decode
Bar bool ////enable bar display
//分割线
//以下为请求参数
Headers any //default headers
WsOption websocket.Option //websocket option
Cookies any // cookies,support :json,map,str，http.Header
Params any //url params，join url query,json,map
@@ -87,136 +109,101 @@ type RequestOption struct {
Host string
Referer string //set headers referer value
ContentType string //headers Content-Type value
ClientOption
Stream bool //disable auto read
DisProxy bool //force disable proxy
Stream bool //disable auto read
DisProxy bool //force disable proxy
readOne bool
orderHeaders *OrderData //order headers
gospiderSpec *GospiderSpec
disBody bool
}
// Upload files with form-data,
type File struct {
Content any
FileName string
ContentType string
}
func randomBoundary() (string, string) {
var buf [30]byte
io.ReadFull(rand.Reader, buf[:])
boundary := fmt.Sprintf("%x", buf[:])
if strings.ContainsAny(boundary, `()<>@,;:\"/[]?= `) {
boundary = `"` + boundary + `"`
func merge(option *RequestOption, clientOption *ClientOption) {
if option.Spec == "" {
option.Spec = clientOption.Spec
}
return "multipart/form-data; boundary=" + boundary, boundary
}
func (obj *RequestOption) initBody(ctx context.Context) (io.Reader, error) {
if obj.Body != nil {
body, orderData, err := obj.newBody(obj.Body)
if err != nil {
return nil, err
}
if body != nil {
return body, nil
}
con, err := orderData.MarshalJSON()
if err != nil {
return nil, err
}
return bytes.NewReader(con), nil
} else if obj.Form != nil {
var boundary string
if obj.ContentType == "" {
obj.ContentType, boundary = randomBoundary()
}
body, orderData, err := obj.newBody(obj.Form)
if err != nil {
return nil, err
}
if body != nil {
return body, nil
}
body, once, err := orderData.parseForm(ctx, boundary)
if err != nil {
return nil, err
}
obj.readOne = once
return body, err
} else if obj.Data != nil {
if obj.ContentType == "" {
obj.ContentType = "application/x-www-form-urlencoded"
}
body, orderData, err := obj.newBody(obj.Data)
if err != nil {
return nil, err
}
if body != nil {
return body, nil
}
return orderData.parseData(), nil
} else if obj.Json != nil {
if obj.ContentType == "" {
obj.ContentType = "application/json"
}
body, orderData, err := obj.newBody(obj.Json)
if err != nil {
return nil, err
}
if body != nil {
return body, nil
}
return orderData.parseJson()
} else if obj.Text != nil {
if obj.ContentType == "" {
obj.ContentType = "text/plain"
}
body, orderData, err := obj.newBody(obj.Text)
if err != nil {
return nil, err
}
if body != nil {
return body, nil
}
return orderData.parseText()
} else {
return nil, nil
if option.OrderHeaders == nil {
option.OrderHeaders = clientOption.OrderHeaders
}
}
func (obj *RequestOption) initParams() (*url.URL, error) {
baseUrl := cloneUrl(obj.Url)
if obj.Params == nil {
return baseUrl, nil
if option.DialOption == nil {
option.DialOption = clientOption.DialOption
}
body, dataData, err := obj.newBody(obj.Params)
if err != nil {
return nil, err
if option.Jar == nil {
option.Jar = clientOption.Jar
}
var query string
if body != nil {
paramsBytes, err := io.ReadAll(body)
if err != nil {
return nil, err
}
query = tools.BytesToString(paramsBytes)
} else {
query = dataData.parseParams().String()
if option.Logger == nil {
option.Logger = clientOption.Logger
}
if query == "" {
return baseUrl, nil
if option.OptionCallBack == nil {
option.OptionCallBack = clientOption.OptionCallBack
}
pquery := baseUrl.Query().Encode()
if pquery == "" {
baseUrl.RawQuery = query
} else {
baseUrl.RawQuery = pquery + "&" + query
if option.ResultCallBack == nil {
option.ResultCallBack = clientOption.ResultCallBack
}
if option.ErrCallBack == nil {
option.ErrCallBack = clientOption.ErrCallBack
}
if option.RequestCallBack == nil {
option.RequestCallBack = clientOption.RequestCallBack
}
if option.GetProxy == nil {
option.GetProxy = clientOption.GetProxy
}
if option.TlsConfig == nil {
option.TlsConfig = clientOption.TlsConfig
}
if option.UtlsConfig == nil {
option.UtlsConfig = clientOption.UtlsConfig
}
if option.QuicConfig == nil {
option.QuicConfig = clientOption.QuicConfig
}
if option.UquicConfig == nil {
option.UquicConfig = clientOption.UquicConfig
}
if option.USpec == nil {
option.USpec = clientOption.USpec
}
if option.UserAgent == "" {
option.UserAgent = clientOption.UserAgent
}
if option.Proxy == nil {
option.Proxy = clientOption.Proxy
}
if option.MaxRetries == 0 {
option.MaxRetries = clientOption.MaxRetries
}
if option.MaxRedirect == 0 {
option.MaxRedirect = clientOption.MaxRedirect
}
if option.Timeout == 0 {
option.Timeout = clientOption.Timeout
}
if option.ResponseHeaderTimeout == 0 {
option.ResponseHeaderTimeout = clientOption.ResponseHeaderTimeout
}
if option.TlsHandshakeTimeout == 0 {
option.TlsHandshakeTimeout = clientOption.TlsHandshakeTimeout
}
if !option.ForceHttp3 {
option.ForceHttp3 = clientOption.ForceHttp3
}
if !option.ForceHttp1 {
option.ForceHttp1 = clientOption.ForceHttp1
}
if !option.DisCookie {
option.DisCookie = clientOption.DisCookie
}
if !option.DisDecode {
option.DisDecode = clientOption.DisDecode
}
if !option.Bar {
option.Bar = clientOption.Bar
}
return baseUrl, nil
}
func (obj *Client) newRequestOption(option RequestOption) (RequestOption, error) {
err := tools.Merge(&option, obj.ClientOption)
// err := tools.Merge(&option, obj.ClientOption)
merge(&option, obj.ClientOption)
//end
if option.MaxRetries < 0 {
option.MaxRetries = 0
@@ -233,12 +220,5 @@ func (obj *Client) newRequestOption(option RequestOption) (RequestOption, error)
if option.Spec == "" {
option.Spec = "16030106f2010006ee03039a2b98d81139db0e128ea09eff6874549c219b543fb6dbaa7e4dbfe9e31602c620ce04c4026f019442affade7fed8ba66e022e186f77f1c670fd992f33c0143f120020aaaa130113021303c02bc02fc02cc030cca9cca8c013c014009c009d002f0035010006851a1a00000010000e000c02683208687474702f312e31002b000706dada03040303002d00020101000d0012001004030804040105030805050108060601001b0003020002ff0100010000230000000a000c000afafa11ec001d001700180000000e000c0000096c6f63616c686f7374003304ef04edfafa00010011ec04c06903195f3660633741f9f5ae64d05a316ac8e717582adb9e58c5c242ba306c99ca8f68c15f261245f9141812383c9265f7d0b5c44a5e0d7633f4f40ab8e820ec01bb6cc74e6ab3168e66b40cc4cd37e96e286b4080552b8b0217f786b7c1a0088fb613cc84471b17a33fbcc68db151df387907a1cf3fb14a0f45c6b84608db5b103131b537255c09559cf1940c3980a7f37959a7f95d32c49923600c76c616af238c579361e1c6a20c251d3d42a50e182ad54b1e5d54a57fe6986e64142815b9478cca8066c9bdcc0eb9022ebe05b0ebd53e7d146761d81aee41cd377611699c536a444d300c152994bfeef3cfb4736cb3d57d683269c1e3c001fba2ac220b2c993cec410f6fa5104d1bcde21c46a9b0be8ab7b51aba15f1745ebbd0a0d3a5224170b3bce456c157937e43390b733375bb96601667f5b36888f9520931bead48bae4723d9ed40af2680746e27eee4328503a280ee8846b37803d6206e0f5248bea4ca4a53ca4e1afdd2b84bce0c83260333bc9b38b86486fb48e18d1ce9187b1b6332b2f4145eca38122ac363210137f52140a57b7976b609d739844fa61f21c1e3c300f3434bc3f8b6856994847e2c9b0f20a24b976f9d552b153246db69f30d4a95301b933b0ba9d48402ddb7863cb4f1923a2c33021fd68634a387bf0f76d87f01b35b6182dd10fe9c14b8e548d7988388308b08ff1585ba18a7615737857a7c23c24ee9b3a2ab1915be18b233acd354c7c6513b8ea617a5cf299f34139756cc1df524292c43ee3364990961b2490ae204634a4461b53c95a11d214503985f27ab85bc7c179ab1ba37a828312cabea8cfc5088616386a83e566279a0a5517b60aca4ec6c30a32191dca3cbb7d33ae5087bbdbab5c42e6293b63ad8e35311d459e1ce57037e65e96283e449c3e012051d247653197834c42613ac377a950607c98cbd5a79fef948d18e99758e12d31d13231e638cfb183623346b231a443f56533d444c7204a63479e4efb34ef97597d858915a8a10a32aa78824c9d2993741176da643be1c6c4d91b6511055b098477e7a3c5b5c312cf7bb4ef5905fd741375e62c8ce942a2117bbe707fcc9871e59c0687507862f3634c871885949fce97612793a30a155e84ac503dc519816a13772c50e1167a7031cb2c8187913108f9a26e55958fd19a6e1ef18ed53a70f1c13d01d71d3b40a22413852c9982daac4ae8071966016c38a60bd5c0258c32b882740c6ed5252093c91e51c50cf037d4f5cb6ca610672710f7ca77b0a76039a9968e368a6b243ee4ca7632855cb568c73f01764c4944fc5879d2c52d7992840863c057db2efec658eeb2a73e02bd62617438d9192911bac1f6b0e55cb38255417af20000d69378c857bb278156f16a684200125906b6c22f3d505bc9e76d75fac3a009332ff98fe6baabe3941cab5271c6d2c0ebc993b944c49bd437353019d1b24d10390e45fa87ad77b329a9025933a11af2af0da44d3ed761722c94d8053242f537624113d7bc0155600573301bd2217c6c481ce63b0944b052c97bcb9d3349258257ff33cccf963a6945119ecab21c25051ce02548f642e0ec1ffd392d60facfdf76bfb7274363b62979231f4996362c85d5ba19d2cab7019750b3443565436867a53b71d875eba3282e6d0ee22076d6b97b7c6c556ae216e8bc1bc9f202ce94c763bfe9afc105fca9372dec2e286a001d00200ee8ac33f1ea3153f6b4a06ab71d21b7ce7955ce64ccfc66b7ec8077d02ffd18fe0d00fa00000100018b00206cada2aa48ee4478c40adad21f147d6bc90d13f6889a9b8a58a02536585a261f00d09306c85aab2a6e424b658f3cd9d1c46f35020839287259d3be605ff97faea0d87b9f7f96529661f08cf3f3899db8e805ee7405e2f9b6abd99bc4f6fa5f99b1ed442ebe53c5b10451c93d1221f662783efc3cc8fcf135ed935bcf02ec32251dd09705f191bd7959afbab5619d8e63cb634a259dd63d1b0e42225ae8c08b5b1620cd59d914857e9f1e8a3b7b892863bdaa05429922d75583059641468d8fc51c01e977a69d3a51d714cd5cceea9a5f404ce4a285fd6647931ed8b1c12db027328f214afdbe2c8102b46fe041b553f8670b00050005010000000044cd0005000302683200170000000b0002010000120000caca000100@@505249202a20485454502f322e300d0a0d0a534d0d0a0d0a00001804000000000000010001000000020000000000040060000000060004000000000408000000000000ef00010001d401250000000180000000ff82418aa0e41d139d09b8f3efbf87845887a47e561cc5801f40874148b1275ad1ffb9fe749d3fd4372ed83aa4fe7efbc1fcbefff3f4a7f388e79a82a97a7b0f497f9fbef07f21659fe7e94fe6f4f61e935b4ff3f7de0fe42cb3fcff408b4148b1275ad1ad49e33505023f30408d4148b1275ad1ad5d034ca7b29f07226d61634f53224092b6b9ac1c8558d520a4b6c2ad617b5a54251f01317ad9d07f66a281b0dae053fad0321aa49d13fda992a49685340c8a6adca7e28104416e277fb521aeba0bc8b1e632586d975765c53facd8f7e8cff4a506ea5531149d4ffda97a7b0f49580b2cae05c0b814dc394761986d975765cf53e5497ca589d34d1f43aeba0c41a4c7a98f33a69a3fdf9a68fa1d75d0620d263d4c79a68fbed00177fe8d48e62b03ee697e8d48e62b1e0b1d7f46a4731581d754df5f2c7cfdf6800bbdf43aeba0c41a4c7a9841a6a8b22c5f249c754c5fbef046cfdf6800bbbf408a4148b4a549275906497f83a8f517408a4148b4a549275a93c85f86a87dcd30d25f408a4148b4a549275ad416cf023f31408a4148b4a549275a42a13f8690e4b692d49f50929bd9abfa5242cb40d25fa523b3e94f684c9f518cf73ad7b4fd7b9fefb4005dff4086aec31ec327d785b6007d286f"
}
return option, err
}
func (obj *Client) newResponse(ctx context.Context, option RequestOption, uhref *url.URL, requestId string) *Response {
option.Url = cloneUrl(uhref)
response := NewResponse(ctx, option)
response.client = obj
response.requestId = requestId
return response
return option, nil
}

3
requests.go
View File

@@ -12,6 +12,7 @@ import (
"net/http"
"github.com/gospider007/http1"
"github.com/gospider007/re"
"github.com/gospider007/tools"
"github.com/gospider007/websocket"
@@ -354,7 +355,7 @@ func (obj *Client) request(ctx *Response) (err error) {
return
}
if ctx.response.Body != nil {
ctx.wrapBody = ctx.response.Body.(*wrapBody)
ctx.rawBody = ctx.response.Body.(*http1.Body)
}
if encoding := ctx.ContentEncoding(); encoding != "" && ctx.response.Body != nil {
var unCompressionBody io.ReadCloser

25
response.go
View File

@@ -16,6 +16,7 @@ import (
"github.com/gospider007/bar"
"github.com/gospider007/bs4"
"github.com/gospider007/gson"
"github.com/gospider007/http1"
"github.com/gospider007/re"
"github.com/gospider007/tools"
"github.com/gospider007/websocket"
@@ -59,7 +60,7 @@ type Response struct {
err error
ctx context.Context
request *http.Request
wrapBody *wrapBody
rawBody *http1.Body
response *http.Response
webSocket *websocket.Conn
sse *SSE
@@ -157,7 +158,7 @@ func (obj *Response) WebSocket() *websocket.Conn {
if obj.StatusCode() != 101 {
return nil
}
obj.webSocket = websocket.NewConn(newFakeConn(obj.wrapBody.connStream()), func() { obj.CloseConn() }, true, obj.Headers().Get("Sec-WebSocket-Extensions"))
obj.webSocket = websocket.NewConn(newFakeConn(obj.rawBody.Stream()), func() { obj.CloseConn() }, true, obj.Headers().Get("Sec-WebSocket-Extensions"))
return obj.webSocket
}
@@ -317,16 +318,10 @@ func (obj *Response) IsNewConn() bool {
return obj.isNewConn
}
// conn proxy
func (obj *Response) Proxys() []Address {
return obj.wrapBody.Proxys()
}
// close
func (obj *Response) CloseConn() {
if obj.wrapBody != nil {
obj.wrapBody.CloseWithError(errors.New("force close conn"))
obj.wrapBody.CloseConn()
if obj.rawBody != nil {
obj.rawBody.CloseWithError(errors.New("force close conn"))
}
obj.cnl()
}
@@ -350,7 +345,7 @@ func (obj *Response) closeBody(i bool, err error) {
}
if err == tools.ErrNoErr {
obj.wrapBody.CloseWithError(err)
obj.rawBody.CloseWithError(err)
} else {
obj.CloseConn()
}
@@ -446,3 +441,11 @@ func (obj *body) closeWithError(i bool, err error) error {
func (obj *Response) Body() *body {
return &body{ctx: obj}
}
func (obj *Client) newResponse(ctx context.Context, option RequestOption, uhref *url.URL, requestId string) *Response {
option.Url = cloneUrl(uhref)
response := NewResponse(ctx, option)
response.client = obj
response.requestId = requestId
return response
}

90
roundTripper.go
View File

@@ -22,7 +22,6 @@ import (
)
type reqTask struct {
bodyCtx context.Context
ctx context.Context
cnl context.CancelCauseFunc
reqCtx *Response
@@ -98,16 +97,6 @@ func (obj *roundTripper) getConnPool(task *reqTask) chan *reqTask {
obj.connPools.Store(task.key, tasks)
return tasks
}
func (obj *roundTripper) putConnPool(task *reqTask, conn *connecotr) {
done := make(chan struct{})
go conn.rwMain(obj.ctx, done, obj.getConnPool(task))
<-done
}
func (obj *roundTripper) newConnecotr() *connecotr {
conne := new(connecotr)
conne.withCancel(obj.ctx)
return conne
}
func (obj *roundTripper) http3Dial(ctx *Response, remtoeAddress Address, proxyAddress ...Address) (udpConn net.PacketConn, err error) {
if len(proxyAddress) > 0 {
@@ -121,7 +110,7 @@ func (obj *roundTripper) http3Dial(ctx *Response, remtoeAddress Address, proxyAd
}
return
}
func (obj *roundTripper) ghttp3Dial(ctx *Response, remoteAddress Address, proxyAddress ...Address) (conn *connecotr, err error) {
func (obj *roundTripper) ghttp3Dial(ctx *Response, remoteAddress Address, proxyAddress ...Address) (conn http1.Conn, err error) {
udpConn, err := obj.http3Dial(ctx, remoteAddress, proxyAddress...)
if err != nil {
return nil, err
@@ -143,22 +132,22 @@ func (obj *roundTripper) ghttp3Dial(ctx *Response, remoteAddress Address, proxyA
if err != nil {
return nil, err
}
conn = obj.newConnecotr()
conn.Conn = http3.NewClient(conn.forceCtx, netConn, udpConn, func() {
conn.forceCnl(errors.New("http3 client close"))
cctx, ccnl := context.WithCancelCause(obj.ctx)
// conn = obj.newConnecotr()
conn = http3.NewClient(cctx, netConn, udpConn, func() {
ccnl(errors.New("http3 client close"))
})
if ct, ok := udpConn.(interface {
SetTcpCloseFunc(f func(error))
}); ok {
ct.SetTcpCloseFunc(func(err error) {
conn.forceCnl(errors.New("http3 client close with udp"))
ccnl(errors.New("http3 client close with udp"))
})
}
return
}
func (obj *roundTripper) uhttp3Dial(ctx *Response, remoteAddress Address, proxyAddress ...Address) (conn *connecotr, err error) {
func (obj *roundTripper) uhttp3Dial(ctx *Response, remoteAddress Address, proxyAddress ...Address) (conn http1.Conn, err error) {
spec, err := ja3.CreateUSpec(ctx.option.USpec)
if err != nil {
return nil, err
@@ -189,21 +178,21 @@ func (obj *roundTripper) uhttp3Dial(ctx *Response, remoteAddress Address, proxyA
if err != nil {
return nil, err
}
conn = obj.newConnecotr()
conn.Conn = http3.NewClient(conn.forceCtx, netConn, udpConn, func() {
conn.forceCnl(errors.New("http3 client close"))
cctx, ccnl := context.WithCancelCause(obj.ctx)
conn = http3.NewClient(cctx, netConn, udpConn, func() {
ccnl(errors.New("http3 client close"))
})
if ct, ok := udpConn.(interface {
SetTcpCloseFunc(f func(error))
}); ok {
ct.SetTcpCloseFunc(func(err error) {
conn.forceCnl(errors.New("uhttp3 client close with udp"))
ccnl(errors.New("uhttp3 client close with udp"))
})
}
return
}
func (obj *roundTripper) dial(ctx *Response) (conn *connecotr, err error) {
func (obj *roundTripper) dial(ctx *Response) (conn http1.Conn, err error) {
proxys, err := obj.initProxys(ctx)
if err != nil {
return nil, err
@@ -247,10 +236,9 @@ func (obj *roundTripper) dial(ctx *Response) (conn *connecotr, err error) {
return nil, err
}
var h2 bool
conne := obj.newConnecotr()
conne.proxys = proxys
var rawConn net.Conn
if ctx.request.URL.Scheme == "https" {
conne.c, h2, err = obj.dialAddTls(ctx.option, ctx.request, rawNetConn)
rawConn, h2, err = obj.dialAddTls(ctx.option, ctx.request, rawNetConn)
if ctx.option.Logger != nil {
ctx.option.Logger(Log{
Id: ctx.requestId,
@@ -263,37 +251,32 @@ func (obj *roundTripper) dial(ctx *Response) (conn *connecotr, err error) {
return nil, err
}
} else {
conne.c = rawNetConn
rawConn = rawNetConn
}
if arch != nil {
conne.c, err = NewCompressionConn(conne.c, arch)
rawConn, err = NewCompressionConn(rawConn, arch)
if err != nil {
return nil, err
}
}
err = obj.dialConnecotr(ctx, conne, h2)
if err != nil {
return nil, err
}
return conne, err
return obj.dialConnecotr(ctx, rawConn, h2)
}
func (obj *roundTripper) dialConnecotr(ctx *Response, conne *connecotr, h2 bool) (err error) {
func (obj *roundTripper) dialConnecotr(ctx *Response, rawCon net.Conn, h2 bool) (conn http1.Conn, err error) {
cctx, ccnl := context.WithCancelCause(obj.ctx)
if h2 {
var spec *http2.Spec
if ctx.option.gospiderSpec != nil {
spec = ctx.option.gospiderSpec.H2Spec
}
if conne.Conn, err = http2.NewClientConn(conne.forceCtx, ctx.Context(), conne.c, spec, func(err error) {
conne.forceCnl(tools.WrapError(err, "http2 client close"))
}); err != nil {
return err
}
conn, err = http2.NewClientConn(cctx, ctx.Context(), rawCon, spec, func(err error) {
ccnl(tools.WrapError(err, "http2 client close"))
})
} else {
conne.Conn = http1.NewClientConn(conne.forceCtx, conne.c, func(err error) {
conne.forceCnl(tools.WrapError(err, "http1 client close"))
conn = http1.NewClientConn(cctx, rawCon, func(err error) {
ccnl(tools.WrapError(err, "http1 client close"))
})
}
return err
return
}
func (obj *roundTripper) dialAddTls(option *RequestOption, req *http.Request, netConn net.Conn) (net.Conn, bool, error) {
ctx, cnl := context.WithTimeout(req.Context(), option.TlsHandshakeTimeout)
@@ -340,22 +323,25 @@ func (obj *roundTripper) initProxys(ctx *Response) ([]Address, error) {
return proxys, nil
}
func (obj *roundTripper) waitTask(task *reqTask) error {
<-task.ctx.Done()
err := context.Cause(task.ctx)
if errors.Is(err, tools.ErrNoErr) {
err = nil
}
return err
}
func (obj *roundTripper) poolRoundTrip(task *reqTask) error {
task.ctx, task.cnl = context.WithCancelCause(task.reqCtx.Context())
select {
case obj.getConnPool(task) <- task:
<-task.ctx.Done()
err := context.Cause(task.ctx)
if errors.Is(err, tools.ErrNoErr) {
err = nil
}
return err
return obj.waitTask(task)
default:
return obj.newRoudTrip(task)
return obj.newRoundTrip(task)
}
}
func (obj *roundTripper) newRoudTrip(task *reqTask) error {
func (obj *roundTripper) newRoundTrip(task *reqTask) error {
task.reqCtx.isNewConn = true
conn, err := obj.dial(task.reqCtx)
if err != nil {
@@ -369,8 +355,8 @@ func (obj *roundTripper) newRoudTrip(task *reqTask) error {
task.enableRetry = true
}
if err == nil {
obj.putConnPool(task, conn)
err = obj.poolRoundTrip(task)
go rwMain(conn, task, obj.getConnPool(task))
return obj.waitTask(task)
}
return err
}

2
test/addType_test.go
View File

@@ -9,7 +9,7 @@ import (
func TestAddType(t *testing.T) {
session, _ := requests.NewClient(nil, requests.ClientOption{
DialOption: requests.DialOption{
DialOption: &requests.DialOption{
AddrType: gtls.Ipv4,
},
})

12
test/cookies/setCookies_test.go
View File

@@ -16,13 +16,11 @@ func TestSetCookies(t *testing.T) {
log.Panic(err)
}
_, err = session.Get(context.TODO(), "https://www.baidu.com", requests.RequestOption{
ClientOption: requests.ClientOption{
RequestCallBack: func(ctx *requests.Response) error {
if ctx.Request().Cookies() == nil {
log.Panic("cookie is nil")
}
return nil
},
RequestCallBack: func(ctx *requests.Response) error {
if ctx.Request().Cookies() == nil {
log.Panic("cookie is nil")
}
return nil
},
})
if err != nil {

10
test/dns_test.go
View File

@@ -9,12 +9,10 @@ import (
func TestDns(t *testing.T) {
resp, err := requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{
ClientOption: requests.ClientOption{
DialOption: requests.DialOption{
Dns: &net.UDPAddr{ //set dns server
IP: net.ParseIP("223.5.5.5"),
Port: 53,
},
DialOption: &requests.DialOption{
Dns: &net.UDPAddr{ //set dns server
IP: net.ParseIP("223.5.5.5"),
Port: 53,
},
},
})

4
test/fingerprint/http2_test.go
View File

@@ -13,9 +13,7 @@ func TestH2(t *testing.T) {
// log.Print(h2Spec)
resp, err := requests.Get(nil, "https://tools.scrapfly.io/api/fp/anything", requests.RequestOption{
ClientOption: requests.ClientOption{
Spec: "16030106f2010006ee03039a2b98d81139db0e128ea09eff6874549c219b543fb6dbaa7e4dbfe9e31602c620ce04c4026f019442affade7fed8ba66e022e186f77f1c670fd992f33c0143f120020aaaa130113021303c02bc02fc02cc030cca9cca8c013c014009c009d002f0035010006851a1a00000010000e000c02683208687474702f312e31002b000706dada03040303002d00020101000d0012001004030804040105030805050108060601001b0003020002ff0100010000230000000a000c000afafa11ec001d001700180000000e000c0000096c6f63616c686f7374003304ef04edfafa00010011ec04c06903195f3660633741f9f5ae64d05a316ac8e717582adb9e58c5c242ba306c99ca8f68c15f261245f9141812383c9265f7d0b5c44a5e0d7633f4f40ab8e820ec01bb6cc74e6ab3168e66b40cc4cd37e96e286b4080552b8b0217f786b7c1a0088fb613cc84471b17a33fbcc68db151df387907a1cf3fb14a0f45c6b84608db5b103131b537255c09559cf1940c3980a7f37959a7f95d32c49923600c76c616af238c579361e1c6a20c251d3d42a50e182ad54b1e5d54a57fe6986e64142815b9478cca8066c9bdcc0eb9022ebe05b0ebd53e7d146761d81aee41cd377611699c536a444d300c152994bfeef3cfb4736cb3d57d683269c1e3c001fba2ac220b2c993cec410f6fa5104d1bcde21c46a9b0be8ab7b51aba15f1745ebbd0a0d3a5224170b3bce456c157937e43390b733375bb96601667f5b36888f9520931bead48bae4723d9ed40af2680746e27eee4328503a280ee8846b37803d6206e0f5248bea4ca4a53ca4e1afdd2b84bce0c83260333bc9b38b86486fb48e18d1ce9187b1b6332b2f4145eca38122ac363210137f52140a57b7976b609d739844fa61f21c1e3c300f3434bc3f8b6856994847e2c9b0f20a24b976f9d552b153246db69f30d4a95301b933b0ba9d48402ddb7863cb4f1923a2c33021fd68634a387bf0f76d87f01b35b6182dd10fe9c14b8e548d7988388308b08ff1585ba18a7615737857a7c23c24ee9b3a2ab1915be18b233acd354c7c6513b8ea617a5cf299f34139756cc1df524292c43ee3364990961b2490ae204634a4461b53c95a11d214503985f27ab85bc7c179ab1ba37a828312cabea8cfc5088616386a83e566279a0a5517b60aca4ec6c30a32191dca3cbb7d33ae5087bbdbab5c42e6293b63ad8e35311d459e1ce57037e65e96283e449c3e012051d247653197834c42613ac377a950607c98cbd5a79fef948d18e99758e12d31d13231e638cfb183623346b231a443f56533d444c7204a63479e4efb34ef97597d858915a8a10a32aa78824c9d2993741176da643be1c6c4d91b6511055b098477e7a3c5b5c312cf7bb4ef5905fd741375e62c8ce942a2117bbe707fcc9871e59c0687507862f3634c871885949fce97612793a30a155e84ac503dc519816a13772c50e1167a7031cb2c8187913108f9a26e55958fd19a6e1ef18ed53a70f1c13d01d71d3b40a22413852c9982daac4ae8071966016c38a60bd5c0258c32b882740c6ed5252093c91e51c50cf037d4f5cb6ca610672710f7ca77b0a76039a9968e368a6b243ee4ca7632855cb568c73f01764c4944fc5879d2c52d7992840863c057db2efec658eeb2a73e02bd62617438d9192911bac1f6b0e55cb38255417af20000d69378c857bb278156f16a684200125906b6c22f3d505bc9e76d75fac3a009332ff98fe6baabe3941cab5271c6d2c0ebc993b944c49bd437353019d1b24d10390e45fa87ad77b329a9025933a11af2af0da44d3ed761722c94d8053242f537624113d7bc0155600573301bd2217c6c481ce63b0944b052c97bcb9d3349258257ff33cccf963a6945119ecab21c25051ce02548f642e0ec1ffd392d60facfdf76bfb7274363b62979231f4996362c85d5ba19d2cab7019750b3443565436867a53b71d875eba3282e6d0ee22076d6b97b7c6c556ae216e8bc1bc9f202ce94c763bfe9afc105fca9372dec2e286a001d00200ee8ac33f1ea3153f6b4a06ab71d21b7ce7955ce64ccfc66b7ec8077d02ffd18fe0d00fa00000100018b00206cada2aa48ee4478c40adad21f147d6bc90d13f6889a9b8a58a02536585a261f00d09306c85aab2a6e424b658f3cd9d1c46f35020839287259d3be605ff97faea0d87b9f7f96529661f08cf3f3899db8e805ee7405e2f9b6abd99bc4f6fa5f99b1ed442ebe53c5b10451c93d1221f662783efc3cc8fcf135ed935bcf02ec32251dd09705f191bd7959afbab5619d8e63cb634a259dd63d1b0e42225ae8c08b5b1620cd59d914857e9f1e8a3b7b892863bdaa05429922d75583059641468d8fc51c01e977a69d3a51d714cd5cceea9a5f404ce4a285fd6647931ed8b1c12db027328f214afdbe2c8102b46fe041b553f8670b00050005010000000044cd0005000302683200170000000b0002010000120000caca000100@@505249202a20485454502f322e300d0a0d0a534d0d0a0d0a00001804000000000000010001000000020000000000040060000000060004000000000408000000000000ef00010001d401250000000180000000ff82418aa0e41d139d09b8f3efbf87845887a47e561cc5801f40874148b1275ad1ffb9fe749d3fd4372ed83aa4fe7efbc1fcbefff3f4a7f388e79a82a97a7b0f497f9fbef07f21659fe7e94fe6f4f61e935b4ff3f7de0fe42cb3fcff408b4148b1275ad1ad49e33505023f30408d4148b1275ad1ad5d034ca7b29f07226d61634f53224092b6b9ac1c8558d520a4b6c2ad617b5a54251f01317ad9d07f66a281b0dae053fad0321aa49d13fda992a49685340c8a6adca7e28104416e277fb521aeba0bc8b1e632586d975765c53facd8f7e8cff4a506ea5531149d4ffda97a7b0f49580b2cae05c0b814dc394761986d975765cf53e5497ca589d34d1f43aeba0c41a4c7a98f33a69a3fdf9a68fa1d75d0620d263d4c79a68fbed00177fe8d48e62b03ee697e8d48e62b1e0b1d7f46a4731581d754df5f2c7cfdf6800bbdf43aeba0c41a4c7a9841a6a8b22c5f249c754c5fbef046cfdf6800bbbf408a4148b4a549275906497f83a8f517408a4148b4a549275a93c85f86a87dcd30d25f408a4148b4a549275ad416cf023f31408a4148b4a549275a42a13f8690e4b692d49f50929bd9abfa5242cb40d25fa523b3e94f684c9f518cf73ad7b4fd7b9fefb4005dff4086aec31ec327d785b6007d286f",
},
Spec: "16030106f2010006ee03039a2b98d81139db0e128ea09eff6874549c219b543fb6dbaa7e4dbfe9e31602c620ce04c4026f019442affade7fed8ba66e022e186f77f1c670fd992f33c0143f120020aaaa130113021303c02bc02fc02cc030cca9cca8c013c014009c009d002f0035010006851a1a00000010000e000c02683208687474702f312e31002b000706dada03040303002d00020101000d0012001004030804040105030805050108060601001b0003020002ff0100010000230000000a000c000afafa11ec001d001700180000000e000c0000096c6f63616c686f7374003304ef04edfafa00010011ec04c06903195f3660633741f9f5ae64d05a316ac8e717582adb9e58c5c242ba306c99ca8f68c15f261245f9141812383c9265f7d0b5c44a5e0d7633f4f40ab8e820ec01bb6cc74e6ab3168e66b40cc4cd37e96e286b4080552b8b0217f786b7c1a0088fb613cc84471b17a33fbcc68db151df387907a1cf3fb14a0f45c6b84608db5b103131b537255c09559cf1940c3980a7f37959a7f95d32c49923600c76c616af238c579361e1c6a20c251d3d42a50e182ad54b1e5d54a57fe6986e64142815b9478cca8066c9bdcc0eb9022ebe05b0ebd53e7d146761d81aee41cd377611699c536a444d300c152994bfeef3cfb4736cb3d57d683269c1e3c001fba2ac220b2c993cec410f6fa5104d1bcde21c46a9b0be8ab7b51aba15f1745ebbd0a0d3a5224170b3bce456c157937e43390b733375bb96601667f5b36888f9520931bead48bae4723d9ed40af2680746e27eee4328503a280ee8846b37803d6206e0f5248bea4ca4a53ca4e1afdd2b84bce0c83260333bc9b38b86486fb48e18d1ce9187b1b6332b2f4145eca38122ac363210137f52140a57b7976b609d739844fa61f21c1e3c300f3434bc3f8b6856994847e2c9b0f20a24b976f9d552b153246db69f30d4a95301b933b0ba9d48402ddb7863cb4f1923a2c33021fd68634a387bf0f76d87f01b35b6182dd10fe9c14b8e548d7988388308b08ff1585ba18a7615737857a7c23c24ee9b3a2ab1915be18b233acd354c7c6513b8ea617a5cf299f34139756cc1df524292c43ee3364990961b2490ae204634a4461b53c95a11d214503985f27ab85bc7c179ab1ba37a828312cabea8cfc5088616386a83e566279a0a5517b60aca4ec6c30a32191dca3cbb7d33ae5087bbdbab5c42e6293b63ad8e35311d459e1ce57037e65e96283e449c3e012051d247653197834c42613ac377a950607c98cbd5a79fef948d18e99758e12d31d13231e638cfb183623346b231a443f56533d444c7204a63479e4efb34ef97597d858915a8a10a32aa78824c9d2993741176da643be1c6c4d91b6511055b098477e7a3c5b5c312cf7bb4ef5905fd741375e62c8ce942a2117bbe707fcc9871e59c0687507862f3634c871885949fce97612793a30a155e84ac503dc519816a13772c50e1167a7031cb2c8187913108f9a26e55958fd19a6e1ef18ed53a70f1c13d01d71d3b40a22413852c9982daac4ae8071966016c38a60bd5c0258c32b882740c6ed5252093c91e51c50cf037d4f5cb6ca610672710f7ca77b0a76039a9968e368a6b243ee4ca7632855cb568c73f01764c4944fc5879d2c52d7992840863c057db2efec658eeb2a73e02bd62617438d9192911bac1f6b0e55cb38255417af20000d69378c857bb278156f16a684200125906b6c22f3d505bc9e76d75fac3a009332ff98fe6baabe3941cab5271c6d2c0ebc993b944c49bd437353019d1b24d10390e45fa87ad77b329a9025933a11af2af0da44d3ed761722c94d8053242f537624113d7bc0155600573301bd2217c6c481ce63b0944b052c97bcb9d3349258257ff33cccf963a6945119ecab21c25051ce02548f642e0ec1ffd392d60facfdf76bfb7274363b62979231f4996362c85d5ba19d2cab7019750b3443565436867a53b71d875eba3282e6d0ee22076d6b97b7c6c556ae216e8bc1bc9f202ce94c763bfe9afc105fca9372dec2e286a001d00200ee8ac33f1ea3153f6b4a06ab71d21b7ce7955ce64ccfc66b7ec8077d02ffd18fe0d00fa00000100018b00206cada2aa48ee4478c40adad21f147d6bc90d13f6889a9b8a58a02536585a261f00d09306c85aab2a6e424b658f3cd9d1c46f35020839287259d3be605ff97faea0d87b9f7f96529661f08cf3f3899db8e805ee7405e2f9b6abd99bc4f6fa5f99b1ed442ebe53c5b10451c93d1221f662783efc3cc8fcf135ed935bcf02ec32251dd09705f191bd7959afbab5619d8e63cb634a259dd63d1b0e42225ae8c08b5b1620cd59d914857e9f1e8a3b7b892863bdaa05429922d75583059641468d8fc51c01e977a69d3a51d714cd5cceea9a5f404ce4a285fd6647931ed8b1c12db027328f214afdbe2c8102b46fe041b553f8670b00050005010000000044cd0005000302683200170000000b0002010000120000caca000100@@505249202a20485454502f322e300d0a0d0a534d0d0a0d0a00001804000000000000010001000000020000000000040060000000060004000000000408000000000000ef00010001d401250000000180000000ff82418aa0e41d139d09b8f3efbf87845887a47e561cc5801f40874148b1275ad1ffb9fe749d3fd4372ed83aa4fe7efbc1fcbefff3f4a7f388e79a82a97a7b0f497f9fbef07f21659fe7e94fe6f4f61e935b4ff3f7de0fe42cb3fcff408b4148b1275ad1ad49e33505023f30408d4148b1275ad1ad5d034ca7b29f07226d61634f53224092b6b9ac1c8558d520a4b6c2ad617b5a54251f01317ad9d07f66a281b0dae053fad0321aa49d13fda992a49685340c8a6adca7e28104416e277fb521aeba0bc8b1e632586d975765c53facd8f7e8cff4a506ea5531149d4ffda97a7b0f49580b2cae05c0b814dc394761986d975765cf53e5497ca589d34d1f43aeba0c41a4c7a98f33a69a3fdf9a68fa1d75d0620d263d4c79a68fbed00177fe8d48e62b03ee697e8d48e62b1e0b1d7f46a4731581d754df5f2c7cfdf6800bbdf43aeba0c41a4c7a9841a6a8b22c5f249c754c5fbef046cfdf6800bbbf408a4148b4a549275906497f83a8f517408a4148b4a549275a93c85f86a87dcd30d25f408a4148b4a549275ad416cf023f31408a4148b4a549275a42a13f8690e4b692d49f50929bd9abfa5242cb40d25fa523b3e94f684c9f518cf73ad7b4fd7b9fefb4005dff4086aec31ec327d785b6007d286f",
})
if err != nil {
t.Fatal(err)

4
test/fingerprint/ja3_test.go
View File

@@ -10,9 +10,7 @@ import (
func TestJa3(t *testing.T) {
// j := "772,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,13-45-5-35-18-23-0-65281-10-65037-51-16-11-27-43-17513,12092-29-23-24,0"
resp, err := requests.Get(nil, "https://tools.scrapfly.io/api/fp/anything", requests.RequestOption{
ClientOption: requests.ClientOption{
Spec: "16030106f2010006ee03039a2b98d81139db0e128ea09eff6874549c219b543fb6dbaa7e4dbfe9e31602c620ce04c4026f019442affade7fed8ba66e022e186f77f1c670fd992f33c0143f120020aaaa130113021303c02bc02fc02cc030cca9cca8c013c014009c009d002f0035010006851a1a00000010000e000c02683208687474702f312e31002b000706dada03040303002d00020101000d0012001004030804040105030805050108060601001b0003020002ff0100010000230000000a000c000afafa11ec001d001700180000000e000c0000096c6f63616c686f7374003304ef04edfafa00010011ec04c06903195f3660633741f9f5ae64d05a316ac8e717582adb9e58c5c242ba306c99ca8f68c15f261245f9141812383c9265f7d0b5c44a5e0d7633f4f40ab8e820ec01bb6cc74e6ab3168e66b40cc4cd37e96e286b4080552b8b0217f786b7c1a0088fb613cc84471b17a33fbcc68db151df387907a1cf3fb14a0f45c6b84608db5b103131b537255c09559cf1940c3980a7f37959a7f95d32c49923600c76c616af238c579361e1c6a20c251d3d42a50e182ad54b1e5d54a57fe6986e64142815b9478cca8066c9bdcc0eb9022ebe05b0ebd53e7d146761d81aee41cd377611699c536a444d300c152994bfeef3cfb4736cb3d57d683269c1e3c001fba2ac220b2c993cec410f6fa5104d1bcde21c46a9b0be8ab7b51aba15f1745ebbd0a0d3a5224170b3bce456c157937e43390b733375bb96601667f5b36888f9520931bead48bae4723d9ed40af2680746e27eee4328503a280ee8846b37803d6206e0f5248bea4ca4a53ca4e1afdd2b84bce0c83260333bc9b38b86486fb48e18d1ce9187b1b6332b2f4145eca38122ac363210137f52140a57b7976b609d739844fa61f21c1e3c300f3434bc3f8b6856994847e2c9b0f20a24b976f9d552b153246db69f30d4a95301b933b0ba9d48402ddb7863cb4f1923a2c33021fd68634a387bf0f76d87f01b35b6182dd10fe9c14b8e548d7988388308b08ff1585ba18a7615737857a7c23c24ee9b3a2ab1915be18b233acd354c7c6513b8ea617a5cf299f34139756cc1df524292c43ee3364990961b2490ae204634a4461b53c95a11d214503985f27ab85bc7c179ab1ba37a828312cabea8cfc5088616386a83e566279a0a5517b60aca4ec6c30a32191dca3cbb7d33ae5087bbdbab5c42e6293b63ad8e35311d459e1ce57037e65e96283e449c3e012051d247653197834c42613ac377a950607c98cbd5a79fef948d18e99758e12d31d13231e638cfb183623346b231a443f56533d444c7204a63479e4efb34ef97597d858915a8a10a32aa78824c9d2993741176da643be1c6c4d91b6511055b098477e7a3c5b5c312cf7bb4ef5905fd741375e62c8ce942a2117bbe707fcc9871e59c0687507862f3634c871885949fce97612793a30a155e84ac503dc519816a13772c50e1167a7031cb2c8187913108f9a26e55958fd19a6e1ef18ed53a70f1c13d01d71d3b40a22413852c9982daac4ae8071966016c38a60bd5c0258c32b882740c6ed5252093c91e51c50cf037d4f5cb6ca610672710f7ca77b0a76039a9968e368a6b243ee4ca7632855cb568c73f01764c4944fc5879d2c52d7992840863c057db2efec658eeb2a73e02bd62617438d9192911bac1f6b0e55cb38255417af20000d69378c857bb278156f16a684200125906b6c22f3d505bc9e76d75fac3a009332ff98fe6baabe3941cab5271c6d2c0ebc993b944c49bd437353019d1b24d10390e45fa87ad77b329a9025933a11af2af0da44d3ed761722c94d8053242f537624113d7bc0155600573301bd2217c6c481ce63b0944b052c97bcb9d3349258257ff33cccf963a6945119ecab21c25051ce02548f642e0ec1ffd392d60facfdf76bfb7274363b62979231f4996362c85d5ba19d2cab7019750b3443565436867a53b71d875eba3282e6d0ee22076d6b97b7c6c556ae216e8bc1bc9f202ce94c763bfe9afc105fca9372dec2e286a001d00200ee8ac33f1ea3153f6b4a06ab71d21b7ce7955ce64ccfc66b7ec8077d02ffd18fe0d00fa00000100018b00206cada2aa48ee4478c40adad21f147d6bc90d13f6889a9b8a58a02536585a261f00d09306c85aab2a6e424b658f3cd9d1c46f35020839287259d3be605ff97faea0d87b9f7f96529661f08cf3f3899db8e805ee7405e2f9b6abd99bc4f6fa5f99b1ed442ebe53c5b10451c93d1221f662783efc3cc8fcf135ed935bcf02ec32251dd09705f191bd7959afbab5619d8e63cb634a259dd63d1b0e42225ae8c08b5b1620cd59d914857e9f1e8a3b7b892863bdaa05429922d75583059641468d8fc51c01e977a69d3a51d714cd5cceea9a5f404ce4a285fd6647931ed8b1c12db027328f214afdbe2c8102b46fe041b553f8670b00050005010000000044cd0005000302683200170000000b0002010000120000caca000100@@505249202a20485454502f322e300d0a0d0a534d0d0a0d0a00001804000000000000010001000000020000000000040060000000060004000000000408000000000000ef00010001d401250000000180000000ff82418aa0e41d139d09b8f3efbf87845887a47e561cc5801f40874148b1275ad1ffb9fe749d3fd4372ed83aa4fe7efbc1fcbefff3f4a7f388e79a82a97a7b0f497f9fbef07f21659fe7e94fe6f4f61e935b4ff3f7de0fe42cb3fcff408b4148b1275ad1ad49e33505023f30408d4148b1275ad1ad5d034ca7b29f07226d61634f53224092b6b9ac1c8558d520a4b6c2ad617b5a54251f01317ad9d07f66a281b0dae053fad0321aa49d13fda992a49685340c8a6adca7e28104416e277fb521aeba0bc8b1e632586d975765c53facd8f7e8cff4a506ea5531149d4ffda97a7b0f49580b2cae05c0b814dc394761986d975765cf53e5497ca589d34d1f43aeba0c41a4c7a98f33a69a3fdf9a68fa1d75d0620d263d4c79a68fbed00177fe8d48e62b03ee697e8d48e62b1e0b1d7f46a4731581d754df5f2c7cfdf6800bbdf43aeba0c41a4c7a9841a6a8b22c5f249c754c5fbef046cfdf6800bbbf408a4148b4a549275906497f83a8f517408a4148b4a549275a93c85f86a87dcd30d25f408a4148b4a549275ad416cf023f31408a4148b4a549275a42a13f8690e4b692d49f50929bd9abfa5242cb40d25fa523b3e94f684c9f518cf73ad7b4fd7b9fefb4005dff4086aec31ec327d785b6007d286f",
},
Spec: "16030106f2010006ee03039a2b98d81139db0e128ea09eff6874549c219b543fb6dbaa7e4dbfe9e31602c620ce04c4026f019442affade7fed8ba66e022e186f77f1c670fd992f33c0143f120020aaaa130113021303c02bc02fc02cc030cca9cca8c013c014009c009d002f0035010006851a1a00000010000e000c02683208687474702f312e31002b000706dada03040303002d00020101000d0012001004030804040105030805050108060601001b0003020002ff0100010000230000000a000c000afafa11ec001d001700180000000e000c0000096c6f63616c686f7374003304ef04edfafa00010011ec04c06903195f3660633741f9f5ae64d05a316ac8e717582adb9e58c5c242ba306c99ca8f68c15f261245f9141812383c9265f7d0b5c44a5e0d7633f4f40ab8e820ec01bb6cc74e6ab3168e66b40cc4cd37e96e286b4080552b8b0217f786b7c1a0088fb613cc84471b17a33fbcc68db151df387907a1cf3fb14a0f45c6b84608db5b103131b537255c09559cf1940c3980a7f37959a7f95d32c49923600c76c616af238c579361e1c6a20c251d3d42a50e182ad54b1e5d54a57fe6986e64142815b9478cca8066c9bdcc0eb9022ebe05b0ebd53e7d146761d81aee41cd377611699c536a444d300c152994bfeef3cfb4736cb3d57d683269c1e3c001fba2ac220b2c993cec410f6fa5104d1bcde21c46a9b0be8ab7b51aba15f1745ebbd0a0d3a5224170b3bce456c157937e43390b733375bb96601667f5b36888f9520931bead48bae4723d9ed40af2680746e27eee4328503a280ee8846b37803d6206e0f5248bea4ca4a53ca4e1afdd2b84bce0c83260333bc9b38b86486fb48e18d1ce9187b1b6332b2f4145eca38122ac363210137f52140a57b7976b609d739844fa61f21c1e3c300f3434bc3f8b6856994847e2c9b0f20a24b976f9d552b153246db69f30d4a95301b933b0ba9d48402ddb7863cb4f1923a2c33021fd68634a387bf0f76d87f01b35b6182dd10fe9c14b8e548d7988388308b08ff1585ba18a7615737857a7c23c24ee9b3a2ab1915be18b233acd354c7c6513b8ea617a5cf299f34139756cc1df524292c43ee3364990961b2490ae204634a4461b53c95a11d214503985f27ab85bc7c179ab1ba37a828312cabea8cfc5088616386a83e566279a0a5517b60aca4ec6c30a32191dca3cbb7d33ae5087bbdbab5c42e6293b63ad8e35311d459e1ce57037e65e96283e449c3e012051d247653197834c42613ac377a950607c98cbd5a79fef948d18e99758e12d31d13231e638cfb183623346b231a443f56533d444c7204a63479e4efb34ef97597d858915a8a10a32aa78824c9d2993741176da643be1c6c4d91b6511055b098477e7a3c5b5c312cf7bb4ef5905fd741375e62c8ce942a2117bbe707fcc9871e59c0687507862f3634c871885949fce97612793a30a155e84ac503dc519816a13772c50e1167a7031cb2c8187913108f9a26e55958fd19a6e1ef18ed53a70f1c13d01d71d3b40a22413852c9982daac4ae8071966016c38a60bd5c0258c32b882740c6ed5252093c91e51c50cf037d4f5cb6ca610672710f7ca77b0a76039a9968e368a6b243ee4ca7632855cb568c73f01764c4944fc5879d2c52d7992840863c057db2efec658eeb2a73e02bd62617438d9192911bac1f6b0e55cb38255417af20000d69378c857bb278156f16a684200125906b6c22f3d505bc9e76d75fac3a009332ff98fe6baabe3941cab5271c6d2c0ebc993b944c49bd437353019d1b24d10390e45fa87ad77b329a9025933a11af2af0da44d3ed761722c94d8053242f537624113d7bc0155600573301bd2217c6c481ce63b0944b052c97bcb9d3349258257ff33cccf963a6945119ecab21c25051ce02548f642e0ec1ffd392d60facfdf76bfb7274363b62979231f4996362c85d5ba19d2cab7019750b3443565436867a53b71d875eba3282e6d0ee22076d6b97b7c6c556ae216e8bc1bc9f202ce94c763bfe9afc105fca9372dec2e286a001d00200ee8ac33f1ea3153f6b4a06ab71d21b7ce7955ce64ccfc66b7ec8077d02ffd18fe0d00fa00000100018b00206cada2aa48ee4478c40adad21f147d6bc90d13f6889a9b8a58a02536585a261f00d09306c85aab2a6e424b658f3cd9d1c46f35020839287259d3be605ff97faea0d87b9f7f96529661f08cf3f3899db8e805ee7405e2f9b6abd99bc4f6fa5f99b1ed442ebe53c5b10451c93d1221f662783efc3cc8fcf135ed935bcf02ec32251dd09705f191bd7959afbab5619d8e63cb634a259dd63d1b0e42225ae8c08b5b1620cd59d914857e9f1e8a3b7b892863bdaa05429922d75583059641468d8fc51c01e977a69d3a51d714cd5cceea9a5f404ce4a285fd6647931ed8b1c12db027328f214afdbe2c8102b46fe041b553f8670b00050005010000000044cd0005000302683200170000000b0002010000120000caca000100@@505249202a20485454502f322e300d0a0d0a534d0d0a0d0a00001804000000000000010001000000020000000000040060000000060004000000000408000000000000ef00010001d401250000000180000000ff82418aa0e41d139d09b8f3efbf87845887a47e561cc5801f40874148b1275ad1ffb9fe749d3fd4372ed83aa4fe7efbc1fcbefff3f4a7f388e79a82a97a7b0f497f9fbef07f21659fe7e94fe6f4f61e935b4ff3f7de0fe42cb3fcff408b4148b1275ad1ad49e33505023f30408d4148b1275ad1ad5d034ca7b29f07226d61634f53224092b6b9ac1c8558d520a4b6c2ad617b5a54251f01317ad9d07f66a281b0dae053fad0321aa49d13fda992a49685340c8a6adca7e28104416e277fb521aeba0bc8b1e632586d975765c53facd8f7e8cff4a506ea5531149d4ffda97a7b0f49580b2cae05c0b814dc394761986d975765cf53e5497ca589d34d1f43aeba0c41a4c7a98f33a69a3fdf9a68fa1d75d0620d263d4c79a68fbed00177fe8d48e62b03ee697e8d48e62b1e0b1d7f46a4731581d754df5f2c7cfdf6800bbdf43aeba0c41a4c7a9841a6a8b22c5f249c754c5fbef046cfdf6800bbbf408a4148b4a549275906497f83a8f517408a4148b4a549275a93c85f86a87dcd30d25f408a4148b4a549275ad416cf023f31408a4148b4a549275a42a13f8690e4b692d49f50929bd9abfa5242cb40d25fa523b3e94f684c9f518cf73ad7b4fd7b9fefb4005dff4086aec31ec327d785b6007d286f",
})
if err != nil {
t.Fatal(err)

5
test/fingerprint/ja4_test.go
View File

@@ -17,10 +17,7 @@ func TestOrderHeaders(t *testing.T) {
}
resp, err := requests.Get(nil, "https://tools.scrapfly.io/api/fp/anything", requests.RequestOption{
ClientOption: requests.ClientOption{
OrderHeaders: orderKeys,
// Headers: headers,
},
OrderHeaders: orderKeys,
// ForceHttp1: true,
})
if err != nil {

11
test/fingerprint/quic_test.go
View File

@@ -10,10 +10,8 @@ import (
func TestHttp3(t *testing.T) {
resp, err := requests.Get(context.TODO(), "https://cloudflare-quic.com/", requests.RequestOption{
ClientOption: requests.ClientOption{
ForceHttp3: true,
Spec: "16030106f2010006ee03039a2b98d81139db0e128ea09eff6874549c219b543fb6dbaa7e4dbfe9e31602c620ce04c4026f019442affade7fed8ba66e022e186f77f1c670fd992f33c0143f120020aaaa130113021303c02bc02fc02cc030cca9cca8c013c014009c009d002f0035010006851a1a00000010000e000c02683208687474702f312e31002b000706dada03040303002d00020101000d0012001004030804040105030805050108060601001b0003020002ff0100010000230000000a000c000afafa11ec001d001700180000000e000c0000096c6f63616c686f7374003304ef04edfafa00010011ec04c06903195f3660633741f9f5ae64d05a316ac8e717582adb9e58c5c242ba306c99ca8f68c15f261245f9141812383c9265f7d0b5c44a5e0d7633f4f40ab8e820ec01bb6cc74e6ab3168e66b40cc4cd37e96e286b4080552b8b0217f786b7c1a0088fb613cc84471b17a33fbcc68db151df387907a1cf3fb14a0f45c6b84608db5b103131b537255c09559cf1940c3980a7f37959a7f95d32c49923600c76c616af238c579361e1c6a20c251d3d42a50e182ad54b1e5d54a57fe6986e64142815b9478cca8066c9bdcc0eb9022ebe05b0ebd53e7d146761d81aee41cd377611699c536a444d300c152994bfeef3cfb4736cb3d57d683269c1e3c001fba2ac220b2c993cec410f6fa5104d1bcde21c46a9b0be8ab7b51aba15f1745ebbd0a0d3a5224170b3bce456c157937e43390b733375bb96601667f5b36888f9520931bead48bae4723d9ed40af2680746e27eee4328503a280ee8846b37803d6206e0f5248bea4ca4a53ca4e1afdd2b84bce0c83260333bc9b38b86486fb48e18d1ce9187b1b6332b2f4145eca38122ac363210137f52140a57b7976b609d739844fa61f21c1e3c300f3434bc3f8b6856994847e2c9b0f20a24b976f9d552b153246db69f30d4a95301b933b0ba9d48402ddb7863cb4f1923a2c33021fd68634a387bf0f76d87f01b35b6182dd10fe9c14b8e548d7988388308b08ff1585ba18a7615737857a7c23c24ee9b3a2ab1915be18b233acd354c7c6513b8ea617a5cf299f34139756cc1df524292c43ee3364990961b2490ae204634a4461b53c95a11d214503985f27ab85bc7c179ab1ba37a828312cabea8cfc5088616386a83e566279a0a5517b60aca4ec6c30a32191dca3cbb7d33ae5087bbdbab5c42e6293b63ad8e35311d459e1ce57037e65e96283e449c3e012051d247653197834c42613ac377a950607c98cbd5a79fef948d18e99758e12d31d13231e638cfb183623346b231a443f56533d444c7204a63479e4efb34ef97597d858915a8a10a32aa78824c9d2993741176da643be1c6c4d91b6511055b098477e7a3c5b5c312cf7bb4ef5905fd741375e62c8ce942a2117bbe707fcc9871e59c0687507862f3634c871885949fce97612793a30a155e84ac503dc519816a13772c50e1167a7031cb2c8187913108f9a26e55958fd19a6e1ef18ed53a70f1c13d01d71d3b40a22413852c9982daac4ae8071966016c38a60bd5c0258c32b882740c6ed5252093c91e51c50cf037d4f5cb6ca610672710f7ca77b0a76039a9968e368a6b243ee4ca7632855cb568c73f01764c4944fc5879d2c52d7992840863c057db2efec658eeb2a73e02bd62617438d9192911bac1f6b0e55cb38255417af20000d69378c857bb278156f16a684200125906b6c22f3d505bc9e76d75fac3a009332ff98fe6baabe3941cab5271c6d2c0ebc993b944c49bd437353019d1b24d10390e45fa87ad77b329a9025933a11af2af0da44d3ed761722c94d8053242f537624113d7bc0155600573301bd2217c6c481ce63b0944b052c97bcb9d3349258257ff33cccf963a6945119ecab21c25051ce02548f642e0ec1ffd392d60facfdf76bfb7274363b62979231f4996362c85d5ba19d2cab7019750b3443565436867a53b71d875eba3282e6d0ee22076d6b97b7c6c556ae216e8bc1bc9f202ce94c763bfe9afc105fca9372dec2e286a001d00200ee8ac33f1ea3153f6b4a06ab71d21b7ce7955ce64ccfc66b7ec8077d02ffd18fe0d00fa00000100018b00206cada2aa48ee4478c40adad21f147d6bc90d13f6889a9b8a58a02536585a261f00d09306c85aab2a6e424b658f3cd9d1c46f35020839287259d3be605ff97faea0d87b9f7f96529661f08cf3f3899db8e805ee7405e2f9b6abd99bc4f6fa5f99b1ed442ebe53c5b10451c93d1221f662783efc3cc8fcf135ed935bcf02ec32251dd09705f191bd7959afbab5619d8e63cb634a259dd63d1b0e42225ae8c08b5b1620cd59d914857e9f1e8a3b7b892863bdaa05429922d75583059641468d8fc51c01e977a69d3a51d714cd5cceea9a5f404ce4a285fd6647931ed8b1c12db027328f214afdbe2c8102b46fe041b553f8670b00050005010000000044cd0005000302683200170000000b0002010000120000caca000100@@505249202a20485454502f322e300d0a0d0a534d0d0a0d0a00001804000000000000010001000000020000000000040060000000060004000000000408000000000000ef00010001d401250000000180000000ff82418aa0e41d139d09b8f3efbf87845887a47e561cc5801f40874148b1275ad1ffb9fe749d3fd4372ed83aa4fe7efbc1fcbefff3f4a7f388e79a82a97a7b0f497f9fbef07f21659fe7e94fe6f4f61e935b4ff3f7de0fe42cb3fcff408b4148b1275ad1ad49e33505023f30408d4148b1275ad1ad5d034ca7b29f07226d61634f53224092b6b9ac1c8558d520a4b6c2ad617b5a54251f01317ad9d07f66a281b0dae053fad0321aa49d13fda992a49685340c8a6adca7e28104416e277fb521aeba0bc8b1e632586d975765c53facd8f7e8cff4a506ea5531149d4ffda97a7b0f49580b2cae05c0b814dc394761986d975765cf53e5497ca589d34d1f43aeba0c41a4c7a98f33a69a3fdf9a68fa1d75d0620d263d4c79a68fbed00177fe8d48e62b03ee697e8d48e62b1e0b1d7f46a4731581d754df5f2c7cfdf6800bbdf43aeba0c41a4c7a9841a6a8b22c5f249c754c5fbef046cfdf6800bbbf408a4148b4a549275906497f83a8f517408a4148b4a549275a93c85f86a87dcd30d25f408a4148b4a549275ad416cf023f31408a4148b4a549275a42a13f8690e4b692d49f50929bd9abfa5242cb40d25fa523b3e94f684c9f518cf73ad7b4fd7b9fefb4005dff4086aec31ec327d785b6007d286f",
},
ForceHttp3: true,
Spec: "16030106f2010006ee03039a2b98d81139db0e128ea09eff6874549c219b543fb6dbaa7e4dbfe9e31602c620ce04c4026f019442affade7fed8ba66e022e186f77f1c670fd992f33c0143f120020aaaa130113021303c02bc02fc02cc030cca9cca8c013c014009c009d002f0035010006851a1a00000010000e000c02683208687474702f312e31002b000706dada03040303002d00020101000d0012001004030804040105030805050108060601001b0003020002ff0100010000230000000a000c000afafa11ec001d001700180000000e000c0000096c6f63616c686f7374003304ef04edfafa00010011ec04c06903195f3660633741f9f5ae64d05a316ac8e717582adb9e58c5c242ba306c99ca8f68c15f261245f9141812383c9265f7d0b5c44a5e0d7633f4f40ab8e820ec01bb6cc74e6ab3168e66b40cc4cd37e96e286b4080552b8b0217f786b7c1a0088fb613cc84471b17a33fbcc68db151df387907a1cf3fb14a0f45c6b84608db5b103131b537255c09559cf1940c3980a7f37959a7f95d32c49923600c76c616af238c579361e1c6a20c251d3d42a50e182ad54b1e5d54a57fe6986e64142815b9478cca8066c9bdcc0eb9022ebe05b0ebd53e7d146761d81aee41cd377611699c536a444d300c152994bfeef3cfb4736cb3d57d683269c1e3c001fba2ac220b2c993cec410f6fa5104d1bcde21c46a9b0be8ab7b51aba15f1745ebbd0a0d3a5224170b3bce456c157937e43390b733375bb96601667f5b36888f9520931bead48bae4723d9ed40af2680746e27eee4328503a280ee8846b37803d6206e0f5248bea4ca4a53ca4e1afdd2b84bce0c83260333bc9b38b86486fb48e18d1ce9187b1b6332b2f4145eca38122ac363210137f52140a57b7976b609d739844fa61f21c1e3c300f3434bc3f8b6856994847e2c9b0f20a24b976f9d552b153246db69f30d4a95301b933b0ba9d48402ddb7863cb4f1923a2c33021fd68634a387bf0f76d87f01b35b6182dd10fe9c14b8e548d7988388308b08ff1585ba18a7615737857a7c23c24ee9b3a2ab1915be18b233acd354c7c6513b8ea617a5cf299f34139756cc1df524292c43ee3364990961b2490ae204634a4461b53c95a11d214503985f27ab85bc7c179ab1ba37a828312cabea8cfc5088616386a83e566279a0a5517b60aca4ec6c30a32191dca3cbb7d33ae5087bbdbab5c42e6293b63ad8e35311d459e1ce57037e65e96283e449c3e012051d247653197834c42613ac377a950607c98cbd5a79fef948d18e99758e12d31d13231e638cfb183623346b231a443f56533d444c7204a63479e4efb34ef97597d858915a8a10a32aa78824c9d2993741176da643be1c6c4d91b6511055b098477e7a3c5b5c312cf7bb4ef5905fd741375e62c8ce942a2117bbe707fcc9871e59c0687507862f3634c871885949fce97612793a30a155e84ac503dc519816a13772c50e1167a7031cb2c8187913108f9a26e55958fd19a6e1ef18ed53a70f1c13d01d71d3b40a22413852c9982daac4ae8071966016c38a60bd5c0258c32b882740c6ed5252093c91e51c50cf037d4f5cb6ca610672710f7ca77b0a76039a9968e368a6b243ee4ca7632855cb568c73f01764c4944fc5879d2c52d7992840863c057db2efec658eeb2a73e02bd62617438d9192911bac1f6b0e55cb38255417af20000d69378c857bb278156f16a684200125906b6c22f3d505bc9e76d75fac3a009332ff98fe6baabe3941cab5271c6d2c0ebc993b944c49bd437353019d1b24d10390e45fa87ad77b329a9025933a11af2af0da44d3ed761722c94d8053242f537624113d7bc0155600573301bd2217c6c481ce63b0944b052c97bcb9d3349258257ff33cccf963a6945119ecab21c25051ce02548f642e0ec1ffd392d60facfdf76bfb7274363b62979231f4996362c85d5ba19d2cab7019750b3443565436867a53b71d875eba3282e6d0ee22076d6b97b7c6c556ae216e8bc1bc9f202ce94c763bfe9afc105fca9372dec2e286a001d00200ee8ac33f1ea3153f6b4a06ab71d21b7ce7955ce64ccfc66b7ec8077d02ffd18fe0d00fa00000100018b00206cada2aa48ee4478c40adad21f147d6bc90d13f6889a9b8a58a02536585a261f00d09306c85aab2a6e424b658f3cd9d1c46f35020839287259d3be605ff97faea0d87b9f7f96529661f08cf3f3899db8e805ee7405e2f9b6abd99bc4f6fa5f99b1ed442ebe53c5b10451c93d1221f662783efc3cc8fcf135ed935bcf02ec32251dd09705f191bd7959afbab5619d8e63cb634a259dd63d1b0e42225ae8c08b5b1620cd59d914857e9f1e8a3b7b892863bdaa05429922d75583059641468d8fc51c01e977a69d3a51d714cd5cceea9a5f404ce4a285fd6647931ed8b1c12db027328f214afdbe2c8102b46fe041b553f8670b00050005010000000044cd0005000302683200170000000b0002010000120000caca000100@@505249202a20485454502f322e300d0a0d0a534d0d0a0d0a00001804000000000000010001000000020000000000040060000000060004000000000408000000000000ef00010001d401250000000180000000ff82418aa0e41d139d09b8f3efbf87845887a47e561cc5801f40874148b1275ad1ffb9fe749d3fd4372ed83aa4fe7efbc1fcbefff3f4a7f388e79a82a97a7b0f497f9fbef07f21659fe7e94fe6f4f61e935b4ff3f7de0fe42cb3fcff408b4148b1275ad1ad49e33505023f30408d4148b1275ad1ad5d034ca7b29f07226d61634f53224092b6b9ac1c8558d520a4b6c2ad617b5a54251f01317ad9d07f66a281b0dae053fad0321aa49d13fda992a49685340c8a6adca7e28104416e277fb521aeba0bc8b1e632586d975765c53facd8f7e8cff4a506ea5531149d4ffda97a7b0f49580b2cae05c0b814dc394761986d975765cf53e5497ca589d34d1f43aeba0c41a4c7a98f33a69a3fdf9a68fa1d75d0620d263d4c79a68fbed00177fe8d48e62b03ee697e8d48e62b1e0b1d7f46a4731581d754df5f2c7cfdf6800bbdf43aeba0c41a4c7a9841a6a8b22c5f249c754c5fbef046cfdf6800bbbf408a4148b4a549275906497f83a8f517408a4148b4a549275a93c85f86a87dcd30d25f408a4148b4a549275ad416cf023f31408a4148b4a549275a42a13f8690e4b692d49f50929bd9abfa5242cb40d25fa523b3e94f684c9f518cf73ad7b4fd7b9fefb4005dff4086aec31ec327d785b6007d286f",
},
)
if err != nil {
@@ -30,10 +28,7 @@ func TestHttp3(t *testing.T) {
func TestHttp32(t *testing.T) {
resp, err := requests.Get(context.TODO(), "https://cloudflare-quic.com/", requests.RequestOption{
ClientOption: requests.ClientOption{
// USpec: true,
ForceHttp3: true,
},
ForceHttp3: true,
},
)
if err != nil {

6
test/logger/logger_test.go
View File

@@ -10,10 +10,8 @@ import (
func TestLogger(t *testing.T) {
for i := 0; i < 2; i++ {
response, err := requests.Get(nil, "https://www.httpbin.org", requests.RequestOption{
ClientOption: requests.ClientOption{
Logger: func(l requests.Log) {
log.Print(l)
},
Logger: func(l requests.Log) {
log.Print(l)
},
})
if err != nil {

22
test/middleware/errCallBack_test.go
View File

@@ -10,19 +10,17 @@ import (
func TestErrCallBack(t *testing.T) {
n := 0
_, err := requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{
ClientOption: requests.ClientOption{
MaxRetries: 3,
ResultCallBack: func(ctx *requests.Response) error {
return errors.New("try")
},
ErrCallBack: func(ctx *requests.Response) error {
if n == 0 {
n++
return nil
}
return errors.New("test")
},
MaxRetries: 3,
ResultCallBack: func(ctx *requests.Response) error {
return errors.New("try")
},
ErrCallBack: func(ctx *requests.Response) error {
if n == 0 {
n++
return nil
}
return errors.New("test")
},
})
if err == nil {

9
test/middleware/optionltCallBack_test.go
View File

@@ -8,12 +8,9 @@ import (
func TestOptionCallBack(t *testing.T) {
resp, err := requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{
ClientOption: requests.ClientOption{
OptionCallBack: func(ctx *requests.Response) error {
ctx.Option().Params = map[string]string{"name": "test"}
return nil
},
OptionCallBack: func(ctx *requests.Response) error {
ctx.Option().Params = map[string]string{"name": "test"}
return nil
},
})
if err != nil {

12
test/middleware/redirectCallBack_test.go
View File

@@ -9,13 +9,11 @@ import (
func TestRedirectCallBack(t *testing.T) {
response, err := requests.Get(context.TODO(), "http://www.baidu.com", requests.RequestOption{
ClientOption: requests.ClientOption{
RequestCallBack: func(ctx *requests.Response) error {
if ctx.Response() != nil {
return requests.ErrUseLastResponse
}
return nil
},
RequestCallBack: func(ctx *requests.Response) error {
if ctx.Response() != nil {
return requests.ErrUseLastResponse
}
return nil
},
})
if err != nil {

16
test/middleware/requestCallback_test.go
View File

@@ -10,16 +10,14 @@ import (
func TestRequestCallBack(t *testing.T) {
_, err := requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{
ClientOption: requests.ClientOption{
RequestCallBack: func(ctx *requests.Response) error {
response := ctx.Response()
if response != nil {
if response.ContentLength > 100 {
return errors.New("max length")
}
RequestCallBack: func(ctx *requests.Response) error {
response := ctx.Response()
if response != nil {
if response.ContentLength > 100 {
return errors.New("max length")
}
return nil
},
}
return nil
},
})
if !strings.Contains(err.Error(), "max length") {

14
test/middleware/resultCallBack_test.go
View File

@@ -10,14 +10,12 @@ import (
func TestResultCallBack(t *testing.T) {
var code int
_, err := requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{
ClientOption: requests.ClientOption{
ResultCallBack: func(ctx *requests.Response) error {
if ctx.StatusCode() != 200 {
return errors.New("resp.StatusCode!= 200")
}
code = ctx.StatusCode()
return nil
},
ResultCallBack: func(ctx *requests.Response) error {
if ctx.StatusCode() != 200 {
return errors.New("resp.StatusCode!= 200")
}
code = ctx.StatusCode()
return nil
},
})
if err != nil {

17
test/middleware/try_test.go
View File

@@ -10,16 +10,13 @@ import (
func TestMaxRetries(t *testing.T) {
n := 0
resp, err := requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{
ClientOption: requests.ClientOption{
MaxRetries: 3,
ResultCallBack: func(ctx *requests.Response) error {
if n == 0 {
n++
return errors.New("try")
}
return nil
},
MaxRetries: 3,
ResultCallBack: func(ctx *requests.Response) error {
if n == 0 {
n++
return errors.New("try")
}
return nil
},
})
if err != nil {

17
test/protocol/http1_test.go
View File

@@ -9,16 +9,13 @@ import (
func TestHttp1(t *testing.T) {
resp, err := requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{
ClientOption: requests.ClientOption{
ForceHttp1: true,
Logger: func(l requests.Log) {
log.Print(l)
},
ErrCallBack: func(ctx *requests.Response) error {
log.Print(ctx.Err())
return nil
},
ForceHttp1: true,
Logger: func(l requests.Log) {
log.Print(l)
},
ErrCallBack: func(ctx *requests.Response) error {
log.Print(ctx.Err())
return nil
},
})
if err != nil {

8
test/protocol/http2_test.go
View File

@@ -34,11 +34,9 @@ func TestHttp2(t *testing.T) {
}
resp, err = session.Post(context.TODO(), "https://mp.weixin.qq.com", requests.RequestOption{
Body: "fasfasfsdfdssdsfasdfasdfsadfsdf对方是大翻身大翻身大翻身对方的身份",
ClientOption: requests.ClientOption{
ErrCallBack: func(ctx *requests.Response) error {
log.Print(ctx.Err())
return nil
},
ErrCallBack: func(ctx *requests.Response) error {
log.Print(ctx.Err())
return nil
},
})
if err != nil {

21
test/protocol/http3_test.go
View File

@@ -13,10 +13,7 @@ import (
func TestHttp3(t *testing.T) {
resp, err := requests.Get(context.TODO(), "https://cloudflare-quic.com/", requests.RequestOption{
ClientOption: requests.ClientOption{
ForceHttp3: true,
},
ForceHttp3: true,
},
)
if err != nil {
@@ -45,17 +42,15 @@ func TestHttp3Proxy(t *testing.T) {
resp, err := requests.Get(context.Background(), href,
requests.RequestOption{
ClientOption: requests.ClientOption{
DialOption: requests.DialOption{
// AddrType: gtls.Ipv4,
GetAddrType: func(host string) gtls.AddrType {
log.Print("我开始喽")
return gtls.Ipv4
},
DialOption: &requests.DialOption{
// AddrType: gtls.Ipv4,
GetAddrType: func(host string) gtls.AddrType {
log.Print("我开始喽")
return gtls.Ipv4
},
Proxy: "socks5://" + proxyAddress,
// ForceHttp3: true,
},
Proxy: "socks5://" + proxyAddress,
// ForceHttp3: true,
})
if err != nil {
log.Panic(err)

5
test/proxy/chain_proxy_test.go
View File

@@ -9,10 +9,7 @@ import (
func TestChainProxy(t *testing.T) {
resp, err := requests.Get(context.TODO(), "https://httpbin.org/anything", requests.RequestOption{
ClientOption: requests.ClientOption{
Proxy: []string{}, //set proxy,ex:"http://127.0.0.1:8080","https://127.0.0.1:8080","socks5://127.0.0.1:8080"
},
Proxy: []string{}, //set proxy,ex:"http://127.0.0.1:8080","https://127.0.0.1:8080","socks5://127.0.0.1:8080"
})
if err != nil {
t.Error(err)

5
test/proxy/disProxy_test.go
View File

@@ -8,10 +8,7 @@ import (
func TestDisProxy(t *testing.T) {
resp, err := requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{
ClientOption: requests.ClientOption{
Proxy: "http://192.368.7.256:9887",
},
Proxy: "http://192.368.7.256:9887",
DisProxy: true,
})
if err != nil {

47
test/proxy/http3_proxy_test.go
View File

@@ -26,15 +26,12 @@ var (
func client() {
for range 5 {
resp, err := requests.Post(nil, "https://"+remoteHost, requests.RequestOption{
ClientOption: requests.ClientOption{
ForceHttp3: true,
Logger: func(l requests.Log) {
log.Print(l)
},
Proxy: "socks5://" + proxyHost,
ForceHttp3: true,
Logger: func(l requests.Log) {
log.Print(l)
},
Body: []byte("hello, server!"),
Proxy: "socks5://" + proxyHost,
Body: []byte("hello, server!"),
})
if err != nil {
fmt.Println(err)
@@ -50,15 +47,13 @@ func client2() {
for range 5 {
resp, err := requests.Post(nil, "https://"+remoteHost, requests.RequestOption{
ClientOption: requests.ClientOption{
ForceHttp3: true,
Logger: func(l requests.Log) {
log.Print(l)
},
Proxy: []string{
"http://" + proxyHost,
"socks5://" + proxyHost,
},
ForceHttp3: true,
Logger: func(l requests.Log) {
log.Print(l)
},
Proxy: []string{
"http://" + proxyHost,
"socks5://" + proxyHost,
},
Body: []byte("hello, server!"),
})
@@ -121,16 +116,14 @@ func TestHttp3Proxy2(t *testing.T) {
for range 5 {
resp, err := requests.Get(context.TODO(), "https://cloudflare-quic.com/", requests.RequestOption{
ClientOption: requests.ClientOption{
USpec: true,
ForceHttp3: true,
// Logger: func(l requests.Log) {
// log.Print(l)
// },
Proxy: []string{
// "http://" + proxyHost,
"socks5://" + proxyHost,
},
USpec: true,
ForceHttp3: true,
// Logger: func(l requests.Log) {
// log.Print(l)
// },
Proxy: []string{
// "http://" + proxyHost,
"socks5://" + proxyHost,
},
Body: []byte("hello, server!"),
})

2
test/proxy/proxy_test.go
View File

@@ -8,7 +8,7 @@ import (
func TestProxy(t *testing.T) {
resp, err := requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{
ClientOption: requests.ClientOption{Proxy: ""}, //set proxy,ex:"http://127.0.0.1:8080","https://127.0.0.1:8080","socks5://127.0.0.1:8080"
Proxy: "", //set proxy,ex:"http://127.0.0.1:8080","https://127.0.0.1:8080","socks5://127.0.0.1:8080"
})
if err != nil {
t.Error(err)

9
test/request/localAddr_test.go
View File

@@ -9,11 +9,10 @@ import (
func TestLocalAddr(t *testing.T) {
resp, err := requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{
ClientOption: requests.ClientOption{
DialOption: requests.DialOption{
LocalAddr: &net.TCPAddr{ //set dns server
IP: net.ParseIP("192.168.1.239"),
},
DialOption: &requests.DialOption{
LocalAddr: &net.TCPAddr{ //set dns server
IP: net.ParseIP("192.168.1.239"),
},
},
})

24
test/request/redirect_test.go
View File

@@ -9,13 +9,11 @@ import (
func TestRedirectBreakWitRequest(t *testing.T) {
href := "https://httpbin.co/absolute-redirect/2"
resp, err := requests.Get(nil, href, requests.RequestOption{
ClientOption: requests.ClientOption{
RequestCallBack: func(ctx *requests.Response) error {
if ctx.Response() != nil {
return requests.ErrUseLastResponse
}
return nil
},
RequestCallBack: func(ctx *requests.Response) error {
if ctx.Response() != nil {
return requests.ErrUseLastResponse
}
return nil
},
})
if err != nil {
@@ -29,13 +27,11 @@ func TestRedirectWithRequest(t *testing.T) {
href := "https://httpbin.co/absolute-redirect/2"
n := 0
resp, err := requests.Get(nil, href, requests.RequestOption{
ClientOption: requests.ClientOption{
RequestCallBack: func(ctx *requests.Response) error {
if ctx.Response() == nil {
n++
}
return nil
},
RequestCallBack: func(ctx *requests.Response) error {
if ctx.Response() == nil {
n++
}
return nil
},
})
if err != nil {

18
test/response/useProxy_test.go
View File

@@ -1,18 +0,0 @@
package main
import (
"testing"
"github.com/gospider007/requests"
)
func TestUseProxy(t *testing.T) {
resp, err := requests.Get(nil, "https://httpbin.org/anything")
if err != nil {
t.Error(err)
}
if resp.Proxys() != nil {
t.Error("proxy error")
}
}

2
test/session_test.go
View File

@@ -50,7 +50,7 @@ func TestSession2(t *testing.T) {
func TestSession3(t *testing.T) {
session, _ := requests.NewClient(nil)
for i := 0; i < 2; i++ {
resp, err := session.Get(nil, "https://cloudflare-quic.com/", requests.RequestOption{ClientOption: requests.ClientOption{ForceHttp3: true}})
resp, err := session.Get(nil, "https://cloudflare-quic.com/", requests.RequestOption{ForceHttp3: true})
if err != nil {
t.Error(err)
}