mirror of
https://github.com/gospider007/requests.git
synced 2025-12-24 13:57:52 +08:00
244 lines
12 KiB
Go
244 lines
12 KiB
Go
package requests
|
||
|
||
import (
|
||
"bytes"
|
||
"context"
|
||
"crypto/rand"
|
||
"crypto/tls"
|
||
"fmt"
|
||
"io"
|
||
"net/url"
|
||
"strings"
|
||
"time"
|
||
|
||
"github.com/gospider007/tools"
|
||
"github.com/gospider007/websocket"
|
||
"github.com/quic-go/quic-go"
|
||
uquic "github.com/refraction-networking/uquic"
|
||
utls "github.com/refraction-networking/utls"
|
||
)
|
||
|
||
type LogType string
|
||
|
||
const (
|
||
LogType_DNSLookup LogType = "DNSLookup"
|
||
LogType_TCPConnect LogType = "TCPConnect"
|
||
LogType_TLSHandshake LogType = "TLSHandshake"
|
||
LogType_ProxyDNSLookup LogType = "ProxyDNSLookup"
|
||
LogType_ProxyTCPConnect LogType = "ProxyTCPConnect"
|
||
LogType_ProxyTLSHandshake LogType = "ProxyTLSHandshake"
|
||
LogType_ProxyConnectRemote LogType = "ProxyConnectRemote"
|
||
LogType_ResponseHeader LogType = "ResponseHeader"
|
||
LogType_ResponseBody LogType = "ResponseBody"
|
||
)
|
||
|
||
type Log struct {
|
||
Time time.Time
|
||
Msg any `json:"msg"`
|
||
Id string `json:"id"`
|
||
Type LogType `json:"type"`
|
||
}
|
||
|
||
// Connection Management Options
|
||
type ClientOption struct {
|
||
Spec string //goSpiderSpec origin : https://github.com/gospider007/fp
|
||
DialOption DialOption
|
||
Headers any //default headers
|
||
Jar Jar //custom cookies
|
||
Logger func(Log) //debuggable
|
||
OptionCallBack func(ctx *Response) error //option callback,if error is returnd, break request
|
||
ResultCallBack func(ctx *Response) error //result callback,if error is returnd,next errCallback
|
||
ErrCallBack func(ctx *Response) error //error callback,if error is returnd,break request
|
||
RequestCallBack func(ctx *Response) error //request and response callback,if error is returnd,reponse is error
|
||
GetProxy func(ctx *Response) (any, error) //proxy callback:support https,http,socks5 proxy
|
||
TlsConfig *tls.Config
|
||
UtlsConfig *utls.Config
|
||
QuicConfig *quic.Config
|
||
UquicConfig *uquic.Config
|
||
USpec any //support ja3.USpec,uquic.QUICID,bool
|
||
UserAgent string //headers User-Agent value
|
||
Proxy any //strong or []string, ,support https,http,socks5
|
||
MaxRetries int //try num
|
||
MaxRedirect int //redirect num ,<0 no redirect,==0 no limit
|
||
Timeout time.Duration //request timeout
|
||
ResponseHeaderTimeout time.Duration //ResponseHeaderTimeout ,default:300
|
||
TlsHandshakeTimeout time.Duration //tls timeout,default:15
|
||
ForceHttp3 bool //force use http3 send requests
|
||
ForceHttp1 bool //force use http1 send requests
|
||
DisCookie bool //disable cookies
|
||
DisDecode bool //disable auto decode
|
||
Bar bool ////enable bar display
|
||
}
|
||
|
||
// Options for sending requests
|
||
type RequestOption struct {
|
||
WsOption websocket.Option //websocket option
|
||
Cookies any // cookies,support :json,map,str，http.Header
|
||
Params any //url params，join url query,json,map
|
||
Json any //send application/json,support io.Reader,：string,[]bytes,json,map
|
||
Data any //send application/x-www-form-urlencoded, support io.Reader, string,[]bytes,json,map
|
||
Form any //send multipart/form-data,file upload,support io.Reader, json,map
|
||
Text any //send text/xml,support: io.Reader, string,[]bytes,json,map
|
||
Body any //not setting context-type,support io.Reader, string,[]bytes,json,map
|
||
Url *url.URL
|
||
// other option
|
||
Method string //method
|
||
Host string
|
||
Referer string //set headers referer value
|
||
ContentType string //headers Content-Type value
|
||
ClientOption
|
||
Stream bool //disable auto read
|
||
DisProxy bool //force disable proxy
|
||
readOne bool
|
||
orderHeaders *OrderData //order headers
|
||
gospiderSpec *GospiderSpec
|
||
disBody bool
|
||
}
|
||
|
||
// Upload files with form-data,
|
||
type File struct {
|
||
Content any
|
||
FileName string
|
||
ContentType string
|
||
}
|
||
|
||
func randomBoundary() (string, string) {
|
||
var buf [30]byte
|
||
io.ReadFull(rand.Reader, buf[:])
|
||
boundary := fmt.Sprintf("%x", buf[:])
|
||
if strings.ContainsAny(boundary, `()<>@,;:\"/[]?= `) {
|
||
boundary = `"` + boundary + `"`
|
||
}
|
||
return "multipart/form-data; boundary=" + boundary, boundary
|
||
}
|
||
|
||
func (obj *RequestOption) initBody(ctx context.Context) (io.Reader, error) {
|
||
if obj.Body != nil {
|
||
body, orderData, err := obj.newBody(obj.Body)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
if body != nil {
|
||
return body, nil
|
||
}
|
||
con, err := orderData.MarshalJSON()
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
return bytes.NewReader(con), nil
|
||
} else if obj.Form != nil {
|
||
var boundary string
|
||
if obj.ContentType == "" {
|
||
obj.ContentType, boundary = randomBoundary()
|
||
}
|
||
body, orderData, err := obj.newBody(obj.Form)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
if body != nil {
|
||
return body, nil
|
||
}
|
||
body, once, err := orderData.parseForm(ctx, boundary)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
obj.readOne = once
|
||
return body, err
|
||
} else if obj.Data != nil {
|
||
if obj.ContentType == "" {
|
||
obj.ContentType = "application/x-www-form-urlencoded"
|
||
}
|
||
body, orderData, err := obj.newBody(obj.Data)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
if body != nil {
|
||
return body, nil
|
||
}
|
||
return orderData.parseData(), nil
|
||
} else if obj.Json != nil {
|
||
if obj.ContentType == "" {
|
||
obj.ContentType = "application/json"
|
||
}
|
||
body, orderData, err := obj.newBody(obj.Json)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
if body != nil {
|
||
return body, nil
|
||
}
|
||
return orderData.parseJson()
|
||
} else if obj.Text != nil {
|
||
if obj.ContentType == "" {
|
||
obj.ContentType = "text/plain"
|
||
}
|
||
body, orderData, err := obj.newBody(obj.Text)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
if body != nil {
|
||
return body, nil
|
||
}
|
||
return orderData.parseText()
|
||
} else {
|
||
return nil, nil
|
||
}
|
||
}
|
||
func (obj *RequestOption) initParams() (*url.URL, error) {
|
||
baseUrl := cloneUrl(obj.Url)
|
||
if obj.Params == nil {
|
||
return baseUrl, nil
|
||
}
|
||
body, dataData, err := obj.newBody(obj.Params)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
var query string
|
||
if body != nil {
|
||
paramsBytes, err := io.ReadAll(body)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
query = tools.BytesToString(paramsBytes)
|
||
} else {
|
||
query = dataData.parseParams().String()
|
||
}
|
||
if query == "" {
|
||
return baseUrl, nil
|
||
}
|
||
pquery := baseUrl.Query().Encode()
|
||
if pquery == "" {
|
||
baseUrl.RawQuery = query
|
||
} else {
|
||
baseUrl.RawQuery = pquery + "&" + query
|
||
}
|
||
return baseUrl, nil
|
||
}
|
||
func (obj *Client) newRequestOption(option RequestOption) (RequestOption, error) {
|
||
err := tools.Merge(&option, obj.ClientOption)
|
||
//end
|
||
if option.MaxRetries < 0 {
|
||
option.MaxRetries = 0
|
||
}
|
||
if option.UserAgent == "" {
|
||
option.UserAgent = obj.ClientOption.UserAgent
|
||
}
|
||
if option.DisCookie {
|
||
option.Jar = nil
|
||
}
|
||
if option.DisProxy {
|
||
option.Proxy = ""
|
||
}
|
||
if option.Spec == "" {
|
||
option.Spec = "16030106f2010006ee03039a2b98d81139db0e128ea09eff6874549c219b543fb6dbaa7e4dbfe9e31602c620ce04c4026f019442affade7fed8ba66e022e186f77f1c670fd992f33c0143f120020aaaa130113021303c02bc02fc02cc030cca9cca8c013c014009c009d002f0035010006851a1a00000010000e000c02683208687474702f312e31002b000706dada03040303002d00020101000d0012001004030804040105030805050108060601001b0003020002ff0100010000230000000a000c000afafa11ec001d001700180000000e000c0000096c6f63616c686f7374003304ef04edfafa00010011ec04c06903195f3660633741f9f5ae64d05a316ac8e717582adb9e58c5c242ba306c99ca8f68c15f261245f9141812383c9265f7d0b5c44a5e0d7633f4f40ab8e820ec01bb6cc74e6ab3168e66b40cc4cd37e96e286b4080552b8b0217f786b7c1a0088fb613cc84471b17a33fbcc68db151df387907a1cf3fb14a0f45c6b84608db5b103131b537255c09559cf1940c3980a7f37959a7f95d32c49923600c76c616af238c579361e1c6a20c251d3d42a50e182ad54b1e5d54a57fe6986e64142815b9478cca8066c9bdcc0eb9022ebe05b0ebd53e7d146761d81aee41cd377611699c536a444d300c152994bfeef3cfb4736cb3d57d683269c1e3c001fba2ac220b2c993cec410f6fa5104d1bcde21c46a9b0be8ab7b51aba15f1745ebbd0a0d3a5224170b3bce456c157937e43390b733375bb96601667f5b36888f9520931bead48bae4723d9ed40af2680746e27eee4328503a280ee8846b37803d6206e0f5248bea4ca4a53ca4e1afdd2b84bce0c83260333bc9b38b86486fb48e18d1ce9187b1b6332b2f4145eca38122ac363210137f52140a57b7976b609d739844fa61f21c1e3c300f3434bc3f8b6856994847e2c9b0f20a24b976f9d552b153246db69f30d4a95301b933b0ba9d48402ddb7863cb4f1923a2c33021fd68634a387bf0f76d87f01b35b6182dd10fe9c14b8e548d7988388308b08ff1585ba18a7615737857a7c23c24ee9b3a2ab1915be18b233acd354c7c6513b8ea617a5cf299f34139756cc1df524292c43ee3364990961b2490ae204634a4461b53c95a11d214503985f27ab85bc7c179ab1ba37a828312cabea8cfc5088616386a83e566279a0a5517b60aca4ec6c30a32191dca3cbb7d33ae5087bbdbab5c42e6293b63ad8e35311d459e1ce57037e65e96283e449c3e012051d247653197834c42613ac377a950607c98cbd5a79fef948d18e99758e12d31d13231e638cfb183623346b231a443f56533d444c7204a63479e4efb34ef97597d858915a8a10a32aa78824c9d2993741176da643be1c6c4d91b6511055b098477e7a3c5b5c312cf7bb4ef5905fd741375e62c8ce942a2117bbe707fcc9871e59c0687507862f3634c871885949fce97612793a30a155e84ac503dc519816a13772c50e1167a7031cb2c8187913108f9a26e55958fd19a6e1ef18ed53a70f1c13d01d71d3b40a22413852c9982daac4ae8071966016c38a60bd5c0258c32b882740c6ed5252093c91e51c50cf037d4f5cb6ca610672710f7ca77b0a76039a9968e368a6b243ee4ca7632855cb568c73f01764c4944fc5879d2c52d7992840863c057db2efec658eeb2a73e02bd62617438d9192911bac1f6b0e55cb38255417af20000d69378c857bb278156f16a684200125906b6c22f3d505bc9e76d75fac3a009332ff98fe6baabe3941cab5271c6d2c0ebc993b944c49bd437353019d1b24d10390e45fa87ad77b329a9025933a11af2af0da44d3ed761722c94d8053242f537624113d7bc0155600573301bd2217c6c481ce63b0944b052c97bcb9d3349258257ff33cccf963a6945119ecab21c25051ce02548f642e0ec1ffd392d60facfdf76bfb7274363b62979231f4996362c85d5ba19d2cab7019750b3443565436867a53b71d875eba3282e6d0ee22076d6b97b7c6c556ae216e8bc1bc9f202ce94c763bfe9afc105fca9372dec2e286a001d00200ee8ac33f1ea3153f6b4a06ab71d21b7ce7955ce64ccfc66b7ec8077d02ffd18fe0d00fa00000100018b00206cada2aa48ee4478c40adad21f147d6bc90d13f6889a9b8a58a02536585a261f00d09306c85aab2a6e424b658f3cd9d1c46f35020839287259d3be605ff97faea0d87b9f7f96529661f08cf3f3899db8e805ee7405e2f9b6abd99bc4f6fa5f99b1ed442ebe53c5b10451c93d1221f662783efc3cc8fcf135ed935bcf02ec32251dd09705f191bd7959afbab5619d8e63cb634a259dd63d1b0e42225ae8c08b5b1620cd59d914857e9f1e8a3b7b892863bdaa05429922d75583059641468d8fc51c01e977a69d3a51d714cd5cceea9a5f404ce4a285fd6647931ed8b1c12db027328f214afdbe2c8102b46fe041b553f8670b00050005010000000044cd0005000302683200170000000b0002010000120000caca000100@@505249202a20485454502f322e300d0a0d0a534d0d0a0d0a00001804000000000000010001000000020000000000040060000000060004000000000408000000000000ef00010001d401250000000180000000ff82418aa0e41d139d09b8f3efbf87845887a47e561cc5801f40874148b1275ad1ffb9fe749d3fd4372ed83aa4fe7efbc1fcbefff3f4a7f388e79a82a97a7b0f497f9fbef07f21659fe7e94fe6f4f61e935b4ff3f7de0fe42cb3fcff408b4148b1275ad1ad49e33505023f30408d4148b1275ad1ad5d034ca7b29f07226d61634f53224092b6b9ac1c8558d520a4b6c2ad617b5a54251f01317ad9d07f66a281b0dae053fad0321aa49d13fda992a49685340c8a6adca7e28104416e277fb521aeba0bc8b1e632586d975765c53facd8f7e8cff4a506ea5531149d4ffda97a7b0f49580b2cae05c0b814dc394761986d975765cf53e5497ca589d34d1f43aeba0c41a4c7a98f33a69a3fdf9a68fa1d75d0620d263d4c79a68fbed00177fe8d48e62b03ee697e8d48e62b1e0b1d7f46a4731581d754df5f2c7cfdf6800bbdf43aeba0c41a4c7a9841a6a8b22c5f249c754c5fbef046cfdf6800bbbf408a4148b4a549275906497f83a8f517408a4148b4a549275a93c85f86a87dcd30d25f408a4148b4a549275ad416cf023f31408a4148b4a549275a42a13f8690e4b692d49f50929bd9abfa5242cb40d25fa523b3e94f684c9f518cf73ad7b4fd7b9fefb4005dff4086aec31ec327d785b6007d286f"
|
||
}
|
||
return option, err
|
||
}
|
||
func (obj *Client) newResponse(ctx context.Context, option RequestOption, uhref *url.URL, requestId string) *Response {
|
||
option.Url = cloneUrl(uhref)
|
||
response := NewResponse(ctx, option)
|
||
response.client = obj
|
||
response.requestId = requestId
|
||
return response
|
||
}
|