+
+
+
Requests - A next-generation HTTP client for Golang.
+ + +Requests is a fully featured HTTP client library for Golang. Network requests can be completed with just a few lines of code +--- +## Features + * GET, POST, PUT, DELETE, HEAD, PATCH, OPTIONS, etc. + * Simple for settings and request + * [Request](https://github.com/gospider007/requests#Request) Body can be `string`, `[]byte`, `struct`, `map`, `slice` and `io.Reader` too + * Auto detects `Content-Type` + * Buffer less processing for `io.Reader` + * [Response](https://github.com/gospider007/requests#Response) object gives you more possibility + * Automatic marshal and unmarshal for content + * Easy to upload one or more file(s) via `multipart/form-data` + * Auto detects file content type + * Request URL Path Params (aka URI Params) + * Backoff Retry Mechanism with retry condition function + * Optionally allows GET request with payload + * Request design + * Have client level settings & options and also override at Request level if you want to + * Request and Response middleware + * goroutine concurrent safe + * Gzip - Go does it automatically also requests has fallback handling too + * Works fine with `HTTP/2` and `HTTP/1.1` + * DNS caching + * Fingerprint + * JA3 + * HTTP2 + * JA4 + * OrderHeaders + * Request header capitalization + * Proxy + * HTTP + * HTTPS + * SOCKS5 + * Protocol + * HTTP + * HTTPS + * WebSocket + * SSE + * Well tested client library + +## Supported Go Versions +Recommended to use `go1.21.3` and above. +Initially Requests started supporting `go modules` + +## Installation + +```bash go get github.com/gospider007/requests ``` -# Function Overview -- No memory leakage -- Only a few lines of code are needed to complete the request -- JA3 Fingerprint, HTTP2 Fingerprint, JA4 Fingerprint -- SOCKS5 proxy, HTTP proxy, HTTPS proxy -- WebSocket protocol, SSE protocol ,HTTPS protocol,HTTP protocol -- Connection pool, Cookies Jar -- Automatic decompression and decoding -- Automatic type conversion -- DNS caching -- Request Retry -- Powerful and convenient request callback - -# quick start -## Quickly send requests +## Usage +```go +import "github.com/gospider007/requests" +``` +## Quick Start +### Quickly send requests ```go package main @@ -51,7 +99,7 @@ func main() { log.Print(resp.Cookies()) // Get cookies } ``` -## use session +### use session ```go package main @@ -74,7 +122,7 @@ func main() { log.Print(resp.StatusCode()) //return status code } ``` -## setting order headers with http1 +### setting order headers with http1 ```go package main @@ -95,7 +143,7 @@ func main() { } ``` -## send websocket +### send websocket ```go package main @@ -126,7 +174,7 @@ func main() { log.Print(string(con)) // Message content } ``` -## IPv4, IPv6 Address Control Parsing +### IPv4, IPv6 Address Control Parsing ```go package main @@ -149,7 +197,7 @@ func main() { log.Print(resp.StatusCode()) } ``` -## Generate Ja3 Fingerprint from String +### Generate Ja3 Fingerprint from String ```go package main @@ -172,7 +220,7 @@ func main() { log.Print(jsonData.Get("ja3").String() == ja3Str) } ``` -## Generate Ja3 Fingerprint from ID +### Generate Ja3 Fingerprint from ID ```go package main @@ -193,7 +241,7 @@ func main() { log.Print(jsonData.Get("ja3").String()) } ``` -## Generate H2 Fingerprint from String +### Generate H2 Fingerprint from String ```go package main @@ -216,7 +264,7 @@ func main() { log.Print(jsonData.Get("akamai_fp").String() == h2ja3Str) } ``` -## Modify H2 Fingerprint +### Modify H2 Fingerprint ```go package main diff --git a/body.go b/body.go index 48c4fde..4819868 100644 --- a/body.go +++ b/body.go @@ -11,13 +11,6 @@ import ( "github.com/gospider007/tools" ) -type File struct { - Key string - Val []byte - - FileName string - ContentType string -} type bodyType = int const ( diff --git a/client.go b/client.go index 2984b2e..229d385 100644 --- a/client.go +++ b/client.go @@ -30,28 +30,29 @@ type ClientOption struct { ErrCallBack func(context.Context, *Client, error) error //error callback,if error is returnd,break request RequestCallBack func(context.Context, *http.Request, *http.Response) error //request and response callback,if error is returnd,reponse is error TryNum int //try num - RedirectNum int //redirect num ,<0 no redirect,==0 no limit + MaxRedirectNum int //redirect num ,<0 no redirect,==0 no limit Headers any //default headers ResponseHeaderTimeout time.Duration //ResponseHeaderTimeout ,default:30 + TlsHandshakeTimeout time.Duration //tls timeout,default:15 - GetProxy func(ctx context.Context, url *url.URL) (string, error) //proxy callback:support https,http,socks5 proxy - LocalAddr *net.TCPAddr //network card ip - DialTimeout time.Duration //dial tcp timeout,default:15 - TlsHandshakeTimeout time.Duration //tls timeout,default:15 - KeepAlive time.Duration //keepalive,default:30 - AddrType AddrType //dns parse addr type - GetAddrType func(string) AddrType - Dns net.IP //dns + GetProxy func(ctx context.Context, url *url.URL) (string, error) //proxy callback:support https,http,socks5 proxy + //network card ip + DialTimeout time.Duration //dial tcp timeout,default:15 + Dns net.IP //dns + KeepAlive time.Duration //keepalive,default:30 + LocalAddr *net.TCPAddr + AddrType AddrType //dns parse addr type + GetAddrType func(string) AddrType } type Client struct { forceHttp1 bool orderHeaders []string - jar *Jar - redirectNum int - disDecode bool - disUnZip bool - disAlive bool + jar *Jar + maxRedirectNum int + disDecode bool + disUnZip bool + disAlive bool tryNum int @@ -63,6 +64,7 @@ type Client struct { timeout time.Duration responseHeaderTimeout time.Duration + tlsHandshakeTimeout time.Duration headers any bar bool @@ -89,37 +91,9 @@ func NewClient(preCtx context.Context, options ...ClientOption) (*Client, error) if len(options) > 0 { option = options[0] } - if option.KeepAlive == 0 { - option.KeepAlive = time.Second * 30 - } - if option.DialTimeout == 0 { - option.DialTimeout = time.Second * 15 - } - if option.TlsHandshakeTimeout == 0 { - option.TlsHandshakeTimeout = time.Second * 15 - } - //cookiesjar - var jar *Jar - if !option.DisCookie { - jar = NewJar() - } - // var http3Transport *http3.RoundTripper - // if option.Http3 { - // http3Transport = &http3.RoundTripper{ - // TLSClientConfig: tlsConfig, - // QuicConfig: &quic.Config{ - // EnableDatagrams: true, - // HandshakeIdleTimeout: option.TLSHandshakeTimeout, - // MaxIdleTimeout: option.IdleConnTimeout, - // KeepAlivePeriod: option.KeepAlive, - // }, - // EnableDatagrams: true, - // } - // } transport := newRoundTripper(ctx, RoundTripperOption{ - TlsHandshakeTimeout: option.TlsHandshakeTimeout, - DialTimeout: option.DialTimeout, - KeepAlive: option.KeepAlive, + DialTimeout: option.DialTimeout, + KeepAlive: option.KeepAlive, LocalAddr: option.LocalAddr, AddrType: option.AddrType, @@ -131,17 +105,13 @@ func NewClient(preCtx context.Context, options ...ClientOption) (*Client, error) Transport: transport, CheckRedirect: func(req *http.Request, via []*http.Request) error { ctxData := req.Context().Value(keyPrincipalID).(*reqCtxData) - if ctxData.redirectNum == 0 || ctxData.redirectNum >= len(via) { + if ctxData.maxRedirectNum == 0 || ctxData.maxRedirectNum >= len(via) { return nil } return http.ErrUseLastResponse }, } - if jar != nil { - client.Jar = jar.jar - } result := &Client{ - jar: jar, ctx: ctx, cnl: cnl, client: client, @@ -150,7 +120,7 @@ func NewClient(preCtx context.Context, options ...ClientOption) (*Client, error) requestCallBack: option.RequestCallBack, orderHeaders: option.OrderHeaders, disCookie: option.DisCookie, - redirectNum: option.RedirectNum, + maxRedirectNum: option.MaxRedirectNum, disDecode: option.DisDecode, disUnZip: option.DisUnZip, disAlive: option.DisAlive, @@ -160,9 +130,15 @@ func NewClient(preCtx context.Context, options ...ClientOption) (*Client, error) errCallBack: option.ErrCallBack, timeout: option.Timeout, responseHeaderTimeout: option.ResponseHeaderTimeout, + tlsHandshakeTimeout: option.TlsHandshakeTimeout, headers: option.Headers, bar: option.Bar, } + //cookiesjar + if !option.DisCookie { + result.jar = NewJar() + result.client.Jar = result.jar.jar + } var err error if option.Proxy != "" { result.proxy, err = gtls.VerifyProxy(option.Proxy) diff --git a/conn.go b/conn.go new file mode 100644 index 0000000..b832c2e --- /dev/null +++ b/conn.go @@ -0,0 +1,258 @@ +package requests + +import ( + "bufio" + "context" + "errors" + "net" + "net/http" + "sync/atomic" + "time" + + "github.com/gospider007/net/http2" + "github.com/gospider007/tools" +) + +type Connecotr struct { + key connKey + err error + deleteCtx context.Context //force close + deleteCnl context.CancelFunc + + closeCtx context.Context //safe close + closeCnl context.CancelFunc + + bodyCtx context.Context //body close + bodyCnl context.CancelFunc + + rawConn net.Conn + h2 bool + r *bufio.Reader + w *bufio.Writer + h2RawConn *http2.ClientConn + rc chan []byte + rn chan int + isRead bool + isPool bool +} + +func (obj *Connecotr) WithCancel(deleteCtx context.Context, closeCtx context.Context) { + obj.deleteCtx, obj.deleteCnl = context.WithCancel(deleteCtx) + obj.closeCtx, obj.closeCnl = context.WithCancel(closeCtx) +} +func (obj *Connecotr) Close() error { + obj.deleteCnl() + if obj.h2RawConn != nil { + obj.h2RawConn.Close() + } + return obj.rawConn.Close() +} +func (obj *Connecotr) read() { + if obj.isRead { + return + } + obj.isRead = true + defer obj.Close() + con := make([]byte, 4096) + var i int + for { + i, obj.err = obj.rawConn.Read(con) + b := con[:i] + for once := true; once || len(b) > 0; once = false { + select { + case obj.rc <- b: + select { + case nw := <-obj.rn: + b = b[nw:] + case <-obj.deleteCtx.Done(): + return + } + case <-obj.deleteCtx.Done(): + return + } + } + if obj.err != nil { + return + } + } +} +func (obj *Connecotr) Read(b []byte) (i int, err error) { + if !obj.isRead { + return obj.rawConn.Read(b) + } + select { + case con := <-obj.rc: + i, err = copy(b, con), obj.err + select { + case obj.rn <- i: + if i < len(con) { + err = nil + } + case <-obj.deleteCtx.Done(): + if err = obj.err; err == nil { + err = tools.WrapError(obj.deleteCtx.Err(), "connecotr close") + } + } + case <-obj.deleteCtx.Done(): + if err = obj.err; err == nil { + err = tools.WrapError(obj.deleteCtx.Err(), "connecotr close") + } + } + return +} +func (obj *Connecotr) Write(b []byte) (int, error) { + return obj.rawConn.Write(b) +} +func (obj *Connecotr) LocalAddr() net.Addr { + return obj.rawConn.LocalAddr() +} +func (obj *Connecotr) RemoteAddr() net.Addr { + return obj.rawConn.RemoteAddr() +} +func (obj *Connecotr) SetDeadline(t time.Time) error { + return obj.rawConn.SetDeadline(t) +} +func (obj *Connecotr) SetReadDeadline(t time.Time) error { + return obj.rawConn.SetReadDeadline(t) +} +func (obj *Connecotr) SetWriteDeadline(t time.Time) error { + return obj.rawConn.SetWriteDeadline(t) +} + +func (obj *Connecotr) h2Closed() bool { + state := obj.h2RawConn.State() + return state.Closed || state.Closing +} +func (obj *Connecotr) wrapBody(task *reqTask) { + body := new(ReadWriteCloser) + obj.bodyCtx, obj.bodyCnl = context.WithCancel(obj.deleteCtx) + body.body = task.res.Body + body.conn = obj + task.res.Body = body +} +func (obj *Connecotr) http1Req(task *reqTask) { + defer task.cnl() + if task.orderHeaders != nil && len(task.orderHeaders) > 0 { + task.err = httpWrite(task.req, obj.w, task.orderHeaders) + } else if task.err = task.req.Write(obj); task.err == nil { + task.err = obj.w.Flush() + } + if task.err == nil { + if task.res, task.err = http.ReadResponse(obj.r, task.req); task.res != nil && task.err == nil { + obj.wrapBody(task) + } else if task.err != nil { + task.err = tools.WrapError(task.err, "http1 read error") + } + } else { + task.err = tools.WrapError(task.err, "http1 write error") + } +} + +func (obj *Connecotr) http2Req(task *reqTask) { + defer task.cnl() + if task.res, task.err = obj.h2RawConn.RoundTrip(task.req); task.res != nil && task.err == nil { + obj.wrapBody(task) + } else if task.err != nil { + task.err = tools.WrapError(task.err, "http2 roundTrip error") + } +} + +func (obj *Connecotr) taskMain(task *reqTask, afterTime *time.Timer) (*http.Response, error, bool) { + if obj.h2 && obj.h2Closed() { + return nil, errors.New("conn is closed"), true + } + select { + case <-obj.closeCtx.Done(): + return nil, tools.WrapError(obj.closeCtx.Err(), "close ctx error: "), true + default: + } + if obj.h2 { + go obj.http2Req(task) + } else { + go obj.http1Req(task) + } + if afterTime == nil { + afterTime = time.NewTimer(task.responseHeaderTimeout) + } else { + afterTime.Reset(task.responseHeaderTimeout) + } + if !obj.isPool { + defer afterTime.Stop() + } + select { + case <-task.ctx.Done(): + if task.res != nil && task.err == nil && obj.isPool { + <-obj.bodyCtx.Done() //wait body close + } + case <-obj.deleteCtx.Done(): //force conn close + task.err = tools.WrapError(obj.deleteCtx.Err(), "delete ctx error: ") + task.cnl() + case <-afterTime.C: + task.err = errors.New("response Header is Timeout") + task.cnl() + } + return task.res, task.err, false +} + +type connPool struct { + deleteCtx context.Context + deleteCnl context.CancelFunc + closeCtx context.Context + closeCnl context.CancelFunc + key connKey + total atomic.Int64 + tasks chan *reqTask + rt *RoundTripper +} + +func (obj *connPool) notice(task *reqTask) { + select { + case obj.tasks <- task: + case task.emptyPool <- struct{}{}: + } +} + +func (obj *connPool) rwMain(conn *Connecotr) { + conn.WithCancel(obj.deleteCtx, obj.closeCtx) + var afterTime *time.Timer + defer func() { + if afterTime != nil { + afterTime.Stop() + } + obj.rt.connsLock.Lock() + defer obj.rt.connsLock.Unlock() + conn.Close() + obj.total.Add(-1) + if obj.total.Load() <= 0 { + obj.Close() + } + }() + select { + case <-conn.deleteCtx.Done(): //force close all conn + return + case <-conn.bodyCtx.Done(): //wait body close + } + for { + select { + case <-conn.closeCtx.Done(): //safe close conn + return + case task := <-obj.tasks: //recv task + res, err, notice := conn.taskMain(task, afterTime) + if notice { + obj.notice(task) + return + } + if res == nil || err != nil { + return + } + } + } +} +func (obj *connPool) ForceClose() { + obj.deleteCnl() + delete(obj.rt.connPools, obj.key) +} +func (obj *connPool) Close() { + obj.closeCnl() + delete(obj.rt.connPools, obj.key) +} diff --git a/cookies.go b/cookies.go index 52cd7bc..d4fefbf 100644 --- a/cookies.go +++ b/cookies.go @@ -2,19 +2,11 @@ package requests import ( "errors" - _ "unsafe" - "net/http" "github.com/gospider007/gson" ) -//go:linkname readCookies net/http.readCookies -func readCookies(h http.Header, filter string) []*http.Cookie - -//go:linkname readSetCookies net/http.readSetCookies -func readSetCookies(h http.Header) []*http.Cookie - // 支持json,map,[]string,http.Header,string func ReadCookies(val any) (Cookies, error) { switch cook := val.(type) { diff --git a/dial.go b/dial.go index b1a9420..0f4f150 100644 --- a/dial.go +++ b/dial.go @@ -64,13 +64,11 @@ func NewDail(ctx context.Context, option DialOption) *DialClient { dialer: &net.Dialer{ Timeout: option.DialTimeout, KeepAlive: option.KeepAlive, + LocalAddr: option.LocalAddr, }, addrType: option.AddrType, getAddrType: option.GetAddrType, } - if option.LocalAddr != nil { - dialCli.dialer.LocalAddr = option.LocalAddr - } if option.Dns != nil { dialCli.dns = &net.UDPAddr{IP: option.Dns, Port: 53} dialCli.dialer.Resolver = &net.Resolver{ @@ -111,7 +109,6 @@ func (obj *DialClient) AddrToIp(ctx context.Context, addr string) (string, error } return net.JoinHostPort(host, port), nil } - func (obj *DialClient) clientVerifySocks5(ctx context.Context, proxyUrl *url.URL, addr string, conn net.Conn) (err error) { if _, err = conn.Write([]byte{5, 2, 0, 2}); err != nil { return @@ -229,10 +226,6 @@ func (obj *DialClient) clientVerifySocks5(ctx context.Context, proxyUrl *url.URL _, err = io.ReadFull(conn, readCon[:2]) return } -func cloneUrl(u *url.URL) *url.URL { - r := *u - return &r -} func (obj *DialClient) lookupIPAddr(ctx context.Context, host string) (net.IP, error) { var addrType int if obj.addrType != 0 { diff --git a/go.mod b/go.mod index fc1f958..f5a15b1 100644 --- a/go.mod +++ b/go.mod @@ -5,8 +5,8 @@ go 1.21.3 require ( github.com/gospider007/bar v0.0.0-20231024075629-3f50832a4cbf github.com/gospider007/bs4 v0.0.0-20231024075735-6bbdac929d8b - github.com/gospider007/gson v0.0.0-20231024092648-c97546a0287d - github.com/gospider007/gtls v0.0.0-20231024092712-01193b9f0404 + github.com/gospider007/gson v0.0.0-20231108025125-ff62d4066dc4 + github.com/gospider007/gtls v0.0.0-20231108025158-443489ca9953 github.com/gospider007/ja3 v0.0.0-20231029025157-38fc2f8f2d91 github.com/gospider007/net v0.0.0-20231028084010-313c148cf0a1 github.com/gospider007/re v0.0.0-20231024115818-adfd03636256 @@ -39,11 +39,11 @@ require ( github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect github.com/zeebo/blake3 v0.2.3 // indirect - go.mongodb.org/mongo-driver v1.12.1 // indirect + go.mongodb.org/mongo-driver v1.13.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.26.0 // indirect golang.org/x/crypto v0.14.0 // indirect - golang.org/x/image v0.13.0 // indirect + golang.org/x/image v0.14.0 // indirect golang.org/x/mod v0.14.0 // indirect golang.org/x/sys v0.14.0 // indirect golang.org/x/text v0.14.0 // indirect diff --git a/go.sum b/go.sum index 60d04d0..c29a3d5 100644 --- a/go.sum +++ b/go.sum @@ -33,8 +33,12 @@ github.com/gospider007/bs4 v0.0.0-20231024075735-6bbdac929d8b h1:S/lDIKvIwj7YyTm github.com/gospider007/bs4 v0.0.0-20231024075735-6bbdac929d8b/go.mod h1:n6GeaqU6PyhuPkKD74xSEC1+vITNpurtzhq/ikwBxcQ= github.com/gospider007/gson v0.0.0-20231024092648-c97546a0287d h1:3+FOSLWmVEq3erpgqtInm+v1me0OIm9xxr18YczoFP0= github.com/gospider007/gson v0.0.0-20231024092648-c97546a0287d/go.mod h1:8CJ5eDQBzhCtArlT6iHA+1TromaaD/N7tag0FuDB1k0= +github.com/gospider007/gson v0.0.0-20231108025125-ff62d4066dc4 h1:5s6Ird9YHFd858jV2DBEgJaVuDacV11VvvFd6a4nqRI= +github.com/gospider007/gson v0.0.0-20231108025125-ff62d4066dc4/go.mod h1:CGrR2aJGWEd96xr9TVCSOePEFWX1F3sBVaf4ZBvCbMI= github.com/gospider007/gtls v0.0.0-20231024092712-01193b9f0404 h1:Qp/o+l3KgWmviEsy0OXlFSjA0lbB8YtlcIOxN1xXyoI= github.com/gospider007/gtls v0.0.0-20231024092712-01193b9f0404/go.mod h1:fLcidMDKVv8b9NvLy0P/ZclltTaXJvTHANWiPCgDbSI= +github.com/gospider007/gtls v0.0.0-20231108025158-443489ca9953 h1:CN2nz4q88S3BhNR1QSmXubOgMGnxzWwdDL2OIASwYBs= +github.com/gospider007/gtls v0.0.0-20231108025158-443489ca9953/go.mod h1:EACbjhpG22ykBPE0xIKxc6D3N3YWTBoWrvsLLyhYqto= github.com/gospider007/ja3 v0.0.0-20231029025157-38fc2f8f2d91 h1:qQokihfTAX+/U8GIMvZauRtE4G+/1Jq8XIJx8xLr04A= github.com/gospider007/ja3 v0.0.0-20231029025157-38fc2f8f2d91/go.mod h1:ur78/uhYDDULSy1ldA/pPpGhjk973Q1VsPnbktXGU/g= github.com/gospider007/kinds v0.0.0-20231024093643-7a4424f2d30e h1:lmX6IQKkrNDbXfHsvrv1Uz0MoG2v5+4VC6Gdh9irUNY= @@ -109,6 +113,8 @@ github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo= github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4= go.mongodb.org/mongo-driver v1.12.1 h1:nLkghSU8fQNaK7oUmDhQFsnrtcoNy7Z6LVFKsEecqgE= go.mongodb.org/mongo-driver v1.12.1/go.mod h1:/rGBTebI3XYboVmgz+Wv3Bcbl3aD0QF9zl6kDDw18rQ= +go.mongodb.org/mongo-driver v1.13.0 h1:67DgFFjYOCMWdtTEmKFpV3ffWlFnh+CYZ8ZS/tXWUfY= +go.mongodb.org/mongo-driver v1.13.0/go.mod h1:/rGBTebI3XYboVmgz+Wv3Bcbl3aD0QF9zl6kDDw18rQ= go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -124,6 +130,8 @@ golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= golang.org/x/image v0.13.0 h1:3cge/F/QTkNLauhf2QoE9zp+7sr+ZcL4HnoZmdwg9sg= golang.org/x/image v0.13.0/go.mod h1:6mmbMOeV28HuMTgA6OSRkdXKYw/t5W9Uwn2Yv1r3Yxk= +golang.org/x/image v0.14.0 h1:tNgSxAFe3jC4uYqvZdTr84SZoM1KfwdC9SKIFrLjFn4= +golang.org/x/image v0.14.0/go.mod h1:HUYqC05R2ZcZ3ejNQsIHQDQiwWM4JBqmm6MKANTp4LE= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY= diff --git a/option.go b/option.go index 989f1a3..c5a9ea8 100644 --- a/option.go +++ b/option.go @@ -34,9 +34,10 @@ type RequestOption struct { ErrCallBack func(context.Context, *Client, error) error //error callback,if error is returnd,break request RequestCallBack func(context.Context, *http.Request, *http.Response) error //request and response callback,if error is returnd,reponse is error TryNum int //try num - RedirectNum int //redirect num ,<0 no redirect,==0 no limit + MaxRedirectNum int //redirect num ,<0 no redirect,==0 no limit Headers any //request headers：json,map，header ResponseHeaderTimeout time.Duration //ResponseHeaderTimeout ,default:30 + TlsHandshakeTimeout time.Duration //tls timeout,default:15 Stream bool //disable auto read Referer string //set headers referer value @@ -63,6 +64,13 @@ type RequestOption struct { body io.Reader once bool } +type File struct { + Key string + Val []byte + + FileName string + ContentType string +} var escapeQuotes = strings.NewReplacer("\\", "\\\\", `"`, "\\\"") @@ -71,7 +79,7 @@ func (obj *RequestOption) fileWrite(writer *multipart.Writer) (err error) { h := make(textproto.MIMEHeader) h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="%s"; filename="%s"`, escapeQuotes.Replace(file.Key), escapeQuotes.Replace(file.FileName))) if file.ContentType == "" { - h.Set("Content-Type", "application/octet-stream") + h.Set("Content-Type", http.DetectContentType(file.Val)) } else { h.Set("Content-Type", file.ContentType) } @@ -192,8 +200,8 @@ func (obj *Client) newRequestOption(option RequestOption) RequestOption { if !option.Bar { option.Bar = obj.bar } - if option.RedirectNum == 0 { - option.RedirectNum = obj.redirectNum + if option.MaxRedirectNum == 0 { + option.MaxRedirectNum = obj.maxRedirectNum } if option.Timeout == 0 { option.Timeout = obj.timeout @@ -201,6 +209,9 @@ func (obj *Client) newRequestOption(option RequestOption) RequestOption { if option.ResponseHeaderTimeout == 0 { option.ResponseHeaderTimeout = obj.responseHeaderTimeout } + if option.TlsHandshakeTimeout == 0 { + option.TlsHandshakeTimeout = obj.tlsHandshakeTimeout + } if !option.DisCookie { option.DisCookie = obj.disCookie } @@ -221,11 +232,12 @@ func (obj *Client) newRequestOption(option RequestOption) RequestOption { if option.OrderHeaders == nil { option.OrderHeaders = obj.orderHeaders } + if !option.Ja3Spec.IsSet() { - if option.Ja3 { - option.Ja3Spec = ja3.DefaultJa3Spec() - } else { + if obj.ja3Spec.IsSet() { option.Ja3Spec = obj.ja3Spec + } else if option.Ja3 { + option.Ja3Spec = ja3.DefaultJa3Spec() } } if !option.H2Ja3Spec.IsSet() { diff --git a/requests.go b/requests.go index ee95e42..27d45dd 100644 --- a/requests.go +++ b/requests.go @@ -1,7 +1,6 @@ package requests import ( - "bufio" "context" "errors" "fmt" @@ -11,7 +10,6 @@ import ( "net/url" "os" "strings" - _ "unsafe" "net/http" @@ -23,26 +21,22 @@ import ( "golang.org/x/exp/slices" ) -//go:linkname ReadRequest net/http.readRequest -func ReadRequest(b *bufio.Reader) (*http.Request, error) - type keyPrincipal string const keyPrincipalID keyPrincipal = "gospiderContextData" -var ( - errFatal = errors.New("Fatal error") -) +var errFatal = errors.New("Fatal error") type reqCtxData struct { isWs bool forceHttp1 bool - redirectNum int + maxRedirectNum int proxy *url.URL disProxy bool disAlive bool orderHeaders []string responseHeaderTimeout time.Duration + tlsHandshakeTimeout time.Duration requestCallBack func(context.Context, *http.Request, *http.Response) error @@ -237,10 +231,20 @@ func (obj *Client) request(preCtx context.Context, option *RequestOption) (respo var reqs *http.Request //init ctxData ctxData := new(reqCtxData) + ctxData.ja3Spec = option.Ja3Spec + ctxData.h2Ja3Spec = option.H2Ja3Spec ctxData.forceHttp1 = option.ForceHttp1 ctxData.disAlive = option.DisAlive + ctxData.maxRedirectNum = option.MaxRedirectNum ctxData.requestCallBack = option.RequestCallBack ctxData.responseHeaderTimeout = option.ResponseHeaderTimeout + + //init tls timeout + if option.TlsHandshakeTimeout == 0 { + ctxData.tlsHandshakeTimeout = time.Second * 15 + } else { + ctxData.tlsHandshakeTimeout = option.TlsHandshakeTimeout + } //init orderHeaders if option.OrderHeaders == nil { if option.Ja3Spec.IsSet() { @@ -274,14 +278,6 @@ func (obj *Client) request(preCtx context.Context, option *RequestOption) (respo ctxData.proxy = obj.proxy } } - //fingerprint - ctxData.ja3Spec = option.Ja3Spec - ctxData.h2Ja3Spec = option.H2Ja3Spec - - //redirect - if option.RedirectNum != 0 { - ctxData.redirectNum = option.RedirectNum - } //init ctx,cnl if option.Timeout > 0 { //超时 response.ctx, response.cnl = context.WithTimeout(context.WithValue(preCtx, keyPrincipalID, ctxData), option.Timeout) diff --git a/response.go b/response.go index 3533817..134ae5d 100644 --- a/response.go +++ b/response.go @@ -16,7 +16,6 @@ import ( "github.com/gospider007/bar" "github.com/gospider007/bs4" "github.com/gospider007/gson" - "github.com/gospider007/gtls" "github.com/gospider007/tools" "github.com/gospider007/websocket" ) @@ -206,11 +205,11 @@ func (obj *Response) Html() *bs4.Client { func (obj *Response) ContentType() string { if obj.filePath != "" { - return gtls.GetContentTypeWithBytes(obj.content) + return http.DetectContentType(obj.content) } contentType := obj.response.Header.Get("Content-Type") if contentType == "" { - contentType = gtls.GetContentTypeWithBytes(obj.content) + contentType = http.DetectContentType(obj.content) } return contentType } diff --git a/roundTripper.go b/roundTripper.go index 0df0bc8..9c93eab 100644 --- a/roundTripper.go +++ b/roundTripper.go @@ -4,30 +4,19 @@ import ( "bufio" "context" "crypto/tls" - "errors" - "fmt" - "io" "net" "net/url" "sync" - "sync/atomic" "time" "net/http" - _ "unsafe" "github.com/gospider007/gtls" "github.com/gospider007/net/http2" "github.com/gospider007/tools" utls "github.com/refraction-networking/utls" - "golang.org/x/exp/slices" - "golang.org/x/net/http/httpguts" ) -type roundTripper interface { - RoundTrip(*http.Request) (*http.Response, error) -} - type reqTask struct { ctx context.Context cnl context.CancelFunc @@ -39,61 +28,21 @@ type reqTask struct { responseHeaderTimeout time.Duration } +func newReqTask(req *http.Request, ctxData *reqCtxData) *reqTask { + if ctxData.responseHeaderTimeout == 0 { + ctxData.responseHeaderTimeout = time.Second * 30 + } + return &reqTask{ + req: req, + emptyPool: make(chan struct{}), + orderHeaders: ctxData.orderHeaders, + responseHeaderTimeout: ctxData.responseHeaderTimeout, + } +} func (obj *reqTask) inPool() bool { return obj.err == nil && obj.res != nil && obj.res.StatusCode != 101 && obj.res.Header.Get("Content-Type") != "text/event-stream" } -type connPool struct { - deleteCtx context.Context - deleteCnl context.CancelFunc - closeCtx context.Context - closeCnl context.CancelFunc - key connKey - total atomic.Int64 - tasks chan *reqTask - rt *RoundTripper -} - -func (obj *connPool) ForceClose() { - obj.deleteCnl() - delete(obj.rt.connPools, obj.key) -} -func (obj *connPool) Close() { - obj.closeCnl() - delete(obj.rt.connPools, obj.key) -} -func getAddr(uurl *url.URL) string { - if uurl == nil { - return "" - } - _, port, _ := net.SplitHostPort(uurl.Host) - if port == "" { - if uurl.Scheme == "https" { - port = "443" - } else { - port = "80" - } - return fmt.Sprintf("%s:%s", uurl.Host, port) - } - return uurl.Host -} -func getHost(req *http.Request) string { - host := req.Host - if host == "" { - host = req.URL.Host - } - _, port, _ := net.SplitHostPort(host) - if port == "" { - if req.URL.Scheme == "https" { - port = "443" - } else { - port = "80" - } - return fmt.Sprintf("%s:%s", host, port) - } - return host -} - type connKey struct { proxy string addr string @@ -113,6 +62,64 @@ func getKey(ctxData *reqCtxData, req *http.Request) connKey { addr: getAddr(req.URL), } } + +type RoundTripper struct { + ctx context.Context + cnl context.CancelFunc + connPools map[connKey]*connPool + connsLock sync.Mutex + dialer *DialClient + tlsConfig *tls.Config + utlsConfig *utls.Config + getProxy func(ctx context.Context, url *url.URL) (string, error) +} + +type RoundTripperOption struct { + DialTimeout time.Duration + KeepAlive time.Duration + LocalAddr *net.TCPAddr //network card ip + AddrType AddrType //first ip type + GetAddrType func(string) AddrType + Dns net.IP + GetProxy func(ctx context.Context, url *url.URL) (string, error) +} + +func newRoundTripper(preCtx context.Context, option RoundTripperOption) *RoundTripper { + if preCtx == nil { + preCtx = context.TODO() + } + ctx, cnl := context.WithCancel(preCtx) + dialClient := NewDail(ctx, DialOption{ + DialTimeout: option.DialTimeout, + Dns: option.Dns, + KeepAlive: option.KeepAlive, + LocalAddr: option.LocalAddr, + AddrType: option.AddrType, + GetAddrType: option.GetAddrType, + }) + tlsConfig := &tls.Config{ + InsecureSkipVerify: true, + SessionTicketKey: [32]byte{}, + ClientSessionCache: tls.NewLRUClientSessionCache(0), + } + utlsConfig := &utls.Config{ + InsecureSkipVerify: true, + InsecureSkipTimeVerify: true, + SessionTicketKey: [32]byte{}, + ClientSessionCache: utls.NewLRUClientSessionCache(0), + OmitEmptyPsk: true, + PreferSkipResumptionOnNilExtension: true, + } + return &RoundTripper{ + tlsConfig: tlsConfig, + utlsConfig: utlsConfig, + ctx: ctx, + cnl: cnl, + connPools: make(map[connKey]*connPool), + dialer: dialClient, + getProxy: option.GetProxy, + } +} func (obj *RoundTripper) newConnPool(conn *Connecotr) *connPool { pool := new(connPool) pool.deleteCtx, pool.deleteCnl = context.WithCancel(obj.ctx) @@ -173,7 +180,7 @@ func (obj *RoundTripper) dial(ctxData *reqCtxData, key *connKey, req *http.Reque conne.rc = make(chan []byte) conne.WithCancel(obj.ctx, obj.ctx) if req.URL.Scheme == "https" { - ctx, cnl := context.WithTimeout(req.Context(), obj.tlsHandshakeTimeout) + ctx, cnl := context.WithTimeout(req.Context(), ctxData.tlsHandshakeTimeout) defer cnl() if ctxData.ja3Spec.IsSet() { tlsConfig := obj.UtlsConfig() @@ -208,423 +215,6 @@ func (obj *RoundTripper) dial(ctxData *reqCtxData, key *connKey, req *http.Reque } return conne, err } - -type Connecotr struct { - key connKey - err error - deleteCtx context.Context //force close - deleteCnl context.CancelFunc - - closeCtx context.Context //safe close - closeCnl context.CancelFunc - - bodyCtx context.Context //body close - bodyCnl context.CancelFunc - - rawConn net.Conn - h2 bool - r *bufio.Reader - w *bufio.Writer - h2RawConn *http2.ClientConn - rc chan []byte - rn chan int - isRead bool - isPool bool -} - -func (obj *Connecotr) WithCancel(deleteCtx context.Context, closeCtx context.Context) { - obj.deleteCtx, obj.deleteCnl = context.WithCancel(deleteCtx) - obj.closeCtx, obj.closeCnl = context.WithCancel(closeCtx) -} -func (obj *Connecotr) Close() error { - obj.deleteCnl() - if obj.h2RawConn != nil { - obj.h2RawConn.Close() - } - return obj.rawConn.Close() -} -func (obj *Connecotr) read() { - if obj.isRead { - return - } - obj.isRead = true - defer obj.Close() - con := make([]byte, 4096) - var i int - for { - i, obj.err = obj.rawConn.Read(con) - b := con[:i] - for once := true; once || len(b) > 0; once = false { - select { - case obj.rc <- b: - select { - case nw := <-obj.rn: - b = b[nw:] - case <-obj.deleteCtx.Done(): - return - } - case <-obj.deleteCtx.Done(): - return - } - } - if obj.err != nil { - return - } - } -} -func (obj *Connecotr) Read(b []byte) (i int, err error) { - if !obj.isRead { - return obj.rawConn.Read(b) - } - select { - case con := <-obj.rc: - i, err = copy(b, con), obj.err - select { - case obj.rn <- i: - if i < len(con) { - err = nil - } - case <-obj.deleteCtx.Done(): - if err = obj.err; err == nil { - err = tools.WrapError(obj.deleteCtx.Err(), "connecotr close") - } - } - case <-obj.deleteCtx.Done(): - if err = obj.err; err == nil { - err = tools.WrapError(obj.deleteCtx.Err(), "connecotr close") - } - } - return -} -func (obj *Connecotr) Write(b []byte) (int, error) { - return obj.rawConn.Write(b) -} -func (obj *Connecotr) LocalAddr() net.Addr { - return obj.rawConn.LocalAddr() -} -func (obj *Connecotr) RemoteAddr() net.Addr { - return obj.rawConn.RemoteAddr() -} -func (obj *Connecotr) SetDeadline(t time.Time) error { - return obj.rawConn.SetDeadline(t) -} -func (obj *Connecotr) SetReadDeadline(t time.Time) error { - return obj.rawConn.SetReadDeadline(t) -} -func (obj *Connecotr) SetWriteDeadline(t time.Time) error { - return obj.rawConn.SetWriteDeadline(t) -} - -func (obj *Connecotr) h2Closed() bool { - state := obj.h2RawConn.State() - return state.Closed || state.Closing -} - -type ReadWriteCloser struct { - body io.ReadCloser - conn *Connecotr -} - -func (obj *ReadWriteCloser) Conn() net.Conn { - return obj.conn -} -func (obj *ReadWriteCloser) Read(p []byte) (n int, err error) { - return obj.body.Read(p) -} -func (obj *ReadWriteCloser) Close() (err error) { - err = obj.body.Close() - if !obj.InPool() { - obj.ForceDelete() - } else { - obj.conn.bodyCnl() - } - return -} -func (obj *ReadWriteCloser) InPool() bool { - return obj.conn.isPool -} -func (obj *ReadWriteCloser) Proxy() string { - return obj.conn.key.proxy -} -func (obj *ReadWriteCloser) Ja3() string { - return obj.conn.key.ja3 -} -func (obj *ReadWriteCloser) H2Ja3() string { - return obj.conn.key.h2Ja3 -} - -// safe close conn -func (obj *ReadWriteCloser) Delete() { - obj.conn.closeCnl() -} - -// force close conn -func (obj *ReadWriteCloser) ForceDelete() { - obj.conn.Close() -} - -func (obj *Connecotr) wrapBody(task *reqTask) { - body := new(ReadWriteCloser) - obj.bodyCtx, obj.bodyCnl = context.WithCancel(obj.deleteCtx) - body.body = task.res.Body - body.conn = obj - task.res.Body = body -} - -var replaceMap = map[string]string{ - "Sec-Ch-Ua": "sec-ch-ua", - "Sec-Ch-Ua-Mobile": "sec-ch-ua-mobile", - "Sec-Ch-Ua-Platform": "sec-ch-ua-platform", -} - -//go:linkname removeZone net/http.removeZone -func removeZone(host string) string - -//go:linkname shouldSendContentLength net/http.(*transferWriter).shouldSendContentLength -func shouldSendContentLength(t *http.Request) bool - -//go:linkname stringContainsCTLByte net/http.stringContainsCTLByte -func stringContainsCTLByte(s string) bool - -func httpWrite(r *http.Request, w *bufio.Writer, orderHeaders []string) (err error) { - host := r.Host - if host == "" { - host = r.URL.Host - } - host, err = httpguts.PunycodeHostPort(host) - if err != nil { - return err - } - if !httpguts.ValidHostHeader(host) { - return errors.New("http: invalid Host header") - } - host = removeZone(host) - ruri := r.URL.RequestURI() - if r.Method == "CONNECT" && r.URL.Path == "" { - // CONNECT requests normally give just the host and port, not a full URL. - ruri = host - if r.URL.Opaque != "" { - ruri = r.URL.Opaque - } - } - if stringContainsCTLByte(ruri) { - return errors.New("net/http: can't write control character in Request.URL") - } - if r.Header.Get("Host") == "" { - r.Header.Set("Host", host) - } - if r.Header.Get("User-Agent") == "" { - r.Header.Set("User-Agent", UserAgent) - } - if r.Header.Get("Content-Length") == "" && shouldSendContentLength(r) { - r.Header.Set("Content-Length", fmt.Sprint(r.ContentLength)) - } - if _, err = fmt.Fprintf(w, "%s %s HTTP/1.1\r\n", r.Method, ruri); err != nil { - return err - } - for _, k := range orderHeaders { - if k2, ok := replaceMap[k]; ok { - k = k2 - } - for _, v := range r.Header.Values(k) { - if _, err = fmt.Fprintf(w, "%s: %s\r\n", k, v); err != nil { - return err - } - } - } - for k, vs := range r.Header { - if !slices.Contains(orderHeaders, k) { - if k2, ok := replaceMap[k]; ok { - k = k2 - } - for _, v := range vs { - if _, err = fmt.Fprintf(w, "%s: %s\r\n", k, v); err != nil { - return err - } - } - } - } - if _, err = w.WriteString("\r\n"); err != nil { - return err - } - if r.Body != nil { - if _, err = io.Copy(w, r.Body); err != nil { - return err - } - } - return w.Flush() -} - -func (obj *Connecotr) http1Req(task *reqTask) { - defer task.cnl() - if task.orderHeaders != nil && len(task.orderHeaders) > 0 { - task.err = httpWrite(task.req, obj.w, task.orderHeaders) - } else if task.err = task.req.Write(obj); task.err == nil { - task.err = obj.w.Flush() - } - if task.err == nil { - if task.res, task.err = http.ReadResponse(obj.r, task.req); task.res != nil && task.err == nil { - obj.wrapBody(task) - } else if task.err != nil { - task.err = tools.WrapError(task.err, "http1 read error") - } - } else { - task.err = tools.WrapError(task.err, "http1 write error") - } -} - -func (obj *Connecotr) http2Req(task *reqTask) { - defer task.cnl() - if task.res, task.err = obj.h2RawConn.RoundTrip(task.req); task.res != nil && task.err == nil { - obj.wrapBody(task) - } else if task.err != nil { - task.err = tools.WrapError(task.err, "http2 roundTrip error") - } -} -func (obj *connPool) notice(task *reqTask) { - select { - case obj.tasks <- task: - case task.emptyPool <- struct{}{}: - } -} -func (obj *Connecotr) taskMain(task *reqTask, afterTime *time.Timer) (*http.Response, error, bool) { - if obj.h2 && obj.h2Closed() { - return nil, errors.New("conn is closed"), true - } - select { - case <-obj.closeCtx.Done(): - return nil, tools.WrapError(obj.closeCtx.Err(), "close ctx error: "), true - default: - } - if obj.h2 { - go obj.http2Req(task) - } else { - go obj.http1Req(task) - } - if afterTime == nil { - afterTime = time.NewTimer(task.responseHeaderTimeout) - } else { - afterTime.Reset(task.responseHeaderTimeout) - } - if !obj.isPool { - defer afterTime.Stop() - } - select { - case <-task.ctx.Done(): - if task.res != nil && task.err == nil && obj.isPool { - <-obj.bodyCtx.Done() //wait body close - } - case <-obj.deleteCtx.Done(): //force conn close - task.err = tools.WrapError(obj.deleteCtx.Err(), "delete ctx error: ") - task.cnl() - case <-afterTime.C: - task.err = errors.New("response Header is Timeout") - task.cnl() - } - return task.res, task.err, false -} - -func (obj *connPool) rwMain(conn *Connecotr) { - conn.WithCancel(obj.deleteCtx, obj.closeCtx) - var afterTime *time.Timer - defer func() { - if afterTime != nil { - afterTime.Stop() - } - obj.rt.connsLock.Lock() - defer obj.rt.connsLock.Unlock() - conn.Close() - obj.total.Add(-1) - if obj.total.Load() <= 0 { - obj.Close() - } - }() - select { - case <-conn.deleteCtx.Done(): //force close all conn - return - case <-conn.bodyCtx.Done(): //wait body close - } - for { - select { - case <-conn.closeCtx.Done(): //safe close conn - return - case task := <-obj.tasks: //recv task - res, err, notice := conn.taskMain(task, afterTime) - if notice { - obj.notice(task) - return - } - if res == nil || err != nil { - return - } - } - } -} - -type RoundTripper struct { - ctx context.Context - cnl context.CancelFunc - connPools map[connKey]*connPool - connsLock sync.Mutex - dialer *DialClient - tlsConfig *tls.Config - utlsConfig *utls.Config - getProxy func(ctx context.Context, url *url.URL) (string, error) - tlsHandshakeTimeout time.Duration -} - -type RoundTripperOption struct { - TlsHandshakeTimeout time.Duration - DialTimeout time.Duration - KeepAlive time.Duration - LocalAddr *net.TCPAddr //network card ip - AddrType AddrType //first ip type - GetAddrType func(string) AddrType - Dns net.IP - GetProxy func(ctx context.Context, url *url.URL) (string, error) -} - -func newRoundTripper(preCtx context.Context, option RoundTripperOption) *RoundTripper { - if preCtx == nil { - preCtx = context.TODO() - } - ctx, cnl := context.WithCancel(preCtx) - - dialClient := NewDail(ctx, DialOption{ - DialTimeout: option.DialTimeout, - Dns: option.Dns, - KeepAlive: option.KeepAlive, - LocalAddr: option.LocalAddr, - AddrType: option.AddrType, - GetAddrType: option.GetAddrType, - }) - if option.TlsHandshakeTimeout == 0 { - option.TlsHandshakeTimeout = time.Second * 15 - } - tlsConfig := &tls.Config{ - InsecureSkipVerify: true, - SessionTicketKey: [32]byte{}, - ClientSessionCache: tls.NewLRUClientSessionCache(0), - } - utlsConfig := &utls.Config{ - InsecureSkipVerify: true, - InsecureSkipTimeVerify: true, - SessionTicketKey: [32]byte{}, - ClientSessionCache: utls.NewLRUClientSessionCache(0), - OmitEmptyPsk: true, - PreferSkipResumptionOnNilExtension: true, - } - return &RoundTripper{ - tlsConfig: tlsConfig, - utlsConfig: utlsConfig, - ctx: ctx, - cnl: cnl, - connPools: make(map[connKey]*connPool), - dialer: dialClient, - getProxy: option.GetProxy, - tlsHandshakeTimeout: option.TlsHandshakeTimeout, - } -} func (obj *RoundTripper) SetGetProxy(getProxy func(ctx context.Context, url *url.URL) (string, error)) { obj.getProxy = getProxy } @@ -678,17 +268,6 @@ func (obj *RoundTripper) CloseConnections() { delete(obj.connPools, key) } } -func newReqTask(req *http.Request, ctxData *reqCtxData) *reqTask { - if ctxData.responseHeaderTimeout == 0 { - ctxData.responseHeaderTimeout = time.Second * 30 - } - return &reqTask{ - req: req, - emptyPool: make(chan struct{}), - orderHeaders: ctxData.orderHeaders, - responseHeaderTimeout: ctxData.responseHeaderTimeout, - } -} func (obj *RoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { ctxData := req.Context().Value(keyPrincipalID).(*reqCtxData) if ctxData.requestCallBack != nil { diff --git a/rw.go b/rw.go new file mode 100644 index 0000000..e4a6632 --- /dev/null +++ b/rw.go @@ -0,0 +1,49 @@ +package requests + +import ( + "io" + "net" +) + +type ReadWriteCloser struct { + body io.ReadCloser + conn *Connecotr +} + +func (obj *ReadWriteCloser) Conn() net.Conn { + return obj.conn +} +func (obj *ReadWriteCloser) Read(p []byte) (n int, err error) { + return obj.body.Read(p) +} +func (obj *ReadWriteCloser) Close() (err error) { + err = obj.body.Close() + if !obj.InPool() { + obj.ForceDelete() + } else { + obj.conn.bodyCnl() + } + return +} +func (obj *ReadWriteCloser) InPool() bool { + return obj.conn.isPool +} +func (obj *ReadWriteCloser) Proxy() string { + return obj.conn.key.proxy +} +func (obj *ReadWriteCloser) Ja3() string { + return obj.conn.key.ja3 +} +func (obj *ReadWriteCloser) H2Ja3() string { + return obj.conn.key.h2Ja3 +} + +// safe close conn +func (obj *ReadWriteCloser) Delete() { + obj.conn.closeCnl() +} + +// force close conn +func (obj *ReadWriteCloser) ForceDelete() { + obj.conn.Close() +} diff --git a/tools.go b/tools.go new file mode 100644 index 0000000..88e487f --- /dev/null +++ b/tools.go @@ -0,0 +1,144 @@ +package requests + +import ( + "bufio" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + _ "unsafe" + + "golang.org/x/exp/slices" + "golang.org/x/net/http/httpguts" +) + +func cloneUrl(u *url.URL) *url.URL { + r := *u + return &r +} +func getAddr(uurl *url.URL) string { + if uurl == nil { + return "" + } + _, port, _ := net.SplitHostPort(uurl.Host) + if port == "" { + if uurl.Scheme == "https" { + port = "443" + } else { + port = "80" + } + return fmt.Sprintf("%s:%s", uurl.Host, port) + } + return uurl.Host +} +func getHost(req *http.Request) string { + host := req.Host + if host == "" { + host = req.URL.Host + } + _, port, _ := net.SplitHostPort(host) + if port == "" { + if req.URL.Scheme == "https" { + port = "443" + } else { + port = "80" + } + return fmt.Sprintf("%s:%s", host, port) + } + return host +} + +var replaceMap = map[string]string{ + "Sec-Ch-Ua": "sec-ch-ua", + "Sec-Ch-Ua-Mobile": "sec-ch-ua-mobile", + "Sec-Ch-Ua-Platform": "sec-ch-ua-platform", +} + +//go:linkname readCookies net/http.readCookies +func readCookies(h http.Header, filter string) []*http.Cookie + +//go:linkname readSetCookies net/http.readSetCookies +func readSetCookies(h http.Header) []*http.Cookie + +//go:linkname ReadRequest net/http.readRequest +func ReadRequest(b *bufio.Reader) (*http.Request, error) + +//go:linkname removeZone net/http.removeZone +func removeZone(host string) string + +//go:linkname shouldSendContentLength net/http.(*transferWriter).shouldSendContentLength +func shouldSendContentLength(t *http.Request) bool + +//go:linkname stringContainsCTLByte net/http.stringContainsCTLByte +func stringContainsCTLByte(s string) bool + +func httpWrite(r *http.Request, w *bufio.Writer, orderHeaders []string) (err error) { + host := r.Host + if host == "" { + host = r.URL.Host + } + host, err = httpguts.PunycodeHostPort(host) + if err != nil { + return err + } + if !httpguts.ValidHostHeader(host) { + return errors.New("http: invalid Host header") + } + host = removeZone(host) + ruri := r.URL.RequestURI() + if r.Method == "CONNECT" && r.URL.Path == "" { + // CONNECT requests normally give just the host and port, not a full URL. + ruri = host + if r.URL.Opaque != "" { + ruri = r.URL.Opaque + } + } + if stringContainsCTLByte(ruri) { + return errors.New("net/http: can't write control character in Request.URL") + } + if r.Header.Get("Host") == "" { + r.Header.Set("Host", host) + } + if r.Header.Get("User-Agent") == "" { + r.Header.Set("User-Agent", UserAgent) + } + if r.Header.Get("Content-Length") == "" && shouldSendContentLength(r) { + r.Header.Set("Content-Length", fmt.Sprint(r.ContentLength)) + } + if _, err = fmt.Fprintf(w, "%s %s HTTP/1.1\r\n", r.Method, ruri); err != nil { + return err + } + for _, k := range orderHeaders { + if k2, ok := replaceMap[k]; ok { + k = k2 + } + for _, v := range r.Header.Values(k) { + if _, err = fmt.Fprintf(w, "%s: %s\r\n", k, v); err != nil { + return err + } + } + } + for k, vs := range r.Header { + if !slices.Contains(orderHeaders, k) { + if k2, ok := replaceMap[k]; ok { + k = k2 + } + for _, v := range vs { + if _, err = fmt.Fprintf(w, "%s: %s\r\n", k, v); err != nil { + return err + } + } + } + } + if _, err = w.WriteString("\r\n"); err != nil { + return err + } + if r.Body != nil { + if _, err = io.Copy(w, r.Body); err != nil { + return err + } + } + return w.Flush() +}
+
+
+