mirror of
https://github.com/gospider007/requests.git
synced 2025-12-24 13:57:52 +08:00
优化代码结构
This commit is contained in:
9
body.go
9
body.go
@@ -129,8 +129,8 @@ func (obj *OrderMap) parseForm(ctx context.Context) (io.Reader, string, bool, er
|
||||
if err == nil {
|
||||
err = io.EOF
|
||||
}
|
||||
pr.Close(err)
|
||||
pw.Close(err)
|
||||
pr.CloseWitError(err)
|
||||
pw.CloseWitError(err)
|
||||
}()
|
||||
return pr, writer.FormDataContentType(), true, nil
|
||||
}
|
||||
@@ -207,7 +207,7 @@ func (obj *OrderMap) parseParams() *bytes.Buffer {
|
||||
}
|
||||
return buf
|
||||
}
|
||||
func (obj *OrderMap) parseData() *bytes.Reader {
|
||||
func (obj *OrderMap) parseData() io.Reader {
|
||||
val := obj.parseParams().Bytes()
|
||||
if val == nil {
|
||||
return nil
|
||||
@@ -285,6 +285,9 @@ func (obj *RequestOption) newBody(val any, valType int) (reader io.Reader, parse
|
||||
}
|
||||
if valType == readType {
|
||||
switch value := val.(type) {
|
||||
case io.ReadCloser:
|
||||
obj.once = true
|
||||
return value, nil, nil, nil
|
||||
case io.Reader:
|
||||
obj.once = true
|
||||
return value, nil, nil, nil
|
||||
|
||||
17
client.go
17
client.go
@@ -2,6 +2,7 @@ package requests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
@@ -9,6 +10,7 @@ import (
|
||||
"net/http"
|
||||
|
||||
"github.com/gospider007/gtls"
|
||||
utls "github.com/refraction-networking/utls"
|
||||
)
|
||||
|
||||
// Connection Management
|
||||
@@ -35,6 +37,21 @@ func NewClient(preCtx context.Context, options ...ClientOption) (*Client, error)
|
||||
result.ctx, result.cnl = context.WithCancel(preCtx)
|
||||
result.transport = newRoundTripper(result.ctx, option)
|
||||
result.option = option
|
||||
if result.option.TlsConfig == nil {
|
||||
result.option.TlsConfig = &tls.Config{
|
||||
InsecureSkipVerify: true,
|
||||
ClientSessionCache: tls.NewLRUClientSessionCache(0),
|
||||
}
|
||||
}
|
||||
if result.option.UtlsConfig == nil {
|
||||
result.option.UtlsConfig = &utls.Config{
|
||||
InsecureSkipVerify: true,
|
||||
ClientSessionCache: utls.NewLRUClientSessionCache(0),
|
||||
InsecureSkipTimeVerify: true,
|
||||
OmitEmptyPsk: true,
|
||||
PreferSkipResumptionOnNilExtension: true,
|
||||
}
|
||||
}
|
||||
//cookiesjar
|
||||
if !result.option.DisCookie {
|
||||
if result.option.Jar == nil {
|
||||
|
||||
216
conn.go
216
conn.go
@@ -5,6 +5,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"iter"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
@@ -12,11 +13,90 @@ import (
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/gospider007/http2"
|
||||
"github.com/gospider007/http3"
|
||||
"github.com/gospider007/tools"
|
||||
)
|
||||
|
||||
type Conn interface {
|
||||
CloseWithError(err error) error
|
||||
DoRequest(*http.Request, []string) (*http.Response, error)
|
||||
}
|
||||
|
||||
type conn struct {
|
||||
r *bufio.Reader
|
||||
w *bufio.Writer
|
||||
pr *pipCon
|
||||
pw *pipCon
|
||||
conn net.Conn
|
||||
closeFunc func()
|
||||
}
|
||||
|
||||
func newConn(ctx context.Context, con net.Conn, closeFunc func()) *conn {
|
||||
c := &conn{
|
||||
conn: con,
|
||||
closeFunc: closeFunc,
|
||||
}
|
||||
c.pr, c.pw = pipe(ctx)
|
||||
c.r = bufio.NewReader(c)
|
||||
c.w = bufio.NewWriter(c)
|
||||
go c.run()
|
||||
return c
|
||||
}
|
||||
func (obj *conn) Close() error {
|
||||
return obj.CloseWithError(nil)
|
||||
}
|
||||
func (obj *conn) CloseWithError(err error) error {
|
||||
if err == nil {
|
||||
err = errors.New("connecotr closeWithError close")
|
||||
} else {
|
||||
err = tools.WrapError(err, "connecotr closeWithError close")
|
||||
}
|
||||
if obj.closeFunc != nil {
|
||||
obj.closeFunc()
|
||||
}
|
||||
obj.conn.Close()
|
||||
return obj.pr.CloseWitError(err)
|
||||
}
|
||||
func (obj *conn) DoRequest(req *http.Request, orderHeaders []string) (*http.Response, error) {
|
||||
err := httpWrite(req, obj.w, orderHeaders)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res, err := http.ReadResponse(obj.r, req)
|
||||
if err != nil {
|
||||
err = tools.WrapError(err, "http1 read error")
|
||||
return nil, err
|
||||
}
|
||||
if res == nil {
|
||||
err = errors.New("response is nil")
|
||||
}
|
||||
return res, err
|
||||
}
|
||||
func (obj *conn) run() (err error) {
|
||||
_, err = io.Copy(obj.pw, obj.conn)
|
||||
return obj.CloseWithError(err)
|
||||
}
|
||||
func (obj *conn) Read(b []byte) (i int, err error) {
|
||||
return obj.pr.Read(b)
|
||||
}
|
||||
func (obj *conn) Write(b []byte) (int, error) {
|
||||
return obj.conn.Write(b)
|
||||
}
|
||||
func (obj *conn) LocalAddr() net.Addr {
|
||||
return obj.conn.LocalAddr()
|
||||
}
|
||||
func (obj *conn) RemoteAddr() net.Addr {
|
||||
return obj.conn.RemoteAddr()
|
||||
}
|
||||
func (obj *conn) SetDeadline(t time.Time) error {
|
||||
return obj.conn.SetDeadline(t)
|
||||
}
|
||||
func (obj *conn) SetReadDeadline(t time.Time) error {
|
||||
return obj.conn.SetReadDeadline(t)
|
||||
}
|
||||
func (obj *conn) SetWriteDeadline(t time.Time) error {
|
||||
return obj.conn.SetWriteDeadline(t)
|
||||
}
|
||||
|
||||
type connecotr struct {
|
||||
parentForceCtx context.Context //parent force close
|
||||
forceCtx context.Context //force close
|
||||
@@ -27,14 +107,9 @@ type connecotr struct {
|
||||
bodyCtx context.Context //body close
|
||||
bodyCnl context.CancelCauseFunc
|
||||
|
||||
rawConn net.Conn
|
||||
h2RawConn *http2.Http2ClientConn
|
||||
h3RawConn http3.RoundTripper
|
||||
proxys []*url.URL
|
||||
r *bufio.Reader
|
||||
w *bufio.Writer
|
||||
pr *pipCon
|
||||
inPool bool
|
||||
rawConn Conn
|
||||
proxys []*url.URL
|
||||
// inPool bool
|
||||
}
|
||||
|
||||
func (obj *connecotr) withCancel(forceCtx context.Context, safeCtx context.Context) {
|
||||
@@ -43,62 +118,7 @@ func (obj *connecotr) withCancel(forceCtx context.Context, safeCtx context.Conte
|
||||
obj.safeCtx, obj.safeCnl = context.WithCancelCause(safeCtx)
|
||||
}
|
||||
func (obj *connecotr) Close() error {
|
||||
return obj.closeWithError(errors.New("connecotr Close close"))
|
||||
}
|
||||
func (obj *connecotr) closeWithError(err error) error {
|
||||
if err == nil {
|
||||
err = errors.New("connecotr closeWithError close")
|
||||
} else {
|
||||
err = tools.WrapError(err, "connecotr closeWithError close")
|
||||
}
|
||||
obj.forceCnl(err)
|
||||
if obj.pr != nil {
|
||||
obj.pr.Close(err)
|
||||
}
|
||||
if obj.h2RawConn != nil {
|
||||
obj.h2RawConn.Close()
|
||||
}
|
||||
if obj.h3RawConn != nil {
|
||||
return obj.h3RawConn.Close(err.Error())
|
||||
}
|
||||
return obj.rawConn.Close()
|
||||
}
|
||||
func (obj *connecotr) read() (err error) {
|
||||
if obj.pr != nil {
|
||||
return nil
|
||||
}
|
||||
var pw *pipCon
|
||||
obj.pr, pw = pipe(obj.forceCtx)
|
||||
if _, err = io.Copy(pw, obj.rawConn); err == nil {
|
||||
err = io.EOF
|
||||
}
|
||||
pw.Close(err)
|
||||
obj.closeWithError(err)
|
||||
return
|
||||
}
|
||||
func (obj *connecotr) Read(b []byte) (i int, err error) {
|
||||
if obj.pr == nil {
|
||||
return obj.rawConn.Read(b)
|
||||
}
|
||||
return obj.pr.Read(b)
|
||||
}
|
||||
func (obj *connecotr) Write(b []byte) (int, error) {
|
||||
return obj.rawConn.Write(b)
|
||||
}
|
||||
func (obj *connecotr) LocalAddr() net.Addr {
|
||||
return obj.rawConn.LocalAddr()
|
||||
}
|
||||
func (obj *connecotr) RemoteAddr() net.Addr {
|
||||
return obj.rawConn.RemoteAddr()
|
||||
}
|
||||
func (obj *connecotr) SetDeadline(t time.Time) error {
|
||||
return obj.rawConn.SetDeadline(t)
|
||||
}
|
||||
func (obj *connecotr) SetReadDeadline(t time.Time) error {
|
||||
return obj.rawConn.SetReadDeadline(t)
|
||||
}
|
||||
func (obj *connecotr) SetWriteDeadline(t time.Time) error {
|
||||
return obj.rawConn.SetWriteDeadline(t)
|
||||
return obj.rawConn.CloseWithError(errors.New("connecotr Close close"))
|
||||
}
|
||||
|
||||
func (obj *connecotr) wrapBody(task *reqTask) {
|
||||
@@ -108,36 +128,15 @@ func (obj *connecotr) wrapBody(task *reqTask) {
|
||||
body.conn = obj
|
||||
task.res.Body = body
|
||||
}
|
||||
func (obj *connecotr) http1Req(task *reqTask, done chan struct{}) {
|
||||
if task.err = httpWrite(task.req, obj.w, task.orderHeaders); task.err == nil {
|
||||
task.res, task.err = http.ReadResponse(obj.r, task.req)
|
||||
if task.err != nil {
|
||||
task.err = tools.WrapError(task.err, "http1 read error")
|
||||
} else if task.res == nil {
|
||||
task.err = errors.New("response is nil")
|
||||
} else {
|
||||
obj.wrapBody(task)
|
||||
}
|
||||
func (obj *connecotr) httpReq(task *reqTask, done chan struct{}) {
|
||||
if task.res, task.err = obj.rawConn.DoRequest(task.req, task.option.OrderHeaders); task.res != nil && task.err == nil {
|
||||
obj.wrapBody(task)
|
||||
} else if task.err != nil {
|
||||
task.err = tools.WrapError(task.err, "http1 roundTrip error")
|
||||
}
|
||||
close(done)
|
||||
}
|
||||
|
||||
func (obj *connecotr) http2Req(task *reqTask, done chan struct{}) {
|
||||
if task.res, task.err = obj.h2RawConn.DoRequest(task.req, task.orderHeaders2); task.res != nil && task.err == nil {
|
||||
obj.wrapBody(task)
|
||||
} else if task.err != nil {
|
||||
task.err = tools.WrapError(task.err, "http2 roundTrip error")
|
||||
}
|
||||
close(done)
|
||||
}
|
||||
func (obj *connecotr) http3Req(task *reqTask, done chan struct{}) {
|
||||
if task.res, task.err = obj.h3RawConn.RoundTrip(task.req); task.res != nil && task.err == nil {
|
||||
obj.wrapBody(task)
|
||||
} else if task.err != nil {
|
||||
task.err = tools.WrapError(task.err, "http2 roundTrip error")
|
||||
}
|
||||
close(done)
|
||||
}
|
||||
func (obj *connecotr) waitBodyClose() error {
|
||||
select {
|
||||
case <-obj.bodyCtx.Done(): //wait body close
|
||||
@@ -155,14 +154,14 @@ func (obj *connecotr) taskMain(task *reqTask, waitBody bool) (retry bool) {
|
||||
defer func() {
|
||||
if retry {
|
||||
task.err = nil
|
||||
obj.closeWithError(errors.New("taskMain retry close"))
|
||||
obj.rawConn.CloseWithError(errors.New("taskMain retry close"))
|
||||
} else {
|
||||
task.cnl()
|
||||
if task.err != nil {
|
||||
obj.closeWithError(task.err)
|
||||
obj.rawConn.CloseWithError(task.err)
|
||||
} else if waitBody {
|
||||
if err := obj.waitBodyClose(); err != nil {
|
||||
obj.closeWithError(err)
|
||||
obj.rawConn.CloseWithError(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -175,13 +174,7 @@ func (obj *connecotr) taskMain(task *reqTask, waitBody bool) (retry bool) {
|
||||
default:
|
||||
}
|
||||
done := make(chan struct{})
|
||||
if obj.h3RawConn != nil {
|
||||
go obj.http3Req(task, done)
|
||||
} else if obj.h2RawConn != nil {
|
||||
go obj.http2Req(task, done)
|
||||
} else {
|
||||
go obj.http1Req(task, done)
|
||||
}
|
||||
go obj.httpReq(task, done)
|
||||
select {
|
||||
case <-task.ctx.Done():
|
||||
task.err = tools.WrapError(context.Cause(task.ctx), "task.ctx error: ")
|
||||
@@ -197,9 +190,9 @@ func (obj *connecotr) taskMain(task *reqTask, waitBody bool) (retry bool) {
|
||||
}
|
||||
return task.suppertRetry()
|
||||
}
|
||||
if task.ctxData.logger != nil {
|
||||
task.ctxData.logger(Log{
|
||||
Id: task.ctxData.requestId,
|
||||
if task.option.Logger != nil {
|
||||
task.option.Logger(Log{
|
||||
Id: task.option.requestId,
|
||||
Time: time.Now(),
|
||||
Type: LogType_ResponseHeader,
|
||||
Msg: "response header",
|
||||
@@ -257,11 +250,12 @@ func (obj *connPools) set(key string, pool *connPool) {
|
||||
func (obj *connPools) del(key string) {
|
||||
obj.connPools.Delete(key)
|
||||
}
|
||||
|
||||
func (obj *connPools) iter(f func(key string, value *connPool) bool) {
|
||||
obj.connPools.Range(func(key, value any) bool {
|
||||
return f(key.(string), value.(*connPool))
|
||||
})
|
||||
func (obj *connPools) Range() iter.Seq2[string, *connPool] {
|
||||
return func(yield func(string, *connPool) bool) {
|
||||
obj.connPools.Range(func(key, value any) bool {
|
||||
return yield(key.(string), value.(*connPool))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (obj *connPool) notice(task *reqTask) {
|
||||
@@ -274,7 +268,7 @@ func (obj *connPool) notice(task *reqTask) {
|
||||
func (obj *connPool) rwMain(conn *connecotr) {
|
||||
conn.withCancel(obj.forceCtx, obj.safeCtx)
|
||||
defer func() {
|
||||
conn.closeWithError(errors.New("connPool rwMain close"))
|
||||
conn.rawConn.CloseWithError(errors.New("connPool rwMain close"))
|
||||
obj.total.Add(-1)
|
||||
if obj.total.Load() <= 0 {
|
||||
obj.safeClose()
|
||||
|
||||
102
dial.go
102
dial.go
@@ -83,9 +83,9 @@ func NewDail(option DialOption) *DialClient {
|
||||
getAddrType: option.GetAddrType,
|
||||
}
|
||||
}
|
||||
func (obj *DialClient) dialContext(ctx context.Context, ctxData *reqCtxData, network string, addr string, isProxy bool) (net.Conn, error) {
|
||||
if ctxData == nil {
|
||||
ctxData = &reqCtxData{}
|
||||
func (obj *DialClient) dialContext(ctx context.Context, option *RequestOption, network string, addr string, isProxy bool) (net.Conn, error) {
|
||||
if option == nil {
|
||||
option = &RequestOption{}
|
||||
}
|
||||
host, port, err := net.SplitHostPort(addr)
|
||||
if err != nil {
|
||||
@@ -95,10 +95,10 @@ func (obj *DialClient) dialContext(ctx context.Context, ctxData *reqCtxData, net
|
||||
if _, ipInt := gtls.ParseHost(host); ipInt == 0 { //domain
|
||||
host, ok := obj.loadHost(host)
|
||||
if !ok { //dns parse
|
||||
dialer = obj.getDialer(ctxData, true)
|
||||
dialer = obj.getDialer(option, true)
|
||||
var addrType gtls.AddrType
|
||||
if ctxData.addrType != 0 {
|
||||
addrType = ctxData.addrType
|
||||
if option.AddrType != 0 {
|
||||
addrType = option.AddrType
|
||||
} else if obj.getAddrType != nil {
|
||||
addrType = obj.getAddrType(host)
|
||||
}
|
||||
@@ -109,17 +109,17 @@ func (obj *DialClient) dialContext(ctx context.Context, ctxData *reqCtxData, net
|
||||
if host, err = obj.addrToIp(host, ips, addrType); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if ctxData.logger != nil {
|
||||
if option.Logger != nil {
|
||||
if isProxy {
|
||||
ctxData.logger(Log{
|
||||
Id: ctxData.requestId,
|
||||
option.Logger(Log{
|
||||
Id: option.requestId,
|
||||
Time: time.Now(),
|
||||
Type: LogType_ProxyDNSLookup,
|
||||
Msg: host,
|
||||
})
|
||||
} else {
|
||||
ctxData.logger(Log{
|
||||
Id: ctxData.requestId,
|
||||
option.Logger(Log{
|
||||
Id: option.requestId,
|
||||
Time: time.Now(),
|
||||
Type: LogType_DNSLookup,
|
||||
Msg: host,
|
||||
@@ -130,20 +130,20 @@ func (obj *DialClient) dialContext(ctx context.Context, ctxData *reqCtxData, net
|
||||
}
|
||||
}
|
||||
if dialer == nil {
|
||||
dialer = obj.getDialer(ctxData, false)
|
||||
dialer = obj.getDialer(option, false)
|
||||
}
|
||||
con, err := dialer.DialContext(ctx, network, addr)
|
||||
if ctxData.logger != nil {
|
||||
if option.Logger != nil {
|
||||
if isProxy {
|
||||
ctxData.logger(Log{
|
||||
Id: ctxData.requestId,
|
||||
option.Logger(Log{
|
||||
Id: option.requestId,
|
||||
Time: time.Now(),
|
||||
Type: LogType_ProxyTCPConnect,
|
||||
Msg: addr,
|
||||
})
|
||||
} else {
|
||||
ctxData.logger(Log{
|
||||
Id: ctxData.requestId,
|
||||
option.Logger(Log{
|
||||
Id: option.requestId,
|
||||
Time: time.Now(),
|
||||
Type: LogType_TCPConnect,
|
||||
Msg: addr,
|
||||
@@ -152,14 +152,14 @@ func (obj *DialClient) dialContext(ctx context.Context, ctxData *reqCtxData, net
|
||||
}
|
||||
return con, err
|
||||
}
|
||||
func (obj *DialClient) DialContext(ctx context.Context, ctxData *reqCtxData, network string, addr string) (net.Conn, error) {
|
||||
func (obj *DialClient) DialContext(ctx context.Context, ctxData *RequestOption, network string, addr string) (net.Conn, error) {
|
||||
return obj.dialContext(ctx, ctxData, network, addr, false)
|
||||
}
|
||||
func (obj *DialClient) ProxyDialContext(ctx context.Context, ctxData *reqCtxData, network string, addr string) (net.Conn, error) {
|
||||
func (obj *DialClient) ProxyDialContext(ctx context.Context, ctxData *RequestOption, network string, addr string) (net.Conn, error) {
|
||||
return obj.dialContext(ctx, ctxData, network, addr, true)
|
||||
}
|
||||
|
||||
func (obj *DialClient) DialProxyContext(ctx context.Context, ctxData *reqCtxData, network string, proxyTlsConfig *tls.Config, proxyUrls ...*url.URL) (net.Conn, error) {
|
||||
func (obj *DialClient) DialProxyContext(ctx context.Context, ctxData *RequestOption, network string, proxyTlsConfig *tls.Config, proxyUrls ...*url.URL) (net.Conn, error) {
|
||||
proxyLen := len(proxyUrls)
|
||||
if proxyLen < 2 {
|
||||
return nil, errors.New("proxyUrls is nil")
|
||||
@@ -196,12 +196,12 @@ func getProxyAddr(proxyUrl *url.URL) (addr string, err error) {
|
||||
}
|
||||
return
|
||||
}
|
||||
func (obj *DialClient) dialProxyContext(ctx context.Context, ctxData *reqCtxData, network string, proxyUrl *url.URL) (net.Conn, error) {
|
||||
func (obj *DialClient) dialProxyContext(ctx context.Context, ctxData *RequestOption, network string, proxyUrl *url.URL) (net.Conn, error) {
|
||||
if proxyUrl == nil {
|
||||
return nil, errors.New("proxyUrl is nil")
|
||||
}
|
||||
if ctxData == nil {
|
||||
ctxData = &reqCtxData{}
|
||||
ctxData = &RequestOption{}
|
||||
}
|
||||
addr, err := getProxyAddr(proxyUrl)
|
||||
if err != nil {
|
||||
@@ -210,15 +210,15 @@ func (obj *DialClient) dialProxyContext(ctx context.Context, ctxData *reqCtxData
|
||||
return obj.ProxyDialContext(ctx, ctxData, network, addr)
|
||||
}
|
||||
|
||||
func (obj *DialClient) verifyProxyToRemote(ctx context.Context, ctxData *reqCtxData, conn net.Conn, proxyTlsConfig *tls.Config, proxyUrl *url.URL, remoteUrl *url.URL) (net.Conn, error) {
|
||||
func (obj *DialClient) verifyProxyToRemote(ctx context.Context, option *RequestOption, conn net.Conn, proxyTlsConfig *tls.Config, proxyUrl *url.URL, remoteUrl *url.URL) (net.Conn, error) {
|
||||
var err error
|
||||
if proxyUrl.Scheme == "https" {
|
||||
if conn, err = obj.addTls(ctx, conn, proxyUrl.Host, true, proxyTlsConfig); err != nil {
|
||||
return conn, err
|
||||
}
|
||||
if ctxData.logger != nil {
|
||||
ctxData.logger(Log{
|
||||
Id: ctxData.requestId,
|
||||
if option.Logger != nil {
|
||||
option.Logger(Log{
|
||||
Id: option.requestId,
|
||||
Time: time.Now(),
|
||||
Type: LogType_ProxyTLSHandshake,
|
||||
Msg: proxyUrl.String(),
|
||||
@@ -228,9 +228,9 @@ func (obj *DialClient) verifyProxyToRemote(ctx context.Context, ctxData *reqCtxD
|
||||
switch proxyUrl.Scheme {
|
||||
case "http", "https":
|
||||
err = obj.clientVerifyHttps(ctx, conn, proxyUrl, remoteUrl)
|
||||
if ctxData.logger != nil {
|
||||
ctxData.logger(Log{
|
||||
Id: ctxData.requestId,
|
||||
if option.Logger != nil {
|
||||
option.Logger(Log{
|
||||
Id: option.requestId,
|
||||
Time: time.Now(),
|
||||
Type: LogType_ProxyConnectRemote,
|
||||
Msg: remoteUrl.String(),
|
||||
@@ -238,9 +238,9 @@ func (obj *DialClient) verifyProxyToRemote(ctx context.Context, ctxData *reqCtxD
|
||||
}
|
||||
case "socks5":
|
||||
err = obj.socks5ProxyWithConn(ctx, conn, proxyUrl, remoteUrl)
|
||||
if ctxData.logger != nil {
|
||||
ctxData.logger(Log{
|
||||
Id: ctxData.requestId,
|
||||
if option.Logger != nil {
|
||||
option.Logger(Log{
|
||||
Id: option.requestId,
|
||||
Time: time.Now(),
|
||||
Type: LogType_ProxyTCPConnect,
|
||||
Msg: remoteUrl.String(),
|
||||
@@ -417,42 +417,42 @@ func (obj *DialClient) lookupIPAddr(ips []net.IPAddr, addrType gtls.AddrType) (n
|
||||
}
|
||||
return nil, errors.New("dns parse host error")
|
||||
}
|
||||
func (obj *DialClient) getDialer(ctxData *reqCtxData, parseDns bool) *net.Dialer {
|
||||
func (obj *DialClient) getDialer(option *RequestOption, parseDns bool) *net.Dialer {
|
||||
var dialOption DialOption
|
||||
var isNew bool
|
||||
if ctxData.dialTimeout == 0 {
|
||||
if option.DialTimeout == 0 {
|
||||
dialOption.DialTimeout = obj.dialer.Timeout
|
||||
} else {
|
||||
dialOption.DialTimeout = ctxData.dialTimeout
|
||||
if ctxData.dialTimeout != obj.dialer.Timeout {
|
||||
dialOption.DialTimeout = option.DialTimeout
|
||||
if option.DialTimeout != obj.dialer.Timeout {
|
||||
isNew = true
|
||||
}
|
||||
}
|
||||
|
||||
if ctxData.keepAlive == 0 {
|
||||
if option.KeepAlive == 0 {
|
||||
dialOption.KeepAlive = obj.dialer.KeepAlive
|
||||
} else {
|
||||
dialOption.KeepAlive = ctxData.keepAlive
|
||||
if ctxData.keepAlive != obj.dialer.KeepAlive {
|
||||
dialOption.KeepAlive = option.KeepAlive
|
||||
if option.KeepAlive != obj.dialer.KeepAlive {
|
||||
isNew = true
|
||||
}
|
||||
}
|
||||
|
||||
if ctxData.localAddr == nil {
|
||||
if option.LocalAddr == nil {
|
||||
if obj.dialer.LocalAddr != nil {
|
||||
dialOption.LocalAddr = obj.dialer.LocalAddr.(*net.TCPAddr)
|
||||
}
|
||||
} else {
|
||||
dialOption.LocalAddr = ctxData.localAddr
|
||||
if ctxData.localAddr.String() != obj.dialer.LocalAddr.String() {
|
||||
dialOption.LocalAddr = option.LocalAddr
|
||||
if option.LocalAddr.String() != obj.dialer.LocalAddr.String() {
|
||||
isNew = true
|
||||
}
|
||||
}
|
||||
if ctxData.dns == nil {
|
||||
if option.Dns == nil {
|
||||
dialOption.Dns = obj.dns
|
||||
} else {
|
||||
dialOption.Dns = ctxData.dns
|
||||
if parseDns && ctxData.dns.String() != obj.dns.String() {
|
||||
dialOption.Dns = option.Dns
|
||||
if parseDns && option.Dns.String() != obj.dns.String() {
|
||||
isNew = true
|
||||
}
|
||||
}
|
||||
@@ -462,10 +462,10 @@ func (obj *DialClient) getDialer(ctxData *reqCtxData, parseDns bool) *net.Dialer
|
||||
return obj.dialer
|
||||
}
|
||||
}
|
||||
func (obj *DialClient) addTls(ctx context.Context, conn net.Conn, host string, disHttp2 bool, tlsConfig *tls.Config) (*tls.Conn, error) {
|
||||
func (obj *DialClient) addTls(ctx context.Context, conn net.Conn, host string, forceHttp1 bool, tlsConfig *tls.Config) (*tls.Conn, error) {
|
||||
var tlsConn *tls.Conn
|
||||
tlsConfig.ServerName = gtls.GetServerName(host)
|
||||
if disHttp2 {
|
||||
if forceHttp1 {
|
||||
tlsConfig.NextProtos = []string{"http/1.1"}
|
||||
} else {
|
||||
tlsConfig.NextProtos = []string{"h2", "http/1.1"}
|
||||
@@ -473,16 +473,16 @@ func (obj *DialClient) addTls(ctx context.Context, conn net.Conn, host string, d
|
||||
tlsConn = tls.Client(conn, tlsConfig)
|
||||
return tlsConn, tlsConn.HandshakeContext(ctx)
|
||||
}
|
||||
func (obj *DialClient) addJa3Tls(ctx context.Context, conn net.Conn, host string, disHttp2 bool, ja3Spec ja3.Ja3Spec, tlsConfig *utls.Config) (*utls.UConn, error) {
|
||||
func (obj *DialClient) addJa3Tls(ctx context.Context, conn net.Conn, host string, forceHttp1 bool, ja3Spec ja3.Ja3Spec, tlsConfig *utls.Config) (*utls.UConn, error) {
|
||||
tlsConfig.ServerName = gtls.GetServerName(host)
|
||||
if disHttp2 {
|
||||
if forceHttp1 {
|
||||
tlsConfig.NextProtos = []string{"http/1.1"}
|
||||
} else {
|
||||
tlsConfig.NextProtos = []string{"h2", "http/1.1"}
|
||||
}
|
||||
return ja3.NewClient(ctx, conn, ja3Spec, disHttp2, tlsConfig)
|
||||
return ja3.NewClient(ctx, conn, ja3Spec, forceHttp1, tlsConfig)
|
||||
}
|
||||
func (obj *DialClient) Socks5Proxy(ctx context.Context, ctxData *reqCtxData, network string, proxyUrl *url.URL, remoteUrl *url.URL) (conn net.Conn, err error) {
|
||||
func (obj *DialClient) Socks5Proxy(ctx context.Context, ctxData *RequestOption, network string, proxyUrl *url.URL, remoteUrl *url.URL) (conn net.Conn, err error) {
|
||||
if conn, err = obj.DialContext(ctx, ctxData, network, net.JoinHostPort(proxyUrl.Hostname(), proxyUrl.Port())); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
8
go.mod
8
go.mod
@@ -14,7 +14,7 @@ require (
|
||||
github.com/gospider007/tools v0.0.0-20241120013952-ff42051bfc9f
|
||||
github.com/gospider007/websocket v0.0.0-20241205004859-e11571a81def
|
||||
github.com/refraction-networking/utls v1.6.7
|
||||
golang.org/x/exp v0.0.0-20241204233417-43b7b7cde48d
|
||||
golang.org/x/exp v0.0.0-20241210194714-1829a127f884
|
||||
golang.org/x/net v0.32.0
|
||||
)
|
||||
|
||||
@@ -34,7 +34,7 @@ require (
|
||||
github.com/gaukas/godicttls v0.0.4 // indirect
|
||||
github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
|
||||
github.com/google/gopacket v1.1.19 // indirect
|
||||
github.com/google/pprof v0.0.0-20241206021119-61a79c692802 // indirect
|
||||
github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad // indirect
|
||||
github.com/gospider007/blog v0.0.0-20241205091827-6bcaf48620d4 // indirect
|
||||
github.com/gospider007/kinds v0.0.0-20240929092451-8f867acde255 // indirect
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
@@ -52,7 +52,7 @@ require (
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/nwaples/rardecode/v2 v2.0.1 // indirect
|
||||
github.com/onsi/ginkgo/v2 v2.22.0 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.21 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.22 // indirect
|
||||
github.com/quic-go/qpack v0.5.1 // indirect
|
||||
github.com/quic-go/quic-go v0.48.2 // indirect
|
||||
github.com/refraction-networking/uquic v0.0.6 // indirect
|
||||
@@ -68,7 +68,7 @@ require (
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
go.uber.org/zap v1.27.0 // indirect
|
||||
go4.org v0.0.0-20230225012048-214862532bf5 // indirect
|
||||
golang.org/x/crypto v0.30.0 // indirect
|
||||
golang.org/x/crypto v0.31.0 // indirect
|
||||
golang.org/x/image v0.23.0 // indirect
|
||||
golang.org/x/mod v0.22.0 // indirect
|
||||
golang.org/x/sync v0.10.0 // indirect
|
||||
|
||||
16
go.sum
16
go.sum
@@ -88,8 +88,8 @@ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXi
|
||||
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
||||
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
||||
github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
|
||||
github.com/google/pprof v0.0.0-20241206021119-61a79c692802 h1:US08AXzP0bLurpzFUV3Poa9ZijrRdd1zAIOVtoHEiS8=
|
||||
github.com/google/pprof v0.0.0-20241206021119-61a79c692802/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144=
|
||||
github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad h1:a6HEuzUHeKH6hwfN/ZoQgRgVIWFJljSWa/zetS2WTvg=
|
||||
github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144=
|
||||
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
|
||||
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
|
||||
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
|
||||
@@ -164,8 +164,8 @@ github.com/onsi/ginkgo/v2 v2.22.0 h1:Yed107/8DjTr0lKCNt7Dn8yQ6ybuDRQoMGrNFKzMfHg
|
||||
github.com/onsi/ginkgo/v2 v2.22.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo=
|
||||
github.com/onsi/gomega v1.34.2 h1:pNCwDkzrsv7MS9kpaQvVb1aVLahQXyJ/Tv5oAZMI3i8=
|
||||
github.com/onsi/gomega v1.34.2/go.mod h1:v1xfxRgk0KIsG+QOdm7p8UosrOzPYRo60fd3B/1Dukc=
|
||||
github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=
|
||||
github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
|
||||
github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU=
|
||||
github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
@@ -233,8 +233,8 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U
|
||||
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.30.0 h1:RwoQn3GkWiMkzlX562cLB7OxWvjH1L8xutO2WoJcRoY=
|
||||
golang.org/x/crypto v0.30.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
|
||||
golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
|
||||
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
|
||||
@@ -243,8 +243,8 @@ golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE
|
||||
golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
|
||||
golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
|
||||
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
|
||||
golang.org/x/exp v0.0.0-20241204233417-43b7b7cde48d h1:0olWaB5pg3+oychR51GUVCEsGkeCU/2JxjBgIo4f3M0=
|
||||
golang.org/x/exp v0.0.0-20241204233417-43b7b7cde48d/go.mod h1:qj5a5QZpwLU2NLQudwIN5koi3beDhSAlJwa67PuM98c=
|
||||
golang.org/x/exp v0.0.0-20241210194714-1829a127f884 h1:Y/Mj/94zIQQGHVSv1tTtQBDaQaJe62U9bkDZKKyhPCU=
|
||||
golang.org/x/exp v0.0.0-20241210194714-1829a127f884/go.mod h1:qj5a5QZpwLU2NLQudwIN5koi3beDhSAlJwa67PuM98c=
|
||||
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
|
||||
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
|
||||
golang.org/x/image v0.23.0 h1:HseQ7c2OpPKTPVzNjG5fwJsOTCiiwS4QdsYi5XU6H68=
|
||||
|
||||
15
headers.go
15
headers.go
@@ -18,25 +18,26 @@ func defaultHeaders() http.Header {
|
||||
}
|
||||
}
|
||||
|
||||
func (obj *RequestOption) initHeaders() (http.Header, []string, error) {
|
||||
func (obj *RequestOption) initHeaders() (http.Header, error) {
|
||||
if obj.Headers == nil {
|
||||
return nil, nil, nil
|
||||
return nil, nil
|
||||
}
|
||||
switch headers := obj.Headers.(type) {
|
||||
case http.Header:
|
||||
return headers.Clone(), nil, nil
|
||||
return headers.Clone(), nil
|
||||
case *OrderMap:
|
||||
head, order := headers.parseHeaders()
|
||||
return head, order, nil
|
||||
obj.OrderHeaders = order
|
||||
return head, nil
|
||||
default:
|
||||
_, dataMap, _, err := obj.newBody(headers, mapType)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
return nil, err
|
||||
}
|
||||
if dataMap == nil {
|
||||
return nil, nil, nil
|
||||
return nil, nil
|
||||
}
|
||||
head, _ := dataMap.parseHeaders()
|
||||
return head, nil, err
|
||||
return head, err
|
||||
}
|
||||
}
|
||||
|
||||
14
option.go
14
option.go
@@ -141,11 +141,15 @@ type RequestOption struct {
|
||||
Text any //send text/xml,support: io.Reader, string,[]bytes,json,map
|
||||
Body any //not setting context-type,support io.Reader, string,[]bytes,json,map
|
||||
|
||||
Stream bool //disable auto read
|
||||
WsOption websocket.Option //websocket option
|
||||
DisProxy bool //force disable proxy
|
||||
once bool
|
||||
client *Client
|
||||
Stream bool //disable auto read
|
||||
WsOption websocket.Option //websocket option
|
||||
DisProxy bool //force disable proxy
|
||||
once bool
|
||||
client *Client
|
||||
requestId string
|
||||
proxy *url.URL
|
||||
proxys []*url.URL
|
||||
isNewConn bool
|
||||
}
|
||||
|
||||
func (obj *RequestOption) Client() *Client {
|
||||
|
||||
5
pip.go
5
pip.go
@@ -48,10 +48,13 @@ func (obj *pipCon) Write(b []byte) (n int, err error) {
|
||||
}
|
||||
return
|
||||
}
|
||||
func (obj *pipCon) Close(err error) error {
|
||||
func (obj *pipCon) CloseWitError(err error) error {
|
||||
obj.cnl(err)
|
||||
return nil
|
||||
}
|
||||
func (obj *pipCon) Close() error {
|
||||
return obj.CloseWitError(nil)
|
||||
}
|
||||
|
||||
func pipe(preCtx context.Context) (*pipCon, *pipCon) {
|
||||
ctx, cnl := context.WithCancelCause(preCtx)
|
||||
|
||||
184
requests.go
184
requests.go
@@ -2,16 +2,14 @@ package requests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"net/url"
|
||||
"os"
|
||||
|
||||
"net/http"
|
||||
|
||||
@@ -20,7 +18,6 @@ import (
|
||||
"github.com/gospider007/re"
|
||||
"github.com/gospider007/tools"
|
||||
"github.com/gospider007/websocket"
|
||||
utls "github.com/refraction-networking/utls"
|
||||
)
|
||||
|
||||
type contextKey string
|
||||
@@ -31,95 +28,15 @@ var errFatal = errors.New("ErrFatal")
|
||||
|
||||
var ErrUseLastResponse = http.ErrUseLastResponse
|
||||
|
||||
type reqCtxData struct {
|
||||
isWs bool
|
||||
h3 bool
|
||||
forceHttp1 bool
|
||||
maxRedirect int
|
||||
proxy *url.URL
|
||||
proxys []*url.URL
|
||||
disProxy bool
|
||||
orderHeaders []string
|
||||
responseHeaderTimeout time.Duration
|
||||
tlsHandshakeTimeout time.Duration
|
||||
tlsConfig *tls.Config
|
||||
utlsConfig *utls.Config
|
||||
|
||||
requestCallBack func(context.Context, *http.Request, *http.Response) error
|
||||
|
||||
h2Ja3Spec ja3.H2Ja3Spec
|
||||
ja3Spec ja3.Ja3Spec
|
||||
|
||||
dialTimeout time.Duration
|
||||
keepAlive time.Duration
|
||||
localAddr *net.TCPAddr //network card ip
|
||||
addrType gtls.AddrType //first ip type
|
||||
dns *net.UDPAddr
|
||||
isNewConn bool
|
||||
logger func(Log)
|
||||
requestId string
|
||||
func CreateReqCtx(ctx context.Context, option *RequestOption) context.Context {
|
||||
return context.WithValue(ctx, gospiderContextKey, option)
|
||||
}
|
||||
|
||||
func NewReqCtxData(ctx context.Context, option *RequestOption) (*reqCtxData, error) {
|
||||
//init ctxData
|
||||
ctxData := new(reqCtxData)
|
||||
ctxData.requestId = tools.NaoId()
|
||||
ctxData.logger = option.Logger
|
||||
ctxData.h3 = option.H3
|
||||
ctxData.tlsConfig = option.TlsConfig
|
||||
ctxData.utlsConfig = option.UtlsConfig
|
||||
ctxData.ja3Spec = option.Ja3Spec
|
||||
ctxData.h2Ja3Spec = option.H2Ja3Spec
|
||||
ctxData.forceHttp1 = option.ForceHttp1
|
||||
ctxData.maxRedirect = option.MaxRedirect
|
||||
ctxData.requestCallBack = option.RequestCallBack
|
||||
ctxData.responseHeaderTimeout = option.ResponseHeaderTimeout
|
||||
ctxData.addrType = option.AddrType
|
||||
ctxData.dialTimeout = option.DialTimeout
|
||||
ctxData.keepAlive = option.KeepAlive
|
||||
ctxData.localAddr = option.LocalAddr
|
||||
ctxData.dns = option.Dns
|
||||
ctxData.disProxy = option.DisProxy
|
||||
ctxData.tlsHandshakeTimeout = option.TlsHandshakeTimeout
|
||||
//init scheme
|
||||
if option.Url != nil {
|
||||
if option.Url.Scheme == "ws" {
|
||||
ctxData.isWs = true
|
||||
option.Url.Scheme = "http"
|
||||
} else if option.Url.Scheme == "wss" {
|
||||
ctxData.isWs = true
|
||||
option.Url.Scheme = "https"
|
||||
}
|
||||
func GetRequestOption(ctx context.Context) *RequestOption {
|
||||
option, ok := ctx.Value(gospiderContextKey).(*RequestOption)
|
||||
if ok {
|
||||
return option
|
||||
}
|
||||
//init tls timeout
|
||||
if option.TlsHandshakeTimeout == 0 {
|
||||
ctxData.tlsHandshakeTimeout = time.Second * 15
|
||||
}
|
||||
//init proxy
|
||||
if option.Proxy != "" {
|
||||
tempProxy, err := gtls.VerifyProxy(option.Proxy)
|
||||
if err != nil {
|
||||
return nil, tools.WrapError(errFatal, errors.New("tempRequest init proxy error"), err)
|
||||
}
|
||||
ctxData.proxy = tempProxy
|
||||
}
|
||||
if l := len(option.Proxys); l > 0 {
|
||||
ctxData.proxys = make([]*url.URL, l)
|
||||
for i, proxy := range option.Proxys {
|
||||
tempProxy, err := gtls.VerifyProxy(proxy)
|
||||
if err != nil {
|
||||
return ctxData, tools.WrapError(errFatal, errors.New("tempRequest init proxy error"), err)
|
||||
}
|
||||
ctxData.proxys[i] = tempProxy
|
||||
}
|
||||
}
|
||||
return ctxData, nil
|
||||
}
|
||||
func CreateReqCtx(ctx context.Context, ctxData *reqCtxData) context.Context {
|
||||
return context.WithValue(ctx, gospiderContextKey, ctxData)
|
||||
}
|
||||
func GetReqCtxData(ctx context.Context) *reqCtxData {
|
||||
return ctx.Value(gospiderContextKey).(*reqCtxData)
|
||||
return nil
|
||||
}
|
||||
|
||||
// sends a GET request and returns the response.
|
||||
@@ -230,6 +147,7 @@ func (obj *Client) Request(ctx context.Context, method string, href string, opti
|
||||
rawOption = options[0]
|
||||
}
|
||||
optionBak := obj.newRequestOption(rawOption)
|
||||
optionBak.requestId = tools.NaoId()
|
||||
if optionBak.Method == "" {
|
||||
optionBak.Method = method
|
||||
}
|
||||
@@ -255,7 +173,7 @@ func (obj *Client) request(ctx context.Context, option *RequestOption) (response
|
||||
response = new(Response)
|
||||
defer func() {
|
||||
//read body
|
||||
if err == nil && !response.IsWebSocket() && !response.IsSse() && !response.IsStream() {
|
||||
if err == nil && !response.IsWebSocket() && !response.IsSSE() && !response.IsStream() {
|
||||
err = response.ReadBody()
|
||||
}
|
||||
//result callback
|
||||
@@ -278,44 +196,75 @@ func (obj *Client) request(ctx context.Context, option *RequestOption) (response
|
||||
}
|
||||
response.requestOption = option
|
||||
//init headers and orderheaders,befor init ctxData
|
||||
headers, orderHeaders, err := option.initHeaders()
|
||||
headers, err := option.initHeaders()
|
||||
if err != nil {
|
||||
return response, tools.WrapError(err, errors.New("tempRequest init headers error"), err)
|
||||
}
|
||||
if headers != nil && option.UserAgent != "" {
|
||||
headers.Set("User-Agent", option.UserAgent)
|
||||
}
|
||||
if orderHeaders == nil {
|
||||
orderHeaders = option.OrderHeaders
|
||||
}
|
||||
//设置 h2 请求头顺序
|
||||
if orderHeaders != nil {
|
||||
if option.OrderHeaders != nil {
|
||||
if !option.H2Ja3Spec.IsSet() {
|
||||
option.H2Ja3Spec = ja3.DefaultH2Ja3Spec()
|
||||
option.H2Ja3Spec.OrderHeaders = orderHeaders
|
||||
option.H2Ja3Spec.OrderHeaders = option.OrderHeaders
|
||||
} else if option.H2Ja3Spec.OrderHeaders == nil {
|
||||
option.H2Ja3Spec.OrderHeaders = orderHeaders
|
||||
option.H2Ja3Spec.OrderHeaders = option.OrderHeaders
|
||||
}
|
||||
}
|
||||
//init ctxData
|
||||
response.reqCtxData, err = NewReqCtxData(ctx, option)
|
||||
if err != nil {
|
||||
return response, tools.WrapError(err, " reqCtxData init error")
|
||||
//init tls timeout
|
||||
if option.TlsHandshakeTimeout == 0 {
|
||||
option.TlsHandshakeTimeout = time.Second * 15
|
||||
}
|
||||
//init proxy
|
||||
if option.Proxy != "" {
|
||||
tempProxy, err := gtls.VerifyProxy(option.Proxy)
|
||||
if err != nil {
|
||||
return nil, tools.WrapError(errFatal, errors.New("tempRequest init proxy error"), err)
|
||||
}
|
||||
option.proxy = tempProxy
|
||||
}
|
||||
if l := len(option.Proxys); l > 0 {
|
||||
option.proxys = make([]*url.URL, l)
|
||||
for i, proxy := range option.Proxys {
|
||||
tempProxy, err := gtls.VerifyProxy(proxy)
|
||||
if err != nil {
|
||||
return response, tools.WrapError(errFatal, errors.New("tempRequest init proxy error"), err)
|
||||
}
|
||||
option.proxys[i] = tempProxy
|
||||
}
|
||||
}
|
||||
//init headers
|
||||
if headers == nil {
|
||||
headers = defaultHeaders()
|
||||
}
|
||||
//设置 h1 请求头顺序
|
||||
if orderHeaders != nil {
|
||||
response.reqCtxData.orderHeaders = orderHeaders
|
||||
} else {
|
||||
response.reqCtxData.orderHeaders = ja3.DefaultOrderHeaders()
|
||||
if option.OrderHeaders == nil {
|
||||
option.OrderHeaders = ja3.DefaultOrderHeaders()
|
||||
}
|
||||
//init ctx,cnl
|
||||
if option.Timeout > 0 { //超时
|
||||
response.ctx, response.cnl = context.WithTimeout(CreateReqCtx(ctx, response.reqCtxData), option.Timeout)
|
||||
response.ctx, response.cnl = context.WithTimeout(CreateReqCtx(ctx, option), option.Timeout)
|
||||
} else {
|
||||
response.ctx, response.cnl = context.WithCancel(CreateReqCtx(ctx, response.reqCtxData))
|
||||
response.ctx, response.cnl = context.WithCancel(CreateReqCtx(ctx, option))
|
||||
}
|
||||
//init Scheme
|
||||
switch option.Url.Scheme {
|
||||
case "file":
|
||||
response.filePath = re.Sub(`^/+`, "", option.Url.Path)
|
||||
response.content, err = os.ReadFile(response.filePath)
|
||||
if err != nil {
|
||||
err = tools.WrapError(errFatal, errors.New("read filePath data error"), err)
|
||||
}
|
||||
return
|
||||
case "ws":
|
||||
option.ForceHttp1 = true
|
||||
option.Url.Scheme = "http"
|
||||
websocket.SetClientHeadersWithOption(headers, option.WsOption)
|
||||
case "wss":
|
||||
option.ForceHttp1 = true
|
||||
option.Url.Scheme = "https"
|
||||
websocket.SetClientHeadersWithOption(headers, option.WsOption)
|
||||
}
|
||||
//init url
|
||||
href, err := option.initParams()
|
||||
@@ -341,7 +290,7 @@ func (obj *Client) request(ctx context.Context, option *RequestOption) (response
|
||||
if reqs.Header.Get("Referer") == "" {
|
||||
if option.Referer != "" {
|
||||
reqs.Header.Set("Referer", option.Referer)
|
||||
} else if reqs.URL.Scheme != "" && reqs.URL.Host != "" {
|
||||
} else {
|
||||
reqs.Header.Set("Referer", fmt.Sprintf("%s://%s", reqs.URL.Scheme, reqs.URL.Host))
|
||||
}
|
||||
}
|
||||
@@ -351,19 +300,6 @@ func (obj *Client) request(ctx context.Context, option *RequestOption) (response
|
||||
reqs.Header.Set("Content-Type", option.ContentType)
|
||||
}
|
||||
|
||||
//init ws
|
||||
if response.reqCtxData.isWs {
|
||||
websocket.SetClientHeadersWithOption(reqs.Header, option.WsOption)
|
||||
}
|
||||
|
||||
if reqs.URL.Scheme == "file" {
|
||||
response.filePath = re.Sub(`^/+`, "", reqs.URL.Path)
|
||||
response.content, err = os.ReadFile(response.filePath)
|
||||
if err != nil {
|
||||
err = tools.WrapError(errFatal, errors.New("read filePath data error"), err)
|
||||
}
|
||||
return
|
||||
}
|
||||
//add host
|
||||
if option.Host != "" {
|
||||
reqs.Host = option.Host
|
||||
@@ -400,7 +336,7 @@ func (obj *Client) request(ctx context.Context, option *RequestOption) (response
|
||||
if response.response.StatusCode == 101 {
|
||||
response.webSocket = websocket.NewClientConn(response.rawConn.Conn(), websocket.GetResponseHeaderOption(response.response.Header))
|
||||
} else if strings.Contains(response.response.Header.Get("Content-Type"), "text/event-stream") {
|
||||
response.sse = newSse(response.Body())
|
||||
response.sse = newSSE(response)
|
||||
} else if !response.requestOption.DisUnZip {
|
||||
var unCompressionBody io.ReadCloser
|
||||
unCompressionBody, err = tools.CompressionDecode(response.Body(), response.ContentEncoding())
|
||||
|
||||
83
response.go
83
response.go
@@ -6,6 +6,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"iter"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -25,21 +26,19 @@ type Response struct {
|
||||
rawConn *readWriteCloser
|
||||
response *http.Response
|
||||
webSocket *websocket.Conn
|
||||
sse *Sse
|
||||
sse *SSE
|
||||
ctx context.Context
|
||||
cnl context.CancelFunc
|
||||
reqCtxData *reqCtxData
|
||||
requestOption *RequestOption
|
||||
|
||||
content []byte
|
||||
encoding string
|
||||
filePath string
|
||||
readBody bool
|
||||
content []byte
|
||||
encoding string
|
||||
filePath string
|
||||
readBody bool
|
||||
}
|
||||
|
||||
type Sse struct {
|
||||
reader *bufio.Reader
|
||||
raw io.ReadCloser
|
||||
type SSE struct {
|
||||
reader *bufio.Reader
|
||||
response *Response
|
||||
}
|
||||
type Event struct {
|
||||
Data string //data
|
||||
@@ -49,12 +48,12 @@ type Event struct {
|
||||
Comment string //comment info
|
||||
}
|
||||
|
||||
func newSse(rd io.ReadCloser) *Sse {
|
||||
return &Sse{raw: rd, reader: bufio.NewReader(rd)}
|
||||
func newSSE(response *Response) *SSE {
|
||||
return &SSE{response: response, reader: bufio.NewReader(response.Body())}
|
||||
}
|
||||
|
||||
// recv sse envent data
|
||||
func (obj *Sse) Recv() (Event, error) {
|
||||
// recv SSE envent data
|
||||
func (obj *SSE) Recv() (Event, error) {
|
||||
var event Event
|
||||
for {
|
||||
readStr, err := obj.reader.ReadString('\n')
|
||||
@@ -93,9 +92,24 @@ func (obj *Sse) Recv() (Event, error) {
|
||||
}
|
||||
}
|
||||
|
||||
// close sse
|
||||
func (obj *Sse) Close() error {
|
||||
return obj.raw.Close()
|
||||
func (obj *SSE) Range() iter.Seq2[Event, error] {
|
||||
return func(yield func(Event, error) bool) {
|
||||
defer obj.Close()
|
||||
for {
|
||||
event, err := obj.Recv()
|
||||
if err == io.EOF {
|
||||
return
|
||||
}
|
||||
if !yield(event, err) || err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// close SSE
|
||||
func (obj *SSE) Close() {
|
||||
obj.response.ForceCloseConn()
|
||||
}
|
||||
|
||||
// return websocket client
|
||||
@@ -103,8 +117,8 @@ func (obj *Response) WebSocket() *websocket.Conn {
|
||||
return obj.webSocket
|
||||
}
|
||||
|
||||
// return sse client
|
||||
func (obj *Response) Sse() *Sse {
|
||||
// return SSE client
|
||||
func (obj *Response) SSE() *SSE {
|
||||
return obj.sse
|
||||
}
|
||||
|
||||
@@ -195,7 +209,7 @@ func (obj *Response) SetContent(val []byte) {
|
||||
|
||||
// return content with []byte
|
||||
func (obj *Response) Content() []byte {
|
||||
if !obj.IsWebSocket() && !obj.IsSse() {
|
||||
if !obj.IsWebSocket() && !obj.IsSSE() {
|
||||
obj.ReadBody()
|
||||
}
|
||||
return obj.content
|
||||
@@ -251,13 +265,6 @@ func (obj *Response) defaultDecode() bool {
|
||||
return strings.Contains(obj.ContentType(), "html")
|
||||
}
|
||||
|
||||
// must stream=true
|
||||
func (obj *Response) Conn() *connecotr {
|
||||
if obj.IsWebSocket() || obj.IsSse() || obj.IsStream() {
|
||||
return obj.rawConn.Conn()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (obj *Response) Body() io.ReadCloser {
|
||||
return obj.response.Body
|
||||
}
|
||||
@@ -271,7 +278,7 @@ func (obj *Response) IsStream() bool {
|
||||
func (obj *Response) IsWebSocket() bool {
|
||||
return obj.webSocket != nil
|
||||
}
|
||||
func (obj *Response) IsSse() bool {
|
||||
func (obj *Response) IsSSE() bool {
|
||||
return obj.sse != nil
|
||||
}
|
||||
|
||||
@@ -280,7 +287,7 @@ func (obj *Response) ReadBody() (err error) {
|
||||
if obj.readBody {
|
||||
return nil
|
||||
}
|
||||
if obj.IsWebSocket() && obj.IsSse() {
|
||||
if obj.IsWebSocket() && obj.IsSSE() {
|
||||
return errors.New("can not read stream")
|
||||
}
|
||||
obj.readBody = true
|
||||
@@ -293,9 +300,9 @@ func (obj *Response) ReadBody() (err error) {
|
||||
} else {
|
||||
_, err = io.Copy(bBody, obj.Body())
|
||||
}
|
||||
if obj.reqCtxData.logger != nil {
|
||||
obj.reqCtxData.logger(Log{
|
||||
Id: obj.reqCtxData.requestId,
|
||||
if obj.requestOption.Logger != nil {
|
||||
obj.requestOption.Logger(Log{
|
||||
Id: obj.requestOption.requestId,
|
||||
Time: time.Now(),
|
||||
Type: LogType_ResponseBody,
|
||||
Msg: "response body",
|
||||
@@ -316,7 +323,7 @@ func (obj *Response) ReadBody() (err error) {
|
||||
|
||||
// conn is new conn
|
||||
func (obj *Response) IsNewConn() bool {
|
||||
return obj.reqCtxData.isNewConn
|
||||
return obj.requestOption.isNewConn
|
||||
}
|
||||
|
||||
// conn proxy
|
||||
@@ -327,14 +334,6 @@ func (obj *Response) Proxys() []*url.URL {
|
||||
return nil
|
||||
}
|
||||
|
||||
// conn is in pool ?
|
||||
func (obj *Response) InPool() bool {
|
||||
if obj.rawConn != nil {
|
||||
return obj.rawConn.InPool()
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// close body
|
||||
func (obj *Response) CloseBody() {
|
||||
obj.close(false)
|
||||
@@ -353,7 +352,7 @@ func (obj *Response) close(closeConn bool) {
|
||||
if obj.sse != nil {
|
||||
obj.sse.Close()
|
||||
}
|
||||
if obj.IsWebSocket() || obj.IsSse() || !obj.readBody {
|
||||
if obj.IsWebSocket() || obj.IsSSE() || !obj.readBody {
|
||||
obj.ForceCloseConn()
|
||||
} else if obj.rawConn != nil {
|
||||
if closeConn {
|
||||
|
||||
229
roundTripper.go
229
roundTripper.go
@@ -1,15 +1,12 @@
|
||||
package requests
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"net/http"
|
||||
@@ -18,51 +15,43 @@ import (
|
||||
"github.com/gospider007/http2"
|
||||
"github.com/gospider007/http3"
|
||||
"github.com/gospider007/tools"
|
||||
utls "github.com/refraction-networking/utls"
|
||||
)
|
||||
|
||||
type reqTask struct {
|
||||
ctxData *reqCtxData
|
||||
ctx context.Context
|
||||
cnl context.CancelFunc
|
||||
req *http.Request
|
||||
res *http.Response
|
||||
emptyPool chan struct{}
|
||||
err error
|
||||
orderHeaders []string
|
||||
orderHeaders2 []string
|
||||
option *RequestOption
|
||||
ctx context.Context
|
||||
cnl context.CancelFunc
|
||||
req *http.Request
|
||||
res *http.Response
|
||||
emptyPool chan struct{}
|
||||
err error
|
||||
}
|
||||
|
||||
func (obj *reqTask) inPool() bool {
|
||||
return obj.err == nil && obj.res != nil && obj.res.StatusCode != 101 && !strings.Contains(obj.res.Header.Get("Content-Type"), "text/event-stream")
|
||||
}
|
||||
func (obj *reqTask) suppertRetry() bool {
|
||||
if obj.req.Body == nil {
|
||||
return true
|
||||
} else if body, ok := obj.req.Body.(*requestBody); ok {
|
||||
if body.Seek(0, io.SeekStart); !body.readed {
|
||||
} else if body, ok := obj.req.Body.(io.Seeker); ok {
|
||||
if i, err := body.Seek(0, io.SeekStart); i == 0 && err == nil {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
func getKey(ctxData *reqCtxData, req *http.Request) (key string) {
|
||||
func getKey(option *RequestOption, req *http.Request) (key string) {
|
||||
var proxyUser string
|
||||
if ctxData.proxy != nil {
|
||||
proxyUser = ctxData.proxy.User.String()
|
||||
if option.proxy != nil {
|
||||
proxyUser = option.proxy.User.String()
|
||||
}
|
||||
return fmt.Sprintf("%s@%s@%s", proxyUser, getAddr(ctxData.proxy), getAddr(req.URL))
|
||||
return fmt.Sprintf("%s@%s@%s", proxyUser, option.proxy.Host, req.URL.Host)
|
||||
}
|
||||
|
||||
type roundTripper struct {
|
||||
ctx context.Context
|
||||
cnl context.CancelFunc
|
||||
connPools *connPools
|
||||
dialer *DialClient
|
||||
tlsConfig *tls.Config
|
||||
utlsConfig *utls.Config
|
||||
getProxy func(ctx context.Context, url *url.URL) (string, error)
|
||||
getProxys func(ctx context.Context, url *url.URL) ([]string, error)
|
||||
ctx context.Context
|
||||
cnl context.CancelFunc
|
||||
connPools *connPools
|
||||
dialer *DialClient
|
||||
getProxy func(ctx context.Context, url *url.URL) (string, error)
|
||||
getProxys func(ctx context.Context, url *url.URL) ([]string, error)
|
||||
}
|
||||
|
||||
func newRoundTripper(preCtx context.Context, option ClientOption) *roundTripper {
|
||||
@@ -78,28 +67,13 @@ func newRoundTripper(preCtx context.Context, option ClientOption) *roundTripper
|
||||
AddrType: option.AddrType,
|
||||
GetAddrType: option.GetAddrType,
|
||||
})
|
||||
tlsConfig := &tls.Config{
|
||||
InsecureSkipVerify: true,
|
||||
// SessionTicketKey: [32]byte{},
|
||||
ClientSessionCache: tls.NewLRUClientSessionCache(0),
|
||||
}
|
||||
utlsConfig := &utls.Config{
|
||||
InsecureSkipVerify: true,
|
||||
InsecureSkipTimeVerify: true,
|
||||
// SessionTicketKey: [32]byte{},
|
||||
ClientSessionCache: utls.NewLRUClientSessionCache(0),
|
||||
OmitEmptyPsk: true,
|
||||
PreferSkipResumptionOnNilExtension: true,
|
||||
}
|
||||
return &roundTripper{
|
||||
tlsConfig: tlsConfig,
|
||||
utlsConfig: utlsConfig,
|
||||
ctx: ctx,
|
||||
cnl: cnl,
|
||||
dialer: dialClient,
|
||||
getProxy: option.GetProxy,
|
||||
getProxys: option.GetProxys,
|
||||
connPools: newConnPools(),
|
||||
ctx: ctx,
|
||||
cnl: cnl,
|
||||
dialer: dialClient,
|
||||
getProxy: option.GetProxy,
|
||||
getProxys: option.GetProxys,
|
||||
connPools: newConnPools(),
|
||||
}
|
||||
}
|
||||
func (obj *roundTripper) newConnPool(conn *connecotr, key string) *connPool {
|
||||
@@ -115,10 +89,6 @@ func (obj *roundTripper) newConnPool(conn *connecotr, key string) *connPool {
|
||||
return pool
|
||||
}
|
||||
func (obj *roundTripper) putConnPool(key string, conn *connecotr) {
|
||||
conn.inPool = true
|
||||
if conn.h2RawConn == nil && conn.h3RawConn == nil {
|
||||
go conn.read()
|
||||
}
|
||||
pool := obj.connPools.get(key)
|
||||
if pool != nil {
|
||||
pool.total.Add(1)
|
||||
@@ -127,67 +97,54 @@ func (obj *roundTripper) putConnPool(key string, conn *connecotr) {
|
||||
obj.connPools.set(key, obj.newConnPool(conn, key))
|
||||
}
|
||||
}
|
||||
func (obj *roundTripper) tlsConfigClone(ctxData *reqCtxData) *tls.Config {
|
||||
if ctxData.tlsConfig != nil {
|
||||
return ctxData.tlsConfig.Clone()
|
||||
}
|
||||
return obj.tlsConfig.Clone()
|
||||
}
|
||||
func (obj *roundTripper) utlsConfigClone(ctxData *reqCtxData) *utls.Config {
|
||||
if ctxData.utlsConfig != nil {
|
||||
return ctxData.utlsConfig.Clone()
|
||||
}
|
||||
return obj.utlsConfig.Clone()
|
||||
}
|
||||
func (obj *roundTripper) newConnecotr(netConn net.Conn) *connecotr {
|
||||
func (obj *roundTripper) newConnecotr() *connecotr {
|
||||
conne := new(connecotr)
|
||||
conne.withCancel(obj.ctx, obj.ctx)
|
||||
conne.rawConn = netConn
|
||||
return conne
|
||||
}
|
||||
|
||||
func (obj *roundTripper) http3Dial(ctxData *reqCtxData, req *http.Request) (conn *connecotr, err error) {
|
||||
tlsConfig := obj.tlsConfigClone(ctxData)
|
||||
func (obj *roundTripper) http3Dial(option *RequestOption, req *http.Request) (conn *connecotr, err error) {
|
||||
tlsConfig := option.TlsConfig.Clone()
|
||||
tlsConfig.NextProtos = []string{http3.NextProtoH3}
|
||||
tlsConfig.ServerName = req.Host
|
||||
netConn, err := http3.Dial(req.Context(), getAddr(req.URL), tlsConfig, nil)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
conn = obj.newConnecotr(nil)
|
||||
conn.h3RawConn = http3.NewClient(netConn)
|
||||
conn = obj.newConnecotr()
|
||||
conn.rawConn = http3.NewClient(netConn)
|
||||
return
|
||||
}
|
||||
|
||||
func (obj *roundTripper) ghttp3Dial(ctxData *reqCtxData, req *http.Request) (conn *connecotr, err error) {
|
||||
tlsConfig := obj.utlsConfigClone(ctxData)
|
||||
func (obj *roundTripper) ghttp3Dial(option *RequestOption, req *http.Request) (conn *connecotr, err error) {
|
||||
tlsConfig := option.UtlsConfig.Clone()
|
||||
tlsConfig.NextProtos = []string{http3.NextProtoH3}
|
||||
tlsConfig.ServerName = req.Host
|
||||
netConn, err := http3.UDial(req.Context(), getAddr(req.URL), tlsConfig, nil)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
conn = obj.newConnecotr(nil)
|
||||
conn.h3RawConn = http3.NewUClient(netConn)
|
||||
conn = obj.newConnecotr()
|
||||
conn.rawConn = http3.NewUClient(netConn)
|
||||
return
|
||||
}
|
||||
|
||||
func (obj *roundTripper) dial(ctxData *reqCtxData, req *http.Request) (conn *connecotr, err error) {
|
||||
if ctxData.h3 {
|
||||
if ctxData.ja3Spec.IsSet() {
|
||||
return obj.ghttp3Dial(ctxData, req)
|
||||
func (obj *roundTripper) dial(option *RequestOption, req *http.Request) (conn *connecotr, err error) {
|
||||
if option.H3 {
|
||||
if option.Ja3Spec.IsSet() {
|
||||
return obj.ghttp3Dial(option, req)
|
||||
} else {
|
||||
return obj.http3Dial(ctxData, req)
|
||||
return obj.http3Dial(option, req)
|
||||
}
|
||||
}
|
||||
var proxys []*url.URL
|
||||
if !ctxData.disProxy {
|
||||
if ctxData.proxy != nil {
|
||||
proxys = []*url.URL{cloneUrl(ctxData.proxy)}
|
||||
if !option.DisProxy {
|
||||
if option.proxy != nil {
|
||||
proxys = []*url.URL{cloneUrl(option.proxy)}
|
||||
}
|
||||
if len(proxys) == 0 && len(ctxData.proxys) > 0 {
|
||||
proxys = make([]*url.URL, len(ctxData.proxys))
|
||||
for i, proxy := range ctxData.proxys {
|
||||
if len(proxys) == 0 && len(option.proxys) > 0 {
|
||||
proxys = make([]*url.URL, len(option.proxys))
|
||||
for i, proxy := range option.proxys {
|
||||
proxys[i] = cloneUrl(proxy)
|
||||
}
|
||||
}
|
||||
@@ -222,52 +179,56 @@ func (obj *roundTripper) dial(ctxData *reqCtxData, req *http.Request) (conn *con
|
||||
host := getHost(req)
|
||||
var netConn net.Conn
|
||||
if len(proxys) > 0 {
|
||||
netConn, err = obj.dialer.DialProxyContext(req.Context(), ctxData, "tcp", obj.tlsConfigClone(ctxData), append(proxys, cloneUrl(req.URL))...)
|
||||
netConn, err = obj.dialer.DialProxyContext(req.Context(), option, "tcp", option.TlsConfig.Clone(), append(proxys, cloneUrl(req.URL))...)
|
||||
} else {
|
||||
netConn, err = obj.dialer.DialContext(req.Context(), ctxData, "tcp", getAddr(req.URL))
|
||||
netConn, err = obj.dialer.DialContext(req.Context(), option, "tcp", getAddr(req.URL))
|
||||
}
|
||||
if err != nil {
|
||||
return conn, err
|
||||
}
|
||||
var h2 bool
|
||||
if req.URL.Scheme == "https" {
|
||||
ctx, cnl := context.WithTimeout(req.Context(), ctxData.tlsHandshakeTimeout)
|
||||
ctx, cnl := context.WithTimeout(req.Context(), option.TlsHandshakeTimeout)
|
||||
defer cnl()
|
||||
disHttp2 := ctxData.isWs || ctxData.forceHttp1
|
||||
if ctxData.ja3Spec.IsSet() {
|
||||
if tlsConn, err := obj.dialer.addJa3Tls(ctx, netConn, host, disHttp2, ctxData.ja3Spec, obj.utlsConfigClone(ctxData)); err != nil {
|
||||
if option.Ja3Spec.IsSet() {
|
||||
if tlsConn, err := obj.dialer.addJa3Tls(ctx, netConn, host, option.ForceHttp1, option.Ja3Spec, option.UtlsConfig.Clone()); err != nil {
|
||||
return conn, tools.WrapError(err, "add ja3 tls error")
|
||||
} else {
|
||||
h2 = tlsConn.ConnectionState().NegotiatedProtocol == "h2"
|
||||
netConn = tlsConn
|
||||
}
|
||||
} else {
|
||||
if tlsConn, err := obj.dialer.addTls(ctx, netConn, host, disHttp2, obj.tlsConfigClone(ctxData)); err != nil {
|
||||
if tlsConn, err := obj.dialer.addTls(ctx, netConn, host, option.ForceHttp1, option.TlsConfig.Clone()); err != nil {
|
||||
return conn, tools.WrapError(err, "add tls error")
|
||||
} else {
|
||||
h2 = tlsConn.ConnectionState().NegotiatedProtocol == "h2"
|
||||
netConn = tlsConn
|
||||
}
|
||||
}
|
||||
if ctxData.logger != nil {
|
||||
ctxData.logger(Log{
|
||||
Id: ctxData.requestId,
|
||||
if option.Logger != nil {
|
||||
option.Logger(Log{
|
||||
Id: option.requestId,
|
||||
Time: time.Now(),
|
||||
Type: LogType_TLSHandshake,
|
||||
Msg: host,
|
||||
})
|
||||
}
|
||||
}
|
||||
conne := obj.newConnecotr(netConn)
|
||||
conne := obj.newConnecotr()
|
||||
conne.proxys = proxys
|
||||
if h2 {
|
||||
if conne.h2RawConn, err = http2.NewClientConn(func() {
|
||||
if option.H2Ja3Spec.OrderHeaders != nil {
|
||||
option.OrderHeaders = option.H2Ja3Spec.OrderHeaders
|
||||
}
|
||||
if conne.rawConn, err = http2.NewClientConn(func() {
|
||||
conne.forceCnl(errors.New("http2 client close"))
|
||||
}, netConn, ctxData.h2Ja3Spec); err != nil {
|
||||
}, netConn, option.H2Ja3Spec); err != nil {
|
||||
return conne, err
|
||||
}
|
||||
} else {
|
||||
conne.r, conne.w = bufio.NewReader(conne), bufio.NewWriter(conne)
|
||||
conne.rawConn = newConn(conne.forceCtx, netConn, func() {
|
||||
conne.forceCnl(errors.New("http1 client close"))
|
||||
})
|
||||
}
|
||||
return conne, err
|
||||
}
|
||||
@@ -278,8 +239,8 @@ func (obj *roundTripper) setGetProxys(getProxys func(ctx context.Context, url *u
|
||||
obj.getProxys = getProxys
|
||||
}
|
||||
|
||||
func (obj *roundTripper) poolRoundTrip(ctxData *reqCtxData, pool *connPool, task *reqTask, key string) (isTry bool) {
|
||||
task.ctx, task.cnl = context.WithTimeout(task.req.Context(), ctxData.responseHeaderTimeout)
|
||||
func (obj *roundTripper) poolRoundTrip(option *RequestOption, pool *connPool, task *reqTask, key string) (isTry bool) {
|
||||
task.ctx, task.cnl = context.WithTimeout(task.req.Context(), option.ResponseHeaderTimeout)
|
||||
select {
|
||||
case pool.tasks <- task:
|
||||
select {
|
||||
@@ -292,71 +253,61 @@ func (obj *roundTripper) poolRoundTrip(ctxData *reqCtxData, pool *connPool, task
|
||||
return false
|
||||
}
|
||||
default:
|
||||
obj.connRoundTripMain(ctxData, task, key)
|
||||
obj.connRoundTripMain(option, task, key)
|
||||
return false
|
||||
}
|
||||
}
|
||||
func (obj *roundTripper) connRoundTripMain(ctxData *reqCtxData, task *reqTask, key string) {
|
||||
func (obj *roundTripper) connRoundTripMain(option *RequestOption, task *reqTask, key string) {
|
||||
for range 10 {
|
||||
if !obj.connRoundTrip(ctxData, task, key) {
|
||||
if !obj.connRoundTrip(option, task, key) {
|
||||
return
|
||||
}
|
||||
}
|
||||
task.err = errors.New("connRoundTripMain retry 5 times")
|
||||
}
|
||||
|
||||
func (obj *roundTripper) connRoundTrip(ctxData *reqCtxData, task *reqTask, key string) (retry bool) {
|
||||
ctxData.isNewConn = true
|
||||
conn, err := obj.dial(ctxData, task.req)
|
||||
func (obj *roundTripper) connRoundTrip(option *RequestOption, task *reqTask, key string) (retry bool) {
|
||||
option.isNewConn = true
|
||||
conn, err := obj.dial(option, task.req)
|
||||
if err != nil {
|
||||
task.err = err
|
||||
return
|
||||
}
|
||||
task.ctx, task.cnl = context.WithTimeout(task.req.Context(), ctxData.responseHeaderTimeout)
|
||||
task.ctx, task.cnl = context.WithTimeout(task.req.Context(), option.ResponseHeaderTimeout)
|
||||
retry = conn.taskMain(task, false)
|
||||
if retry || task.err != nil {
|
||||
return retry
|
||||
}
|
||||
if task.inPool() {
|
||||
obj.putConnPool(key, conn)
|
||||
}
|
||||
obj.putConnPool(key, conn)
|
||||
return retry
|
||||
}
|
||||
|
||||
func (obj *roundTripper) closeConns() {
|
||||
obj.connPools.iter(func(key string, pool *connPool) bool {
|
||||
for key, pool := range obj.connPools.Range() {
|
||||
pool.safeClose()
|
||||
obj.connPools.del(key)
|
||||
return true
|
||||
})
|
||||
}
|
||||
}
|
||||
func (obj *roundTripper) forceCloseConns() {
|
||||
obj.connPools.iter(func(key string, pool *connPool) bool {
|
||||
for key, pool := range obj.connPools.Range() {
|
||||
pool.forceClose()
|
||||
obj.connPools.del(key)
|
||||
return true
|
||||
})
|
||||
}
|
||||
}
|
||||
func (obj *roundTripper) newReqTask(req *http.Request, ctxData *reqCtxData) *reqTask {
|
||||
if ctxData.responseHeaderTimeout == 0 {
|
||||
ctxData.responseHeaderTimeout = time.Second * 300
|
||||
func (obj *roundTripper) newReqTask(req *http.Request, option *RequestOption) *reqTask {
|
||||
if option.ResponseHeaderTimeout == 0 {
|
||||
option.ResponseHeaderTimeout = time.Second * 300
|
||||
}
|
||||
task := new(reqTask)
|
||||
task.req = req
|
||||
task.ctxData = ctxData
|
||||
task.option = option
|
||||
task.emptyPool = make(chan struct{})
|
||||
task.orderHeaders = ctxData.orderHeaders
|
||||
if ctxData.h2Ja3Spec.OrderHeaders != nil {
|
||||
task.orderHeaders2 = ctxData.h2Ja3Spec.OrderHeaders
|
||||
} else {
|
||||
task.orderHeaders2 = ctxData.orderHeaders
|
||||
}
|
||||
return task
|
||||
}
|
||||
func (obj *roundTripper) RoundTrip(req *http.Request) (response *http.Response, err error) {
|
||||
ctxData := GetReqCtxData(req.Context())
|
||||
if ctxData.requestCallBack != nil {
|
||||
if err = ctxData.requestCallBack(req.Context(), req, nil); err != nil {
|
||||
option := GetRequestOption(req.Context())
|
||||
if option.RequestCallBack != nil {
|
||||
if err = option.RequestCallBack(req.Context(), req, nil); err != nil {
|
||||
if err == http.ErrUseLastResponse {
|
||||
if req.Response == nil {
|
||||
return nil, errors.New("errUseLastResponse response is nil")
|
||||
@@ -367,20 +318,20 @@ func (obj *roundTripper) RoundTrip(req *http.Request) (response *http.Response,
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
key := getKey(ctxData, req) //pool key
|
||||
task := obj.newReqTask(req, ctxData)
|
||||
key := getKey(option, req) //pool key
|
||||
task := obj.newReqTask(req, option)
|
||||
for {
|
||||
pool := obj.connPools.get(key)
|
||||
if pool == nil {
|
||||
obj.connRoundTripMain(ctxData, task, key)
|
||||
obj.connRoundTripMain(option, task, key)
|
||||
break
|
||||
}
|
||||
if !obj.poolRoundTrip(ctxData, pool, task, key) {
|
||||
if !obj.poolRoundTrip(option, pool, task, key) {
|
||||
break
|
||||
}
|
||||
}
|
||||
if task.err == nil && ctxData.requestCallBack != nil {
|
||||
if err = ctxData.requestCallBack(task.req.Context(), task.req, task.res); err != nil {
|
||||
if task.err == nil && option.RequestCallBack != nil {
|
||||
if err = option.RequestCallBack(task.req.Context(), task.req, task.res); err != nil {
|
||||
task.err = err
|
||||
}
|
||||
}
|
||||
|
||||
26
rw.go
26
rw.go
@@ -3,6 +3,7 @@ package requests
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"net/url"
|
||||
)
|
||||
|
||||
@@ -11,8 +12,8 @@ type readWriteCloser struct {
|
||||
conn *connecotr
|
||||
}
|
||||
|
||||
func (obj *readWriteCloser) Conn() *connecotr {
|
||||
return obj.conn
|
||||
func (obj *readWriteCloser) Conn() net.Conn {
|
||||
return obj.conn.rawConn.(net.Conn)
|
||||
}
|
||||
func (obj *readWriteCloser) Read(p []byte) (n int, err error) {
|
||||
i, err := obj.body.Read(p)
|
||||
@@ -21,9 +22,6 @@ func (obj *readWriteCloser) Read(p []byte) (n int, err error) {
|
||||
}
|
||||
return i, err
|
||||
}
|
||||
func (obj *readWriteCloser) InPool() bool {
|
||||
return obj.conn.inPool
|
||||
}
|
||||
func (obj *readWriteCloser) Proxys() []*url.URL {
|
||||
if l := len(obj.conn.proxys); l > 0 {
|
||||
proxys := make([]*url.URL, l)
|
||||
@@ -37,26 +35,18 @@ func (obj *readWriteCloser) Proxys() []*url.URL {
|
||||
var errGospiderBodyClose = errors.New("gospider body close error")
|
||||
|
||||
func (obj *readWriteCloser) Close() (err error) {
|
||||
if !obj.InPool() {
|
||||
obj.ForceCloseConn()
|
||||
} else {
|
||||
obj.conn.bodyCnl(errGospiderBodyClose)
|
||||
err = obj.body.Close() //reuse conn
|
||||
}
|
||||
obj.conn.bodyCnl(errGospiderBodyClose)
|
||||
err = obj.body.Close() //reuse conn
|
||||
return
|
||||
}
|
||||
|
||||
// safe close conn
|
||||
func (obj *readWriteCloser) CloseConn() {
|
||||
if !obj.InPool() {
|
||||
obj.ForceCloseConn()
|
||||
} else {
|
||||
obj.conn.bodyCnl(errors.New("readWriterCloser close conn"))
|
||||
obj.conn.safeCnl(errors.New("readWriterCloser close conn"))
|
||||
}
|
||||
obj.conn.bodyCnl(errors.New("readWriterCloser close conn"))
|
||||
obj.conn.safeCnl(errors.New("readWriterCloser close conn"))
|
||||
}
|
||||
|
||||
// force close conn
|
||||
func (obj *readWriteCloser) ForceCloseConn() {
|
||||
obj.conn.closeWithError(errConnectionForceClosed)
|
||||
obj.conn.rawConn.CloseWithError(errConnectionForceClosed)
|
||||
}
|
||||
|
||||
@@ -52,7 +52,8 @@ func TestSse(t *testing.T) {
|
||||
t.Error(err)
|
||||
}
|
||||
defer response.CloseBody()
|
||||
sseCli := response.Sse()
|
||||
sseCli := response.SSE()
|
||||
defer sseCli.Close()
|
||||
if sseCli == nil {
|
||||
t.Error("not is sseCli")
|
||||
}
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/gospider007/requests"
|
||||
@@ -8,26 +10,28 @@ import (
|
||||
)
|
||||
|
||||
func TestWebSocket(t *testing.T) {
|
||||
response, err := requests.Get(nil, "ws://82.157.123.54:9010/ajaxchattest", requests.RequestOption{Headers: map[string]string{
|
||||
"Origin": "http://coolaf.com",
|
||||
}}) // Send WebSocket request
|
||||
response, err := requests.Get(nil, "ws://124.222.224.186:8800", requests.RequestOption{}) // Send WebSocket request
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
log.Panic(err)
|
||||
}
|
||||
defer response.CloseBody()
|
||||
wsCli := response.WebSocket()
|
||||
defer wsCli.Close()
|
||||
if err = wsCli.WriteMessage(websocket.TextMessage, "test"); err != nil { // Send text message
|
||||
t.Error(err)
|
||||
if err = wsCli.WriteMessage(websocket.TextMessage, "test1122332211"); err != nil { // Send text message
|
||||
log.Panic(err)
|
||||
}
|
||||
msgType, con, err := wsCli.ReadMessage() // Receive message
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
if msgType != websocket.TextMessage {
|
||||
t.Error("Message type is not text")
|
||||
}
|
||||
if string(con) != "test" {
|
||||
t.Error("Message content is not test")
|
||||
for {
|
||||
|
||||
msgType, con, err := wsCli.ReadMessage() // Receive message
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
if msgType != websocket.TextMessage {
|
||||
log.Panic("Message type is not text")
|
||||
}
|
||||
log.Print(string(con))
|
||||
if strings.Contains(string(con), "test1122332211") {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,14 +11,14 @@ func TestRawConn(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
if resp.Conn() != nil {
|
||||
if resp.Body() != nil {
|
||||
t.Error("conn is not nil")
|
||||
}
|
||||
resp, err = requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{Stream: true})
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
if resp.Conn() == nil {
|
||||
if resp.Body() == nil {
|
||||
t.Error("conn is nil")
|
||||
}
|
||||
}
|
||||
|
||||
44
tools.go
44
tools.go
@@ -34,6 +34,7 @@ func getHost(req *http.Request) string {
|
||||
}
|
||||
return host
|
||||
}
|
||||
|
||||
func getAddr(uurl *url.URL) (addr string) {
|
||||
if uurl == nil {
|
||||
return ""
|
||||
@@ -239,38 +240,21 @@ func httpWrite(r *http.Request, w *bufio.Writer, orderHeaders []string) (err err
|
||||
}
|
||||
|
||||
type requestBody struct {
|
||||
r io.Reader
|
||||
readed bool
|
||||
closed bool
|
||||
r io.Reader
|
||||
}
|
||||
|
||||
func (obj *requestBody) Read(p []byte) (n int, err error) {
|
||||
obj.readed = true
|
||||
return obj.r.Read(p)
|
||||
}
|
||||
func (obj *requestBody) Close() (err error) {
|
||||
obj.closed = true
|
||||
obj.readed = true
|
||||
if closer, ok := obj.r.(io.Closer); ok {
|
||||
return closer.Close()
|
||||
func (obj *requestBody) Close() error {
|
||||
b, ok := obj.r.(io.Closer)
|
||||
if ok {
|
||||
return b.Close()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (obj *requestBody) Read(p []byte) (n int, err error) {
|
||||
return obj.r.Read(p)
|
||||
}
|
||||
func (obj *requestBody) Seek(offset int64, whence int) (int64, error) {
|
||||
if obj.closed {
|
||||
return 0, fmt.Errorf("unsupported operation: closed")
|
||||
}
|
||||
if !obj.readed {
|
||||
return 0, nil
|
||||
}
|
||||
if seeker, ok := obj.r.(io.Seeker); ok {
|
||||
i, err := seeker.Seek(offset, whence)
|
||||
if i == 0 && err == nil {
|
||||
obj.readed = false
|
||||
}
|
||||
return i, err
|
||||
}
|
||||
return 0, fmt.Errorf("unsupported operation: seeker")
|
||||
return obj.r.(io.Seeker).Seek(0, whence)
|
||||
}
|
||||
|
||||
func NewRequestWithContext(ctx context.Context, method string, u *url.URL, body io.Reader) (*http.Request, error) {
|
||||
@@ -290,7 +274,13 @@ func NewRequestWithContext(ctx context.Context, method string, u *url.URL, body
|
||||
if v, ok := body.(interface{ Len() int }); ok {
|
||||
req.ContentLength = int64(v.Len())
|
||||
}
|
||||
req.Body = &requestBody{r: body}
|
||||
if _, ok := body.(io.Seeker); ok {
|
||||
req.Body = &requestBody{body}
|
||||
} else if b, ok := body.(io.ReadCloser); ok {
|
||||
req.Body = b
|
||||
} else {
|
||||
req.Body = io.NopCloser(body)
|
||||
}
|
||||
}
|
||||
return req, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user