optimize doc info and ja3 tls

2025-12-24 13:57:52 +08:00 · 2023-10-16 22:50:59 +08:00
parent 6116418d27
commit 94d85d779c
7 changed files with 142 additions and 26 deletions
--- a/README.md
+++ b/README.md
@@ -0,0 +1,89 @@
+# introduce
+Lightning fast HTTP client
+# get started
+## install
+```
+go get github.com/gospider007/requests
+```
+# Function Overview
+- Only a few lines of code are needed to complete the request
+- HTTP2 Fingerprint, JA3 Fingerprint
+- SOCKS5 proxy, HTTP proxy, HTTPS proxy
+- WebSocket protocol, SSE protocol ,HTTPS protocol,HTTP protocol
+- Cookie switch
+- Connection pool
+- Automatic decompression and decoding
+- DNS caching
+- Automatic type conversion
+- Retry attempts, request callbacks
+- Force connection closure
+- Powerful and convenient request callback, very convenient to obtain every request detail
+
+# quick start
+## send request,support :"GET","HEAD","POST","PUT","PATCH"
+```go
+resp,error:=request.Get(nil,"http://httpbin.org/get")
+```
+## use proxy
+```go
+resp, error := request.Get(nil, "http://httpbin.org/get", requests.RequestOption{Proxy: "http://127.0.0.1:8888"})
+```
+## use ja3 fingerprint
+```go
+resp, _ := request.Get(nil, "http://httpbin.org/get", requests.RequestOption{Ja3: true})
+```
+## use session
+```go
+session, _ := requests.NewClient(nil)
+resp, _ := session.Get(nil, "http://httpbin.org/get")
+```
+## close body
+```go
+resp,_:=request.Get(nil,"http://httpbin.org/get")
+resp.Close()
+```
+## safe delete net.conn
+```go
+resp,_:=request.Get(nil,"http://httpbin.org/get")
+resp.Delete()
+```
+## force delete net.conn
+```go
+resp,_:=request.Get(nil,"http://httpbin.org/get")
+resp.ForceDelete()
+```
+## websocket
+```go
+response, _ := request.Get(nil, "ws://82.157.123.54:9010/ajaxchattest", requests.RequestOption{Headers: map[string]string{"Origin": "http://coolaf.com"}})
+defer response.Close()
+wsCli := response.WebSocket()
+```
+
+
+
+
+
+
+# Contributing
+If you have a bug report or feature request, you can [open an issue](../../issues/new)
+# Contact
+If you have questions, feel free to reach out to us in the following ways:
+* QQ Group (Chinese): 939111384 - <a href="http://qm.qq.com/cgi-bin/qm/qr?_wv=1027&k=yI72QqgPExDqX6u_uEbzAE_XfMW6h_d3&jump_from=webapi"><img src="https://pub.idqqimg.com/wpa/images/group.png"></a>
+* WeChat (Chinese): gospider007
+
+## Sponsors
+If you like and it really helps you, feel free to reward me with a cup of coffee, and don't forget to mention your github id.
+<table>
+    <tr>
+        <td align="center">
+            <img src="https://github.com/gospider007/tools/blob/master/play/wx.jpg?raw=true" height="200px" width="200px"   alt=""/>
+            <br />
+            <sub><b>Wechat</b></sub>
+        </td>
+        <td align="center">
+            <img src="https://github.com/gospider007/tools/blob/master/play/qq.jpg?raw=true" height="200px" width="200px"   alt=""/>
+            <br />
+            <sub><b>Alipay</b></sub>
+        </td>
+    </tr>
+</table>
--- a/client.go
+++ b/client.go
@@ -34,9 +34,10 @@ type ClientOption struct {

 	RedirectNum int //redirect num ,<0 no redirect,==0 no limit

-	DisDecode       bool                                                       //disable auto decode                               //关闭自动编码
-	DisUnZip        bool                                                       //disable auto zip decode                                      //关闭自动解压
-	TryNum          int                                                        //try num                                       //重试次数
+	DisDecode       bool                                                       //disable auto decode
+	DisUnZip        bool                                                       //disable auto zip decode
+	DisAlive        bool                                                       //disable  keepalive
+	TryNum          int                                                        //try num
 	OptionCallBack  func(context.Context, *Client, *RequestOption) error       //option callback,if error is returnd, break request
 	ResultCallBack  func(context.Context, *Client, *Response) error            //result callback,if error is returnd,next errCallback
 	ErrCallBack     func(context.Context, *Client, error) error                //error callback,if error is returnd,break request
@@ -51,7 +52,9 @@ type Client struct {
 	redirectNum int
 	disDecode   bool
 	disUnZip    bool
-	tryNum      int
+	disAlive    bool
+
+	tryNum int

 	requestCallBack func(context.Context, *http.Request, *http.Response) error

@@ -96,7 +99,7 @@ func NewClient(preCtx context.Context, options ...ClientOption) (*Client, error)
 		option.DialTimeout = time.Second * 15
 	}
 	if option.TlsHandshakeTimeout == 0 {
-		option.TlsHandshakeTimeout = time.Second * 5
+		option.TlsHandshakeTimeout = time.Second * 15
 	}
 	//cookiesjar
 	var jar *Jar
@@ -116,7 +119,7 @@ func NewClient(preCtx context.Context, options ...ClientOption) (*Client, error)
 	// 		EnableDatagrams: true,
 	// 	}
 	// }
-	transport := NewRoundTripper(ctx, RoundTripperOption{
+	transport := newRoundTripper(ctx, RoundTripperOption{
 		TlsHandshakeTimeout:   option.TlsHandshakeTimeout,
 		DialTimeout:           option.DialTimeout,
 		KeepAlive:             option.KeepAlive,
@@ -161,6 +164,7 @@ func NewClient(preCtx context.Context, options ...ClientOption) (*Client, error)
 		redirectNum:    option.RedirectNum,
 		disDecode:      option.DisDecode,
 		disUnZip:       option.DisUnZip,
+		disAlive:       option.DisAlive,
 		tryNum:         option.TryNum,
 		optionCallBack: option.OptionCallBack,
 		resultCallBack: option.ResultCallBack,
--- a/go.mod
+++ b/go.mod
@@ -6,11 +6,11 @@ require (
 	github.com/gospider007/bar v0.0.0-20231015040703-7d4e67e1741a
 	github.com/gospider007/bs4 v0.0.0-20231015041001-064efde5bd08
 	github.com/gospider007/gson v0.0.0-20231015042048-4fae1418931b
-	github.com/gospider007/gtls v0.0.0-20231015025737-e18d927ad339
-	github.com/gospider007/ja3 v0.0.0-20231015030157-fdfe60187c35
+	github.com/gospider007/gtls v0.0.0-20231015064159-7a2b27c0a8ef
+	github.com/gospider007/ja3 v0.0.0-20231016144937-e8ea455fa9c5
 	github.com/gospider007/net v0.0.0-20231015031122-61ca42e0006f
 	github.com/gospider007/re v0.0.0-20231015023348-717c984874af
-	github.com/gospider007/tools v0.0.0-20231015025637-9b1d99c470f4
+	github.com/gospider007/tools v0.0.0-20231015122431-2acd695704d6
 	github.com/gospider007/websocket v0.0.0-20231015042221-9176c1ebbdfc
 	github.com/refraction-networking/utls v1.5.4
 	golang.org/x/net v0.17.0
@@ -18,7 +18,7 @@ require (

 require (
 	github.com/PuerkitoBio/goquery v1.8.1 // indirect
-	github.com/andybalholm/brotli v1.0.5 // indirect
+	github.com/andybalholm/brotli v1.0.6 // indirect
 	github.com/andybalholm/cascadia v1.3.2 // indirect
 	github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect
 	github.com/cloudflare/circl v1.3.3 // indirect
@@ -31,7 +31,7 @@ require (
 	github.com/gospider007/blog v0.0.0-20231015040505-4b08187c11ee // indirect
 	github.com/gospider007/kinds v0.0.0-20231015025159-69ab7d8d9b94 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/klauspost/compress v1.17.0 // indirect
+	github.com/klauspost/compress v1.17.1 // indirect
 	github.com/leodido/go-urn v1.2.4 // indirect
 	github.com/mattn/go-isatty v0.0.19 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
--- a/go.sum
+++ b/go.sum
@@ -2,6 +2,8 @@ github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAc
 github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ=
 github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
 github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
+github.com/andybalholm/brotli v1.0.6 h1:Yf9fFpf49Zrxb9NlQaluyE92/+X7UVHlhMNJN2sxfOI=
+github.com/andybalholm/brotli v1.0.6/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
 github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
 github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss=
 github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
@@ -79,8 +81,14 @@ github.com/gospider007/gson v0.0.0-20231015042048-4fae1418931b h1:Gd5tOuEAlrs+5I
 github.com/gospider007/gson v0.0.0-20231015042048-4fae1418931b/go.mod h1:frJXgZBXIUJlfsIB7LrEGEIU2SeQu7xP/1xs/Tphlwg=
 github.com/gospider007/gtls v0.0.0-20231015025737-e18d927ad339 h1:YmZe17AF0QZQp0K0QlArKa0UPoULgQepI5BXeHGs8/8=
 github.com/gospider007/gtls v0.0.0-20231015025737-e18d927ad339/go.mod h1:HNrQHMSTsG73mweP5p010QAgXVaZ6RgcfxfJOdyieUY=
+github.com/gospider007/gtls v0.0.0-20231015064159-7a2b27c0a8ef h1:lGPyThRy3XZTRX3+Z0WHhCByv3TAPd22UTNyVNApPVc=
+github.com/gospider007/gtls v0.0.0-20231015064159-7a2b27c0a8ef/go.mod h1:HNrQHMSTsG73mweP5p010QAgXVaZ6RgcfxfJOdyieUY=
 github.com/gospider007/ja3 v0.0.0-20231015030157-fdfe60187c35 h1:Eyfpv7ZG9+l//KIKeEs0CJRTGPIeP5sfZ0kJCspfakU=
 github.com/gospider007/ja3 v0.0.0-20231015030157-fdfe60187c35/go.mod h1:BNtOUDtO8NF8aR7rtl3CAvE/LG9+RWtUiLssswjGLTI=
+github.com/gospider007/ja3 v0.0.0-20231016035739-bf5e78522517 h1:QrFF3PCfIJmc9sphHVafBo433fdBE5pilzTs6fmWZDs=
+github.com/gospider007/ja3 v0.0.0-20231016035739-bf5e78522517/go.mod h1:BNtOUDtO8NF8aR7rtl3CAvE/LG9+RWtUiLssswjGLTI=
+github.com/gospider007/ja3 v0.0.0-20231016144937-e8ea455fa9c5 h1:m5HyZ07ExPHI/ooVNs9Ri1p8hH1UPpjPwMG2iO94Ojo=
+github.com/gospider007/ja3 v0.0.0-20231016144937-e8ea455fa9c5/go.mod h1:BNtOUDtO8NF8aR7rtl3CAvE/LG9+RWtUiLssswjGLTI=
 github.com/gospider007/kinds v0.0.0-20231015025159-69ab7d8d9b94 h1:K2XYy6B6M62o35KMabuzl9DmA4WQftanUgKomb2YYU4=
 github.com/gospider007/kinds v0.0.0-20231015025159-69ab7d8d9b94/go.mod h1:5e4rIpZmhwqOLVVRZa0uZFAuMPH15dO0qqzKf1s3IgI=
 github.com/gospider007/net v0.0.0-20231015031122-61ca42e0006f h1:Cj8eyFdQnOxTIt8VbyYwAwUZRg/sTX+9hL8jvg4ksqs=
@@ -89,6 +97,8 @@ github.com/gospider007/re v0.0.0-20231015023348-717c984874af h1:bYip5he4jUPslsEY
 github.com/gospider007/re v0.0.0-20231015023348-717c984874af/go.mod h1:fOcqR9yJHQjUmpLerdxJSoMJ4pFyMfRirM12rTMEH38=
 github.com/gospider007/tools v0.0.0-20231015025637-9b1d99c470f4 h1:XeDsy/Qy/uH+MhRPeBK44Owl5V/zotyklIgP9rS2ONc=
 github.com/gospider007/tools v0.0.0-20231015025637-9b1d99c470f4/go.mod h1:UAIsaldUoH2zZqg4i0es5Q6x/GS0BkgCeeQ/sLnnZJk=
+github.com/gospider007/tools v0.0.0-20231015122431-2acd695704d6 h1:0yKzCr0+F6RTKVppEP5o1PzmrSzy5OrtwDKVEW4uOx4=
+github.com/gospider007/tools v0.0.0-20231015122431-2acd695704d6/go.mod h1:UAIsaldUoH2zZqg4i0es5Q6x/GS0BkgCeeQ/sLnnZJk=
 github.com/gospider007/websocket v0.0.0-20231015042221-9176c1ebbdfc h1:eLhD3+H6ads14/yDR8L6UQ/7ZCAjO5QC5fHLGTpqCiw=
 github.com/gospider007/websocket v0.0.0-20231015042221-9176c1ebbdfc/go.mod h1:xfgcqdRvRpjIyPAhO5/o+ubuTmJAey/wGnY8DyiNHRc=
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
@@ -98,6 +108,8 @@ github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYs
 github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
 github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM=
 github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/klauspost/compress v1.17.1 h1:NE3C767s2ak2bweCZo3+rdP4U/HoyVXLv/X9f2gPS5g=
+github.com/klauspost/compress v1.17.1/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg=
 github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
--- a/option.go
+++ b/option.go
@@ -42,6 +42,7 @@ type RequestOption struct {
 	DisCookie bool //disable cookies,not use cookies
 	DisDecode bool //disable auto decode
 	DisRead   bool //disable auto read
+	DisAlive  bool //disable  keepalive

 	Bar      bool //enable bar display
 	DisProxy bool //force disable proxy
@@ -219,6 +220,10 @@ func (obj *Client) newRequestOption(option RequestOption) RequestOption {
 		option.DisUnZip = obj.disUnZip
 	}

+	if !option.DisAlive {
+		option.DisAlive = obj.disAlive
+	}
+
 	if !option.Ja3Spec.IsSet() {
 		if option.Ja3 {
 			option.Ja3Spec = ja3.DefaultJa3Spec()
--- a/requests.go
+++ b/requests.go
@@ -139,6 +139,7 @@ type reqCtxData struct {
 	redirectNum int
 	proxy       *url.URL
 	disProxy    bool
+	disAlive    bool

 	requestCallBack func(context.Context, *http.Request, *http.Response) error

@@ -335,6 +336,7 @@ func (obj *Client) request(preCtx context.Context, option RequestOption) (respon
 	var reqs *http.Request
 	//init ctxData
 	ctxData := new(reqCtxData)
+	ctxData.disAlive = option.DisAlive
 	ctxData.requestCallBack = option.RequestCallBack
 	//init proxy
 	ctxData.disProxy = option.DisProxy
--- a/roundTripper.go
+++ b/roundTripper.go
@@ -7,6 +7,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"log"
 	"net"
 	"net/url"
 	"sync"
@@ -147,14 +148,15 @@ func (obj *RoundTripper) dial(ctxData *reqCtxData, addr string, key string, req
 		if ctxData.ja3Spec.IsSet() {
 			tlsConn, err := obj.dialer.AddJa3Tls(ctx, netConn, host, isWebSocket, ctxData.ja3Spec, obj.UtlsConfig())
 			if err != nil {
-				return conne, err
+				return conne, tools.WrapError(err, "add tls error")
 			}
+			log.Print(tlsConn.ConnectionState().NegotiatedProtocol)
 			conne.h2 = tlsConn.ConnectionState().NegotiatedProtocol == "h2"
 			netConn = tlsConn
 		} else {
 			tlsConn, err := obj.dialer.AddTls(ctx, netConn, host, isWebSocket, obj.TlsConfig())
 			if err != nil {
-				return conne, err
+				return conne, tools.WrapError(err, "add tls error")
 			}
 			conne.h2 = tlsConn.ConnectionState().NegotiatedProtocol == "h2"
 			netConn = tlsConn
@@ -343,7 +345,7 @@ func (obj *Connecotr) taskMain(task *reqTask, afterTime *time.Timer, responseHea
 	}
 	select {
 	case <-obj.closeCtx.Done():
-		return nil, obj.closeCtx.Err(), true
+		return nil, tools.WrapError(obj.closeCtx.Err(), "close ctx error: "), true
 	default:
 	}
 	if obj.h2 {
@@ -365,7 +367,7 @@ func (obj *Connecotr) taskMain(task *reqTask, afterTime *time.Timer, responseHea
 			<-obj.bodyCtx.Done() //wait body close
 		}
 	case <-obj.deleteCtx.Done(): //force conn close
-		task.err = obj.deleteCtx.Err()
+		task.err = tools.WrapError(obj.deleteCtx.Err(), "delete ctx error: ")
 		task.cnl()
 	case <-afterTime.C:
 		task.err = errors.New("response Header is Timeout")
@@ -437,7 +439,7 @@ type RoundTripperOption struct {
 	GetProxy              func(ctx context.Context, url *url.URL) (string, error)
 }

-func NewRoundTripper(preCtx context.Context, option RoundTripperOption) *RoundTripper {
+func newRoundTripper(preCtx context.Context, option RoundTripperOption) *RoundTripper {
 	if preCtx == nil {
 		preCtx = context.TODO()
 	}
@@ -503,13 +505,13 @@ func (obj *RoundTripper) poolRoundTrip(task *reqTask, key string) (bool, error)
 	}
 	select {
 	case <-obj.ctx.Done():
-		return false, obj.ctx.Err()
+		return false, tools.WrapError(obj.ctx.Err(), "roundTripper close ctx error: ")
 	case pool.tasks <- task:
 		select {
 		case <-task.emptyPool:
 		case <-task.ctx.Done():
 			if task.err == nil && task.res == nil {
-				task.err = obj.ctx.Err()
+				task.err = tools.WrapError(task.ctx.Err(), "task close ctx error: ")
 			}
 			return true, nil
 		}
@@ -538,15 +540,17 @@ func (obj *RoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
 	task.ctx, task.cnl = context.WithCancel(obj.ctx)
 	defer task.cnl()
 	//get pool conn
-	if ok, err := obj.poolRoundTrip(task, key); err != nil {
-		return nil, err
-	} else if ok { //is conn multi
-		if ctxData.requestCallBack != nil {
-			if err = ctxData.requestCallBack(task.req.Context(), req, task.res); err != nil {
-				task.err = err
+	if !ctxData.disAlive {
+		if ok, err := obj.poolRoundTrip(task, key); err != nil {
+			return nil, err
+		} else if ok { //is conn multi
+			if ctxData.requestCallBack != nil {
+				if err = ctxData.requestCallBack(task.req.Context(), req, task.res); err != nil {
+					task.err = err
+				}
 			}
+			return task.res, task.err
 		}
-		return task.res, task.err
 	}
 newConn:
 	addr := getAddr(req.URL)
@@ -560,7 +564,7 @@ newConn:
 	if task.err == nil && task.res == nil {
 		task.err = obj.ctx.Err()
 	}
-	if task.isPool() {
+	if task.isPool() && !ctxData.disAlive {
 		obj.putConnPool(key, conn)
 	}
 	if ctxData.requestCallBack != nil {