From de912c24b7151cd1ec6cb94f02c9ac33d1413a43 Mon Sep 17 00:00:00 2001 From: bxd <2216403312@qq.com> Date: Tue, 17 Oct 2023 23:12:24 +0800 Subject: [PATCH] add http1 with order headers --- README.md | 21 +++++++++++++++ client.go | 4 +++ option.go | 5 +++- requests.go | 10 ++++--- roundTripper.go | 71 +++++++++++++++++++++++++++++++++++++++++++------ 5 files changed, 98 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 3a5e0fb..80b1bff 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,27 @@ func main() { log.Print(resp.StatusCode()) //return status code } ``` +## setting order headers with http1 +```go +package main + +import ( + "log" + + "github.com/gospider007/requests" +) + +func main() { + resp, err := requests.Get(nil, "http://httpbin.org/anything", requests.RequestOption{ + OrderHeaders: []string{"accept-encoding"}}, + ) + if err != nil { + log.Panic(err) + } + log.Print(resp.Text()) +} +``` + ## send websocket ```go package main diff --git a/client.go b/client.go index 72237fb..174b0d2 100644 --- a/client.go +++ b/client.go @@ -13,6 +13,7 @@ import ( ) type ClientOption struct { + OrderHeaders []string //order headers with http1 Ja3 bool //enable ja3 fingerprint Ja3Spec ja3.Ja3Spec //custom ja3Spec,use ja3.CreateSpecWithStr or ja3.CreateSpecWithId create H2Ja3Spec ja3.H2Ja3Spec //h2 fingerprint @@ -42,6 +43,8 @@ type ClientOption struct { Dns net.IP //dns } type Client struct { + orderHeaders []string + jar *Jar redirectNum int disDecode bool @@ -154,6 +157,7 @@ func NewClient(preCtx context.Context, options ...ClientOption) (*Client, error) noJarClient: noJarClient, requestCallBack: option.RequestCallBack, + orderHeaders: option.OrderHeaders, disCookie: option.DisCookie, redirectNum: option.RedirectNum, disDecode: option.DisDecode, diff --git a/option.go b/option.go index 7f6a9c8..08be290 100644 --- a/option.go +++ b/option.go @@ -17,6 +17,7 @@ import ( ) type RequestOption struct { + OrderHeaders []string //order headers with http1 Ja3 bool //enable ja3 fingerprint Ja3Spec ja3.Ja3Spec //custom ja3Spec,use ja3.CreateSpecWithStr or ja3.CreateSpecWithId create H2Ja3Spec ja3.H2Ja3Spec //custom h2 fingerprint @@ -221,7 +222,9 @@ func (obj *Client) newRequestOption(option RequestOption) RequestOption { if !option.DisAlive { option.DisAlive = obj.disAlive } - + if option.OrderHeaders == nil { + option.OrderHeaders = obj.orderHeaders + } if !option.Ja3Spec.IsSet() { if option.Ja3 { option.Ja3Spec = ja3.DefaultJa3Spec() diff --git a/requests.go b/requests.go index 3de8946..e3013eb 100644 --- a/requests.go +++ b/requests.go @@ -136,10 +136,11 @@ var ( ) type reqCtxData struct { - redirectNum int - proxy *url.URL - disProxy bool - disAlive bool + redirectNum int + proxy *url.URL + disProxy bool + disAlive bool + orderHeaders []string requestCallBack func(context.Context, *http.Request, *http.Response) error @@ -338,6 +339,7 @@ func (obj *Client) request(preCtx context.Context, option RequestOption) (respon ctxData := new(reqCtxData) ctxData.disAlive = option.DisAlive ctxData.requestCallBack = option.RequestCallBack + ctxData.orderHeaders = option.OrderHeaders //init proxy ctxData.disProxy = option.DisProxy if !ctxData.disProxy { diff --git a/roundTripper.go b/roundTripper.go index 2fa75af..4341c9b 100644 --- a/roundTripper.go +++ b/roundTripper.go @@ -2,6 +2,7 @@ package requests import ( "bufio" + "bytes" "context" "crypto/tls" "errors" @@ -9,6 +10,7 @@ import ( "io" "net" "net/url" + "sort" "sync" "sync/atomic" "time" @@ -19,6 +21,9 @@ import ( "github.com/gospider007/net/http2" "github.com/gospider007/tools" utls "github.com/refraction-networking/utls" + "golang.org/x/exp/slices" + "golang.org/x/text/cases" + "golang.org/x/text/language" ) type roundTripper interface { @@ -26,12 +31,13 @@ type roundTripper interface { } type reqTask struct { - ctx context.Context - cnl context.CancelFunc - req *http.Request - res *http.Response - emptyPool chan struct{} - err error + ctx context.Context + cnl context.CancelFunc + req *http.Request + res *http.Response + emptyPool chan struct{} + err error + orderHeaders []string } func (obj *reqTask) isPool() bool { @@ -316,9 +322,58 @@ func (obj *Connecotr) wrapBody(task *reqTask) { body.conn = obj task.res.Body = body } + +type orderHeadersConn struct { + w io.Writer + raw []byte + fin bool + orderHeaders []string +} + +func (obj *orderHeadersConn) Write(p []byte) (n int, err error) { + if obj.fin { + return obj.w.Write(p) + } + obj.raw = append(obj.raw, p...) + if lastIndex := bytes.Index(obj.raw, []byte{'\r', '\n', '\r', '\n'}); lastIndex != -1 { + if firstIndex := bytes.Index(obj.raw, []byte{'\r', '\n'}) + 2; firstIndex < lastIndex { + kvs := bytes.Split(obj.raw[firstIndex:lastIndex], []byte{'\r', '\n'}) + sort.Slice(kvs, func(i, j int) bool { + iIndex := bytes.Index(kvs[i], []byte{':'}) + jIndex := bytes.Index(kvs[j], []byte{':'}) + if iIndex == -1 && jIndex == -1 { + return false + } else if iIndex == -1 { + return false + } else if jIndex == -1 { + return true + } else { + return slices.Index(obj.orderHeaders, tools.BytesToString(kvs[i][:iIndex])) > slices.Index(obj.orderHeaders, tools.BytesToString(kvs[j][:jIndex])) + } + }) + copy(obj.raw[firstIndex:lastIndex], bytes.Join(kvs, []byte{'\r', '\n'})) + } + _, err = obj.w.Write(obj.raw) + obj.fin = true + obj.raw = nil + } + return len(p), err +} + func (obj *Connecotr) http1Req(task *reqTask) { defer task.cnl() - if task.err = task.req.Write(obj); task.err == nil { + if task.orderHeaders != nil { + total := len(task.orderHeaders) * 2 + orderHeaders := make([]string, total) + for i, val := range task.orderHeaders { + orderHeaders[total-(i*2+1)] = val + orderHeaders[total-(i*2+2)] = cases.Title(language.Und, cases.NoLower).String(val) + } + task.err = task.req.Write(&orderHeadersConn{w: obj, raw: []byte{}, orderHeaders: orderHeaders}) + } else { + task.err = task.req.Write(obj) + } + if task.err == nil { if task.res, task.err = http.ReadResponse(obj.r, task.req); task.res != nil && task.err == nil { obj.wrapBody(task) } @@ -534,7 +589,7 @@ func (obj *RoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { } } key := getKey(ctxData, req) //pool key - task := &reqTask{req: req, emptyPool: make(chan struct{})} + task := &reqTask{req: req, emptyPool: make(chan struct{}), orderHeaders: ctxData.orderHeaders} task.ctx, task.cnl = context.WithCancel(obj.ctx) defer task.cnl() //get pool conn