package/requests
1
0
Fork 0
mirror of https://github.com/gospider007/requests.git synced 2025-12-24 13:57:52 +08:00
Code Issues Packages Projects Releases Wiki Activity

add order headers option

Browse Source
This commit is contained in:
gospider
2024-07-26 18:10:35 +08:00
parent 9bf5cb6bb2
commit 71cb593bf6
9 changed files with 157 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
conn.go
View File

@@ -6,7 +6,7 @@ import (
"errors"
"io"
"net"
"net/textproto"
"net/http"
"sync"
"sync/atomic"
"time"
@@ -27,7 +27,7 @@ type connecotr struct {
rawConn net.Conn
h2RawConn *http2.ClientConn
proxy string
r *textproto.Reader
r *bufio.Reader
w *bufio.Writer
pr *pipCon
inPool bool
@@ -101,7 +101,7 @@ func (obj *connecotr) wrapBody(task *reqTask) {
}
func (obj *connecotr) http1Req(task *reqTask) {
if task.err = httpWrite(task.req, obj.w, task.orderHeaders); task.err == nil {
task.res, task.err = readResponse(obj.r, task.req)
task.res, task.err = http.ReadResponse(obj.r, task.req)
if task.err != nil {
task.err = tools.WrapError(task.err, "http1 read error")
} else if task.res == nil {

3
dial.go
View File

@@ -7,7 +7,6 @@ import (
"errors"
"io"
"net"
"net/textproto"
"net/url"
"sync"
"time"
@@ -407,7 +406,7 @@ func (obj *DialClient) clientVerifyHttps(ctx context.Context, scheme string, pro
if err = connectReq.Write(conn); err != nil {
return err
}
resp, err := readResponse(textproto.NewReader(bufio.NewReader(conn)), connectReq)
resp, err := http.ReadResponse(bufio.NewReader(conn), connectReq)
if err != nil {
return err
}

3
go.mod
View File

@@ -1,6 +1,6 @@
module github.com/gospider007/requests
go 1.21.3
go 1.22.5
require (
github.com/gospider007/bar v0.0.0-20231215084215-956cfa59ce61
@@ -34,6 +34,7 @@ require (
github.com/miekg/dns v1.1.61 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/stretchr/testify v1.9.0 // indirect
github.com/tidwall/gjson v1.17.1 // indirect
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.1 // indirect

3
go.sum
View File

@@ -64,8 +64,7 @@ github.com/refraction-networking/utls v1.6.6 h1:igFsYBUJPYM8Rno9xUuDoM5GQrVEqY4l
github.com/refraction-networking/utls v1.6.6/go.mod h1:BC3O4vQzye5hqpmDTWUqi4P5DDhzJfkV1tdqtawQIH0=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U=
github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=

5
option.go
View File

@@ -15,6 +15,7 @@ import (
// Connection Management Options
type ClientOption struct {
OrderHeaders []string //order headers
Ja3Spec ja3.Ja3Spec //custom ja3Spec,use ja3.CreateSpecWithStr or ja3.CreateSpecWithId create
H2Ja3Spec ja3.H2Ja3Spec //h2 fingerprint
Proxy string //proxy,support https,http,socks5
@@ -51,6 +52,7 @@ type ClientOption struct {
// Options for sending requests
type RequestOption struct {
OrderHeaders []string //order headers
Ja3Spec ja3.Ja3Spec //custom ja3Spec,use ja3.CreateSpecWithStr or ja3.CreateSpecWithId create
H2Ja3Spec ja3.H2Ja3Spec //custom h2 fingerprint
Proxy string //proxy,support http,https,socks5,example：http://127.0.0.1:7005
@@ -242,6 +244,9 @@ func (obj *Client) newRequestOption(option RequestOption) RequestOption {
if !option.Bar {
option.Bar = obj.option.Bar
}
if option.OrderHeaders == nil {
option.OrderHeaders = obj.option.OrderHeaders
}
if option.OptionCallBack == nil {
option.OptionCallBack = obj.option.OptionCallBack
}

3
requests.go
View File

@@ -266,6 +266,9 @@ func (obj *Client) request(ctx context.Context, option *RequestOption) (response
if err != nil {
return response, tools.WrapError(err, errors.New("tempRequest init headers error"), err)
}
if orderHeaders == nil {
orderHeaders = option.OrderHeaders
}
//设置 h2 请求头顺序
if orderHeaders != nil {
if !option.H2Ja3Spec.IsSet() {

3
roundTripper.go
View File

@@ -7,7 +7,6 @@ import (
"errors"
"fmt"
"net"
"net/textproto"
"net/url"
"strings"
"time"
@@ -177,7 +176,7 @@ func (obj *roundTripper) dial(ctxData *reqCtxData, req *http.Request) (conn *con
return conne, err
}
} else {
conne.r, conne.w = textproto.NewReader(bufio.NewReader(conne)), bufio.NewWriter(conne)
conne.r, conne.w = bufio.NewReader(conne), bufio.NewWriter(conne)
}
return conne, err
}

52
test/fingerprint/ja4_test.go
View File

@@ -50,3 +50,55 @@ func TestOrderHeaders(t *testing.T) {
}
}
}
func TestOrderHeaders2(t *testing.T) {
headers := map[string]any{
"Accept-Encoding": "gzip, deflate, br",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
"User-Agent": requests.UserAgent,
"Accept-Language": requests.AcceptLanguage,
"Sec-Ch-Ua": requests.SecChUa,
"Sec-Ch-Ua-Mobile": "?0",
"Sec-Ch-Ua-Platform": `"Windows"`,
}
orderHeaders := []string{
"Accept-Encoding",
"Accept",
"User-Agent",
"Accept-Language",
"Sec-Ch-Ua",
"Sec-Ch-Ua-Mobile",
"Sec-Ch-Ua-Platform",
}
resp, err := requests.Get(nil, "https://tools.scrapfly.io/api/fp/anything", requests.RequestOption{
Headers: headers,
OrderHeaders: orderHeaders,
// ForceHttp1: true,
})
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json()
header_order := jsonData.Find("ordered_headers_key")
if !header_order.Exists() {
t.Fatal("not found akamai")
}
i := -1
log.Print(header_order)
// log.Print(headers.Keys())
kks := []string{}
for _, kk := range orderHeaders {
kks = append(kks, textproto.CanonicalMIMEHeaderKey(kk))
}
for _, key := range header_order.Array() {
kk := textproto.CanonicalMIMEHeaderKey(key.String())
if slices.Contains(kks, kk) {
i2 := slices.Index(kks, textproto.CanonicalMIMEHeaderKey(kk))
if i2 < i {
log.Print(header_order)
t.Fatal("not equal")
}
i = i2
}
}
}

142
tools.go
View File

@@ -3,14 +3,12 @@ package requests
import (
"bufio"
"context"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/textproto"
"net/url"
"strconv"
"strings"
_ "unsafe"
@@ -63,9 +61,11 @@ var replaceMap = map[string]string{
"Sec-Ch-Ua-Mobile": "sec-ch-ua-mobile",
"Sec-Ch-Ua-Platform": "sec-ch-ua-platform",
}
var quoteEscaper = strings.NewReplacer("\\", "\\\\", `"`, "\\\"")
//go:linkname escapeQuotes mime/multipart.escapeQuotes
func escapeQuotes(string) string
func escapeQuotes(s string) string {
return quoteEscaper.Replace(s)
}
//go:linkname readCookies net/http.readCookies
func readCookies(h http.Header, filter string) []*http.Cookie
@@ -73,23 +73,96 @@ func readCookies(h http.Header, filter string) []*http.Cookie
//go:linkname readSetCookies net/http.readSetCookies
func readSetCookies(h http.Header) []*http.Cookie
//go:linkname ReadRequest net/http.readRequest
func ReadRequest(b *bufio.Reader) (*http.Request, error)
func removeZone(host string) string {
if !strings.HasPrefix(host, "[") {
return host
}
i := strings.LastIndex(host, "]")
if i < 0 {
return host
}
j := strings.LastIndex(host[:i], "%")
if j < 0 {
return host
}
return host[:j] + host[i:]
}
//go:linkname removeZone net/http.removeZone
func removeZone(host string) string
func chunked(te []string) bool { return len(te) > 0 && te[0] == "chunked" }
func isIdentity(te []string) bool { return len(te) == 1 && te[0] == "identity" }
func shouldSendContentLength(t *http.Request) bool {
if chunked(t.TransferEncoding) {
return false
}
if t.ContentLength > 0 {
return true
}
if t.ContentLength < 0 {
return false
}
// Many servers expect a Content-Length for these methods
if t.Method == "POST" || t.Method == "PUT" || t.Method == "PATCH" {
return true
}
if t.ContentLength == 0 && isIdentity(t.TransferEncoding) {
if t.Method == "GET" || t.Method == "HEAD" {
return false
}
return true
}
//go:linkname shouldSendContentLength net/http.(*transferWriter).shouldSendContentLength
func shouldSendContentLength(t *http.Request) bool
return false
}
//go:linkname removeEmptyPort net/http.removeEmptyPort
func removeEmptyPort(host string) string
func hasPort(s string) bool { return strings.LastIndex(s, ":") > strings.LastIndex(s, "]") }
//go:linkname redirectBehavior net/http.redirectBehavior
func redirectBehavior(reqMethod string, resp *http.Response, ireq *http.Request) (redirectMethod string, shouldRedirect, includeBody bool)
// removeEmptyPort strips the empty port in ":port" to ""
// as mandated by RFC 3986 Section 6.2.3.
func removeEmptyPort(host string) string {
if hasPort(host) {
return strings.TrimSuffix(host, ":")
}
return host
}
//go:linkname readTransfer net/http.readTransfer
func readTransfer(msg any, r *bufio.Reader) (err error)
func outgoingLength(r *http.Request) int64 {
if r.Body == nil || r.Body == http.NoBody {
return 0
}
if r.ContentLength != 0 {
return r.ContentLength
}
return -1
}
func redirectBehavior(reqMethod string, resp *http.Response, ireq *http.Request) (redirectMethod string, shouldRedirect, includeBody bool) {
switch resp.StatusCode {
case 301, 302, 303:
redirectMethod = reqMethod
shouldRedirect = true
includeBody = false
// RFC 2616 allowed automatic redirection only with GET and
// HEAD requests. RFC 7231 lifts this restriction, but we still
// restrict other methods to GET to maintain compatibility.
// See Issue 18570.
if reqMethod != "GET" && reqMethod != "HEAD" {
redirectMethod = "GET"
}
case 307, 308:
redirectMethod = reqMethod
shouldRedirect = true
includeBody = true
if ireq.GetBody == nil && outgoingLength(ireq) != 0 {
// We had a request body, and 307/308 require
// re-sending it, but GetBody is not defined. So just
// return this response to the user instead of an
// error, like we did in Go 1.7 and earlier.
shouldRedirect = false
}
}
return redirectMethod, shouldRedirect, includeBody
}
var filterHeaderKeys = ja3.DefaultOrderHeadersWithH2()
@@ -195,43 +268,6 @@ func NewRequestWithContext(ctx context.Context, method string, u *url.URL, body
return req, nil
}
func readResponse(tp *textproto.Reader, req *http.Request) (*http.Response, error) {
resp := &http.Response{
Request: req,
}
// Parse the first line of the response.
line, err := tp.ReadLine()
if err != nil {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
return nil, err
}
proto, status, ok := strings.Cut(line, " ")
if !ok {
return nil, errors.New("malformed HTTP response")
}
resp.Proto = proto
resp.Status = strings.TrimLeft(status, " ")
statusCode, _, _ := strings.Cut(resp.Status, " ")
if resp.StatusCode, err = strconv.Atoi(statusCode); err != nil {
return nil, errors.New("malformed HTTP status code")
}
if resp.ProtoMajor, resp.ProtoMinor, ok = http.ParseHTTPVersion(resp.Proto); !ok {
return nil, errors.New("malformed HTTP version")
}
// Parse the response headers.
mimeHeader, err := tp.ReadMIMEHeader()
if err != nil {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
return nil, err
}
resp.Header = http.Header(mimeHeader)
return resp, readTransfer(resp, tp.R)
}
func addCookie(req *http.Request, cookies Cookies) {
cooks := Cookies(readCookies(req.Header, ""))
for _, cook := range cookies {