diff --git a/client.go b/client.go index 0371557..bab2b60 100644 --- a/client.go +++ b/client.go @@ -108,7 +108,7 @@ func (obj *Client) do(ctx *Response) (err error) { return nil } ctx.request.Response = ctx.response - ctx.request.Header = defaultHeaders() + ctx.request.Header = make(http.Header) ctx.request.Header.Set("Referer", ctx.Request().URL.String()) for key := range ctx.request.Header { if val := ctx.Request().Header.Get(key); val != "" { diff --git a/dial.go b/dial.go index 2bbfe5b..e4e6278 100644 --- a/dial.go +++ b/dial.go @@ -468,7 +468,6 @@ func (obj *Dialer) Socks5UdpProxy(ctx *Response, proxyAddress Address, remoteAdd } func (obj *Dialer) clientVerifyHttps(ctx context.Context, conn net.Conn, proxyAddress Address, remoteAddress Address) (err error) { hdr := make(http.Header) - hdr.Set("User-Agent", tools.UserAgent) if proxyAddress.User != "" && proxyAddress.Password != "" { hdr.Set("Proxy-Authorization", "Basic "+tools.Base64Encode(proxyAddress.User+":"+proxyAddress.Password)) } diff --git a/headers.go b/headers.go index 64672fd..f44c19f 100644 --- a/headers.go +++ b/headers.go @@ -4,26 +4,12 @@ import ( "fmt" "net/http" - "github.com/gospider007/tools" "gopkg.in/errgo.v2/fmt/errors" ) -func defaultHeaders() http.Header { - return http.Header{ - "User-Agent": []string{tools.UserAgent}, - "Connection": []string{"keep-alive"}, - "Accept": []string{"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7"}, - "Accept-Encoding": []string{"gzip, deflate, br, zstd"}, - "Accept-Language": []string{tools.AcceptLanguage}, - "Sec-Ch-Ua": []string{tools.SecChUa}, - "Sec-Ch-Ua-Mobile": []string{"?0"}, - "Sec-Ch-Ua-Platform": []string{`"Windows"`}, - } -} - func (obj *RequestOption) initOrderHeaders() (http.Header, error) { if obj.Headers == nil { - return defaultHeaders(), nil + return make(http.Header), nil } switch headers := obj.Headers.(type) { case http.Header: diff --git a/sepc.go b/sepc.go index ae1f473..d0c988e 100644 --- a/sepc.go +++ b/sepc.go @@ -4,7 +4,7 @@ import ( "bytes" "encoding/hex" "errors" - "log" + "net/textproto" "slices" "strings" @@ -30,16 +30,6 @@ func (obj *Spec) String() string { return tools.BytesToString(obj.raw) } -var defaultHeaderKeys = []string{ - "Accept", - "Accept-Encoding", - "Accept-Language", - "Sec-Ch-Ua", - "Sec-Ch-Ua-Mobile", - "Sec-Ch-Ua-Platform", - "User-Agent", -} - type Spec struct { OrderHeaders [][2]string raw []byte @@ -91,7 +81,6 @@ func ParseGospiderSpec(value string) (*GospiderSpec, error) { if spec.TLSSpec, err = ja3.ParseSpec(b); err != nil { return nil, err } - log.Print("发送请求:", spec.TLSSpec.CipherSuites) } if specs[1] != "" { b, err := hex.DecodeString(specs[1]) @@ -127,7 +116,7 @@ func (obj *RequestOption) initSpec() error { if obj.orderHeaders == nil { orderData := NewOrderData() for _, kv := range gospiderSpec.H1Spec.OrderHeaders { - if slices.Contains(defaultHeaderKeys, strings.ToLower(kv[0])) { + if slices.Contains(tools.DefaultHeaderKeys, kv[0]) { orderData.Add(kv[0], kv[1]) } else { orderData.Add(kv[0], nil) @@ -140,10 +129,11 @@ func (obj *RequestOption) initSpec() error { if obj.orderHeaders == nil { orderData := NewOrderData() for _, kv := range gospiderSpec.H2Spec.OrderHeaders { - if slices.Contains(defaultHeaderKeys, strings.ToLower(kv[0])) { - orderData.Add(kv[0], kv[1]) + key := textproto.CanonicalMIMEHeaderKey(kv[0]) + if slices.Contains(tools.DefaultHeaderKeys, key) { + orderData.Add(key, kv[1]) } else { - orderData.Add(kv[0], nil) + orderData.Add(key, nil) } } obj.orderHeaders = orderData diff --git a/test/fingerprint/ja4_test.go b/test/fingerprint/ja4_test.go index 97334a6..fb3f394 100644 --- a/test/fingerprint/ja4_test.go +++ b/test/fingerprint/ja4_test.go @@ -7,7 +7,6 @@ import ( "testing" "github.com/gospider007/requests" - "github.com/gospider007/tools" ) func TestOrderHeaders(t *testing.T) { @@ -15,9 +14,6 @@ func TestOrderHeaders(t *testing.T) { headers := requests.NewOrderData() headers.Add("Accept-Encoding", "gzip, deflate, br") headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7") - headers.Add("User-Agent", tools.UserAgent) - headers.Add("Accept-Language", tools.AcceptLanguage) - headers.Add("Sec-Ch-Ua", tools.SecChUa) headers.Add("Sec-Ch-Ua-Mobile", "?0") headers.Add("Sec-Ch-Ua-Platform", `"Windows"`) resp, err := requests.Get(nil, "https://tools.scrapfly.io/api/fp/anything", requests.RequestOption{ @@ -58,9 +54,6 @@ func TestOrderHeaders2(t *testing.T) { headers := map[string]any{ "Accept-Encoding": "gzip, deflate, br", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7", - "User-Agent": tools.UserAgent, - "Accept-Language": tools.AcceptLanguage, - "Sec-Ch-Ua": tools.SecChUa, "Sec-Ch-Ua-Mobile": "?0", "Sec-Ch-Ua-Platform": `"Windows"`, }