package/requests
1
0
Fork 0
mirror of https://github.com/gospider007/requests.git synced 2025-12-24 13:57:52 +08:00
Code Issues Packages Projects Releases Wiki Activity

optimize cookies

Browse Source
This commit is contained in:
bxd
2023-11-09 15:38:12 +08:00
parent 534d537108
commit a8a1e0dc56
10 changed files with 52 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
client.go
View File

@@ -39,11 +39,11 @@ type ClientOption struct {
DialTimeout time.Duration //dial tcp timeout,default:15
KeepAlive time.Duration //keepalive,default:30
LocalAddr *net.TCPAddr
Dns *net.UDPAddr //dns
AddrType AddrType //dns parse addr type
Dns *net.UDPAddr //dns
AddrType gtls.AddrType //dns parse addr type
GetProxy func(ctx context.Context, url *url.URL) (string, error) //proxy callback:support https,http,socks5 proxy
GetAddrType func(string) AddrType
GetAddrType func(string) gtls.AddrType
}
type Client struct {
forceHttp1 bool
@@ -81,7 +81,7 @@ type Client struct {
ja3Spec ja3.Ja3Spec
h2Ja3Spec ja3.H2Ja3Spec
addrType AddrType
addrType gtls.AddrType
}
func NewClient(preCtx context.Context, options ...ClientOption) (*Client, error) {
@@ -147,7 +147,6 @@ func NewClient(preCtx context.Context, options ...ClientOption) (*Client, error)
if option.Proxy != "" {
result.proxy, err = gtls.VerifyProxy(option.Proxy)
}
if option.Ja3Spec.IsSet() {
result.ja3Spec = option.Ja3Spec
} else if option.Ja3 {

31
dial.go
View File

@@ -25,35 +25,28 @@ type DialClient struct {
dnsIpData sync.Map
dns *net.UDPAddr
localAddr *net.TCPAddr
getAddrType func(string) AddrType
getAddrType func(string) gtls.AddrType
}
type msgClient struct {
time time.Time
host string
}
type AddrType int
const (
Auto AddrType = 0
Ipv4 AddrType = 4
Ipv6 AddrType = 6
)
type DialOption struct {
DialTimeout time.Duration
KeepAlive time.Duration
LocalAddr *net.TCPAddr //network card ip
AddrType AddrType //first ip type
LocalAddr *net.TCPAddr //network card ip
AddrType gtls.AddrType //first ip type
Dns *net.UDPAddr
GetAddrType func(string) AddrType
GetAddrType func(string) gtls.AddrType
}
type DialerOption struct {
DialTimeout time.Duration
KeepAlive time.Duration
LocalAddr *net.TCPAddr //network card ip
AddrType AddrType //first ip type
LocalAddr *net.TCPAddr //network card ip
AddrType gtls.AddrType //first ip type
Dns *net.UDPAddr
GetAddrType func(string) AddrType
GetAddrType func(string) gtls.AddrType
}
func NewDialer(option DialerOption) *net.Dialer {
@@ -110,7 +103,7 @@ func (obj *DialClient) loadHost(host string) (string, bool) {
}
return host, false
}
func (obj *DialClient) AddrToIp(ctx context.Context, host string, ips []net.IPAddr, addrType AddrType) (string, error) {
func (obj *DialClient) AddrToIp(ctx context.Context, host string, ips []net.IPAddr, addrType gtls.AddrType) (string, error) {
ip, err := obj.lookupIPAddr(ctx, host, ips, addrType)
if err != nil {
return host, tools.WrapError(err, "addrToIp error,lookupIPAddr")
@@ -235,10 +228,10 @@ func (obj *DialClient) clientVerifySocks5(ctx context.Context, proxyUrl *url.URL
_, err = io.ReadFull(conn, readCon[:2])
return
}
func (obj *DialClient) lookupIPAddr(ctx context.Context, host string, ips []net.IPAddr, addrType AddrType) (net.IP, error) {
func (obj *DialClient) lookupIPAddr(ctx context.Context, host string, ips []net.IPAddr, addrType gtls.AddrType) (net.IP, error) {
for _, ipAddr := range ips {
ip := ipAddr.IP
if ipType := ParseIp(ip); ipType == 4 || ipType == 6 {
if ipType := gtls.ParseIp(ip); ipType == 4 || ipType == 6 {
if addrType == 0 || addrType == ipType {
return ip, nil
}
@@ -246,7 +239,7 @@ func (obj *DialClient) lookupIPAddr(ctx context.Context, host string, ips []net.
}
for _, ipAddr := range ips {
ip := ipAddr.IP
if ipType := ParseIp(ip); ipType == 4 || ipType == 6 {
if ipType := gtls.ParseIp(ip); ipType == 4 || ipType == 6 {
return ip, nil
}
}
@@ -307,7 +300,7 @@ func (obj *DialClient) DialContext(ctx context.Context, network string, addr str
host, ok := obj.loadHost(host)
if !ok { //dns parse
dialer = obj.getDialer(ctxData, true)
var addrType AddrType
var addrType gtls.AddrType
if ctxData.addrType != 0 {
addrType = ctxData.addrType
} else if obj.getAddrType != nil {

2
go.mod
View File

@@ -6,7 +6,7 @@ require (
github.com/gospider007/bar v0.0.0-20231024075629-3f50832a4cbf
github.com/gospider007/bs4 v0.0.0-20231024075735-6bbdac929d8b
github.com/gospider007/gson v0.0.0-20231108025125-ff62d4066dc4
github.com/gospider007/gtls v0.0.0-20231108025158-443489ca9953
github.com/gospider007/gtls v0.0.0-20231109073720-32fe741c06d3
github.com/gospider007/ja3 v0.0.0-20231029025157-38fc2f8f2d91
github.com/gospider007/net v0.0.0-20231028084010-313c148cf0a1
github.com/gospider007/re v0.0.0-20231024115818-adfd03636256

2
go.sum
View File

@@ -39,6 +39,8 @@ github.com/gospider007/gtls v0.0.0-20231024092712-01193b9f0404 h1:Qp/o+l3KgWmviE
github.com/gospider007/gtls v0.0.0-20231024092712-01193b9f0404/go.mod h1:fLcidMDKVv8b9NvLy0P/ZclltTaXJvTHANWiPCgDbSI=
github.com/gospider007/gtls v0.0.0-20231108025158-443489ca9953 h1:CN2nz4q88S3BhNR1QSmXubOgMGnxzWwdDL2OIASwYBs=
github.com/gospider007/gtls v0.0.0-20231108025158-443489ca9953/go.mod h1:EACbjhpG22ykBPE0xIKxc6D3N3YWTBoWrvsLLyhYqto=
github.com/gospider007/gtls v0.0.0-20231109073720-32fe741c06d3 h1:vyqxzWp1OQapjYY3St9CsI6Kln6QZrxwOM7cqgNlt9Y=
github.com/gospider007/gtls v0.0.0-20231109073720-32fe741c06d3/go.mod h1:8GOdq5AxbOweohgOQ7YNWwEqzIZHLtM5gHHQOPdwfFI=
github.com/gospider007/ja3 v0.0.0-20231029025157-38fc2f8f2d91 h1:qQokihfTAX+/U8GIMvZauRtE4G+/1Jq8XIJx8xLr04A=
github.com/gospider007/ja3 v0.0.0-20231029025157-38fc2f8f2d91/go.mod h1:ur78/uhYDDULSy1ldA/pPpGhjk973Q1VsPnbktXGU/g=
github.com/gospider007/kinds v0.0.0-20231024093643-7a4424f2d30e h1:lmX6IQKkrNDbXfHsvrv1Uz0MoG2v5+4VC6Gdh9irUNY=

2
headers.go
View File

@@ -15,7 +15,7 @@ const (
AcceptLanguage = "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6"
)
// 请求操作========================================================================= start
// get default headers
func DefaultHeaders() http.Header {
return http.Header{
"User-Agent": []string{UserAgent},

12
jar.go
View File

@@ -5,6 +5,7 @@ import (
"net/http/cookiejar"
"net/url"
"github.com/gospider007/gtls"
"golang.org/x/net/publicsuffix"
)
@@ -50,6 +51,12 @@ func (obj *Jar) SetCookies(href string, cookies ...any) error {
if err != nil {
return err
}
domain := u.Hostname()
if _, addType := gtls.ParseHost(domain); addType == 0 {
if domain, err = publicsuffix.EffectiveTLDPlusOne(domain); err != nil {
return err
}
}
for _, cookie := range cookies {
cooks, err := ReadCookies(cookie)
if err != nil {
@@ -60,10 +67,7 @@ func (obj *Jar) SetCookies(href string, cookies ...any) error {
cook.Path = "/"
}
if cook.Domain == "" {
cook.Domain, _ = publicsuffix.PublicSuffix(u.Hostname())
if err != nil {
return err
}
cook.Domain = domain
}
}
obj.jar.SetCookies(u, cooks)

5
option.go
View File

@@ -12,6 +12,7 @@ import (
"net/url"
"time"
"github.com/gospider007/gtls"
"github.com/gospider007/ja3"
"github.com/gospider007/websocket"
)
@@ -43,8 +44,8 @@ type RequestOption struct {
DialTimeout time.Duration //dial tcp timeout,default:15
KeepAlive time.Duration //keepalive,default:30
LocalAddr *net.TCPAddr
Dns *net.UDPAddr //dns
AddrType AddrType //dns parse addr type //tls timeout,default:15
Dns *net.UDPAddr //dns
AddrType gtls.AddrType //dns parse addr type //tls timeout,default:15
Stream bool //disable auto read
Referer string //set headers referer value

4
requests.go
View File

@@ -46,8 +46,8 @@ type reqCtxData struct {
dialTimeout time.Duration
keepAlive time.Duration
localAddr *net.TCPAddr //network card ip
addrType AddrType //first ip type
localAddr *net.TCPAddr //network card ip
addrType gtls.AddrType //first ip type
dns *net.UDPAddr
}

6
roundTripper.go
View File

@@ -77,9 +77,9 @@ type RoundTripper struct {
type RoundTripperOption struct {
DialTimeout time.Duration
KeepAlive time.Duration
LocalAddr *net.TCPAddr //network card ip
AddrType AddrType //first ip type
GetAddrType func(string) AddrType
LocalAddr *net.TCPAddr //network card ip
AddrType gtls.AddrType //first ip type
GetAddrType func(string) gtls.AddrType
Dns *net.UDPAddr
GetProxy func(ctx context.Context, url *url.URL) (string, error)
}

70
tools.go
View File

@@ -14,47 +14,21 @@ import (
"golang.org/x/net/http/httpguts"
)
func cloneUrl(u *url.URL) *url.URL {
r := *u
return &r
}
func ParseIp(ip net.IP) AddrType {
if ip != nil {
if ip4 := ip.To4(); ip4 != nil {
return 4
} else if ip6 := ip.To16(); ip6 != nil {
return 6
func getHost(req *http.Request) string {
host := req.Host
if host == "" {
host = req.URL.Host
}
_, port, _ := net.SplitHostPort(host)
if port == "" {
if req.URL.Scheme == "https" {
port = "443"
} else {
port = "80"
}
return fmt.Sprintf("%s:%s", host, port)
}
return 0
}
func GetHost(addrTypes ...AddrType) net.IP {
hosts := GetHosts(addrTypes...)
if len(hosts) == 0 {
return nil
} else {
return hosts[0]
}
}
func GetHosts(addrTypes ...AddrType) []net.IP {
var addrType AddrType
if len(addrTypes) > 0 {
addrType = addrTypes[0]
}
result := []net.IP{}
lls, err := net.InterfaceAddrs()
if err != nil {
return result
}
for _, ll := range lls {
mm, ok := ll.(*net.IPNet)
if ok && mm.IP.IsPrivate() {
if addrType == 0 || ParseIp(mm.IP) == addrType {
result = append(result, mm.IP)
}
}
}
return result
return host
}
func getAddr(uurl *url.URL) string {
if uurl == nil {
@@ -71,21 +45,9 @@ func getAddr(uurl *url.URL) string {
}
return uurl.Host
}
func getHost(req *http.Request) string {
host := req.Host
if host == "" {
host = req.URL.Host
}
_, port, _ := net.SplitHostPort(host)
if port == "" {
if req.URL.Scheme == "https" {
port = "443"
} else {
port = "80"
}
return fmt.Sprintf("%s:%s", host, port)
}
return host
func cloneUrl(u *url.URL) *url.URL {
r := *u
return &r
}
var replaceMap = map[string]string{