package/requests
1
0
Fork 0
mirror of https://github.com/gospider007/requests.git synced 2025-12-24 13:57:52 +08:00
Code Issues Packages Projects Releases Wiki Activity

sync

Browse Source
This commit is contained in:
gospider
2025-06-21 16:23:41 +08:00
parent 79aaa7b6e3
commit 48eac55e95
9 changed files with 50 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
compressConn.go
View File

@@ -116,7 +116,6 @@ func NewCompressionConn(conn net.Conn, arch Compression) (net.Conn, error) {
r: r,
w: w,
oneFunc: sync.OnceFunc(func() {
defer recover()
if snW, ok := w.(*snappy.Writer); ok {
putSnappyWriter(snW)
}

4
go.mod
View File

@@ -8,7 +8,7 @@ require (
github.com/gospider007/bs4 v0.0.0-20250413121342-fed910fb00c9
github.com/gospider007/gson v0.0.0-20250611163241-fa021e9c5531
github.com/gospider007/gtls v0.0.0-20250610060422-446e017b9858
github.com/gospider007/http2 v0.0.0-20250611163252-ee5823d0f04a
github.com/gospider007/http2 v0.0.0-20250616085600-a49f075f841e
github.com/gospider007/http3 v0.0.0-20250611163115-77b77de7a3f9
github.com/gospider007/ja3 v0.0.0-20250615121404-6d7f30dfa700
github.com/gospider007/re v0.0.0-20250217075352-bcb79f285d6c
@@ -80,7 +80,7 @@ require (
go.uber.org/zap v1.27.0 // indirect
go.uber.org/zap/exp v0.3.0 // indirect
go4.org v0.0.0-20230225012048-214862532bf5 // indirect
golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476 // indirect
golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b // indirect
golang.org/x/image v0.28.0 // indirect
golang.org/x/mod v0.25.0 // indirect
golang.org/x/sync v0.15.0 // indirect

8
go.sum
View File

@@ -114,8 +114,8 @@ github.com/gospider007/gson v0.0.0-20250611163241-fa021e9c5531 h1:dzUU4p3u9eaY1v
github.com/gospider007/gson v0.0.0-20250611163241-fa021e9c5531/go.mod h1:YO6h5iR/nAFAaZ34JJbkJPuMG2nq2mShKWI7llEWRQA=
github.com/gospider007/gtls v0.0.0-20250610060422-446e017b9858 h1:Tb+T5YfysjR7Mp22AgrJhMPF28KBgjWkbKj2zi8AyBU=
github.com/gospider007/gtls v0.0.0-20250610060422-446e017b9858/go.mod h1:SxhMpxq4EYHlUre2EdQ4dKKGZyvj7+1PYZ2LQ9RO0y4=
github.com/gospider007/http2 v0.0.0-20250611163252-ee5823d0f04a h1:OtgLN7N2rFHxF1HXutXeY33bzNAZ/npzQQ4JlaDZk+8=
github.com/gospider007/http2 v0.0.0-20250611163252-ee5823d0f04a/go.mod h1:r0XfXyaHynGBLi0o7SN6KxPL0V7q5sX2JVvk2ISDyH0=
github.com/gospider007/http2 v0.0.0-20250616085600-a49f075f841e h1:fO2FFvKS/BAX7saBk6YKuJD28wD/UZ078fkfBetcB2s=
github.com/gospider007/http2 v0.0.0-20250616085600-a49f075f841e/go.mod h1:Uz/rGu9oVkQmroHDeDSS1yhWqaLGHj5UbJdvAXu645E=
github.com/gospider007/http3 v0.0.0-20250611163115-77b77de7a3f9 h1:6OiYSTsyrabSDV0dj5SpAGenpPm8VA20fD/FVzrx/TM=
github.com/gospider007/http3 v0.0.0-20250611163115-77b77de7a3f9/go.mod h1:FObAlfJRtytX4nI47Ra+j2mKoJlUmxfraqpeOBX2Bi0=
github.com/gospider007/ja3 v0.0.0-20250615121404-6d7f30dfa700 h1:UsY1LwHsnvZXdXjoExpwKK/bAiqvfowpwUoLUCG25WU=
@@ -283,8 +283,8 @@ golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE
golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476 h1:bsqhLWFR6G6xiQcb+JoGqdKdRU6WzPWmK8E0jxTjzo4=
golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8=
golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b h1:M2rDM6z3Fhozi9O7NWsxAkg/yqS/lQJ6PmkyIV3YP+o=
golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/image v0.28.0 h1:gdem5JW1OLS4FbkWgLO+7ZeFzYtL3xClb97GaUzYMFE=

5
http.go
View File

@@ -68,13 +68,14 @@ func (obj *clientConn) send(req *http.Request, orderHeaders []interface {
return
}
rawBody := res.Body
isStream := res.StatusCode == 101 || strings.Contains(res.Header.Get("Content-Type"), "text/event-stream")
pr, pw := io.Pipe()
go func() {
var readErr error
defer func() {
obj.readWriteCtx.readCnl(readErr)
}()
if res.Body != nil {
if rawBody != nil {
_, readErr = tools.Copy(pw, rawBody)
}
if readErr != nil && readErr != io.EOF && readErr != io.ErrUnexpectedEOF {
@@ -88,7 +89,7 @@ func (obj *clientConn) send(req *http.Request, orderHeaders []interface {
} else {
select {
case <-obj.readWriteCtx.writeCtx.Done():
if res.StatusCode == 101 || strings.Contains(res.Header.Get("Content-Type"), "text/event-stream") {
if isStream {
<-obj.ctx.Done()
return
}

1
option.go
View File

@@ -42,6 +42,7 @@ type Log struct {
// Connection Management Options
type ClientOption struct {
Spec string //goSpiderSpec origin : https://github.com/gospider007/fp
OrderHeaders []string
DialOption DialOption
Headers any //default headers
Jar Jar //custom cookies

3
requests.go
View File

@@ -201,6 +201,9 @@ func (obj *Client) Request(ctx context.Context, method string, href string, opti
}
func (obj *Client) request(ctx *Response) (err error) {
defer func() {
if ctx.filePath != "" {
return
}
//read body
if err == nil && ctx.sse == nil && !ctx.option.Stream {
err = ctx.ReadBody()

5
response.go
View File

@@ -161,6 +161,9 @@ func (obj *Response) SSE() *SSE {
// return URL redirected address
func (obj *Response) Location() (*url.URL, error) {
if obj.filePath != "" {
return nil, nil
}
u, err := obj.response.Location()
if err == http.ErrNoLocation {
err = nil
@@ -246,7 +249,7 @@ func (obj *Response) SetContent(val []byte) {
// return content with []byte
func (obj *Response) Content() []byte {
if obj.webSocket == nil && obj.sse == nil {
if obj.webSocket == nil && obj.sse == nil && obj.filePath == "" {
obj.ReadBody()
}
return obj.content

43
sepc.go
View File

@@ -112,31 +112,46 @@ func (obj *RequestOption) initSpec() error {
return err
}
obj.gospiderSpec = gospiderSpec
if gospiderSpec.H1Spec != nil {
if obj.orderHeaders == nil {
orderData := NewOrderData()
if obj.orderHeaders == nil {
if len(obj.OrderHeaders) > 0 {
obj.orderHeaders = NewOrderData()
var ods [][2]string
if gospiderSpec.H1Spec != nil {
ods = gospiderSpec.H1Spec.OrderHeaders
} else if gospiderSpec.H2Spec != nil {
ods = gospiderSpec.H2Spec.OrderHeaders
}
for _, key := range obj.OrderHeaders {
key = textproto.CanonicalMIMEHeaderKey(key)
var val any
for _, kv := range ods {
if key == kv[0] && slices.Contains(tools.DefaultHeaderKeys, kv[0]) {
val = kv[1]
break
}
}
obj.orderHeaders.Add(key, val)
}
} else if gospiderSpec.H1Spec != nil {
obj.orderHeaders = NewOrderData()
for _, kv := range gospiderSpec.H1Spec.OrderHeaders {
if slices.Contains(tools.DefaultHeaderKeys, kv[0]) {
orderData.Add(kv[0], kv[1])
obj.orderHeaders.Add(kv[0], kv[1])
} else {
orderData.Add(kv[0], nil)
obj.orderHeaders.Add(kv[0], nil)
}
}
obj.orderHeaders = orderData
}
}
if gospiderSpec.H2Spec != nil {
if obj.orderHeaders == nil {
orderData := NewOrderData()
} else if gospiderSpec.H2Spec != nil {
obj.orderHeaders = NewOrderData()
for _, kv := range gospiderSpec.H2Spec.OrderHeaders {
key := textproto.CanonicalMIMEHeaderKey(kv[0])
if slices.Contains(tools.DefaultHeaderKeys, key) {
orderData.Add(key, kv[1])
obj.orderHeaders.Add(key, kv[1])
} else {
orderData.Add(key, nil)
obj.orderHeaders.Add(key, nil)
}
}
obj.orderHeaders = orderData
}
}
return nil

5
snappy.go
View File

@@ -32,6 +32,8 @@ func getSnappyWriter(w io.Writer) *snappy.Writer {
// 释放 snappy.Writer
func putSnappyWriter(sw *snappy.Writer) {
defer recover()
// sw.Reset(nil)
snappyWriterPool.Put(sw)
}
@@ -44,6 +46,7 @@ func getSnappyReader(r io.Reader) *snappy.Reader {
// 释放 snappy.Reader
func putSnappyReader(sr *snappy.Reader) {
// snappy.Reader 没有 Close 方法，直接放回池
defer recover()
// sr.Reset(nil)
snappyReaderPool.Put(sr)
}