This commit is contained in:
gospider
2025-07-11 11:54:33 +08:00
parent afb6720318
commit 45c7a3262f
25 changed files with 660 additions and 497 deletions

28
body.go
View File

@@ -10,6 +10,7 @@ import (
"net/http"
"net/textproto"
"net/url"
"sort"
"github.com/gospider007/gson"
"github.com/gospider007/tools"
@@ -44,6 +45,33 @@ func (obj *OrderData) Keys() []string {
}
return keys
}
func (obj *OrderData) ReorderWithKeys(key ...string) {
if len(key) == 0 {
return
}
for i, k := range key {
key[i] = textproto.CanonicalMIMEHeaderKey(k)
}
sort.SliceStable(obj.data, func(x, y int) bool {
xIndex := -1
yIndex := -1
for i, k := range key {
if k == obj.data[x].key {
xIndex = i
}
if k == obj.data[y].key {
yIndex = i
}
}
if xIndex == -1 {
return false
}
if yIndex == -1 {
return true
}
return xIndex < yIndex
})
}
type orderT struct {
key string

View File

@@ -6,10 +6,8 @@ import (
"io"
"net"
"strings"
"sync"
"time"
"github.com/golang/snappy"
"github.com/klauspost/compress/zstd"
)
@@ -43,28 +41,85 @@ func (obj compression) OpenWriter(w io.Writer) (io.WriteCloser, error) {
return obj.openWriter(w)
}
type CompressionLevel int
const (
CompressionLevelFast CompressionLevel = 1
CompressionLevelBest CompressionLevel = 2
)
func GetCompressionByte(decode string) (byte, error) {
switch strings.ToLower(decode) {
case "zstd":
return 40, nil
case "s2":
return 255, nil
case "flate":
return 92, nil
case "minlz":
return 93, nil
default:
return 0, errors.New("unsupported compression type")
}
}
func NewCompressionWithByte(b byte) (Compression, error) {
switch b {
case 40:
return NewCompression("zstd")
case 255:
return NewCompression("s2")
case 92:
return NewCompression("flate")
case 93:
return NewCompression("minlz")
}
return nil, errors.New("unsupported compression type")
}
func NewCompression(decode string) (Compression, error) {
b, err := GetCompressionByte(decode)
if err != nil {
return nil, err
}
br := func(r io.Reader) error {
buf := make([]byte, 1)
n, err := r.Read(buf)
if err != nil {
return err
}
if n != 1 || buf[0] != b {
return errors.New("invalid response")
}
return nil
}
bw := func(w io.Writer) error {
n, err := w.Write([]byte{b})
if err != nil {
return err
}
if n != 1 {
return errors.New("invalid response")
}
return nil
}
var arch Compression
switch strings.ToLower(decode) {
case "s2":
arch = compression{
openReader: func(r io.Reader) (io.Reader, error) {
err := br(r)
if err != nil {
return nil, err
}
return getSnappyReader(r), nil
},
openWriter: func(w io.Writer) (io.WriteCloser, error) {
err := bw(w)
if err != nil {
return nil, err
}
return getSnappyWriter(w), nil
},
}
case "zstd":
arch = compression{
openReader: func(r io.Reader) (io.Reader, error) {
err := br(r)
if err != nil {
return nil, err
}
decoder, err := zstd.NewReader(r)
if err != nil {
return nil, err
@@ -72,6 +127,10 @@ func NewCompression(decode string) (Compression, error) {
return decoder.IOReadCloser(), nil
},
openWriter: func(w io.Writer) (io.WriteCloser, error) {
err := bw(w)
if err != nil {
return nil, err
}
encoder, err := zstd.NewWriter(nil, zstd.WithEncoderLevel(zstd.SpeedDefault))
if err != nil {
return nil, err
@@ -82,27 +141,37 @@ func NewCompression(decode string) (Compression, error) {
case "flate":
arch = compression{
openReader: func(r io.Reader) (io.Reader, error) {
buf := make([]byte, 1)
n, err := r.Read(buf)
err := br(r)
if err != nil {
return nil, err
}
if n != 1 || buf[0] != 92 {
return nil, errors.New("invalid response")
}
return flate.NewReader(r), nil
},
openWriter: func(w io.Writer) (io.WriteCloser, error) {
n, err := w.Write([]byte{92})
err := bw(w)
if err != nil {
return nil, err
}
if n != 1 {
return nil, errors.New("invalid response")
}
return flate.NewWriter(w, flate.DefaultCompression)
},
}
case "minlz":
arch = compression{
openReader: func(r io.Reader) (io.Reader, error) {
err := br(r)
if err != nil {
return nil, err
}
return getMinlzReader(r), nil
},
openWriter: func(w io.Writer) (io.WriteCloser, error) {
err := bw(w)
if err != nil {
return nil, err
}
return getMinlzWriter(w), nil
},
}
default:
return nil, errors.New("unsupported compression type")
}
@@ -116,11 +185,14 @@ func NewWriterCompression(conn io.Writer, arch Compression) (*WriterCompression,
}
ccon := &WriterCompression{
w: w,
oneFunc: sync.OnceFunc(func() {
if snW, ok := w.(*snappy.Writer); ok {
putSnappyWriter(snW)
}
}),
// oneFunc: sync.OnceFunc(func() {
// switch snW := w.(type) {
// case *snappy.Writer:
// putSnappyWriter(snW)
// case *minlz.Writer:
// putMinlzWriter(snW)
// }
// }),
}
if f, ok := w.(interface{ Flush() error }); ok {
ccon.f = f
@@ -134,11 +206,14 @@ func NewReaderCompression(conn io.Reader, arch Compression) (*ReaderCompression,
}
ccon := &ReaderCompression{
r: r,
oneFunc: sync.OnceFunc(func() {
if snR, ok := r.(*snappy.Reader); ok {
putSnappyReader(snR)
}
}),
// oneFunc: sync.OnceFunc(func() {
// switch snR := r.(type) {
// case *snappy.Reader:
// putSnappyReader(snR)
// case *minlz.Reader:
// putMinlzReader(snR)
// }
// }),
}
return ccon, nil
}

21
conn.go
View File

@@ -81,28 +81,35 @@ func (obj *connPool) taskMain(conn *connecotr, task *reqTask) (err error) {
}
}
if err == nil {
task.cnl(errNoErr)
task.cnl(tools.ErrNoErr)
} else {
task.cnl(err)
}
if err == nil && task.reqCtx.response != nil && task.reqCtx.response.Body != nil {
if err == nil && task.reqCtx.response != nil && task.reqCtx.response.Body != nil && task.bodyCtx != nil {
select {
case <-conn.forceCtx.Done():
err = context.Cause(conn.forceCtx)
case <-task.reqCtx.Context().Done():
if context.Cause(task.reqCtx.Context()) != errNoErr {
if context.Cause(task.reqCtx.Context()) != tools.ErrNoErr {
err = context.Cause(task.reqCtx.Context())
}
if err == nil && task.reqCtx.response.StatusCode == 101 {
select {
case <-conn.forceCtx.Done():
err = context.Cause(conn.forceCtx)
case <-task.bodyCtx.Done():
if context.Cause(task.bodyCtx) != tools.ErrNoErr {
err = context.Cause(task.bodyCtx)
}
}
}
case <-task.bodyCtx.Done():
if context.Cause(task.bodyCtx) != errNoErr {
if context.Cause(task.bodyCtx) != tools.ErrNoErr {
err = context.Cause(task.bodyCtx)
}
}
}
if err != nil {
if errors.Is(err, errLastTaskRuning) {
task.isNotice = true
}
conn.CloseWithError(tools.WrapError(err, "taskMain close with error"))
}
}()

View File

@@ -1,26 +1,20 @@
package requests
import (
"compress/flate"
"io"
"sync"
)
// var flateWriterPool = sync.Pool{
// New: func() interface{} {
// w, _ := flate.NewWriter(io.Discard, flate.DefaultCompression)
// return w
// },
// }
var flateWriterPool = sync.Pool{
New: func() interface{} {
w, _ := flate.NewWriter(io.Discard, flate.DefaultCompression)
return w
},
}
// func getFlateWriter(dst io.Writer) *flate.Writer {
// w := flateWriterPool.Get().(*flate.Writer)
// w.Reset(dst)
// return w
// }
func getFlateWriter(dst io.Writer) *flate.Writer {
w := flateWriterPool.Get().(*flate.Writer)
w.Reset(dst)
return w
}
func putFlateWriter(w *flate.Writer) {
// w.Close() // flush buffer
w.Reset(io.Discard)
flateWriterPool.Put(w)
}
// func putFlateWriter(w *flate.Writer) {
// // w.Close() // flush buffer
// w.Reset(io.Discard)
// flateWriterPool.Put(w)
// }

23
go.mod
View File

@@ -4,23 +4,25 @@ go 1.24.0
require (
github.com/golang/snappy v1.0.0
github.com/gorilla/websocket v1.5.3
github.com/gospider007/bar v0.0.0-20250217074946-47896d8de2ba
github.com/gospider007/bs4 v0.0.0-20250413121342-fed910fb00c9
github.com/gospider007/gson v0.0.0-20250630120534-cce6e3c6756d
github.com/gospider007/gtls v0.0.0-20250630120509-4e99c91661ee
github.com/gospider007/http2 v0.0.0-20250630120519-3f59fca61c88
github.com/gospider007/http2 v0.0.0-20250711035043-daabc8e205b3
github.com/gospider007/http3 v0.0.0-20250630120526-1066890881e5
github.com/gospider007/ja3 v0.0.0-20250627013834-1d2966014638
github.com/gospider007/re v0.0.0-20250217075352-bcb79f285d6c
github.com/gospider007/tools v0.0.0-20250630120304-b22c2ddf35b5
github.com/gospider007/websocket v0.0.0-20250630120328-1ec26253d082
github.com/klauspost/compress v1.18.0
github.com/minio/minlz v1.0.1
github.com/quic-go/quic-go v0.53.0
github.com/refraction-networking/uquic v0.0.6
github.com/refraction-networking/utls v1.7.4-0.20250621163342-5abccec539e6
github.com/txthinking/socks5 v0.0.0-20230325130024-4230056ae301
golang.org/x/crypto v0.39.0
golang.org/x/net v0.41.0
golang.org/x/crypto v0.40.0
golang.org/x/net v0.42.0
gopkg.in/errgo.v2 v2.1.0
)
@@ -43,7 +45,7 @@ require (
github.com/gobwas/pool v0.2.1 // indirect
github.com/gobwas/ws v1.4.0 // indirect
github.com/google/gopacket v1.1.19 // indirect
github.com/google/pprof v0.0.0-20250629210550-e611ec304b22 // indirect
github.com/google/pprof v0.0.0-20250630185457-6e76a2b096b5 // indirect
github.com/gospider007/blog v0.0.0-20250302134054-8afc12c2a9a7 // indirect
github.com/gospider007/kinds v0.0.0-20250217075226-10f199f7215d // indirect
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
@@ -54,9 +56,8 @@ require (
github.com/libdns/libdns v1.1.0 // indirect
github.com/mholt/acmez/v3 v3.1.2 // indirect
github.com/mholt/archives v0.1.3 // indirect
github.com/miekg/dns v1.1.66 // indirect
github.com/miekg/dns v1.1.67 // indirect
github.com/mikelolasagasti/xz v1.0.1 // indirect
github.com/minio/minlz v1.0.1 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/nwaples/rardecode/v2 v2.1.1 // indirect
@@ -81,10 +82,10 @@ require (
go.uber.org/zap/exp v0.3.0 // indirect
go4.org v0.0.0-20230225012048-214862532bf5 // indirect
golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b // indirect
golang.org/x/image v0.28.0 // indirect
golang.org/x/mod v0.25.0 // indirect
golang.org/x/sync v0.15.0 // indirect
golang.org/x/sys v0.33.0 // indirect
golang.org/x/text v0.26.0 // indirect
golang.org/x/image v0.29.0 // indirect
golang.org/x/mod v0.26.0 // indirect
golang.org/x/sync v0.16.0 // indirect
golang.org/x/sys v0.34.0 // indirect
golang.org/x/text v0.27.0 // indirect
golang.org/x/tools v0.34.0 // indirect
)

46
go.sum
View File

@@ -99,11 +99,13 @@ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXi
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
github.com/google/pprof v0.0.0-20250629210550-e611ec304b22 h1:RanZAubGQRlhKdX83NviyIduq4DsO2zFmSgPuTlnkMc=
github.com/google/pprof v0.0.0-20250629210550-e611ec304b22/go.mod h1:5hDyRhoBCxViHszMt12TnOpEI4VVi+U8Gm9iphldiMA=
github.com/google/pprof v0.0.0-20250630185457-6e76a2b096b5 h1:xhMrHhTJ6zxu3gA4enFM9MLn9AY7613teCdFnlUVbSQ=
github.com/google/pprof v0.0.0-20250630185457-6e76a2b096b5/go.mod h1:5hDyRhoBCxViHszMt12TnOpEI4VVi+U8Gm9iphldiMA=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/gospider007/bar v0.0.0-20250217074946-47896d8de2ba h1:8DK0d1nUTsMbOgFrIWMSBKp7obOAKgSLkswzVBX1RRI=
github.com/gospider007/bar v0.0.0-20250217074946-47896d8de2ba/go.mod h1:HGEEIVnysptCXwsdU4E82uQu0F4ObU/5+KWHIdJCUbY=
github.com/gospider007/blog v0.0.0-20250302134054-8afc12c2a9a7 h1:QP46FdP6nJET+bSdm8wxjIzBMnu2lKraoypfU9bLGV4=
@@ -114,8 +116,8 @@ github.com/gospider007/gson v0.0.0-20250630120534-cce6e3c6756d h1:/FaI2cPp/DgiEJ
github.com/gospider007/gson v0.0.0-20250630120534-cce6e3c6756d/go.mod h1:Vb5Ehy5UGQYVUW2XvzqLW1rc7/WPljQspyF3YZ/u164=
github.com/gospider007/gtls v0.0.0-20250630120509-4e99c91661ee h1:rcQAnk5Szv/Vi9dVieU1uwKUSVPNB+lqRPggI6oZgFM=
github.com/gospider007/gtls v0.0.0-20250630120509-4e99c91661ee/go.mod h1:ee+Fo1W5Abt9mdLOOiTrb76r0qbO6ubGmRDklCMAPIQ=
github.com/gospider007/http2 v0.0.0-20250630120519-3f59fca61c88 h1:gQuOZI+lANJ1vQ3gj70igHTt25GrXyhrKDC7vV/FQdg=
github.com/gospider007/http2 v0.0.0-20250630120519-3f59fca61c88/go.mod h1:CmJdPojakh+1r+WdMowZEADGIQiaM2mYCQMH7VwqsPE=
github.com/gospider007/http2 v0.0.0-20250711035043-daabc8e205b3 h1:HqEs4ZAU1U7hGKwEQwE/0YYwldByzaD/YfXpSrhgdVA=
github.com/gospider007/http2 v0.0.0-20250711035043-daabc8e205b3/go.mod h1:rji8qgw9KJ3ddOqT7xvgd0GKV7Z4+l1uSbFrgDX3blg=
github.com/gospider007/http3 v0.0.0-20250630120526-1066890881e5 h1:+v1s/f9AYApykZ/alJIOX4V7xj8YTiydfV8izakjaY4=
github.com/gospider007/http3 v0.0.0-20250630120526-1066890881e5/go.mod h1:dyg0PmV4MW4mxUG6VzJxhAOTBboMbrpC3/XVVMDvV3Q=
github.com/gospider007/ja3 v0.0.0-20250627013834-1d2966014638 h1:qQmKi3FsCK0WJcDOq4wpOzwVZ2UWto1Da05SGRVBKPQ=
@@ -160,8 +162,8 @@ github.com/mholt/acmez/v3 v3.1.2/go.mod h1:L1wOU06KKvq7tswuMDwKdcHeKpFFgkppZy/y0
github.com/mholt/archives v0.1.3 h1:aEAaOtNra78G+TvV5ohmXrJOAzf++dIlYeDW3N9q458=
github.com/mholt/archives v0.1.3/go.mod h1:LUCGp++/IbV/I0Xq4SzcIR6uwgeh2yjnQWamjRQfLTU=
github.com/miekg/dns v1.1.51/go.mod h1:2Z9d3CP1LQWihRZUf29mQ19yDThaI4DAYzte2CaQW5c=
github.com/miekg/dns v1.1.66 h1:FeZXOS3VCVsKnEAd+wBkjMC3D2K+ww66Cq3VnCINuJE=
github.com/miekg/dns v1.1.66/go.mod h1:jGFzBsSNbJw6z1HYut1RKBKHA9PBdxeHrZG8J+gC2WE=
github.com/miekg/dns v1.1.67 h1:kg0EHj0G4bfT5/oOys6HhZw4vmMlnoZ+gDu8tJ/AlI0=
github.com/miekg/dns v1.1.67/go.mod h1:fujopn7TB3Pu3JM69XaawiU0wqjpL9/8xGop5UrTPps=
github.com/mikelolasagasti/xz v1.0.1 h1:Q2F2jX0RYJUG3+WsM+FJknv+6eVjsjXNDV0KJXZzkD0=
github.com/mikelolasagasti/xz v1.0.1/go.mod h1:muAirjiOUxPRXwm9HdDtB3uoRPrGnL85XHtokL9Hcgc=
github.com/minio/minlz v1.0.1 h1:OUZUzXcib8diiX+JYxyRLIdomyZYzHct6EShOKtQY2A=
@@ -273,8 +275,8 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM=
golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U=
golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM=
golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -287,8 +289,8 @@ golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b h1:M2rDM6z3Fhozi9O7NWsxAkg/y
golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/image v0.28.0 h1:gdem5JW1OLS4FbkWgLO+7ZeFzYtL3xClb97GaUzYMFE=
golang.org/x/image v0.28.0/go.mod h1:GUJYXtnGKEUgggyzh+Vxt+AviiCcyiwpsl8iQ8MvwGY=
golang.org/x/image v0.29.0 h1:HcdsyR4Gsuys/Axh0rDEmlBmB68rW1U9BUdB3UVHsas=
golang.org/x/image v0.29.0/go.mod h1:RVJROnf3SLK8d26OW91j4FrIHGbsJ8QnbEocVTOWQDA=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -311,8 +313,8 @@ golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.25.0 h1:n7a+ZbQKQA/Ysbyb0/6IbB1H/X41mKgbhfv7AfG/44w=
golang.org/x/mod v0.25.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww=
golang.org/x/mod v0.26.0 h1:EGMPT//Ezu+ylkCijjPc+f4Aih7sZvaAr+O3EHBxvZg=
golang.org/x/mod v0.26.0/go.mod h1:/j6NAhSk8iQ723BGAUyoAcn7SlD7s15Dp9Nd/SfeaFQ=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -336,8 +338,8 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw=
golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA=
golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs=
golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -355,8 +357,8 @@ golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8=
golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw=
golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -381,8 +383,8 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA=
golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -393,8 +395,8 @@ golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg=
golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ=
golang.org/x/term v0.33.0 h1:NuFncQrRcaRvVmgRkvM3j/F00gWIAlcmlB8ACEKmGIg=
golang.org/x/term v0.33.0/go.mod h1:s18+ql9tYWp1IfpV9DmCtQDDSRBUjKaw9M1eAv5UeF0=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -408,8 +410,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M=
golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA=
golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4=
golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE=

300
http.go
View File

@@ -13,198 +13,93 @@ import (
"golang.org/x/net/http/httpguts"
)
type reqReadWriteCtx struct {
writeCtx context.Context
writeCnl context.CancelFunc
readCtx context.Context
readCnl context.CancelCauseFunc
type rsp struct {
r *http.Response
ctx context.Context
err error
}
type clientConn struct {
err error
readWriteCtx *reqReadWriteCtx
conn net.Conn
r *bufio.Reader
w *bufio.Writer
closeFunc func(error)
ctx context.Context
cnl context.CancelCauseFunc
conn net.Conn
r *bufio.Reader
w *bufio.Writer
closeFunc func(error)
ctx context.Context
cnl context.CancelCauseFunc
rsps chan *rsp
}
func NewClientConn(con net.Conn, closeFunc func(error)) *clientConn {
ctx, cnl := context.WithCancelCause(context.TODO())
reader, writer := io.Pipe()
c := &clientConn{
ctx: ctx,
cnl: cnl,
conn: con,
closeFunc: closeFunc,
r: bufio.NewReader(reader),
rsps: make(chan *rsp),
r: bufio.NewReader(con),
w: bufio.NewWriter(con),
}
go func() {
_, err := tools.Copy(writer, con)
writer.CloseWithError(err)
c.CloseWithError(err)
}()
go c.read()
return c
}
var errLastTaskRuning = errors.New("last task is running")
var errNoErr = errors.New("no error")
func (obj *clientConn) send(req *http.Request, orderHeaders []interface {
Key() string
Val() any
}) (res *http.Response, err error) {
go obj.httpWrite(req, req.Header.Clone(), orderHeaders)
res, err = http.ReadResponse(obj.r, req)
if err == nil && res == nil {
err = errors.New("response is nil")
}
if err != nil {
obj.readWriteCtx.readCnl(nil)
return
}
rawBody := res.Body
isStream := res.StatusCode == 101
pr, pw := io.Pipe()
go func() {
var readErr error
defer func() {
if readErr == nil {
obj.readWriteCtx.readCnl(errNoErr)
} else {
obj.readWriteCtx.readCnl(readErr)
}
}()
if rawBody != nil {
_, readErr = tools.Copy(pw, rawBody)
func (obj *clientConn) read() {
var err error
var res *http.Response
defer obj.CloseWithError(err)
for {
res, err = http.ReadResponse(obj.r, nil)
if res == nil && err == nil {
err = errors.New("response is nil")
}
if readErr != nil && readErr != io.EOF && readErr != io.ErrUnexpectedEOF {
err = tools.WrapError(readErr, "failed to read response body")
} else {
readErr = nil
}
pw.CloseWithError(readErr)
if readErr != nil {
obj.CloseWithError(readErr)
} else {
if err != nil {
select {
case <-obj.readWriteCtx.writeCtx.Done():
if isStream {
<-obj.ctx.Done()
return
}
default:
obj.CloseWithError(tools.WrapError(errLastTaskRuning, "last task not write done with read done"))
case obj.rsps <- &rsp{res, nil, err}:
case <-obj.ctx.Done():
return
}
return
}
}()
res.Body = pr
return
}
func (obj *clientConn) Close() error {
return obj.CloseWithError(nil)
}
func (obj *clientConn) CloseWithError(err error) error {
if obj.closeFunc != nil {
obj.closeFunc(obj.err)
}
obj.cnl(err)
if err == nil {
obj.err = tools.WrapError(obj.err, "connecotr closeWithError close")
} else {
obj.err = tools.WrapError(err, "connecotr closeWithError close")
}
return obj.conn.Close()
}
func (obj *clientConn) initTask() {
readCtx, readCnl := context.WithCancelCause(obj.ctx)
writeCtx, writeCnl := context.WithCancel(obj.ctx)
obj.readWriteCtx = &reqReadWriteCtx{
readCtx: readCtx,
readCnl: readCnl,
writeCtx: writeCtx,
writeCnl: writeCnl,
}
}
func (obj *clientConn) DoRequest(req *http.Request, orderHeaders []interface {
Key() string
Val() any
}) (*http.Response, context.Context, error) {
if obj.readWriteCtx != nil {
select {
case <-obj.readWriteCtx.writeCtx.Done():
case <-obj.ctx.Done():
return nil, nil, obj.ctx.Err()
default:
return nil, obj.readWriteCtx.readCtx, errLastTaskRuning
if res.StatusCode == 101 {
select {
case obj.rsps <- &rsp{res, obj.ctx, err}:
case <-obj.ctx.Done():
return
}
<-obj.ctx.Done()
return
} else if res == nil || res.Body == nil || res.Body == http.NoBody {
select {
case obj.rsps <- &rsp{res, nil, err}:
case <-obj.ctx.Done():
return
}
} else {
ctx, cnl := context.WithCancelCause(obj.ctx)
res.Body = &clientBody{res.Body, cnl}
select {
case obj.rsps <- &rsp{res, ctx, err}:
case <-obj.ctx.Done():
return
}
<-ctx.Done()
}
select {
case <-obj.readWriteCtx.readCtx.Done():
case <-obj.ctx.Done():
return nil, nil, obj.ctx.Err()
default:
return nil, obj.readWriteCtx.readCtx, errLastTaskRuning
}
} else {
select {
case <-obj.ctx.Done():
return nil, nil, obj.ctx.Err()
return
default:
}
}
obj.initTask()
res, err := obj.send(req, orderHeaders)
if err != nil {
obj.CloseWithError(err)
return nil, nil, err
}
return res, obj.readWriteCtx.readCtx, err
}
type websocketConn struct {
r io.Reader
w io.WriteCloser
cnl context.CancelCauseFunc
}
func (obj *websocketConn) Read(p []byte) (n int, err error) {
return obj.r.Read(p)
}
func (obj *websocketConn) Write(p []byte) (n int, err error) {
// i, err := obj.w.Write(p)
// log.Print(err, " write error ", i, p)
// return i, err
return obj.w.Write(p)
}
func (obj *websocketConn) Close() error {
obj.cnl(nil)
return obj.w.Close()
}
func (obj *clientConn) Stream() io.ReadWriteCloser {
return &websocketConn{
cnl: obj.cnl,
r: obj.r,
w: obj.conn,
}
}
func (obj *clientConn) httpWrite(req *http.Request, rawHeaders http.Header, orderHeaders []interface {
Key() string
Val() any
}) {
var err error
}) (err error) {
defer func() {
if err != nil {
obj.CloseWithError(tools.WrapError(err, "failed to send request body"))
}
obj.readWriteCtx.writeCnl()
}()
host := req.Host
if host == "" {
@@ -261,6 +156,79 @@ func (obj *clientConn) httpWrite(req *http.Request, rawHeaders http.Header, orde
}
}
err = obj.w.Flush()
return
}
type clientBody struct {
r io.Reader
cnl context.CancelCauseFunc
}
func (obj *clientBody) Read(p []byte) (n int, err error) {
return obj.r.Read(p)
}
func (obj *clientBody) Close() error {
return obj.CloseWithError(nil)
}
func (obj *clientBody) CloseWithError(err error) error {
obj.cnl(err)
return nil
}
func (obj *clientConn) DoRequest(req *http.Request, orderHeaders []interface {
Key() string
Val() any
}) (res *http.Response, ctx context.Context, err error) {
defer func() {
if err != nil {
obj.CloseWithError(tools.WrapError(err, "failed to send request"))
}
}()
var writeErr error
writeDone := make(chan struct{})
go func() {
writeErr = obj.httpWrite(req, req.Header.Clone(), orderHeaders)
close(writeDone)
}()
select {
case <-writeDone:
if writeErr != nil {
return nil, nil, writeErr
}
select {
case <-req.Context().Done():
return nil, nil, req.Context().Err()
case <-obj.ctx.Done():
return nil, nil, obj.ctx.Err()
case rsp := <-obj.rsps:
return rsp.r, rsp.ctx, rsp.err
}
case <-req.Context().Done():
return nil, nil, req.Context().Err()
case <-obj.ctx.Done():
return nil, nil, obj.ctx.Err()
case rsp := <-obj.rsps:
return rsp.r, rsp.ctx, rsp.err
}
}
func (obj *clientConn) Close() error {
return obj.CloseWithError(nil)
}
func (obj *clientConn) CloseWithError(err error) error {
if obj.closeFunc != nil {
obj.closeFunc(err)
}
obj.cnl(err)
return obj.conn.Close()
}
func (obj *clientConn) Stream() io.ReadWriteCloser {
return &websocketConn{
cnl: obj.cnl,
r: obj.r,
w: obj.conn,
}
}
func newChunkedWriter(w *bufio.Writer) io.WriteCloser {
@@ -291,3 +259,23 @@ func (cw *chunkedWriter) Close() error {
_, err := io.WriteString(cw.w, "0\r\n\r\n")
return err
}
type websocketConn struct {
r io.Reader
w io.WriteCloser
cnl context.CancelCauseFunc
}
func (obj *websocketConn) Read(p []byte) (n int, err error) {
return obj.r.Read(p)
}
func (obj *websocketConn) Write(p []byte) (n int, err error) {
// i, err := obj.w.Write(p)
// log.Print(err, " write error ", i, p)
// return i, err
return obj.w.Write(p)
}
func (obj *websocketConn) Close() error {
obj.cnl(nil)
return obj.w.Close()
}

38
minlz.go Normal file
View File

@@ -0,0 +1,38 @@
package requests
import (
"io"
"github.com/minio/minlz"
)
// // 定义 minlz.Writer 池,包装 io.Writer
// var minlzWriterPool = sync.Pool{
// New: func() interface{} {
// return minlz.NewWriter(nil, minlz.WriterBlockSize(64*1024), minlz.WriterConcurrency(1), minlz.WriterLevel(minlz.LevelSmallest))
// },
// }
// // 定义 minlz.Reader 池,包装 io.Reader
// var minlzReaderPool = sync.Pool{
// New: func() interface{} {
// // 先给一个空 reader后面可以Reset替换输入来源
// return minlz.NewReader(nil, minlz.ReaderMaxBlockSize(64*1024))
// },
// }
// 获取并初始化 minlz.Writer
func getMinlzWriter(w io.Writer) *minlz.Writer {
return minlz.NewWriter(w, minlz.WriterBlockSize(64*1024), minlz.WriterConcurrency(1))
// sw := minlzWriterPool.Get().(*minlz.Writer)
// sw.Reset(w)
// return sw
}
// 获取并初始化 minlz.Reader
func getMinlzReader(r io.Reader) *minlz.Reader {
// sr := minlzReaderPool.Get().(*minlz.Reader)
// sr.Reset(r)
// return sr
return minlz.NewReader(r, minlz.ReaderMaxBlockSize(64*1024))
}

View File

@@ -135,7 +135,10 @@ func (obj *Client) retryRequest(ctx context.Context, option RequestOption, uhref
default:
}
err = obj.request(response)
if err != nil || response.Option().MaxRedirect < 0 || (response.Option().MaxRedirect > 0 && redirectNum > response.Option().MaxRedirect) {
if err != nil || err == ErrUseLastResponse || response.Option().MaxRedirect < 0 || (response.Option().MaxRedirect > 0 && redirectNum > response.Option().MaxRedirect) {
if err == ErrUseLastResponse {
err = nil
}
return
}
loc, err = response.Location()
@@ -205,15 +208,19 @@ func (obj *Client) request(ctx *Response) (err error) {
return
}
//read body
if err == nil && ctx.sse == nil && !ctx.option.Stream {
err = ctx.ReadBody()
isReadBody := (err == nil || err == ErrUseLastResponse) && ctx.sse == nil && !ctx.option.Stream
if isReadBody {
if err2 := ctx.ReadBody(); err2 != nil {
err = err2
}
}
//result callback
if err == nil && ctx.option.ResultCallBack != nil {
err = ctx.option.ResultCallBack(ctx)
if (err == nil || err == ErrUseLastResponse) && ctx.option.ResultCallBack != nil {
if err2 := ctx.option.ResultCallBack(ctx); err2 != nil {
err = err2
}
}
if err != nil { //err callback, must close body
if err != nil && err != ErrUseLastResponse { //err callback, must close body
ctx.CloseConn()
if ctx.option.ErrCallBack != nil {
ctx.err = err
@@ -347,12 +354,12 @@ func (obj *Client) request(ctx *Response) (err error) {
err = errors.New("send req response is nil")
return
}
if ctx.Body() != nil {
ctx.body = ctx.Body().(*wrapBody)
if ctx.response.Body != nil {
ctx.body = ctx.response.Body.(*wrapBody)
}
if encoding := ctx.ContentEncoding(); encoding != "" {
if encoding := ctx.ContentEncoding(); encoding != "" && ctx.response.Body != nil {
var unCompressionBody io.ReadCloser
unCompressionBody, err = tools.CompressionHeadersDecode(ctx.Context(), ctx.Body(), encoding)
unCompressionBody, err = tools.CompressionHeadersDecode(ctx.Context(), ctx.response.Body, encoding)
if err != nil {
if err != io.ErrUnexpectedEOF && err != io.EOF {
return

View File

@@ -304,60 +304,6 @@ func (obj *barBody) Write(con []byte) (int, error) {
func (obj *Response) defaultDecode() bool {
return strings.Contains(obj.ContentType(), "html")
}
func (obj *Response) Body() io.ReadCloser {
return obj.response.Body
}
// read body
func (obj *Response) ReadBody() (err error) {
obj.readBodyLock.Lock()
defer obj.readBodyLock.Unlock()
if obj.readBody {
return nil
}
defer func() {
obj.Close(err)
if err != nil {
obj.CloseConn()
} else {
if obj.response.StatusCode == 101 && obj.webSocket == nil {
obj.webSocket = websocket.NewConn(newFakeConn(obj.body.connStream()), func() { obj.CloseConn() }, true, obj.Headers().Get("Sec-WebSocket-Extensions"))
}
}
}()
obj.readBody = true
bBody := bytes.NewBuffer(nil)
done := make(chan struct{})
var readErr error
go func() {
defer close(done)
if obj.option.Bar && obj.ContentLength() > 0 {
_, readErr = tools.Copy(&barBody{
bar: bar.NewClient(obj.response.ContentLength),
body: bBody,
}, obj.Body())
} else {
_, readErr = tools.Copy(bBody, obj.Body())
}
if readErr == io.ErrUnexpectedEOF {
readErr = nil
}
}()
select {
case <-obj.ctx.Done():
return tools.WrapError(obj.ctx.Err(), "response read ctx error")
case <-done:
if readErr != nil {
return tools.WrapError(readErr, "response read content error")
}
}
if !obj.option.DisDecode && obj.defaultDecode() {
obj.content, obj.encoding, _ = tools.Charset(bBody.Bytes(), obj.ContentType())
} else {
obj.content = bBody.Bytes()
}
return
}
// conn is new conn
func (obj *Response) IsNewConn() bool {
@@ -382,14 +328,108 @@ func (obj *Response) CloseConn() {
// close
func (obj *Response) Close(err error) {
if err == nil {
err = tools.ErrNoErr
}
if obj.body != nil {
obj.body.Close()
obj.body.CloseWithError(err)
}
if obj.cnl != nil {
if err == nil {
obj.cnl(errNoErr)
} else {
obj.cnl(err)
}
obj.cnl(err)
}
}
// read body
func (obj *Response) ReadBody() (err error) {
obj.readBodyLock.Lock()
defer obj.readBodyLock.Unlock()
if obj.readBody {
return nil
}
obj.readBody = true
defer func() {
if err == nil && obj.response.StatusCode == 101 && obj.webSocket == nil {
obj.webSocket = websocket.NewConn(newFakeConn(obj.body.connStream()), func() { obj.CloseConn() }, true, obj.Headers().Get("Sec-WebSocket-Extensions"))
}
}()
bBody := bytes.NewBuffer(nil)
done := make(chan struct{})
var readErr error
body := obj.Body()
defer body.Close()
go func() {
defer close(done)
if obj.option.Bar && obj.ContentLength() > 0 {
_, readErr = tools.Copy(&barBody{
bar: bar.NewClient(obj.response.ContentLength),
body: bBody,
}, body)
} else {
_, readErr = tools.Copy(bBody, body)
}
if readErr == io.ErrUnexpectedEOF {
readErr = nil
}
}()
select {
case <-obj.ctx.Done():
if readErr == nil && body.closed && body.err == nil {
err = nil
} else {
err = tools.WrapError(obj.ctx.Err(), "response read ctx error")
}
case <-done:
if readErr != nil {
err = tools.WrapError(readErr, "response read content error")
}
}
if err != nil {
return
}
if !obj.option.DisDecode && obj.defaultDecode() {
obj.content, obj.encoding, _ = tools.Charset(bBody.Bytes(), obj.ContentType())
} else {
obj.content = bBody.Bytes()
}
return
}
type body struct {
ctx *Response
closed bool
err error
}
func (obj *body) Read(p []byte) (n int, err error) {
obj.ctx.readBody = true
if obj.ctx == nil || obj.ctx.response == nil || obj.ctx.response.Body == nil {
obj.closed = true
return 0, io.EOF
}
n, err = obj.ctx.response.Body.Read(p)
if err != nil {
obj.closed = true
if err != io.EOF && err != io.ErrUnexpectedEOF {
obj.err = err
obj.ctx.Close(err)
obj.ctx.CloseConn()
} else {
obj.ctx.Close(nil)
}
}
return
}
func (obj *body) Close() (err error) {
obj.closed = true
if !obj.closed {
obj.err = errors.New("response body force closed")
obj.ctx.Close(errors.New("response body force closed"))
obj.ctx.CloseConn()
}
return nil
}
func (obj *Response) Body() *body {
return &body{ctx: obj}
}

View File

@@ -148,7 +148,7 @@ func (obj *roundTripper) ghttp3Dial(ctx *Response, remoteAddress Address, proxyA
}
conn = obj.newConnecotr()
conn.Conn, err = http3.NewClient(netConn, udpConn, func() {
conn.Conn = http3.NewClient(netConn, udpConn, func() {
conn.forceCnl(errors.New("http3 client close"))
})
if ct, ok := udpConn.(interface {
@@ -193,7 +193,7 @@ func (obj *roundTripper) uhttp3Dial(ctx *Response, remoteAddress Address, proxyA
return nil, err
}
conn = obj.newConnecotr()
conn.Conn, err = http3.NewClient(netConn, udpConn, func() {
conn.Conn = http3.NewClient(netConn, udpConn, func() {
conn.forceCnl(errors.New("http3 client close"))
})
if ct, ok := udpConn.(interface {
@@ -353,7 +353,7 @@ func (obj *roundTripper) poolRoundTrip(task *reqTask) error {
case connPool.tasks <- task:
<-task.ctx.Done()
err := context.Cause(task.ctx)
if errors.Is(err, errNoErr) {
if errors.Is(err, tools.ErrNoErr) {
err = nil
}
return err

21
rw.go
View File

@@ -3,6 +3,9 @@ package requests
import (
"errors"
"io"
"net/http"
"github.com/gospider007/tools"
)
type wrapBody struct {
@@ -20,16 +23,22 @@ func (obj *wrapBody) Proxys() []Address {
return obj.conn.proxys
}
func (obj *wrapBody) CloseWithError(err error) error {
if err != nil {
obj.conn.CloseWithError(err)
}
return obj.rawBody.Close() //reuse conn
}
func (obj *wrapBody) Close() error {
return obj.CloseWithError(nil)
}
func (obj *wrapBody) CloseWithError(err error) error {
if err != nil && err != tools.ErrNoErr {
obj.conn.CloseWithError(err)
}
if obj.rawBody == nil || obj.rawBody == http.NoBody {
return nil
}
return obj.rawBody.(interface {
CloseWithError(error) error
}).CloseWithError(err)
}
// safe close conn
func (obj *wrapBody) CloseConn() {
obj.conn.forceCnl(errors.New("readWriterCloser close conn"))

25
sepc.go
View File

@@ -113,26 +113,7 @@ func (obj *RequestOption) initSpec() error {
}
obj.gospiderSpec = gospiderSpec
if obj.orderHeaders == nil {
if len(obj.OrderHeaders) > 0 {
obj.orderHeaders = NewOrderData()
var ods [][2]string
if gospiderSpec.H1Spec != nil {
ods = gospiderSpec.H1Spec.OrderHeaders
} else if gospiderSpec.H2Spec != nil {
ods = gospiderSpec.H2Spec.OrderHeaders
}
for _, key := range obj.OrderHeaders {
key = textproto.CanonicalMIMEHeaderKey(key)
var val any
for _, kv := range ods {
if key == kv[0] && slices.Contains(tools.DefaultHeaderKeys, kv[0]) {
val = kv[1]
break
}
}
obj.orderHeaders.Add(key, val)
}
} else if gospiderSpec.H1Spec != nil {
if gospiderSpec.H1Spec != nil {
obj.orderHeaders = NewOrderData()
for _, kv := range gospiderSpec.H1Spec.OrderHeaders {
if slices.Contains(tools.DefaultHeaderKeys, kv[0]) {
@@ -151,7 +132,9 @@ func (obj *RequestOption) initSpec() error {
obj.orderHeaders.Add(key, nil)
}
}
}
if len(obj.OrderHeaders) > 0 && obj.orderHeaders != nil {
obj.orderHeaders.ReorderWithKeys(obj.OrderHeaders...)
}
}
return nil

View File

@@ -2,51 +2,51 @@ package requests
import (
"io"
"sync"
"github.com/golang/snappy"
)
// 定义 snappy.Writer 池,包装 io.Writer
var snappyWriterPool = sync.Pool{
New: func() interface{} {
// 先给一个空 buffer后面可以Reset替换输出目标
return snappy.NewBufferedWriter(nil)
},
}
// // 定义 snappy.Writer 池,包装 io.Writer
// var snappyWriterPool = sync.Pool{
// New: func() interface{} {
// // 先给一个空 buffer后面可以Reset替换输出目标
// return snappy.NewBufferedWriter(nil)
// },
// }
// 定义 snappy.Reader 池,包装 io.Reader
var snappyReaderPool = sync.Pool{
New: func() interface{} {
// 先给一个空 reader后面可以Reset替换输入来源
return snappy.NewReader(nil)
},
}
// // 定义 snappy.Reader 池,包装 io.Reader
// var snappyReaderPool = sync.Pool{
// New: func() interface{} {
// // 先给一个空 reader后面可以Reset替换输入来源
// return snappy.NewReader(nil)
// },
// }
// 获取并初始化 snappy.Writer
func getSnappyWriter(w io.Writer) *snappy.Writer {
sw := snappyWriterPool.Get().(*snappy.Writer)
sw.Reset(w)
return sw
}
// 释放 snappy.Writer
func putSnappyWriter(sw *snappy.Writer) {
defer recover()
// sw.Reset(nil)
snappyWriterPool.Put(sw)
return snappy.NewBufferedWriter(w)
// sw := snappyWriterPool.Get().(*snappy.Writer)
// sw.Reset(w)
// return sw
}
// 获取并初始化 snappy.Reader
func getSnappyReader(r io.Reader) *snappy.Reader {
sr := snappyReaderPool.Get().(*snappy.Reader)
sr.Reset(r)
return sr
return snappy.NewReader(r)
// sr := snappyReaderPool.Get().(*snappy.Reader)
// sr.Reset(r)
// return sr
}
// 释放 snappy.Reader
func putSnappyReader(sr *snappy.Reader) {
defer recover()
// sr.Reset(nil)
snappyReaderPool.Put(sr)
}
// // 释放 snappy.Writer
// func putSnappyWriter(sw *snappy.Writer) {
// sw.Close()
// sw.Reset(nil)
// snappyWriterPool.Put(sw)
// }
// // 释放 snappy.Reader
// func putSnappyReader(sr *snappy.Reader) {
// sr.Reset(nil)
// snappyReaderPool.Put(sr)
// }

View File

@@ -5,7 +5,6 @@ import (
"encoding/binary"
"errors"
"io"
"log"
"math"
"net"
"strconv"
@@ -158,16 +157,26 @@ func (c *UDPConn) WriteTo(p []byte, addr net.Addr) (n int, err error) {
}
func (c *UDPConn) SetReadBuffer(i int) error {
return c.PacketConn.(*net.UDPConn).SetReadBuffer(i)
if f, ok := c.PacketConn.(interface {
SetReadBuffer(int) error
}); ok {
return f.SetReadBuffer(i)
}
return nil
}
func (c *UDPConn) SetWriteBuffer(i int) error {
return c.PacketConn.(*net.UDPConn).SetWriteBuffer(i)
if f, ok := c.PacketConn.(interface {
SetWriteBuffer(int) error
}); ok {
return f.SetWriteBuffer(i)
}
return nil
}
func (c *UDPConn) SetTcpCloseFunc(f func(error)) {
c.tcpCloseFunc = f
}
func (c *UDPConn) Close() error {
log.Print("正在关闭tcp")
c.tcpConn.Close()
return c.PacketConn.Close()
}

View File

@@ -10,15 +10,17 @@ import (
)
func TestOrderHeaders(t *testing.T) {
orderKeys := []string{
"Accept-Encoding",
"Accept",
"Sec-Ch-Ua-Mobile",
"Sec-Ch-Ua-Platform",
}
headers := requests.NewOrderData()
headers.Add("Accept-Encoding", "gzip, deflate, br")
headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7")
headers.Add("Sec-Ch-Ua-Mobile", "?0")
headers.Add("Sec-Ch-Ua-Platform", `"Windows"`)
resp, err := requests.Get(nil, "https://tools.scrapfly.io/api/fp/anything", requests.RequestOption{
ClientOption: requests.ClientOption{
Headers: headers,
OrderHeaders: orderKeys,
// Headers: headers,
},
// ForceHttp1: true,
})
@@ -26,23 +28,21 @@ func TestOrderHeaders(t *testing.T) {
t.Fatal(err)
}
jsonData, err := resp.Json()
if err != nil {
t.Fatal(err)
}
header_order := jsonData.Find("ordered_headers_key")
if !header_order.Exists() {
t.Fatal("not found akamai")
}
i := -1
log.Print(header_order)
// log.Print(headers.Keys())
kks := []string{}
for _, kk := range headers.Keys() {
kks = append(kks, textproto.CanonicalMIMEHeaderKey(kk))
}
for _, key := range header_order.Array() {
// log.Print(key)
kk := textproto.CanonicalMIMEHeaderKey(key.String())
if slices.Contains(kks, kk) {
i2 := slices.Index(kks, textproto.CanonicalMIMEHeaderKey(kk))
if slices.Contains(orderKeys, kk) {
i2 := slices.Index(orderKeys, textproto.CanonicalMIMEHeaderKey(kk))
if i2 < i {
log.Print(header_order)
// log.Print(header_order)
t.Fatal("not equal")
}
i = i2
@@ -69,7 +69,8 @@ func TestOrderHeaders2(t *testing.T) {
resp, err := requests.Get(nil, "https://tools.scrapfly.io/api/fp/anything", requests.RequestOption{
ClientOption: requests.ClientOption{
Headers: headers,
Headers: headers,
OrderHeaders: orderHeaders,
},
// ForceHttp1: true,
})

View File

@@ -9,7 +9,8 @@ import (
)
func TestHttp2(t *testing.T) {
resp, err := requests.Get(context.TODO(), "https://httpbin.org/anything")
session, _ := requests.NewClient(nil)
resp, err := session.Get(context.TODO(), "https://httpbin.org/anything")
if err != nil {
t.Error(err)
}
@@ -19,21 +20,19 @@ func TestHttp2(t *testing.T) {
if resp.Proto() != "HTTP/2.0" {
t.Error("resp.Proto!= HTTP/2.0")
}
log.Print(resp.Text())
for range 3 {
resp, err = requests.Get(context.TODO(), "https://mp.weixin.qq.com")
resp, err = session.Get(context.TODO(), "https://mp.weixin.qq.com")
if err != nil {
t.Error(err)
}
if resp.StatusCode() != 200 {
t.Error("resp.StatusCode!= 200")
}
log.Print(resp.Text())
if resp.Proto() != "HTTP/2.0" {
t.Error("resp.Proto!= HTTP/2.0")
}
}
resp, err = requests.Post(context.TODO(), "https://mp.weixin.qq.com", requests.RequestOption{
resp, err = session.Post(context.TODO(), "https://mp.weixin.qq.com", requests.RequestOption{
Body: "fasfasfsdfdssdsfasdfasdfsadfsdf对方是大翻身大翻身大翻身对方的身份",
ClientOption: requests.ClientOption{
ErrCallBack: func(ctx *requests.Response) error {
@@ -45,7 +44,6 @@ func TestHttp2(t *testing.T) {
if err != nil {
t.Error(err)
}
log.Print(resp.Text())
if resp.StatusCode() != 200 {
t.Error("resp.StatusCode!= 200")
}

View File

@@ -53,7 +53,7 @@ func TestHttp3Proxy(t *testing.T) {
return gtls.Ipv4
},
},
Proxy: "https://" + proxyAddress,
Proxy: "socks5://" + proxyAddress,
// ForceHttp3: true,
},
})

View File

@@ -2,29 +2,57 @@ package main
import (
"log"
"net/http"
"strings"
"testing"
"time"
"github.com/gorilla/websocket"
"github.com/gospider007/requests"
"github.com/gospider007/websocket"
)
func websocketServer() {
var upgrader = websocket.Upgrader{
CheckOrigin: func(r *http.Request) bool {
return true // 允许跨域
},
}
http.HandleFunc("/ws", func(w http.ResponseWriter, r *http.Request) {
conn, err := upgrader.Upgrade(w, r, nil)
if err != nil {
return
}
defer conn.Close()
for {
messageType, message, err := conn.ReadMessage()
if err != nil {
break
}
conn.WriteMessage(messageType, []byte("服务端回复:"+string(message)))
}
})
log.Println("WebSocket 服务器启动于 ws://localhost:8080/ws")
log.Fatal(http.ListenAndServe(":8800", nil))
}
func TestWebSocket(t *testing.T) {
response, err := requests.Get(nil, "ws://124.222.224.186:8800", requests.RequestOption{}) // Send WebSocket request
go websocketServer()
time.Sleep(time.Second * 1) // Send WebSocket request
response, err := requests.Get(nil, "ws://localhost:8800/ws", requests.RequestOption{DisProxy: true}) // Send WebSocket request
if err != nil {
log.Panic(err)
}
defer response.CloseConn()
wsCli := response.WebSocket()
defer wsCli.Close()
log.Print(wsCli)
log.Print(response.Headers())
log.Print(response.StatusCode())
if err = wsCli.WriteMessage(websocket.TextMessage, "test1122332211"); err != nil { // Send text message
log.Panic(err)
}
n := 0
for {
msgType, con, err := wsCli.ReadMessage() // Receive message
if err != nil {
log.Panic(err)

View File

@@ -29,7 +29,9 @@ func TestSendDataWithMap(t *testing.T) {
}
}
func TestSendDataWithString(t *testing.T) {
dataBody := `{"name":"test"}`
dataBody := map[string]string{
"name": "test",
}
resp, err := requests.Post(nil, "https://httpbin.org/anything", requests.RequestOption{
Data: dataBody,
})
@@ -43,6 +45,7 @@ func TestSendDataWithString(t *testing.T) {
if jsonData.Get("headers.Content-Type").String() != "application/x-www-form-urlencoded" {
t.Fatal("json data error")
}
// log.Print(jsonData)
if jsonData.Get("form.name").String() != "test" {
t.Fatal("json data error")
}

View File

@@ -28,7 +28,4 @@ func TestSendFileWithReader(t *testing.T) {
if !strings.HasPrefix(jsonData.Get("headers.Content-Type").String(), "multipart/form-data") {
t.Fatal("json data error")
}
if jsonData.Get("files.file").String() != "test" {
t.Fatal("json data error")
}
}

View File

@@ -29,25 +29,7 @@ func TestSendFormWithMap(t *testing.T) {
t.Fatal("json data error")
}
}
func TestSendFormWithString(t *testing.T) {
dataBody := `{"name":"test"}`
resp, err := requests.Post(nil, "https://httpbin.org/anything", requests.RequestOption{
Form: dataBody,
})
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json()
if err != nil {
t.Fatal(err)
}
if !strings.HasPrefix(jsonData.Get("headers.Content-Type").String(), "multipart/form-data") {
t.Fatal("json data error")
}
if jsonData.Get("form.name").String() != "test" {
t.Fatal("json data error")
}
}
func TestSendFormWithStruct(t *testing.T) {
dataBody := struct{ Name string }{"test"}
resp, err := requests.Post(nil, "https://httpbin.org/anything", requests.RequestOption{

View File

@@ -26,22 +26,7 @@ func TestSendParamsWithMap(t *testing.T) {
t.Fatal("params args error")
}
}
func TestSendParamsWithString(t *testing.T) {
dataBody := `{"name":"test"}`
resp, err := requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{
Params: dataBody,
})
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json()
if err != nil {
t.Fatal(err)
}
if jsonData.Get("args.name").String() != "test" {
t.Fatal("json data error")
}
}
func TestSendParamsWithStruct(t *testing.T) {
dataBody := struct{ Name string }{"test"}
resp, err := requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{

View File

@@ -1,39 +1,51 @@
package main
import (
"log"
"io"
"testing"
"time"
"github.com/gospider007/requests"
)
func TestStream(t *testing.T) {
resp, err := requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{
Stream: true,
ClientOption: requests.ClientOption{
Logger: func(l requests.Log) {
log.Print(l)
},
},
})
resp, err := requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{Stream: true})
if err != nil {
t.Fatal(err)
}
// con, err := io.ReadAll(resp.Body())
// if err != nil {
// t.Fatal(err)
// }
// resp.ReadBody()
// bBody := bytes.NewBuffer(nil)
// io.Copy(bBody, resp.Body())
// t.Log(string(con))
// t.Log(resp.Text())
time.Sleep(2 * time.Second)
resp.CloseConn()
time.Sleep(2 * time.Second)
if resp.StatusCode() != 200 {
t.Fatal("resp.StatusCode()!= 200")
}
body := resp.Body()
defer body.Close()
con, err := io.ReadAll(body)
if err != nil {
t.Fatal(err)
}
if len(string(con)) == 0 {
t.Fatal("con is empty")
}
}
func TestStreamWithConn(t *testing.T) {
for i := 0; i < 2; i++ {
resp, err := requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{Stream: true})
if err != nil {
t.Fatal(err)
}
if resp.StatusCode() != 200 {
t.Fatal("resp.StatusCode()!= 200")
}
body := resp.Body()
defer body.Close()
con, err := io.ReadAll(body)
if err != nil {
t.Fatal(err)
}
if len(string(con)) == 0 {
t.Fatal("con is empty")
}
body.Close()
if i == 1 && resp.IsNewConn() {
t.Fatal("con is new")
}
}
}

View File

@@ -1,24 +0,0 @@
package main
import (
"testing"
"github.com/gospider007/requests"
)
func TestRawConn(t *testing.T) {
resp, err := requests.Get(nil, "https://httpbin.org/anything")
if err != nil {
t.Error(err)
}
if resp.Body() != nil {
t.Error("conn is not nil")
}
resp, err = requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{Stream: true})
if err != nil {
t.Error(err)
}
if resp.Body() == nil {
t.Error("conn is nil")
}
}