mirror of
https://github.com/gospider007/requests.git
synced 2025-12-24 13:57:52 +08:00
sync
This commit is contained in:
355
compressConn.go
355
compressConn.go
@@ -1,40 +1,34 @@
|
||||
package requests
|
||||
|
||||
import (
|
||||
"compress/flate"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/klauspost/compress/zstd"
|
||||
)
|
||||
|
||||
type CompressionConn struct {
|
||||
conn net.Conn
|
||||
w *WriterCompression
|
||||
r *ReaderCompression
|
||||
}
|
||||
type ReaderCompression struct {
|
||||
oneFunc func()
|
||||
r io.Reader
|
||||
}
|
||||
type WriterCompression struct {
|
||||
oneFunc func()
|
||||
w io.WriteCloser
|
||||
f interface{ Flush() error }
|
||||
w io.WriteCloser
|
||||
r io.ReadCloser
|
||||
}
|
||||
type Compression interface {
|
||||
OpenReader(r io.Reader) (io.Reader, error)
|
||||
String() string
|
||||
OpenReader(r io.Reader) (io.ReadCloser, error)
|
||||
OpenWriter(w io.Writer) (io.WriteCloser, error)
|
||||
}
|
||||
type compression struct {
|
||||
openReader func(r io.Reader) (io.Reader, error)
|
||||
name string
|
||||
openReader func(r io.Reader) (io.ReadCloser, error)
|
||||
openWriter func(w io.Writer) (io.WriteCloser, error)
|
||||
}
|
||||
|
||||
func (obj compression) OpenReader(r io.Reader) (io.Reader, error) {
|
||||
func (obj compression) String() string {
|
||||
return obj.name
|
||||
}
|
||||
func (obj compression) OpenReader(r io.Reader) (io.ReadCloser, error) {
|
||||
return obj.openReader(r)
|
||||
}
|
||||
func (obj compression) OpenWriter(w io.Writer) (io.WriteCloser, error) {
|
||||
@@ -55,174 +49,80 @@ func GetCompressionByte(decode string) (byte, error) {
|
||||
return 0, errors.New("unsupported compression type")
|
||||
}
|
||||
}
|
||||
|
||||
var compressionData = map[byte]compression{
|
||||
40: {
|
||||
name: "zstd",
|
||||
openReader: newZstdReader,
|
||||
openWriter: newZstdWriter,
|
||||
},
|
||||
255: {
|
||||
name: "s2",
|
||||
openReader: newSnappyReader,
|
||||
openWriter: newSnappyWriter,
|
||||
},
|
||||
92: {
|
||||
name: "flate",
|
||||
openReader: newFlateReader,
|
||||
openWriter: newFlateWriter,
|
||||
},
|
||||
93: {
|
||||
name: "minlz",
|
||||
openReader: newMinlzReader,
|
||||
openWriter: newMinlzWriter,
|
||||
},
|
||||
}
|
||||
|
||||
func NewCompressionWithByte(b byte) (Compression, error) {
|
||||
switch b {
|
||||
case 40:
|
||||
return NewCompression("zstd")
|
||||
case 255:
|
||||
return NewCompression("s2")
|
||||
case 92:
|
||||
return NewCompression("flate")
|
||||
case 93:
|
||||
return NewCompression("minlz")
|
||||
c, ok := compressionData[b]
|
||||
if !ok {
|
||||
return nil, errors.New("unsupported compression type")
|
||||
}
|
||||
return compression{
|
||||
name: c.name,
|
||||
openReader: func(r io.Reader) (io.ReadCloser, error) {
|
||||
buf := make([]byte, 1)
|
||||
n, err := r.Read(buf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if n != 1 || buf[0] != b {
|
||||
return nil, errors.New("invalid response")
|
||||
}
|
||||
return c.openReader(r)
|
||||
},
|
||||
openWriter: func(w io.Writer) (io.WriteCloser, error) {
|
||||
n, err := w.Write([]byte{b})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if n != 1 {
|
||||
return nil, errors.New("invalid response")
|
||||
}
|
||||
return c.openWriter(w)
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
func NewCompression(decode string) (Compression, error) {
|
||||
decode = strings.ToLower(decode)
|
||||
for b, c := range compressionData {
|
||||
if c.String() == decode {
|
||||
return NewCompressionWithByte(b)
|
||||
}
|
||||
}
|
||||
return nil, errors.New("unsupported compression type")
|
||||
}
|
||||
func NewCompression(decode string) (Compression, error) {
|
||||
b, err := GetCompressionByte(decode)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
br := func(r io.Reader) error {
|
||||
buf := make([]byte, 1)
|
||||
n, err := r.Read(buf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n != 1 || buf[0] != b {
|
||||
return errors.New("invalid response")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
bw := func(w io.Writer) error {
|
||||
n, err := w.Write([]byte{b})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n != 1 {
|
||||
return errors.New("invalid response")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
var arch Compression
|
||||
switch strings.ToLower(decode) {
|
||||
case "s2":
|
||||
arch = compression{
|
||||
openReader: func(r io.Reader) (io.Reader, error) {
|
||||
err := br(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return getSnappyReader(r), nil
|
||||
},
|
||||
openWriter: func(w io.Writer) (io.WriteCloser, error) {
|
||||
err := bw(w)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return getSnappyWriter(w), nil
|
||||
},
|
||||
}
|
||||
case "zstd":
|
||||
arch = compression{
|
||||
openReader: func(r io.Reader) (io.Reader, error) {
|
||||
err := br(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
decoder, err := zstd.NewReader(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return decoder.IOReadCloser(), nil
|
||||
},
|
||||
openWriter: func(w io.Writer) (io.WriteCloser, error) {
|
||||
err := bw(w)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
encoder, err := zstd.NewWriter(nil, zstd.WithEncoderLevel(zstd.SpeedDefault))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return encoder, nil
|
||||
},
|
||||
}
|
||||
case "flate":
|
||||
arch = compression{
|
||||
openReader: func(r io.Reader) (io.Reader, error) {
|
||||
err := br(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return flate.NewReader(r), nil
|
||||
},
|
||||
openWriter: func(w io.Writer) (io.WriteCloser, error) {
|
||||
err := bw(w)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return flate.NewWriter(w, flate.DefaultCompression)
|
||||
},
|
||||
}
|
||||
case "minlz":
|
||||
arch = compression{
|
||||
openReader: func(r io.Reader) (io.Reader, error) {
|
||||
err := br(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return getMinlzReader(r), nil
|
||||
},
|
||||
openWriter: func(w io.Writer) (io.WriteCloser, error) {
|
||||
err := bw(w)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return getMinlzWriter(w), nil
|
||||
},
|
||||
}
|
||||
default:
|
||||
return nil, errors.New("unsupported compression type")
|
||||
}
|
||||
return arch, nil
|
||||
}
|
||||
|
||||
func NewWriterCompression(conn io.Writer, arch Compression) (*WriterCompression, error) {
|
||||
w, err := arch.OpenWriter(conn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ccon := &WriterCompression{
|
||||
w: w,
|
||||
// oneFunc: sync.OnceFunc(func() {
|
||||
// switch snW := w.(type) {
|
||||
// case *snappy.Writer:
|
||||
// putSnappyWriter(snW)
|
||||
// case *minlz.Writer:
|
||||
// putMinlzWriter(snW)
|
||||
// }
|
||||
// }),
|
||||
}
|
||||
if f, ok := w.(interface{ Flush() error }); ok {
|
||||
ccon.f = f
|
||||
}
|
||||
return ccon, nil
|
||||
}
|
||||
func NewReaderCompression(conn io.Reader, arch Compression) (*ReaderCompression, error) {
|
||||
r, err := arch.OpenReader(conn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ccon := &ReaderCompression{
|
||||
r: r,
|
||||
// oneFunc: sync.OnceFunc(func() {
|
||||
// switch snR := r.(type) {
|
||||
// case *snappy.Reader:
|
||||
// putSnappyReader(snR)
|
||||
// case *minlz.Reader:
|
||||
// putMinlzReader(snR)
|
||||
// }
|
||||
// }),
|
||||
}
|
||||
return ccon, nil
|
||||
}
|
||||
func NewCompressionConn(conn net.Conn, arch Compression) (net.Conn, error) {
|
||||
w, err := NewWriterCompression(conn, arch)
|
||||
func NewCompressionConn(conn net.Conn, decode string) (net.Conn, error) {
|
||||
arch, err := NewCompression(decode)
|
||||
if err != nil {
|
||||
return conn, err
|
||||
}
|
||||
r, err := NewReaderCompression(conn, arch)
|
||||
w, err := arch.OpenWriter(conn)
|
||||
if err != nil {
|
||||
return conn, err
|
||||
}
|
||||
r, err := arch.OpenReader(conn)
|
||||
if err != nil {
|
||||
return conn, err
|
||||
}
|
||||
@@ -233,34 +133,6 @@ func NewCompressionConn(conn net.Conn, arch Compression) (net.Conn, error) {
|
||||
}
|
||||
return ccon, nil
|
||||
}
|
||||
func (obj *WriterCompression) Write(b []byte) (n int, err error) {
|
||||
n, err = obj.w.Write(b)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if obj.f != nil {
|
||||
err = obj.f.Flush()
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (obj *WriterCompression) Close() error {
|
||||
if obj.oneFunc != nil {
|
||||
obj.oneFunc()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (obj *ReaderCompression) Read(b []byte) (n int, err error) {
|
||||
return obj.r.Read(b)
|
||||
}
|
||||
func (obj *ReaderCompression) Close() error {
|
||||
if obj.oneFunc != nil {
|
||||
obj.oneFunc()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (obj *CompressionConn) Read(b []byte) (n int, err error) {
|
||||
return obj.r.Read(b)
|
||||
}
|
||||
@@ -268,8 +140,6 @@ func (obj *CompressionConn) Write(b []byte) (n int, err error) {
|
||||
return obj.w.Write(b)
|
||||
}
|
||||
func (obj *CompressionConn) Close() error {
|
||||
obj.w.Close()
|
||||
obj.r.Close()
|
||||
return obj.conn.Close()
|
||||
}
|
||||
|
||||
@@ -289,3 +159,74 @@ func (obj *CompressionConn) SetReadDeadline(t time.Time) error {
|
||||
func (obj *CompressionConn) SetWriteDeadline(t time.Time) error {
|
||||
return obj.conn.SetWriteDeadline(t)
|
||||
}
|
||||
|
||||
type ReaderCompression struct {
|
||||
c io.ReadCloser
|
||||
closed bool
|
||||
lock sync.Mutex
|
||||
closeFunc func()
|
||||
}
|
||||
|
||||
func (obj *ReaderCompression) Read(p []byte) (int, error) {
|
||||
obj.lock.Lock()
|
||||
defer obj.lock.Unlock()
|
||||
if obj.closed {
|
||||
return 0, errors.New("closed")
|
||||
}
|
||||
return obj.c.Read(p)
|
||||
}
|
||||
func (obj *ReaderCompression) Close() error {
|
||||
obj.lock.Lock()
|
||||
defer obj.lock.Unlock()
|
||||
if obj.closed {
|
||||
return nil
|
||||
}
|
||||
obj.closed = true
|
||||
obj.c.Close()
|
||||
obj.closeFunc()
|
||||
return nil
|
||||
}
|
||||
|
||||
type WriterCompression struct {
|
||||
c io.WriteCloser
|
||||
closed bool
|
||||
lock sync.Mutex
|
||||
flush interface{ Flush() error }
|
||||
closeFunc func()
|
||||
}
|
||||
|
||||
func (obj *WriterCompression) Write(p []byte) (int, error) {
|
||||
obj.lock.Lock()
|
||||
defer obj.lock.Unlock()
|
||||
if obj.closed {
|
||||
return 0, errors.New("closed")
|
||||
}
|
||||
n, err := obj.c.Write(p)
|
||||
if err != nil {
|
||||
return n, err
|
||||
}
|
||||
if obj.flush != nil {
|
||||
err = obj.flush.Flush()
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (obj *WriterCompression) Close() error {
|
||||
obj.lock.Lock()
|
||||
defer obj.lock.Unlock()
|
||||
if obj.closed {
|
||||
return nil
|
||||
}
|
||||
obj.closed = true
|
||||
obj.c.Close()
|
||||
obj.closeFunc()
|
||||
return nil
|
||||
}
|
||||
|
||||
func newWriterCompression(c io.WriteCloser, closeFunc func()) *WriterCompression {
|
||||
flush, _ := c.(interface{ Flush() error })
|
||||
return &WriterCompression{c: c, closeFunc: closeFunc, flush: flush}
|
||||
}
|
||||
func newReaderCompression(c io.ReadCloser, closeFunc func()) *ReaderCompression {
|
||||
return &ReaderCompression{c: c, closeFunc: closeFunc}
|
||||
}
|
||||
|
||||
139
compressionPool.go
Normal file
139
compressionPool.go
Normal file
@@ -0,0 +1,139 @@
|
||||
package requests
|
||||
|
||||
import (
|
||||
"compress/flate"
|
||||
"io"
|
||||
"sync"
|
||||
|
||||
"github.com/golang/snappy"
|
||||
"github.com/klauspost/compress/zstd"
|
||||
"github.com/minio/minlz"
|
||||
)
|
||||
|
||||
// zstd pool
|
||||
var zstdWriterPool = sync.Pool{
|
||||
New: func() any {
|
||||
c, _ := zstd.NewWriter(nil, zstd.WithWindowSize(32*1024))
|
||||
return c
|
||||
},
|
||||
}
|
||||
|
||||
func newZstdWriter(w io.Writer) (io.WriteCloser, error) {
|
||||
z := zstdWriterPool.Get().(*zstd.Encoder)
|
||||
z.Reset(w)
|
||||
return newWriterCompression(z, func() {
|
||||
z.Reset(nil)
|
||||
zstdWriterPool.Put(z)
|
||||
}), nil
|
||||
}
|
||||
|
||||
var zstdReaderPool = sync.Pool{
|
||||
New: func() any {
|
||||
w, _ := zstd.NewReader(nil)
|
||||
return w
|
||||
},
|
||||
}
|
||||
|
||||
func newZstdReader(w io.Reader) (io.ReadCloser, error) {
|
||||
z := zstdReaderPool.Get().(*zstd.Decoder)
|
||||
z.Reset(w)
|
||||
return newReaderCompression(io.NopCloser(z), func() {
|
||||
z.Reset(nil)
|
||||
zstdReaderPool.Put(z)
|
||||
}), nil
|
||||
}
|
||||
|
||||
// snappy pool
|
||||
var snappyWriterPool = sync.Pool{
|
||||
New: func() any {
|
||||
return snappy.NewBufferedWriter(nil)
|
||||
},
|
||||
}
|
||||
|
||||
func newSnappyWriter(w io.Writer) (io.WriteCloser, error) {
|
||||
s := snappyWriterPool.Get().(*snappy.Writer)
|
||||
s.Reset(w)
|
||||
return newWriterCompression(s, func() {
|
||||
s.Reset(nil)
|
||||
snappyWriterPool.Put(s)
|
||||
}), nil
|
||||
}
|
||||
|
||||
var snappyReaderPool = sync.Pool{
|
||||
New: func() any {
|
||||
return snappy.NewReader(nil)
|
||||
},
|
||||
}
|
||||
|
||||
func newSnappyReader(w io.Reader) (io.ReadCloser, error) {
|
||||
s := snappyReaderPool.Get().(*snappy.Reader)
|
||||
s.Reset(w)
|
||||
return newReaderCompression(io.NopCloser(s), func() {
|
||||
s.Reset(nil)
|
||||
snappyReaderPool.Put(s)
|
||||
}), nil
|
||||
}
|
||||
|
||||
// flate pool
|
||||
var flateWriterPool = sync.Pool{
|
||||
New: func() any {
|
||||
w, _ := flate.NewWriter(nil, flate.DefaultCompression)
|
||||
return w
|
||||
},
|
||||
}
|
||||
|
||||
func newFlateWriter(w io.Writer) (io.WriteCloser, error) {
|
||||
f := flateWriterPool.Get().(*flate.Writer)
|
||||
f.Reset(w)
|
||||
return newWriterCompression(f, func() {
|
||||
f.Reset(nil)
|
||||
flateWriterPool.Put(f)
|
||||
}), nil
|
||||
}
|
||||
|
||||
var flateReaderPool = sync.Pool{
|
||||
New: func() any {
|
||||
return flate.NewReader(nil)
|
||||
},
|
||||
}
|
||||
|
||||
func newFlateReader(w io.Reader) (io.ReadCloser, error) {
|
||||
r := flateReaderPool.Get().(io.ReadCloser)
|
||||
f := r.(flate.Resetter)
|
||||
err := f.Reset(w, nil)
|
||||
return newReaderCompression(r, func() {
|
||||
f.Reset(nil, nil)
|
||||
flateReaderPool.Put(r)
|
||||
}), err
|
||||
}
|
||||
|
||||
// minlz pool
|
||||
var minlzWriterPool = sync.Pool{
|
||||
New: func() any {
|
||||
return minlz.NewWriter(nil, minlz.WriterBlockSize(32*1024))
|
||||
},
|
||||
}
|
||||
|
||||
func newMinlzWriter(w io.Writer) (io.WriteCloser, error) {
|
||||
m := minlzWriterPool.Get().(*minlz.Writer)
|
||||
m.Reset(w)
|
||||
return newWriterCompression(m, func() {
|
||||
m.Reset(nil)
|
||||
minlzWriterPool.Put(m)
|
||||
}), nil
|
||||
}
|
||||
|
||||
var minlzReaderPool = sync.Pool{
|
||||
New: func() any {
|
||||
return minlz.NewReader(nil, minlz.ReaderMaxBlockSize(32*1024))
|
||||
},
|
||||
}
|
||||
|
||||
func newMinlzReader(w io.Reader) (io.ReadCloser, error) {
|
||||
m := minlzReaderPool.Get().(*minlz.Reader)
|
||||
m.Reset(w)
|
||||
return newReaderCompression(io.NopCloser(m), func() {
|
||||
m.Reset(nil)
|
||||
minlzReaderPool.Put(m)
|
||||
}), nil
|
||||
}
|
||||
18
go.mod
18
go.mod
@@ -11,7 +11,7 @@ require (
|
||||
github.com/gospider007/gtls v0.0.0-20250801033740-1ebcdc3aaa9a
|
||||
github.com/gospider007/http1 v0.0.0-20250806001143-d0669acd99ef
|
||||
github.com/gospider007/http2 v0.0.0-20250805092410-87c07ed55dc7
|
||||
github.com/gospider007/http3 v0.0.0-20250805092453-0bfc34e041e4
|
||||
github.com/gospider007/http3 v0.0.0-20250811065920-4ec9b14f9686
|
||||
github.com/gospider007/ja3 v0.0.0-20250724085013-aa7e0a527d6c
|
||||
github.com/gospider007/re v0.0.0-20250217075352-bcb79f285d6c
|
||||
github.com/gospider007/tools v0.0.0-20250730001814-2c13a4717064
|
||||
@@ -22,8 +22,8 @@ require (
|
||||
github.com/refraction-networking/uquic v0.0.6
|
||||
github.com/refraction-networking/utls v1.8.0
|
||||
github.com/txthinking/socks5 v0.0.0-20250803014457-d3594b14cf12
|
||||
golang.org/x/crypto v0.40.0
|
||||
golang.org/x/net v0.42.0
|
||||
golang.org/x/crypto v0.41.0
|
||||
golang.org/x/net v0.43.0
|
||||
gopkg.in/errgo.v2 v2.1.0
|
||||
)
|
||||
|
||||
@@ -81,11 +81,11 @@ require (
|
||||
go.uber.org/zap v1.27.0 // indirect
|
||||
go.uber.org/zap/exp v0.3.0 // indirect
|
||||
go4.org v0.0.0-20230225012048-214862532bf5 // indirect
|
||||
golang.org/x/exp v0.0.0-20250718183923-645b1fa84792 // indirect
|
||||
golang.org/x/image v0.29.0 // indirect
|
||||
golang.org/x/mod v0.26.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20250808145144-a408d31f581a // indirect
|
||||
golang.org/x/image v0.30.0 // indirect
|
||||
golang.org/x/mod v0.27.0 // indirect
|
||||
golang.org/x/sync v0.16.0 // indirect
|
||||
golang.org/x/sys v0.34.0 // indirect
|
||||
golang.org/x/text v0.27.0 // indirect
|
||||
golang.org/x/tools v0.35.0 // indirect
|
||||
golang.org/x/sys v0.35.0 // indirect
|
||||
golang.org/x/text v0.28.0 // indirect
|
||||
golang.org/x/tools v0.36.0 // indirect
|
||||
)
|
||||
|
||||
40
go.sum
40
go.sum
@@ -118,8 +118,8 @@ github.com/gospider007/http1 v0.0.0-20250806001143-d0669acd99ef h1:Ag8TrazSH1sVx
|
||||
github.com/gospider007/http1 v0.0.0-20250806001143-d0669acd99ef/go.mod h1:fnyx2omi7IYVdfKxfkAUR240Au/EKZ+HruhXw2EOrdg=
|
||||
github.com/gospider007/http2 v0.0.0-20250805092410-87c07ed55dc7 h1:o0S6ZZwnTrDwuDnRZkA0dIPsRuBe9vTlazb6+j/taR0=
|
||||
github.com/gospider007/http2 v0.0.0-20250805092410-87c07ed55dc7/go.mod h1:9heGY5Ue/tgXRZWbPYjP7+0ccZcv+4FBH1gKANCy5Q4=
|
||||
github.com/gospider007/http3 v0.0.0-20250805092453-0bfc34e041e4 h1:Em8k4roCfuYqOkYMCKOD+SxaMbyJpGIKP2e27FBCdEY=
|
||||
github.com/gospider007/http3 v0.0.0-20250805092453-0bfc34e041e4/go.mod h1:6/TrJFJVirAzDReKWE8GOAOavXcqXLJp4E23Fh/ZM3k=
|
||||
github.com/gospider007/http3 v0.0.0-20250811065920-4ec9b14f9686 h1:1DgqAbiUoZueBEO4Ti5uqqKE60SagebEzKNh1tzpWcI=
|
||||
github.com/gospider007/http3 v0.0.0-20250811065920-4ec9b14f9686/go.mod h1:g5v7nxLoHAXT5FMbU+m2Q43oiNiAp0MDdcI+lqb2+vs=
|
||||
github.com/gospider007/ja3 v0.0.0-20250724085013-aa7e0a527d6c h1:Ksw8S50iGK0dqYmScPRn/oOuKegHJ2j9DupSURZZmoI=
|
||||
github.com/gospider007/ja3 v0.0.0-20250724085013-aa7e0a527d6c/go.mod h1:zXTX/Yt8S9fChnNwUFrHaQJpZYi0xyFQ2qKP6iHvPWI=
|
||||
github.com/gospider007/kinds v0.0.0-20250217075226-10f199f7215d h1:+Fih572EdNmYCK1L82MyWOj9tAwIgZ2JqepmasAclg8=
|
||||
@@ -275,8 +275,8 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY
|
||||
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
|
||||
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
|
||||
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
|
||||
golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM=
|
||||
golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY=
|
||||
golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4=
|
||||
golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
|
||||
@@ -285,12 +285,12 @@ golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE
|
||||
golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
|
||||
golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
|
||||
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
|
||||
golang.org/x/exp v0.0.0-20250718183923-645b1fa84792 h1:R9PFI6EUdfVKgwKjZef7QIwGcBKu86OEFpJ9nUEP2l4=
|
||||
golang.org/x/exp v0.0.0-20250718183923-645b1fa84792/go.mod h1:A+z0yzpGtvnG90cToK5n2tu8UJVP2XUATh+r+sfOOOc=
|
||||
golang.org/x/exp v0.0.0-20250808145144-a408d31f581a h1:Y+7uR/b1Mw2iSXZ3G//1haIiSElDQZ8KWh0h+sZPG90=
|
||||
golang.org/x/exp v0.0.0-20250808145144-a408d31f581a/go.mod h1:rT6SFzZ7oxADUDx58pcaKFTcZ+inxAa9fTrYx/uVYwg=
|
||||
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
|
||||
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
|
||||
golang.org/x/image v0.29.0 h1:HcdsyR4Gsuys/Axh0rDEmlBmB68rW1U9BUdB3UVHsas=
|
||||
golang.org/x/image v0.29.0/go.mod h1:RVJROnf3SLK8d26OW91j4FrIHGbsJ8QnbEocVTOWQDA=
|
||||
golang.org/x/image v0.30.0 h1:jD5RhkmVAnjqaCUXfbGBrn3lpxbknfN9w2UhHHU+5B4=
|
||||
golang.org/x/image v0.30.0/go.mod h1:SAEUTxCCMWSrJcCy/4HwavEsfZZJlYxeHLc6tTiAe/c=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
@@ -313,8 +313,8 @@ golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/mod v0.26.0 h1:EGMPT//Ezu+ylkCijjPc+f4Aih7sZvaAr+O3EHBxvZg=
|
||||
golang.org/x/mod v0.26.0/go.mod h1:/j6NAhSk8iQ723BGAUyoAcn7SlD7s15Dp9Nd/SfeaFQ=
|
||||
golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ=
|
||||
golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
@@ -338,8 +338,8 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
|
||||
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
|
||||
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
|
||||
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
|
||||
golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs=
|
||||
golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8=
|
||||
golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE=
|
||||
golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
@@ -383,8 +383,8 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA=
|
||||
golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
|
||||
golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI=
|
||||
golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
|
||||
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
@@ -395,8 +395,8 @@ golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
|
||||
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
|
||||
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
|
||||
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
|
||||
golang.org/x/term v0.33.0 h1:NuFncQrRcaRvVmgRkvM3j/F00gWIAlcmlB8ACEKmGIg=
|
||||
golang.org/x/term v0.33.0/go.mod h1:s18+ql9tYWp1IfpV9DmCtQDDSRBUjKaw9M1eAv5UeF0=
|
||||
golang.org/x/term v0.34.0 h1:O/2T7POpk0ZZ7MAzMeWFSg6S5IpWd/RXDlM9hgM3DR4=
|
||||
golang.org/x/term v0.34.0/go.mod h1:5jC53AEywhIVebHgPVeg0mj8OD3VO9OzclacVrqpaAw=
|
||||
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
@@ -410,8 +410,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
|
||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
|
||||
golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4=
|
||||
golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU=
|
||||
golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng=
|
||||
golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE=
|
||||
@@ -445,8 +445,8 @@ golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k=
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
|
||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
|
||||
golang.org/x/tools v0.35.0 h1:mBffYraMEf7aa0sB+NuKnuCy8qI/9Bughn8dC2Gu5r0=
|
||||
golang.org/x/tools v0.35.0/go.mod h1:NKdj5HkL/73byiZSJjqJgKn3ep7KjFkBOkR/Hps3VPw=
|
||||
golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg=
|
||||
golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
|
||||
38
minlz.go
38
minlz.go
@@ -1,38 +0,0 @@
|
||||
package requests
|
||||
|
||||
import (
|
||||
"io"
|
||||
|
||||
"github.com/minio/minlz"
|
||||
)
|
||||
|
||||
// // 定义 minlz.Writer 池,包装 io.Writer
|
||||
// var minlzWriterPool = sync.Pool{
|
||||
// New: func() interface{} {
|
||||
// return minlz.NewWriter(nil, minlz.WriterBlockSize(64*1024), minlz.WriterConcurrency(1), minlz.WriterLevel(minlz.LevelSmallest))
|
||||
// },
|
||||
// }
|
||||
|
||||
// // 定义 minlz.Reader 池,包装 io.Reader
|
||||
// var minlzReaderPool = sync.Pool{
|
||||
// New: func() interface{} {
|
||||
// // 先给一个空 reader,后面可以Reset替换输入来源
|
||||
// return minlz.NewReader(nil, minlz.ReaderMaxBlockSize(64*1024))
|
||||
// },
|
||||
// }
|
||||
|
||||
// 获取并初始化 minlz.Writer
|
||||
func getMinlzWriter(w io.Writer) *minlz.Writer {
|
||||
return minlz.NewWriter(w, minlz.WriterBlockSize(64*1024), minlz.WriterConcurrency(1))
|
||||
// sw := minlzWriterPool.Get().(*minlz.Writer)
|
||||
// sw.Reset(w)
|
||||
// return sw
|
||||
}
|
||||
|
||||
// 获取并初始化 minlz.Reader
|
||||
func getMinlzReader(r io.Reader) *minlz.Reader {
|
||||
// sr := minlzReaderPool.Get().(*minlz.Reader)
|
||||
// sr.Reset(r)
|
||||
// return sr
|
||||
return minlz.NewReader(r, minlz.ReaderMaxBlockSize(64*1024))
|
||||
}
|
||||
@@ -213,15 +213,9 @@ func (obj *roundTripper) dial(ctx *Response) (conn http1.Conn, err error) {
|
||||
}
|
||||
}
|
||||
var rawNetConn net.Conn
|
||||
var arch Compression
|
||||
var arch string
|
||||
if len(proxys) > 0 {
|
||||
comp := proxys[len(proxys)-1]
|
||||
if comp.Compression != "" {
|
||||
arch, err = NewCompression(comp.Compression)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
arch = proxys[len(proxys)-1].Compression
|
||||
_, rawNetConn, err = obj.dialer.DialProxyContext(ctx, "tcp", ctx.option.TlsConfig.Clone(), append(proxys, remoteAddress)...)
|
||||
} else {
|
||||
var remoteAddress Address
|
||||
@@ -264,7 +258,7 @@ func (obj *roundTripper) dial(ctx *Response) (conn http1.Conn, err error) {
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
if arch != nil {
|
||||
if arch != "" {
|
||||
rawConn, err = NewCompressionConn(rawConn, arch)
|
||||
}
|
||||
if err != nil {
|
||||
|
||||
52
snappy.go
52
snappy.go
@@ -1,52 +0,0 @@
|
||||
package requests
|
||||
|
||||
import (
|
||||
"io"
|
||||
|
||||
"github.com/golang/snappy"
|
||||
)
|
||||
|
||||
// // 定义 snappy.Writer 池,包装 io.Writer
|
||||
// var snappyWriterPool = sync.Pool{
|
||||
// New: func() interface{} {
|
||||
// // 先给一个空 buffer,后面可以Reset替换输出目标
|
||||
// return snappy.NewBufferedWriter(nil)
|
||||
// },
|
||||
// }
|
||||
|
||||
// // 定义 snappy.Reader 池,包装 io.Reader
|
||||
// var snappyReaderPool = sync.Pool{
|
||||
// New: func() interface{} {
|
||||
// // 先给一个空 reader,后面可以Reset替换输入来源
|
||||
// return snappy.NewReader(nil)
|
||||
// },
|
||||
// }
|
||||
|
||||
// 获取并初始化 snappy.Writer
|
||||
func getSnappyWriter(w io.Writer) *snappy.Writer {
|
||||
return snappy.NewBufferedWriter(w)
|
||||
// sw := snappyWriterPool.Get().(*snappy.Writer)
|
||||
// sw.Reset(w)
|
||||
// return sw
|
||||
}
|
||||
|
||||
// 获取并初始化 snappy.Reader
|
||||
func getSnappyReader(r io.Reader) *snappy.Reader {
|
||||
return snappy.NewReader(r)
|
||||
// sr := snappyReaderPool.Get().(*snappy.Reader)
|
||||
// sr.Reset(r)
|
||||
// return sr
|
||||
}
|
||||
|
||||
// // 释放 snappy.Writer
|
||||
// func putSnappyWriter(sw *snappy.Writer) {
|
||||
// sw.Close()
|
||||
// sw.Reset(nil)
|
||||
// snappyWriterPool.Put(sw)
|
||||
// }
|
||||
|
||||
// // 释放 snappy.Reader
|
||||
// func putSnappyReader(sr *snappy.Reader) {
|
||||
// sr.Reset(nil)
|
||||
// snappyReaderPool.Put(sr)
|
||||
// }
|
||||
@@ -2,6 +2,7 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"testing"
|
||||
|
||||
"github.com/gospider007/requests"
|
||||
@@ -20,6 +21,7 @@ func TestRedirectCallBack(t *testing.T) {
|
||||
t.Error(err)
|
||||
}
|
||||
if response.StatusCode() != 302 {
|
||||
log.Print(response.StatusCode())
|
||||
t.Error("redirect failed")
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user