diff --git a/README.md b/README.md index 91799b9..981e72c 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,10 @@

Requests is a fully featured HTTP client library for Golang. Network requests can be completed with just a few lines of code. Unified support for http1, http2, http3, websocket, sse, utls, uquic

+ +## If you want to use a multilingual version, please use the project at [fingerproxy](https://github.com/gospider007/fingerproxy). It is a crawler-focused forward proxy with fingerprint spoofing. + + ## Innovative Features * Unlimited chained proxy * HTTP/3 fingerprint spoofing protection diff --git a/compressConn.go b/compressConn.go deleted file mode 100644 index 320e694..0000000 --- a/compressConn.go +++ /dev/null @@ -1,213 +0,0 @@ -package requests - -import ( - "errors" - "io" - "net" - "strings" - "sync" - "time" -) - -type CompressionConn struct { - conn net.Conn - w io.WriteCloser - r io.ReadCloser -} -type Compression interface { - String() string - OpenReader(r io.Reader) (io.ReadCloser, error) - OpenWriter(w io.Writer) (io.WriteCloser, error) -} -type compression struct { - name string - openReader func(r io.Reader) (io.ReadCloser, error) - openWriter func(w io.Writer) (io.WriteCloser, error) -} - -func (obj compression) String() string { - return obj.name -} -func (obj compression) OpenReader(r io.Reader) (io.ReadCloser, error) { - return obj.openReader(r) -} -func (obj compression) OpenWriter(w io.Writer) (io.WriteCloser, error) { - return obj.openWriter(w) -} - -func GetCompressionByte(decode string) (byte, error) { - switch strings.ToLower(decode) { - case "zstd": - return 40, nil - case "s2": - return 255, nil - case "flate": - return 92, nil - case "minlz": - return 93, nil - default: - return 0, errors.New("unsupported compression type") - } -} - -func NewCompressionWithByte(b byte) (Compression, error) { - c, ok := compressionData[b] - if !ok { - return nil, errors.New("unsupported compression type") - } - return compression{ - name: c.name, - openReader: func(r io.Reader) (io.ReadCloser, error) { - buf := make([]byte, 1) - n, err := r.Read(buf) - if err != nil { - return nil, err - } - if n != 1 || buf[0] != b { - return nil, errors.New("invalid response") - } - return c.openReader(r) - }, - openWriter: func(w io.Writer) (io.WriteCloser, error) { - n, err := w.Write([]byte{b}) - if err != nil { - return nil, err - } - if n != 1 { - return nil, errors.New("invalid response") - } - return c.openWriter(w) - }, - }, nil -} -func NewCompression(decode string) (Compression, error) { - decode = strings.ToLower(decode) - for b, c := range compressionData { - if c.name == decode { - return NewCompressionWithByte(b) - } - } - return nil, errors.New("unsupported compression type") -} - -func NewCompressionConn(conn net.Conn, decode string) (net.Conn, error) { - arch, err := NewCompression(decode) - if err != nil { - return conn, err - } - w, err := arch.OpenWriter(conn) - if err != nil { - return conn, err - } - r, err := arch.OpenReader(conn) - if err != nil { - return conn, err - } - ccon := &CompressionConn{ - conn: conn, - r: r, - w: w, - } - return ccon, nil -} -func (obj *CompressionConn) Read(b []byte) (n int, err error) { - return obj.r.Read(b) -} -func (obj *CompressionConn) Write(b []byte) (n int, err error) { - return obj.w.Write(b) -} -func (obj *CompressionConn) Close() error { - err := obj.conn.Close() - obj.w.Close() - obj.r.Close() - return err -} - -func (obj *CompressionConn) LocalAddr() net.Addr { - return obj.conn.LocalAddr() -} -func (obj *CompressionConn) RemoteAddr() net.Addr { - return obj.conn.RemoteAddr() -} -func (obj *CompressionConn) SetDeadline(t time.Time) error { - return obj.conn.SetDeadline(t) -} - -func (obj *CompressionConn) SetReadDeadline(t time.Time) error { - return obj.conn.SetReadDeadline(t) -} -func (obj *CompressionConn) SetWriteDeadline(t time.Time) error { - return obj.conn.SetWriteDeadline(t) -} - -type ReaderCompression struct { - c io.ReadCloser - closed bool - lock sync.Mutex - closeFunc func() -} - -func (obj *ReaderCompression) Read(p []byte) (n int, err error) { - obj.lock.Lock() - defer obj.lock.Unlock() - if obj.closed { - return 0, errors.New("read closed") - } - n, err = obj.c.Read(p) - return -} -func (obj *ReaderCompression) Close() error { - obj.lock.Lock() - defer obj.lock.Unlock() - if obj.closed { - return nil - } - obj.closed = true - obj.c.Close() - obj.closeFunc() - return nil -} - -type WriterCompression struct { - c io.WriteCloser - closed bool - lock sync.Mutex - flush interface{ Flush() error } - closeFunc func() -} - -func (obj *WriterCompression) Write(p []byte) (int, error) { - obj.lock.Lock() - defer obj.lock.Unlock() - if obj.closed { - return 0, errors.New("write closed") - } - n, err := obj.c.Write(p) - if err != nil { - return n, err - } - if obj.flush != nil { - err = obj.flush.Flush() - } - return n, err -} - -func (obj *WriterCompression) Close() error { - obj.lock.Lock() - defer obj.lock.Unlock() - if obj.closed { - return nil - } - obj.closed = true - obj.c.Close() - obj.closeFunc() - return nil -} - -func newWriterCompression(c io.WriteCloser, closeFunc func()) *WriterCompression { - flush, _ := c.(interface{ Flush() error }) - return &WriterCompression{c: c, closeFunc: closeFunc, flush: flush} -} -func newReaderCompression(c io.ReadCloser, closeFunc func()) *ReaderCompression { - return &ReaderCompression{c: c, closeFunc: closeFunc} -} diff --git a/compressionPool.go b/compressionPool.go deleted file mode 100644 index 76ac6f7..0000000 --- a/compressionPool.go +++ /dev/null @@ -1,200 +0,0 @@ -package requests - -import ( - "compress/flate" - "io" - "sync" - - "github.com/golang/snappy" - "github.com/klauspost/compress/zstd" - "github.com/minio/minlz" -) - -type compreData struct { - rpool *sync.Pool - wpool *sync.Pool - name string - openReader func(r io.Reader) (io.ReadCloser, error) - openWriter func(w io.Writer) (io.WriteCloser, error) -} - -var compressionData map[byte]compreData - -func init() { - compressionData = map[byte]compreData{ - 40: { - name: "zstd", - rpool: &sync.Pool{New: func() any { return nil }}, - wpool: &sync.Pool{New: func() any { return nil }}, - openReader: newZstdReader, - openWriter: newZstdWriter, - }, - 255: { - name: "s2", - rpool: &sync.Pool{New: func() any { return nil }}, - wpool: &sync.Pool{New: func() any { return nil }}, - openReader: newSnappyReader, - openWriter: newSnappyWriter, - }, - 92: { - name: "flate", - rpool: &sync.Pool{New: func() any { return nil }}, - wpool: &sync.Pool{New: func() any { return nil }}, - openReader: newFlateReader, - openWriter: newFlateWriter, - }, - 93: { - name: "minlz", - rpool: &sync.Pool{New: func() any { return nil }}, - wpool: &sync.Pool{New: func() any { return nil }}, - openReader: newMinlzReader, - openWriter: newMinlzWriter, - }, - } -} - -func newZstdWriter(w io.Writer) (io.WriteCloser, error) { - pool := compressionData[40].wpool - cp := pool.Get() - var z *zstd.Encoder - var err error - if cp == nil { - z, err = zstd.NewWriter(w, zstd.WithWindowSize(32*1024)) - } else { - z = cp.(*zstd.Encoder) - z.Reset(w) - } - if err != nil { - return nil, err - } - return newWriterCompression(z, func() { - z.Reset(nil) - pool.Put(z) - }), nil -} -func newZstdReader(w io.Reader) (io.ReadCloser, error) { - pool := compressionData[40].rpool - cp := pool.Get() - var z *zstd.Decoder - var err error - if cp == nil { - z, err = zstd.NewReader(w) - } else { - z = cp.(*zstd.Decoder) - z.Reset(w) - } - if err != nil { - return nil, err - } - return newReaderCompression(io.NopCloser(z), func() { - z.Reset(nil) - pool.Put(z) - }), nil -} - -// snappy pool - -func newSnappyWriter(w io.Writer) (io.WriteCloser, error) { - pool := compressionData[255].wpool - cp := pool.Get() - var z *snappy.Writer - if cp == nil { - z = snappy.NewBufferedWriter(w) - } else { - z = cp.(*snappy.Writer) - z.Reset(w) - } - return newWriterCompression(z, func() { - z.Reset(nil) - pool.Put(z) - }), nil -} -func newSnappyReader(w io.Reader) (io.ReadCloser, error) { - pool := compressionData[255].rpool - cp := pool.Get() - var z *snappy.Reader - if cp == nil { - z = snappy.NewReader(w) - } else { - z = cp.(*snappy.Reader) - z.Reset(w) - } - return newReaderCompression(io.NopCloser(z), func() { - z.Reset(nil) - pool.Put(z) - }), nil -} - -// flate pool -func newFlateWriter(w io.Writer) (io.WriteCloser, error) { - pool := compressionData[92].wpool - cp := pool.Get() - var z *flate.Writer - var err error - if cp == nil { - z, err = flate.NewWriter(w, flate.DefaultCompression) - } else { - z = cp.(*flate.Writer) - z.Reset(w) - } - if err != nil { - return nil, err - } - return newWriterCompression(z, func() { - z.Reset(nil) - pool.Put(z) - }), nil -} - -func newFlateReader(w io.Reader) (io.ReadCloser, error) { - pool := compressionData[92].rpool - cp := pool.Get() - var z io.ReadCloser - var f flate.Resetter - if cp == nil { - z = flate.NewReader(w) - f = z.(flate.Resetter) - } else { - z = cp.(io.ReadCloser) - f = z.(flate.Resetter) - f.Reset(w, nil) - } - return newReaderCompression(z, func() { - f.Reset(nil, nil) - pool.Put(z) - }), nil -} - -// minlz pool - -func newMinlzWriter(w io.Writer) (io.WriteCloser, error) { - pool := compressionData[93].wpool - cp := pool.Get() - var z *minlz.Writer - if cp == nil { - z = minlz.NewWriter(w, minlz.WriterBlockSize(32*1024)) - } else { - z = cp.(*minlz.Writer) - z.Reset(w) - } - return newWriterCompression(z, func() { - z.Reset(nil) - pool.Put(z) - }), nil -} - -func newMinlzReader(w io.Reader) (io.ReadCloser, error) { - pool := compressionData[93].rpool - cp := pool.Get() - var z *minlz.Reader - if cp == nil { - z = minlz.NewReader(w, minlz.ReaderMaxBlockSize(32*1024)) - } else { - z = cp.(*minlz.Reader) - z.Reset(w) - } - return newReaderCompression(io.NopCloser(z), func() { - z.Reset(nil) - pool.Put(z) - }), nil -} diff --git a/conn.go b/conn.go index d1a60a9..7179387 100644 --- a/conn.go +++ b/conn.go @@ -23,20 +23,18 @@ func taskMain(conn http1.Conn, task *reqTask) (err error) { if err != nil { task.cnl(err) } else { - var bodyContext context.Context - if task.reqCtx.response != nil && task.reqCtx.response.Body != nil { - bodyContext = task.reqCtx.response.Body.(*http1.Body).Context() - } task.cnl(tools.ErrNoErr) - if bodyContext != nil { - select { - case <-task.reqCtx.Context().Done(): - if context.Cause(task.reqCtx.Context()) != tools.ErrNoErr { - err = context.Cause(task.reqCtx.Context()) - } - case <-bodyContext.Done(): - if context.Cause(bodyContext) != tools.ErrNoErr { - err = context.Cause(bodyContext) + if task.reqCtx.response != nil && task.reqCtx.response.Body != nil { + if bodyContext := task.reqCtx.response.Body.(*http1.Body).Context(); bodyContext != nil { + select { + case <-task.reqCtx.Context().Done(): + if context.Cause(task.reqCtx.Context()) != tools.ErrNoErr { + err = context.Cause(task.reqCtx.Context()) + } + case <-bodyContext.Done(): + if context.Cause(bodyContext) != tools.ErrNoErr { + err = context.Cause(bodyContext) + } } } } diff --git a/go.mod b/go.mod index 9579f6d..a40ee31 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,6 @@ module github.com/gospider007/requests go 1.25 require ( - github.com/golang/snappy v1.0.0 github.com/gorilla/websocket v1.5.3 github.com/gospider007/bar v0.0.0-20250815030902-4f5b5d6312cf github.com/gospider007/bs4 v0.0.0-20250815030800-a352d3ad57ee @@ -14,10 +13,8 @@ require ( github.com/gospider007/http3 v0.0.0-20250817123336-07d66db6dbb3 github.com/gospider007/ja3 v0.0.0-20250815031055-0948dc3bbe0b github.com/gospider007/re v0.0.0-20250815031101-a57caeff73bf - github.com/gospider007/tools v0.0.0-20250815031258-8a81d680917c - github.com/gospider007/websocket v0.0.0-20250815031315-cde1470d5ace - github.com/klauspost/compress v1.18.0 - github.com/minio/minlz v1.0.1 + github.com/gospider007/tools v0.0.0-20250819094836-a81233312764 + github.com/gospider007/websocket v0.0.0-20250819094917-c00c0a99815f github.com/quic-go/quic-go v0.54.0 github.com/refraction-networking/uquic v0.0.6 github.com/refraction-networking/utls v1.8.0 @@ -45,12 +42,14 @@ require ( github.com/gobwas/httphead v0.1.0 // indirect github.com/gobwas/pool v0.2.1 // indirect github.com/gobwas/ws v1.4.0 // indirect + github.com/golang/snappy v1.0.0 // indirect github.com/google/gopacket v1.1.19 // indirect github.com/google/pprof v0.0.0-20250630185457-6e76a2b096b5 // indirect github.com/gospider007/blog v0.0.0-20250815030743-f2af6b9013ab // indirect github.com/gospider007/kinds v0.0.0-20250815031133-b2282666f69c // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.18.0 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/klauspost/pgzip v1.2.6 // indirect github.com/kr/pretty v0.3.1 // indirect @@ -59,6 +58,7 @@ require ( github.com/mholt/archives v0.1.3 // indirect github.com/miekg/dns v1.1.68 // indirect github.com/mikelolasagasti/xz v1.0.1 // indirect + github.com/minio/minlz v1.0.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/nwaples/rardecode/v2 v2.1.1 // indirect @@ -89,4 +89,5 @@ require ( golang.org/x/sys v0.35.0 // indirect golang.org/x/text v0.28.0 // indirect golang.org/x/tools v0.36.0 // indirect + google.golang.org/protobuf v1.36.7 // indirect ) diff --git a/go.sum b/go.sum index 572b2d2..c4e9915 100644 --- a/go.sum +++ b/go.sum @@ -128,10 +128,10 @@ github.com/gospider007/kinds v0.0.0-20250815031133-b2282666f69c h1:WnFZf0v+Du5nH github.com/gospider007/kinds v0.0.0-20250815031133-b2282666f69c/go.mod h1:8eAmhxyW7p81nsYlD5Wj8I0HuZpVGci+Q9wr6vfY8pg= github.com/gospider007/re v0.0.0-20250815031101-a57caeff73bf h1:+3g9qmg+XLbYcY4vvbsDM7xHkXF+5U8V/haAYDImGTA= github.com/gospider007/re v0.0.0-20250815031101-a57caeff73bf/go.mod h1:5fVj9OYkqlOgBdz+kH4jgdiSZ6jdfbmexLIFqtMa9v8= -github.com/gospider007/tools v0.0.0-20250815031258-8a81d680917c h1:2NOEZaRbICBWs07Hs9hm4wPhXf+zTtoYnZV2aTcuvRw= -github.com/gospider007/tools v0.0.0-20250815031258-8a81d680917c/go.mod h1:hT2nPJQoUL5nwturslhBjykigIA27qxujkLGSEPWb3o= -github.com/gospider007/websocket v0.0.0-20250815031315-cde1470d5ace h1:phCUKTZU8SNtLjEZfK7SAFWenemHD10FH6MU8M5aeVg= -github.com/gospider007/websocket v0.0.0-20250815031315-cde1470d5ace/go.mod h1:cj1g1pNi5VOR+aed4Q37UEoEla+EgrmMNoTi0f5EYUA= +github.com/gospider007/tools v0.0.0-20250819094836-a81233312764 h1:NYVqv0I6pE7yV5Enee9VOWmqjgJFZqQT66p+yZ1CwO4= +github.com/gospider007/tools v0.0.0-20250819094836-a81233312764/go.mod h1:Imryilbsdhsx+zIE+s6mIB+DNrAIig0ixupk45exsts= +github.com/gospider007/websocket v0.0.0-20250819094917-c00c0a99815f h1:nyklmiKQ1vLOmC2DCPL8/RZHN22or/CFRi0xCGTiZiU= +github.com/gospider007/websocket v0.0.0-20250819094917-c00c0a99815f/go.mod h1:olruUjPKXJ74OlFipbKedfK0LMuDu/UarSWIVhEtOX4= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= @@ -485,8 +485,8 @@ google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= -google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= +google.golang.org/protobuf v1.36.7 h1:IgrO7UwFQGJdRNXH/sQux4R1Dj1WAKcLElzeeRaXV2A= +google.golang.org/protobuf v1.36.7/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= diff --git a/requests.go b/requests.go index 7b4a145..fb5d537 100644 --- a/requests.go +++ b/requests.go @@ -349,19 +349,20 @@ func (obj *Client) request(ctx *Response) (err error) { err = errors.New("send req response is nil") return } - if ctx.response.Body != nil { - ctx.rawBody = ctx.response.Body.(*http1.Body) - } + ctx.rawBody = ctx.response.Body.(*http1.Body) if encoding := ctx.ContentEncoding(); encoding != "" && ctx.response.Body != nil { - var unCompressionBody io.ReadCloser - unCompressionBody, err = tools.CompressionHeadersDecode(ctx.Context(), ctx.response.Body, encoding) - if err != nil { - if err != io.ErrUnexpectedEOF && err != io.EOF { - return + arch, cerr := tools.NewRawCompression(encoding) + if cerr != nil { + return cerr + } + unCompressionBody, cerr := arch.OpenReader(ctx.rawBody.GetReader()) + if cerr != nil { + if cerr != io.ErrUnexpectedEOF && cerr != io.EOF { + return cerr } } if unCompressionBody != nil { - ctx.response.Body = unCompressionBody + ctx.rawBody.SetReader(unCompressionBody) } } if strings.Contains(ctx.response.Header.Get("Content-Type"), "text/event-stream") { diff --git a/roundTripper.go b/roundTripper.go index bbfd073..28e7b34 100644 --- a/roundTripper.go +++ b/roundTripper.go @@ -215,7 +215,7 @@ func (obj *roundTripper) dial(ctx *Response) (conn http1.Conn, err error) { rawConn = rawNetConn } if arch != "" { - rawConn, err = NewCompressionConn(rawConn, arch) + rawConn, err = tools.NewCompressionConn(rawConn, arch) } if err != nil { if rawConn != nil { diff --git a/socks_js.go b/socks_js.go new file mode 100644 index 0000000..2c7f86a --- /dev/null +++ b/socks_js.go @@ -0,0 +1,11 @@ +//go:build js + +package requests + +import ( + "syscall" +) + +func Control(network, address string, c syscall.RawConn) error { + return c.Control(func(fd uintptr) {}) +}