sync

README.md

@@ -19,6 +19,10 @@
 
 <h2 align="center">Requests is a fully featured HTTP client library for Golang. Network requests can be completed with just a few lines of code. Unified support for http1, http2, http3, websocket, sse, utls, uquic</h2>
 
+
+## If you want to use a multilingual version, please use the project at [fingerproxy](https://github.com/gospider007/fingerproxy). It is a crawler-focused forward proxy with fingerprint spoofing.
+
+
 ## Innovative Features
 * Unlimited chained proxy
 * HTTP/3 fingerprint spoofing protection

compressConn.go

@@ -1,213 +0,0 @@
-package requests
-
-import (
-	"errors"
-	"io"
-	"net"
-	"strings"
-	"sync"
-	"time"
-)
-
-type CompressionConn struct {
-	conn net.Conn
-	w    io.WriteCloser
-	r    io.ReadCloser
-}
-type Compression interface {
-	String() string
-	OpenReader(r io.Reader) (io.ReadCloser, error)
-	OpenWriter(w io.Writer) (io.WriteCloser, error)
-}
-type compression struct {
-	name       string
-	openReader func(r io.Reader) (io.ReadCloser, error)
-	openWriter func(w io.Writer) (io.WriteCloser, error)
-}
-
-func (obj compression) String() string {
-	return obj.name
-}
-func (obj compression) OpenReader(r io.Reader) (io.ReadCloser, error) {
-	return obj.openReader(r)
-}
-func (obj compression) OpenWriter(w io.Writer) (io.WriteCloser, error) {
-	return obj.openWriter(w)
-}
-
-func GetCompressionByte(decode string) (byte, error) {
-	switch strings.ToLower(decode) {
-	case "zstd":
-		return 40, nil
-	case "s2":
-		return 255, nil
-	case "flate":
-		return 92, nil
-	case "minlz":
-		return 93, nil
-	default:
-		return 0, errors.New("unsupported compression type")
-	}
-}
-
-func NewCompressionWithByte(b byte) (Compression, error) {
-	c, ok := compressionData[b]
-	if !ok {
-		return nil, errors.New("unsupported compression type")
-	}
-	return compression{
-		name: c.name,
-		openReader: func(r io.Reader) (io.ReadCloser, error) {
-			buf := make([]byte, 1)
-			n, err := r.Read(buf)
-			if err != nil {
-				return nil, err
-			}
-			if n != 1 || buf[0] != b {
-				return nil, errors.New("invalid response")
-			}
-			return c.openReader(r)
-		},
-		openWriter: func(w io.Writer) (io.WriteCloser, error) {
-			n, err := w.Write([]byte{b})
-			if err != nil {
-				return nil, err
-			}
-			if n != 1 {
-				return nil, errors.New("invalid response")
-			}
-			return c.openWriter(w)
-		},
-	}, nil
-}
-func NewCompression(decode string) (Compression, error) {
-	decode = strings.ToLower(decode)
-	for b, c := range compressionData {
-		if c.name == decode {
-			return NewCompressionWithByte(b)
-		}
-	}
-	return nil, errors.New("unsupported compression type")
-}
-
-func NewCompressionConn(conn net.Conn, decode string) (net.Conn, error) {
-	arch, err := NewCompression(decode)
-	if err != nil {
-		return conn, err
-	}
-	w, err := arch.OpenWriter(conn)
-	if err != nil {
-		return conn, err
-	}
-	r, err := arch.OpenReader(conn)
-	if err != nil {
-		return conn, err
-	}
-	ccon := &CompressionConn{
-		conn: conn,
-		r:    r,
-		w:    w,
-	}
-	return ccon, nil
-}
-func (obj *CompressionConn) Read(b []byte) (n int, err error) {
-	return obj.r.Read(b)
-}
-func (obj *CompressionConn) Write(b []byte) (n int, err error) {
-	return obj.w.Write(b)
-}
-func (obj *CompressionConn) Close() error {
-	err := obj.conn.Close()
-	obj.w.Close()
-	obj.r.Close()
-	return err
-}
-
-func (obj *CompressionConn) LocalAddr() net.Addr {
-	return obj.conn.LocalAddr()
-}
-func (obj *CompressionConn) RemoteAddr() net.Addr {
-	return obj.conn.RemoteAddr()
-}
-func (obj *CompressionConn) SetDeadline(t time.Time) error {
-	return obj.conn.SetDeadline(t)
-}
-
-func (obj *CompressionConn) SetReadDeadline(t time.Time) error {
-	return obj.conn.SetReadDeadline(t)
-}
-func (obj *CompressionConn) SetWriteDeadline(t time.Time) error {
-	return obj.conn.SetWriteDeadline(t)
-}
-
-type ReaderCompression struct {
-	c         io.ReadCloser
-	closed    bool
-	lock      sync.Mutex
-	closeFunc func()
-}
-
-func (obj *ReaderCompression) Read(p []byte) (n int, err error) {
-	obj.lock.Lock()
-	defer obj.lock.Unlock()
-	if obj.closed {
-		return 0, errors.New("read closed")
-	}
-	n, err = obj.c.Read(p)
-	return
-}
-func (obj *ReaderCompression) Close() error {
-	obj.lock.Lock()
-	defer obj.lock.Unlock()
-	if obj.closed {
-		return nil
-	}
-	obj.closed = true
-	obj.c.Close()
-	obj.closeFunc()
-	return nil
-}
-
-type WriterCompression struct {
-	c         io.WriteCloser
-	closed    bool
-	lock      sync.Mutex
-	flush     interface{ Flush() error }
-	closeFunc func()
-}
-
-func (obj *WriterCompression) Write(p []byte) (int, error) {
-	obj.lock.Lock()
-	defer obj.lock.Unlock()
-	if obj.closed {
-		return 0, errors.New("write closed")
-	}
-	n, err := obj.c.Write(p)
-	if err != nil {
-		return n, err
-	}
-	if obj.flush != nil {
-		err = obj.flush.Flush()
-	}
-	return n, err
-}
-
-func (obj *WriterCompression) Close() error {
-	obj.lock.Lock()
-	defer obj.lock.Unlock()
-	if obj.closed {
-		return nil
-	}
-	obj.closed = true
-	obj.c.Close()
-	obj.closeFunc()
-	return nil
-}
-
-func newWriterCompression(c io.WriteCloser, closeFunc func()) *WriterCompression {
-	flush, _ := c.(interface{ Flush() error })
-	return &WriterCompression{c: c, closeFunc: closeFunc, flush: flush}
-}
-func newReaderCompression(c io.ReadCloser, closeFunc func()) *ReaderCompression {
-	return &ReaderCompression{c: c, closeFunc: closeFunc}
-}

compressionPool.go

@@ -1,200 +0,0 @@
-package requests
-
-import (
-	"compress/flate"
-	"io"
-	"sync"
-
-	"github.com/golang/snappy"
-	"github.com/klauspost/compress/zstd"
-	"github.com/minio/minlz"
-)
-
-type compreData struct {
-	rpool      *sync.Pool
-	wpool      *sync.Pool
-	name       string
-	openReader func(r io.Reader) (io.ReadCloser, error)
-	openWriter func(w io.Writer) (io.WriteCloser, error)
-}
-
-var compressionData map[byte]compreData
-
-func init() {
-	compressionData = map[byte]compreData{
-		40: {
-			name:       "zstd",
-			rpool:      &sync.Pool{New: func() any { return nil }},
-			wpool:      &sync.Pool{New: func() any { return nil }},
-			openReader: newZstdReader,
-			openWriter: newZstdWriter,
-		},
-		255: {
-			name:       "s2",
-			rpool:      &sync.Pool{New: func() any { return nil }},
-			wpool:      &sync.Pool{New: func() any { return nil }},
-			openReader: newSnappyReader,
-			openWriter: newSnappyWriter,
-		},
-		92: {
-			name:       "flate",
-			rpool:      &sync.Pool{New: func() any { return nil }},
-			wpool:      &sync.Pool{New: func() any { return nil }},
-			openReader: newFlateReader,
-			openWriter: newFlateWriter,
-		},
-		93: {
-			name:       "minlz",
-			rpool:      &sync.Pool{New: func() any { return nil }},
-			wpool:      &sync.Pool{New: func() any { return nil }},
-			openReader: newMinlzReader,
-			openWriter: newMinlzWriter,
-		},
-	}
-}
-
-func newZstdWriter(w io.Writer) (io.WriteCloser, error) {
-	pool := compressionData[40].wpool
-	cp := pool.Get()
-	var z *zstd.Encoder
-	var err error
-	if cp == nil {
-		z, err = zstd.NewWriter(w, zstd.WithWindowSize(32*1024))
-	} else {
-		z = cp.(*zstd.Encoder)
-		z.Reset(w)
-	}
-	if err != nil {
-		return nil, err
-	}
-	return newWriterCompression(z, func() {
-		z.Reset(nil)
-		pool.Put(z)
-	}), nil
-}
-func newZstdReader(w io.Reader) (io.ReadCloser, error) {
-	pool := compressionData[40].rpool
-	cp := pool.Get()
-	var z *zstd.Decoder
-	var err error
-	if cp == nil {
-		z, err = zstd.NewReader(w)
-	} else {
-		z = cp.(*zstd.Decoder)
-		z.Reset(w)
-	}
-	if err != nil {
-		return nil, err
-	}
-	return newReaderCompression(io.NopCloser(z), func() {
-		z.Reset(nil)
-		pool.Put(z)
-	}), nil
-}
-
-// snappy pool
-
-func newSnappyWriter(w io.Writer) (io.WriteCloser, error) {
-	pool := compressionData[255].wpool
-	cp := pool.Get()
-	var z *snappy.Writer
-	if cp == nil {
-		z = snappy.NewBufferedWriter(w)
-	} else {
-		z = cp.(*snappy.Writer)
-		z.Reset(w)
-	}
-	return newWriterCompression(z, func() {
-		z.Reset(nil)
-		pool.Put(z)
-	}), nil
-}
-func newSnappyReader(w io.Reader) (io.ReadCloser, error) {
-	pool := compressionData[255].rpool
-	cp := pool.Get()
-	var z *snappy.Reader
-	if cp == nil {
-		z = snappy.NewReader(w)
-	} else {
-		z = cp.(*snappy.Reader)
-		z.Reset(w)
-	}
-	return newReaderCompression(io.NopCloser(z), func() {
-		z.Reset(nil)
-		pool.Put(z)
-	}), nil
-}
-
-// flate pool
-func newFlateWriter(w io.Writer) (io.WriteCloser, error) {
-	pool := compressionData[92].wpool
-	cp := pool.Get()
-	var z *flate.Writer
-	var err error
-	if cp == nil {
-		z, err = flate.NewWriter(w, flate.DefaultCompression)
-	} else {
-		z = cp.(*flate.Writer)
-		z.Reset(w)
-	}
-	if err != nil {
-		return nil, err
-	}
-	return newWriterCompression(z, func() {
-		z.Reset(nil)
-		pool.Put(z)
-	}), nil
-}
-
-func newFlateReader(w io.Reader) (io.ReadCloser, error) {
-	pool := compressionData[92].rpool
-	cp := pool.Get()
-	var z io.ReadCloser
-	var f flate.Resetter
-	if cp == nil {
-		z = flate.NewReader(w)
-		f = z.(flate.Resetter)
-	} else {
-		z = cp.(io.ReadCloser)
-		f = z.(flate.Resetter)
-		f.Reset(w, nil)
-	}
-	return newReaderCompression(z, func() {
-		f.Reset(nil, nil)
-		pool.Put(z)
-	}), nil
-}
-
-// minlz pool
-
-func newMinlzWriter(w io.Writer) (io.WriteCloser, error) {
-	pool := compressionData[93].wpool
-	cp := pool.Get()
-	var z *minlz.Writer
-	if cp == nil {
-		z = minlz.NewWriter(w, minlz.WriterBlockSize(32*1024))
-	} else {
-		z = cp.(*minlz.Writer)
-		z.Reset(w)
-	}
-	return newWriterCompression(z, func() {
-		z.Reset(nil)
-		pool.Put(z)
-	}), nil
-}
-
-func newMinlzReader(w io.Reader) (io.ReadCloser, error) {
-	pool := compressionData[93].rpool
-	cp := pool.Get()
-	var z *minlz.Reader
-	if cp == nil {
-		z = minlz.NewReader(w, minlz.ReaderMaxBlockSize(32*1024))
-	} else {
-		z = cp.(*minlz.Reader)
-		z.Reset(w)
-	}
-	return newReaderCompression(io.NopCloser(z), func() {
-		z.Reset(nil)
-		pool.Put(z)
-	}), nil
-}

conn.go

@@ -23,20 +23,18 @@ func taskMain(conn http1.Conn, task *reqTask) (err error) {
 		if err != nil {
 			task.cnl(err)
 		} else {
-			var bodyContext context.Context
-			if task.reqCtx.response != nil && task.reqCtx.response.Body != nil {
-				bodyContext = task.reqCtx.response.Body.(*http1.Body).Context()
-			}
 			task.cnl(tools.ErrNoErr)
-			if bodyContext != nil {
-				select {
-				case <-task.reqCtx.Context().Done():
-					if context.Cause(task.reqCtx.Context()) != tools.ErrNoErr {
-						err = context.Cause(task.reqCtx.Context())
-					}
-				case <-bodyContext.Done():
-					if context.Cause(bodyContext) != tools.ErrNoErr {
-						err = context.Cause(bodyContext)
+			if task.reqCtx.response != nil && task.reqCtx.response.Body != nil {
+				if bodyContext := task.reqCtx.response.Body.(*http1.Body).Context(); bodyContext != nil {
+					select {
+					case <-task.reqCtx.Context().Done():
+						if context.Cause(task.reqCtx.Context()) != tools.ErrNoErr {
+							err = context.Cause(task.reqCtx.Context())
+						}
+					case <-bodyContext.Done():
+						if context.Cause(bodyContext) != tools.ErrNoErr {
+							err = context.Cause(bodyContext)
+						}
 					}
 				}
 			}

go.mod

@@ -3,7 +3,6 @@ module github.com/gospider007/requests
 go 1.25
 
 require (
-	github.com/golang/snappy v1.0.0
 	github.com/gorilla/websocket v1.5.3
 	github.com/gospider007/bar v0.0.0-20250815030902-4f5b5d6312cf
 	github.com/gospider007/bs4 v0.0.0-20250815030800-a352d3ad57ee
@@ -14,10 +13,8 @@ require (
 	github.com/gospider007/http3 v0.0.0-20250817123336-07d66db6dbb3
 	github.com/gospider007/ja3 v0.0.0-20250815031055-0948dc3bbe0b
 	github.com/gospider007/re v0.0.0-20250815031101-a57caeff73bf
-	github.com/gospider007/tools v0.0.0-20250815031258-8a81d680917c
-	github.com/gospider007/websocket v0.0.0-20250815031315-cde1470d5ace
-	github.com/klauspost/compress v1.18.0
-	github.com/minio/minlz v1.0.1
+	github.com/gospider007/tools v0.0.0-20250819094836-a81233312764
+	github.com/gospider007/websocket v0.0.0-20250819094917-c00c0a99815f
 	github.com/quic-go/quic-go v0.54.0
 	github.com/refraction-networking/uquic v0.0.6
 	github.com/refraction-networking/utls v1.8.0
@@ -45,12 +42,14 @@ require (
 	github.com/gobwas/httphead v0.1.0 // indirect
 	github.com/gobwas/pool v0.2.1 // indirect
 	github.com/gobwas/ws v1.4.0 // indirect
+	github.com/golang/snappy v1.0.0 // indirect
 	github.com/google/gopacket v1.1.19 // indirect
 	github.com/google/pprof v0.0.0-20250630185457-6e76a2b096b5 // indirect
 	github.com/gospider007/blog v0.0.0-20250815030743-f2af6b9013ab // indirect
 	github.com/gospider007/kinds v0.0.0-20250815031133-b2282666f69c // indirect
 	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/klauspost/compress v1.18.0 // indirect
 	github.com/klauspost/cpuid/v2 v2.3.0 // indirect
 	github.com/klauspost/pgzip v1.2.6 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
@@ -59,6 +58,7 @@ require (
 	github.com/mholt/archives v0.1.3 // indirect
 	github.com/miekg/dns v1.1.68 // indirect
 	github.com/mikelolasagasti/xz v1.0.1 // indirect
+	github.com/minio/minlz v1.0.1 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/nwaples/rardecode/v2 v2.1.1 // indirect
@@ -89,4 +89,5 @@ require (
 	golang.org/x/sys v0.35.0 // indirect
 	golang.org/x/text v0.28.0 // indirect
 	golang.org/x/tools v0.36.0 // indirect
+	google.golang.org/protobuf v1.36.7 // indirect
 )

go.sum

@@ -128,10 +128,10 @@ github.com/gospider007/kinds v0.0.0-20250815031133-b2282666f69c h1:WnFZf0v+Du5nH
 github.com/gospider007/kinds v0.0.0-20250815031133-b2282666f69c/go.mod h1:8eAmhxyW7p81nsYlD5Wj8I0HuZpVGci+Q9wr6vfY8pg=
 github.com/gospider007/re v0.0.0-20250815031101-a57caeff73bf h1:+3g9qmg+XLbYcY4vvbsDM7xHkXF+5U8V/haAYDImGTA=
 github.com/gospider007/re v0.0.0-20250815031101-a57caeff73bf/go.mod h1:5fVj9OYkqlOgBdz+kH4jgdiSZ6jdfbmexLIFqtMa9v8=
-github.com/gospider007/tools v0.0.0-20250815031258-8a81d680917c h1:2NOEZaRbICBWs07Hs9hm4wPhXf+zTtoYnZV2aTcuvRw=
-github.com/gospider007/tools v0.0.0-20250815031258-8a81d680917c/go.mod h1:hT2nPJQoUL5nwturslhBjykigIA27qxujkLGSEPWb3o=
-github.com/gospider007/websocket v0.0.0-20250815031315-cde1470d5ace h1:phCUKTZU8SNtLjEZfK7SAFWenemHD10FH6MU8M5aeVg=
-github.com/gospider007/websocket v0.0.0-20250815031315-cde1470d5ace/go.mod h1:cj1g1pNi5VOR+aed4Q37UEoEla+EgrmMNoTi0f5EYUA=
+github.com/gospider007/tools v0.0.0-20250819094836-a81233312764 h1:NYVqv0I6pE7yV5Enee9VOWmqjgJFZqQT66p+yZ1CwO4=
+github.com/gospider007/tools v0.0.0-20250819094836-a81233312764/go.mod h1:Imryilbsdhsx+zIE+s6mIB+DNrAIig0ixupk45exsts=
+github.com/gospider007/websocket v0.0.0-20250819094917-c00c0a99815f h1:nyklmiKQ1vLOmC2DCPL8/RZHN22or/CFRi0xCGTiZiU=
+github.com/gospider007/websocket v0.0.0-20250819094917-c00c0a99815f/go.mod h1:olruUjPKXJ74OlFipbKedfK0LMuDu/UarSWIVhEtOX4=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
@@ -485,8 +485,8 @@ google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac
 google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
 google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
 google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY=
-google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY=
+google.golang.org/protobuf v1.36.7 h1:IgrO7UwFQGJdRNXH/sQux4R1Dj1WAKcLElzeeRaXV2A=
+google.golang.org/protobuf v1.36.7/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=

requests.go

@@ -349,19 +349,20 @@ func (obj *Client) request(ctx *Response) (err error) {
 		err = errors.New("send req response is nil")
 		return
 	}
-	if ctx.response.Body != nil {
-		ctx.rawBody = ctx.response.Body.(*http1.Body)
-	}
+	ctx.rawBody = ctx.response.Body.(*http1.Body)
 	if encoding := ctx.ContentEncoding(); encoding != "" && ctx.response.Body != nil {
-		var unCompressionBody io.ReadCloser
-		unCompressionBody, err = tools.CompressionHeadersDecode(ctx.Context(), ctx.response.Body, encoding)
-		if err != nil {
-			if err != io.ErrUnexpectedEOF && err != io.EOF {
-				return
+		arch, cerr := tools.NewRawCompression(encoding)
+		if cerr != nil {
+			return cerr
+		}
+		unCompressionBody, cerr := arch.OpenReader(ctx.rawBody.GetReader())
+		if cerr != nil {
+			if cerr != io.ErrUnexpectedEOF && cerr != io.EOF {
+				return cerr
 			}
 		}
 		if unCompressionBody != nil {
-			ctx.response.Body = unCompressionBody
+			ctx.rawBody.SetReader(unCompressionBody)
 		}
 	}
 	if strings.Contains(ctx.response.Header.Get("Content-Type"), "text/event-stream") {

roundTripper.go

@@ -215,7 +215,7 @@ func (obj *roundTripper) dial(ctx *Response) (conn http1.Conn, err error) {
 		rawConn = rawNetConn
 	}
 	if arch != "" {
-		rawConn, err = NewCompressionConn(rawConn, arch)
+		rawConn, err = tools.NewCompressionConn(rawConn, arch)
 	}
 	if err != nil {
 		if rawConn != nil {

socks_js.go

@@ -0,0 +1,11 @@
+//go:build js
+
+package requests
+
+import (
+	"syscall"
+)
+
+func Control(network, address string, c syscall.RawConn) error {
+	return c.Control(func(fd uintptr) {})
+}