diff --git a/README.md b/README.md
index 91799b9..981e72c 100644
--- a/README.md
+++ b/README.md
@@ -19,6 +19,10 @@
Requests is a fully featured HTTP client library for Golang. Network requests can be completed with just a few lines of code. Unified support for http1, http2, http3, websocket, sse, utls, uquic
+
+## If you want to use a multilingual version, please use the project at [fingerproxy](https://github.com/gospider007/fingerproxy). It is a crawler-focused forward proxy with fingerprint spoofing.
+
+
## Innovative Features
* Unlimited chained proxy
* HTTP/3 fingerprint spoofing protection
diff --git a/compressConn.go b/compressConn.go
deleted file mode 100644
index 320e694..0000000
--- a/compressConn.go
+++ /dev/null
@@ -1,213 +0,0 @@
-package requests
-
-import (
- "errors"
- "io"
- "net"
- "strings"
- "sync"
- "time"
-)
-
-type CompressionConn struct {
- conn net.Conn
- w io.WriteCloser
- r io.ReadCloser
-}
-type Compression interface {
- String() string
- OpenReader(r io.Reader) (io.ReadCloser, error)
- OpenWriter(w io.Writer) (io.WriteCloser, error)
-}
-type compression struct {
- name string
- openReader func(r io.Reader) (io.ReadCloser, error)
- openWriter func(w io.Writer) (io.WriteCloser, error)
-}
-
-func (obj compression) String() string {
- return obj.name
-}
-func (obj compression) OpenReader(r io.Reader) (io.ReadCloser, error) {
- return obj.openReader(r)
-}
-func (obj compression) OpenWriter(w io.Writer) (io.WriteCloser, error) {
- return obj.openWriter(w)
-}
-
-func GetCompressionByte(decode string) (byte, error) {
- switch strings.ToLower(decode) {
- case "zstd":
- return 40, nil
- case "s2":
- return 255, nil
- case "flate":
- return 92, nil
- case "minlz":
- return 93, nil
- default:
- return 0, errors.New("unsupported compression type")
- }
-}
-
-func NewCompressionWithByte(b byte) (Compression, error) {
- c, ok := compressionData[b]
- if !ok {
- return nil, errors.New("unsupported compression type")
- }
- return compression{
- name: c.name,
- openReader: func(r io.Reader) (io.ReadCloser, error) {
- buf := make([]byte, 1)
- n, err := r.Read(buf)
- if err != nil {
- return nil, err
- }
- if n != 1 || buf[0] != b {
- return nil, errors.New("invalid response")
- }
- return c.openReader(r)
- },
- openWriter: func(w io.Writer) (io.WriteCloser, error) {
- n, err := w.Write([]byte{b})
- if err != nil {
- return nil, err
- }
- if n != 1 {
- return nil, errors.New("invalid response")
- }
- return c.openWriter(w)
- },
- }, nil
-}
-func NewCompression(decode string) (Compression, error) {
- decode = strings.ToLower(decode)
- for b, c := range compressionData {
- if c.name == decode {
- return NewCompressionWithByte(b)
- }
- }
- return nil, errors.New("unsupported compression type")
-}
-
-func NewCompressionConn(conn net.Conn, decode string) (net.Conn, error) {
- arch, err := NewCompression(decode)
- if err != nil {
- return conn, err
- }
- w, err := arch.OpenWriter(conn)
- if err != nil {
- return conn, err
- }
- r, err := arch.OpenReader(conn)
- if err != nil {
- return conn, err
- }
- ccon := &CompressionConn{
- conn: conn,
- r: r,
- w: w,
- }
- return ccon, nil
-}
-func (obj *CompressionConn) Read(b []byte) (n int, err error) {
- return obj.r.Read(b)
-}
-func (obj *CompressionConn) Write(b []byte) (n int, err error) {
- return obj.w.Write(b)
-}
-func (obj *CompressionConn) Close() error {
- err := obj.conn.Close()
- obj.w.Close()
- obj.r.Close()
- return err
-}
-
-func (obj *CompressionConn) LocalAddr() net.Addr {
- return obj.conn.LocalAddr()
-}
-func (obj *CompressionConn) RemoteAddr() net.Addr {
- return obj.conn.RemoteAddr()
-}
-func (obj *CompressionConn) SetDeadline(t time.Time) error {
- return obj.conn.SetDeadline(t)
-}
-
-func (obj *CompressionConn) SetReadDeadline(t time.Time) error {
- return obj.conn.SetReadDeadline(t)
-}
-func (obj *CompressionConn) SetWriteDeadline(t time.Time) error {
- return obj.conn.SetWriteDeadline(t)
-}
-
-type ReaderCompression struct {
- c io.ReadCloser
- closed bool
- lock sync.Mutex
- closeFunc func()
-}
-
-func (obj *ReaderCompression) Read(p []byte) (n int, err error) {
- obj.lock.Lock()
- defer obj.lock.Unlock()
- if obj.closed {
- return 0, errors.New("read closed")
- }
- n, err = obj.c.Read(p)
- return
-}
-func (obj *ReaderCompression) Close() error {
- obj.lock.Lock()
- defer obj.lock.Unlock()
- if obj.closed {
- return nil
- }
- obj.closed = true
- obj.c.Close()
- obj.closeFunc()
- return nil
-}
-
-type WriterCompression struct {
- c io.WriteCloser
- closed bool
- lock sync.Mutex
- flush interface{ Flush() error }
- closeFunc func()
-}
-
-func (obj *WriterCompression) Write(p []byte) (int, error) {
- obj.lock.Lock()
- defer obj.lock.Unlock()
- if obj.closed {
- return 0, errors.New("write closed")
- }
- n, err := obj.c.Write(p)
- if err != nil {
- return n, err
- }
- if obj.flush != nil {
- err = obj.flush.Flush()
- }
- return n, err
-}
-
-func (obj *WriterCompression) Close() error {
- obj.lock.Lock()
- defer obj.lock.Unlock()
- if obj.closed {
- return nil
- }
- obj.closed = true
- obj.c.Close()
- obj.closeFunc()
- return nil
-}
-
-func newWriterCompression(c io.WriteCloser, closeFunc func()) *WriterCompression {
- flush, _ := c.(interface{ Flush() error })
- return &WriterCompression{c: c, closeFunc: closeFunc, flush: flush}
-}
-func newReaderCompression(c io.ReadCloser, closeFunc func()) *ReaderCompression {
- return &ReaderCompression{c: c, closeFunc: closeFunc}
-}
diff --git a/compressionPool.go b/compressionPool.go
deleted file mode 100644
index 76ac6f7..0000000
--- a/compressionPool.go
+++ /dev/null
@@ -1,200 +0,0 @@
-package requests
-
-import (
- "compress/flate"
- "io"
- "sync"
-
- "github.com/golang/snappy"
- "github.com/klauspost/compress/zstd"
- "github.com/minio/minlz"
-)
-
-type compreData struct {
- rpool *sync.Pool
- wpool *sync.Pool
- name string
- openReader func(r io.Reader) (io.ReadCloser, error)
- openWriter func(w io.Writer) (io.WriteCloser, error)
-}
-
-var compressionData map[byte]compreData
-
-func init() {
- compressionData = map[byte]compreData{
- 40: {
- name: "zstd",
- rpool: &sync.Pool{New: func() any { return nil }},
- wpool: &sync.Pool{New: func() any { return nil }},
- openReader: newZstdReader,
- openWriter: newZstdWriter,
- },
- 255: {
- name: "s2",
- rpool: &sync.Pool{New: func() any { return nil }},
- wpool: &sync.Pool{New: func() any { return nil }},
- openReader: newSnappyReader,
- openWriter: newSnappyWriter,
- },
- 92: {
- name: "flate",
- rpool: &sync.Pool{New: func() any { return nil }},
- wpool: &sync.Pool{New: func() any { return nil }},
- openReader: newFlateReader,
- openWriter: newFlateWriter,
- },
- 93: {
- name: "minlz",
- rpool: &sync.Pool{New: func() any { return nil }},
- wpool: &sync.Pool{New: func() any { return nil }},
- openReader: newMinlzReader,
- openWriter: newMinlzWriter,
- },
- }
-}
-
-func newZstdWriter(w io.Writer) (io.WriteCloser, error) {
- pool := compressionData[40].wpool
- cp := pool.Get()
- var z *zstd.Encoder
- var err error
- if cp == nil {
- z, err = zstd.NewWriter(w, zstd.WithWindowSize(32*1024))
- } else {
- z = cp.(*zstd.Encoder)
- z.Reset(w)
- }
- if err != nil {
- return nil, err
- }
- return newWriterCompression(z, func() {
- z.Reset(nil)
- pool.Put(z)
- }), nil
-}
-func newZstdReader(w io.Reader) (io.ReadCloser, error) {
- pool := compressionData[40].rpool
- cp := pool.Get()
- var z *zstd.Decoder
- var err error
- if cp == nil {
- z, err = zstd.NewReader(w)
- } else {
- z = cp.(*zstd.Decoder)
- z.Reset(w)
- }
- if err != nil {
- return nil, err
- }
- return newReaderCompression(io.NopCloser(z), func() {
- z.Reset(nil)
- pool.Put(z)
- }), nil
-}
-
-// snappy pool
-
-func newSnappyWriter(w io.Writer) (io.WriteCloser, error) {
- pool := compressionData[255].wpool
- cp := pool.Get()
- var z *snappy.Writer
- if cp == nil {
- z = snappy.NewBufferedWriter(w)
- } else {
- z = cp.(*snappy.Writer)
- z.Reset(w)
- }
- return newWriterCompression(z, func() {
- z.Reset(nil)
- pool.Put(z)
- }), nil
-}
-func newSnappyReader(w io.Reader) (io.ReadCloser, error) {
- pool := compressionData[255].rpool
- cp := pool.Get()
- var z *snappy.Reader
- if cp == nil {
- z = snappy.NewReader(w)
- } else {
- z = cp.(*snappy.Reader)
- z.Reset(w)
- }
- return newReaderCompression(io.NopCloser(z), func() {
- z.Reset(nil)
- pool.Put(z)
- }), nil
-}
-
-// flate pool
-func newFlateWriter(w io.Writer) (io.WriteCloser, error) {
- pool := compressionData[92].wpool
- cp := pool.Get()
- var z *flate.Writer
- var err error
- if cp == nil {
- z, err = flate.NewWriter(w, flate.DefaultCompression)
- } else {
- z = cp.(*flate.Writer)
- z.Reset(w)
- }
- if err != nil {
- return nil, err
- }
- return newWriterCompression(z, func() {
- z.Reset(nil)
- pool.Put(z)
- }), nil
-}
-
-func newFlateReader(w io.Reader) (io.ReadCloser, error) {
- pool := compressionData[92].rpool
- cp := pool.Get()
- var z io.ReadCloser
- var f flate.Resetter
- if cp == nil {
- z = flate.NewReader(w)
- f = z.(flate.Resetter)
- } else {
- z = cp.(io.ReadCloser)
- f = z.(flate.Resetter)
- f.Reset(w, nil)
- }
- return newReaderCompression(z, func() {
- f.Reset(nil, nil)
- pool.Put(z)
- }), nil
-}
-
-// minlz pool
-
-func newMinlzWriter(w io.Writer) (io.WriteCloser, error) {
- pool := compressionData[93].wpool
- cp := pool.Get()
- var z *minlz.Writer
- if cp == nil {
- z = minlz.NewWriter(w, minlz.WriterBlockSize(32*1024))
- } else {
- z = cp.(*minlz.Writer)
- z.Reset(w)
- }
- return newWriterCompression(z, func() {
- z.Reset(nil)
- pool.Put(z)
- }), nil
-}
-
-func newMinlzReader(w io.Reader) (io.ReadCloser, error) {
- pool := compressionData[93].rpool
- cp := pool.Get()
- var z *minlz.Reader
- if cp == nil {
- z = minlz.NewReader(w, minlz.ReaderMaxBlockSize(32*1024))
- } else {
- z = cp.(*minlz.Reader)
- z.Reset(w)
- }
- return newReaderCompression(io.NopCloser(z), func() {
- z.Reset(nil)
- pool.Put(z)
- }), nil
-}
diff --git a/conn.go b/conn.go
index d1a60a9..7179387 100644
--- a/conn.go
+++ b/conn.go
@@ -23,20 +23,18 @@ func taskMain(conn http1.Conn, task *reqTask) (err error) {
if err != nil {
task.cnl(err)
} else {
- var bodyContext context.Context
- if task.reqCtx.response != nil && task.reqCtx.response.Body != nil {
- bodyContext = task.reqCtx.response.Body.(*http1.Body).Context()
- }
task.cnl(tools.ErrNoErr)
- if bodyContext != nil {
- select {
- case <-task.reqCtx.Context().Done():
- if context.Cause(task.reqCtx.Context()) != tools.ErrNoErr {
- err = context.Cause(task.reqCtx.Context())
- }
- case <-bodyContext.Done():
- if context.Cause(bodyContext) != tools.ErrNoErr {
- err = context.Cause(bodyContext)
+ if task.reqCtx.response != nil && task.reqCtx.response.Body != nil {
+ if bodyContext := task.reqCtx.response.Body.(*http1.Body).Context(); bodyContext != nil {
+ select {
+ case <-task.reqCtx.Context().Done():
+ if context.Cause(task.reqCtx.Context()) != tools.ErrNoErr {
+ err = context.Cause(task.reqCtx.Context())
+ }
+ case <-bodyContext.Done():
+ if context.Cause(bodyContext) != tools.ErrNoErr {
+ err = context.Cause(bodyContext)
+ }
}
}
}
diff --git a/go.mod b/go.mod
index 9579f6d..a40ee31 100644
--- a/go.mod
+++ b/go.mod
@@ -3,7 +3,6 @@ module github.com/gospider007/requests
go 1.25
require (
- github.com/golang/snappy v1.0.0
github.com/gorilla/websocket v1.5.3
github.com/gospider007/bar v0.0.0-20250815030902-4f5b5d6312cf
github.com/gospider007/bs4 v0.0.0-20250815030800-a352d3ad57ee
@@ -14,10 +13,8 @@ require (
github.com/gospider007/http3 v0.0.0-20250817123336-07d66db6dbb3
github.com/gospider007/ja3 v0.0.0-20250815031055-0948dc3bbe0b
github.com/gospider007/re v0.0.0-20250815031101-a57caeff73bf
- github.com/gospider007/tools v0.0.0-20250815031258-8a81d680917c
- github.com/gospider007/websocket v0.0.0-20250815031315-cde1470d5ace
- github.com/klauspost/compress v1.18.0
- github.com/minio/minlz v1.0.1
+ github.com/gospider007/tools v0.0.0-20250819094836-a81233312764
+ github.com/gospider007/websocket v0.0.0-20250819094917-c00c0a99815f
github.com/quic-go/quic-go v0.54.0
github.com/refraction-networking/uquic v0.0.6
github.com/refraction-networking/utls v1.8.0
@@ -45,12 +42,14 @@ require (
github.com/gobwas/httphead v0.1.0 // indirect
github.com/gobwas/pool v0.2.1 // indirect
github.com/gobwas/ws v1.4.0 // indirect
+ github.com/golang/snappy v1.0.0 // indirect
github.com/google/gopacket v1.1.19 // indirect
github.com/google/pprof v0.0.0-20250630185457-6e76a2b096b5 // indirect
github.com/gospider007/blog v0.0.0-20250815030743-f2af6b9013ab // indirect
github.com/gospider007/kinds v0.0.0-20250815031133-b2282666f69c // indirect
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
github.com/json-iterator/go v1.1.12 // indirect
+ github.com/klauspost/compress v1.18.0 // indirect
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
github.com/klauspost/pgzip v1.2.6 // indirect
github.com/kr/pretty v0.3.1 // indirect
@@ -59,6 +58,7 @@ require (
github.com/mholt/archives v0.1.3 // indirect
github.com/miekg/dns v1.1.68 // indirect
github.com/mikelolasagasti/xz v1.0.1 // indirect
+ github.com/minio/minlz v1.0.1 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/nwaples/rardecode/v2 v2.1.1 // indirect
@@ -89,4 +89,5 @@ require (
golang.org/x/sys v0.35.0 // indirect
golang.org/x/text v0.28.0 // indirect
golang.org/x/tools v0.36.0 // indirect
+ google.golang.org/protobuf v1.36.7 // indirect
)
diff --git a/go.sum b/go.sum
index 572b2d2..c4e9915 100644
--- a/go.sum
+++ b/go.sum
@@ -128,10 +128,10 @@ github.com/gospider007/kinds v0.0.0-20250815031133-b2282666f69c h1:WnFZf0v+Du5nH
github.com/gospider007/kinds v0.0.0-20250815031133-b2282666f69c/go.mod h1:8eAmhxyW7p81nsYlD5Wj8I0HuZpVGci+Q9wr6vfY8pg=
github.com/gospider007/re v0.0.0-20250815031101-a57caeff73bf h1:+3g9qmg+XLbYcY4vvbsDM7xHkXF+5U8V/haAYDImGTA=
github.com/gospider007/re v0.0.0-20250815031101-a57caeff73bf/go.mod h1:5fVj9OYkqlOgBdz+kH4jgdiSZ6jdfbmexLIFqtMa9v8=
-github.com/gospider007/tools v0.0.0-20250815031258-8a81d680917c h1:2NOEZaRbICBWs07Hs9hm4wPhXf+zTtoYnZV2aTcuvRw=
-github.com/gospider007/tools v0.0.0-20250815031258-8a81d680917c/go.mod h1:hT2nPJQoUL5nwturslhBjykigIA27qxujkLGSEPWb3o=
-github.com/gospider007/websocket v0.0.0-20250815031315-cde1470d5ace h1:phCUKTZU8SNtLjEZfK7SAFWenemHD10FH6MU8M5aeVg=
-github.com/gospider007/websocket v0.0.0-20250815031315-cde1470d5ace/go.mod h1:cj1g1pNi5VOR+aed4Q37UEoEla+EgrmMNoTi0f5EYUA=
+github.com/gospider007/tools v0.0.0-20250819094836-a81233312764 h1:NYVqv0I6pE7yV5Enee9VOWmqjgJFZqQT66p+yZ1CwO4=
+github.com/gospider007/tools v0.0.0-20250819094836-a81233312764/go.mod h1:Imryilbsdhsx+zIE+s6mIB+DNrAIig0ixupk45exsts=
+github.com/gospider007/websocket v0.0.0-20250819094917-c00c0a99815f h1:nyklmiKQ1vLOmC2DCPL8/RZHN22or/CFRi0xCGTiZiU=
+github.com/gospider007/websocket v0.0.0-20250819094917-c00c0a99815f/go.mod h1:olruUjPKXJ74OlFipbKedfK0LMuDu/UarSWIVhEtOX4=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
@@ -485,8 +485,8 @@ google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac
google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY=
-google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY=
+google.golang.org/protobuf v1.36.7 h1:IgrO7UwFQGJdRNXH/sQux4R1Dj1WAKcLElzeeRaXV2A=
+google.golang.org/protobuf v1.36.7/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
diff --git a/requests.go b/requests.go
index 7b4a145..fb5d537 100644
--- a/requests.go
+++ b/requests.go
@@ -349,19 +349,20 @@ func (obj *Client) request(ctx *Response) (err error) {
err = errors.New("send req response is nil")
return
}
- if ctx.response.Body != nil {
- ctx.rawBody = ctx.response.Body.(*http1.Body)
- }
+ ctx.rawBody = ctx.response.Body.(*http1.Body)
if encoding := ctx.ContentEncoding(); encoding != "" && ctx.response.Body != nil {
- var unCompressionBody io.ReadCloser
- unCompressionBody, err = tools.CompressionHeadersDecode(ctx.Context(), ctx.response.Body, encoding)
- if err != nil {
- if err != io.ErrUnexpectedEOF && err != io.EOF {
- return
+ arch, cerr := tools.NewRawCompression(encoding)
+ if cerr != nil {
+ return cerr
+ }
+ unCompressionBody, cerr := arch.OpenReader(ctx.rawBody.GetReader())
+ if cerr != nil {
+ if cerr != io.ErrUnexpectedEOF && cerr != io.EOF {
+ return cerr
}
}
if unCompressionBody != nil {
- ctx.response.Body = unCompressionBody
+ ctx.rawBody.SetReader(unCompressionBody)
}
}
if strings.Contains(ctx.response.Header.Get("Content-Type"), "text/event-stream") {
diff --git a/roundTripper.go b/roundTripper.go
index bbfd073..28e7b34 100644
--- a/roundTripper.go
+++ b/roundTripper.go
@@ -215,7 +215,7 @@ func (obj *roundTripper) dial(ctx *Response) (conn http1.Conn, err error) {
rawConn = rawNetConn
}
if arch != "" {
- rawConn, err = NewCompressionConn(rawConn, arch)
+ rawConn, err = tools.NewCompressionConn(rawConn, arch)
}
if err != nil {
if rawConn != nil {
diff --git a/socks_js.go b/socks_js.go
new file mode 100644
index 0000000..2c7f86a
--- /dev/null
+++ b/socks_js.go
@@ -0,0 +1,11 @@
+//go:build js
+
+package requests
+
+import (
+ "syscall"
+)
+
+func Control(network, address string, c syscall.RawConn) error {
+ return c.Control(func(fd uintptr) {})
+}