package/requests
1
0
Fork 0
mirror of https://github.com/gospider007/requests.git synced 2025-12-24 13:57:52 +08:00
Code Issues Packages Projects Releases Wiki Activity

sync

Browse Source
This commit is contained in:
gospider
2024-12-06 21:25:11 +08:00
parent ccfedb27ae
commit 8ab64c5d00
13 changed files with 80 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
README.md
View File

@@ -44,6 +44,7 @@ Requests is a fully featured HTTP client library for Golang. Network requests ca
* [Middleware](https://github.com/gospider007/requests/tree/master/test/middleware)
* [Option Callback Method](https://github.com/gospider007/requests/blob/master/test/middleware/optionltCallBack_test.go)
* [Result Callback Method](https://github.com/gospider007/requests/blob/master/test/middleware/resultCallBack_test.go)
* [Redirect Callback Method](https://github.com/gospider007/requests/blob/master/test/middleware/redirectCallBack_test.go)
* [Error Callback Method](https://github.com/gospider007/requests/blob/master/test/middleware/errCallBack_test.go)
* [Request Callback Method](https://github.com/gospider007/requests/blob/master/test/middleware/requestCallback_test.go)
* [Retry](https://github.com/gospider007/requests/blob/master/test/middleware/try_test.go)

15
client.go
View File

@@ -96,14 +96,6 @@ func (obj *Client) Close() {
obj.cnl()
}
// func checkRedirect(req *http.Request, via []*http.Request) error {
// ctxData := GetReqCtxData(req.Context())
// if ctxData.maxRedirect == 0 || ctxData.maxRedirect >= len(via) {
// return nil
// }
// return http.ErrUseLastResponse
// }
func (obj *Client) do(req *http.Request, option *RequestOption) (resp *http.Response, err error) {
var redirectNum int
for {
@@ -166,13 +158,10 @@ func (obj *Client) send(req *http.Request, option *RequestOption) (resp *http.Re
addCookie(req, option.Jar.GetCookies(req.URL))
}
resp, err = obj.transport.RoundTrip(req)
if err != nil {
return nil, err
}
if option.Jar != nil {
if option.Jar != nil && resp != nil {
if rc := resp.Cookies(); len(rc) > 0 {
option.Jar.SetCookies(req.URL, rc)
}
}
return resp, nil
return resp, err
}

2
conn.go
View File

@@ -141,7 +141,7 @@ func (obj *connecotr) http3Req(task *reqTask, done chan struct{}) {
func (obj *connecotr) waitBodyClose() error {
select {
case <-obj.bodyCtx.Done(): //wait body close
if err := context.Cause(obj.bodyCtx); errors.Is(err, ErrgospiderBodyClose) {
if err := context.Cause(obj.bodyCtx); errors.Is(err, errGospiderBodyClose) {
return nil
} else {
return err

4
option.go
View File

@@ -102,7 +102,6 @@ type RequestOption struct {
DisCookie bool //disable cookies,not use cookies
DisDecode bool //disable auto decode
DisUnZip bool //disable auto zip decode
DisAlive bool //disable keepalive
Bar bool //enable bar display
OptionCallBack func(ctx context.Context, option *RequestOption) error //option callback,if error is returnd, break request
ResultCallBack func(ctx context.Context, option *RequestOption, response *Response) error //result callback,if error is returnd,next errCallback
@@ -285,9 +284,6 @@ func (obj *Client) newRequestOption(option RequestOption) RequestOption {
if !option.DisUnZip {
option.DisUnZip = obj.option.DisUnZip
}
if !option.DisAlive {
option.DisAlive = obj.option.DisAlive
}
if !option.Bar {
option.Bar = obj.option.Bar
}

8
requests.go
View File

@@ -29,6 +29,8 @@ const gospiderContextKey contextKey = "GospiderContextKey"
var errFatal = errors.New("ErrFatal")
var ErrUseLastResponse = http.ErrUseLastResponse
type reqCtxData struct {
isWs bool
h3 bool
@@ -37,7 +39,6 @@ type reqCtxData struct {
proxy *url.URL
proxys []*url.URL
disProxy bool
disAlive bool
orderHeaders []string
responseHeaderTimeout time.Duration
tlsHandshakeTimeout time.Duration
@@ -70,7 +71,6 @@ func NewReqCtxData(ctx context.Context, option *RequestOption) (*reqCtxData, err
ctxData.ja3Spec = option.Ja3Spec
ctxData.h2Ja3Spec = option.H2Ja3Spec
ctxData.forceHttp1 = option.ForceHttp1
ctxData.disAlive = option.DisAlive
ctxData.maxRedirect = option.MaxRedirect
ctxData.requestCallBack = option.RequestCallBack
ctxData.responseHeaderTimeout = option.ResponseHeaderTimeout
@@ -383,7 +383,7 @@ func (obj *Client) request(ctx context.Context, option *RequestOption) (response
}
//send req
response.response, err = obj.do(reqs, option)
if err != nil {
if err != nil && err != ErrUseLastResponse {
err = tools.WrapError(err, "client do error")
return
}
@@ -392,7 +392,7 @@ func (obj *Client) request(ctx context.Context, option *RequestOption) (response
return
}
if response.Body() != nil {
response.rawConn = response.Body().(*readWriteCloser)
response.rawConn = response.response.Body.(*readWriteCloser)
}
if !response.requestOption.DisUnZip {
response.requestOption.DisUnZip = response.response.Uncompressed

32
response.go
View File

@@ -33,12 +33,7 @@ type Response struct {
content []byte
encoding string
// stream bool
// disDecode bool
// disUnzip bool
filePath string
// bar bool
// isNewConn bool
readBody bool
}
@@ -202,14 +197,7 @@ func (obj *Response) SetContent(val []byte) {
func (obj *Response) Content() []byte {
if !obj.IsWebSocket() && !obj.IsSse() {
obj.ReadBody()
// err := obj.ReadBody()
// if err != nil {
// // log.Panic(err, " Response Content read body error")
// // log.Print(err, " Response Content read body error")
// // return nil
// }
}
// log.Print(len(obj.content))
return obj.content
}
@@ -271,8 +259,24 @@ func (obj *Response) Conn() *connecotr {
return nil
}
type Body struct {
response *Response
}
func (obj *Body) Read(p []byte) (n int, err error) {
i, err := obj.response.response.Body.Read(p)
if err == io.EOF {
obj.Close()
}
return i, err
}
func (obj *Body) Close() (err error) {
obj.response.CloseBody()
return nil
}
func (obj *Response) Body() io.ReadCloser {
return obj.response.Body
return &Body{response: obj}
}
// return true if response is stream
@@ -291,7 +295,7 @@ func (obj *Response) IsSse() bool {
// read body
func (obj *Response) ReadBody() (err error) {
if obj.readBody {
return errors.New("already read body")
return nil
}
if obj.IsWebSocket() && obj.IsSse() {
return errors.New("can not read stream")

22
roundTripper.go
View File

@@ -316,7 +316,7 @@ func (obj *roundTripper) connRoundTrip(ctxData *reqCtxData, task *reqTask, key s
if retry || task.err != nil {
return retry
}
if task.inPool() && !ctxData.disAlive {
if task.inPool() {
obj.putConnPool(key, conn)
}
return retry
@@ -368,18 +368,14 @@ func (obj *roundTripper) RoundTrip(req *http.Request) (response *http.Response,
}
key := getKey(ctxData, req) //pool key
task := obj.newReqTask(req, ctxData)
if ctxData.disAlive {
obj.connRoundTripMain(ctxData, task, key)
} else {
for {
pool := obj.connPools.get(key)
if pool == nil {
obj.connRoundTripMain(ctxData, task, key)
break
}
if !obj.poolRoundTrip(ctxData, pool, task, key) {
break
}
for {
pool := obj.connPools.get(key)
if pool == nil {
obj.connRoundTripMain(ctxData, task, key)
break
}
if !obj.poolRoundTrip(ctxData, pool, task, key) {
break
}
}
if task.err == nil && ctxData.requestCallBack != nil {

10
rw.go
View File

@@ -15,7 +15,11 @@ func (obj *readWriteCloser) Conn() *connecotr {
return obj.conn
}
func (obj *readWriteCloser) Read(p []byte) (n int, err error) {
return obj.body.Read(p)
i, err := obj.body.Read(p)
if err == io.EOF {
obj.Close()
}
return i, err
}
func (obj *readWriteCloser) InPool() bool {
return obj.conn.inPool
@@ -30,13 +34,13 @@ func (obj *readWriteCloser) Proxys() []*url.URL {
return obj.conn.proxys
}
var ErrgospiderBodyClose = errors.New("gospider body close error")
var errGospiderBodyClose = errors.New("gospider body close error")
func (obj *readWriteCloser) Close() (err error) {
if !obj.InPool() {
obj.ForceCloseConn()
} else {
obj.conn.bodyCnl(ErrgospiderBodyClose)
obj.conn.bodyCnl(errGospiderBodyClose)
err = obj.body.Close() //reuse conn
}
return

26
test/middleware/redirectCallBack_test.go Normal file
View File

@@ -0,0 +1,26 @@
package main
import (
"context"
"net/http"
"testing"
"github.com/gospider007/requests"
)
func TestRedirectCallBack(t *testing.T) {
response, err := requests.Get(context.TODO(), "http://www.baidu.com", requests.RequestOption{
RequestCallBack: func(ctx context.Context, request *http.Request, response *http.Response) error {
if response != nil {
return requests.ErrUseLastResponse
}
return nil
},
})
if err != nil {
t.Error(err)
}
if response.StatusCode() != 302 {
t.Error("redirect failed")
}
}

3
test/middleware/requestCallback_test.go
View File

@@ -21,9 +21,6 @@ func TestRequestCallBack(t *testing.T) {
return nil
},
})
if err == nil {
t.Error("err is nil")
}
if !strings.Contains(err.Error(), "max length") {
t.Error("err is not max length")
}

1
test/protocol/http1_test.go
View File

@@ -9,7 +9,6 @@ import (
func TestHttp1(t *testing.T) {
resp, err := requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{
ForceHttp1: true,
DisAlive: true,
})
if err != nil {
t.Error(err)

3
test/protocol/http2_test.go
View File

@@ -1,13 +1,14 @@
package main
import (
"context"
"testing"
"github.com/gospider007/requests"
)
func TestHttp2(t *testing.T) {
resp, err := requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{DisAlive: true})
resp, err := requests.Get(context.TODO(), "https://httpbin.org/anything")
if err != nil {
t.Error(err)
}

17
test/request/stream_test.go
View File

@@ -1,7 +1,6 @@
package main
import (
"io"
"log"
"testing"
"time"
@@ -20,15 +19,19 @@ func TestStream(t *testing.T) {
t.Fatal(err)
}
if resp.IsStream() {
con, err := io.ReadAll(resp.Body())
if err != nil {
t.Fatal(err)
}
t.Log(string(con))
// con, err := io.ReadAll(resp.Body())
// if err != nil {
// t.Fatal(err)
// }
// resp.ReadBody()
// bBody := bytes.NewBuffer(nil)
// io.Copy(bBody, resp.Body())
// t.Log(string(con))
// t.Log(resp.Text())
time.Sleep(2 * time.Second)
resp.CloseBody()
time.Sleep(2 * time.Second)
if resp.StatusCode() != 200 {
t.Fatal("resp.StatusCode()!= 200")
}