package/requests
1
0
Fork 0
mirror of https://github.com/gospider007/requests.git synced 2025-12-24 13:57:52 +08:00
Code Issues Packages Projects Releases Wiki Activity

sync

Browse Source
This commit is contained in:
赵小小
2024-06-11 22:46:32 +08:00
parent 99bd7b84ca
commit ee64501285
47 changed files with 5341 additions and 5317 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1348
LICENSE
View File

File diff suppressed because it is too large Load Diff

224
README.md
View File

@@ -1,113 +1,113 @@
<p align="center">
<a href="https://github.com/gospider007/requests"><img src="https://go.dev/images/favicon-gopher.png"></a>
</p>
<p align="center"><strong>Requests</strong> <em>- A next-generation HTTP client for Golang.</em></p>
<p align="center">
<a href="https://github.com/gospider007/requests">
<img src="https://img.shields.io/github/last-commit/gospider007/requests">
</a>
<a href="https://github.com/gospider007/requests">
<img src="https://img.shields.io/badge/build-passing-brightgreen">
</a>
<a href="https://github.com/gospider007/requests">
<img src="https://img.shields.io/badge/language-golang-brightgreen">
</a>
</p>
Requests is a fully featured HTTP client library for Golang. Network requests can be completed with just a few lines of code
---
## Features
* [Simple for settings and Request](https://github.com/gospider007/requests#quickly-send-requests)
* [Request](https://github.com/gospider007/requests/tree/master/test/request) Support Automatic type conversion, Support orderly map
* [Json Request](https://github.com/gospider007/requests/blob/master/test/request/json_test.go) with `application/json`
* [Data Request](https://github.com/gospider007/requests/blob/master/test/request/data_test.go) with `application/x-www-form-urlencoded`
* [Form Request](https://github.com/gospider007/requests/blob/master/test/request/form_test.go) with `multipart/form-data`
* [Upload File Request](https://github.com/gospider007/requests/blob/master/test/request/file_test.go) with `multipart/form-data`
* [Flow Request](https://github.com/gospider007/requests/blob/master/test/request/stream_test.go)
* [Request URL Path Params](https://github.com/gospider007/requests/blob/master/test/request/params_test.go)
* [Local network card](https://github.com/gospider007/requests/blob/master/test/request/localAddr_test.go)
* [Response](https://github.com/gospider007/requests/tree/master/test/response)
* [Return whether to reuse connections](https://github.com/gospider007/requests/blob/master/test/response/isNewConn_test.go)
* [Return Raw Connection](https://github.com/gospider007/requests/blob/master/test/response/rawConn_test.go)
* [Return Proxy](https://github.com/gospider007/requests/blob/master/test/response/useProxy_test.go)
* [Middleware](https://github.com/gospider007/requests/tree/master/test/middleware)
* [Option Callback Method](https://github.com/gospider007/requests/blob/master/test/middleware/optionltCallBack_test.go)
* [Result Callback Method](https://github.com/gospider007/requests/blob/master/test/middleware/resultCallBack_test.go)
* [Error Callback Method](https://github.com/gospider007/requests/blob/master/test/middleware/errCallBack_test.go)
* [Request Callback Method](https://github.com/gospider007/requests/blob/master/test/middleware/requestCallback_test.go)
* [Retry](https://github.com/gospider007/requests/blob/master/test/middleware/try_test.go)
* [Protocol](https://github.com/gospider007/requests/tree/master/test/protocol)
* [HTTP1](https://github.com/gospider007/requests/blob/master/test/protocol/http1_test.go)
* [HTTP2](https://github.com/gospider007/requests/blob/master/test/protocol/http2_test.go)
* [WebSocket](https://github.com/gospider007/requests/blob/master/test/protocol/websocket_test.go)
* [SSE](https://github.com/gospider007/requests/blob/master/test/protocol/sse_test.go)
* [Fingerprint](https://github.com/gospider007/requests/tree/master/test/fingerprint)
* [Ja3 Fingerprint](https://github.com/gospider007/requests/blob/master/test/fingerprint/ja3_test.go)
* [Http2 Fingerprint](https://github.com/gospider007/requests/blob/master/test/fingerprint/http2_test.go)
* [Ja4 Fingerprint](https://github.com/gospider007/requests/blob/master/test/fingerprint/ja4_test.go)
* [Session](https://github.com/gospider007/requests/blob/master/test/session_test.go)
* [IPv4, IPv6 Address Control Parsing](https://github.com/gospider007/requests/blob/master/test/addType_test.go)
* [DNS Settings](https://github.com/gospider007/requests/blob/master/test/dns_test.go)
* [Proxy](https://github.com/gospider007/requests/blob/master/test/proxy_test.go)
* [Well tested client library](https://github.com/gospider007/requests/tree/master/test)
## [Benchmark](https://github.com/gospider007/benchmark)
[gospider007/requests](https://github.com/gospider007/requests) > [imroc/req](github.com/imroc/req) > [go-resty](github.com/go-resty/resty) > [wangluozhe/requests](github.com/wangluozhe/requests) > [curl_cffi](https://github.com/yifeikong/curl_cffi) > [httpx](https://github.com/encode/httpx) > [psf/requests](https://github.com/psf/requests)
## Supported Go Versions
Recommended to use `go1.21.3` and above.
Initially Requests started supporting `go modules`
## Installation
```bash
go get github.com/gospider007/requests
```
## Usage
```go
import "github.com/gospider007/requests"
```
### Quickly Send Requests
```go
package main
import (
"log"
"time"
"github.com/gospider007/requests"
)
func main() {
resp, err := requests.Get(nil, "http://httpbin.org/anything")
if err != nil {
log.Panic(err)
}
log.Print(resp.Text()) // Get content and parse as string
log.Print(resp.Content()) // Get content as bytes
log.Print(resp.Json()) // Get content and parse as gjson JSON
log.Print(resp.Html()) // Get content and parse as goquery DOM
log.Print(resp.Cookies()) // Get cookies
}
```
# Contributing
If you have a bug report or feature request, you can [open an issue](../../issues/new)
# Contact
If you have questions, feel free to reach out to us in the following ways:
* QQ Group (Chinese): 939111384 - <a href="http://qm.qq.com/cgi-bin/qm/qr?_wv=1027&k=yI72QqgPExDqX6u_uEbzAE_XfMW6h_d3&jump_from=webapi"><img src="https://pub.idqqimg.com/wpa/images/group.png"></a>
* WeChat (Chinese): gospider007
## Sponsors
If you like and it really helps you, feel free to reward me with a cup of coffee, and don't forget to mention your github id.
<table>
<tr>
<td align="center">
<img src="https://github.com/gospider007/tools/blob/master/play/wx.jpg?raw=true" height="200px" width="200px" alt=""/>
<br />
<sub><b>Wechat</b></sub>
</td>
<td align="center">
<img src="https://github.com/gospider007/tools/blob/master/play/qq.jpg?raw=true" height="200px" width="200px" alt=""/>
<br />
<sub><b>Alipay</b></sub>
</td>
</tr>
<p align="center">
<a href="https://github.com/gospider007/requests"><img src="https://go.dev/images/favicon-gopher.png"></a>
</p>
<p align="center"><strong>Requests</strong> <em>- A next-generation HTTP client for Golang.</em></p>
<p align="center">
<a href="https://github.com/gospider007/requests">
<img src="https://img.shields.io/github/last-commit/gospider007/requests">
</a>
<a href="https://github.com/gospider007/requests">
<img src="https://img.shields.io/badge/build-passing-brightgreen">
</a>
<a href="https://github.com/gospider007/requests">
<img src="https://img.shields.io/badge/language-golang-brightgreen">
</a>
</p>
Requests is a fully featured HTTP client library for Golang. Network requests can be completed with just a few lines of code
---
## Features
* [Simple for settings and Request](https://github.com/gospider007/requests#quickly-send-requests)
* [Request](https://github.com/gospider007/requests/tree/master/test/request) Support Automatic type conversion, Support orderly map
* [Json Request](https://github.com/gospider007/requests/blob/master/test/request/json_test.go) with `application/json`
* [Data Request](https://github.com/gospider007/requests/blob/master/test/request/data_test.go) with `application/x-www-form-urlencoded`
* [Form Request](https://github.com/gospider007/requests/blob/master/test/request/form_test.go) with `multipart/form-data`
* [Upload File Request](https://github.com/gospider007/requests/blob/master/test/request/file_test.go) with `multipart/form-data`
* [Flow Request](https://github.com/gospider007/requests/blob/master/test/request/stream_test.go)
* [Request URL Path Params](https://github.com/gospider007/requests/blob/master/test/request/params_test.go)
* [Local network card](https://github.com/gospider007/requests/blob/master/test/request/localAddr_test.go)
* [Response](https://github.com/gospider007/requests/tree/master/test/response)
* [Return whether to reuse connections](https://github.com/gospider007/requests/blob/master/test/response/isNewConn_test.go)
* [Return Raw Connection](https://github.com/gospider007/requests/blob/master/test/response/rawConn_test.go)
* [Return Proxy](https://github.com/gospider007/requests/blob/master/test/response/useProxy_test.go)
* [Middleware](https://github.com/gospider007/requests/tree/master/test/middleware)
* [Option Callback Method](https://github.com/gospider007/requests/blob/master/test/middleware/optionltCallBack_test.go)
* [Result Callback Method](https://github.com/gospider007/requests/blob/master/test/middleware/resultCallBack_test.go)
* [Error Callback Method](https://github.com/gospider007/requests/blob/master/test/middleware/errCallBack_test.go)
* [Request Callback Method](https://github.com/gospider007/requests/blob/master/test/middleware/requestCallback_test.go)
* [Retry](https://github.com/gospider007/requests/blob/master/test/middleware/try_test.go)
* [Protocol](https://github.com/gospider007/requests/tree/master/test/protocol)
* [HTTP1](https://github.com/gospider007/requests/blob/master/test/protocol/http1_test.go)
* [HTTP2](https://github.com/gospider007/requests/blob/master/test/protocol/http2_test.go)
* [WebSocket](https://github.com/gospider007/requests/blob/master/test/protocol/websocket_test.go)
* [SSE](https://github.com/gospider007/requests/blob/master/test/protocol/sse_test.go)
* [Fingerprint](https://github.com/gospider007/requests/tree/master/test/fingerprint)
* [Ja3 Fingerprint](https://github.com/gospider007/requests/blob/master/test/fingerprint/ja3_test.go)
* [Http2 Fingerprint](https://github.com/gospider007/requests/blob/master/test/fingerprint/http2_test.go)
* [Ja4 Fingerprint](https://github.com/gospider007/requests/blob/master/test/fingerprint/ja4_test.go)
* [Session](https://github.com/gospider007/requests/blob/master/test/session_test.go)
* [IPv4, IPv6 Address Control Parsing](https://github.com/gospider007/requests/blob/master/test/addType_test.go)
* [DNS Settings](https://github.com/gospider007/requests/blob/master/test/dns_test.go)
* [Proxy](https://github.com/gospider007/requests/blob/master/test/proxy_test.go)
* [Well tested client library](https://github.com/gospider007/requests/tree/master/test)
## [Benchmark](https://github.com/gospider007/benchmark)
[gospider007/requests](https://github.com/gospider007/requests) > [imroc/req](github.com/imroc/req) > [go-resty](github.com/go-resty/resty) > [wangluozhe/requests](github.com/wangluozhe/requests) > [curl_cffi](https://github.com/yifeikong/curl_cffi) > [httpx](https://github.com/encode/httpx) > [psf/requests](https://github.com/psf/requests)
## Supported Go Versions
Recommended to use `go1.21.3` and above.
Initially Requests started supporting `go modules`
## Installation
```bash
go get github.com/gospider007/requests
```
## Usage
```go
import "github.com/gospider007/requests"
```
### Quickly Send Requests
```go
package main
import (
"log"
"time"
"github.com/gospider007/requests"
)
func main() {
resp, err := requests.Get(nil, "http://httpbin.org/anything")
if err != nil {
log.Panic(err)
}
log.Print(resp.Text()) // Get content and parse as string
log.Print(resp.Content()) // Get content as bytes
log.Print(resp.Json()) // Get content and parse as gjson JSON
log.Print(resp.Html()) // Get content and parse as goquery DOM
log.Print(resp.Cookies()) // Get cookies
}
```
# Contributing
If you have a bug report or feature request, you can [open an issue](../../issues/new)
# Contact
If you have questions, feel free to reach out to us in the following ways:
* QQ Group (Chinese): 939111384 - <a href="http://qm.qq.com/cgi-bin/qm/qr?_wv=1027&k=yI72QqgPExDqX6u_uEbzAE_XfMW6h_d3&jump_from=webapi"><img src="https://pub.idqqimg.com/wpa/images/group.png"></a>
* WeChat (Chinese): gospider007
## Sponsors
If you like and it really helps you, feel free to reward me with a cup of coffee, and don't forget to mention your github id.
<table>
<tr>
<td align="center">
<img src="https://github.com/gospider007/tools/blob/master/play/wx.jpg?raw=true" height="200px" width="200px" alt=""/>
<br />
<sub><b>Wechat</b></sub>
</td>
<td align="center">
<img src="https://github.com/gospider007/tools/blob/master/play/qq.jpg?raw=true" height="200px" width="200px" alt=""/>
<br />
<sub><b>Alipay</b></sub>
</td>
</tr>
</table>

692
body.go
View File

@@ -1,346 +1,346 @@
package requests
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"mime/multipart"
"net/http"
"net/textproto"
"net/url"
"reflect"
"github.com/gospider007/gson"
"github.com/gospider007/tools"
"golang.org/x/exp/maps"
)
const (
readType = iota
mapType
)
type OrderMap struct {
data map[string]any
keys []string
}
func NewOrderMap() *OrderMap {
return &OrderMap{
data: make(map[string]any),
keys: []string{},
}
}
func (obj *OrderMap) Set(key string, val any) {
obj.Del(key)
obj.data[key] = val
obj.keys = append(obj.keys, key)
}
func (obj *OrderMap) Del(key string) {
delete(obj.data, key)
obj.keys = tools.DelSliceVals(obj.keys, key)
}
func (obj *OrderMap) Keys() []string {
return obj.keys
}
func (obj *OrderMap) parseHeaders() (map[string][]string, []string) {
head := make(http.Header)
for _, kk := range obj.keys {
if vvs, ok := obj.data[kk].([]any); ok {
for _, vv := range vvs {
head.Add(kk, fmt.Sprint(vv))
}
} else {
head.Add(kk, fmt.Sprint(obj.data[kk]))
}
}
return head, obj.keys
}
func formWrite(writer *multipart.Writer, key string, val any) (err error) {
switch value := val.(type) {
case File:
h := make(textproto.MIMEHeader)
h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="%s"; filename="%s"`, escapeQuotes(key), escapeQuotes(value.FileName)))
if value.ContentType == "" {
switch content := value.Content.(type) {
case []byte:
h.Set("Content-Type", http.DetectContentType(content))
case string:
h.Set("Content-Type", http.DetectContentType(tools.StringToBytes(content)))
case io.Reader:
h.Set("Content-Type", "application/octet-stream")
default:
con, err := gson.Encode(content)
if err != nil {
return err
}
h.Set("Content-Type", http.DetectContentType(con))
}
} else {
h.Set("Content-Type", value.ContentType)
}
var wp io.Writer
if wp, err = writer.CreatePart(h); err != nil {
return
}
switch content := value.Content.(type) {
case []byte:
_, err = wp.Write(content)
case string:
_, err = wp.Write(tools.StringToBytes(content))
case io.Reader:
_, err = io.Copy(wp, content)
default:
con, err := gson.Encode(content)
if err != nil {
return err
}
_, err = wp.Write(con)
if err != nil {
return err
}
}
case []byte:
err = writer.WriteField(key, tools.BytesToString(value))
case string:
err = writer.WriteField(key, value)
default:
con, err := gson.Encode(val)
if err != nil {
return err
}
err = writer.WriteField(key, tools.BytesToString(con))
if err != nil {
return err
}
}
return
}
func (obj *OrderMap) parseForm(ctx context.Context) (io.Reader, string, bool, error) {
if len(obj.keys) == 0 || len(obj.data) == 0 {
return nil, "", false, nil
}
if obj.isformPip() {
pr, pw := pipe(ctx)
writer := multipart.NewWriter(pw)
go func() {
err := obj.formWriteMain(writer)
if err == nil {
err = io.EOF
}
pr.Close(err)
pw.Close(err)
}()
return pr, writer.FormDataContentType(), true, nil
}
body := bytes.NewBuffer(nil)
writer := multipart.NewWriter(body)
err := obj.formWriteMain(writer)
if err != nil {
return nil, writer.FormDataContentType(), false, err
}
return body, writer.FormDataContentType(), false, err
}
func (obj *OrderMap) isformPip() bool {
if len(obj.keys) == 0 || len(obj.data) == 0 {
return false
}
for _, key := range obj.keys {
if vals, ok := obj.data[key].([]any); ok {
for _, val := range vals {
if file, ok := val.(File); ok {
if _, ok := file.Content.(io.Reader); ok {
return true
}
}
}
} else {
if file, ok := obj.data[key].(File); ok {
if _, ok := file.Content.(io.Reader); ok {
return true
}
}
}
}
return false
}
func (obj *OrderMap) formWriteMain(writer *multipart.Writer) (err error) {
for _, key := range obj.keys {
if vals, ok := obj.data[key].([]any); ok {
for _, val := range vals {
if err = formWrite(writer, key, val); err != nil {
return
}
}
} else {
if err = formWrite(writer, key, obj.data[key]); err != nil {
return
}
}
}
return writer.Close()
}
func paramsWrite(buf *bytes.Buffer, key string, val any) {
if buf.Len() > 0 {
buf.WriteByte('&')
}
buf.WriteString(url.QueryEscape(key))
buf.WriteByte('=')
buf.WriteString(url.QueryEscape(fmt.Sprint(val)))
}
func (obj *OrderMap) parseParams() *bytes.Buffer {
buf := bytes.NewBuffer(nil)
for _, k := range obj.keys {
if vals, ok := obj.data[k].([]any); ok {
for _, v := range vals {
paramsWrite(buf, k, v)
}
} else {
paramsWrite(buf, k, obj.data[k])
}
}
return buf
}
func (obj *OrderMap) parseData() *bytes.Reader {
val := obj.parseParams().Bytes()
if val == nil {
return nil
}
return bytes.NewReader(val)
}
func (obj *OrderMap) MarshalJSON() ([]byte, error) {
buf := bytes.NewBuffer(nil)
err := buf.WriteByte('{')
if err != nil {
return nil, err
}
for i, k := range obj.keys {
if i > 0 {
if err = buf.WriteByte(','); err != nil {
return nil, err
}
}
key, err := gson.Encode(k)
if err != nil {
return nil, err
}
if _, err = buf.Write(key); err != nil {
return nil, err
}
if err = buf.WriteByte(':'); err != nil {
return nil, err
}
val, err := gson.Encode(obj.data[k])
if err != nil {
return nil, err
}
if _, err = buf.Write(val); err != nil {
return nil, err
}
}
if err = buf.WriteByte('}'); err != nil {
return nil, err
}
return buf.Bytes(), nil
}
func any2Map(val any) map[string]any {
if reflect.TypeOf(val).Kind() != reflect.Map {
return nil
}
mapValue := reflect.ValueOf(val)
result := make(map[string]any)
for _, key := range mapValue.MapKeys() {
valueData := mapValue.MapIndex(key).Interface()
sliceValue := reflect.ValueOf(valueData)
if sliceValue.Kind() == reflect.Slice {
valueData2 := []any{}
for i := 0; i < sliceValue.Len(); i++ {
valueData2 = append(valueData2, sliceValue.Index(i).Interface())
}
result[fmt.Sprint(key.Interface())] = valueData2
} else {
result[fmt.Sprint(key.Interface())] = valueData
}
}
return result
}
func (obj *RequestOption) newBody(val any, valType int) (reader io.Reader, parseOrderMap *OrderMap, orderKey []string, err error) {
var isOrderMap bool
parseOrderMap, isOrderMap = val.(*OrderMap)
if isOrderMap {
if valType == readType {
readCon, err := parseOrderMap.MarshalJSON()
return bytes.NewReader(readCon), nil, nil, err
} else {
return nil, parseOrderMap, parseOrderMap.Keys(), nil
}
}
if valType == readType {
switch value := val.(type) {
case io.Reader:
obj.once = true
return value, nil, nil, nil
case string:
return bytes.NewReader(tools.StringToBytes(value)), nil, nil, nil
case []byte:
return bytes.NewReader(value), nil, nil, nil
default:
enData, err := gson.Encode(val)
if err != nil {
return nil, nil, nil, err
}
return bytes.NewReader(enData), nil, nil, nil
}
}
if mapData := any2Map(val); mapData != nil {
val = mapData
}
mapL:
switch value := val.(type) {
case *gson.Client:
if !value.IsObject() {
return nil, nil, nil, errors.New("body-type error")
}
orderMap := NewOrderMap()
for kk, vv := range value.Map() {
if vv.IsArray() {
valData := make([]any, len(vv.Array()))
for i, v := range vv.Array() {
valData[i] = v.Value()
}
orderMap.Set(kk, valData)
} else {
orderMap.Set(kk, vv.Value())
}
}
return nil, orderMap, nil, nil
case *OrderMap:
if mapData := any2Map(value.data); mapData != nil {
value.data = mapData
}
return nil, value, nil, nil
case map[string]any:
orderMap := NewOrderMap()
orderMap.data = value
orderMap.keys = maps.Keys(value)
return nil, orderMap, nil, nil
}
if val, err = gson.Decode(val); err != nil {
switch value := val.(type) {
case string:
return bytes.NewReader(tools.StringToBytes(value)), nil, nil, nil
case []byte:
return bytes.NewReader(value), nil, nil, nil
default:
return nil, nil, nil, err
}
}
goto mapL
}
package requests
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"mime/multipart"
"net/http"
"net/textproto"
"net/url"
"reflect"
"github.com/gospider007/gson"
"github.com/gospider007/tools"
"golang.org/x/exp/maps"
)
const (
readType = iota
mapType
)
type OrderMap struct {
data map[string]any
keys []string
}
func NewOrderMap() *OrderMap {
return &OrderMap{
data: make(map[string]any),
keys: []string{},
}
}
func (obj *OrderMap) Set(key string, val any) {
obj.Del(key)
obj.data[key] = val
obj.keys = append(obj.keys, key)
}
func (obj *OrderMap) Del(key string) {
delete(obj.data, key)
obj.keys = tools.DelSliceVals(obj.keys, key)
}
func (obj *OrderMap) Keys() []string {
return obj.keys
}
func (obj *OrderMap) parseHeaders() (map[string][]string, []string) {
head := make(http.Header)
for _, kk := range obj.keys {
if vvs, ok := obj.data[kk].([]any); ok {
for _, vv := range vvs {
head.Add(kk, fmt.Sprint(vv))
}
} else {
head.Add(kk, fmt.Sprint(obj.data[kk]))
}
}
return head, obj.keys
}
func formWrite(writer *multipart.Writer, key string, val any) (err error) {
switch value := val.(type) {
case File:
h := make(textproto.MIMEHeader)
h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="%s"; filename="%s"`, escapeQuotes(key), escapeQuotes(value.FileName)))
if value.ContentType == "" {
switch content := value.Content.(type) {
case []byte:
h.Set("Content-Type", http.DetectContentType(content))
case string:
h.Set("Content-Type", http.DetectContentType(tools.StringToBytes(content)))
case io.Reader:
h.Set("Content-Type", "application/octet-stream")
default:
con, err := gson.Encode(content)
if err != nil {
return err
}
h.Set("Content-Type", http.DetectContentType(con))
}
} else {
h.Set("Content-Type", value.ContentType)
}
var wp io.Writer
if wp, err = writer.CreatePart(h); err != nil {
return
}
switch content := value.Content.(type) {
case []byte:
_, err = wp.Write(content)
case string:
_, err = wp.Write(tools.StringToBytes(content))
case io.Reader:
_, err = io.Copy(wp, content)
default:
con, err := gson.Encode(content)
if err != nil {
return err
}
_, err = wp.Write(con)
if err != nil {
return err
}
}
case []byte:
err = writer.WriteField(key, tools.BytesToString(value))
case string:
err = writer.WriteField(key, value)
default:
con, err := gson.Encode(val)
if err != nil {
return err
}
err = writer.WriteField(key, tools.BytesToString(con))
if err != nil {
return err
}
}
return
}
func (obj *OrderMap) parseForm(ctx context.Context) (io.Reader, string, bool, error) {
if len(obj.keys) == 0 || len(obj.data) == 0 {
return nil, "", false, nil
}
if obj.isformPip() {
pr, pw := pipe(ctx)
writer := multipart.NewWriter(pw)
go func() {
err := obj.formWriteMain(writer)
if err == nil {
err = io.EOF
}
pr.Close(err)
pw.Close(err)
}()
return pr, writer.FormDataContentType(), true, nil
}
body := bytes.NewBuffer(nil)
writer := multipart.NewWriter(body)
err := obj.formWriteMain(writer)
if err != nil {
return nil, writer.FormDataContentType(), false, err
}
return body, writer.FormDataContentType(), false, err
}
func (obj *OrderMap) isformPip() bool {
if len(obj.keys) == 0 || len(obj.data) == 0 {
return false
}
for _, key := range obj.keys {
if vals, ok := obj.data[key].([]any); ok {
for _, val := range vals {
if file, ok := val.(File); ok {
if _, ok := file.Content.(io.Reader); ok {
return true
}
}
}
} else {
if file, ok := obj.data[key].(File); ok {
if _, ok := file.Content.(io.Reader); ok {
return true
}
}
}
}
return false
}
func (obj *OrderMap) formWriteMain(writer *multipart.Writer) (err error) {
for _, key := range obj.keys {
if vals, ok := obj.data[key].([]any); ok {
for _, val := range vals {
if err = formWrite(writer, key, val); err != nil {
return
}
}
} else {
if err = formWrite(writer, key, obj.data[key]); err != nil {
return
}
}
}
return writer.Close()
}
func paramsWrite(buf *bytes.Buffer, key string, val any) {
if buf.Len() > 0 {
buf.WriteByte('&')
}
buf.WriteString(url.QueryEscape(key))
buf.WriteByte('=')
buf.WriteString(url.QueryEscape(fmt.Sprint(val)))
}
func (obj *OrderMap) parseParams() *bytes.Buffer {
buf := bytes.NewBuffer(nil)
for _, k := range obj.keys {
if vals, ok := obj.data[k].([]any); ok {
for _, v := range vals {
paramsWrite(buf, k, v)
}
} else {
paramsWrite(buf, k, obj.data[k])
}
}
return buf
}
func (obj *OrderMap) parseData() *bytes.Reader {
val := obj.parseParams().Bytes()
if val == nil {
return nil
}
return bytes.NewReader(val)
}
func (obj *OrderMap) MarshalJSON() ([]byte, error) {
buf := bytes.NewBuffer(nil)
err := buf.WriteByte('{')
if err != nil {
return nil, err
}
for i, k := range obj.keys {
if i > 0 {
if err = buf.WriteByte(','); err != nil {
return nil, err
}
}
key, err := gson.Encode(k)
if err != nil {
return nil, err
}
if _, err = buf.Write(key); err != nil {
return nil, err
}
if err = buf.WriteByte(':'); err != nil {
return nil, err
}
val, err := gson.Encode(obj.data[k])
if err != nil {
return nil, err
}
if _, err = buf.Write(val); err != nil {
return nil, err
}
}
if err = buf.WriteByte('}'); err != nil {
return nil, err
}
return buf.Bytes(), nil
}
func any2Map(val any) map[string]any {
if reflect.TypeOf(val).Kind() != reflect.Map {
return nil
}
mapValue := reflect.ValueOf(val)
result := make(map[string]any)
for _, key := range mapValue.MapKeys() {
valueData := mapValue.MapIndex(key).Interface()
sliceValue := reflect.ValueOf(valueData)
if sliceValue.Kind() == reflect.Slice {
valueData2 := []any{}
for i := 0; i < sliceValue.Len(); i++ {
valueData2 = append(valueData2, sliceValue.Index(i).Interface())
}
result[fmt.Sprint(key.Interface())] = valueData2
} else {
result[fmt.Sprint(key.Interface())] = valueData
}
}
return result
}
func (obj *RequestOption) newBody(val any, valType int) (reader io.Reader, parseOrderMap *OrderMap, orderKey []string, err error) {
var isOrderMap bool
parseOrderMap, isOrderMap = val.(*OrderMap)
if isOrderMap {
if valType == readType {
readCon, err := parseOrderMap.MarshalJSON()
return bytes.NewReader(readCon), nil, nil, err
} else {
return nil, parseOrderMap, parseOrderMap.Keys(), nil
}
}
if valType == readType {
switch value := val.(type) {
case io.Reader:
obj.once = true
return value, nil, nil, nil
case string:
return bytes.NewReader(tools.StringToBytes(value)), nil, nil, nil
case []byte:
return bytes.NewReader(value), nil, nil, nil
default:
enData, err := gson.Encode(val)
if err != nil {
return nil, nil, nil, err
}
return bytes.NewReader(enData), nil, nil, nil
}
}
if mapData := any2Map(val); mapData != nil {
val = mapData
}
mapL:
switch value := val.(type) {
case *gson.Client:
if !value.IsObject() {
return nil, nil, nil, errors.New("body-type error")
}
orderMap := NewOrderMap()
for kk, vv := range value.Map() {
if vv.IsArray() {
valData := make([]any, len(vv.Array()))
for i, v := range vv.Array() {
valData[i] = v.Value()
}
orderMap.Set(kk, valData)
} else {
orderMap.Set(kk, vv.Value())
}
}
return nil, orderMap, nil, nil
case *OrderMap:
if mapData := any2Map(value.data); mapData != nil {
value.data = mapData
}
return nil, value, nil, nil
case map[string]any:
orderMap := NewOrderMap()
orderMap.data = value
orderMap.keys = maps.Keys(value)
return nil, orderMap, nil, nil
}
if val, err = gson.Decode(val); err != nil {
switch value := val.(type) {
case string:
return bytes.NewReader(tools.StringToBytes(value)), nil, nil, nil
case []byte:
return bytes.NewReader(value), nil, nil, nil
default:
return nil, nil, nil, err
}
}
goto mapL
}

322
client.go
View File

@@ -1,161 +1,161 @@
package requests
import (
"context"
"fmt"
"io"
"net/url"
"net/http"
"github.com/gospider007/gtls"
)
// Connection Management
type Client struct {
option ClientOption
transport *roundTripper
ctx context.Context
cnl context.CancelFunc
closed bool
}
var defaultClient, _ = NewClient(context.TODO())
// New Connection Management
func NewClient(preCtx context.Context, options ...ClientOption) (*Client, error) {
if preCtx == nil {
preCtx = context.TODO()
}
var option ClientOption
if len(options) > 0 {
option = options[0]
}
result := new(Client)
result.ctx, result.cnl = context.WithCancel(preCtx)
result.transport = newRoundTripper(result.ctx, option)
result.option = option
//cookiesjar
if !result.option.DisCookie {
if result.option.Jar == nil {
result.option.Jar = NewJar()
}
}
var err error
if result.option.Proxy != "" {
_, err = gtls.VerifyProxy(result.option.Proxy)
}
return result, err
}
// Modifying the client's proxy
func (obj *Client) SetProxy(proxyUrl string) (err error) {
_, err = gtls.VerifyProxy(proxyUrl)
if err == nil {
obj.option.Proxy = proxyUrl
}
return
}
// Modify the proxy method of the client
func (obj *Client) SetGetProxy(getProxy func(ctx context.Context, url *url.URL) (string, error)) {
obj.transport.setGetProxy(getProxy)
}
// Close idle connections. If the connection is in use, wait until it ends before closing
func (obj *Client) CloseConns() {
obj.transport.closeConns()
}
// Close the connection, even if it is in use, it will be closed
func (obj *Client) ForceCloseConns() {
obj.transport.forceCloseConns()
}
// Close the client and cannot be used again after shutdown
func (obj *Client) Close() {
obj.closed = true
obj.ForceCloseConns()
obj.cnl()
}
// func checkRedirect(req *http.Request, via []*http.Request) error {
// ctxData := GetReqCtxData(req.Context())
// if ctxData.maxRedirect == 0 || ctxData.maxRedirect >= len(via) {
// return nil
// }
// return http.ErrUseLastResponse
// }
func (obj *Client) do(req *http.Request, option *RequestOption) (resp *http.Response, err error) {
var redirectNum int
for {
redirectNum++
resp, err = obj.send(req, option)
if req.Body != nil {
req.Body.Close()
}
if err != nil {
return
}
if option.MaxRedirect < 0 { //dis redirect
return
}
if option.MaxRedirect > 0 && redirectNum > option.MaxRedirect {
return
}
loc := resp.Header.Get("Location")
if loc == "" {
return resp, nil
}
u, err := req.URL.Parse(loc)
if err != nil {
return resp, fmt.Errorf("failed to parse Location header %q: %v", loc, err)
}
ireq, err := NewRequestWithContext(req.Context(), http.MethodGet, u, nil)
if err != nil {
return resp, err
}
var shouldRedirect bool
ireq.Method, shouldRedirect, _ = redirectBehavior(req.Method, resp, ireq)
if !shouldRedirect {
return resp, nil
}
ireq.Response = resp
ireq.Header = defaultHeaders()
ireq.Header.Set("Referer", req.URL.String())
for key := range ireq.Header {
if val := req.Header.Get(key); val != "" {
ireq.Header.Set(key, val)
}
}
if getDomain(u) == getDomain(req.URL) {
if Authorization := req.Header.Get("Authorization"); Authorization != "" {
ireq.Header.Set("Authorization", Authorization)
}
cookies := Cookies(req.Cookies()).String()
if cookies != "" {
ireq.Header.Set("Cookie", cookies)
}
addCookie(ireq, resp.Cookies())
}
io.Copy(io.Discard, resp.Body)
resp.Body.Close()
req = ireq
}
}
func (obj *Client) send(req *http.Request, option *RequestOption) (resp *http.Response, err error) {
if option.Jar != nil {
addCookie(req, option.Jar.GetCookies(req.URL))
}
resp, err = obj.transport.RoundTrip(req)
if err != nil {
return nil, err
}
if option.Jar != nil {
if rc := resp.Cookies(); len(rc) > 0 {
option.Jar.SetCookies(req.URL, rc)
}
}
return resp, nil
}
package requests
import (
"context"
"fmt"
"io"
"net/url"
"net/http"
"github.com/gospider007/gtls"
)
// Connection Management
type Client struct {
option ClientOption
transport *roundTripper
ctx context.Context
cnl context.CancelFunc
closed bool
}
var defaultClient, _ = NewClient(context.TODO())
// New Connection Management
func NewClient(preCtx context.Context, options ...ClientOption) (*Client, error) {
if preCtx == nil {
preCtx = context.TODO()
}
var option ClientOption
if len(options) > 0 {
option = options[0]
}
result := new(Client)
result.ctx, result.cnl = context.WithCancel(preCtx)
result.transport = newRoundTripper(result.ctx, option)
result.option = option
//cookiesjar
if !result.option.DisCookie {
if result.option.Jar == nil {
result.option.Jar = NewJar()
}
}
var err error
if result.option.Proxy != "" {
_, err = gtls.VerifyProxy(result.option.Proxy)
}
return result, err
}
// Modifying the client's proxy
func (obj *Client) SetProxy(proxyUrl string) (err error) {
_, err = gtls.VerifyProxy(proxyUrl)
if err == nil {
obj.option.Proxy = proxyUrl
}
return
}
// Modify the proxy method of the client
func (obj *Client) SetGetProxy(getProxy func(ctx context.Context, url *url.URL) (string, error)) {
obj.transport.setGetProxy(getProxy)
}
// Close idle connections. If the connection is in use, wait until it ends before closing
func (obj *Client) CloseConns() {
obj.transport.closeConns()
}
// Close the connection, even if it is in use, it will be closed
func (obj *Client) ForceCloseConns() {
obj.transport.forceCloseConns()
}
// Close the client and cannot be used again after shutdown
func (obj *Client) Close() {
obj.closed = true
obj.ForceCloseConns()
obj.cnl()
}
// func checkRedirect(req *http.Request, via []*http.Request) error {
// ctxData := GetReqCtxData(req.Context())
// if ctxData.maxRedirect == 0 || ctxData.maxRedirect >= len(via) {
// return nil
// }
// return http.ErrUseLastResponse
// }
func (obj *Client) do(req *http.Request, option *RequestOption) (resp *http.Response, err error) {
var redirectNum int
for {
redirectNum++
resp, err = obj.send(req, option)
if req.Body != nil {
req.Body.Close()
}
if err != nil {
return
}
if option.MaxRedirect < 0 { //dis redirect
return
}
if option.MaxRedirect > 0 && redirectNum > option.MaxRedirect {
return
}
loc := resp.Header.Get("Location")
if loc == "" {
return resp, nil
}
u, err := req.URL.Parse(loc)
if err != nil {
return resp, fmt.Errorf("failed to parse Location header %q: %v", loc, err)
}
ireq, err := NewRequestWithContext(req.Context(), http.MethodGet, u, nil)
if err != nil {
return resp, err
}
var shouldRedirect bool
ireq.Method, shouldRedirect, _ = redirectBehavior(req.Method, resp, ireq)
if !shouldRedirect {
return resp, nil
}
ireq.Response = resp
ireq.Header = defaultHeaders()
ireq.Header.Set("Referer", req.URL.String())
for key := range ireq.Header {
if val := req.Header.Get(key); val != "" {
ireq.Header.Set(key, val)
}
}
if getDomain(u) == getDomain(req.URL) {
if Authorization := req.Header.Get("Authorization"); Authorization != "" {
ireq.Header.Set("Authorization", Authorization)
}
cookies := Cookies(req.Cookies()).String()
if cookies != "" {
ireq.Header.Set("Cookie", cookies)
}
addCookie(ireq, resp.Cookies())
}
io.Copy(io.Discard, resp.Body)
resp.Body.Close()
req = ireq
}
}
func (obj *Client) send(req *http.Request, option *RequestOption) (resp *http.Response, err error) {
if option.Jar != nil {
addCookie(req, option.Jar.GetCookies(req.URL))
}
resp, err = obj.transport.RoundTrip(req)
if err != nil {
return nil, err
}
if option.Jar != nil {
if rc := resp.Cookies(); len(rc) > 0 {
option.Jar.SetCookies(req.URL, rc)
}
}
return resp, nil
}

528
conn.go
View File

@@ -1,264 +1,264 @@
package requests
import (
"bufio"
"context"
"errors"
"io"
"net"
"net/textproto"
"sync"
"sync/atomic"
"time"
"github.com/gospider007/net/http2"
"github.com/gospider007/tools"
)
type connecotr struct {
deleteCtx context.Context //force close
deleteCnl context.CancelCauseFunc
closeCtx context.Context //safe close
closeCnl context.CancelCauseFunc
bodyCtx context.Context //body close
bodyCnl context.CancelCauseFunc
rawConn net.Conn
h2RawConn *http2.ClientConn
proxy string
r *textproto.Reader
w *bufio.Writer
pr *pipCon
inPool bool
}
func (obj *connecotr) withCancel(deleteCtx context.Context, closeCtx context.Context) {
obj.deleteCtx, obj.deleteCnl = context.WithCancelCause(deleteCtx)
obj.closeCtx, obj.closeCnl = context.WithCancelCause(closeCtx)
}
func (obj *connecotr) Close() error {
obj.deleteCnl(errors.New("connecotr close"))
if obj.pr != nil {
obj.pr.Close(errors.New("connecotr close"))
}
if obj.h2RawConn != nil {
obj.h2RawConn.Close()
}
return obj.rawConn.Close()
}
func (obj *connecotr) read() (err error) {
if obj.pr != nil {
return nil
}
var pw *pipCon
obj.pr, pw = pipe(obj.deleteCtx)
if _, err = io.Copy(pw, obj.rawConn); err == nil {
err = io.EOF
}
pw.Close(err)
obj.Close()
return
}
func (obj *connecotr) Read(b []byte) (i int, err error) {
if obj.pr == nil {
return obj.rawConn.Read(b)
}
return obj.pr.Read(b)
}
func (obj *connecotr) Write(b []byte) (int, error) {
return obj.rawConn.Write(b)
}
func (obj *connecotr) LocalAddr() net.Addr {
return obj.rawConn.LocalAddr()
}
func (obj *connecotr) RemoteAddr() net.Addr {
return obj.rawConn.RemoteAddr()
}
func (obj *connecotr) SetDeadline(t time.Time) error {
return obj.rawConn.SetDeadline(t)
}
func (obj *connecotr) SetReadDeadline(t time.Time) error {
return obj.rawConn.SetReadDeadline(t)
}
func (obj *connecotr) SetWriteDeadline(t time.Time) error {
return obj.rawConn.SetWriteDeadline(t)
}
func (obj *connecotr) h2Closed() bool {
if obj.h2RawConn == nil {
return false
}
state := obj.h2RawConn.State()
return state.Closed || state.Closing
}
func (obj *connecotr) wrapBody(task *reqTask) {
body := new(readWriteCloser)
obj.bodyCtx, obj.bodyCnl = context.WithCancelCause(task.req.Context())
body.body = task.res.Body
body.conn = obj
task.res.Body = body
}
func (obj *connecotr) http1Req(task *reqTask) {
if task.err = httpWrite(task.req, obj.w, task.orderHeaders); task.err == nil {
task.res, task.err = readResponse(obj.r, task.req)
if task.err != nil {
task.err = tools.WrapError(task.err, "http1 read error")
} else if task.res == nil {
task.err = errors.New("response is nil")
} else {
obj.wrapBody(task)
}
}
task.cnl()
}
func (obj *connecotr) http2Req(task *reqTask) {
if task.res, task.err = obj.h2RawConn.RoundTripWithOrderHeaders(task.req, task.orderHeaders2); task.res != nil && task.err == nil {
obj.wrapBody(task)
} else if task.err != nil {
task.err = tools.WrapError(task.err, "http2 roundTrip error")
}
task.cnl()
}
func (obj *connecotr) waitBodyClose() error {
select {
case <-obj.bodyCtx.Done(): //wait body close
if err := context.Cause(obj.bodyCtx); errors.Is(err, ErrgospiderBodyClose) {
return nil
} else {
return err
}
case <-obj.deleteCtx.Done(): //force conn close
return tools.WrapError(context.Cause(obj.deleteCtx), "delete ctx error: ")
}
}
func (obj *connecotr) taskMain(task *reqTask, waitBody bool) (retry bool) {
defer func() {
if retry || task.err != nil {
obj.Close()
}
}()
if obj.h2Closed() {
return true
}
select {
case <-obj.closeCtx.Done():
return true
default:
}
if obj.h2RawConn != nil {
go obj.http2Req(task)
} else {
go obj.http1Req(task)
}
select {
case <-task.ctx.Done():
if task.err != nil {
return false
}
if task.res == nil {
task.err = task.ctx.Err()
if task.err == nil {
task.err = errors.New("response is nil")
}
return false
}
if waitBody {
task.err = obj.waitBodyClose()
}
return false
case <-obj.deleteCtx.Done(): //force conn close
task.err = tools.WrapError(obj.deleteCtx.Err(), "delete ctx error: ")
task.cnl()
return false
}
}
type connPool struct {
deleteCtx context.Context
deleteCnl context.CancelCauseFunc
closeCtx context.Context
closeCnl context.CancelCauseFunc
connKey string
total atomic.Int64
tasks chan *reqTask
connPools *connPools
}
type connPools struct {
connPools sync.Map
}
func newConnPools() *connPools {
return new(connPools)
}
func (obj *connPools) get(key string) *connPool {
val, ok := obj.connPools.Load(key)
if !ok {
return nil
}
return val.(*connPool)
}
func (obj *connPools) set(key string, pool *connPool) {
obj.connPools.Store(key, pool)
}
func (obj *connPools) del(key string) {
obj.connPools.Delete(key)
}
func (obj *connPools) iter(f func(key string, value *connPool) bool) {
obj.connPools.Range(func(key, value any) bool {
return f(key.(string), value.(*connPool))
})
}
func (obj *connPool) notice(task *reqTask) {
select {
case obj.tasks <- task:
case task.emptyPool <- struct{}{}:
}
}
func (obj *connPool) rwMain(conn *connecotr) {
conn.withCancel(obj.deleteCtx, obj.closeCtx)
defer func() {
conn.Close()
obj.total.Add(-1)
if obj.total.Load() <= 0 {
obj.close()
}
}()
if err := conn.waitBodyClose(); err != nil {
return
}
for {
select {
case <-conn.closeCtx.Done(): //safe close conn
return
case task := <-obj.tasks: //recv task
if task == nil {
return
}
if conn.taskMain(task, true) {
obj.notice(task)
return
}
if task.err != nil {
return
}
}
}
}
func (obj *connPool) forceClose() {
obj.close()
obj.deleteCnl(errors.New("connPool forceClose"))
}
func (obj *connPool) close() {
obj.connPools.del(obj.connKey)
obj.closeCnl(errors.New("connPool close"))
}
package requests
import (
"bufio"
"context"
"errors"
"io"
"net"
"net/textproto"
"sync"
"sync/atomic"
"time"
"github.com/gospider007/net/http2"
"github.com/gospider007/tools"
)
type connecotr struct {
deleteCtx context.Context //force close
deleteCnl context.CancelCauseFunc
closeCtx context.Context //safe close
closeCnl context.CancelCauseFunc
bodyCtx context.Context //body close
bodyCnl context.CancelCauseFunc
rawConn net.Conn
h2RawConn *http2.ClientConn
proxy string
r *textproto.Reader
w *bufio.Writer
pr *pipCon
inPool bool
}
func (obj *connecotr) withCancel(deleteCtx context.Context, closeCtx context.Context) {
obj.deleteCtx, obj.deleteCnl = context.WithCancelCause(deleteCtx)
obj.closeCtx, obj.closeCnl = context.WithCancelCause(closeCtx)
}
func (obj *connecotr) Close() error {
obj.deleteCnl(errors.New("connecotr close"))
if obj.pr != nil {
obj.pr.Close(errors.New("connecotr close"))
}
if obj.h2RawConn != nil {
obj.h2RawConn.Close()
}
return obj.rawConn.Close()
}
func (obj *connecotr) read() (err error) {
if obj.pr != nil {
return nil
}
var pw *pipCon
obj.pr, pw = pipe(obj.deleteCtx)
if _, err = io.Copy(pw, obj.rawConn); err == nil {
err = io.EOF
}
pw.Close(err)
obj.Close()
return
}
func (obj *connecotr) Read(b []byte) (i int, err error) {
if obj.pr == nil {
return obj.rawConn.Read(b)
}
return obj.pr.Read(b)
}
func (obj *connecotr) Write(b []byte) (int, error) {
return obj.rawConn.Write(b)
}
func (obj *connecotr) LocalAddr() net.Addr {
return obj.rawConn.LocalAddr()
}
func (obj *connecotr) RemoteAddr() net.Addr {
return obj.rawConn.RemoteAddr()
}
func (obj *connecotr) SetDeadline(t time.Time) error {
return obj.rawConn.SetDeadline(t)
}
func (obj *connecotr) SetReadDeadline(t time.Time) error {
return obj.rawConn.SetReadDeadline(t)
}
func (obj *connecotr) SetWriteDeadline(t time.Time) error {
return obj.rawConn.SetWriteDeadline(t)
}
func (obj *connecotr) h2Closed() bool {
if obj.h2RawConn == nil {
return false
}
state := obj.h2RawConn.State()
return state.Closed || state.Closing
}
func (obj *connecotr) wrapBody(task *reqTask) {
body := new(readWriteCloser)
obj.bodyCtx, obj.bodyCnl = context.WithCancelCause(task.req.Context())
body.body = task.res.Body
body.conn = obj
task.res.Body = body
}
func (obj *connecotr) http1Req(task *reqTask) {
if task.err = httpWrite(task.req, obj.w, task.orderHeaders); task.err == nil {
task.res, task.err = readResponse(obj.r, task.req)
if task.err != nil {
task.err = tools.WrapError(task.err, "http1 read error")
} else if task.res == nil {
task.err = errors.New("response is nil")
} else {
obj.wrapBody(task)
}
}
task.cnl()
}
func (obj *connecotr) http2Req(task *reqTask) {
if task.res, task.err = obj.h2RawConn.RoundTripWithOrderHeaders(task.req, task.orderHeaders2); task.res != nil && task.err == nil {
obj.wrapBody(task)
} else if task.err != nil {
task.err = tools.WrapError(task.err, "http2 roundTrip error")
}
task.cnl()
}
func (obj *connecotr) waitBodyClose() error {
select {
case <-obj.bodyCtx.Done(): //wait body close
if err := context.Cause(obj.bodyCtx); errors.Is(err, ErrgospiderBodyClose) {
return nil
} else {
return err
}
case <-obj.deleteCtx.Done(): //force conn close
return tools.WrapError(context.Cause(obj.deleteCtx), "delete ctx error: ")
}
}
func (obj *connecotr) taskMain(task *reqTask, waitBody bool) (retry bool) {
defer func() {
if retry || task.err != nil {
obj.Close()
}
}()
if obj.h2Closed() {
return true
}
select {
case <-obj.closeCtx.Done():
return true
default:
}
if obj.h2RawConn != nil {
go obj.http2Req(task)
} else {
go obj.http1Req(task)
}
select {
case <-task.ctx.Done():
if task.err != nil {
return false
}
if task.res == nil {
task.err = task.ctx.Err()
if task.err == nil {
task.err = errors.New("response is nil")
}
return false
}
if waitBody {
task.err = obj.waitBodyClose()
}
return false
case <-obj.deleteCtx.Done(): //force conn close
task.err = tools.WrapError(obj.deleteCtx.Err(), "delete ctx error: ")
task.cnl()
return false
}
}
type connPool struct {
deleteCtx context.Context
deleteCnl context.CancelCauseFunc
closeCtx context.Context
closeCnl context.CancelCauseFunc
connKey string
total atomic.Int64
tasks chan *reqTask
connPools *connPools
}
type connPools struct {
connPools sync.Map
}
func newConnPools() *connPools {
return new(connPools)
}
func (obj *connPools) get(key string) *connPool {
val, ok := obj.connPools.Load(key)
if !ok {
return nil
}
return val.(*connPool)
}
func (obj *connPools) set(key string, pool *connPool) {
obj.connPools.Store(key, pool)
}
func (obj *connPools) del(key string) {
obj.connPools.Delete(key)
}
func (obj *connPools) iter(f func(key string, value *connPool) bool) {
obj.connPools.Range(func(key, value any) bool {
return f(key.(string), value.(*connPool))
})
}
func (obj *connPool) notice(task *reqTask) {
select {
case obj.tasks <- task:
case task.emptyPool <- struct{}{}:
}
}
func (obj *connPool) rwMain(conn *connecotr) {
conn.withCancel(obj.deleteCtx, obj.closeCtx)
defer func() {
conn.Close()
obj.total.Add(-1)
if obj.total.Load() <= 0 {
obj.close()
}
}()
if err := conn.waitBodyClose(); err != nil {
return
}
for {
select {
case <-conn.closeCtx.Done(): //safe close conn
return
case task := <-obj.tasks: //recv task
if task == nil {
return
}
if conn.taskMain(task, true) {
obj.notice(task)
return
}
if task.err != nil {
return
}
}
}
}
func (obj *connPool) forceClose() {
obj.close()
obj.deleteCnl(errors.New("connPool forceClose"))
}
func (obj *connPool) close() {
obj.connPools.del(obj.connKey)
obj.closeCnl(errors.New("connPool close"))
}

368
cookies.go
View File

@@ -1,184 +1,184 @@
package requests
import (
"errors"
"fmt"
"net/http"
"strings"
"github.com/gospider007/gson"
)
// cookies
type Cookies []*http.Cookie
// return cookies with string,join with '; '
func (obj Cookies) String() string {
cooks := []string{}
for _, cook := range obj {
cooks = append(cooks, fmt.Sprintf("%s=%s", cook.Name, cook.Value))
}
return strings.Join(cooks, "; ")
}
// get cookies by name
func (obj Cookies) Gets(name string) Cookies {
var result Cookies
for _, cook := range obj {
if cook.Name == name {
result = append(result, cook)
}
}
return result
}
// get cookie by name
func (obj Cookies) Get(name string) *http.Cookie {
vals := obj.Gets(name)
if i := len(vals); i == 0 {
return nil
} else {
return vals[i-1]
}
}
// get cookie values by name, return []string
func (obj Cookies) GetVals(name string) []string {
var result []string
for _, cook := range obj {
if cook.Name == name {
result = append(result, cook.Value)
}
}
return result
}
// get cookie value by name,return string
func (obj Cookies) GetVal(name string) string {
vals := obj.GetVals(name)
if i := len(vals); i == 0 {
return ""
} else {
return vals[i-1]
}
}
func (obj Cookies) append(cook *http.Cookie) Cookies {
return append(obj, cook)
}
// read cookies or parse cookies,support json,map,[]string,http.Header,string
func ReadCookies(val any) (Cookies, error) {
switch cook := val.(type) {
case *http.Cookie:
return Cookies{
cook,
}, nil
case http.Cookie:
return Cookies{
&cook,
}, nil
case Cookies:
return cook, nil
case []*http.Cookie:
return Cookies(cook), nil
case string:
return readCookies(http.Header{"Cookie": []string{cook}}, ""), nil
case http.Header:
return readCookies(cook, ""), nil
case []string:
return readCookies(http.Header{"Cookie": cook}, ""), nil
default:
return any2Cookies(cook)
}
}
// read set cookies or parse set cookies,support json,map,[]string,http.Header,string
func ReadSetCookies(val any) (Cookies, error) {
switch cook := val.(type) {
case Cookies:
return cook, nil
case []*http.Cookie:
return Cookies(cook), nil
case string:
return readSetCookies(http.Header{"Set-Cookie": []string{cook}}), nil
case http.Header:
return readSetCookies(cook), nil
case []string:
return readSetCookies(http.Header{"Set-Cookie": cook}), nil
default:
return any2Cookies(cook)
}
}
func any2Cookies(val any) (Cookies, error) {
switch cooks := val.(type) {
case map[string]string:
cookies := Cookies{}
for kk, vv := range cooks {
cookies = append(cookies, &http.Cookie{
Name: kk,
Value: vv,
})
}
return cookies, nil
case map[string][]string:
cookies := Cookies{}
for kk, vvs := range cooks {
for _, vv := range vvs {
cookies = append(cookies, &http.Cookie{
Name: kk,
Value: vv,
})
}
}
return cookies, nil
case *gson.Client:
if !cooks.IsObject() {
return nil, errors.New("cookies not support type")
}
cookies := Cookies{}
for kk, vvs := range cooks.Map() {
if vvs.IsArray() {
for _, vv := range vvs.Array() {
cookies = append(cookies, &http.Cookie{
Name: kk,
Value: vv.String(),
})
}
} else {
cookies = append(cookies, &http.Cookie{
Name: kk,
Value: vvs.String(),
})
}
}
return cookies, nil
default:
jsonData, err := gson.Decode(cooks)
if err != nil {
return nil, err
}
cookies := Cookies{}
for kk, vvs := range jsonData.Map() {
if vvs.IsArray() {
for _, vv := range vvs.Array() {
cookies = append(cookies, &http.Cookie{
Name: kk,
Value: vv.String(),
})
}
} else {
cookies = append(cookies, &http.Cookie{
Name: kk,
Value: vvs.String(),
})
}
}
return cookies, nil
}
}
func (obj *RequestOption) initCookies() (Cookies, error) {
if obj.Cookies == nil {
return nil, nil
}
return ReadCookies(obj.Cookies)
}
package requests
import (
"errors"
"fmt"
"net/http"
"strings"
"github.com/gospider007/gson"
)
// cookies
type Cookies []*http.Cookie
// return cookies with string,join with '; '
func (obj Cookies) String() string {
cooks := []string{}
for _, cook := range obj {
cooks = append(cooks, fmt.Sprintf("%s=%s", cook.Name, cook.Value))
}
return strings.Join(cooks, "; ")
}
// get cookies by name
func (obj Cookies) Gets(name string) Cookies {
var result Cookies
for _, cook := range obj {
if cook.Name == name {
result = append(result, cook)
}
}
return result
}
// get cookie by name
func (obj Cookies) Get(name string) *http.Cookie {
vals := obj.Gets(name)
if i := len(vals); i == 0 {
return nil
} else {
return vals[i-1]
}
}
// get cookie values by name, return []string
func (obj Cookies) GetVals(name string) []string {
var result []string
for _, cook := range obj {
if cook.Name == name {
result = append(result, cook.Value)
}
}
return result
}
// get cookie value by name,return string
func (obj Cookies) GetVal(name string) string {
vals := obj.GetVals(name)
if i := len(vals); i == 0 {
return ""
} else {
return vals[i-1]
}
}
func (obj Cookies) append(cook *http.Cookie) Cookies {
return append(obj, cook)
}
// read cookies or parse cookies,support json,map,[]string,http.Header,string
func ReadCookies(val any) (Cookies, error) {
switch cook := val.(type) {
case *http.Cookie:
return Cookies{
cook,
}, nil
case http.Cookie:
return Cookies{
&cook,
}, nil
case Cookies:
return cook, nil
case []*http.Cookie:
return Cookies(cook), nil
case string:
return readCookies(http.Header{"Cookie": []string{cook}}, ""), nil
case http.Header:
return readCookies(cook, ""), nil
case []string:
return readCookies(http.Header{"Cookie": cook}, ""), nil
default:
return any2Cookies(cook)
}
}
// read set cookies or parse set cookies,support json,map,[]string,http.Header,string
func ReadSetCookies(val any) (Cookies, error) {
switch cook := val.(type) {
case Cookies:
return cook, nil
case []*http.Cookie:
return Cookies(cook), nil
case string:
return readSetCookies(http.Header{"Set-Cookie": []string{cook}}), nil
case http.Header:
return readSetCookies(cook), nil
case []string:
return readSetCookies(http.Header{"Set-Cookie": cook}), nil
default:
return any2Cookies(cook)
}
}
func any2Cookies(val any) (Cookies, error) {
switch cooks := val.(type) {
case map[string]string:
cookies := Cookies{}
for kk, vv := range cooks {
cookies = append(cookies, &http.Cookie{
Name: kk,
Value: vv,
})
}
return cookies, nil
case map[string][]string:
cookies := Cookies{}
for kk, vvs := range cooks {
for _, vv := range vvs {
cookies = append(cookies, &http.Cookie{
Name: kk,
Value: vv,
})
}
}
return cookies, nil
case *gson.Client:
if !cooks.IsObject() {
return nil, errors.New("cookies not support type")
}
cookies := Cookies{}
for kk, vvs := range cooks.Map() {
if vvs.IsArray() {
for _, vv := range vvs.Array() {
cookies = append(cookies, &http.Cookie{
Name: kk,
Value: vv.String(),
})
}
} else {
cookies = append(cookies, &http.Cookie{
Name: kk,
Value: vvs.String(),
})
}
}
return cookies, nil
default:
jsonData, err := gson.Decode(cooks)
if err != nil {
return nil, err
}
cookies := Cookies{}
for kk, vvs := range jsonData.Map() {
if vvs.IsArray() {
for _, vv := range vvs.Array() {
cookies = append(cookies, &http.Cookie{
Name: kk,
Value: vv.String(),
})
}
} else {
cookies = append(cookies, &http.Cookie{
Name: kk,
Value: vvs.String(),
})
}
}
return cookies, nil
}
}
func (obj *RequestOption) initCookies() (Cookies, error) {
if obj.Cookies == nil {
return nil, nil
}
return ReadCookies(obj.Cookies)
}

836
dial.go
View File

@@ -1,418 +1,418 @@
package requests
import (
"bufio"
"context"
"crypto/tls"
"errors"
"io"
"net"
"net/textproto"
"net/url"
"sync"
"time"
"net/http"
"github.com/gospider007/gtls"
"github.com/gospider007/ja3"
"github.com/gospider007/tools"
utls "github.com/refraction-networking/utls"
)
type DialClient struct {
dialer *net.Dialer
dnsIpData sync.Map
dns *net.UDPAddr
getAddrType func(host string) gtls.AddrType
}
type msgClient struct {
time time.Time
host string
}
type DialOption struct {
DialTimeout time.Duration
KeepAlive time.Duration
LocalAddr *net.TCPAddr //network card ip
AddrType gtls.AddrType //first ip type
Dns *net.UDPAddr
GetAddrType func(host string) gtls.AddrType
}
func NewDialer(option DialOption) *net.Dialer {
if option.KeepAlive == 0 {
option.KeepAlive = time.Second * 30
}
if option.DialTimeout == 0 {
option.DialTimeout = time.Second * 15
}
dialer := &net.Dialer{
Timeout: option.DialTimeout,
KeepAlive: option.KeepAlive,
LocalAddr: option.LocalAddr,
}
if option.LocalAddr != nil {
dialer.LocalAddr = option.LocalAddr
}
if option.Dns != nil {
dialer.Resolver = &net.Resolver{
PreferGo: true,
Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
return (&net.Dialer{
Timeout: option.DialTimeout,
KeepAlive: option.KeepAlive,
}).DialContext(ctx, network, option.Dns.String())
},
}
}
dialer.SetMultipathTCP(true)
return dialer
}
func NewDail(option DialOption) *DialClient {
return &DialClient{
dialer: NewDialer(option),
dns: option.Dns,
getAddrType: option.GetAddrType,
}
}
func (obj *DialClient) DialContext(ctx context.Context, ctxData *reqCtxData, network string, addr string) (net.Conn, error) {
if ctxData == nil {
ctxData = &reqCtxData{}
}
host, port, err := net.SplitHostPort(addr)
if err != nil {
return nil, tools.WrapError(err, "addrToIp error,SplitHostPort")
}
var dialer *net.Dialer
if _, ipInt := gtls.ParseHost(host); ipInt == 0 { //domain
host, ok := obj.loadHost(host)
if !ok { //dns parse
dialer = obj.getDialer(ctxData, true)
var addrType gtls.AddrType
if ctxData.addrType != 0 {
addrType = ctxData.addrType
} else if obj.getAddrType != nil {
addrType = obj.getAddrType(host)
}
ips, err := dialer.Resolver.LookupIPAddr(ctx, host)
if err != nil {
return nil, err
}
if host, err = obj.addrToIp(host, ips, addrType); err != nil {
return nil, err
}
addr = net.JoinHostPort(host, port)
}
}
if dialer == nil {
dialer = obj.getDialer(ctxData, false)
}
return dialer.DialContext(ctx, network, addr)
}
func (obj *DialClient) DialContextWithProxy(ctx context.Context, ctxData *reqCtxData, network string, scheme string, addr string, host string, proxyUrl *url.URL, tlsConfig *tls.Config) (net.Conn, error) {
if ctxData == nil {
ctxData = &reqCtxData{}
}
if proxyUrl == nil {
return obj.DialContext(ctx, ctxData, network, addr)
}
if proxyUrl.Port() == "" {
if proxyUrl.Scheme == "http" {
proxyUrl.Host = net.JoinHostPort(proxyUrl.Hostname(), "80")
} else if proxyUrl.Scheme == "https" {
proxyUrl.Host = net.JoinHostPort(proxyUrl.Hostname(), "443")
}
}
switch proxyUrl.Scheme {
case "http", "https":
conn, err := obj.DialContext(ctx, ctxData, network, net.JoinHostPort(proxyUrl.Hostname(), proxyUrl.Port()))
if err != nil {
return conn, err
} else if proxyUrl.Scheme == "https" {
if conn, err = obj.addTls(ctx, conn, proxyUrl.Host, true, tlsConfig); err != nil {
return conn, err
}
}
return conn, obj.clientVerifyHttps(ctx, scheme, proxyUrl, addr, host, conn)
case "socks5":
return obj.Socks5Proxy(ctx, ctxData, network, addr, proxyUrl)
default:
return nil, errors.New("proxyUrl Scheme error")
}
}
func (obj *DialClient) loadHost(host string) (string, bool) {
msgDataAny, ok := obj.dnsIpData.Load(host)
if ok {
msgdata := msgDataAny.(msgClient)
if time.Since(msgdata.time) < time.Second*60*5 {
return msgdata.host, true
}
}
return host, false
}
func (obj *DialClient) addrToIp(host string, ips []net.IPAddr, addrType gtls.AddrType) (string, error) {
ip, err := obj.lookupIPAddr(ips, addrType)
if err != nil {
return host, tools.WrapError(err, "addrToIp error,lookupIPAddr")
}
obj.dnsIpData.Store(host, msgClient{time: time.Now(), host: ip.String()})
return ip.String(), nil
}
func (obj *DialClient) clientVerifySocks5(proxyUrl *url.URL, addr string, conn net.Conn) (err error) {
if _, err = conn.Write([]byte{5, 2, 0, 2}); err != nil {
return
}
readCon := make([]byte, 4)
if _, err = io.ReadFull(conn, readCon[:2]); err != nil {
return
}
switch readCon[1] {
case 2:
if proxyUrl.User == nil {
err = errors.New("socks5 need auth")
return
}
pwd, pwdOk := proxyUrl.User.Password()
if !pwdOk {
err = errors.New("socks5 auth error")
return
}
usr := proxyUrl.User.Username()
if usr == "" {
err = errors.New("socks5 auth user format error")
return
}
if _, err = conn.Write(append(
append(
[]byte{1, byte(len(usr))},
tools.StringToBytes(usr)...,
),
append(
[]byte{byte(len(pwd))},
tools.StringToBytes(pwd)...,
)...,
)); err != nil {
return
}
if _, err = io.ReadFull(conn, readCon[:2]); err != nil {
return
}
switch readCon[1] {
case 0:
default:
err = errors.New("socks5 auth error")
return
}
case 0:
default:
err = errors.New("not support auth format")
return
}
var host string
var port int
if host, port, err = gtls.SplitHostPort(addr); err != nil {
return
}
writeCon := []byte{5, 1, 0}
ip, ipInt := gtls.ParseHost(host)
switch ipInt {
case 4:
writeCon = append(writeCon, 1)
writeCon = append(writeCon, ip...)
case 6:
writeCon = append(writeCon, 4)
writeCon = append(writeCon, ip...)
case 0:
if len(host) > 255 {
err = errors.New("FQDN too long")
return
}
writeCon = append(writeCon, 3)
writeCon = append(writeCon, byte(len(host)))
writeCon = append(writeCon, host...)
}
writeCon = append(writeCon, byte(port>>8), byte(port))
if _, err = conn.Write(writeCon); err != nil {
return
}
if _, err = io.ReadFull(conn, readCon); err != nil {
return
}
if readCon[0] != 5 {
err = errors.New("socks version error")
return
}
if readCon[1] != 0 {
err = errors.New("socks conn error")
return
}
if readCon[3] != 1 {
err = errors.New("socks conn type error")
return
}
switch readCon[3] {
case 1: //ipv4
if _, err = io.ReadFull(conn, readCon); err != nil {
return
}
case 3: //domain
if _, err = io.ReadFull(conn, readCon[:1]); err != nil {
return
}
if _, err = io.ReadFull(conn, make([]byte, readCon[0])); err != nil {
return
}
case 4: //IPv6
if _, err = io.ReadFull(conn, make([]byte, 16)); err != nil {
return
}
default:
err = errors.New("invalid atyp")
return
}
_, err = io.ReadFull(conn, readCon[:2])
return
}
func (obj *DialClient) lookupIPAddr(ips []net.IPAddr, addrType gtls.AddrType) (net.IP, error) {
for _, ipAddr := range ips {
ip := ipAddr.IP
if ipType := gtls.ParseIp(ip); ipType == 4 || ipType == 6 {
if addrType == 0 || addrType == ipType {
return ip, nil
}
}
}
for _, ipAddr := range ips {
ip := ipAddr.IP
if ipType := gtls.ParseIp(ip); ipType == 4 || ipType == 6 {
return ip, nil
}
}
return nil, errors.New("dns parse host error")
}
func (obj *DialClient) getDialer(ctxData *reqCtxData, parseDns bool) *net.Dialer {
var dialOption DialOption
var isNew bool
if ctxData.dialTimeout == 0 {
dialOption.DialTimeout = obj.dialer.Timeout
} else {
dialOption.DialTimeout = ctxData.dialTimeout
if ctxData.dialTimeout != obj.dialer.Timeout {
isNew = true
}
}
if ctxData.keepAlive == 0 {
dialOption.KeepAlive = obj.dialer.KeepAlive
} else {
dialOption.KeepAlive = ctxData.keepAlive
if ctxData.keepAlive != obj.dialer.KeepAlive {
isNew = true
}
}
if ctxData.localAddr == nil {
if obj.dialer.LocalAddr != nil {
dialOption.LocalAddr = obj.dialer.LocalAddr.(*net.TCPAddr)
}
} else {
dialOption.LocalAddr = ctxData.localAddr
if ctxData.localAddr.String() != obj.dialer.LocalAddr.String() {
isNew = true
}
}
if ctxData.dns == nil {
dialOption.Dns = obj.dns
} else {
dialOption.Dns = ctxData.dns
if parseDns && ctxData.dns.String() != obj.dns.String() {
isNew = true
}
}
if isNew {
return NewDialer(dialOption)
} else {
return obj.dialer
}
}
func (obj *DialClient) addTls(ctx context.Context, conn net.Conn, host string, disHttp2 bool, tlsConfig *tls.Config) (*tls.Conn, error) {
var tlsConn *tls.Conn
tlsConfig.ServerName = gtls.GetServerName(host)
if disHttp2 {
tlsConfig.NextProtos = []string{"http/1.1"}
} else {
tlsConfig.NextProtos = []string{"h2", "http/1.1"}
}
tlsConn = tls.Client(conn, tlsConfig)
return tlsConn, tlsConn.HandshakeContext(ctx)
}
func (obj *DialClient) addJa3Tls(ctx context.Context, conn net.Conn, host string, disHttp2 bool, ja3Spec ja3.Ja3Spec, tlsConfig *utls.Config) (*utls.UConn, error) {
tlsConfig.ServerName = gtls.GetServerName(host)
if disHttp2 {
tlsConfig.NextProtos = []string{"http/1.1"}
} else {
tlsConfig.NextProtos = []string{"h2", "http/1.1"}
}
return ja3.NewClient(ctx, conn, ja3Spec, disHttp2, tlsConfig)
}
func (obj *DialClient) Socks5Proxy(ctx context.Context, ctxData *reqCtxData, network string, addr string, proxyUrl *url.URL) (conn net.Conn, err error) {
if conn, err = obj.DialContext(ctx, ctxData, network, net.JoinHostPort(proxyUrl.Hostname(), proxyUrl.Port())); err != nil {
return
}
didVerify := make(chan struct{})
go func() {
if err = obj.clientVerifySocks5(proxyUrl, addr, conn); err != nil {
conn.Close()
}
close(didVerify)
}()
select {
case <-ctx.Done():
return conn, ctx.Err()
case <-didVerify:
return
}
}
func (obj *DialClient) clientVerifyHttps(ctx context.Context, scheme string, proxyUrl *url.URL, addr string, host string, conn net.Conn) (err error) {
hdr := make(http.Header)
hdr.Set("User-Agent", UserAgent)
if proxyUrl.User != nil {
if password, ok := proxyUrl.User.Password(); ok {
hdr.Set("Proxy-Authorization", "Basic "+tools.Base64Encode(proxyUrl.User.Username()+":"+password))
}
}
cHost := host
_, hport, _ := net.SplitHostPort(host)
if hport == "" {
_, aport, _ := net.SplitHostPort(addr)
if aport != "" {
cHost = net.JoinHostPort(cHost, aport)
} else if scheme == "http" {
cHost = net.JoinHostPort(cHost, "80")
} else if scheme == "https" {
cHost = net.JoinHostPort(cHost, "443")
} else {
return errors.New("clientVerifyHttps not found port")
}
}
connectReq, err := NewRequestWithContext(ctx, http.MethodConnect, &url.URL{Opaque: addr}, nil)
if err != nil {
return err
}
connectReq.Header = hdr
connectReq.Host = cHost
if err = connectReq.Write(conn); err != nil {
return err
}
resp, err := readResponse(textproto.NewReader(bufio.NewReader(conn)), connectReq)
if err != nil {
return err
}
if resp.StatusCode != 200 {
return errors.New(resp.Status)
}
return
}
package requests
import (
"bufio"
"context"
"crypto/tls"
"errors"
"io"
"net"
"net/textproto"
"net/url"
"sync"
"time"
"net/http"
"github.com/gospider007/gtls"
"github.com/gospider007/ja3"
"github.com/gospider007/tools"
utls "github.com/refraction-networking/utls"
)
type DialClient struct {
dialer *net.Dialer
dnsIpData sync.Map
dns *net.UDPAddr
getAddrType func(host string) gtls.AddrType
}
type msgClient struct {
time time.Time
host string
}
type DialOption struct {
DialTimeout time.Duration
KeepAlive time.Duration
LocalAddr *net.TCPAddr //network card ip
AddrType gtls.AddrType //first ip type
Dns *net.UDPAddr
GetAddrType func(host string) gtls.AddrType
}
func NewDialer(option DialOption) *net.Dialer {
if option.KeepAlive == 0 {
option.KeepAlive = time.Second * 30
}
if option.DialTimeout == 0 {
option.DialTimeout = time.Second * 15
}
dialer := &net.Dialer{
Timeout: option.DialTimeout,
KeepAlive: option.KeepAlive,
LocalAddr: option.LocalAddr,
}
if option.LocalAddr != nil {
dialer.LocalAddr = option.LocalAddr
}
if option.Dns != nil {
dialer.Resolver = &net.Resolver{
PreferGo: true,
Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
return (&net.Dialer{
Timeout: option.DialTimeout,
KeepAlive: option.KeepAlive,
}).DialContext(ctx, network, option.Dns.String())
},
}
}
dialer.SetMultipathTCP(true)
return dialer
}
func NewDail(option DialOption) *DialClient {
return &DialClient{
dialer: NewDialer(option),
dns: option.Dns,
getAddrType: option.GetAddrType,
}
}
func (obj *DialClient) DialContext(ctx context.Context, ctxData *reqCtxData, network string, addr string) (net.Conn, error) {
if ctxData == nil {
ctxData = &reqCtxData{}
}
host, port, err := net.SplitHostPort(addr)
if err != nil {
return nil, tools.WrapError(err, "addrToIp error,SplitHostPort")
}
var dialer *net.Dialer
if _, ipInt := gtls.ParseHost(host); ipInt == 0 { //domain
host, ok := obj.loadHost(host)
if !ok { //dns parse
dialer = obj.getDialer(ctxData, true)
var addrType gtls.AddrType
if ctxData.addrType != 0 {
addrType = ctxData.addrType
} else if obj.getAddrType != nil {
addrType = obj.getAddrType(host)
}
ips, err := dialer.Resolver.LookupIPAddr(ctx, host)
if err != nil {
return nil, err
}
if host, err = obj.addrToIp(host, ips, addrType); err != nil {
return nil, err
}
addr = net.JoinHostPort(host, port)
}
}
if dialer == nil {
dialer = obj.getDialer(ctxData, false)
}
return dialer.DialContext(ctx, network, addr)
}
func (obj *DialClient) DialContextWithProxy(ctx context.Context, ctxData *reqCtxData, network string, scheme string, addr string, host string, proxyUrl *url.URL, tlsConfig *tls.Config) (net.Conn, error) {
if ctxData == nil {
ctxData = &reqCtxData{}
}
if proxyUrl == nil {
return obj.DialContext(ctx, ctxData, network, addr)
}
if proxyUrl.Port() == "" {
if proxyUrl.Scheme == "http" {
proxyUrl.Host = net.JoinHostPort(proxyUrl.Hostname(), "80")
} else if proxyUrl.Scheme == "https" {
proxyUrl.Host = net.JoinHostPort(proxyUrl.Hostname(), "443")
}
}
switch proxyUrl.Scheme {
case "http", "https":
conn, err := obj.DialContext(ctx, ctxData, network, net.JoinHostPort(proxyUrl.Hostname(), proxyUrl.Port()))
if err != nil {
return conn, err
} else if proxyUrl.Scheme == "https" {
if conn, err = obj.addTls(ctx, conn, proxyUrl.Host, true, tlsConfig); err != nil {
return conn, err
}
}
return conn, obj.clientVerifyHttps(ctx, scheme, proxyUrl, addr, host, conn)
case "socks5":
return obj.Socks5Proxy(ctx, ctxData, network, addr, proxyUrl)
default:
return nil, errors.New("proxyUrl Scheme error")
}
}
func (obj *DialClient) loadHost(host string) (string, bool) {
msgDataAny, ok := obj.dnsIpData.Load(host)
if ok {
msgdata := msgDataAny.(msgClient)
if time.Since(msgdata.time) < time.Second*60*5 {
return msgdata.host, true
}
}
return host, false
}
func (obj *DialClient) addrToIp(host string, ips []net.IPAddr, addrType gtls.AddrType) (string, error) {
ip, err := obj.lookupIPAddr(ips, addrType)
if err != nil {
return host, tools.WrapError(err, "addrToIp error,lookupIPAddr")
}
obj.dnsIpData.Store(host, msgClient{time: time.Now(), host: ip.String()})
return ip.String(), nil
}
func (obj *DialClient) clientVerifySocks5(proxyUrl *url.URL, addr string, conn net.Conn) (err error) {
if _, err = conn.Write([]byte{5, 2, 0, 2}); err != nil {
return
}
readCon := make([]byte, 4)
if _, err = io.ReadFull(conn, readCon[:2]); err != nil {
return
}
switch readCon[1] {
case 2:
if proxyUrl.User == nil {
err = errors.New("socks5 need auth")
return
}
pwd, pwdOk := proxyUrl.User.Password()
if !pwdOk {
err = errors.New("socks5 auth error")
return
}
usr := proxyUrl.User.Username()
if usr == "" {
err = errors.New("socks5 auth user format error")
return
}
if _, err = conn.Write(append(
append(
[]byte{1, byte(len(usr))},
tools.StringToBytes(usr)...,
),
append(
[]byte{byte(len(pwd))},
tools.StringToBytes(pwd)...,
)...,
)); err != nil {
return
}
if _, err = io.ReadFull(conn, readCon[:2]); err != nil {
return
}
switch readCon[1] {
case 0:
default:
err = errors.New("socks5 auth error")
return
}
case 0:
default:
err = errors.New("not support auth format")
return
}
var host string
var port int
if host, port, err = gtls.SplitHostPort(addr); err != nil {
return
}
writeCon := []byte{5, 1, 0}
ip, ipInt := gtls.ParseHost(host)
switch ipInt {
case 4:
writeCon = append(writeCon, 1)
writeCon = append(writeCon, ip...)
case 6:
writeCon = append(writeCon, 4)
writeCon = append(writeCon, ip...)
case 0:
if len(host) > 255 {
err = errors.New("FQDN too long")
return
}
writeCon = append(writeCon, 3)
writeCon = append(writeCon, byte(len(host)))
writeCon = append(writeCon, host...)
}
writeCon = append(writeCon, byte(port>>8), byte(port))
if _, err = conn.Write(writeCon); err != nil {
return
}
if _, err = io.ReadFull(conn, readCon); err != nil {
return
}
if readCon[0] != 5 {
err = errors.New("socks version error")
return
}
if readCon[1] != 0 {
err = errors.New("socks conn error")
return
}
if readCon[3] != 1 {
err = errors.New("socks conn type error")
return
}
switch readCon[3] {
case 1: //ipv4
if _, err = io.ReadFull(conn, readCon); err != nil {
return
}
case 3: //domain
if _, err = io.ReadFull(conn, readCon[:1]); err != nil {
return
}
if _, err = io.ReadFull(conn, make([]byte, readCon[0])); err != nil {
return
}
case 4: //IPv6
if _, err = io.ReadFull(conn, make([]byte, 16)); err != nil {
return
}
default:
err = errors.New("invalid atyp")
return
}
_, err = io.ReadFull(conn, readCon[:2])
return
}
func (obj *DialClient) lookupIPAddr(ips []net.IPAddr, addrType gtls.AddrType) (net.IP, error) {
for _, ipAddr := range ips {
ip := ipAddr.IP
if ipType := gtls.ParseIp(ip); ipType == 4 || ipType == 6 {
if addrType == 0 || addrType == ipType {
return ip, nil
}
}
}
for _, ipAddr := range ips {
ip := ipAddr.IP
if ipType := gtls.ParseIp(ip); ipType == 4 || ipType == 6 {
return ip, nil
}
}
return nil, errors.New("dns parse host error")
}
func (obj *DialClient) getDialer(ctxData *reqCtxData, parseDns bool) *net.Dialer {
var dialOption DialOption
var isNew bool
if ctxData.dialTimeout == 0 {
dialOption.DialTimeout = obj.dialer.Timeout
} else {
dialOption.DialTimeout = ctxData.dialTimeout
if ctxData.dialTimeout != obj.dialer.Timeout {
isNew = true
}
}
if ctxData.keepAlive == 0 {
dialOption.KeepAlive = obj.dialer.KeepAlive
} else {
dialOption.KeepAlive = ctxData.keepAlive
if ctxData.keepAlive != obj.dialer.KeepAlive {
isNew = true
}
}
if ctxData.localAddr == nil {
if obj.dialer.LocalAddr != nil {
dialOption.LocalAddr = obj.dialer.LocalAddr.(*net.TCPAddr)
}
} else {
dialOption.LocalAddr = ctxData.localAddr
if ctxData.localAddr.String() != obj.dialer.LocalAddr.String() {
isNew = true
}
}
if ctxData.dns == nil {
dialOption.Dns = obj.dns
} else {
dialOption.Dns = ctxData.dns
if parseDns && ctxData.dns.String() != obj.dns.String() {
isNew = true
}
}
if isNew {
return NewDialer(dialOption)
} else {
return obj.dialer
}
}
func (obj *DialClient) addTls(ctx context.Context, conn net.Conn, host string, disHttp2 bool, tlsConfig *tls.Config) (*tls.Conn, error) {
var tlsConn *tls.Conn
tlsConfig.ServerName = gtls.GetServerName(host)
if disHttp2 {
tlsConfig.NextProtos = []string{"http/1.1"}
} else {
tlsConfig.NextProtos = []string{"h2", "http/1.1"}
}
tlsConn = tls.Client(conn, tlsConfig)
return tlsConn, tlsConn.HandshakeContext(ctx)
}
func (obj *DialClient) addJa3Tls(ctx context.Context, conn net.Conn, host string, disHttp2 bool, ja3Spec ja3.Ja3Spec, tlsConfig *utls.Config) (*utls.UConn, error) {
tlsConfig.ServerName = gtls.GetServerName(host)
if disHttp2 {
tlsConfig.NextProtos = []string{"http/1.1"}
} else {
tlsConfig.NextProtos = []string{"h2", "http/1.1"}
}
return ja3.NewClient(ctx, conn, ja3Spec, disHttp2, tlsConfig)
}
func (obj *DialClient) Socks5Proxy(ctx context.Context, ctxData *reqCtxData, network string, addr string, proxyUrl *url.URL) (conn net.Conn, err error) {
if conn, err = obj.DialContext(ctx, ctxData, network, net.JoinHostPort(proxyUrl.Hostname(), proxyUrl.Port())); err != nil {
return
}
didVerify := make(chan struct{})
go func() {
if err = obj.clientVerifySocks5(proxyUrl, addr, conn); err != nil {
conn.Close()
}
close(didVerify)
}()
select {
case <-ctx.Done():
return conn, ctx.Err()
case <-didVerify:
return
}
}
func (obj *DialClient) clientVerifyHttps(ctx context.Context, scheme string, proxyUrl *url.URL, addr string, host string, conn net.Conn) (err error) {
hdr := make(http.Header)
hdr.Set("User-Agent", UserAgent)
if proxyUrl.User != nil {
if password, ok := proxyUrl.User.Password(); ok {
hdr.Set("Proxy-Authorization", "Basic "+tools.Base64Encode(proxyUrl.User.Username()+":"+password))
}
}
cHost := host
_, hport, _ := net.SplitHostPort(host)
if hport == "" {
_, aport, _ := net.SplitHostPort(addr)
if aport != "" {
cHost = net.JoinHostPort(cHost, aport)
} else if scheme == "http" {
cHost = net.JoinHostPort(cHost, "80")
} else if scheme == "https" {
cHost = net.JoinHostPort(cHost, "443")
} else {
return errors.New("clientVerifyHttps not found port")
}
}
connectReq, err := NewRequestWithContext(ctx, http.MethodConnect, &url.URL{Opaque: addr}, nil)
if err != nil {
return err
}
connectReq.Header = hdr
connectReq.Host = cHost
if err = connectReq.Write(conn); err != nil {
return err
}
resp, err := readResponse(textproto.NewReader(bufio.NewReader(conn)), connectReq)
if err != nil {
return err
}
if resp.StatusCode != 200 {
return errors.New(resp.Status)
}
return
}

24
go.mod
View File

@@ -4,31 +4,31 @@ go 1.21.3
require (
github.com/gospider007/bar v0.0.0-20231215084215-956cfa59ce61
github.com/gospider007/bs4 v0.0.0-20240507072418-204e8cc763ae
github.com/gospider007/bs4 v0.0.0-20240531060354-fe6c0582dfd9
github.com/gospider007/gson v0.0.0-20240528092941-f4f87ed18978
github.com/gospider007/gtls v0.0.0-20240527084326-e580531eb89e
github.com/gospider007/ja3 v0.0.0-20240527084619-200356d63175
github.com/gospider007/ja3 v0.0.0-20240531015859-2479d5d5388e
github.com/gospider007/net v0.0.0-20240527131652-176df16a2ba2
github.com/gospider007/re v0.0.0-20240227100911-e27255e48eff
github.com/gospider007/tools v0.0.0-20240506025307-b6a12cb9b897
github.com/gospider007/websocket v0.0.0-20240331132617-8217ca7a8439
github.com/refraction-networking/utls v1.6.6
golang.org/x/exp v0.0.0-20240525044651-4c93da0ed11d
golang.org/x/net v0.25.0
golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8
golang.org/x/net v0.26.0
)
require (
github.com/PuerkitoBio/goquery v1.9.2 // indirect
github.com/andybalholm/brotli v1.1.0 // indirect
github.com/andybalholm/cascadia v1.3.2 // indirect
github.com/caddyserver/certmagic v0.21.2 // indirect
github.com/caddyserver/certmagic v0.21.3 // indirect
github.com/caddyserver/zerossl v0.1.3 // indirect
github.com/cloudflare/circl v1.3.8 // indirect
github.com/gospider007/blog v0.0.0-20231121084103-59a004dafccf // indirect
github.com/gospider007/kinds v0.0.0-20231024093643-7a4424f2d30e // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.17.8 // indirect
github.com/klauspost/cpuid/v2 v2.2.7 // indirect
github.com/klauspost/cpuid/v2 v2.2.8 // indirect
github.com/libdns/libdns v0.2.2 // indirect
github.com/mholt/acmez/v2 v2.0.1 // indirect
github.com/miekg/dns v1.1.59 // indirect
@@ -41,11 +41,11 @@ require (
go.mongodb.org/mongo-driver v1.15.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.27.0 // indirect
golang.org/x/crypto v0.23.0 // indirect
golang.org/x/image v0.16.0 // indirect
golang.org/x/mod v0.17.0 // indirect
golang.org/x/crypto v0.24.0 // indirect
golang.org/x/image v0.17.0 // indirect
golang.org/x/mod v0.18.0 // indirect
golang.org/x/sync v0.7.0 // indirect
golang.org/x/sys v0.20.0 // indirect
golang.org/x/text v0.15.0 // indirect
golang.org/x/tools v0.21.0 // indirect
golang.org/x/sys v0.21.0 // indirect
golang.org/x/text v0.16.0 // indirect
golang.org/x/tools v0.22.0 // indirect
)

24
go.sum
View File

@@ -6,6 +6,8 @@ github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsVi
github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
github.com/caddyserver/certmagic v0.21.2 h1:O18LtaYBGDooyy257cYePnhp4lPfz6TaJELil6Q1fDg=
github.com/caddyserver/certmagic v0.21.2/go.mod h1:Zq6pklO9nVRl3DIFUw9gVUfXKdpc/0qwTUAQMBlfgtI=
github.com/caddyserver/certmagic v0.21.3 h1:pqRRry3yuB4CWBVq9+cUqu+Y6E2z8TswbhNx1AZeYm0=
github.com/caddyserver/certmagic v0.21.3/go.mod h1:Zq6pklO9nVRl3DIFUw9gVUfXKdpc/0qwTUAQMBlfgtI=
github.com/caddyserver/zerossl v0.1.3 h1:onS+pxp3M8HnHpN5MMbOMyNjmTheJyWRaZYwn+YTAyA=
github.com/caddyserver/zerossl v0.1.3/go.mod h1:CxA0acn7oEGO6//4rtrRjYgEoa4MFw/XofZnrYwGqG4=
github.com/cloudflare/circl v1.3.8 h1:j+V8jJt09PoeMFIu2uh5JUyEaIHTXVOHslFoLNAKqwI=
@@ -24,12 +26,16 @@ github.com/gospider007/blog v0.0.0-20231121084103-59a004dafccf h1:1laTsuH/wl5pZ5
github.com/gospider007/blog v0.0.0-20231121084103-59a004dafccf/go.mod h1:CCJ+hvQ0kxL+qB/Wfr1xt7xspsG4XiczhnAPVxG2m3M=
github.com/gospider007/bs4 v0.0.0-20240507072418-204e8cc763ae h1:f+OlG2F/9zI35lk5SVL5LlVb2kMh82u9HAtyvy8GgA8=
github.com/gospider007/bs4 v0.0.0-20240507072418-204e8cc763ae/go.mod h1:kFlkn2u1ulhiybnhC9I840oS98tRSLJVwoBI5qNfZ2U=
github.com/gospider007/bs4 v0.0.0-20240531060354-fe6c0582dfd9 h1:SiePZGEq9yUj97mSzNAGO9wuKnGrAui9VI4jgUUZEts=
github.com/gospider007/bs4 v0.0.0-20240531060354-fe6c0582dfd9/go.mod h1:kFlkn2u1ulhiybnhC9I840oS98tRSLJVwoBI5qNfZ2U=
github.com/gospider007/gson v0.0.0-20240528092941-f4f87ed18978 h1:0kEPjlM9Vc5uCekI4TRsWzDukonjFt0g00KTaD9WUrs=
github.com/gospider007/gson v0.0.0-20240528092941-f4f87ed18978/go.mod h1:CGwmeOi7NGQFKsWlFQM2o4tHoASgKdrSipXT/wCvU/g=
github.com/gospider007/gtls v0.0.0-20240527084326-e580531eb89e h1:3bTAtZx+JstbWbTVRd4DivK7QDf6cJwBpWUJpVAgyjY=
github.com/gospider007/gtls v0.0.0-20240527084326-e580531eb89e/go.mod h1:pUD3WKesQAdqD1W8O3v03qRYyq760iPNE8IPEILf52Y=
github.com/gospider007/ja3 v0.0.0-20240527084619-200356d63175 h1:za1DnNWOmdRh/TD8weU7wjjWdL4B0uSbUjazCiMFmL8=
github.com/gospider007/ja3 v0.0.0-20240527084619-200356d63175/go.mod h1:GQ/Eojogp6s7wdOpDxvydQEhRaM7xj0jj+cmTMlLN/Y=
github.com/gospider007/ja3 v0.0.0-20240531015859-2479d5d5388e h1:RQJsZofUupnaRQ/9MR5LDHQwjo3JiJSlJT8qsrLyuNk=
github.com/gospider007/ja3 v0.0.0-20240531015859-2479d5d5388e/go.mod h1:iqvTWt8Ywjw95oaGB8lKjMat92vKMHNT7eAISIF+LO0=
github.com/gospider007/kinds v0.0.0-20231024093643-7a4424f2d30e h1:lmX6IQKkrNDbXfHsvrv1Uz0MoG2v5+4VC6Gdh9irUNY=
github.com/gospider007/kinds v0.0.0-20231024093643-7a4424f2d30e/go.mod h1:nB4OMmd8Ji92yEmgjbHcqLcBHTAhSSmlGNb2JpTYK9A=
github.com/gospider007/net v0.0.0-20240527131652-176df16a2ba2 h1:72qTKliOoXwD3inHG+tOCRGE5RlTulaaG8ZQ1UfWZRg=
@@ -47,6 +53,8 @@ github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ib
github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM=
github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM=
github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
github.com/libdns/libdns v0.2.2 h1:O6ws7bAfRPaBsgAYt8MDe2HcNBGC29hkZ9MX2eUSX3s=
github.com/libdns/libdns v0.2.2/go.mod h1:4Bj9+5CQiNMVGf87wjX4CY3HQJypUHRuLvlsfsZqLWQ=
github.com/mholt/acmez/v2 v2.0.1 h1:3/3N0u1pLjMK4sNEAFSI+bcvzbPhRpY383sy1kLHJ6k=
@@ -92,14 +100,22 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI=
golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM=
golang.org/x/exp v0.0.0-20240525044651-4c93da0ed11d h1:N0hmiNbwsSNwHBAvR3QB5w25pUwH4tK0Y/RltD1j1h4=
golang.org/x/exp v0.0.0-20240525044651-4c93da0ed11d/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc=
golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8 h1:LoYXNGAShUG3m/ehNk4iFctuhGX/+R1ZpfJ4/ia80JM=
golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI=
golang.org/x/image v0.16.0 h1:9kloLAKhUufZhA12l5fwnx2NZW39/we1UhBesW433jw=
golang.org/x/image v0.16.0/go.mod h1:ugSZItdV4nOxyqp56HmXwH0Ry0nBCpjnZdpDaIHdoPs=
golang.org/x/image v0.17.0 h1:nTRVVdajgB8zCMZVsViyzhnMKPwYeroEERRC64JuLco=
golang.org/x/image v0.17.0/go.mod h1:4yyo5vMFQjVjUcVk4jEQcU9MGy/rulF5WvUILseCM2E=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA=
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0=
golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
@@ -107,6 +123,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ=
golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -121,6 +139,8 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
@@ -132,12 +152,16 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.21.0 h1:qc0xYgIbsSDt9EyWz05J5wfa7LOVW0YTLOXrqdLAWIw=
golang.org/x/tools v0.21.0/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA=
golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

96
headers.go
View File

@@ -1,48 +1,48 @@
package requests
import (
"net/http"
)
const (
chromeV = "120"
chromeE = ".0.6099.28"
UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/" + chromeV + ".0.0.0 Safari/537.36 Edg/" + chromeV + chromeE
SecChUa = `"Chromium";v="` + chromeV + `", "Microsoft Edge";v="` + chromeV + `", "Not=A?Brand";v="99"`
AcceptLanguage = "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6"
)
func defaultHeaders() http.Header {
return http.Header{
"User-Agent": []string{UserAgent},
"Accept": []string{"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7"},
"Accept-Encoding": []string{"gzip, deflate, br"},
"Accept-Language": []string{AcceptLanguage},
"Sec-Ch-Ua": []string{SecChUa},
"Sec-Ch-Ua-Mobile": []string{"?0"},
"Sec-Ch-Ua-Platform": []string{`"Windows"`},
}
}
func (obj *RequestOption) initHeaders() (http.Header, []string, error) {
if obj.Headers == nil {
return nil, nil, nil
}
switch headers := obj.Headers.(type) {
case http.Header:
return headers.Clone(), nil, nil
case *OrderMap:
head, order := headers.parseHeaders()
return head, order, nil
default:
_, dataMap, _, err := obj.newBody(headers, mapType)
if err != nil {
return nil, nil, err
}
if dataMap == nil {
return nil, nil, nil
}
head, _ := dataMap.parseHeaders()
return head, nil, err
}
}
package requests
import (
"net/http"
)
const (
chromeV = "120"
chromeE = ".0.6099.28"
UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/" + chromeV + ".0.0.0 Safari/537.36 Edg/" + chromeV + chromeE
SecChUa = `"Chromium";v="` + chromeV + `", "Microsoft Edge";v="` + chromeV + `", "Not=A?Brand";v="99"`
AcceptLanguage = "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6"
)
func defaultHeaders() http.Header {
return http.Header{
"User-Agent": []string{UserAgent},
"Accept": []string{"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7"},
"Accept-Encoding": []string{"gzip, deflate, br"},
"Accept-Language": []string{AcceptLanguage},
"Sec-Ch-Ua": []string{SecChUa},
"Sec-Ch-Ua-Mobile": []string{"?0"},
"Sec-Ch-Ua-Platform": []string{`"Windows"`},
}
}
func (obj *RequestOption) initHeaders() (http.Header, []string, error) {
if obj.Headers == nil {
return nil, nil, nil
}
switch headers := obj.Headers.(type) {
case http.Header:
return headers.Clone(), nil, nil
case *OrderMap:
head, order := headers.parseHeaders()
return head, order, nil
default:
_, dataMap, _, err := obj.newBody(headers, mapType)
if err != nil {
return nil, nil, err
}
if dataMap == nil {
return nil, nil, nil
}
head, _ := dataMap.parseHeaders()
return head, nil, err
}
}

174
jar.go
View File

@@ -1,87 +1,87 @@
package requests
import (
"net/http/cookiejar"
"net/url"
"github.com/gospider007/gtls"
"golang.org/x/net/publicsuffix"
)
// cookies jar
type Jar struct {
jar *cookiejar.Jar
}
// new cookies jar
func NewJar() *Jar {
j, _ := cookiejar.New(nil)
return &Jar{
jar: j,
}
}
// get cookies
func (obj *Client) GetCookies(href *url.URL) Cookies {
if obj.option.Jar == nil {
return nil
}
return obj.option.Jar.GetCookies(href)
}
// set cookies
func (obj *Client) SetCookies(href *url.URL, cookies ...any) error {
if obj.option.Jar == nil {
return nil
}
return obj.option.Jar.SetCookies(href, cookies...)
}
// clear cookies
func (obj *Client) ClearCookies() {
if obj.option.Jar == nil {
return
}
obj.option.Jar.ClearCookies()
}
// Get cookies
func (obj *Jar) GetCookies(u *url.URL) Cookies {
return obj.jar.Cookies(u)
}
func getDomain(u *url.URL) string {
domain := u.Hostname()
if _, addType := gtls.ParseHost(domain); addType == 0 {
if tlp, err := publicsuffix.EffectiveTLDPlusOne(domain); err == nil {
domain = tlp
}
}
return domain
}
// Set cookies
func (obj *Jar) SetCookies(u *url.URL, cookies ...any) error {
domain := getDomain(u)
for _, cookie := range cookies {
cooks, err := ReadCookies(cookie)
if err != nil {
return err
}
for _, cook := range cooks {
if cook.Path == "" {
cook.Path = "/"
}
if cook.Domain == "" {
cook.Domain = domain
}
}
obj.jar.SetCookies(u, cooks)
}
return nil
}
// Clear cookies
func (obj *Jar) ClearCookies() {
jar, _ := cookiejar.New(nil)
obj.jar = jar
}
package requests
import (
"net/http/cookiejar"
"net/url"
"github.com/gospider007/gtls"
"golang.org/x/net/publicsuffix"
)
// cookies jar
type Jar struct {
jar *cookiejar.Jar
}
// new cookies jar
func NewJar() *Jar {
j, _ := cookiejar.New(nil)
return &Jar{
jar: j,
}
}
// get cookies
func (obj *Client) GetCookies(href *url.URL) Cookies {
if obj.option.Jar == nil {
return nil
}
return obj.option.Jar.GetCookies(href)
}
// set cookies
func (obj *Client) SetCookies(href *url.URL, cookies ...any) error {
if obj.option.Jar == nil {
return nil
}
return obj.option.Jar.SetCookies(href, cookies...)
}
// clear cookies
func (obj *Client) ClearCookies() {
if obj.option.Jar == nil {
return
}
obj.option.Jar.ClearCookies()
}
// Get cookies
func (obj *Jar) GetCookies(u *url.URL) Cookies {
return obj.jar.Cookies(u)
}
func getDomain(u *url.URL) string {
domain := u.Hostname()
if _, addType := gtls.ParseHost(domain); addType == 0 {
if tlp, err := publicsuffix.EffectiveTLDPlusOne(domain); err == nil {
domain = tlp
}
}
return domain
}
// Set cookies
func (obj *Jar) SetCookies(u *url.URL, cookies ...any) error {
domain := getDomain(u)
for _, cookie := range cookies {
cooks, err := ReadCookies(cookie)
if err != nil {
return err
}
for _, cook := range cooks {
if cook.Path == "" {
cook.Path = "/"
}
if cook.Domain == "" {
cook.Domain = domain
}
}
obj.jar.SetCookies(u, cooks)
}
return nil
}
// Clear cookies
func (obj *Jar) ClearCookies() {
jar, _ := cookiejar.New(nil)
obj.jar = jar
}

614
option.go
View File

@@ -1,307 +1,307 @@
package requests
import (
"context"
"io"
"net"
"net/http"
"net/url"
"time"
"github.com/gospider007/gtls"
"github.com/gospider007/ja3"
"github.com/gospider007/websocket"
)
// Connection Management Options
type ClientOption struct {
Ja3Spec ja3.Ja3Spec //custom ja3Spec,use ja3.CreateSpecWithStr or ja3.CreateSpecWithId create
H2Ja3Spec ja3.H2Ja3Spec //h2 fingerprint
Proxy string //proxy,support https,http,socks5
ForceHttp1 bool //force use http1 send requests
Ja3 bool //enable ja3 fingerprint
DisCookie bool //disable cookies
DisDecode bool //disable auto decode
DisUnZip bool //disable auto zip decode
DisAlive bool //disable keepalive
Bar bool ////enable bar display
OptionCallBack func(ctx context.Context, option *RequestOption) error //option callback,if error is returnd, break request
ResultCallBack func(ctx context.Context, option *RequestOption, response *Response) error //result callback,if error is returnd,next errCallback
ErrCallBack func(ctx context.Context, option *RequestOption, response *Response, err error) error //error callback,if error is returnd,break request
RequestCallBack func(ctx context.Context, request *http.Request, response *http.Response) error //request and response callback,if error is returnd,reponse is error
MaxRetries int //try num
MaxRedirect int //redirect num ,<0 no redirect,==0 no limit
Headers any //default headers
Timeout time.Duration //request timeout
ResponseHeaderTimeout time.Duration //ResponseHeaderTimeout ,default:300
TlsHandshakeTimeout time.Duration //tls timeout,default:15
//network card ip
DialTimeout time.Duration //dial tcp timeout,default:15
KeepAlive time.Duration //keepalive,default:30
LocalAddr *net.TCPAddr
Dns *net.UDPAddr //dns
AddrType gtls.AddrType //dns parse addr type
Jar *Jar //custom cookies
//other option
GetProxy func(ctx context.Context, url *url.URL) (string, error) //proxy callback:support https,http,socks5 proxy
GetAddrType func(host string) gtls.AddrType
}
// Options for sending requests
type RequestOption struct {
Ja3Spec ja3.Ja3Spec //custom ja3Spec,use ja3.CreateSpecWithStr or ja3.CreateSpecWithId create
H2Ja3Spec ja3.H2Ja3Spec //custom h2 fingerprint
Proxy string //proxy,support http,https,socks5,example：http://127.0.0.1:7005
ForceHttp1 bool //force use http1 send requests
Ja3 bool //enable ja3 fingerprint
DisCookie bool //disable cookies,not use cookies
DisDecode bool //disable auto decode
DisUnZip bool //disable auto zip decode
DisAlive bool //disable keepalive
Bar bool //enable bar display
OptionCallBack func(ctx context.Context, option *RequestOption) error //option callback,if error is returnd, break request
ResultCallBack func(ctx context.Context, option *RequestOption, response *Response) error //result callback,if error is returnd,next errCallback
ErrCallBack func(ctx context.Context, option *RequestOption, response *Response, err error) error //error callback,if error is returnd,break request
RequestCallBack func(ctx context.Context, request *http.Request, response *http.Response) error //request and response callback,if error is returnd,reponse is error
MaxRetries int //try num
MaxRedirect int //redirect num ,<0 no redirect,==0 no limit
Headers any //request headers：json,map，header
Timeout time.Duration //request timeout
ResponseHeaderTimeout time.Duration //ResponseHeaderTimeout ,default:300
TlsHandshakeTimeout time.Duration
//network card ip
DialTimeout time.Duration //dial tcp timeout,default:15
KeepAlive time.Duration //keepalive,default:30
LocalAddr *net.TCPAddr
Dns *net.UDPAddr //dns
AddrType gtls.AddrType //dns parse addr type //tls timeout,default:15
Jar *Jar //custom cookies
// other option
Method string //method
Url *url.URL
Host string
Referer string //set headers referer value
ContentType string //headers Content-Type value
Cookies any // cookies,support :json,map,str，http.Header
Params any //url params，join url query,json,map
Json any //send application/json,support io.Reader,：string,[]bytes,json,map
Data any //send application/x-www-form-urlencoded, support io.Reader, string,[]bytes,json,map
Form any //send multipart/form-data,file upload,support io.Reader, json,map
Text any //send text/xml,support: io.Reader, string,[]bytes,json,map
Body any //not setting context-type,support io.Reader, string,[]bytes,json,map
Stream bool //disable auto read
WsOption websocket.Option //websocket option
DisProxy bool //force disable proxy
once bool
client *Client
}
func (obj *RequestOption) Client() *Client {
return obj.client
}
// Upload files with form-data,
type File struct {
FileName string
ContentType string
Content any
}
func (obj *RequestOption) initBody(ctx context.Context) (io.Reader, error) {
if obj.Body != nil {
body, _, _, err := obj.newBody(obj.Body, readType)
if err != nil || body == nil {
return nil, err
}
return body, err
} else if obj.Form != nil {
var orderMap *OrderMap
_, orderMap, _, err := obj.newBody(obj.Form, mapType)
if err != nil {
return nil, err
}
if orderMap == nil {
return nil, nil
}
body, contentType, once, err := orderMap.parseForm(ctx)
if err != nil {
return nil, err
}
obj.once = once
if obj.ContentType == "" {
obj.ContentType = contentType
}
if body == nil {
return nil, nil
}
return body, nil
} else if obj.Data != nil {
body, orderMap, _, err := obj.newBody(obj.Data, mapType)
if err != nil {
return body, err
}
if obj.ContentType == "" {
obj.ContentType = "application/x-www-form-urlencoded"
}
if body != nil {
return body, nil
}
if orderMap == nil {
return nil, nil
}
body2 := orderMap.parseData()
if body2 == nil {
return nil, nil
}
return body2, nil
} else if obj.Json != nil {
body, _, _, err := obj.newBody(obj.Json, readType)
if err != nil {
return nil, err
}
if obj.ContentType == "" {
obj.ContentType = "application/json"
}
if body == nil {
return nil, nil
}
return body, nil
} else if obj.Text != nil {
body, _, _, err := obj.newBody(obj.Text, readType)
if err != nil {
return nil, err
}
if obj.ContentType == "" {
obj.ContentType = "text/plain"
}
if body == nil {
return nil, nil
}
return body, nil
} else {
return nil, nil
}
}
func (obj *RequestOption) initParams() (*url.URL, error) {
baseUrl := cloneUrl(obj.Url)
if obj.Params == nil {
return baseUrl, nil
}
_, dataMap, _, err := obj.newBody(obj.Params, mapType)
if err != nil {
return nil, err
}
query := dataMap.parseParams().String()
if query == "" {
return baseUrl, nil
}
pquery := baseUrl.Query().Encode()
if pquery == "" {
baseUrl.RawQuery = query
} else {
baseUrl.RawQuery = pquery + "&" + query
}
return baseUrl, nil
}
func (obj *Client) newRequestOption(option RequestOption) RequestOption {
// start
if !option.Ja3Spec.IsSet() {
option.Ja3Spec = obj.option.Ja3Spec
}
if !option.H2Ja3Spec.IsSet() {
option.H2Ja3Spec = obj.option.H2Ja3Spec
}
if option.Proxy == "" {
option.Proxy = obj.option.Proxy
}
if !option.ForceHttp1 {
option.ForceHttp1 = obj.option.ForceHttp1
}
if !option.Ja3 {
option.Ja3 = obj.option.Ja3
}
if !option.DisCookie {
option.DisCookie = obj.option.DisCookie
}
if !option.DisDecode {
option.DisDecode = obj.option.DisDecode
}
if !option.DisUnZip {
option.DisUnZip = obj.option.DisUnZip
}
if !option.DisAlive {
option.DisAlive = obj.option.DisAlive
}
if !option.Bar {
option.Bar = obj.option.Bar
}
if option.OptionCallBack == nil {
option.OptionCallBack = obj.option.OptionCallBack
}
if option.ResultCallBack == nil {
option.ResultCallBack = obj.option.ResultCallBack
}
if option.ErrCallBack == nil {
option.ErrCallBack = obj.option.ErrCallBack
}
if option.RequestCallBack == nil {
option.RequestCallBack = obj.option.RequestCallBack
}
if option.MaxRetries == 0 {
option.MaxRetries = obj.option.MaxRetries
}
if option.MaxRedirect == 0 {
option.MaxRedirect = obj.option.MaxRedirect
}
if option.Headers == nil {
option.Headers = obj.option.Headers
}
if option.Timeout == 0 {
option.Timeout = obj.option.Timeout
}
if option.ResponseHeaderTimeout == 0 {
option.ResponseHeaderTimeout = obj.option.ResponseHeaderTimeout
}
if option.TlsHandshakeTimeout == 0 {
option.TlsHandshakeTimeout = obj.option.TlsHandshakeTimeout
}
if option.DialTimeout == 0 {
option.DialTimeout = obj.option.DialTimeout
}
if option.KeepAlive == 0 {
option.KeepAlive = obj.option.KeepAlive
}
if option.LocalAddr == nil {
option.LocalAddr = obj.option.LocalAddr
}
if option.Dns == nil {
option.Dns = obj.option.Dns
}
if option.AddrType == 0 {
option.AddrType = obj.option.AddrType
}
if option.Jar == nil {
option.Jar = obj.option.Jar
}
//end
if option.MaxRetries < 0 {
option.MaxRetries = 0
}
if !option.Ja3Spec.IsSet() && option.Ja3 {
option.Ja3Spec = ja3.DefaultJa3Spec()
}
if option.DisCookie {
option.Jar = nil
}
if option.DisProxy {
option.Proxy = ""
}
return option
}
package requests
import (
"context"
"io"
"net"
"net/http"
"net/url"
"time"
"github.com/gospider007/gtls"
"github.com/gospider007/ja3"
"github.com/gospider007/websocket"
)
// Connection Management Options
type ClientOption struct {
Ja3Spec ja3.Ja3Spec //custom ja3Spec,use ja3.CreateSpecWithStr or ja3.CreateSpecWithId create
H2Ja3Spec ja3.H2Ja3Spec //h2 fingerprint
Proxy string //proxy,support https,http,socks5
ForceHttp1 bool //force use http1 send requests
Ja3 bool //enable ja3 fingerprint
DisCookie bool //disable cookies
DisDecode bool //disable auto decode
DisUnZip bool //disable auto zip decode
DisAlive bool //disable keepalive
Bar bool ////enable bar display
OptionCallBack func(ctx context.Context, option *RequestOption) error //option callback,if error is returnd, break request
ResultCallBack func(ctx context.Context, option *RequestOption, response *Response) error //result callback,if error is returnd,next errCallback
ErrCallBack func(ctx context.Context, option *RequestOption, response *Response, err error) error //error callback,if error is returnd,break request
RequestCallBack func(ctx context.Context, request *http.Request, response *http.Response) error //request and response callback,if error is returnd,reponse is error
MaxRetries int //try num
MaxRedirect int //redirect num ,<0 no redirect,==0 no limit
Headers any //default headers
Timeout time.Duration //request timeout
ResponseHeaderTimeout time.Duration //ResponseHeaderTimeout ,default:300
TlsHandshakeTimeout time.Duration //tls timeout,default:15
//network card ip
DialTimeout time.Duration //dial tcp timeout,default:15
KeepAlive time.Duration //keepalive,default:30
LocalAddr *net.TCPAddr
Dns *net.UDPAddr //dns
AddrType gtls.AddrType //dns parse addr type
Jar *Jar //custom cookies
//other option
GetProxy func(ctx context.Context, url *url.URL) (string, error) //proxy callback:support https,http,socks5 proxy
GetAddrType func(host string) gtls.AddrType
}
// Options for sending requests
type RequestOption struct {
Ja3Spec ja3.Ja3Spec //custom ja3Spec,use ja3.CreateSpecWithStr or ja3.CreateSpecWithId create
H2Ja3Spec ja3.H2Ja3Spec //custom h2 fingerprint
Proxy string //proxy,support http,https,socks5,example：http://127.0.0.1:7005
ForceHttp1 bool //force use http1 send requests
Ja3 bool //enable ja3 fingerprint
DisCookie bool //disable cookies,not use cookies
DisDecode bool //disable auto decode
DisUnZip bool //disable auto zip decode
DisAlive bool //disable keepalive
Bar bool //enable bar display
OptionCallBack func(ctx context.Context, option *RequestOption) error //option callback,if error is returnd, break request
ResultCallBack func(ctx context.Context, option *RequestOption, response *Response) error //result callback,if error is returnd,next errCallback
ErrCallBack func(ctx context.Context, option *RequestOption, response *Response, err error) error //error callback,if error is returnd,break request
RequestCallBack func(ctx context.Context, request *http.Request, response *http.Response) error //request and response callback,if error is returnd,reponse is error
MaxRetries int //try num
MaxRedirect int //redirect num ,<0 no redirect,==0 no limit
Headers any //request headers：json,map，header
Timeout time.Duration //request timeout
ResponseHeaderTimeout time.Duration //ResponseHeaderTimeout ,default:300
TlsHandshakeTimeout time.Duration
//network card ip
DialTimeout time.Duration //dial tcp timeout,default:15
KeepAlive time.Duration //keepalive,default:30
LocalAddr *net.TCPAddr
Dns *net.UDPAddr //dns
AddrType gtls.AddrType //dns parse addr type //tls timeout,default:15
Jar *Jar //custom cookies
// other option
Method string //method
Url *url.URL
Host string
Referer string //set headers referer value
ContentType string //headers Content-Type value
Cookies any // cookies,support :json,map,str，http.Header
Params any //url params，join url query,json,map
Json any //send application/json,support io.Reader,：string,[]bytes,json,map
Data any //send application/x-www-form-urlencoded, support io.Reader, string,[]bytes,json,map
Form any //send multipart/form-data,file upload,support io.Reader, json,map
Text any //send text/xml,support: io.Reader, string,[]bytes,json,map
Body any //not setting context-type,support io.Reader, string,[]bytes,json,map
Stream bool //disable auto read
WsOption websocket.Option //websocket option
DisProxy bool //force disable proxy
once bool
client *Client
}
func (obj *RequestOption) Client() *Client {
return obj.client
}
// Upload files with form-data,
type File struct {
FileName string
ContentType string
Content any
}
func (obj *RequestOption) initBody(ctx context.Context) (io.Reader, error) {
if obj.Body != nil {
body, _, _, err := obj.newBody(obj.Body, readType)
if err != nil || body == nil {
return nil, err
}
return body, err
} else if obj.Form != nil {
var orderMap *OrderMap
_, orderMap, _, err := obj.newBody(obj.Form, mapType)
if err != nil {
return nil, err
}
if orderMap == nil {
return nil, nil
}
body, contentType, once, err := orderMap.parseForm(ctx)
if err != nil {
return nil, err
}
obj.once = once
if obj.ContentType == "" {
obj.ContentType = contentType
}
if body == nil {
return nil, nil
}
return body, nil
} else if obj.Data != nil {
body, orderMap, _, err := obj.newBody(obj.Data, mapType)
if err != nil {
return body, err
}
if obj.ContentType == "" {
obj.ContentType = "application/x-www-form-urlencoded"
}
if body != nil {
return body, nil
}
if orderMap == nil {
return nil, nil
}
body2 := orderMap.parseData()
if body2 == nil {
return nil, nil
}
return body2, nil
} else if obj.Json != nil {
body, _, _, err := obj.newBody(obj.Json, readType)
if err != nil {
return nil, err
}
if obj.ContentType == "" {
obj.ContentType = "application/json"
}
if body == nil {
return nil, nil
}
return body, nil
} else if obj.Text != nil {
body, _, _, err := obj.newBody(obj.Text, readType)
if err != nil {
return nil, err
}
if obj.ContentType == "" {
obj.ContentType = "text/plain"
}
if body == nil {
return nil, nil
}
return body, nil
} else {
return nil, nil
}
}
func (obj *RequestOption) initParams() (*url.URL, error) {
baseUrl := cloneUrl(obj.Url)
if obj.Params == nil {
return baseUrl, nil
}
_, dataMap, _, err := obj.newBody(obj.Params, mapType)
if err != nil {
return nil, err
}
query := dataMap.parseParams().String()
if query == "" {
return baseUrl, nil
}
pquery := baseUrl.Query().Encode()
if pquery == "" {
baseUrl.RawQuery = query
} else {
baseUrl.RawQuery = pquery + "&" + query
}
return baseUrl, nil
}
func (obj *Client) newRequestOption(option RequestOption) RequestOption {
// start
if !option.Ja3Spec.IsSet() {
option.Ja3Spec = obj.option.Ja3Spec
}
if !option.H2Ja3Spec.IsSet() {
option.H2Ja3Spec = obj.option.H2Ja3Spec
}
if option.Proxy == "" {
option.Proxy = obj.option.Proxy
}
if !option.ForceHttp1 {
option.ForceHttp1 = obj.option.ForceHttp1
}
if !option.Ja3 {
option.Ja3 = obj.option.Ja3
}
if !option.DisCookie {
option.DisCookie = obj.option.DisCookie
}
if !option.DisDecode {
option.DisDecode = obj.option.DisDecode
}
if !option.DisUnZip {
option.DisUnZip = obj.option.DisUnZip
}
if !option.DisAlive {
option.DisAlive = obj.option.DisAlive
}
if !option.Bar {
option.Bar = obj.option.Bar
}
if option.OptionCallBack == nil {
option.OptionCallBack = obj.option.OptionCallBack
}
if option.ResultCallBack == nil {
option.ResultCallBack = obj.option.ResultCallBack
}
if option.ErrCallBack == nil {
option.ErrCallBack = obj.option.ErrCallBack
}
if option.RequestCallBack == nil {
option.RequestCallBack = obj.option.RequestCallBack
}
if option.MaxRetries == 0 {
option.MaxRetries = obj.option.MaxRetries
}
if option.MaxRedirect == 0 {
option.MaxRedirect = obj.option.MaxRedirect
}
if option.Headers == nil {
option.Headers = obj.option.Headers
}
if option.Timeout == 0 {
option.Timeout = obj.option.Timeout
}
if option.ResponseHeaderTimeout == 0 {
option.ResponseHeaderTimeout = obj.option.ResponseHeaderTimeout
}
if option.TlsHandshakeTimeout == 0 {
option.TlsHandshakeTimeout = obj.option.TlsHandshakeTimeout
}
if option.DialTimeout == 0 {
option.DialTimeout = obj.option.DialTimeout
}
if option.KeepAlive == 0 {
option.KeepAlive = obj.option.KeepAlive
}
if option.LocalAddr == nil {
option.LocalAddr = obj.option.LocalAddr
}
if option.Dns == nil {
option.Dns = obj.option.Dns
}
if option.AddrType == 0 {
option.AddrType = obj.option.AddrType
}
if option.Jar == nil {
option.Jar = obj.option.Jar
}
//end
if option.MaxRetries < 0 {
option.MaxRetries = 0
}
if !option.Ja3Spec.IsSet() && option.Ja3 {
option.Ja3Spec = ja3.DefaultJa3Spec()
}
if option.DisCookie {
option.Jar = nil
}
if option.DisProxy {
option.Proxy = ""
}
return option
}

160
pip.go
View File

@@ -1,80 +1,80 @@
package requests
import (
"context"
"sync"
)
type pipCon struct {
reader <-chan []byte
writer chan<- []byte
readerI <-chan int
writerI chan<- int
lock sync.Mutex
ctx context.Context
cnl context.CancelCauseFunc
}
func (obj *pipCon) Read(b []byte) (n int, err error) {
select {
case con := <-obj.reader:
n = copy(b, con)
select {
case obj.writerI <- n:
return
case <-obj.ctx.Done():
return n, context.Cause(obj.ctx)
}
case <-obj.ctx.Done():
return n, context.Cause(obj.ctx)
}
}
func (obj *pipCon) Write(b []byte) (n int, err error) {
obj.lock.Lock()
defer obj.lock.Unlock()
for once := true; once || len(b) > 0; once = false {
select {
case obj.writer <- b:
select {
case i := <-obj.readerI:
b = b[i:]
n += i
case <-obj.ctx.Done():
return n, context.Cause(obj.ctx)
}
case <-obj.ctx.Done():
return n, context.Cause(obj.ctx)
}
}
return
}
func (obj *pipCon) Close(err error) error {
obj.cnl(err)
return nil
}
func pipe(preCtx context.Context) (*pipCon, *pipCon) {
ctx, cnl := context.WithCancelCause(preCtx)
readerCha := make(chan []byte)
writerCha := make(chan []byte)
readerI := make(chan int)
writerI := make(chan int)
localpipCon := &pipCon{
reader: readerCha,
readerI: readerI,
writer: writerCha,
writerI: writerI,
ctx: ctx,
cnl: cnl,
}
remotepipCon := &pipCon{
reader: writerCha,
readerI: writerI,
writer: readerCha,
writerI: readerI,
ctx: ctx,
cnl: cnl,
}
return localpipCon, remotepipCon
}
package requests
import (
"context"
"sync"
)
type pipCon struct {
reader <-chan []byte
writer chan<- []byte
readerI <-chan int
writerI chan<- int
lock sync.Mutex
ctx context.Context
cnl context.CancelCauseFunc
}
func (obj *pipCon) Read(b []byte) (n int, err error) {
select {
case con := <-obj.reader:
n = copy(b, con)
select {
case obj.writerI <- n:
return
case <-obj.ctx.Done():
return n, context.Cause(obj.ctx)
}
case <-obj.ctx.Done():
return n, context.Cause(obj.ctx)
}
}
func (obj *pipCon) Write(b []byte) (n int, err error) {
obj.lock.Lock()
defer obj.lock.Unlock()
for once := true; once || len(b) > 0; once = false {
select {
case obj.writer <- b:
select {
case i := <-obj.readerI:
b = b[i:]
n += i
case <-obj.ctx.Done():
return n, context.Cause(obj.ctx)
}
case <-obj.ctx.Done():
return n, context.Cause(obj.ctx)
}
}
return
}
func (obj *pipCon) Close(err error) error {
obj.cnl(err)
return nil
}
func pipe(preCtx context.Context) (*pipCon, *pipCon) {
ctx, cnl := context.WithCancelCause(preCtx)
readerCha := make(chan []byte)
writerCha := make(chan []byte)
readerI := make(chan int)
writerI := make(chan int)
localpipCon := &pipCon{
reader: readerCha,
readerI: readerI,
writer: writerCha,
writerI: writerI,
ctx: ctx,
cnl: cnl,
}
remotepipCon := &pipCon{
reader: writerCha,
readerI: writerI,
writer: readerCha,
writerI: readerI,
ctx: ctx,
cnl: cnl,
}
return localpipCon, remotepipCon
}

792
requests.go
View File

@@ -1,396 +1,396 @@
package requests
import (
"context"
"errors"
"fmt"
"io"
"net"
"strings"
"time"
"net/url"
"os"
"net/http"
"github.com/gospider007/gtls"
"github.com/gospider007/ja3"
"github.com/gospider007/re"
"github.com/gospider007/tools"
"github.com/gospider007/websocket"
)
type contextKey string
const gospiderContextKey contextKey = "GospiderContextKey"
var errFatal = errors.New("ErrFatal")
type reqCtxData struct {
isWs bool
forceHttp1 bool
maxRedirect int
proxy *url.URL
disProxy bool
disAlive bool
orderHeaders []string
responseHeaderTimeout time.Duration
tlsHandshakeTimeout time.Duration
requestCallBack func(context.Context, *http.Request, *http.Response) error
h2Ja3Spec ja3.H2Ja3Spec
ja3Spec ja3.Ja3Spec
dialTimeout time.Duration
keepAlive time.Duration
localAddr *net.TCPAddr //network card ip
addrType gtls.AddrType //first ip type
dns *net.UDPAddr
isNewConn bool
}
func NewReqCtxData(ctx context.Context, option *RequestOption) (*reqCtxData, error) {
//init ctxData
ctxData := new(reqCtxData)
ctxData.ja3Spec = option.Ja3Spec
ctxData.h2Ja3Spec = option.H2Ja3Spec
ctxData.forceHttp1 = option.ForceHttp1
ctxData.disAlive = option.DisAlive
ctxData.maxRedirect = option.MaxRedirect
ctxData.requestCallBack = option.RequestCallBack
ctxData.responseHeaderTimeout = option.ResponseHeaderTimeout
ctxData.addrType = option.AddrType
ctxData.dialTimeout = option.DialTimeout
ctxData.keepAlive = option.KeepAlive
ctxData.localAddr = option.LocalAddr
ctxData.dns = option.Dns
ctxData.disProxy = option.DisProxy
ctxData.tlsHandshakeTimeout = option.TlsHandshakeTimeout
//init scheme
if option.Url != nil {
if option.Url.Scheme == "ws" {
ctxData.isWs = true
option.Url.Scheme = "http"
} else if option.Url.Scheme == "wss" {
ctxData.isWs = true
option.Url.Scheme = "https"
}
}
//init tls timeout
if option.TlsHandshakeTimeout == 0 {
ctxData.tlsHandshakeTimeout = time.Second * 15
}
//init proxy
if option.Proxy != "" {
tempProxy, err := gtls.VerifyProxy(option.Proxy)
if err != nil {
return nil, tools.WrapError(errFatal, errors.New("tempRequest init proxy error"), err)
}
ctxData.proxy = tempProxy
}
return ctxData, nil
}
func CreateReqCtx(ctx context.Context, ctxData *reqCtxData) context.Context {
return context.WithValue(ctx, gospiderContextKey, ctxData)
}
func GetReqCtxData(ctx context.Context) *reqCtxData {
return ctx.Value(gospiderContextKey).(*reqCtxData)
}
// sends a GET request and returns the response.
func Get(ctx context.Context, href string, options ...RequestOption) (resp *Response, err error) {
return defaultClient.Request(ctx, http.MethodGet, href, options...)
}
// sends a Head request and returns the response.
func Head(ctx context.Context, href string, options ...RequestOption) (resp *Response, err error) {
return defaultClient.Request(ctx, http.MethodHead, href, options...)
}
// sends a Post request and returns the response.
func Post(ctx context.Context, href string, options ...RequestOption) (resp *Response, err error) {
return defaultClient.Request(ctx, http.MethodPost, href, options...)
}
// sends a Put request and returns the response.
func Put(ctx context.Context, href string, options ...RequestOption) (resp *Response, err error) {
return defaultClient.Request(ctx, http.MethodPut, href, options...)
}
// sends a Patch request and returns the response.
func Patch(ctx context.Context, href string, options ...RequestOption) (resp *Response, err error) {
return defaultClient.Request(ctx, http.MethodPatch, href, options...)
}
// sends a Delete request and returns the response.
func Delete(ctx context.Context, href string, options ...RequestOption) (resp *Response, err error) {
return defaultClient.Request(ctx, http.MethodDelete, href, options...)
}
// sends a Connect request and returns the response.
func Connect(ctx context.Context, href string, options ...RequestOption) (resp *Response, err error) {
return defaultClient.Request(ctx, http.MethodConnect, href, options...)
}
// sends a Options request and returns the response.
func Options(ctx context.Context, href string, options ...RequestOption) (resp *Response, err error) {
return defaultClient.Request(ctx, http.MethodOptions, href, options...)
}
// sends a Trace request and returns the response.
func Trace(ctx context.Context, href string, options ...RequestOption) (resp *Response, err error) {
return defaultClient.Request(ctx, http.MethodTrace, href, options...)
}
// Define a function named Request that takes in four parameters:
func Request(ctx context.Context, method string, href string, options ...RequestOption) (resp *Response, err error) {
return defaultClient.Request(ctx, method, href, options...)
}
// sends a Get request and returns the response.
func (obj *Client) Get(ctx context.Context, href string, options ...RequestOption) (*Response, error) {
return obj.Request(ctx, http.MethodGet, href, options...)
}
// sends a Head request and returns the response.
func (obj *Client) Head(ctx context.Context, href string, options ...RequestOption) (*Response, error) {
return obj.Request(ctx, http.MethodHead, href, options...)
}
// sends a Post request and returns the response.
func (obj *Client) Post(ctx context.Context, href string, options ...RequestOption) (*Response, error) {
return obj.Request(ctx, http.MethodPost, href, options...)
}
// sends a Put request and returns the response.
func (obj *Client) Put(ctx context.Context, href string, options ...RequestOption) (*Response, error) {
return obj.Request(ctx, http.MethodPut, href, options...)
}
// sends a Patch request and returns the response.
func (obj *Client) Patch(ctx context.Context, href string, options ...RequestOption) (*Response, error) {
return obj.Request(ctx, http.MethodPatch, href, options...)
}
// sends a Delete request and returns the response.
func (obj *Client) Delete(ctx context.Context, href string, options ...RequestOption) (*Response, error) {
return obj.Request(ctx, http.MethodDelete, href, options...)
}
// sends a Connect request and returns the response.
func (obj *Client) Connect(ctx context.Context, href string, options ...RequestOption) (*Response, error) {
return obj.Request(ctx, http.MethodConnect, href, options...)
}
// sends a Options request and returns the response.
func (obj *Client) Options(ctx context.Context, href string, options ...RequestOption) (*Response, error) {
return obj.Request(ctx, http.MethodOptions, href, options...)
}
// sends a Trace request and returns the response.
func (obj *Client) Trace(ctx context.Context, href string, options ...RequestOption) (*Response, error) {
return obj.Request(ctx, http.MethodTrace, href, options...)
}
// Define a function named Request that takes in four parameters:
func (obj *Client) Request(ctx context.Context, method string, href string, options ...RequestOption) (response *Response, err error) {
if obj.closed {
return nil, errors.New("client is closed")
}
if ctx == nil {
ctx = obj.ctx
}
var rawOption RequestOption
if len(options) > 0 {
rawOption = options[0]
}
optionBak := obj.newRequestOption(rawOption)
if optionBak.Method == "" {
optionBak.Method = method
}
uhref := optionBak.Url
if uhref == nil {
if uhref, err = url.Parse(href); err != nil {
err = tools.WrapError(err, "url parse error")
return
}
}
for maxRetries := 0; maxRetries <= optionBak.MaxRetries; maxRetries++ {
option := optionBak
option.Url = cloneUrl(uhref)
option.client = obj
response, err = obj.request(ctx, &option)
if err == nil || errors.Is(err, errFatal) || option.once {
return
}
}
return
}
func (obj *Client) request(ctx context.Context, option *RequestOption) (response *Response, err error) {
response = new(Response)
defer func() {
//read body
if err == nil && !response.IsStream() {
err = response.ReadBody()
}
//result callback
if err == nil && option.ResultCallBack != nil {
err = option.ResultCallBack(ctx, option, response)
}
if err != nil { //err callback, must close body
response.CloseBody()
if option.ErrCallBack != nil {
if err2 := option.ErrCallBack(ctx, option, response, err); err2 != nil {
err = tools.WrapError(errFatal, err2)
}
}
} else if !response.IsStream() { //is not is stream must close body
response.CloseBody()
}
}()
if option.OptionCallBack != nil {
if err = option.OptionCallBack(ctx, option); err != nil {
return
}
}
response.bar = option.Bar
response.disUnzip = option.DisUnZip
response.disDecode = option.DisDecode
response.stream = option.Stream
//init headers and orderheaders,befor init ctxData
headers, orderHeaders, err := option.initHeaders()
if err != nil {
return response, tools.WrapError(err, errors.New("tempRequest init headers error"), err)
}
//设置 h2 请求头顺序
if orderHeaders != nil {
if !option.H2Ja3Spec.IsSet() {
option.H2Ja3Spec = ja3.DefaultH2Ja3Spec()
option.H2Ja3Spec.OrderHeaders = orderHeaders
} else if option.H2Ja3Spec.OrderHeaders == nil {
option.H2Ja3Spec.OrderHeaders = orderHeaders
}
}
//init ctxData
ctxData, err := NewReqCtxData(ctx, option)
if err != nil {
return response, tools.WrapError(err, " reqCtxData init error")
}
if headers == nil {
headers = defaultHeaders()
}
//设置 h1 请求头顺序
if orderHeaders != nil {
ctxData.orderHeaders = orderHeaders
} else {
ctxData.orderHeaders = ja3.DefaultOrderHeaders()
}
//init ctx,cnl
if option.Timeout > 0 { //超时
response.ctx, response.cnl = context.WithTimeout(CreateReqCtx(ctx, ctxData), option.Timeout)
} else {
response.ctx, response.cnl = context.WithCancel(CreateReqCtx(ctx, ctxData))
}
//init url
href, err := option.initParams()
if err != nil {
err = tools.WrapError(err, "url init error")
return
}
if href.User != nil {
headers.Set("Authorization", "Basic "+tools.Base64Encode(href.User.String()))
}
//init body
body, err := option.initBody(response.ctx)
if err != nil {
return response, tools.WrapError(err, errors.New("tempRequest init body error"), err)
}
//create request
reqs, err := NewRequestWithContext(response.ctx, option.Method, href, body)
if err != nil {
return response, tools.WrapError(errFatal, errors.New("tempRequest 构造request失败"), err)
}
reqs.Header = headers
//add Referer
if reqs.Header.Get("Referer") == "" {
if option.Referer != "" {
reqs.Header.Set("Referer", option.Referer)
} else if reqs.URL.Scheme != "" && reqs.URL.Host != "" {
reqs.Header.Set("Referer", fmt.Sprintf("%s://%s", reqs.URL.Scheme, reqs.URL.Host))
}
}
//set ContentType
if option.ContentType != "" && reqs.Header.Get("Content-Type") == "" {
reqs.Header.Set("Content-Type", option.ContentType)
}
//init ws
if ctxData.isWs {
websocket.SetClientHeadersWithOption(reqs.Header, option.WsOption)
}
if reqs.URL.Scheme == "file" {
response.filePath = re.Sub(`^/+`, "", reqs.URL.Path)
response.content, err = os.ReadFile(response.filePath)
if err != nil {
err = tools.WrapError(errFatal, errors.New("read filePath data error"), err)
}
return
}
//add host
if option.Host != "" {
reqs.Host = option.Host
} else if reqs.Header.Get("Host") != "" {
reqs.Host = reqs.Header.Get("Host")
} else {
reqs.Host = reqs.URL.Host
}
//init cookies
cookies, err := option.initCookies()
if err != nil {
return response, tools.WrapError(err, errors.New("tempRequest init cookies error"), err)
}
if cookies != nil {
addCookie(reqs, cookies)
}
//send req
response.response, err = obj.do(reqs, option)
response.isNewConn = ctxData.isNewConn
if err != nil {
err = tools.WrapError(err, "client do error")
return
}
if response.response == nil {
err = errors.New("response is nil")
return
}
if response.response.Body != nil {
response.rawConn = response.response.Body.(*readWriteCloser)
}
if !response.disUnzip {
response.disUnzip = response.response.Uncompressed
}
if response.response.StatusCode == 101 {
response.webSocket = websocket.NewClientConn(response.rawConn.Conn(), websocket.GetResponseHeaderOption(response.response.Header))
} else if strings.Contains(response.response.Header.Get("Content-Type"), "text/event-stream") {
response.sse = newSse(response.response.Body)
} else if !response.disUnzip {
var unCompressionBody io.ReadCloser
unCompressionBody, err = tools.CompressionDecode(response.response.Body, response.ContentEncoding())
if err != nil {
if err != io.ErrUnexpectedEOF && err != io.EOF {
return
}
}
if unCompressionBody != nil {
response.response.Body = unCompressionBody
}
}
return
}
package requests
import (
"context"
"errors"
"fmt"
"io"
"net"
"strings"
"time"
"net/url"
"os"
"net/http"
"github.com/gospider007/gtls"
"github.com/gospider007/ja3"
"github.com/gospider007/re"
"github.com/gospider007/tools"
"github.com/gospider007/websocket"
)
type contextKey string
const gospiderContextKey contextKey = "GospiderContextKey"
var errFatal = errors.New("ErrFatal")
type reqCtxData struct {
isWs bool
forceHttp1 bool
maxRedirect int
proxy *url.URL
disProxy bool
disAlive bool
orderHeaders []string
responseHeaderTimeout time.Duration
tlsHandshakeTimeout time.Duration
requestCallBack func(context.Context, *http.Request, *http.Response) error
h2Ja3Spec ja3.H2Ja3Spec
ja3Spec ja3.Ja3Spec
dialTimeout time.Duration
keepAlive time.Duration
localAddr *net.TCPAddr //network card ip
addrType gtls.AddrType //first ip type
dns *net.UDPAddr
isNewConn bool
}
func NewReqCtxData(ctx context.Context, option *RequestOption) (*reqCtxData, error) {
//init ctxData
ctxData := new(reqCtxData)
ctxData.ja3Spec = option.Ja3Spec
ctxData.h2Ja3Spec = option.H2Ja3Spec
ctxData.forceHttp1 = option.ForceHttp1
ctxData.disAlive = option.DisAlive
ctxData.maxRedirect = option.MaxRedirect
ctxData.requestCallBack = option.RequestCallBack
ctxData.responseHeaderTimeout = option.ResponseHeaderTimeout
ctxData.addrType = option.AddrType
ctxData.dialTimeout = option.DialTimeout
ctxData.keepAlive = option.KeepAlive
ctxData.localAddr = option.LocalAddr
ctxData.dns = option.Dns
ctxData.disProxy = option.DisProxy
ctxData.tlsHandshakeTimeout = option.TlsHandshakeTimeout
//init scheme
if option.Url != nil {
if option.Url.Scheme == "ws" {
ctxData.isWs = true
option.Url.Scheme = "http"
} else if option.Url.Scheme == "wss" {
ctxData.isWs = true
option.Url.Scheme = "https"
}
}
//init tls timeout
if option.TlsHandshakeTimeout == 0 {
ctxData.tlsHandshakeTimeout = time.Second * 15
}
//init proxy
if option.Proxy != "" {
tempProxy, err := gtls.VerifyProxy(option.Proxy)
if err != nil {
return nil, tools.WrapError(errFatal, errors.New("tempRequest init proxy error"), err)
}
ctxData.proxy = tempProxy
}
return ctxData, nil
}
func CreateReqCtx(ctx context.Context, ctxData *reqCtxData) context.Context {
return context.WithValue(ctx, gospiderContextKey, ctxData)
}
func GetReqCtxData(ctx context.Context) *reqCtxData {
return ctx.Value(gospiderContextKey).(*reqCtxData)
}
// sends a GET request and returns the response.
func Get(ctx context.Context, href string, options ...RequestOption) (resp *Response, err error) {
return defaultClient.Request(ctx, http.MethodGet, href, options...)
}
// sends a Head request and returns the response.
func Head(ctx context.Context, href string, options ...RequestOption) (resp *Response, err error) {
return defaultClient.Request(ctx, http.MethodHead, href, options...)
}
// sends a Post request and returns the response.
func Post(ctx context.Context, href string, options ...RequestOption) (resp *Response, err error) {
return defaultClient.Request(ctx, http.MethodPost, href, options...)
}
// sends a Put request and returns the response.
func Put(ctx context.Context, href string, options ...RequestOption) (resp *Response, err error) {
return defaultClient.Request(ctx, http.MethodPut, href, options...)
}
// sends a Patch request and returns the response.
func Patch(ctx context.Context, href string, options ...RequestOption) (resp *Response, err error) {
return defaultClient.Request(ctx, http.MethodPatch, href, options...)
}
// sends a Delete request and returns the response.
func Delete(ctx context.Context, href string, options ...RequestOption) (resp *Response, err error) {
return defaultClient.Request(ctx, http.MethodDelete, href, options...)
}
// sends a Connect request and returns the response.
func Connect(ctx context.Context, href string, options ...RequestOption) (resp *Response, err error) {
return defaultClient.Request(ctx, http.MethodConnect, href, options...)
}
// sends a Options request and returns the response.
func Options(ctx context.Context, href string, options ...RequestOption) (resp *Response, err error) {
return defaultClient.Request(ctx, http.MethodOptions, href, options...)
}
// sends a Trace request and returns the response.
func Trace(ctx context.Context, href string, options ...RequestOption) (resp *Response, err error) {
return defaultClient.Request(ctx, http.MethodTrace, href, options...)
}
// Define a function named Request that takes in four parameters:
func Request(ctx context.Context, method string, href string, options ...RequestOption) (resp *Response, err error) {
return defaultClient.Request(ctx, method, href, options...)
}
// sends a Get request and returns the response.
func (obj *Client) Get(ctx context.Context, href string, options ...RequestOption) (*Response, error) {
return obj.Request(ctx, http.MethodGet, href, options...)
}
// sends a Head request and returns the response.
func (obj *Client) Head(ctx context.Context, href string, options ...RequestOption) (*Response, error) {
return obj.Request(ctx, http.MethodHead, href, options...)
}
// sends a Post request and returns the response.
func (obj *Client) Post(ctx context.Context, href string, options ...RequestOption) (*Response, error) {
return obj.Request(ctx, http.MethodPost, href, options...)
}
// sends a Put request and returns the response.
func (obj *Client) Put(ctx context.Context, href string, options ...RequestOption) (*Response, error) {
return obj.Request(ctx, http.MethodPut, href, options...)
}
// sends a Patch request and returns the response.
func (obj *Client) Patch(ctx context.Context, href string, options ...RequestOption) (*Response, error) {
return obj.Request(ctx, http.MethodPatch, href, options...)
}
// sends a Delete request and returns the response.
func (obj *Client) Delete(ctx context.Context, href string, options ...RequestOption) (*Response, error) {
return obj.Request(ctx, http.MethodDelete, href, options...)
}
// sends a Connect request and returns the response.
func (obj *Client) Connect(ctx context.Context, href string, options ...RequestOption) (*Response, error) {
return obj.Request(ctx, http.MethodConnect, href, options...)
}
// sends a Options request and returns the response.
func (obj *Client) Options(ctx context.Context, href string, options ...RequestOption) (*Response, error) {
return obj.Request(ctx, http.MethodOptions, href, options...)
}
// sends a Trace request and returns the response.
func (obj *Client) Trace(ctx context.Context, href string, options ...RequestOption) (*Response, error) {
return obj.Request(ctx, http.MethodTrace, href, options...)
}
// Define a function named Request that takes in four parameters:
func (obj *Client) Request(ctx context.Context, method string, href string, options ...RequestOption) (response *Response, err error) {
if obj.closed {
return nil, errors.New("client is closed")
}
if ctx == nil {
ctx = obj.ctx
}
var rawOption RequestOption
if len(options) > 0 {
rawOption = options[0]
}
optionBak := obj.newRequestOption(rawOption)
if optionBak.Method == "" {
optionBak.Method = method
}
uhref := optionBak.Url
if uhref == nil {
if uhref, err = url.Parse(href); err != nil {
err = tools.WrapError(err, "url parse error")
return
}
}
for maxRetries := 0; maxRetries <= optionBak.MaxRetries; maxRetries++ {
option := optionBak
option.Url = cloneUrl(uhref)
option.client = obj
response, err = obj.request(ctx, &option)
if err == nil || errors.Is(err, errFatal) || option.once {
return
}
}
return
}
func (obj *Client) request(ctx context.Context, option *RequestOption) (response *Response, err error) {
response = new(Response)
defer func() {
//read body
if err == nil && !response.IsStream() {
err = response.ReadBody()
}
//result callback
if err == nil && option.ResultCallBack != nil {
err = option.ResultCallBack(ctx, option, response)
}
if err != nil { //err callback, must close body
response.CloseBody()
if option.ErrCallBack != nil {
if err2 := option.ErrCallBack(ctx, option, response, err); err2 != nil {
err = tools.WrapError(errFatal, err2)
}
}
} else if !response.IsStream() { //is not is stream must close body
response.CloseBody()
}
}()
if option.OptionCallBack != nil {
if err = option.OptionCallBack(ctx, option); err != nil {
return
}
}
response.bar = option.Bar
response.disUnzip = option.DisUnZip
response.disDecode = option.DisDecode
response.stream = option.Stream
//init headers and orderheaders,befor init ctxData
headers, orderHeaders, err := option.initHeaders()
if err != nil {
return response, tools.WrapError(err, errors.New("tempRequest init headers error"), err)
}
//设置 h2 请求头顺序
if orderHeaders != nil {
if !option.H2Ja3Spec.IsSet() {
option.H2Ja3Spec = ja3.DefaultH2Ja3Spec()
option.H2Ja3Spec.OrderHeaders = orderHeaders
} else if option.H2Ja3Spec.OrderHeaders == nil {
option.H2Ja3Spec.OrderHeaders = orderHeaders
}
}
//init ctxData
ctxData, err := NewReqCtxData(ctx, option)
if err != nil {
return response, tools.WrapError(err, " reqCtxData init error")
}
if headers == nil {
headers = defaultHeaders()
}
//设置 h1 请求头顺序
if orderHeaders != nil {
ctxData.orderHeaders = orderHeaders
} else {
ctxData.orderHeaders = ja3.DefaultOrderHeaders()
}
//init ctx,cnl
if option.Timeout > 0 { //超时
response.ctx, response.cnl = context.WithTimeout(CreateReqCtx(ctx, ctxData), option.Timeout)
} else {
response.ctx, response.cnl = context.WithCancel(CreateReqCtx(ctx, ctxData))
}
//init url
href, err := option.initParams()
if err != nil {
err = tools.WrapError(err, "url init error")
return
}
if href.User != nil {
headers.Set("Authorization", "Basic "+tools.Base64Encode(href.User.String()))
}
//init body
body, err := option.initBody(response.ctx)
if err != nil {
return response, tools.WrapError(err, errors.New("tempRequest init body error"), err)
}
//create request
reqs, err := NewRequestWithContext(response.ctx, option.Method, href, body)
if err != nil {
return response, tools.WrapError(errFatal, errors.New("tempRequest 构造request失败"), err)
}
reqs.Header = headers
//add Referer
if reqs.Header.Get("Referer") == "" {
if option.Referer != "" {
reqs.Header.Set("Referer", option.Referer)
} else if reqs.URL.Scheme != "" && reqs.URL.Host != "" {
reqs.Header.Set("Referer", fmt.Sprintf("%s://%s", reqs.URL.Scheme, reqs.URL.Host))
}
}
//set ContentType
if option.ContentType != "" && reqs.Header.Get("Content-Type") == "" {
reqs.Header.Set("Content-Type", option.ContentType)
}
//init ws
if ctxData.isWs {
websocket.SetClientHeadersWithOption(reqs.Header, option.WsOption)
}
if reqs.URL.Scheme == "file" {
response.filePath = re.Sub(`^/+`, "", reqs.URL.Path)
response.content, err = os.ReadFile(response.filePath)
if err != nil {
err = tools.WrapError(errFatal, errors.New("read filePath data error"), err)
}
return
}
//add host
if option.Host != "" {
reqs.Host = option.Host
} else if reqs.Header.Get("Host") != "" {
reqs.Host = reqs.Header.Get("Host")
} else {
reqs.Host = reqs.URL.Host
}
//init cookies
cookies, err := option.initCookies()
if err != nil {
return response, tools.WrapError(err, errors.New("tempRequest init cookies error"), err)
}
if cookies != nil {
addCookie(reqs, cookies)
}
//send req
response.response, err = obj.do(reqs, option)
response.isNewConn = ctxData.isNewConn
if err != nil {
err = tools.WrapError(err, "client do error")
return
}
if response.response == nil {
err = errors.New("response is nil")
return
}
if response.response.Body != nil {
response.rawConn = response.response.Body.(*readWriteCloser)
}
if !response.disUnzip {
response.disUnzip = response.response.Uncompressed
}
if response.response.StatusCode == 101 {
response.webSocket = websocket.NewClientConn(response.rawConn.Conn(), websocket.GetResponseHeaderOption(response.response.Header))
} else if strings.Contains(response.response.Header.Get("Content-Type"), "text/event-stream") {
response.sse = newSse(response.response.Body)
} else if !response.disUnzip {
var unCompressionBody io.ReadCloser
unCompressionBody, err = tools.CompressionDecode(response.response.Body, response.ContentEncoding())
if err != nil {
if err != io.ErrUnexpectedEOF && err != io.EOF {
return
}
}
if unCompressionBody != nil {
response.response.Body = unCompressionBody
}
}
return
}

702
response.go
View File

@@ -1,351 +1,351 @@
package requests
import (
"bufio"
"bytes"
"context"
"errors"
"io"
"net/url"
"strconv"
"strings"
"net/http"
"github.com/gospider007/bar"
"github.com/gospider007/bs4"
"github.com/gospider007/gson"
"github.com/gospider007/re"
"github.com/gospider007/tools"
"github.com/gospider007/websocket"
)
type Response struct {
rawConn *readWriteCloser
response *http.Response
webSocket *websocket.Conn
sse *Sse
ctx context.Context
cnl context.CancelFunc
content []byte
encoding string
stream bool
disDecode bool
disUnzip bool
filePath string
bar bool
isNewConn bool
readBody bool
}
type Sse struct {
reader *bufio.Reader
raw io.ReadCloser
}
type Event struct {
Data string //data
Event string //event
Id string //id
Retry int //retry num
Comment string //comment info
}
func newSse(rd io.ReadCloser) *Sse {
return &Sse{raw: rd, reader: bufio.NewReader(rd)}
}
// recv sse envent data
func (obj *Sse) Recv() (Event, error) {
var event Event
for {
readStr, err := obj.reader.ReadString('\n')
if err != nil || readStr == "\n" {
return event, err
}
reResult := re.Search(`data:\s?(.*)`, readStr)
if reResult != nil {
event.Data += reResult.Group(1)
continue
}
reResult = re.Search(`event:\s?(.*)`, readStr)
if reResult != nil {
event.Event = reResult.Group(1)
continue
}
reResult = re.Search(`id:\s?(.*)`, readStr)
if reResult != nil {
event.Id = reResult.Group(1)
continue
}
reResult = re.Search(`retry:\s?(.*)`, readStr)
if reResult != nil {
if event.Retry, err = strconv.Atoi(reResult.Group(1)); err != nil {
return event, err
}
continue
}
reResult = re.Search(`:\s?(.*)`, readStr)
if reResult != nil {
event.Comment = reResult.Group(1)
continue
}
return event, errors.New("content parse error:" + readStr)
}
}
// close sse
func (obj *Sse) Close() error {
return obj.raw.Close()
}
// return websocket client
func (obj *Response) WebSocket() *websocket.Conn {
return obj.webSocket
}
// return sse client
func (obj *Response) Sse() *Sse {
return obj.sse
}
// return URL redirected address
func (obj *Response) Location() (*url.URL, error) {
u, err := obj.response.Location()
if err == http.ErrNoLocation {
err = nil
}
return u, err
}
// return response Proto
func (obj *Response) Proto() string {
return obj.response.Proto
}
// return response cookies
func (obj *Response) Cookies() Cookies {
if obj.filePath != "" {
return nil
}
return obj.response.Cookies()
}
// return response status code
func (obj *Response) StatusCode() int {
if obj.filePath != "" {
return 200
}
return obj.response.StatusCode
}
// return response status
func (obj *Response) Status() string {
if obj.filePath != "" {
return "200 OK"
}
return obj.response.Status
}
// return response url
func (obj *Response) Url() *url.URL {
if obj.filePath != "" {
return nil
}
return obj.response.Request.URL
}
// return response headers
func (obj *Response) Headers() http.Header {
if obj.filePath != "" {
return http.Header{
"Content-Type": []string{obj.ContentType()},
}
}
return obj.response.Header
}
// change decoding with content
func (obj *Response) Decode(encoding string) {
if obj.encoding != encoding {
obj.encoding = encoding
obj.SetContent(tools.Decode(obj.Content(), encoding))
}
}
// return content with map[string]any
func (obj *Response) Map() (data map[string]any, err error) {
_, err = gson.Decode(obj.Content(), &data)
return
}
// return content with json and you can parse struct
func (obj *Response) Json(vals ...any) (*gson.Client, error) {
return gson.Decode(obj.Content(), vals...)
}
// return content with string
func (obj *Response) Text() string {
return tools.BytesToString(obj.Content())
}
// set response content with []byte
func (obj *Response) SetContent(val []byte) {
obj.content = val
}
// return content with []byte
func (obj *Response) Content() []byte {
return obj.content
}
// return content with parse html
func (obj *Response) Html() *bs4.Client {
return bs4.NewClient(obj.Text(), obj.Url().String())
}
// return content type
func (obj *Response) ContentType() string {
if obj.filePath != "" {
return http.DetectContentType(obj.content)
}
contentType := obj.response.Header.Get("Content-Type")
if contentType == "" {
contentType = http.DetectContentType(obj.content)
}
return contentType
}
// return content encoding
func (obj *Response) ContentEncoding() string {
if obj.filePath != "" {
return ""
}
return obj.response.Header.Get("Content-Encoding")
}
// return content length
func (obj *Response) ContentLength() int64 {
if obj.filePath != "" {
return int64(len(obj.content))
}
if obj.response.ContentLength >= 0 {
return obj.response.ContentLength
}
return int64(len(obj.content))
}
type barBody struct {
body *bytes.Buffer
bar *bar.Client
}
func (obj *barBody) Write(con []byte) (int, error) {
l, err := obj.body.Write(con)
obj.bar.Add(int64(l))
return l, err
}
func (obj *Response) defaultDecode() bool {
return strings.Contains(obj.ContentType(), "html")
}
// must stream=true
func (obj *Response) Conn() *connecotr {
if obj.IsStream() {
return obj.rawConn.Conn()
}
return nil
}
// return true if response is stream
func (obj *Response) IsStream() bool {
return obj.webSocket != nil || obj.sse != nil || obj.stream
}
// read body
func (obj *Response) ReadBody() (err error) {
if obj.IsStream() {
return errors.New("can not read stream")
}
if obj.readBody {
return errors.New("already read body")
}
obj.readBody = true
bBody := bytes.NewBuffer(nil)
if obj.bar && obj.ContentLength() > 0 {
_, err = io.Copy(&barBody{
bar: bar.NewClient(obj.response.ContentLength),
body: bBody,
}, obj.response.Body)
} else {
_, err = io.Copy(bBody, obj.response.Body)
}
if err != nil {
obj.ForceCloseConn()
return errors.New("response read content error: " + err.Error())
}
if !obj.disDecode && obj.defaultDecode() {
obj.content, obj.encoding, _ = tools.Charset(bBody.Bytes(), obj.ContentType())
} else {
obj.content = bBody.Bytes()
}
return
}
// conn is new conn
func (obj *Response) IsNewConn() bool {
return obj.isNewConn
}
// conn proxy
func (obj *Response) Proxy() string {
if obj.rawConn != nil {
return obj.rawConn.Proxy()
}
return ""
}
// conn is in pool ?
func (obj *Response) InPool() bool {
if obj.rawConn != nil {
return obj.rawConn.InPool()
}
return false
}
// close body
func (obj *Response) CloseBody() {
obj.close(false)
}
// safe close conn
func (obj *Response) CloseConn() {
obj.close(true)
}
// close
func (obj *Response) close(closeConn bool) {
if obj.webSocket != nil {
obj.webSocket.Close()
}
if obj.sse != nil {
obj.sse.Close()
}
if obj.IsStream() || !obj.readBody {
obj.ForceCloseConn()
} else if obj.rawConn != nil {
if closeConn {
obj.rawConn.CloseConn()
} else {
obj.rawConn.Close()
}
}
obj.cnl() //must later
}
// force close conn
func (obj *Response) ForceCloseConn() {
if obj.rawConn != nil {
obj.rawConn.ForceCloseConn()
}
}
package requests
import (
"bufio"
"bytes"
"context"
"errors"
"io"
"net/url"
"strconv"
"strings"
"net/http"
"github.com/gospider007/bar"
"github.com/gospider007/bs4"
"github.com/gospider007/gson"
"github.com/gospider007/re"
"github.com/gospider007/tools"
"github.com/gospider007/websocket"
)
type Response struct {
rawConn *readWriteCloser
response *http.Response
webSocket *websocket.Conn
sse *Sse
ctx context.Context
cnl context.CancelFunc
content []byte
encoding string
stream bool
disDecode bool
disUnzip bool
filePath string
bar bool
isNewConn bool
readBody bool
}
type Sse struct {
reader *bufio.Reader
raw io.ReadCloser
}
type Event struct {
Data string //data
Event string //event
Id string //id
Retry int //retry num
Comment string //comment info
}
func newSse(rd io.ReadCloser) *Sse {
return &Sse{raw: rd, reader: bufio.NewReader(rd)}
}
// recv sse envent data
func (obj *Sse) Recv() (Event, error) {
var event Event
for {
readStr, err := obj.reader.ReadString('\n')
if err != nil || readStr == "\n" {
return event, err
}
reResult := re.Search(`data:\s?(.*)`, readStr)
if reResult != nil {
event.Data += reResult.Group(1)
continue
}
reResult = re.Search(`event:\s?(.*)`, readStr)
if reResult != nil {
event.Event = reResult.Group(1)
continue
}
reResult = re.Search(`id:\s?(.*)`, readStr)
if reResult != nil {
event.Id = reResult.Group(1)
continue
}
reResult = re.Search(`retry:\s?(.*)`, readStr)
if reResult != nil {
if event.Retry, err = strconv.Atoi(reResult.Group(1)); err != nil {
return event, err
}
continue
}
reResult = re.Search(`:\s?(.*)`, readStr)
if reResult != nil {
event.Comment = reResult.Group(1)
continue
}
return event, errors.New("content parse error:" + readStr)
}
}
// close sse
func (obj *Sse) Close() error {
return obj.raw.Close()
}
// return websocket client
func (obj *Response) WebSocket() *websocket.Conn {
return obj.webSocket
}
// return sse client
func (obj *Response) Sse() *Sse {
return obj.sse
}
// return URL redirected address
func (obj *Response) Location() (*url.URL, error) {
u, err := obj.response.Location()
if err == http.ErrNoLocation {
err = nil
}
return u, err
}
// return response Proto
func (obj *Response) Proto() string {
return obj.response.Proto
}
// return response cookies
func (obj *Response) Cookies() Cookies {
if obj.filePath != "" {
return nil
}
return obj.response.Cookies()
}
// return response status code
func (obj *Response) StatusCode() int {
if obj.filePath != "" {
return 200
}
return obj.response.StatusCode
}
// return response status
func (obj *Response) Status() string {
if obj.filePath != "" {
return "200 OK"
}
return obj.response.Status
}
// return response url
func (obj *Response) Url() *url.URL {
if obj.filePath != "" {
return nil
}
return obj.response.Request.URL
}
// return response headers
func (obj *Response) Headers() http.Header {
if obj.filePath != "" {
return http.Header{
"Content-Type": []string{obj.ContentType()},
}
}
return obj.response.Header
}
// change decoding with content
func (obj *Response) Decode(encoding string) {
if obj.encoding != encoding {
obj.encoding = encoding
obj.SetContent(tools.Decode(obj.Content(), encoding))
}
}
// return content with map[string]any
func (obj *Response) Map() (data map[string]any, err error) {
_, err = gson.Decode(obj.Content(), &data)
return
}
// return content with json and you can parse struct
func (obj *Response) Json(vals ...any) (*gson.Client, error) {
return gson.Decode(obj.Content(), vals...)
}
// return content with string
func (obj *Response) Text() string {
return tools.BytesToString(obj.Content())
}
// set response content with []byte
func (obj *Response) SetContent(val []byte) {
obj.content = val
}
// return content with []byte
func (obj *Response) Content() []byte {
return obj.content
}
// return content with parse html
func (obj *Response) Html() *bs4.Client {
return bs4.NewClient(obj.Text(), obj.Url().String())
}
// return content type
func (obj *Response) ContentType() string {
if obj.filePath != "" {
return http.DetectContentType(obj.content)
}
contentType := obj.response.Header.Get("Content-Type")
if contentType == "" {
contentType = http.DetectContentType(obj.content)
}
return contentType
}
// return content encoding
func (obj *Response) ContentEncoding() string {
if obj.filePath != "" {
return ""
}
return obj.response.Header.Get("Content-Encoding")
}
// return content length
func (obj *Response) ContentLength() int64 {
if obj.filePath != "" {
return int64(len(obj.content))
}
if obj.response.ContentLength >= 0 {
return obj.response.ContentLength
}
return int64(len(obj.content))
}
type barBody struct {
body *bytes.Buffer
bar *bar.Client
}
func (obj *barBody) Write(con []byte) (int, error) {
l, err := obj.body.Write(con)
obj.bar.Add(int64(l))
return l, err
}
func (obj *Response) defaultDecode() bool {
return strings.Contains(obj.ContentType(), "html")
}
// must stream=true
func (obj *Response) Conn() *connecotr {
if obj.IsStream() {
return obj.rawConn.Conn()
}
return nil
}
// return true if response is stream
func (obj *Response) IsStream() bool {
return obj.webSocket != nil || obj.sse != nil || obj.stream
}
// read body
func (obj *Response) ReadBody() (err error) {
if obj.IsStream() {
return errors.New("can not read stream")
}
if obj.readBody {
return errors.New("already read body")
}
obj.readBody = true
bBody := bytes.NewBuffer(nil)
if obj.bar && obj.ContentLength() > 0 {
_, err = io.Copy(&barBody{
bar: bar.NewClient(obj.response.ContentLength),
body: bBody,
}, obj.response.Body)
} else {
_, err = io.Copy(bBody, obj.response.Body)
}
if err != nil {
obj.ForceCloseConn()
return errors.New("response read content error: " + err.Error())
}
if !obj.disDecode && obj.defaultDecode() {
obj.content, obj.encoding, _ = tools.Charset(bBody.Bytes(), obj.ContentType())
} else {
obj.content = bBody.Bytes()
}
return
}
// conn is new conn
func (obj *Response) IsNewConn() bool {
return obj.isNewConn
}
// conn proxy
func (obj *Response) Proxy() string {
if obj.rawConn != nil {
return obj.rawConn.Proxy()
}
return ""
}
// conn is in pool ?
func (obj *Response) InPool() bool {
if obj.rawConn != nil {
return obj.rawConn.InPool()
}
return false
}
// close body
func (obj *Response) CloseBody() {
obj.close(false)
}
// safe close conn
func (obj *Response) CloseConn() {
obj.close(true)
}
// close
func (obj *Response) close(closeConn bool) {
if obj.webSocket != nil {
obj.webSocket.Close()
}
if obj.sse != nil {
obj.sse.Close()
}
if obj.IsStream() || !obj.readBody {
obj.ForceCloseConn()
} else if obj.rawConn != nil {
if closeConn {
obj.rawConn.CloseConn()
} else {
obj.rawConn.Close()
}
}
obj.cnl() //must later
}
// force close conn
func (obj *Response) ForceCloseConn() {
if obj.rawConn != nil {
obj.rawConn.ForceCloseConn()
}
}

582
roundTripper.go
View File

@@ -1,291 +1,291 @@
package requests
import (
"bufio"
"context"
"crypto/tls"
"errors"
"fmt"
"net"
"net/textproto"
"net/url"
"strings"
"time"
"net/http"
"github.com/gospider007/gtls"
"github.com/gospider007/net/http2"
"github.com/gospider007/tools"
utls "github.com/refraction-networking/utls"
)
type reqTask struct {
ctx context.Context
cnl context.CancelFunc
req *http.Request
res *http.Response
emptyPool chan struct{}
err error
orderHeaders []string
orderHeaders2 []string
}
func (obj *reqTask) inPool() bool {
return obj.err == nil && obj.res != nil && obj.res.StatusCode != 101 && !strings.Contains(obj.res.Header.Get("Content-Type"), "text/event-stream")
}
func getKey(ctxData *reqCtxData, req *http.Request) (key string) {
var proxyUser string
if ctxData.proxy != nil {
proxyUser = ctxData.proxy.User.String()
}
return fmt.Sprintf("%s@%s@%s", proxyUser, getAddr(ctxData.proxy), getAddr(req.URL))
}
type roundTripper struct {
ctx context.Context
cnl context.CancelFunc
connPools *connPools
dialer *DialClient
tlsConfig *tls.Config
utlsConfig *utls.Config
getProxy func(ctx context.Context, url *url.URL) (string, error)
}
func newRoundTripper(preCtx context.Context, option ClientOption) *roundTripper {
if preCtx == nil {
preCtx = context.TODO()
}
ctx, cnl := context.WithCancel(preCtx)
dialClient := NewDail(DialOption{
DialTimeout: option.DialTimeout,
Dns: option.Dns,
KeepAlive: option.KeepAlive,
LocalAddr: option.LocalAddr,
AddrType: option.AddrType,
GetAddrType: option.GetAddrType,
})
tlsConfig := &tls.Config{
InsecureSkipVerify: true,
SessionTicketKey: [32]byte{},
ClientSessionCache: tls.NewLRUClientSessionCache(0),
}
utlsConfig := &utls.Config{
InsecureSkipVerify: true,
InsecureSkipTimeVerify: true,
SessionTicketKey: [32]byte{},
ClientSessionCache: utls.NewLRUClientSessionCache(0),
OmitEmptyPsk: true,
PreferSkipResumptionOnNilExtension: true,
}
return &roundTripper{
tlsConfig: tlsConfig,
utlsConfig: utlsConfig,
ctx: ctx,
cnl: cnl,
dialer: dialClient,
getProxy: option.GetProxy,
connPools: newConnPools(),
}
}
func (obj *roundTripper) newConnPool(conn *connecotr, key string) *connPool {
pool := new(connPool)
pool.connKey = key
pool.deleteCtx, pool.deleteCnl = context.WithCancelCause(obj.ctx)
pool.closeCtx, pool.closeCnl = context.WithCancelCause(pool.deleteCtx)
pool.tasks = make(chan *reqTask)
pool.connPools = obj.connPools
pool.total.Add(1)
go pool.rwMain(conn)
return pool
}
func (obj *roundTripper) putConnPool(key string, conn *connecotr) {
conn.inPool = true
if conn.h2RawConn == nil {
go conn.read()
}
pool := obj.connPools.get(key)
if pool != nil {
pool.total.Add(1)
go pool.rwMain(conn)
} else {
obj.connPools.set(key, obj.newConnPool(conn, key))
}
}
func (obj *roundTripper) tlsConfigClone() *tls.Config {
return obj.tlsConfig.Clone()
}
func (obj *roundTripper) utlsConfigClone() *utls.Config {
return obj.utlsConfig.Clone()
}
func (obj *roundTripper) newConnecotr(netConn net.Conn) *connecotr {
conne := new(connecotr)
conne.withCancel(obj.ctx, obj.ctx)
conne.rawConn = netConn
return conne
}
func (obj *roundTripper) dial(ctxData *reqCtxData, req *http.Request) (conn *connecotr, err error) {
var proxy *url.URL
if !ctxData.disProxy {
if proxy = cloneUrl(ctxData.proxy); proxy == nil && obj.getProxy != nil {
proxyStr, err := obj.getProxy(req.Context(), proxy)
if err != nil {
return conn, err
}
if proxy, err = gtls.VerifyProxy(proxyStr); err != nil {
return conn, err
}
}
}
netConn, err := obj.dialer.DialContextWithProxy(req.Context(), ctxData, "tcp", req.URL.Scheme, getAddr(req.URL), getHost(req), proxy, obj.tlsConfigClone())
if err != nil {
return conn, err
}
var h2 bool
if req.URL.Scheme == "https" {
ctx, cnl := context.WithTimeout(req.Context(), ctxData.tlsHandshakeTimeout)
defer cnl()
if ctxData.ja3Spec.IsSet() {
tlsConfig := obj.utlsConfigClone()
if ctxData.forceHttp1 {
tlsConfig.NextProtos = []string{"http/1.1"}
}
tlsConn, err := obj.dialer.addJa3Tls(ctx, netConn, getHost(req), ctxData.isWs || ctxData.forceHttp1, ctxData.ja3Spec, tlsConfig)
if err != nil {
return conn, tools.WrapError(err, "add ja3 tls error")
}
h2 = tlsConn.ConnectionState().NegotiatedProtocol == "h2"
netConn = tlsConn
} else {
tlsConn, err := obj.dialer.addTls(ctx, netConn, getHost(req), ctxData.isWs || ctxData.forceHttp1, obj.tlsConfigClone())
if err != nil {
return conn, tools.WrapError(err, "add tls error")
}
h2 = tlsConn.ConnectionState().NegotiatedProtocol == "h2"
netConn = tlsConn
}
}
conne := obj.newConnecotr(netConn)
if proxy != nil {
conne.proxy = proxy.String()
}
if h2 {
if conne.h2RawConn, err = http2.NewClientConn(func() {
conne.closeCnl(errors.New("http2 client close"))
}, netConn, ctxData.h2Ja3Spec); err != nil {
return conne, err
}
} else {
conne.r, conne.w = textproto.NewReader(bufio.NewReader(conne)), bufio.NewWriter(conne)
}
return conne, err
}
func (obj *roundTripper) setGetProxy(getProxy func(ctx context.Context, url *url.URL) (string, error)) {
obj.getProxy = getProxy
}
func (obj *roundTripper) poolRoundTrip(ctxData *reqCtxData, task *reqTask, key string) (newConn bool) {
pool := obj.connPools.get(key)
if pool == nil {
return true
}
task.ctx, task.cnl = context.WithTimeout(task.req.Context(), ctxData.responseHeaderTimeout)
select {
case pool.tasks <- task:
select {
case <-task.emptyPool:
return true
case <-task.ctx.Done():
if task.err == nil && task.res == nil {
task.err = task.ctx.Err()
}
return false
}
default:
return true
}
}
func (obj *roundTripper) connRoundTrip(ctxData *reqCtxData, task *reqTask, key string) (retry bool) {
conn, err := obj.dial(ctxData, task.req)
if err != nil {
task.err = err
return
}
task.ctx, task.cnl = context.WithTimeout(task.req.Context(), ctxData.responseHeaderTimeout)
retry = conn.taskMain(task, false)
if retry || task.err != nil {
return retry
}
if task.inPool() && !ctxData.disAlive {
obj.putConnPool(key, conn)
}
return retry
}
func (obj *roundTripper) closeConns() {
obj.connPools.iter(func(key string, pool *connPool) bool {
pool.close()
obj.connPools.del(key)
return true
})
}
func (obj *roundTripper) forceCloseConns() {
obj.connPools.iter(func(key string, pool *connPool) bool {
pool.forceClose()
obj.connPools.del(key)
return true
})
}
func (obj *roundTripper) newReqTask(req *http.Request, ctxData *reqCtxData) *reqTask {
if ctxData.responseHeaderTimeout == 0 {
ctxData.responseHeaderTimeout = time.Second * 300
}
task := new(reqTask)
task.req = req
task.emptyPool = make(chan struct{})
task.orderHeaders = ctxData.orderHeaders
if ctxData.h2Ja3Spec.OrderHeaders != nil {
task.orderHeaders2 = ctxData.h2Ja3Spec.OrderHeaders
} else {
task.orderHeaders2 = ctxData.orderHeaders
}
return task
}
func (obj *roundTripper) RoundTrip(req *http.Request) (response *http.Response, err error) {
ctxData := GetReqCtxData(req.Context())
if ctxData.requestCallBack != nil {
if err = ctxData.requestCallBack(req.Context(), req, nil); err != nil {
if err == http.ErrUseLastResponse {
if req.Response == nil {
return nil, errors.New("errUseLastResponse response is nil")
} else {
return req.Response, nil
}
}
return nil, err
}
}
key := getKey(ctxData, req) //pool key
task := obj.newReqTask(req, ctxData)
//get pool conn
var isNewConn bool
if !ctxData.disAlive {
isNewConn = obj.poolRoundTrip(ctxData, task, key)
}
if ctxData.disAlive || isNewConn {
ctxData.isNewConn = true
for {
retry := obj.connRoundTrip(ctxData, task, key)
if !retry {
break
}
}
}
if task.err == nil && ctxData.requestCallBack != nil {
if err = ctxData.requestCallBack(task.req.Context(), task.req, task.res); err != nil {
task.err = err
}
}
return task.res, task.err
}
package requests
import (
"bufio"
"context"
"crypto/tls"
"errors"
"fmt"
"net"
"net/textproto"
"net/url"
"strings"
"time"
"net/http"
"github.com/gospider007/gtls"
"github.com/gospider007/net/http2"
"github.com/gospider007/tools"
utls "github.com/refraction-networking/utls"
)
type reqTask struct {
ctx context.Context
cnl context.CancelFunc
req *http.Request
res *http.Response
emptyPool chan struct{}
err error
orderHeaders []string
orderHeaders2 []string
}
func (obj *reqTask) inPool() bool {
return obj.err == nil && obj.res != nil && obj.res.StatusCode != 101 && !strings.Contains(obj.res.Header.Get("Content-Type"), "text/event-stream")
}
func getKey(ctxData *reqCtxData, req *http.Request) (key string) {
var proxyUser string
if ctxData.proxy != nil {
proxyUser = ctxData.proxy.User.String()
}
return fmt.Sprintf("%s@%s@%s", proxyUser, getAddr(ctxData.proxy), getAddr(req.URL))
}
type roundTripper struct {
ctx context.Context
cnl context.CancelFunc
connPools *connPools
dialer *DialClient
tlsConfig *tls.Config
utlsConfig *utls.Config
getProxy func(ctx context.Context, url *url.URL) (string, error)
}
func newRoundTripper(preCtx context.Context, option ClientOption) *roundTripper {
if preCtx == nil {
preCtx = context.TODO()
}
ctx, cnl := context.WithCancel(preCtx)
dialClient := NewDail(DialOption{
DialTimeout: option.DialTimeout,
Dns: option.Dns,
KeepAlive: option.KeepAlive,
LocalAddr: option.LocalAddr,
AddrType: option.AddrType,
GetAddrType: option.GetAddrType,
})
tlsConfig := &tls.Config{
InsecureSkipVerify: true,
SessionTicketKey: [32]byte{},
ClientSessionCache: tls.NewLRUClientSessionCache(0),
}
utlsConfig := &utls.Config{
InsecureSkipVerify: true,
InsecureSkipTimeVerify: true,
SessionTicketKey: [32]byte{},
ClientSessionCache: utls.NewLRUClientSessionCache(0),
OmitEmptyPsk: true,
PreferSkipResumptionOnNilExtension: true,
}
return &roundTripper{
tlsConfig: tlsConfig,
utlsConfig: utlsConfig,
ctx: ctx,
cnl: cnl,
dialer: dialClient,
getProxy: option.GetProxy,
connPools: newConnPools(),
}
}
func (obj *roundTripper) newConnPool(conn *connecotr, key string) *connPool {
pool := new(connPool)
pool.connKey = key
pool.deleteCtx, pool.deleteCnl = context.WithCancelCause(obj.ctx)
pool.closeCtx, pool.closeCnl = context.WithCancelCause(pool.deleteCtx)
pool.tasks = make(chan *reqTask)
pool.connPools = obj.connPools
pool.total.Add(1)
go pool.rwMain(conn)
return pool
}
func (obj *roundTripper) putConnPool(key string, conn *connecotr) {
conn.inPool = true
if conn.h2RawConn == nil {
go conn.read()
}
pool := obj.connPools.get(key)
if pool != nil {
pool.total.Add(1)
go pool.rwMain(conn)
} else {
obj.connPools.set(key, obj.newConnPool(conn, key))
}
}
func (obj *roundTripper) tlsConfigClone() *tls.Config {
return obj.tlsConfig.Clone()
}
func (obj *roundTripper) utlsConfigClone() *utls.Config {
return obj.utlsConfig.Clone()
}
func (obj *roundTripper) newConnecotr(netConn net.Conn) *connecotr {
conne := new(connecotr)
conne.withCancel(obj.ctx, obj.ctx)
conne.rawConn = netConn
return conne
}
func (obj *roundTripper) dial(ctxData *reqCtxData, req *http.Request) (conn *connecotr, err error) {
var proxy *url.URL
if !ctxData.disProxy {
if proxy = cloneUrl(ctxData.proxy); proxy == nil && obj.getProxy != nil {
proxyStr, err := obj.getProxy(req.Context(), proxy)
if err != nil {
return conn, err
}
if proxy, err = gtls.VerifyProxy(proxyStr); err != nil {
return conn, err
}
}
}
netConn, err := obj.dialer.DialContextWithProxy(req.Context(), ctxData, "tcp", req.URL.Scheme, getAddr(req.URL), getHost(req), proxy, obj.tlsConfigClone())
if err != nil {
return conn, err
}
var h2 bool
if req.URL.Scheme == "https" {
ctx, cnl := context.WithTimeout(req.Context(), ctxData.tlsHandshakeTimeout)
defer cnl()
if ctxData.ja3Spec.IsSet() {
tlsConfig := obj.utlsConfigClone()
if ctxData.forceHttp1 {
tlsConfig.NextProtos = []string{"http/1.1"}
}
tlsConn, err := obj.dialer.addJa3Tls(ctx, netConn, getHost(req), ctxData.isWs || ctxData.forceHttp1, ctxData.ja3Spec, tlsConfig)
if err != nil {
return conn, tools.WrapError(err, "add ja3 tls error")
}
h2 = tlsConn.ConnectionState().NegotiatedProtocol == "h2"
netConn = tlsConn
} else {
tlsConn, err := obj.dialer.addTls(ctx, netConn, getHost(req), ctxData.isWs || ctxData.forceHttp1, obj.tlsConfigClone())
if err != nil {
return conn, tools.WrapError(err, "add tls error")
}
h2 = tlsConn.ConnectionState().NegotiatedProtocol == "h2"
netConn = tlsConn
}
}
conne := obj.newConnecotr(netConn)
if proxy != nil {
conne.proxy = proxy.String()
}
if h2 {
if conne.h2RawConn, err = http2.NewClientConn(func() {
conne.closeCnl(errors.New("http2 client close"))
}, netConn, ctxData.h2Ja3Spec); err != nil {
return conne, err
}
} else {
conne.r, conne.w = textproto.NewReader(bufio.NewReader(conne)), bufio.NewWriter(conne)
}
return conne, err
}
func (obj *roundTripper) setGetProxy(getProxy func(ctx context.Context, url *url.URL) (string, error)) {
obj.getProxy = getProxy
}
func (obj *roundTripper) poolRoundTrip(ctxData *reqCtxData, task *reqTask, key string) (newConn bool) {
pool := obj.connPools.get(key)
if pool == nil {
return true
}
task.ctx, task.cnl = context.WithTimeout(task.req.Context(), ctxData.responseHeaderTimeout)
select {
case pool.tasks <- task:
select {
case <-task.emptyPool:
return true
case <-task.ctx.Done():
if task.err == nil && task.res == nil {
task.err = task.ctx.Err()
}
return false
}
default:
return true
}
}
func (obj *roundTripper) connRoundTrip(ctxData *reqCtxData, task *reqTask, key string) (retry bool) {
conn, err := obj.dial(ctxData, task.req)
if err != nil {
task.err = err
return
}
task.ctx, task.cnl = context.WithTimeout(task.req.Context(), ctxData.responseHeaderTimeout)
retry = conn.taskMain(task, false)
if retry || task.err != nil {
return retry
}
if task.inPool() && !ctxData.disAlive {
obj.putConnPool(key, conn)
}
return retry
}
func (obj *roundTripper) closeConns() {
obj.connPools.iter(func(key string, pool *connPool) bool {
pool.close()
obj.connPools.del(key)
return true
})
}
func (obj *roundTripper) forceCloseConns() {
obj.connPools.iter(func(key string, pool *connPool) bool {
pool.forceClose()
obj.connPools.del(key)
return true
})
}
func (obj *roundTripper) newReqTask(req *http.Request, ctxData *reqCtxData) *reqTask {
if ctxData.responseHeaderTimeout == 0 {
ctxData.responseHeaderTimeout = time.Second * 300
}
task := new(reqTask)
task.req = req
task.emptyPool = make(chan struct{})
task.orderHeaders = ctxData.orderHeaders
if ctxData.h2Ja3Spec.OrderHeaders != nil {
task.orderHeaders2 = ctxData.h2Ja3Spec.OrderHeaders
} else {
task.orderHeaders2 = ctxData.orderHeaders
}
return task
}
func (obj *roundTripper) RoundTrip(req *http.Request) (response *http.Response, err error) {
ctxData := GetReqCtxData(req.Context())
if ctxData.requestCallBack != nil {
if err = ctxData.requestCallBack(req.Context(), req, nil); err != nil {
if err == http.ErrUseLastResponse {
if req.Response == nil {
return nil, errors.New("errUseLastResponse response is nil")
} else {
return req.Response, nil
}
}
return nil, err
}
}
key := getKey(ctxData, req) //pool key
task := obj.newReqTask(req, ctxData)
//get pool conn
var isNewConn bool
if !ctxData.disAlive {
isNewConn = obj.poolRoundTrip(ctxData, task, key)
}
if ctxData.disAlive || isNewConn {
ctxData.isNewConn = true
for {
retry := obj.connRoundTrip(ctxData, task, key)
if !retry {
break
}
}
}
if task.err == nil && ctxData.requestCallBack != nil {
if err = ctxData.requestCallBack(task.req.Context(), task.req, task.res); err != nil {
task.err = err
}
}
return task.res, task.err
}

102
rw.go
View File

@@ -1,51 +1,51 @@
package requests
import (
"errors"
"io"
)
type readWriteCloser struct {
body io.ReadCloser
conn *connecotr
}
func (obj *readWriteCloser) Conn() *connecotr {
return obj.conn
}
func (obj *readWriteCloser) Read(p []byte) (n int, err error) {
return obj.body.Read(p)
}
func (obj *readWriteCloser) InPool() bool {
return obj.conn.inPool
}
func (obj *readWriteCloser) Proxy() string {
return obj.conn.proxy
}
var ErrgospiderBodyClose = errors.New("gospider body close error")
func (obj *readWriteCloser) Close() (err error) {
if !obj.InPool() {
obj.ForceCloseConn()
} else {
err = obj.body.Close() //reuse conn
obj.conn.bodyCnl(ErrgospiderBodyClose)
}
return
}
// safe close conn
func (obj *readWriteCloser) CloseConn() {
if !obj.InPool() {
obj.ForceCloseConn()
} else {
obj.conn.bodyCnl(errors.New("readWriterCloser close conn"))
obj.conn.closeCnl(errors.New("readWriterCloser close conn"))
}
}
// force close conn
func (obj *readWriteCloser) ForceCloseConn() {
obj.conn.Close()
}
package requests
import (
"errors"
"io"
)
type readWriteCloser struct {
body io.ReadCloser
conn *connecotr
}
func (obj *readWriteCloser) Conn() *connecotr {
return obj.conn
}
func (obj *readWriteCloser) Read(p []byte) (n int, err error) {
return obj.body.Read(p)
}
func (obj *readWriteCloser) InPool() bool {
return obj.conn.inPool
}
func (obj *readWriteCloser) Proxy() string {
return obj.conn.proxy
}
var ErrgospiderBodyClose = errors.New("gospider body close error")
func (obj *readWriteCloser) Close() (err error) {
if !obj.InPool() {
obj.ForceCloseConn()
} else {
err = obj.body.Close() //reuse conn
obj.conn.bodyCnl(ErrgospiderBodyClose)
}
return
}
// safe close conn
func (obj *readWriteCloser) CloseConn() {
if !obj.InPool() {
obj.ForceCloseConn()
} else {
obj.conn.bodyCnl(errors.New("readWriterCloser close conn"))
obj.conn.closeCnl(errors.New("readWriterCloser close conn"))
}
}
// force close conn
func (obj *readWriteCloser) ForceCloseConn() {
obj.conn.Close()
}

42
test/addType_test.go
View File

@@ -1,21 +1,21 @@
package main
import (
"testing"
"github.com/gospider007/gtls"
"github.com/gospider007/requests"
)
func TestAddType(t *testing.T) {
session, _ := requests.NewClient(nil, requests.ClientOption{
AddrType: gtls.Ipv4, // Prioritize parsing IPv4 addresses
})
resp, err := session.Get(nil, "https://test.ipw.cn")
if err != nil {
t.Error(err)
}
if resp.StatusCode() != 200 {
t.Errorf("status code error, expected 200, got %d", resp.StatusCode())
}
}
package main
import (
"testing"
"github.com/gospider007/gtls"
"github.com/gospider007/requests"
)
func TestAddType(t *testing.T) {
session, _ := requests.NewClient(nil, requests.ClientOption{
AddrType: gtls.Ipv4, // Prioritize parsing IPv4 addresses
})
resp, err := session.Get(nil, "https://test.ipw.cn")
if err != nil {
t.Error(err)
}
if resp.StatusCode() != 200 {
t.Errorf("status code error, expected 200, got %d", resp.StatusCode())
}
}

60
test/cookies/setCookies_test.go
View File

@@ -1,30 +1,30 @@
package main
import (
"context"
"log"
"net/http"
"testing"
"github.com/gospider007/requests"
)
func TestSetCookies(t *testing.T) {
session, _ := requests.NewClient(context.TODO())
_, err := session.Get(context.TODO(), "https://www.baidu.com")
if err != nil {
log.Panic(err)
}
_, err = session.Get(context.TODO(), "https://www.baidu.com", requests.RequestOption{
RequestCallBack: func(ctx context.Context, request *http.Request, response *http.Response) error {
if request.Cookies() == nil {
log.Panic("cookie is nil")
}
return nil
},
})
if err != nil {
log.Panic(err)
}
}
package main
import (
"context"
"log"
"net/http"
"testing"
"github.com/gospider007/requests"
)
func TestSetCookies(t *testing.T) {
session, _ := requests.NewClient(context.TODO())
_, err := session.Get(context.TODO(), "https://www.baidu.com")
if err != nil {
log.Panic(err)
}
_, err = session.Get(context.TODO(), "https://www.baidu.com", requests.RequestOption{
RequestCallBack: func(ctx context.Context, request *http.Request, response *http.Response) error {
if request.Cookies() == nil {
log.Panic("cookie is nil")
}
return nil
},
})
if err != nil {
log.Panic(err)
}
}

46
test/dns_test.go
View File

@@ -1,23 +1,23 @@
package main
import (
"net"
"testing"
"github.com/gospider007/requests"
)
func TestDns(t *testing.T) {
resp, err := requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{
Dns: &net.UDPAddr{ //set dns server
IP: net.ParseIP("223.5.5.5"),
Port: 53,
},
})
if err != nil {
t.Fatal(err)
}
if resp.StatusCode() != 200 {
t.Fatal("http status code is not 200")
}
}
package main
import (
"net"
"testing"
"github.com/gospider007/requests"
)
func TestDns(t *testing.T) {
resp, err := requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{
Dns: &net.UDPAddr{ //set dns server
IP: net.ParseIP("223.5.5.5"),
Port: 53,
},
})
if err != nil {
t.Fatal(err)
}
if resp.StatusCode() != 200 {
t.Fatal("http status code is not 200")
}
}

74
test/fingerprint/http2_test.go
View File

@@ -1,37 +1,37 @@
package main
import (
"log"
"testing"
"github.com/gospider007/ja3"
"github.com/gospider007/requests"
)
func TestH2(t *testing.T) {
j := "1:65536,2:0,4:6291456,6:262144|15663105|0|m,a,s,p"
j2 := "1:65536;2:0;4:6291456;6:262144|15663105|0|m,a,s,p"
h2Spec, err := ja3.CreateH2SpecWithStr(j) //create h2 spec with string
if err != nil {
t.Fatal(err)
}
// log.Print(h2Spec)
resp, err := requests.Get(nil, "https://tools.scrapfly.io/api/fp/anything", requests.RequestOption{
H2Ja3Spec: h2Spec, //set h2 spec
})
// log.Print(resp.Text())
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json()
ja3 := jsonData.Get("http2.fingerprint")
if !ja3.Exists() {
t.Fatal("not found http2")
}
if j2 != ja3.String() {
log.Print(j)
log.Print(ja3)
t.Fatal("not equal")
}
}
package main
import (
"log"
"testing"
"github.com/gospider007/ja3"
"github.com/gospider007/requests"
)
func TestH2(t *testing.T) {
j := "1:65536,2:0,4:6291456,6:262144|15663105|0|m,a,s,p"
j2 := "1:65536;2:0;4:6291456;6:262144|15663105|0|m,a,s,p"
h2Spec, err := ja3.CreateH2SpecWithStr(j) //create h2 spec with string
if err != nil {
t.Fatal(err)
}
// log.Print(h2Spec)
resp, err := requests.Get(nil, "https://tools.scrapfly.io/api/fp/anything", requests.RequestOption{
H2Ja3Spec: h2Spec, //set h2 spec
})
// log.Print(resp.Text())
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json()
ja3 := jsonData.Get("http2.fingerprint")
if !ja3.Exists() {
t.Fatal("not found http2")
}
if j2 != ja3.String() {
log.Print(j)
log.Print(ja3)
t.Fatal("not equal")
}
}

182
test/fingerprint/ja3_test.go
View File

@@ -1,91 +1,91 @@
package main
import (
"log"
"testing"
"github.com/gospider007/ja3"
"github.com/gospider007/requests"
)
func TestJa3(t *testing.T) {
// j := "772,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,13-45-5-35-18-23-0-65281-10-65037-51-16-11-27-43-17513,12092-29-23-24,0"
j := "772,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,35-51-27-5-0-65281-13-65037-23-10-16-45-18-17513-43-11,29-23-24,0"
// j := "772,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,43-16-27-18-5-45-0-23-11-17513-10-65281-65037-35-13-51,29-23-24,0"
ja3Spec, err := ja3.CreateSpecWithStr(j) //create ja3 spec with string
// ja3Spec, err := ja3.CreateSpecWithId(ja3.HelloChrome_100)
if err != nil {
t.Fatal(err)
}
resp, err := requests.Get(nil, "https://tools.scrapfly.io/api/fp/anything", requests.RequestOption{
Ja3Spec: ja3Spec,
})
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json() //parse json
log.Print(jsonData.Find("scrapfly_fp"))
ja3 := jsonData.Get("tls.ja3") //get ja3 value
if ja3 == nil {
t.Fatal("not found ja3")
}
if j != ja3.String() {
log.Print(j)
log.Print(ja3)
t.Fatal("not equal")
}
}
func TestJa3Psk(t *testing.T) {
// j := "772,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,5-27-13-35-16-18-43-17513-65281-51-45-11-0-10-23-41,12092-29-23-24,0"
j := "772,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,5-27-13-35-16-18-43-17513-65281-51-45-11-0-10-23,12092-29-23-24,0"
j2 := "772,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,5-27-13-35-16-18-43-17513-65281-51-45-11-0-10-23,12092-29-23-24,0"
ja3Spec, err := ja3.CreateSpecWithStr(j) //create ja3 spec with string
if err != nil {
t.Fatal(err)
}
session, _ := requests.NewClient(nil)
for i := 0; i < 2; i++ {
resp, err := session.Get(nil, "https://tools.scrapfly.io/api/fp/anything", requests.RequestOption{
Ja3Spec: ja3Spec, //set ja3 spec
})
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json()
ja3 := jsonData.Get("tls.ja3")
if ja3 == nil {
t.Fatal("not found ja3")
}
var eqJ string
if i == 0 {
eqJ = j2
} else {
eqJ = j
}
if eqJ != ja3.String() {
log.Print(j)
log.Print(ja3)
t.Fatal("not equal")
}
}
}
func TestJa3ClientHello(t *testing.T) {
hexStream := "16030107da010007d60303f6e7130bee8e1362fb26e166e87da2e00d2ac2212370464b6dbb04a615b39eff20037c899a1ee9bec636722374c44308d7d40d8953994cec5dd44eb06786898f1e00206a6a130113021303c02bc02fc02cc030cca9cca8c013c014009c009d002f00350100076dcaca0000ff0100010000170000000a000c000acaca2f3c001d0017001800050005010000000000230000fe0d00fa00000100016e0020ab82cd7ab430aae210d15b45225c1be0b861d67b15428565458023468a48680c00d0427920f67aa56fc94e41a1b93be0d5d1596977a8113bd5eb9b32c93c7a75a47f93294a8b37e228fc64d4959224351efaacba5ddb7b3a19f8704501d8df49af7d7d03ee3c59f6a52cd8734d5317c8cc107411816aacaede9d5fc000e326ef05a8676fe8bc41fe8137aab39b9006d13ce768e731dc1e783862cecdf7d6ae8b2494f827f7e4cb4b22b5b2b8c98a8cb002620f0cf0a89e0303fe296392c10aa724a90c95cf48e9cfee49b0efbb7ac8ad1ac001c8e84d7c2ba68a1498b07594b8c547447b04d1c8b6c8717d91020f49c543c5001b0003020002000d0012001004030804040105030805050108060601446900050003026832000b00020100002b0007064a4a03040303001200000010000e000c02683208687474702f312e3100330530052ecaca0001002f3c050104e734eef6ce9d4e7dc128b8e01bd5602713f452286397fe21373a0a34a2cd04686c2434fa4bde624d401cf47ab703233cc47728b8efbce586c728103260ccf4c4ed5feb9c18a8318a6542766e318b8e61a109b68a242f205dd1759913332e14b2713b890b5c47e6e27c4c1021686827fa606554773a2b87060972241845a085c634536ca1a59a6e9ca40bcc543348a7745d581d4612545005068829bc37a41ec874177137368ac67c7df353d2674d62a1ab5c92ca5b489d8842844194aac5c853ba0b7b76139333a072e041419f9cc2f182cf1312519cc8192c129da8a5641924a5e2f06f19d6b9a5c9a0d3d65451d82383704579237d48159bb66214c5559738c95eebe595cbf81893c378c3f5a202660c8c0946012b42ed93bf17b7a43f44ae843735de7bbcf6643b0a03a7a3c0c71a068cf1f9b9d3265d016a3b5e94cf7173b698baa7f1a256224c36a71263768c6078c74bf6b8ba65550bb43cad4a04a407588aaef76a799cc4290c0044b893f4b5aa34176e648c515d11330d7b7a599922f8834efba00b4b7573c8c9b0a3367c237875f9f46e4298a26d02286b8c16ab4249847985fac03647bb686d132d9153186d50608dd19888161c899098b174037d6985401356ea92092eb91e6e6c30d3e05e1e5068d55a0b39d692675bcbb59b3c40a8733b59483e403922544bc8d63960ac35e3c7430f26235a159df153747887a259d81ab1811a236b57a2f4227d009fa9a988f9e45acae5128e4a4d5c1b31860454fd3012806568df489cfb389da087bae8e3bc68da62d86b77e7115acf761be3384bb6bc9f12fa62cb4810b2336d11c4b99e7458e95a32918364d846408acb779414663f19b6dcd6a58d65afdd899925141bf4692ee654690a3c6b906c6d4d892356175f8526cf183b6721e034d4a69a52f0148ff82b80babae073908bc60efa51c85d77cf183c2f75b1c675d79465474a5518c41c97511265c1dc67ad41c5320fa8c12b5cca15061a5292453fa18f35e42d7e6a4ce960c8bca6102a592a78c75a7ac41a95830a257441832c12c034b9cb95394078a6abe971262b000b341838eb748ce1cad878325a534da0016a0a924b306b872819235e182281c6a3c9a604e0ac262dfa30a4e4664b2c00f526cecb14125ff21f49c727db781f791573d39876956889ea990002bb74f180c26933b1428a998dd2bb1a8b40fb906b5680c08393cfc934520ad81ac2c388af06ad0a93125226a157d605e1f955c643113d387a7becbba50b5626351036ec329621ce4f9a9e39eab80ff6341151213deb66bd4256be415fc14093eda75f660b78997155fa6079afe3c10fd25a956a6e6c449f4275bf1ee40f3fb930b4130073718d06123b96f3c3f6227e3ff81d86f53b1025546df22f4303907fa07f344a086b1a34e6645f3e05b9b4242606f5c83935624ff5c6e7c49401c7725404bf2792709e0390ca082806189ca88799e71b2f1943761df228320c4c9b245b12b5565574967eb31c0993a059f5c6ccd1820ef2a994958085a866351333c17a676d5b0808d00dc1fa9912833eb1745f5b461f5407692053a912d43315aab1f4b69abd1986a9621f3c3722c633c36115460a47325a33335c68410f80a4e0fb4a79d75458032d285ba7d9f167340952fed75cce99a80a74661db32949072082978e7586598d4c2c82500a205760f821728587c1339105203711a2c55267e734569c915be2937409436dca2b098ac216492b57708d41986962e105957c6ea05ef0aac995722671961507ea6b21e8bcd687db34d24a8786ed001d00207e52cc0bfb373444c54aeb19720f42b2379d7db8910e2451321e440c8da6df360000001d001b000018676f737069646572322e676f737069646572622e61736961002d000201016a6a00010000290094006f0069b4f2a27d7236f2b6e8d4b6db52a2450dd48a8d833b2479a9f8be35157c9ab0b971f349d1eaf577a7046540a4f1ded29d88b9f3aef7ec4225de9e622981351e0e8eadc7ab0318f74704f63d3056f9fa399fb1fc757bb7c73f5f66e652db9b9f613fe30288436979337443cfd0fb002120f06d72218279775978c6be6fb2d36dc9e362768f4bb353b71141cf95e3ae7750"
ja3Spec, err := ja3.CreateSpecWithClientHello(hexStream)
if err != nil {
t.Fatal(err)
}
// log.Print(ja3Spec)
resp, err := requests.Get(nil, "https://tools.scrapfly.io/api/fp/anything", requests.RequestOption{
Ja3Spec: ja3Spec,
})
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json() //parse json
ja3 := jsonData.Get("tls.ja3") //get ja3 value
if ja3 == nil {
t.Fatal("not found ja3")
}
}
package main
import (
"log"
"testing"
"github.com/gospider007/ja3"
"github.com/gospider007/requests"
)
func TestJa3(t *testing.T) {
// j := "772,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,13-45-5-35-18-23-0-65281-10-65037-51-16-11-27-43-17513,12092-29-23-24,0"
j := "772,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,35-51-27-5-0-65281-13-65037-23-10-16-45-18-17513-43-11,29-23-24,0"
// j := "772,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,43-16-27-18-5-45-0-23-11-17513-10-65281-65037-35-13-51,29-23-24,0"
ja3Spec, err := ja3.CreateSpecWithStr(j) //create ja3 spec with string
// ja3Spec, err := ja3.CreateSpecWithId(ja3.HelloChrome_100)
if err != nil {
t.Fatal(err)
}
resp, err := requests.Get(nil, "https://tools.scrapfly.io/api/fp/anything", requests.RequestOption{
Ja3Spec: ja3Spec,
})
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json() //parse json
log.Print(jsonData.Find("scrapfly_fp"))
ja3 := jsonData.Get("tls.ja3") //get ja3 value
if ja3 == nil {
t.Fatal("not found ja3")
}
if j != ja3.String() {
log.Print(j)
log.Print(ja3)
t.Fatal("not equal")
}
}
func TestJa3Psk(t *testing.T) {
// j := "772,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,5-27-13-35-16-18-43-17513-65281-51-45-11-0-10-23-41,12092-29-23-24,0"
j := "772,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,5-27-13-35-16-18-43-17513-65281-51-45-11-0-10-23,12092-29-23-24,0"
j2 := "772,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,5-27-13-35-16-18-43-17513-65281-51-45-11-0-10-23,12092-29-23-24,0"
ja3Spec, err := ja3.CreateSpecWithStr(j) //create ja3 spec with string
if err != nil {
t.Fatal(err)
}
session, _ := requests.NewClient(nil)
for i := 0; i < 2; i++ {
resp, err := session.Get(nil, "https://tools.scrapfly.io/api/fp/anything", requests.RequestOption{
Ja3Spec: ja3Spec, //set ja3 spec
})
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json()
ja3 := jsonData.Get("tls.ja3")
if ja3 == nil {
t.Fatal("not found ja3")
}
var eqJ string
if i == 0 {
eqJ = j2
} else {
eqJ = j
}
if eqJ != ja3.String() {
log.Print(j)
log.Print(ja3)
t.Fatal("not equal")
}
}
}
func TestJa3ClientHello(t *testing.T) {
hexStream := "16030107da010007d60303f6e7130bee8e1362fb26e166e87da2e00d2ac2212370464b6dbb04a615b39eff20037c899a1ee9bec636722374c44308d7d40d8953994cec5dd44eb06786898f1e00206a6a130113021303c02bc02fc02cc030cca9cca8c013c014009c009d002f00350100076dcaca0000ff0100010000170000000a000c000acaca2f3c001d0017001800050005010000000000230000fe0d00fa00000100016e0020ab82cd7ab430aae210d15b45225c1be0b861d67b15428565458023468a48680c00d0427920f67aa56fc94e41a1b93be0d5d1596977a8113bd5eb9b32c93c7a75a47f93294a8b37e228fc64d4959224351efaacba5ddb7b3a19f8704501d8df49af7d7d03ee3c59f6a52cd8734d5317c8cc107411816aacaede9d5fc000e326ef05a8676fe8bc41fe8137aab39b9006d13ce768e731dc1e783862cecdf7d6ae8b2494f827f7e4cb4b22b5b2b8c98a8cb002620f0cf0a89e0303fe296392c10aa724a90c95cf48e9cfee49b0efbb7ac8ad1ac001c8e84d7c2ba68a1498b07594b8c547447b04d1c8b6c8717d91020f49c543c5001b0003020002000d0012001004030804040105030805050108060601446900050003026832000b00020100002b0007064a4a03040303001200000010000e000c02683208687474702f312e3100330530052ecaca0001002f3c050104e734eef6ce9d4e7dc128b8e01bd5602713f452286397fe21373a0a34a2cd04686c2434fa4bde624d401cf47ab703233cc47728b8efbce586c728103260ccf4c4ed5feb9c18a8318a6542766e318b8e61a109b68a242f205dd1759913332e14b2713b890b5c47e6e27c4c1021686827fa606554773a2b87060972241845a085c634536ca1a59a6e9ca40bcc543348a7745d581d4612545005068829bc37a41ec874177137368ac67c7df353d2674d62a1ab5c92ca5b489d8842844194aac5c853ba0b7b76139333a072e041419f9cc2f182cf1312519cc8192c129da8a5641924a5e2f06f19d6b9a5c9a0d3d65451d82383704579237d48159bb66214c5559738c95eebe595cbf81893c378c3f5a202660c8c0946012b42ed93bf17b7a43f44ae843735de7bbcf6643b0a03a7a3c0c71a068cf1f9b9d3265d016a3b5e94cf7173b698baa7f1a256224c36a71263768c6078c74bf6b8ba65550bb43cad4a04a407588aaef76a799cc4290c0044b893f4b5aa34176e648c515d11330d7b7a599922f8834efba00b4b7573c8c9b0a3367c237875f9f46e4298a26d02286b8c16ab4249847985fac03647bb686d132d9153186d50608dd19888161c899098b174037d6985401356ea92092eb91e6e6c30d3e05e1e5068d55a0b39d692675bcbb59b3c40a8733b59483e403922544bc8d63960ac35e3c7430f26235a159df153747887a259d81ab1811a236b57a2f4227d009fa9a988f9e45acae5128e4a4d5c1b31860454fd3012806568df489cfb389da087bae8e3bc68da62d86b77e7115acf761be3384bb6bc9f12fa62cb4810b2336d11c4b99e7458e95a32918364d846408acb779414663f19b6dcd6a58d65afdd899925141bf4692ee654690a3c6b906c6d4d892356175f8526cf183b6721e034d4a69a52f0148ff82b80babae073908bc60efa51c85d77cf183c2f75b1c675d79465474a5518c41c97511265c1dc67ad41c5320fa8c12b5cca15061a5292453fa18f35e42d7e6a4ce960c8bca6102a592a78c75a7ac41a95830a257441832c12c034b9cb95394078a6abe971262b000b341838eb748ce1cad878325a534da0016a0a924b306b872819235e182281c6a3c9a604e0ac262dfa30a4e4664b2c00f526cecb14125ff21f49c727db781f791573d39876956889ea990002bb74f180c26933b1428a998dd2bb1a8b40fb906b5680c08393cfc934520ad81ac2c388af06ad0a93125226a157d605e1f955c643113d387a7becbba50b5626351036ec329621ce4f9a9e39eab80ff6341151213deb66bd4256be415fc14093eda75f660b78997155fa6079afe3c10fd25a956a6e6c449f4275bf1ee40f3fb930b4130073718d06123b96f3c3f6227e3ff81d86f53b1025546df22f4303907fa07f344a086b1a34e6645f3e05b9b4242606f5c83935624ff5c6e7c49401c7725404bf2792709e0390ca082806189ca88799e71b2f1943761df228320c4c9b245b12b5565574967eb31c0993a059f5c6ccd1820ef2a994958085a866351333c17a676d5b0808d00dc1fa9912833eb1745f5b461f5407692053a912d43315aab1f4b69abd1986a9621f3c3722c633c36115460a47325a33335c68410f80a4e0fb4a79d75458032d285ba7d9f167340952fed75cce99a80a74661db32949072082978e7586598d4c2c82500a205760f821728587c1339105203711a2c55267e734569c915be2937409436dca2b098ac216492b57708d41986962e105957c6ea05ef0aac995722671961507ea6b21e8bcd687db34d24a8786ed001d00207e52cc0bfb373444c54aeb19720f42b2379d7db8910e2451321e440c8da6df360000001d001b000018676f737069646572322e676f737069646572622e61736961002d000201016a6a00010000290094006f0069b4f2a27d7236f2b6e8d4b6db52a2450dd48a8d833b2479a9f8be35157c9ab0b971f349d1eaf577a7046540a4f1ded29d88b9f3aef7ec4225de9e622981351e0e8eadc7ab0318f74704f63d3056f9fa399fb1fc757bb7c73f5f66e652db9b9f613fe30288436979337443cfd0fb002120f06d72218279775978c6be6fb2d36dc9e362768f4bb353b71141cf95e3ae7750"
ja3Spec, err := ja3.CreateSpecWithClientHello(hexStream)
if err != nil {
t.Fatal(err)
}
// log.Print(ja3Spec)
resp, err := requests.Get(nil, "https://tools.scrapfly.io/api/fp/anything", requests.RequestOption{
Ja3Spec: ja3Spec,
})
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json() //parse json
ja3 := jsonData.Get("tls.ja3") //get ja3 value
if ja3 == nil {
t.Fatal("not found ja3")
}
}

104
test/fingerprint/ja4_test.go
View File

@@ -1,52 +1,52 @@
package main
import (
"log"
"net/textproto"
"slices"
"testing"
"github.com/gospider007/requests"
)
func TestOrderHeaders(t *testing.T) {
headers := requests.NewOrderMap()
headers.Set("Accept-Encoding", "gzip, deflate, br")
headers.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7")
headers.Set("User-Agent", requests.UserAgent)
headers.Set("Accept-Language", requests.AcceptLanguage)
headers.Set("Sec-Ch-Ua", requests.SecChUa)
headers.Set("Sec-Ch-Ua-Mobile", "?0")
headers.Set("Sec-Ch-Ua-Platform", `"Windows"`)
resp, err := requests.Get(nil, "https://tools.scrapfly.io/api/fp/anything", requests.RequestOption{
Headers: headers,
// ForceHttp1: true,
})
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json()
header_order := jsonData.Find("ordered_headers_key")
if !header_order.Exists() {
t.Fatal("not found akamai")
}
i := -1
log.Print(header_order)
// log.Print(headers.Keys())
kks := []string{}
for _, kk := range headers.Keys() {
kks = append(kks, textproto.CanonicalMIMEHeaderKey(kk))
}
for _, key := range header_order.Array() {
kk := textproto.CanonicalMIMEHeaderKey(key.String())
if slices.Contains(kks, kk) {
i2 := slices.Index(kks, textproto.CanonicalMIMEHeaderKey(kk))
if i2 < i {
log.Print(header_order)
t.Fatal("not equal")
}
i = i2
}
}
}
package main
import (
"log"
"net/textproto"
"slices"
"testing"
"github.com/gospider007/requests"
)
func TestOrderHeaders(t *testing.T) {
headers := requests.NewOrderMap()
headers.Set("Accept-Encoding", "gzip, deflate, br")
headers.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7")
headers.Set("User-Agent", requests.UserAgent)
headers.Set("Accept-Language", requests.AcceptLanguage)
headers.Set("Sec-Ch-Ua", requests.SecChUa)
headers.Set("Sec-Ch-Ua-Mobile", "?0")
headers.Set("Sec-Ch-Ua-Platform", `"Windows"`)
resp, err := requests.Get(nil, "https://tools.scrapfly.io/api/fp/anything", requests.RequestOption{
Headers: headers,
// ForceHttp1: true,
})
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json()
header_order := jsonData.Find("ordered_headers_key")
if !header_order.Exists() {
t.Fatal("not found akamai")
}
i := -1
log.Print(header_order)
// log.Print(headers.Keys())
kks := []string{}
for _, kk := range headers.Keys() {
kks = append(kks, textproto.CanonicalMIMEHeaderKey(kk))
}
for _, key := range header_order.Array() {
kk := textproto.CanonicalMIMEHeaderKey(key.String())
if slices.Contains(kks, kk) {
i2 := slices.Index(kks, textproto.CanonicalMIMEHeaderKey(kk))
if i2 < i {
log.Print(header_order)
t.Fatal("not equal")
}
i = i2
}
}
}

64
test/middleware/errCallBack_test.go
View File

@@ -1,32 +1,32 @@
package main
import (
"context"
"errors"
"testing"
"github.com/gospider007/requests"
)
func TestErrCallBack(t *testing.T) {
n := 0
_, err := requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{
MaxRetries: 3,
ResultCallBack: func(ctx context.Context, option *requests.RequestOption, response *requests.Response) error {
return errors.New("try")
},
ErrCallBack: func(ctx context.Context, option *requests.RequestOption, response *requests.Response, err error) error {
if n == 0 {
n++
return nil
}
return errors.New("test")
},
})
if err == nil {
t.Error("callback error")
}
if n != 1 {
t.Error("n!=1")
}
}
package main
import (
"context"
"errors"
"testing"
"github.com/gospider007/requests"
)
func TestErrCallBack(t *testing.T) {
n := 0
_, err := requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{
MaxRetries: 3,
ResultCallBack: func(ctx context.Context, option *requests.RequestOption, response *requests.Response) error {
return errors.New("try")
},
ErrCallBack: func(ctx context.Context, option *requests.RequestOption, response *requests.Response, err error) error {
if n == 0 {
n++
return nil
}
return errors.New("test")
},
})
if err == nil {
t.Error("callback error")
}
if n != 1 {
t.Error("n!=1")
}
}

60
test/middleware/optionltCallBack_test.go
View File

@@ -1,30 +1,30 @@
package main
import (
"context"
"testing"
"github.com/gospider007/requests"
)
func TestOptionCallBack(t *testing.T) {
resp, err := requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{
OptionCallBack: func(ctx context.Context, option *requests.RequestOption) error {
option.Params = map[string]string{"name": "test"}
return nil
},
})
if err != nil {
t.Error(err)
}
if resp.StatusCode() != 200 {
t.Error("resp.StatusCode!= 200")
}
jsonData, err := resp.Json()
if err != nil {
t.Error(err)
}
if jsonData.Get("args.name").String() != "test" {
t.Error("jsonData.Get(\"args.name\").String()!= test")
}
}
package main
import (
"context"
"testing"
"github.com/gospider007/requests"
)
func TestOptionCallBack(t *testing.T) {
resp, err := requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{
OptionCallBack: func(ctx context.Context, option *requests.RequestOption) error {
option.Params = map[string]string{"name": "test"}
return nil
},
})
if err != nil {
t.Error(err)
}
if resp.StatusCode() != 200 {
t.Error("resp.StatusCode!= 200")
}
jsonData, err := resp.Json()
if err != nil {
t.Error(err)
}
if jsonData.Get("args.name").String() != "test" {
t.Error("jsonData.Get(\"args.name\").String()!= test")
}
}

60
test/middleware/requestCallback_test.go
View File

@@ -1,30 +1,30 @@
package main
import (
"context"
"errors"
"net/http"
"strings"
"testing"
"github.com/gospider007/requests"
)
func TestRequestCallBack(t *testing.T) {
_, err := requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{
RequestCallBack: func(ctx context.Context, request *http.Request, response *http.Response) error {
if response != nil {
if response.ContentLength > 100 {
return errors.New("max length")
}
}
return nil
},
})
if err == nil {
t.Error("err is nil")
}
if !strings.Contains(err.Error(), "max length") {
t.Error("err is not max length")
}
}
package main
import (
"context"
"errors"
"net/http"
"strings"
"testing"
"github.com/gospider007/requests"
)
func TestRequestCallBack(t *testing.T) {
_, err := requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{
RequestCallBack: func(ctx context.Context, request *http.Request, response *http.Response) error {
if response != nil {
if response.ContentLength > 100 {
return errors.New("max length")
}
}
return nil
},
})
if err == nil {
t.Error("err is nil")
}
if !strings.Contains(err.Error(), "max length") {
t.Error("err is not max length")
}
}

56
test/middleware/resultCallBack_test.go
View File

@@ -1,28 +1,28 @@
package main
import (
"context"
"errors"
"testing"
"github.com/gospider007/requests"
)
func TestResultCallBack(t *testing.T) {
var code int
_, err := requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{
ResultCallBack: func(ctx context.Context, option *requests.RequestOption, response *requests.Response) error {
if response.StatusCode() != 200 {
return errors.New("resp.StatusCode!= 200")
}
code = response.StatusCode()
return nil
},
})
if err != nil {
t.Error(err)
}
if code != 200 {
t.Error("code!= 200")
}
}
package main
import (
"context"
"errors"
"testing"
"github.com/gospider007/requests"
)
func TestResultCallBack(t *testing.T) {
var code int
_, err := requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{
ResultCallBack: func(ctx context.Context, option *requests.RequestOption, response *requests.Response) error {
if response.StatusCode() != 200 {
return errors.New("resp.StatusCode!= 200")
}
code = response.StatusCode()
return nil
},
})
if err != nil {
t.Error(err)
}
if code != 200 {
t.Error("code!= 200")
}
}

66
test/middleware/try_test.go
View File

@@ -1,33 +1,33 @@
package main
import (
"context"
"errors"
"testing"
"github.com/gospider007/requests"
)
func TestMaxRetries(t *testing.T) {
n := 0
resp, err := requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{
MaxRetries: 3,
ResultCallBack: func(ctx context.Context, option *requests.RequestOption, response *requests.Response) error {
if n == 0 {
n++
return errors.New("try")
}
return nil
},
})
if err != nil {
t.Error(err)
}
if resp.StatusCode() != 200 {
t.Error("resp.StatusCode!= 200")
}
if n != 1 {
t.Error("n!=1")
}
}
package main
import (
"context"
"errors"
"testing"
"github.com/gospider007/requests"
)
func TestMaxRetries(t *testing.T) {
n := 0
resp, err := requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{
MaxRetries: 3,
ResultCallBack: func(ctx context.Context, option *requests.RequestOption, response *requests.Response) error {
if n == 0 {
n++
return errors.New("try")
}
return nil
},
})
if err != nil {
t.Error(err)
}
if resp.StatusCode() != 200 {
t.Error("resp.StatusCode!= 200")
}
if n != 1 {
t.Error("n!=1")
}
}

46
test/protocol/http1_test.go
View File

@@ -1,23 +1,23 @@
package main
import (
"testing"
"github.com/gospider007/requests"
)
func TestHttp1(t *testing.T) {
resp, err := requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{
ForceHttp1: true,
DisAlive: true,
})
if err != nil {
t.Error(err)
}
if resp.StatusCode() != 200 {
t.Error("resp.StatusCode!= 200")
}
if resp.Proto() != "HTTP/1.1" {
t.Error("resp.Proto!= HTTP/1.1")
}
}
package main
import (
"testing"
"github.com/gospider007/requests"
)
func TestHttp1(t *testing.T) {
resp, err := requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{
ForceHttp1: true,
DisAlive: true,
})
if err != nil {
t.Error(err)
}
if resp.StatusCode() != 200 {
t.Error("resp.StatusCode!= 200")
}
if resp.Proto() != "HTTP/1.1" {
t.Error("resp.Proto!= HTTP/1.1")
}
}

40
test/protocol/http2_test.go
View File

@@ -1,20 +1,20 @@
package main
import (
"testing"
"github.com/gospider007/requests"
)
func TestHttp2(t *testing.T) {
resp, err := requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{DisAlive: true})
if err != nil {
t.Error(err)
}
if resp.StatusCode() != 200 {
t.Error("resp.StatusCode!= 200")
}
if resp.Proto() != "HTTP/2.0" {
t.Error("resp.Proto!= HTTP/2.0")
}
}
package main
import (
"testing"
"github.com/gospider007/requests"
)
func TestHttp2(t *testing.T) {
resp, err := requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{DisAlive: true})
if err != nil {
t.Error(err)
}
if resp.StatusCode() != 200 {
t.Error("resp.StatusCode!= 200")
}
if resp.Proto() != "HTTP/2.0" {
t.Error("resp.Proto!= HTTP/2.0")
}
}

142
test/protocol/sse_test.go
View File

@@ -1,71 +1,71 @@
package main
import (
"io"
"log"
"net/http"
"testing"
"time"
"github.com/gospider007/requests"
)
func TestSse(t *testing.T) {
// Start the server
go func() {
err := http.ListenAndServe(":3333", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/event-stream")
w.Header().Set("Cache-Control", "no-cache")
w.Header().Set("Connection", "keep-alive")
w.Header().Set("Access-Control-Allow-Origin", "*")
// SSE event format
event := "message"
data := "testing"
// Start SSE loop
for i := 0; i < 3; i++ {
// Send SSE event
_, err := w.Write([]byte("event: " + event + "\n"))
if err != nil {
log.Println("Error writing SSE event:", err)
return
}
_, err = w.Write([]byte("data: " + data + "\n\n"))
if err != nil {
log.Println("Error writing SSE data:", err)
return
}
// Flush the response writer
if f, ok := w.(http.Flusher); ok {
f.Flush()
}
// Delay before sending the next event
time.Sleep(1 * time.Second)
}
}))
if err != nil {
t.Error(err)
}
}()
time.Sleep(time.Second * 3)
response, err := requests.Get(nil, "http://127.0.0.1:3333/events") // Send WebSocket request
if err != nil {
t.Error(err)
}
defer response.CloseBody()
sseCli := response.Sse()
if sseCli == nil {
t.Error("not is sseCli")
}
for {
data, err := sseCli.Recv()
if err != nil {
if err != io.EOF {
t.Error(err)
}
break
}
if data.Data != "testing" {
t.Error("testing")
}
}
}
package main
import (
"io"
"log"
"net/http"
"testing"
"time"
"github.com/gospider007/requests"
)
func TestSse(t *testing.T) {
// Start the server
go func() {
err := http.ListenAndServe(":3333", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/event-stream")
w.Header().Set("Cache-Control", "no-cache")
w.Header().Set("Connection", "keep-alive")
w.Header().Set("Access-Control-Allow-Origin", "*")
// SSE event format
event := "message"
data := "testing"
// Start SSE loop
for i := 0; i < 3; i++ {
// Send SSE event
_, err := w.Write([]byte("event: " + event + "\n"))
if err != nil {
log.Println("Error writing SSE event:", err)
return
}
_, err = w.Write([]byte("data: " + data + "\n\n"))
if err != nil {
log.Println("Error writing SSE data:", err)
return
}
// Flush the response writer
if f, ok := w.(http.Flusher); ok {
f.Flush()
}
// Delay before sending the next event
time.Sleep(1 * time.Second)
}
}))
if err != nil {
t.Error(err)
}
}()
time.Sleep(time.Second * 3)
response, err := requests.Get(nil, "http://127.0.0.1:3333/events") // Send WebSocket request
if err != nil {
t.Error(err)
}
defer response.CloseBody()
sseCli := response.Sse()
if sseCli == nil {
t.Error("not is sseCli")
}
for {
data, err := sseCli.Recv()
if err != nil {
if err != io.EOF {
t.Error(err)
}
break
}
if data.Data != "testing" {
t.Error("testing")
}
}
}

66
test/protocol/websocket_test.go
View File

@@ -1,33 +1,33 @@
package main
import (
"testing"
"github.com/gospider007/requests"
"github.com/gospider007/websocket"
)
func TestWebSocket(t *testing.T) {
response, err := requests.Get(nil, "ws://82.157.123.54:9010/ajaxchattest", requests.RequestOption{Headers: map[string]string{
"Origin": "http://coolaf.com",
}}) // Send WebSocket request
if err != nil {
t.Error(err)
}
defer response.CloseBody()
wsCli := response.WebSocket()
defer wsCli.Close()
if err = wsCli.WriteMessage(websocket.TextMessage, "test"); err != nil { // Send text message
t.Error(err)
}
msgType, con, err := wsCli.ReadMessage() // Receive message
if err != nil {
t.Error(err)
}
if msgType != websocket.TextMessage {
t.Error("Message type is not text")
}
if string(con) != "test" {
t.Error("Message content is not test")
}
}
package main
import (
"testing"
"github.com/gospider007/requests"
"github.com/gospider007/websocket"
)
func TestWebSocket(t *testing.T) {
response, err := requests.Get(nil, "ws://82.157.123.54:9010/ajaxchattest", requests.RequestOption{Headers: map[string]string{
"Origin": "http://coolaf.com",
}}) // Send WebSocket request
if err != nil {
t.Error(err)
}
defer response.CloseBody()
wsCli := response.WebSocket()
defer wsCli.Close()
if err = wsCli.WriteMessage(websocket.TextMessage, "test"); err != nil { // Send text message
t.Error(err)
}
msgType, con, err := wsCli.ReadMessage() // Receive message
if err != nil {
t.Error(err)
}
if msgType != websocket.TextMessage {
t.Error("Message type is not text")
}
if string(con) != "test" {
t.Error("Message content is not test")
}
}

40
test/proxy/disProxy_test.go
View File

@@ -1,20 +1,20 @@
package main
import (
"testing"
"github.com/gospider007/requests"
)
func TestDisProxy(t *testing.T) {
resp, err := requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{
Proxy: "http://192.368.7.256:9887",
DisProxy: true,
})
if err != nil {
t.Error(err)
}
if resp.StatusCode() != 200 {
t.Error("status code is not 200")
}
}
package main
import (
"testing"
"github.com/gospider007/requests"
)
func TestDisProxy(t *testing.T) {
resp, err := requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{
Proxy: "http://192.368.7.256:9887",
DisProxy: true,
})
if err != nil {
t.Error(err)
}
if resp.StatusCode() != 200 {
t.Error("status code is not 200")
}
}

72
test/proxy_test.go
View File

@@ -1,36 +1,36 @@
package main
import (
"context"
"net/url"
"testing"
"github.com/gospider007/requests"
)
func TestProxy(t *testing.T) {
resp, err := requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{
Proxy: "", //set proxy,ex:"http://127.0.0.1:8080","https://127.0.0.1:8080","socks5://127.0.0.1:8080"
})
if err != nil {
t.Error(err)
}
if resp.StatusCode() != 200 {
t.Error("status code is not 200")
}
}
func TestGetProxy(t *testing.T) {
session, _ := requests.NewClient(nil, requests.ClientOption{
GetProxy: func(ctx context.Context, url *url.URL) (string, error) { //Penalty when creating a new connection
proxy := "" //set proxy,ex:"http://127.0.0.1:8080","https://127.0.0.1:8080","socks5://127.0.0.1:8080"
return proxy, nil
},
})
resp, err := session.Get(nil, "https://httpbin.org/anything")
if err != nil {
t.Error(err)
}
if resp.StatusCode() != 200 {
t.Error("status code is not 200")
}
}
package main
import (
"context"
"net/url"
"testing"
"github.com/gospider007/requests"
)
func TestProxy(t *testing.T) {
resp, err := requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{
Proxy: "", //set proxy,ex:"http://127.0.0.1:8080","https://127.0.0.1:8080","socks5://127.0.0.1:8080"
})
if err != nil {
t.Error(err)
}
if resp.StatusCode() != 200 {
t.Error("status code is not 200")
}
}
func TestGetProxy(t *testing.T) {
session, _ := requests.NewClient(nil, requests.ClientOption{
GetProxy: func(ctx context.Context, url *url.URL) (string, error) { //Penalty when creating a new connection
proxy := "" //set proxy,ex:"http://127.0.0.1:8080","https://127.0.0.1:8080","socks5://127.0.0.1:8080"
return proxy, nil
},
})
resp, err := session.Get(nil, "https://httpbin.org/anything")
if err != nil {
t.Error(err)
}
if resp.StatusCode() != 200 {
t.Error("status code is not 200")
}
}

210
test/request/data_test.go
View File

@@ -1,105 +1,105 @@
package main
import (
"testing"
"github.com/gospider007/gson"
"github.com/gospider007/requests"
)
func TestSendDataWithMap(t *testing.T) {
dataBody := map[string]any{
"name": "test",
}
resp, err := requests.Post(nil, "https://httpbin.org/anything", requests.RequestOption{
Data: dataBody,
})
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json()
if err != nil {
t.Fatal(err)
}
if jsonData.Get("headers.Content-Type").String() != "application/x-www-form-urlencoded" {
t.Fatal("json data error")
}
if jsonData.Get("form.name").String() != "test" {
t.Fatal("json data error")
}
}
func TestSendDataWithString(t *testing.T) {
dataBody := `{"name":"test"}`
resp, err := requests.Post(nil, "https://httpbin.org/anything", requests.RequestOption{
Data: dataBody,
})
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json()
if err != nil {
t.Fatal(err)
}
if jsonData.Get("headers.Content-Type").String() != "application/x-www-form-urlencoded" {
t.Fatal("json data error")
}
if jsonData.Get("form.name").String() != "test" {
t.Fatal("json data error")
}
}
func TestSendDataWithStruct(t *testing.T) {
dataBody := struct{ Name string }{"test"}
resp, err := requests.Post(nil, "https://httpbin.org/anything", requests.RequestOption{
Data: dataBody,
})
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json()
if err != nil {
t.Fatal(err)
}
if jsonData.Get("headers.Content-Type").String() != "application/x-www-form-urlencoded" {
t.Fatal("json data error")
}
if jsonData.Get("form.Name").String() != "test" {
t.Fatal("json data error")
}
}
func TestSendDataWithGson(t *testing.T) {
dataBody, err := gson.Decode(struct{ Name string }{"test"})
if err != nil {
t.Fatal(err)
}
resp, err := requests.Post(nil, "https://httpbin.org/anything", requests.RequestOption{
Data: dataBody,
})
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json()
if err != nil {
t.Fatal(err)
}
if jsonData.Get("headers.Content-Type").String() != "application/x-www-form-urlencoded" {
t.Fatal("json data error")
}
if jsonData.Get("form.Name").String() != "test" {
t.Fatal("json data error")
}
}
func TestSendDataWithEmptiyMap(t *testing.T) {
resp, err := requests.Post(nil, "https://httpbin.org/anything", requests.RequestOption{
Data: map[string]string{},
})
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json()
if err != nil {
t.Fatal(err)
}
if jsonData.Get("headers.Content-Type").String() != "application/x-www-form-urlencoded" {
t.Fatal("json data error")
}
}
package main
import (
"testing"
"github.com/gospider007/gson"
"github.com/gospider007/requests"
)
func TestSendDataWithMap(t *testing.T) {
dataBody := map[string]any{
"name": "test",
}
resp, err := requests.Post(nil, "https://httpbin.org/anything", requests.RequestOption{
Data: dataBody,
})
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json()
if err != nil {
t.Fatal(err)
}
if jsonData.Get("headers.Content-Type").String() != "application/x-www-form-urlencoded" {
t.Fatal("json data error")
}
if jsonData.Get("form.name").String() != "test" {
t.Fatal("json data error")
}
}
func TestSendDataWithString(t *testing.T) {
dataBody := `{"name":"test"}`
resp, err := requests.Post(nil, "https://httpbin.org/anything", requests.RequestOption{
Data: dataBody,
})
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json()
if err != nil {
t.Fatal(err)
}
if jsonData.Get("headers.Content-Type").String() != "application/x-www-form-urlencoded" {
t.Fatal("json data error")
}
if jsonData.Get("form.name").String() != "test" {
t.Fatal("json data error")
}
}
func TestSendDataWithStruct(t *testing.T) {
dataBody := struct{ Name string }{"test"}
resp, err := requests.Post(nil, "https://httpbin.org/anything", requests.RequestOption{
Data: dataBody,
})
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json()
if err != nil {
t.Fatal(err)
}
if jsonData.Get("headers.Content-Type").String() != "application/x-www-form-urlencoded" {
t.Fatal("json data error")
}
if jsonData.Get("form.Name").String() != "test" {
t.Fatal("json data error")
}
}
func TestSendDataWithGson(t *testing.T) {
dataBody, err := gson.Decode(struct{ Name string }{"test"})
if err != nil {
t.Fatal(err)
}
resp, err := requests.Post(nil, "https://httpbin.org/anything", requests.RequestOption{
Data: dataBody,
})
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json()
if err != nil {
t.Fatal(err)
}
if jsonData.Get("headers.Content-Type").String() != "application/x-www-form-urlencoded" {
t.Fatal("json data error")
}
if jsonData.Get("form.Name").String() != "test" {
t.Fatal("json data error")
}
}
func TestSendDataWithEmptiyMap(t *testing.T) {
resp, err := requests.Post(nil, "https://httpbin.org/anything", requests.RequestOption{
Data: map[string]string{},
})
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json()
if err != nil {
t.Fatal(err)
}
if jsonData.Get("headers.Content-Type").String() != "application/x-www-form-urlencoded" {
t.Fatal("json data error")
}
}

68
test/request/file_test.go
View File

@@ -1,34 +1,34 @@
package main
import (
"bytes"
"strings"
"testing"
"github.com/gospider007/requests"
)
func TestSendFileWithReader(t *testing.T) {
resp, err := requests.Post(nil, "https://httpbin.org/anything", requests.RequestOption{
Form: map[string]any{
"file": requests.File{
Content: bytes.NewBuffer([]byte("test")), //support: io.Reader, string, []byte
FileName: "test.txt",
ContentType: "text/plain",
},
},
})
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json()
if err != nil {
t.Fatal(err)
}
if !strings.HasPrefix(jsonData.Get("headers.Content-Type").String(), "multipart/form-data") {
t.Fatal("json data error")
}
if jsonData.Get("files.file").String() != "test" {
t.Fatal("json data error")
}
}
package main
import (
"bytes"
"strings"
"testing"
"github.com/gospider007/requests"
)
func TestSendFileWithReader(t *testing.T) {
resp, err := requests.Post(nil, "https://httpbin.org/anything", requests.RequestOption{
Form: map[string]any{
"file": requests.File{
Content: bytes.NewBuffer([]byte("test")), //support: io.Reader, string, []byte
FileName: "test.txt",
ContentType: "text/plain",
},
},
})
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json()
if err != nil {
t.Fatal(err)
}
if !strings.HasPrefix(jsonData.Get("headers.Content-Type").String(), "multipart/form-data") {
t.Fatal("json data error")
}
if jsonData.Get("files.file").String() != "test" {
t.Fatal("json data error")
}
}

254
test/request/form_test.go
View File

@@ -1,127 +1,127 @@
package main
import (
"strings"
"testing"
"github.com/gospider007/gson"
"github.com/gospider007/requests"
)
func TestSendFormWithMap(t *testing.T) {
dataBody := map[string]any{
"name": "test",
}
resp, err := requests.Post(nil, "https://httpbin.org/anything", requests.RequestOption{
Form: dataBody,
})
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json()
if err != nil {
t.Fatal(err)
}
if !strings.HasPrefix(jsonData.Get("headers.Content-Type").String(), "multipart/form-data") {
t.Fatal("json data error")
}
if jsonData.Get("form.name").String() != "test" {
t.Fatal("json data error")
}
}
func TestSendFormWithString(t *testing.T) {
dataBody := `{"name":"test"}`
resp, err := requests.Post(nil, "https://httpbin.org/anything", requests.RequestOption{
Form: dataBody,
})
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json()
if err != nil {
t.Fatal(err)
}
if !strings.HasPrefix(jsonData.Get("headers.Content-Type").String(), "multipart/form-data") {
t.Fatal("json data error")
}
if jsonData.Get("form.name").String() != "test" {
t.Fatal("json data error")
}
}
func TestSendFormWithStruct(t *testing.T) {
dataBody := struct{ Name string }{"test"}
resp, err := requests.Post(nil, "https://httpbin.org/anything", requests.RequestOption{
Form: dataBody,
})
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json()
if err != nil {
t.Fatal(err)
}
if !strings.HasPrefix(jsonData.Get("headers.Content-Type").String(), "multipart/form-data") {
t.Fatal("json data error")
}
if jsonData.Get("form.Name").String() != "test" {
t.Fatal("json data error")
}
}
func TestSendFormWithGson(t *testing.T) {
dataBody, err := gson.Decode(struct{ Name string }{"test"})
if err != nil {
t.Fatal(err)
}
resp, err := requests.Post(nil, "https://httpbin.org/anything", requests.RequestOption{
Form: dataBody,
})
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json()
if err != nil {
t.Fatal(err)
}
if !strings.HasPrefix(jsonData.Get("headers.Content-Type").String(), "multipart/form-data") {
t.Fatal("json data error")
}
if jsonData.Get("form.Name").String() != "test" {
t.Fatal("json data error")
}
}
func TestSendFormWithOrderMap(t *testing.T) {
orderMap := requests.NewOrderMap()
orderMap.Set("name", "test")
orderMap.Set("age", 11)
orderMap.Set("sex", "boy")
resp, err := requests.Post(nil, "https://httpbin.org/anything", requests.RequestOption{
Form: orderMap,
})
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json()
if err != nil {
t.Fatal(err)
}
if !strings.HasPrefix(jsonData.Get("headers.Content-Type").String(), "multipart/form-data") {
t.Fatal("json data error")
}
// log.Print(jsonData)
if jsonData.Get("form.name").String() != "test" {
t.Fatal("json data error")
}
}
func TestSendFileWithEmptiyMap(t *testing.T) {
resp, err := requests.Post(nil, "https://httpbin.org/anything", requests.RequestOption{
Form: map[string]string{},
})
if err != nil {
t.Fatal(err)
}
if resp.StatusCode() != 200 {
t.Fatal("status code error")
}
}
package main
import (
"strings"
"testing"
"github.com/gospider007/gson"
"github.com/gospider007/requests"
)
func TestSendFormWithMap(t *testing.T) {
dataBody := map[string]any{
"name": "test",
}
resp, err := requests.Post(nil, "https://httpbin.org/anything", requests.RequestOption{
Form: dataBody,
})
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json()
if err != nil {
t.Fatal(err)
}
if !strings.HasPrefix(jsonData.Get("headers.Content-Type").String(), "multipart/form-data") {
t.Fatal("json data error")
}
if jsonData.Get("form.name").String() != "test" {
t.Fatal("json data error")
}
}
func TestSendFormWithString(t *testing.T) {
dataBody := `{"name":"test"}`
resp, err := requests.Post(nil, "https://httpbin.org/anything", requests.RequestOption{
Form: dataBody,
})
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json()
if err != nil {
t.Fatal(err)
}
if !strings.HasPrefix(jsonData.Get("headers.Content-Type").String(), "multipart/form-data") {
t.Fatal("json data error")
}
if jsonData.Get("form.name").String() != "test" {
t.Fatal("json data error")
}
}
func TestSendFormWithStruct(t *testing.T) {
dataBody := struct{ Name string }{"test"}
resp, err := requests.Post(nil, "https://httpbin.org/anything", requests.RequestOption{
Form: dataBody,
})
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json()
if err != nil {
t.Fatal(err)
}
if !strings.HasPrefix(jsonData.Get("headers.Content-Type").String(), "multipart/form-data") {
t.Fatal("json data error")
}
if jsonData.Get("form.Name").String() != "test" {
t.Fatal("json data error")
}
}
func TestSendFormWithGson(t *testing.T) {
dataBody, err := gson.Decode(struct{ Name string }{"test"})
if err != nil {
t.Fatal(err)
}
resp, err := requests.Post(nil, "https://httpbin.org/anything", requests.RequestOption{
Form: dataBody,
})
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json()
if err != nil {
t.Fatal(err)
}
if !strings.HasPrefix(jsonData.Get("headers.Content-Type").String(), "multipart/form-data") {
t.Fatal("json data error")
}
if jsonData.Get("form.Name").String() != "test" {
t.Fatal("json data error")
}
}
func TestSendFormWithOrderMap(t *testing.T) {
orderMap := requests.NewOrderMap()
orderMap.Set("name", "test")
orderMap.Set("age", 11)
orderMap.Set("sex", "boy")
resp, err := requests.Post(nil, "https://httpbin.org/anything", requests.RequestOption{
Form: orderMap,
})
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json()
if err != nil {
t.Fatal(err)
}
if !strings.HasPrefix(jsonData.Get("headers.Content-Type").String(), "multipart/form-data") {
t.Fatal("json data error")
}
// log.Print(jsonData)
if jsonData.Get("form.name").String() != "test" {
t.Fatal("json data error")
}
}
func TestSendFileWithEmptiyMap(t *testing.T) {
resp, err := requests.Post(nil, "https://httpbin.org/anything", requests.RequestOption{
Form: map[string]string{},
})
if err != nil {
t.Fatal(err)
}
if resp.StatusCode() != 200 {
t.Fatal("status code error")
}
}

284
test/request/json_test.go
View File

@@ -1,142 +1,142 @@
package main
import (
"log"
"testing"
"github.com/gospider007/gson"
"github.com/gospider007/requests"
"github.com/gospider007/tools"
)
func TestSendJsonWithMap(t *testing.T) {
jsonBody := map[string]any{
"name": "test",
}
resp, err := requests.Post(nil, "https://httpbin.org/anything", requests.RequestOption{
Json: jsonBody,
})
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json()
if err != nil {
t.Fatal(err)
}
if jsonData.Get("headers.Content-Type").String() != "application/json" {
t.Fatal("json data error")
}
bodyJson, err := gson.Decode(jsonBody)
if err != nil {
t.Fatal(err)
}
if bodyJson.String() != jsonData.Get("data").String() {
t.Fatal("json data error")
}
}
func TestSendJsonWithString(t *testing.T) {
jsonBody := `{"name":"test"}`
resp, err := requests.Post(nil, "https://httpbin.org/anything", requests.RequestOption{
Json: jsonBody,
})
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json()
if err != nil {
t.Fatal(err)
}
if jsonData.Get("headers.Content-Type").String() != "application/json" {
t.Fatal("json data error")
}
if jsonBody != jsonData.Get("data").String() {
t.Fatal("json data error")
}
}
func TestSendJsonWithStruct(t *testing.T) {
jsonBody := struct{ Name string }{"test"}
resp, err := requests.Post(nil, "https://httpbin.org/anything", requests.RequestOption{
Json: jsonBody,
})
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json()
if err != nil {
t.Fatal(err)
}
if jsonData.Get("headers.Content-Type").String() != "application/json" {
t.Fatal("json data error")
}
bodyJson, err := gson.Decode(jsonBody)
if err != nil {
t.Fatal(err)
}
if bodyJson.String() != jsonData.Get("data").String() {
t.Fatal("json data error")
}
}
func TestSendJsonWithGson(t *testing.T) {
bodyJson, err := gson.Decode(struct{ Name string }{"test"})
if err != nil {
t.Fatal(err)
}
resp, err := requests.Post(nil, "https://httpbin.org/anything", requests.RequestOption{
Json: bodyJson,
})
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json()
if err != nil {
t.Fatal(err)
}
if jsonData.Get("headers.Content-Type").String() != "application/json" {
t.Fatal("json data error")
}
if bodyJson.String() != jsonData.Get("data").String() {
t.Fatal("json data error")
}
}
func TestSendJsonWithOrder(t *testing.T) {
orderMap := requests.NewOrderMap()
orderMap.Set("age", "1")
orderMap.Set("age4", "4")
orderMap.Set("Name", "test")
orderMap.Set("age2", "2")
orderMap.Set("age3", []string{"22", "121"})
bodyJson, err := gson.Encode(orderMap)
if err != nil {
t.Fatal(err)
}
resp, err := requests.Post(nil, "https://httpbin.org/anything", requests.RequestOption{
Json: orderMap,
})
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json()
if err != nil {
t.Fatal(err)
}
if jsonData.Get("headers.Content-Type").String() != "application/json" {
t.Fatal("json data error")
}
if tools.BytesToString(bodyJson) != jsonData.Get("data").String() {
log.Print(jsonData.Get("data").String())
t.Fatal("json data error")
}
}
func TestSendJsonWithEmptiyMap(t *testing.T) {
resp, err := requests.Post(nil, "https://httpbin.org/anything", requests.RequestOption{
Form: map[string]string{},
})
if err != nil {
t.Fatal(err)
}
if resp.StatusCode() != 200 {
t.Fatal("status code error")
}
}
package main
import (
"log"
"testing"
"github.com/gospider007/gson"
"github.com/gospider007/requests"
"github.com/gospider007/tools"
)
func TestSendJsonWithMap(t *testing.T) {
jsonBody := map[string]any{
"name": "test",
}
resp, err := requests.Post(nil, "https://httpbin.org/anything", requests.RequestOption{
Json: jsonBody,
})
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json()
if err != nil {
t.Fatal(err)
}
if jsonData.Get("headers.Content-Type").String() != "application/json" {
t.Fatal("json data error")
}
bodyJson, err := gson.Decode(jsonBody)
if err != nil {
t.Fatal(err)
}
if bodyJson.String() != jsonData.Get("data").String() {
t.Fatal("json data error")
}
}
func TestSendJsonWithString(t *testing.T) {
jsonBody := `{"name":"test"}`
resp, err := requests.Post(nil, "https://httpbin.org/anything", requests.RequestOption{
Json: jsonBody,
})
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json()
if err != nil {
t.Fatal(err)
}
if jsonData.Get("headers.Content-Type").String() != "application/json" {
t.Fatal("json data error")
}
if jsonBody != jsonData.Get("data").String() {
t.Fatal("json data error")
}
}
func TestSendJsonWithStruct(t *testing.T) {
jsonBody := struct{ Name string }{"test"}
resp, err := requests.Post(nil, "https://httpbin.org/anything", requests.RequestOption{
Json: jsonBody,
})
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json()
if err != nil {
t.Fatal(err)
}
if jsonData.Get("headers.Content-Type").String() != "application/json" {
t.Fatal("json data error")
}
bodyJson, err := gson.Decode(jsonBody)
if err != nil {
t.Fatal(err)
}
if bodyJson.String() != jsonData.Get("data").String() {
t.Fatal("json data error")
}
}
func TestSendJsonWithGson(t *testing.T) {
bodyJson, err := gson.Decode(struct{ Name string }{"test"})
if err != nil {
t.Fatal(err)
}
resp, err := requests.Post(nil, "https://httpbin.org/anything", requests.RequestOption{
Json: bodyJson,
})
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json()
if err != nil {
t.Fatal(err)
}
if jsonData.Get("headers.Content-Type").String() != "application/json" {
t.Fatal("json data error")
}
if bodyJson.String() != jsonData.Get("data").String() {
t.Fatal("json data error")
}
}
func TestSendJsonWithOrder(t *testing.T) {
orderMap := requests.NewOrderMap()
orderMap.Set("age", "1")
orderMap.Set("age4", "4")
orderMap.Set("Name", "test")
orderMap.Set("age2", "2")
orderMap.Set("age3", []string{"22", "121"})
bodyJson, err := gson.Encode(orderMap)
if err != nil {
t.Fatal(err)
}
resp, err := requests.Post(nil, "https://httpbin.org/anything", requests.RequestOption{
Json: orderMap,
})
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json()
if err != nil {
t.Fatal(err)
}
if jsonData.Get("headers.Content-Type").String() != "application/json" {
t.Fatal("json data error")
}
if tools.BytesToString(bodyJson) != jsonData.Get("data").String() {
log.Print(jsonData.Get("data").String())
t.Fatal("json data error")
}
}
func TestSendJsonWithEmptiyMap(t *testing.T) {
resp, err := requests.Post(nil, "https://httpbin.org/anything", requests.RequestOption{
Form: map[string]string{},
})
if err != nil {
t.Fatal(err)
}
if resp.StatusCode() != 200 {
t.Fatal("status code error")
}
}

44
test/request/localAddr_test.go
View File

@@ -1,22 +1,22 @@
package main
import (
"net"
"testing"
"github.com/gospider007/requests"
)
func TestLocalAddr(t *testing.T) {
resp, err := requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{
LocalAddr: &net.TCPAddr{ //set dns server
IP: net.ParseIP("192.168.1.239"),
},
})
if err != nil {
t.Fatal(err)
}
if resp.StatusCode() != 200 {
t.Fatal("http status code is not 200")
}
}
package main
import (
"net"
"testing"
"github.com/gospider007/requests"
)
func TestLocalAddr(t *testing.T) {
resp, err := requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{
LocalAddr: &net.TCPAddr{ //set dns server
IP: net.ParseIP("192.168.1.239"),
},
})
if err != nil {
t.Fatal(err)
}
if resp.StatusCode() != 200 {
t.Fatal("http status code is not 200")
}
}

96
test/request/method_test.go
View File

@@ -1,48 +1,48 @@
package main
import (
"net/http"
"testing"
"github.com/gospider007/requests"
)
func TestMethodGet(t *testing.T) {
resp, err := requests.Get(nil, "https://httpbin.org/anything")
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json()
if err != nil {
t.Fatal(err)
}
if jsonData.Get("method").String() != http.MethodGet {
t.Fatal("method error")
}
}
func TestMethodPost(t *testing.T) {
resp, err := requests.Post(nil, "https://httpbin.org/anything")
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json()
if err != nil {
t.Fatal(err)
}
if jsonData.Get("method").String() != http.MethodPost {
t.Fatal("method error")
}
}
func TestMethodPost2(t *testing.T) {
resp, err := requests.Request(nil, "post", "https://httpbin.org/anything")
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json()
if err != nil {
t.Fatal(err)
}
if jsonData.Get("method").String() != http.MethodPost {
t.Fatal("method error")
}
}
package main
import (
"net/http"
"testing"
"github.com/gospider007/requests"
)
func TestMethodGet(t *testing.T) {
resp, err := requests.Get(nil, "https://httpbin.org/anything")
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json()
if err != nil {
t.Fatal(err)
}
if jsonData.Get("method").String() != http.MethodGet {
t.Fatal("method error")
}
}
func TestMethodPost(t *testing.T) {
resp, err := requests.Post(nil, "https://httpbin.org/anything")
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json()
if err != nil {
t.Fatal(err)
}
if jsonData.Get("method").String() != http.MethodPost {
t.Fatal("method error")
}
}
func TestMethodPost2(t *testing.T) {
resp, err := requests.Request(nil, "post", "https://httpbin.org/anything")
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json()
if err != nil {
t.Fatal(err)
}
if jsonData.Get("method").String() != http.MethodPost {
t.Fatal("method error")
}
}

182
test/request/params_test.go
View File

@@ -1,91 +1,91 @@
package main
import (
"testing"
"github.com/gospider007/gson"
"github.com/gospider007/requests"
)
func TestSendParamsWithMap(t *testing.T) {
dataBody := map[string]any{
"name": "test",
}
resp, err := requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{
Params: dataBody,
})
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json()
if err != nil {
t.Fatal(err)
}
// log.Print(jsonData)
if jsonData.Get("args.name").String() != "test" {
t.Fatal("params args error")
}
}
func TestSendParamsWithString(t *testing.T) {
dataBody := `{"name":"test"}`
resp, err := requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{
Params: dataBody,
})
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json()
if err != nil {
t.Fatal(err)
}
if jsonData.Get("args.name").String() != "test" {
t.Fatal("json data error")
}
}
func TestSendParamsWithStruct(t *testing.T) {
dataBody := struct{ Name string }{"test"}
resp, err := requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{
Params: dataBody,
})
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json()
if err != nil {
t.Fatal(err)
}
if jsonData.Get("args.Name").String() != "test" {
t.Fatal("json data error")
}
}
func TestSendParamsWithGson(t *testing.T) {
dataBody, err := gson.Decode(struct{ Name string }{"test"})
if err != nil {
t.Fatal(err)
}
resp, err := requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{
Params: dataBody,
})
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json()
if err != nil {
t.Fatal(err)
}
if jsonData.Get("args.Name").String() != "test" {
t.Fatal("json data error")
}
}
func TestSendParamsWithEmptiyMap(t *testing.T) {
resp, err := requests.Post(nil, "https://httpbin.org/anything", requests.RequestOption{
Params: map[string]string{},
})
if err != nil {
t.Fatal(err)
}
if resp.StatusCode() != 200 {
t.Fatal("status code error")
}
}
package main
import (
"testing"
"github.com/gospider007/gson"
"github.com/gospider007/requests"
)
func TestSendParamsWithMap(t *testing.T) {
dataBody := map[string]any{
"name": "test",
}
resp, err := requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{
Params: dataBody,
})
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json()
if err != nil {
t.Fatal(err)
}
// log.Print(jsonData)
if jsonData.Get("args.name").String() != "test" {
t.Fatal("params args error")
}
}
func TestSendParamsWithString(t *testing.T) {
dataBody := `{"name":"test"}`
resp, err := requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{
Params: dataBody,
})
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json()
if err != nil {
t.Fatal(err)
}
if jsonData.Get("args.name").String() != "test" {
t.Fatal("json data error")
}
}
func TestSendParamsWithStruct(t *testing.T) {
dataBody := struct{ Name string }{"test"}
resp, err := requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{
Params: dataBody,
})
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json()
if err != nil {
t.Fatal(err)
}
if jsonData.Get("args.Name").String() != "test" {
t.Fatal("json data error")
}
}
func TestSendParamsWithGson(t *testing.T) {
dataBody, err := gson.Decode(struct{ Name string }{"test"})
if err != nil {
t.Fatal(err)
}
resp, err := requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{
Params: dataBody,
})
if err != nil {
t.Fatal(err)
}
jsonData, err := resp.Json()
if err != nil {
t.Fatal(err)
}
if jsonData.Get("args.Name").String() != "test" {
t.Fatal("json data error")
}
}
func TestSendParamsWithEmptiyMap(t *testing.T) {
resp, err := requests.Post(nil, "https://httpbin.org/anything", requests.RequestOption{
Params: map[string]string{},
})
if err != nil {
t.Fatal(err)
}
if resp.StatusCode() != 200 {
t.Fatal("status code error")
}
}

50
test/request/stream_test.go
View File

@@ -1,25 +1,25 @@
package main
import (
"testing"
"github.com/gospider007/requests"
)
func TestStream(t *testing.T) {
resp, err := requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{
Stream: true,
})
if err != nil {
t.Fatal(err)
}
if resp.IsStream() {
if resp.StatusCode() != 200 {
t.Fatal("resp.StatusCode()!= 200")
}
resp.CloseBody()
resp.CloseBody()
} else {
t.Fatal("resp.IsStream() is false")
}
}
package main
import (
"testing"
"github.com/gospider007/requests"
)
func TestStream(t *testing.T) {
resp, err := requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{
Stream: true,
})
if err != nil {
t.Fatal(err)
}
if resp.IsStream() {
if resp.StatusCode() != 200 {
t.Fatal("resp.StatusCode()!= 200")
}
resp.CloseBody()
resp.CloseBody()
} else {
t.Fatal("resp.IsStream() is false")
}
}

38
test/response/isNewConn_test.go
View File

@@ -1,19 +1,19 @@
package main
import (
"testing"
"github.com/gospider007/requests"
)
func TestDefaultClient(t *testing.T) {
for i := 0; i < 2; i++ {
resp, err := requests.Get(nil, "https://httpbin.org/anything")
if err != nil {
t.Error(err)
}
if i == 1 && resp.IsNewConn() {
t.Error("new conn error")
}
}
}
package main
import (
"testing"
"github.com/gospider007/requests"
)
func TestDefaultClient(t *testing.T) {
for i := 0; i < 2; i++ {
resp, err := requests.Get(nil, "https://httpbin.org/anything")
if err != nil {
t.Error(err)
}
if i == 1 && resp.IsNewConn() {
t.Error("new conn error")
}
}
}

48
test/response/rawConn_test.go
View File

@@ -1,24 +1,24 @@
package main
import (
"testing"
"github.com/gospider007/requests"
)
func TestRawConn(t *testing.T) {
resp, err := requests.Get(nil, "https://httpbin.org/anything")
if err != nil {
t.Error(err)
}
if resp.Conn() != nil {
t.Error("conn is not nil")
}
resp, err = requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{Stream: true})
if err != nil {
t.Error(err)
}
if resp.Conn() == nil {
t.Error("conn is nil")
}
}
package main
import (
"testing"
"github.com/gospider007/requests"
)
func TestRawConn(t *testing.T) {
resp, err := requests.Get(nil, "https://httpbin.org/anything")
if err != nil {
t.Error(err)
}
if resp.Conn() != nil {
t.Error("conn is not nil")
}
resp, err = requests.Get(nil, "https://httpbin.org/anything", requests.RequestOption{Stream: true})
if err != nil {
t.Error(err)
}
if resp.Conn() == nil {
t.Error("conn is nil")
}
}

34
test/response/useProxy_test.go
View File

@@ -1,17 +1,17 @@
package main
import (
"testing"
"github.com/gospider007/requests"
)
func TestUseProxy(t *testing.T) {
resp, err := requests.Get(nil, "https://httpbin.org/anything")
if err != nil {
t.Error(err)
}
if resp.Proxy() != "" {
t.Error("proxy error")
}
}
package main
import (
"testing"
"github.com/gospider007/requests"
)
func TestUseProxy(t *testing.T) {
resp, err := requests.Get(nil, "https://httpbin.org/anything")
if err != nil {
t.Error(err)
}
if resp.Proxy() != "" {
t.Error("proxy error")
}
}

52
test/session_test.go
View File

@@ -1,26 +1,26 @@
package main
import (
"testing"
"github.com/gospider007/requests"
)
func TestSession(t *testing.T) {
session, _ := requests.NewClient(nil)
for i := 0; i < 2; i++ {
resp, err := session.Get(nil, "https://httpbin.org/anything")
if err != nil {
t.Error(err)
}
if i == 0 {
if !resp.IsNewConn() { //return is NewConn
t.Error("new conn error: ", i)
}
} else {
if resp.IsNewConn() {
t.Error("new conn error: ", i)
}
}
}
}
package main
import (
"testing"
"github.com/gospider007/requests"
)
func TestSession(t *testing.T) {
session, _ := requests.NewClient(nil)
for i := 0; i < 2; i++ {
resp, err := session.Get(nil, "https://httpbin.org/anything")
if err != nil {
t.Error(err)
}
if i == 0 {
if !resp.IsNewConn() { //return is NewConn
t.Error("new conn error: ", i)
}
} else {
if resp.IsNewConn() {
t.Error("new conn error: ", i)
}
}
}
}

490
tools.go
View File

@@ -1,245 +1,245 @@
package requests
import (
"bufio"
"context"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/textproto"
"net/url"
"strconv"
"strings"
_ "unsafe"
"github.com/gospider007/ja3"
"golang.org/x/exp/slices"
"golang.org/x/net/http/httpguts"
)
func getHost(req *http.Request) string {
host := req.Host
if host == "" {
host = req.URL.Host
}
_, port, _ := net.SplitHostPort(host)
if port == "" {
if req.URL.Scheme == "https" {
port = "443"
} else {
port = "80"
}
return fmt.Sprintf("%s:%s", host, port)
}
return host
}
func getAddr(uurl *url.URL) (addr string) {
if uurl == nil {
return ""
}
_, port, _ := net.SplitHostPort(uurl.Host)
if port == "" {
if uurl.Scheme == "https" {
port = "443"
} else {
port = "80"
}
return fmt.Sprintf("%s:%s", uurl.Host, port)
}
return uurl.Host
}
func cloneUrl(u *url.URL) *url.URL {
if u == nil {
return nil
}
r := *u
return &r
}
var replaceMap = map[string]string{
"Sec-Ch-Ua": "sec-ch-ua",
"Sec-Ch-Ua-Mobile": "sec-ch-ua-mobile",
"Sec-Ch-Ua-Platform": "sec-ch-ua-platform",
}
//go:linkname escapeQuotes mime/multipart.escapeQuotes
func escapeQuotes(string) string
//go:linkname readCookies net/http.readCookies
func readCookies(h http.Header, filter string) []*http.Cookie
//go:linkname readSetCookies net/http.readSetCookies
func readSetCookies(h http.Header) []*http.Cookie
//go:linkname ReadRequest net/http.readRequest
func ReadRequest(b *bufio.Reader) (*http.Request, error)
//go:linkname removeZone net/http.removeZone
func removeZone(host string) string
//go:linkname shouldSendContentLength net/http.(*transferWriter).shouldSendContentLength
func shouldSendContentLength(t *http.Request) bool
//go:linkname removeEmptyPort net/http.removeEmptyPort
func removeEmptyPort(host string) string
//go:linkname redirectBehavior net/http.redirectBehavior
func redirectBehavior(reqMethod string, resp *http.Response, ireq *http.Request) (redirectMethod string, shouldRedirect, includeBody bool)
//go:linkname readTransfer net/http.readTransfer
func readTransfer(msg any, r *bufio.Reader) (err error)
var filterHeaderKeys = ja3.DefaultOrderHeadersWithH2()
func httpWrite(r *http.Request, w *bufio.Writer, orderHeaders []string) (err error) {
for i := range orderHeaders {
orderHeaders[i] = textproto.CanonicalMIMEHeaderKey(orderHeaders[i])
}
host := r.Host
if host == "" {
host = r.URL.Host
}
host, err = httpguts.PunycodeHostPort(host)
if err != nil {
return err
}
host = removeZone(host)
ruri := r.URL.RequestURI()
if r.Method == "CONNECT" && r.URL.Path == "" {
if r.URL.Opaque != "" {
ruri = r.URL.Opaque
} else {
ruri = host
}
}
if r.Header.Get("Host") == "" {
r.Header.Set("Host", host)
}
if r.Header.Get("Connection") == "" {
r.Header.Set("Connection", "keep-alive")
}
if r.Header.Get("User-Agent") == "" {
r.Header.Set("User-Agent", UserAgent)
}
if r.Header.Get("Content-Length") == "" && r.ContentLength != 0 && shouldSendContentLength(r) {
r.Header.Set("Content-Length", fmt.Sprint(r.ContentLength))
}
if _, err = w.WriteString(fmt.Sprintf("%s %s %s\r\n", r.Method, ruri, r.Proto)); err != nil {
return err
}
for _, k := range orderHeaders {
if vs, ok := r.Header[k]; ok {
if k2, ok := replaceMap[k]; ok {
k = k2
}
if slices.Contains(filterHeaderKeys, k) {
continue
}
for _, v := range vs {
if _, err = w.WriteString(fmt.Sprintf("%s: %s\r\n", k, v)); err != nil {
return err
}
}
}
}
for k, vs := range r.Header {
if !slices.Contains(orderHeaders, k) {
if k2, ok := replaceMap[k]; ok {
k = k2
}
if slices.Contains(filterHeaderKeys, k) {
continue
}
for _, v := range vs {
if _, err = w.WriteString(fmt.Sprintf("%s: %s\r\n", k, v)); err != nil {
return err
}
}
}
}
if _, err = w.WriteString("\r\n"); err != nil {
return err
}
if r.Body != nil {
if _, err = io.Copy(w, r.Body); err != nil {
return err
}
}
return w.Flush()
}
func NewRequestWithContext(ctx context.Context, method string, u *url.URL, body io.Reader) (*http.Request, error) {
req := (&http.Request{}).WithContext(ctx)
if method == "" {
req.Method = http.MethodGet
} else {
req.Method = strings.ToUpper(method)
}
req.URL = u
req.Proto = "HTTP/1.1"
req.ProtoMajor = 1
req.ProtoMinor = 1
req.Host = u.Host
u.Host = removeEmptyPort(u.Host)
if body != nil {
if v, ok := body.(interface{ Len() int }); ok {
req.ContentLength = int64(v.Len())
}
rc, ok := body.(io.ReadCloser)
if !ok {
rc = io.NopCloser(body)
}
req.Body = rc
}
return req, nil
}
func readResponse(tp *textproto.Reader, req *http.Request) (*http.Response, error) {
resp := &http.Response{
Request: req,
}
// Parse the first line of the response.
line, err := tp.ReadLine()
if err != nil {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
return nil, err
}
proto, status, ok := strings.Cut(line, " ")
if !ok {
return nil, errors.New("malformed HTTP response")
}
resp.Proto = proto
resp.Status = strings.TrimLeft(status, " ")
statusCode, _, _ := strings.Cut(resp.Status, " ")
if resp.StatusCode, err = strconv.Atoi(statusCode); err != nil {
return nil, errors.New("malformed HTTP status code")
}
if resp.ProtoMajor, resp.ProtoMinor, ok = http.ParseHTTPVersion(resp.Proto); !ok {
return nil, errors.New("malformed HTTP version")
}
// Parse the response headers.
mimeHeader, err := tp.ReadMIMEHeader()
if err != nil {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
return nil, err
}
resp.Header = http.Header(mimeHeader)
return resp, readTransfer(resp, tp.R)
}
func addCookie(req *http.Request, cookies Cookies) {
cooks := Cookies(readCookies(req.Header, ""))
for _, cook := range cookies {
if val := cooks.Get(cook.Name); val == nil {
cooks = cooks.append(cook)
}
}
if result := cooks.String(); result != "" {
req.Header.Set("Cookie", result)
}
}
package requests
import (
"bufio"
"context"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/textproto"
"net/url"
"strconv"
"strings"
_ "unsafe"
"github.com/gospider007/ja3"
"golang.org/x/exp/slices"
"golang.org/x/net/http/httpguts"
)
func getHost(req *http.Request) string {
host := req.Host
if host == "" {
host = req.URL.Host
}
_, port, _ := net.SplitHostPort(host)
if port == "" {
if req.URL.Scheme == "https" {
port = "443"
} else {
port = "80"
}
return fmt.Sprintf("%s:%s", host, port)
}
return host
}
func getAddr(uurl *url.URL) (addr string) {
if uurl == nil {
return ""
}
_, port, _ := net.SplitHostPort(uurl.Host)
if port == "" {
if uurl.Scheme == "https" {
port = "443"
} else {
port = "80"
}
return fmt.Sprintf("%s:%s", uurl.Host, port)
}
return uurl.Host
}
func cloneUrl(u *url.URL) *url.URL {
if u == nil {
return nil
}
r := *u
return &r
}
var replaceMap = map[string]string{
"Sec-Ch-Ua": "sec-ch-ua",
"Sec-Ch-Ua-Mobile": "sec-ch-ua-mobile",
"Sec-Ch-Ua-Platform": "sec-ch-ua-platform",
}
//go:linkname escapeQuotes mime/multipart.escapeQuotes
func escapeQuotes(string) string
//go:linkname readCookies net/http.readCookies
func readCookies(h http.Header, filter string) []*http.Cookie
//go:linkname readSetCookies net/http.readSetCookies
func readSetCookies(h http.Header) []*http.Cookie
//go:linkname ReadRequest net/http.readRequest
func ReadRequest(b *bufio.Reader) (*http.Request, error)
//go:linkname removeZone net/http.removeZone
func removeZone(host string) string
//go:linkname shouldSendContentLength net/http.(*transferWriter).shouldSendContentLength
func shouldSendContentLength(t *http.Request) bool
//go:linkname removeEmptyPort net/http.removeEmptyPort
func removeEmptyPort(host string) string
//go:linkname redirectBehavior net/http.redirectBehavior
func redirectBehavior(reqMethod string, resp *http.Response, ireq *http.Request) (redirectMethod string, shouldRedirect, includeBody bool)
//go:linkname readTransfer net/http.readTransfer
func readTransfer(msg any, r *bufio.Reader) (err error)
var filterHeaderKeys = ja3.DefaultOrderHeadersWithH2()
func httpWrite(r *http.Request, w *bufio.Writer, orderHeaders []string) (err error) {
for i := range orderHeaders {
orderHeaders[i] = textproto.CanonicalMIMEHeaderKey(orderHeaders[i])
}
host := r.Host
if host == "" {
host = r.URL.Host
}
host, err = httpguts.PunycodeHostPort(host)
if err != nil {
return err
}
host = removeZone(host)
ruri := r.URL.RequestURI()
if r.Method == "CONNECT" && r.URL.Path == "" {
if r.URL.Opaque != "" {
ruri = r.URL.Opaque
} else {
ruri = host
}
}
if r.Header.Get("Host") == "" {
r.Header.Set("Host", host)
}
if r.Header.Get("Connection") == "" {
r.Header.Set("Connection", "keep-alive")
}
if r.Header.Get("User-Agent") == "" {
r.Header.Set("User-Agent", UserAgent)
}
if r.Header.Get("Content-Length") == "" && r.ContentLength != 0 && shouldSendContentLength(r) {
r.Header.Set("Content-Length", fmt.Sprint(r.ContentLength))
}
if _, err = w.WriteString(fmt.Sprintf("%s %s %s\r\n", r.Method, ruri, r.Proto)); err != nil {
return err
}
for _, k := range orderHeaders {
if vs, ok := r.Header[k]; ok {
if k2, ok := replaceMap[k]; ok {
k = k2
}
if slices.Contains(filterHeaderKeys, k) {
continue
}
for _, v := range vs {
if _, err = w.WriteString(fmt.Sprintf("%s: %s\r\n", k, v)); err != nil {
return err
}
}
}
}
for k, vs := range r.Header {
if !slices.Contains(orderHeaders, k) {
if k2, ok := replaceMap[k]; ok {
k = k2
}
if slices.Contains(filterHeaderKeys, k) {
continue
}
for _, v := range vs {
if _, err = w.WriteString(fmt.Sprintf("%s: %s\r\n", k, v)); err != nil {
return err
}
}
}
}
if _, err = w.WriteString("\r\n"); err != nil {
return err
}
if r.Body != nil {
if _, err = io.Copy(w, r.Body); err != nil {
return err
}
}
return w.Flush()
}
func NewRequestWithContext(ctx context.Context, method string, u *url.URL, body io.Reader) (*http.Request, error) {
req := (&http.Request{}).WithContext(ctx)
if method == "" {
req.Method = http.MethodGet
} else {
req.Method = strings.ToUpper(method)
}
req.URL = u
req.Proto = "HTTP/1.1"
req.ProtoMajor = 1
req.ProtoMinor = 1
req.Host = u.Host
u.Host = removeEmptyPort(u.Host)
if body != nil {
if v, ok := body.(interface{ Len() int }); ok {
req.ContentLength = int64(v.Len())
}
rc, ok := body.(io.ReadCloser)
if !ok {
rc = io.NopCloser(body)
}
req.Body = rc
}
return req, nil
}
func readResponse(tp *textproto.Reader, req *http.Request) (*http.Response, error) {
resp := &http.Response{
Request: req,
}
// Parse the first line of the response.
line, err := tp.ReadLine()
if err != nil {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
return nil, err
}
proto, status, ok := strings.Cut(line, " ")
if !ok {
return nil, errors.New("malformed HTTP response")
}
resp.Proto = proto
resp.Status = strings.TrimLeft(status, " ")
statusCode, _, _ := strings.Cut(resp.Status, " ")
if resp.StatusCode, err = strconv.Atoi(statusCode); err != nil {
return nil, errors.New("malformed HTTP status code")
}
if resp.ProtoMajor, resp.ProtoMinor, ok = http.ParseHTTPVersion(resp.Proto); !ok {
return nil, errors.New("malformed HTTP version")
}
// Parse the response headers.
mimeHeader, err := tp.ReadMIMEHeader()
if err != nil {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
return nil, err
}
resp.Header = http.Header(mimeHeader)
return resp, readTransfer(resp, tp.R)
}
func addCookie(req *http.Request, cookies Cookies) {
cooks := Cookies(readCookies(req.Header, ""))
for _, cook := range cookies {
if val := cooks.Get(cook.Name); val == nil {
cooks = cooks.append(cook)
}
}
if result := cooks.String(); result != "" {
req.Header.Set("Cookie", result)
}
}