mirror of
https://github.com/gospider007/requests.git
synced 2025-12-24 13:57:52 +08:00
first
This commit is contained in:
bxd
368
README.md
368
README.md
@@ -1,368 +0,0 @@
|
||||
# 功能概述
|
||||
- cookies 开关，连接池，http2，ja3
|
||||
- 自实现socks五,http代理,https代理
|
||||
- 自动解压缩,解码
|
||||
- dns缓存
|
||||
- 类型自动转化
|
||||
- 尝试重试，请求回调
|
||||
- websocket 协议
|
||||
- sse协议
|
||||
# 设置代理
|
||||
## 代理设置的优先级
|
||||
```
|
||||
全局代理方法 < 全局代理字符串 < 局部代理字符串
|
||||
```
|
||||
## 设置并修改全局代理方法 (只会在新建连接的时候调用获取代理的方法,复用连接的时候不会调用)
|
||||
```golang
|
||||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
|
||||
"gitee.com/baixudong/requests"
|
||||
)
|
||||
|
||||
func main() {
|
||||
//创建请求客户端
|
||||
reqCli, err := requests.NewClient(nil, requests.ClientOption{
|
||||
GetProxy: func(ctx context.Context, url *url.URL) (string, error) { //设置全局代理方法
|
||||
return "http://127.0.0.1:7005", nil
|
||||
}})
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
response, err := reqCli.Request(nil, "get", "http://myip.top") //发送get请求
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
reqCli.SetGetProxy(func(ctx context.Context, url *url.URL) (string, error) { //修改全局代理方法
|
||||
return "http://127.0.0.1:7006", nil
|
||||
})
|
||||
log.Print(response.Text()) //获取内容,解析为字符串
|
||||
}
|
||||
```
|
||||
## 设置并修改全局代理
|
||||
```golang
|
||||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
|
||||
"gitee.com/baixudong/requests"
|
||||
)
|
||||
|
||||
func main() {
|
||||
//创建请求客户端
|
||||
reqCli, err := requests.NewClient(nil, requests.ClientOption{
|
||||
Proxy: "http://127.0.0.1:7005", //设置全局代理
|
||||
})
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
//发送get请求
|
||||
response, err := reqCli.Request(nil, "get", "http://myip.top")
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
err = reqCli.SetProxy("http://127.0.0.1:7006") //修改全局代理
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
log.Print(response.Text()) //获取内容,解析为字符串
|
||||
}
|
||||
```
|
||||
## 设置局部代理
|
||||
```golang
|
||||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
|
||||
"gitee.com/baixudong/requests"
|
||||
)
|
||||
|
||||
func main() {
|
||||
//创建请求客户端
|
||||
reqCli, err := requests.NewClient(nil)
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
//发送get请求
|
||||
response, err := reqCli.Request(nil, "get", "http://myip.top", requests.RequestOption{
|
||||
Proxy: "http://127.0.0.1:7005",
|
||||
})
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
log.Print(response.Text()) //获取内容,解析为字符串
|
||||
}
|
||||
```
|
||||
## 强制关闭代理，走本地网络
|
||||
```golang
|
||||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
|
||||
"gitee.com/baixudong/requests"
|
||||
)
|
||||
|
||||
func main() {
|
||||
//创建请求客户端
|
||||
reqCli, err := requests.NewClient(nil)
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
//发送get请求
|
||||
response, err := reqCli.Request(nil, "get", "http://myip.top", requests.RequestOption{
|
||||
DisProxy: true, //强制走本地代理
|
||||
})
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
log.Print(response.Text()) //获取内容,解析为字符串
|
||||
}
|
||||
```
|
||||
|
||||
# 发送http请求
|
||||
|
||||
```golang
|
||||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
|
||||
"gitee.com/baixudong/requests"
|
||||
)
|
||||
|
||||
func main() {
|
||||
reqCli, err := requests.NewClient(nil) //创建请求客户端
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
response, err := reqCli.Request(nil, "get", "http://myip.top") //发送get请求
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
log.Print(response.Text()) //获取内容,解析为字符串
|
||||
log.Print(response.Content()) //获取内容,解析为字节
|
||||
log.Print(response.Json()) //获取json,解析为gjson
|
||||
log.Print(response.Html()) //获取内容,解析为dom
|
||||
log.Print(response.Cookies()) //获取cookies
|
||||
}
|
||||
|
||||
```
|
||||
|
||||
# 发送websocket 请求
|
||||
|
||||
```golang
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
|
||||
"gitee.com/baixudong/requests"
|
||||
"gitee.com/baixudong/websocket"
|
||||
)
|
||||
|
||||
func main() {
|
||||
reqCli, err := requests.NewClient(nil) //创建请求客户端
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
response, err := reqCli.Request(nil, "get", "ws://82.157.123.54:9010/ajaxchattest", requests.RequestOption{Headers: map[string]string{
|
||||
"Origin": "http://coolaf.com",
|
||||
}}) //发送websocket请求
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
defer response.Close()
|
||||
wsCli := response.WebSocket()
|
||||
if err = wsCli.Send(context.TODO(), websocket.MessageText, "测试"); err != nil { //发送txt 消息
|
||||
log.Panic(err)
|
||||
}
|
||||
msgType, con, err := wsCli.Recv(context.TODO()) //接收消息
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
log.Print(msgType) //消息类型
|
||||
log.Print(string(con)) //消息内容
|
||||
}
|
||||
```
|
||||
# ipv4,ipv6 地址控制解析
|
||||
```go
|
||||
func main() {
|
||||
reqCli, err := requests.NewClient(nil, requests.ClientOption{
|
||||
AddrType: requests.Ipv4, //优先解析ipv4地址
|
||||
// AddrType: requests.Ipv6,//优先解析ipv6地址
|
||||
})
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
href := "https://test.ipw.cn"
|
||||
resp, err := reqCli.Request(nil, "get", href)
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
log.Print(resp.Text())
|
||||
log.Print(resp.StatusCode())
|
||||
}
|
||||
```
|
||||
# ja3 伪造指纹
|
||||
## 根据字符串生成指纹
|
||||
```go
|
||||
func main() {
|
||||
ja3Str := "772,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,0-23-65281-10-11-35-16-5-13-18-51-45-43-27-17513,29-23-24,0"
|
||||
Ja3Spec, err := ja3.CreateSpecWithStr(ja3Str)//根据字符串生成指纹
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
reqCli, err := requests.NewClient(nil, requests.ClientOption{Ja3Spec: Ja3Spec})
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
response, err := reqCli.Request(nil, "get", "https://tools.scrapfly.io/api/fp/ja3?extended=1")
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
jsonData,_:=response.Json()
|
||||
log.Print(jsonData.Get("ja3").String())
|
||||
log.Print(jsonData.Get("ja3").String() == ja3Str)
|
||||
}
|
||||
```
|
||||
## 根据id 生成指纹
|
||||
```go
|
||||
func main() {
|
||||
Ja3Spec, err := ja3.CreateSpecWithId(ja3.HelloChrome_Auto) //根据id 生成指纹
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
reqCli, err := requests.NewClient(nil, requests.ClientOption{Ja3Spec: Ja3Spec})
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
response, err := reqCli.Request(nil, "get", "https://tools.scrapfly.io/api/fp/ja3?extended=1")
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
jsonData,_:=response.Json()
|
||||
log.Print(jsonData.Get("ja3").String())
|
||||
}
|
||||
```
|
||||
## ja3 开关
|
||||
```go
|
||||
func main() {
|
||||
reqCli, err := requests.NewClient(nil)
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
response, err := reqCli.Request(nil, "get", "https://tools.scrapfly.io/api/fp/ja3?extended=1", requests.RequestOption{Ja3: true})//使用最新chrome 指纹
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
jsonData,_:=response.Json()
|
||||
log.Print(jsonData.Get("ja3").String())
|
||||
}
|
||||
```
|
||||
## h2 指纹开关
|
||||
```go
|
||||
func main() {
|
||||
reqCli, err := requests.NewClient(nil, requests.ClientOption{
|
||||
H2Ja3: true,
|
||||
})
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
href := "https://tools.scrapfly.io/api/fp/anything"
|
||||
resp, err := reqCli.Request(nil, "get", href)
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
log.Print(resp.Text())
|
||||
}
|
||||
```
|
||||
## 修改h2指纹
|
||||
```go
|
||||
func main() {
|
||||
reqCli, err := requests.NewClient(nil, requests.ClientOption{
|
||||
H2Ja3Spec: ja3.H2Ja3Spec{
|
||||
InitialSetting: []ja3.Setting{
|
||||
{Id: 1, Val: 65555},
|
||||
{Id: 2, Val: 1},
|
||||
{Id: 3, Val: 2000},
|
||||
{Id: 4, Val: 6291457},
|
||||
{Id: 6, Val: 262145},
|
||||
},
|
||||
ConnFlow: 15663106,
|
||||
OrderHeaders: []string{
|
||||
":method",
|
||||
":path",
|
||||
":scheme",
|
||||
":authority",
|
||||
},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
href := "https://tools.scrapfly.io/api/fp/anything"
|
||||
resp, err := reqCli.Request(nil, "get", href)
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
log.Print(resp.Text())
|
||||
}
|
||||
```
|
||||
# 采集全国公共资源网和中国政府采购网的列表页的标题
|
||||
```go
|
||||
package main
|
||||
import (
|
||||
"log"
|
||||
"gitee.com/baixudong/requests"
|
||||
)
|
||||
func main() {
|
||||
reqCli, err := requests.NewClient(nil)
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
resp, err := reqCli.Request(nil, "get", "http://www.ccgp.gov.cn/cggg/zygg/")
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
html := resp.Html()
|
||||
lis := html.Finds("ul.c_list_bid li")
|
||||
for _, li := range lis {
|
||||
title := li.Find("a").Get("title")
|
||||
log.Print(title)
|
||||
}
|
||||
resp, err = reqCli.Request(nil, "post", "http://deal.ggzy.gov.cn/ds/deal/dealList_find.jsp", requests.RequestOption{
|
||||
Data: map[string]string{
|
||||
"TIMEBEGIN_SHOW": "2023-04-26",
|
||||
"TIMEEND_SHOW": "2023-05-05",
|
||||
"TIMEBEGIN": "2023-04-26",
|
||||
"TIMEEND": "2023-05-05",
|
||||
"SOURCE_TYPE": "1",
|
||||
"DEAL_TIME": "02",
|
||||
"DEAL_CLASSIFY": "01",
|
||||
"DEAL_STAGE": "0100",
|
||||
"DEAL_PROVINCE": "0",
|
||||
"DEAL_CITY": "0",
|
||||
"DEAL_PLATFORM": "0",
|
||||
"BID_PLATFORM": "0",
|
||||
"DEAL_TRADE": "0",
|
||||
"isShowAll": "1",
|
||||
"PAGENUMBER": "2",
|
||||
"FINDTXT": "",
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
jsonData,_ := resp.Json()
|
||||
lls := jsonData.Get("data").Array()
|
||||
for _, ll := range lls {
|
||||
log.Print(ll.Get("title"))
|
||||
}
|
||||
}
|
||||
```
|
||||
4
body.go
4
body.go
@@ -6,8 +6,8 @@ import (
|
||||
"fmt"
|
||||
"net/url"
|
||||
|
||||
"gitee.com/baixudong/gson"
|
||||
"gitee.com/baixudong/tools"
|
||||
"github.com/gospider007/gson"
|
||||
"github.com/gospider007/tools"
|
||||
)
|
||||
|
||||
// 构造一个文件
|
||||
|
||||
28
client.go
28
client.go
@@ -8,7 +8,8 @@ import (
|
||||
|
||||
"net/http"
|
||||
|
||||
"gitee.com/baixudong/ja3"
|
||||
"github.com/gospider007/gtls"
|
||||
"github.com/gospider007/ja3"
|
||||
)
|
||||
|
||||
type ClientOption struct {
|
||||
@@ -43,8 +44,7 @@ type ClientOption struct {
|
||||
Headers any //请求头
|
||||
Bar bool //是否开启请求进度条
|
||||
|
||||
RequestCallBack func(context.Context, *http.Request) error
|
||||
ResponseCallBack func(context.Context, *http.Request, *http.Response) error
|
||||
RequestCallBack func(context.Context, *http.Request, *http.Response) error
|
||||
}
|
||||
type Client struct {
|
||||
jar *Jar
|
||||
@@ -53,8 +53,7 @@ type Client struct {
|
||||
disUnZip bool //变比自动解压
|
||||
tryNum int //重试次数
|
||||
|
||||
requestCallBack func(context.Context, *http.Request) error
|
||||
responseCallBack func(context.Context, *http.Request, *http.Response) error
|
||||
requestCallBack func(context.Context, *http.Request, *http.Response) error
|
||||
|
||||
optionCallBack func(context.Context, *Client, *RequestOption) error //请求参数回调,用于对请求参数进行修改。返回error,中断重试请求,返回nil继续
|
||||
resultCallBack func(context.Context, *Client, *Response) error //结果回调,用于对结果进行校验。返回nil，直接返回,返回err的话，如果有errCallBack 走errCallBack，没有继续try
|
||||
@@ -151,14 +150,13 @@ func NewClient(preCtx context.Context, options ...ClientOption) (*Client, error)
|
||||
}
|
||||
}
|
||||
result := &Client{
|
||||
jar: jar,
|
||||
ctx: ctx,
|
||||
cnl: cnl,
|
||||
client: client,
|
||||
transport: transport,
|
||||
noJarClient: noJarClient,
|
||||
requestCallBack: option.RequestCallBack,
|
||||
responseCallBack: option.ResponseCallBack,
|
||||
jar: jar,
|
||||
ctx: ctx,
|
||||
cnl: cnl,
|
||||
client: client,
|
||||
transport: transport,
|
||||
noJarClient: noJarClient,
|
||||
requestCallBack: option.RequestCallBack,
|
||||
|
||||
disCookie: option.DisCookie,
|
||||
redirectNum: option.RedirectNum,
|
||||
@@ -174,7 +172,7 @@ func NewClient(preCtx context.Context, options ...ClientOption) (*Client, error)
|
||||
}
|
||||
var err error
|
||||
if option.Proxy != "" {
|
||||
result.proxy, err = VerifyProxy(option.Proxy)
|
||||
result.proxy, err = gtls.VerifyProxy(option.Proxy)
|
||||
}
|
||||
|
||||
if option.Ja3Spec.IsSet() {
|
||||
@@ -189,7 +187,7 @@ func (obj *Client) HttpClient() *http.Client {
|
||||
return obj.client
|
||||
}
|
||||
func (obj *Client) SetProxy(proxyUrl string) (err error) {
|
||||
obj.proxy, err = VerifyProxy(proxyUrl)
|
||||
obj.proxy, err = gtls.VerifyProxy(proxyUrl)
|
||||
return
|
||||
}
|
||||
func (obj *Client) SetGetProxy(getProxy func(ctx context.Context, url *url.URL) (string, error)) {
|
||||
|
||||
@@ -6,7 +6,7 @@ import (
|
||||
|
||||
"net/http"
|
||||
|
||||
"gitee.com/baixudong/gson"
|
||||
"github.com/gospider007/gson"
|
||||
)
|
||||
|
||||
//go:linkname readCookies net/http.readCookies
|
||||
|
||||
19
dial.go
19
dial.go
@@ -14,8 +14,9 @@ import (
|
||||
|
||||
"net/http"
|
||||
|
||||
"gitee.com/baixudong/ja3"
|
||||
"gitee.com/baixudong/tools"
|
||||
"github.com/gospider007/gtls"
|
||||
"github.com/gospider007/ja3"
|
||||
"github.com/gospider007/tools"
|
||||
utls "github.com/refraction-networking/utls"
|
||||
)
|
||||
|
||||
@@ -95,7 +96,7 @@ func (obj *DialClient) AddrToIp(ctx context.Context, addr string) (string, error
|
||||
if err != nil {
|
||||
return addr, tools.WrapError(err, "addrToIp 错误,SplitHostPort")
|
||||
}
|
||||
_, ipInt := ParseHost(host)
|
||||
_, ipInt := gtls.ParseHost(host)
|
||||
if ipInt == 4 || ipInt == 6 {
|
||||
return addr, nil
|
||||
}
|
||||
@@ -164,11 +165,11 @@ func (obj *DialClient) clientVerifySocks5(ctx context.Context, proxyUrl *url.URL
|
||||
}
|
||||
var host string
|
||||
var port int
|
||||
if host, port, err = SplitHostPort(addr); err != nil {
|
||||
if host, port, err = gtls.SplitHostPort(addr); err != nil {
|
||||
return
|
||||
}
|
||||
writeCon := []byte{5, 1, 0}
|
||||
ip, ipInt := ParseHost(host)
|
||||
ip, ipInt := gtls.ParseHost(host)
|
||||
switch ipInt {
|
||||
case 4:
|
||||
writeCon = append(writeCon, 1)
|
||||
@@ -245,7 +246,7 @@ func (obj *DialClient) lookupIPAddr(ctx context.Context, host string) (net.IP, e
|
||||
}
|
||||
for _, ipAddr := range ips {
|
||||
ip := ipAddr.IP
|
||||
if ipType := ParseIp(ip); ipType == 4 || ipType == 6 {
|
||||
if ipType := gtls.ParseIp(ip); ipType == 4 || ipType == 6 {
|
||||
if addrType == 0 || addrType == ipType {
|
||||
return ip, nil
|
||||
}
|
||||
@@ -253,7 +254,7 @@ func (obj *DialClient) lookupIPAddr(ctx context.Context, host string) (net.IP, e
|
||||
}
|
||||
for _, ipAddr := range ips {
|
||||
ip := ipAddr.IP
|
||||
if ipType := ParseIp(ip); ipType == 4 || ipType == 6 {
|
||||
if ipType := gtls.ParseIp(ip); ipType == 4 || ipType == 6 {
|
||||
return ip, nil
|
||||
}
|
||||
}
|
||||
@@ -277,7 +278,7 @@ func (obj *DialClient) Dialer() *net.Dialer {
|
||||
}
|
||||
func (obj *DialClient) AddTls(ctx context.Context, conn net.Conn, host string, disHttp2 bool, tlsConfig *tls.Config) (*tls.Conn, error) {
|
||||
var tlsConn *tls.Conn
|
||||
tlsConfig.ServerName = GetServerName(host)
|
||||
tlsConfig.ServerName = gtls.GetServerName(host)
|
||||
if disHttp2 {
|
||||
tlsConfig.NextProtos = []string{"http/1.1"}
|
||||
} else {
|
||||
@@ -287,7 +288,7 @@ func (obj *DialClient) AddTls(ctx context.Context, conn net.Conn, host string, d
|
||||
return tlsConn, tlsConn.HandshakeContext(ctx)
|
||||
}
|
||||
func (obj *DialClient) AddJa3Tls(ctx context.Context, conn net.Conn, host string, disHttp2 bool, ja3Spec ja3.Ja3Spec, tlsConfig *utls.Config) (*utls.UConn, error) {
|
||||
tlsConfig.ServerName = GetServerName(host)
|
||||
tlsConfig.ServerName = gtls.GetServerName(host)
|
||||
if disHttp2 {
|
||||
tlsConfig.NextProtos = []string{"http/1.1"}
|
||||
} else {
|
||||
|
||||
26
go.mod
26
go.mod
@@ -1,23 +1,22 @@
|
||||
module gitee.com/baixudong/requests
|
||||
module github.com/gospider007/requests
|
||||
|
||||
go 1.21.1
|
||||
|
||||
require (
|
||||
gitee.com/baixudong/bar v0.0.0-20230923032414-c19cc384edeb
|
||||
gitee.com/baixudong/bs4 v0.0.0-20230927091031-00dd545f5784
|
||||
gitee.com/baixudong/gson v0.0.0-20230927090937-556b58fb6899
|
||||
gitee.com/baixudong/ja3 v0.0.0-20231010095622-1f26d89ef419
|
||||
gitee.com/baixudong/net v0.0.0-20230927083058-4a6a7a20f917
|
||||
gitee.com/baixudong/re v0.0.0-20230809033040-360c1d945e59
|
||||
gitee.com/baixudong/tools v0.0.0-20230926010219-77f29cbf5935
|
||||
gitee.com/baixudong/websocket v0.0.0-20230927082325-5b4eb178c3ec
|
||||
github.com/gospider007/bar v0.0.0-20231015040703-7d4e67e1741a
|
||||
github.com/gospider007/bs4 v0.0.0-20231015041001-064efde5bd08
|
||||
github.com/gospider007/gson v0.0.0-20231015042048-4fae1418931b
|
||||
github.com/gospider007/gtls v0.0.0-20231015025737-e18d927ad339
|
||||
github.com/gospider007/ja3 v0.0.0-20231015030157-fdfe60187c35
|
||||
github.com/gospider007/net v0.0.0-20231015031122-61ca42e0006f
|
||||
github.com/gospider007/re v0.0.0-20231015023348-717c984874af
|
||||
github.com/gospider007/tools v0.0.0-20231015025637-9b1d99c470f4
|
||||
github.com/gospider007/websocket v0.0.0-20231015042221-9176c1ebbdfc
|
||||
github.com/refraction-networking/utls v1.5.4
|
||||
golang.org/x/net v0.16.0
|
||||
golang.org/x/net v0.17.0
|
||||
)
|
||||
|
||||
require (
|
||||
gitee.com/baixudong/blog v0.0.0-20230923032533-d83e7cb672f7 // indirect
|
||||
gitee.com/baixudong/kinds v0.0.0-20230809033013-c3d6d3479f8c // indirect
|
||||
github.com/PuerkitoBio/goquery v1.8.1 // indirect
|
||||
github.com/andybalholm/brotli v1.0.5 // indirect
|
||||
github.com/andybalholm/cascadia v1.3.2 // indirect
|
||||
@@ -29,7 +28,8 @@ require (
|
||||
github.com/gobwas/httphead v0.1.0 // indirect
|
||||
github.com/gobwas/pool v0.2.1 // indirect
|
||||
github.com/goccy/go-json v0.10.2 // indirect
|
||||
github.com/google/uuid v1.3.1 // indirect
|
||||
github.com/gospider007/blog v0.0.0-20231015040505-4b08187c11ee // indirect
|
||||
github.com/gospider007/kinds v0.0.0-20231015025159-69ab7d8d9b94 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/klauspost/compress v1.17.0 // indirect
|
||||
github.com/leodido/go-urn v1.2.4 // indirect
|
||||
|
||||
74
go.sum
74
go.sum
@@ -1,25 +1,3 @@
|
||||
gitee.com/baixudong/bar v0.0.0-20230923032414-c19cc384edeb h1:mqsOumc9CnMEYOefPzVxedRWaXP8XmpuN3loom1zLno=
|
||||
gitee.com/baixudong/bar v0.0.0-20230923032414-c19cc384edeb/go.mod h1:vrjSAS99QLVH3l/S7ZJ/flF+4e+9ayHZ3h9SC1mDVv4=
|
||||
gitee.com/baixudong/blog v0.0.0-20230923032533-d83e7cb672f7 h1:QDS4QkH6BHy95RnoGJVSlOdpwpSjZrdne0jqIuIEEd4=
|
||||
gitee.com/baixudong/blog v0.0.0-20230923032533-d83e7cb672f7/go.mod h1:CRRC/3rq5rR/0+3MUuuxP3BX2gj8aR0SM0gumsd0pUk=
|
||||
gitee.com/baixudong/bs4 v0.0.0-20230927091031-00dd545f5784 h1:mPmPbhSf61sSa8BlztLqu904ceay0qAwyzvoagNBRik=
|
||||
gitee.com/baixudong/bs4 v0.0.0-20230927091031-00dd545f5784/go.mod h1:lVGpwtBVvx5LuWTtWjoNxNw7XQDl9lvkDdFKX4VumVI=
|
||||
gitee.com/baixudong/gson v0.0.0-20230927090937-556b58fb6899 h1:JFV4d4J07fYDmp5hyZirQNSv4FmE2JQliZVSYoa4ieM=
|
||||
gitee.com/baixudong/gson v0.0.0-20230927090937-556b58fb6899/go.mod h1:1DBHleeT45+7ogNCRkJYKhq28Krc3oC5Hr4pHmd+GM0=
|
||||
gitee.com/baixudong/ja3 v0.0.0-20230927070510-152bc4a3675a h1:IONxzaxrHVCEINNw6oMcguyYqAu5DD9DjmYq2DoOuhY=
|
||||
gitee.com/baixudong/ja3 v0.0.0-20230927070510-152bc4a3675a/go.mod h1:Y3tcqTP6YapuS8pWDT7zkHgoKV14Sx652+OuJJyssk4=
|
||||
gitee.com/baixudong/ja3 v0.0.0-20231010095622-1f26d89ef419 h1:EUeBpoYVKI/rK0FQ8I8g50wx510HrQNju1orZ9RF25M=
|
||||
gitee.com/baixudong/ja3 v0.0.0-20231010095622-1f26d89ef419/go.mod h1:SiOf/JBnmWJ/aTQKVUHFKSDLF+gnlV2rIZ73Ecy/RD8=
|
||||
gitee.com/baixudong/kinds v0.0.0-20230809033013-c3d6d3479f8c h1:MZewpjU0+82TcA+nHrcAbbgZhTCqVH2I5zwYBWJ4v54=
|
||||
gitee.com/baixudong/kinds v0.0.0-20230809033013-c3d6d3479f8c/go.mod h1:J3r4FWnZOMgk7/1EWEEDaPQnx5uH4OLCRzpTOyKg5cA=
|
||||
gitee.com/baixudong/net v0.0.0-20230927083058-4a6a7a20f917 h1:/eAVC7BfyvZCMNMhYXtAw50RezEREx5baecMQZHGFAs=
|
||||
gitee.com/baixudong/net v0.0.0-20230927083058-4a6a7a20f917/go.mod h1:q4sczw+EvXxYspUBQESB+Kr1lwkyOZ2Kc0bPItIgoTo=
|
||||
gitee.com/baixudong/re v0.0.0-20230809033040-360c1d945e59 h1:HtdoUzBIqepeVTeeZ0QcPKoQSDqgWQn/fs84fsByJv0=
|
||||
gitee.com/baixudong/re v0.0.0-20230809033040-360c1d945e59/go.mod h1:i5L3z/q42lHZvd3JSk3390XdujEY2QHXFMEcaO/uZD0=
|
||||
gitee.com/baixudong/tools v0.0.0-20230926010219-77f29cbf5935 h1:+RccqXXfx7yPpwc7Q13z8YZlXFTtqh+A8tlsfd00jS4=
|
||||
gitee.com/baixudong/tools v0.0.0-20230926010219-77f29cbf5935/go.mod h1:nGjhspfGKu9Z6knMz0i0riNzk8Q7kmv46qRl+6Wx+rY=
|
||||
gitee.com/baixudong/websocket v0.0.0-20230927082325-5b4eb178c3ec h1:MAHSoPaI3qKYVXvd7eSSo4McWH2tgaZ5qDS9zG+vR0c=
|
||||
gitee.com/baixudong/websocket v0.0.0-20230927082325-5b4eb178c3ec/go.mod h1:oKErVWktOk6crTjCHDDioz08/9oY8H3EvZQ7roNmIJ4=
|
||||
github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM=
|
||||
github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ=
|
||||
github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
|
||||
@@ -29,8 +7,8 @@ github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsVi
|
||||
github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
|
||||
github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM=
|
||||
github.com/bytedance/sonic v1.10.0-rc/go.mod h1:ElCzW+ufi8qKqNW0FY314xriJhyJhuoJ3gFZdAHF7NM=
|
||||
github.com/bytedance/sonic v1.10.1 h1:7a1wuFXL1cMy7a3f7/VFcEtriuXQnUBhtoVfOZiaysc=
|
||||
github.com/bytedance/sonic v1.10.1/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4=
|
||||
github.com/bytedance/sonic v1.10.2 h1:GQebETVBxYB7JGWJtLBi07OVzWwt+8dWA00gEVW2ZFE=
|
||||
github.com/bytedance/sonic v1.10.2/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4=
|
||||
github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY=
|
||||
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk=
|
||||
github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d h1:77cEq6EriyTZ0g/qfRdp61a3Uu/AWrgIq2s0ClJV1g0=
|
||||
@@ -61,8 +39,8 @@ github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+
|
||||
github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
|
||||
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
|
||||
github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI=
|
||||
github.com/go-playground/validator/v10 v10.15.4 h1:zMXza4EpOdooxPel5xDqXEdXG5r+WggpvnAKMsalBjs=
|
||||
github.com/go-playground/validator/v10 v10.15.4/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
|
||||
github.com/go-playground/validator/v10 v10.15.5 h1:LEBecTWb/1j5TNY1YYG2RcOUN3R7NLylN+x8TTueE24=
|
||||
github.com/go-playground/validator/v10 v10.15.5/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
|
||||
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
|
||||
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
|
||||
github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo=
|
||||
@@ -88,11 +66,31 @@ github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98 h1:pUa4ghanp6q4IJHwE9RwLgmVFfReJN+KbQ8ExNEUUoQ=
|
||||
github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik=
|
||||
github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4=
|
||||
github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
|
||||
github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/gospider007/bar v0.0.0-20231015040703-7d4e67e1741a h1:CK3MzqDX3TaT+WCoVTALF7ssj8fufZxtrzICK2/Wpu8=
|
||||
github.com/gospider007/bar v0.0.0-20231015040703-7d4e67e1741a/go.mod h1:Yv0zw5KpG9gFwXVUs2dIauORHbUgzX+sR9UTVdgdWzo=
|
||||
github.com/gospider007/blog v0.0.0-20231015040505-4b08187c11ee h1:F+MircZJ9ysDFsq4Mea43+MbvbDcnHQFq/lOHLSO0IE=
|
||||
github.com/gospider007/blog v0.0.0-20231015040505-4b08187c11ee/go.mod h1:y4zOnf7HvNA5TJst6YEQP3kJllg5PBjl6yMogqAMGq0=
|
||||
github.com/gospider007/bs4 v0.0.0-20231015041001-064efde5bd08 h1:1xP5y0hLlvRQJ1TW58hEx+oPlh2tKm88wMKtFHaQeHo=
|
||||
github.com/gospider007/bs4 v0.0.0-20231015041001-064efde5bd08/go.mod h1:R4MKplqr25t6ewFY6LqzH4Oggur81IegmRNJEzuhwfw=
|
||||
github.com/gospider007/gson v0.0.0-20231015042048-4fae1418931b h1:Gd5tOuEAlrs+5IWE/Q7WmJFmNmWA+tarN1MZe1G4oPw=
|
||||
github.com/gospider007/gson v0.0.0-20231015042048-4fae1418931b/go.mod h1:frJXgZBXIUJlfsIB7LrEGEIU2SeQu7xP/1xs/Tphlwg=
|
||||
github.com/gospider007/gtls v0.0.0-20231015025737-e18d927ad339 h1:YmZe17AF0QZQp0K0QlArKa0UPoULgQepI5BXeHGs8/8=
|
||||
github.com/gospider007/gtls v0.0.0-20231015025737-e18d927ad339/go.mod h1:HNrQHMSTsG73mweP5p010QAgXVaZ6RgcfxfJOdyieUY=
|
||||
github.com/gospider007/ja3 v0.0.0-20231015030157-fdfe60187c35 h1:Eyfpv7ZG9+l//KIKeEs0CJRTGPIeP5sfZ0kJCspfakU=
|
||||
github.com/gospider007/ja3 v0.0.0-20231015030157-fdfe60187c35/go.mod h1:BNtOUDtO8NF8aR7rtl3CAvE/LG9+RWtUiLssswjGLTI=
|
||||
github.com/gospider007/kinds v0.0.0-20231015025159-69ab7d8d9b94 h1:K2XYy6B6M62o35KMabuzl9DmA4WQftanUgKomb2YYU4=
|
||||
github.com/gospider007/kinds v0.0.0-20231015025159-69ab7d8d9b94/go.mod h1:5e4rIpZmhwqOLVVRZa0uZFAuMPH15dO0qqzKf1s3IgI=
|
||||
github.com/gospider007/net v0.0.0-20231015031122-61ca42e0006f h1:Cj8eyFdQnOxTIt8VbyYwAwUZRg/sTX+9hL8jvg4ksqs=
|
||||
github.com/gospider007/net v0.0.0-20231015031122-61ca42e0006f/go.mod h1:hadIeF78Sy2/gCvOAr/ABCKe7cmudBh9w2a5YhCaRXQ=
|
||||
github.com/gospider007/re v0.0.0-20231015023348-717c984874af h1:bYip5he4jUPslsEY3GtBndUUJppR84Ml9areZ/wG0Hk=
|
||||
github.com/gospider007/re v0.0.0-20231015023348-717c984874af/go.mod h1:fOcqR9yJHQjUmpLerdxJSoMJ4pFyMfRirM12rTMEH38=
|
||||
github.com/gospider007/tools v0.0.0-20231015025637-9b1d99c470f4 h1:XeDsy/Qy/uH+MhRPeBK44Owl5V/zotyklIgP9rS2ONc=
|
||||
github.com/gospider007/tools v0.0.0-20231015025637-9b1d99c470f4/go.mod h1:UAIsaldUoH2zZqg4i0es5Q6x/GS0BkgCeeQ/sLnnZJk=
|
||||
github.com/gospider007/websocket v0.0.0-20231015042221-9176c1ebbdfc h1:eLhD3+H6ads14/yDR8L6UQ/7ZCAjO5QC5fHLGTpqCiw=
|
||||
github.com/gospider007/websocket v0.0.0-20231015042221-9176c1ebbdfc/go.mod h1:xfgcqdRvRpjIyPAhO5/o+ubuTmJAey/wGnY8DyiNHRc=
|
||||
github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
|
||||
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
|
||||
@@ -127,8 +125,6 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/quic-go/quic-go v0.39.0 h1:AgP40iThFMY0bj8jGxROhw3S0FMGa8ryqsmi9tBH3So=
|
||||
github.com/quic-go/quic-go v0.39.0/go.mod h1:T09QsDQWjLiQ74ZmacDfqZmhY/NLnw5BC40MANNNZ1Q=
|
||||
github.com/refraction-networking/utls v1.5.3 h1:Ds5Ocg1+MC1ahNx5iBEcHe0jHeLaA/fLey61EENm7ro=
|
||||
github.com/refraction-networking/utls v1.5.3/go.mod h1:SPuDbBmgLGp8s+HLNc83FuavwZCFoMmExj+ltUHiHUw=
|
||||
github.com/refraction-networking/utls v1.5.4 h1:9k6EO2b8TaOGsQ7Pl7p9w6PUhx18/ZCeT0WNTZ7Uw4o=
|
||||
github.com/refraction-networking/utls v1.5.4/go.mod h1:SPuDbBmgLGp8s+HLNc83FuavwZCFoMmExj+ltUHiHUw=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
@@ -175,16 +171,10 @@ golang.org/x/arch v0.5.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||
golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck=
|
||||
golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
|
||||
golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
|
||||
golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
|
||||
golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g=
|
||||
golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k=
|
||||
golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
|
||||
golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
|
||||
golang.org/x/image v0.12.0 h1:w13vZbU4o5rKOFFR8y7M+c4A5jXDC0uXTdHYRP8X2DQ=
|
||||
golang.org/x/image v0.12.0/go.mod h1:Lu90jvHG7GfemOIcldsh9A2hS01ocl6oNO7ype5mEnk=
|
||||
golang.org/x/image v0.13.0 h1:3cge/F/QTkNLauhf2QoE9zp+7sr+ZcL4HnoZmdwg9sg=
|
||||
golang.org/x/image v0.13.0/go.mod h1:6mmbMOeV28HuMTgA6OSRkdXKYw/t5W9Uwn2Yv1r3Yxk=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
@@ -197,10 +187,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug
|
||||
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
|
||||
golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8=
|
||||
golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
|
||||
golang.org/x/net v0.16.0 h1:7eBu7KsSvFDtSXUIDbh3aqlK4DPsZ1rByC8PFfBThos=
|
||||
golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
|
||||
golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
|
||||
golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
@@ -214,8 +202,6 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o=
|
||||
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
|
||||
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
@@ -237,8 +223,8 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ=
|
||||
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
|
||||
golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc=
|
||||
golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
|
||||
|
||||
@@ -5,7 +5,7 @@ import (
|
||||
|
||||
"net/http"
|
||||
|
||||
"gitee.com/baixudong/gson"
|
||||
"github.com/gospider007/gson"
|
||||
)
|
||||
|
||||
var UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36 Edg/117.0.2045.31"
|
||||
|
||||
17
option.go
17
option.go
@@ -12,8 +12,8 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitee.com/baixudong/ja3"
|
||||
"gitee.com/baixudong/websocket"
|
||||
"github.com/gospider007/ja3"
|
||||
"github.com/gospider007/websocket"
|
||||
)
|
||||
|
||||
// 请求参数选项
|
||||
@@ -48,12 +48,10 @@ type RequestOption struct {
|
||||
DisProxy bool //是否关闭代理,强制关闭代理
|
||||
TryNum int //重试次数
|
||||
|
||||
OptionCallBack func(context.Context, *Client, *RequestOption) error //请求参数回调,用于对请求参数进行修改。返回error,中断重试请求,返回nil继续
|
||||
ResultCallBack func(context.Context, *Client, *Response) error //结果回调,用于对结果进行校验。返回nil，直接返回,返回err的话，如果有errCallBack 走errCallBack，没有继续try
|
||||
ErrCallBack func(context.Context, *Client, error) error //错误回调,返回error,中断重试请求,返回nil继续
|
||||
|
||||
RequestCallBack func(context.Context, *http.Request) error
|
||||
ResponseCallBack func(context.Context, *http.Request, *http.Response) error
|
||||
OptionCallBack func(context.Context, *Client, *RequestOption) error //请求参数回调,用于对请求参数进行修改。返回error,中断重试请求,返回nil继续
|
||||
ResultCallBack func(context.Context, *Client, *Response) error //结果回调,用于对结果进行校验。返回nil，直接返回,返回err的话，如果有errCallBack 走errCallBack，没有继续try
|
||||
ErrCallBack func(context.Context, *Client, error) error //错误回调,返回error,中断重试请求,返回nil继续
|
||||
RequestCallBack func(context.Context, *http.Request, *http.Response) error
|
||||
|
||||
Jar *Jar //自定义临时cookies 管理
|
||||
|
||||
@@ -239,8 +237,5 @@ func (obj *Client) newRequestOption(option RequestOption) RequestOption {
|
||||
if option.RequestCallBack == nil {
|
||||
option.RequestCallBack = obj.requestCallBack
|
||||
}
|
||||
if option.ResponseCallBack == nil {
|
||||
option.ResponseCallBack = obj.responseCallBack
|
||||
}
|
||||
return option
|
||||
}
|
||||
|
||||
177
requests.go
177
requests.go
@@ -14,10 +14,11 @@ import (
|
||||
|
||||
"net/http"
|
||||
|
||||
"gitee.com/baixudong/ja3"
|
||||
"gitee.com/baixudong/re"
|
||||
"gitee.com/baixudong/tools"
|
||||
"gitee.com/baixudong/websocket"
|
||||
"github.com/gospider007/gtls"
|
||||
"github.com/gospider007/ja3"
|
||||
"github.com/gospider007/re"
|
||||
"github.com/gospider007/tools"
|
||||
"github.com/gospider007/websocket"
|
||||
)
|
||||
|
||||
//go:linkname ReadRequest net/http.readRequest
|
||||
@@ -139,8 +140,7 @@ type reqCtxData struct {
|
||||
proxy *url.URL
|
||||
disProxy bool
|
||||
|
||||
requestCallBack func(context.Context, *http.Request) error
|
||||
responseCallBack func(context.Context, *http.Request, *http.Response) error
|
||||
requestCallBack func(context.Context, *http.Request, *http.Response) error
|
||||
|
||||
h2Ja3Spec ja3.H2Ja3Spec
|
||||
ja3Spec ja3.Ja3Spec
|
||||
@@ -148,93 +148,83 @@ type reqCtxData struct {
|
||||
|
||||
func Get(preCtx context.Context, href string, options ...RequestOption) (resp *Response, err error) {
|
||||
client, _ := NewClient(preCtx)
|
||||
defer func() {
|
||||
if err != nil || (resp.webSocket == nil && resp.sseClient == nil) {
|
||||
client.Close()
|
||||
}
|
||||
}()
|
||||
return client.Request(preCtx, http.MethodGet, href, options...)
|
||||
resp, err = client.Request(preCtx, http.MethodGet, href, options...)
|
||||
if err != nil || resp == nil || !resp.oneceAlive() {
|
||||
client.Close()
|
||||
}
|
||||
return
|
||||
}
|
||||
func Head(preCtx context.Context, href string, options ...RequestOption) (resp *Response, err error) {
|
||||
client, _ := NewClient(preCtx)
|
||||
defer func() {
|
||||
if err != nil || (resp.webSocket == nil && resp.sseClient == nil) {
|
||||
client.Close()
|
||||
}
|
||||
}()
|
||||
return client.Request(preCtx, http.MethodHead, href, options...)
|
||||
resp, err = client.Request(preCtx, http.MethodHead, href, options...)
|
||||
if err != nil || resp == nil || !resp.oneceAlive() {
|
||||
client.Close()
|
||||
}
|
||||
return
|
||||
}
|
||||
func Post(preCtx context.Context, href string, options ...RequestOption) (resp *Response, err error) {
|
||||
client, _ := NewClient(preCtx)
|
||||
defer func() {
|
||||
if err != nil || (resp.webSocket == nil && resp.sseClient == nil) {
|
||||
client.Close()
|
||||
}
|
||||
}()
|
||||
return client.Request(preCtx, http.MethodPost, href, options...)
|
||||
resp, err = client.Request(preCtx, http.MethodPost, href, options...)
|
||||
if err != nil || resp == nil || !resp.oneceAlive() {
|
||||
client.Close()
|
||||
}
|
||||
return
|
||||
}
|
||||
func Put(preCtx context.Context, href string, options ...RequestOption) (resp *Response, err error) {
|
||||
client, _ := NewClient(preCtx)
|
||||
defer func() {
|
||||
if err != nil || (resp.webSocket == nil && resp.sseClient == nil) {
|
||||
client.Close()
|
||||
}
|
||||
}()
|
||||
return client.Request(preCtx, http.MethodPut, href, options...)
|
||||
resp, err = client.Request(preCtx, http.MethodPut, href, options...)
|
||||
if err != nil || resp == nil || !resp.oneceAlive() {
|
||||
client.Close()
|
||||
}
|
||||
return
|
||||
}
|
||||
func Patch(preCtx context.Context, href string, options ...RequestOption) (resp *Response, err error) {
|
||||
client, _ := NewClient(preCtx)
|
||||
defer func() {
|
||||
if err != nil || (resp.webSocket == nil && resp.sseClient == nil) {
|
||||
client.Close()
|
||||
}
|
||||
}()
|
||||
return client.Request(preCtx, http.MethodPatch, href, options...)
|
||||
resp, err = client.Request(preCtx, http.MethodPatch, href, options...)
|
||||
if err != nil || resp == nil || !resp.oneceAlive() {
|
||||
client.Close()
|
||||
}
|
||||
return
|
||||
}
|
||||
func Delete(preCtx context.Context, href string, options ...RequestOption) (resp *Response, err error) {
|
||||
client, _ := NewClient(preCtx)
|
||||
defer func() {
|
||||
if err != nil || (resp.webSocket == nil && resp.sseClient == nil) {
|
||||
client.Close()
|
||||
}
|
||||
}()
|
||||
return client.Request(preCtx, http.MethodDelete, href, options...)
|
||||
resp, err = client.Request(preCtx, http.MethodDelete, href, options...)
|
||||
if err != nil || resp == nil || !resp.oneceAlive() {
|
||||
client.Close()
|
||||
}
|
||||
return
|
||||
}
|
||||
func Connect(preCtx context.Context, href string, options ...RequestOption) (resp *Response, err error) {
|
||||
client, _ := NewClient(preCtx)
|
||||
defer func() {
|
||||
if err != nil || (resp.webSocket == nil && resp.sseClient == nil) {
|
||||
client.Close()
|
||||
}
|
||||
}()
|
||||
return client.Request(preCtx, http.MethodConnect, href, options...)
|
||||
resp, err = client.Request(preCtx, http.MethodConnect, href, options...)
|
||||
if err != nil || resp == nil || !resp.oneceAlive() {
|
||||
client.Close()
|
||||
}
|
||||
return
|
||||
}
|
||||
func Options(preCtx context.Context, href string, options ...RequestOption) (resp *Response, err error) {
|
||||
client, _ := NewClient(preCtx)
|
||||
defer func() {
|
||||
if err != nil || (resp.webSocket == nil && resp.sseClient == nil) {
|
||||
client.Close()
|
||||
}
|
||||
}()
|
||||
return client.Request(preCtx, http.MethodOptions, href, options...)
|
||||
resp, err = client.Request(preCtx, http.MethodOptions, href, options...)
|
||||
if err != nil || resp == nil || !resp.oneceAlive() {
|
||||
client.Close()
|
||||
}
|
||||
return
|
||||
}
|
||||
func Trace(preCtx context.Context, href string, options ...RequestOption) (resp *Response, err error) {
|
||||
client, _ := NewClient(preCtx)
|
||||
defer func() {
|
||||
if err != nil || (resp.webSocket == nil && resp.sseClient == nil) {
|
||||
client.Close()
|
||||
}
|
||||
}()
|
||||
return client.Request(preCtx, http.MethodTrace, href, options...)
|
||||
resp, err = client.Request(preCtx, http.MethodTrace, href, options...)
|
||||
if err != nil || resp == nil || !resp.oneceAlive() {
|
||||
client.Close()
|
||||
}
|
||||
return
|
||||
}
|
||||
func Request(preCtx context.Context, method string, href string, options ...RequestOption) (resp *Response, err error) {
|
||||
client, _ := NewClient(preCtx)
|
||||
defer func() {
|
||||
if err != nil || (resp.webSocket == nil && resp.sseClient == nil) {
|
||||
client.Close()
|
||||
}
|
||||
}()
|
||||
return client.Request(preCtx, method, href, options...)
|
||||
resp, err = client.Request(preCtx, method, href, options...)
|
||||
if err != nil || resp == nil || !resp.oneceAlive() {
|
||||
client.Close()
|
||||
}
|
||||
return
|
||||
}
|
||||
func (obj *Client) Get(preCtx context.Context, href string, options ...RequestOption) (*Response, error) {
|
||||
return obj.Request(preCtx, http.MethodGet, href, options...)
|
||||
@@ -313,7 +303,7 @@ func (obj *Client) Request(preCtx context.Context, method string, href string, o
|
||||
func (obj *Client) request(preCtx context.Context, option RequestOption) (response *Response, err error) {
|
||||
response = new(Response)
|
||||
defer func() {
|
||||
if err == nil && response.webSocket == nil && response.sseClient == nil && !option.DisRead { //判断是否读取body,和对body的处理
|
||||
if err == nil && !response.oneceAlive() && !option.DisRead { //判断是否读取body,和对body的处理
|
||||
err = response.ReadBody()
|
||||
defer response.Close()
|
||||
}
|
||||
@@ -349,12 +339,11 @@ func (obj *Client) request(preCtx context.Context, option RequestOption) (respon
|
||||
ctxData := new(reqCtxData)
|
||||
|
||||
ctxData.requestCallBack = option.RequestCallBack
|
||||
ctxData.responseCallBack = option.ResponseCallBack
|
||||
//构造代理
|
||||
ctxData.disProxy = option.DisProxy
|
||||
if !ctxData.disProxy {
|
||||
if option.Proxy != "" { //代理相关构造
|
||||
tempProxy, err := VerifyProxy(option.Proxy)
|
||||
tempProxy, err := gtls.VerifyProxy(option.Proxy)
|
||||
if err != nil {
|
||||
return response, tools.WrapError(errFatal, errors.New("tempRequest 构造代理失败"), err)
|
||||
}
|
||||
@@ -386,24 +375,6 @@ func (obj *Client) request(preCtx context.Context, option RequestOption) (respon
|
||||
if err != nil {
|
||||
return response, tools.WrapError(errFatal, errors.New("tempRequest 构造request失败"), err)
|
||||
}
|
||||
|
||||
//解析Scheme
|
||||
var isWebSocket bool
|
||||
switch reqs.URL.Scheme {
|
||||
case "ws", "wss":
|
||||
isWebSocket = true
|
||||
case "file":
|
||||
response.filePath = re.Sub(`^/+`, "", reqs.URL.Path)
|
||||
response.content, err = os.ReadFile(response.filePath)
|
||||
if err != nil {
|
||||
err = tools.WrapError(errFatal, errors.New("read filePath data error"), err)
|
||||
}
|
||||
return
|
||||
case "http", "https":
|
||||
default:
|
||||
err = tools.WrapError(errFatal, fmt.Errorf("url scheme error: %s", reqs.URL.Scheme))
|
||||
return
|
||||
}
|
||||
//添加headers
|
||||
var headOk bool
|
||||
if reqs.Header, headOk = option.Headers.(http.Header); !headOk {
|
||||
@@ -418,6 +389,24 @@ func (obj *Client) request(preCtx context.Context, option RequestOption) (respon
|
||||
if reqs.Header.Get("Content-Type") == "" && reqs.Header.Get("content-type") == "" && option.ContentType != "" {
|
||||
reqs.Header.Set("Content-Type", option.ContentType)
|
||||
}
|
||||
|
||||
//解析Scheme
|
||||
switch reqs.URL.Scheme {
|
||||
case "ws", "wss":
|
||||
websocket.SetClientHeadersOption(reqs.Header, option.WsOption)
|
||||
case "file":
|
||||
response.filePath = re.Sub(`^/+`, "", reqs.URL.Path)
|
||||
response.content, err = os.ReadFile(response.filePath)
|
||||
if err != nil {
|
||||
err = tools.WrapError(errFatal, errors.New("read filePath data error"), err)
|
||||
}
|
||||
return
|
||||
case "http", "https":
|
||||
default:
|
||||
err = tools.WrapError(errFatal, fmt.Errorf("url scheme error: %s", reqs.URL.Scheme))
|
||||
return
|
||||
}
|
||||
|
||||
//host构造
|
||||
if option.Host != "" {
|
||||
reqs.Host = option.Host
|
||||
@@ -434,10 +423,7 @@ func (obj *Client) request(preCtx context.Context, option RequestOption) (respon
|
||||
reqs.AddCookie(vv)
|
||||
}
|
||||
}
|
||||
//开始发送请求
|
||||
if isWebSocket { //设置websocket headers
|
||||
websocket.SetClientHeadersOption(reqs.Header, option.WsOption)
|
||||
}
|
||||
|
||||
if response.response, err = obj.getClient(option).Do(reqs); err != nil {
|
||||
return
|
||||
} else if response.response == nil {
|
||||
@@ -447,13 +433,8 @@ func (obj *Client) request(preCtx context.Context, option RequestOption) (respon
|
||||
if !response.disUnzip {
|
||||
response.disUnzip = response.response.Uncompressed
|
||||
}
|
||||
|
||||
if isWebSocket { //判断ws 的状态码是否正确
|
||||
if response.response.StatusCode == 101 {
|
||||
response.webSocket, err = websocket.NewClientConn(response.response)
|
||||
} else {
|
||||
err = errors.New("statusCode not 101, url为websocket链接，但是对方服务器没有将请求升级到websocket")
|
||||
}
|
||||
if response.response.StatusCode == 101 {
|
||||
response.webSocket, err = websocket.NewClientConn(response.response)
|
||||
} else if response.response.Header.Get("Content-Type") == "text/event-stream" { //如果为sse协议就关闭读取
|
||||
response.sseClient = newSseClient(response)
|
||||
}
|
||||
|
||||
22
response.go
22
response.go
@@ -14,11 +14,12 @@ import (
|
||||
|
||||
"net/http"
|
||||
|
||||
"gitee.com/baixudong/bar"
|
||||
"gitee.com/baixudong/bs4"
|
||||
"gitee.com/baixudong/gson"
|
||||
"gitee.com/baixudong/tools"
|
||||
"gitee.com/baixudong/websocket"
|
||||
"github.com/gospider007/bar"
|
||||
"github.com/gospider007/bs4"
|
||||
"github.com/gospider007/gson"
|
||||
"github.com/gospider007/gtls"
|
||||
"github.com/gospider007/tools"
|
||||
"github.com/gospider007/websocket"
|
||||
)
|
||||
|
||||
type Response struct {
|
||||
@@ -230,11 +231,11 @@ func (obj *Response) Html() *bs4.Client {
|
||||
// 获取headers 的Content-Type
|
||||
func (obj *Response) ContentType() string {
|
||||
if obj.filePath != "" {
|
||||
return GetContentTypeWithBytes(obj.content)
|
||||
return gtls.GetContentTypeWithBytes(obj.content)
|
||||
}
|
||||
contentType := obj.response.Header.Get("Content-Type")
|
||||
if contentType == "" {
|
||||
contentType = GetContentTypeWithBytes(obj.content)
|
||||
contentType = gtls.GetContentTypeWithBytes(obj.content)
|
||||
}
|
||||
return contentType
|
||||
}
|
||||
@@ -303,7 +304,14 @@ func (obj *Response) Read(con []byte) (i int, err error) { //读取body
|
||||
}
|
||||
}
|
||||
|
||||
func (obj *Response) oneceAlive() bool { //读取body,对body 解压，解码操作
|
||||
return obj.webSocket != nil || obj.sseClient != nil
|
||||
}
|
||||
|
||||
func (obj *Response) ReadBody() error { //读取body,对body 解压，解码操作
|
||||
if obj.oneceAlive() {
|
||||
return errors.New("ws 或 sse 不能读取")
|
||||
}
|
||||
var bBody *bytes.Buffer
|
||||
var err error
|
||||
if obj.bar && obj.ContentLength() > 0 { //是否打印进度条,读取内容
|
||||
|
||||
@@ -15,8 +15,9 @@ import (
|
||||
|
||||
"net/http"
|
||||
|
||||
"gitee.com/baixudong/net/http2"
|
||||
"gitee.com/baixudong/tools"
|
||||
"github.com/gospider007/gtls"
|
||||
"github.com/gospider007/net/http2"
|
||||
"github.com/gospider007/tools"
|
||||
utls "github.com/refraction-networking/utls"
|
||||
)
|
||||
|
||||
@@ -491,7 +492,7 @@ func (obj *RoundTripper) GetProxy(ctx context.Context, proxyUrl *url.URL) (*url.
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return VerifyProxy(proxy)
|
||||
return gtls.VerifyProxy(proxy)
|
||||
}
|
||||
|
||||
func (obj *RoundTripper) poolRoundTrip(task *reqTask, key string) (bool, error) {
|
||||
@@ -527,7 +528,7 @@ func (obj *RoundTripper) CloseIdleConnections() {
|
||||
func (obj *RoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
ctxData := req.Context().Value(keyPrincipalID).(*reqCtxData)
|
||||
if ctxData.requestCallBack != nil {
|
||||
if err := ctxData.requestCallBack(req.Context(), req); err != nil {
|
||||
if err := ctxData.requestCallBack(req.Context(), req, nil); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
@@ -539,8 +540,8 @@ func (obj *RoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
if ok, err := obj.poolRoundTrip(task, key); err != nil {
|
||||
return nil, err
|
||||
} else if ok { //is conn multi
|
||||
if ctxData.responseCallBack != nil {
|
||||
if err = ctxData.responseCallBack(task.req.Context(), req, task.res); err != nil {
|
||||
if ctxData.requestCallBack != nil {
|
||||
if err = ctxData.requestCallBack(task.req.Context(), req, task.res); err != nil {
|
||||
task.err = err
|
||||
}
|
||||
}
|
||||
@@ -561,8 +562,8 @@ newConn:
|
||||
if task.isPool() {
|
||||
obj.putConnPool(key, conn)
|
||||
}
|
||||
if ctxData.responseCallBack != nil {
|
||||
if err = ctxData.responseCallBack(task.req.Context(), req, task.res); err != nil {
|
||||
if ctxData.requestCallBack != nil {
|
||||
if err = ctxData.requestCallBack(task.req.Context(), req, task.res); err != nil {
|
||||
task.err = err
|
||||
conn.Close()
|
||||
}
|
||||
|
||||
@@ -1,14 +0,0 @@
|
||||
-----BEGIN CERTIFICATE-----
|
||||
MIICKzCCAdGgAwIBAgIEZCFPpzAKBggqhkjOPQQDAjB5MQswCQYDVQQGEwJDTjER
|
||||
MA8GA1UECBMIU2hhbmdoYWkxETAPBgNVBAcTCFNoYW5naGFpMREwDwYDVQQKEwhH
|
||||
b1NwaWRlcjEWMBQGA1UECxMNR29TcGlkZXJQcm94eTEZMBcGA1UEAxMQR29zcGlk
|
||||
ZXIgUm9vdCBDQTAgFw0yMzAzMTkxNjAwMDBaGA8zMDIzMDMxOTE2MDAwMFoweTEL
|
||||
MAkGA1UEBhMCQ04xETAPBgNVBAgTCFNoYW5naGFpMREwDwYDVQQHEwhTaGFuZ2hh
|
||||
aTERMA8GA1UEChMIR29TcGlkZXIxFjAUBgNVBAsTDUdvU3BpZGVyUHJveHkxGTAX
|
||||
BgNVBAMTEEdvc3BpZGVyIFJvb3QgQ0EwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNC
|
||||
AASvquXLARToWOkMCtyujW+yu9Amq69hYuO0wsEMfDy35LRGcKDKXBYQAlzI4oK/
|
||||
QG/jHTe/aq1sS4ee5tHtJsAdo0UwQzAOBgNVHQ8BAf8EBAMCAQYwEgYDVR0TAQH/
|
||||
BAgwBgEB/wIBATAdBgNVHQ4EFgQUFgJ50m8ArIS37WgP6lL7NqmsPMcwCgYIKoZI
|
||||
zj0EAwIDSAAwRQIgWzRUfg2baj/hFyCJJIbnrwk9H4XfzVEJqXJHMb0dbD4CIQC6
|
||||
JA5hlVq6PW49+/R90EZLn51NhSuRHgW9bmosKUMkZw==
|
||||
-----END CERTIFICATE-----
|
||||
@@ -1,5 +0,0 @@
|
||||
-----BEGIN EC PRIVATE KEY-----
|
||||
MHcCAQEEIEIygwBaoy/fcxW08s9EJMAR2sXaFVNZ2uLrJO53zegvoAoGCCqGSM49
|
||||
AwEHoUQDQgAEr6rlywEU6FjpDArcro1vsrvQJquvYWLjtMLBDHw8t+S0RnCgylwW
|
||||
EAJcyOKCv0Bv4x03v2qtbEuHnubR7SbAHQ==
|
||||
-----END EC PRIVATE KEY-----
|
||||
268
tools.go
268
tools.go
@@ -1,268 +0,0 @@
|
||||
package requests
|
||||
|
||||
import (
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
_ "embed"
|
||||
"encoding/pem"
|
||||
"errors"
|
||||
"math/big"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
//go:embed ssl/gospider.crt
|
||||
var CrtFile []byte
|
||||
|
||||
//go:embed ssl/gospider.key
|
||||
var KeyFile []byte
|
||||
|
||||
func ParseIp(ip net.IP) int {
|
||||
if ip != nil {
|
||||
if ip4 := ip.To4(); ip4 != nil {
|
||||
return 4
|
||||
} else if ip6 := ip.To16(); ip6 != nil {
|
||||
return 6
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
func SplitHostPort(address string) (string, int, error) {
|
||||
host, port, err := net.SplitHostPort(address)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
portnum, err := strconv.Atoi(port)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
if 1 > portnum || portnum > 0xffff {
|
||||
return "", 0, errors.New("port number out of range " + port)
|
||||
}
|
||||
return host, portnum, nil
|
||||
}
|
||||
func ParseHost(host string) (net.IP, int) {
|
||||
if ip := net.ParseIP(host); ip != nil {
|
||||
if ip4 := ip.To4(); ip4 != nil {
|
||||
return ip4, 4
|
||||
} else if ip6 := ip.To16(); ip6 != nil {
|
||||
return ip6, 6
|
||||
}
|
||||
}
|
||||
return nil, 0
|
||||
}
|
||||
|
||||
func GetHost(addrTypes ...int) net.IP {
|
||||
hosts := GetHosts(addrTypes...)
|
||||
if len(hosts) == 0 {
|
||||
return nil
|
||||
} else {
|
||||
return hosts[0]
|
||||
}
|
||||
}
|
||||
func GetHosts(addrTypes ...int) []net.IP {
|
||||
var addrType int
|
||||
if len(addrTypes) > 0 {
|
||||
addrType = addrTypes[0]
|
||||
}
|
||||
result := []net.IP{}
|
||||
lls, err := net.InterfaceAddrs()
|
||||
if err != nil {
|
||||
return result
|
||||
}
|
||||
for _, ll := range lls {
|
||||
mm, ok := ll.(*net.IPNet)
|
||||
if ok && mm.IP.IsPrivate() {
|
||||
if addrType == 0 || ParseIp(mm.IP) == addrType {
|
||||
result = append(result, mm.IP)
|
||||
}
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
func VerifyProxy(proxyUrl string) (*url.URL, error) {
|
||||
proxy, err := url.Parse(proxyUrl)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch proxy.Scheme {
|
||||
case "http", "socks5", "https":
|
||||
return proxy, nil
|
||||
default:
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
func GetContentTypeWithBytes(content []byte) string {
|
||||
return http.DetectContentType(content)
|
||||
}
|
||||
func GetServerName(addr string) string {
|
||||
host, _, err := net.SplitHostPort(addr)
|
||||
if err != nil {
|
||||
return addr
|
||||
}
|
||||
return host
|
||||
}
|
||||
|
||||
// 生成根证书
|
||||
func CreateRootCert(key *ecdsa.PrivateKey) (*x509.Certificate, error) {
|
||||
beforDate, err := time.ParseInLocation(time.DateOnly, "2023-03-20", time.Local)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
afterDate, err := time.ParseInLocation(time.DateOnly, "3023-03-20", time.Local)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rootCsr := &x509.Certificate{
|
||||
Version: 3,
|
||||
SerialNumber: big.NewInt(time.Now().Unix()),
|
||||
Subject: pkix.Name{
|
||||
Country: []string{"CN"},
|
||||
Province: []string{"Shanghai"},
|
||||
Locality: []string{"Shanghai"},
|
||||
Organization: []string{"GoSpider"},
|
||||
OrganizationalUnit: []string{"GoSpiderProxy"},
|
||||
CommonName: "Gospider Root CA",
|
||||
},
|
||||
NotBefore: beforDate,
|
||||
NotAfter: afterDate,
|
||||
BasicConstraintsValid: true,
|
||||
IsCA: true,
|
||||
MaxPathLen: 1,
|
||||
MaxPathLenZero: false,
|
||||
KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign,
|
||||
}
|
||||
rootDer, err := x509.CreateCertificate(rand.Reader, rootCsr, rootCsr, key.Public(), key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return x509.ParseCertificate(rootDer)
|
||||
}
|
||||
|
||||
// 生成私钥
|
||||
func CreateCertKey() (*ecdsa.PrivateKey, error) {
|
||||
return ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
}
|
||||
func GetCertWithCN(rootCert *x509.Certificate, key *ecdsa.PrivateKey, commonName string) (*x509.Certificate, error) {
|
||||
csr := &x509.Certificate{
|
||||
Version: 3,
|
||||
SerialNumber: big.NewInt(time.Now().Unix()),
|
||||
Subject: rootCert.Subject,
|
||||
IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1)},
|
||||
NotBefore: time.Now(),
|
||||
NotAfter: time.Now().AddDate(1000, 0, 0),
|
||||
BasicConstraintsValid: true,
|
||||
IsCA: false,
|
||||
KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
|
||||
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
|
||||
}
|
||||
csr.IPAddresses = []net.IP{}
|
||||
if commonName != "" {
|
||||
if ip, ipType := ParseHost(commonName); ipType == 0 {
|
||||
csr.Subject.CommonName = commonName
|
||||
csr.DNSNames = []string{commonName}
|
||||
} else {
|
||||
csr.IPAddresses = append(csr.IPAddresses, ip)
|
||||
}
|
||||
}
|
||||
der, err := x509.CreateCertificate(rand.Reader, csr, rootCert, key.Public(), key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return x509.ParseCertificate(der)
|
||||
}
|
||||
|
||||
func CreateCertWithCert(rootCert *x509.Certificate, key *ecdsa.PrivateKey, preCert *x509.Certificate) (*x509.Certificate, error) {
|
||||
if preCert.DNSNames == nil && preCert.Subject.CommonName != "" {
|
||||
preCert.DNSNames = []string{preCert.Subject.CommonName}
|
||||
}
|
||||
rootCert.Subject.CommonName = preCert.Subject.CommonName
|
||||
csr := &x509.Certificate{
|
||||
Version: 3,
|
||||
SerialNumber: big.NewInt(time.Now().Unix()),
|
||||
Subject: rootCert.Subject,
|
||||
DNSNames: preCert.DNSNames,
|
||||
IPAddresses: preCert.IPAddresses,
|
||||
NotBefore: time.Now(),
|
||||
NotAfter: time.Now().AddDate(1000, 0, 0),
|
||||
BasicConstraintsValid: true,
|
||||
IsCA: false,
|
||||
KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
|
||||
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
|
||||
}
|
||||
if len(preCert.DNSNames) > 0 {
|
||||
csr.Subject.CommonName = preCert.DNSNames[0]
|
||||
}
|
||||
der, err := x509.CreateCertificate(rand.Reader, csr, rootCert, key.Public(), key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return x509.ParseCertificate(der)
|
||||
}
|
||||
func CreateProxyCertWithName(serverName string) (tlsCert tls.Certificate, err error) {
|
||||
crt, err := LoadCert(CrtFile)
|
||||
if err != nil {
|
||||
return tlsCert, err
|
||||
}
|
||||
key, err := LoadCertKey(KeyFile)
|
||||
if err != nil {
|
||||
return tlsCert, err
|
||||
}
|
||||
cert, err := GetCertWithCN(crt, key, serverName)
|
||||
if err != nil {
|
||||
return tlsCert, err
|
||||
}
|
||||
return GetTlsCert(cert, key)
|
||||
}
|
||||
func CreateProxyCertWithCert(crt *x509.Certificate, key *ecdsa.PrivateKey, preCert *x509.Certificate) (tlsCert tls.Certificate, err error) {
|
||||
if crt == nil {
|
||||
crt, err = LoadCert(CrtFile)
|
||||
if err != nil {
|
||||
return tlsCert, err
|
||||
}
|
||||
}
|
||||
if key == nil {
|
||||
key, err = LoadCertKey(KeyFile)
|
||||
if err != nil {
|
||||
return tlsCert, err
|
||||
}
|
||||
}
|
||||
cert, err := CreateCertWithCert(crt, key, preCert)
|
||||
if err != nil {
|
||||
return tlsCert, err
|
||||
}
|
||||
return GetTlsCert(cert, key)
|
||||
}
|
||||
func GetTlsCert(cert *x509.Certificate, key *ecdsa.PrivateKey) (tls.Certificate, error) {
|
||||
keyFile, err := GetCertKeyData(key)
|
||||
if err != nil {
|
||||
return tls.Certificate{}, err
|
||||
}
|
||||
return tls.X509KeyPair(GetCertData(cert), keyFile)
|
||||
}
|
||||
func GetCertData(cert *x509.Certificate) []byte {
|
||||
return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: cert.Raw})
|
||||
}
|
||||
func GetCertKeyData(key *ecdsa.PrivateKey) ([]byte, error) {
|
||||
keyDer, err := x509.MarshalECPrivateKey(key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDer}), nil
|
||||
}
|
||||
func LoadCertKey(data []byte) (*ecdsa.PrivateKey, error) {
|
||||
block, _ := pem.Decode(data)
|
||||
return x509.ParseECPrivateKey(block.Bytes)
|
||||
}
|
||||
func LoadCert(data []byte) (*x509.Certificate, error) {
|
||||
block, _ := pem.Decode(data)
|
||||
return x509.ParseCertificate(block.Bytes)
|
||||
}
|
||||
Reference in New Issue
Block a user