新增文件上传解析、文件下载解析插件

This commit is contained in:
Liujian
2023-11-01 11:04:25 +08:00
parent 6b30f51ddd
commit 62aaeda4ed
19 changed files with 580 additions and 18 deletions

View File

@@ -2,6 +2,7 @@ package main
import (
"github.com/eolinker/apinto/drivers/plugins/app"
body_record_truncation "github.com/eolinker/apinto/drivers/plugins/body-record-truncation"
"github.com/eolinker/apinto/drivers/plugins/cors"
data_transform "github.com/eolinker/apinto/drivers/plugins/data-transform"
dubbo2_proxy_rewrite "github.com/eolinker/apinto/drivers/plugins/dubbo2-proxy-rewrite"
@@ -10,7 +11,9 @@ import (
"github.com/eolinker/apinto/drivers/plugins/gzip"
params_check "github.com/eolinker/apinto/drivers/plugins/params-check"
"github.com/eolinker/apinto/drivers/plugins/prometheus"
request_file_parse "github.com/eolinker/apinto/drivers/plugins/request-file-parse"
request_interception "github.com/eolinker/apinto/drivers/plugins/request-interception"
response_file_parse "github.com/eolinker/apinto/drivers/plugins/response-file-parse"
response_filter "github.com/eolinker/apinto/drivers/plugins/response-filter"
response_rewrite_v2 "github.com/eolinker/apinto/drivers/plugins/response-rewrite_v2"
@@ -71,12 +74,14 @@ func pluginRegister(extenderRegister eosc.IExtenderDriverRegister) {
params_check.Register(extenderRegister)
data_transform.Register(extenderRegister)
request_interception.Register(extenderRegister)
request_file_parse.Register(extenderRegister)
// 响应处理插件
response_rewrite.Register(extenderRegister)
response_rewrite_v2.Register(extenderRegister)
response_filter.Register(extenderRegister)
gzip.Register(extenderRegister)
response_file_parse.Register(extenderRegister)
// 安全相关插件
ip_restriction.Register(extenderRegister)
@@ -90,6 +95,7 @@ func pluginRegister(extenderRegister eosc.IExtenderDriverRegister) {
prometheus.Register(extenderRegister)
monitor.Register(extenderRegister)
proxy_mirror.Register(extenderRegister)
body_record_truncation.Register(extenderRegister)
// 计数插件
counter.Register(extenderRegister)

View File

@@ -11,5 +11,11 @@ type Config struct {
Period string `json:"period" yaml:"period" enum:"hour,day" label:"日志分割周期"`
Expire int `json:"expire" yaml:"expire" label:"日志保存时间" description:"单位:天" default:"7" minimum:"1"`
Type string `json:"type" yaml:"type" enum:"json,line" label:"输出格式"`
//BodyConfig BodyConfig `json:"body_config" yaml:"body_config" label:"请求体/响应体配置" description:"请求体/响应体配置" switch:"type===json"`
Formatter eosc.FormatterConfig `json:"formatter" yaml:"formatter" label:"格式化配置"`
}
//type BodyConfig struct {
// BodySize int `json:"body_size" label:"请求体/响应体截取长度" description:"单位M" default:"10" minimum:"0"`
// BodyCode string `json:"body_code" label:"请求体/响应体编码" enum:"latin,utf8,gbk" default:"utf8"`
//}

View File

@@ -2,11 +2,12 @@ package fileoutput
import (
"fmt"
"net/http"
"github.com/eolinker/eosc"
"github.com/eolinker/eosc/formatter"
"github.com/eolinker/eosc/log/filelog"
"github.com/eolinker/eosc/router"
"net/http"
"time"
)
@@ -15,7 +16,6 @@ type FileWriter struct {
formatter eosc.IFormatter
transport *filelog.FileWriterByPeriod
//id string
fileHandler http.Handler
}
@@ -37,7 +37,10 @@ func (a *FileWriter) reset(cfg *Config, name string) (err error) {
if !has {
return errorFormatterType
}
//var extendCfg []byte
//if cfg.Type == "json" {
// extendCfg, _ = json.Marshal(cfg.BodyConfig)
//}
fm, err := factory.Create(cfg.Formatter)
if err != nil {
return err

View File

@@ -73,6 +73,7 @@ func create(config *Config) (formatter.ITransport, eosc.IFormatter, error) {
if !has {
return nil, nil, errFormatterType
}
fm, err := factory.Create(config.Formatter)
if err != nil {
return nil, nil, err

View File

@@ -32,6 +32,7 @@ func (o *tProducer) reset(cfg *ProducerConfig) (err error) {
if !has {
return errorFormatterType
}
o.formatter, err = factory.Create(cfg.Formatter)
if o.producer != nil {

View File

@@ -54,6 +54,7 @@ func (n *Writer) reset(config *Config) error {
if !has {
return errFormatterType
}
fm, err := factory.Create(config.Formatter)
if err != nil {
return err

View File

@@ -30,7 +30,8 @@ func (b *BodyCheck) DoHttpFilter(ctx http_service.IHttpContext, next eocontext.I
if err != nil {
return err
}
bodySize := len([]rune(string(body)))
// 计算body大小
bodySize := len(body)
if !b.isEmpty && bodySize < 1 {
ctx.Response().SetStatus(400, "400")
ctx.Response().SetBody([]byte("Body is required"))

View File

@@ -0,0 +1,5 @@
package body_record_truncation
type Config struct {
BodySize int64 `json:"body_size" label:"截断大小"`
}

View File

@@ -0,0 +1,73 @@
package body_record_truncation
import (
"github.com/eolinker/apinto/drivers"
"github.com/eolinker/eosc"
"github.com/eolinker/eosc/eocontext"
http_service "github.com/eolinker/eosc/eocontext/http-context"
)
var ()
var _ http_service.HttpFilter = (*executor)(nil)
var _ eocontext.IFilter = (*executor)(nil)
var _ eosc.IWorker = (*executor)(nil)
type executor struct {
drivers.WorkerBase
bodySize int64
}
func (e *executor) DoFilter(ctx eocontext.EoContext, next eocontext.IChain) (err error) {
return http_service.DoHttpFilter(e, ctx, next)
}
func (e *executor) DoHttpFilter(ctx http_service.IHttpContext, next eocontext.IChain) (err error) {
if ctx.Request().Method() == "POST" || ctx.Request().Method() == "PUT" || ctx.Request().Method() == "PATCH" {
if e.bodySize != 0 && int64(ctx.Request().ContentLength()) > e.bodySize {
// 当请求体大小大于限制时,截断请求体
entry := ctx.GetEntry()
body := entry.Read("ctx_request_body")
v, _ := body.(string)
ctx.SetLabel("request_body", v[:e.bodySize])
ctx.WithValue("request_body_complete", 0)
} else {
ctx.WithValue("request_body_complete", 1)
}
}
if next != nil {
err = next.DoChain(ctx)
}
if e.bodySize != 0 && int64(ctx.Response().ContentLength()) > e.bodySize {
// 当响应体大小大于限制时,截断响应体
entry := ctx.GetEntry()
body := entry.Read("ctx_response_body")
v, _ := body.(string)
ctx.SetLabel("response_body", v[:e.bodySize])
ctx.WithValue("response_body_complete", 0)
} else {
ctx.WithValue("response_body_complete", 1)
}
return err
}
func (e *executor) Destroy() {
return
}
func (e *executor) Start() error {
return nil
}
func (e *executor) Reset(conf interface{}, workers map[eosc.RequireId]eosc.IWorker) error {
return nil
}
func (e *executor) Stop() error {
e.Destroy()
return nil
}
func (e *executor) CheckSkill(skill string) bool {
return http_service.FilterSkillName == skill
}

View File

@@ -0,0 +1,26 @@
package body_record_truncation
import (
"github.com/eolinker/apinto/drivers"
"github.com/eolinker/eosc"
)
const (
Name = "body-record-truncation"
)
func Register(register eosc.IExtenderDriverRegister) {
register.RegisterExtenderDriver(Name, NewFactory())
}
func NewFactory() eosc.IExtenderDriverFactory {
return drivers.NewFactory[Config](Create)
}
func Create(id, name string, conf *Config, workers map[eosc.RequireId]eosc.IWorker) (eosc.IWorker, error) {
return &executor{
WorkerBase: drivers.Worker(id, name),
bodySize: conf.BodySize << 20,
}, nil
}

View File

@@ -0,0 +1,8 @@
package request_file_parse
type Config struct {
FileKey string `json:"file_key" label:"文件Key"`
FileSuffix []string `json:"file_suffix" label:"文件有效后缀列表"`
LargeWarn int64 `json:"large_warn" label:"文件大小警告阈值"`
LargeWarnText string `json:"large_warn_text" label:"文件大小警告标签值"`
}

View File

@@ -0,0 +1,158 @@
package request_file_parse
import (
"errors"
"io"
"mime"
"mime/multipart"
"strings"
"golang.org/x/text/encoding/charmap"
"github.com/eolinker/eosc/log"
"github.com/eolinker/apinto/drivers"
"github.com/eolinker/eosc"
"github.com/eolinker/eosc/eocontext"
http_service "github.com/eolinker/eosc/eocontext/http-context"
)
var (
MultipartForm = "multipart/form-data"
//csv,tar,bz2,xz,jar,pdf,doc,docx,xls,ppt,xlsx,pptx,zip,txt,rar,gz,dot
defaultValidSuf = map[string]struct{}{
"csv": {},
"tar": {},
"bz2": {},
"xz": {},
"jar": {},
"pdf": {},
"doc": {},
"docx": {},
"xls": {},
"ppt": {},
"xlsx": {},
"pptx": {},
"zip": {},
"txt": {},
"rar": {},
"gz": {},
"dot": {},
}
)
var _ http_service.HttpFilter = (*executor)(nil)
var _ eocontext.IFilter = (*executor)(nil)
var _ eosc.IWorker = (*executor)(nil)
type executor struct {
drivers.WorkerBase
fileKey string
validSuf map[string]struct{}
largeWarn int64
largeWarnStr string
}
func (e *executor) DoFilter(ctx eocontext.EoContext, next eocontext.IChain) (err error) {
return http_service.DoHttpFilter(e, ctx, next)
}
func (e *executor) DoHttpFilter(ctx http_service.IHttpContext, next eocontext.IChain) (err error) {
if ctx.Request().Method() == "POST" || ctx.Request().Method() == "PUT" || ctx.Request().Method() == "PATCH" {
contentType, _, err := mime.ParseMediaType(ctx.Request().ContentType())
if err != nil {
return err
}
if contentType == MultipartForm {
// 当请求为文件请求时,解析文件
fh, has := ctx.Request().Body().GetFile(e.fileKey)
if has {
for _, h := range fh {
suffix, err := getFileSuffix(h)
if err != nil {
log.Errorf("get file suffix error: %v,name is %s", err, e.fileKey)
continue
}
if _, ok := e.validSuf[suffix]; !ok {
log.Errorf("file suffix is not valid,name is %s,suffix is %s", e.fileKey, suffix)
continue
}
f, err := h.Open()
if err != nil {
log.Errorf("file open error: %v,name is %s", err, e.fileKey)
continue
}
body, err := io.ReadAll(f)
if err != nil {
log.Errorf("read file body error: %v,name is %s", err, e.fileKey)
f.Close()
continue
}
f.Close()
// body此处要做latin1编码
out := make([]byte, 0, len(body))
for _, t := range body {
if v, ok := charmap.ISO8859_1.EncodeRune(rune(t)); ok {
out = append(out, v)
}
}
//if e.largeWarn > 0 && h.Size > e.largeWarn {
// ctx.WithValue("file_size_warn", e.largeWarnStr)
// out = out[:e.largeWarn]
// ctx.WithValue("request_body_complete", 0)
//}
ctx.SetLabel("request_body", string(out))
ctx.SetLabel("file_direction", "upload")
ctx.SetLabel("file_name", h.Filename)
ctx.SetLabel("file_suffix", suffix)
ctx.WithValue("file_size", h.Size)
break
}
}
}
}
if next != nil {
return next.DoChain(ctx)
}
return nil
}
func (e *executor) Destroy() {
return
}
func (e *executor) Start() error {
return nil
}
func (e *executor) Reset(conf interface{}, workers map[eosc.RequireId]eosc.IWorker) error {
return nil
}
func (e *executor) Stop() error {
e.Destroy()
return nil
}
func (e *executor) CheckSkill(skill string) bool {
return http_service.FilterSkillName == skill
}
func getFileSuffix(f *multipart.FileHeader) (string, error) {
// 获取文件后缀
fileName := f.Filename
// 获取文件后缀
suffix := fileName[strings.LastIndex(fileName, ".")+1:]
if len(suffix) == 0 {
contentType := f.Header.Get("Content-Type")
if len(contentType) == 0 {
return "", errors.New("file suffix is empty")
}
}
return suffix, nil
}

View File

@@ -0,0 +1,39 @@
package request_file_parse
import (
"github.com/eolinker/apinto/drivers"
"github.com/eolinker/eosc"
)
const (
Name = "request_file_parse"
)
func Register(register eosc.IExtenderDriverRegister) {
register.RegisterExtenderDriver(Name, NewFactory())
}
func NewFactory() eosc.IExtenderDriverFactory {
return drivers.NewFactory[Config](Create)
}
func Create(id, name string, conf *Config, workers map[eosc.RequireId]eosc.IWorker) (eosc.IWorker, error) {
largeWarnText := "large"
if conf.LargeWarnText != "" {
largeWarnText = conf.LargeWarnText
}
validSuffix := make(map[string]struct{})
for key := range defaultValidSuf {
validSuffix[key] = struct{}{}
}
for _, s := range conf.FileSuffix {
validSuffix[s] = struct{}{}
}
return &executor{
WorkerBase: drivers.Worker(id, name),
fileKey: conf.FileKey,
validSuf: validSuffix,
largeWarn: conf.LargeWarn << 20,
largeWarnStr: largeWarnText,
}, nil
}

View File

@@ -0,0 +1,8 @@
package response_file_parse
type Config struct {
FileKey string `json:"file_key" label:"文件Key"`
FileSuffix []string `json:"file_suffix" label:"文件有效后缀列表"`
LargeWarn int64 `json:"large_warn" label:"文件大小警告阈值"`
LargeWarnText string `json:"large_warn_text" label:"文件大小警告标签值"`
}

View File

@@ -0,0 +1,130 @@
package response_file_parse
import (
"strings"
"golang.org/x/text/encoding/charmap"
"github.com/eolinker/eosc/log"
"github.com/eolinker/apinto/drivers"
"github.com/eolinker/eosc"
"github.com/eolinker/eosc/eocontext"
http_service "github.com/eolinker/eosc/eocontext/http-context"
)
var (
//csv,tar,bz2,xz,jar,pdf,doc,docx,xls,ppt,xlsx,pptx,zip,txt,rar,gz,dot
defaultValidSuf = map[string]struct{}{
"csv": {},
"tar": {},
"bz2": {},
"xz": {},
"jar": {},
"pdf": {},
"doc": {},
"docx": {},
"xls": {},
"ppt": {},
"xlsx": {},
"pptx": {},
"zip": {},
"txt": {},
"rar": {},
"gz": {},
"dot": {},
}
)
var _ http_service.HttpFilter = (*executor)(nil)
var _ eocontext.IFilter = (*executor)(nil)
var _ eosc.IWorker = (*executor)(nil)
type executor struct {
drivers.WorkerBase
fileKey string
validSuf map[string]struct{}
largeWarn int64
largeWarnStr string
}
func (e *executor) DoFilter(ctx eocontext.EoContext, next eocontext.IChain) (err error) {
return http_service.DoHttpFilter(e, ctx, next)
}
func (e *executor) DoHttpFilter(ctx http_service.IHttpContext, next eocontext.IChain) (err error) {
if next != nil {
err = next.DoChain(ctx)
if err != nil {
return err
}
}
contentDisposition := ctx.Response().Headers().Get("Content-Disposition")
if contentDisposition != "" {
params := strings.Split(contentDisposition, ";")
paramsMap := make(map[string]string, len(params))
for _, param := range params {
param = strings.TrimSpace(param)
ps := strings.Split(param, "=")
if ps[0] != "" {
if len(ps) > 1 {
paramsMap[ps[0]] = ps[1]
} else {
paramsMap[ps[0]] = ""
}
}
}
if err != nil {
log.Errorf("parse content disposition error: %v", err)
return nil
}
if fileName, ok := paramsMap[e.fileKey]; ok {
if fileName != "" {
suffix := fileName[strings.LastIndex(fileName, ".")+1:]
if _, ok := e.validSuf[suffix]; !ok {
log.Errorf("file suffix is not valid,name is %s,suffix is %s", e.fileKey, suffix)
return nil
}
body := ctx.Response().GetBody()
// body此处要做latin1编码
out := make([]byte, 0, len(body))
for _, t := range body {
if v, ok := charmap.ISO8859_1.EncodeRune(rune(t)); ok {
out = append(out, v)
}
}
size := len(out)
ctx.WithValue("response_body", string(out))
ctx.WithValue("file_direction", "download")
ctx.WithValue("file_name", fileName)
ctx.WithValue("file_suffix", suffix)
ctx.WithValue("file_size", size)
if int64(size) > e.largeWarn {
ctx.WithValue("file_large_warn", e.largeWarnStr)
}
}
}
}
return nil
}
func (e *executor) Destroy() {
return
}
func (e *executor) Start() error {
return nil
}
func (e *executor) Reset(conf interface{}, workers map[eosc.RequireId]eosc.IWorker) error {
return nil
}
func (e *executor) Stop() error {
e.Destroy()
return nil
}
func (e *executor) CheckSkill(skill string) bool {
return http_service.FilterSkillName == skill
}

View File

@@ -0,0 +1,39 @@
package response_file_parse
import (
"github.com/eolinker/apinto/drivers"
"github.com/eolinker/eosc"
)
const (
Name = "response_file_parse"
)
func Register(register eosc.IExtenderDriverRegister) {
register.RegisterExtenderDriver(Name, NewFactory())
}
func NewFactory() eosc.IExtenderDriverFactory {
return drivers.NewFactory[Config](Create)
}
func Create(id, name string, conf *Config, workers map[eosc.RequireId]eosc.IWorker) (eosc.IWorker, error) {
largeWarnText := "large"
if conf.LargeWarnText != "" {
largeWarnText = conf.LargeWarnText
}
validSuffix := make(map[string]struct{})
for key := range defaultValidSuf {
validSuffix[key] = struct{}{}
}
for _, s := range conf.FileSuffix {
validSuffix[s] = struct{}{}
}
return &executor{
WorkerBase: drivers.Worker(id, name),
fileKey: conf.FileKey,
validSuf: validSuffix,
largeWarn: conf.LargeWarn << 20,
largeWarnStr: largeWarnText,
}, nil
}

View File

@@ -26,6 +26,7 @@ func (e *Entry) Read(pattern string) interface{} {
if !ok {
return ""
}
return v
}

View File

@@ -47,7 +47,31 @@ func (f Fields) Read(name string, ctx http_service.IHttpContext) (interface{}, b
return label, label != ""
}
type CtxRule struct {
fields Fields
}
func (l *CtxRule) Read(name string, ctx http_service.IHttpContext) (interface{}, bool) {
value := ctx.Value(name)
if value != nil {
return value, true
}
// 先从Label中获取值
value = ctx.GetLabel(name)
if value != "" {
return value, true
}
return l.fields.Read(name, ctx)
}
func init() {
ctxRule = &CtxRule{
fields: rule,
}
}
var (
ctxRule *CtxRule
rule Fields = map[string]IReader{
"request_id": ReadFunc(func(name string, ctx http_service.IHttpContext) (interface{}, bool) {
return ctx.RequestId(), true
@@ -58,15 +82,18 @@ var (
"cluster": ReadFunc(func(name string, ctx http_service.IHttpContext) (interface{}, bool) {
return os.Getenv("cluster_id"), true
}),
"api_id": ReadFunc(func(name string, ctx http_service.IHttpContext) (interface{}, bool) {
return ctx.GetLabel("api_id"), true
}),
"query": ReadFunc(func(name string, ctx http_service.IHttpContext) (interface{}, bool) {
if name == "" {
return utils.QueryUrlEncode(ctx.Request().URI().RawQuery()), true
}
return url.QueryEscape(ctx.Request().URI().GetQuery(name)), true
}),
"src_ip": ReadFunc(func(name string, ctx http_service.IHttpContext) (interface{}, bool) {
return ctx.Request().RealIp(), true
}),
"src_port": ReadFunc(func(name string, ctx http_service.IHttpContext) (interface{}, bool) {
return ctx.Request().RemotePort(), true
}),
"uri": ReadFunc(func(name string, ctx http_service.IHttpContext) (interface{}, bool) {
//不带请求参数的uri
return ctx.Request().URI().Path(), true
@@ -95,6 +122,10 @@ var (
"remote_port": ReadFunc(func(name string, ctx http_service.IHttpContext) (interface{}, bool) {
return ctx.Request().RemotePort(), true
}),
"ctx": ReadFunc(func(name string, ctx http_service.IHttpContext) (interface{}, bool) {
return ctxRule.Read(name, ctx)
}),
"request": Fields{
"body": ReadFunc(func(name string, ctx http_service.IHttpContext) (interface{}, bool) {
@@ -104,6 +135,17 @@ var (
}
return string(body), true
}),
"body_filter": ReadFunc(func(name string, ctx http_service.IHttpContext) (interface{}, bool) {
value := ctx.GetLabel("xxx")
if value == "" {
body, err := ctx.Request().Body().RawBody()
if err != nil {
return "", false
}
return string(body), true
}
return ctx.GetLabel("xxx"), true
}),
"length": ReadFunc(func(name string, ctx http_service.IHttpContext) (interface{}, bool) {
return ctx.Request().ContentLength(), true
@@ -139,12 +181,18 @@ var (
//return time.Now().Format("2006-01-02 15:04:05"), true
return ctx.AcceptTime().Format("2006-01-02 15:04:05"), true
}),
"timestamp": ReadFunc(func(name string, ctx http_service.IHttpContext) (interface{}, bool) {
return ctx.AcceptTime().Unix(), true
}),
"header": ReadFunc(func(name string, ctx http_service.IHttpContext) (interface{}, bool) {
if name == "" {
return url.Values(ctx.Request().Header().Headers()).Encode(), true
}
return ctx.Request().Header().GetHeader(strings.Replace(name, "_", "-", -1)), true
}),
"headers": ReadFunc(func(name string, ctx http_service.IHttpContext) (interface{}, bool) {
return ctx.Request().Header().Headers(), true
}),
"http": ReadFunc(func(name string, ctx http_service.IHttpContext) (interface{}, bool) {
return ctx.Request().Header().GetHeader(strings.Replace(name, "_", "-", -1)), true
}),
@@ -160,15 +208,20 @@ var (
"": ReadFunc(func(name string, ctx http_service.IHttpContext) (interface{}, bool) {
return ctx.Response().String(), true
}),
"body": ReadFunc(func(name string, ctx http_service.IHttpContext) (interface{}, bool) {
"body": Fields{
"": ReadFunc(func(name string, ctx http_service.IHttpContext) (interface{}, bool) {
return string(ctx.Response().GetBody()), true
}),
},
"header": ReadFunc(func(name string, ctx http_service.IHttpContext) (interface{}, bool) {
if name == "" {
return url.Values(ctx.Response().Headers()).Encode(), true
}
return ctx.Response().GetHeader(strings.Replace(name, "_", "-", -1)), true
}),
"headers": ReadFunc(func(name string, ctx http_service.IHttpContext) (interface{}, bool) {
return ctx.Response().Headers(), true
}),
"status": ReadFunc(func(name string, ctx http_service.IHttpContext) (interface{}, bool) {
return ctx.Response().ProxyStatus(), true
}),
@@ -179,6 +232,9 @@ var (
return strconv.Itoa(ctx.Response().ContentLength()), true
}),
},
"set_cookies": ReadFunc(func(name string, ctx http_service.IHttpContext) (interface{}, bool) {
return strings.Split(ctx.Response().GetHeader("Set-Cookie"), "; "), true
}),
"proxy": proxyFields,
}

2
go.mod
View File

@@ -167,4 +167,4 @@ require (
gopkg.in/yaml.v2 v2.4.0 // indirect
)
//replace github.com/eolinker/eosc => ../eosc
replace github.com/eolinker/eosc => ../eosc