diff --git a/dataflow/cleanpath.go b/dataflow/cleanpath.go new file mode 100644 index 0000000..03d9bf4 --- /dev/null +++ b/dataflow/cleanpath.go @@ -0,0 +1,47 @@ +package dataflow + +import ( + "path" + "strings" +) + +const ( + httpProto = "http://" + httpsProto = "https://" +) + +func cleanPaths(rv string) string { + if rv == "" { + return rv + } + + if strings.HasPrefix(rv, httpProto) || strings.HasPrefix(rv, httpsProto) { + var proto, domainPath, query string + protoEndIdx := strings.Index(rv, "//") + 2 + queryIdx := strings.Index(rv, "?") + + proto = rv[:protoEndIdx] + if queryIdx != -1 { + domainPath = rv[protoEndIdx:queryIdx] + query = rv[queryIdx:] + } else if rv != proto { + domainPath = rv[protoEndIdx:] + } + if domainPath != "" { + appendSlash := len(domainPath) > 0 && domainPath[len(domainPath)-1] == '/' + domainPath = path.Clean(domainPath) + if appendSlash { + domainPath += "/" + } + } + return proto + domainPath + query + } + + appendSlash := len(rv) > 0 && rv[len(rv)-1] == '/' + cleanPath := path.Clean(rv) + if appendSlash { + return cleanPath + "/" + } + + return cleanPath +} diff --git a/dataflow/cleanpath_test.go b/dataflow/cleanpath_test.go new file mode 100644 index 0000000..0604310 --- /dev/null +++ b/dataflow/cleanpath_test.go @@ -0,0 +1,39 @@ +package dataflow + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +type cleanPathTest struct { + path string + need string +} + +func TestUtil_cleanPaths(t *testing.T) { + test := []cleanPathTest{ + {path: "", need: ""}, + {path: "https://www.aa.com/a", need: "https://www.aa.com/a"}, + {path: "http://www.bb.com/b", need: "http://www.bb.com/b"}, + {path: "www.bb.com/b", need: "www.bb.com/b"}, + {path: "www.bb.com", need: "www.bb.com"}, + {path: "/a", need: "/a"}, + {path: "/a/", need: "/a/"}, + {path: "http://", need: "http://"}, + {path: "https://", need: "https://"}, + {path: "http://www.aa.com/urls?", need: "http://www.aa.com/urls?"}, + {path: "https://www.aa.com/urls?bb", need: "https://www.aa.com/urls?bb"}, + {path: "http://www.aa.com/urls?site=https://bb.com", need: "http://www.aa.com/urls?site=https://bb.com"}, + {path: "https://www.aa.com/urls?site=http://bb.com", need: "https://www.aa.com/urls?site=http://bb.com"}, + {path: "http://www.aa.com/urls/./a?site=https://bb.com", need: "http://www.aa.com/urls/a?site=https://bb.com"}, + {path: "https://www.aa.com/urls/../a?site=https://bb.com", need: "https://www.aa.com/a?site=https://bb.com"}, + {path: "https://api.map.baidu.com/weather/v1/?district_id=310100&data_type=all&ak=ffyu0pP8P6Ao0KYr8FTZwDgsOFiA1oYA", need: "https://api.map.baidu.com/weather/v1/?district_id=310100&data_type=all&ak=ffyu0pP8P6Ao0KYr8FTZwDgsOFiA1oYA"}, + } + + for index, v := range test { + rv := cleanPaths(v.path) + assert.Equal(t, v.need, rv, fmt.Sprintf("index:%d", index)) + } +} diff --git a/dataflow/dataflow.go b/dataflow/dataflow.go index d749e4f..32a2158 100644 --- a/dataflow/dataflow.go +++ b/dataflow/dataflow.go @@ -35,43 +35,43 @@ type DataFlow struct { // GET send HTTP GET method func (df *DataFlow) GET(url string) *DataFlow { - df.Req = reqDef(get, joinPaths("", url), df.out) + df.Req = reqDef(get, cleanPaths(url), df.out) return df } // POST send HTTP POST method func (df *DataFlow) POST(url string) *DataFlow { - df.Req = reqDef(post, joinPaths("", url), df.out) + df.Req = reqDef(post, cleanPaths(url), df.out) return df } // PUT send HTTP PUT method func (df *DataFlow) PUT(url string) *DataFlow { - df.Req = reqDef(put, joinPaths("", url), df.out) + df.Req = reqDef(put, cleanPaths(url), df.out) return df } // DELETE send HTTP DELETE method func (df *DataFlow) DELETE(url string) *DataFlow { - df.Req = reqDef(delete2, joinPaths("", url), df.out) + df.Req = reqDef(delete2, cleanPaths(url), df.out) return df } // PATCH send HTTP PATCH method func (df *DataFlow) PATCH(url string) *DataFlow { - df.Req = reqDef(patch, joinPaths("", url), df.out) + df.Req = reqDef(patch, cleanPaths(url), df.out) return df } // HEAD send HTTP HEAD method func (df *DataFlow) HEAD(url string) *DataFlow { - df.Req = reqDef(head, joinPaths("", url), df.out) + df.Req = reqDef(head, cleanPaths(url), df.out) return df } // OPTIONS send HTTP OPTIONS method func (df *DataFlow) OPTIONS(url string) *DataFlow { - df.Req = reqDef(options, joinPaths("", url), df.out) + df.Req = reqDef(options, cleanPaths(url), df.out) return df } @@ -132,11 +132,11 @@ func (df *DataFlow) SetURL(url string) *DataFlow { } if df.Req.url == "" && df.Req.req == nil && df.Req.method == "" { - df.Req = reqDef("", joinPaths("", url), df.out) + df.Req = reqDef("", cleanPaths(url), df.out) return df } - df.Req.url = modifyURL(joinPaths("", url)) + df.Req.url = modifyURL(cleanPaths(url)) return df } diff --git a/dataflow/req.go b/dataflow/req.go index 6d82a8b..0ecfa36 100644 --- a/dataflow/req.go +++ b/dataflow/req.go @@ -163,7 +163,7 @@ func (r *Req) selectRequest(body *bytes.Buffer) (req *http.Request, err error) { } if len(r.host) > 0 { - urlStr := modifyURL(joinPaths("", r.host)) + urlStr := modifyURL(cleanPaths(r.host)) URL, err := url.Parse(urlStr) if err != nil { r.Err = err diff --git a/dataflow/utils.go b/dataflow/utils.go deleted file mode 100644 index 7f10e82..0000000 --- a/dataflow/utils.go +++ /dev/null @@ -1,64 +0,0 @@ -package dataflow - -import ( - "path" - "strings" -) - -const ( - httpProto = "http://" - httpsProto = "https://" -) - -func lastChar(str string) uint8 { - if str == "" { - panic("The length of the string can't be 0") - } - return str[len(str)-1] -} - -func join(elem ...string) (rv string) { - - defer func() { - if strings.HasPrefix(rv, httpProto) || strings.HasPrefix(rv, httpsProto) { - var proto, domainPath, query string - protoEndIdx := strings.Index(rv, "//") + 2 - queryIdx := strings.Index(rv, "?") - - proto = rv[:protoEndIdx] - if queryIdx != -1 { - domainPath = rv[protoEndIdx:queryIdx] - query = rv[queryIdx:] - } else if rv != proto { - domainPath = rv[protoEndIdx:] - } - if domainPath != "" { - domainPath = path.Clean(domainPath) - } - rv = proto + domainPath + query - return - } - - rv = path.Clean(rv) - }() - - for i, e := range elem { - if e != "" { - return strings.Join(elem[i:], "/") - } - } - return "" -} - -func joinPaths(absolutePath, relativePath string) string { - if relativePath == "" { - return absolutePath - } - - finalPath := join(absolutePath, relativePath) - appendSlash := lastChar(relativePath) == '/' && lastChar(finalPath) != '/' - if appendSlash { - return finalPath + "/" - } - return finalPath -} diff --git a/dataflow/utils_test.go b/dataflow/utils_test.go deleted file mode 100644 index 17612ae..0000000 --- a/dataflow/utils_test.go +++ /dev/null @@ -1,53 +0,0 @@ -package dataflow - -import ( - "github.com/stretchr/testify/assert" - "testing" -) - -func TestUtil_Join(t *testing.T) { - urls := []string{ - "http://127.0.0.1:43471/v1", - } - - want := []string{ - "http://127.0.0.1:43471/v1", - } - - assert.Equal(t, joinPaths("", urls[0]), want[0]) -} - -type joinTest struct { - absolutePath string - relativePath string - need string -} - -func TestUtil_join(t *testing.T) { - //实际调用不会发生 join函数absolutePath和relativePath参数都为空的情况 - assert.Equal(t, join("", ""), ".") -} - -func TestUtil_joinPaths(t *testing.T) { - test := []joinTest{ - {absolutePath: "", relativePath: "", need: ""}, - {absolutePath: "https://www.aa.com", relativePath: "/a", need: "https://www.aa.com/a"}, - {absolutePath: "http://www.bb.com", relativePath: "/b", need: "http://www.bb.com/b"}, - {absolutePath: "www.bb.com", relativePath: "/b", need: "www.bb.com/b"}, - {absolutePath: "www.bb.com", relativePath: "", need: "www.bb.com"}, - {absolutePath: "", relativePath: "/a", need: "/a"}, - {absolutePath: "", relativePath: "http://", need: "http://"}, - {absolutePath: "", relativePath: "https://", need: "https://"}, - {absolutePath: "", relativePath: "http://www.aa.com/urls?", need: "http://www.aa.com/urls?"}, - {absolutePath: "", relativePath: "https://www.aa.com/urls?bb", need: "https://www.aa.com/urls?bb"}, - {absolutePath: "", relativePath: "http://www.aa.com/urls?site=https://bb.com", need: "http://www.aa.com/urls?site=https://bb.com"}, - {absolutePath: "", relativePath: "https://www.aa.com/urls?site=http://bb.com", need: "https://www.aa.com/urls?site=http://bb.com"}, - {absolutePath: "", relativePath: "http://www.aa.com/urls/./a?site=https://bb.com", need: "http://www.aa.com/urls/a?site=https://bb.com"}, - {absolutePath: "", relativePath: "https://www.aa.com/urls/../a?site=https://bb.com", need: "https://www.aa.com/a?site=https://bb.com"}, - } - - for _, v := range test { - rv := joinPaths(v.absolutePath, v.relativePath) - assert.Equal(t, v.need, rv) - } -}