fix-316 (#320)

* bugfix * 优化清理路径的函数
2025-12-24 12:58:00 +08:00 · 2021-09-18 21:18:02 +08:00
parent d943327e53
commit 891749bb41
6 changed files with 96 additions and 127 deletions
--- a/dataflow/cleanpath.go
+++ b/dataflow/cleanpath.go
@@ -0,0 +1,47 @@
+package dataflow
+
+import (
+	"path"
+	"strings"
+)
+
+const (
+	httpProto  = "http://"
+	httpsProto = "https://"
+)
+
+func cleanPaths(rv string) string {
+	if rv == "" {
+		return rv
+	}
+
+	if strings.HasPrefix(rv, httpProto) || strings.HasPrefix(rv, httpsProto) {
+		var proto, domainPath, query string
+		protoEndIdx := strings.Index(rv, "//") + 2
+		queryIdx := strings.Index(rv, "?")
+
+		proto = rv[:protoEndIdx]
+		if queryIdx != -1 {
+			domainPath = rv[protoEndIdx:queryIdx]
+			query = rv[queryIdx:]
+		} else if rv != proto {
+			domainPath = rv[protoEndIdx:]
+		}
+		if domainPath != "" {
+			appendSlash := len(domainPath) > 0 && domainPath[len(domainPath)-1] == '/'
+			domainPath = path.Clean(domainPath)
+			if appendSlash {
+				domainPath += "/"
+			}
+		}
+		return proto + domainPath + query
+	}
+
+	appendSlash := len(rv) > 0 && rv[len(rv)-1] == '/'
+	cleanPath := path.Clean(rv)
+	if appendSlash {
+		return cleanPath + "/"
+	}
+
+	return cleanPath
+}
--- a/dataflow/cleanpath_test.go
+++ b/dataflow/cleanpath_test.go
@@ -0,0 +1,39 @@
+package dataflow
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+type cleanPathTest struct {
+	path string
+	need string
+}
+
+func TestUtil_cleanPaths(t *testing.T) {
+	test := []cleanPathTest{
+		{path: "", need: ""},
+		{path: "https://www.aa.com/a", need: "https://www.aa.com/a"},
+		{path: "http://www.bb.com/b", need: "http://www.bb.com/b"},
+		{path: "www.bb.com/b", need: "www.bb.com/b"},
+		{path: "www.bb.com", need: "www.bb.com"},
+		{path: "/a", need: "/a"},
+		{path: "/a/", need: "/a/"},
+		{path: "http://", need: "http://"},
+		{path: "https://", need: "https://"},
+		{path: "http://www.aa.com/urls?", need: "http://www.aa.com/urls?"},
+		{path: "https://www.aa.com/urls?bb", need: "https://www.aa.com/urls?bb"},
+		{path: "http://www.aa.com/urls?site=https://bb.com", need: "http://www.aa.com/urls?site=https://bb.com"},
+		{path: "https://www.aa.com/urls?site=http://bb.com", need: "https://www.aa.com/urls?site=http://bb.com"},
+		{path: "http://www.aa.com/urls/./a?site=https://bb.com", need: "http://www.aa.com/urls/a?site=https://bb.com"},
+		{path: "https://www.aa.com/urls/../a?site=https://bb.com", need: "https://www.aa.com/a?site=https://bb.com"},
+		{path: "https://api.map.baidu.com/weather/v1/?district_id=310100&data_type=all&ak=ffyu0pP8P6Ao0KYr8FTZwDgsOFiA1oYA", need: "https://api.map.baidu.com/weather/v1/?district_id=310100&data_type=all&ak=ffyu0pP8P6Ao0KYr8FTZwDgsOFiA1oYA"},
+	}
+
+	for index, v := range test {
+		rv := cleanPaths(v.path)
+		assert.Equal(t, v.need, rv, fmt.Sprintf("index:%d", index))
+	}
+}
--- a/dataflow/dataflow.go
+++ b/dataflow/dataflow.go
@@ -35,43 +35,43 @@ type DataFlow struct {

 // GET send HTTP GET method
 func (df *DataFlow) GET(url string) *DataFlow {
-	df.Req = reqDef(get, joinPaths("", url), df.out)
+	df.Req = reqDef(get, cleanPaths(url), df.out)
 	return df
 }

 // POST send HTTP POST method
 func (df *DataFlow) POST(url string) *DataFlow {
-	df.Req = reqDef(post, joinPaths("", url), df.out)
+	df.Req = reqDef(post, cleanPaths(url), df.out)
 	return df
 }

 // PUT send HTTP PUT method
 func (df *DataFlow) PUT(url string) *DataFlow {
-	df.Req = reqDef(put, joinPaths("", url), df.out)
+	df.Req = reqDef(put, cleanPaths(url), df.out)
 	return df
 }

 // DELETE send HTTP DELETE method
 func (df *DataFlow) DELETE(url string) *DataFlow {
-	df.Req = reqDef(delete2, joinPaths("", url), df.out)
+	df.Req = reqDef(delete2, cleanPaths(url), df.out)
 	return df
 }

 // PATCH send HTTP PATCH method
 func (df *DataFlow) PATCH(url string) *DataFlow {
-	df.Req = reqDef(patch, joinPaths("", url), df.out)
+	df.Req = reqDef(patch, cleanPaths(url), df.out)
 	return df
 }

 // HEAD send HTTP HEAD method
 func (df *DataFlow) HEAD(url string) *DataFlow {
-	df.Req = reqDef(head, joinPaths("", url), df.out)
+	df.Req = reqDef(head, cleanPaths(url), df.out)
 	return df
 }

 // OPTIONS send HTTP OPTIONS method
 func (df *DataFlow) OPTIONS(url string) *DataFlow {
-	df.Req = reqDef(options, joinPaths("", url), df.out)
+	df.Req = reqDef(options, cleanPaths(url), df.out)
 	return df
 }

@@ -132,11 +132,11 @@ func (df *DataFlow) SetURL(url string) *DataFlow {
 	}

 	if df.Req.url == "" && df.Req.req == nil && df.Req.method == "" {
-		df.Req = reqDef("", joinPaths("", url), df.out)
+		df.Req = reqDef("", cleanPaths(url), df.out)
 		return df
 	}

-	df.Req.url = modifyURL(joinPaths("", url))
+	df.Req.url = modifyURL(cleanPaths(url))

 	return df
 }
--- a/dataflow/req.go
+++ b/dataflow/req.go
@@ -163,7 +163,7 @@ func (r *Req) selectRequest(body *bytes.Buffer) (req *http.Request, err error) {
 		}

 		if len(r.host) > 0 {
-			urlStr := modifyURL(joinPaths("", r.host))
+			urlStr := modifyURL(cleanPaths(r.host))
 			URL, err := url.Parse(urlStr)
 			if err != nil {
 				r.Err = err
--- a/dataflow/utils.go
+++ b/dataflow/utils.go
@@ -1,64 +0,0 @@
-package dataflow
-
-import (
-	"path"
-	"strings"
-)
-
-const (
-	httpProto  = "http://"
-	httpsProto = "https://"
-)
-
-func lastChar(str string) uint8 {
-	if str == "" {
-		panic("The length of the string can't be 0")
-	}
-	return str[len(str)-1]
-}
-
-func join(elem ...string) (rv string) {
-
-	defer func() {
-		if strings.HasPrefix(rv, httpProto) || strings.HasPrefix(rv, httpsProto) {
-			var proto, domainPath, query string
-			protoEndIdx := strings.Index(rv, "//") + 2
-			queryIdx := strings.Index(rv, "?")
-
-			proto = rv[:protoEndIdx]
-			if queryIdx != -1 {
-				domainPath = rv[protoEndIdx:queryIdx]
-				query = rv[queryIdx:]
-			} else if rv != proto {
-				domainPath = rv[protoEndIdx:]
-			}
-			if domainPath != "" {
-				domainPath = path.Clean(domainPath)
-			}
-			rv = proto + domainPath + query
-			return
-		}
-
-		rv = path.Clean(rv)
-	}()
-
-	for i, e := range elem {
-		if e != "" {
-			return strings.Join(elem[i:], "/")
-		}
-	}
-	return ""
-}
-
-func joinPaths(absolutePath, relativePath string) string {
-	if relativePath == "" {
-		return absolutePath
-	}
-
-	finalPath := join(absolutePath, relativePath)
-	appendSlash := lastChar(relativePath) == '/' && lastChar(finalPath) != '/'
-	if appendSlash {
-		return finalPath + "/"
-	}
-	return finalPath
-}
--- a/dataflow/utils_test.go
+++ b/dataflow/utils_test.go
@@ -1,53 +0,0 @@
-package dataflow
-
-import (
-	"github.com/stretchr/testify/assert"
-	"testing"
-)
-
-func TestUtil_Join(t *testing.T) {
-	urls := []string{
-		"http://127.0.0.1:43471/v1",
-	}
-
-	want := []string{
-		"http://127.0.0.1:43471/v1",
-	}
-
-	assert.Equal(t, joinPaths("", urls[0]), want[0])
-}
-
-type joinTest struct {
-	absolutePath string
-	relativePath string
-	need         string
-}
-
-func TestUtil_join(t *testing.T) {
-	//实际调用不会发生 join函数absolutePath和relativePath参数都为空的情况
-	assert.Equal(t, join("", ""), ".")
-}
-
-func TestUtil_joinPaths(t *testing.T) {
-	test := []joinTest{
-		{absolutePath: "", relativePath: "", need: ""},
-		{absolutePath: "https://www.aa.com", relativePath: "/a", need: "https://www.aa.com/a"},
-		{absolutePath: "http://www.bb.com", relativePath: "/b", need: "http://www.bb.com/b"},
-		{absolutePath: "www.bb.com", relativePath: "/b", need: "www.bb.com/b"},
-		{absolutePath: "www.bb.com", relativePath: "", need: "www.bb.com"},
-		{absolutePath: "", relativePath: "/a", need: "/a"},
-		{absolutePath: "", relativePath: "http://", need: "http://"},
-		{absolutePath: "", relativePath: "https://", need: "https://"},
-		{absolutePath: "", relativePath: "http://www.aa.com/urls?", need: "http://www.aa.com/urls?"},
-		{absolutePath: "", relativePath: "https://www.aa.com/urls?bb", need: "https://www.aa.com/urls?bb"},
-		{absolutePath: "", relativePath: "http://www.aa.com/urls?site=https://bb.com", need: "http://www.aa.com/urls?site=https://bb.com"},
-		{absolutePath: "", relativePath: "https://www.aa.com/urls?site=http://bb.com", need: "https://www.aa.com/urls?site=http://bb.com"},
-		{absolutePath: "", relativePath: "http://www.aa.com/urls/./a?site=https://bb.com", need: "http://www.aa.com/urls/a?site=https://bb.com"},
-		{absolutePath: "", relativePath: "https://www.aa.com/urls/../a?site=https://bb.com", need: "https://www.aa.com/a?site=https://bb.com"},
-	}
-
-	for _, v := range test {
-		rv := joinPaths(v.absolutePath, v.relativePath)
-		assert.Equal(t, v.need, rv)
-	}
-}