令ws在upgrade失败时也解析X-forwared-for;unix监听不打印@,关联 #191

advLayer/grpcSimple/server.go

@@ -268,7 +268,7 @@ func newServerConn(rw http.ResponseWriter, rq *http.Request) (sc *ServerConn) {
 			if e == nil {
 				sc.ra = ta
 			} else {
-				if ce := utils.CanLogErr("Failed in grpcSimple parse X-Forwarded-For"); ce != nil {
+				if ce := utils.CanLogWarn("Failed in grpcSimple parse X-Forwarded-For"); ce != nil {
 					ce.Write(zap.Error(e), zap.Any(httpLayer.XForwardStr, xffs))
 				}
 			}

advLayer/ws/server.go

@@ -107,7 +107,7 @@ func (s *Server) Handshake(underlay net.Conn) (net.Conn, error) {
 				}
 
 			} else {
-				return nil, utils.ErrInErr{ErrDesc: "Failed in WS check parse http", ErrDetail: re, ExtraIs: []error{httpLayer.ErrNotHTTP_Request}}
+				return nil, utils.ErrInErr{ErrDesc: "Failed in WS check parse http", ErrDetail: re, ExtraIs: []error{httpLayer.ErrNotHTTP_Request}, Data: rp.Failreason}
 
 				//return nil, utils.ErrInErr{ErrDesc: "Failed in WS check parse http", ErrDetail: re, Data: rp.WholeRequestBuf.String(), ExtraIs: []error{httpLayer.ErrNotHTTP_Request}}
 
@@ -124,6 +124,7 @@ func (s *Server) Handshake(underlay net.Conn) (net.Conn, error) {
 
 	notWsRequest := false
 	notReason := ""
+	var realAddr net.Addr
 
 	//因为 gobwas 会先自行给错误的连接 返回 错误信息，而这不行，所以我们先过滤一遍。
 	//header 我们只过滤一个 connection 就行. 要是怕攻击者用 “对的path,method 和错误的header” 进行探测,
@@ -134,7 +135,11 @@ func (s *Server) Handshake(underlay net.Conn) (net.Conn, error) {
 		notReason = `rp.Method != "GET" || s.Thepath != rp.Path || len(rp.Headers) == 0`
 
 	} else {
+		//不在这里检查header，可以让后面ws部分检查一下X-forwarded-for 找出实际客户端地址
+		//但是必须要在这里检查header，所以也在这里检查 X-forwarded-for
+
 		hasUpgrade := false
+		gotXForward := false
 		for _, rh := range rp.Headers {
 			httpLayer.CanonicalizeHeaderKey(rh.Head)
 			if bytes.Equal(rh.Head, connectionBs) {
@@ -143,9 +148,31 @@ func (s *Server) Handshake(underlay net.Conn) (net.Conn, error) {
 				if bytes.Equal(rh.Value, upgradeBs) {
 
 					hasUpgrade = true
-					break
+					if gotXForward {
+						break
+					}
 				}
 			}
+
+			if string(rh.Head) == httpLayer.XForwardStr {
+				gotXForward = true
+
+				realV := string(rh.Value)
+				xffs := strings.SplitN(realV, ",", 2)
+
+				if len(xffs) > 0 {
+					ta, e := net.ResolveIPAddr("ip", strings.TrimSpace(xffs[0]))
+					if e == nil {
+						realAddr = ta
+					} else {
+						if ce := utils.CanLogWarn("Failed in ws parse X-Forwarded-For"); ce != nil {
+							ce.Write(zap.Error(e), zap.Any(httpLayer.XForwardStr, xffs))
+						}
+					}
+				}
+
+			}
+
 		}
 		if !hasUpgrade {
 			notWsRequest = true
@@ -161,6 +188,7 @@ func (s *Server) Handshake(underlay net.Conn) (net.Conn, error) {
 			Path:         rp.Path,
 			Method:       rp.Method,
 			Reason:       notReason,
+			XFF:          realAddr,
 		}, httpLayer.ErrShouldFallback
 	}
 
@@ -168,8 +196,6 @@ func (s *Server) Handshake(underlay net.Conn) (net.Conn, error) {
 
 	requestHeaderNotGivenCount := s.requestHeaderCheckCount
 
-	var realAddr net.Addr
-
 	var theUpgrader *ws.Upgrader = &ws.Upgrader{
 
 		//因为我们vs的架构，先统一监听tcp；然后再调用Handshake函数
@@ -186,22 +212,6 @@ func (s *Server) Handshake(underlay net.Conn) (net.Conn, error) {
 		OnHeader: func(key, value []byte) error {
 			sk := string(key)
 
-			if sk == httpLayer.XForwardStr {
-				realV := string(value)
-				xffs := strings.SplitN(realV, ",", 2)
-
-				if len(xffs) > 0 {
-					ta, e := net.ResolveIPAddr("ip", strings.TrimLeft(xffs[0], " "))
-					if e == nil {
-						realAddr = ta
-					} else {
-						if ce := utils.CanLogErr("Failed in ws parse X-Forwarded-For"); ce != nil {
-							ce.Write(zap.Error(e), zap.Any(httpLayer.XForwardStr, xffs))
-						}
-					}
-				}
-
-			}
 			if s.noNeedToCheckRequestHeaders {
 				return nil
 			}
@@ -284,6 +294,7 @@ func (s *Server) Handshake(underlay net.Conn) (net.Conn, error) {
 			Path:         rp.Path,
 			Method:       rp.Method,
 			Reason:       err.Error(),
+			XFF:          realAddr,
 		}, httpLayer.ErrShouldFallback
 	}
 

advLayer/ws/ws.go

@@ -1,6 +1,7 @@
-/*Package ws implements websocket for advLayer.
+/*
+Package ws implements websocket for advLayer.
 
-Reference
+# Reference
 
 websocket rfc: https://datatracker.ietf.org/doc/html/rfc6455/
 
@@ -8,22 +9,22 @@ Below is a real websocket handshake progress:
 
 Request
 
-    GET /chat HTTP/1.1
-        Host: server.example.com
-        Upgrade: websocket
-        Connection: Upgrade
-        Sec-WebSocket-Key: x3JJHMbDL1EzLkh9GBhXDw==
-        Sec-WebSocket-Protocol: chat, superchat
-        Sec-WebSocket-Version: 13
-        Origin: http://example.com
+	GET /chat HTTP/1.1
+	    Host: server.example.com
+	    Upgrade: websocket
+	    Connection: Upgrade
+	    Sec-WebSocket-Key: x3JJHMbDL1EzLkh9GBhXDw==
+	    Sec-WebSocket-Protocol: chat, superchat
+	    Sec-WebSocket-Version: 13
+	    Origin: http://example.com
 
 Response
 
-    HTTP/1.1 101 Switching Protocols
-        Upgrade: websocket
-        Connection: Upgrade
-        Sec-WebSocket-Accept: HSmrc0sMlYUkAGmm5OPpG2HaGWk=
-        Sec-WebSocket-Protocol: chat
+	HTTP/1.1 101 Switching Protocols
+	    Upgrade: websocket
+	    Connection: Upgrade
+	    Sec-WebSocket-Accept: HSmrc0sMlYUkAGmm5OPpG2HaGWk=
+	    Sec-WebSocket-Protocol: chat
 
 websocket packages comparison:
 https://yalantis.com/blog/how-to-build-websockets-in-go/
@@ -32,6 +33,8 @@ https://yalantis.com/blog/how-to-build-websockets-in-go/
 https://tonybai.com/2019/09/28/how-to-build-websockets-in-go/
 
 All in all gobwas/ws is the best package. We use gobwas/ws.
+
+gobwas包只支持http1.1, 所以如果使用nginx前置，确保 proxy_http_version 1.1;
 */
 package ws
 

httpLayer/fallback.go

@@ -47,6 +47,7 @@ type FallbackMeta struct {
 	Reason       string
 
 	H2Request *http.Request
+	XFF       net.Addr
 }
 
 func getfallbacktype_byindex(i int) byte {

iics.go

@@ -300,7 +300,6 @@ func (iics *incomingInserverConnState) getRealRAddr() (raddr string) {
 			//大部分情况下，realRA.String() == iics.cachedRemoteAddr
 			// 但是在 ws/grpc 下，我们的代码 会读取 X-Forwarded-For, 来试图找出反代之前的客户端真实ip
 			//此时 RemoteAddr就 不相等了
-
 			raddr = realRA.String()
 		}
 	}

main.go

@@ -267,8 +267,15 @@ func handleNewIncomeConnection(inServer proxy.Server, defaultClientForThis proxy
 	wrappedConn := thisLocalConnectionInstance
 
 	if ce := iics.CanLogInfo("New Accepted Conn"); ce != nil {
+		var addrstr string
 
-		addrstr := wrappedConn.RemoteAddr().String()
+		if inServer.Network() == "unix" {
+			addrstr = inServer.AddrStr()
+
+		} else {
+			addrstr = wrappedConn.RemoteAddr().String()
+
+		}
 		ce.Write(
 			zap.String("from", addrstr),
 			zap.String("handler", proxy.GetVSI_url(inServer, "")),
@@ -423,6 +430,10 @@ func handleNewIncomeConnection(inServer proxy.Server, defaultClientForThis proxy
 				iics.fallbackRequestPath = meta.Path
 				iics.fallbackFirstBuffer = meta.H1RequestBuf
 				iics.wrappedConn = meta.Conn
+				if meta.XFF != nil {
+					iics.cachedRemoteAddr = meta.XFF.String()
+
+				}
 
 				if ce := iics.CanLogDebug("Single AdvLayer Check failed, will fallback."); ce != nil {
 
@@ -433,6 +444,7 @@ func handleNewIncomeConnection(inServer proxy.Server, defaultClientForThis proxy
 						zap.String("validPath", advSer.GetPath()),
 						zap.String("gotMethod", meta.Method),
 						zap.String("gotPath", meta.Path),
+						zap.String("from", iics.cachedRemoteAddr),
 					)
 				}
 
@@ -739,7 +751,7 @@ func passToOutClient(iics incomingInserverConnState, isfallback bool, wlc net.Co
 	if wlc == nil && udp_wlc == nil {
 
 		if ce := iics.CanLogWarn("Invalid request and no matched fallback, hung up"); ce != nil {
-			ce.Write(zap.String("client RemoteAddr", iics.getRealRAddr()))
+			ce.Write(zap.String("client RemoteAddr", iics.cachedRemoteAddr))
 		}
 
 		if wc := iics.wrappedConn; wc != nil {