diff --git a/advLayer/grpcSimple/server.go b/advLayer/grpcSimple/server.go index 17b658e..ff7ee09 100644 --- a/advLayer/grpcSimple/server.go +++ b/advLayer/grpcSimple/server.go @@ -268,7 +268,7 @@ func newServerConn(rw http.ResponseWriter, rq *http.Request) (sc *ServerConn) { if e == nil { sc.ra = ta } else { - if ce := utils.CanLogErr("Failed in grpcSimple parse X-Forwarded-For"); ce != nil { + if ce := utils.CanLogWarn("Failed in grpcSimple parse X-Forwarded-For"); ce != nil { ce.Write(zap.Error(e), zap.Any(httpLayer.XForwardStr, xffs)) } } diff --git a/advLayer/ws/server.go b/advLayer/ws/server.go index b520977..9cdd0a0 100644 --- a/advLayer/ws/server.go +++ b/advLayer/ws/server.go @@ -107,7 +107,7 @@ func (s *Server) Handshake(underlay net.Conn) (net.Conn, error) { } } else { - return nil, utils.ErrInErr{ErrDesc: "Failed in WS check parse http", ErrDetail: re, ExtraIs: []error{httpLayer.ErrNotHTTP_Request}} + return nil, utils.ErrInErr{ErrDesc: "Failed in WS check parse http", ErrDetail: re, ExtraIs: []error{httpLayer.ErrNotHTTP_Request}, Data: rp.Failreason} //return nil, utils.ErrInErr{ErrDesc: "Failed in WS check parse http", ErrDetail: re, Data: rp.WholeRequestBuf.String(), ExtraIs: []error{httpLayer.ErrNotHTTP_Request}} @@ -124,6 +124,7 @@ func (s *Server) Handshake(underlay net.Conn) (net.Conn, error) { notWsRequest := false notReason := "" + var realAddr net.Addr //因为 gobwas 会先自行给错误的连接 返回 错误信息，而这不行，所以我们先过滤一遍。 //header 我们只过滤一个 connection 就行. 要是怕攻击者用 “对的path,method 和错误的header” 进行探测, @@ -134,7 +135,11 @@ func (s *Server) Handshake(underlay net.Conn) (net.Conn, error) { notReason = `rp.Method != "GET" || s.Thepath != rp.Path || len(rp.Headers) == 0` } else { + //不在这里检查header，可以让后面ws部分检查一下X-forwarded-for 找出实际客户端地址 + //但是必须要在这里检查header，所以也在这里检查 X-forwarded-for + hasUpgrade := false + gotXForward := false for _, rh := range rp.Headers { httpLayer.CanonicalizeHeaderKey(rh.Head) if bytes.Equal(rh.Head, connectionBs) { @@ -143,9 +148,31 @@ func (s *Server) Handshake(underlay net.Conn) (net.Conn, error) { if bytes.Equal(rh.Value, upgradeBs) { hasUpgrade = true - break + if gotXForward { + break + } } } + + if string(rh.Head) == httpLayer.XForwardStr { + gotXForward = true + + realV := string(rh.Value) + xffs := strings.SplitN(realV, ",", 2) + + if len(xffs) > 0 { + ta, e := net.ResolveIPAddr("ip", strings.TrimSpace(xffs[0])) + if e == nil { + realAddr = ta + } else { + if ce := utils.CanLogWarn("Failed in ws parse X-Forwarded-For"); ce != nil { + ce.Write(zap.Error(e), zap.Any(httpLayer.XForwardStr, xffs)) + } + } + } + + } + } if !hasUpgrade { notWsRequest = true @@ -161,6 +188,7 @@ func (s *Server) Handshake(underlay net.Conn) (net.Conn, error) { Path: rp.Path, Method: rp.Method, Reason: notReason, + XFF: realAddr, }, httpLayer.ErrShouldFallback } @@ -168,8 +196,6 @@ func (s *Server) Handshake(underlay net.Conn) (net.Conn, error) { requestHeaderNotGivenCount := s.requestHeaderCheckCount - var realAddr net.Addr - var theUpgrader *ws.Upgrader = &ws.Upgrader{ //因为我们vs的架构，先统一监听tcp；然后再调用Handshake函数 @@ -186,22 +212,6 @@ func (s *Server) Handshake(underlay net.Conn) (net.Conn, error) { OnHeader: func(key, value []byte) error { sk := string(key) - if sk == httpLayer.XForwardStr { - realV := string(value) - xffs := strings.SplitN(realV, ",", 2) - - if len(xffs) > 0 { - ta, e := net.ResolveIPAddr("ip", strings.TrimLeft(xffs[0], " ")) - if e == nil { - realAddr = ta - } else { - if ce := utils.CanLogErr("Failed in ws parse X-Forwarded-For"); ce != nil { - ce.Write(zap.Error(e), zap.Any(httpLayer.XForwardStr, xffs)) - } - } - } - - } if s.noNeedToCheckRequestHeaders { return nil } @@ -284,6 +294,7 @@ func (s *Server) Handshake(underlay net.Conn) (net.Conn, error) { Path: rp.Path, Method: rp.Method, Reason: err.Error(), + XFF: realAddr, }, httpLayer.ErrShouldFallback } diff --git a/advLayer/ws/ws.go b/advLayer/ws/ws.go index cea959d..81ab4ea 100644 --- a/advLayer/ws/ws.go +++ b/advLayer/ws/ws.go @@ -1,6 +1,7 @@ -/*Package ws implements websocket for advLayer. +/* +Package ws implements websocket for advLayer. -Reference +# Reference websocket rfc: https://datatracker.ietf.org/doc/html/rfc6455/ @@ -8,22 +9,22 @@ Below is a real websocket handshake progress: Request - GET /chat HTTP/1.1 - Host: server.example.com - Upgrade: websocket - Connection: Upgrade - Sec-WebSocket-Key: x3JJHMbDL1EzLkh9GBhXDw== - Sec-WebSocket-Protocol: chat, superchat - Sec-WebSocket-Version: 13 - Origin: http://example.com + GET /chat HTTP/1.1 + Host: server.example.com + Upgrade: websocket + Connection: Upgrade + Sec-WebSocket-Key: x3JJHMbDL1EzLkh9GBhXDw== + Sec-WebSocket-Protocol: chat, superchat + Sec-WebSocket-Version: 13 + Origin: http://example.com Response - HTTP/1.1 101 Switching Protocols - Upgrade: websocket - Connection: Upgrade - Sec-WebSocket-Accept: HSmrc0sMlYUkAGmm5OPpG2HaGWk= - Sec-WebSocket-Protocol: chat + HTTP/1.1 101 Switching Protocols + Upgrade: websocket + Connection: Upgrade + Sec-WebSocket-Accept: HSmrc0sMlYUkAGmm5OPpG2HaGWk= + Sec-WebSocket-Protocol: chat websocket packages comparison: https://yalantis.com/blog/how-to-build-websockets-in-go/ @@ -32,6 +33,8 @@ https://yalantis.com/blog/how-to-build-websockets-in-go/ https://tonybai.com/2019/09/28/how-to-build-websockets-in-go/ All in all gobwas/ws is the best package. We use gobwas/ws. + +gobwas包只支持http1.1, 所以如果使用nginx前置，确保 proxy_http_version 1.1; */ package ws diff --git a/httpLayer/fallback.go b/httpLayer/fallback.go index 9858e41..cba2b15 100644 --- a/httpLayer/fallback.go +++ b/httpLayer/fallback.go @@ -47,6 +47,7 @@ type FallbackMeta struct { Reason string H2Request *http.Request + XFF net.Addr } func getfallbacktype_byindex(i int) byte { diff --git a/iics.go b/iics.go index ec9f282..fcf15da 100644 --- a/iics.go +++ b/iics.go @@ -300,7 +300,6 @@ func (iics *incomingInserverConnState) getRealRAddr() (raddr string) { //大部分情况下，realRA.String() == iics.cachedRemoteAddr // 但是在 ws/grpc 下，我们的代码 会读取 X-Forwarded-For, 来试图找出反代之前的客户端真实ip //此时 RemoteAddr就 不相等了 - raddr = realRA.String() } } diff --git a/main.go b/main.go index 3064b79..72a681d 100644 --- a/main.go +++ b/main.go @@ -267,8 +267,15 @@ func handleNewIncomeConnection(inServer proxy.Server, defaultClientForThis proxy wrappedConn := thisLocalConnectionInstance if ce := iics.CanLogInfo("New Accepted Conn"); ce != nil { + var addrstr string - addrstr := wrappedConn.RemoteAddr().String() + if inServer.Network() == "unix" { + addrstr = inServer.AddrStr() + + } else { + addrstr = wrappedConn.RemoteAddr().String() + + } ce.Write( zap.String("from", addrstr), zap.String("handler", proxy.GetVSI_url(inServer, "")), @@ -423,6 +430,10 @@ func handleNewIncomeConnection(inServer proxy.Server, defaultClientForThis proxy iics.fallbackRequestPath = meta.Path iics.fallbackFirstBuffer = meta.H1RequestBuf iics.wrappedConn = meta.Conn + if meta.XFF != nil { + iics.cachedRemoteAddr = meta.XFF.String() + + } if ce := iics.CanLogDebug("Single AdvLayer Check failed, will fallback."); ce != nil { @@ -433,6 +444,7 @@ func handleNewIncomeConnection(inServer proxy.Server, defaultClientForThis proxy zap.String("validPath", advSer.GetPath()), zap.String("gotMethod", meta.Method), zap.String("gotPath", meta.Path), + zap.String("from", iics.cachedRemoteAddr), ) } @@ -739,7 +751,7 @@ func passToOutClient(iics incomingInserverConnState, isfallback bool, wlc net.Co if wlc == nil && udp_wlc == nil { if ce := iics.CanLogWarn("Invalid request and no matched fallback, hung up"); ce != nil { - ce.Write(zap.String("client RemoteAddr", iics.getRealRAddr())) + ce.Write(zap.String("client RemoteAddr", iics.cachedRemoteAddr)) } if wc := iics.wrappedConn; wc != nil {