diff --git a/client.go b/client.go
index 020cc8b3..787730ab 100644
--- a/client.go
+++ b/client.go
@@ -1425,7 +1425,7 @@ func (c *Client) doSetup(
 		rtpListener.remoteReadIP = c.nconn.RemoteAddr().(*net.TCPAddr).IP
 		rtpListener.remoteWriteIP = c.nconn.RemoteAddr().(*net.TCPAddr).IP
 		rtpListener.remoteZone = c.nconn.RemoteAddr().(*net.TCPAddr).Zone
-		if thRes.ServerPorts != nil {
+		if !c.AnyPortEnable {
 			rtpListener.remotePort = thRes.ServerPorts[0]
 		}
 		rtpListener.trackID = trackID
@@ -1435,7 +1435,7 @@ func (c *Client) doSetup(
 		rtcpListener.remoteReadIP = c.nconn.RemoteAddr().(*net.TCPAddr).IP
 		rtcpListener.remoteWriteIP = c.nconn.RemoteAddr().(*net.TCPAddr).IP
 		rtcpListener.remoteZone = c.nconn.RemoteAddr().(*net.TCPAddr).Zone
-		if thRes.ServerPorts != nil {
+		if !c.AnyPortEnable {
 			rtcpListener.remotePort = thRes.ServerPorts[1]
 		}
 		rtcpListener.trackID = trackID
diff --git a/client_read_test.go b/client_read_test.go
index faa5a7dd..1f2f3731 100644
--- a/client_read_test.go
+++ b/client_read_test.go
@@ -792,6 +792,7 @@ func TestClientReadAnyPort(t *testing.T) {
 		"zero",
 		"zero_one",
 		"no",
+		"random",
 	} {
 		t.Run(ca, func(t *testing.T) {
 			l, err := net.Listen("tcp", "localhost:8554")
@@ -868,8 +869,13 @@ func TestClientReadAnyPort(t *testing.T) {
 
 								case "zero_one":
 									return &[2]int{0, 1}
+
+								case "no":
+									return nil
+
+								default: // random
+									return &[2]int{23040, 23041}
 								}
-								return nil
 							}(),
 						}.Write(),
 					},
@@ -887,7 +893,7 @@ func TestClientReadAnyPort(t *testing.T) {
 
 				time.Sleep(1 * time.Second)
 
-				l1, err := net.ListenPacket("udp", "localhost:0")
+				l1, err := net.ListenPacket("udp", "localhost:13344")
 				require.NoError(t, err)
 				defer l1.Close()
 
diff --git a/clientudpl.go b/clientudpl.go
index 2651b694..d08a0f41 100644
--- a/clientudpl.go
+++ b/clientudpl.go
@@ -170,7 +170,7 @@ func (l *clientUDPListener) run() {
 
 		uaddr := addr.(*net.UDPAddr)
 
-		if !l.remoteReadIP.Equal(uaddr.IP) || (!isAnyPort(l.remotePort) && l.remotePort != uaddr.Port) {
+		if !l.remoteReadIP.Equal(uaddr.IP) || (l.remotePort != 0 && l.remotePort != uaddr.Port) {
 			continue
 		}