diff --git a/server.go b/server.go
index cf764731..9015f806 100644
--- a/server.go
+++ b/server.go
@@ -37,7 +37,10 @@ func newServer(conf ServerConf, address string) (*Server, error) {
 
 	if conf.UDPRTPListener != nil {
 		conf.UDPRTPListener.streamType = StreamTypeRTP
+		conf.UDPRTPListener.writeTimeout = conf.WriteTimeout
+
 		conf.UDPRTCPListener.streamType = StreamTypeRTCP
+		conf.UDPRTCPListener.writeTimeout = conf.WriteTimeout
 	}
 
 	listener, err := conf.Listen("tcp", address)
diff --git a/serverconn.go b/serverconn.go
index 0c28c9bf..7c45818a 100644
--- a/serverconn.go
+++ b/serverconn.go
@@ -809,14 +809,14 @@ func (sc *ServerConn) WriteFrame(trackID int, streamType StreamType, payload []b
 		track := sc.tracks[trackID]
 
 		if streamType == StreamTypeRTP {
-			return sc.conf.UDPRTPListener.write(sc.conf.WriteTimeout, payload, &net.UDPAddr{
+			return sc.conf.UDPRTPListener.write(payload, &net.UDPAddr{
 				IP:   sc.ip(),
 				Zone: sc.zone(),
 				Port: track.rtpPort,
 			})
 		}
 
-		return sc.conf.UDPRTCPListener.write(sc.conf.WriteTimeout, payload, &net.UDPAddr{
+		return sc.conf.UDPRTCPListener.write(payload, &net.UDPAddr{
 			IP:   sc.ip(),
 			Zone: sc.zone(),
 			Port: track.rtcpPort,
diff --git a/serverudpl.go b/serverudpl.go
index 3617acef..4f1cdb31 100644
--- a/serverudpl.go
+++ b/serverudpl.go
@@ -39,7 +39,8 @@ func (p *publisherAddr) fill(ip net.IP, port int) {
 
 // ServerUDPListener is a UDP server that can be used to send and receive RTP and RTCP packets.
 type ServerUDPListener struct {
-	streamType StreamType
+	streamType   StreamType
+	writeTimeout time.Duration
 
 	pc              *net.UDPConn
 	readBuf         *multibuffer.MultiBuffer
@@ -116,11 +117,11 @@ func (s *ServerUDPListener) port() int {
 	return s.pc.LocalAddr().(*net.UDPAddr).Port
 }
 
-func (s *ServerUDPListener) write(writeTimeout time.Duration, buf []byte, addr *net.UDPAddr) error {
+func (s *ServerUDPListener) write(buf []byte, addr *net.UDPAddr) error {
 	s.writeMutex.Lock()
 	defer s.writeMutex.Unlock()
 
-	s.pc.SetWriteDeadline(time.Now().Add(writeTimeout))
+	s.pc.SetWriteDeadline(time.Now().Add(s.writeTimeout))
 	_, err := s.pc.WriteTo(buf, addr)
 	return err
 }