diff --git a/internal/ntpestimator/estimator.go b/internal/ntpestimator/estimator.go
index fa234683..ea4fb82f 100644
--- a/internal/ntpestimator/estimator.go
+++ b/internal/ntpestimator/estimator.go
@@ -5,6 +5,10 @@ import (
 	"time"
 )
 
+const (
+	maxTimeDifference = 5 * time.Second
+)
+
 var timeNow = time.Now
 
 func multiplyAndDivide(v, m, d time.Duration) time.Duration {
@@ -35,7 +39,7 @@ func (e *Estimator) Estimate(pts int64) time.Time {
 
 	computed := e.refNTP.Add((multiplyAndDivide(time.Duration(pts-e.refPTS), time.Second, time.Duration(e.ClockRate))))
 
-	if computed.After(now) {
+	if computed.After(now) || computed.Before(now.Add(-maxTimeDifference)) {
 		e.refNTP = now
 		e.refPTS = pts
 		return now
diff --git a/internal/ntpestimator/estimator_test.go b/internal/ntpestimator/estimator_test.go
index 4929fcf2..5a62d64d 100644
--- a/internal/ntpestimator/estimator_test.go
+++ b/internal/ntpestimator/estimator_test.go
@@ -29,4 +29,8 @@ func TestEstimator(t *testing.T) {
 	timeNow = func() time.Time { return time.Date(2003, 11, 4, 23, 15, 15, 0, time.UTC) }
 	ntp = e.Estimate(5 * 90000)
 	require.Equal(t, time.Date(2003, 11, 4, 23, 15, 10, 0, time.UTC), ntp)
+
+	timeNow = func() time.Time { return time.Date(2003, 11, 4, 23, 15, 20, 0, time.UTC) }
+	ntp = e.Estimate(6 * 90000)
+	require.Equal(t, time.Date(2003, 11, 4, 23, 15, 20, 0, time.UTC), ntp)
 }