diff --git a/rtpconn/rtpconn.go b/rtpconn/rtpconn.go
index 1df9c1f..f8e03d5 100644
--- a/rtpconn/rtpconn.go
+++ b/rtpconn/rtpconn.go
@@ -271,11 +271,13 @@ func (down *rtpDownTrack) Write(buf []byte) (int, error) {
 			if flags.Keyframe {
 				layer.sid = layer.wantedSid
 				down.setLayerInfo(layer)
-			} else if flags.SidUpSync {
-				layer.sid = layer.sid + 1
-				down.setLayerInfo(layer)
-			} else {
-				down.remote.RequestKeyframe()
+			} else if flags.Sid == layer.sid + 1 {
+				if flags.SidUpSync {
+					layer.sid = layer.sid + 1
+					down.setLayerInfo(layer)
+				} else {
+					down.remote.RequestKeyframe()
+				}
 			}
 		}
 	}