From fdfe1c96a3f98fb4ad5dd589fd31bb2ea61237d2 Mon Sep 17 00:00:00 2001
From: Ilya Nikolaevskiy <ilnik@webrtc.org>
Date: Fri, 18 Jan 2019 13:33:38 +0100
Subject: [PATCH] Update jitter delay on per-superframe level from FrameBuffer

Current way with updates on each frame caused a bogus jitter estimate
and lots of dropped frames in unfiltered KSVC stream.

Bug: chromium:912122
Change-Id: I4a1af71a242af3f9b5f5a411b194331b2df24f68
Reviewed-on: https://webrtc-review.googlesource.com/c/117566
Commit-Queue: Ilya Nikolaevskiy <ilnik@webrtc.org>
Reviewed-by: Philip Eliasson <philipel@webrtc.org>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#26322}
---
 modules/video_coding/frame_buffer2.cc | 66 +++++++++++++++++----------
 1 file changed, 41 insertions(+), 25 deletions(-)

diff --git a/modules/video_coding/frame_buffer2.cc b/modules/video_coding/frame_buffer2.cc
index 00a8ab07d8..640111726c 100644
--- a/modules/video_coding/frame_buffer2.cc
+++ b/modules/video_coding/frame_buffer2.cc
@@ -190,17 +190,50 @@ FrameBuffer::ReturnReason FrameBuffer::NextFrame(
   {
     rtc::CritScope lock(&crit_);
     now_ms = clock_->TimeInMilliseconds();
+    // TODO(ilnik): remove |frames_out| use frames_to_decode_ directly.
     std::vector<EncodedFrame*> frames_out;
-    for (FrameMap::iterator& frame_it : frames_to_decode_) {
-      RTC_DCHECK(frame_it != frames_.end());
-      EncodedFrame* frame = frame_it->second.frame.release();
 
-      if (!frame->delayed_by_retransmission()) {
+    if (!frames_to_decode_.empty()) {
+      bool superframe_delayed_by_retransmission = false;
+      size_t superframe_size = 0;
+      EncodedFrame* first_frame = frames_to_decode_[0]->second.frame.get();
+      int64_t render_time_ms = first_frame->RenderTime();
+      int64_t receive_time_ms = first_frame->ReceivedTime();
+      // Gracefully handle bad RTP timestamps and render time issues.
+      if (HasBadRenderTiming(*first_frame, now_ms)) {
+        jitter_estimator_->Reset();
+        timing_->Reset();
+        render_time_ms =
+            timing_->RenderTimeMs(first_frame->Timestamp(), now_ms);
+      }
+
+      for (FrameMap::iterator& frame_it : frames_to_decode_) {
+        RTC_DCHECK(frame_it != frames_.end());
+        EncodedFrame* frame = frame_it->second.frame.release();
+
+        frame->SetRenderTime(render_time_ms);
+
+        superframe_delayed_by_retransmission |=
+            frame->delayed_by_retransmission();
+        receive_time_ms = std::max(receive_time_ms, frame->ReceivedTime());
+        superframe_size += frame->size();
+
+        PropagateDecodability(frame_it->second);
+        decoded_frames_history_.InsertDecoded(frame_it->first,
+                                              frame->Timestamp());
+
+        // Remove decoded frame and all undecoded frames before it.
+        frames_.erase(frames_.begin(), ++frame_it);
+
+        frames_out.push_back(frame);
+      }
+
+      if (!superframe_delayed_by_retransmission) {
         int64_t frame_delay;
 
-        if (inter_frame_delay_.CalculateDelay(frame->Timestamp(), &frame_delay,
-                                              frame->ReceivedTime())) {
-          jitter_estimator_->UpdateEstimate(frame_delay, frame->size());
+        if (inter_frame_delay_.CalculateDelay(first_frame->Timestamp(),
+                                              &frame_delay, receive_time_ms)) {
+          jitter_estimator_->UpdateEstimate(frame_delay, superframe_size);
         }
 
         float rtt_mult = protection_mode_ == kProtectionNackFEC ? 0.0 : 1.0;
@@ -208,32 +241,15 @@ FrameBuffer::ReturnReason FrameBuffer::NextFrame(
           rtt_mult = RttMultExperiment::GetRttMultValue();
         }
         timing_->SetJitterDelay(jitter_estimator_->GetJitterEstimate(rtt_mult));
-        timing_->UpdateCurrentDelay(frame->RenderTime(), now_ms);
+        timing_->UpdateCurrentDelay(render_time_ms, now_ms);
       } else {
         if (RttMultExperiment::RttMultEnabled() || add_rtt_to_playout_delay_)
           jitter_estimator_->FrameNacked();
       }
 
-      // Gracefully handle bad RTP timestamps and render time issues.
-      if (HasBadRenderTiming(*frame, now_ms)) {
-        jitter_estimator_->Reset();
-        timing_->Reset();
-        frame->SetRenderTime(timing_->RenderTimeMs(frame->Timestamp(), now_ms));
-      }
-
       UpdateJitterDelay();
       UpdateTimingFrameInfo();
-      PropagateDecodability(frame_it->second);
-
-      decoded_frames_history_.InsertDecoded(frame_it->first,
-                                            frame->Timestamp());
-
-      // Remove decoded frame and all undecoded frames before it.
-      frames_.erase(frames_.begin(), ++frame_it);
-
-      frames_out.push_back(frame);
     }
-
     if (!frames_out.empty()) {
       if (frames_out.size() == 1) {
         frame_out->reset(frames_out[0]);