Wire up VCMTiming in FrameBuffer2.

The VCMTiming class is now used to set the render time for frames.

BUG=webrtc:5514
R=stefan@webrtc.org

Review URL: https://codereview.webrtc.org/2138873003 .

Cr-Commit-Position: refs/heads/master@{#13621}

webrtc/modules/video_coding/frame_buffer2.cc

@@ -42,13 +42,15 @@ bool FrameBuffer::FrameComp::operator()(const FrameKey& f1,
 
 FrameBuffer::FrameBuffer(Clock* clock,
                          VCMJitterEstimator* jitter_estimator,
-                         const VCMTiming* timing)
+                         VCMTiming* timing)
     : clock_(clock),
       frame_inserted_event_(false, false),
       jitter_estimator_(jitter_estimator),
       timing_(timing),
+      inter_frame_delay_(clock_->TimeInMilliseconds()),
       newest_picture_id_(-1),
-      stopped_(false) {}
+      stopped_(false),
+      protection_mode_(kProtectionNack) {}
 
 std::unique_ptr<FrameObject> FrameBuffer::NextFrame(int64_t max_wait_time_ms) {
   int64_t latest_return_time = clock_->TimeInMilliseconds() + max_wait_time_ms;
@@ -56,7 +58,7 @@ std::unique_ptr<FrameObject> FrameBuffer::NextFrame(int64_t max_wait_time_ms) {
   int64_t wait_ms = max_wait_time_ms;
   while (true) {
     std::map<FrameKey, std::unique_ptr<FrameObject>, FrameComp>::iterator
-        next_frame;
+        next_frame_it;
     {
       rtc::CritScope lock(&crit_);
       frame_inserted_event_.Reset();
@@ -65,14 +67,18 @@ std::unique_ptr<FrameObject> FrameBuffer::NextFrame(int64_t max_wait_time_ms) {
 
       now = clock_->TimeInMilliseconds();
       wait_ms = max_wait_time_ms;
-      next_frame = frames_.end();
+      next_frame_it = frames_.end();
       for (auto frame_it = frames_.begin(); frame_it != frames_.end();
            ++frame_it) {
         const FrameObject& frame = *frame_it->second;
         if (IsContinuous(frame)) {
-          next_frame = frame_it;
+          next_frame_it = frame_it;
-          int64_t render_time = timing_->RenderTimeMs(frame.timestamp, now);
+          int64_t render_time =
+              next_frame_it->second->RenderTime() == -1
+                  ? timing_->RenderTimeMs(frame.timestamp, now)
+                  : next_frame_it->second->RenderTime();
           wait_ms = timing_->MaxWaitingTime(render_time, now);
+          frame_it->second->SetRenderTime(render_time);
 
           // This will cause the frame buffer to prefer high framerate rather
           // than high resolution in the case of the decoder not decoding fast
@@ -85,19 +91,30 @@ std::unique_ptr<FrameObject> FrameBuffer::NextFrame(int64_t max_wait_time_ms) {
       }
     }
 
-    // If the timout occures, return. Otherwise a new frame has been inserted
-    // and the best frame to decode next will be selected again.
     wait_ms = std::min<int64_t>(wait_ms, latest_return_time - now);
     wait_ms = std::max<int64_t>(wait_ms, 0);
+    // If the timeout occurs, return. Otherwise a new frame has been inserted
+    // and the best frame to decode next will be selected again.
     if (!frame_inserted_event_.Wait(wait_ms)) {
       rtc::CritScope lock(&crit_);
-      if (next_frame != frames_.end()) {
+      if (next_frame_it != frames_.end()) {
-        // TODO(philipel): update jitter estimator with correct values.
+        int64_t received_timestamp = next_frame_it->second->ReceivedTime();
-        jitter_estimator_->UpdateEstimate(100, 100);
+        uint32_t timestamp = next_frame_it->second->Timestamp();
 
-        decoded_frames_.insert(next_frame->first);
+        int64_t frame_delay;
-        std::unique_ptr<FrameObject> frame = std::move(next_frame->second);
+        if (inter_frame_delay_.CalculateDelay(timestamp, &frame_delay,
-        frames_.erase(frames_.begin(), ++next_frame);
+                                              received_timestamp)) {
+          jitter_estimator_->UpdateEstimate(frame_delay,
+                                            next_frame_it->second->size);
+        }
+        float rtt_mult = protection_mode_ == kProtectionNackFEC ? 0.0 : 1.0;
+        timing_->SetJitterDelay(jitter_estimator_->GetJitterEstimate(rtt_mult));
+        timing_->UpdateCurrentDelay(next_frame_it->second->RenderTime(),
+                                    clock_->TimeInMilliseconds());
+
+        decoded_frames_.insert(next_frame_it->first);
+        std::unique_ptr<FrameObject> frame = std::move(next_frame_it->second);
+        frames_.erase(frames_.begin(), ++next_frame_it);
         return frame;
       } else {
         return std::unique_ptr<FrameObject>();
@@ -106,6 +123,11 @@ std::unique_ptr<FrameObject> FrameBuffer::NextFrame(int64_t max_wait_time_ms) {
   }
 }
 
+void FrameBuffer::SetProtectionMode(VCMVideoProtection mode) {
+  rtc::CritScope lock(&crit_);
+  protection_mode_ = mode;
+}
+
 void FrameBuffer::Start() {
   rtc::CritScope lock(&crit_);
   stopped_ = false;
@@ -119,6 +141,7 @@ void FrameBuffer::Stop() {
 
 void FrameBuffer::InsertFrame(std::unique_ptr<FrameObject> frame) {
   rtc::CritScope lock(&crit_);
+  // If |newest_picture_id_| is -1 then this is the first frame we received.
   if (newest_picture_id_ == -1)
     newest_picture_id_ = frame->picture_id;
 
@@ -129,7 +152,7 @@ void FrameBuffer::InsertFrame(std::unique_ptr<FrameObject> frame) {
   while (decoded_frames_.size() > kMaxNumHistoryFrames)
     decoded_frames_.erase(decoded_frames_.begin());
 
-  // Remove frames that are too old, |kMaxNumHistoryFrames|.
+  // Remove frames that are too old.
   uint16_t old_picture_id = Subtract<1 << 16>(newest_picture_id_, kMaxFrameAge);
   auto old_decoded_it =
       decoded_frames_.lower_bound(FrameKey(old_picture_id, 0));

webrtc/modules/video_coding/frame_buffer2.h

@@ -21,6 +21,8 @@
 #include "webrtc/base/criticalsection.h"
 #include "webrtc/base/event.h"
 #include "webrtc/base/thread_annotations.h"
+#include "webrtc/modules/video_coding/include/video_coding_defines.h"
+#include "webrtc/modules/video_coding/inter_frame_delay.h"
 
 namespace webrtc {
 
@@ -36,7 +38,7 @@ class FrameBuffer {
  public:
   FrameBuffer(Clock* clock,
               VCMJitterEstimator* jitter_estimator,
-              const VCMTiming* timing);
+              VCMTiming* timing);
 
   // Insert a frame into the frame buffer.
   void InsertFrame(std::unique_ptr<FrameObject> frame);
@@ -46,6 +48,12 @@ class FrameBuffer {
   // unique ptr if there is no available frame for decoding.
   std::unique_ptr<FrameObject> NextFrame(int64_t max_wait_time_ms);
 
+  // Tells the FrameBuffer which protection mode that is in use. Affects
+  // the frame timing.
+  // TODO(philipel): Remove this when new timing calculations has been
+  //                 implemented.
+  void SetProtectionMode(VCMVideoProtection mode);
+
   // Start the frame buffer, has no effect if the frame buffer is started.
   // The frame buffer is started upon construction.
   void Start();
@@ -78,10 +86,12 @@ class FrameBuffer {
   rtc::CriticalSection crit_;
   Clock* const clock_;
   rtc::Event frame_inserted_event_;
-  VCMJitterEstimator* const jitter_estimator_;
+  VCMJitterEstimator* const jitter_estimator_ GUARDED_BY(crit_);
-  const VCMTiming* const timing_;
+  VCMTiming* const timing_ GUARDED_BY(crit_);
+  VCMInterFrameDelay inter_frame_delay_ GUARDED_BY(crit_);
   int newest_picture_id_ GUARDED_BY(crit_);
   bool stopped_ GUARDED_BY(crit_);
+  VCMVideoProtection protection_mode_ GUARDED_BY(crit_);
 
   RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(FrameBuffer);
 };

webrtc/modules/video_coding/frame_buffer2_unittest.cc

@@ -70,6 +70,7 @@ class VCMJitterEstimatorMock : public VCMJitterEstimator {
                void(int64_t frameDelayMs,
                     uint32_t frameSizeBytes,
                     bool incompleteFrame));
+  MOCK_METHOD1(GetJitterEstimate, int(double rttMultiplier));
 };
 
 class FrameObjectFake : public FrameObject {
@@ -178,7 +179,7 @@ class TestFrameBuffer2 : public ::testing::Test {
 
   SimulatedClock clock_;
   VCMTimingFake timing_;
-  VCMJitterEstimatorMock jitter_estimator_;
+  ::testing::NiceMock<VCMJitterEstimatorMock> jitter_estimator_;
   FrameBuffer buffer_;
   std::vector<std::unique_ptr<FrameObject>> frames_;
   Random rand_;
@@ -336,5 +337,19 @@ TEST_F(TestFrameBuffer2, InsertLateFrame) {
   CheckNoFrame(2);
 }
 
+TEST_F(TestFrameBuffer2, ProtectionMode) {
+  uint16_t pid = Rand();
+  uint32_t ts = Rand();
+
+  EXPECT_CALL(jitter_estimator_, GetJitterEstimate(1.0));
+  InsertFrame(pid, 0, ts, false);
+  ExtractFrame();
+
+  buffer_.SetProtectionMode(kProtectionNackFEC);
+  EXPECT_CALL(jitter_estimator_, GetJitterEstimate(0.0));
+  InsertFrame(pid + 1, 0, ts, false);
+  ExtractFrame();
+}
+
 }  // namespace video_coding
 }  // namespace webrtc

webrtc/modules/video_coding/jitter_estimator.h

@@ -50,7 +50,7 @@ class VCMJitterEstimator {
   //          - rttMultiplier  : RTT param multiplier (when applicable).
   //
   // Return value                   : Jitter estimate in milliseconds
-  int GetJitterEstimate(double rttMultiplier);
+  virtual int GetJitterEstimate(double rttMultiplier);
 
   // Updates the nack counter.
   void FrameNacked();