Move VP9 frame rate controller to separate class.

Bug: webrtc:9669
Change-Id: I6f30587778e9783182af11d2410464024918e171
Reviewed-on: https://webrtc-review.googlesource.com/96201
Commit-Queue: Sergey Silkin <ssilkin@webrtc.org>
Reviewed-by: Erik Språng <sprang@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#24487}

modules/video_coding/BUILD.gn

@@ -234,6 +234,8 @@ rtc_source_set("video_coding_utility") {
     "utility/default_video_bitrate_allocator.h",
     "utility/frame_dropper.cc",
     "utility/frame_dropper.h",
+    "utility/framerate_controller.cc",
+    "utility/framerate_controller.h",
     "utility/ivf_file_writer.cc",
     "utility/ivf_file_writer.h",
     "utility/moving_average.cc",
@@ -799,6 +801,7 @@ if (rtc_include_tests) {
       "timing_unittest.cc",
       "utility/default_video_bitrate_allocator_unittest.cc",
       "utility/frame_dropper_unittest.cc",
+      "utility/framerate_controller_unittest.cc",
       "utility/ivf_file_writer_unittest.cc",
       "utility/mock/mock_frame_dropper.h",
       "utility/moving_average_unittest.cc",

modules/video_coding/codecs/vp9/vp9_impl.cc

@@ -157,8 +157,7 @@ VP9EncoderImpl::VP9EncoderImpl(const cricket::VideoCodec& codec)
       num_spatial_layers_(0),
       is_svc_(false),
       inter_layer_pred_(InterLayerPredMode::kOn),
-      output_framerate_(1000.0, 1000.0),
+      framerate_controller_(kMaxScreenSharingFramerateFps),
-      last_encoded_frame_rtp_timestamp_(0),
       is_flexible_mode_(false) {
   memset(&codec_, 0, sizeof(codec_));
   memset(&svc_params_, 0, sizeof(vpx_svc_extra_cfg_t));
@@ -358,11 +357,9 @@ int VP9EncoderImpl::InitEncode(const VideoCodec* inst,
     num_temporal_layers_ = 1;
 
   // Init framerate controller.
-  output_framerate_.Reset();
   if (codec_.mode == VideoCodecMode::kScreensharing) {
-    target_framerate_fps_ = kMaxScreenSharingFramerateFps;
+    framerate_controller_.Reset();
-  } else {
+    framerate_controller_.SetTargetRate(kMaxScreenSharingFramerateFps);
-    target_framerate_fps_.reset();
   }
 
   is_svc_ = (num_spatial_layers_ > 1 || num_temporal_layers_ > 1);
@@ -672,7 +669,8 @@ int VP9EncoderImpl::Encode(const VideoFrame& input_image,
   }
 
   if (VideoCodecMode::kScreensharing == codec_.mode && !force_key_frame_) {
-    if (DropFrame(input_image.timestamp())) {
+    if (framerate_controller_.DropFrame(1000 * input_image.timestamp() /
+                                        kVideoPayloadTypeFrequency)) {
       return WEBRTC_VIDEO_CODEC_OK;
     }
   }
@@ -734,8 +732,10 @@ int VP9EncoderImpl::Encode(const VideoFrame& input_image,
   }
 
   RTC_CHECK_GT(codec_.maxFramerate, 0);
-  uint32_t duration =
+  uint32_t target_framerate_fps = codec_.mode == VideoCodecMode::kScreensharing
-      90000 / target_framerate_fps_.value_or(codec_.maxFramerate);
+                                      ? kMaxScreenSharingFramerateFps
+                                      : codec_.maxFramerate;
+  uint32_t duration = 90000 / target_framerate_fps;
   const vpx_codec_err_t rv = vpx_codec_encode(encoder_, raw_, timestamp_,
                                               duration, flags, VPX_DL_REALTIME);
   if (rv != VPX_CODEC_OK) {
@@ -1064,45 +1064,14 @@ void VP9EncoderImpl::DeliverBufferedFrame(bool end_of_picture) {
                                                &frag_info);
     encoded_image_._length = 0;
 
-    if (end_of_picture) {
+    if (end_of_picture && codec_.mode == VideoCodecMode::kScreensharing) {
       const uint32_t timestamp_ms =
           1000 * encoded_image_.Timestamp() / kVideoPayloadTypeFrequency;
-      output_framerate_.Update(1, timestamp_ms);
+      framerate_controller_.AddFrame(timestamp_ms);
-      last_encoded_frame_rtp_timestamp_ = encoded_image_.Timestamp();
     }
   }
 }
 
-bool VP9EncoderImpl::DropFrame(uint32_t rtp_timestamp) {
-  if (target_framerate_fps_) {
-    if (rtp_timestamp < last_encoded_frame_rtp_timestamp_) {
-      // Timestamp has wrapped around. Reset framerate statistic.
-      output_framerate_.Reset();
-      return false;
-    }
-
-    const uint32_t timestamp_ms =
-        1000 * rtp_timestamp / kVideoPayloadTypeFrequency;
-    const uint32_t framerate_fps =
-        output_framerate_.Rate(timestamp_ms).value_or(0);
-    if (framerate_fps > *target_framerate_fps_) {
-      return true;
-    }
-
-    // Primarily check if frame interval is too short using frame timestamps,
-    // as if they are correct they won't be affected by queuing in webrtc.
-    const uint32_t expected_frame_interval =
-        kVideoPayloadTypeFrequency / *target_framerate_fps_;
-
-    const uint32_t ts_diff = rtp_timestamp - last_encoded_frame_rtp_timestamp_;
-    if (ts_diff < 85 * expected_frame_interval / 100) {
-      return true;
-    }
-  }
-
-  return false;
-}
-
 int VP9EncoderImpl::SetChannelParameters(uint32_t packet_loss, int64_t rtt) {
   return WEBRTC_VIDEO_CODEC_OK;
 }

modules/video_coding/codecs/vp9/vp9_impl.h

@@ -20,7 +20,7 @@
 
 #include "media/base/vp9_profile.h"
 #include "modules/video_coding/codecs/vp9/vp9_frame_buffer_pool.h"
-#include "rtc_base/rate_statistics.h"
+#include "modules/video_coding/utility/framerate_controller.h"
 
 #include "vpx/vp8cx.h"
 #include "vpx/vpx_decoder.h"
@@ -82,7 +82,7 @@ class VP9EncoderImpl : public VP9Encoder {
 
   void DeliverBufferedFrame(bool end_of_picture);
 
-  bool DropFrame(uint32_t rtp_timestamp);
+  bool DropFrame(uint8_t spatial_idx, uint32_t rtp_timestamp);
 
   // Determine maximum target for Intra frames
   //
@@ -117,9 +117,7 @@ class VP9EncoderImpl : public VP9Encoder {
   InterLayerPredMode inter_layer_pred_;
 
   // Framerate controller.
-  absl::optional<float> target_framerate_fps_;
+  FramerateController framerate_controller_;
-  RateStatistics output_framerate_;
-  uint32_t last_encoded_frame_rtp_timestamp_;
 
   // Used for flexible mode.
   bool is_flexible_mode_;

modules/video_coding/utility/framerate_controller.cc

@@ -0,0 +1,81 @@
+/*
+ *  Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/video_coding/utility/framerate_controller.h"
+
+#include "rtc_base/checks.h"
+
+namespace webrtc {
+
+FramerateController::FramerateController(float target_framerate_fps)
+    : min_frame_interval_ms_(0), framerate_estimator_(1000.0, 1000.0) {
+  SetTargetRate(target_framerate_fps);
+}
+
+void FramerateController::SetTargetRate(float target_framerate_fps) {
+  if (target_framerate_fps_ != target_framerate_fps) {
+    framerate_estimator_.Reset();
+    if (last_timestamp_ms_) {
+      framerate_estimator_.Update(1, *last_timestamp_ms_);
+    }
+
+    const size_t target_frame_interval_ms = 1000 / target_framerate_fps;
+    target_framerate_fps_ = target_framerate_fps;
+    min_frame_interval_ms_ = 85 * target_frame_interval_ms / 100;
+  }
+}
+
+float FramerateController::GetTargetRate() {
+  return *target_framerate_fps_;
+}
+
+void FramerateController::Reset() {
+  framerate_estimator_.Reset();
+  last_timestamp_ms_.reset();
+}
+
+bool FramerateController::DropFrame(uint32_t timestamp_ms) const {
+  if (timestamp_ms < last_timestamp_ms_) {
+    // Timestamp jumps backward. We can't make adequate drop decision. Don't
+    // drop this frame. Stats will be reset in AddFrame().
+    return false;
+  }
+
+  if (Rate(timestamp_ms).value_or(*target_framerate_fps_) >
+      target_framerate_fps_) {
+    return true;
+  }
+
+  if (last_timestamp_ms_) {
+    const int64_t diff_ms =
+        static_cast<int64_t>(timestamp_ms) - *last_timestamp_ms_;
+    if (diff_ms < min_frame_interval_ms_) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
+void FramerateController::AddFrame(uint32_t timestamp_ms) {
+  if (timestamp_ms < last_timestamp_ms_) {
+    // Timestamp jumps backward.
+    Reset();
+  }
+
+  framerate_estimator_.Update(1, timestamp_ms);
+  last_timestamp_ms_ = timestamp_ms;
+}
+
+absl::optional<float> FramerateController::Rate(uint32_t timestamp_ms) const {
+  return framerate_estimator_.Rate(timestamp_ms);
+}
+
+}  // namespace webrtc

modules/video_coding/utility/framerate_controller.h

@@ -0,0 +1,44 @@
+/*
+ *  Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_VIDEO_CODING_UTILITY_FRAMERATE_CONTROLLER_H_
+#define MODULES_VIDEO_CODING_UTILITY_FRAMERATE_CONTROLLER_H_
+
+#include "absl/types/optional.h"
+#include "rtc_base/rate_statistics.h"
+
+namespace webrtc {
+
+class FramerateController {
+ public:
+  explicit FramerateController(float target_framerate_fps);
+
+  void SetTargetRate(float target_framerate_fps);
+  float GetTargetRate();
+
+  // Advices user to drop next frame in order to reach target framerate.
+  bool DropFrame(uint32_t timestamp_ms) const;
+
+  void AddFrame(uint32_t timestamp_ms);
+
+  void Reset();
+
+ private:
+  absl::optional<float> Rate(uint32_t timestamp_ms) const;
+
+  absl::optional<float> target_framerate_fps_;
+  absl::optional<uint32_t> last_timestamp_ms_;
+  uint32_t min_frame_interval_ms_;
+  RateStatistics framerate_estimator_;
+};
+
+}  // namespace webrtc
+
+#endif  // MODULES_VIDEO_CODING_UTILITY_FRAMERATE_CONTROLLER_H_

modules/video_coding/utility/framerate_controller_unittest.cc

@@ -0,0 +1,88 @@
+/*
+ *  Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/video_coding/utility/framerate_controller.h"
+
+#include "test/gtest.h"
+
+namespace webrtc {
+
+TEST(FramerateController, KeepTargetFramerate) {
+  const float input_framerate_fps = 20;
+  const float target_framerate_fps = 5;
+  const float max_abs_framerate_error_fps = target_framerate_fps * 0.1f;
+  const size_t input_duration_secs = 3;
+  const size_t num_input_frames = input_duration_secs * input_framerate_fps;
+
+  FramerateController framerate_controller(target_framerate_fps);
+  size_t num_dropped_frames = 0;
+  for (size_t frame_num = 0; frame_num < num_input_frames; ++frame_num) {
+    const uint32_t timestamp_ms =
+        static_cast<uint32_t>(1000 * frame_num / input_framerate_fps);
+    if (framerate_controller.DropFrame(timestamp_ms)) {
+      ++num_dropped_frames;
+    } else {
+      framerate_controller.AddFrame(timestamp_ms);
+    }
+  }
+
+  const float output_framerate_fps =
+      static_cast<float>(num_input_frames - num_dropped_frames) /
+      input_duration_secs;
+  EXPECT_NEAR(output_framerate_fps, target_framerate_fps,
+              max_abs_framerate_error_fps);
+}
+
+TEST(FramerateController, DoNotDropAnyFramesIfTargerEqualsInput) {
+  const float input_framerate_fps = 30;
+  const size_t input_duration_secs = 3;
+  const size_t num_input_frames = input_duration_secs * input_framerate_fps;
+
+  FramerateController framerate_controller(input_framerate_fps);
+  size_t num_dropped_frames = 0;
+  for (size_t frame_num = 0; frame_num < num_input_frames; ++frame_num) {
+    const uint32_t timestamp_ms =
+        static_cast<uint32_t>(1000 * frame_num / input_framerate_fps);
+    if (framerate_controller.DropFrame(timestamp_ms)) {
+      ++num_dropped_frames;
+    } else {
+      framerate_controller.AddFrame(timestamp_ms);
+    }
+  }
+
+  EXPECT_EQ(num_dropped_frames, 0U);
+}
+
+TEST(FramerateController, DoNotDropFrameWhenTimestampJumpsBackward) {
+  FramerateController framerate_controller(30);
+  ASSERT_FALSE(framerate_controller.DropFrame(66));
+  framerate_controller.AddFrame(66);
+  EXPECT_FALSE(framerate_controller.DropFrame(33));
+}
+
+TEST(FramerateController, DropFrameIfItIsTooCloseToPreviousFrame) {
+  FramerateController framerate_controller(30);
+  ASSERT_FALSE(framerate_controller.DropFrame(33));
+  framerate_controller.AddFrame(33);
+  EXPECT_TRUE(framerate_controller.DropFrame(34));
+}
+
+TEST(FramerateController, FrameDroppingStartsFromSecondInputFrame) {
+  const float input_framerate_fps = 23;
+  const float target_framerate_fps = 19;
+  const uint32_t input_frame_duration_ms =
+      static_cast<uint32_t>(1000 / input_framerate_fps);
+  FramerateController framerate_controller(target_framerate_fps);
+  ASSERT_FALSE(framerate_controller.DropFrame(1 * input_frame_duration_ms));
+  framerate_controller.AddFrame(1 * input_frame_duration_ms);
+  EXPECT_TRUE(framerate_controller.DropFrame(2 * input_frame_duration_ms));
+}
+
+}  // namespace webrtc