diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn index 0b760209d6..d4594f66d1 100644 --- a/modules/video_coding/BUILD.gn +++ b/modules/video_coding/BUILD.gn @@ -234,6 +234,8 @@ rtc_source_set("video_coding_utility") { "utility/default_video_bitrate_allocator.h", "utility/frame_dropper.cc", "utility/frame_dropper.h", + "utility/framerate_controller.cc", + "utility/framerate_controller.h", "utility/ivf_file_writer.cc", "utility/ivf_file_writer.h", "utility/moving_average.cc", @@ -799,6 +801,7 @@ if (rtc_include_tests) { "timing_unittest.cc", "utility/default_video_bitrate_allocator_unittest.cc", "utility/frame_dropper_unittest.cc", + "utility/framerate_controller_unittest.cc", "utility/ivf_file_writer_unittest.cc", "utility/mock/mock_frame_dropper.h", "utility/moving_average_unittest.cc", diff --git a/modules/video_coding/codecs/vp9/vp9_impl.cc b/modules/video_coding/codecs/vp9/vp9_impl.cc index b4c13f9ccf..580556392b 100644 --- a/modules/video_coding/codecs/vp9/vp9_impl.cc +++ b/modules/video_coding/codecs/vp9/vp9_impl.cc @@ -157,8 +157,7 @@ VP9EncoderImpl::VP9EncoderImpl(const cricket::VideoCodec& codec) num_spatial_layers_(0), is_svc_(false), inter_layer_pred_(InterLayerPredMode::kOn), - output_framerate_(1000.0, 1000.0), - last_encoded_frame_rtp_timestamp_(0), + framerate_controller_(kMaxScreenSharingFramerateFps), is_flexible_mode_(false) { memset(&codec_, 0, sizeof(codec_)); memset(&svc_params_, 0, sizeof(vpx_svc_extra_cfg_t)); @@ -358,11 +357,9 @@ int VP9EncoderImpl::InitEncode(const VideoCodec* inst, num_temporal_layers_ = 1; // Init framerate controller. - output_framerate_.Reset(); if (codec_.mode == VideoCodecMode::kScreensharing) { - target_framerate_fps_ = kMaxScreenSharingFramerateFps; - } else { - target_framerate_fps_.reset(); + framerate_controller_.Reset(); + framerate_controller_.SetTargetRate(kMaxScreenSharingFramerateFps); } is_svc_ = (num_spatial_layers_ > 1 || num_temporal_layers_ > 1); @@ -672,7 +669,8 @@ int VP9EncoderImpl::Encode(const VideoFrame& input_image, } if (VideoCodecMode::kScreensharing == codec_.mode && !force_key_frame_) { - if (DropFrame(input_image.timestamp())) { + if (framerate_controller_.DropFrame(1000 * input_image.timestamp() / + kVideoPayloadTypeFrequency)) { return WEBRTC_VIDEO_CODEC_OK; } } @@ -734,8 +732,10 @@ int VP9EncoderImpl::Encode(const VideoFrame& input_image, } RTC_CHECK_GT(codec_.maxFramerate, 0); - uint32_t duration = - 90000 / target_framerate_fps_.value_or(codec_.maxFramerate); + uint32_t target_framerate_fps = codec_.mode == VideoCodecMode::kScreensharing + ? kMaxScreenSharingFramerateFps + : codec_.maxFramerate; + uint32_t duration = 90000 / target_framerate_fps; const vpx_codec_err_t rv = vpx_codec_encode(encoder_, raw_, timestamp_, duration, flags, VPX_DL_REALTIME); if (rv != VPX_CODEC_OK) { @@ -1064,45 +1064,14 @@ void VP9EncoderImpl::DeliverBufferedFrame(bool end_of_picture) { &frag_info); encoded_image_._length = 0; - if (end_of_picture) { + if (end_of_picture && codec_.mode == VideoCodecMode::kScreensharing) { const uint32_t timestamp_ms = 1000 * encoded_image_.Timestamp() / kVideoPayloadTypeFrequency; - output_framerate_.Update(1, timestamp_ms); - last_encoded_frame_rtp_timestamp_ = encoded_image_.Timestamp(); + framerate_controller_.AddFrame(timestamp_ms); } } } -bool VP9EncoderImpl::DropFrame(uint32_t rtp_timestamp) { - if (target_framerate_fps_) { - if (rtp_timestamp < last_encoded_frame_rtp_timestamp_) { - // Timestamp has wrapped around. Reset framerate statistic. - output_framerate_.Reset(); - return false; - } - - const uint32_t timestamp_ms = - 1000 * rtp_timestamp / kVideoPayloadTypeFrequency; - const uint32_t framerate_fps = - output_framerate_.Rate(timestamp_ms).value_or(0); - if (framerate_fps > *target_framerate_fps_) { - return true; - } - - // Primarily check if frame interval is too short using frame timestamps, - // as if they are correct they won't be affected by queuing in webrtc. - const uint32_t expected_frame_interval = - kVideoPayloadTypeFrequency / *target_framerate_fps_; - - const uint32_t ts_diff = rtp_timestamp - last_encoded_frame_rtp_timestamp_; - if (ts_diff < 85 * expected_frame_interval / 100) { - return true; - } - } - - return false; -} - int VP9EncoderImpl::SetChannelParameters(uint32_t packet_loss, int64_t rtt) { return WEBRTC_VIDEO_CODEC_OK; } diff --git a/modules/video_coding/codecs/vp9/vp9_impl.h b/modules/video_coding/codecs/vp9/vp9_impl.h index d4483060ee..e8fd6062b8 100644 --- a/modules/video_coding/codecs/vp9/vp9_impl.h +++ b/modules/video_coding/codecs/vp9/vp9_impl.h @@ -20,7 +20,7 @@ #include "media/base/vp9_profile.h" #include "modules/video_coding/codecs/vp9/vp9_frame_buffer_pool.h" -#include "rtc_base/rate_statistics.h" +#include "modules/video_coding/utility/framerate_controller.h" #include "vpx/vp8cx.h" #include "vpx/vpx_decoder.h" @@ -82,7 +82,7 @@ class VP9EncoderImpl : public VP9Encoder { void DeliverBufferedFrame(bool end_of_picture); - bool DropFrame(uint32_t rtp_timestamp); + bool DropFrame(uint8_t spatial_idx, uint32_t rtp_timestamp); // Determine maximum target for Intra frames // @@ -117,9 +117,7 @@ class VP9EncoderImpl : public VP9Encoder { InterLayerPredMode inter_layer_pred_; // Framerate controller. - absl::optional target_framerate_fps_; - RateStatistics output_framerate_; - uint32_t last_encoded_frame_rtp_timestamp_; + FramerateController framerate_controller_; // Used for flexible mode. bool is_flexible_mode_; diff --git a/modules/video_coding/utility/framerate_controller.cc b/modules/video_coding/utility/framerate_controller.cc new file mode 100644 index 0000000000..ce8ab12712 --- /dev/null +++ b/modules/video_coding/utility/framerate_controller.cc @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/utility/framerate_controller.h" + +#include "rtc_base/checks.h" + +namespace webrtc { + +FramerateController::FramerateController(float target_framerate_fps) + : min_frame_interval_ms_(0), framerate_estimator_(1000.0, 1000.0) { + SetTargetRate(target_framerate_fps); +} + +void FramerateController::SetTargetRate(float target_framerate_fps) { + if (target_framerate_fps_ != target_framerate_fps) { + framerate_estimator_.Reset(); + if (last_timestamp_ms_) { + framerate_estimator_.Update(1, *last_timestamp_ms_); + } + + const size_t target_frame_interval_ms = 1000 / target_framerate_fps; + target_framerate_fps_ = target_framerate_fps; + min_frame_interval_ms_ = 85 * target_frame_interval_ms / 100; + } +} + +float FramerateController::GetTargetRate() { + return *target_framerate_fps_; +} + +void FramerateController::Reset() { + framerate_estimator_.Reset(); + last_timestamp_ms_.reset(); +} + +bool FramerateController::DropFrame(uint32_t timestamp_ms) const { + if (timestamp_ms < last_timestamp_ms_) { + // Timestamp jumps backward. We can't make adequate drop decision. Don't + // drop this frame. Stats will be reset in AddFrame(). + return false; + } + + if (Rate(timestamp_ms).value_or(*target_framerate_fps_) > + target_framerate_fps_) { + return true; + } + + if (last_timestamp_ms_) { + const int64_t diff_ms = + static_cast(timestamp_ms) - *last_timestamp_ms_; + if (diff_ms < min_frame_interval_ms_) { + return true; + } + } + + return false; +} + +void FramerateController::AddFrame(uint32_t timestamp_ms) { + if (timestamp_ms < last_timestamp_ms_) { + // Timestamp jumps backward. + Reset(); + } + + framerate_estimator_.Update(1, timestamp_ms); + last_timestamp_ms_ = timestamp_ms; +} + +absl::optional FramerateController::Rate(uint32_t timestamp_ms) const { + return framerate_estimator_.Rate(timestamp_ms); +} + +} // namespace webrtc diff --git a/modules/video_coding/utility/framerate_controller.h b/modules/video_coding/utility/framerate_controller.h new file mode 100644 index 0000000000..63493eaae2 --- /dev/null +++ b/modules/video_coding/utility/framerate_controller.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CODING_UTILITY_FRAMERATE_CONTROLLER_H_ +#define MODULES_VIDEO_CODING_UTILITY_FRAMERATE_CONTROLLER_H_ + +#include "absl/types/optional.h" +#include "rtc_base/rate_statistics.h" + +namespace webrtc { + +class FramerateController { + public: + explicit FramerateController(float target_framerate_fps); + + void SetTargetRate(float target_framerate_fps); + float GetTargetRate(); + + // Advices user to drop next frame in order to reach target framerate. + bool DropFrame(uint32_t timestamp_ms) const; + + void AddFrame(uint32_t timestamp_ms); + + void Reset(); + + private: + absl::optional Rate(uint32_t timestamp_ms) const; + + absl::optional target_framerate_fps_; + absl::optional last_timestamp_ms_; + uint32_t min_frame_interval_ms_; + RateStatistics framerate_estimator_; +}; + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_UTILITY_FRAMERATE_CONTROLLER_H_ diff --git a/modules/video_coding/utility/framerate_controller_unittest.cc b/modules/video_coding/utility/framerate_controller_unittest.cc new file mode 100644 index 0000000000..6c06ea3cec --- /dev/null +++ b/modules/video_coding/utility/framerate_controller_unittest.cc @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/utility/framerate_controller.h" + +#include "test/gtest.h" + +namespace webrtc { + +TEST(FramerateController, KeepTargetFramerate) { + const float input_framerate_fps = 20; + const float target_framerate_fps = 5; + const float max_abs_framerate_error_fps = target_framerate_fps * 0.1f; + const size_t input_duration_secs = 3; + const size_t num_input_frames = input_duration_secs * input_framerate_fps; + + FramerateController framerate_controller(target_framerate_fps); + size_t num_dropped_frames = 0; + for (size_t frame_num = 0; frame_num < num_input_frames; ++frame_num) { + const uint32_t timestamp_ms = + static_cast(1000 * frame_num / input_framerate_fps); + if (framerate_controller.DropFrame(timestamp_ms)) { + ++num_dropped_frames; + } else { + framerate_controller.AddFrame(timestamp_ms); + } + } + + const float output_framerate_fps = + static_cast(num_input_frames - num_dropped_frames) / + input_duration_secs; + EXPECT_NEAR(output_framerate_fps, target_framerate_fps, + max_abs_framerate_error_fps); +} + +TEST(FramerateController, DoNotDropAnyFramesIfTargerEqualsInput) { + const float input_framerate_fps = 30; + const size_t input_duration_secs = 3; + const size_t num_input_frames = input_duration_secs * input_framerate_fps; + + FramerateController framerate_controller(input_framerate_fps); + size_t num_dropped_frames = 0; + for (size_t frame_num = 0; frame_num < num_input_frames; ++frame_num) { + const uint32_t timestamp_ms = + static_cast(1000 * frame_num / input_framerate_fps); + if (framerate_controller.DropFrame(timestamp_ms)) { + ++num_dropped_frames; + } else { + framerate_controller.AddFrame(timestamp_ms); + } + } + + EXPECT_EQ(num_dropped_frames, 0U); +} + +TEST(FramerateController, DoNotDropFrameWhenTimestampJumpsBackward) { + FramerateController framerate_controller(30); + ASSERT_FALSE(framerate_controller.DropFrame(66)); + framerate_controller.AddFrame(66); + EXPECT_FALSE(framerate_controller.DropFrame(33)); +} + +TEST(FramerateController, DropFrameIfItIsTooCloseToPreviousFrame) { + FramerateController framerate_controller(30); + ASSERT_FALSE(framerate_controller.DropFrame(33)); + framerate_controller.AddFrame(33); + EXPECT_TRUE(framerate_controller.DropFrame(34)); +} + +TEST(FramerateController, FrameDroppingStartsFromSecondInputFrame) { + const float input_framerate_fps = 23; + const float target_framerate_fps = 19; + const uint32_t input_frame_duration_ms = + static_cast(1000 / input_framerate_fps); + FramerateController framerate_controller(target_framerate_fps); + ASSERT_FALSE(framerate_controller.DropFrame(1 * input_frame_duration_ms)); + framerate_controller.AddFrame(1 * input_frame_duration_ms); + EXPECT_TRUE(framerate_controller.DropFrame(2 * input_frame_duration_ms)); +} + +} // namespace webrtc