From 639602ace347cdc6e221835d46caf2b31de1f46f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?K=C3=A1ri=20Tristan=20Helgason?= Date: Thu, 2 Aug 2018 10:51:40 +0200 Subject: [PATCH] Activate initial framedrop when first BW estimate arrives MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bug: webrtc:9176, webrtc:6086 Change-Id: I420f8a0c6191697f7b50aaf780cf90a4ea365739 Reviewed-on: https://webrtc-review.googlesource.com/79580 Commit-Queue: Kári Helgason Reviewed-by: Åsa Persson Reviewed-by: Sebastian Jansson Reviewed-by: Niels Moller Cr-Commit-Position: refs/heads/master@{#24306} --- video/video_stream_encoder.cc | 40 ++++++++++++++++++++------ video/video_stream_encoder.h | 6 ++-- video/video_stream_encoder_unittest.cc | 25 ++++++++++++++++ 3 files changed, 61 insertions(+), 10 deletions(-) diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index b0d450f67e..4bf3a86016 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -27,6 +27,7 @@ #include "rtc_base/system/fallthrough.h" #include "rtc_base/timeutils.h" #include "rtc_base/trace_event.h" +#include "system_wrappers/include/field_trial.h" #include "video/overuse_frame_detector.h" namespace webrtc { @@ -41,9 +42,14 @@ const int kMaxFramerateFps = 120; // Time to keep a single cached pending frame in paused state. const int64_t kPendingFrameTimeoutMs = 1000; +const char kInitialFramedropFieldTrial[] = "WebRTC-InitialFramedrop"; + // The maximum number of frames to drop at beginning of stream // to try and achieve desired bitrate. const int kMaxInitialFramedrop = 4; +// When the first change in BWE above this threshold occurs, +// enable DropFrameDueToSize logic. +const float kFramedropThreshold = 0.3; // Initial limits for BALANCED degradation preference. int MinFps(int pixels) { @@ -68,6 +74,10 @@ int MaxFps(int pixels) { } } +uint32_t abs_diff(uint32_t a, uint32_t b) { + return (a < b) ? b - a : a - b; +} + bool IsResolutionScalingEnabled(DegradationPreference degradation_preference) { return degradation_preference == DegradationPreference::MAINTAIN_FRAMERATE || degradation_preference == DegradationPreference::BALANCED; @@ -318,7 +328,9 @@ VideoStreamEncoder::VideoStreamEncoder( std::unique_ptr overuse_detector) : shutdown_event_(true /* manual_reset */, false), number_of_cores_(number_of_cores), - initial_rampup_(0), + initial_framedrop_(0), + initial_framedrop_on_bwe_enabled_( + webrtc::field_trial::IsEnabled(kInitialFramedropFieldTrial)), quality_scaling_experiment_enabled_(QualityScalingExperiment::Enabled()), source_proxy_(new VideoSourceProxy(this)), sink_(nullptr), @@ -401,8 +413,6 @@ void VideoStreamEncoder::SetSource( } } degradation_preference_ = degradation_preference; - bool allow_scaling = IsResolutionScalingEnabled(degradation_preference_); - initial_rampup_ = allow_scaling ? 0 : kMaxInitialFramedrop; if (encoder_) ConfigureQualityScaler(); @@ -600,7 +610,6 @@ void VideoStreamEncoder::ConfigureQualityScaler() { if (quality_scaling_allowed) { if (quality_scaler_.get() == nullptr) { // Quality scaler has not already been configured. - // Drop frames and scale down until desired quality is achieved. // Use experimental thresholds if available. absl::optional experimental_thresholds; @@ -614,10 +623,12 @@ void VideoStreamEncoder::ConfigureQualityScaler() { quality_scaler_ = absl::make_unique( observer, experimental_thresholds ? *experimental_thresholds : *(scaling_settings.thresholds)); + has_seen_first_significant_bwe_change_ = false; + initial_framedrop_ = 0; } } else { quality_scaler_.reset(nullptr); - initial_rampup_ = kMaxInitialFramedrop; + initial_framedrop_ = kMaxInitialFramedrop; } encoder_stats_observer_->OnAdaptationChanged( @@ -782,7 +793,7 @@ void VideoStreamEncoder::MaybeEncodeVideoFrame(const VideoFrame& video_frame, if (GetConstAdaptCounter().ResolutionCount(kQuality) > count) { encoder_stats_observer_->OnInitialQualityResolutionAdaptDown(); } - ++initial_rampup_; + ++initial_framedrop_; // Storing references to a native buffer risks blocking frame capture. if (video_frame.video_frame_buffer()->type() != VideoFrameBuffer::Type::kNative) { @@ -794,7 +805,7 @@ void VideoStreamEncoder::MaybeEncodeVideoFrame(const VideoFrame& video_frame, } return; } - initial_rampup_ = kMaxInitialFramedrop; + initial_framedrop_ = kMaxInitialFramedrop; if (EncoderPaused()) { // Storing references to a native buffer risks blocking frame capture. @@ -941,6 +952,19 @@ void VideoStreamEncoder::OnBitrateUpdated(uint32_t bitrate_bps, RTC_LOG(LS_VERBOSE) << "OnBitrateUpdated, bitrate " << bitrate_bps << " packet loss " << static_cast(fraction_lost) << " rtt " << round_trip_time_ms; + // On significant changes to BWE at the start of the call, + // enable frame drops to quickly react to jumps in available bandwidth. + if (encoder_start_bitrate_bps_ != 0 && + !has_seen_first_significant_bwe_change_ && quality_scaler_ && + initial_framedrop_on_bwe_enabled_ && + abs_diff(bitrate_bps, encoder_start_bitrate_bps_) >= + kFramedropThreshold * encoder_start_bitrate_bps_) { + // Reset initial framedrop feature when first real BW estimate arrives. + // TODO(kthelgason): Update BitrateAllocator to not call OnBitrateUpdated + // without an actual BW estimate. + initial_framedrop_ = 0; + has_seen_first_significant_bwe_change_ = true; + } video_sender_.SetChannelParameters(bitrate_bps, fraction_lost, round_trip_time_ms, rate_allocator_.get(), @@ -967,7 +991,7 @@ void VideoStreamEncoder::OnBitrateUpdated(uint32_t bitrate_bps, } bool VideoStreamEncoder::DropDueToSize(uint32_t pixel_count) const { - if (initial_rampup_ < kMaxInitialFramedrop && + if (initial_framedrop_ < kMaxInitialFramedrop && encoder_start_bitrate_bps_ > 0) { if (encoder_start_bitrate_bps_ < 300000 /* qvga */) { return pixel_count > 320 * 240; diff --git a/video/video_stream_encoder.h b/video/video_stream_encoder.h index 8d228a43f6..4008426bac 100644 --- a/video/video_stream_encoder.h +++ b/video/video_stream_encoder.h @@ -178,8 +178,10 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, rtc::Event shutdown_event_; const uint32_t number_of_cores_; - // Counts how many frames we've dropped in the initial rampup phase. - int initial_rampup_; + // Counts how many frames we've dropped in the initial framedrop phase. + int initial_framedrop_; + const bool initial_framedrop_on_bwe_enabled_; + bool has_seen_first_significant_bwe_change_ = false; const bool quality_scaling_experiment_enabled_; diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc index 32053abf54..cda8a05edc 100644 --- a/video/video_stream_encoder_unittest.cc +++ b/video/video_stream_encoder_unittest.cc @@ -25,6 +25,7 @@ #include "test/encoder_proxy_factory.h" #include "test/encoder_settings.h" #include "test/fake_encoder.h" +#include "test/field_trial.h" #include "test/frame_generator.h" #include "test/gmock.h" #include "test/gtest.h" @@ -2334,6 +2335,30 @@ TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) { fake_encoder_.SetQualityScaling(true); } +TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBWEstimateReady) { + webrtc::test::ScopedFieldTrials field_trials( + "WebRTC-InitialFramedrop/Enabled/"); + // Reset encoder for field trials to take effect. + ConfigureEncoder(video_encoder_config_.Copy()); + const int kTooLowBitrateForFrameSizeBps = 10000; + const int kWidth = 640; + const int kHeight = 360; + + video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0); + video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight)); + // Frame should not be dropped. + WaitForEncodedFrame(1); + + video_stream_encoder_->OnBitrateUpdated(kTooLowBitrateForFrameSizeBps, 0, 0); + video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight)); + // Expect to drop this frame, the wait should time out. + ExpectDroppedFrame(); + + // Expect the sink_wants to specify a scaled frame. + EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight); + video_stream_encoder_->Stop(); +} + TEST_F(VideoStreamEncoderTest, ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) { const int kTooSmallWidth = 10;