From 139f4dc7acc9066599702c4bfb73619c6767d1bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=85sa=20Persson?= Date: Fri, 2 Aug 2019 09:29:58 +0200 Subject: [PATCH] QualityScaler: Add option to try fast adapt down at start up based on initial bw estimates. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit optional initial_bitrate_interval_ms: time interval since start of call where fast adapt down is allowed. optional initial_bitrate_factor: try fast adapt down if bw estimate is below initial bitrate * factor. Bug: none Change-Id: I63e1fdaac6556d8e9a961a42e11c925f9ecb9771 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/147725 Reviewed-by: Sergey Silkin Commit-Queue: Åsa Persson Cr-Commit-Position: refs/heads/master@{#28753} --- .../experiments/quality_scaler_settings.cc | 25 ++++++++++- .../experiments/quality_scaler_settings.h | 4 ++ .../quality_scaler_settings_unittest.cc | 41 +++++++++++++++---- video/BUILD.gn | 1 + video/video_stream_encoder.cc | 22 ++++++++++ video/video_stream_encoder.h | 5 +++ video/video_stream_encoder_unittest.cc | 36 ++++++++++++++++ 7 files changed, 125 insertions(+), 9 deletions(-) diff --git a/rtc_base/experiments/quality_scaler_settings.cc b/rtc_base/experiments/quality_scaler_settings.cc index f32a2965a9..c8d83ebe4d 100644 --- a/rtc_base/experiments/quality_scaler_settings.cc +++ b/rtc_base/experiments/quality_scaler_settings.cc @@ -23,9 +23,12 @@ QualityScalerSettings::QualityScalerSettings( const WebRtcKeyValueConfig* const key_value_config) : min_frames_("min_frames"), initial_scale_factor_("initial_scale_factor"), - scale_factor_("scale_factor") { + scale_factor_("scale_factor"), + initial_bitrate_interval_ms_("initial_bitrate_interval_ms"), + initial_bitrate_factor_("initial_bitrate_factor") { ParseFieldTrial( - {&min_frames_, &initial_scale_factor_, &scale_factor_}, + {&min_frames_, &initial_scale_factor_, &scale_factor_, + &initial_bitrate_interval_ms_, &initial_bitrate_factor_}, key_value_config->Lookup("WebRTC-Video-QualityScalerSettings")); } @@ -59,4 +62,22 @@ absl::optional QualityScalerSettings::ScaleFactor() const { return scale_factor_.GetOptional(); } +absl::optional QualityScalerSettings::InitialBitrateIntervalMs() const { + if (initial_bitrate_interval_ms_ && + initial_bitrate_interval_ms_.Value() < 0) { + RTC_LOG(LS_WARNING) << "Unsupported bitrate_interval value, ignored."; + return absl::nullopt; + } + return initial_bitrate_interval_ms_.GetOptional(); +} + +absl::optional QualityScalerSettings::InitialBitrateFactor() const { + if (initial_bitrate_factor_ && + initial_bitrate_factor_.Value() < kMinScaleFactor) { + RTC_LOG(LS_WARNING) << "Unsupported initial_bitrate_factor value, ignored."; + return absl::nullopt; + } + return initial_bitrate_factor_.GetOptional(); +} + } // namespace webrtc diff --git a/rtc_base/experiments/quality_scaler_settings.h b/rtc_base/experiments/quality_scaler_settings.h index 0b26d82332..e3b12c54e3 100644 --- a/rtc_base/experiments/quality_scaler_settings.h +++ b/rtc_base/experiments/quality_scaler_settings.h @@ -24,6 +24,8 @@ class QualityScalerSettings final { absl::optional MinFrames() const; absl::optional InitialScaleFactor() const; absl::optional ScaleFactor() const; + absl::optional InitialBitrateIntervalMs() const; + absl::optional InitialBitrateFactor() const; private: explicit QualityScalerSettings( @@ -32,6 +34,8 @@ class QualityScalerSettings final { FieldTrialOptional min_frames_; FieldTrialOptional initial_scale_factor_; FieldTrialOptional scale_factor_; + FieldTrialOptional initial_bitrate_interval_ms_; + FieldTrialOptional initial_bitrate_factor_; }; } // namespace webrtc diff --git a/rtc_base/experiments/quality_scaler_settings_unittest.cc b/rtc_base/experiments/quality_scaler_settings_unittest.cc index 497c078f85..9da770c1b5 100644 --- a/rtc_base/experiments/quality_scaler_settings_unittest.cc +++ b/rtc_base/experiments/quality_scaler_settings_unittest.cc @@ -17,10 +17,12 @@ namespace webrtc { namespace { TEST(QualityScalerSettingsTest, ValuesNotSetByDefault) { - EXPECT_FALSE(QualityScalerSettings::ParseFromFieldTrials().MinFrames()); - EXPECT_FALSE( - QualityScalerSettings::ParseFromFieldTrials().InitialScaleFactor()); - EXPECT_FALSE(QualityScalerSettings::ParseFromFieldTrials().ScaleFactor()); + const auto settings = QualityScalerSettings::ParseFromFieldTrials(); + EXPECT_FALSE(settings.MinFrames()); + EXPECT_FALSE(settings.InitialScaleFactor()); + EXPECT_FALSE(settings.ScaleFactor()); + EXPECT_FALSE(settings.InitialBitrateIntervalMs()); + EXPECT_FALSE(settings.InitialBitrateFactor()); } TEST(QualityScalerSettingsTest, ParseMinFrames) { @@ -42,34 +44,59 @@ TEST(QualityScalerSettingsTest, ParseScaleFactor) { EXPECT_EQ(1.1, QualityScalerSettings::ParseFromFieldTrials().ScaleFactor()); } +TEST(QualityScalerSettingsTest, ParseInitialBitrateInterval) { + test::ScopedFieldTrials field_trials( + "WebRTC-Video-QualityScalerSettings/initial_bitrate_interval_ms:1000/"); + EXPECT_EQ( + 1000, + QualityScalerSettings::ParseFromFieldTrials().InitialBitrateIntervalMs()); +} + +TEST(QualityScalerSettingsTest, ParseInitialBitrateFactor) { + test::ScopedFieldTrials field_trials( + "WebRTC-Video-QualityScalerSettings/initial_bitrate_factor:0.75/"); + EXPECT_EQ( + 0.75, + QualityScalerSettings::ParseFromFieldTrials().InitialBitrateFactor()); +} + TEST(QualityScalerSettingsTest, ParseAll) { test::ScopedFieldTrials field_trials( "WebRTC-Video-QualityScalerSettings/" - "min_frames:100,initial_scale_factor:1.5,scale_factor:0.9/"); + "min_frames:100,initial_scale_factor:1.5,scale_factor:0.9," + "initial_bitrate_interval_ms:5500,initial_bitrate_factor:0.7/"); const auto settings = QualityScalerSettings::ParseFromFieldTrials(); EXPECT_EQ(100, settings.MinFrames()); EXPECT_EQ(1.5, settings.InitialScaleFactor()); EXPECT_EQ(0.9, settings.ScaleFactor()); + EXPECT_EQ(5500, settings.InitialBitrateIntervalMs()); + EXPECT_EQ(0.7, settings.InitialBitrateFactor()); } TEST(QualityScalerSettingsTest, DoesNotParseIncorrectValue) { test::ScopedFieldTrials field_trials( "WebRTC-Video-QualityScalerSettings/" - "min_frames:a,initial_scale_factor:b,scale_factor:c/"); + "min_frames:a,initial_scale_factor:b,scale_factor:c," + "initial_bitrate_interval_ms:d,initial_bitrate_factor:e/"); const auto settings = QualityScalerSettings::ParseFromFieldTrials(); EXPECT_FALSE(settings.MinFrames()); EXPECT_FALSE(settings.InitialScaleFactor()); EXPECT_FALSE(settings.ScaleFactor()); + EXPECT_FALSE(settings.InitialBitrateIntervalMs()); + EXPECT_FALSE(settings.InitialBitrateFactor()); } TEST(QualityScalerSettingsTest, DoesNotReturnTooSmallValue) { test::ScopedFieldTrials field_trials( "WebRTC-Video-QualityScalerSettings/" - "min_frames:0,initial_scale_factor:0.0,scale_factor:0.0/"); + "min_frames:0,initial_scale_factor:0.0,scale_factor:0.0," + "initial_bitrate_interval_ms:-1,initial_bitrate_factor:0.0/"); const auto settings = QualityScalerSettings::ParseFromFieldTrials(); EXPECT_FALSE(settings.MinFrames()); EXPECT_FALSE(settings.InitialScaleFactor()); EXPECT_FALSE(settings.ScaleFactor()); + EXPECT_FALSE(settings.InitialBitrateIntervalMs()); + EXPECT_FALSE(settings.InitialBitrateFactor()); } } // namespace diff --git a/video/BUILD.gn b/video/BUILD.gn index 7d83ffe7e2..62210538b8 100644 --- a/video/BUILD.gn +++ b/video/BUILD.gn @@ -213,6 +213,7 @@ rtc_source_set("video_stream_encoder_impl") { "../rtc_base/experiments:alr_experiment", "../rtc_base/experiments:balanced_degradation_settings", "../rtc_base/experiments:field_trial_parser", + "../rtc_base/experiments:quality_scaler_settings", "../rtc_base/experiments:quality_scaling_experiment", "../rtc_base/experiments:rate_control_settings", "../rtc_base/synchronization:sequence_checker", diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index f81a178a3b..2bf2d5b248 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -479,6 +479,7 @@ VideoStreamEncoder::VideoStreamEncoder( sink_(nullptr), settings_(settings), rate_control_settings_(RateControlSettings::ParseFromFieldTrials()), + quality_scaler_settings_(QualityScalerSettings::ParseFromFieldTrials()), overuse_detector_(std::move(overuse_detector)), encoder_stats_observer_(encoder_stats_observer), encoder_initialized_(false), @@ -488,6 +489,9 @@ VideoStreamEncoder::VideoStreamEncoder( crop_width_(0), crop_height_(0), encoder_start_bitrate_bps_(0), + set_start_bitrate_bps_(0), + set_start_bitrate_time_ms_(0), + has_seen_first_bwe_drop_(false), max_data_payload_length_(0), encoder_paused_and_dropped_frame_(false), was_encode_called_since_last_initialization_(false), @@ -612,6 +616,8 @@ void VideoStreamEncoder::SetStartBitrate(int start_bitrate_bps) { encoder_queue_.PostTask([this, start_bitrate_bps] { RTC_DCHECK_RUN_ON(&encoder_queue_); encoder_start_bitrate_bps_ = start_bitrate_bps; + set_start_bitrate_bps_ = start_bitrate_bps; + set_start_bitrate_time_ms_ = clock_->TimeInMilliseconds(); }); } @@ -1688,6 +1694,7 @@ void VideoStreamEncoder::OnBitrateUpdated(DataRate target_bitrate, << " link allocation bitrate = " << link_allocation.bps() << " packet loss " << static_cast(fraction_lost) << " rtt " << round_trip_time_ms; + // On significant changes to BWE at the start of the call, // enable frame drops to quickly react to jumps in available bandwidth. if (encoder_start_bitrate_bps_ != 0 && @@ -1701,6 +1708,21 @@ void VideoStreamEncoder::OnBitrateUpdated(DataRate target_bitrate, initial_framedrop_ = 0; has_seen_first_significant_bwe_change_ = true; } + if (set_start_bitrate_bps_ > 0 && !has_seen_first_bwe_drop_ && + quality_scaler_ && quality_scaler_settings_.InitialBitrateIntervalMs() && + quality_scaler_settings_.InitialBitrateFactor()) { + int64_t diff_ms = clock_->TimeInMilliseconds() - set_start_bitrate_time_ms_; + if (diff_ms < quality_scaler_settings_.InitialBitrateIntervalMs().value() && + (target_bitrate.bps() < + (set_start_bitrate_bps_ * + quality_scaler_settings_.InitialBitrateFactor().value()))) { + RTC_LOG(LS_INFO) << "Reset initial_framedrop_. Start bitrate: " + << set_start_bitrate_bps_ + << ", target bitrate: " << target_bitrate.bps(); + initial_framedrop_ = 0; + has_seen_first_bwe_drop_ = true; + } + } if (encoder_) { encoder_->OnPacketLossRateUpdate(static_cast(fraction_lost) / 256.f); diff --git a/video/video_stream_encoder.h b/video/video_stream_encoder.h index 22293ce5a3..3b589bfc76 100644 --- a/video/video_stream_encoder.h +++ b/video/video_stream_encoder.h @@ -31,6 +31,7 @@ #include "rtc_base/critical_section.h" #include "rtc_base/event.h" #include "rtc_base/experiments/balanced_degradation_settings.h" +#include "rtc_base/experiments/quality_scaler_settings.h" #include "rtc_base/experiments/rate_control_settings.h" #include "rtc_base/race_checker.h" #include "rtc_base/rate_statistics.h" @@ -238,6 +239,7 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, EncoderSink* sink_; const VideoStreamEncoderSettings settings_; const RateControlSettings rate_control_settings_; + const QualityScalerSettings quality_scaler_settings_; const std::unique_ptr overuse_detector_ RTC_PT_GUARDED_BY(&encoder_queue_); @@ -271,6 +273,9 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, int crop_width_ RTC_GUARDED_BY(&encoder_queue_); int crop_height_ RTC_GUARDED_BY(&encoder_queue_); uint32_t encoder_start_bitrate_bps_ RTC_GUARDED_BY(&encoder_queue_); + int set_start_bitrate_bps_ RTC_GUARDED_BY(&encoder_queue_); + int64_t set_start_bitrate_time_ms_ RTC_GUARDED_BY(&encoder_queue_); + bool has_seen_first_bwe_drop_ RTC_GUARDED_BY(&encoder_queue_); size_t max_data_payload_length_ RTC_GUARDED_BY(&encoder_queue_); absl::optional last_encoder_rate_settings_ RTC_GUARDED_BY(&encoder_queue_); diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc index 06f7b183e9..fe059c2d12 100644 --- a/video/video_stream_encoder_unittest.cc +++ b/video/video_stream_encoder_unittest.cc @@ -3093,6 +3093,42 @@ TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBWEstimateReady) { video_stream_encoder_->Stop(); } +TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) { + webrtc::test::ScopedFieldTrials field_trials( + "WebRTC-Video-QualityScalerSettings/" + "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/"); + // Reset encoder for field trials to take effect. + ConfigureEncoder(video_encoder_config_.Copy()); + const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.2; + const int kTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.19; + const int kWidth = 640; + const int kHeight = 360; + + video_stream_encoder_->OnBitrateUpdated( + DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0); + video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight)); + // Frame should not be dropped. + WaitForEncodedFrame(1); + + video_stream_encoder_->OnBitrateUpdated( + DataRate::bps(kNotTooLowBitrateForFrameSizeBps), + DataRate::bps(kNotTooLowBitrateForFrameSizeBps), 0, 0); + video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight)); + // Frame should not be dropped. + WaitForEncodedFrame(2); + + video_stream_encoder_->OnBitrateUpdated( + DataRate::bps(kTooLowBitrateForFrameSizeBps), + DataRate::bps(kTooLowBitrateForFrameSizeBps), 0, 0); + video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight)); + // Expect to drop this frame, the wait should time out. + ExpectDroppedFrame(); + + // Expect the sink_wants to specify a scaled frame. + EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight); + video_stream_encoder_->Stop(); +} + TEST_F(VideoStreamEncoderTest, ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) { const int kTooSmallWidth = 10;