From cf9cbf5edb0e18c7be8244985feb8a1f81c0306e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20Spr=C3=A5ng?= Date: Wed, 4 Sep 2019 14:30:57 +0200 Subject: [PATCH] Add support for stable bitrate target in SvcRateAllocator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bug: webrtc:10126 Change-Id: I1362d183bb91510db4e2763a779bcdf681d855ef Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/149069 Reviewed-by: Florent Castelli Commit-Queue: Erik Språng Cr-Commit-Position: refs/heads/master@{#29066} --- modules/video_coding/BUILD.gn | 3 + .../codecs/vp9/svc_rate_allocator.cc | 331 ++++++++++++------ .../codecs/vp9/svc_rate_allocator.h | 23 +- .../codecs/vp9/svc_rate_allocator_unittest.cc | 198 ++++++++++- .../stable_target_rate_experiment.cc | 2 + .../stable_target_rate_experiment.h | 1 + video/video_stream_encoder.cc | 7 +- 7 files changed, 429 insertions(+), 136 deletions(-) diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn index 33c893d9f4..eafbd39ed2 100644 --- a/modules/video_coding/BUILD.gn +++ b/modules/video_coding/BUILD.gn @@ -444,9 +444,12 @@ rtc_static_library("webrtc_vp9_helpers") { "../..:webrtc_common", "../../api/video:video_bitrate_allocation", "../../api/video:video_bitrate_allocator", + "../../api/video:video_codec_constants", "../../api/video_codecs:video_codecs_api", "../../common_video", "../../rtc_base:checks", + "../../rtc_base/experiments:stable_target_rate_experiment", + "//third_party/abseil-cpp/absl/container:inlined_vector", ] } diff --git a/modules/video_coding/codecs/vp9/svc_rate_allocator.cc b/modules/video_coding/codecs/vp9/svc_rate_allocator.cc index 5aa414ea41..45fc9864ed 100644 --- a/modules/video_coding/codecs/vp9/svc_rate_allocator.cc +++ b/modules/video_coding/codecs/vp9/svc_rate_allocator.cc @@ -14,10 +14,13 @@ #include #include #include +#include +#include "absl/container/inlined_vector.h" #include "rtc_base/checks.h" namespace webrtc { +namespace { const float kSpatialLayeringRateScalingFactor = 0.55f; const float kTemporalLayeringRateScalingFactor = 0.55f; @@ -40,40 +43,43 @@ static size_t GetNumActiveSpatialLayers(const VideoCodec& codec) { return num_spatial_layers; } -static bool AdjustAndVerify(const VideoCodec& codec, - std::vector* spatial_layer_bitrate_bps) { - bool enough_bitrate = true; - size_t excess_rate = 0; - for (size_t sl_idx = 0; - sl_idx < spatial_layer_bitrate_bps->size() && enough_bitrate; ++sl_idx) { - RTC_DCHECK_GT(codec.spatialLayers[sl_idx].maxBitrate, 0); - RTC_DCHECK_GE(codec.spatialLayers[sl_idx].maxBitrate, - codec.spatialLayers[sl_idx].minBitrate); +std::vector AdjustAndVerify( + const VideoCodec& codec, + const std::vector& spatial_layer_rates) { + std::vector adjusted_spatial_layer_rates; + // Keep track of rate that couldn't be applied to the previous layer due to + // max bitrate constraint, try to pass it forward to the next one. + DataRate excess_rate = DataRate::Zero(); + for (size_t sl_idx = 0; sl_idx < spatial_layer_rates.size(); ++sl_idx) { + DataRate min_rate = DataRate::kbps(codec.spatialLayers[sl_idx].minBitrate); + DataRate max_rate = DataRate::kbps(codec.spatialLayers[sl_idx].maxBitrate); - const size_t min_bitrate_bps = - codec.spatialLayers[sl_idx].minBitrate * 1000; - const size_t max_bitrate_bps = - codec.spatialLayers[sl_idx].maxBitrate * 1000; - - spatial_layer_bitrate_bps->at(sl_idx) += excess_rate; - if (spatial_layer_bitrate_bps->at(sl_idx) < max_bitrate_bps) { - excess_rate = 0; - } else { - excess_rate = spatial_layer_bitrate_bps->at(sl_idx) - max_bitrate_bps; - spatial_layer_bitrate_bps->at(sl_idx) = max_bitrate_bps; + DataRate layer_rate = spatial_layer_rates[sl_idx] + excess_rate; + if (layer_rate < min_rate) { + // Not enough rate to reach min bitrate for desired number of layers, + // abort allocation. + if (spatial_layer_rates.size() == 1) { + return spatial_layer_rates; + } + return adjusted_spatial_layer_rates; } - size_t bitrate_bps = spatial_layer_bitrate_bps->at(sl_idx); - enough_bitrate = (bitrate_bps >= min_bitrate_bps); + if (layer_rate <= max_rate) { + excess_rate = DataRate::Zero(); + adjusted_spatial_layer_rates.push_back(layer_rate); + } else { + excess_rate = layer_rate - max_rate; + adjusted_spatial_layer_rates.push_back(max_rate); + } } - return enough_bitrate; + return adjusted_spatial_layer_rates; } -static std::vector SplitBitrate(size_t num_layers, - size_t total_bitrate, - float rate_scaling_factor) { - std::vector bitrates; +static std::vector SplitBitrate(size_t num_layers, + DataRate total_bitrate, + float rate_scaling_factor) { + std::vector bitrates; double denominator = 0.0; for (size_t layer_idx = 0; layer_idx < num_layers; ++layer_idx) { @@ -86,21 +92,89 @@ static std::vector SplitBitrate(size_t num_layers, numerator /= rate_scaling_factor; } - const size_t sum = std::accumulate(bitrates.begin(), bitrates.end(), 0); - // Ensure the sum of split bitrates doesn't exceed the total bitrate. - RTC_DCHECK_LE(sum, total_bitrate); + const DataRate sum = + std::accumulate(bitrates.begin(), bitrates.end(), DataRate::Zero()); - // Keep the sum of split bitrates equal to the total bitrate by adding bits, - // which were lost due to rounding, to the latest layer. - bitrates.back() += total_bitrate - sum; + // Keep the sum of split bitrates equal to the total bitrate by adding or + // subtracting bits, which were lost due to rounding, to the latest layer. + if (total_bitrate > sum) { + bitrates.back() += total_bitrate - sum; + } else if (total_bitrate < sum) { + bitrates.back() -= sum - total_bitrate; + } return bitrates; } -SvcRateAllocator::SvcRateAllocator(const VideoCodec& codec) : codec_(codec) { +// Returns the minimum bitrate needed for |num_active_layers| spatial layers to +// become active using the configuration specified by |codec|. +DataRate FindLayerTogglingThreshold(const VideoCodec& codec, + size_t num_active_layers) { + if (num_active_layers == 1) { + return DataRate::kbps(codec.spatialLayers[0].minBitrate); + } + + if (codec.mode == VideoCodecMode::kRealtimeVideo) { + DataRate lower_bound = DataRate::Zero(); + DataRate upper_bound = DataRate::Zero(); + if (num_active_layers > 1) { + for (size_t i = 0; i < num_active_layers - 1; ++i) { + lower_bound += DataRate::kbps(codec.spatialLayers[i].minBitrate); + upper_bound += DataRate::kbps(codec.spatialLayers[i].maxBitrate); + } + } + upper_bound += + DataRate::kbps(codec.spatialLayers[num_active_layers - 1].minBitrate); + + // Do a binary search until upper and lower bound is the highest bitrate for + // |num_active_layers| - 1 layers and lowest bitrate for |num_active_layers| + // layers respectively. + while (upper_bound - lower_bound > DataRate::bps(1)) { + DataRate try_rate = (lower_bound + upper_bound) / 2; + if (AdjustAndVerify(codec, + SplitBitrate(num_active_layers, try_rate, + kSpatialLayeringRateScalingFactor)) + .size() == num_active_layers) { + upper_bound = try_rate; + } else { + lower_bound = try_rate; + } + } + return upper_bound; + } else { + DataRate toggling_rate = DataRate::Zero(); + for (size_t i = 0; i < num_active_layers - 1; ++i) { + toggling_rate += DataRate::kbps(codec.spatialLayers[i].targetBitrate); + } + toggling_rate += + DataRate::kbps(codec.spatialLayers[num_active_layers - 1].minBitrate); + return toggling_rate; + } +} + +} // namespace + +SvcRateAllocator::SvcRateAllocator(const VideoCodec& codec) + : codec_(codec), + experiment_settings_(StableTargetRateExperiment::ParseFromFieldTrials()), + cumulative_layer_start_bitrates_(GetLayerStartBitrates(codec)), + last_active_layer_count_(0) { RTC_DCHECK_EQ(codec.codecType, kVideoCodecVP9); RTC_DCHECK_GT(codec.VP9().numberOfSpatialLayers, 0u); RTC_DCHECK_GT(codec.VP9().numberOfTemporalLayers, 0u); + for (size_t layer_idx = 0; layer_idx < codec.VP9().numberOfSpatialLayers; + ++layer_idx) { + // Verify min <= target <= max. + if (codec.spatialLayers[layer_idx].active) { + RTC_DCHECK_GT(codec.spatialLayers[layer_idx].maxBitrate, 0); + RTC_DCHECK_GE(codec.spatialLayers[layer_idx].maxBitrate, + codec.spatialLayers[layer_idx].minBitrate); + RTC_DCHECK_GE(codec.spatialLayers[layer_idx].targetBitrate, + codec.spatialLayers[layer_idx].minBitrate); + RTC_DCHECK_GE(codec.spatialLayers[layer_idx].maxBitrate, + codec.spatialLayers[layer_idx].targetBitrate); + } + } } VideoBitrateAllocation SvcRateAllocator::Allocate( @@ -123,50 +197,76 @@ VideoBitrateAllocation SvcRateAllocator::Allocate( return VideoBitrateAllocation(); // All layers are deactivated. } - if (codec_.mode == VideoCodecMode::kRealtimeVideo) { - return GetAllocationNormalVideo(total_bitrate.bps(), num_spatial_layers); + // Figure out how many spatial layers should be active. + if (experiment_settings_.IsEnabled() && + parameters.stable_bitrate > DataRate::Zero()) { + double hysteresis_factor = 1.0; + if (codec_.mode == VideoCodecMode::kScreensharing) { + hysteresis_factor = + experiment_settings_.GetScreenshareHysteresisFactor().value_or( + hysteresis_factor); + } else { + hysteresis_factor = + experiment_settings_.GetVideoHysteresisFactor().value_or( + hysteresis_factor); + } + + DataRate stable_rate = + std::min(parameters.total_bitrate, parameters.stable_bitrate); + // First check if bitrate has grown large enough to enable new layers. + size_t num_enabled_with_hysteresis = + FindNumEnabledLayers(stable_rate / hysteresis_factor); + if (num_enabled_with_hysteresis >= last_active_layer_count_) { + num_spatial_layers = num_enabled_with_hysteresis; + } else { + // We could not enable new layers, check if any should be disabled. + num_spatial_layers = + std::min(last_active_layer_count_, FindNumEnabledLayers(stable_rate)); + } } else { - return GetAllocationScreenSharing(total_bitrate.bps(), num_spatial_layers); + num_spatial_layers = FindNumEnabledLayers(parameters.total_bitrate); + } + last_active_layer_count_ = num_spatial_layers; + + if (codec_.mode == VideoCodecMode::kRealtimeVideo) { + return GetAllocationNormalVideo(total_bitrate, num_spatial_layers); + } else { + return GetAllocationScreenSharing(total_bitrate, num_spatial_layers); } } VideoBitrateAllocation SvcRateAllocator::GetAllocationNormalVideo( - uint32_t total_bitrate_bps, + DataRate total_bitrate, size_t num_spatial_layers) const { - std::vector spatial_layer_bitrate_bps; - - // Distribute total bitrate across spatial layers. If there is not enough - // bitrate to provide all layers with at least minimum required bitrate - // then number of layers is reduced by one and distribution is repeated - // until that condition is met or if number of layers is reduced to one. - for (;; --num_spatial_layers) { - spatial_layer_bitrate_bps = - SplitBitrate(num_spatial_layers, total_bitrate_bps, - kSpatialLayeringRateScalingFactor); - - const bool enough_bitrate = - AdjustAndVerify(codec_, &spatial_layer_bitrate_bps); - if (enough_bitrate || num_spatial_layers == 1) { - break; - } + std::vector spatial_layer_rates; + if (num_spatial_layers == 0) { + // Not enough rate for even the base layer. Force allocation at the total + // bitrate anyway. + num_spatial_layers = 1; + spatial_layer_rates.push_back(total_bitrate); + } else { + spatial_layer_rates = AdjustAndVerify( + codec_, SplitBitrate(num_spatial_layers, total_bitrate, + kSpatialLayeringRateScalingFactor)); + RTC_DCHECK_EQ(spatial_layer_rates.size(), num_spatial_layers); } VideoBitrateAllocation bitrate_allocation; const size_t num_temporal_layers = codec_.VP9().numberOfTemporalLayers; for (size_t sl_idx = 0; sl_idx < num_spatial_layers; ++sl_idx) { - std::vector temporal_layer_bitrate_bps = - SplitBitrate(num_temporal_layers, spatial_layer_bitrate_bps[sl_idx], + std::vector temporal_layer_rates = + SplitBitrate(num_temporal_layers, spatial_layer_rates[sl_idx], kTemporalLayeringRateScalingFactor); // Distribute rate across temporal layers. Allocate more bits to lower // layers since they are used for prediction of higher layers and their // references are far apart. if (num_temporal_layers == 1) { - bitrate_allocation.SetBitrate(sl_idx, 0, temporal_layer_bitrate_bps[0]); + bitrate_allocation.SetBitrate(sl_idx, 0, temporal_layer_rates[0].bps()); } else if (num_temporal_layers == 2) { - bitrate_allocation.SetBitrate(sl_idx, 0, temporal_layer_bitrate_bps[1]); - bitrate_allocation.SetBitrate(sl_idx, 1, temporal_layer_bitrate_bps[0]); + bitrate_allocation.SetBitrate(sl_idx, 0, temporal_layer_rates[1].bps()); + bitrate_allocation.SetBitrate(sl_idx, 1, temporal_layer_rates[0].bps()); } else { RTC_CHECK_EQ(num_temporal_layers, 3); // In case of three temporal layers the high layer has two frames and the @@ -174,9 +274,9 @@ VideoBitrateAllocation SvcRateAllocator::GetAllocationNormalVideo( // layer frames). Thus high layer requires more bits (comparing pure // bitrate of layer, excluding bitrate of base layers) to keep quality on // par with lower layers. - bitrate_allocation.SetBitrate(sl_idx, 0, temporal_layer_bitrate_bps[2]); - bitrate_allocation.SetBitrate(sl_idx, 1, temporal_layer_bitrate_bps[0]); - bitrate_allocation.SetBitrate(sl_idx, 2, temporal_layer_bitrate_bps[1]); + bitrate_allocation.SetBitrate(sl_idx, 0, temporal_layer_rates[2].bps()); + bitrate_allocation.SetBitrate(sl_idx, 1, temporal_layer_rates[0].bps()); + bitrate_allocation.SetBitrate(sl_idx, 2, temporal_layer_rates[1].bps()); } } @@ -187,87 +287,98 @@ VideoBitrateAllocation SvcRateAllocator::GetAllocationNormalVideo( // between min and max bitrate, and all others will have exactly target // bit-rate allocated. VideoBitrateAllocation SvcRateAllocator::GetAllocationScreenSharing( - uint32_t total_bitrate_bps, + DataRate total_bitrate, size_t num_spatial_layers) const { if (num_spatial_layers == 0 || - total_bitrate_bps < codec_.spatialLayers[0].minBitrate * 1000) { + total_bitrate < DataRate::kbps(codec_.spatialLayers[0].minBitrate)) { return VideoBitrateAllocation(); } VideoBitrateAllocation bitrate_allocation; - size_t left_bitrate_bps = total_bitrate_bps; + DataRate allocated_rate = DataRate::Zero(); + DataRate top_layer_rate = DataRate::Zero(); size_t sl_idx; for (sl_idx = 0; sl_idx < num_spatial_layers; ++sl_idx) { - const size_t min_bitrate_bps = - codec_.spatialLayers[sl_idx].minBitrate * 1000; - const size_t target_bitrate_bps = - codec_.spatialLayers[sl_idx].targetBitrate * 1000; - RTC_DCHECK_LE(min_bitrate_bps, target_bitrate_bps); + const DataRate min_rate = + DataRate::kbps(codec_.spatialLayers[sl_idx].minBitrate); + const DataRate target_rate = + DataRate::kbps(codec_.spatialLayers[sl_idx].targetBitrate); - const size_t bitrate_bps = std::min(left_bitrate_bps, target_bitrate_bps); - if (bitrate_bps >= min_bitrate_bps) { - bitrate_allocation.SetBitrate(sl_idx, 0, bitrate_bps); - } else { + if (allocated_rate + min_rate > total_bitrate) { + // Use stable rate to determine if layer should be enabled. break; } - left_bitrate_bps -= bitrate_bps; + top_layer_rate = std::min(target_rate, total_bitrate - allocated_rate); + bitrate_allocation.SetBitrate(sl_idx, 0, top_layer_rate.bps()); + allocated_rate += top_layer_rate; } - if (left_bitrate_bps > 0 && sl_idx > 0) { + if (sl_idx > 0 && total_bitrate - allocated_rate > DataRate::Zero()) { // Add leftover to the last allocated layer. - const size_t max_bitrate_bps = - codec_.spatialLayers[sl_idx - 1].maxBitrate * 1000; - - const size_t bitrate_bps = std::min( - bitrate_allocation.GetBitrate(sl_idx - 1, 0) + left_bitrate_bps, - max_bitrate_bps); - bitrate_allocation.SetBitrate(sl_idx - 1, 0, bitrate_bps); + top_layer_rate = + std::min(top_layer_rate + (total_bitrate - allocated_rate), + DataRate::kbps(codec_.spatialLayers[sl_idx - 1].maxBitrate)); + bitrate_allocation.SetBitrate(sl_idx - 1, 0, top_layer_rate.bps()); } return bitrate_allocation; } -uint32_t SvcRateAllocator::GetMaxBitrateBps(const VideoCodec& codec) { +size_t SvcRateAllocator::FindNumEnabledLayers(DataRate target_rate) const { + if (cumulative_layer_start_bitrates_.empty()) { + return 0; + } + + size_t num_enabled_layers = 0; + for (DataRate start_rate : cumulative_layer_start_bitrates_) { + // First layer is always enabled. + if (num_enabled_layers == 0 || start_rate <= target_rate) { + ++num_enabled_layers; + } else { + break; + } + } + + return num_enabled_layers; +} + +DataRate SvcRateAllocator::GetMaxBitrate(const VideoCodec& codec) { const size_t num_spatial_layers = GetNumActiveSpatialLayers(codec); - uint32_t max_bitrate_kbps = 0; + DataRate max_bitrate = DataRate::Zero(); for (size_t sl_idx = 0; sl_idx < num_spatial_layers; ++sl_idx) { - max_bitrate_kbps += codec.spatialLayers[sl_idx].maxBitrate; + max_bitrate += DataRate::kbps(codec.spatialLayers[sl_idx].maxBitrate); } if (codec.maxBitrate != 0) { - max_bitrate_kbps = std::min(max_bitrate_kbps, codec.maxBitrate); + max_bitrate = std::min(max_bitrate, DataRate::kbps(codec.maxBitrate)); } - return max_bitrate_kbps * 1000; + return max_bitrate; } -uint32_t SvcRateAllocator::GetPaddingBitrateBps(const VideoCodec& codec) { - const size_t num_spatial_layers = GetNumActiveSpatialLayers(codec); - if (num_spatial_layers == 0) { - return 0; // All layers are deactivated. +DataRate SvcRateAllocator::GetPaddingBitrate(const VideoCodec& codec) { + auto start_bitrate = GetLayerStartBitrates(codec); + if (start_bitrate.empty()) { + return DataRate::Zero(); // All layers are deactivated. } - if (codec.mode == VideoCodecMode::kRealtimeVideo) { - float scale_factor = 0.0; - for (size_t sl_idx = 0; sl_idx < num_spatial_layers; ++sl_idx) { - scale_factor += std::pow(kSpatialLayeringRateScalingFactor, sl_idx); - } - uint32_t min_bitrate_bps = - codec.spatialLayers[num_spatial_layers - 1].minBitrate * 1000; - return static_cast(min_bitrate_bps * scale_factor); + return start_bitrate.back(); +} + +absl::InlinedVector +SvcRateAllocator::GetLayerStartBitrates(const VideoCodec& codec) { + absl::InlinedVector start_bitrates; + size_t num_layers = GetNumActiveSpatialLayers(codec); + DataRate last_rate = DataRate::Zero(); + for (size_t i = 1; i <= num_layers; ++i) { + DataRate layer_toggling_rate = FindLayerTogglingThreshold(codec, i); + start_bitrates.push_back(layer_toggling_rate); + RTC_DCHECK_LE(last_rate, layer_toggling_rate); + last_rate = layer_toggling_rate; } - - RTC_DCHECK(codec.mode == VideoCodecMode::kScreensharing); - - uint32_t min_bitrate_kbps = 0; - for (size_t sl_idx = 0; sl_idx < num_spatial_layers - 1; ++sl_idx) { - min_bitrate_kbps += codec.spatialLayers[sl_idx].targetBitrate; - } - min_bitrate_kbps += codec.spatialLayers[num_spatial_layers - 1].minBitrate; - - return min_bitrate_kbps * 1000; + return start_bitrates; } } // namespace webrtc diff --git a/modules/video_coding/codecs/vp9/svc_rate_allocator.h b/modules/video_coding/codecs/vp9/svc_rate_allocator.h index e410964969..1b14dd6c69 100644 --- a/modules/video_coding/codecs/vp9/svc_rate_allocator.h +++ b/modules/video_coding/codecs/vp9/svc_rate_allocator.h @@ -14,11 +14,12 @@ #include #include -#include - +#include "absl/container/inlined_vector.h" #include "api/video/video_bitrate_allocation.h" #include "api/video/video_bitrate_allocator.h" +#include "api/video/video_codec_constants.h" #include "api/video_codecs/video_codec.h" +#include "rtc_base/experiments/stable_target_rate_experiment.h" namespace webrtc { @@ -29,19 +30,29 @@ class SvcRateAllocator : public VideoBitrateAllocator { VideoBitrateAllocation Allocate( VideoBitrateAllocationParameters parameters) override; - static uint32_t GetMaxBitrateBps(const VideoCodec& codec); - static uint32_t GetPaddingBitrateBps(const VideoCodec& codec); + static DataRate GetMaxBitrate(const VideoCodec& codec); + static DataRate GetPaddingBitrate(const VideoCodec& codec); + static absl::InlinedVector GetLayerStartBitrates( + const VideoCodec& codec); private: VideoBitrateAllocation GetAllocationNormalVideo( - uint32_t total_bitrate_bps, + DataRate total_bitrate, size_t num_spatial_layers) const; VideoBitrateAllocation GetAllocationScreenSharing( - uint32_t total_bitrate_bps, + DataRate total_bitrate, size_t num_spatial_layers) const; + // Returns the number of layers that are active and have enough bitrate to + // actually be enabled. + size_t FindNumEnabledLayers(DataRate target_rate) const; + const VideoCodec codec_; + const StableTargetRateExperiment experiment_settings_; + const absl::InlinedVector + cumulative_layer_start_bitrates_; + size_t last_active_layer_count_; }; } // namespace webrtc diff --git a/modules/video_coding/codecs/vp9/svc_rate_allocator_unittest.cc b/modules/video_coding/codecs/vp9/svc_rate_allocator_unittest.cc index c0febb8bf8..f721608365 100644 --- a/modules/video_coding/codecs/vp9/svc_rate_allocator_unittest.cc +++ b/modules/video_coding/codecs/vp9/svc_rate_allocator_unittest.cc @@ -11,12 +11,15 @@ #include "modules/video_coding/codecs/vp9/svc_rate_allocator.h" #include +#include #include "modules/video_coding/codecs/vp9/svc_config.h" #include "rtc_base/checks.h" +#include "test/field_trial.h" #include "test/gtest.h" namespace webrtc { +namespace test { namespace { static VideoCodec Configure(size_t width, size_t height, @@ -46,6 +49,7 @@ static VideoCodec Configure(size_t width, return codec; } + } // namespace TEST(SvcRateAllocatorTest, SingleLayerFor320x180Input) { @@ -198,8 +202,25 @@ TEST(SvcRateAllocatorTest, NoPaddingIfAllLayersAreDeactivated) { EXPECT_EQ(codec.VP9()->numberOfSpatialLayers, 3U); // Deactivation of base layer deactivates all layers. codec.spatialLayers[0].active = false; - uint32_t padding_bps = SvcRateAllocator::GetPaddingBitrateBps(codec); - EXPECT_EQ(padding_bps, 0U); + DataRate padding_rate = SvcRateAllocator::GetPaddingBitrate(codec); + EXPECT_EQ(padding_rate, DataRate::Zero()); +} + +TEST(SvcRateAllocatorTest, FindLayerTogglingThreshold) { + // Let's unit test a utility method of the unit test... + + // Predetermined constants indicating the min bitrate needed for two and three + // layers to be enabled respectively, using the config from Configure() with + // 1280x720 resolution and three spatial layers. + const DataRate kTwoLayerMinRate = DataRate::bps(299150); + const DataRate kThreeLayerMinRate = DataRate::bps(891052); + + VideoCodec codec = Configure(1280, 720, 3, 1, false); + absl::InlinedVector layer_start_bitrates = + SvcRateAllocator::GetLayerStartBitrates(codec); + ASSERT_EQ(layer_start_bitrates.size(), 3u); + EXPECT_EQ(layer_start_bitrates[1], kTwoLayerMinRate); + EXPECT_EQ(layer_start_bitrates[2], kThreeLayerMinRate); } class SvcRateAllocatorTestParametrizedContentType @@ -214,33 +235,32 @@ class SvcRateAllocatorTestParametrizedContentType TEST_P(SvcRateAllocatorTestParametrizedContentType, MaxBitrate) { VideoCodec codec = Configure(1280, 720, 3, 1, is_screen_sharing_); - EXPECT_EQ( - SvcRateAllocator::GetMaxBitrateBps(codec), - (codec.spatialLayers[0].maxBitrate + codec.spatialLayers[1].maxBitrate + - codec.spatialLayers[2].maxBitrate) * - 1000); + EXPECT_EQ(SvcRateAllocator::GetMaxBitrate(codec), + DataRate::kbps(codec.spatialLayers[0].maxBitrate + + codec.spatialLayers[1].maxBitrate + + codec.spatialLayers[2].maxBitrate)); // Deactivate middle layer. This causes deactivation of top layer as well. codec.spatialLayers[1].active = false; - EXPECT_EQ(SvcRateAllocator::GetMaxBitrateBps(codec), - codec.spatialLayers[0].maxBitrate * 1000); + EXPECT_EQ(SvcRateAllocator::GetMaxBitrate(codec), + DataRate::kbps(codec.spatialLayers[0].maxBitrate)); } TEST_P(SvcRateAllocatorTestParametrizedContentType, PaddingBitrate) { VideoCodec codec = Configure(1280, 720, 3, 1, is_screen_sharing_); SvcRateAllocator allocator = SvcRateAllocator(codec); - uint32_t padding_bitrate_bps = SvcRateAllocator::GetPaddingBitrateBps(codec); + DataRate padding_bitrate = SvcRateAllocator::GetPaddingBitrate(codec); - VideoBitrateAllocation allocation = allocator.Allocate( - VideoBitrateAllocationParameters(padding_bitrate_bps, 30)); + VideoBitrateAllocation allocation = + allocator.Allocate(VideoBitrateAllocationParameters(padding_bitrate, 30)); EXPECT_GT(allocation.GetSpatialLayerSum(0), 0UL); EXPECT_GT(allocation.GetSpatialLayerSum(1), 0UL); EXPECT_GT(allocation.GetSpatialLayerSum(2), 0UL); // Allocate 90% of padding bitrate. Top layer should be disabled. allocation = allocator.Allocate( - VideoBitrateAllocationParameters(9 * padding_bitrate_bps / 10, 30)); + VideoBitrateAllocationParameters(9 * padding_bitrate / 10, 30)); EXPECT_GT(allocation.GetSpatialLayerSum(0), 0UL); EXPECT_GT(allocation.GetSpatialLayerSum(1), 0UL); EXPECT_EQ(allocation.GetSpatialLayerSum(2), 0UL); @@ -248,22 +268,166 @@ TEST_P(SvcRateAllocatorTestParametrizedContentType, PaddingBitrate) { // Deactivate top layer. codec.spatialLayers[2].active = false; - padding_bitrate_bps = SvcRateAllocator::GetPaddingBitrateBps(codec); - allocation = allocator.Allocate( - VideoBitrateAllocationParameters(padding_bitrate_bps, 30)); + padding_bitrate = SvcRateAllocator::GetPaddingBitrate(codec); + allocation = + allocator.Allocate(VideoBitrateAllocationParameters(padding_bitrate, 30)); EXPECT_GT(allocation.GetSpatialLayerSum(0), 0UL); EXPECT_GT(allocation.GetSpatialLayerSum(1), 0UL); EXPECT_EQ(allocation.GetSpatialLayerSum(2), 0UL); allocation = allocator.Allocate( - VideoBitrateAllocationParameters(9 * padding_bitrate_bps / 10, 30)); + VideoBitrateAllocationParameters(9 * padding_bitrate / 10, 30)); EXPECT_GT(allocation.GetSpatialLayerSum(0), 0UL); EXPECT_EQ(allocation.GetSpatialLayerSum(1), 0UL); EXPECT_EQ(allocation.GetSpatialLayerSum(2), 0UL); } +TEST_P(SvcRateAllocatorTestParametrizedContentType, StableBitrate) { + ScopedFieldTrials field_trial("WebRTC-StableTargetRate/enabled:true/"); + + const VideoCodec codec = Configure(1280, 720, 3, 1, is_screen_sharing_); + const auto start_rates = SvcRateAllocator::GetLayerStartBitrates(codec); + const DataRate min_rate_two_layers = start_rates[1]; + const DataRate min_rate_three_layers = start_rates[2]; + + const DataRate max_rate_one_layer = + DataRate::kbps(codec.spatialLayers[0].maxBitrate); + const DataRate max_rate_two_layers = + is_screen_sharing_ ? DataRate::kbps(codec.spatialLayers[0].targetBitrate + + codec.spatialLayers[1].maxBitrate) + : DataRate::kbps(codec.spatialLayers[0].maxBitrate + + codec.spatialLayers[1].maxBitrate); + + SvcRateAllocator allocator = SvcRateAllocator(codec); + + // Two layers, stable and target equal. + auto allocation = allocator.Allocate(VideoBitrateAllocationParameters( + /*total_bitrate=*/min_rate_two_layers, + /*stable_bitrate=*/min_rate_two_layers, /*fps=*/30.0)); + EXPECT_TRUE(allocation.IsSpatialLayerUsed(1)); + EXPECT_EQ(allocation.get_sum_bps(), min_rate_two_layers.bps()); + + // Two layers, stable bitrate too low for two layers. + allocation = allocator.Allocate(VideoBitrateAllocationParameters( + /*total_bitrate=*/min_rate_two_layers, + /*stable_bitrate=*/min_rate_two_layers - DataRate::bps(1), + /*fps=*/30.0)); + EXPECT_FALSE(allocation.IsSpatialLayerUsed(1)); + EXPECT_EQ( + DataRate::bps(allocation.get_sum_bps()), + std::min(min_rate_two_layers - DataRate::bps(1), max_rate_one_layer)); + + // Three layers, stable and target equal. + allocation = allocator.Allocate(VideoBitrateAllocationParameters( + /*total_bitrate=*/min_rate_three_layers, + /*stable_bitrate=*/min_rate_three_layers, /*fps=*/30.0)); + EXPECT_TRUE(allocation.IsSpatialLayerUsed(2)); + EXPECT_EQ(allocation.get_sum_bps(), min_rate_three_layers.bps()); + + // Three layers, stable bitrate too low for three layers. + allocation = allocator.Allocate(VideoBitrateAllocationParameters( + /*total_bitrate=*/min_rate_three_layers, + /*stable_bitrate=*/min_rate_three_layers - DataRate::bps(1), + /*fps=*/30.0)); + EXPECT_FALSE(allocation.IsSpatialLayerUsed(2)); + EXPECT_EQ( + DataRate::bps(allocation.get_sum_bps()), + std::min(min_rate_three_layers - DataRate::bps(1), max_rate_two_layers)); +} + +TEST_P(SvcRateAllocatorTestParametrizedContentType, + StableBitrateWithHysteresis) { + const VideoCodec codec = Configure(1280, 720, 3, 1, is_screen_sharing_); + const auto start_rates = SvcRateAllocator::GetLayerStartBitrates(codec); + const DataRate min_rate_single_layer = start_rates[0]; + const DataRate min_rate_two_layers = start_rates[1]; + const DataRate min_rate_three_layers = start_rates[2]; + + ScopedFieldTrials field_trial( + "WebRTC-StableTargetRate/enabled:true,video_hysteresis_factor:1.1," + "screenshare_hysteresis_factor:1.1/"); + SvcRateAllocator allocator = SvcRateAllocator(codec); + // Always use max bitrate as target, verify only stable is used for layer + // count selection. + const DataRate max_bitrate = allocator.GetMaxBitrate(codec); + + // Start with a single layer. + auto allocation = allocator.Allocate(VideoBitrateAllocationParameters( + /*total_bitrate=*/max_bitrate, + /*stable_bitrate=*/min_rate_single_layer, /*fps=*/30.0)); + EXPECT_TRUE(allocation.IsSpatialLayerUsed(0)); + EXPECT_FALSE(allocation.IsSpatialLayerUsed(1)); + EXPECT_FALSE(allocation.IsSpatialLayerUsed(2)); + + // Min bitrate not enough to enable second layer due to 10% hysteresis. + allocation = allocator.Allocate(VideoBitrateAllocationParameters( + /*total_bitrate=*/max_bitrate, + /*stable_bitrate=*/min_rate_two_layers, /*fps=*/30.0)); + EXPECT_TRUE(allocation.IsSpatialLayerUsed(0)); + EXPECT_FALSE(allocation.IsSpatialLayerUsed(1)); + EXPECT_FALSE(allocation.IsSpatialLayerUsed(2)); + + // Add hysteresis, second layer should turn on. + allocation = allocator.Allocate(VideoBitrateAllocationParameters( + /*total_bitrate=*/max_bitrate, + /*stable_bitrate=*/min_rate_two_layers * 1.1, /*fps=*/30.0)); + EXPECT_TRUE(allocation.IsSpatialLayerUsed(0)); + EXPECT_TRUE(allocation.IsSpatialLayerUsed(1)); + EXPECT_FALSE(allocation.IsSpatialLayerUsed(2)); + + // Remove hysteresis, second layer should stay on. + allocation = allocator.Allocate(VideoBitrateAllocationParameters( + /*total_bitrate=*/max_bitrate, + /*stable_bitrate=*/min_rate_two_layers, /*fps=*/30.0)); + EXPECT_TRUE(allocation.IsSpatialLayerUsed(0)); + EXPECT_TRUE(allocation.IsSpatialLayerUsed(1)); + EXPECT_FALSE(allocation.IsSpatialLayerUsed(2)); + + // Going below min for two layers, second layer should turn off again. + allocation = allocator.Allocate(VideoBitrateAllocationParameters( + /*total_bitrate=*/max_bitrate, + /*stable_bitrate=*/min_rate_two_layers - DataRate::bps(1), /*fps=*/30.0)); + EXPECT_TRUE(allocation.IsSpatialLayerUsed(0)); + EXPECT_FALSE(allocation.IsSpatialLayerUsed(1)); + EXPECT_FALSE(allocation.IsSpatialLayerUsed(2)); + + // Min bitrate not enough to enable third layer due to 10% hysteresis. + allocation = allocator.Allocate(VideoBitrateAllocationParameters( + /*total_bitrate=*/max_bitrate, + /*stable_bitrate=*/min_rate_three_layers, /*fps=*/30.0)); + EXPECT_TRUE(allocation.IsSpatialLayerUsed(0)); + EXPECT_TRUE(allocation.IsSpatialLayerUsed(1)); + EXPECT_FALSE(allocation.IsSpatialLayerUsed(2)); + + // Add hysteresis, third layer should turn on. + allocation = allocator.Allocate(VideoBitrateAllocationParameters( + /*total_bitrate=*/max_bitrate, + /*stable_bitrate=*/min_rate_three_layers * 1.1, /*fps=*/30.0)); + EXPECT_TRUE(allocation.IsSpatialLayerUsed(0)); + EXPECT_TRUE(allocation.IsSpatialLayerUsed(1)); + EXPECT_TRUE(allocation.IsSpatialLayerUsed(2)); + + // Remove hysteresis, third layer should stay on. + allocation = allocator.Allocate(VideoBitrateAllocationParameters( + /*total_bitrate=*/max_bitrate, + /*stable_bitrate=*/min_rate_three_layers, /*fps=*/30.0)); + EXPECT_TRUE(allocation.IsSpatialLayerUsed(0)); + EXPECT_TRUE(allocation.IsSpatialLayerUsed(1)); + EXPECT_TRUE(allocation.IsSpatialLayerUsed(2)); + + // Going below min for three layers, third layer should turn off again. + allocation = allocator.Allocate(VideoBitrateAllocationParameters( + /*total_bitrate=*/max_bitrate, + /*stable_bitrate=*/min_rate_three_layers - DataRate::bps(1), + /*fps=*/30.0)); + EXPECT_TRUE(allocation.IsSpatialLayerUsed(0)); + EXPECT_TRUE(allocation.IsSpatialLayerUsed(1)); + EXPECT_FALSE(allocation.IsSpatialLayerUsed(2)); +} + INSTANTIATE_TEST_SUITE_P(_, SvcRateAllocatorTestParametrizedContentType, ::testing::Bool()); +} // namespace test } // namespace webrtc diff --git a/rtc_base/experiments/stable_target_rate_experiment.cc b/rtc_base/experiments/stable_target_rate_experiment.cc index 28b541320b..185bd40960 100644 --- a/rtc_base/experiments/stable_target_rate_experiment.cc +++ b/rtc_base/experiments/stable_target_rate_experiment.cc @@ -32,6 +32,8 @@ StableTargetRateExperiment::StableTargetRateExperiment( key_value_config->Lookup(kFieldTrialName)); } +StableTargetRateExperiment::StableTargetRateExperiment( + const StableTargetRateExperiment&) = default; StableTargetRateExperiment::StableTargetRateExperiment( StableTargetRateExperiment&&) = default; diff --git a/rtc_base/experiments/stable_target_rate_experiment.h b/rtc_base/experiments/stable_target_rate_experiment.h index b56108d797..7a2c06c4ba 100644 --- a/rtc_base/experiments/stable_target_rate_experiment.h +++ b/rtc_base/experiments/stable_target_rate_experiment.h @@ -18,6 +18,7 @@ namespace webrtc { class StableTargetRateExperiment { public: + StableTargetRateExperiment(const StableTargetRateExperiment&); StableTargetRateExperiment(StableTargetRateExperiment&&); static StableTargetRateExperiment ParseFromFieldTrials(); static StableTargetRateExperiment ParseFromKeyValueConfig( diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index 703f470b31..0af1628c0f 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -781,12 +781,13 @@ void VideoStreamEncoder::ReconfigureEncoder() { // Set min_bitrate_bps, max_bitrate_bps, and max padding bit rate for VP9. if (encoder_config_.codec_type == kVideoCodecVP9) { // Lower max bitrate to the level codec actually can produce. - streams[0].max_bitrate_bps = std::min( - streams[0].max_bitrate_bps, SvcRateAllocator::GetMaxBitrateBps(codec)); + streams[0].max_bitrate_bps = + std::min(streams[0].max_bitrate_bps, + SvcRateAllocator::GetMaxBitrate(codec).bps()); streams[0].min_bitrate_bps = codec.spatialLayers[0].minBitrate * 1000; // target_bitrate_bps specifies the maximum padding bitrate. streams[0].target_bitrate_bps = - SvcRateAllocator::GetPaddingBitrateBps(codec); + SvcRateAllocator::GetPaddingBitrate(codec).bps(); } codec.startBitrate =