Revert "VP9 encoder: handle disabled layers correctly"
This reverts commit 88fe84b7fbcb8dffe07b98d21d8a11572259c0d0. Reason for revert: Downstream project isn't updated to the latest libvpx roll yet, thus some tests are broken. Original change's description: > VP9 encoder: handle disabled layers correctly > > Now vp9 screenshare would enable new layers as soon as requested and will > force all spatial layers present on the next frame, even if they should be > dropped because of frame-rate limiting. > > This might cause frame-rate liming to be exceeded if layer is toggling on > and off very often, but this situation is bad itself. E.g. in realtime video > it will cause too many key-frames. > > Now SvcRateAllocator and VP9EncoderImpl are aware that there may be some skipped > layers before the first enabled. Key-frames and ss_info triggering logic is also > updated. > > Bug: webrtc:10977 > Change-Id: Ie2555210c0368a1d3c51ddf6670d0052e6d679de > Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/153483 > Commit-Queue: Ilya Nikolaevskiy <ilnik@webrtc.org> > Reviewed-by: Sergey Silkin <ssilkin@webrtc.org> > Cr-Commit-Position: refs/heads/master@{#29296} TBR=ilnik@webrtc.org,ssilkin@webrtc.org Change-Id: If33886a5f8a0c3b33168dcadfe45c11a6f4387c1 No-Presubmit: true No-Tree-Checks: true No-Try: true Bug: webrtc:10977 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/154354 Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org> Commit-Queue: Ilya Nikolaevskiy <ilnik@webrtc.org> Cr-Commit-Position: refs/heads/master@{#29299}
This commit is contained in:
Ilya Nikolaevskiy
committed by
Commit Bot
Commit Bot
parent
7911d3705c
commit
90d6efbd4e
@ -25,47 +25,34 @@ namespace {
|
||||
const float kSpatialLayeringRateScalingFactor = 0.55f;
|
||||
const float kTemporalLayeringRateScalingFactor = 0.55f;
|
||||
|
||||
// Returns numberOfSpatialLayers if no layers are active.
|
||||
size_t GetFirstActiveLayer(const VideoCodec& codec) {
|
||||
RTC_DCHECK_EQ(codec.codecType, kVideoCodecVP9);
|
||||
RTC_DCHECK_GT(codec.VP9().numberOfSpatialLayers, 0u);
|
||||
size_t layer = 0;
|
||||
for (; layer < codec.VP9().numberOfSpatialLayers; ++layer) {
|
||||
if (codec.spatialLayers[layer].active) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return layer;
|
||||
}
|
||||
|
||||
static size_t GetNumActiveSpatialLayers(const VideoCodec& codec) {
|
||||
RTC_DCHECK_EQ(codec.codecType, kVideoCodecVP9);
|
||||
RTC_DCHECK_GT(codec.VP9().numberOfSpatialLayers, 0u);
|
||||
|
||||
const size_t first_active_layer = GetFirstActiveLayer(codec);
|
||||
size_t last_active_layer = first_active_layer;
|
||||
for (; last_active_layer < codec.VP9().numberOfSpatialLayers;
|
||||
++last_active_layer) {
|
||||
if (!codec.spatialLayers[last_active_layer].active) {
|
||||
size_t num_spatial_layers = 0;
|
||||
for (; num_spatial_layers < codec.VP9().numberOfSpatialLayers;
|
||||
++num_spatial_layers) {
|
||||
if (!codec.spatialLayers[num_spatial_layers].active) {
|
||||
// TODO(bugs.webrtc.org/9350): Deactivation of middle layer is not
|
||||
// implemented. For now deactivation of a VP9 layer deactivates all
|
||||
// layers above the deactivated one.
|
||||
break;
|
||||
}
|
||||
}
|
||||
return last_active_layer - first_active_layer;
|
||||
|
||||
return num_spatial_layers;
|
||||
}
|
||||
|
||||
std::vector<DataRate> AdjustAndVerify(
|
||||
const VideoCodec& codec,
|
||||
size_t first_active_layer,
|
||||
const std::vector<DataRate>& spatial_layer_rates) {
|
||||
std::vector<DataRate> adjusted_spatial_layer_rates;
|
||||
// Keep track of rate that couldn't be applied to the previous layer due to
|
||||
// max bitrate constraint, try to pass it forward to the next one.
|
||||
DataRate excess_rate = DataRate::Zero();
|
||||
for (size_t sl_idx = 0; sl_idx < spatial_layer_rates.size(); ++sl_idx) {
|
||||
DataRate min_rate = DataRate::kbps(
|
||||
codec.spatialLayers[first_active_layer + sl_idx].minBitrate);
|
||||
DataRate max_rate = DataRate::kbps(
|
||||
codec.spatialLayers[first_active_layer + sl_idx].maxBitrate);
|
||||
DataRate min_rate = DataRate::kbps(codec.spatialLayers[sl_idx].minBitrate);
|
||||
DataRate max_rate = DataRate::kbps(codec.spatialLayers[sl_idx].maxBitrate);
|
||||
|
||||
DataRate layer_rate = spatial_layer_rates[sl_idx] + excess_rate;
|
||||
if (layer_rate < min_rate) {
|
||||
@ -122,7 +109,6 @@ static std::vector<DataRate> SplitBitrate(size_t num_layers,
|
||||
// Returns the minimum bitrate needed for |num_active_layers| spatial layers to
|
||||
// become active using the configuration specified by |codec|.
|
||||
DataRate FindLayerTogglingThreshold(const VideoCodec& codec,
|
||||
size_t first_active_layer,
|
||||
size_t num_active_layers) {
|
||||
if (num_active_layers == 1) {
|
||||
return DataRate::kbps(codec.spatialLayers[0].minBitrate);
|
||||
@ -133,10 +119,8 @@ DataRate FindLayerTogglingThreshold(const VideoCodec& codec,
|
||||
DataRate upper_bound = DataRate::Zero();
|
||||
if (num_active_layers > 1) {
|
||||
for (size_t i = 0; i < num_active_layers - 1; ++i) {
|
||||
lower_bound += DataRate::kbps(
|
||||
codec.spatialLayers[first_active_layer + i].minBitrate);
|
||||
upper_bound += DataRate::kbps(
|
||||
codec.spatialLayers[first_active_layer + i].maxBitrate);
|
||||
lower_bound += DataRate::kbps(codec.spatialLayers[i].minBitrate);
|
||||
upper_bound += DataRate::kbps(codec.spatialLayers[i].maxBitrate);
|
||||
}
|
||||
}
|
||||
upper_bound +=
|
||||
@ -147,7 +131,7 @@ DataRate FindLayerTogglingThreshold(const VideoCodec& codec,
|
||||
// layers respectively.
|
||||
while (upper_bound - lower_bound > DataRate::bps(1)) {
|
||||
DataRate try_rate = (lower_bound + upper_bound) / 2;
|
||||
if (AdjustAndVerify(codec, first_active_layer,
|
||||
if (AdjustAndVerify(codec,
|
||||
SplitBitrate(num_active_layers, try_rate,
|
||||
kSpatialLayeringRateScalingFactor))
|
||||
.size() == num_active_layers) {
|
||||
@ -160,12 +144,10 @@ DataRate FindLayerTogglingThreshold(const VideoCodec& codec,
|
||||
} else {
|
||||
DataRate toggling_rate = DataRate::Zero();
|
||||
for (size_t i = 0; i < num_active_layers - 1; ++i) {
|
||||
toggling_rate += DataRate::kbps(
|
||||
codec.spatialLayers[first_active_layer + i].targetBitrate);
|
||||
toggling_rate += DataRate::kbps(codec.spatialLayers[i].targetBitrate);
|
||||
}
|
||||
toggling_rate += DataRate::kbps(
|
||||
codec.spatialLayers[first_active_layer + num_active_layers - 1]
|
||||
.minBitrate);
|
||||
toggling_rate +=
|
||||
DataRate::kbps(codec.spatialLayers[num_active_layers - 1].minBitrate);
|
||||
return toggling_rate;
|
||||
}
|
||||
}
|
||||
@ -210,9 +192,7 @@ VideoBitrateAllocation SvcRateAllocator::Allocate(
|
||||
return bitrate_allocation;
|
||||
}
|
||||
|
||||
const size_t first_active_layer = GetFirstActiveLayer(codec_);
|
||||
size_t num_spatial_layers = GetNumActiveSpatialLayers(codec_);
|
||||
|
||||
if (num_spatial_layers == 0) {
|
||||
return VideoBitrateAllocation(); // All layers are deactivated.
|
||||
}
|
||||
@ -245,17 +225,14 @@ VideoBitrateAllocation SvcRateAllocator::Allocate(
|
||||
last_active_layer_count_ = num_spatial_layers;
|
||||
|
||||
if (codec_.mode == VideoCodecMode::kRealtimeVideo) {
|
||||
return GetAllocationNormalVideo(total_bitrate, first_active_layer,
|
||||
num_spatial_layers);
|
||||
return GetAllocationNormalVideo(total_bitrate, num_spatial_layers);
|
||||
} else {
|
||||
return GetAllocationScreenSharing(total_bitrate, first_active_layer,
|
||||
num_spatial_layers);
|
||||
return GetAllocationScreenSharing(total_bitrate, num_spatial_layers);
|
||||
}
|
||||
}
|
||||
|
||||
VideoBitrateAllocation SvcRateAllocator::GetAllocationNormalVideo(
|
||||
DataRate total_bitrate,
|
||||
size_t first_active_layer,
|
||||
size_t num_spatial_layers) const {
|
||||
std::vector<DataRate> spatial_layer_rates;
|
||||
if (num_spatial_layers == 0) {
|
||||
@ -264,10 +241,9 @@ VideoBitrateAllocation SvcRateAllocator::GetAllocationNormalVideo(
|
||||
num_spatial_layers = 1;
|
||||
spatial_layer_rates.push_back(total_bitrate);
|
||||
} else {
|
||||
spatial_layer_rates =
|
||||
AdjustAndVerify(codec_, first_active_layer,
|
||||
SplitBitrate(num_spatial_layers, total_bitrate,
|
||||
kSpatialLayeringRateScalingFactor));
|
||||
spatial_layer_rates = AdjustAndVerify(
|
||||
codec_, SplitBitrate(num_spatial_layers, total_bitrate,
|
||||
kSpatialLayeringRateScalingFactor));
|
||||
RTC_DCHECK_EQ(spatial_layer_rates.size(), num_spatial_layers);
|
||||
}
|
||||
|
||||
@ -283,13 +259,10 @@ VideoBitrateAllocation SvcRateAllocator::GetAllocationNormalVideo(
|
||||
// layers since they are used for prediction of higher layers and their
|
||||
// references are far apart.
|
||||
if (num_temporal_layers == 1) {
|
||||
bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 0,
|
||||
temporal_layer_rates[0].bps());
|
||||
bitrate_allocation.SetBitrate(sl_idx, 0, temporal_layer_rates[0].bps());
|
||||
} else if (num_temporal_layers == 2) {
|
||||
bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 0,
|
||||
temporal_layer_rates[1].bps());
|
||||
bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 1,
|
||||
temporal_layer_rates[0].bps());
|
||||
bitrate_allocation.SetBitrate(sl_idx, 0, temporal_layer_rates[1].bps());
|
||||
bitrate_allocation.SetBitrate(sl_idx, 1, temporal_layer_rates[0].bps());
|
||||
} else {
|
||||
RTC_CHECK_EQ(num_temporal_layers, 3);
|
||||
// In case of three temporal layers the high layer has two frames and the
|
||||
@ -297,12 +270,9 @@ VideoBitrateAllocation SvcRateAllocator::GetAllocationNormalVideo(
|
||||
// layer frames). Thus high layer requires more bits (comparing pure
|
||||
// bitrate of layer, excluding bitrate of base layers) to keep quality on
|
||||
// par with lower layers.
|
||||
bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 0,
|
||||
temporal_layer_rates[2].bps());
|
||||
bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 1,
|
||||
temporal_layer_rates[0].bps());
|
||||
bitrate_allocation.SetBitrate(sl_idx + first_active_layer, 2,
|
||||
temporal_layer_rates[1].bps());
|
||||
bitrate_allocation.SetBitrate(sl_idx, 0, temporal_layer_rates[2].bps());
|
||||
bitrate_allocation.SetBitrate(sl_idx, 1, temporal_layer_rates[0].bps());
|
||||
bitrate_allocation.SetBitrate(sl_idx, 2, temporal_layer_rates[1].bps());
|
||||
}
|
||||
}
|
||||
|
||||
@ -314,11 +284,9 @@ VideoBitrateAllocation SvcRateAllocator::GetAllocationNormalVideo(
|
||||
// bit-rate allocated.
|
||||
VideoBitrateAllocation SvcRateAllocator::GetAllocationScreenSharing(
|
||||
DataRate total_bitrate,
|
||||
size_t first_active_layer,
|
||||
size_t num_spatial_layers) const {
|
||||
if (num_spatial_layers == 0 ||
|
||||
total_bitrate <
|
||||
DataRate::kbps(codec_.spatialLayers[first_active_layer].minBitrate)) {
|
||||
total_bitrate < DataRate::kbps(codec_.spatialLayers[0].minBitrate)) {
|
||||
return VideoBitrateAllocation();
|
||||
}
|
||||
VideoBitrateAllocation bitrate_allocation;
|
||||
@ -326,8 +294,7 @@ VideoBitrateAllocation SvcRateAllocator::GetAllocationScreenSharing(
|
||||
DataRate allocated_rate = DataRate::Zero();
|
||||
DataRate top_layer_rate = DataRate::Zero();
|
||||
size_t sl_idx;
|
||||
for (sl_idx = first_active_layer;
|
||||
sl_idx < first_active_layer + num_spatial_layers; ++sl_idx) {
|
||||
for (sl_idx = 0; sl_idx < num_spatial_layers; ++sl_idx) {
|
||||
const DataRate min_rate =
|
||||
DataRate::kbps(codec_.spatialLayers[sl_idx].minBitrate);
|
||||
const DataRate target_rate =
|
||||
@ -373,13 +340,11 @@ size_t SvcRateAllocator::FindNumEnabledLayers(DataRate target_rate) const {
|
||||
}
|
||||
|
||||
DataRate SvcRateAllocator::GetMaxBitrate(const VideoCodec& codec) {
|
||||
const size_t first_active_layer = GetFirstActiveLayer(codec);
|
||||
const size_t num_spatial_layers = GetNumActiveSpatialLayers(codec);
|
||||
|
||||
DataRate max_bitrate = DataRate::Zero();
|
||||
for (size_t sl_idx = 0; sl_idx < num_spatial_layers; ++sl_idx) {
|
||||
max_bitrate += DataRate::kbps(
|
||||
codec.spatialLayers[first_active_layer + sl_idx].maxBitrate);
|
||||
max_bitrate += DataRate::kbps(codec.spatialLayers[sl_idx].maxBitrate);
|
||||
}
|
||||
|
||||
if (codec.maxBitrate != 0) {
|
||||
@ -401,12 +366,10 @@ DataRate SvcRateAllocator::GetPaddingBitrate(const VideoCodec& codec) {
|
||||
absl::InlinedVector<DataRate, kMaxSpatialLayers>
|
||||
SvcRateAllocator::GetLayerStartBitrates(const VideoCodec& codec) {
|
||||
absl::InlinedVector<DataRate, kMaxSpatialLayers> start_bitrates;
|
||||
const size_t first_active_layer = GetFirstActiveLayer(codec);
|
||||
const size_t num_layers = GetNumActiveSpatialLayers(codec);
|
||||
size_t num_layers = GetNumActiveSpatialLayers(codec);
|
||||
DataRate last_rate = DataRate::Zero();
|
||||
for (size_t i = 1; i <= num_layers; ++i) {
|
||||
DataRate layer_toggling_rate =
|
||||
FindLayerTogglingThreshold(codec, first_active_layer, i);
|
||||
DataRate layer_toggling_rate = FindLayerTogglingThreshold(codec, i);
|
||||
start_bitrates.push_back(layer_toggling_rate);
|
||||
RTC_DCHECK_LE(last_rate, layer_toggling_rate);
|
||||
last_rate = layer_toggling_rate;
|
||||
|
||||
@ -38,12 +38,10 @@ class SvcRateAllocator : public VideoBitrateAllocator {
|
||||
private:
|
||||
VideoBitrateAllocation GetAllocationNormalVideo(
|
||||
DataRate total_bitrate,
|
||||
size_t first_active_layer,
|
||||
size_t num_spatial_layers) const;
|
||||
|
||||
VideoBitrateAllocation GetAllocationScreenSharing(
|
||||
DataRate total_bitrate,
|
||||
size_t first_active_layer,
|
||||
size_t num_spatial_layers) const;
|
||||
|
||||
// Returns the number of layers that are active and have enough bitrate to
|
||||
|
||||
@ -173,13 +173,12 @@ TEST(SvcRateAllocatorTest, MinBitrateToGetQualityLayer) {
|
||||
EXPECT_EQ(allocation.GetSpatialLayerSum(1) / 1000, layers[1].minBitrate);
|
||||
}
|
||||
|
||||
TEST(SvcRateAllocatorTest, DeactivateHigherLayers) {
|
||||
TEST(SvcRateAllocatorTest, DeativateLayers) {
|
||||
for (int deactivated_idx = 2; deactivated_idx >= 0; --deactivated_idx) {
|
||||
VideoCodec codec = Configure(1280, 720, 3, 1, false);
|
||||
EXPECT_LE(codec.VP9()->numberOfSpatialLayers, 3U);
|
||||
|
||||
for (int i = deactivated_idx; i < 3; ++i)
|
||||
codec.spatialLayers[i].active = false;
|
||||
codec.spatialLayers[deactivated_idx].active = false;
|
||||
|
||||
SvcRateAllocator allocator = SvcRateAllocator(codec);
|
||||
|
||||
@ -198,39 +197,11 @@ TEST(SvcRateAllocatorTest, DeactivateHigherLayers) {
|
||||
}
|
||||
}
|
||||
|
||||
TEST(SvcRateAllocatorTest, DeactivateLowerLayers) {
|
||||
for (int deactivated_idx = 0; deactivated_idx < 3; ++deactivated_idx) {
|
||||
VideoCodec codec = Configure(1280, 720, 3, 1, false);
|
||||
EXPECT_LE(codec.VP9()->numberOfSpatialLayers, 3U);
|
||||
|
||||
for (int i = deactivated_idx; i >= 0; --i)
|
||||
codec.spatialLayers[i].active = false;
|
||||
|
||||
SvcRateAllocator allocator = SvcRateAllocator(codec);
|
||||
|
||||
VideoBitrateAllocation allocation = allocator.Allocate(
|
||||
VideoBitrateAllocationParameters(10 * 1000 * 1000, 30));
|
||||
|
||||
// Ensure layers spatial_idx <= deactivated_idx are deactivated.
|
||||
for (int spatial_idx = 0; spatial_idx <= deactivated_idx; ++spatial_idx) {
|
||||
EXPECT_EQ(allocation.GetSpatialLayerSum(spatial_idx), 0UL);
|
||||
}
|
||||
|
||||
// Ensure layers spatial_idx > deactivated_idx are activated.
|
||||
for (int spatial_idx = deactivated_idx + 1; spatial_idx < 3;
|
||||
++spatial_idx) {
|
||||
EXPECT_GT(allocation.GetSpatialLayerSum(spatial_idx), 0UL);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST(SvcRateAllocatorTest, NoPaddingIfAllLayersAreDeactivated) {
|
||||
VideoCodec codec = Configure(1280, 720, 3, 1, false);
|
||||
EXPECT_EQ(codec.VP9()->numberOfSpatialLayers, 3U);
|
||||
// Deactivation of base layer deactivates all layers.
|
||||
codec.spatialLayers[0].active = false;
|
||||
codec.spatialLayers[1].active = false;
|
||||
codec.spatialLayers[2].active = false;
|
||||
DataRate padding_rate = SvcRateAllocator::GetPaddingBitrate(codec);
|
||||
EXPECT_EQ(padding_rate, DataRate::Zero());
|
||||
}
|
||||
@ -309,15 +280,6 @@ TEST_P(SvcRateAllocatorTestParametrizedContentType, PaddingBitrate) {
|
||||
EXPECT_GT(allocation.GetSpatialLayerSum(0), 0UL);
|
||||
EXPECT_EQ(allocation.GetSpatialLayerSum(1), 0UL);
|
||||
EXPECT_EQ(allocation.GetSpatialLayerSum(2), 0UL);
|
||||
|
||||
// Deactivate all layers.
|
||||
codec.spatialLayers[0].active = false;
|
||||
codec.spatialLayers[1].active = false;
|
||||
codec.spatialLayers[2].active = false;
|
||||
|
||||
padding_bitrate = SvcRateAllocator::GetPaddingBitrate(codec);
|
||||
// No padding expected.
|
||||
EXPECT_EQ(DataRate::Zero(), padding_bitrate);
|
||||
}
|
||||
|
||||
TEST_P(SvcRateAllocatorTestParametrizedContentType, StableBitrate) {
|
||||
|
||||
@ -352,120 +352,6 @@ TEST_F(TestVp9Impl, EnableDisableSpatialLayers) {
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(TestVp9Impl, DisableEnableBaseLayerTriggersKeyFrame) {
|
||||
// Configure encoder to produce N spatial layers. Encode frames for all
|
||||
// layers. Then disable all but the last layer. Then reenable all back again.
|
||||
const size_t num_spatial_layers = 3;
|
||||
const size_t num_frames_to_encode = 5;
|
||||
|
||||
ConfigureSvc(num_spatial_layers);
|
||||
codec_settings_.VP9()->frameDroppingOn = false;
|
||||
|
||||
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
|
||||
encoder_->InitEncode(&codec_settings_, kSettings));
|
||||
|
||||
VideoBitrateAllocation bitrate_allocation;
|
||||
for (size_t sl_idx = 0; sl_idx < num_spatial_layers; ++sl_idx) {
|
||||
// Allocate high bit rate to avoid frame dropping due to rate control.
|
||||
bitrate_allocation.SetBitrate(
|
||||
sl_idx, 0,
|
||||
codec_settings_.spatialLayers[sl_idx].targetBitrate * 1000 * 2);
|
||||
}
|
||||
encoder_->SetRates(VideoEncoder::RateControlParameters(
|
||||
bitrate_allocation, codec_settings_.maxFramerate));
|
||||
|
||||
for (size_t frame_num = 0; frame_num < num_frames_to_encode; ++frame_num) {
|
||||
SetWaitForEncodedFramesThreshold(num_spatial_layers);
|
||||
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
|
||||
encoder_->Encode(*NextInputFrame(), nullptr));
|
||||
std::vector<EncodedImage> encoded_frame;
|
||||
std::vector<CodecSpecificInfo> codec_specific_info;
|
||||
ASSERT_TRUE(WaitForEncodedFrames(&encoded_frame, &codec_specific_info));
|
||||
EXPECT_EQ(codec_specific_info[0].codecSpecific.VP9.ss_data_available,
|
||||
frame_num == 0);
|
||||
}
|
||||
|
||||
// Disable all but top layer.
|
||||
for (size_t sl_idx = 0; sl_idx < num_spatial_layers - 1; ++sl_idx) {
|
||||
bitrate_allocation.SetBitrate(sl_idx, 0, 0);
|
||||
}
|
||||
encoder_->SetRates(VideoEncoder::RateControlParameters(
|
||||
bitrate_allocation, codec_settings_.maxFramerate));
|
||||
|
||||
for (size_t frame_num = 0; frame_num < num_frames_to_encode; ++frame_num) {
|
||||
SetWaitForEncodedFramesThreshold(1);
|
||||
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
|
||||
encoder_->Encode(*NextInputFrame(), nullptr));
|
||||
std::vector<EncodedImage> encoded_frame;
|
||||
std::vector<CodecSpecificInfo> codec_specific_info;
|
||||
ASSERT_TRUE(WaitForEncodedFrames(&encoded_frame, &codec_specific_info));
|
||||
// SS available immediatly after switching off.
|
||||
EXPECT_EQ(codec_specific_info[0].codecSpecific.VP9.ss_data_available,
|
||||
frame_num == 0);
|
||||
// No key-frames generated for disabling layers.
|
||||
EXPECT_EQ(encoded_frame[0]._frameType, VideoFrameType::kVideoFrameDelta);
|
||||
}
|
||||
|
||||
// Force key-frame.
|
||||
std::vector<VideoFrameType> frame_types = {VideoFrameType::kVideoFrameKey};
|
||||
SetWaitForEncodedFramesThreshold(1);
|
||||
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
|
||||
encoder_->Encode(*NextInputFrame(), &frame_types));
|
||||
std::vector<EncodedImage> encoded_frame;
|
||||
std::vector<CodecSpecificInfo> codec_specific_info;
|
||||
ASSERT_TRUE(WaitForEncodedFrames(&encoded_frame, &codec_specific_info));
|
||||
// Key-frame should be produced.
|
||||
EXPECT_EQ(encoded_frame[0]._frameType, VideoFrameType::kVideoFrameKey);
|
||||
|
||||
// Enable the second layer back.
|
||||
// Allocate high bit rate to avoid frame dropping due to rate control.
|
||||
bitrate_allocation.SetBitrate(
|
||||
1, 0, codec_settings_.spatialLayers[0].targetBitrate * 1000 * 2);
|
||||
encoder_->SetRates(VideoEncoder::RateControlParameters(
|
||||
bitrate_allocation, codec_settings_.maxFramerate));
|
||||
|
||||
for (size_t frame_num = 0; frame_num < num_frames_to_encode; ++frame_num) {
|
||||
SetWaitForEncodedFramesThreshold(2);
|
||||
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
|
||||
encoder_->Encode(*NextInputFrame(), nullptr));
|
||||
std::vector<EncodedImage> encoded_frame;
|
||||
std::vector<CodecSpecificInfo> codec_specific_info;
|
||||
ASSERT_TRUE(WaitForEncodedFrames(&encoded_frame, &codec_specific_info));
|
||||
// SS available immediatly after switching on.
|
||||
EXPECT_EQ(codec_specific_info[0].codecSpecific.VP9.ss_data_available,
|
||||
frame_num == 0);
|
||||
// Keyframe should be generated when enabling lower layers.
|
||||
const VideoFrameType expected_type = frame_num == 0
|
||||
? VideoFrameType::kVideoFrameKey
|
||||
: VideoFrameType::kVideoFrameDelta;
|
||||
EXPECT_EQ(encoded_frame[0]._frameType, expected_type);
|
||||
}
|
||||
|
||||
// Enable the first layer back.
|
||||
// Allocate high bit rate to avoid frame dropping due to rate control.
|
||||
bitrate_allocation.SetBitrate(
|
||||
0, 0, codec_settings_.spatialLayers[1].targetBitrate * 1000 * 2);
|
||||
encoder_->SetRates(VideoEncoder::RateControlParameters(
|
||||
bitrate_allocation, codec_settings_.maxFramerate));
|
||||
|
||||
for (size_t frame_num = 0; frame_num < num_frames_to_encode; ++frame_num) {
|
||||
SetWaitForEncodedFramesThreshold(num_spatial_layers);
|
||||
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
|
||||
encoder_->Encode(*NextInputFrame(), nullptr));
|
||||
std::vector<EncodedImage> encoded_frame;
|
||||
std::vector<CodecSpecificInfo> codec_specific_info;
|
||||
ASSERT_TRUE(WaitForEncodedFrames(&encoded_frame, &codec_specific_info));
|
||||
// SS available immediatly after switching on.
|
||||
EXPECT_EQ(codec_specific_info[0].codecSpecific.VP9.ss_data_available,
|
||||
frame_num == 0);
|
||||
// Keyframe should be generated when enabling lower layers.
|
||||
const VideoFrameType expected_type = frame_num == 0
|
||||
? VideoFrameType::kVideoFrameKey
|
||||
: VideoFrameType::kVideoFrameDelta;
|
||||
EXPECT_EQ(encoded_frame[0]._frameType, expected_type);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(TestVp9Impl, EndOfPicture) {
|
||||
const size_t num_spatial_layers = 2;
|
||||
ConfigureSvc(num_spatial_layers);
|
||||
@ -846,11 +732,14 @@ TEST_F(TestVp9Impl, EnablingDisablingUpperLayerAccrossGof) {
|
||||
false);
|
||||
}
|
||||
|
||||
TEST_F(TestVp9Impl, EnablingNewLayerInScreenshareForcesAllLayersWithSS) {
|
||||
TEST_F(TestVp9Impl, EnablingNewLayerIsDelayedInScreenshareAndAddsSsInfo) {
|
||||
const size_t num_spatial_layers = 3;
|
||||
// Chosen by hand, the 2nd frame is dropped with configured per-layer max
|
||||
// framerate.
|
||||
const size_t num_frames_to_encode_before_drop = 1;
|
||||
// Chosen by hand, exactly 5 frames are dropped for input fps=30 and max
|
||||
// framerate = 5.
|
||||
const size_t num_dropped_frames = 5;
|
||||
|
||||
codec_settings_.maxFramerate = 30;
|
||||
ConfigureSvc(num_spatial_layers);
|
||||
@ -895,8 +784,18 @@ TEST_F(TestVp9Impl, EnablingNewLayerInScreenshareForcesAllLayersWithSS) {
|
||||
encoder_->SetRates(VideoEncoder::RateControlParameters(
|
||||
bitrate_allocation, codec_settings_.maxFramerate));
|
||||
|
||||
// All layers are encoded, even though frame dropping should happen.
|
||||
SetWaitForEncodedFramesThreshold(num_spatial_layers);
|
||||
for (size_t frame_num = 0; frame_num < num_dropped_frames; ++frame_num) {
|
||||
SetWaitForEncodedFramesThreshold(1);
|
||||
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
|
||||
encoder_->Encode(*NextInputFrame(), nullptr));
|
||||
// First layer is dropped due to frame rate cap. The last layer should not
|
||||
// be enabled yet.
|
||||
std::vector<EncodedImage> encoded_frames;
|
||||
std::vector<CodecSpecificInfo> codec_specific_info;
|
||||
ASSERT_TRUE(WaitForEncodedFrames(&encoded_frames, &codec_specific_info));
|
||||
}
|
||||
|
||||
SetWaitForEncodedFramesThreshold(2);
|
||||
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
|
||||
encoder_->Encode(*NextInputFrame(), nullptr));
|
||||
// Now all 3 layers should be encoded.
|
||||
|
||||
@ -15,7 +15,6 @@
|
||||
|
||||
#include <algorithm>
|
||||
#include <limits>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "absl/memory/memory.h"
|
||||
@ -138,19 +137,15 @@ ColorSpace ExtractVP9ColorSpace(vpx_color_space_t space_t,
|
||||
return ColorSpace(primaries, transfer, matrix, range);
|
||||
}
|
||||
|
||||
std::pair<size_t, size_t> GetActiveLayers(
|
||||
const VideoBitrateAllocation& allocation) {
|
||||
bool MoreLayersEnabled(const VideoBitrateAllocation& first,
|
||||
const VideoBitrateAllocation& second) {
|
||||
for (size_t sl_idx = 0; sl_idx < kMaxSpatialLayers; ++sl_idx) {
|
||||
if (allocation.GetSpatialLayerSum(sl_idx) > 0) {
|
||||
size_t last_layer = sl_idx + 1;
|
||||
while (last_layer < kMaxSpatialLayers &&
|
||||
allocation.GetSpatialLayerSum(last_layer) > 0) {
|
||||
++last_layer;
|
||||
}
|
||||
return std::make_pair(sl_idx, last_layer);
|
||||
if (first.GetSpatialLayerSum(sl_idx) > 0 &&
|
||||
second.GetSpatialLayerSum(sl_idx) == 0) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return {0, 0};
|
||||
return false;
|
||||
}
|
||||
|
||||
uint32_t Interpolate(uint32_t low,
|
||||
@ -229,7 +224,6 @@ VP9EncoderImpl::VP9EncoderImpl(const cricket::VideoCodec& codec)
|
||||
num_temporal_layers_(0),
|
||||
num_spatial_layers_(0),
|
||||
num_active_spatial_layers_(0),
|
||||
first_active_layer_(0),
|
||||
layer_deactivation_requires_key_frame_(
|
||||
field_trial::IsEnabled("WebRTC-Vp9IssueKeyFrameOnLayerDeactivation")),
|
||||
is_svc_(false),
|
||||
@ -243,7 +237,6 @@ VP9EncoderImpl::VP9EncoderImpl(const cricket::VideoCodec& codec)
|
||||
full_superframe_drop_(true),
|
||||
first_frame_in_picture_(true),
|
||||
ss_info_needed_(false),
|
||||
force_all_active_layers_(false),
|
||||
is_flexible_mode_(false),
|
||||
variable_framerate_experiment_(ParseVariableFramerateConfig(
|
||||
"WebRTC-VP9VariableFramerateScreenshare")),
|
||||
@ -296,31 +289,13 @@ bool VP9EncoderImpl::ExplicitlyConfiguredSpatialLayers() const {
|
||||
|
||||
bool VP9EncoderImpl::SetSvcRates(
|
||||
const VideoBitrateAllocation& bitrate_allocation) {
|
||||
std::pair<size_t, size_t> current_layers =
|
||||
GetActiveLayers(current_bitrate_allocation_);
|
||||
std::pair<size_t, size_t> new_layers = GetActiveLayers(bitrate_allocation);
|
||||
|
||||
const bool layer_activation_requires_key_frame =
|
||||
inter_layer_pred_ == InterLayerPredMode::kOff ||
|
||||
inter_layer_pred_ == InterLayerPredMode::kOnKeyPic;
|
||||
const bool lower_layers_enabled = new_layers.first < current_layers.first;
|
||||
const bool higher_layers_enabled = new_layers.second > current_layers.second;
|
||||
const bool disabled_layers = new_layers.first > current_layers.first ||
|
||||
new_layers.second < current_layers.second;
|
||||
|
||||
if (lower_layers_enabled ||
|
||||
(higher_layers_enabled && layer_activation_requires_key_frame) ||
|
||||
(disabled_layers && layer_deactivation_requires_key_frame_)) {
|
||||
force_key_frame_ = true;
|
||||
}
|
||||
|
||||
if (current_layers != new_layers) {
|
||||
ss_info_needed_ = true;
|
||||
}
|
||||
|
||||
config_->rc_target_bitrate = bitrate_allocation.get_sum_kbps();
|
||||
|
||||
if (ExplicitlyConfiguredSpatialLayers()) {
|
||||
const bool layer_activation_requires_key_frame =
|
||||
inter_layer_pred_ == InterLayerPredMode::kOff ||
|
||||
inter_layer_pred_ == InterLayerPredMode::kOnKeyPic;
|
||||
|
||||
for (size_t sl_idx = 0; sl_idx < num_spatial_layers_; ++sl_idx) {
|
||||
const bool was_layer_active = (config_->ss_target_bitrate[sl_idx] > 0);
|
||||
config_->ss_target_bitrate[sl_idx] =
|
||||
@ -331,6 +306,15 @@ bool VP9EncoderImpl::SetSvcRates(
|
||||
bitrate_allocation.GetTemporalLayerSum(sl_idx, tl_idx) / 1000;
|
||||
}
|
||||
|
||||
const bool is_active_layer = (config_->ss_target_bitrate[sl_idx] > 0);
|
||||
if (!was_layer_active && is_active_layer &&
|
||||
layer_activation_requires_key_frame) {
|
||||
force_key_frame_ = true;
|
||||
} else if (was_layer_active && !is_active_layer &&
|
||||
layer_deactivation_requires_key_frame_) {
|
||||
force_key_frame_ = true;
|
||||
}
|
||||
|
||||
if (!was_layer_active) {
|
||||
// Reset frame rate controller if layer is resumed after pause.
|
||||
framerate_controller_[sl_idx].Reset();
|
||||
@ -383,34 +367,13 @@ bool VP9EncoderImpl::SetSvcRates(
|
||||
}
|
||||
|
||||
num_active_spatial_layers_ = 0;
|
||||
first_active_layer_ = 0;
|
||||
bool seen_active_layer = false;
|
||||
bool expect_no_more_active_layers = false;
|
||||
for (int i = 0; i < num_spatial_layers_; ++i) {
|
||||
if (config_->ss_target_bitrate[i] > 0) {
|
||||
RTC_DCHECK(!expect_no_more_active_layers) << "Only middle layer is "
|
||||
"deactivated.";
|
||||
if (!seen_active_layer) {
|
||||
first_active_layer_ = i;
|
||||
}
|
||||
num_active_spatial_layers_ = i + 1;
|
||||
seen_active_layer = true;
|
||||
} else {
|
||||
expect_no_more_active_layers = seen_active_layer;
|
||||
++num_active_spatial_layers_;
|
||||
}
|
||||
}
|
||||
RTC_DCHECK_GT(num_active_spatial_layers_, 0);
|
||||
|
||||
if (higher_layers_enabled && !force_key_frame_) {
|
||||
// Prohibit drop of all layers for the next frame, so newly enabled
|
||||
// layer would have a valid spatial reference.
|
||||
for (size_t i = 0; i < num_spatial_layers_; ++i) {
|
||||
svc_drop_frame_.framedrop_thresh[i] = 0;
|
||||
}
|
||||
force_all_active_layers_ = true;
|
||||
}
|
||||
|
||||
current_bitrate_allocation_ = bitrate_allocation;
|
||||
return true;
|
||||
}
|
||||
|
||||
@ -430,16 +393,7 @@ void VP9EncoderImpl::SetRates(const RateControlParameters& parameters) {
|
||||
}
|
||||
|
||||
codec_.maxFramerate = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
|
||||
|
||||
if (dynamic_rate_settings_) {
|
||||
// Tweak rate control settings based on available network headroom.
|
||||
UpdateRateSettings(
|
||||
config_, GetRateSettings(parameters.bandwidth_allocation.bps<double>() /
|
||||
parameters.bitrate.get_sum_bps()));
|
||||
}
|
||||
|
||||
bool res = SetSvcRates(parameters.bitrate);
|
||||
RTC_DCHECK(res) << "Failed to set new bitrate allocation";
|
||||
requested_rate_settings_ = parameters;
|
||||
}
|
||||
|
||||
// TODO(eladalon): s/inst/codec_settings/g.
|
||||
@ -876,10 +830,6 @@ int VP9EncoderImpl::Encode(const VideoFrame& input_image,
|
||||
num_steady_state_frames_ >=
|
||||
variable_framerate_experiment_.frames_before_steady_state;
|
||||
|
||||
// Need to check all frame limiters, even if lower layers are disabled,
|
||||
// because variable frame-rate limiter should be checked after the first
|
||||
// layer. It's easier to overwrite active layers after, then check all
|
||||
// cases.
|
||||
for (uint8_t sl_idx = 0; sl_idx < num_active_spatial_layers_; ++sl_idx) {
|
||||
const float layer_fps =
|
||||
framerate_controller_[layer_id.spatial_layer_id].GetTargetRate();
|
||||
@ -906,11 +856,6 @@ int VP9EncoderImpl::Encode(const VideoFrame& input_image,
|
||||
}
|
||||
}
|
||||
|
||||
if (force_all_active_layers_) {
|
||||
layer_id.spatial_layer_id = first_active_layer_;
|
||||
force_all_active_layers_ = false;
|
||||
}
|
||||
|
||||
RTC_DCHECK_LE(layer_id.spatial_layer_id, num_active_spatial_layers_);
|
||||
if (layer_id.spatial_layer_id >= num_active_spatial_layers_) {
|
||||
// Drop entire picture.
|
||||
@ -922,12 +867,50 @@ int VP9EncoderImpl::Encode(const VideoFrame& input_image,
|
||||
layer_id.temporal_layer_id_per_spatial[sl_idx] = layer_id.temporal_layer_id;
|
||||
}
|
||||
|
||||
if (layer_id.spatial_layer_id < first_active_layer_) {
|
||||
layer_id.spatial_layer_id = first_active_layer_;
|
||||
}
|
||||
|
||||
vpx_codec_control(encoder_, VP9E_SET_SVC_LAYER_ID, &layer_id);
|
||||
|
||||
if (requested_rate_settings_) {
|
||||
if (dynamic_rate_settings_) {
|
||||
// Tweak rate control settings based on available network headroom.
|
||||
UpdateRateSettings(
|
||||
config_,
|
||||
GetRateSettings(
|
||||
requested_rate_settings_->bandwidth_allocation.bps<double>() /
|
||||
requested_rate_settings_->bitrate.get_sum_bps()));
|
||||
}
|
||||
|
||||
bool more_layers_requested = MoreLayersEnabled(
|
||||
requested_rate_settings_->bitrate, current_bitrate_allocation_);
|
||||
bool less_layers_requested = MoreLayersEnabled(
|
||||
current_bitrate_allocation_, requested_rate_settings_->bitrate);
|
||||
// In SVC can enable new layers only if all lower layers are encoded and at
|
||||
// the base temporal layer.
|
||||
// This will delay rate allocation change until the next frame on the base
|
||||
// spatial layer.
|
||||
// In KSVC or simulcast modes KF will be generated for a new layer, so can
|
||||
// update allocation any time.
|
||||
bool can_upswitch =
|
||||
inter_layer_pred_ != InterLayerPredMode::kOn ||
|
||||
(layer_id.spatial_layer_id == 0 && layer_id.temporal_layer_id == 0);
|
||||
if (!more_layers_requested || can_upswitch) {
|
||||
current_bitrate_allocation_ = requested_rate_settings_->bitrate;
|
||||
requested_rate_settings_ = absl::nullopt;
|
||||
if (!SetSvcRates(current_bitrate_allocation_)) {
|
||||
return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
|
||||
}
|
||||
if (less_layers_requested || more_layers_requested) {
|
||||
ss_info_needed_ = true;
|
||||
}
|
||||
if (more_layers_requested && !force_key_frame_) {
|
||||
// Prohibit drop of all layers for the next frame, so newly enabled
|
||||
// layer would have a valid spatial reference.
|
||||
for (size_t i = 0; i < num_spatial_layers_; ++i) {
|
||||
svc_drop_frame_.framedrop_thresh[i] = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (num_spatial_layers_ > 1) {
|
||||
// Update frame dropping settings as they may change on per-frame basis.
|
||||
vpx_codec_control(encoder_, VP9E_SET_SVC_FRAME_DROP_LAYER,
|
||||
@ -1134,15 +1117,10 @@ void VP9EncoderImpl::PopulateCodecSpecific(CodecSpecificInfo* codec_specific,
|
||||
// of key picture (inter-layer prediction is enabled).
|
||||
const bool is_key_frame = is_key_pic && !vp9_info->inter_layer_predicted;
|
||||
if (is_key_frame || (ss_info_needed_ && layer_id.temporal_layer_id == 0 &&
|
||||
layer_id.spatial_layer_id == first_active_layer_)) {
|
||||
layer_id.spatial_layer_id == 0)) {
|
||||
vp9_info->ss_data_available = true;
|
||||
vp9_info->spatial_layer_resolution_present = true;
|
||||
// Signal disabled layers.
|
||||
for (size_t i = 0; i < first_active_layer_; ++i) {
|
||||
vp9_info->width[i] = 0;
|
||||
vp9_info->height[i] = 0;
|
||||
}
|
||||
for (size_t i = first_active_layer_; i < num_active_spatial_layers_; ++i) {
|
||||
for (size_t i = 0; i < num_active_spatial_layers_; ++i) {
|
||||
vp9_info->width[i] = codec_.width * svc_params_.scaling_factor_num[i] /
|
||||
svc_params_.scaling_factor_den[i];
|
||||
vp9_info->height[i] = codec_.height * svc_params_.scaling_factor_num[i] /
|
||||
|
||||
@ -119,7 +119,6 @@ class VP9EncoderImpl : public VP9Encoder {
|
||||
uint8_t num_temporal_layers_;
|
||||
uint8_t num_spatial_layers_; // Number of configured SLs
|
||||
uint8_t num_active_spatial_layers_; // Number of actively encoded SLs
|
||||
uint8_t first_active_layer_;
|
||||
bool layer_deactivation_requires_key_frame_;
|
||||
bool is_svc_;
|
||||
InterLayerPredMode inter_layer_pred_;
|
||||
@ -131,8 +130,8 @@ class VP9EncoderImpl : public VP9Encoder {
|
||||
vpx_svc_frame_drop_t svc_drop_frame_;
|
||||
bool first_frame_in_picture_;
|
||||
VideoBitrateAllocation current_bitrate_allocation_;
|
||||
absl::optional<RateControlParameters> requested_rate_settings_;
|
||||
bool ss_info_needed_;
|
||||
bool force_all_active_layers_;
|
||||
|
||||
std::vector<FramerateController> framerate_controller_;
|
||||
|
||||
|
||||
Reference in New Issue
Block a user