zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Apply simulcast resolution normalization before scaling.

Browse Source 
With this CL, we normalize the resolution coming from the
capturer, before applying the requested scaling factors.
That has the benefit that the actual scale factor between
two layers will be the fraction of the requested scale
factors of the two layers.

Prior to this CL, when the normalization was done per layer,
the actual scale factor between two layers might not
have been the fraction of the requested scale factors
of the two layers.

Bug: webrtc:10069
Change-Id: I9ca4d394f259d5d37faee96a41204ff8df898907
Reviewed-on: https://webrtc-review.googlesource.com/c/121425
Commit-Queue: Rasmus Brandt <brandtr@webrtc.org>
Reviewed-by: Mirta Dvornicic <mirtad@webrtc.org>
Reviewed-by: Florent Castelli <orphis@webrtc.org>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#26550}
This commit is contained in:
Rasmus Brandt
2019-02-05 14:23:26 +01:00
committed by Commit Bot
parent 1f0a84a2ec
commit 9387b52297
2 changed files with 122 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
media/engine/webrtc_video_engine.cc
View File

@ -2698,22 +2698,26 @@ std::vector<webrtc::VideoStream> EncoderStreamFactory::CreateEncoderStreams(
((absl::EqualsIgnoreCase(codec_name_, kVp8CodecName) ||
absl::EqualsIgnoreCase(codec_name_, kH264CodecName)) &&
is_screenshare_ && screenshare_config_explicitly_enabled_)) {
bool temporal_layers_supported =
const bool temporal_layers_supported =
absl::EqualsIgnoreCase(codec_name_, kVp8CodecName);
layers = GetSimulcastConfig(encoder_config.number_of_streams, width, height,
0 /*not used*/, encoder_config.bitrate_priority,
max_qp_, 0 /*not_used*/, is_screenshare_,
temporal_layers_supported);
// The maximum |max_framerate| is currently used for video.
int max_framerate = GetMaxFramerate(encoder_config, layers.size());
const int max_framerate = GetMaxFramerate(encoder_config, layers.size());
// Update the active simulcast layers and configured bitrates.
bool is_highest_layer_max_bitrate_configured = false;
bool has_scale_resolution_down_by =
const bool has_scale_resolution_down_by =
std::any_of(encoder_config.simulcast_layers.begin(),
encoder_config.simulcast_layers.end(),
[](const webrtc::VideoStream& layer) {
return layer.scale_resolution_down_by != -1.;
});
const int normalized_width =
NormalizeSimulcastSize(width, encoder_config.number_of_streams);
const int normalized_height =
NormalizeSimulcastSize(height, encoder_config.number_of_streams);
for (size_t i = 0; i < layers.size(); ++i) {
layers[i].active = encoder_config.simulcast_layers[i].active;
if (!is_screenshare_) {
@ -2727,16 +2731,14 @@ std::vector<webrtc::VideoStream> EncoderStreamFactory::CreateEncoderStreams(
*encoder_config.simulcast_layers[i].num_temporal_layers;
}
if (has_scale_resolution_down_by) {
double scale_resolution_down_by = std::max(
const double scale_resolution_down_by = std::max(
encoder_config.simulcast_layers[i].scale_resolution_down_by, 1.0);
layers[i].width =
std::max(NormalizeSimulcastSize(width / scale_resolution_down_by,
encoder_config.number_of_streams),
kMinLayerSize);
layers[i].height =
std::max(NormalizeSimulcastSize(height / scale_resolution_down_by,
encoder_config.number_of_streams),
kMinLayerSize);
layers[i].width = std::max(
static_cast<int>(normalized_width / scale_resolution_down_by),
kMinLayerSize);
layers[i].height = std::max(
static_cast<int>(normalized_height / scale_resolution_down_by),
kMinLayerSize);
}
// Update simulcast bitrates with configured min and max bitrate.
if (encoder_config.simulcast_layers[i].min_bitrate_bps > 0) {

117
media/engine/webrtc_video_engine_unittest.cc
View File

@ -5549,14 +5549,13 @@ TEST_F(WebRtcVideoChannelTest, SetRtpSendParametersInvalidNetworkPriority) {
TEST_F(WebRtcVideoChannelTest,
GetAndSetRtpSendParametersScaleResolutionDownByVP8) {
cricket::VideoSendParameters parameters;
parameters.codecs.push_back(cricket::VideoCodec("VP8"));
VideoSendParameters parameters;
parameters.codecs.push_back(VideoCodec(kVp8CodecName));
ASSERT_TRUE(channel_->SetSendParameters(parameters));
FakeVideoSendStream* stream = SetUpSimulcast(true, false);
webrtc::test::FrameForwarder frame_forwarder;
cricket::FakeFrameSource frame_source(1280, 720,
rtc::kNumMicrosecsPerSec / 30);
FakeFrameSource frame_source(1280, 720, rtc::kNumMicrosecsPerSec / 30);
VideoOptions options;
EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, &options, &frame_forwarder));
@ -5653,17 +5652,66 @@ TEST_F(WebRtcVideoChannelTest,
EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, nullptr, nullptr));
}
TEST_F(WebRtcVideoChannelTest,
GetAndSetRtpSendParametersScaleResolutionDownByVP8WithOddResolution) {
// Ensure that the top layer has width and height divisible by 2^3,
// so that the bottom layer has width and height divisible by 2.
// TODO(bugs.webrtc.org/8785): Remove this field trial when we fully trust
// the number of simulcast layers set by the app.
webrtc::test::ScopedFieldTrials field_trial(
"WebRTC-NormalizeSimulcastResolution/Enabled-3/");
// Set up WebRtcVideoChannel for 3-layer VP8 simulcast.
VideoSendParameters parameters;
parameters.codecs.push_back(VideoCodec(kVp8CodecName));
ASSERT_TRUE(channel_->SetSendParameters(parameters));
FakeVideoSendStream* stream = SetUpSimulcast(true, false);
webrtc::test::FrameForwarder frame_forwarder;
EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, /*options=*/nullptr,
&frame_forwarder));
channel_->SetSend(true);
// Set |scale_resolution_down_by|'s.
auto rtp_parameters = channel_->GetRtpSendParameters(last_ssrc_);
ASSERT_EQ(rtp_parameters.encodings.size(), 3u);
rtp_parameters.encodings[0].scale_resolution_down_by = 1.0;
rtp_parameters.encodings[1].scale_resolution_down_by = 2.0;
rtp_parameters.encodings[2].scale_resolution_down_by = 4.0;
const auto result =
channel_->SetRtpSendParameters(last_ssrc_, rtp_parameters);
ASSERT_TRUE(result.ok());
// Use a capture resolution whose width and height are not divisible by 2^3.
// (See field trial set at the top of the test.)
FakeFrameSource frame_source(2007, 1207, rtc::kNumMicrosecsPerSec / 30);
frame_forwarder.IncomingCapturedFrame(frame_source.GetFrame());
// Ensure the scaling is correct.
const auto video_streams = stream->GetVideoStreams();
ASSERT_EQ(video_streams.size(), 3u);
// Ensure that we round the capture resolution down for the top layer...
EXPECT_EQ(video_streams[0].width, 2000u);
EXPECT_EQ(video_streams[0].height, 1200u);
EXPECT_EQ(video_streams[1].width, 1000u);
EXPECT_EQ(video_streams[1].height, 600u);
// ...and that the bottom layer has a width/height divisible by 2.
EXPECT_EQ(video_streams[2].width, 500u);
EXPECT_EQ(video_streams[2].height, 300u);
// Tear down.
EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, nullptr, nullptr));
}
TEST_F(WebRtcVideoChannelTest,
GetAndSetRtpSendParametersScaleResolutionDownByH264) {
encoder_factory_->AddSupportedVideoCodecType("H264");
cricket::VideoSendParameters parameters;
parameters.codecs.push_back(cricket::VideoCodec("H264"));
encoder_factory_->AddSupportedVideoCodecType(kH264CodecName);
VideoSendParameters parameters;
parameters.codecs.push_back(VideoCodec(kH264CodecName));
ASSERT_TRUE(channel_->SetSendParameters(parameters));
FakeVideoSendStream* stream = SetUpSimulcast(true, false);
webrtc::test::FrameForwarder frame_forwarder;
cricket::FakeFrameSource frame_source(1280, 720,
rtc::kNumMicrosecsPerSec / 30);
FakeFrameSource frame_source(1280, 720, rtc::kNumMicrosecsPerSec / 30);
VideoOptions options;
EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, &options, &frame_forwarder));
@ -5759,6 +5807,57 @@ TEST_F(WebRtcVideoChannelTest,
EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, nullptr, nullptr));
}
TEST_F(WebRtcVideoChannelTest,
GetAndSetRtpSendParametersScaleResolutionDownByH264WithOddResolution) {
// Ensure that the top layer has width and height divisible by 2^3,
// so that the bottom layer has width and height divisible by 2.
// TODO(bugs.webrtc.org/8785): Remove this field trial when we fully trust
// the number of simulcast layers set by the app.
webrtc::test::ScopedFieldTrials field_trial(
"WebRTC-NormalizeSimulcastResolution/Enabled-3/");
// Set up WebRtcVideoChannel for 3-layer H264 simulcast.
encoder_factory_->AddSupportedVideoCodecType(kH264CodecName);
VideoSendParameters parameters;
parameters.codecs.push_back(VideoCodec(kH264CodecName));
ASSERT_TRUE(channel_->SetSendParameters(parameters));
FakeVideoSendStream* stream = SetUpSimulcast(true, false);
webrtc::test::FrameForwarder frame_forwarder;
EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, /*options=*/nullptr,
&frame_forwarder));
channel_->SetSend(true);
// Set |scale_resolution_down_by|'s.
auto rtp_parameters = channel_->GetRtpSendParameters(last_ssrc_);
ASSERT_EQ(rtp_parameters.encodings.size(), 3u);
rtp_parameters.encodings[0].scale_resolution_down_by = 1.0;
rtp_parameters.encodings[1].scale_resolution_down_by = 2.0;
rtp_parameters.encodings[2].scale_resolution_down_by = 4.0;
const auto result =
channel_->SetRtpSendParameters(last_ssrc_, rtp_parameters);
ASSERT_TRUE(result.ok());
// Use a capture resolution whose width and height are not divisible by 2^3.
// (See field trial set at the top of the test.)
FakeFrameSource frame_source(2007, 1207, rtc::kNumMicrosecsPerSec / 30);
frame_forwarder.IncomingCapturedFrame(frame_source.GetFrame());
// Ensure the scaling is correct.
const auto video_streams = stream->GetVideoStreams();
ASSERT_EQ(video_streams.size(), 3u);
// Ensure that we round the capture resolution down for the top layer...
EXPECT_EQ(video_streams[0].width, 2000u);
EXPECT_EQ(video_streams[0].height, 1200u);
EXPECT_EQ(video_streams[1].width, 1000u);
EXPECT_EQ(video_streams[1].height, 600u);
// ...and that the bottom layer has a width/height divisible by 2.
EXPECT_EQ(video_streams[2].width, 500u);
EXPECT_EQ(video_streams[2].height, 300u);
// Tear down.
EXPECT_TRUE(channel_->SetVideoSend(last_ssrc_, nullptr, nullptr));
}
TEST_F(WebRtcVideoChannelTest, GetAndSetRtpSendParametersMaxFramerate) {
const size_t kNumSimulcastStreams = 3;
SetUpSimulcast(true, false);