zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote tracking branch 'upstream-master'

Browse Source 
Bug: 153469641
Test: run cuttlefish locally
Change-Id: Ida3bfe62ef5c6549278f4c155a1f690b008e9b9d
This commit is contained in:
Jorge E. Moreira
2020-07-23 13:07:40 -07:00
parent 889b21a2e1 708b22cd03
commit 206ccd0b36
1236 changed files with 50564 additions and 32463 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
modules/video_coding/codecs/vp9/include/vp9.h
View File

@ -25,6 +25,10 @@ namespace webrtc {
// negotiate in SDP, in order of preference.
std::vector<SdpVideoFormat> SupportedVP9Codecs();
// Returns a vector with all supported internal VP9 decode profiles in order of
// preference. These will be availble for receive-only connections.
std::vector<SdpVideoFormat> SupportedVP9DecoderCodecs();
class VP9Encoder : public VideoEncoder {
public:
// Deprecated. Returns default implementation using VP9 Profile 0.

4
modules/video_coding/codecs/vp9/include/vp9_globals.h
View File

@ -30,8 +30,8 @@ const size_t kMaxVp9RefPics = 3;
const size_t kMaxVp9FramesInGof = 0xFF; // 8 bits
const size_t kMaxVp9NumberOfSpatialLayers = 8;
const size_t kMinVp9SpatialLayerWidth = 320;
const size_t kMinVp9SpatialLayerHeight = 180;
const size_t kMinVp9SpatialLayerWidth = 240;
const size_t kMinVp9SpatialLayerHeight = 135;
enum TemporalStructureMode {
kTemporalStructureMode1, // 1 temporal layer structure - i.e., IPPP...

30
modules/video_coding/codecs/vp9/svc_config.cc
View File

@ -16,6 +16,7 @@
#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
namespace webrtc {
@ -74,11 +75,23 @@ std::vector<SpatialLayer> ConfigureSvcNormalVideo(size_t input_width,
const size_t num_layers_fit_vert = static_cast<size_t>(
std::floor(1 + std::max(0.0f, std::log2(1.0f * input_height /
kMinVp9SpatialLayerHeight))));
num_spatial_layers =
std::min({num_spatial_layers, num_layers_fit_horz, num_layers_fit_vert});
const size_t limited_num_spatial_layers =
std::min(num_layers_fit_horz, num_layers_fit_vert);
if (limited_num_spatial_layers < num_spatial_layers) {
RTC_LOG(LS_WARNING) << "Reducing number of spatial layers from "
<< num_spatial_layers << " to "
<< limited_num_spatial_layers
<< " due to low input resolution.";
num_spatial_layers = limited_num_spatial_layers;
}
// First active layer must be configured.
num_spatial_layers = std::max(num_spatial_layers, first_active_layer + 1);
// Ensure top layer is even enough.
int required_divisiblity = 1 << (num_spatial_layers - first_active_layer - 1);
input_width = input_width - input_width % required_divisiblity;
input_height = input_height - input_height % required_divisiblity;
for (size_t sl_idx = first_active_layer; sl_idx < num_spatial_layers;
++sl_idx) {
SpatialLayer spatial_layer = {0};
@ -108,6 +121,19 @@ std::vector<SpatialLayer> ConfigureSvcNormalVideo(size_t input_width,
spatial_layers.push_back(spatial_layer);
}
// A workaround for sitiation when single HD layer is left with minBitrate
// about 500kbps. This would mean that there will always be at least 500kbps
// allocated to video regardless of how low is the actual BWE.
// Also, boost maxBitrate for the first layer to account for lost ability to
// predict from previous layers.
if (first_active_layer > 0) {
spatial_layers[0].minBitrate = kMinVp9SvcBitrateKbps;
// TODO(ilnik): tune this value or come up with a different formula to
// ensure that all singlecast configurations look good and not too much
// bitrate is added.
spatial_layers[0].maxBitrate *= 1.1;
}
return spatial_layers;
}

26
modules/video_coding/codecs/vp9/svc_config_unittest.cc
View File

@ -41,6 +41,32 @@ TEST(SvcConfig, AlwaysSendsAtLeastOneLayer) {
EXPECT_EQ(spatial_layers.back().width, kMinVp9SpatialLayerWidth);
}
TEST(SvcConfig, EnforcesMinimalRequiredParity) {
const size_t max_num_spatial_layers = 3;
const size_t kOddSize = 1023;
std::vector<SpatialLayer> spatial_layers =
GetSvcConfig(kOddSize, kOddSize, 30,
/*first_active_layer=*/1, max_num_spatial_layers, 1, false);
// Since there are 2 layers total (1, 2), divisiblity by 2 is required.
EXPECT_EQ(spatial_layers.back().width, kOddSize - 1);
EXPECT_EQ(spatial_layers.back().width, kOddSize - 1);
spatial_layers =
GetSvcConfig(kOddSize, kOddSize, 30,
/*first_active_layer=*/0, max_num_spatial_layers, 1, false);
// Since there are 3 layers total (0, 1, 2), divisiblity by 4 is required.
EXPECT_EQ(spatial_layers.back().width, kOddSize - 3);
EXPECT_EQ(spatial_layers.back().width, kOddSize - 3);
spatial_layers =
GetSvcConfig(kOddSize, kOddSize, 30,
/*first_active_layer=*/2, max_num_spatial_layers, 1, false);
// Since there is only 1 layer active (2), divisiblity by 1 is required.
EXPECT_EQ(spatial_layers.back().width, kOddSize);
EXPECT_EQ(spatial_layers.back().width, kOddSize);
}
TEST(SvcConfig, SkipsInactiveLayers) {
const size_t num_spatial_layers = 4;
const size_t first_active_layer = 2;

3
modules/video_coding/codecs/vp9/svc_rate_allocator.cc
View File

@ -140,7 +140,8 @@ DataRate FindLayerTogglingThreshold(const VideoCodec& codec,
}
}
upper_bound += DataRate::KilobitsPerSec(
codec.spatialLayers[num_active_layers - 1].minBitrate);
codec.spatialLayers[first_active_layer + num_active_layers - 1]
.minBitrate);
// Do a binary search until upper and lower bound is the highest bitrate for
// |num_active_layers| - 1 layers and lowest bitrate for |num_active_layers|

275
modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc
View File

@ -16,6 +16,7 @@
#include "common_video/libyuv/include/webrtc_libyuv.h"
#include "media/base/vp9_profile.h"
#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h"
#include "modules/video_coding/codecs/test/encoded_video_frame_producer.h"
#include "modules/video_coding/codecs/test/video_codec_unittest.h"
#include "modules/video_coding/codecs/vp9/include/vp9.h"
#include "modules/video_coding/codecs/vp9/svc_config.h"
@ -25,20 +26,33 @@
#include "test/video_codec_settings.h"
namespace webrtc {
namespace {
using ::testing::ElementsAreArray;
using ::testing::SizeIs;
using ::testing::UnorderedElementsAreArray;
using EncoderInfo = webrtc::VideoEncoder::EncoderInfo;
using FramerateFractions =
absl::InlinedVector<uint8_t, webrtc::kMaxTemporalStreams>;
namespace {
const size_t kWidth = 1280;
const size_t kHeight = 720;
constexpr size_t kWidth = 1280;
constexpr size_t kHeight = 720;
const VideoEncoder::Capabilities kCapabilities(false);
const VideoEncoder::Settings kSettings(kCapabilities,
/*number_of_cores=*/1,
/*max_payload_size=*/0);
VideoCodec DefaultCodecSettings() {
VideoCodec codec_settings;
webrtc::test::CodecSettings(kVideoCodecVP9, &codec_settings);
codec_settings.width = kWidth;
codec_settings.height = kHeight;
codec_settings.VP9()->numberOfTemporalLayers = 1;
codec_settings.VP9()->numberOfSpatialLayers = 1;
return codec_settings;
}
} // namespace
class TestVp9Impl : public VideoCodecUnitTest {
@ -59,53 +73,6 @@ class TestVp9Impl : public VideoCodecUnitTest {
codec_settings->VP9()->numberOfSpatialLayers = 1;
}
void ExpectFrameWith(uint8_t temporal_idx) {
EncodedImage encoded_frame;
CodecSpecificInfo codec_specific_info;
ASSERT_TRUE(WaitForEncodedFrame(&encoded_frame, &codec_specific_info));
EXPECT_EQ(temporal_idx, codec_specific_info.codecSpecific.VP9.temporal_idx);
}
void ExpectFrameWith(size_t num_spatial_layers,
uint8_t temporal_idx,
bool temporal_up_switch,
uint8_t num_ref_pics,
const std::vector<uint8_t>& p_diff) {
std::vector<EncodedImage> encoded_frame;
std::vector<CodecSpecificInfo> codec_specific;
ASSERT_TRUE(WaitForEncodedFrames(&encoded_frame, &codec_specific));
for (size_t spatial_idx = 0; spatial_idx < num_spatial_layers;
++spatial_idx) {
const CodecSpecificInfoVP9& vp9 =
codec_specific[spatial_idx].codecSpecific.VP9;
if (vp9.temporal_idx == kNoTemporalIdx) {
EXPECT_EQ(temporal_idx, 0);
} else {
EXPECT_EQ(vp9.temporal_idx, temporal_idx);
}
if (num_spatial_layers == 1) {
EXPECT_FALSE(encoded_frame[spatial_idx].SpatialIndex());
} else {
EXPECT_EQ(encoded_frame[spatial_idx].SpatialIndex(),
static_cast<int>(spatial_idx));
}
EXPECT_EQ(vp9.temporal_up_switch, temporal_up_switch);
// Ensure there are no duplicates in reference list.
std::vector<uint8_t> vp9_p_diff(vp9.p_diff,
vp9.p_diff + vp9.num_ref_pics);
std::sort(vp9_p_diff.begin(), vp9_p_diff.end());
EXPECT_EQ(std::unique(vp9_p_diff.begin(), vp9_p_diff.end()),
vp9_p_diff.end());
for (size_t ref_pic_num = 0; ref_pic_num < num_ref_pics; ++ref_pic_num) {
EXPECT_NE(
std::find(p_diff.begin(), p_diff.end(), vp9.p_diff[ref_pic_num]),
p_diff.end());
}
}
}
void ConfigureSvc(size_t num_spatial_layers, size_t num_temporal_layers = 1) {
codec_settings_.VP9()->numberOfSpatialLayers =
static_cast<unsigned char>(num_spatial_layers);
@ -187,57 +154,61 @@ TEST_F(TestVp9Impl, DecodedQpEqualsEncodedQp) {
EXPECT_EQ(encoded_frame.qp_, *decoded_qp);
}
TEST_F(TestVp9Impl, ParserQpEqualsEncodedQp) {
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(NextInputFrame(), nullptr));
EncodedImage encoded_frame;
CodecSpecificInfo codec_specific_info;
ASSERT_TRUE(WaitForEncodedFrame(&encoded_frame, &codec_specific_info));
TEST(Vp9ImplTest, ParserQpEqualsEncodedQp) {
std::unique_ptr<VideoEncoder> encoder = VP9Encoder::Create();
VideoCodec codec_settings = DefaultCodecSettings();
encoder->InitEncode(&codec_settings, kSettings);
std::vector<EncodedVideoFrameProducer::EncodedFrame> frames =
EncodedVideoFrameProducer(*encoder)
.SetNumInputFrames(1)
.SetResolution({kWidth, kHeight})
.Encode();
ASSERT_THAT(frames, SizeIs(1));
const auto& encoded_frame = frames.front().encoded_image;
int qp = 0;
ASSERT_TRUE(vp9::GetQp(encoded_frame.data(), encoded_frame.size(), &qp));
EXPECT_EQ(encoded_frame.qp_, qp);
}
TEST_F(TestVp9Impl, EncoderWith2TemporalLayers) {
// Override default settings.
codec_settings_.VP9()->numberOfTemporalLayers = 2;
TEST(Vp9ImplTest, EncoderWith2TemporalLayers) {
std::unique_ptr<VideoEncoder> encoder = VP9Encoder::Create();
VideoCodec codec_settings = DefaultCodecSettings();
codec_settings.VP9()->numberOfTemporalLayers = 2;
// Tl0PidIdx is only used in non-flexible mode.
codec_settings_.VP9()->flexibleMode = false;
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
encoder_->InitEncode(&codec_settings_, kSettings));
codec_settings.VP9()->flexibleMode = false;
EXPECT_EQ(encoder->InitEncode(&codec_settings, kSettings),
WEBRTC_VIDEO_CODEC_OK);
// Temporal layer 0.
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(NextInputFrame(), nullptr));
EncodedImage encoded_frame;
CodecSpecificInfo codec_specific_info;
ASSERT_TRUE(WaitForEncodedFrame(&encoded_frame, &codec_specific_info));
EXPECT_EQ(0, codec_specific_info.codecSpecific.VP9.temporal_idx);
std::vector<EncodedVideoFrameProducer::EncodedFrame> frames =
EncodedVideoFrameProducer(*encoder)
.SetNumInputFrames(4)
.SetResolution({kWidth, kHeight})
.Encode();
// Temporal layer 1.
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(NextInputFrame(), nullptr));
ExpectFrameWith(1);
// Temporal layer 0.
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(NextInputFrame(), nullptr));
ExpectFrameWith(0);
// Temporal layer 1.
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(NextInputFrame(), nullptr));
ExpectFrameWith(1);
ASSERT_THAT(frames, SizeIs(4));
EXPECT_EQ(frames[0].codec_specific_info.codecSpecific.VP9.temporal_idx, 0);
EXPECT_EQ(frames[1].codec_specific_info.codecSpecific.VP9.temporal_idx, 1);
EXPECT_EQ(frames[2].codec_specific_info.codecSpecific.VP9.temporal_idx, 0);
EXPECT_EQ(frames[3].codec_specific_info.codecSpecific.VP9.temporal_idx, 1);
}
TEST_F(TestVp9Impl, EncoderWith2SpatialLayers) {
codec_settings_.VP9()->numberOfSpatialLayers = 2;
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
encoder_->InitEncode(&codec_settings_, kSettings));
TEST(Vp9ImplTest, EncoderWith2SpatialLayers) {
std::unique_ptr<VideoEncoder> encoder = VP9Encoder::Create();
VideoCodec codec_settings = DefaultCodecSettings();
codec_settings.VP9()->numberOfSpatialLayers = 2;
EXPECT_EQ(encoder->InitEncode(&codec_settings, kSettings),
WEBRTC_VIDEO_CODEC_OK);
SetWaitForEncodedFramesThreshold(2);
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(NextInputFrame(), nullptr));
std::vector<EncodedImage> encoded_frame;
std::vector<CodecSpecificInfo> codec_info;
ASSERT_TRUE(WaitForEncodedFrames(&encoded_frame, &codec_info));
EXPECT_EQ(encoded_frame[0].SpatialIndex(), 0);
EXPECT_EQ(encoded_frame[1].SpatialIndex(), 1);
std::vector<EncodedVideoFrameProducer::EncodedFrame> frames =
EncodedVideoFrameProducer(*encoder)
.SetNumInputFrames(1)
.SetResolution({kWidth, kHeight})
.Encode();
ASSERT_THAT(frames, SizeIs(2));
EXPECT_EQ(frames[0].encoded_image.SpatialIndex(), 0);
EXPECT_EQ(frames[1].encoded_image.SpatialIndex(), 1);
}
TEST_F(TestVp9Impl, EncoderExplicitLayering) {
@ -1421,29 +1392,34 @@ TEST_F(TestVp9Impl, EncoderInfoFpsAllocationFlexibleMode) {
::testing::ElementsAreArray(expected_fps_allocation));
}
class TestVp9ImplWithLayering
: public TestVp9Impl,
public ::testing::WithParamInterface<::testing::tuple<uint8_t, uint8_t>> {
class Vp9ImplWithLayeringTest
: public ::testing::TestWithParam<std::tuple<int, int, bool>> {
protected:
TestVp9ImplWithLayering()
: num_spatial_layers_(::testing::get<0>(GetParam())),
num_temporal_layers_(::testing::get<1>(GetParam())) {}
Vp9ImplWithLayeringTest()
: num_spatial_layers_(std::get<0>(GetParam())),
num_temporal_layers_(std::get<1>(GetParam())),
override_field_trials_(std::get<2>(GetParam())
? "WebRTC-Vp9ExternalRefCtrl/Enabled/"
: "") {}
const uint8_t num_spatial_layers_;
const uint8_t num_temporal_layers_;
const test::ScopedFieldTrials override_field_trials_;
};
TEST_P(TestVp9ImplWithLayering, FlexibleMode) {
TEST_P(Vp9ImplWithLayeringTest, FlexibleMode) {
// In flexible mode encoder wrapper obtains actual list of references from
// encoder and writes it into RTP payload descriptor. Check that reference
// list in payload descriptor matches the predefined one, which is used
// in non-flexible mode.
codec_settings_.VP9()->flexibleMode = true;
codec_settings_.VP9()->frameDroppingOn = false;
codec_settings_.VP9()->numberOfSpatialLayers = num_spatial_layers_;
codec_settings_.VP9()->numberOfTemporalLayers = num_temporal_layers_;
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
encoder_->InitEncode(&codec_settings_, kSettings));
std::unique_ptr<VideoEncoder> encoder = VP9Encoder::Create();
VideoCodec codec_settings = DefaultCodecSettings();
codec_settings.VP9()->flexibleMode = true;
codec_settings.VP9()->frameDroppingOn = false;
codec_settings.VP9()->numberOfSpatialLayers = num_spatial_layers_;
codec_settings.VP9()->numberOfTemporalLayers = num_temporal_layers_;
EXPECT_EQ(encoder->InitEncode(&codec_settings, kSettings),
WEBRTC_VIDEO_CODEC_OK);
GofInfoVP9 gof;
if (num_temporal_layers_ == 1) {
@ -1456,65 +1432,48 @@ TEST_P(TestVp9ImplWithLayering, FlexibleMode) {
// Encode at least (num_frames_in_gof + 1) frames to verify references
// of non-key frame with gof_idx = 0.
for (size_t frame_num = 0; frame_num < gof.num_frames_in_gof + 1;
++frame_num) {
SetWaitForEncodedFramesThreshold(num_spatial_layers_);
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
encoder_->Encode(NextInputFrame(), nullptr));
int num_input_frames = gof.num_frames_in_gof + 1;
std::vector<EncodedVideoFrameProducer::EncodedFrame> frames =
EncodedVideoFrameProducer(*encoder)
.SetNumInputFrames(num_input_frames)
.SetResolution({kWidth, kHeight})
.Encode();
ASSERT_THAT(frames, SizeIs(num_input_frames * num_spatial_layers_));
const bool is_key_frame = frame_num == 0;
const size_t gof_idx = frame_num % gof.num_frames_in_gof;
const std::vector<uint8_t> p_diff(std::begin(gof.pid_diff[gof_idx]),
std::end(gof.pid_diff[gof_idx]));
for (size_t i = 0; i < frames.size(); ++i) {
const EncodedVideoFrameProducer::EncodedFrame& frame = frames[i];
const size_t picture_idx = i / num_spatial_layers_;
const size_t gof_idx = picture_idx % gof.num_frames_in_gof;
ExpectFrameWith(num_spatial_layers_, gof.temporal_idx[gof_idx],
gof.temporal_up_switch[gof_idx],
is_key_frame ? 0 : gof.num_ref_pics[gof_idx], p_diff);
}
}
TEST_P(TestVp9ImplWithLayering, ExternalRefControl) {
test::ScopedFieldTrials override_field_trials(
"WebRTC-Vp9ExternalRefCtrl/Enabled/");
codec_settings_.VP9()->flexibleMode = true;
codec_settings_.VP9()->frameDroppingOn = false;
codec_settings_.VP9()->numberOfSpatialLayers = num_spatial_layers_;
codec_settings_.VP9()->numberOfTemporalLayers = num_temporal_layers_;
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
encoder_->InitEncode(&codec_settings_, kSettings));
GofInfoVP9 gof;
if (num_temporal_layers_ == 1) {
gof.SetGofInfoVP9(kTemporalStructureMode1);
} else if (num_temporal_layers_ == 2) {
gof.SetGofInfoVP9(kTemporalStructureMode2);
} else if (num_temporal_layers_ == 3) {
gof.SetGofInfoVP9(kTemporalStructureMode3);
}
// Encode at least (num_frames_in_gof + 1) frames to verify references
// of non-key frame with gof_idx = 0.
for (size_t frame_num = 0; frame_num < gof.num_frames_in_gof + 1;
++frame_num) {
SetWaitForEncodedFramesThreshold(num_spatial_layers_);
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
encoder_->Encode(NextInputFrame(), nullptr));
const bool is_key_frame = frame_num == 0;
const size_t gof_idx = frame_num % gof.num_frames_in_gof;
const std::vector<uint8_t> p_diff(std::begin(gof.pid_diff[gof_idx]),
std::end(gof.pid_diff[gof_idx]));
ExpectFrameWith(num_spatial_layers_, gof.temporal_idx[gof_idx],
gof.temporal_up_switch[gof_idx],
is_key_frame ? 0 : gof.num_ref_pics[gof_idx], p_diff);
const CodecSpecificInfoVP9& vp9 =
frame.codec_specific_info.codecSpecific.VP9;
EXPECT_EQ(frame.encoded_image.SpatialIndex(),
num_spatial_layers_ == 1
? absl::nullopt
: absl::optional<int>(i % num_spatial_layers_))
<< "Frame " << i;
EXPECT_EQ(vp9.temporal_idx, num_temporal_layers_ == 1
? kNoTemporalIdx
: gof.temporal_idx[gof_idx])
<< "Frame " << i;
EXPECT_EQ(vp9.temporal_up_switch, gof.temporal_up_switch[gof_idx])
<< "Frame " << i;
if (picture_idx == 0) {
EXPECT_EQ(vp9.num_ref_pics, 0) << "Frame " << i;
} else {
EXPECT_THAT(rtc::MakeArrayView(vp9.p_diff, vp9.num_ref_pics),
UnorderedElementsAreArray(gof.pid_diff[gof_idx],
gof.num_ref_pics[gof_idx]))
<< "Frame " << i;
}
}
}
INSTANTIATE_TEST_SUITE_P(All,
TestVp9ImplWithLayering,
Vp9ImplWithLayeringTest,
::testing::Combine(::testing::Values(1, 2, 3),
::testing::Values(1, 2, 3)));
::testing::Values(1, 2, 3),
::testing::Bool()));
class TestVp9ImplFrameDropping : public TestVp9Impl {
protected:
@ -1774,4 +1733,12 @@ TEST_F(TestVp9Impl, ReenablingUpperLayerAfterKFWithInterlayerPredIsEnabled) {
EXPECT_EQ(encoded_frames[0]._frameType, VideoFrameType::kVideoFrameDelta);
}
TEST_F(TestVp9Impl, HandlesEmptyInitDecode) {
std::unique_ptr<VideoDecoder> decoder = CreateDecoder();
// Check that nullptr settings are ok for decoder.
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
decoder->InitDecode(/*codec_settings=*/nullptr, 1));
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, decoder->Release());
}
} // namespace webrtc

16
modules/video_coding/codecs/vp9/vp9.cc
View File

@ -39,6 +39,22 @@ std::vector<SdpVideoFormat> SupportedVP9Codecs() {
cricket::kVp9CodecName,
{{kVP9FmtpProfileId, VP9ProfileToString(VP9Profile::kProfile2)}}));
}
return supported_formats;
#else
return std::vector<SdpVideoFormat>();
#endif
}
std::vector<SdpVideoFormat> SupportedVP9DecoderCodecs() {
#ifdef RTC_ENABLE_VP9
std::vector<SdpVideoFormat> supported_formats = SupportedVP9Codecs();
// The WebRTC internal decoder supports VP9 profile 1. However, there's
// currently no way of sending VP9 profile 1 using the internal encoder.
// It would require extended support for I444, I422, and I440 buffers.
supported_formats.push_back(SdpVideoFormat(
cricket::kVp9CodecName,
{{kVP9FmtpProfileId, VP9ProfileToString(VP9Profile::kProfile1)}}));
return supported_formats;
#else
return std::vector<SdpVideoFormat>();

8
modules/video_coding/codecs/vp9/vp9_frame_buffer_pool.cc
View File

@ -58,7 +58,7 @@ Vp9FrameBufferPool::GetFrameBuffer(size_t min_size) {
RTC_DCHECK_GT(min_size, 0);
rtc::scoped_refptr<Vp9FrameBuffer> available_buffer = nullptr;
{
rtc::CritScope cs(&buffers_lock_);
MutexLock lock(&buffers_lock_);
// Do we have a buffer we can recycle?
for (const auto& buffer : allocated_buffers_) {
if (buffer->HasOneRef()) {
@ -91,7 +91,7 @@ Vp9FrameBufferPool::GetFrameBuffer(size_t min_size) {
int Vp9FrameBufferPool::GetNumBuffersInUse() const {
int num_buffers_in_use = 0;
rtc::CritScope cs(&buffers_lock_);
MutexLock lock(&buffers_lock_);
for (const auto& buffer : allocated_buffers_) {
if (!buffer->HasOneRef())
++num_buffers_in_use;
@ -100,7 +100,7 @@ int Vp9FrameBufferPool::GetNumBuffersInUse() const {
}
bool Vp9FrameBufferPool::Resize(size_t max_number_of_buffers) {
rtc::CritScope cs(&buffers_lock_);
MutexLock lock(&buffers_lock_);
size_t used_buffers_count = 0;
for (const auto& buffer : allocated_buffers_) {
// If the buffer is in use, the ref count will be >= 2, one from the list we
@ -130,7 +130,7 @@ bool Vp9FrameBufferPool::Resize(size_t max_number_of_buffers) {
}
void Vp9FrameBufferPool::ClearPool() {
rtc::CritScope cs(&buffers_lock_);
MutexLock lock(&buffers_lock_);
allocated_buffers_.clear();
}

4
modules/video_coding/codecs/vp9/vp9_frame_buffer_pool.h
View File

@ -18,8 +18,8 @@
#include "api/scoped_refptr.h"
#include "rtc_base/buffer.h"
#include "rtc_base/critical_section.h"
#include "rtc_base/ref_count.h"
#include "rtc_base/synchronization/mutex.h"
struct vpx_codec_ctx;
struct vpx_codec_frame_buffer;
@ -119,7 +119,7 @@ class Vp9FrameBufferPool {
private:
// Protects |allocated_buffers_|.
rtc::CriticalSection buffers_lock_;
mutable Mutex buffers_lock_;
// All buffers, in use or ready to be recycled.
std::vector<rtc::scoped_refptr<Vp9FrameBuffer>> allocated_buffers_
RTC_GUARDED_BY(buffers_lock_);

124
modules/video_coding/codecs/vp9/vp9_impl.cc
View File

@ -25,6 +25,7 @@
#include "common_video/libyuv/include/webrtc_libyuv.h"
#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h"
#include "modules/video_coding/codecs/vp9/svc_rate_allocator.h"
#include "modules/video_coding/utility/vp9_uncompressed_header_parser.h"
#include "rtc_base/checks.h"
#include "rtc_base/experiments/rate_control_settings.h"
#include "rtc_base/keep_ref_until_done.h"
@ -45,14 +46,21 @@ namespace {
uint8_t kRefBufIdx[4] = {0, 0, 0, 1};
uint8_t kUpdBufIdx[4] = {0, 0, 1, 0};
int kMaxNumTiles4kVideo = 8;
// Maximum allowed PID difference for differnet per-layer frame-rate case.
const int kMaxAllowedPidDiff = 30;
constexpr double kLowRateFactor = 1.0;
constexpr double kHighRateFactor = 2.0;
// TODO(ilink): Tune these thresholds further.
// Selected using ConverenceMotion_1280_720_50.yuv clip.
// No toggling observed on any link capacity from 100-2000kbps.
// HD was reached consistently when link capacity was 1500kbps.
// Set resolutions are a bit more conservative than svc_config.cc sets, e.g.
// for 300kbps resolution converged to 270p instead of 360p.
constexpr int kLowVp9QpThreshold = 149;
constexpr int kHighVp9QpThreshold = 205;
// These settings correspond to the settings in vpx_codec_enc_cfg.
struct Vp9RateSettings {
uint32_t rc_undershoot_pct;
@ -249,6 +257,8 @@ VP9EncoderImpl::VP9EncoderImpl(const cricket::VideoCodec& codec)
"WebRTC-VP9VariableFramerateScreenshare")),
variable_framerate_controller_(
variable_framerate_experiment_.framerate_limit),
quality_scaler_experiment_(
ParseQualityScalerConfig("WebRTC-VP9QualityScaler")),
num_steady_state_frames_(0),
config_changed_(true) {
codec_ = {};
@ -399,7 +409,6 @@ bool VP9EncoderImpl::SetSvcRates(
expect_no_more_active_layers = seen_active_layer;
}
}
RTC_DCHECK_GT(num_active_spatial_layers_, 0);
if (higher_layers_enabled && !force_key_frame_) {
// Prohibit drop of all layers for the next frame, so newly enabled
@ -517,6 +526,11 @@ int VP9EncoderImpl::InitEncode(const VideoCodec* inst,
config_->g_profile = 0;
config_->g_input_bit_depth = 8;
break;
case VP9Profile::kProfile1:
// Encoding of profile 1 is not implemented. It would require extended
// support for I444, I422, and I440 buffers.
RTC_NOTREACHED();
break;
case VP9Profile::kProfile2:
img_fmt = VPX_IMG_FMT_I42016;
bits_for_storage = 16;
@ -562,7 +576,13 @@ int VP9EncoderImpl::InitEncode(const VideoCodec* inst,
// put some key-frames at will even in VPX_KF_DISABLED kf_mode.
config_->kf_max_dist = inst->VP9().keyFrameInterval;
config_->kf_min_dist = config_->kf_max_dist;
config_->rc_resize_allowed = inst->VP9().automaticResizeOn ? 1 : 0;
if (quality_scaler_experiment_.enabled) {
// In that experiment webrtc wide quality scaler is used instead of libvpx
// internal scaler.
config_->rc_resize_allowed = 0;
} else {
config_->rc_resize_allowed = inst->VP9().automaticResizeOn ? 1 : 0;
}
// Determine number of threads based on the image size and #cores.
config_->g_threads =
NumberOfThreads(config_->g_w, config_->g_h, settings.number_of_cores);
@ -587,13 +607,6 @@ int VP9EncoderImpl::InitEncode(const VideoCodec* inst,
(num_spatial_layers_ > 1 &&
codec_.mode == VideoCodecMode::kScreensharing) ||
inter_layer_pred_ == InterLayerPredMode::kOn;
// TODO(ilnik): Remove this workaround once external reference control works
// nicely with simulcast SVC mode.
// Simlucast SVC mode is currently only used in some tests and is impossible
// to trigger for users without using some field trials.
if (inter_layer_pred_ == InterLayerPredMode::kOff) {
external_ref_control_ = false;
}
if (num_temporal_layers_ == 1) {
gof_.SetGofInfoVP9(kTemporalStructureMode1);
@ -987,6 +1000,10 @@ int VP9EncoderImpl::Encode(const VideoFrame& input_image,
raw_->stride[VPX_PLANE_V] = i420_buffer->StrideV();
break;
}
case VP9Profile::kProfile1: {
RTC_NOTREACHED();
break;
}
case VP9Profile::kProfile2: {
// We can inject kI010 frames directly for encode. All other formats
// should be converted to it.
@ -1560,7 +1577,12 @@ VideoEncoder::EncoderInfo VP9EncoderImpl::GetEncoderInfo() const {
EncoderInfo info;
info.supports_native_handle = false;
info.implementation_name = "libvpx";
info.scaling_settings = VideoEncoder::ScalingSettings::kOff;
if (quality_scaler_experiment_.enabled) {
info.scaling_settings = VideoEncoder::ScalingSettings(
quality_scaler_experiment_.low_qp, quality_scaler_experiment_.high_qp);
} else {
info.scaling_settings = VideoEncoder::ScalingSettings::kOff;
}
info.has_trusted_rate_controller = trusted_rate_controller_;
info.is_hardware_accelerated = false;
info.has_internal_source = false;
@ -1633,6 +1655,24 @@ VP9EncoderImpl::ParseVariableFramerateConfig(std::string group_name) {
return config;
}
// static
VP9EncoderImpl::QualityScalerExperiment
VP9EncoderImpl::ParseQualityScalerConfig(std::string group_name) {
FieldTrialFlag disabled = FieldTrialFlag("Disabled");
FieldTrialParameter<int> low_qp("low_qp", kLowVp9QpThreshold);
FieldTrialParameter<int> high_qp("hihg_qp", kHighVp9QpThreshold);
ParseFieldTrial({&disabled, &low_qp, &high_qp},
field_trial::FindFullName(group_name));
QualityScalerExperiment config;
config.enabled = !disabled.Get();
RTC_LOG(LS_INFO) << "Webrtc quality scaler for vp9 is "
<< (config.enabled ? "enabled." : "disabled");
config.low_qp = low_qp.Get();
config.high_qp = high_qp.Get();
return config;
}
VP9DecoderImpl::VP9DecoderImpl()
: decode_complete_callback_(nullptr),
inited_(false),
@ -1672,14 +1712,32 @@ int VP9DecoderImpl::InitDecode(const VideoCodec* inst, int number_of_cores) {
// errors earlier than the multi-threads version.
// - Make peak CPU usage under control (not depending on input)
cfg.threads = 1;
(void)kMaxNumTiles4kVideo; // unused
#else
// We want to use multithreading when decoding high resolution videos. But,
// since we don't know resolution of input stream at this stage, we always
// enable it.
cfg.threads = std::min(number_of_cores, kMaxNumTiles4kVideo);
if (!inst) {
// No config provided - don't know resolution to decode yet.
// Set thread count to one in the meantime.
cfg.threads = 1;
} else {
// We want to use multithreading when decoding high resolution videos. But
// not too many in order to avoid overhead when many stream are decoded
// concurrently.
// Set 2 thread as target for 1280x720 pixel count, and then scale up
// linearly from there - but cap at physical core count.
// For common resolutions this results in:
// 1 for 360p
// 2 for 720p
// 4 for 1080p
// 8 for 1440p
// 18 for 4K
int num_threads =
std::max(1, 2 * (inst->width * inst->height) / (1280 * 720));
cfg.threads = std::min(number_of_cores, num_threads);
current_codec_ = *inst;
}
#endif
num_cores_ = number_of_cores;
vpx_codec_flags_t flags = 0;
if (vpx_codec_dec_init(decoder_, vpx_codec_vp9_dx(), &cfg, flags)) {
return WEBRTC_VIDEO_CODEC_MEMORY;
@ -1697,6 +1755,15 @@ int VP9DecoderImpl::InitDecode(const VideoCodec* inst, int number_of_cores) {
return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
}
}
vpx_codec_err_t status =
vpx_codec_control(decoder_, VP9D_SET_LOOP_FILTER_OPT, 1);
if (status != VPX_CODEC_OK) {
RTC_LOG(LS_ERROR) << "Failed to enable VP9D_SET_LOOP_FILTER_OPT. "
<< vpx_codec_error(decoder_);
return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
}
return WEBRTC_VIDEO_CODEC_OK;
}
@ -1709,6 +1776,29 @@ int VP9DecoderImpl::Decode(const EncodedImage& input_image,
if (decode_complete_callback_ == nullptr) {
return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
}
if (input_image._frameType == VideoFrameType::kVideoFrameKey) {
absl::optional<vp9::FrameInfo> frame_info =
vp9::ParseIntraFrameInfo(input_image.data(), input_image.size());
if (frame_info) {
if (frame_info->frame_width != current_codec_.width ||
frame_info->frame_height != current_codec_.height) {
// Resolution has changed, tear down and re-init a new decoder in
// order to get correct sizing.
Release();
current_codec_.width = frame_info->frame_width;
current_codec_.height = frame_info->frame_height;
int reinit_status = InitDecode(&current_codec_, num_cores_);
if (reinit_status != WEBRTC_VIDEO_CODEC_OK) {
RTC_LOG(LS_WARNING) << "Failed to re-init decoder.";
return reinit_status;
}
}
} else {
RTC_LOG(LS_WARNING) << "Failed to parse VP9 header from key-frame.";
}
}
// Always start with a complete key frame.
if (key_frame_required_) {
if (input_image._frameType != VideoFrameType::kVideoFrameKey)

11
modules/video_coding/codecs/vp9/vp9_impl.h
View File

@ -175,6 +175,15 @@ class VP9EncoderImpl : public VP9Encoder {
static VariableFramerateExperiment ParseVariableFramerateConfig(
std::string group_name);
FramerateController variable_framerate_controller_;
const struct QualityScalerExperiment {
int low_qp;
int high_qp;
bool enabled;
} quality_scaler_experiment_;
static QualityScalerExperiment ParseQualityScalerConfig(
std::string group_name);
int num_steady_state_frames_;
// Only set config when this flag is set.
bool config_changed_;
@ -210,6 +219,8 @@ class VP9DecoderImpl : public VP9Decoder {
bool inited_;
vpx_codec_ctx_t* decoder_;
bool key_frame_required_;
VideoCodec current_codec_;
int num_cores_;
};
} // namespace webrtc