zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add ability for VideoEncoder to signal frame rate allocation.

Browse Source 
This CL add new data to the VideoEncoder::EncoderInfo struct, indicating
how the encoder intends to allocate frames across spatial and temporal
layers.

This metadata will be used in upcoming CLs to control how the encoder's
rate controller performs.

Bug: webrtc:10155
Change-Id: Id56fae04bae5f230d1a985171097d7ca83a3be8a
Reviewed-on: https://webrtc-review.googlesource.com/c/117900
Reviewed-by: Niels Moller <nisse@webrtc.org>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Commit-Queue: Erik Språng <sprang@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#26300}
This commit is contained in:
Erik Språng
2019-01-17 15:27:50 +01:00
committed by Commit Bot
parent f93eda1705
commit dbdd8395f7
9 changed files with 302 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc
View File

@ -963,6 +963,31 @@ VideoEncoder::EncoderInfo LibvpxVp8Encoder::GetEncoderInfo() const {
? VideoEncoder::ScalingSettings(
kLowVp8QpThreshold, kHighVp8QpThreshold)
: VideoEncoder::ScalingSettings::kOff;
// |encoder_idx| is libvpx index where 0 is highest resolution.
// |si| is simulcast index, where 0 is lowest resolution.
for (size_t si = 0, encoder_idx = encoders_.size() - 1; si < encoders_.size();
++si, --encoder_idx) {
info.fps_allocation[si].clear();
if ((codec_.numberOfSimulcastStreams > si &&
!codec_.simulcastStream[si].active) ||
(si == 0 && SimulcastUtility::IsConferenceModeScreenshare(codec_))) {
// No defined frame rate fractions if not active or if using
// ScreenshareLayers, leave vector empty and continue;
continue;
}
if (configurations_[encoder_idx].ts_number_layers <= 1) {
info.fps_allocation[si].push_back(EncoderInfo::kMaxFramerateFraction);
} else {
for (size_t ti = 0; ti < configurations_[encoder_idx].ts_number_layers;
++ti) {
RTC_DCHECK_GT(configurations_[encoder_idx].ts_rate_decimator[ti], 0);
info.fps_allocation[si].push_back(rtc::saturated_cast<uint8_t>(
EncoderInfo::kMaxFramerateFraction /
configurations_[encoder_idx].ts_rate_decimator[ti] +
0.5));
}
}
}
return info;
}

105
modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc
View File

@ -27,12 +27,18 @@
namespace webrtc {
using testing::_;
using testing::ElementsAreArray;
using testing::Invoke;
using testing::NiceMock;
using testing::Return;
using testing::_;
using EncoderInfo = webrtc::VideoEncoder::EncoderInfo;
using FramerateFractions =
absl::InlinedVector<uint8_t, webrtc::kMaxTemporalStreams>;
namespace {
constexpr uint32_t kLegacyScreenshareTl0BitrateKbps = 200;
constexpr uint32_t kLegacyScreenshareTl1BitrateKbps = 1000;
constexpr uint32_t kInitialTimestampRtp = 123;
constexpr int64_t kTestNtpTimeMs = 456;
constexpr int64_t kInitialTimestampMs = 789;
@ -472,4 +478,101 @@ TEST_F(TestVp8Impl, KeepsTimestampOnReencode) {
encoder.Encode(*NextInputFrame(), nullptr, &delta_frame);
}
TEST_F(TestVp8Impl, GetEncoderInfoFpsAllocationNoLayers) {
FramerateFractions expected_fps_allocation[kMaxSpatialLayers] = {
FramerateFractions(1, EncoderInfo::kMaxFramerateFraction)};
EXPECT_THAT(encoder_->GetEncoderInfo().fps_allocation,
::testing::ElementsAreArray(expected_fps_allocation));
}
TEST_F(TestVp8Impl, GetEncoderInfoFpsAllocationTwoTemporalLayers) {
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release());
codec_settings_.numberOfSimulcastStreams = 1;
codec_settings_.simulcastStream[0].active = true;
codec_settings_.simulcastStream[0].targetBitrate = 100;
codec_settings_.simulcastStream[0].maxBitrate = 100;
codec_settings_.simulcastStream[0].numberOfTemporalLayers = 2;
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
encoder_->InitEncode(&codec_settings_, kNumCores, kMaxPayloadSize));
FramerateFractions expected_fps_allocation[kMaxSpatialLayers];
expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction / 2);
expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction);
EXPECT_THAT(encoder_->GetEncoderInfo().fps_allocation,
::testing::ElementsAreArray(expected_fps_allocation));
}
TEST_F(TestVp8Impl, GetEncoderInfoFpsAllocationThreeTemporalLayers) {
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release());
codec_settings_.numberOfSimulcastStreams = 1;
codec_settings_.simulcastStream[0].active = true;
codec_settings_.simulcastStream[0].targetBitrate = 100;
codec_settings_.simulcastStream[0].maxBitrate = 100;
codec_settings_.simulcastStream[0].numberOfTemporalLayers = 3;
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
encoder_->InitEncode(&codec_settings_, kNumCores, kMaxPayloadSize));
FramerateFractions expected_fps_allocation[kMaxSpatialLayers];
expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction / 4);
expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction / 2);
expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction);
EXPECT_THAT(encoder_->GetEncoderInfo().fps_allocation,
::testing::ElementsAreArray(expected_fps_allocation));
}
TEST_F(TestVp8Impl, GetEncoderInfoFpsAllocationScreenshareLayers) {
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release());
codec_settings_.numberOfSimulcastStreams = 1;
codec_settings_.mode = VideoCodecMode::kScreensharing;
codec_settings_.simulcastStream[0].active = true;
codec_settings_.simulcastStream[0].minBitrate = 30;
codec_settings_.simulcastStream[0].targetBitrate =
kLegacyScreenshareTl0BitrateKbps;
codec_settings_.simulcastStream[0].maxBitrate =
kLegacyScreenshareTl1BitrateKbps;
codec_settings_.simulcastStream[0].numberOfTemporalLayers = 2;
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
encoder_->InitEncode(&codec_settings_, kNumCores, kMaxPayloadSize));
// Expect empty vector, since this mode doesn't have a fixed framerate.
FramerateFractions expected_fps_allocation[kMaxSpatialLayers];
EXPECT_THAT(encoder_->GetEncoderInfo().fps_allocation,
::testing::ElementsAreArray(expected_fps_allocation));
}
TEST_F(TestVp8Impl, GetEncoderInfoFpsAllocationSimulcastVideo) {
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release());
// Set up three simulcast streams with three temporal layers each.
codec_settings_.numberOfSimulcastStreams = 3;
for (int i = 0; i < codec_settings_.numberOfSimulcastStreams; ++i) {
codec_settings_.simulcastStream[i].active = true;
codec_settings_.simulcastStream[i].minBitrate = 30;
codec_settings_.simulcastStream[i].targetBitrate = 30;
codec_settings_.simulcastStream[i].maxBitrate = 30;
codec_settings_.simulcastStream[i].numberOfTemporalLayers = 3;
codec_settings_.simulcastStream[i].width =
codec_settings_.width >>
(codec_settings_.numberOfSimulcastStreams - i - 1);
codec_settings_.simulcastStream[i].height =
codec_settings_.height >>
(codec_settings_.numberOfSimulcastStreams - i - 1);
}
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
encoder_->InitEncode(&codec_settings_, kNumCores, kMaxPayloadSize));
FramerateFractions expected_fps_allocation[kMaxSpatialLayers];
expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction / 4);
expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction / 2);
expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction);
expected_fps_allocation[1] = expected_fps_allocation[0];
expected_fps_allocation[2] = expected_fps_allocation[0];
EXPECT_THAT(encoder_->GetEncoderInfo().fps_allocation,
::testing::ElementsAreArray(expected_fps_allocation));
}
} // namespace webrtc

80
modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc
View File

@ -18,10 +18,17 @@
#include "modules/video_coding/codecs/vp9/include/vp9.h"
#include "modules/video_coding/codecs/vp9/svc_config.h"
#include "test/field_trial.h"
#include "test/gmock.h"
#include "test/gtest.h"
#include "test/video_codec_settings.h"
namespace webrtc {
using testing::ElementsAreArray;
using EncoderInfo = webrtc::VideoEncoder::EncoderInfo;
using FramerateFractions =
absl::InlinedVector<uint8_t, webrtc::kMaxTemporalStreams>;
namespace {
const size_t kWidth = 1280;
const size_t kHeight = 720;
@ -848,6 +855,79 @@ TEST_F(TestVp9Impl, ScalabilityStructureIsAvailableInFlexibleMode) {
EXPECT_TRUE(codec_specific_info.codecSpecific.VP9.ss_data_available);
}
TEST_F(TestVp9Impl, EncoderInfoFpsAllocation) {
const uint8_t kNumSpatialLayers = 3;
const uint8_t kNumTemporalLayers = 3;
codec_settings_.maxFramerate = 30;
codec_settings_.VP9()->numberOfSpatialLayers = kNumSpatialLayers;
codec_settings_.VP9()->numberOfTemporalLayers = kNumTemporalLayers;
for (uint8_t sl_idx = 0; sl_idx < kNumSpatialLayers; ++sl_idx) {
codec_settings_.spatialLayers[sl_idx].width = codec_settings_.width;
codec_settings_.spatialLayers[sl_idx].height = codec_settings_.height;
codec_settings_.spatialLayers[sl_idx].minBitrate =
codec_settings_.startBitrate;
codec_settings_.spatialLayers[sl_idx].maxBitrate =
codec_settings_.startBitrate;
codec_settings_.spatialLayers[sl_idx].targetBitrate =
codec_settings_.startBitrate;
codec_settings_.spatialLayers[sl_idx].active = true;
codec_settings_.spatialLayers[sl_idx].maxFramerate =
codec_settings_.maxFramerate;
}
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
encoder_->InitEncode(&codec_settings_, 1 /* number of cores */,
0 /* max payload size (unused) */));
FramerateFractions expected_fps_allocation[kMaxSpatialLayers];
expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction / 4);
expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction / 2);
expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction);
expected_fps_allocation[1] = expected_fps_allocation[0];
expected_fps_allocation[2] = expected_fps_allocation[0];
EXPECT_THAT(encoder_->GetEncoderInfo().fps_allocation,
::testing::ElementsAreArray(expected_fps_allocation));
}
TEST_F(TestVp9Impl, EncoderInfoFpsAllocationFlexibleMode) {
const uint8_t kNumSpatialLayers = 3;
codec_settings_.maxFramerate = 30;
codec_settings_.VP9()->numberOfSpatialLayers = kNumSpatialLayers;
codec_settings_.VP9()->numberOfTemporalLayers = 1;
codec_settings_.VP9()->flexibleMode = true;
for (uint8_t sl_idx = 0; sl_idx < kNumSpatialLayers; ++sl_idx) {
codec_settings_.spatialLayers[sl_idx].width = codec_settings_.width;
codec_settings_.spatialLayers[sl_idx].height = codec_settings_.height;
codec_settings_.spatialLayers[sl_idx].minBitrate =
codec_settings_.startBitrate;
codec_settings_.spatialLayers[sl_idx].maxBitrate =
codec_settings_.startBitrate;
codec_settings_.spatialLayers[sl_idx].targetBitrate =
codec_settings_.startBitrate;
codec_settings_.spatialLayers[sl_idx].active = true;
// Force different frame rates for different layers, to verify that total
// fraction is correct.
codec_settings_.spatialLayers[sl_idx].maxFramerate =
codec_settings_.maxFramerate / (kNumSpatialLayers - sl_idx);
}
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
encoder_->InitEncode(&codec_settings_, 1 /* number of cores */,
0 /* max payload size (unused) */));
// No temporal layers allowed when spatial layers have different fps targets.
FramerateFractions expected_fps_allocation[kMaxSpatialLayers];
expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction / 3);
expected_fps_allocation[1].push_back(EncoderInfo::kMaxFramerateFraction / 2);
expected_fps_allocation[2].push_back(EncoderInfo::kMaxFramerateFraction);
EXPECT_THAT(encoder_->GetEncoderInfo().fps_allocation,
::testing::ElementsAreArray(expected_fps_allocation));
}
class TestVp9ImplWithLayering
: public TestVp9Impl,
public ::testing::WithParamInterface<::testing::tuple<uint8_t, uint8_t>> {

16
modules/video_coding/codecs/vp9/vp9_impl.cc
View File

@ -1354,6 +1354,22 @@ VideoEncoder::EncoderInfo VP9EncoderImpl::GetEncoderInfo() const {
info.has_trusted_rate_controller = trusted_rate_controller_;
info.is_hardware_accelerated = false;
info.has_internal_source = false;
for (size_t si = 0; si < num_spatial_layers_; ++si) {
info.fps_allocation[si].clear();
if (!codec_.spatialLayers[si].active) {
continue;
}
// This spatial layer may already use a fraction of the total frame rate.
const float sl_fps_fraction =
codec_.spatialLayers[si].maxFramerate / codec_.maxFramerate;
for (size_t ti = 0; ti < num_temporal_layers_; ++ti) {
const uint32_t decimator =
num_temporal_layers_ <= 1 ? 1 : config_->ts_rate_decimator[ti];
RTC_DCHECK_GT(decimator, 0);
info.fps_allocation[si].push_back(rtc::saturated_cast<uint8_t>(
EncoderInfo::kMaxFramerateFraction * (sl_fps_fraction / decimator)));
}
}
return info;
}