zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add ability for VideoEncoder to signal frame rate allocation.

Browse Source 
This CL add new data to the VideoEncoder::EncoderInfo struct, indicating
how the encoder intends to allocate frames across spatial and temporal
layers.

This metadata will be used in upcoming CLs to control how the encoder's
rate controller performs.

Bug: webrtc:10155
Change-Id: Id56fae04bae5f230d1a985171097d7ca83a3be8a
Reviewed-on: https://webrtc-review.googlesource.com/c/117900
Reviewed-by: Niels Moller <nisse@webrtc.org>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Commit-Queue: Erik Språng <sprang@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#26300}
This commit is contained in:
Erik Språng
2019-01-17 15:27:50 +01:00
committed by Commit Bot
parent f93eda1705
commit dbdd8395f7
9 changed files with 302 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
api/video_codecs/BUILD.gn
View File

@ -40,6 +40,7 @@ rtc_source_set("video_codecs_api") {
"../video:video_bitrate_allocation",
"../video:video_codec_constants",
"../video:video_frame",
"//third_party/abseil-cpp/absl/container:inlined_vector",
"//third_party/abseil-cpp/absl/strings",
"//third_party/abseil-cpp/absl/types:optional",
]

7
api/video_codecs/video_encoder.cc
View File

@ -82,6 +82,8 @@ VideoEncoder::ScalingSettings::~ScalingSettings() {}
// static
constexpr VideoEncoder::ScalingSettings::KOff
VideoEncoder::ScalingSettings::kOff;
// static
constexpr uint8_t VideoEncoder::EncoderInfo::kMaxFramerateFraction;
VideoEncoder::EncoderInfo::EncoderInfo()
: scaling_settings(VideoEncoder::ScalingSettings::kOff),
@ -89,7 +91,10 @@ VideoEncoder::EncoderInfo::EncoderInfo()
implementation_name("unknown"),
has_trusted_rate_controller(false),
is_hardware_accelerated(true),
has_internal_source(false) {}
has_internal_source(false),
fps_allocation{absl::InlinedVector<uint8_t, kMaxTemporalStreams>(
1,
kMaxFramerateFraction)} {}
VideoEncoder::EncoderInfo::EncoderInfo(const EncoderInfo&) = default;

31
api/video_codecs/video_encoder.h
View File

@ -11,10 +11,12 @@
#ifndef API_VIDEO_CODECS_VIDEO_ENCODER_H_
#define API_VIDEO_CODECS_VIDEO_ENCODER_H_
#include <limits>
#include <memory>
#include <string>
#include <vector>
#include "absl/container/inlined_vector.h"
#include "absl/types/optional.h"
#include "api/video/encoded_image.h"
#include "api/video/video_bitrate_allocation.h"
@ -120,6 +122,9 @@ class RTC_EXPORT VideoEncoder {
// Struct containing metadata about the encoder implementing this interface.
struct EncoderInfo {
static constexpr uint8_t kMaxFramerateFraction =
std::numeric_limits<uint8_t>::max();
EncoderInfo();
EncoderInfo(const EncoderInfo&);
@ -159,6 +164,32 @@ class RTC_EXPORT VideoEncoder {
// Internal source encoders are deprecated and support for them will be
// phased out.
bool has_internal_source;
// For each spatial layer (simulcast stream or SVC layer), represented as an
// element in |fps_allocation| a vector indicates how many temporal layers
// the encoder is using for that spatial layer.
// For each spatial/temporal layer pair, the frame rate fraction is given as
// an 8bit unsigned integer where 0 = 0% and 255 = 100%.
//
// If the vector is empty for a given spatial layer, it indicates that frame
// rates are not defined and we can't count on any specific frame rate to be
// generated. Likely this indicates Vp8TemporalLayersType::kBitrateDynamic.
//
// The encoder may update this on a per-frame basis in response to both
// internal and external signals.
//
// Spatial layers are treated independently, but temporal layers are
// cumulative. For instance, if:
// fps_allocation[0][0] = kFullFramerate / 2;
// fps_allocation[0][1] = kFullFramerate;
// Then half of the frames are in the base layer and half is in TL1, but
// since TL1 is assumed to depend on the base layer, the frame rate is
// indicated as the full 100% for the top layer.
//
// Defaults to a single spatial layer containing a single temporal layer
// with a 100% frame rate fraction.
absl::InlinedVector<uint8_t, kMaxTemporalStreams>
fps_allocation[kMaxSpatialLayers];
};
static VideoCodecVP8 GetDefaultVp8Settings();

1
media/engine/simulcast_encoder_adapter.cc
View File

@ -312,6 +312,7 @@ int SimulcastEncoderAdapter::InitEncode(const VideoCodec* inst,
encoder_info_.has_internal_source &=
encoder_impl_info.has_internal_source;
}
encoder_info_.fps_allocation[i] = encoder_impl_info.fps_allocation[0];
}
}

38
media/engine/simulcast_encoder_adapter_unittest.cc
View File

@ -31,6 +31,9 @@
using ::testing::_;
using ::testing::Return;
using EncoderInfo = webrtc::VideoEncoder::EncoderInfo;
using FramerateFractions =
absl::InlinedVector<uint8_t, webrtc::kMaxTemporalStreams>;
namespace webrtc {
namespace test {
@ -219,6 +222,7 @@ class MockVideoEncoder : public VideoEncoder {
info.has_trusted_rate_controller = has_trusted_rate_controller_;
info.is_hardware_accelerated = is_hardware_accelerated_;
info.has_internal_source = has_internal_source_;
info.fps_allocation[0] = fps_allocation_;
return info;
}
@ -265,6 +269,10 @@ class MockVideoEncoder : public VideoEncoder {
has_internal_source_ = has_internal_source;
}
void set_fps_allocation(const FramerateFractions& fps_allocation) {
fps_allocation_ = fps_allocation;
}
VideoBitrateAllocation last_set_bitrate() const { return last_set_bitrate_; }
private:
@ -277,6 +285,7 @@ class MockVideoEncoder : public VideoEncoder {
bool has_internal_source_ = false;
int32_t init_encode_return_value_ = 0;
VideoBitrateAllocation last_set_bitrate_;
FramerateFractions fps_allocation_;
VideoCodec codec_;
EncodedImageCallback* callback_;
@ -1085,5 +1094,34 @@ TEST_F(TestSimulcastEncoderAdapterFake, ReportsInternalSource) {
EXPECT_FALSE(adapter_->GetEncoderInfo().has_internal_source);
}
TEST_F(TestSimulcastEncoderAdapterFake, ReportsFpsAllocation) {
SimulcastTestFixtureImpl::DefaultSettings(
&codec_, static_cast<const int*>(kTestTemporalLayerProfile),
kVideoCodecVP8);
codec_.numberOfSimulcastStreams = 3;
adapter_->RegisterEncodeCompleteCallback(this);
EXPECT_EQ(0, adapter_->InitEncode(&codec_, 1, 1200));
ASSERT_EQ(3u, helper_->factory()->encoders().size());
// Combination of three different supported mode:
// Simulcast stream 0 has undefined fps behavior.
// Simulcast stream 1 has three temporal layers.
// Simulcast stream 2 has 1 temporal layer.
FramerateFractions expected_fps_allocation[kMaxSpatialLayers];
expected_fps_allocation[1].push_back(EncoderInfo::kMaxFramerateFraction / 4);
expected_fps_allocation[1].push_back(EncoderInfo::kMaxFramerateFraction / 2);
expected_fps_allocation[1].push_back(EncoderInfo::kMaxFramerateFraction);
expected_fps_allocation[2].push_back(EncoderInfo::kMaxFramerateFraction);
// All encoders have internal source, simulcast adapter reports true.
for (size_t i = 0; i < codec_.numberOfSimulcastStreams; ++i) {
MockVideoEncoder* encoder = helper_->factory()->encoders()[i];
encoder->set_fps_allocation(expected_fps_allocation[i]);
}
EXPECT_EQ(0, adapter_->InitEncode(&codec_, 1, 1200));
EXPECT_THAT(adapter_->GetEncoderInfo().fps_allocation,
::testing::ElementsAreArray(expected_fps_allocation));
}
} // namespace test
} // namespace webrtc

25
modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc
View File

@ -963,6 +963,31 @@ VideoEncoder::EncoderInfo LibvpxVp8Encoder::GetEncoderInfo() const {
? VideoEncoder::ScalingSettings(
kLowVp8QpThreshold, kHighVp8QpThreshold)
: VideoEncoder::ScalingSettings::kOff;
// |encoder_idx| is libvpx index where 0 is highest resolution.
// |si| is simulcast index, where 0 is lowest resolution.
for (size_t si = 0, encoder_idx = encoders_.size() - 1; si < encoders_.size();
++si, --encoder_idx) {
info.fps_allocation[si].clear();
if ((codec_.numberOfSimulcastStreams > si &&
!codec_.simulcastStream[si].active) ||
(si == 0 && SimulcastUtility::IsConferenceModeScreenshare(codec_))) {
// No defined frame rate fractions if not active or if using
// ScreenshareLayers, leave vector empty and continue;
continue;
}
if (configurations_[encoder_idx].ts_number_layers <= 1) {
info.fps_allocation[si].push_back(EncoderInfo::kMaxFramerateFraction);
} else {
for (size_t ti = 0; ti < configurations_[encoder_idx].ts_number_layers;
++ti) {
RTC_DCHECK_GT(configurations_[encoder_idx].ts_rate_decimator[ti], 0);
info.fps_allocation[si].push_back(rtc::saturated_cast<uint8_t>(
EncoderInfo::kMaxFramerateFraction /
configurations_[encoder_idx].ts_rate_decimator[ti] +
0.5));
}
}
}
return info;
}

105
modules/video_coding/codecs/vp8/test/vp8_impl_unittest.cc
View File

@ -27,12 +27,18 @@
namespace webrtc {
using testing::_;
using testing::ElementsAreArray;
using testing::Invoke;
using testing::NiceMock;
using testing::Return;
using testing::_;
using EncoderInfo = webrtc::VideoEncoder::EncoderInfo;
using FramerateFractions =
absl::InlinedVector<uint8_t, webrtc::kMaxTemporalStreams>;
namespace {
constexpr uint32_t kLegacyScreenshareTl0BitrateKbps = 200;
constexpr uint32_t kLegacyScreenshareTl1BitrateKbps = 1000;
constexpr uint32_t kInitialTimestampRtp = 123;
constexpr int64_t kTestNtpTimeMs = 456;
constexpr int64_t kInitialTimestampMs = 789;
@ -472,4 +478,101 @@ TEST_F(TestVp8Impl, KeepsTimestampOnReencode) {
encoder.Encode(*NextInputFrame(), nullptr, &delta_frame);
}
TEST_F(TestVp8Impl, GetEncoderInfoFpsAllocationNoLayers) {
FramerateFractions expected_fps_allocation[kMaxSpatialLayers] = {
FramerateFractions(1, EncoderInfo::kMaxFramerateFraction)};
EXPECT_THAT(encoder_->GetEncoderInfo().fps_allocation,
::testing::ElementsAreArray(expected_fps_allocation));
}
TEST_F(TestVp8Impl, GetEncoderInfoFpsAllocationTwoTemporalLayers) {
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release());
codec_settings_.numberOfSimulcastStreams = 1;
codec_settings_.simulcastStream[0].active = true;
codec_settings_.simulcastStream[0].targetBitrate = 100;
codec_settings_.simulcastStream[0].maxBitrate = 100;
codec_settings_.simulcastStream[0].numberOfTemporalLayers = 2;
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
encoder_->InitEncode(&codec_settings_, kNumCores, kMaxPayloadSize));
FramerateFractions expected_fps_allocation[kMaxSpatialLayers];
expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction / 2);
expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction);
EXPECT_THAT(encoder_->GetEncoderInfo().fps_allocation,
::testing::ElementsAreArray(expected_fps_allocation));
}
TEST_F(TestVp8Impl, GetEncoderInfoFpsAllocationThreeTemporalLayers) {
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release());
codec_settings_.numberOfSimulcastStreams = 1;
codec_settings_.simulcastStream[0].active = true;
codec_settings_.simulcastStream[0].targetBitrate = 100;
codec_settings_.simulcastStream[0].maxBitrate = 100;
codec_settings_.simulcastStream[0].numberOfTemporalLayers = 3;
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
encoder_->InitEncode(&codec_settings_, kNumCores, kMaxPayloadSize));
FramerateFractions expected_fps_allocation[kMaxSpatialLayers];
expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction / 4);
expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction / 2);
expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction);
EXPECT_THAT(encoder_->GetEncoderInfo().fps_allocation,
::testing::ElementsAreArray(expected_fps_allocation));
}
TEST_F(TestVp8Impl, GetEncoderInfoFpsAllocationScreenshareLayers) {
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release());
codec_settings_.numberOfSimulcastStreams = 1;
codec_settings_.mode = VideoCodecMode::kScreensharing;
codec_settings_.simulcastStream[0].active = true;
codec_settings_.simulcastStream[0].minBitrate = 30;
codec_settings_.simulcastStream[0].targetBitrate =
kLegacyScreenshareTl0BitrateKbps;
codec_settings_.simulcastStream[0].maxBitrate =
kLegacyScreenshareTl1BitrateKbps;
codec_settings_.simulcastStream[0].numberOfTemporalLayers = 2;
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
encoder_->InitEncode(&codec_settings_, kNumCores, kMaxPayloadSize));
// Expect empty vector, since this mode doesn't have a fixed framerate.
FramerateFractions expected_fps_allocation[kMaxSpatialLayers];
EXPECT_THAT(encoder_->GetEncoderInfo().fps_allocation,
::testing::ElementsAreArray(expected_fps_allocation));
}
TEST_F(TestVp8Impl, GetEncoderInfoFpsAllocationSimulcastVideo) {
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Release());
// Set up three simulcast streams with three temporal layers each.
codec_settings_.numberOfSimulcastStreams = 3;
for (int i = 0; i < codec_settings_.numberOfSimulcastStreams; ++i) {
codec_settings_.simulcastStream[i].active = true;
codec_settings_.simulcastStream[i].minBitrate = 30;
codec_settings_.simulcastStream[i].targetBitrate = 30;
codec_settings_.simulcastStream[i].maxBitrate = 30;
codec_settings_.simulcastStream[i].numberOfTemporalLayers = 3;
codec_settings_.simulcastStream[i].width =
codec_settings_.width >>
(codec_settings_.numberOfSimulcastStreams - i - 1);
codec_settings_.simulcastStream[i].height =
codec_settings_.height >>
(codec_settings_.numberOfSimulcastStreams - i - 1);
}
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
encoder_->InitEncode(&codec_settings_, kNumCores, kMaxPayloadSize));
FramerateFractions expected_fps_allocation[kMaxSpatialLayers];
expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction / 4);
expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction / 2);
expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction);
expected_fps_allocation[1] = expected_fps_allocation[0];
expected_fps_allocation[2] = expected_fps_allocation[0];
EXPECT_THAT(encoder_->GetEncoderInfo().fps_allocation,
::testing::ElementsAreArray(expected_fps_allocation));
}
} // namespace webrtc

80
modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc
View File

@ -18,10 +18,17 @@
#include "modules/video_coding/codecs/vp9/include/vp9.h"
#include "modules/video_coding/codecs/vp9/svc_config.h"
#include "test/field_trial.h"
#include "test/gmock.h"
#include "test/gtest.h"
#include "test/video_codec_settings.h"
namespace webrtc {
using testing::ElementsAreArray;
using EncoderInfo = webrtc::VideoEncoder::EncoderInfo;
using FramerateFractions =
absl::InlinedVector<uint8_t, webrtc::kMaxTemporalStreams>;
namespace {
const size_t kWidth = 1280;
const size_t kHeight = 720;
@ -848,6 +855,79 @@ TEST_F(TestVp9Impl, ScalabilityStructureIsAvailableInFlexibleMode) {
EXPECT_TRUE(codec_specific_info.codecSpecific.VP9.ss_data_available);
}
TEST_F(TestVp9Impl, EncoderInfoFpsAllocation) {
const uint8_t kNumSpatialLayers = 3;
const uint8_t kNumTemporalLayers = 3;
codec_settings_.maxFramerate = 30;
codec_settings_.VP9()->numberOfSpatialLayers = kNumSpatialLayers;
codec_settings_.VP9()->numberOfTemporalLayers = kNumTemporalLayers;
for (uint8_t sl_idx = 0; sl_idx < kNumSpatialLayers; ++sl_idx) {
codec_settings_.spatialLayers[sl_idx].width = codec_settings_.width;
codec_settings_.spatialLayers[sl_idx].height = codec_settings_.height;
codec_settings_.spatialLayers[sl_idx].minBitrate =
codec_settings_.startBitrate;
codec_settings_.spatialLayers[sl_idx].maxBitrate =
codec_settings_.startBitrate;
codec_settings_.spatialLayers[sl_idx].targetBitrate =
codec_settings_.startBitrate;
codec_settings_.spatialLayers[sl_idx].active = true;
codec_settings_.spatialLayers[sl_idx].maxFramerate =
codec_settings_.maxFramerate;
}
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
encoder_->InitEncode(&codec_settings_, 1 /* number of cores */,
0 /* max payload size (unused) */));
FramerateFractions expected_fps_allocation[kMaxSpatialLayers];
expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction / 4);
expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction / 2);
expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction);
expected_fps_allocation[1] = expected_fps_allocation[0];
expected_fps_allocation[2] = expected_fps_allocation[0];
EXPECT_THAT(encoder_->GetEncoderInfo().fps_allocation,
::testing::ElementsAreArray(expected_fps_allocation));
}
TEST_F(TestVp9Impl, EncoderInfoFpsAllocationFlexibleMode) {
const uint8_t kNumSpatialLayers = 3;
codec_settings_.maxFramerate = 30;
codec_settings_.VP9()->numberOfSpatialLayers = kNumSpatialLayers;
codec_settings_.VP9()->numberOfTemporalLayers = 1;
codec_settings_.VP9()->flexibleMode = true;
for (uint8_t sl_idx = 0; sl_idx < kNumSpatialLayers; ++sl_idx) {
codec_settings_.spatialLayers[sl_idx].width = codec_settings_.width;
codec_settings_.spatialLayers[sl_idx].height = codec_settings_.height;
codec_settings_.spatialLayers[sl_idx].minBitrate =
codec_settings_.startBitrate;
codec_settings_.spatialLayers[sl_idx].maxBitrate =
codec_settings_.startBitrate;
codec_settings_.spatialLayers[sl_idx].targetBitrate =
codec_settings_.startBitrate;
codec_settings_.spatialLayers[sl_idx].active = true;
// Force different frame rates for different layers, to verify that total
// fraction is correct.
codec_settings_.spatialLayers[sl_idx].maxFramerate =
codec_settings_.maxFramerate / (kNumSpatialLayers - sl_idx);
}
EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK,
encoder_->InitEncode(&codec_settings_, 1 /* number of cores */,
0 /* max payload size (unused) */));
// No temporal layers allowed when spatial layers have different fps targets.
FramerateFractions expected_fps_allocation[kMaxSpatialLayers];
expected_fps_allocation[0].push_back(EncoderInfo::kMaxFramerateFraction / 3);
expected_fps_allocation[1].push_back(EncoderInfo::kMaxFramerateFraction / 2);
expected_fps_allocation[2].push_back(EncoderInfo::kMaxFramerateFraction);
EXPECT_THAT(encoder_->GetEncoderInfo().fps_allocation,
::testing::ElementsAreArray(expected_fps_allocation));
}
class TestVp9ImplWithLayering
: public TestVp9Impl,
public ::testing::WithParamInterface<::testing::tuple<uint8_t, uint8_t>> {

16
modules/video_coding/codecs/vp9/vp9_impl.cc
View File

@ -1354,6 +1354,22 @@ VideoEncoder::EncoderInfo VP9EncoderImpl::GetEncoderInfo() const {
info.has_trusted_rate_controller = trusted_rate_controller_;
info.is_hardware_accelerated = false;
info.has_internal_source = false;
for (size_t si = 0; si < num_spatial_layers_; ++si) {
info.fps_allocation[si].clear();
if (!codec_.spatialLayers[si].active) {
continue;
}
// This spatial layer may already use a fraction of the total frame rate.
const float sl_fps_fraction =
codec_.spatialLayers[si].maxFramerate / codec_.maxFramerate;
for (size_t ti = 0; ti < num_temporal_layers_; ++ti) {
const uint32_t decimator =
num_temporal_layers_ <= 1 ? 1 : config_->ts_rate_decimator[ti];
RTC_DCHECK_GT(decimator, 0);
info.fps_allocation[si].push_back(rtc::saturated_cast<uint8_t>(
EncoderInfo::kMaxFramerateFraction * (sl_fps_fraction / decimator)));
}
}
return info;
}