zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Derive current layer from TL frame config.

Browse Source 
Partially removes "current frame" state from TemporalLayers, aiming to
tie more state to the frame config being encoded. This is necessary for
having several outstanding frames being encoded.

Also renames several structs, since TemporalReferences contains more
than references it's renamed TemporalLayers::FrameConfig.

BUG=chromium:702017, webrtc:7349
R=sprang@webrtc.org

Review-Url: https://codereview.webrtc.org/2853073004
Cr-Commit-Position: refs/heads/master@{#18012}
This commit is contained in:
pbos
2017-05-04 05:04:46 -07:00
committed by Commit bot
parent 173658968e
commit 18ad1d4008
10 changed files with 299 additions and 195 deletions
Download Patch File Download Diff File Expand all files Collapse all files

218
webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.cc
View File

@ -25,27 +25,31 @@
namespace webrtc {
TemporalReferences::TemporalReferences(TemporalBufferFlags last,
TemporalBufferFlags golden,
TemporalBufferFlags arf)
: TemporalReferences(last, golden, arf, false, false) {}
TemporalLayers::FrameConfig::FrameConfig() {}
TemporalReferences::TemporalReferences(TemporalBufferFlags last,
TemporalBufferFlags golden,
TemporalBufferFlags arf,
int extra_flags)
: TemporalReferences(last,
golden,
arf,
(extra_flags & kLayerSync) != 0,
(extra_flags & kFreezeEntropy) != 0) {}
TemporalLayers::FrameConfig::FrameConfig(TemporalLayers::BufferFlags last,
TemporalLayers::BufferFlags golden,
TemporalLayers::BufferFlags arf)
: FrameConfig(last, golden, arf, false, false) {}
TemporalReferences::TemporalReferences(TemporalBufferFlags last,
TemporalBufferFlags golden,
TemporalBufferFlags arf,
bool layer_sync,
bool freeze_entropy)
: drop_frame(last == kNone && golden == kNone && arf == kNone),
TemporalLayers::FrameConfig::FrameConfig(TemporalLayers::BufferFlags last,
TemporalLayers::BufferFlags golden,
TemporalLayers::BufferFlags arf,
int extra_flags)
: FrameConfig(last,
golden,
arf,
(extra_flags & kLayerSync) != 0,
(extra_flags & kFreezeEntropy) != 0) {}
TemporalLayers::FrameConfig::FrameConfig(TemporalLayers::BufferFlags last,
TemporalLayers::BufferFlags golden,
TemporalLayers::BufferFlags arf,
bool layer_sync,
bool freeze_entropy)
: drop_frame(last == TemporalLayers::kNone &&
golden == TemporalLayers::kNone &&
arf == TemporalLayers::kNone),
last_buffer_flags(last),
golden_buffer_flags(golden),
arf_buffer_flags(arf),
@ -86,8 +90,7 @@ std::vector<unsigned int> GetTemporalIds(size_t num_layers) {
return {0};
}
std::vector<TemporalReferences> GetTemporalPattern(
size_t num_layers) {
std::vector<TemporalLayers::FrameConfig> GetTemporalPattern(size_t num_layers) {
// For indexing in the patterns described below (which temporal layers they
// belong to), see the diagram above.
// Layer sync is done similarly for all patterns (except single stream) and
@ -104,74 +107,128 @@ std::vector<TemporalReferences> GetTemporalPattern(
switch (num_layers) {
case 1:
// All frames reference all buffers and the 'last' buffer is updated.
return {TemporalReferences(kReferenceAndUpdate, kReference, kReference)};
return {TemporalLayers::FrameConfig(TemporalLayers::kReferenceAndUpdate,
TemporalLayers::kReference,
TemporalLayers::kReference)};
case 2:
// All layers can reference but not update the 'alt' buffer, this means
// that the 'alt' buffer reference is effectively the last keyframe.
// TL0 also references and updates the 'last' buffer.
// TL1 also references 'last' and references and updates 'golden'.
return {TemporalReferences(kReferenceAndUpdate, kUpdate, kReference),
TemporalReferences(kReference, kUpdate, kReference, kLayerSync),
TemporalReferences(kReferenceAndUpdate, kNone, kReference),
TemporalReferences(kReference, kReferenceAndUpdate, kReference),
TemporalReferences(kReferenceAndUpdate, kNone, kReference),
TemporalReferences(kReference, kReferenceAndUpdate, kReference),
TemporalReferences(kReferenceAndUpdate, kNone, kReference),
TemporalReferences(kReference, kReference, kReference,
kFreezeEntropy)};
return {TemporalLayers::FrameConfig(TemporalLayers::kReferenceAndUpdate,
TemporalLayers::kUpdate,
TemporalLayers::kReference),
TemporalLayers::FrameConfig(
TemporalLayers::kReference, TemporalLayers::kUpdate,
TemporalLayers::kReference, kLayerSync),
TemporalLayers::FrameConfig(TemporalLayers::kReferenceAndUpdate,
TemporalLayers::kNone,
TemporalLayers::kReference),
TemporalLayers::FrameConfig(TemporalLayers::kReference,
TemporalLayers::kReferenceAndUpdate,
TemporalLayers::kReference),
TemporalLayers::FrameConfig(TemporalLayers::kReferenceAndUpdate,
TemporalLayers::kNone,
TemporalLayers::kReference),
TemporalLayers::FrameConfig(TemporalLayers::kReference,
TemporalLayers::kReferenceAndUpdate,
TemporalLayers::kReference),
TemporalLayers::FrameConfig(TemporalLayers::kReferenceAndUpdate,
TemporalLayers::kNone,
TemporalLayers::kReference),
TemporalLayers::FrameConfig(
TemporalLayers::kReference, TemporalLayers::kReference,
TemporalLayers::kReference, kFreezeEntropy)};
case 3:
// All layers can reference but not update the 'alt' buffer, this means
// that the 'alt' buffer reference is effectively the last keyframe.
// TL0 also references and updates the 'last' buffer.
// TL1 also references 'last' and references and updates 'golden'.
// TL2 references both 'last' and 'golden' but updates no buffer.
return {TemporalReferences(kReferenceAndUpdate, kUpdate, kReference),
TemporalReferences(kReference, kNone, kReference,
kLayerSync | kFreezeEntropy),
TemporalReferences(kReference, kUpdate, kReference, kLayerSync),
TemporalReferences(kReference, kReference, kReference,
kFreezeEntropy),
TemporalReferences(kReferenceAndUpdate, kNone, kReference),
TemporalReferences(kReference, kReference, kReference,
kFreezeEntropy),
TemporalReferences(kReference, kReferenceAndUpdate, kReference),
TemporalReferences(kReference, kReference, kReference,
kFreezeEntropy)};
return {TemporalLayers::FrameConfig(TemporalLayers::kReferenceAndUpdate,
TemporalLayers::kUpdate,
TemporalLayers::kReference),
TemporalLayers::FrameConfig(
TemporalLayers::kReference, TemporalLayers::kNone,
TemporalLayers::kReference, kLayerSync | kFreezeEntropy),
TemporalLayers::FrameConfig(
TemporalLayers::kReference, TemporalLayers::kUpdate,
TemporalLayers::kReference, kLayerSync),
TemporalLayers::FrameConfig(
TemporalLayers::kReference, TemporalLayers::kReference,
TemporalLayers::kReference, kFreezeEntropy),
TemporalLayers::FrameConfig(TemporalLayers::kReferenceAndUpdate,
TemporalLayers::kNone,
TemporalLayers::kReference),
TemporalLayers::FrameConfig(
TemporalLayers::kReference, TemporalLayers::kReference,
TemporalLayers::kReference, kFreezeEntropy),
TemporalLayers::FrameConfig(TemporalLayers::kReference,
TemporalLayers::kReferenceAndUpdate,
TemporalLayers::kReference),
TemporalLayers::FrameConfig(
TemporalLayers::kReference, TemporalLayers::kReference,
TemporalLayers::kReference, kFreezeEntropy)};
case 4:
// TL0 references and updates only the 'last' buffer.
// TL1 references 'last' and updates and references 'golden'.
// TL2 references 'last' and 'golden', and references and updates 'arf'.
// TL3 references all buffers but update none of them.
return {TemporalReferences(kReferenceAndUpdate, kNone, kNone),
TemporalReferences(kReference, kReference, kReference,
kLayerSync | kFreezeEntropy),
TemporalReferences(kReference, kNone, kUpdate, kLayerSync),
TemporalReferences(kReference, kReference, kReference,
kLayerSync | kFreezeEntropy),
TemporalReferences(kReference, kUpdate, kNone, kLayerSync),
TemporalReferences(kReference, kReference, kReference,
kLayerSync | kFreezeEntropy),
TemporalReferences(kReference, kReference, kReferenceAndUpdate),
TemporalReferences(kReference, kReference, kReference,
kLayerSync | kFreezeEntropy),
TemporalReferences(kReferenceAndUpdate, kNone, kNone),
TemporalReferences(kReference, kReference, kReference,
kLayerSync | kFreezeEntropy),
TemporalReferences(kReference, kReference, kReferenceAndUpdate),
TemporalReferences(kReference, kReference, kReference,
kLayerSync | kFreezeEntropy),
TemporalReferences(kReference, kReferenceAndUpdate, kNone),
TemporalReferences(kReference, kReference, kReference,
kLayerSync | kFreezeEntropy),
TemporalReferences(kReference, kReference, kReferenceAndUpdate),
TemporalReferences(kReference, kReference, kReference,
kLayerSync | kFreezeEntropy)};
return {TemporalLayers::FrameConfig(TemporalLayers::kReferenceAndUpdate,
TemporalLayers::kNone,
TemporalLayers::kNone),
TemporalLayers::FrameConfig(
TemporalLayers::kReference, TemporalLayers::kReference,
TemporalLayers::kReference, kLayerSync | kFreezeEntropy),
TemporalLayers::FrameConfig(TemporalLayers::kReference,
TemporalLayers::kNone,
TemporalLayers::kUpdate, kLayerSync),
TemporalLayers::FrameConfig(
TemporalLayers::kReference, TemporalLayers::kReference,
TemporalLayers::kReference, kLayerSync | kFreezeEntropy),
TemporalLayers::FrameConfig(TemporalLayers::kReference,
TemporalLayers::kUpdate,
TemporalLayers::kNone, kLayerSync),
TemporalLayers::FrameConfig(
TemporalLayers::kReference, TemporalLayers::kReference,
TemporalLayers::kReference, kLayerSync | kFreezeEntropy),
TemporalLayers::FrameConfig(TemporalLayers::kReference,
TemporalLayers::kReference,
TemporalLayers::kReferenceAndUpdate),
TemporalLayers::FrameConfig(
TemporalLayers::kReference, TemporalLayers::kReference,
TemporalLayers::kReference, kLayerSync | kFreezeEntropy),
TemporalLayers::FrameConfig(TemporalLayers::kReferenceAndUpdate,
TemporalLayers::kNone,
TemporalLayers::kNone),
TemporalLayers::FrameConfig(
TemporalLayers::kReference, TemporalLayers::kReference,
TemporalLayers::kReference, kLayerSync | kFreezeEntropy),
TemporalLayers::FrameConfig(TemporalLayers::kReference,
TemporalLayers::kReference,
TemporalLayers::kReferenceAndUpdate),
TemporalLayers::FrameConfig(
TemporalLayers::kReference, TemporalLayers::kReference,
TemporalLayers::kReference, kLayerSync | kFreezeEntropy),
TemporalLayers::FrameConfig(TemporalLayers::kReference,
TemporalLayers::kReferenceAndUpdate,
TemporalLayers::kNone),
TemporalLayers::FrameConfig(
TemporalLayers::kReference, TemporalLayers::kReference,
TemporalLayers::kReference, kLayerSync | kFreezeEntropy),
TemporalLayers::FrameConfig(TemporalLayers::kReference,
TemporalLayers::kReference,
TemporalLayers::kReferenceAndUpdate),
TemporalLayers::FrameConfig(
TemporalLayers::kReference, TemporalLayers::kReference,
TemporalLayers::kReference, kLayerSync | kFreezeEntropy)};
default:
RTC_NOTREACHED();
break;
}
RTC_NOTREACHED();
return {TemporalReferences(kNone, kNone, kNone)};
return {TemporalLayers::FrameConfig(
TemporalLayers::kNone, TemporalLayers::kNone, TemporalLayers::kNone)};
}
} // namespace
@ -188,10 +245,16 @@ DefaultTemporalLayers::DefaultTemporalLayers(int number_of_temporal_layers,
RTC_CHECK_GE(kMaxTemporalStreams, number_of_temporal_layers);
RTC_CHECK_GE(number_of_temporal_layers, 0);
RTC_CHECK_LE(number_of_temporal_layers, 4);
// pattern_idx_ wraps around temporal_pattern_.size, this is incorrect if
// temporal_ids_ are ever longer. If this is no longer correct it needs to
// wrap at max(temporal_ids_.size(), temporal_pattern_.size()).
RTC_DCHECK_LE(temporal_ids_.size(), temporal_pattern_.size());
}
int DefaultTemporalLayers::CurrentLayerId() const {
return temporal_ids_[pattern_idx_ % temporal_ids_.size()];
int DefaultTemporalLayers::GetTemporalLayerId(
const TemporalLayers::FrameConfig& tl_config) const {
RTC_DCHECK(!tl_config.drop_frame);
return temporal_ids_[tl_config.pattern_idx % temporal_ids_.size()];
}
uint8_t DefaultTemporalLayers::Tl0PicIdx() const {
@ -248,16 +311,19 @@ bool DefaultTemporalLayers::UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) {
return true;
}
// TODO(pbos): Name method so that it's obvious that it updates state.
TemporalReferences DefaultTemporalLayers::UpdateLayerConfig(
TemporalLayers::FrameConfig DefaultTemporalLayers::UpdateLayerConfig(
uint32_t timestamp) {
RTC_DCHECK_GT(num_layers_, 0);
RTC_DCHECK_LT(0, temporal_pattern_.size());
return temporal_pattern_[++pattern_idx_ % temporal_pattern_.size()];
pattern_idx_ = (pattern_idx_ + 1) % temporal_pattern_.size();
TemporalLayers::FrameConfig tl_config = temporal_pattern_[pattern_idx_];
tl_config.pattern_idx = pattern_idx_;
return tl_config;
}
void DefaultTemporalLayers::PopulateCodecSpecific(
bool frame_is_keyframe,
const TemporalLayers::FrameConfig& tl_config,
CodecSpecificInfoVP8* vp8_info,
uint32_t timestamp) {
RTC_DCHECK_GT(num_layers_, 0);
@ -271,11 +337,9 @@ void DefaultTemporalLayers::PopulateCodecSpecific(
vp8_info->temporalIdx = 0;
vp8_info->layerSync = true;
} else {
vp8_info->temporalIdx = CurrentLayerId();
TemporalReferences temporal_reference =
temporal_pattern_[pattern_idx_ % temporal_pattern_.size()];
vp8_info->temporalIdx = GetTemporalLayerId(tl_config);
vp8_info->layerSync = temporal_reference.layer_sync;
vp8_info->layerSync = tl_config.layer_sync;
}
if (last_base_layer_sync_ && vp8_info->temporalIdx != 0) {
// Regardless of pattern the frame after a base layer sync will always

8
webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.h
View File

@ -28,7 +28,7 @@ class DefaultTemporalLayers : public TemporalLayers {
// Returns the recommended VP8 encode flags needed. May refresh the decoder
// and/or update the reference buffers.
TemporalReferences UpdateLayerConfig(uint32_t timestamp) override;
TemporalLayers::FrameConfig UpdateLayerConfig(uint32_t timestamp) override;
// Update state based on new bitrate target and incoming framerate.
// Returns the bitrate allocation for the active temporal layers.
@ -39,19 +39,21 @@ class DefaultTemporalLayers : public TemporalLayers {
bool UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) override;
void PopulateCodecSpecific(bool frame_is_keyframe,
const TemporalLayers::FrameConfig& tl_config,
CodecSpecificInfoVP8* vp8_info,
uint32_t timestamp) override;
void FrameEncoded(unsigned int size, int qp) override {}
int CurrentLayerId() const override;
int GetTemporalLayerId(
const TemporalLayers::FrameConfig& references) const override;
uint8_t Tl0PicIdx() const override;
private:
const size_t num_layers_;
const std::vector<unsigned int> temporal_ids_;
const std::vector<TemporalReferences> temporal_pattern_;
const std::vector<TemporalLayers::FrameConfig> temporal_pattern_;
uint8_t tl0_pic_idx_;
uint8_t pattern_idx_;

20
webrtc/modules/video_coding/codecs/vp8/default_temporal_layers_unittest.cc
View File

@ -84,9 +84,9 @@ TEST(TemporalLayersTest, 2Layers) {
uint32_t timestamp = 0;
for (int i = 0; i < 16; ++i) {
TemporalReferences tl_config = tl.UpdateLayerConfig(timestamp);
TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp);
EXPECT_EQ(expected_flags[i], VP8EncoderImpl::EncodeFlags(tl_config));
tl.PopulateCodecSpecific(false, &vp8_info, 0);
tl.PopulateCodecSpecific(false, tl_config, &vp8_info, 0);
EXPECT_EQ(expected_temporal_idx[i], vp8_info.temporalIdx);
EXPECT_EQ(expected_layer_sync[i], vp8_info.layerSync);
timestamp += 3000;
@ -127,9 +127,9 @@ TEST(TemporalLayersTest, 3Layers) {
unsigned int timestamp = 0;
for (int i = 0; i < 16; ++i) {
TemporalReferences tl_config = tl.UpdateLayerConfig(timestamp);
TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp);
EXPECT_EQ(expected_flags[i], VP8EncoderImpl::EncodeFlags(tl_config));
tl.PopulateCodecSpecific(false, &vp8_info, 0);
tl.PopulateCodecSpecific(false, tl_config, &vp8_info, 0);
EXPECT_EQ(expected_temporal_idx[i], vp8_info.temporalIdx);
EXPECT_EQ(expected_layer_sync[i], vp8_info.layerSync);
timestamp += 3000;
@ -169,9 +169,9 @@ TEST(TemporalLayersTest, 4Layers) {
uint32_t timestamp = 0;
for (int i = 0; i < 16; ++i) {
TemporalReferences tl_config = tl.UpdateLayerConfig(timestamp);
TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp);
EXPECT_EQ(expected_flags[i], VP8EncoderImpl::EncodeFlags(tl_config));
tl.PopulateCodecSpecific(false, &vp8_info, 0);
tl.PopulateCodecSpecific(false, tl_config, &vp8_info, 0);
EXPECT_EQ(expected_temporal_idx[i], vp8_info.temporalIdx);
EXPECT_EQ(expected_layer_sync[i], vp8_info.layerSync);
timestamp += 3000;
@ -199,16 +199,16 @@ TEST(TemporalLayersTest, KeyFrame) {
uint32_t timestamp = 0;
for (int i = 0; i < 7; ++i) {
TemporalReferences tl_config = tl.UpdateLayerConfig(timestamp);
TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp);
EXPECT_EQ(expected_flags[i], VP8EncoderImpl::EncodeFlags(tl_config));
tl.PopulateCodecSpecific(true, &vp8_info, 0);
tl.PopulateCodecSpecific(true, tl_config, &vp8_info, 0);
EXPECT_EQ(expected_temporal_idx[i], vp8_info.temporalIdx);
EXPECT_EQ(true, vp8_info.layerSync);
timestamp += 3000;
}
TemporalReferences tl_config = tl.UpdateLayerConfig(timestamp);
TemporalLayers::FrameConfig tl_config = tl.UpdateLayerConfig(timestamp);
EXPECT_EQ(expected_flags[7], VP8EncoderImpl::EncodeFlags(tl_config));
tl.PopulateCodecSpecific(false, &vp8_info, 0);
tl.PopulateCodecSpecific(false, tl_config, &vp8_info, 0);
EXPECT_EQ(expected_temporal_idx[7], vp8_info.temporalIdx);
EXPECT_EQ(true, vp8_info.layerSync);
}

34
webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc
View File

@ -78,7 +78,8 @@ ScreenshareLayers::~ScreenshareLayers() {
UpdateHistograms();
}
int ScreenshareLayers::CurrentLayerId() const {
int ScreenshareLayers::GetTemporalLayerId(
const TemporalLayers::FrameConfig& tl_config) const {
// Codec does not use temporal layers for screenshare.
return 0;
}
@ -87,19 +88,20 @@ uint8_t ScreenshareLayers::Tl0PicIdx() const {
return tl0_pic_idx_;
}
TemporalReferences ScreenshareLayers::UpdateLayerConfig(uint32_t timestamp) {
TemporalLayers::FrameConfig ScreenshareLayers::UpdateLayerConfig(
uint32_t timestamp) {
if (number_of_temporal_layers_ <= 1) {
// No flags needed for 1 layer screenshare.
// TODO(pbos): Consider updating only last, and not all buffers.
return TemporalReferences(kReferenceAndUpdate, kReferenceAndUpdate,
kReferenceAndUpdate);
return TemporalLayers::FrameConfig(kReferenceAndUpdate, kReferenceAndUpdate,
kReferenceAndUpdate);
}
const int64_t now_ms = clock_->TimeInMilliseconds();
if (target_framerate_.value_or(0) > 0 &&
encode_framerate_.Rate(now_ms).value_or(0) > *target_framerate_) {
// Max framerate exceeded, drop frame.
return TemporalReferences(kNone, kNone, kNone);
return TemporalLayers::FrameConfig(kNone, kNone, kNone);
}
if (stats_.first_frame_time_ms_ == -1)
@ -165,21 +167,23 @@ TemporalReferences ScreenshareLayers::UpdateLayerConfig(uint32_t timestamp) {
// layers.
switch (layer_state) {
case kDrop:
return TemporalReferences(kNone, kNone, kNone);
return TemporalLayers::FrameConfig(kNone, kNone, kNone);
case kTl0:
// TL0 only references and updates 'last'.
return TemporalReferences(kReferenceAndUpdate, kNone, kNone);
return TemporalLayers::FrameConfig(kReferenceAndUpdate, kNone, kNone);
case kTl1:
// TL1 references both 'last' and 'golden' but only updates 'golden'.
return TemporalReferences(kReference, kReferenceAndUpdate, kNone);
return TemporalLayers::FrameConfig(kReference, kReferenceAndUpdate,
kNone);
case kTl1Sync:
// Predict from only TL0 to allow participants to switch to the high
// bitrate stream. Updates 'golden' so that TL1 can continue to refer to
// and update 'golden' from this point on.
return TemporalReferences(kReference, kUpdate, kNone, kLayerSync);
return TemporalLayers::FrameConfig(kReference, kUpdate, kNone,
kLayerSync);
}
RTC_NOTREACHED();
return TemporalReferences(kNone, kNone, kNone);
return TemporalLayers::FrameConfig(kNone, kNone, kNone);
}
std::vector<uint32_t> ScreenshareLayers::OnRatesUpdated(int bitrate_kbps,
@ -250,15 +254,19 @@ void ScreenshareLayers::FrameEncoded(unsigned int size, int qp) {
}
}
void ScreenshareLayers::PopulateCodecSpecific(bool frame_is_keyframe,
CodecSpecificInfoVP8* vp8_info,
uint32_t timestamp) {
void ScreenshareLayers::PopulateCodecSpecific(
bool frame_is_keyframe,
const TemporalLayers::FrameConfig& tl_config,
CodecSpecificInfoVP8* vp8_info,
uint32_t timestamp) {
int64_t unwrapped_timestamp = time_wrap_handler_.Unwrap(timestamp);
if (number_of_temporal_layers_ == 1) {
vp8_info->temporalIdx = kNoTemporalIdx;
vp8_info->layerSync = false;
vp8_info->tl0PicIdx = kNoTl0PicIdx;
} else {
// TODO(pbos): Add active_layer_ to TemporalLayers::FrameConfig (as
// pattern_idx) and make function const.
RTC_DCHECK_NE(-1, active_layer_);
vp8_info->temporalIdx = active_layer_;
if (frame_is_keyframe) {

6
webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h
View File

@ -35,7 +35,7 @@ class ScreenshareLayers : public TemporalLayers {
// Returns the recommended VP8 encode flags needed. May refresh the decoder
// and/or update the reference buffers.
TemporalReferences UpdateLayerConfig(uint32_t timestamp) override;
TemporalLayers::FrameConfig UpdateLayerConfig(uint32_t timestamp) override;
// Update state based on new bitrate target and incoming framerate.
// Returns the bitrate allocation for the active temporal layers.
@ -48,12 +48,14 @@ class ScreenshareLayers : public TemporalLayers {
bool UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) override;
void PopulateCodecSpecific(bool base_layer_sync,
const TemporalLayers::FrameConfig& tl_config,
CodecSpecificInfoVP8* vp8_info,
uint32_t timestamp) override;
void FrameEncoded(unsigned int size, int qp) override;
int CurrentLayerId() const override;
int GetTemporalLayerId(
const TemporalLayers::FrameConfig& tl_config) const override;
uint8_t Tl0PicIdx() const override;

59
webrtc/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc
View File

@ -58,13 +58,14 @@ class ScreenshareLayerTest : public ::testing::Test {
bool base_sync,
CodecSpecificInfoVP8* vp8_info,
int* flags) {
TemporalReferences tl_config = layers_->UpdateLayerConfig(timestamp);
TemporalLayers::FrameConfig tl_config =
layers_->UpdateLayerConfig(timestamp);
if (tl_config.drop_frame) {
*flags = -1;
return;
}
*flags = VP8EncoderImpl::EncodeFlags(tl_config);
layers_->PopulateCodecSpecific(base_sync, vp8_info, timestamp);
layers_->PopulateCodecSpecific(base_sync, tl_config, vp8_info, timestamp);
ASSERT_NE(-1, frame_size_);
layers_->FrameEncoded(frame_size_, kDefaultQp);
}
@ -111,10 +112,11 @@ class ScreenshareLayerTest : public ::testing::Test {
int SkipUntilTl(int layer, int timestamp) {
CodecSpecificInfoVP8 vp8_info;
for (int i = 0; i < 5; ++i) {
TemporalReferences tl_config = layers_->UpdateLayerConfig(timestamp);
TemporalLayers::FrameConfig tl_config =
layers_->UpdateLayerConfig(timestamp);
VP8EncoderImpl::EncodeFlags(tl_config);
timestamp += kTimestampDelta5Fps;
layers_->PopulateCodecSpecific(false, &vp8_info, timestamp);
layers_->PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp);
if (vp8_info.temporalIdx != layer) {
layers_->FrameEncoded(frame_size_, kDefaultQp);
} else {
@ -149,17 +151,20 @@ TEST_F(ScreenshareLayerTest, 1Layer) {
// One layer screenshare should not use the frame dropper as all frames will
// belong to the base layer.
const int kSingleLayerFlags = 0;
flags = VP8EncoderImpl::EncodeFlags(layers_->UpdateLayerConfig(timestamp));
TemporalLayers::FrameConfig tl_config;
tl_config = layers_->UpdateLayerConfig(timestamp);
flags = VP8EncoderImpl::EncodeFlags(tl_config);
EXPECT_EQ(kSingleLayerFlags, flags);
layers_->PopulateCodecSpecific(false, &vp8_info, timestamp);
layers_->PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp);
EXPECT_EQ(static_cast<uint8_t>(kNoTemporalIdx), vp8_info.temporalIdx);
EXPECT_FALSE(vp8_info.layerSync);
EXPECT_EQ(kNoTl0PicIdx, vp8_info.tl0PicIdx);
layers_->FrameEncoded(frame_size_, kDefaultQp);
flags = VP8EncoderImpl::EncodeFlags(layers_->UpdateLayerConfig(timestamp));
tl_config = layers_->UpdateLayerConfig(timestamp);
flags = VP8EncoderImpl::EncodeFlags(tl_config);
EXPECT_EQ(kSingleLayerFlags, flags);
timestamp += kTimestampDelta5Fps;
layers_->PopulateCodecSpecific(false, &vp8_info, timestamp);
layers_->PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp);
EXPECT_EQ(static_cast<uint8_t>(kNoTemporalIdx), vp8_info.temporalIdx);
EXPECT_FALSE(vp8_info.layerSync);
EXPECT_EQ(kNoTl0PicIdx, vp8_info.tl0PicIdx);
@ -244,8 +249,9 @@ TEST_F(ScreenshareLayerTest, 2LayersSyncAfterTimeout) {
const int kNumFrames = kMaxSyncPeriodSeconds * kFrameRate * 2 - 1;
for (int i = 0; i < kNumFrames; ++i) {
timestamp += kTimestampDelta5Fps;
layers_->UpdateLayerConfig(timestamp);
layers_->PopulateCodecSpecific(false, &vp8_info, timestamp);
TemporalLayers::FrameConfig tl_config =
layers_->UpdateLayerConfig(timestamp);
layers_->PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp);
// Simulate TL1 being at least 8 qp steps better.
if (vp8_info.temporalIdx == 0) {
@ -273,8 +279,9 @@ TEST_F(ScreenshareLayerTest, 2LayersSyncAfterSimilarQP) {
kFrameRate;
for (int i = 0; i < kNumFrames; ++i) {
timestamp += kTimestampDelta5Fps;
layers_->UpdateLayerConfig(timestamp);
layers_->PopulateCodecSpecific(false, &vp8_info, timestamp);
TemporalLayers::FrameConfig tl_config =
layers_->UpdateLayerConfig(timestamp);
layers_->PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp);
// Simulate TL1 being at least 8 qp steps better.
if (vp8_info.temporalIdx == 0) {
@ -292,9 +299,10 @@ TEST_F(ScreenshareLayerTest, 2LayersSyncAfterSimilarQP) {
bool bumped_tl0_quality = false;
for (int i = 0; i < 3; ++i) {
timestamp += kTimestampDelta5Fps;
TemporalReferences tl_config = layers_->UpdateLayerConfig(timestamp);
TemporalLayers::FrameConfig tl_config =
layers_->UpdateLayerConfig(timestamp);
int flags = VP8EncoderImpl::EncodeFlags(tl_config);
layers_->PopulateCodecSpecific(false, &vp8_info, timestamp);
layers_->PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp);
if (vp8_info.temporalIdx == 0) {
// Bump TL0 to same quality as TL1.
@ -442,9 +450,9 @@ TEST_F(ScreenshareLayerTest, EncoderDrop) {
layers_->FrameEncoded(0, kDefaultQp);
timestamp += kTimestampDelta5Fps;
EXPECT_FALSE(layers_->UpdateConfiguration(&cfg));
EXPECT_EQ(kTl0Flags,
VP8EncoderImpl::EncodeFlags(layers_->UpdateLayerConfig(timestamp)));
layers_->PopulateCodecSpecific(false, &vp8_info, timestamp);
TemporalLayers::FrameConfig tl_config = layers_->UpdateLayerConfig(timestamp);
EXPECT_EQ(kTl0Flags, VP8EncoderImpl::EncodeFlags(tl_config));
layers_->PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp);
layers_->FrameEncoded(frame_size_, kDefaultQp);
timestamp = SkipUntilTl(0, timestamp);
@ -452,10 +460,10 @@ TEST_F(ScreenshareLayerTest, EncoderDrop) {
EXPECT_LT(cfg.rc_max_quantizer, static_cast<unsigned int>(kDefaultQp));
layers_->FrameEncoded(frame_size_, kDefaultQp);
layers_->UpdateLayerConfig(timestamp);
tl_config = layers_->UpdateLayerConfig(timestamp);
timestamp += kTimestampDelta5Fps;
EXPECT_TRUE(layers_->UpdateConfiguration(&cfg));
layers_->PopulateCodecSpecific(false, &vp8_info, timestamp);
layers_->PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp);
EXPECT_EQ(cfg.rc_max_quantizer, static_cast<unsigned int>(kDefaultQp));
layers_->FrameEncoded(frame_size_, kDefaultQp);
@ -465,9 +473,9 @@ TEST_F(ScreenshareLayerTest, EncoderDrop) {
layers_->FrameEncoded(0, kDefaultQp);
timestamp += kTimestampDelta5Fps;
EXPECT_FALSE(layers_->UpdateConfiguration(&cfg));
EXPECT_EQ(kTl1Flags,
VP8EncoderImpl::EncodeFlags(layers_->UpdateLayerConfig(timestamp)));
layers_->PopulateCodecSpecific(false, &vp8_info, timestamp);
tl_config = layers_->UpdateLayerConfig(timestamp);
EXPECT_EQ(kTl1Flags, VP8EncoderImpl::EncodeFlags(tl_config));
layers_->PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp);
layers_->FrameEncoded(frame_size_, kDefaultQp);
timestamp = SkipUntilTl(1, timestamp);
@ -475,10 +483,10 @@ TEST_F(ScreenshareLayerTest, EncoderDrop) {
EXPECT_LT(cfg.rc_max_quantizer, static_cast<unsigned int>(kDefaultQp));
layers_->FrameEncoded(frame_size_, kDefaultQp);
layers_->UpdateLayerConfig(timestamp);
tl_config = layers_->UpdateLayerConfig(timestamp);
timestamp += kTimestampDelta5Fps;
EXPECT_TRUE(layers_->UpdateConfiguration(&cfg));
layers_->PopulateCodecSpecific(false, &vp8_info, timestamp);
layers_->PopulateCodecSpecific(false, tl_config, &vp8_info, timestamp);
EXPECT_EQ(cfg.rc_max_quantizer, static_cast<unsigned int>(kDefaultQp));
layers_->FrameEncoded(frame_size_, kDefaultQp);
}
@ -522,7 +530,8 @@ TEST_F(ScreenshareLayerTest, UpdatesHistograms) {
for (int64_t timestamp = 0;
timestamp < kTimestampDelta5Fps * 5 * metrics::kMinRunTimeInSeconds;
timestamp += kTimestampDelta5Fps) {
TemporalReferences tl_config = layers_->UpdateLayerConfig(timestamp);
TemporalLayers::FrameConfig tl_config =
layers_->UpdateLayerConfig(timestamp);
if (tl_config.drop_frame) {
dropped_frame = true;
continue;

89
webrtc/modules/video_coding/codecs/vp8/temporal_layers.h
View File

@ -24,52 +24,57 @@ namespace webrtc {
struct CodecSpecificInfoVP8;
enum TemporalBufferFlags {
kNone = 0,
kReference = 1,
kUpdate = 2,
kReferenceAndUpdate = kReference | kUpdate,
};
// TODO(pbos): Remove along with layer_sync and freeze_entropy, they are
// derivable from picture_idx and should be exposed with
// TL::IsLayerSync(TL::FrameConfig).
enum TemporalFlags { kLayerSync = 1, kFreezeEntropy = 2 };
struct TemporalReferences {
TemporalReferences(TemporalBufferFlags last,
TemporalBufferFlags golden,
TemporalBufferFlags arf);
TemporalReferences(TemporalBufferFlags last,
TemporalBufferFlags golden,
TemporalBufferFlags arf,
int extra_flags);
const bool drop_frame;
const TemporalBufferFlags last_buffer_flags;
const TemporalBufferFlags golden_buffer_flags;
const TemporalBufferFlags arf_buffer_flags;
// TODO(pbos): Consider breaking these out of here and returning only a
// pattern index that needs to be returned to fill CodecSpecificInfoVP8 or
// EncodeFlags.
const bool layer_sync;
const bool freeze_entropy;
private:
TemporalReferences(TemporalBufferFlags last,
TemporalBufferFlags golden,
TemporalBufferFlags arf,
bool layer_sync,
bool freeze_entropy);
};
class TemporalLayers {
public:
enum BufferFlags {
kNone = 0,
kReference = 1,
kUpdate = 2,
kReferenceAndUpdate = kReference | kUpdate,
};
struct FrameConfig {
FrameConfig();
FrameConfig(BufferFlags last, BufferFlags golden, BufferFlags arf);
FrameConfig(BufferFlags last,
BufferFlags golden,
BufferFlags arf,
int extra_flags);
bool drop_frame;
BufferFlags last_buffer_flags;
BufferFlags golden_buffer_flags;
BufferFlags arf_buffer_flags;
// TODO(pbos): Consider breaking these out of here and returning only a
// pattern index that needs to be returned to fill CodecSpecificInfoVP8 or
// EncodeFlags.
bool layer_sync;
bool freeze_entropy;
int pattern_idx;
private:
FrameConfig(BufferFlags last,
BufferFlags golden,
BufferFlags arf,
bool layer_sync,
bool freeze_entropy);
};
// Factory for TemporalLayer strategy. Default behavior is a fixed pattern
// of temporal layers. See default_temporal_layers.cc
virtual ~TemporalLayers() {}
// Returns the recommended VP8 encode flags needed. May refresh the decoder
// and/or update the reference buffers.
virtual TemporalReferences UpdateLayerConfig(uint32_t timestamp) = 0;
virtual FrameConfig UpdateLayerConfig(uint32_t timestamp) = 0;
// Update state based on new bitrate target and incoming framerate.
// Returns the bitrate allocation for the active temporal layers.
@ -81,17 +86,19 @@ class TemporalLayers {
// Returns true iff the configuration was actually modified.
virtual bool UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) = 0;
virtual void PopulateCodecSpecific(bool is_keyframe,
CodecSpecificInfoVP8* vp8_info,
uint32_t timestamp) = 0;
virtual void PopulateCodecSpecific(
bool is_keyframe,
const TemporalLayers::FrameConfig& tl_config,
CodecSpecificInfoVP8* vp8_info,
uint32_t timestamp) = 0;
virtual void FrameEncoded(unsigned int size, int qp) = 0;
virtual int CurrentLayerId() const = 0;
// Returns the current tl0_pic_idx, so it can be reused in future
// instantiations.
virtual uint8_t Tl0PicIdx() const = 0;
virtual int GetTemporalLayerId(
const TemporalLayers::FrameConfig& tl_config) const = 0;
};
class TemporalLayersListener;

41
webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc
View File

@ -133,22 +133,22 @@ VP8Decoder* VP8Decoder::Create() {
}
vpx_enc_frame_flags_t VP8EncoderImpl::EncodeFlags(
TemporalReferences references) {
const TemporalLayers::FrameConfig& references) {
RTC_DCHECK(!references.drop_frame);
vpx_enc_frame_flags_t flags = 0;
if ((references.last_buffer_flags & kReference) == 0)
if ((references.last_buffer_flags & TemporalLayers::kReference) == 0)
flags |= VP8_EFLAG_NO_REF_LAST;
if ((references.last_buffer_flags & kUpdate) == 0)
if ((references.last_buffer_flags & TemporalLayers::kUpdate) == 0)
flags |= VP8_EFLAG_NO_UPD_LAST;
if ((references.golden_buffer_flags & kReference) == 0)
if ((references.golden_buffer_flags & TemporalLayers::kReference) == 0)
flags |= VP8_EFLAG_NO_REF_GF;
if ((references.golden_buffer_flags & kUpdate) == 0)
if ((references.golden_buffer_flags & TemporalLayers::kUpdate) == 0)
flags |= VP8_EFLAG_NO_UPD_GF;
if ((references.arf_buffer_flags & kReference) == 0)
if ((references.arf_buffer_flags & TemporalLayers::kReference) == 0)
flags |= VP8_EFLAG_NO_REF_ARF;
if ((references.arf_buffer_flags & kUpdate) == 0)
if ((references.arf_buffer_flags & TemporalLayers::kUpdate) == 0)
flags |= VP8_EFLAG_NO_UPD_ARF;
if (references.freeze_entropy)
flags |= VP8_EFLAG_NO_UPD_ENTROPY;
@ -703,15 +703,15 @@ int VP8EncoderImpl::Encode(const VideoFrame& frame,
raw_images_[i].d_h, libyuv::kFilterBilinear);
}
vpx_enc_frame_flags_t flags[kMaxSimulcastStreams];
TemporalLayers::FrameConfig tl_configs[kMaxSimulcastStreams];
for (size_t i = 0; i < encoders_.size(); ++i) {
TemporalReferences tl_config =
temporal_layers_[i]->UpdateLayerConfig(frame.timestamp());
tl_configs[i] = temporal_layers_[i]->UpdateLayerConfig(frame.timestamp());
if (tl_config.drop_frame) {
if (tl_configs[i].drop_frame) {
// Drop this frame.
return WEBRTC_VIDEO_CODEC_OK;
}
flags[i] = EncodeFlags(tl_config);
flags[i] = EncodeFlags(tl_configs[i]);
}
bool send_key_frame = false;
for (size_t i = 0; i < key_frame_request_.size() && i < send_stream_.size();
@ -764,8 +764,9 @@ int VP8EncoderImpl::Encode(const VideoFrame& frame,
}
vpx_codec_control(&encoders_[i], VP8E_SET_FRAME_FLAGS, flags[stream_idx]);
vpx_codec_control(&encoders_[i], VP8E_SET_TEMPORAL_LAYER_ID,
temporal_layers_[stream_idx]->CurrentLayerId());
vpx_codec_control(
&encoders_[i], VP8E_SET_TEMPORAL_LAYER_ID,
temporal_layers_[stream_idx]->GetTemporalLayerId(tl_configs[i]));
}
// TODO(holmer): Ideally the duration should be the timestamp diff of this
// frame and the next frame to be encoded, which we don't have. Instead we
@ -788,11 +789,12 @@ int VP8EncoderImpl::Encode(const VideoFrame& frame,
return WEBRTC_VIDEO_CODEC_ERROR;
timestamp_ += duration;
// Examines frame timestamps only.
return GetEncodedPartitions(frame);
return GetEncodedPartitions(tl_configs, frame);
}
void VP8EncoderImpl::PopulateCodecSpecific(
CodecSpecificInfo* codec_specific,
const TemporalLayers::FrameConfig& tl_config,
const vpx_codec_cx_pkt_t& pkt,
int stream_idx,
uint32_t timestamp) {
@ -805,12 +807,15 @@ void VP8EncoderImpl::PopulateCodecSpecific(
vp8Info->keyIdx = kNoKeyIdx; // TODO(hlundin) populate this
vp8Info->nonReference = (pkt.data.frame.flags & VPX_FRAME_IS_DROPPABLE) != 0;
temporal_layers_[stream_idx]->PopulateCodecSpecific(
(pkt.data.frame.flags & VPX_FRAME_IS_KEY) != 0, vp8Info, timestamp);
(pkt.data.frame.flags & VPX_FRAME_IS_KEY) != 0, tl_config, vp8Info,
timestamp);
// Prepare next.
picture_id_[stream_idx] = (picture_id_[stream_idx] + 1) & 0x7FFF;
}
int VP8EncoderImpl::GetEncodedPartitions(const VideoFrame& input_image) {
int VP8EncoderImpl::GetEncodedPartitions(
const TemporalLayers::FrameConfig tl_configs[],
const VideoFrame& input_image) {
int bw_resolutions_disabled =
(encoders_.size() > 1) ? NumStreamsDisabled(send_stream_) : -1;
@ -860,8 +865,8 @@ int VP8EncoderImpl::GetEncodedPartitions(const VideoFrame& input_image) {
if (pkt->data.frame.flags & VPX_FRAME_IS_KEY) {
encoded_images_[encoder_idx]._frameType = kVideoFrameKey;
}
PopulateCodecSpecific(&codec_specific, *pkt, stream_idx,
input_image.timestamp());
PopulateCodecSpecific(&codec_specific, tl_configs[stream_idx], *pkt,
stream_idx, input_image.timestamp());
break;
}
}

7
webrtc/modules/video_coding/codecs/vp8/vp8_impl.h
View File

@ -62,7 +62,8 @@ class VP8EncoderImpl : public VP8Encoder {
const char* ImplementationName() const override;
static vpx_enc_frame_flags_t EncodeFlags(TemporalReferences references);
static vpx_enc_frame_flags_t EncodeFlags(
const TemporalLayers::FrameConfig& references);
private:
void SetupTemporalLayers(int num_streams,
@ -79,11 +80,13 @@ class VP8EncoderImpl : public VP8Encoder {
int InitAndSetControlSettings();
void PopulateCodecSpecific(CodecSpecificInfo* codec_specific,
const TemporalLayers::FrameConfig& tl_config,
const vpx_codec_cx_pkt& pkt,
int stream_idx,
uint32_t timestamp);
int GetEncodedPartitions(const VideoFrame& input_image);
int GetEncodedPartitions(const TemporalLayers::FrameConfig tl_configs[],
const VideoFrame& input_image);
// Set the stream state for stream |stream_idx|.
void SetStreamState(bool send_stream, int stream_idx);