zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Base screenshare layers on TemporalReferences.

Browse Source 
Decouples encode flags and calculates them the same for both default and
screencast temporal layers.

With this change encoders could start using TemporalReferences for
temporal-layers flags, but they can not be used by asynchronous encoders
(hardware encoders) yet.

Also removes 'timestamp' as a dead parameter to FrameEncoded().

BUG=chromium:702017, webrtc:7349
R=marpan@google.com, sprang@webrtc.org, marpan@webrtc.org

Review-Url: https://codereview.webrtc.org/2769263002 .
Cr-Commit-Position: refs/heads/master@{#17397}
This commit is contained in:
Peter Boström
2017-03-27 15:01:49 -04:00
parent d8cfa1af38
commit 1436c83cd1
8 changed files with 166 additions and 161 deletions
Download Patch File Download Diff File Expand all files Collapse all files

64
webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.cc
View File

@ -25,14 +25,14 @@
namespace webrtc { namespace webrtc {
TemporalReferences::TemporalReferences(TemporalBufferUsage last, TemporalReferences::TemporalReferences(TemporalBufferFlags last,
TemporalBufferUsage golden, TemporalBufferFlags golden,
TemporalBufferUsage arf) TemporalBufferFlags arf)
: TemporalReferences(last, golden, arf, false, false) {} : TemporalReferences(last, golden, arf, false, false) {}
TemporalReferences::TemporalReferences(TemporalBufferUsage last, TemporalReferences::TemporalReferences(TemporalBufferFlags last,
TemporalBufferUsage golden, TemporalBufferFlags golden,
TemporalBufferUsage arf, TemporalBufferFlags arf,
int extra_flags) int extra_flags)
: TemporalReferences(last, : TemporalReferences(last,
golden, golden,
@ -40,17 +40,15 @@ TemporalReferences::TemporalReferences(TemporalBufferUsage last,
(extra_flags & kLayerSync) != 0, (extra_flags & kLayerSync) != 0,
(extra_flags & kFreezeEntropy) != 0) {} (extra_flags & kFreezeEntropy) != 0) {}
TemporalReferences::TemporalReferences(TemporalBufferUsage last, TemporalReferences::TemporalReferences(TemporalBufferFlags last,
TemporalBufferUsage golden, TemporalBufferFlags golden,
TemporalBufferUsage arf, TemporalBufferFlags arf,
bool layer_sync, bool layer_sync,
bool freeze_entropy) bool freeze_entropy)
: reference_last((last & kReference) != 0), : drop_frame(last == kNone && golden == kNone && arf == kNone),
update_last((last & kUpdate) != 0), last_buffer_flags(last),
reference_golden((golden & kReference) != 0), golden_buffer_flags(golden),
update_golden((golden & kUpdate) != 0), arf_buffer_flags(arf),
reference_arf((arf & kReference) != 0),
update_arf((arf & kUpdate) != 0),
layer_sync(layer_sync), layer_sync(layer_sync),
freeze_entropy(freeze_entropy) {} freeze_entropy(freeze_entropy) {}
@ -246,26 +244,32 @@ bool DefaultTemporalLayers::UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) {
return true; return true;
} }
int DefaultTemporalLayers::EncodeFlags(uint32_t timestamp) { // TODO(pbos): Name method so that it's obvious that it updates state.
TemporalReferences DefaultTemporalLayers::UpdateLayerConfig(
uint32_t timestamp) {
RTC_DCHECK_GT(num_layers_, 0); RTC_DCHECK_GT(num_layers_, 0);
RTC_DCHECK_LT(0, temporal_pattern_.size()); RTC_DCHECK_LT(0, temporal_pattern_.size());
int flags = 0; return temporal_pattern_[++pattern_idx_ % temporal_pattern_.size()];
// TODO(pbos): Move pattern-update out of EncodeFlags. It's not obvious that }
// EncodeFlags() is non-const.
const TemporalReferences& references =
temporal_pattern_[++pattern_idx_ % temporal_pattern_.size()];
if (!references.reference_last) int TemporalLayers::EncodeFlags(uint32_t timestamp) {
TemporalReferences references = UpdateLayerConfig(timestamp);
if (references.drop_frame)
return -1;
int flags = 0;
if ((references.last_buffer_flags & kReference) == 0)
flags |= VP8_EFLAG_NO_REF_LAST; flags |= VP8_EFLAG_NO_REF_LAST;
if (!references.update_last) if ((references.last_buffer_flags & kUpdate) == 0)
flags |= VP8_EFLAG_NO_UPD_LAST; flags |= VP8_EFLAG_NO_UPD_LAST;
if (!references.reference_golden) if ((references.golden_buffer_flags & kReference) == 0)
flags |= VP8_EFLAG_NO_REF_GF; flags |= VP8_EFLAG_NO_REF_GF;
if (!references.update_golden) if ((references.golden_buffer_flags & kUpdate) == 0)
flags |= VP8_EFLAG_NO_UPD_GF; flags |= VP8_EFLAG_NO_UPD_GF;
if (!references.reference_arf) if ((references.arf_buffer_flags & kReference) == 0)
flags |= VP8_EFLAG_NO_REF_ARF; flags |= VP8_EFLAG_NO_REF_ARF;
if (!references.update_arf) if ((references.arf_buffer_flags & kUpdate) == 0)
flags |= VP8_EFLAG_NO_UPD_ARF; flags |= VP8_EFLAG_NO_UPD_ARF;
if (references.freeze_entropy) if (references.freeze_entropy)
flags |= VP8_EFLAG_NO_UPD_ENTROPY; flags |= VP8_EFLAG_NO_UPD_ENTROPY;
@ -274,7 +278,7 @@ int DefaultTemporalLayers::EncodeFlags(uint32_t timestamp) {
} }
void DefaultTemporalLayers::PopulateCodecSpecific( void DefaultTemporalLayers::PopulateCodecSpecific(
bool base_layer_sync, bool frame_is_keyframe,
CodecSpecificInfoVP8* vp8_info, CodecSpecificInfoVP8* vp8_info,
uint32_t timestamp) { uint32_t timestamp) {
RTC_DCHECK_GT(num_layers_, 0); RTC_DCHECK_GT(num_layers_, 0);
@ -284,7 +288,7 @@ void DefaultTemporalLayers::PopulateCodecSpecific(
vp8_info->layerSync = false; vp8_info->layerSync = false;
vp8_info->tl0PicIdx = kNoTl0PicIdx; vp8_info->tl0PicIdx = kNoTl0PicIdx;
} else { } else {
if (base_layer_sync) { if (frame_is_keyframe) {
vp8_info->temporalIdx = 0; vp8_info->temporalIdx = 0;
vp8_info->layerSync = true; vp8_info->layerSync = true;
} else { } else {
@ -303,7 +307,7 @@ void DefaultTemporalLayers::PopulateCodecSpecific(
timestamp_ = timestamp; timestamp_ = timestamp;
tl0_pic_idx_++; tl0_pic_idx_++;
} }
last_base_layer_sync_ = base_layer_sync; last_base_layer_sync_ = frame_is_keyframe;
vp8_info->tl0PicIdx = tl0_pic_idx_; vp8_info->tl0PicIdx = tl0_pic_idx_;
} }
} }

44
webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.h
View File

@ -20,44 +20,6 @@
namespace webrtc { namespace webrtc {
enum TemporalBufferUsage {
kNone = 0,
kReference = 1,
kUpdate = 2,
kReferenceAndUpdate = kReference | kUpdate,
};
enum TemporalFlags { kLayerSync = 1, kFreezeEntropy = 2 };
struct TemporalReferences {
TemporalReferences(TemporalBufferUsage last,
TemporalBufferUsage golden,
TemporalBufferUsage arf);
TemporalReferences(TemporalBufferUsage last,
TemporalBufferUsage golden,
TemporalBufferUsage arf,
int extra_flags);
const bool reference_last;
const bool update_last;
const bool reference_golden;
const bool update_golden;
const bool reference_arf;
const bool update_arf;
// TODO(pbos): Consider breaking these out of here and returning only a
// pattern index that needs to be returned to fill CodecSpecificInfoVP8 or
// EncodeFlags.
const bool layer_sync;
const bool freeze_entropy;
private:
TemporalReferences(TemporalBufferUsage last,
TemporalBufferUsage golden,
TemporalBufferUsage arf,
bool layer_sync,
bool freeze_entropy);
};
class DefaultTemporalLayers : public TemporalLayers { class DefaultTemporalLayers : public TemporalLayers {
public: public:
DefaultTemporalLayers(int number_of_temporal_layers, DefaultTemporalLayers(int number_of_temporal_layers,
@ -66,7 +28,7 @@ class DefaultTemporalLayers : public TemporalLayers {
// Returns the recommended VP8 encode flags needed. May refresh the decoder // Returns the recommended VP8 encode flags needed. May refresh the decoder
// and/or update the reference buffers. // and/or update the reference buffers.
int EncodeFlags(uint32_t timestamp) override; TemporalReferences UpdateLayerConfig(uint32_t timestamp) override;
// Update state based on new bitrate target and incoming framerate. // Update state based on new bitrate target and incoming framerate.
// Returns the bitrate allocation for the active temporal layers. // Returns the bitrate allocation for the active temporal layers.
@ -76,11 +38,11 @@ class DefaultTemporalLayers : public TemporalLayers {
bool UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) override; bool UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) override;
void PopulateCodecSpecific(bool base_layer_sync, void PopulateCodecSpecific(bool frame_is_keyframe,
CodecSpecificInfoVP8* vp8_info, CodecSpecificInfoVP8* vp8_info,
uint32_t timestamp) override; uint32_t timestamp) override;
void FrameEncoded(unsigned int size, uint32_t timestamp, int qp) override {} void FrameEncoded(unsigned int size, int qp) override {}
int CurrentLayerId() const override; int CurrentLayerId() const override;

67
webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc
View File

@ -32,23 +32,6 @@ const double ScreenshareLayers::kAcceptableTargetOvershoot = 2.0;
constexpr int ScreenshareLayers::kMaxNumTemporalLayers; constexpr int ScreenshareLayers::kMaxNumTemporalLayers;
// Since this is TL0 we only allow updating and predicting from the LAST
// reference frame.
const int ScreenshareLayers::kTl0Flags =
VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_REF_GF |
VP8_EFLAG_NO_REF_ARF;
// Allow predicting from both TL0 and TL1.
const int ScreenshareLayers::kTl1Flags =
VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST;
// Allow predicting from only TL0 to allow participants to switch to the high
// bitrate stream. This means predicting only from the LAST reference frame, but
// only updating GF to not corrupt TL0.
const int ScreenshareLayers::kTl1SyncFlags =
VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_UPD_ARF |
VP8_EFLAG_NO_UPD_LAST;
// Always emit a frame with certain interval, even if bitrate targets have // Always emit a frame with certain interval, even if bitrate targets have
// been exceeded. // been exceeded.
const int ScreenshareLayers::kMaxFrameIntervalMs = 2000; const int ScreenshareLayers::kMaxFrameIntervalMs = 2000;
@ -100,24 +83,27 @@ int ScreenshareLayers::CurrentLayerId() const {
return 0; return 0;
} }
int ScreenshareLayers::EncodeFlags(uint32_t timestamp) { TemporalReferences ScreenshareLayers::UpdateLayerConfig(uint32_t timestamp) {
if (number_of_temporal_layers_ <= 1) { if (number_of_temporal_layers_ <= 1) {
// No flags needed for 1 layer screenshare. // No flags needed for 1 layer screenshare.
return 0; // TODO(pbos): Consider updating only last, and not all buffers.
return TemporalReferences(kReferenceAndUpdate, kReferenceAndUpdate,
kReferenceAndUpdate);
} }
const int64_t now_ms = clock_->TimeInMilliseconds(); const int64_t now_ms = clock_->TimeInMilliseconds();
if (target_framerate_.value_or(0) > 0 && if (target_framerate_.value_or(0) > 0 &&
encode_framerate_.Rate(now_ms).value_or(0) > *target_framerate_) { encode_framerate_.Rate(now_ms).value_or(0) > *target_framerate_) {
// Max framerate exceeded, drop frame. // Max framerate exceeded, drop frame.
return -1; return TemporalReferences(kNone, kNone, kNone);
} }
if (stats_.first_frame_time_ms_ == -1) if (stats_.first_frame_time_ms_ == -1)
stats_.first_frame_time_ms_ = now_ms; stats_.first_frame_time_ms_ = now_ms;
int64_t unwrapped_timestamp = time_wrap_handler_.Unwrap(timestamp); int64_t unwrapped_timestamp = time_wrap_handler_.Unwrap(timestamp);
int flags = 0; enum TemporalLayerState { kDrop, kTl0, kTl1, kTl1Sync };
enum TemporalLayerState layer_state = kDrop;
if (active_layer_ == -1 || if (active_layer_ == -1 ||
layers_[active_layer_].state != TemporalLayer::State::kDropped) { layers_[active_layer_].state != TemporalLayer::State::kDropped) {
if (last_emitted_tl0_timestamp_ != -1 && if (last_emitted_tl0_timestamp_ != -1 &&
@ -142,23 +128,22 @@ int ScreenshareLayers::EncodeFlags(uint32_t timestamp) {
switch (active_layer_) { switch (active_layer_) {
case 0: case 0:
flags = kTl0Flags; layer_state = kTl0;
last_emitted_tl0_timestamp_ = unwrapped_timestamp; last_emitted_tl0_timestamp_ = unwrapped_timestamp;
break; break;
case 1: case 1:
if (TimeToSync(unwrapped_timestamp)) { if (TimeToSync(unwrapped_timestamp)) {
last_sync_timestamp_ = unwrapped_timestamp; last_sync_timestamp_ = unwrapped_timestamp;
flags = kTl1SyncFlags; layer_state = kTl1Sync;
} else { } else {
flags = kTl1Flags; layer_state = kTl1;
} }
break; break;
case -1: case -1:
flags = -1; layer_state = kDrop;
++stats_.num_dropped_frames_; ++stats_.num_dropped_frames_;
break; break;
default: default:
flags = -1;
RTC_NOTREACHED(); RTC_NOTREACHED();
} }
@ -172,7 +157,25 @@ int ScreenshareLayers::EncodeFlags(uint32_t timestamp) {
layers_[0].UpdateDebt(ts_diff / 90); layers_[0].UpdateDebt(ts_diff / 90);
layers_[1].UpdateDebt(ts_diff / 90); layers_[1].UpdateDebt(ts_diff / 90);
last_timestamp_ = timestamp; last_timestamp_ = timestamp;
return flags; // TODO(pbos): Consider referencing but not updating the 'alt' buffer for all
// layers.
switch (layer_state) {
case kDrop:
return TemporalReferences(kNone, kNone, kNone);
case kTl0:
// TL0 only references and updates 'last'.
return TemporalReferences(kReferenceAndUpdate, kNone, kNone);
case kTl1:
// TL1 references both 'last' and 'golden' but only updates 'golden'.
return TemporalReferences(kReference, kReferenceAndUpdate, kNone);
case kTl1Sync:
// Predict from only TL0 to allow participants to switch to the high
// bitrate stream. Updates 'golden' so that TL1 can continue to refer to
// and update 'golden' from this point on.
return TemporalReferences(kReference, kUpdate, kNone, kLayerSync);
}
RTC_NOTREACHED();
return TemporalReferences(kNone, kNone, kNone);
} }
std::vector<uint32_t> ScreenshareLayers::OnRatesUpdated(int bitrate_kbps, std::vector<uint32_t> ScreenshareLayers::OnRatesUpdated(int bitrate_kbps,
@ -208,9 +211,7 @@ std::vector<uint32_t> ScreenshareLayers::OnRatesUpdated(int bitrate_kbps,
return allocation; return allocation;
} }
void ScreenshareLayers::FrameEncoded(unsigned int size, void ScreenshareLayers::FrameEncoded(unsigned int size, int qp) {
uint32_t timestamp,
int qp) {
if (size > 0) if (size > 0)
encode_framerate_.Update(1, clock_->TimeInMilliseconds()); encode_framerate_.Update(1, clock_->TimeInMilliseconds());
@ -245,7 +246,7 @@ void ScreenshareLayers::FrameEncoded(unsigned int size,
} }
} }
void ScreenshareLayers::PopulateCodecSpecific(bool base_layer_sync, void ScreenshareLayers::PopulateCodecSpecific(bool frame_is_keyframe,
CodecSpecificInfoVP8* vp8_info, CodecSpecificInfoVP8* vp8_info,
uint32_t timestamp) { uint32_t timestamp) {
int64_t unwrapped_timestamp = time_wrap_handler_.Unwrap(timestamp); int64_t unwrapped_timestamp = time_wrap_handler_.Unwrap(timestamp);
@ -256,7 +257,7 @@ void ScreenshareLayers::PopulateCodecSpecific(bool base_layer_sync,
} else { } else {
RTC_DCHECK_NE(-1, active_layer_); RTC_DCHECK_NE(-1, active_layer_);
vp8_info->temporalIdx = active_layer_; vp8_info->temporalIdx = active_layer_;
if (base_layer_sync) { if (frame_is_keyframe) {
vp8_info->temporalIdx = 0; vp8_info->temporalIdx = 0;
last_sync_timestamp_ = unwrapped_timestamp; last_sync_timestamp_ = unwrapped_timestamp;
} else if (last_base_layer_sync_ && vp8_info->temporalIdx != 0) { } else if (last_base_layer_sync_ && vp8_info->temporalIdx != 0) {
@ -269,7 +270,7 @@ void ScreenshareLayers::PopulateCodecSpecific(bool base_layer_sync,
if (vp8_info->temporalIdx == 0) { if (vp8_info->temporalIdx == 0) {
tl0_pic_idx_++; tl0_pic_idx_++;
} }
last_base_layer_sync_ = base_layer_sync; last_base_layer_sync_ = frame_is_keyframe;
vp8_info->tl0PicIdx = tl0_pic_idx_; vp8_info->tl0PicIdx = tl0_pic_idx_;
} }
} }

7
webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h
View File

@ -26,9 +26,6 @@ class ScreenshareLayers : public TemporalLayers {
public: public:
static const double kMaxTL0FpsReduction; static const double kMaxTL0FpsReduction;
static const double kAcceptableTargetOvershoot; static const double kAcceptableTargetOvershoot;
static const int kTl0Flags;
static const int kTl1Flags;
static const int kTl1SyncFlags;
static const int kMaxFrameIntervalMs; static const int kMaxFrameIntervalMs;
ScreenshareLayers(int num_temporal_layers, ScreenshareLayers(int num_temporal_layers,
@ -38,7 +35,7 @@ class ScreenshareLayers : public TemporalLayers {
// Returns the recommended VP8 encode flags needed. May refresh the decoder // Returns the recommended VP8 encode flags needed. May refresh the decoder
// and/or update the reference buffers. // and/or update the reference buffers.
int EncodeFlags(uint32_t timestamp) override; TemporalReferences UpdateLayerConfig(uint32_t timestamp) override;
// Update state based on new bitrate target and incoming framerate. // Update state based on new bitrate target and incoming framerate.
// Returns the bitrate allocation for the active temporal layers. // Returns the bitrate allocation for the active temporal layers.
@ -54,7 +51,7 @@ class ScreenshareLayers : public TemporalLayers {
CodecSpecificInfoVP8* vp8_info, CodecSpecificInfoVP8* vp8_info,
uint32_t timestamp) override; uint32_t timestamp) override;
void FrameEncoded(unsigned int size, uint32_t timestamp, int qp) override; void FrameEncoded(unsigned int size, int qp) override;
int CurrentLayerId() const override; int CurrentLayerId() const override;

85
webrtc/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc
View File

@ -37,6 +37,14 @@ const int kFrameRate = 5;
const int kSyncPeriodSeconds = 5; const int kSyncPeriodSeconds = 5;
const int kMaxSyncPeriodSeconds = 10; const int kMaxSyncPeriodSeconds = 10;
// Expected flags for corresponding temporal layers.
const int kTl0Flags = VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF |
VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_REF_ARF;
const int kTl1Flags =
VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST;
const int kTl1SyncFlags = VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_REF_GF |
VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST;
class ScreenshareLayerTest : public ::testing::Test { class ScreenshareLayerTest : public ::testing::Test {
protected: protected:
ScreenshareLayerTest() ScreenshareLayerTest()
@ -54,7 +62,7 @@ class ScreenshareLayerTest : public ::testing::Test {
return; return;
layers_->PopulateCodecSpecific(base_sync, vp8_info, timestamp); layers_->PopulateCodecSpecific(base_sync, vp8_info, timestamp);
ASSERT_NE(-1, frame_size_); ASSERT_NE(-1, frame_size_);
layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp); layers_->FrameEncoded(frame_size_, kDefaultQp);
} }
void ConfigureBitrates() { void ConfigureBitrates() {
@ -103,7 +111,7 @@ class ScreenshareLayerTest : public ::testing::Test {
timestamp += kTimestampDelta5Fps; timestamp += kTimestampDelta5Fps;
layers_->PopulateCodecSpecific(false, &vp8_info, timestamp); layers_->PopulateCodecSpecific(false, &vp8_info, timestamp);
if (vp8_info.temporalIdx != layer) { if (vp8_info.temporalIdx != layer) {
layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp); layers_->FrameEncoded(frame_size_, kDefaultQp);
} else { } else {
return timestamp; return timestamp;
} }
@ -142,7 +150,7 @@ TEST_F(ScreenshareLayerTest, 1Layer) {
EXPECT_EQ(static_cast<uint8_t>(kNoTemporalIdx), vp8_info.temporalIdx); EXPECT_EQ(static_cast<uint8_t>(kNoTemporalIdx), vp8_info.temporalIdx);
EXPECT_FALSE(vp8_info.layerSync); EXPECT_FALSE(vp8_info.layerSync);
EXPECT_EQ(kNoTl0PicIdx, vp8_info.tl0PicIdx); EXPECT_EQ(kNoTl0PicIdx, vp8_info.tl0PicIdx);
layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp); layers_->FrameEncoded(frame_size_, kDefaultQp);
flags = layers_->EncodeFlags(timestamp); flags = layers_->EncodeFlags(timestamp);
EXPECT_EQ(kSingleLayerFlags, flags); EXPECT_EQ(kSingleLayerFlags, flags);
timestamp += kTimestampDelta5Fps; timestamp += kTimestampDelta5Fps;
@ -150,7 +158,7 @@ TEST_F(ScreenshareLayerTest, 1Layer) {
EXPECT_EQ(static_cast<uint8_t>(kNoTemporalIdx), vp8_info.temporalIdx); EXPECT_EQ(static_cast<uint8_t>(kNoTemporalIdx), vp8_info.temporalIdx);
EXPECT_FALSE(vp8_info.layerSync); EXPECT_FALSE(vp8_info.layerSync);
EXPECT_EQ(kNoTl0PicIdx, vp8_info.tl0PicIdx); EXPECT_EQ(kNoTl0PicIdx, vp8_info.tl0PicIdx);
layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp); layers_->FrameEncoded(frame_size_, kDefaultQp);
} }
TEST_F(ScreenshareLayerTest, 2Layer) { TEST_F(ScreenshareLayerTest, 2Layer) {
@ -160,7 +168,7 @@ TEST_F(ScreenshareLayerTest, 2Layer) {
uint8_t expected_tl0_idx = 0; uint8_t expected_tl0_idx = 0;
CodecSpecificInfoVP8 vp8_info; CodecSpecificInfoVP8 vp8_info;
EncodeFrame(timestamp, false, &vp8_info, &flags); EncodeFrame(timestamp, false, &vp8_info, &flags);
EXPECT_EQ(ScreenshareLayers::kTl0Flags, flags); EXPECT_EQ(kTl0Flags, flags);
EXPECT_EQ(0, vp8_info.temporalIdx); EXPECT_EQ(0, vp8_info.temporalIdx);
EXPECT_FALSE(vp8_info.layerSync); EXPECT_FALSE(vp8_info.layerSync);
++expected_tl0_idx; ++expected_tl0_idx;
@ -179,7 +187,7 @@ TEST_F(ScreenshareLayerTest, 2Layer) {
// First frame in TL0. // First frame in TL0.
timestamp += kTimestampDelta5Fps; timestamp += kTimestampDelta5Fps;
EncodeFrame(timestamp, false, &vp8_info, &flags); EncodeFrame(timestamp, false, &vp8_info, &flags);
EXPECT_EQ(ScreenshareLayers::kTl0Flags, flags); EXPECT_EQ(kTl0Flags, flags);
EXPECT_EQ(0, vp8_info.temporalIdx); EXPECT_EQ(0, vp8_info.temporalIdx);
EXPECT_FALSE(vp8_info.layerSync); EXPECT_FALSE(vp8_info.layerSync);
++expected_tl0_idx; ++expected_tl0_idx;
@ -189,14 +197,14 @@ TEST_F(ScreenshareLayerTest, 2Layer) {
timestamp += kTimestampDelta5Fps; timestamp += kTimestampDelta5Fps;
EncodeFrame(timestamp, false, &vp8_info, &flags); EncodeFrame(timestamp, false, &vp8_info, &flags);
// First frame is sync frame. // First frame is sync frame.
EXPECT_EQ(ScreenshareLayers::kTl1SyncFlags, flags); EXPECT_EQ(kTl1SyncFlags, flags);
EXPECT_EQ(1, vp8_info.temporalIdx); EXPECT_EQ(1, vp8_info.temporalIdx);
EXPECT_TRUE(vp8_info.layerSync); EXPECT_TRUE(vp8_info.layerSync);
EXPECT_EQ(expected_tl0_idx, vp8_info.tl0PicIdx); EXPECT_EQ(expected_tl0_idx, vp8_info.tl0PicIdx);
timestamp += kTimestampDelta5Fps; timestamp += kTimestampDelta5Fps;
EncodeFrame(timestamp, false, &vp8_info, &flags); EncodeFrame(timestamp, false, &vp8_info, &flags);
EXPECT_EQ(ScreenshareLayers::kTl1Flags, flags); EXPECT_EQ(kTl1Flags, flags);
EXPECT_EQ(1, vp8_info.temporalIdx); EXPECT_EQ(1, vp8_info.temporalIdx);
EXPECT_FALSE(vp8_info.layerSync); EXPECT_FALSE(vp8_info.layerSync);
EXPECT_EQ(expected_tl0_idx, vp8_info.tl0PicIdx); EXPECT_EQ(expected_tl0_idx, vp8_info.tl0PicIdx);
@ -236,9 +244,9 @@ TEST_F(ScreenshareLayerTest, 2LayersSyncAfterTimeout) {
// Simulate TL1 being at least 8 qp steps better. // Simulate TL1 being at least 8 qp steps better.
if (vp8_info.temporalIdx == 0) { if (vp8_info.temporalIdx == 0) {
layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp); layers_->FrameEncoded(frame_size_, kDefaultQp);
} else { } else {
layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp - 8); layers_->FrameEncoded(frame_size_, kDefaultQp - 8);
} }
if (vp8_info.temporalIdx == 1 && vp8_info.layerSync) if (vp8_info.temporalIdx == 1 && vp8_info.layerSync)
@ -265,9 +273,9 @@ TEST_F(ScreenshareLayerTest, 2LayersSyncAfterSimilarQP) {
// Simulate TL1 being at least 8 qp steps better. // Simulate TL1 being at least 8 qp steps better.
if (vp8_info.temporalIdx == 0) { if (vp8_info.temporalIdx == 0) {
layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp); layers_->FrameEncoded(frame_size_, kDefaultQp);
} else { } else {
layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp - 8); layers_->FrameEncoded(frame_size_, kDefaultQp - 8);
} }
if (vp8_info.temporalIdx == 1 && vp8_info.layerSync) if (vp8_info.temporalIdx == 1 && vp8_info.layerSync)
@ -284,13 +292,13 @@ TEST_F(ScreenshareLayerTest, 2LayersSyncAfterSimilarQP) {
if (vp8_info.temporalIdx == 0) { if (vp8_info.temporalIdx == 0) {
// Bump TL0 to same quality as TL1. // Bump TL0 to same quality as TL1.
layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp - 8); layers_->FrameEncoded(frame_size_, kDefaultQp - 8);
bumped_tl0_quality = true; bumped_tl0_quality = true;
} else { } else {
layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp - 8); layers_->FrameEncoded(frame_size_, kDefaultQp - 8);
if (bumped_tl0_quality) { if (bumped_tl0_quality) {
EXPECT_TRUE(vp8_info.layerSync); EXPECT_TRUE(vp8_info.layerSync);
EXPECT_EQ(ScreenshareLayers::kTl1SyncFlags, flags); EXPECT_EQ(kTl1SyncFlags, flags);
return; return;
} }
} }
@ -336,7 +344,7 @@ TEST_F(ScreenshareLayerTest, AllFitsLayer0) {
for (int i = 0; i < 50; ++i) { for (int i = 0; i < 50; ++i) {
EncodeFrame(timestamp, false, &vp8_info, &flags); EncodeFrame(timestamp, false, &vp8_info, &flags);
timestamp += kTimestampDelta5Fps; timestamp += kTimestampDelta5Fps;
EXPECT_EQ(ScreenshareLayers::kTl0Flags, flags); EXPECT_EQ(kTl0Flags, flags);
EXPECT_EQ(0, vp8_info.temporalIdx); EXPECT_EQ(0, vp8_info.temporalIdx);
} }
} }
@ -425,46 +433,46 @@ TEST_F(ScreenshareLayerTest, EncoderDrop) {
timestamp = SkipUntilTl(0, timestamp); timestamp = SkipUntilTl(0, timestamp);
// Size 0 indicates dropped frame. // Size 0 indicates dropped frame.
layers_->FrameEncoded(0, timestamp, kDefaultQp); layers_->FrameEncoded(0, kDefaultQp);
timestamp += kTimestampDelta5Fps; timestamp += kTimestampDelta5Fps;
EXPECT_FALSE(layers_->UpdateConfiguration(&cfg)); EXPECT_FALSE(layers_->UpdateConfiguration(&cfg));
EXPECT_EQ(ScreenshareLayers::kTl0Flags, layers_->EncodeFlags(timestamp)); EXPECT_EQ(kTl0Flags, layers_->EncodeFlags(timestamp));
layers_->PopulateCodecSpecific(false, &vp8_info, timestamp); layers_->PopulateCodecSpecific(false, &vp8_info, timestamp);
layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp); layers_->FrameEncoded(frame_size_, kDefaultQp);
timestamp = SkipUntilTl(0, timestamp); timestamp = SkipUntilTl(0, timestamp);
EXPECT_TRUE(layers_->UpdateConfiguration(&cfg)); EXPECT_TRUE(layers_->UpdateConfiguration(&cfg));
EXPECT_LT(cfg.rc_max_quantizer, static_cast<unsigned int>(kDefaultQp)); EXPECT_LT(cfg.rc_max_quantizer, static_cast<unsigned int>(kDefaultQp));
layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp); layers_->FrameEncoded(frame_size_, kDefaultQp);
layers_->EncodeFlags(timestamp); layers_->EncodeFlags(timestamp);
timestamp += kTimestampDelta5Fps; timestamp += kTimestampDelta5Fps;
EXPECT_TRUE(layers_->UpdateConfiguration(&cfg)); EXPECT_TRUE(layers_->UpdateConfiguration(&cfg));
layers_->PopulateCodecSpecific(false, &vp8_info, timestamp); layers_->PopulateCodecSpecific(false, &vp8_info, timestamp);
EXPECT_EQ(cfg.rc_max_quantizer, static_cast<unsigned int>(kDefaultQp)); EXPECT_EQ(cfg.rc_max_quantizer, static_cast<unsigned int>(kDefaultQp));
layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp); layers_->FrameEncoded(frame_size_, kDefaultQp);
// Next drop in TL1. // Next drop in TL1.
timestamp = SkipUntilTl(1, timestamp); timestamp = SkipUntilTl(1, timestamp);
layers_->FrameEncoded(0, timestamp, kDefaultQp); layers_->FrameEncoded(0, kDefaultQp);
timestamp += kTimestampDelta5Fps; timestamp += kTimestampDelta5Fps;
EXPECT_FALSE(layers_->UpdateConfiguration(&cfg)); EXPECT_FALSE(layers_->UpdateConfiguration(&cfg));
EXPECT_EQ(ScreenshareLayers::kTl1Flags, layers_->EncodeFlags(timestamp)); EXPECT_EQ(kTl1Flags, layers_->EncodeFlags(timestamp));
layers_->PopulateCodecSpecific(false, &vp8_info, timestamp); layers_->PopulateCodecSpecific(false, &vp8_info, timestamp);
layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp); layers_->FrameEncoded(frame_size_, kDefaultQp);
timestamp = SkipUntilTl(1, timestamp); timestamp = SkipUntilTl(1, timestamp);
EXPECT_TRUE(layers_->UpdateConfiguration(&cfg)); EXPECT_TRUE(layers_->UpdateConfiguration(&cfg));
EXPECT_LT(cfg.rc_max_quantizer, static_cast<unsigned int>(kDefaultQp)); EXPECT_LT(cfg.rc_max_quantizer, static_cast<unsigned int>(kDefaultQp));
layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp); layers_->FrameEncoded(frame_size_, kDefaultQp);
layers_->EncodeFlags(timestamp); layers_->EncodeFlags(timestamp);
timestamp += kTimestampDelta5Fps; timestamp += kTimestampDelta5Fps;
EXPECT_TRUE(layers_->UpdateConfiguration(&cfg)); EXPECT_TRUE(layers_->UpdateConfiguration(&cfg));
layers_->PopulateCodecSpecific(false, &vp8_info, timestamp); layers_->PopulateCodecSpecific(false, &vp8_info, timestamp);
EXPECT_EQ(cfg.rc_max_quantizer, static_cast<unsigned int>(kDefaultQp)); EXPECT_EQ(cfg.rc_max_quantizer, static_cast<unsigned int>(kDefaultQp));
layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp); layers_->FrameEncoded(frame_size_, kDefaultQp);
} }
TEST_F(ScreenshareLayerTest, RespectsMaxIntervalBetweenFrames) { TEST_F(ScreenshareLayerTest, RespectsMaxIntervalBetweenFrames) {
@ -476,9 +484,8 @@ TEST_F(ScreenshareLayerTest, RespectsMaxIntervalBetweenFrames) {
layers_->OnRatesUpdated(kLowBitrateKbps, kLowBitrateKbps, 5); layers_->OnRatesUpdated(kLowBitrateKbps, kLowBitrateKbps, 5);
layers_->UpdateConfiguration(&cfg); layers_->UpdateConfiguration(&cfg);
EXPECT_EQ(ScreenshareLayers::kTl0Flags, EXPECT_EQ(kTl0Flags, layers_->EncodeFlags(kStartTimestamp));
layers_->EncodeFlags(kStartTimestamp)); layers_->FrameEncoded(kLargeFrameSizeBytes, kDefaultQp);
layers_->FrameEncoded(kLargeFrameSizeBytes, kStartTimestamp, kDefaultQp);
const uint32_t kTwoSecondsLater = const uint32_t kTwoSecondsLater =
kStartTimestamp + (ScreenshareLayers::kMaxFrameIntervalMs * 90); kStartTimestamp + (ScreenshareLayers::kMaxFrameIntervalMs * 90);
@ -490,8 +497,7 @@ TEST_F(ScreenshareLayerTest, RespectsMaxIntervalBetweenFrames) {
EXPECT_EQ(-1, layers_->EncodeFlags(kTwoSecondsLater)); EXPECT_EQ(-1, layers_->EncodeFlags(kTwoSecondsLater));
// More than two seconds has passed since last frame, one should be emitted // More than two seconds has passed since last frame, one should be emitted
// even if bitrate target is then exceeded. // even if bitrate target is then exceeded.
EXPECT_EQ(ScreenshareLayers::kTl0Flags, EXPECT_EQ(kTl0Flags, layers_->EncodeFlags(kTwoSecondsLater + 90));
layers_->EncodeFlags(kTwoSecondsLater + 90));
} }
TEST_F(ScreenshareLayerTest, UpdatesHistograms) { TEST_F(ScreenshareLayerTest, UpdatesHistograms) {
@ -512,22 +518,21 @@ TEST_F(ScreenshareLayerTest, UpdatesHistograms) {
if (timestamp >= kTimestampDelta5Fps * 5 && !overshoot && flags != -1) { if (timestamp >= kTimestampDelta5Fps * 5 && !overshoot && flags != -1) {
// Simulate one overshoot. // Simulate one overshoot.
layers_->FrameEncoded(0, timestamp, 0); layers_->FrameEncoded(0, 0);
overshoot = true; overshoot = true;
flags = layers_->EncodeFlags(timestamp); flags = layers_->EncodeFlags(timestamp);
} }
if (flags == ScreenshareLayers::kTl0Flags) { if (flags == kTl0Flags) {
if (timestamp >= kTimestampDelta5Fps * 20 && !trigger_drop) { if (timestamp >= kTimestampDelta5Fps * 20 && !trigger_drop) {
// Simulate a too large frame, to cause frame drop. // Simulate a too large frame, to cause frame drop.
layers_->FrameEncoded(frame_size_ * 5, timestamp, kTl0Qp); layers_->FrameEncoded(frame_size_ * 5, kTl0Qp);
trigger_drop = true; trigger_drop = true;
} else { } else {
layers_->FrameEncoded(frame_size_, timestamp, kTl0Qp); layers_->FrameEncoded(frame_size_, kTl0Qp);
} }
} else if (flags == ScreenshareLayers::kTl1Flags || } else if (flags == kTl1Flags || flags == kTl1SyncFlags) {
flags == ScreenshareLayers::kTl1SyncFlags) { layers_->FrameEncoded(frame_size_, kTl1Qp);
layers_->FrameEncoded(frame_size_, timestamp, kTl1Qp);
} else if (flags == -1) { } else if (flags == -1) {
dropped_frame = true; dropped_frame = true;
} else { } else {
@ -593,7 +598,7 @@ TEST_F(ScreenshareLayerTest, RespectsConfiguredFramerate) {
++num_discarded_frames; ++num_discarded_frames;
} else { } else {
size_t frame_size_bytes = kDefaultTl0BitrateKbps * kFrameIntervalsMs / 8; size_t frame_size_bytes = kDefaultTl0BitrateKbps * kFrameIntervalsMs / 8;
layers_->FrameEncoded(frame_size_bytes, timestamp, kDefaultQp); layers_->FrameEncoded(frame_size_bytes, kDefaultQp);
} }
timestamp += kFrameIntervalsMs * 90; timestamp += kFrameIntervalsMs * 90;
clock_.AdvanceTimeMilliseconds(kFrameIntervalsMs); clock_.AdvanceTimeMilliseconds(kFrameIntervalsMs);
@ -609,7 +614,7 @@ TEST_F(ScreenshareLayerTest, RespectsConfiguredFramerate) {
++num_discarded_frames; ++num_discarded_frames;
} else { } else {
size_t frame_size_bytes = kDefaultTl0BitrateKbps * kFrameIntervalsMs / 8; size_t frame_size_bytes = kDefaultTl0BitrateKbps * kFrameIntervalsMs / 8;
layers_->FrameEncoded(frame_size_bytes, timestamp, kDefaultQp); layers_->FrameEncoded(frame_size_bytes, kDefaultQp);
} }
timestamp += kFrameIntervalsMs * 90 / 2; timestamp += kFrameIntervalsMs * 90 / 2;
clock_.AdvanceTimeMilliseconds(kFrameIntervalsMs / 2); clock_.AdvanceTimeMilliseconds(kFrameIntervalsMs / 2);

45
webrtc/modules/video_coding/codecs/vp8/temporal_layers.h
View File

@ -24,6 +24,43 @@ namespace webrtc {
struct CodecSpecificInfoVP8; struct CodecSpecificInfoVP8;
enum TemporalBufferFlags {
kNone = 0,
kReference = 1,
kUpdate = 2,
kReferenceAndUpdate = kReference | kUpdate,
};
enum TemporalFlags { kLayerSync = 1, kFreezeEntropy = 2 };
struct TemporalReferences {
TemporalReferences(TemporalBufferFlags last,
TemporalBufferFlags golden,
TemporalBufferFlags arf);
TemporalReferences(TemporalBufferFlags last,
TemporalBufferFlags golden,
TemporalBufferFlags arf,
int extra_flags);
const bool drop_frame;
const TemporalBufferFlags last_buffer_flags;
const TemporalBufferFlags golden_buffer_flags;
const TemporalBufferFlags arf_buffer_flags;
// TODO(pbos): Consider breaking these out of here and returning only a
// pattern index that needs to be returned to fill CodecSpecificInfoVP8 or
// EncodeFlags.
const bool layer_sync;
const bool freeze_entropy;
private:
TemporalReferences(TemporalBufferFlags last,
TemporalBufferFlags golden,
TemporalBufferFlags arf,
bool layer_sync,
bool freeze_entropy);
};
class TemporalLayers { class TemporalLayers {
public: public:
// Factory for TemporalLayer strategy. Default behavior is a fixed pattern // Factory for TemporalLayer strategy. Default behavior is a fixed pattern
@ -32,7 +69,9 @@ class TemporalLayers {
// Returns the recommended VP8 encode flags needed. May refresh the decoder // Returns the recommended VP8 encode flags needed. May refresh the decoder
// and/or update the reference buffers. // and/or update the reference buffers.
virtual int EncodeFlags(uint32_t timestamp) = 0; virtual TemporalReferences UpdateLayerConfig(uint32_t timestamp) = 0;
int EncodeFlags(uint32_t timestamp);
// Update state based on new bitrate target and incoming framerate. // Update state based on new bitrate target and incoming framerate.
// Returns the bitrate allocation for the active temporal layers. // Returns the bitrate allocation for the active temporal layers.
@ -44,11 +83,11 @@ class TemporalLayers {
// Returns true iff the configuration was actually modified. // Returns true iff the configuration was actually modified.
virtual bool UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) = 0; virtual bool UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) = 0;
virtual void PopulateCodecSpecific(bool base_layer_sync, virtual void PopulateCodecSpecific(bool is_keyframe,
CodecSpecificInfoVP8* vp8_info, CodecSpecificInfoVP8* vp8_info,
uint32_t timestamp) = 0; uint32_t timestamp) = 0;
virtual void FrameEncoded(unsigned int size, uint32_t timestamp, int qp) = 0; virtual void FrameEncoded(unsigned int size, int qp) = 0;
virtual int CurrentLayerId() const = 0; virtual int CurrentLayerId() const = 0;
}; };

11
webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc
View File

@ -812,11 +812,9 @@ void VP8EncoderImpl::PopulateCodecSpecific(
} }
vp8Info->simulcastIdx = stream_idx; vp8Info->simulcastIdx = stream_idx;
vp8Info->keyIdx = kNoKeyIdx; // TODO(hlundin) populate this vp8Info->keyIdx = kNoKeyIdx; // TODO(hlundin) populate this
vp8Info->nonReference = vp8Info->nonReference = (pkt.data.frame.flags & VPX_FRAME_IS_DROPPABLE) != 0;
(pkt.data.frame.flags & VPX_FRAME_IS_DROPPABLE) ? true : false; temporal_layers_[stream_idx]->PopulateCodecSpecific(
bool base_layer_sync_point = pkt.data.frame.flags & VPX_FRAME_IS_KEY; (pkt.data.frame.flags & VPX_FRAME_IS_KEY) != 0, vp8Info, timestamp);
temporal_layers_[stream_idx]->PopulateCodecSpecific(base_layer_sync_point,
vp8Info, timestamp);
// Prepare next. // Prepare next.
picture_id_[stream_idx] = (picture_id_[stream_idx] + 1) & 0x7FFF; picture_id_[stream_idx] = (picture_id_[stream_idx] + 1) & 0x7FFF;
} }
@ -884,8 +882,7 @@ int VP8EncoderImpl::GetEncodedPartitions(const VideoFrame& input_image) {
int qp = -1; int qp = -1;
vpx_codec_control(&encoders_[encoder_idx], VP8E_GET_LAST_QUANTIZER_64, &qp); vpx_codec_control(&encoders_[encoder_idx], VP8E_GET_LAST_QUANTIZER_64, &qp);
temporal_layers_[stream_idx]->FrameEncoded( temporal_layers_[stream_idx]->FrameEncoded(
encoded_images_[encoder_idx]._length, encoded_images_[encoder_idx]._length, qp);
encoded_images_[encoder_idx]._timeStamp, qp);
if (send_stream_[stream_idx]) { if (send_stream_[stream_idx]) {
if (encoded_images_[encoder_idx]._length > 0) { if (encoded_images_[encoder_idx]._length > 0) {
TRACE_COUNTER_ID1("webrtc", "EncodedFrameSize", encoder_idx, TRACE_COUNTER_ID1("webrtc", "EncodedFrameSize", encoder_idx,

4
webrtc/modules/video_coding/utility/simulcast_rate_allocator_unittest.cc
View File

@ -29,12 +29,12 @@ constexpr uint32_t kFramerateFps = 5;
class MockTemporalLayers : public TemporalLayers { class MockTemporalLayers : public TemporalLayers {
public: public:
MOCK_METHOD1(EncodeFlags, int(uint32_t)); MOCK_METHOD1(UpdateLayerConfig, TemporalReferences(uint32_t));
MOCK_METHOD3(OnRatesUpdated, std::vector<uint32_t>(int, int, int)); MOCK_METHOD3(OnRatesUpdated, std::vector<uint32_t>(int, int, int));
MOCK_METHOD1(UpdateConfiguration, bool(vpx_codec_enc_cfg_t*)); MOCK_METHOD1(UpdateConfiguration, bool(vpx_codec_enc_cfg_t*));
MOCK_METHOD3(PopulateCodecSpecific, MOCK_METHOD3(PopulateCodecSpecific,
void(bool, CodecSpecificInfoVP8*, uint32_t)); void(bool, CodecSpecificInfoVP8*, uint32_t));
MOCK_METHOD3(FrameEncoded, void(unsigned int, uint32_t, int)); MOCK_METHOD2(FrameEncoded, void(unsigned int, int));
MOCK_CONST_METHOD0(CurrentLayerId, int()); MOCK_CONST_METHOD0(CurrentLayerId, int());
}; };
} // namespace } // namespace