diff --git a/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.cc b/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.cc index 2af8f41cd7..e2246b06ad 100644 --- a/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.cc +++ b/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.cc @@ -25,14 +25,14 @@ namespace webrtc { -TemporalReferences::TemporalReferences(TemporalBufferUsage last, - TemporalBufferUsage golden, - TemporalBufferUsage arf) +TemporalReferences::TemporalReferences(TemporalBufferFlags last, + TemporalBufferFlags golden, + TemporalBufferFlags arf) : TemporalReferences(last, golden, arf, false, false) {} -TemporalReferences::TemporalReferences(TemporalBufferUsage last, - TemporalBufferUsage golden, - TemporalBufferUsage arf, +TemporalReferences::TemporalReferences(TemporalBufferFlags last, + TemporalBufferFlags golden, + TemporalBufferFlags arf, int extra_flags) : TemporalReferences(last, golden, @@ -40,17 +40,15 @@ TemporalReferences::TemporalReferences(TemporalBufferUsage last, (extra_flags & kLayerSync) != 0, (extra_flags & kFreezeEntropy) != 0) {} -TemporalReferences::TemporalReferences(TemporalBufferUsage last, - TemporalBufferUsage golden, - TemporalBufferUsage arf, +TemporalReferences::TemporalReferences(TemporalBufferFlags last, + TemporalBufferFlags golden, + TemporalBufferFlags arf, bool layer_sync, bool freeze_entropy) - : reference_last((last & kReference) != 0), - update_last((last & kUpdate) != 0), - reference_golden((golden & kReference) != 0), - update_golden((golden & kUpdate) != 0), - reference_arf((arf & kReference) != 0), - update_arf((arf & kUpdate) != 0), + : drop_frame(last == kNone && golden == kNone && arf == kNone), + last_buffer_flags(last), + golden_buffer_flags(golden), + arf_buffer_flags(arf), layer_sync(layer_sync), freeze_entropy(freeze_entropy) {} @@ -246,26 +244,32 @@ bool DefaultTemporalLayers::UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) { return true; } -int DefaultTemporalLayers::EncodeFlags(uint32_t timestamp) { +// TODO(pbos): Name method so that it's obvious that it updates state. +TemporalReferences DefaultTemporalLayers::UpdateLayerConfig( + uint32_t timestamp) { RTC_DCHECK_GT(num_layers_, 0); RTC_DCHECK_LT(0, temporal_pattern_.size()); - int flags = 0; - // TODO(pbos): Move pattern-update out of EncodeFlags. It's not obvious that - // EncodeFlags() is non-const. - const TemporalReferences& references = - temporal_pattern_[++pattern_idx_ % temporal_pattern_.size()]; + return temporal_pattern_[++pattern_idx_ % temporal_pattern_.size()]; +} - if (!references.reference_last) +int TemporalLayers::EncodeFlags(uint32_t timestamp) { + TemporalReferences references = UpdateLayerConfig(timestamp); + if (references.drop_frame) + return -1; + + int flags = 0; + + if ((references.last_buffer_flags & kReference) == 0) flags |= VP8_EFLAG_NO_REF_LAST; - if (!references.update_last) + if ((references.last_buffer_flags & kUpdate) == 0) flags |= VP8_EFLAG_NO_UPD_LAST; - if (!references.reference_golden) + if ((references.golden_buffer_flags & kReference) == 0) flags |= VP8_EFLAG_NO_REF_GF; - if (!references.update_golden) + if ((references.golden_buffer_flags & kUpdate) == 0) flags |= VP8_EFLAG_NO_UPD_GF; - if (!references.reference_arf) + if ((references.arf_buffer_flags & kReference) == 0) flags |= VP8_EFLAG_NO_REF_ARF; - if (!references.update_arf) + if ((references.arf_buffer_flags & kUpdate) == 0) flags |= VP8_EFLAG_NO_UPD_ARF; if (references.freeze_entropy) flags |= VP8_EFLAG_NO_UPD_ENTROPY; @@ -274,7 +278,7 @@ int DefaultTemporalLayers::EncodeFlags(uint32_t timestamp) { } void DefaultTemporalLayers::PopulateCodecSpecific( - bool base_layer_sync, + bool frame_is_keyframe, CodecSpecificInfoVP8* vp8_info, uint32_t timestamp) { RTC_DCHECK_GT(num_layers_, 0); @@ -284,7 +288,7 @@ void DefaultTemporalLayers::PopulateCodecSpecific( vp8_info->layerSync = false; vp8_info->tl0PicIdx = kNoTl0PicIdx; } else { - if (base_layer_sync) { + if (frame_is_keyframe) { vp8_info->temporalIdx = 0; vp8_info->layerSync = true; } else { @@ -303,7 +307,7 @@ void DefaultTemporalLayers::PopulateCodecSpecific( timestamp_ = timestamp; tl0_pic_idx_++; } - last_base_layer_sync_ = base_layer_sync; + last_base_layer_sync_ = frame_is_keyframe; vp8_info->tl0PicIdx = tl0_pic_idx_; } } diff --git a/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.h b/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.h index 446d162c3e..45096e0c0c 100644 --- a/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.h +++ b/webrtc/modules/video_coding/codecs/vp8/default_temporal_layers.h @@ -20,44 +20,6 @@ namespace webrtc { -enum TemporalBufferUsage { - kNone = 0, - kReference = 1, - kUpdate = 2, - kReferenceAndUpdate = kReference | kUpdate, -}; -enum TemporalFlags { kLayerSync = 1, kFreezeEntropy = 2 }; - -struct TemporalReferences { - TemporalReferences(TemporalBufferUsage last, - TemporalBufferUsage golden, - TemporalBufferUsage arf); - TemporalReferences(TemporalBufferUsage last, - TemporalBufferUsage golden, - TemporalBufferUsage arf, - int extra_flags); - - const bool reference_last; - const bool update_last; - const bool reference_golden; - const bool update_golden; - const bool reference_arf; - const bool update_arf; - - // TODO(pbos): Consider breaking these out of here and returning only a - // pattern index that needs to be returned to fill CodecSpecificInfoVP8 or - // EncodeFlags. - const bool layer_sync; - const bool freeze_entropy; - - private: - TemporalReferences(TemporalBufferUsage last, - TemporalBufferUsage golden, - TemporalBufferUsage arf, - bool layer_sync, - bool freeze_entropy); -}; - class DefaultTemporalLayers : public TemporalLayers { public: DefaultTemporalLayers(int number_of_temporal_layers, @@ -66,7 +28,7 @@ class DefaultTemporalLayers : public TemporalLayers { // Returns the recommended VP8 encode flags needed. May refresh the decoder // and/or update the reference buffers. - int EncodeFlags(uint32_t timestamp) override; + TemporalReferences UpdateLayerConfig(uint32_t timestamp) override; // Update state based on new bitrate target and incoming framerate. // Returns the bitrate allocation for the active temporal layers. @@ -76,11 +38,11 @@ class DefaultTemporalLayers : public TemporalLayers { bool UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) override; - void PopulateCodecSpecific(bool base_layer_sync, + void PopulateCodecSpecific(bool frame_is_keyframe, CodecSpecificInfoVP8* vp8_info, uint32_t timestamp) override; - void FrameEncoded(unsigned int size, uint32_t timestamp, int qp) override {} + void FrameEncoded(unsigned int size, int qp) override {} int CurrentLayerId() const override; diff --git a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc index 05b81b8440..6d9511f9dd 100644 --- a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc +++ b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.cc @@ -32,23 +32,6 @@ const double ScreenshareLayers::kAcceptableTargetOvershoot = 2.0; constexpr int ScreenshareLayers::kMaxNumTemporalLayers; -// Since this is TL0 we only allow updating and predicting from the LAST -// reference frame. -const int ScreenshareLayers::kTl0Flags = - VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_REF_GF | - VP8_EFLAG_NO_REF_ARF; - -// Allow predicting from both TL0 and TL1. -const int ScreenshareLayers::kTl1Flags = - VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST; - -// Allow predicting from only TL0 to allow participants to switch to the high -// bitrate stream. This means predicting only from the LAST reference frame, but -// only updating GF to not corrupt TL0. -const int ScreenshareLayers::kTl1SyncFlags = - VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_UPD_ARF | - VP8_EFLAG_NO_UPD_LAST; - // Always emit a frame with certain interval, even if bitrate targets have // been exceeded. const int ScreenshareLayers::kMaxFrameIntervalMs = 2000; @@ -100,24 +83,27 @@ int ScreenshareLayers::CurrentLayerId() const { return 0; } -int ScreenshareLayers::EncodeFlags(uint32_t timestamp) { +TemporalReferences ScreenshareLayers::UpdateLayerConfig(uint32_t timestamp) { if (number_of_temporal_layers_ <= 1) { // No flags needed for 1 layer screenshare. - return 0; + // TODO(pbos): Consider updating only last, and not all buffers. + return TemporalReferences(kReferenceAndUpdate, kReferenceAndUpdate, + kReferenceAndUpdate); } const int64_t now_ms = clock_->TimeInMilliseconds(); if (target_framerate_.value_or(0) > 0 && encode_framerate_.Rate(now_ms).value_or(0) > *target_framerate_) { // Max framerate exceeded, drop frame. - return -1; + return TemporalReferences(kNone, kNone, kNone); } if (stats_.first_frame_time_ms_ == -1) stats_.first_frame_time_ms_ = now_ms; int64_t unwrapped_timestamp = time_wrap_handler_.Unwrap(timestamp); - int flags = 0; + enum TemporalLayerState { kDrop, kTl0, kTl1, kTl1Sync }; + enum TemporalLayerState layer_state = kDrop; if (active_layer_ == -1 || layers_[active_layer_].state != TemporalLayer::State::kDropped) { if (last_emitted_tl0_timestamp_ != -1 && @@ -142,23 +128,22 @@ int ScreenshareLayers::EncodeFlags(uint32_t timestamp) { switch (active_layer_) { case 0: - flags = kTl0Flags; + layer_state = kTl0; last_emitted_tl0_timestamp_ = unwrapped_timestamp; break; case 1: if (TimeToSync(unwrapped_timestamp)) { last_sync_timestamp_ = unwrapped_timestamp; - flags = kTl1SyncFlags; + layer_state = kTl1Sync; } else { - flags = kTl1Flags; + layer_state = kTl1; } break; case -1: - flags = -1; + layer_state = kDrop; ++stats_.num_dropped_frames_; break; default: - flags = -1; RTC_NOTREACHED(); } @@ -172,7 +157,25 @@ int ScreenshareLayers::EncodeFlags(uint32_t timestamp) { layers_[0].UpdateDebt(ts_diff / 90); layers_[1].UpdateDebt(ts_diff / 90); last_timestamp_ = timestamp; - return flags; + // TODO(pbos): Consider referencing but not updating the 'alt' buffer for all + // layers. + switch (layer_state) { + case kDrop: + return TemporalReferences(kNone, kNone, kNone); + case kTl0: + // TL0 only references and updates 'last'. + return TemporalReferences(kReferenceAndUpdate, kNone, kNone); + case kTl1: + // TL1 references both 'last' and 'golden' but only updates 'golden'. + return TemporalReferences(kReference, kReferenceAndUpdate, kNone); + case kTl1Sync: + // Predict from only TL0 to allow participants to switch to the high + // bitrate stream. Updates 'golden' so that TL1 can continue to refer to + // and update 'golden' from this point on. + return TemporalReferences(kReference, kUpdate, kNone, kLayerSync); + } + RTC_NOTREACHED(); + return TemporalReferences(kNone, kNone, kNone); } std::vector ScreenshareLayers::OnRatesUpdated(int bitrate_kbps, @@ -208,9 +211,7 @@ std::vector ScreenshareLayers::OnRatesUpdated(int bitrate_kbps, return allocation; } -void ScreenshareLayers::FrameEncoded(unsigned int size, - uint32_t timestamp, - int qp) { +void ScreenshareLayers::FrameEncoded(unsigned int size, int qp) { if (size > 0) encode_framerate_.Update(1, clock_->TimeInMilliseconds()); @@ -245,7 +246,7 @@ void ScreenshareLayers::FrameEncoded(unsigned int size, } } -void ScreenshareLayers::PopulateCodecSpecific(bool base_layer_sync, +void ScreenshareLayers::PopulateCodecSpecific(bool frame_is_keyframe, CodecSpecificInfoVP8* vp8_info, uint32_t timestamp) { int64_t unwrapped_timestamp = time_wrap_handler_.Unwrap(timestamp); @@ -256,7 +257,7 @@ void ScreenshareLayers::PopulateCodecSpecific(bool base_layer_sync, } else { RTC_DCHECK_NE(-1, active_layer_); vp8_info->temporalIdx = active_layer_; - if (base_layer_sync) { + if (frame_is_keyframe) { vp8_info->temporalIdx = 0; last_sync_timestamp_ = unwrapped_timestamp; } else if (last_base_layer_sync_ && vp8_info->temporalIdx != 0) { @@ -269,7 +270,7 @@ void ScreenshareLayers::PopulateCodecSpecific(bool base_layer_sync, if (vp8_info->temporalIdx == 0) { tl0_pic_idx_++; } - last_base_layer_sync_ = base_layer_sync; + last_base_layer_sync_ = frame_is_keyframe; vp8_info->tl0PicIdx = tl0_pic_idx_; } } diff --git a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h index 53818a2a97..27f641acad 100644 --- a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h +++ b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers.h @@ -26,9 +26,6 @@ class ScreenshareLayers : public TemporalLayers { public: static const double kMaxTL0FpsReduction; static const double kAcceptableTargetOvershoot; - static const int kTl0Flags; - static const int kTl1Flags; - static const int kTl1SyncFlags; static const int kMaxFrameIntervalMs; ScreenshareLayers(int num_temporal_layers, @@ -38,7 +35,7 @@ class ScreenshareLayers : public TemporalLayers { // Returns the recommended VP8 encode flags needed. May refresh the decoder // and/or update the reference buffers. - int EncodeFlags(uint32_t timestamp) override; + TemporalReferences UpdateLayerConfig(uint32_t timestamp) override; // Update state based on new bitrate target and incoming framerate. // Returns the bitrate allocation for the active temporal layers. @@ -54,7 +51,7 @@ class ScreenshareLayers : public TemporalLayers { CodecSpecificInfoVP8* vp8_info, uint32_t timestamp) override; - void FrameEncoded(unsigned int size, uint32_t timestamp, int qp) override; + void FrameEncoded(unsigned int size, int qp) override; int CurrentLayerId() const override; diff --git a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc index cbbf1e33ad..8a227a23fa 100644 --- a/webrtc/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc +++ b/webrtc/modules/video_coding/codecs/vp8/screenshare_layers_unittest.cc @@ -37,6 +37,14 @@ const int kFrameRate = 5; const int kSyncPeriodSeconds = 5; const int kMaxSyncPeriodSeconds = 10; +// Expected flags for corresponding temporal layers. +const int kTl0Flags = VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF | + VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_REF_ARF; +const int kTl1Flags = + VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST; +const int kTl1SyncFlags = VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_REF_GF | + VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST; + class ScreenshareLayerTest : public ::testing::Test { protected: ScreenshareLayerTest() @@ -54,7 +62,7 @@ class ScreenshareLayerTest : public ::testing::Test { return; layers_->PopulateCodecSpecific(base_sync, vp8_info, timestamp); ASSERT_NE(-1, frame_size_); - layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp); + layers_->FrameEncoded(frame_size_, kDefaultQp); } void ConfigureBitrates() { @@ -103,7 +111,7 @@ class ScreenshareLayerTest : public ::testing::Test { timestamp += kTimestampDelta5Fps; layers_->PopulateCodecSpecific(false, &vp8_info, timestamp); if (vp8_info.temporalIdx != layer) { - layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp); + layers_->FrameEncoded(frame_size_, kDefaultQp); } else { return timestamp; } @@ -142,7 +150,7 @@ TEST_F(ScreenshareLayerTest, 1Layer) { EXPECT_EQ(static_cast(kNoTemporalIdx), vp8_info.temporalIdx); EXPECT_FALSE(vp8_info.layerSync); EXPECT_EQ(kNoTl0PicIdx, vp8_info.tl0PicIdx); - layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp); + layers_->FrameEncoded(frame_size_, kDefaultQp); flags = layers_->EncodeFlags(timestamp); EXPECT_EQ(kSingleLayerFlags, flags); timestamp += kTimestampDelta5Fps; @@ -150,7 +158,7 @@ TEST_F(ScreenshareLayerTest, 1Layer) { EXPECT_EQ(static_cast(kNoTemporalIdx), vp8_info.temporalIdx); EXPECT_FALSE(vp8_info.layerSync); EXPECT_EQ(kNoTl0PicIdx, vp8_info.tl0PicIdx); - layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp); + layers_->FrameEncoded(frame_size_, kDefaultQp); } TEST_F(ScreenshareLayerTest, 2Layer) { @@ -160,7 +168,7 @@ TEST_F(ScreenshareLayerTest, 2Layer) { uint8_t expected_tl0_idx = 0; CodecSpecificInfoVP8 vp8_info; EncodeFrame(timestamp, false, &vp8_info, &flags); - EXPECT_EQ(ScreenshareLayers::kTl0Flags, flags); + EXPECT_EQ(kTl0Flags, flags); EXPECT_EQ(0, vp8_info.temporalIdx); EXPECT_FALSE(vp8_info.layerSync); ++expected_tl0_idx; @@ -179,7 +187,7 @@ TEST_F(ScreenshareLayerTest, 2Layer) { // First frame in TL0. timestamp += kTimestampDelta5Fps; EncodeFrame(timestamp, false, &vp8_info, &flags); - EXPECT_EQ(ScreenshareLayers::kTl0Flags, flags); + EXPECT_EQ(kTl0Flags, flags); EXPECT_EQ(0, vp8_info.temporalIdx); EXPECT_FALSE(vp8_info.layerSync); ++expected_tl0_idx; @@ -189,14 +197,14 @@ TEST_F(ScreenshareLayerTest, 2Layer) { timestamp += kTimestampDelta5Fps; EncodeFrame(timestamp, false, &vp8_info, &flags); // First frame is sync frame. - EXPECT_EQ(ScreenshareLayers::kTl1SyncFlags, flags); + EXPECT_EQ(kTl1SyncFlags, flags); EXPECT_EQ(1, vp8_info.temporalIdx); EXPECT_TRUE(vp8_info.layerSync); EXPECT_EQ(expected_tl0_idx, vp8_info.tl0PicIdx); timestamp += kTimestampDelta5Fps; EncodeFrame(timestamp, false, &vp8_info, &flags); - EXPECT_EQ(ScreenshareLayers::kTl1Flags, flags); + EXPECT_EQ(kTl1Flags, flags); EXPECT_EQ(1, vp8_info.temporalIdx); EXPECT_FALSE(vp8_info.layerSync); EXPECT_EQ(expected_tl0_idx, vp8_info.tl0PicIdx); @@ -236,9 +244,9 @@ TEST_F(ScreenshareLayerTest, 2LayersSyncAfterTimeout) { // Simulate TL1 being at least 8 qp steps better. if (vp8_info.temporalIdx == 0) { - layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp); + layers_->FrameEncoded(frame_size_, kDefaultQp); } else { - layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp - 8); + layers_->FrameEncoded(frame_size_, kDefaultQp - 8); } if (vp8_info.temporalIdx == 1 && vp8_info.layerSync) @@ -265,9 +273,9 @@ TEST_F(ScreenshareLayerTest, 2LayersSyncAfterSimilarQP) { // Simulate TL1 being at least 8 qp steps better. if (vp8_info.temporalIdx == 0) { - layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp); + layers_->FrameEncoded(frame_size_, kDefaultQp); } else { - layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp - 8); + layers_->FrameEncoded(frame_size_, kDefaultQp - 8); } if (vp8_info.temporalIdx == 1 && vp8_info.layerSync) @@ -284,13 +292,13 @@ TEST_F(ScreenshareLayerTest, 2LayersSyncAfterSimilarQP) { if (vp8_info.temporalIdx == 0) { // Bump TL0 to same quality as TL1. - layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp - 8); + layers_->FrameEncoded(frame_size_, kDefaultQp - 8); bumped_tl0_quality = true; } else { - layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp - 8); + layers_->FrameEncoded(frame_size_, kDefaultQp - 8); if (bumped_tl0_quality) { EXPECT_TRUE(vp8_info.layerSync); - EXPECT_EQ(ScreenshareLayers::kTl1SyncFlags, flags); + EXPECT_EQ(kTl1SyncFlags, flags); return; } } @@ -336,7 +344,7 @@ TEST_F(ScreenshareLayerTest, AllFitsLayer0) { for (int i = 0; i < 50; ++i) { EncodeFrame(timestamp, false, &vp8_info, &flags); timestamp += kTimestampDelta5Fps; - EXPECT_EQ(ScreenshareLayers::kTl0Flags, flags); + EXPECT_EQ(kTl0Flags, flags); EXPECT_EQ(0, vp8_info.temporalIdx); } } @@ -425,46 +433,46 @@ TEST_F(ScreenshareLayerTest, EncoderDrop) { timestamp = SkipUntilTl(0, timestamp); // Size 0 indicates dropped frame. - layers_->FrameEncoded(0, timestamp, kDefaultQp); + layers_->FrameEncoded(0, kDefaultQp); timestamp += kTimestampDelta5Fps; EXPECT_FALSE(layers_->UpdateConfiguration(&cfg)); - EXPECT_EQ(ScreenshareLayers::kTl0Flags, layers_->EncodeFlags(timestamp)); + EXPECT_EQ(kTl0Flags, layers_->EncodeFlags(timestamp)); layers_->PopulateCodecSpecific(false, &vp8_info, timestamp); - layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp); + layers_->FrameEncoded(frame_size_, kDefaultQp); timestamp = SkipUntilTl(0, timestamp); EXPECT_TRUE(layers_->UpdateConfiguration(&cfg)); EXPECT_LT(cfg.rc_max_quantizer, static_cast(kDefaultQp)); - layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp); + layers_->FrameEncoded(frame_size_, kDefaultQp); layers_->EncodeFlags(timestamp); timestamp += kTimestampDelta5Fps; EXPECT_TRUE(layers_->UpdateConfiguration(&cfg)); layers_->PopulateCodecSpecific(false, &vp8_info, timestamp); EXPECT_EQ(cfg.rc_max_quantizer, static_cast(kDefaultQp)); - layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp); + layers_->FrameEncoded(frame_size_, kDefaultQp); // Next drop in TL1. timestamp = SkipUntilTl(1, timestamp); - layers_->FrameEncoded(0, timestamp, kDefaultQp); + layers_->FrameEncoded(0, kDefaultQp); timestamp += kTimestampDelta5Fps; EXPECT_FALSE(layers_->UpdateConfiguration(&cfg)); - EXPECT_EQ(ScreenshareLayers::kTl1Flags, layers_->EncodeFlags(timestamp)); + EXPECT_EQ(kTl1Flags, layers_->EncodeFlags(timestamp)); layers_->PopulateCodecSpecific(false, &vp8_info, timestamp); - layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp); + layers_->FrameEncoded(frame_size_, kDefaultQp); timestamp = SkipUntilTl(1, timestamp); EXPECT_TRUE(layers_->UpdateConfiguration(&cfg)); EXPECT_LT(cfg.rc_max_quantizer, static_cast(kDefaultQp)); - layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp); + layers_->FrameEncoded(frame_size_, kDefaultQp); layers_->EncodeFlags(timestamp); timestamp += kTimestampDelta5Fps; EXPECT_TRUE(layers_->UpdateConfiguration(&cfg)); layers_->PopulateCodecSpecific(false, &vp8_info, timestamp); EXPECT_EQ(cfg.rc_max_quantizer, static_cast(kDefaultQp)); - layers_->FrameEncoded(frame_size_, timestamp, kDefaultQp); + layers_->FrameEncoded(frame_size_, kDefaultQp); } TEST_F(ScreenshareLayerTest, RespectsMaxIntervalBetweenFrames) { @@ -476,9 +484,8 @@ TEST_F(ScreenshareLayerTest, RespectsMaxIntervalBetweenFrames) { layers_->OnRatesUpdated(kLowBitrateKbps, kLowBitrateKbps, 5); layers_->UpdateConfiguration(&cfg); - EXPECT_EQ(ScreenshareLayers::kTl0Flags, - layers_->EncodeFlags(kStartTimestamp)); - layers_->FrameEncoded(kLargeFrameSizeBytes, kStartTimestamp, kDefaultQp); + EXPECT_EQ(kTl0Flags, layers_->EncodeFlags(kStartTimestamp)); + layers_->FrameEncoded(kLargeFrameSizeBytes, kDefaultQp); const uint32_t kTwoSecondsLater = kStartTimestamp + (ScreenshareLayers::kMaxFrameIntervalMs * 90); @@ -490,8 +497,7 @@ TEST_F(ScreenshareLayerTest, RespectsMaxIntervalBetweenFrames) { EXPECT_EQ(-1, layers_->EncodeFlags(kTwoSecondsLater)); // More than two seconds has passed since last frame, one should be emitted // even if bitrate target is then exceeded. - EXPECT_EQ(ScreenshareLayers::kTl0Flags, - layers_->EncodeFlags(kTwoSecondsLater + 90)); + EXPECT_EQ(kTl0Flags, layers_->EncodeFlags(kTwoSecondsLater + 90)); } TEST_F(ScreenshareLayerTest, UpdatesHistograms) { @@ -512,22 +518,21 @@ TEST_F(ScreenshareLayerTest, UpdatesHistograms) { if (timestamp >= kTimestampDelta5Fps * 5 && !overshoot && flags != -1) { // Simulate one overshoot. - layers_->FrameEncoded(0, timestamp, 0); + layers_->FrameEncoded(0, 0); overshoot = true; flags = layers_->EncodeFlags(timestamp); } - if (flags == ScreenshareLayers::kTl0Flags) { + if (flags == kTl0Flags) { if (timestamp >= kTimestampDelta5Fps * 20 && !trigger_drop) { // Simulate a too large frame, to cause frame drop. - layers_->FrameEncoded(frame_size_ * 5, timestamp, kTl0Qp); + layers_->FrameEncoded(frame_size_ * 5, kTl0Qp); trigger_drop = true; } else { - layers_->FrameEncoded(frame_size_, timestamp, kTl0Qp); + layers_->FrameEncoded(frame_size_, kTl0Qp); } - } else if (flags == ScreenshareLayers::kTl1Flags || - flags == ScreenshareLayers::kTl1SyncFlags) { - layers_->FrameEncoded(frame_size_, timestamp, kTl1Qp); + } else if (flags == kTl1Flags || flags == kTl1SyncFlags) { + layers_->FrameEncoded(frame_size_, kTl1Qp); } else if (flags == -1) { dropped_frame = true; } else { @@ -593,7 +598,7 @@ TEST_F(ScreenshareLayerTest, RespectsConfiguredFramerate) { ++num_discarded_frames; } else { size_t frame_size_bytes = kDefaultTl0BitrateKbps * kFrameIntervalsMs / 8; - layers_->FrameEncoded(frame_size_bytes, timestamp, kDefaultQp); + layers_->FrameEncoded(frame_size_bytes, kDefaultQp); } timestamp += kFrameIntervalsMs * 90; clock_.AdvanceTimeMilliseconds(kFrameIntervalsMs); @@ -609,7 +614,7 @@ TEST_F(ScreenshareLayerTest, RespectsConfiguredFramerate) { ++num_discarded_frames; } else { size_t frame_size_bytes = kDefaultTl0BitrateKbps * kFrameIntervalsMs / 8; - layers_->FrameEncoded(frame_size_bytes, timestamp, kDefaultQp); + layers_->FrameEncoded(frame_size_bytes, kDefaultQp); } timestamp += kFrameIntervalsMs * 90 / 2; clock_.AdvanceTimeMilliseconds(kFrameIntervalsMs / 2); diff --git a/webrtc/modules/video_coding/codecs/vp8/temporal_layers.h b/webrtc/modules/video_coding/codecs/vp8/temporal_layers.h index bf3e49aed6..ba7f406b31 100644 --- a/webrtc/modules/video_coding/codecs/vp8/temporal_layers.h +++ b/webrtc/modules/video_coding/codecs/vp8/temporal_layers.h @@ -24,6 +24,43 @@ namespace webrtc { struct CodecSpecificInfoVP8; +enum TemporalBufferFlags { + kNone = 0, + kReference = 1, + kUpdate = 2, + kReferenceAndUpdate = kReference | kUpdate, +}; + +enum TemporalFlags { kLayerSync = 1, kFreezeEntropy = 2 }; + +struct TemporalReferences { + TemporalReferences(TemporalBufferFlags last, + TemporalBufferFlags golden, + TemporalBufferFlags arf); + TemporalReferences(TemporalBufferFlags last, + TemporalBufferFlags golden, + TemporalBufferFlags arf, + int extra_flags); + + const bool drop_frame; + const TemporalBufferFlags last_buffer_flags; + const TemporalBufferFlags golden_buffer_flags; + const TemporalBufferFlags arf_buffer_flags; + + // TODO(pbos): Consider breaking these out of here and returning only a + // pattern index that needs to be returned to fill CodecSpecificInfoVP8 or + // EncodeFlags. + const bool layer_sync; + const bool freeze_entropy; + + private: + TemporalReferences(TemporalBufferFlags last, + TemporalBufferFlags golden, + TemporalBufferFlags arf, + bool layer_sync, + bool freeze_entropy); +}; + class TemporalLayers { public: // Factory for TemporalLayer strategy. Default behavior is a fixed pattern @@ -32,7 +69,9 @@ class TemporalLayers { // Returns the recommended VP8 encode flags needed. May refresh the decoder // and/or update the reference buffers. - virtual int EncodeFlags(uint32_t timestamp) = 0; + virtual TemporalReferences UpdateLayerConfig(uint32_t timestamp) = 0; + + int EncodeFlags(uint32_t timestamp); // Update state based on new bitrate target and incoming framerate. // Returns the bitrate allocation for the active temporal layers. @@ -44,11 +83,11 @@ class TemporalLayers { // Returns true iff the configuration was actually modified. virtual bool UpdateConfiguration(vpx_codec_enc_cfg_t* cfg) = 0; - virtual void PopulateCodecSpecific(bool base_layer_sync, + virtual void PopulateCodecSpecific(bool is_keyframe, CodecSpecificInfoVP8* vp8_info, uint32_t timestamp) = 0; - virtual void FrameEncoded(unsigned int size, uint32_t timestamp, int qp) = 0; + virtual void FrameEncoded(unsigned int size, int qp) = 0; virtual int CurrentLayerId() const = 0; }; diff --git a/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc b/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc index b1282c392e..41fd7ff384 100644 --- a/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc +++ b/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc @@ -812,11 +812,9 @@ void VP8EncoderImpl::PopulateCodecSpecific( } vp8Info->simulcastIdx = stream_idx; vp8Info->keyIdx = kNoKeyIdx; // TODO(hlundin) populate this - vp8Info->nonReference = - (pkt.data.frame.flags & VPX_FRAME_IS_DROPPABLE) ? true : false; - bool base_layer_sync_point = pkt.data.frame.flags & VPX_FRAME_IS_KEY; - temporal_layers_[stream_idx]->PopulateCodecSpecific(base_layer_sync_point, - vp8Info, timestamp); + vp8Info->nonReference = (pkt.data.frame.flags & VPX_FRAME_IS_DROPPABLE) != 0; + temporal_layers_[stream_idx]->PopulateCodecSpecific( + (pkt.data.frame.flags & VPX_FRAME_IS_KEY) != 0, vp8Info, timestamp); // Prepare next. picture_id_[stream_idx] = (picture_id_[stream_idx] + 1) & 0x7FFF; } @@ -884,8 +882,7 @@ int VP8EncoderImpl::GetEncodedPartitions(const VideoFrame& input_image) { int qp = -1; vpx_codec_control(&encoders_[encoder_idx], VP8E_GET_LAST_QUANTIZER_64, &qp); temporal_layers_[stream_idx]->FrameEncoded( - encoded_images_[encoder_idx]._length, - encoded_images_[encoder_idx]._timeStamp, qp); + encoded_images_[encoder_idx]._length, qp); if (send_stream_[stream_idx]) { if (encoded_images_[encoder_idx]._length > 0) { TRACE_COUNTER_ID1("webrtc", "EncodedFrameSize", encoder_idx, diff --git a/webrtc/modules/video_coding/utility/simulcast_rate_allocator_unittest.cc b/webrtc/modules/video_coding/utility/simulcast_rate_allocator_unittest.cc index a0dd7c8817..670ad31483 100644 --- a/webrtc/modules/video_coding/utility/simulcast_rate_allocator_unittest.cc +++ b/webrtc/modules/video_coding/utility/simulcast_rate_allocator_unittest.cc @@ -29,12 +29,12 @@ constexpr uint32_t kFramerateFps = 5; class MockTemporalLayers : public TemporalLayers { public: - MOCK_METHOD1(EncodeFlags, int(uint32_t)); + MOCK_METHOD1(UpdateLayerConfig, TemporalReferences(uint32_t)); MOCK_METHOD3(OnRatesUpdated, std::vector(int, int, int)); MOCK_METHOD1(UpdateConfiguration, bool(vpx_codec_enc_cfg_t*)); MOCK_METHOD3(PopulateCodecSpecific, void(bool, CodecSpecificInfoVP8*, uint32_t)); - MOCK_METHOD3(FrameEncoded, void(unsigned int, uint32_t, int)); + MOCK_METHOD2(FrameEncoded, void(unsigned int, int)); MOCK_CONST_METHOD0(CurrentLayerId, int()); }; } // namespace