From b19f27a5f93698b46844b82b348575afc99bc79d Mon Sep 17 00:00:00 2001 From: philipel Date: Thu, 28 Mar 2019 16:09:52 +0100 Subject: [PATCH] Decode Target Information for VP8 libvpx encoder. In this CL: - Created static helper function GenericFrameInfo::DecodeTargetInfo to convert DTI symbols to a list of GenericFrameInfo::OperatingPointIndication. - Added per frame DTI information for the different stream structures. Bug: webrtc:10342 Change-Id: I62ff2e9fc9b380fe1d0447ff071e86b6b35ab249 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/129923 Reviewed-by: Danil Chapovalov Commit-Queue: Philip Eliasson Cr-Commit-Position: refs/heads/master@{#27350} --- .../generic_frame_info.cc | 32 ++-- .../generic_frame_info.h | 11 +- .../codecs/vp8/default_temporal_layers.cc | 147 +++++++++--------- .../codecs/vp8/default_temporal_layers.h | 24 ++- .../codecs/vp8/screenshare_layers.cc | 60 ++++--- .../codecs/vp8/screenshare_layers.h | 16 +- 6 files changed, 168 insertions(+), 122 deletions(-) diff --git a/common_video/generic_frame_descriptor/generic_frame_info.cc b/common_video/generic_frame_descriptor/generic_frame_info.cc index 94a6192835..1c01b6c948 100644 --- a/common_video/generic_frame_descriptor/generic_frame_info.cc +++ b/common_video/generic_frame_descriptor/generic_frame_info.cc @@ -15,6 +15,24 @@ namespace webrtc { +absl::InlinedVector +GenericFrameInfo::DecodeTargetInfo(absl::string_view indication_symbols) { + absl::InlinedVector decode_targets; + for (char symbol : indication_symbols) { + DecodeTargetIndication indication; + switch (symbol) { + case '-': indication = DecodeTargetIndication::kNotPresent; break; + case 'D': indication = DecodeTargetIndication::kDiscardable; break; + case 'R': indication = DecodeTargetIndication::kRequired; break; + case 'S': indication = DecodeTargetIndication::kSwitch; break; + default: RTC_NOTREACHED(); + } + decode_targets.push_back(indication); + } + + return decode_targets; +} + GenericFrameInfo::GenericFrameInfo() = default; GenericFrameInfo::GenericFrameInfo(const GenericFrameInfo&) = default; GenericFrameInfo::~GenericFrameInfo() = default; @@ -38,19 +56,7 @@ GenericFrameInfo::Builder& GenericFrameInfo::Builder::S(int spatial_id) { GenericFrameInfo::Builder& GenericFrameInfo::Builder::Dtis( absl::string_view indication_symbols) { - for (const auto& symbol : indication_symbols) { - DecodeTargetIndication indication; - switch (symbol) { - case '-': indication = DecodeTargetIndication::kNotPresent; break; - case 'D': indication = DecodeTargetIndication::kDiscardable; break; - case 'R': indication = DecodeTargetIndication::kRequired; break; - case 'S': indication = DecodeTargetIndication::kSwitch; break; - default: RTC_NOTREACHED(); - } - - info_.decode_target_indications.push_back(indication); - } - + info_.decode_target_indications = DecodeTargetInfo(indication_symbols); return *this; } diff --git a/common_video/generic_frame_descriptor/generic_frame_info.h b/common_video/generic_frame_descriptor/generic_frame_info.h index 91a0868b15..790929106e 100644 --- a/common_video/generic_frame_descriptor/generic_frame_info.h +++ b/common_video/generic_frame_descriptor/generic_frame_info.h @@ -22,12 +22,15 @@ namespace webrtc { struct GenericFrameInfo { enum class DecodeTargetIndication { - kNotPresent, // GenericFrameInfo::Builder symbol '-' - kDiscardable, // GenericFrameInfo::Builder symbol 'D' - kSwitch, // GenericFrameInfo::Builder symbol 'S' - kRequired // GenericFrameInfo::Builder symbol 'R' + kNotPresent, // DecodeTargetInfo symbol '-' + kDiscardable, // DecodeTargetInfo symbol 'D' + kSwitch, // DecodeTargetInfo symbol 'S' + kRequired // DecodeTargetInfo symbol 'R' }; + static absl::InlinedVector DecodeTargetInfo( + absl::string_view indication_symbols); + class Builder; GenericFrameInfo(); diff --git a/modules/video_coding/codecs/vp8/default_temporal_layers.cc b/modules/video_coding/codecs/vp8/default_temporal_layers.cc index d47317e5a4..abab490797 100644 --- a/modules/video_coding/codecs/vp8/default_temporal_layers.cc +++ b/modules/video_coding/codecs/vp8/default_temporal_layers.cc @@ -31,10 +31,10 @@ DefaultTemporalLayers::PendingFrame::PendingFrame() = default; DefaultTemporalLayers::PendingFrame::PendingFrame( bool expired, uint8_t updated_buffers_mask, - const Vp8FrameConfig& frame_config) + const DependencyInfo& dependency_info) : expired(expired), updated_buffer_mask(updated_buffers_mask), - frame_config(frame_config) {} + dependency_info(dependency_info) {} namespace { using Buffer = Vp8FrameConfig::Buffer; @@ -99,27 +99,10 @@ uint8_t GetUpdatedBuffers(const Vp8FrameConfig& config) { } return flags; } - -// Find the set of buffers that are never updated by the given pattern. -std::set FindKfBuffers( - const std::vector& frame_configs) { - std::set kf_buffers(kAllBuffers.begin(), - kAllBuffers.end()); - for (Vp8FrameConfig config : frame_configs) { - // Get bit-masked set of update buffers for this frame config. - uint8_t updated_buffers = GetUpdatedBuffers(config); - for (Vp8BufferReference buffer : kAllBuffers) { - if (static_cast(buffer) & updated_buffers) { - kf_buffers.erase(buffer); - } - } - } - return kf_buffers; -} } // namespace -std::vector DefaultTemporalLayers::GetTemporalPattern( - size_t num_layers) { +std::vector +DefaultTemporalLayers::GetDependencyInfo(size_t num_layers) { // For indexing in the patterns described below (which temporal layers they // belong to), see the diagram above. // Layer sync is done similarly for all patterns (except single stream) and @@ -133,10 +116,11 @@ std::vector DefaultTemporalLayers::GetTemporalPattern( // so that if scene changes occur (user walks between rooms or rotates webcam) // the 'arf' (or 'golden' respectively) is not stuck on a no-longer relevant // keyframe. + switch (num_layers) { case 1: // Always reference and update the same buffer. - return {Vp8FrameConfig(kReferenceAndUpdate, kNone, kNone)}; + return {{"S", {kReferenceAndUpdate, kNone, kNone}}}; case 2: // All layers can reference but not update the 'alt' buffer, this means // that the 'alt' buffer reference is effectively the last keyframe. @@ -147,23 +131,23 @@ std::vector DefaultTemporalLayers::GetTemporalPattern( // 1---1 1---1 ... // / / / / // 0---0---0---0 ... - return {Vp8FrameConfig(kReferenceAndUpdate, kNone, kNone), - Vp8FrameConfig(kReference, kUpdate, kNone), - Vp8FrameConfig(kReferenceAndUpdate, kNone, kNone), - Vp8FrameConfig(kReference, kReference, kNone, kFreezeEntropy)}; + return {{"SS", {kReferenceAndUpdate, kNone, kNone}}, + {"-S", {kReference, kUpdate, kNone}}, + {"SR", {kReferenceAndUpdate, kNone, kNone}}, + {"-D", {kReference, kReference, kNone, kFreezeEntropy}}}; } else { // "Default" 8-frame pattern: // 1---1---1---1 1---1---1---1 ... // / / / / / / / / // 0---0---0---0---0---0---0---0 ... - return {Vp8FrameConfig(kReferenceAndUpdate, kNone, kNone), - Vp8FrameConfig(kReference, kUpdate, kNone), - Vp8FrameConfig(kReferenceAndUpdate, kNone, kNone), - Vp8FrameConfig(kReference, kReferenceAndUpdate, kNone), - Vp8FrameConfig(kReferenceAndUpdate, kNone, kNone), - Vp8FrameConfig(kReference, kReferenceAndUpdate, kNone), - Vp8FrameConfig(kReferenceAndUpdate, kNone, kNone), - Vp8FrameConfig(kReference, kReference, kNone, kFreezeEntropy)}; + return {{"SS", {kReferenceAndUpdate, kNone, kNone}}, + {"-S", {kReference, kUpdate, kNone}}, + {"SR", {kReferenceAndUpdate, kNone, kNone}}, + {"-R", {kReference, kReferenceAndUpdate, kNone}}, + {"SR", {kReferenceAndUpdate, kNone, kNone}}, + {"-R", {kReference, kReferenceAndUpdate, kNone}}, + {"SR", {kReferenceAndUpdate, kNone, kNone}}, + {"-D", {kReference, kReference, kNone, kFreezeEntropy}}}; } case 3: if (field_trial::IsEnabled("WebRTC-UseShortVP8TL3Pattern")) { @@ -183,62 +167,59 @@ std::vector DefaultTemporalLayers::GetTemporalPattern( // TL1 references 'last' and references and updates 'golden'. // TL2 references both 'last' & 'golden' and references and updates // 'arf'. - return { - Vp8FrameConfig(kReferenceAndUpdate, kNone, kNone), - Vp8FrameConfig(kReference, kNone, kUpdate), - Vp8FrameConfig(kReference, kUpdate, kNone), - Vp8FrameConfig(kReference, kReference, kReference, kFreezeEntropy)}; + return {{"SSS", {kReferenceAndUpdate, kNone, kNone}}, + {"--S", {kReference, kNone, kUpdate}}, + {"-DR", {kReference, kUpdate, kNone}}, + {"--D", {kReference, kReference, kReference, kFreezeEntropy}}}; } else { // All layers can reference but not update the 'alt' buffer, this means // that the 'alt' buffer reference is effectively the last keyframe. // TL0 also references and updates the 'last' buffer. // TL1 also references 'last' and references and updates 'golden'. // TL2 references both 'last' and 'golden' but updates no buffer. - return { - Vp8FrameConfig(kReferenceAndUpdate, kNone, kReference), - Vp8FrameConfig(kReference, kNone, kReference, kFreezeEntropy), - Vp8FrameConfig(kReference, kUpdate, kReference), - Vp8FrameConfig(kReference, kReference, kReference, kFreezeEntropy), - Vp8FrameConfig(kReferenceAndUpdate, kNone, kReference), - Vp8FrameConfig(kReference, kReference, kReference, kFreezeEntropy), - Vp8FrameConfig(kReference, kReferenceAndUpdate, kReference), - Vp8FrameConfig(kReference, kReference, kReference, kFreezeEntropy)}; + return {{"SSS", {kReferenceAndUpdate, kNone, kReference}}, + {"--D", {kReference, kNone, kReference, kFreezeEntropy}}, + {"-SS", {kReference, kUpdate, kReference}}, + {"--D", {kReference, kReference, kReference, kFreezeEntropy}}, + {"SRR", {kReferenceAndUpdate, kNone, kReference}}, + {"--D", {kReference, kReference, kReference, kFreezeEntropy}}, + {"-DS", {kReference, kReferenceAndUpdate, kReference}}, + {"--D", {kReference, kReference, kReference, kFreezeEntropy}}}; } case 4: // TL0 references and updates only the 'last' buffer. // TL1 references 'last' and updates and references 'golden'. // TL2 references 'last' and 'golden', and references and updates 'arf'. // TL3 references all buffers but update none of them. - return { - Vp8FrameConfig(kReferenceAndUpdate, kNone, kNone), - Vp8FrameConfig(kReference, kNone, kNone, kFreezeEntropy), - Vp8FrameConfig(kReference, kNone, kUpdate), - Vp8FrameConfig(kReference, kNone, kReference, kFreezeEntropy), - Vp8FrameConfig(kReference, kUpdate, kNone), - Vp8FrameConfig(kReference, kReference, kReference, kFreezeEntropy), - Vp8FrameConfig(kReference, kReference, kReferenceAndUpdate), - Vp8FrameConfig(kReference, kReference, kReference, kFreezeEntropy), - Vp8FrameConfig(kReferenceAndUpdate, kNone, kNone), - Vp8FrameConfig(kReference, kReference, kReference, kFreezeEntropy), - Vp8FrameConfig(kReference, kReference, kReferenceAndUpdate), - Vp8FrameConfig(kReference, kReference, kReference, kFreezeEntropy), - Vp8FrameConfig(kReference, kReferenceAndUpdate, kNone), - Vp8FrameConfig(kReference, kReference, kReference, kFreezeEntropy), - Vp8FrameConfig(kReference, kReference, kReferenceAndUpdate), - Vp8FrameConfig(kReference, kReference, kReference, kFreezeEntropy)}; + // TODO(philipel): Set decode target information for this structure. + return {{"----", {kReferenceAndUpdate, kNone, kNone}}, + {"----", {kReference, kNone, kNone, kFreezeEntropy}}, + {"----", {kReference, kNone, kUpdate}}, + {"----", {kReference, kNone, kReference, kFreezeEntropy}}, + {"----", {kReference, kUpdate, kNone}}, + {"----", {kReference, kReference, kReference, kFreezeEntropy}}, + {"----", {kReference, kReference, kReferenceAndUpdate}}, + {"----", {kReference, kReference, kReference, kFreezeEntropy}}, + {"----", {kReferenceAndUpdate, kNone, kNone}}, + {"----", {kReference, kReference, kReference, kFreezeEntropy}}, + {"----", {kReference, kReference, kReferenceAndUpdate}}, + {"----", {kReference, kReference, kReference, kFreezeEntropy}}, + {"----", {kReference, kReferenceAndUpdate, kNone}}, + {"----", {kReference, kReference, kReference, kFreezeEntropy}}, + {"----", {kReference, kReference, kReferenceAndUpdate}}, + {"----", {kReference, kReference, kReference, kFreezeEntropy}}}; default: RTC_NOTREACHED(); break; } RTC_NOTREACHED(); - return {Vp8FrameConfig(kNone, kNone, kNone)}; + return {{"", {kNone, kNone, kNone}}}; } DefaultTemporalLayers::DefaultTemporalLayers(int number_of_temporal_layers) : num_layers_(std::max(1, number_of_temporal_layers)), temporal_ids_(GetTemporalIds(num_layers_)), - temporal_pattern_(GetTemporalPattern(num_layers_)), - kf_buffers_(FindKfBuffers(temporal_pattern_)), + temporal_pattern_(GetDependencyInfo(num_layers_)), pattern_idx_(kUninitializedPatternIndex) { RTC_CHECK_GE(kMaxTemporalStreams, number_of_temporal_layers); RTC_CHECK_GE(number_of_temporal_layers, 0); @@ -257,6 +238,16 @@ DefaultTemporalLayers::DefaultTemporalLayers(int number_of_temporal_layers) for (Vp8BufferReference buffer : kAllBuffers) { frames_since_buffer_refresh_[buffer] = 0; } + + kf_buffers_ = {kAllBuffers.begin(), kAllBuffers.end()}; + for (DependencyInfo info : temporal_pattern_) { + uint8_t updated_buffers = GetUpdatedBuffers(info.frame_config); + + for (Vp8BufferReference buffer : kAllBuffers) { + if (static_cast(buffer) & updated_buffers) + kf_buffers_.erase(buffer); + } + } } DefaultTemporalLayers::~DefaultTemporalLayers() = default; @@ -347,7 +338,8 @@ Vp8FrameConfig DefaultTemporalLayers::UpdateLayerConfig(size_t stream_index, RTC_DCHECK_GT(temporal_pattern_.size(), 0); pattern_idx_ = (pattern_idx_ + 1) % temporal_pattern_.size(); - Vp8FrameConfig tl_config = temporal_pattern_[pattern_idx_]; + DependencyInfo dependency_info = temporal_pattern_[pattern_idx_]; + Vp8FrameConfig& tl_config = dependency_info.frame_config; tl_config.encoder_layer_id = tl_config.packetizer_temporal_idx = temporal_ids_[pattern_idx_ % temporal_ids_.size()]; @@ -386,7 +378,7 @@ Vp8FrameConfig DefaultTemporalLayers::UpdateLayerConfig(size_t stream_index, // Add frame to set of pending frames, awaiting completion. pending_frames_[timestamp] = - PendingFrame{false, GetUpdatedBuffers(tl_config), tl_config}; + PendingFrame{false, GetUpdatedBuffers(tl_config), dependency_info}; #if RTC_DCHECK_IS_ON // Checker does not yet support encoder frame dropping, so validate flags @@ -472,10 +464,11 @@ void DefaultTemporalLayers::OnEncodeDone(size_t stream_index, } PendingFrame& frame = pending_frame->second; + const Vp8FrameConfig& frame_config = frame.dependency_info.frame_config; #if RTC_DCHECK_IS_ON if (is_keyframe) { // Signal key-frame so checker resets state. - RTC_DCHECK(checker_->CheckTemporalConfig(true, frame.frame_config)); + RTC_DCHECK(checker_->CheckTemporalConfig(true, frame_config)); } #endif @@ -503,8 +496,8 @@ void DefaultTemporalLayers::OnEncodeDone(size_t stream_index, } } else { // Delta frame, update codec specifics with temporal id and sync flag. - vp8_info.temporalIdx = frame.frame_config.packetizer_temporal_idx; - vp8_info.layerSync = frame.frame_config.layer_sync; + vp8_info.temporalIdx = frame_config.packetizer_temporal_idx; + vp8_info.layerSync = frame_config.layer_sync; } } @@ -513,13 +506,13 @@ void DefaultTemporalLayers::OnEncodeDone(size_t stream_index, RTC_DCHECK_EQ(vp8_info.updatedBuffersCount, 0u); for (int i = 0; i < static_cast(Buffer::kCount); ++i) { - if (!is_keyframe && frame.frame_config.References(static_cast(i))) { + if (!is_keyframe && frame_config.References(static_cast(i))) { RTC_DCHECK_LT(vp8_info.referencedBuffersCount, arraysize(CodecSpecificInfoVP8::referencedBuffers)); vp8_info.referencedBuffers[vp8_info.referencedBuffersCount++] = i; } - if (is_keyframe || frame.frame_config.Updates(static_cast(i))) { + if (is_keyframe || frame_config.Updates(static_cast(i))) { RTC_DCHECK_LT(vp8_info.updatedBuffersCount, arraysize(CodecSpecificInfoVP8::updatedBuffers)); vp8_info.updatedBuffers[vp8_info.updatedBuffersCount++] = i; @@ -532,6 +525,10 @@ void DefaultTemporalLayers::OnEncodeDone(size_t stream_index, info->template_structure = GetTemplateStructure(num_layers_); } + GenericFrameInfo& generic_frame_info = info->generic_frame_info.emplace(); + generic_frame_info.decode_target_indications = + frame.dependency_info.decode_target_indications; + if (!frame.expired) { for (Vp8BufferReference buffer : kAllBuffers) { if (frame.updated_buffer_mask & static_cast(buffer)) { diff --git a/modules/video_coding/codecs/vp8/default_temporal_layers.h b/modules/video_coding/codecs/vp8/default_temporal_layers.h index ee2333ec2d..9aa95db49b 100644 --- a/modules/video_coding/codecs/vp8/default_temporal_layers.h +++ b/modules/video_coding/codecs/vp8/default_temporal_layers.h @@ -18,6 +18,7 @@ #include #include #include +#include #include #include "absl/types/optional.h" @@ -61,7 +62,20 @@ class DefaultTemporalLayers final : public Vp8FrameBufferController { void OnRttUpdate(int64_t rtt_ms) override; private: - static std::vector GetTemporalPattern(size_t num_layers); + struct DependencyInfo { + DependencyInfo() = default; + DependencyInfo(absl::string_view indication_symbols, + Vp8FrameConfig frame_config) + : decode_target_indications( + GenericFrameInfo::DecodeTargetInfo(indication_symbols)), + frame_config(frame_config) {} + + absl::InlinedVector + decode_target_indications; + Vp8FrameConfig frame_config; + }; + + static std::vector GetDependencyInfo(size_t num_layers); bool IsSyncFrame(const Vp8FrameConfig& config) const; void ValidateReferences(Vp8FrameConfig::BufferFlags* flags, Vp8FrameConfig::Vp8BufferReference ref) const; @@ -69,9 +83,9 @@ class DefaultTemporalLayers final : public Vp8FrameBufferController { const size_t num_layers_; const std::vector temporal_ids_; - const std::vector temporal_pattern_; + const std::vector temporal_pattern_; // Set of buffers that are never updated except by keyframes. - const std::set kf_buffers_; + std::set kf_buffers_; TemplateStructure GetTemplateStructure(int num_layers) const; uint8_t pattern_idx_; @@ -82,7 +96,7 @@ class DefaultTemporalLayers final : public Vp8FrameBufferController { PendingFrame(); PendingFrame(bool expired, uint8_t updated_buffers_mask, - const Vp8FrameConfig& frame_config); + const DependencyInfo& dependency_info); // Flag indicating if this frame has expired, ie it belongs to a previous // iteration of the temporal pattern. bool expired = false; @@ -90,7 +104,7 @@ class DefaultTemporalLayers final : public Vp8FrameBufferController { // updates. uint8_t updated_buffer_mask = 0; // The frame config return by UpdateLayerConfig() for this frame. - Vp8FrameConfig frame_config; + DependencyInfo dependency_info; }; // Map from rtp timestamp to pending frame status. Reset on pattern loop. std::map pending_frames_; diff --git a/modules/video_coding/codecs/vp8/screenshare_layers.cc b/modules/video_coding/codecs/vp8/screenshare_layers.cc index 9412ba43d5..11a1cbd228 100644 --- a/modules/video_coding/codecs/vp8/screenshare_layers.cc +++ b/modules/video_coding/codecs/vp8/screenshare_layers.cc @@ -90,16 +90,16 @@ Vp8FrameConfig ScreenshareLayers::UpdateLayerConfig(size_t stream_index, auto it = pending_frame_configs_.find(timestamp); if (it != pending_frame_configs_.end()) { // Drop and re-encode, reuse the previous config. - return it->second; + return it->second.frame_config; } if (number_of_temporal_layers_ <= 1) { // No flags needed for 1 layer screenshare. // TODO(pbos): Consider updating only last, and not all buffers. - Vp8FrameConfig tl_config(kReferenceAndUpdate, kReferenceAndUpdate, - kReferenceAndUpdate); - pending_frame_configs_[timestamp] = tl_config; - return tl_config; + DependencyInfo dependency_info{ + "S", {kReferenceAndUpdate, kReferenceAndUpdate, kReferenceAndUpdate}}; + pending_frame_configs_[timestamp] = dependency_info; + return dependency_info.frame_config; } const int64_t now_ms = rtc::TimeMillis(); @@ -199,35 +199,35 @@ Vp8FrameConfig ScreenshareLayers::UpdateLayerConfig(size_t stream_index, RTC_NOTREACHED(); } - Vp8FrameConfig tl_config; + DependencyInfo dependency_info; // TODO(pbos): Consider referencing but not updating the 'alt' buffer for all // layers. switch (layer_state) { case TemporalLayerState::kDrop: - tl_config = Vp8FrameConfig(kNone, kNone, kNone); + dependency_info = {"", {kNone, kNone, kNone}}; break; case TemporalLayerState::kTl0: // TL0 only references and updates 'last'. - tl_config = Vp8FrameConfig(kReferenceAndUpdate, kNone, kNone); - tl_config.packetizer_temporal_idx = 0; + dependency_info = {"SS", {kReferenceAndUpdate, kNone, kNone}}; + dependency_info.frame_config.packetizer_temporal_idx = 0; break; case TemporalLayerState::kTl1: // TL1 references both 'last' and 'golden' but only updates 'golden'. - tl_config = Vp8FrameConfig(kReference, kReferenceAndUpdate, kNone); - tl_config.packetizer_temporal_idx = 1; + dependency_info = {"-R", {kReference, kReferenceAndUpdate, kNone}}; + dependency_info.frame_config.packetizer_temporal_idx = 1; break; case TemporalLayerState::kTl1Sync: // Predict from only TL0 to allow participants to switch to the high // bitrate stream. Updates 'golden' so that TL1 can continue to refer to // and update 'golden' from this point on. - tl_config = Vp8FrameConfig(kReference, kUpdate, kNone); - tl_config.packetizer_temporal_idx = 1; + dependency_info = {"-S", {kReference, kUpdate, kNone}}; + dependency_info.frame_config.packetizer_temporal_idx = 1; + dependency_info.frame_config.layer_sync = true; break; } - tl_config.layer_sync = layer_state == TemporalLayerState::kTl1Sync; - pending_frame_configs_[timestamp] = tl_config; - return tl_config; + pending_frame_configs_[timestamp] = dependency_info; + return dependency_info.frame_config; } void ScreenshareLayers::OnRatesUpdated( @@ -286,28 +286,38 @@ void ScreenshareLayers::OnEncodeDone(size_t stream_index, return; } - absl::optional frame_config; + absl::optional dependency_info; auto it = pending_frame_configs_.find(rtp_timestamp); if (it != pending_frame_configs_.end()) { - frame_config = it->second; + dependency_info = it->second; pending_frame_configs_.erase(it); if (checker_) { - RTC_DCHECK(checker_->CheckTemporalConfig(is_keyframe, *frame_config)); + RTC_DCHECK(checker_->CheckTemporalConfig(is_keyframe, + dependency_info->frame_config)); } } CodecSpecificInfoVP8& vp8_info = info->codecSpecific.VP8; + GenericFrameInfo& generic_frame_info = info->generic_frame_info.emplace(); + if (number_of_temporal_layers_ == 1) { vp8_info.temporalIdx = kNoTemporalIdx; vp8_info.layerSync = false; + generic_frame_info.decode_target_indications = + GenericFrameInfo::DecodeTargetInfo("S"); } else { int64_t unwrapped_timestamp = time_wrap_handler_.Unwrap(rtp_timestamp); - if (frame_config) { - vp8_info.temporalIdx = frame_config->packetizer_temporal_idx; - vp8_info.layerSync = frame_config->layer_sync; + if (dependency_info) { + vp8_info.temporalIdx = + dependency_info->frame_config.packetizer_temporal_idx; + vp8_info.layerSync = dependency_info->frame_config.layer_sync; + generic_frame_info.decode_target_indications = + dependency_info->decode_target_indications; } else { RTC_DCHECK(is_keyframe); + generic_frame_info.decode_target_indications = + GenericFrameInfo::DecodeTargetInfo("SS"); } if (is_keyframe) { @@ -328,13 +338,15 @@ void ScreenshareLayers::OnEncodeDone(size_t stream_index, // Note that |frame_config| is not derefernced if |is_keyframe|, // meaning it's never dereferenced if the optional may be unset. for (int i = 0; i < static_cast(Buffer::kCount); ++i) { - if (!is_keyframe && frame_config->References(static_cast(i))) { + if (!is_keyframe && + dependency_info->frame_config.References(static_cast(i))) { RTC_DCHECK_LT(vp8_info.referencedBuffersCount, arraysize(CodecSpecificInfoVP8::referencedBuffers)); vp8_info.referencedBuffers[vp8_info.referencedBuffersCount++] = i; } - if (is_keyframe || frame_config->Updates(static_cast(i))) { + if (is_keyframe || + dependency_info->frame_config.Updates(static_cast(i))) { RTC_DCHECK_LT(vp8_info.updatedBuffersCount, arraysize(CodecSpecificInfoVP8::updatedBuffers)); vp8_info.updatedBuffers[vp8_info.updatedBuffersCount++] = i; diff --git a/modules/video_coding/codecs/vp8/screenshare_layers.h b/modules/video_coding/codecs/vp8/screenshare_layers.h index 6b1bc7ec25..8fe5fa3556 100644 --- a/modules/video_coding/codecs/vp8/screenshare_layers.h +++ b/modules/video_coding/codecs/vp8/screenshare_layers.h @@ -11,6 +11,7 @@ #include #include +#include #include #include "api/video_codecs/vp8_frame_config.h" @@ -67,6 +68,19 @@ class ScreenshareLayers final : public Vp8FrameBufferController { private: enum class TemporalLayerState : int { kDrop, kTl0, kTl1, kTl1Sync }; + struct DependencyInfo { + DependencyInfo() = default; + DependencyInfo(absl::string_view indication_symbols, + Vp8FrameConfig frame_config) + : decode_target_indications( + GenericFrameInfo::DecodeTargetInfo(indication_symbols)), + frame_config(frame_config) {} + + absl::InlinedVector + decode_target_indications; + Vp8FrameConfig frame_config; + }; + bool TimeToSync(int64_t timestamp) const; uint32_t GetCodecTargetBitrateKbps() const; @@ -81,7 +95,7 @@ class ScreenshareLayers final : public Vp8FrameBufferController { int max_qp_; uint32_t max_debt_bytes_; - std::map pending_frame_configs_; + std::map pending_frame_configs_; // Configured max framerate. absl::optional target_framerate_;