zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Break RtpFrameReferenceFinder into descriptor specific parts.

Browse Source 
This CL breaks out descriptor specific parts into separate classes. All logic in the newly added classes is just copy pasted from the (previously massive) RtpFrameReferenceFinder with the exception of how frames are being returned, which is now done via return value rather than a callback. Basically, all interesting changes have been made in the RtpFrameReferenceFinder.

Bug: webrtc:12221
Change-Id: I5f958d2fbf4b77ba11c3c6c01d8d0d80e325be60
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/195448
Commit-Queue: Philip Eliasson <philipel@webrtc.org>
Reviewed-by: Erik Språng <sprang@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#32717}
This commit is contained in:
philipel
2020-11-27 17:56:37 +01:00
committed by Commit Bot
parent e99b6ccb9b
commit 4e702169e4
13 changed files with 1351 additions and 811 deletions
Download Patch File Download Diff File Expand all files Collapse all files

801
modules/video_coding/rtp_frame_reference_finder.cc
View File

@ -10,15 +10,141 @@
#include "modules/video_coding/rtp_frame_reference_finder.h"
#include <algorithm>
#include <utility>
#include "absl/base/macros.h"
#include "absl/types/variant.h"
#include "modules/video_coding/frame_object.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
#include "modules/video_coding/rtp_frame_id_only_ref_finder.h"
#include "modules/video_coding/rtp_generic_ref_finder.h"
#include "modules/video_coding/rtp_seq_num_only_ref_finder.h"
#include "modules/video_coding/rtp_vp8_ref_finder.h"
#include "modules/video_coding/rtp_vp9_ref_finder.h"
namespace webrtc {
namespace video_coding {
namespace internal {
class RtpFrameReferenceFinderImpl {
public:
RtpFrameReferenceFinderImpl() = default;
RtpFrameReferenceFinder::ReturnVector ManageFrame(
std::unique_ptr<RtpFrameObject> frame);
RtpFrameReferenceFinder::ReturnVector PaddingReceived(uint16_t seq_num);
void ClearTo(uint16_t seq_num);
private:
using RefFinder = absl::variant<absl::monostate,
RtpGenericFrameRefFinder,
RtpFrameIdOnlyRefFinder,
RtpSeqNumOnlyRefFinder,
RtpVp8RefFinder,
RtpVp9RefFinder>;
template <typename T>
T& GetRefFinderAs();
RefFinder ref_finder_;
};
RtpFrameReferenceFinder::ReturnVector RtpFrameReferenceFinderImpl::ManageFrame(
std::unique_ptr<RtpFrameObject> frame) {
const RTPVideoHeader& video_header = frame->GetRtpVideoHeader();
if (video_header.generic.has_value()) {
return GetRefFinderAs<RtpGenericFrameRefFinder>().ManageFrame(
std::move(frame), *video_header.generic);
}
switch (frame->codec_type()) {
case kVideoCodecVP8: {
const RTPVideoHeaderVP8& vp8_header =
absl::get<RTPVideoHeaderVP8>(video_header.video_type_header);
if (vp8_header.temporalIdx == kNoTemporalIdx ||
vp8_header.tl0PicIdx == kNoTl0PicIdx) {
if (vp8_header.pictureId == kNoPictureId) {
return GetRefFinderAs<RtpSeqNumOnlyRefFinder>().ManageFrame(
std::move(frame));
}
return GetRefFinderAs<RtpFrameIdOnlyRefFinder>().ManageFrame(
std::move(frame), vp8_header.pictureId);
}
return GetRefFinderAs<RtpVp8RefFinder>().ManageFrame(std::move(frame));
}
case kVideoCodecVP9: {
const RTPVideoHeaderVP9& vp9_header =
absl::get<RTPVideoHeaderVP9>(video_header.video_type_header);
if (vp9_header.temporal_idx == kNoTemporalIdx) {
if (vp9_header.picture_id == kNoPictureId) {
return GetRefFinderAs<RtpSeqNumOnlyRefFinder>().ManageFrame(
std::move(frame));
}
return GetRefFinderAs<RtpFrameIdOnlyRefFinder>().ManageFrame(
std::move(frame), vp9_header.picture_id);
}
return GetRefFinderAs<RtpVp9RefFinder>().ManageFrame(std::move(frame));
}
case kVideoCodecH264: {
return GetRefFinderAs<RtpSeqNumOnlyRefFinder>().ManageFrame(
std::move(frame));
}
case kVideoCodecGeneric: {
if (auto* generic_header = absl::get_if<RTPVideoHeaderLegacyGeneric>(
&video_header.video_type_header)) {
return GetRefFinderAs<RtpFrameIdOnlyRefFinder>().ManageFrame(
std::move(frame), generic_header->picture_id);
}
return GetRefFinderAs<RtpSeqNumOnlyRefFinder>().ManageFrame(
std::move(frame));
}
default: {
RTC_CHECK_NOTREACHED();
}
}
}
RtpFrameReferenceFinder::ReturnVector
RtpFrameReferenceFinderImpl::PaddingReceived(uint16_t seq_num) {
if (auto* ref_finder = absl::get_if<RtpSeqNumOnlyRefFinder>(&ref_finder_)) {
return ref_finder->PaddingReceived(seq_num);
}
return {};
}
void RtpFrameReferenceFinderImpl::ClearTo(uint16_t seq_num) {
struct ClearToVisitor {
void operator()(absl::monostate& ref_finder) {}
void operator()(RtpGenericFrameRefFinder& ref_finder) {}
void operator()(RtpFrameIdOnlyRefFinder& ref_finder) {}
void operator()(RtpSeqNumOnlyRefFinder& ref_finder) {
ref_finder.ClearTo(seq_num);
}
void operator()(RtpVp8RefFinder& ref_finder) {
ref_finder.ClearTo(seq_num);
}
void operator()(RtpVp9RefFinder& ref_finder) {
ref_finder.ClearTo(seq_num);
}
uint16_t seq_num;
};
absl::visit(ClearToVisitor{seq_num}, ref_finder_);
}
template <typename T>
T& RtpFrameReferenceFinderImpl::GetRefFinderAs() {
if (auto* ref_finder = absl::get_if<T>(&ref_finder_)) {
return *ref_finder;
}
return ref_finder_.emplace<T>();
}
} // namespace internal
RtpFrameReferenceFinder::RtpFrameReferenceFinder(
OnCompleteFrameCallback* frame_callback)
@ -27,11 +153,9 @@ RtpFrameReferenceFinder::RtpFrameReferenceFinder(
RtpFrameReferenceFinder::RtpFrameReferenceFinder(
OnCompleteFrameCallback* frame_callback,
int64_t picture_id_offset)
: last_picture_id_(-1),
current_ss_idx_(0),
cleared_to_seq_num_(-1),
: picture_id_offset_(picture_id_offset),
frame_callback_(frame_callback),
picture_id_offset_(picture_id_offset) {}
impl_(std::make_unique<internal::RtpFrameReferenceFinderImpl>()) {}
RtpFrameReferenceFinder::~RtpFrameReferenceFinder() = default;
@ -42,672 +166,27 @@ void RtpFrameReferenceFinder::ManageFrame(
AheadOf<uint16_t>(cleared_to_seq_num_, frame->first_seq_num())) {
return;
}
FrameDecision decision = ManageFrameInternal(frame.get());
switch (decision) {
case kStash:
if (stashed_frames_.size() > kMaxStashedFrames)
stashed_frames_.pop_back();
stashed_frames_.push_front(std::move(frame));
break;
case kHandOff:
HandOffFrame(std::move(frame));
RetryStashedFrames();
break;
case kDrop:
break;
}
}
void RtpFrameReferenceFinder::RetryStashedFrames() {
bool complete_frame = false;
do {
complete_frame = false;
for (auto frame_it = stashed_frames_.begin();
frame_it != stashed_frames_.end();) {
FrameDecision decision = ManageFrameInternal(frame_it->get());
switch (decision) {
case kStash:
++frame_it;
break;
case kHandOff:
complete_frame = true;
HandOffFrame(std::move(*frame_it));
ABSL_FALLTHROUGH_INTENDED;
case kDrop:
frame_it = stashed_frames_.erase(frame_it);
}
}
} while (complete_frame);
}
void RtpFrameReferenceFinder::HandOffFrame(
std::unique_ptr<RtpFrameObject> frame) {
frame->id.picture_id += picture_id_offset_;
for (size_t i = 0; i < frame->num_references; ++i) {
frame->references[i] += picture_id_offset_;
}
frame_callback_->OnCompleteFrame(std::move(frame));
}
RtpFrameReferenceFinder::FrameDecision
RtpFrameReferenceFinder::ManageFrameInternal(RtpFrameObject* frame) {
if (const absl::optional<RTPVideoHeader::GenericDescriptorInfo>&
generic_descriptor = frame->GetRtpVideoHeader().generic) {
return ManageFrameGeneric(frame, *generic_descriptor);
}
switch (frame->codec_type()) {
case kVideoCodecVP8:
return ManageFrameVp8(frame);
case kVideoCodecVP9:
return ManageFrameVp9(frame);
case kVideoCodecGeneric:
if (auto* generic_header = absl::get_if<RTPVideoHeaderLegacyGeneric>(
&frame->GetRtpVideoHeader().video_type_header)) {
return ManageFramePidOrSeqNum(frame, generic_header->picture_id);
}
ABSL_FALLTHROUGH_INTENDED;
default:
return ManageFramePidOrSeqNum(frame, kNoPictureId);
}
HandOffFrames(impl_->ManageFrame(std::move(frame)));
}
void RtpFrameReferenceFinder::PaddingReceived(uint16_t seq_num) {
auto clean_padding_to =
stashed_padding_.lower_bound(seq_num - kMaxPaddingAge);
stashed_padding_.erase(stashed_padding_.begin(), clean_padding_to);
stashed_padding_.insert(seq_num);
UpdateLastPictureIdWithPadding(seq_num);
RetryStashedFrames();
HandOffFrames(impl_->PaddingReceived(seq_num));
}
void RtpFrameReferenceFinder::ClearTo(uint16_t seq_num) {
cleared_to_seq_num_ = seq_num;
auto it = stashed_frames_.begin();
while (it != stashed_frames_.end()) {
if (AheadOf<uint16_t>(cleared_to_seq_num_, (*it)->first_seq_num())) {
it = stashed_frames_.erase(it);
} else {
++it;
}
}
impl_->ClearTo(seq_num);
}
void RtpFrameReferenceFinder::UpdateLastPictureIdWithPadding(uint16_t seq_num) {
auto gop_seq_num_it = last_seq_num_gop_.upper_bound(seq_num);
// If this padding packet "belongs" to a group of pictures that we don't track
// anymore, do nothing.
if (gop_seq_num_it == last_seq_num_gop_.begin())
return;
--gop_seq_num_it;
// Calculate the next contiuous sequence number and search for it in
// the padding packets we have stashed.
uint16_t next_seq_num_with_padding = gop_seq_num_it->second.second + 1;
auto padding_seq_num_it =
stashed_padding_.lower_bound(next_seq_num_with_padding);
// While there still are padding packets and those padding packets are
// continuous, then advance the "last-picture-id-with-padding" and remove
// the stashed padding packet.
while (padding_seq_num_it != stashed_padding_.end() &&
*padding_seq_num_it == next_seq_num_with_padding) {
gop_seq_num_it->second.second = next_seq_num_with_padding;
++next_seq_num_with_padding;
padding_seq_num_it = stashed_padding_.erase(padding_seq_num_it);
}
// In the case where the stream has been continuous without any new keyframes
// for a while there is a risk that new frames will appear to be older than
// the keyframe they belong to due to wrapping sequence number. In order
// to prevent this we advance the picture id of the keyframe every so often.
if (ForwardDiff(gop_seq_num_it->first, seq_num) > 10000) {
auto save = gop_seq_num_it->second;
last_seq_num_gop_.clear();
last_seq_num_gop_[seq_num] = save;
}
}
RtpFrameReferenceFinder::FrameDecision
RtpFrameReferenceFinder::ManageFrameGeneric(
RtpFrameObject* frame,
const RTPVideoHeader::GenericDescriptorInfo& descriptor) {
frame->id.picture_id = descriptor.frame_id;
frame->SetSpatialIndex(descriptor.spatial_index);
if (EncodedFrame::kMaxFrameReferences < descriptor.dependencies.size()) {
RTC_LOG(LS_WARNING) << "Too many dependencies in generic descriptor.";
return kDrop;
}
frame->num_references = descriptor.dependencies.size();
for (size_t i = 0; i < descriptor.dependencies.size(); ++i)
frame->references[i] = descriptor.dependencies[i];
return kHandOff;
}
RtpFrameReferenceFinder::FrameDecision
RtpFrameReferenceFinder::ManageFramePidOrSeqNum(RtpFrameObject* frame,
int picture_id) {
// If |picture_id| is specified then we use that to set the frame references,
// otherwise we use sequence number.
if (picture_id != kNoPictureId) {
frame->id.picture_id = unwrapper_.Unwrap(picture_id & 0x7FFF);
frame->num_references =
frame->frame_type() == VideoFrameType::kVideoFrameKey ? 0 : 1;
frame->references[0] = frame->id.picture_id - 1;
return kHandOff;
}
if (frame->frame_type() == VideoFrameType::kVideoFrameKey) {
last_seq_num_gop_.insert(std::make_pair(
frame->last_seq_num(),
std::make_pair(frame->last_seq_num(), frame->last_seq_num())));
}
// We have received a frame but not yet a keyframe, stash this frame.
if (last_seq_num_gop_.empty())
return kStash;
// Clean up info for old keyframes but make sure to keep info
// for the last keyframe.
auto clean_to = last_seq_num_gop_.lower_bound(frame->last_seq_num() - 100);
for (auto it = last_seq_num_gop_.begin();
it != clean_to && last_seq_num_gop_.size() > 1;) {
it = last_seq_num_gop_.erase(it);
}
// Find the last sequence number of the last frame for the keyframe
// that this frame indirectly references.
auto seq_num_it = last_seq_num_gop_.upper_bound(frame->last_seq_num());
if (seq_num_it == last_seq_num_gop_.begin()) {
RTC_LOG(LS_WARNING) << "Generic frame with packet range ["
<< frame->first_seq_num() << ", "
<< frame->last_seq_num()
<< "] has no GoP, dropping frame.";
return kDrop;
}
seq_num_it--;
// Make sure the packet sequence numbers are continuous, otherwise stash
// this frame.
uint16_t last_picture_id_gop = seq_num_it->second.first;
uint16_t last_picture_id_with_padding_gop = seq_num_it->second.second;
if (frame->frame_type() == VideoFrameType::kVideoFrameDelta) {
uint16_t prev_seq_num = frame->first_seq_num() - 1;
if (prev_seq_num != last_picture_id_with_padding_gop)
return kStash;
}
RTC_DCHECK(AheadOrAt(frame->last_seq_num(), seq_num_it->first));
// Since keyframes can cause reordering we can't simply assign the
// picture id according to some incrementing counter.
frame->id.picture_id = frame->last_seq_num();
frame->num_references =
frame->frame_type() == VideoFrameType::kVideoFrameDelta;
frame->references[0] = rtp_seq_num_unwrapper_.Unwrap(last_picture_id_gop);
if (AheadOf<uint16_t>(frame->id.picture_id, last_picture_id_gop)) {
seq_num_it->second.first = frame->id.picture_id;
seq_num_it->second.second = frame->id.picture_id;
}
UpdateLastPictureIdWithPadding(frame->id.picture_id);
frame->id.picture_id = rtp_seq_num_unwrapper_.Unwrap(frame->id.picture_id);
return kHandOff;
}
RtpFrameReferenceFinder::FrameDecision RtpFrameReferenceFinder::ManageFrameVp8(
RtpFrameObject* frame) {
const RTPVideoHeader& video_header = frame->GetRtpVideoHeader();
const RTPVideoHeaderVP8& codec_header =
absl::get<RTPVideoHeaderVP8>(video_header.video_type_header);
if (codec_header.pictureId == kNoPictureId ||
codec_header.temporalIdx == kNoTemporalIdx ||
codec_header.tl0PicIdx == kNoTl0PicIdx) {
return ManageFramePidOrSeqNum(frame, codec_header.pictureId);
}
// Protect against corrupted packets with arbitrary large temporal idx.
if (codec_header.temporalIdx >= kMaxTemporalLayers)
return kDrop;
frame->id.picture_id = codec_header.pictureId & 0x7FFF;
if (last_picture_id_ == -1)
last_picture_id_ = frame->id.picture_id;
// Clean up info about not yet received frames that are too old.
uint16_t old_picture_id =
Subtract<kPicIdLength>(frame->id.picture_id, kMaxNotYetReceivedFrames);
auto clean_frames_to = not_yet_received_frames_.lower_bound(old_picture_id);
not_yet_received_frames_.erase(not_yet_received_frames_.begin(),
clean_frames_to);
// Avoid re-adding picture ids that were just erased.
if (AheadOf<uint16_t, kPicIdLength>(old_picture_id, last_picture_id_)) {
last_picture_id_ = old_picture_id;
}
// Find if there has been a gap in fully received frames and save the picture
// id of those frames in |not_yet_received_frames_|.
if (AheadOf<uint16_t, kPicIdLength>(frame->id.picture_id, last_picture_id_)) {
do {
last_picture_id_ = Add<kPicIdLength>(last_picture_id_, 1);
not_yet_received_frames_.insert(last_picture_id_);
} while (last_picture_id_ != frame->id.picture_id);
}
int64_t unwrapped_tl0 = tl0_unwrapper_.Unwrap(codec_header.tl0PicIdx & 0xFF);
// Clean up info for base layers that are too old.
int64_t old_tl0_pic_idx = unwrapped_tl0 - kMaxLayerInfo;
auto clean_layer_info_to = layer_info_.lower_bound(old_tl0_pic_idx);
layer_info_.erase(layer_info_.begin(), clean_layer_info_to);
if (frame->frame_type() == VideoFrameType::kVideoFrameKey) {
if (codec_header.temporalIdx != 0) {
return kDrop;
}
frame->num_references = 0;
layer_info_[unwrapped_tl0].fill(-1);
UpdateLayerInfoVp8(frame, unwrapped_tl0, codec_header.temporalIdx);
return kHandOff;
}
auto layer_info_it = layer_info_.find(
codec_header.temporalIdx == 0 ? unwrapped_tl0 - 1 : unwrapped_tl0);
// If we don't have the base layer frame yet, stash this frame.
if (layer_info_it == layer_info_.end())
return kStash;
// A non keyframe base layer frame has been received, copy the layer info
// from the previous base layer frame and set a reference to the previous
// base layer frame.
if (codec_header.temporalIdx == 0) {
layer_info_it =
layer_info_.emplace(unwrapped_tl0, layer_info_it->second).first;
frame->num_references = 1;
int64_t last_pid_on_layer = layer_info_it->second[0];
// Is this an old frame that has already been used to update the state? If
// so, drop it.
if (AheadOrAt<uint16_t, kPicIdLength>(last_pid_on_layer,
frame->id.picture_id)) {
return kDrop;
}
frame->references[0] = last_pid_on_layer;
UpdateLayerInfoVp8(frame, unwrapped_tl0, codec_header.temporalIdx);
return kHandOff;
}
// Layer sync frame, this frame only references its base layer frame.
if (codec_header.layerSync) {
frame->num_references = 1;
int64_t last_pid_on_layer = layer_info_it->second[codec_header.temporalIdx];
// Is this an old frame that has already been used to update the state? If
// so, drop it.
if (last_pid_on_layer != -1 &&
AheadOrAt<uint16_t, kPicIdLength>(last_pid_on_layer,
frame->id.picture_id)) {
return kDrop;
}
frame->references[0] = layer_info_it->second[0];
UpdateLayerInfoVp8(frame, unwrapped_tl0, codec_header.temporalIdx);
return kHandOff;
}
// Find all references for this frame.
frame->num_references = 0;
for (uint8_t layer = 0; layer <= codec_header.temporalIdx; ++layer) {
// If we have not yet received a previous frame on this temporal layer,
// stash this frame.
if (layer_info_it->second[layer] == -1)
return kStash;
// If the last frame on this layer is ahead of this frame it means that
// a layer sync frame has been received after this frame for the same
// base layer frame, drop this frame.
if (AheadOf<uint16_t, kPicIdLength>(layer_info_it->second[layer],
frame->id.picture_id)) {
return kDrop;
}
// If we have not yet received a frame between this frame and the referenced
// frame then we have to wait for that frame to be completed first.
auto not_received_frame_it =
not_yet_received_frames_.upper_bound(layer_info_it->second[layer]);
if (not_received_frame_it != not_yet_received_frames_.end() &&
AheadOf<uint16_t, kPicIdLength>(frame->id.picture_id,
*not_received_frame_it)) {
return kStash;
}
if (!(AheadOf<uint16_t, kPicIdLength>(frame->id.picture_id,
layer_info_it->second[layer]))) {
RTC_LOG(LS_WARNING) << "Frame with picture id " << frame->id.picture_id
<< " and packet range [" << frame->first_seq_num()
<< ", " << frame->last_seq_num()
<< "] already received, "
" dropping frame.";
return kDrop;
}
++frame->num_references;
frame->references[layer] = layer_info_it->second[layer];
}
UpdateLayerInfoVp8(frame, unwrapped_tl0, codec_header.temporalIdx);
return kHandOff;
}
void RtpFrameReferenceFinder::UpdateLayerInfoVp8(RtpFrameObject* frame,
int64_t unwrapped_tl0,
uint8_t temporal_idx) {
auto layer_info_it = layer_info_.find(unwrapped_tl0);
// Update this layer info and newer.
while (layer_info_it != layer_info_.end()) {
if (layer_info_it->second[temporal_idx] != -1 &&
AheadOf<uint16_t, kPicIdLength>(layer_info_it->second[temporal_idx],
frame->id.picture_id)) {
// The frame was not newer, then no subsequent layer info have to be
// update.
break;
}
layer_info_it->second[temporal_idx] = frame->id.picture_id;
++unwrapped_tl0;
layer_info_it = layer_info_.find(unwrapped_tl0);
}
not_yet_received_frames_.erase(frame->id.picture_id);
UnwrapPictureIds(frame);
}
RtpFrameReferenceFinder::FrameDecision RtpFrameReferenceFinder::ManageFrameVp9(
RtpFrameObject* frame) {
const RTPVideoHeader& video_header = frame->GetRtpVideoHeader();
const RTPVideoHeaderVP9& codec_header =
absl::get<RTPVideoHeaderVP9>(video_header.video_type_header);
if (codec_header.picture_id == kNoPictureId ||
codec_header.temporal_idx == kNoTemporalIdx) {
return ManageFramePidOrSeqNum(frame, codec_header.picture_id);
}
// Protect against corrupted packets with arbitrary large temporal idx.
if (codec_header.temporal_idx >= kMaxTemporalLayers ||
codec_header.spatial_idx >= kMaxSpatialLayers)
return kDrop;
frame->id.spatial_layer = codec_header.spatial_idx;
frame->inter_layer_predicted = codec_header.inter_layer_predicted;
frame->id.picture_id = codec_header.picture_id & 0x7FFF;
if (last_picture_id_ == -1)
last_picture_id_ = frame->id.picture_id;
if (codec_header.flexible_mode) {
if (codec_header.num_ref_pics > EncodedFrame::kMaxFrameReferences) {
return kDrop;
}
frame->num_references = codec_header.num_ref_pics;
void RtpFrameReferenceFinder::HandOffFrames(ReturnVector frames) {
for (auto& frame : frames) {
frame->id.picture_id += picture_id_offset_;
for (size_t i = 0; i < frame->num_references; ++i) {
frame->references[i] = Subtract<kPicIdLength>(frame->id.picture_id,
codec_header.pid_diff[i]);
frame->references[i] += picture_id_offset_;
}
UnwrapPictureIds(frame);
return kHandOff;
frame_callback_->OnCompleteFrame(std::move(frame));
}
if (codec_header.tl0_pic_idx == kNoTl0PicIdx) {
RTC_LOG(LS_WARNING) << "TL0PICIDX is expected to be present in "
"non-flexible mode.";
return kDrop;
}
GofInfo* info;
int64_t unwrapped_tl0 =
tl0_unwrapper_.Unwrap(codec_header.tl0_pic_idx & 0xFF);
if (codec_header.ss_data_available) {
if (codec_header.temporal_idx != 0) {
RTC_LOG(LS_WARNING) << "Received scalability structure on a non base "
"layer frame. Scalability structure ignored.";
} else {
if (codec_header.gof.num_frames_in_gof > kMaxVp9FramesInGof) {
return kDrop;
}
for (size_t i = 0; i < codec_header.gof.num_frames_in_gof; ++i) {
if (codec_header.gof.num_ref_pics[i] > kMaxVp9RefPics) {
return kDrop;
}
}
GofInfoVP9 gof = codec_header.gof;
if (gof.num_frames_in_gof == 0) {
RTC_LOG(LS_WARNING) << "Number of frames in GOF is zero. Assume "
"that stream has only one temporal layer.";
gof.SetGofInfoVP9(kTemporalStructureMode1);
}
current_ss_idx_ = Add<kMaxGofSaved>(current_ss_idx_, 1);
scalability_structures_[current_ss_idx_] = gof;
scalability_structures_[current_ss_idx_].pid_start = frame->id.picture_id;
gof_info_.emplace(unwrapped_tl0,
GofInfo(&scalability_structures_[current_ss_idx_],
frame->id.picture_id));
}
const auto gof_info_it = gof_info_.find(unwrapped_tl0);
if (gof_info_it == gof_info_.end())
return kStash;
info = &gof_info_it->second;
if (frame->frame_type() == VideoFrameType::kVideoFrameKey) {
frame->num_references = 0;
FrameReceivedVp9(frame->id.picture_id, info);
UnwrapPictureIds(frame);
return kHandOff;
}
} else if (frame->frame_type() == VideoFrameType::kVideoFrameKey) {
if (frame->id.spatial_layer == 0) {
RTC_LOG(LS_WARNING) << "Received keyframe without scalability structure";
return kDrop;
}
const auto gof_info_it = gof_info_.find(unwrapped_tl0);
if (gof_info_it == gof_info_.end())
return kStash;
info = &gof_info_it->second;
if (frame->frame_type() == VideoFrameType::kVideoFrameKey) {
frame->num_references = 0;
FrameReceivedVp9(frame->id.picture_id, info);
UnwrapPictureIds(frame);
return kHandOff;
}
} else {
auto gof_info_it = gof_info_.find(
(codec_header.temporal_idx == 0) ? unwrapped_tl0 - 1 : unwrapped_tl0);
// Gof info for this frame is not available yet, stash this frame.
if (gof_info_it == gof_info_.end())
return kStash;
if (codec_header.temporal_idx == 0) {
gof_info_it = gof_info_
.emplace(unwrapped_tl0, GofInfo(gof_info_it->second.gof,
frame->id.picture_id))
.first;
}
info = &gof_info_it->second;
}
// Clean up info for base layers that are too old.
int64_t old_tl0_pic_idx = unwrapped_tl0 - kMaxGofSaved;
auto clean_gof_info_to = gof_info_.lower_bound(old_tl0_pic_idx);
gof_info_.erase(gof_info_.begin(), clean_gof_info_to);
FrameReceivedVp9(frame->id.picture_id, info);
// Make sure we don't miss any frame that could potentially have the
// up switch flag set.
if (MissingRequiredFrameVp9(frame->id.picture_id, *info))
return kStash;
if (codec_header.temporal_up_switch)
up_switch_.emplace(frame->id.picture_id, codec_header.temporal_idx);
// Clean out old info about up switch frames.
uint16_t old_picture_id = Subtract<kPicIdLength>(frame->id.picture_id, 50);
auto up_switch_erase_to = up_switch_.lower_bound(old_picture_id);
up_switch_.erase(up_switch_.begin(), up_switch_erase_to);
size_t diff = ForwardDiff<uint16_t, kPicIdLength>(info->gof->pid_start,
frame->id.picture_id);
size_t gof_idx = diff % info->gof->num_frames_in_gof;
if (info->gof->num_ref_pics[gof_idx] > EncodedFrame::kMaxFrameReferences) {
return kDrop;
}
// Populate references according to the scalability structure.
frame->num_references = info->gof->num_ref_pics[gof_idx];
for (size_t i = 0; i < frame->num_references; ++i) {
frame->references[i] = Subtract<kPicIdLength>(
frame->id.picture_id, info->gof->pid_diff[gof_idx][i]);
// If this is a reference to a frame earlier than the last up switch point,
// then ignore this reference.
if (UpSwitchInIntervalVp9(frame->id.picture_id, codec_header.temporal_idx,
frame->references[i])) {
--frame->num_references;
}
}
// Override GOF references.
if (!codec_header.inter_pic_predicted) {
frame->num_references = 0;
}
UnwrapPictureIds(frame);
return kHandOff;
}
bool RtpFrameReferenceFinder::MissingRequiredFrameVp9(uint16_t picture_id,
const GofInfo& info) {
size_t diff =
ForwardDiff<uint16_t, kPicIdLength>(info.gof->pid_start, picture_id);
size_t gof_idx = diff % info.gof->num_frames_in_gof;
size_t temporal_idx = info.gof->temporal_idx[gof_idx];
if (temporal_idx >= kMaxTemporalLayers) {
RTC_LOG(LS_WARNING) << "At most " << kMaxTemporalLayers
<< " temporal "
"layers are supported.";
return true;
}
// For every reference this frame has, check if there is a frame missing in
// the interval (|ref_pid|, |picture_id|) in any of the lower temporal
// layers. If so, we are missing a required frame.
uint8_t num_references = info.gof->num_ref_pics[gof_idx];
for (size_t i = 0; i < num_references; ++i) {
uint16_t ref_pid =
Subtract<kPicIdLength>(picture_id, info.gof->pid_diff[gof_idx][i]);
for (size_t l = 0; l < temporal_idx; ++l) {
auto missing_frame_it = missing_frames_for_layer_[l].lower_bound(ref_pid);
if (missing_frame_it != missing_frames_for_layer_[l].end() &&
AheadOf<uint16_t, kPicIdLength>(picture_id, *missing_frame_it)) {
return true;
}
}
}
return false;
}
void RtpFrameReferenceFinder::FrameReceivedVp9(uint16_t picture_id,
GofInfo* info) {
int last_picture_id = info->last_picture_id;
size_t gof_size = std::min(info->gof->num_frames_in_gof, kMaxVp9FramesInGof);
// If there is a gap, find which temporal layer the missing frames
// belong to and add the frame as missing for that temporal layer.
// Otherwise, remove this frame from the set of missing frames.
if (AheadOf<uint16_t, kPicIdLength>(picture_id, last_picture_id)) {
size_t diff = ForwardDiff<uint16_t, kPicIdLength>(info->gof->pid_start,
last_picture_id);
size_t gof_idx = diff % gof_size;
last_picture_id = Add<kPicIdLength>(last_picture_id, 1);
while (last_picture_id != picture_id) {
gof_idx = (gof_idx + 1) % gof_size;
RTC_CHECK(gof_idx < kMaxVp9FramesInGof);
size_t temporal_idx = info->gof->temporal_idx[gof_idx];
if (temporal_idx >= kMaxTemporalLayers) {
RTC_LOG(LS_WARNING) << "At most " << kMaxTemporalLayers
<< " temporal "
"layers are supported.";
return;
}
missing_frames_for_layer_[temporal_idx].insert(last_picture_id);
last_picture_id = Add<kPicIdLength>(last_picture_id, 1);
}
info->last_picture_id = last_picture_id;
} else {
size_t diff =
ForwardDiff<uint16_t, kPicIdLength>(info->gof->pid_start, picture_id);
size_t gof_idx = diff % gof_size;
RTC_CHECK(gof_idx < kMaxVp9FramesInGof);
size_t temporal_idx = info->gof->temporal_idx[gof_idx];
if (temporal_idx >= kMaxTemporalLayers) {
RTC_LOG(LS_WARNING) << "At most " << kMaxTemporalLayers
<< " temporal "
"layers are supported.";
return;
}
missing_frames_for_layer_[temporal_idx].erase(picture_id);
}
}
bool RtpFrameReferenceFinder::UpSwitchInIntervalVp9(uint16_t picture_id,
uint8_t temporal_idx,
uint16_t pid_ref) {
for (auto up_switch_it = up_switch_.upper_bound(pid_ref);
up_switch_it != up_switch_.end() &&
AheadOf<uint16_t, kPicIdLength>(picture_id, up_switch_it->first);
++up_switch_it) {
if (up_switch_it->second < temporal_idx)
return true;
}
return false;
}
void RtpFrameReferenceFinder::UnwrapPictureIds(RtpFrameObject* frame) {
for (size_t i = 0; i < frame->num_references; ++i)
frame->references[i] = unwrapper_.Unwrap(frame->references[i]);
frame->id.picture_id = unwrapper_.Unwrap(frame->id.picture_id);
}
} // namespace video_coding