zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Bugfix for Vp9 GOF and missing frames.

Browse Source 
In order to correctly determine the references of a frame when using Vp9
with GOF one has to wait for all frames on the lower temporal layers
to make sure no up-switch point is missed.

This patch fix a bug where upon receiving a frame the RtpFrameReferenceFinder
would try to add missing frame for a group with a not yet knows scalability
structure.

BUG=webrtc:5514

Review-Url: https://codereview.webrtc.org/2127073002
Cr-Commit-Position: refs/heads/master@{#13487}
This commit is contained in:
philipel
2016-07-15 06:50:27 -07:00
committed by Commit bot
parent 136778252d
commit c9b27d56c0
3 changed files with 132 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

83
webrtc/modules/video_coding/rtp_frame_reference_finder.cc
View File

@ -368,9 +368,9 @@ void RtpFrameReferenceFinder::ManageFrameVp9(
scalability_structures_[current_ss_idx_] = codec_header.gof; scalability_structures_[current_ss_idx_] = codec_header.gof;
scalability_structures_[current_ss_idx_].pid_start = frame->picture_id; scalability_structures_[current_ss_idx_].pid_start = frame->picture_id;
auto pid_and_gof = std::make_pair( GofInfo info(&scalability_structures_[current_ss_idx_],
frame->picture_id, &scalability_structures_[current_ss_idx_]); frame->picture_id);
gof_info_.insert(std::make_pair(codec_header.tl0_pic_idx, pid_and_gof)); gof_info_.insert(std::make_pair(codec_header.tl0_pic_idx, info));
} }
} }
@ -385,8 +385,8 @@ void RtpFrameReferenceFinder::ManageFrameVp9(
LOG(LS_WARNING) << "Received keyframe without scalability structure"; LOG(LS_WARNING) << "Received keyframe without scalability structure";
frame->num_references = 0; frame->num_references = 0;
GofInfoVP9* gof = gof_info_.find(codec_header.tl0_pic_idx)->second.second; GofInfo info = gof_info_.find(codec_header.tl0_pic_idx)->second;
FrameReceivedVp9(frame->picture_id, *gof); FrameReceivedVp9(frame->picture_id, &info);
CompletedFrameVp9(std::move(frame)); CompletedFrameVp9(std::move(frame));
return; return;
} }
@ -402,14 +402,12 @@ void RtpFrameReferenceFinder::ManageFrameVp9(
return; return;
} }
GofInfoVP9* gof = gof_info_it->second.second; GofInfo* info = &gof_info_it->second;
uint16_t picture_id_tl0 = gof_info_it->second.first; FrameReceivedVp9(frame->picture_id, info);
FrameReceivedVp9(frame->picture_id, *gof);
// Make sure we don't miss any frame that could potentially have the // Make sure we don't miss any frame that could potentially have the
// up switch flag set. // up switch flag set.
if (MissingRequiredFrameVp9(frame->picture_id, *gof)) { if (MissingRequiredFrameVp9(frame->picture_id, *info)) {
stashed_frames_.emplace(std::move(frame)); stashed_frames_.emplace(std::move(frame));
return; return;
} }
@ -424,27 +422,24 @@ void RtpFrameReferenceFinder::ManageFrameVp9(
// then gof info has already been inserted earlier, so we only want to // then gof info has already been inserted earlier, so we only want to
// insert if we haven't done so already. // insert if we haven't done so already.
if (codec_header.temporal_idx == 0 && !codec_header.ss_data_available) { if (codec_header.temporal_idx == 0 && !codec_header.ss_data_available) {
auto pid_and_gof = std::make_pair(frame->picture_id, gof); GofInfo new_info(info->gof, frame->picture_id);
gof_info_.insert(std::make_pair(codec_header.tl0_pic_idx, pid_and_gof)); gof_info_.insert(std::make_pair(codec_header.tl0_pic_idx, new_info));
} }
// Clean out old info about up switch frames. // Clean out old info about up switch frames.
uint16_t old_picture_id = Subtract<kPicIdLength>(last_picture_id_, 50); uint16_t old_picture_id = Subtract<kPicIdLength>(frame->picture_id, 50);
auto up_switch_erase_to = up_switch_.lower_bound(old_picture_id); auto up_switch_erase_to = up_switch_.lower_bound(old_picture_id);
up_switch_.erase(up_switch_.begin(), up_switch_erase_to); up_switch_.erase(up_switch_.begin(), up_switch_erase_to);
RTC_DCHECK( size_t diff = ForwardDiff<uint16_t, kPicIdLength>(info->gof->pid_start,
(AheadOrAt<uint16_t, kPicIdLength>(frame->picture_id, picture_id_tl0))); frame->picture_id);
size_t gof_idx = diff % info->gof->num_frames_in_gof;
size_t diff =
ForwardDiff<uint16_t, kPicIdLength>(gof->pid_start, frame->picture_id);
size_t gof_idx = diff % gof->num_frames_in_gof;
// Populate references according to the scalability structure. // Populate references according to the scalability structure.
frame->num_references = gof->num_ref_pics[gof_idx]; frame->num_references = info->gof->num_ref_pics[gof_idx];
for (size_t i = 0; i < frame->num_references; ++i) { for (size_t i = 0; i < frame->num_references; ++i) {
frame->references[i] = frame->references[i] = Subtract<kPicIdLength>(
Subtract<kPicIdLength>(frame->picture_id, gof->pid_diff[gof_idx][i]); frame->picture_id, info->gof->pid_diff[gof_idx][i]);
// If this is a reference to a frame earlier than the last up switch point, // If this is a reference to a frame earlier than the last up switch point,
// then ignore this reference. // then ignore this reference.
@ -458,18 +453,19 @@ void RtpFrameReferenceFinder::ManageFrameVp9(
} }
bool RtpFrameReferenceFinder::MissingRequiredFrameVp9(uint16_t picture_id, bool RtpFrameReferenceFinder::MissingRequiredFrameVp9(uint16_t picture_id,
const GofInfoVP9& gof) { const GofInfo& info) {
size_t diff = ForwardDiff<uint16_t, kPicIdLength>(gof.pid_start, picture_id); size_t diff =
size_t gof_idx = diff % gof.num_frames_in_gof; ForwardDiff<uint16_t, kPicIdLength>(info.gof->pid_start, picture_id);
size_t temporal_idx = gof.temporal_idx[gof_idx]; size_t gof_idx = diff % info.gof->num_frames_in_gof;
size_t temporal_idx = info.gof->temporal_idx[gof_idx];
// For every reference this frame has, check if there is a frame missing in // For every reference this frame has, check if there is a frame missing in
// the interval (|ref_pid|, |picture_id|) in any of the lower temporal // the interval (|ref_pid|, |picture_id|) in any of the lower temporal
// layers. If so, we are missing a required frame. // layers. If so, we are missing a required frame.
uint8_t num_references = gof.num_ref_pics[gof_idx]; uint8_t num_references = info.gof->num_ref_pics[gof_idx];
for (size_t i = 0; i < num_references; ++i) { for (size_t i = 0; i < num_references; ++i) {
uint16_t ref_pid = uint16_t ref_pid =
Subtract<kPicIdLength>(picture_id, gof.pid_diff[gof_idx][i]); Subtract<kPicIdLength>(picture_id, info.gof->pid_diff[gof_idx][i]);
for (size_t l = 0; l < temporal_idx; ++l) { for (size_t l = 0; l < temporal_idx; ++l) {
auto missing_frame_it = missing_frames_for_layer_[l].lower_bound(ref_pid); auto missing_frame_it = missing_frames_for_layer_[l].lower_bound(ref_pid);
if (missing_frame_it != missing_frames_for_layer_[l].end() && if (missing_frame_it != missing_frames_for_layer_[l].end() &&
@ -482,30 +478,31 @@ bool RtpFrameReferenceFinder::MissingRequiredFrameVp9(uint16_t picture_id,
} }
void RtpFrameReferenceFinder::FrameReceivedVp9(uint16_t picture_id, void RtpFrameReferenceFinder::FrameReceivedVp9(uint16_t picture_id,
const GofInfoVP9& gof) { GofInfo* info) {
RTC_DCHECK_NE(-1, last_picture_id_); int last_picture_id = info->last_picture_id;
// If there is a gap, find which temporal layer the missing frames // If there is a gap, find which temporal layer the missing frames
// belong to and add the frame as missing for that temporal layer. // belong to and add the frame as missing for that temporal layer.
// Otherwise, remove this frame from the set of missing frames. // Otherwise, remove this frame from the set of missing frames.
if (AheadOf<uint16_t, kPicIdLength>(picture_id, last_picture_id_)) { if (AheadOf<uint16_t, kPicIdLength>(picture_id, last_picture_id)) {
size_t diff = size_t diff = ForwardDiff<uint16_t, kPicIdLength>(info->gof->pid_start,
ForwardDiff<uint16_t, kPicIdLength>(gof.pid_start, last_picture_id_); last_picture_id);
size_t gof_idx = diff % gof.num_frames_in_gof; size_t gof_idx = diff % info->gof->num_frames_in_gof;
last_picture_id_ = Add<kPicIdLength>(last_picture_id_, 1); last_picture_id = Add<kPicIdLength>(last_picture_id, 1);
while (last_picture_id_ != picture_id) { while (last_picture_id != picture_id) {
++gof_idx; ++gof_idx;
RTC_DCHECK_NE(0ul, gof_idx % gof.num_frames_in_gof); RTC_DCHECK_NE(0ul, gof_idx % info->gof->num_frames_in_gof);
size_t temporal_idx = gof.temporal_idx[gof_idx]; size_t temporal_idx = info->gof->temporal_idx[gof_idx];
missing_frames_for_layer_[temporal_idx].insert(last_picture_id_); missing_frames_for_layer_[temporal_idx].insert(last_picture_id);
last_picture_id_ = Add<kPicIdLength>(last_picture_id_, 1); last_picture_id = Add<kPicIdLength>(last_picture_id, 1);
} }
info->last_picture_id = last_picture_id;
} else { } else {
size_t diff = size_t diff =
ForwardDiff<uint16_t, kPicIdLength>(gof.pid_start, picture_id); ForwardDiff<uint16_t, kPicIdLength>(info->gof->pid_start, picture_id);
size_t gof_idx = diff % gof.num_frames_in_gof; size_t gof_idx = diff % info->gof->num_frames_in_gof;
size_t temporal_idx = gof.temporal_idx[gof_idx]; size_t temporal_idx = info->gof->temporal_idx[gof_idx];
missing_frames_for_layer_[temporal_idx].erase(picture_id); missing_frames_for_layer_[temporal_idx].erase(picture_id);
} }
} }

23
webrtc/modules/video_coding/rtp_frame_reference_finder.h
View File

@ -44,6 +44,14 @@ class RtpFrameReferenceFinder {
static const int kMaxGofSaved = 15; static const int kMaxGofSaved = 15;
static const int kMaxPaddingAge = 100; static const int kMaxPaddingAge = 100;
struct GofInfo {
GofInfo(GofInfoVP9* gof, uint16_t last_picture_id)
: gof(gof), last_picture_id(last_picture_id) {}
GofInfoVP9* gof;
uint16_t last_picture_id;
};
rtc::CriticalSection crit_; rtc::CriticalSection crit_;
// Find the relevant group of pictures and update its "last-picture-id-with // Find the relevant group of pictures and update its "last-picture-id-with
@ -82,13 +90,13 @@ class RtpFrameReferenceFinder {
// Check if we are missing a frame necessary to determine the references // Check if we are missing a frame necessary to determine the references
// for this frame. // for this frame.
bool MissingRequiredFrameVp9(uint16_t picture_id, const GofInfoVP9& gof) bool MissingRequiredFrameVp9(uint16_t picture_id, const GofInfo& info)
EXCLUSIVE_LOCKS_REQUIRED(crit_); EXCLUSIVE_LOCKS_REQUIRED(crit_);
// Updates which frames that have been received. If there is a gap, // Updates which frames that have been received. If there is a gap,
// missing frames will be added to |missing_frames_for_layer_| or // missing frames will be added to |missing_frames_for_layer_| or
// if this is an already missing frame then it will be removed. // if this is an already missing frame then it will be removed.
void FrameReceivedVp9(uint16_t picture_id, const GofInfoVP9& gof) void FrameReceivedVp9(uint16_t picture_id, GofInfo* info)
EXCLUSIVE_LOCKS_REQUIRED(crit_); EXCLUSIVE_LOCKS_REQUIRED(crit_);
// Check if there is a frame with the up-switch flag set in the interval // Check if there is a frame with the up-switch flag set in the interval
@ -147,15 +155,14 @@ class RtpFrameReferenceFinder {
std::array<GofInfoVP9, kMaxGofSaved> scalability_structures_ std::array<GofInfoVP9, kMaxGofSaved> scalability_structures_
GUARDED_BY(crit_); GUARDED_BY(crit_);
// Holds the picture id and the Gof information for a given TL0 picture index. // Holds the the Gof information for a given TL0 picture index.
std::map<uint8_t, std::map<uint8_t, GofInfo, DescendingSeqNumComp<uint8_t>> gof_info_
std::pair<uint16_t, GofInfoVP9*>, GUARDED_BY(crit_);
DescendingSeqNumComp<uint8_t>>
gof_info_ GUARDED_BY(crit_);
// Keep track of which picture id and which temporal layer that had the // Keep track of which picture id and which temporal layer that had the
// up switch flag set. // up switch flag set.
std::map<uint16_t, uint8_t> up_switch_ GUARDED_BY(crit_); std::map<uint16_t, uint8_t, DescendingSeqNumComp<uint16_t, kPicIdLength>>
up_switch_ GUARDED_BY(crit_);
// For every temporal layer, keep a set of which frames that are missing. // For every temporal layer, keep a set of which frames that are missing.
std::array<std::set<uint16_t, DescendingSeqNumComp<uint16_t, kPicIdLength>>, std::array<std::set<uint16_t, DescendingSeqNumComp<uint16_t, kPicIdLength>>,

77
webrtc/modules/video_coding/video_packet_buffer_unittest.cc
View File

@ -1118,6 +1118,83 @@ TEST_F(TestPacketBuffer, Vp9GofTemporalLayersReordered_0) {
CheckReferencesVp9(pid + 19, 0, pid + 18); CheckReferencesVp9(pid + 19, 0, pid + 18);
} }
TEST_F(TestPacketBuffer, Vp9GofSkipFramesTemporalLayers_01) {
uint16_t pid = Rand();
uint16_t sn = Rand();
GofInfoVP9 ss;
ss.SetGofInfoVP9(kTemporalStructureMode2); // 0101 pattern
// sn , kf, frst, lst, up, pid , sid, tid, tl0, ss
InsertVp9Gof(sn , kT, kT , kT , kF, pid , 0 , 0 , 0 , &ss);
InsertVp9Gof(sn + 1 , kF, kT , kT , kF, pid + 1 , 0 , 1 , 0);
// Skip GOF with tl0 1
InsertVp9Gof(sn + 4 , kT, kT , kT , kF, pid + 4 , 0 , 0 , 2 , &ss);
InsertVp9Gof(sn + 5 , kF, kT , kT , kF, pid + 5 , 0 , 1 , 2);
// Skip GOF with tl0 3
// Skip GOF with tl0 4
InsertVp9Gof(sn + 10, kF, kT , kT , kF, pid + 10, 0 , 0 , 5 , &ss);
InsertVp9Gof(sn + 11, kF, kT , kT , kF, pid + 11, 0 , 1 , 5);
ASSERT_EQ(6UL, frames_from_callback_.size());
CheckReferencesVp9(pid, 0);
CheckReferencesVp9(pid + 1 , 0, pid);
CheckReferencesVp9(pid + 4 , 0);
CheckReferencesVp9(pid + 5 , 0, pid + 4);
CheckReferencesVp9(pid + 10, 0, pid + 8);
CheckReferencesVp9(pid + 11, 0, pid + 10);
}
TEST_F(TestPacketBuffer, Vp9GofSkipFramesTemporalLayers_0212) {
uint16_t pid = Rand();
uint16_t sn = Rand();
GofInfoVP9 ss;
ss.SetGofInfoVP9(kTemporalStructureMode3); // 02120212 pattern
// sn , kf, frst, lst, up, pid , sid, tid, tl0, ss
InsertVp9Gof(sn , kT, kT , kT , kF, pid , 0 , 0 , 0 , &ss);
InsertVp9Gof(sn + 1 , kF, kT , kT , kF, pid + 1 , 0 , 2 , 0);
InsertVp9Gof(sn + 2 , kF, kT , kT , kF, pid + 2 , 0 , 1 , 0);
InsertVp9Gof(sn + 3 , kF, kT , kT , kF, pid + 3 , 0 , 2 , 0);
ASSERT_EQ(4UL, frames_from_callback_.size());
CheckReferencesVp9(pid, 0);
CheckReferencesVp9(pid + 1 , 0, pid);
CheckReferencesVp9(pid + 2 , 0, pid);
CheckReferencesVp9(pid + 3 , 0, pid + 1, pid + 2);
// Skip frames with tl0 = 1
// sn , kf, frst, lst, up, pid , sid, tid, tl0, ss
InsertVp9Gof(sn + 8 , kT, kT , kT , kF, pid + 8 , 0 , 0 , 2 , &ss);
InsertVp9Gof(sn + 9 , kF, kT , kT , kF, pid + 9 , 0 , 2 , 2);
InsertVp9Gof(sn + 10, kF, kT , kT , kF, pid + 10, 0 , 1 , 2);
InsertVp9Gof(sn + 11, kF, kT , kT , kF, pid + 11, 0 , 2 , 2);
ASSERT_EQ(8UL, frames_from_callback_.size());
CheckReferencesVp9(pid + 8, 0);
CheckReferencesVp9(pid + 9 , 0, pid + 8);
CheckReferencesVp9(pid + 10, 0, pid + 8);
CheckReferencesVp9(pid + 11, 0, pid + 9, pid + 10);
// Now insert frames with tl0 = 1
// sn , kf, frst, lst, up, pid , sid, tid, tl0, ss
InsertVp9Gof(sn + 4 , kT, kT , kT , kF, pid + 4 , 0 , 0 , 1 , &ss);
InsertVp9Gof(sn + 7 , kF, kT , kT , kF, pid + 7 , 0 , 2 , 1);
ASSERT_EQ(9UL, frames_from_callback_.size());
CheckReferencesVp9(pid + 4, 0);
// Rest of frames belonging to tl0 = 1
// sn , kf, frst, lst, up, pid , sid, tid, tl0, ss
InsertVp9Gof(sn + 5 , kF, kT , kT , kF, pid + 5 , 0 , 2 , 1);
InsertVp9Gof(sn + 6 , kF, kT , kT , kT, pid + 6 , 0 , 1 , 1); // up-switch
ASSERT_EQ(12UL, frames_from_callback_.size());
CheckReferencesVp9(pid + 5 , 0, pid + 4);
CheckReferencesVp9(pid + 6 , 0, pid + 4);
CheckReferencesVp9(pid + 7 , 0, pid + 6);
}
TEST_F(TestPacketBuffer, Vp9GofTemporalLayers_01) { TEST_F(TestPacketBuffer, Vp9GofTemporalLayers_01) {
uint16_t pid = Rand(); uint16_t pid = Rand();
uint16_t sn = Rand(); uint16_t sn = Rand();