zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Revert "Revert of Use sps and pps to determine decodability of H.264 frames. (patchset #4 id:60001 of https://codereview.webrtc.org/2341713002/ )"

Browse Source 
This reverts commit 3cdfcd88a14449a9b116cb6149e1348d3a1e4cb2.

NOPRESUBMIT=true
BUG=webrtc:6208

Review-Url: https://codereview.webrtc.org/2385143002
Cr-Commit-Position: refs/heads/master@{#14551}
This commit is contained in:
stefan
2016-10-06 05:04:52 -07:00
committed by Commit bot
parent 0e7c7ce35d
commit a669a3a0dc
9 changed files with 243 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

78
webrtc/modules/video_coding/decoding_state.cc
View File

@ -10,6 +10,8 @@
#include "webrtc/modules/video_coding/decoding_state.h"
#include "webrtc/base/logging.h"
#include "webrtc/common_video/h264/h264_common.h"
#include "webrtc/modules/include/module_common_types.h"
#include "webrtc/modules/video_coding/frame_buffer.h"
#include "webrtc/modules/video_coding/jitter_buffer_common.h"
@ -40,6 +42,8 @@ void VCMDecodingState::Reset() {
full_sync_ = true;
in_initial_state_ = true;
memset(frame_decoded_, 0, sizeof(frame_decoded_));
received_sps_.clear();
received_pps_.clear();
}
uint32_t VCMDecodingState::time_stamp() const {
@ -74,6 +78,24 @@ void VCMDecodingState::SetState(const VCMFrameBuffer* frame) {
temporal_id_ = frame->TemporalId();
tl0_pic_id_ = frame->Tl0PicId();
for (const NaluInfo& nalu : frame->GetNaluInfos()) {
if (nalu.type == H264::NaluType::kPps) {
if (nalu.pps_id < 0) {
LOG(LS_WARNING) << "Received pps without pps id.";
} else if (nalu.sps_id < 0) {
LOG(LS_WARNING) << "Received pps without sps id.";
} else {
received_pps_[nalu.pps_id] = nalu.sps_id;
}
} else if (nalu.type == H264::NaluType::kSps) {
if (nalu.sps_id < 0) {
LOG(LS_WARNING) << "Received sps without sps id.";
} else {
received_sps_.insert(nalu.sps_id);
}
}
}
if (UsingFlexibleMode(frame)) {
uint16_t frame_index = picture_id_ % kFrameDecodedLength;
if (in_initial_state_) {
@ -106,6 +128,8 @@ void VCMDecodingState::CopyFrom(const VCMDecodingState& state) {
in_initial_state_ = state.in_initial_state_;
frame_decoded_cleared_to_ = state.frame_decoded_cleared_to_;
memcpy(frame_decoded_, state.frame_decoded_, sizeof(frame_decoded_));
received_sps_ = state.received_sps_;
received_pps_ = state.received_pps_;
}
bool VCMDecodingState::UpdateEmptyFrame(const VCMFrameBuffer* frame) {
@ -183,8 +207,10 @@ bool VCMDecodingState::ContinuousFrame(const VCMFrameBuffer* frame) const {
// A key frame is always considered continuous as it doesn't refer to any
// frames and therefore won't introduce any errors even if prior frames are
// missing.
if (frame->FrameType() == kVideoFrameKey)
if (frame->FrameType() == kVideoFrameKey &&
HaveSpsAndPps(frame->GetNaluInfos())) {
return true;
}
// When in the initial state we always require a key frame to start decoding.
if (in_initial_state_)
return false;
@ -205,7 +231,8 @@ bool VCMDecodingState::ContinuousFrame(const VCMFrameBuffer* frame) const {
return ContinuousPictureId(frame->PictureId());
}
} else {
return ContinuousSeqNum(static_cast<uint16_t>(frame->GetLowSeqNum()));
return ContinuousSeqNum(static_cast<uint16_t>(frame->GetLowSeqNum())) &&
HaveSpsAndPps(frame->GetNaluInfos());
}
}
@ -282,4 +309,51 @@ bool VCMDecodingState::AheadOfFramesDecodedClearedTo(uint16_t index) const {
return diff > kFrameDecodedLength / 2;
}
bool VCMDecodingState::HaveSpsAndPps(const std::vector<NaluInfo>& nalus) const {
std::set<int> new_sps;
std::map<int, int> new_pps;
for (const NaluInfo& nalu : nalus) {
// Check if this nalu actually contains sps/pps information or dependencies.
if (nalu.sps_id == -1 && nalu.pps_id == -1)
continue;
switch (nalu.type) {
case H264::NaluType::kPps:
if (nalu.pps_id < 0) {
LOG(LS_WARNING) << "Received pps without pps id.";
} else if (nalu.sps_id < 0) {
LOG(LS_WARNING) << "Received pps without sps id.";
} else {
new_pps[nalu.pps_id] = nalu.sps_id;
}
break;
case H264::NaluType::kSps:
if (nalu.sps_id < 0) {
LOG(LS_WARNING) << "Received sps without sps id.";
} else {
new_sps.insert(nalu.sps_id);
}
break;
default: {
int needed_sps = -1;
auto pps_it = new_pps.find(nalu.pps_id);
if (pps_it != new_pps.end()) {
needed_sps = pps_it->second;
} else {
auto pps_it2 = received_pps_.find(nalu.pps_id);
if (pps_it2 == received_pps_.end()) {
return false;
}
needed_sps = pps_it2->second;
}
if (new_sps.find(needed_sps) == new_sps.end() &&
received_sps_.find(needed_sps) == received_sps_.end()) {
return false;
}
break;
}
}
}
return true;
}
} // namespace webrtc

8
webrtc/modules/video_coding/decoding_state.h
View File

@ -11,11 +11,16 @@
#ifndef WEBRTC_MODULES_VIDEO_CODING_DECODING_STATE_H_
#define WEBRTC_MODULES_VIDEO_CODING_DECODING_STATE_H_
#include <map>
#include <set>
#include <vector>
#include "webrtc/typedefs.h"
namespace webrtc {
// Forward declarations
struct NaluInfo;
class VCMFrameBuffer;
class VCMPacket;
@ -61,6 +66,7 @@ class VCMDecodingState {
bool UsingPictureId(const VCMFrameBuffer* frame) const;
bool UsingFlexibleMode(const VCMFrameBuffer* frame) const;
bool AheadOfFramesDecodedClearedTo(uint16_t index) const;
bool HaveSpsAndPps(const std::vector<NaluInfo>& nalus) const;
// Keep state of last decoded frame.
// TODO(mikhal/stefan): create designated classes to handle these types.
@ -75,6 +81,8 @@ class VCMDecodingState {
// Used to check references in flexible mode.
bool frame_decoded_[kFrameDecodedLength];
uint16_t frame_decoded_cleared_to_;
std::set<int> received_sps_;
std::map<int, int> received_pps_;
};
} // namespace webrtc

4
webrtc/modules/video_coding/frame_buffer.cc
View File

@ -66,6 +66,10 @@ bool VCMFrameBuffer::NonReference() const {
return _sessionInfo.NonReference();
}
std::vector<NaluInfo> VCMFrameBuffer::GetNaluInfos() const {
return _sessionInfo.GetNaluInfos();
}
void VCMFrameBuffer::SetGofInfo(const GofInfoVP9& gof_info, size_t idx) {
_sessionInfo.SetGofInfo(gof_info, idx);
// TODO(asapersson): Consider adding hdr->VP9.ref_picture_id for testing.

4
webrtc/modules/video_coding/frame_buffer.h
View File

@ -11,6 +11,8 @@
#ifndef WEBRTC_MODULES_VIDEO_CODING_FRAME_BUFFER_H_
#define WEBRTC_MODULES_VIDEO_CODING_FRAME_BUFFER_H_
#include <vector>
#include "webrtc/modules/include/module_common_types.h"
#include "webrtc/modules/video_coding/include/video_coding.h"
#include "webrtc/modules/video_coding/encoded_frame.h"
@ -61,6 +63,8 @@ class VCMFrameBuffer : public VCMEncodedFrame {
int Tl0PicId() const;
bool NonReference() const;
std::vector<NaluInfo> GetNaluInfos() const;
void SetGofInfo(const GofInfoVP9& gof_info, size_t idx);
// Increments a counter to keep track of the number of packets of this frame

86
webrtc/modules/video_coding/jitter_buffer_unittest.cc
View File

@ -12,7 +12,9 @@
#include <list>
#include <memory>
#include <vector>
#include "webrtc/common_video/h264/h264_common.h"
#include "webrtc/modules/video_coding/frame_buffer.h"
#include "webrtc/modules/video_coding/jitter_buffer.h"
#include "webrtc/modules/video_coding/media_opt_util.h"
@ -1160,6 +1162,89 @@ TEST_F(TestBasicJitterBuffer, H264InsertStartCode) {
jitter_buffer_->ReleaseFrame(frame_out);
}
TEST_F(TestBasicJitterBuffer, SpsAndPpsHandling) {
jitter_buffer_->SetDecodeErrorMode(kNoErrors);
packet_->timestamp = timestamp_;
packet_->frameType = kVideoFrameKey;
packet_->isFirstPacket = true;
packet_->markerBit = true;
packet_->codec = kVideoCodecH264;
packet_->video_header.codec = kRtpVideoH264;
packet_->video_header.codecHeader.H264.nalu_type = H264::NaluType::kIdr;
packet_->video_header.codecHeader.H264.nalus[0].type = H264::NaluType::kIdr;
packet_->video_header.codecHeader.H264.nalus[0].sps_id = -1;
packet_->video_header.codecHeader.H264.nalus[0].pps_id = 0;
packet_->video_header.codecHeader.H264.nalus_length = 1;
bool retransmitted = false;
EXPECT_EQ(kCompleteSession,
jitter_buffer_->InsertPacket(*packet_, &retransmitted));
// Not decodable since sps and pps are missing.
EXPECT_EQ(nullptr, DecodeCompleteFrame());
timestamp_ += 3000;
packet_->timestamp = timestamp_;
++seq_num_;
packet_->seqNum = seq_num_;
packet_->frameType = kVideoFrameKey;
packet_->isFirstPacket = true;
packet_->markerBit = false;
packet_->codec = kVideoCodecH264;
packet_->video_header.codec = kRtpVideoH264;
packet_->video_header.codecHeader.H264.nalu_type = H264::NaluType::kStapA;
packet_->video_header.codecHeader.H264.nalus[0].type = H264::NaluType::kSps;
packet_->video_header.codecHeader.H264.nalus[0].sps_id = 0;
packet_->video_header.codecHeader.H264.nalus[0].pps_id = -1;
packet_->video_header.codecHeader.H264.nalus[1].type = H264::NaluType::kPps;
packet_->video_header.codecHeader.H264.nalus[1].sps_id = 0;
packet_->video_header.codecHeader.H264.nalus[1].pps_id = 0;
packet_->video_header.codecHeader.H264.nalus_length = 2;
// Not complete since the marker bit hasn't been received.
EXPECT_EQ(kIncomplete,
jitter_buffer_->InsertPacket(*packet_, &retransmitted));
++seq_num_;
packet_->seqNum = seq_num_;
packet_->frameType = kVideoFrameKey;
packet_->isFirstPacket = false;
packet_->markerBit = true;
packet_->codec = kVideoCodecH264;
packet_->video_header.codec = kRtpVideoH264;
packet_->video_header.codecHeader.H264.nalu_type = H264::NaluType::kIdr;
packet_->video_header.codecHeader.H264.nalus[0].type = H264::NaluType::kIdr;
packet_->video_header.codecHeader.H264.nalus[0].sps_id = -1;
packet_->video_header.codecHeader.H264.nalus[0].pps_id = 0;
packet_->video_header.codecHeader.H264.nalus_length = 1;
// Complete and decodable since the pps and sps are received in the first
// packet of this frame.
EXPECT_EQ(kCompleteSession,
jitter_buffer_->InsertPacket(*packet_, &retransmitted));
VCMEncodedFrame* frame_out = DecodeCompleteFrame();
ASSERT_NE(nullptr, frame_out);
jitter_buffer_->ReleaseFrame(frame_out);
timestamp_ += 3000;
packet_->timestamp = timestamp_;
++seq_num_;
packet_->seqNum = seq_num_;
packet_->frameType = kVideoFrameDelta;
packet_->isFirstPacket = true;
packet_->markerBit = true;
packet_->codec = kVideoCodecH264;
packet_->video_header.codec = kRtpVideoH264;
packet_->video_header.codecHeader.H264.nalu_type = H264::NaluType::kSlice;
packet_->video_header.codecHeader.H264.nalus[0].type = H264::NaluType::kSlice;
packet_->video_header.codecHeader.H264.nalus[0].sps_id = -1;
packet_->video_header.codecHeader.H264.nalus[0].pps_id = 0;
packet_->video_header.codecHeader.H264.nalus_length = 1;
// Complete and decodable since sps, pps and key frame has been received.
EXPECT_EQ(kCompleteSession,
jitter_buffer_->InsertPacket(*packet_, &retransmitted));
frame_out = DecodeCompleteFrame();
ASSERT_NE(nullptr, frame_out);
jitter_buffer_->ReleaseFrame(frame_out);
}
// Test threshold conditions of decodable state.
TEST_F(TestBasicJitterBuffer, PacketLossWithSelectiveErrorsThresholdCheck) {
jitter_buffer_->SetDecodeErrorMode(kSelectiveErrors);
@ -2633,5 +2718,4 @@ TEST_F(TestJitterBufferNack, ResetByFutureKeyFrameDoesntError) {
nack_list = jitter_buffer_->GetNackList(&extended);
EXPECT_EQ(0u, nack_list.size());
}
} // namespace webrtc

13
webrtc/modules/video_coding/session_info.cc
View File

@ -111,6 +111,19 @@ bool VCMSessionInfo::NonReference() const {
return packets_.front().video_header.codecHeader.VP8.nonReference;
}
std::vector<NaluInfo> VCMSessionInfo::GetNaluInfos() const {
if (packets_.empty() || packets_.front().video_header.codec != kRtpVideoH264)
return std::vector<NaluInfo>();
std::vector<NaluInfo> nalu_infos;
for (const VCMPacket& packet : packets_) {
for (size_t i = 0; i < packet.video_header.codecHeader.H264.nalus_length;
++i) {
nalu_infos.push_back(packet.video_header.codecHeader.H264.nalus[i]);
}
}
return nalu_infos;
}
void VCMSessionInfo::SetGofInfo(const GofInfoVP9& gof_info, size_t idx) {
if (packets_.empty() || packets_.front().video_header.codec != kRtpVideoVp9 ||
packets_.front().video_header.codecHeader.VP9.flexible_mode) {

3
webrtc/modules/video_coding/session_info.h
View File

@ -12,6 +12,7 @@
#define WEBRTC_MODULES_VIDEO_CODING_SESSION_INFO_H_
#include <list>
#include <vector>
#include "webrtc/modules/include/module_common_types.h"
#include "webrtc/modules/video_coding/include/video_coding.h"
@ -81,6 +82,8 @@ class VCMSessionInfo {
int Tl0PicId() const;
bool NonReference() const;
std::vector<NaluInfo> GetNaluInfos() const;
void SetGofInfo(const GofInfoVP9& gof_info, size_t idx);
// The number of packets discarded because the decoder can't make use of