zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move AssembleFrame from PacketBuffer to RtpVideoStreamReceiver

Browse Source 
this is a step towards resolving own todo: making AssembleFrame part of
the VideoRtpDepacketizer interface and replacing codec check with a
call to a virtual function.
RtpVideoStreamReceiver has access to the VideoRtpDepacketizers,
PacketBuffer - hasn't.

Bug: None
Change-Id: I83df09975c092bdb71bab270ced356d79a50683d
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/168056
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Reviewed-by: Philip Eliasson <philipel@webrtc.org>
Commit-Queue: Danil Chapovalov <danilchap@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#30833}
This commit is contained in:
Danil Chapovalov
2020-03-19 13:56:11 +01:00
committed by Commit Bot
parent dc5522b4bf
commit 810b4ca386
4 changed files with 203 additions and 365 deletions
Download Patch File Download Diff File Expand all files Collapse all files

121
modules/video_coding/packet_buffer.cc
View File

@ -21,15 +21,12 @@
#include "absl/types/variant.h"
#include "api/array_view.h"
#include "api/rtp_packet_info.h"
#include "api/video/encoded_frame.h"
#include "api/video/video_frame_type.h"
#include "common_video/h264/h264_common.h"
#include "modules/rtp_rtcp/source/rtp_header_extensions.h"
#include "modules/rtp_rtcp/source/rtp_packet_received.h"
#include "modules/rtp_rtcp/source/rtp_video_header.h"
#include "modules/rtp_rtcp/source/video_rtp_depacketizer_av1.h"
#include "modules/video_coding/codecs/h264/include/h264_globals.h"
#include "modules/video_coding/frame_object.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
#include "rtc_base/numerics/mod_ops.h"
@ -135,7 +132,7 @@ PacketBuffer::InsertResult PacketBuffer::InsertPacket(
UpdateMissingPackets(seq_num);
result.frames = FindFrames(seq_num);
result.packets = FindFrames(seq_num);
return result;
}
@ -176,20 +173,6 @@ void PacketBuffer::ClearTo(uint16_t seq_num) {
}
}
void PacketBuffer::ClearInterval(uint16_t start_seq_num,
uint16_t stop_seq_num) {
size_t iterations = ForwardDiff<uint16_t>(start_seq_num, stop_seq_num + 1);
RTC_DCHECK_LE(iterations, buffer_.size());
uint16_t seq_num = start_seq_num;
for (size_t i = 0; i < iterations; ++i) {
size_t index = seq_num % buffer_.size();
RTC_DCHECK_EQ(buffer_[index].seq_num(), seq_num);
buffer_[index].packet = nullptr;
++seq_num;
}
}
void PacketBuffer::Clear() {
rtc::CritScope lock(&crit_);
for (StoredPacket& entry : buffer_) {
@ -208,7 +191,7 @@ PacketBuffer::InsertResult PacketBuffer::InsertPadding(uint16_t seq_num) {
PacketBuffer::InsertResult result;
rtc::CritScope lock(&crit_);
UpdateMissingPackets(seq_num);
result.frames = FindFrames(static_cast<uint16_t>(seq_num + 1));
result.packets = FindFrames(static_cast<uint16_t>(seq_num + 1));
return result;
}
@ -265,15 +248,15 @@ bool PacketBuffer::PotentialNewFrame(uint16_t seq_num) const {
return false;
}
std::vector<std::unique_ptr<RtpFrameObject>> PacketBuffer::FindFrames(
std::vector<std::unique_ptr<PacketBuffer::Packet>> PacketBuffer::FindFrames(
uint16_t seq_num) {
std::vector<std::unique_ptr<RtpFrameObject>> found_frames;
std::vector<std::unique_ptr<PacketBuffer::Packet>> found_frames;
for (size_t i = 0; i < buffer_.size() && PotentialNewFrame(seq_num); ++i) {
size_t index = seq_num % buffer_.size();
buffer_[index].continuous = true;
// If all packets of the frame is continuous, find the first packet of the
// frame and create an RtpFrameObject.
// frame and add all packets of the frame to the returned packets.
if (buffer_[index].frame_end()) {
uint16_t start_seq_num = seq_num;
@ -393,97 +376,29 @@ std::vector<std::unique_ptr<RtpFrameObject>> PacketBuffer::FindFrames(
}
}
if (auto frame = AssembleFrame(start_seq_num, seq_num)) {
found_frames.push_back(std::move(frame));
} else {
RTC_LOG(LS_ERROR) << "Failed to assemble frame from packets "
<< start_seq_num << "-" << seq_num;
const uint16_t end_seq_num = seq_num + 1;
// Use uint16_t type to handle sequence number wrap around case.
uint16_t num_packets = end_seq_num - start_seq_num;
found_frames.reserve(found_frames.size() + num_packets);
for (uint16_t i = start_seq_num; i != end_seq_num; ++i) {
StoredPacket& entry = buffer_[i % buffer_.size()];
RTC_DCHECK(entry.used());
RTC_DCHECK_EQ(i, entry.seq_num());
// Ensure frame boundary flags are properly set.
entry.packet->video_header.is_first_packet_in_frame =
(i == start_seq_num);
entry.packet->video_header.is_last_packet_in_frame = (i == seq_num);
found_frames.push_back(std::move(entry.packet));
}
missing_packets_.erase(missing_packets_.begin(),
missing_packets_.upper_bound(seq_num));
ClearInterval(start_seq_num, seq_num);
}
++seq_num;
}
return found_frames;
}
std::unique_ptr<RtpFrameObject> PacketBuffer::AssembleFrame(
uint16_t first_seq_num,
uint16_t last_seq_num) {
const uint16_t end_seq_num = last_seq_num + 1;
const uint16_t num_packets = end_seq_num - first_seq_num;
int max_nack_count = -1;
int64_t min_recv_time = std::numeric_limits<int64_t>::max();
int64_t max_recv_time = std::numeric_limits<int64_t>::min();
size_t frame_size = 0;
std::vector<rtc::ArrayView<const uint8_t>> payloads;
RtpPacketInfos::vector_type packet_infos;
payloads.reserve(num_packets);
packet_infos.reserve(num_packets);
for (uint16_t seq_num = first_seq_num; seq_num != end_seq_num; ++seq_num) {
const Packet& packet = GetPacket(seq_num);
max_nack_count = std::max(max_nack_count, packet.times_nacked);
min_recv_time =
std::min(min_recv_time, packet.packet_info.receive_time_ms());
max_recv_time =
std::max(max_recv_time, packet.packet_info.receive_time_ms());
frame_size += packet.video_payload.size();
payloads.emplace_back(packet.video_payload);
packet_infos.push_back(packet.packet_info);
}
const Packet& first_packet = GetPacket(first_seq_num);
rtc::scoped_refptr<EncodedImageBuffer> bitstream;
// TODO(danilchap): Hide codec-specific code paths behind an interface.
if (first_packet.codec() == VideoCodecType::kVideoCodecAV1) {
bitstream = VideoRtpDepacketizerAv1::AssembleFrame(payloads);
if (!bitstream) {
// Failed to assemble a frame. Discard and continue.
return nullptr;
}
} else {
bitstream = EncodedImageBuffer::Create(frame_size);
uint8_t* write_at = bitstream->data();
for (rtc::ArrayView<const uint8_t> payload : payloads) {
memcpy(write_at, payload.data(), payload.size());
write_at += payload.size();
}
RTC_DCHECK_EQ(write_at - bitstream->data(), bitstream->size());
}
const Packet& last_packet = GetPacket(last_seq_num);
return std::make_unique<RtpFrameObject>(
first_seq_num, //
last_seq_num, //
last_packet.marker_bit, //
max_nack_count, //
min_recv_time, //
max_recv_time, //
first_packet.timestamp, //
first_packet.ntp_time_ms, //
last_packet.video_header.video_timing, //
first_packet.payload_type, //
first_packet.codec(), //
last_packet.video_header.rotation, //
last_packet.video_header.content_type, //
first_packet.video_header, //
last_packet.video_header.color_space, //
RtpPacketInfos(std::move(packet_infos)), //
std::move(bitstream));
}
const PacketBuffer::Packet& PacketBuffer::GetPacket(uint16_t seq_num) const {
const StoredPacket& entry = buffer_[seq_num % buffer_.size()];
RTC_DCHECK(entry.used());
RTC_DCHECK_EQ(seq_num, entry.seq_num());
return *entry.packet;
}
void PacketBuffer::UpdateMissingPackets(uint16_t seq_num) {
if (!newest_inserted_seq_num_)
newest_inserted_seq_num_ = seq_num;

22
modules/video_coding/packet_buffer.h
View File

@ -21,7 +21,6 @@
#include "api/video/encoded_image.h"
#include "modules/rtp_rtcp/source/rtp_packet_received.h"
#include "modules/rtp_rtcp/source/rtp_video_header.h"
#include "modules/video_coding/frame_object.h"
#include "rtc_base/copy_on_write_buffer.h"
#include "rtc_base/critical_section.h"
#include "rtc_base/numerics/sequence_number_util.h"
@ -70,7 +69,7 @@ class PacketBuffer {
RtpPacketInfo packet_info;
};
struct InsertResult {
std::vector<std::unique_ptr<RtpFrameObject>> frames;
std::vector<std::unique_ptr<Packet>> packets;
// Indicates if the packet buffer was cleared, which means that a key
// frame request should be sent.
bool buffer_cleared = false;
@ -118,22 +117,9 @@ class PacketBuffer {
bool PotentialNewFrame(uint16_t seq_num) const
RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_);
// Test if all packets of a frame has arrived, and if so, creates a frame.
// Returns a vector of received frames.
std::vector<std::unique_ptr<RtpFrameObject>> FindFrames(uint16_t seq_num)
RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_);
std::unique_ptr<RtpFrameObject> AssembleFrame(uint16_t first_seq_num,
uint16_t last_seq_num)
RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_);
// Get the packet with sequence number |seq_num|.
const Packet& GetPacket(uint16_t seq_num) const
RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_);
// Clears the packet buffer from |start_seq_num| to |stop_seq_num| where the
// endpoints are inclusive.
void ClearInterval(uint16_t start_seq_num, uint16_t stop_seq_num)
// Test if all packets of a frame has arrived, and if so, returns packets to
// create frames.
std::vector<std::unique_ptr<Packet>> FindFrames(uint16_t seq_num)
RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_);
void UpdateMissingPackets(uint16_t seq_num)

349
modules/video_coding/packet_buffer_unittest.cc
View File

@ -40,29 +40,39 @@ constexpr int kMaxSize = 64;
void IgnoreResult(PacketBuffer::InsertResult /*result*/) {}
// Validates frame boundaries are valid and returns first sequence_number for
// each frame.
std::vector<uint16_t> StartSeqNums(
rtc::ArrayView<const std::unique_ptr<RtpFrameObject>> frames) {
rtc::ArrayView<const std::unique_ptr<PacketBuffer::Packet>> packets) {
std::vector<uint16_t> result;
for (const auto& frame : frames) {
result.push_back(frame->first_seq_num());
bool frame_boundary = true;
for (const auto& packet : packets) {
EXPECT_EQ(frame_boundary, packet->is_first_packet_in_frame());
if (packet->is_first_packet_in_frame()) {
result.push_back(packet->seq_num);
}
frame_boundary = packet->is_last_packet_in_frame();
}
EXPECT_TRUE(frame_boundary);
return result;
}
MATCHER_P(StartSeqNumsAre, seq_num, "") {
return Matches(ElementsAre(seq_num))(StartSeqNums(arg.frames));
return Matches(ElementsAre(seq_num))(StartSeqNums(arg.packets));
}
MATCHER_P2(StartSeqNumsAre, seq_num1, seq_num2, "") {
return Matches(ElementsAre(seq_num1, seq_num2))(StartSeqNums(arg.frames));
return Matches(ElementsAre(seq_num1, seq_num2))(StartSeqNums(arg.packets));
}
MATCHER(KeyFrame, "") {
return arg->frame_type() == VideoFrameType::kVideoFrameKey;
return arg->is_first_packet_in_frame() &&
arg->video_header.frame_type == VideoFrameType::kVideoFrameKey;
}
MATCHER(DeltaFrame, "") {
return arg->frame_type() == VideoFrameType::kVideoFrameDelta;
return arg->is_first_packet_in_frame() &&
arg->video_header.frame_type == VideoFrameType::kVideoFrameDelta;
}
struct PacketBufferInsertResult : public PacketBuffer::InsertResult {
@ -72,18 +82,15 @@ struct PacketBufferInsertResult : public PacketBuffer::InsertResult {
void PrintTo(const PacketBufferInsertResult& result, std::ostream* os) {
*os << "frames: { ";
for (size_t i = 0; i < result.frames.size(); ++i) {
const RtpFrameObject& frame = *result.frames[i];
if (i > 0) {
*os << ", ";
for (const auto& packet : result.packets) {
if (packet->is_first_packet_in_frame() &&
packet->is_last_packet_in_frame()) {
*os << "{sn: " << packet->seq_num << " }";
} else if (packet->is_first_packet_in_frame()) {
*os << "{sn: [" << packet->seq_num << "-";
} else if (packet->is_last_packet_in_frame()) {
*os << packet->seq_num << "] }, ";
}
*os << "{sn: ";
if (frame.first_seq_num() == frame.last_seq_num()) {
*os << frame.first_seq_num();
} else {
*os << "[" << frame.first_seq_num() << "-" << frame.last_seq_num() << "]";
}
*os << "}";
}
*os << " }";
if (result.buffer_cleared) {
@ -134,23 +141,23 @@ class PacketBufferTest : public ::testing::Test {
TEST_F(PacketBufferTest, InsertOnePacket) {
const uint16_t seq_num = Rand();
EXPECT_THAT(Insert(seq_num, kKeyFrame, kFirst, kLast).frames, SizeIs(1));
EXPECT_THAT(Insert(seq_num, kKeyFrame, kFirst, kLast).packets, SizeIs(1));
}
TEST_F(PacketBufferTest, InsertMultiplePackets) {
const uint16_t seq_num = Rand();
EXPECT_THAT(Insert(seq_num, kKeyFrame, kFirst, kLast).frames, SizeIs(1));
EXPECT_THAT(Insert(seq_num + 1, kKeyFrame, kFirst, kLast).frames, SizeIs(1));
EXPECT_THAT(Insert(seq_num + 2, kKeyFrame, kFirst, kLast).frames, SizeIs(1));
EXPECT_THAT(Insert(seq_num + 3, kKeyFrame, kFirst, kLast).frames, SizeIs(1));
EXPECT_THAT(Insert(seq_num, kKeyFrame, kFirst, kLast).packets, SizeIs(1));
EXPECT_THAT(Insert(seq_num + 1, kKeyFrame, kFirst, kLast).packets, SizeIs(1));
EXPECT_THAT(Insert(seq_num + 2, kKeyFrame, kFirst, kLast).packets, SizeIs(1));
EXPECT_THAT(Insert(seq_num + 3, kKeyFrame, kFirst, kLast).packets, SizeIs(1));
}
TEST_F(PacketBufferTest, InsertDuplicatePacket) {
const uint16_t seq_num = Rand();
EXPECT_THAT(Insert(seq_num, kKeyFrame, kFirst, kNotLast).frames, IsEmpty());
EXPECT_THAT(Insert(seq_num, kKeyFrame, kFirst, kNotLast).frames, IsEmpty());
EXPECT_THAT(Insert(seq_num + 1, kKeyFrame, kNotFirst, kLast).frames,
SizeIs(1));
EXPECT_THAT(Insert(seq_num, kKeyFrame, kFirst, kNotLast).packets, IsEmpty());
EXPECT_THAT(Insert(seq_num, kKeyFrame, kFirst, kNotLast).packets, IsEmpty());
EXPECT_THAT(Insert(seq_num + 1, kKeyFrame, kNotFirst, kLast).packets,
SizeIs(2));
}
TEST_F(PacketBufferTest, SeqNumWrapOneFrame) {
@ -166,57 +173,17 @@ TEST_F(PacketBufferTest, SeqNumWrapTwoFrames) {
}
TEST_F(PacketBufferTest, InsertOldPackets) {
EXPECT_THAT(Insert(100, kKeyFrame, kFirst, kNotLast).frames, IsEmpty());
EXPECT_THAT(Insert(102, kDeltaFrame, kFirst, kLast).frames, SizeIs(1));
EXPECT_THAT(Insert(101, kKeyFrame, kNotFirst, kLast).frames, SizeIs(1));
EXPECT_THAT(Insert(100, kKeyFrame, kFirst, kNotLast).packets, IsEmpty());
EXPECT_THAT(Insert(102, kDeltaFrame, kFirst, kLast).packets, SizeIs(1));
EXPECT_THAT(Insert(101, kKeyFrame, kNotFirst, kLast).packets, SizeIs(2));
EXPECT_THAT(Insert(100, kKeyFrame, kFirst, kNotLast).frames, IsEmpty());
EXPECT_THAT(Insert(100, kKeyFrame, kFirst, kNotLast).frames, IsEmpty());
EXPECT_THAT(Insert(102, kDeltaFrame, kFirst, kLast).frames, SizeIs(1));
EXPECT_THAT(Insert(100, kKeyFrame, kFirst, kNotLast).packets, IsEmpty());
EXPECT_THAT(Insert(100, kKeyFrame, kFirst, kNotLast).packets, IsEmpty());
EXPECT_THAT(Insert(102, kDeltaFrame, kFirst, kLast).packets, SizeIs(1));
packet_buffer_.ClearTo(102);
EXPECT_THAT(Insert(102, kDeltaFrame, kFirst, kLast).frames, IsEmpty());
EXPECT_THAT(Insert(103, kDeltaFrame, kFirst, kLast).frames, SizeIs(1));
}
TEST_F(PacketBufferTest, NackCount) {
const uint16_t seq_num = Rand();
auto packet = std::make_unique<PacketBuffer::Packet>();
packet->video_header.codec = kVideoCodecGeneric;
packet->seq_num = seq_num;
packet->video_header.frame_type = VideoFrameType::kVideoFrameKey;
packet->video_header.is_first_packet_in_frame = true;
packet->video_header.is_last_packet_in_frame = false;
packet->times_nacked = 0;
IgnoreResult(packet_buffer_.InsertPacket(std::move(packet)));
packet = std::make_unique<PacketBuffer::Packet>();
packet->seq_num = seq_num + 1;
packet->video_header.frame_type = VideoFrameType::kVideoFrameKey;
packet->video_header.is_first_packet_in_frame = false;
packet->video_header.is_last_packet_in_frame = false;
packet->times_nacked = 1;
IgnoreResult(packet_buffer_.InsertPacket(std::move(packet)));
packet = std::make_unique<PacketBuffer::Packet>();
packet->seq_num = seq_num + 2;
packet->video_header.frame_type = VideoFrameType::kVideoFrameKey;
packet->video_header.is_first_packet_in_frame = false;
packet->video_header.is_last_packet_in_frame = false;
packet->times_nacked = 3;
IgnoreResult(packet_buffer_.InsertPacket(std::move(packet)));
packet = std::make_unique<PacketBuffer::Packet>();
packet->seq_num = seq_num + 3;
packet->video_header.frame_type = VideoFrameType::kVideoFrameKey;
packet->video_header.is_first_packet_in_frame = false;
packet->video_header.is_last_packet_in_frame = true;
packet->times_nacked = 1;
auto frames = packet_buffer_.InsertPacket(std::move(packet)).frames;
ASSERT_THAT(frames, SizeIs(1));
EXPECT_EQ(frames.front()->times_nacked(), 3);
EXPECT_THAT(Insert(102, kDeltaFrame, kFirst, kLast).packets, IsEmpty());
EXPECT_THAT(Insert(103, kDeltaFrame, kFirst, kLast).packets, SizeIs(1));
}
TEST_F(PacketBufferTest, FrameSize) {
@ -229,8 +196,11 @@ TEST_F(PacketBufferTest, FrameSize) {
Insert(seq_num, kKeyFrame, kFirst, kNotLast, data1);
Insert(seq_num + 1, kKeyFrame, kNotFirst, kNotLast, data2);
Insert(seq_num + 2, kKeyFrame, kNotFirst, kNotLast, data3);
EXPECT_THAT(Insert(seq_num + 3, kKeyFrame, kNotFirst, kLast, data4).frames,
ElementsAre(Pointee(SizeIs(20))));
auto packets =
Insert(seq_num + 3, kKeyFrame, kNotFirst, kLast, data4).packets;
// Expect one frame of 4 packets.
EXPECT_THAT(StartSeqNums(packets), ElementsAre(seq_num));
EXPECT_THAT(packets, SizeIs(4));
}
TEST_F(PacketBufferTest, ExpandBuffer) {
@ -289,7 +259,7 @@ TEST_F(PacketBufferTest, TwoPacketsTwoFrames) {
TEST_F(PacketBufferTest, TwoPacketsOneFrames) {
const uint16_t seq_num = Rand();
EXPECT_THAT(Insert(seq_num, kKeyFrame, kFirst, kNotLast).frames, IsEmpty());
EXPECT_THAT(Insert(seq_num, kKeyFrame, kFirst, kNotLast).packets, IsEmpty());
EXPECT_THAT(Insert(seq_num + 1, kKeyFrame, kNotFirst, kLast),
StartSeqNumsAre(seq_num));
}
@ -297,8 +267,8 @@ TEST_F(PacketBufferTest, TwoPacketsOneFrames) {
TEST_F(PacketBufferTest, ThreePacketReorderingOneFrame) {
const uint16_t seq_num = Rand();
EXPECT_THAT(Insert(seq_num, kKeyFrame, kFirst, kNotLast).frames, IsEmpty());
EXPECT_THAT(Insert(seq_num + 2, kKeyFrame, kNotFirst, kLast).frames,
EXPECT_THAT(Insert(seq_num, kKeyFrame, kFirst, kNotLast).packets, IsEmpty());
EXPECT_THAT(Insert(seq_num + 2, kKeyFrame, kNotFirst, kLast).packets,
IsEmpty());
EXPECT_THAT(Insert(seq_num + 1, kKeyFrame, kNotFirst, kNotLast),
StartSeqNumsAre(seq_num));
@ -343,7 +313,7 @@ TEST_F(PacketBufferTest, DontClearNewerPacket) {
packet_buffer_.ClearTo(0);
EXPECT_THAT(Insert(2 * kStartSize, kKeyFrame, kFirst, kLast),
StartSeqNumsAre(2 * kStartSize));
EXPECT_THAT(Insert(3 * kStartSize + 1, kKeyFrame, kFirst, kNotLast).frames,
EXPECT_THAT(Insert(3 * kStartSize + 1, kKeyFrame, kFirst, kNotLast).packets,
IsEmpty());
packet_buffer_.ClearTo(2 * kStartSize);
EXPECT_THAT(Insert(3 * kStartSize + 2, kKeyFrame, kNotFirst, kLast),
@ -353,10 +323,11 @@ TEST_F(PacketBufferTest, DontClearNewerPacket) {
TEST_F(PacketBufferTest, OneIncompleteFrame) {
const uint16_t seq_num = Rand();
EXPECT_THAT(Insert(seq_num, kDeltaFrame, kFirst, kNotLast).frames, IsEmpty());
EXPECT_THAT(Insert(seq_num, kDeltaFrame, kFirst, kNotLast).packets,
IsEmpty());
EXPECT_THAT(Insert(seq_num + 1, kDeltaFrame, kNotFirst, kLast),
StartSeqNumsAre(seq_num));
EXPECT_THAT(Insert(seq_num - 1, kDeltaFrame, kNotFirst, kLast).frames,
EXPECT_THAT(Insert(seq_num - 1, kDeltaFrame, kNotFirst, kLast).packets,
IsEmpty());
}
@ -365,8 +336,9 @@ TEST_F(PacketBufferTest, TwoIncompleteFramesFullBuffer) {
for (int i = 1; i < kMaxSize - 1; ++i)
Insert(seq_num + i, kDeltaFrame, kNotFirst, kNotLast);
EXPECT_THAT(Insert(seq_num, kDeltaFrame, kFirst, kNotLast).frames, IsEmpty());
EXPECT_THAT(Insert(seq_num - 1, kDeltaFrame, kNotFirst, kLast).frames,
EXPECT_THAT(Insert(seq_num, kDeltaFrame, kFirst, kNotLast).packets,
IsEmpty());
EXPECT_THAT(Insert(seq_num - 1, kDeltaFrame, kNotFirst, kLast).packets,
IsEmpty());
}
@ -383,111 +355,6 @@ TEST_F(PacketBufferTest, FramesReordered) {
StartSeqNumsAre(seq_num + 2));
}
TEST_F(PacketBufferTest, GetBitstream) {
// "many bitstream, such data" with null termination.
uint8_t many[] = {0x6d, 0x61, 0x6e, 0x79, 0x20};
uint8_t bitstream[] = {0x62, 0x69, 0x74, 0x73, 0x74, 0x72,
0x65, 0x61, 0x6d, 0x2c, 0x20};
uint8_t such[] = {0x73, 0x75, 0x63, 0x68, 0x20};
uint8_t data[] = {0x64, 0x61, 0x74, 0x61, 0x0};
const uint16_t seq_num = Rand();
Insert(seq_num, kKeyFrame, kFirst, kNotLast, many);
Insert(seq_num + 1, kDeltaFrame, kNotFirst, kNotLast, bitstream);
Insert(seq_num + 2, kDeltaFrame, kNotFirst, kNotLast, such);
auto frames = Insert(seq_num + 3, kDeltaFrame, kNotFirst, kLast, data).frames;
ASSERT_THAT(frames, SizeIs(1));
EXPECT_EQ(frames[0]->first_seq_num(), seq_num);
EXPECT_THAT(rtc::MakeArrayView(frames[0]->data(), frames[0]->size()),
ElementsAreArray("many bitstream, such data"));
}
TEST_F(PacketBufferTest, GetBitstreamOneFrameOnePacket) {
uint8_t bitstream[] = "All the bitstream data for this frame!";
auto frames = Insert(0, kKeyFrame, kFirst, kLast, bitstream).frames;
ASSERT_THAT(StartSeqNums(frames), ElementsAre(0));
EXPECT_THAT(rtc::MakeArrayView(frames[0]->data(), frames[0]->size()),
ElementsAreArray(bitstream));
}
TEST_F(PacketBufferTest, GetBitstreamOneFrameFullBuffer) {
uint8_t data_arr[kStartSize][1];
uint8_t expected[kStartSize];
for (uint8_t i = 0; i < kStartSize; ++i) {
data_arr[i][0] = i;
expected[i] = i;
}
Insert(0, kKeyFrame, kFirst, kNotLast, data_arr[0]);
for (uint8_t i = 1; i < kStartSize - 1; ++i)
Insert(i, kKeyFrame, kNotFirst, kNotLast, data_arr[i]);
auto frames = Insert(kStartSize - 1, kKeyFrame, kNotFirst, kLast,
data_arr[kStartSize - 1])
.frames;
ASSERT_THAT(StartSeqNums(frames), ElementsAre(0));
EXPECT_THAT(rtc::MakeArrayView(frames[0]->data(), frames[0]->size()),
ElementsAreArray(expected));
}
TEST_F(PacketBufferTest, GetBitstreamAv1) {
const uint8_t data1[] = {0b01'01'0000, 0b0'0100'000, 'm', 'a', 'n', 'y', ' '};
const uint8_t data2[] = {0b10'01'0000, 'b', 'i', 't', 's', 0};
auto packet1 = std::make_unique<PacketBuffer::Packet>();
packet1->video_header.codec = kVideoCodecAV1;
packet1->seq_num = 13;
packet1->video_header.is_first_packet_in_frame = true;
packet1->video_header.is_last_packet_in_frame = false;
packet1->video_payload = data1;
auto frames = packet_buffer_.InsertPacket(std::move(packet1)).frames;
EXPECT_THAT(frames, IsEmpty());
auto packet2 = std::make_unique<PacketBuffer::Packet>();
packet2->video_header.codec = kVideoCodecAV1;
packet2->seq_num = 14;
packet2->video_header.is_first_packet_in_frame = false;
packet2->video_header.is_last_packet_in_frame = true;
packet2->video_payload = data2;
frames = packet_buffer_.InsertPacket(std::move(packet2)).frames;
ASSERT_THAT(frames, SizeIs(1));
EXPECT_EQ(frames[0]->first_seq_num(), 13);
EXPECT_THAT(rtc::MakeArrayView(frames[0]->data(), 2),
ElementsAre(0b0'0100'010, 10)); // obu_header and obu_size.
EXPECT_THAT(rtc::MakeArrayView(frames[0]->data() + 2, frames[0]->size() - 2),
ElementsAreArray("many bits"));
}
TEST_F(PacketBufferTest, GetBitstreamInvalidAv1) {
// Two av1 payloads that can't be combined into proper frame.
const uint8_t data1[] = {0b01'01'0000, 0b0'0100'000, 'm', 'a', 'n', 'y', ' '};
const uint8_t data2[] = {0b00'01'0000, 'b', 'i', 't', 's', 0};
auto packet1 = std::make_unique<PacketBuffer::Packet>();
packet1->video_header.codec = kVideoCodecAV1;
packet1->seq_num = 13;
packet1->video_header.is_first_packet_in_frame = true;
packet1->video_header.is_last_packet_in_frame = false;
packet1->video_payload = data1;
auto frames = packet_buffer_.InsertPacket(std::move(packet1)).frames;
EXPECT_THAT(frames, IsEmpty());
auto packet2 = std::make_unique<PacketBuffer::Packet>();
packet2->video_header.codec = kVideoCodecAV1;
packet2->seq_num = 14;
packet2->video_header.is_first_packet_in_frame = false;
packet2->video_header.is_last_packet_in_frame = true;
packet2->video_payload = data2;
frames = packet_buffer_.InsertPacket(std::move(packet2)).frames;
EXPECT_THAT(frames, IsEmpty());
}
TEST_F(PacketBufferTest, InsertPacketAfterSequenceNumberWrapAround) {
uint16_t kFirstSeqNum = 0;
uint32_t kTimestampDelta = 100;
@ -510,9 +377,11 @@ TEST_F(PacketBufferTest, InsertPacketAfterSequenceNumberWrapAround) {
for (int i = 0; i < 5; ++i) {
Insert(seq_num++, kKeyFrame, kNotFirst, kNotLast, {}, timestamp);
}
EXPECT_THAT(
Insert(seq_num++, kKeyFrame, kNotFirst, kLast, {}, timestamp).frames,
SizeIs(1));
auto packets =
Insert(seq_num++, kKeyFrame, kNotFirst, kLast, {}, timestamp).packets;
// One frame of 7 packets.
EXPECT_THAT(StartSeqNums(packets), SizeIs(1));
EXPECT_THAT(packets, SizeIs(7));
}
// If |sps_pps_idr_is_keyframe| is true, we require keyframes to contain
@ -614,7 +483,7 @@ TEST_P(PacketBufferH264ParameterizedTest, DontRemoveMissingPacketOnClearTo) {
InsertH264(2, kDeltaFrame, kFirst, kNotLast, 2);
packet_buffer_.ClearTo(0);
// Expect no frame because of missing of packet #1
EXPECT_THAT(InsertH264(3, kDeltaFrame, kNotFirst, kLast, 2).frames,
EXPECT_THAT(InsertH264(3, kDeltaFrame, kNotFirst, kLast, 2).packets,
IsEmpty());
}
@ -632,17 +501,19 @@ TEST_P(PacketBufferH264ParameterizedTest, GetBitstreamOneFrameFullBuffer) {
InsertH264(i, kKeyFrame, kNotFirst, kNotLast, 1, data_arr[i]);
}
auto frames = InsertH264(kStartSize - 1, kKeyFrame, kNotFirst, kLast, 1,
data_arr[kStartSize - 1])
.frames;
ASSERT_THAT(StartSeqNums(frames), ElementsAre(0));
EXPECT_THAT(rtc::MakeArrayView(frames[0]->data(), frames[0]->size()),
ElementsAreArray(expected));
auto packets = InsertH264(kStartSize - 1, kKeyFrame, kNotFirst, kLast, 1,
data_arr[kStartSize - 1])
.packets;
ASSERT_THAT(StartSeqNums(packets), ElementsAre(0));
EXPECT_THAT(packets, SizeIs(kStartSize));
for (size_t i = 0; i < packets.size(); ++i) {
EXPECT_THAT(packets[i]->video_payload, SizeIs(1)) << "Packet #" << i;
}
}
TEST_P(PacketBufferH264ParameterizedTest, GetBitstreamBufferPadding) {
uint16_t seq_num = Rand();
uint8_t data[] = "some plain old data";
rtc::CopyOnWriteBuffer data = "some plain old data";
auto packet = std::make_unique<PacketBuffer::Packet>();
auto& h264_header =
@ -655,14 +526,11 @@ TEST_P(PacketBufferH264ParameterizedTest, GetBitstreamBufferPadding) {
packet->video_payload = data;
packet->video_header.is_first_packet_in_frame = true;
packet->video_header.is_last_packet_in_frame = true;
auto frames = packet_buffer_.InsertPacket(std::move(packet)).frames;
auto frames = packet_buffer_.InsertPacket(std::move(packet)).packets;
ASSERT_THAT(frames, SizeIs(1));
EXPECT_EQ(frames[0]->first_seq_num(), seq_num);
EXPECT_EQ(frames[0]->EncodedImage().size(), sizeof(data));
EXPECT_EQ(frames[0]->EncodedImage().capacity(), sizeof(data));
EXPECT_THAT(rtc::MakeArrayView(frames[0]->data(), frames[0]->size()),
ElementsAreArray(data));
EXPECT_EQ(frames[0]->seq_num, seq_num);
EXPECT_EQ(frames[0]->video_payload, data);
}
TEST_P(PacketBufferH264ParameterizedTest, FrameResolution) {
@ -672,15 +540,13 @@ TEST_P(PacketBufferH264ParameterizedTest, FrameResolution) {
uint32_t height = 360;
uint32_t timestamp = 1000;
auto frames = InsertH264(seq_num, kKeyFrame, kFirst, kLast, timestamp, data,
width, height)
.frames;
auto packets = InsertH264(seq_num, kKeyFrame, kFirst, kLast, timestamp, data,
width, height)
.packets;
ASSERT_THAT(frames, SizeIs(1));
EXPECT_THAT(rtc::MakeArrayView(frames[0]->data(), frames[0]->size()),
ElementsAreArray(data));
EXPECT_EQ(frames[0]->EncodedImage()._encodedWidth, width);
EXPECT_EQ(frames[0]->EncodedImage()._encodedHeight, height);
ASSERT_THAT(packets, SizeIs(1));
EXPECT_EQ(packets[0]->video_header.width, width);
EXPECT_EQ(packets[0]->video_header.height, height);
}
TEST_P(PacketBufferH264ParameterizedTest, FrameResolutionNaluBeforeSPS) {
@ -690,18 +556,13 @@ TEST_P(PacketBufferH264ParameterizedTest, FrameResolutionNaluBeforeSPS) {
uint32_t height = 360;
uint32_t timestamp = 1000;
auto frames = InsertH264KeyFrameWithAud(seq_num, kKeyFrame, kFirst, kLast,
timestamp, data, width, height)
.frames;
auto packets = InsertH264KeyFrameWithAud(seq_num, kKeyFrame, kFirst, kLast,
timestamp, data, width, height)
.packets;
ASSERT_THAT(StartSeqNums(frames), ElementsAre(seq_num));
EXPECT_EQ(frames[0]->EncodedImage().size(), sizeof(data));
EXPECT_EQ(frames[0]->EncodedImage().capacity(), sizeof(data));
EXPECT_EQ(frames[0]->EncodedImage()._encodedWidth, width);
EXPECT_EQ(frames[0]->EncodedImage()._encodedHeight, height);
EXPECT_THAT(rtc::MakeArrayView(frames[0]->data(), frames[0]->size()),
ElementsAreArray(data));
ASSERT_THAT(StartSeqNums(packets), ElementsAre(seq_num));
EXPECT_EQ(packets[0]->video_header.width, width);
EXPECT_EQ(packets[0]->video_header.height, height);
}
TEST_F(PacketBufferTest, FreeSlotsOnFrameCreation) {
@ -740,20 +601,20 @@ TEST_F(PacketBufferTest, FramesAfterClear) {
Insert(9025, kDeltaFrame, kFirst, kLast);
Insert(9024, kKeyFrame, kFirst, kLast);
packet_buffer_.ClearTo(9025);
EXPECT_THAT(Insert(9057, kDeltaFrame, kFirst, kLast).frames, SizeIs(1));
EXPECT_THAT(Insert(9026, kDeltaFrame, kFirst, kLast).frames, SizeIs(1));
EXPECT_THAT(Insert(9057, kDeltaFrame, kFirst, kLast).packets, SizeIs(1));
EXPECT_THAT(Insert(9026, kDeltaFrame, kFirst, kLast).packets, SizeIs(1));
}
TEST_F(PacketBufferTest, SameFrameDifferentTimestamps) {
Insert(0, kKeyFrame, kFirst, kNotLast, {}, 1000);
EXPECT_THAT(Insert(1, kKeyFrame, kNotFirst, kLast, {}, 1001).frames,
EXPECT_THAT(Insert(1, kKeyFrame, kNotFirst, kLast, {}, 1001).packets,
IsEmpty());
}
TEST_F(PacketBufferTest, ContinuousSeqNumDoubleMarkerBit) {
Insert(2, kKeyFrame, kNotFirst, kNotLast);
Insert(1, kKeyFrame, kFirst, kLast);
EXPECT_THAT(Insert(3, kKeyFrame, kNotFirst, kLast).frames, IsEmpty());
EXPECT_THAT(Insert(3, kKeyFrame, kNotFirst, kLast).packets, IsEmpty());
}
TEST_F(PacketBufferTest, PacketTimestamps) {
@ -826,7 +687,8 @@ TEST_F(PacketBufferTest, IncomingCodecChange) {
packet->timestamp = 1;
packet->seq_num = 1;
packet->video_header.frame_type = VideoFrameType::kVideoFrameKey;
EXPECT_THAT(packet_buffer_.InsertPacket(std::move(packet)).frames, SizeIs(1));
EXPECT_THAT(packet_buffer_.InsertPacket(std::move(packet)).packets,
SizeIs(1));
packet = std::make_unique<PacketBuffer::Packet>();
packet->video_header.is_first_packet_in_frame = true;
@ -838,7 +700,8 @@ TEST_F(PacketBufferTest, IncomingCodecChange) {
packet->timestamp = 3;
packet->seq_num = 3;
packet->video_header.frame_type = VideoFrameType::kVideoFrameKey;
EXPECT_THAT(packet_buffer_.InsertPacket(std::move(packet)).frames, IsEmpty());
EXPECT_THAT(packet_buffer_.InsertPacket(std::move(packet)).packets,
IsEmpty());
packet = std::make_unique<PacketBuffer::Packet>();
packet->video_header.is_first_packet_in_frame = true;
@ -848,7 +711,8 @@ TEST_F(PacketBufferTest, IncomingCodecChange) {
packet->timestamp = 2;
packet->seq_num = 2;
packet->video_header.frame_type = VideoFrameType::kVideoFrameDelta;
EXPECT_THAT(packet_buffer_.InsertPacket(std::move(packet)).frames, SizeIs(2));
EXPECT_THAT(packet_buffer_.InsertPacket(std::move(packet)).packets,
SizeIs(2));
}
TEST_F(PacketBufferTest, TooManyNalusInPacket) {
@ -862,7 +726,8 @@ TEST_F(PacketBufferTest, TooManyNalusInPacket) {
auto& h264_header =
packet->video_header.video_type_header.emplace<RTPVideoHeaderH264>();
h264_header.nalus_length = kMaxNalusPerPacket;
EXPECT_THAT(packet_buffer_.InsertPacket(std::move(packet)).frames, IsEmpty());
EXPECT_THAT(packet_buffer_.InsertPacket(std::move(packet)).packets,
IsEmpty());
}
TEST_P(PacketBufferH264ParameterizedTest, OneFrameFillBuffer) {
@ -874,7 +739,7 @@ TEST_P(PacketBufferH264ParameterizedTest, OneFrameFillBuffer) {
}
TEST_P(PacketBufferH264ParameterizedTest, CreateFramesAfterFilledBuffer) {
EXPECT_THAT(InsertH264(kStartSize - 2, kKeyFrame, kFirst, kLast, 0).frames,
EXPECT_THAT(InsertH264(kStartSize - 2, kKeyFrame, kFirst, kLast, 0).packets,
SizeIs(1));
InsertH264(kStartSize, kDeltaFrame, kFirst, kNotLast, 2000);
@ -882,7 +747,7 @@ TEST_P(PacketBufferH264ParameterizedTest, CreateFramesAfterFilledBuffer) {
InsertH264(kStartSize + i, kDeltaFrame, kNotFirst, kNotLast, 2000);
EXPECT_THAT(
InsertH264(kStartSize + kStartSize, kDeltaFrame, kNotFirst, kLast, 2000)
.frames,
.packets,
IsEmpty());
EXPECT_THAT(InsertH264(kStartSize - 1, kKeyFrame, kFirst, kLast, 1000),
@ -908,7 +773,7 @@ TEST_P(PacketBufferH264ParameterizedTest, ClearMissingPacketsOnKeyframe) {
TEST_P(PacketBufferH264ParameterizedTest, FindFramesOnPadding) {
EXPECT_THAT(InsertH264(0, kKeyFrame, kFirst, kLast, 1000),
StartSeqNumsAre(0));
EXPECT_THAT(InsertH264(2, kDeltaFrame, kFirst, kLast, 1000).frames,
EXPECT_THAT(InsertH264(2, kDeltaFrame, kFirst, kLast, 1000).packets,
IsEmpty());
EXPECT_THAT(packet_buffer_.InsertPadding(1), StartSeqNumsAre(2));
@ -945,7 +810,7 @@ TEST_F(PacketBufferH264IdrIsKeyframeTest, IdrIsKeyframe) {
packet->video_header.video_type_header.emplace<RTPVideoHeaderH264>();
h264_header.nalus[0].type = H264::NaluType::kIdr;
h264_header.nalus_length = 1;
EXPECT_THAT(packet_buffer_.InsertPacket(std::move(packet)).frames,
EXPECT_THAT(packet_buffer_.InsertPacket(std::move(packet)).packets,
ElementsAre(KeyFrame()));
}
@ -958,7 +823,7 @@ TEST_F(PacketBufferH264IdrIsKeyframeTest, SpsPpsIdrIsKeyframe) {
h264_header.nalus[2].type = H264::NaluType::kIdr;
h264_header.nalus_length = 3;
EXPECT_THAT(packet_buffer_.InsertPacket(std::move(packet)).frames,
EXPECT_THAT(packet_buffer_.InsertPacket(std::move(packet)).packets,
ElementsAre(KeyFrame()));
}
@ -976,7 +841,7 @@ TEST_F(PacketBufferH264SpsPpsIdrIsKeyframeTest, IdrIsNotKeyframe) {
h264_header.nalus[0].type = H264::NaluType::kIdr;
h264_header.nalus_length = 1;
EXPECT_THAT(packet_buffer_.InsertPacket(std::move(packet)).frames,
EXPECT_THAT(packet_buffer_.InsertPacket(std::move(packet)).packets,
ElementsAre(DeltaFrame()));
}
@ -988,7 +853,7 @@ TEST_F(PacketBufferH264SpsPpsIdrIsKeyframeTest, SpsPpsIsNotKeyframe) {
h264_header.nalus[1].type = H264::NaluType::kPps;
h264_header.nalus_length = 2;
EXPECT_THAT(packet_buffer_.InsertPacket(std::move(packet)).frames,
EXPECT_THAT(packet_buffer_.InsertPacket(std::move(packet)).packets,
ElementsAre(DeltaFrame()));
}
@ -1001,7 +866,7 @@ TEST_F(PacketBufferH264SpsPpsIdrIsKeyframeTest, SpsPpsIdrIsKeyframe) {
h264_header.nalus[2].type = H264::NaluType::kIdr;
h264_header.nalus_length = 3;
EXPECT_THAT(packet_buffer_.InsertPacket(std::move(packet)).frames,
EXPECT_THAT(packet_buffer_.InsertPacket(std::move(packet)).packets,
ElementsAre(KeyFrame()));
}

76
video/rtp_video_stream_receiver.cc
View File

@ -36,6 +36,7 @@
#include "modules/rtp_rtcp/source/rtp_packet_received.h"
#include "modules/rtp_rtcp/source/rtp_rtcp_config.h"
#include "modules/rtp_rtcp/source/video_rtp_depacketizer.h"
#include "modules/rtp_rtcp/source/video_rtp_depacketizer_av1.h"
#include "modules/rtp_rtcp/source/video_rtp_depacketizer_raw.h"
#include "modules/utility/include/process_thread.h"
#include "modules/video_coding/frame_object.h"
@ -707,9 +708,80 @@ bool RtpVideoStreamReceiver::IsDecryptable() const {
void RtpVideoStreamReceiver::OnInsertedPacket(
video_coding::PacketBuffer::InsertResult result) {
for (std::unique_ptr<video_coding::RtpFrameObject>& frame : result.frames) {
OnAssembledFrame(std::move(frame));
video_coding::PacketBuffer::Packet* first_packet = nullptr;
int max_nack_count;
int64_t min_recv_time;
int64_t max_recv_time;
int frame_size;
std::vector<rtc::ArrayView<const uint8_t>> payloads;
RtpPacketInfos::vector_type packet_infos;
bool frame_boundary = true;
for (auto& packet : result.packets) {
// PacketBuffer promisses frame boundaries are correctly set on each
// packet. Document that assumption with the DCHECKs.
RTC_DCHECK_EQ(frame_boundary, packet->is_first_packet_in_frame());
if (packet->is_first_packet_in_frame()) {
first_packet = packet.get();
max_nack_count = packet->times_nacked;
min_recv_time = packet->packet_info.receive_time_ms();
max_recv_time = packet->packet_info.receive_time_ms();
frame_size = packet->video_payload.size();
payloads.clear();
packet_infos.clear();
} else {
max_nack_count = std::max(max_nack_count, packet->times_nacked);
min_recv_time =
std::min(min_recv_time, packet->packet_info.receive_time_ms());
max_recv_time =
std::max(max_recv_time, packet->packet_info.receive_time_ms());
frame_size += packet->video_payload.size();
}
payloads.emplace_back(packet->video_payload);
packet_infos.push_back(packet->packet_info);
frame_boundary = packet->is_last_packet_in_frame();
if (packet->is_last_packet_in_frame()) {
rtc::scoped_refptr<EncodedImageBuffer> bitstream;
// TODO(danilchap): Hide codec-specific code paths behind an interface.
if (first_packet->codec() == VideoCodecType::kVideoCodecAV1) {
bitstream = VideoRtpDepacketizerAv1::AssembleFrame(payloads);
if (!bitstream) {
// Failed to assemble a frame. Discard and continue.
continue;
}
} else {
bitstream = EncodedImageBuffer::Create(frame_size);
uint8_t* write_at = bitstream->data();
for (rtc::ArrayView<const uint8_t> payload : payloads) {
memcpy(write_at, payload.data(), payload.size());
write_at += payload.size();
}
RTC_DCHECK_EQ(write_at - bitstream->data(), bitstream->size());
}
const video_coding::PacketBuffer::Packet& last_packet = *packet;
OnAssembledFrame(std::make_unique<video_coding::RtpFrameObject>(
first_packet->seq_num, //
last_packet.seq_num, //
last_packet.marker_bit, //
max_nack_count, //
min_recv_time, //
max_recv_time, //
first_packet->timestamp, //
first_packet->ntp_time_ms, //
last_packet.video_header.video_timing, //
first_packet->payload_type, //
first_packet->codec(), //
last_packet.video_header.rotation, //
last_packet.video_header.content_type, //
first_packet->video_header, //
last_packet.video_header.color_space, //
RtpPacketInfos(std::move(packet_infos)), //
std::move(bitstream)));
}
}
RTC_DCHECK(frame_boundary);
if (result.buffer_cleared) {
RequestKeyFrame();
}