New H264PacketBuffer consolidating a bunch of H264 specific hacks into one class.
Bug: webrtc:12579 Change-Id: Idea35983e204e4a3f8628d5b4eb587bbdbff5877 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/227286 Reviewed-by: Niels Moller <nisse@webrtc.org> Reviewed-by: Erik Språng <sprang@webrtc.org> Reviewed-by: Danil Chapovalov <danilchap@webrtc.org> Commit-Queue: Philip Eliasson <philipel@webrtc.org> Cr-Commit-Position: refs/heads/main@{#34999}
This commit is contained in:
@ -159,6 +159,7 @@ rtc_library("rtp_video_frame_assembler") {
|
|||||||
":encoded_frame",
|
":encoded_frame",
|
||||||
"../../modules/rtp_rtcp:rtp_rtcp",
|
"../../modules/rtp_rtcp:rtp_rtcp",
|
||||||
"../../modules/rtp_rtcp:rtp_rtcp_format",
|
"../../modules/rtp_rtcp:rtp_rtcp_format",
|
||||||
|
"../../modules/video_coding:packet_buffer",
|
||||||
"../../modules/video_coding:video_coding",
|
"../../modules/video_coding:video_coding",
|
||||||
"../../rtc_base:logging",
|
"../../rtc_base:logging",
|
||||||
]
|
]
|
||||||
|
@ -98,6 +98,56 @@ rtc_library("nack_requester") {
|
|||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|
||||||
|
rtc_library("packet_buffer") {
|
||||||
|
sources = [
|
||||||
|
"packet_buffer.cc",
|
||||||
|
"packet_buffer.h",
|
||||||
|
]
|
||||||
|
deps = [
|
||||||
|
":codec_globals_headers",
|
||||||
|
"../../api:array_view",
|
||||||
|
"../../api:rtp_packet_info",
|
||||||
|
"../../api/units:timestamp",
|
||||||
|
"../../api/video:encoded_image",
|
||||||
|
"../../api/video:video_frame_type",
|
||||||
|
"../../common_video",
|
||||||
|
"../../rtc_base:checks",
|
||||||
|
"../../rtc_base:logging",
|
||||||
|
"../../rtc_base:rtc_base_approved",
|
||||||
|
"../../rtc_base:rtc_numerics",
|
||||||
|
"../rtp_rtcp:rtp_rtcp_format",
|
||||||
|
"../rtp_rtcp:rtp_video_header",
|
||||||
|
]
|
||||||
|
absl_deps = [
|
||||||
|
"//third_party/abseil-cpp/absl/base:core_headers",
|
||||||
|
"//third_party/abseil-cpp/absl/types:variant",
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
rtc_library("h264_packet_buffer") {
|
||||||
|
sources = [
|
||||||
|
"h264_packet_buffer.cc",
|
||||||
|
"h264_packet_buffer.h",
|
||||||
|
]
|
||||||
|
deps = [
|
||||||
|
":codec_globals_headers",
|
||||||
|
":packet_buffer",
|
||||||
|
"../../api:array_view",
|
||||||
|
"../../api:rtp_packet_info",
|
||||||
|
"../../api/units:timestamp",
|
||||||
|
"../../api/video:encoded_image",
|
||||||
|
"../../api/video:video_frame_type",
|
||||||
|
"../../common_video",
|
||||||
|
"../../rtc_base:checks",
|
||||||
|
"../../rtc_base:logging",
|
||||||
|
"../../rtc_base:rtc_base_approved",
|
||||||
|
"../../rtc_base:rtc_numerics",
|
||||||
|
"../rtp_rtcp:rtp_rtcp_format",
|
||||||
|
"../rtp_rtcp:rtp_video_header",
|
||||||
|
]
|
||||||
|
absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
|
||||||
|
}
|
||||||
|
|
||||||
rtc_library("video_coding") {
|
rtc_library("video_coding") {
|
||||||
visibility = [ "*" ]
|
visibility = [ "*" ]
|
||||||
sources = [
|
sources = [
|
||||||
@ -128,8 +178,6 @@ rtc_library("video_coding") {
|
|||||||
"loss_notification_controller.h",
|
"loss_notification_controller.h",
|
||||||
"media_opt_util.cc",
|
"media_opt_util.cc",
|
||||||
"media_opt_util.h",
|
"media_opt_util.h",
|
||||||
"packet_buffer.cc",
|
|
||||||
"packet_buffer.h",
|
|
||||||
"rtp_frame_id_only_ref_finder.cc",
|
"rtp_frame_id_only_ref_finder.cc",
|
||||||
"rtp_frame_id_only_ref_finder.h",
|
"rtp_frame_id_only_ref_finder.h",
|
||||||
"rtp_frame_reference_finder.cc",
|
"rtp_frame_reference_finder.cc",
|
||||||
@ -158,6 +206,7 @@ rtc_library("video_coding") {
|
|||||||
deps = [
|
deps = [
|
||||||
":codec_globals_headers",
|
":codec_globals_headers",
|
||||||
":encoded_frame",
|
":encoded_frame",
|
||||||
|
":packet_buffer",
|
||||||
":video_codec_interface",
|
":video_codec_interface",
|
||||||
":video_coding_utility",
|
":video_coding_utility",
|
||||||
":webrtc_vp9_helpers",
|
":webrtc_vp9_helpers",
|
||||||
@ -962,6 +1011,7 @@ if (rtc_include_tests) {
|
|||||||
"frame_buffer2_unittest.cc",
|
"frame_buffer2_unittest.cc",
|
||||||
"frame_dependencies_calculator_unittest.cc",
|
"frame_dependencies_calculator_unittest.cc",
|
||||||
"generic_decoder_unittest.cc",
|
"generic_decoder_unittest.cc",
|
||||||
|
"h264_packet_buffer_unittest.cc",
|
||||||
"h264_sprop_parameter_sets_unittest.cc",
|
"h264_sprop_parameter_sets_unittest.cc",
|
||||||
"h264_sps_pps_tracker_unittest.cc",
|
"h264_sps_pps_tracker_unittest.cc",
|
||||||
"histogram_unittest.cc",
|
"histogram_unittest.cc",
|
||||||
@ -1005,7 +1055,9 @@ if (rtc_include_tests) {
|
|||||||
":codec_globals_headers",
|
":codec_globals_headers",
|
||||||
":encoded_frame",
|
":encoded_frame",
|
||||||
":frame_dependencies_calculator",
|
":frame_dependencies_calculator",
|
||||||
|
":h264_packet_buffer",
|
||||||
":nack_requester",
|
":nack_requester",
|
||||||
|
":packet_buffer",
|
||||||
":simulcast_test_fixture_impl",
|
":simulcast_test_fixture_impl",
|
||||||
":video_codec_interface",
|
":video_codec_interface",
|
||||||
":video_codecs_test_framework",
|
":video_codecs_test_framework",
|
||||||
@ -1033,6 +1085,7 @@ if (rtc_include_tests) {
|
|||||||
"../../api/test/video:function_video_factory",
|
"../../api/test/video:function_video_factory",
|
||||||
"../../api/video:builtin_video_bitrate_allocator_factory",
|
"../../api/video:builtin_video_bitrate_allocator_factory",
|
||||||
"../../api/video:encoded_frame",
|
"../../api/video:encoded_frame",
|
||||||
|
"../../api/video:render_resolution",
|
||||||
"../../api/video:video_adaptation",
|
"../../api/video:video_adaptation",
|
||||||
"../../api/video:video_bitrate_allocation",
|
"../../api/video:video_bitrate_allocation",
|
||||||
"../../api/video:video_bitrate_allocator",
|
"../../api/video:video_bitrate_allocator",
|
||||||
@ -1055,6 +1108,7 @@ if (rtc_include_tests) {
|
|||||||
"../../rtc_base:task_queue_for_test",
|
"../../rtc_base:task_queue_for_test",
|
||||||
"../../rtc_base/experiments:jitter_upper_bound_experiment",
|
"../../rtc_base/experiments:jitter_upper_bound_experiment",
|
||||||
"../../rtc_base/synchronization:mutex",
|
"../../rtc_base/synchronization:mutex",
|
||||||
|
"../../rtc_base/system:unused",
|
||||||
"../../system_wrappers",
|
"../../system_wrappers",
|
||||||
"../../system_wrappers:field_trial",
|
"../../system_wrappers:field_trial",
|
||||||
"../../system_wrappers:metrics",
|
"../../system_wrappers:metrics",
|
||||||
|
287
modules/video_coding/h264_packet_buffer.cc
Normal file
287
modules/video_coding/h264_packet_buffer.cc
Normal file
@ -0,0 +1,287 @@
|
|||||||
|
/*
|
||||||
|
* Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
|
||||||
|
*
|
||||||
|
* Use of this source code is governed by a BSD-style license
|
||||||
|
* that can be found in the LICENSE file in the root of the source
|
||||||
|
* tree. An additional intellectual property rights grant can be found
|
||||||
|
* in the file PATENTS. All contributing project authors may
|
||||||
|
* be found in the AUTHORS file in the root of the source tree.
|
||||||
|
*/
|
||||||
|
|
||||||
|
#include "modules/video_coding/h264_packet_buffer.h"
|
||||||
|
|
||||||
|
#include <algorithm>
|
||||||
|
#include <cstdint>
|
||||||
|
#include <utility>
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
|
#include "api/array_view.h"
|
||||||
|
#include "api/rtp_packet_info.h"
|
||||||
|
#include "api/video/video_frame_type.h"
|
||||||
|
#include "common_video/h264/h264_common.h"
|
||||||
|
#include "modules/rtp_rtcp/source/rtp_header_extensions.h"
|
||||||
|
#include "modules/rtp_rtcp/source/rtp_packet_received.h"
|
||||||
|
#include "modules/rtp_rtcp/source/rtp_video_header.h"
|
||||||
|
#include "modules/video_coding/codecs/h264/include/h264_globals.h"
|
||||||
|
#include "rtc_base/checks.h"
|
||||||
|
#include "rtc_base/copy_on_write_buffer.h"
|
||||||
|
#include "rtc_base/logging.h"
|
||||||
|
#include "rtc_base/numerics/sequence_number_util.h"
|
||||||
|
|
||||||
|
namespace webrtc {
|
||||||
|
namespace {
|
||||||
|
int64_t EuclideanMod(int64_t n, int64_t div) {
|
||||||
|
RTC_DCHECK_GT(div, 0);
|
||||||
|
return (n %= div) < 0 ? n + div : n;
|
||||||
|
}
|
||||||
|
|
||||||
|
rtc::ArrayView<const NaluInfo> GetNaluInfos(
|
||||||
|
const RTPVideoHeaderH264& h264_header) {
|
||||||
|
if (h264_header.nalus_length > kMaxNalusPerPacket) {
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
|
||||||
|
return rtc::MakeArrayView(h264_header.nalus, h264_header.nalus_length);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool IsFirstPacketOfFragment(const RTPVideoHeaderH264& h264_header) {
|
||||||
|
return h264_header.nalus_length > 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool BeginningOfIdr(const H264PacketBuffer::Packet& packet) {
|
||||||
|
const auto& h264_header =
|
||||||
|
absl::get<RTPVideoHeaderH264>(packet.video_header.video_type_header);
|
||||||
|
const bool contains_idr_nalu =
|
||||||
|
absl::c_any_of(GetNaluInfos(h264_header), [](const auto& nalu_info) {
|
||||||
|
return nalu_info.type == H264::NaluType::kIdr;
|
||||||
|
});
|
||||||
|
switch (h264_header.packetization_type) {
|
||||||
|
case kH264StapA:
|
||||||
|
case kH264SingleNalu: {
|
||||||
|
return contains_idr_nalu;
|
||||||
|
}
|
||||||
|
case kH264FuA: {
|
||||||
|
return contains_idr_nalu && IsFirstPacketOfFragment(h264_header);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
bool HasSps(const H264PacketBuffer::Packet& packet) {
|
||||||
|
auto& h264_header =
|
||||||
|
absl::get<RTPVideoHeaderH264>(packet.video_header.video_type_header);
|
||||||
|
return absl::c_any_of(GetNaluInfos(h264_header), [](const auto& nalu_info) {
|
||||||
|
return nalu_info.type == H264::NaluType::kSps;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// TODO(bugs.webrtc.org/13157): Update the H264 depacketizer so we don't have to
|
||||||
|
// fiddle with the payload at this point.
|
||||||
|
rtc::CopyOnWriteBuffer FixVideoPayload(rtc::ArrayView<const uint8_t> payload,
|
||||||
|
const RTPVideoHeader& video_header) {
|
||||||
|
constexpr uint8_t kStartCode[] = {0, 0, 0, 1};
|
||||||
|
|
||||||
|
const auto& h264_header =
|
||||||
|
absl::get<RTPVideoHeaderH264>(video_header.video_type_header);
|
||||||
|
|
||||||
|
rtc::CopyOnWriteBuffer result;
|
||||||
|
switch (h264_header.packetization_type) {
|
||||||
|
case kH264StapA: {
|
||||||
|
const uint8_t* payload_end = payload.data() + payload.size();
|
||||||
|
const uint8_t* nalu_ptr = payload.data() + 1;
|
||||||
|
while (nalu_ptr < payload_end - 1) {
|
||||||
|
// The first two bytes describe the length of the segment, where a
|
||||||
|
// segment is the nalu type plus nalu payload.
|
||||||
|
uint16_t segment_length = nalu_ptr[0] << 8 | nalu_ptr[1];
|
||||||
|
nalu_ptr += 2;
|
||||||
|
|
||||||
|
if (nalu_ptr + segment_length <= payload_end) {
|
||||||
|
result.AppendData(kStartCode);
|
||||||
|
result.AppendData(nalu_ptr, segment_length);
|
||||||
|
}
|
||||||
|
nalu_ptr += segment_length;
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
case kH264FuA: {
|
||||||
|
if (IsFirstPacketOfFragment(h264_header)) {
|
||||||
|
result.AppendData(kStartCode);
|
||||||
|
}
|
||||||
|
result.AppendData(payload.data(), payload.size());
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
case kH264SingleNalu: {
|
||||||
|
result.AppendData(kStartCode);
|
||||||
|
result.AppendData(payload.data(), payload.size());
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
RTC_NOTREACHED();
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace
|
||||||
|
|
||||||
|
H264PacketBuffer::H264PacketBuffer(bool idr_only_keyframes_allowed)
|
||||||
|
: idr_only_keyframes_allowed_(idr_only_keyframes_allowed) {}
|
||||||
|
|
||||||
|
H264PacketBuffer::InsertResult H264PacketBuffer::InsertPacket(
|
||||||
|
std::unique_ptr<Packet> packet) {
|
||||||
|
RTC_DCHECK(packet->video_header.codec == kVideoCodecH264);
|
||||||
|
|
||||||
|
InsertResult result;
|
||||||
|
if (!absl::holds_alternative<RTPVideoHeaderH264>(
|
||||||
|
packet->video_header.video_type_header)) {
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
int64_t unwrapped_seq_num = seq_num_unwrapper_.Unwrap(packet->seq_num);
|
||||||
|
auto& packet_slot = GetPacket(unwrapped_seq_num);
|
||||||
|
if (packet_slot != nullptr &&
|
||||||
|
AheadOrAt(packet_slot->timestamp, packet->timestamp)) {
|
||||||
|
// The incoming `packet` is old or a duplicate.
|
||||||
|
return result;
|
||||||
|
} else {
|
||||||
|
packet_slot = std::move(packet);
|
||||||
|
}
|
||||||
|
|
||||||
|
result.packets = FindFrames(unwrapped_seq_num);
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::unique_ptr<H264PacketBuffer::Packet>& H264PacketBuffer::GetPacket(
|
||||||
|
int64_t unwrapped_seq_num) {
|
||||||
|
return buffer_[EuclideanMod(unwrapped_seq_num, kBufferSize)];
|
||||||
|
}
|
||||||
|
|
||||||
|
bool H264PacketBuffer::BeginningOfStream(
|
||||||
|
const H264PacketBuffer::Packet& packet) const {
|
||||||
|
return HasSps(packet) ||
|
||||||
|
(idr_only_keyframes_allowed_ && BeginningOfIdr(packet));
|
||||||
|
}
|
||||||
|
|
||||||
|
std::vector<std::unique_ptr<H264PacketBuffer::Packet>>
|
||||||
|
H264PacketBuffer::FindFrames(int64_t unwrapped_seq_num) {
|
||||||
|
std::vector<std::unique_ptr<Packet>> found_frames;
|
||||||
|
|
||||||
|
Packet* packet = GetPacket(unwrapped_seq_num).get();
|
||||||
|
RTC_CHECK(packet != nullptr);
|
||||||
|
|
||||||
|
// Check if the packet is continuous or the beginning of a new coded video
|
||||||
|
// sequence.
|
||||||
|
if (unwrapped_seq_num - 1 != last_continuous_unwrapped_seq_num_) {
|
||||||
|
if (unwrapped_seq_num <= last_continuous_unwrapped_seq_num_ ||
|
||||||
|
!BeginningOfStream(*packet)) {
|
||||||
|
return found_frames;
|
||||||
|
}
|
||||||
|
|
||||||
|
last_continuous_unwrapped_seq_num_ = unwrapped_seq_num;
|
||||||
|
}
|
||||||
|
|
||||||
|
for (int64_t seq_num = unwrapped_seq_num;
|
||||||
|
seq_num < unwrapped_seq_num + kBufferSize;) {
|
||||||
|
RTC_DCHECK_GE(seq_num, *last_continuous_unwrapped_seq_num_);
|
||||||
|
|
||||||
|
// Packets that were never assembled into a completed frame will stay in
|
||||||
|
// the 'buffer_'. Check that the `packet` sequence number match the expected
|
||||||
|
// unwrapped sequence number.
|
||||||
|
if (static_cast<uint16_t>(seq_num) != packet->seq_num) {
|
||||||
|
return found_frames;
|
||||||
|
}
|
||||||
|
|
||||||
|
last_continuous_unwrapped_seq_num_ = seq_num;
|
||||||
|
// Last packet of the frame, try to assemble the frame.
|
||||||
|
if (packet->marker_bit) {
|
||||||
|
uint32_t rtp_timestamp = packet->timestamp;
|
||||||
|
|
||||||
|
// Iterate backwards to find where the frame starts.
|
||||||
|
for (int64_t seq_num_start = seq_num;
|
||||||
|
seq_num_start > seq_num - kBufferSize; --seq_num_start) {
|
||||||
|
auto& prev_packet = GetPacket(seq_num_start - 1);
|
||||||
|
|
||||||
|
if (prev_packet == nullptr || prev_packet->timestamp != rtp_timestamp) {
|
||||||
|
if (MaybeAssembleFrame(seq_num_start, seq_num, found_frames)) {
|
||||||
|
// Frame was assembled, continue to look for more frames.
|
||||||
|
break;
|
||||||
|
} else {
|
||||||
|
// Frame was not assembled, no subsequent frame will be continuous.
|
||||||
|
return found_frames;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
seq_num++;
|
||||||
|
packet = GetPacket(seq_num).get();
|
||||||
|
if (packet == nullptr) {
|
||||||
|
return found_frames;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return found_frames;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool H264PacketBuffer::MaybeAssembleFrame(
|
||||||
|
int64_t start_seq_num_unwrapped,
|
||||||
|
int64_t end_sequence_number_unwrapped,
|
||||||
|
std::vector<std::unique_ptr<Packet>>& frames) {
|
||||||
|
bool has_sps = false;
|
||||||
|
bool has_pps = false;
|
||||||
|
bool has_idr = false;
|
||||||
|
|
||||||
|
int width = -1;
|
||||||
|
int height = -1;
|
||||||
|
|
||||||
|
for (int64_t seq_num = start_seq_num_unwrapped;
|
||||||
|
seq_num <= end_sequence_number_unwrapped; ++seq_num) {
|
||||||
|
const auto& packet = GetPacket(seq_num);
|
||||||
|
const auto& h264_header =
|
||||||
|
absl::get<RTPVideoHeaderH264>(packet->video_header.video_type_header);
|
||||||
|
for (const auto& nalu : GetNaluInfos(h264_header)) {
|
||||||
|
has_idr |= nalu.type == H264::NaluType::kIdr;
|
||||||
|
has_sps |= nalu.type == H264::NaluType::kSps;
|
||||||
|
has_pps |= nalu.type == H264::NaluType::kPps;
|
||||||
|
}
|
||||||
|
|
||||||
|
width = std::max<int>(packet->video_header.width, width);
|
||||||
|
height = std::max<int>(packet->video_header.height, height);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (has_idr) {
|
||||||
|
if (!idr_only_keyframes_allowed_ && (!has_sps || !has_pps)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for (int64_t seq_num = start_seq_num_unwrapped;
|
||||||
|
seq_num <= end_sequence_number_unwrapped; ++seq_num) {
|
||||||
|
auto& packet = GetPacket(seq_num);
|
||||||
|
|
||||||
|
packet->video_header.is_first_packet_in_frame =
|
||||||
|
(seq_num == start_seq_num_unwrapped);
|
||||||
|
packet->video_header.is_last_packet_in_frame =
|
||||||
|
(seq_num == end_sequence_number_unwrapped);
|
||||||
|
|
||||||
|
if (packet->video_header.is_first_packet_in_frame) {
|
||||||
|
if (width > 0 && height > 0) {
|
||||||
|
packet->video_header.width = width;
|
||||||
|
packet->video_header.height = height;
|
||||||
|
}
|
||||||
|
|
||||||
|
packet->video_header.frame_type = has_idr
|
||||||
|
? VideoFrameType::kVideoFrameKey
|
||||||
|
: VideoFrameType::kVideoFrameDelta;
|
||||||
|
}
|
||||||
|
|
||||||
|
packet->video_payload =
|
||||||
|
FixVideoPayload(packet->video_payload, packet->video_header);
|
||||||
|
|
||||||
|
frames.push_back(std::move(packet));
|
||||||
|
}
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace webrtc
|
56
modules/video_coding/h264_packet_buffer.h
Normal file
56
modules/video_coding/h264_packet_buffer.h
Normal file
@ -0,0 +1,56 @@
|
|||||||
|
/*
|
||||||
|
* Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
|
||||||
|
*
|
||||||
|
* Use of this source code is governed by a BSD-style license
|
||||||
|
* that can be found in the LICENSE file in the root of the source
|
||||||
|
* tree. An additional intellectual property rights grant can be found
|
||||||
|
* in the file PATENTS. All contributing project authors may
|
||||||
|
* be found in the AUTHORS file in the root of the source tree.
|
||||||
|
*/
|
||||||
|
|
||||||
|
#ifndef MODULES_VIDEO_CODING_H264_PACKET_BUFFER_H_
|
||||||
|
#define MODULES_VIDEO_CODING_H264_PACKET_BUFFER_H_
|
||||||
|
|
||||||
|
#include <array>
|
||||||
|
#include <memory>
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
|
#include "absl/base/attributes.h"
|
||||||
|
#include "absl/types/optional.h"
|
||||||
|
#include "modules/video_coding/packet_buffer.h"
|
||||||
|
#include "rtc_base/numerics/sequence_number_util.h"
|
||||||
|
|
||||||
|
namespace webrtc {
|
||||||
|
|
||||||
|
class H264PacketBuffer {
|
||||||
|
public:
|
||||||
|
// The H264PacketBuffer does the same job as the PacketBuffer but for H264
|
||||||
|
// only. To make it fit in with surronding code the PacketBuffer input/output
|
||||||
|
// classes are used.
|
||||||
|
using Packet = video_coding::PacketBuffer::Packet;
|
||||||
|
using InsertResult = video_coding::PacketBuffer::InsertResult;
|
||||||
|
|
||||||
|
explicit H264PacketBuffer(bool idr_only_keyframes_allowed);
|
||||||
|
|
||||||
|
ABSL_MUST_USE_RESULT InsertResult
|
||||||
|
InsertPacket(std::unique_ptr<Packet> packet);
|
||||||
|
|
||||||
|
private:
|
||||||
|
static constexpr int kBufferSize = 2048;
|
||||||
|
|
||||||
|
std::unique_ptr<Packet>& GetPacket(int64_t unwrapped_seq_num);
|
||||||
|
bool BeginningOfStream(const Packet& packet) const;
|
||||||
|
std::vector<std::unique_ptr<Packet>> FindFrames(int64_t unwrapped_seq_num);
|
||||||
|
bool MaybeAssembleFrame(int64_t start_seq_num_unwrapped,
|
||||||
|
int64_t end_sequence_number_unwrapped,
|
||||||
|
std::vector<std::unique_ptr<Packet>>& packets);
|
||||||
|
|
||||||
|
const bool idr_only_keyframes_allowed_;
|
||||||
|
std::array<std::unique_ptr<Packet>, kBufferSize> buffer_;
|
||||||
|
absl::optional<int64_t> last_continuous_unwrapped_seq_num_;
|
||||||
|
SeqNumUnwrapper<uint16_t> seq_num_unwrapper_;
|
||||||
|
};
|
||||||
|
|
||||||
|
} // namespace webrtc
|
||||||
|
|
||||||
|
#endif // MODULES_VIDEO_CODING_H264_PACKET_BUFFER_H_
|
779
modules/video_coding/h264_packet_buffer_unittest.cc
Normal file
779
modules/video_coding/h264_packet_buffer_unittest.cc
Normal file
@ -0,0 +1,779 @@
|
|||||||
|
/*
|
||||||
|
* Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
|
||||||
|
*
|
||||||
|
* Use of this source code is governed by a BSD-style license
|
||||||
|
* that can be found in the LICENSE file in the root of the source
|
||||||
|
* tree. An additional intellectual property rights grant can be found
|
||||||
|
* in the file PATENTS. All contributing project authors may
|
||||||
|
* be found in the AUTHORS file in the root of the source tree.
|
||||||
|
*/
|
||||||
|
#include "modules/video_coding/h264_packet_buffer.h"
|
||||||
|
|
||||||
|
#include <cstring>
|
||||||
|
#include <limits>
|
||||||
|
#include <ostream>
|
||||||
|
#include <string>
|
||||||
|
#include <utility>
|
||||||
|
|
||||||
|
#include "api/array_view.h"
|
||||||
|
#include "api/video/render_resolution.h"
|
||||||
|
#include "common_video/h264/h264_common.h"
|
||||||
|
#include "rtc_base/system/unused.h"
|
||||||
|
#include "test/gmock.h"
|
||||||
|
#include "test/gtest.h"
|
||||||
|
|
||||||
|
namespace webrtc {
|
||||||
|
namespace {
|
||||||
|
|
||||||
|
using ::testing::ElementsAreArray;
|
||||||
|
using ::testing::Eq;
|
||||||
|
using ::testing::IsEmpty;
|
||||||
|
using ::testing::SizeIs;
|
||||||
|
|
||||||
|
using H264::NaluType::kAud;
|
||||||
|
using H264::NaluType::kFuA;
|
||||||
|
using H264::NaluType::kIdr;
|
||||||
|
using H264::NaluType::kPps;
|
||||||
|
using H264::NaluType::kSlice;
|
||||||
|
using H264::NaluType::kSps;
|
||||||
|
using H264::NaluType::kStapA;
|
||||||
|
|
||||||
|
constexpr int kBufferSize = 2048;
|
||||||
|
|
||||||
|
std::vector<uint8_t> StartCode() {
|
||||||
|
return {0, 0, 0, 1};
|
||||||
|
}
|
||||||
|
|
||||||
|
NaluInfo MakeNaluInfo(uint8_t type) {
|
||||||
|
NaluInfo res;
|
||||||
|
res.type = type;
|
||||||
|
res.sps_id = -1;
|
||||||
|
res.pps_id = -1;
|
||||||
|
return res;
|
||||||
|
}
|
||||||
|
|
||||||
|
class Packet {
|
||||||
|
public:
|
||||||
|
explicit Packet(H264PacketizationTypes type);
|
||||||
|
|
||||||
|
Packet& Idr(std::vector<uint8_t> payload = {9, 9, 9});
|
||||||
|
Packet& Slice(std::vector<uint8_t> payload = {9, 9, 9});
|
||||||
|
Packet& Sps(std::vector<uint8_t> payload = {9, 9, 9});
|
||||||
|
Packet& SpsWithResolution(RenderResolution resolution,
|
||||||
|
std::vector<uint8_t> payload = {9, 9, 9});
|
||||||
|
Packet& Pps(std::vector<uint8_t> payload = {9, 9, 9});
|
||||||
|
Packet& Aud();
|
||||||
|
Packet& Marker();
|
||||||
|
Packet& AsFirstFragment();
|
||||||
|
Packet& Time(uint32_t rtp_timestamp);
|
||||||
|
Packet& SeqNum(uint16_t rtp_seq_num);
|
||||||
|
|
||||||
|
std::unique_ptr<H264PacketBuffer::Packet> Build();
|
||||||
|
|
||||||
|
private:
|
||||||
|
rtc::CopyOnWriteBuffer BuildFuaPayload() const;
|
||||||
|
rtc::CopyOnWriteBuffer BuildSingleNaluPayload() const;
|
||||||
|
rtc::CopyOnWriteBuffer BuildStapAPayload() const;
|
||||||
|
|
||||||
|
RTPVideoHeaderH264& H264Header() {
|
||||||
|
return absl::get<RTPVideoHeaderH264>(video_header_.video_type_header);
|
||||||
|
}
|
||||||
|
const RTPVideoHeaderH264& H264Header() const {
|
||||||
|
return absl::get<RTPVideoHeaderH264>(video_header_.video_type_header);
|
||||||
|
}
|
||||||
|
|
||||||
|
H264PacketizationTypes type_;
|
||||||
|
RTPVideoHeader video_header_;
|
||||||
|
bool first_fragment_ = false;
|
||||||
|
bool marker_bit_ = false;
|
||||||
|
uint32_t rtp_timestamp_ = 0;
|
||||||
|
uint16_t rtp_seq_num_ = 0;
|
||||||
|
std::vector<std::vector<uint8_t>> nalu_payloads_;
|
||||||
|
};
|
||||||
|
|
||||||
|
Packet::Packet(H264PacketizationTypes type) : type_(type) {
|
||||||
|
video_header_.video_type_header.emplace<RTPVideoHeaderH264>();
|
||||||
|
}
|
||||||
|
|
||||||
|
Packet& Packet::Idr(std::vector<uint8_t> payload) {
|
||||||
|
auto& h264_header = H264Header();
|
||||||
|
h264_header.nalus[h264_header.nalus_length++] = MakeNaluInfo(kIdr);
|
||||||
|
nalu_payloads_.push_back(std::move(payload));
|
||||||
|
return *this;
|
||||||
|
}
|
||||||
|
|
||||||
|
Packet& Packet::Slice(std::vector<uint8_t> payload) {
|
||||||
|
auto& h264_header = H264Header();
|
||||||
|
h264_header.nalus[h264_header.nalus_length++] = MakeNaluInfo(kSlice);
|
||||||
|
nalu_payloads_.push_back(std::move(payload));
|
||||||
|
return *this;
|
||||||
|
}
|
||||||
|
|
||||||
|
Packet& Packet::Sps(std::vector<uint8_t> payload) {
|
||||||
|
auto& h264_header = H264Header();
|
||||||
|
h264_header.nalus[h264_header.nalus_length++] = MakeNaluInfo(kSps);
|
||||||
|
nalu_payloads_.push_back(std::move(payload));
|
||||||
|
return *this;
|
||||||
|
}
|
||||||
|
|
||||||
|
Packet& Packet::SpsWithResolution(RenderResolution resolution,
|
||||||
|
std::vector<uint8_t> payload) {
|
||||||
|
auto& h264_header = H264Header();
|
||||||
|
h264_header.nalus[h264_header.nalus_length++] = MakeNaluInfo(kSps);
|
||||||
|
video_header_.width = resolution.Width();
|
||||||
|
video_header_.height = resolution.Height();
|
||||||
|
nalu_payloads_.push_back(std::move(payload));
|
||||||
|
return *this;
|
||||||
|
}
|
||||||
|
|
||||||
|
Packet& Packet::Pps(std::vector<uint8_t> payload) {
|
||||||
|
auto& h264_header = H264Header();
|
||||||
|
h264_header.nalus[h264_header.nalus_length++] = MakeNaluInfo(kPps);
|
||||||
|
nalu_payloads_.push_back(std::move(payload));
|
||||||
|
return *this;
|
||||||
|
}
|
||||||
|
|
||||||
|
Packet& Packet::Aud() {
|
||||||
|
auto& h264_header = H264Header();
|
||||||
|
h264_header.nalus[h264_header.nalus_length++] = MakeNaluInfo(kAud);
|
||||||
|
nalu_payloads_.push_back({});
|
||||||
|
return *this;
|
||||||
|
}
|
||||||
|
|
||||||
|
Packet& Packet::Marker() {
|
||||||
|
marker_bit_ = true;
|
||||||
|
return *this;
|
||||||
|
}
|
||||||
|
|
||||||
|
Packet& Packet::AsFirstFragment() {
|
||||||
|
first_fragment_ = true;
|
||||||
|
return *this;
|
||||||
|
}
|
||||||
|
|
||||||
|
Packet& Packet::Time(uint32_t rtp_timestamp) {
|
||||||
|
rtp_timestamp_ = rtp_timestamp;
|
||||||
|
return *this;
|
||||||
|
}
|
||||||
|
|
||||||
|
Packet& Packet::SeqNum(uint16_t rtp_seq_num) {
|
||||||
|
rtp_seq_num_ = rtp_seq_num;
|
||||||
|
return *this;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::unique_ptr<H264PacketBuffer::Packet> Packet::Build() {
|
||||||
|
auto res = std::make_unique<H264PacketBuffer::Packet>();
|
||||||
|
|
||||||
|
auto& h264_header = H264Header();
|
||||||
|
switch (type_) {
|
||||||
|
case kH264FuA: {
|
||||||
|
RTC_CHECK_EQ(h264_header.nalus_length, 1);
|
||||||
|
res->video_payload = BuildFuaPayload();
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case kH264SingleNalu: {
|
||||||
|
RTC_CHECK_EQ(h264_header.nalus_length, 1);
|
||||||
|
res->video_payload = BuildSingleNaluPayload();
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case kH264StapA: {
|
||||||
|
RTC_CHECK_GT(h264_header.nalus_length, 1);
|
||||||
|
RTC_CHECK_LE(h264_header.nalus_length, kMaxNalusPerPacket);
|
||||||
|
res->video_payload = BuildStapAPayload();
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (type_ == kH264FuA && !first_fragment_) {
|
||||||
|
h264_header.nalus_length = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
h264_header.packetization_type = type_;
|
||||||
|
res->marker_bit = marker_bit_;
|
||||||
|
res->video_header = video_header_;
|
||||||
|
res->timestamp = rtp_timestamp_;
|
||||||
|
res->seq_num = rtp_seq_num_;
|
||||||
|
res->video_header.codec = kVideoCodecH264;
|
||||||
|
|
||||||
|
return res;
|
||||||
|
}
|
||||||
|
|
||||||
|
rtc::CopyOnWriteBuffer Packet::BuildFuaPayload() const {
|
||||||
|
return rtc::CopyOnWriteBuffer(nalu_payloads_[0].data(),
|
||||||
|
nalu_payloads_[0].size());
|
||||||
|
}
|
||||||
|
|
||||||
|
rtc::CopyOnWriteBuffer Packet::BuildSingleNaluPayload() const {
|
||||||
|
rtc::CopyOnWriteBuffer res;
|
||||||
|
auto& h264_header = H264Header();
|
||||||
|
res.AppendData(&h264_header.nalus[0].type, 1);
|
||||||
|
res.AppendData(nalu_payloads_[0].data(), nalu_payloads_[0].size());
|
||||||
|
return res;
|
||||||
|
}
|
||||||
|
|
||||||
|
rtc::CopyOnWriteBuffer Packet::BuildStapAPayload() const {
|
||||||
|
rtc::CopyOnWriteBuffer res;
|
||||||
|
|
||||||
|
const uint8_t indicator = H264::NaluType::kStapA;
|
||||||
|
res.AppendData(&indicator, 1);
|
||||||
|
|
||||||
|
auto& h264_header = H264Header();
|
||||||
|
for (size_t i = 0; i < h264_header.nalus_length; ++i) {
|
||||||
|
// The two first bytes indicates the nalu segment size.
|
||||||
|
uint8_t length_as_array[2] = {
|
||||||
|
0, static_cast<uint8_t>(nalu_payloads_[i].size() + 1)};
|
||||||
|
res.AppendData(length_as_array);
|
||||||
|
|
||||||
|
res.AppendData(&h264_header.nalus[i].type, 1);
|
||||||
|
res.AppendData(nalu_payloads_[i].data(), nalu_payloads_[i].size());
|
||||||
|
}
|
||||||
|
return res;
|
||||||
|
}
|
||||||
|
|
||||||
|
rtc::ArrayView<const uint8_t> PacketPayload(
|
||||||
|
const std::unique_ptr<H264PacketBuffer::Packet>& packet) {
|
||||||
|
return packet->video_payload;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::vector<uint8_t> FlatVector(
|
||||||
|
const std::vector<std::vector<uint8_t>>& elems) {
|
||||||
|
std::vector<uint8_t> res;
|
||||||
|
for (const auto& elem : elems) {
|
||||||
|
res.insert(res.end(), elem.begin(), elem.end());
|
||||||
|
}
|
||||||
|
return res;
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST(H264PacketBufferTest, IdrIsKeyframe) {
|
||||||
|
H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/true);
|
||||||
|
|
||||||
|
EXPECT_THAT(
|
||||||
|
packet_buffer.InsertPacket(Packet(kH264SingleNalu).Idr().Marker().Build())
|
||||||
|
.packets,
|
||||||
|
SizeIs(1));
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST(H264PacketBufferTest, IdrIsNotKeyframe) {
|
||||||
|
H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false);
|
||||||
|
|
||||||
|
EXPECT_THAT(
|
||||||
|
packet_buffer.InsertPacket(Packet(kH264SingleNalu).Idr().Marker().Build())
|
||||||
|
.packets,
|
||||||
|
IsEmpty());
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST(H264PacketBufferTest, IdrIsKeyframeFuaRequiresFirstFragmet) {
|
||||||
|
H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/true);
|
||||||
|
|
||||||
|
// Not marked as the first fragment
|
||||||
|
EXPECT_THAT(
|
||||||
|
packet_buffer
|
||||||
|
.InsertPacket(Packet(kH264FuA).Idr().SeqNum(0).Time(0).Build())
|
||||||
|
.packets,
|
||||||
|
IsEmpty());
|
||||||
|
|
||||||
|
EXPECT_THAT(packet_buffer
|
||||||
|
.InsertPacket(
|
||||||
|
Packet(kH264FuA).Idr().SeqNum(1).Time(0).Marker().Build())
|
||||||
|
.packets,
|
||||||
|
IsEmpty());
|
||||||
|
|
||||||
|
// Marked as first fragment
|
||||||
|
EXPECT_THAT(packet_buffer
|
||||||
|
.InsertPacket(Packet(kH264FuA)
|
||||||
|
.Idr()
|
||||||
|
.SeqNum(2)
|
||||||
|
.Time(1)
|
||||||
|
.AsFirstFragment()
|
||||||
|
.Build())
|
||||||
|
.packets,
|
||||||
|
IsEmpty());
|
||||||
|
|
||||||
|
EXPECT_THAT(packet_buffer
|
||||||
|
.InsertPacket(
|
||||||
|
Packet(kH264FuA).Idr().SeqNum(3).Time(1).Marker().Build())
|
||||||
|
.packets,
|
||||||
|
SizeIs(2));
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST(H264PacketBufferTest, SpsPpsIdrIsKeyframeSingleNalus) {
|
||||||
|
H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false);
|
||||||
|
|
||||||
|
RTC_UNUSED(packet_buffer.InsertPacket(
|
||||||
|
Packet(kH264SingleNalu).Sps().SeqNum(0).Time(0).Build()));
|
||||||
|
RTC_UNUSED(packet_buffer.InsertPacket(
|
||||||
|
Packet(kH264SingleNalu).Pps().SeqNum(1).Time(0).Build()));
|
||||||
|
EXPECT_THAT(
|
||||||
|
packet_buffer
|
||||||
|
.InsertPacket(
|
||||||
|
Packet(kH264SingleNalu).Idr().SeqNum(2).Time(0).Marker().Build())
|
||||||
|
.packets,
|
||||||
|
SizeIs(3));
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST(H264PacketBufferTest, PpsIdrIsNotKeyframeSingleNalus) {
|
||||||
|
H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false);
|
||||||
|
|
||||||
|
RTC_UNUSED(packet_buffer.InsertPacket(
|
||||||
|
Packet(kH264SingleNalu).Pps().SeqNum(0).Time(0).Build()));
|
||||||
|
EXPECT_THAT(
|
||||||
|
packet_buffer
|
||||||
|
.InsertPacket(
|
||||||
|
Packet(kH264SingleNalu).Idr().SeqNum(1).Time(0).Marker().Build())
|
||||||
|
.packets,
|
||||||
|
IsEmpty());
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST(H264PacketBufferTest, SpsIdrIsNotKeyframeSingleNalus) {
|
||||||
|
H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false);
|
||||||
|
|
||||||
|
RTC_UNUSED(packet_buffer.InsertPacket(
|
||||||
|
Packet(kH264SingleNalu).Sps().SeqNum(0).Time(0).Build()));
|
||||||
|
EXPECT_THAT(
|
||||||
|
packet_buffer
|
||||||
|
.InsertPacket(
|
||||||
|
Packet(kH264SingleNalu).Idr().SeqNum(1).Time(0).Marker().Build())
|
||||||
|
.packets,
|
||||||
|
IsEmpty());
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST(H264PacketBufferTest, SpsPpsIdrIsKeyframeStapA) {
|
||||||
|
H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false);
|
||||||
|
|
||||||
|
EXPECT_THAT(packet_buffer
|
||||||
|
.InsertPacket(Packet(kH264StapA)
|
||||||
|
.Sps()
|
||||||
|
.Pps()
|
||||||
|
.Idr()
|
||||||
|
.SeqNum(0)
|
||||||
|
.Time(0)
|
||||||
|
.Marker()
|
||||||
|
.Build())
|
||||||
|
.packets,
|
||||||
|
SizeIs(1));
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST(H264PacketBufferTest, PpsIdrIsNotKeyframeStapA) {
|
||||||
|
H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false);
|
||||||
|
|
||||||
|
EXPECT_THAT(
|
||||||
|
packet_buffer
|
||||||
|
.InsertPacket(
|
||||||
|
Packet(kH264StapA).Pps().Idr().SeqNum(0).Time(0).Marker().Build())
|
||||||
|
.packets,
|
||||||
|
IsEmpty());
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST(H264PacketBufferTest, SpsIdrIsNotKeyframeStapA) {
|
||||||
|
H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false);
|
||||||
|
|
||||||
|
EXPECT_THAT(
|
||||||
|
packet_buffer
|
||||||
|
.InsertPacket(
|
||||||
|
Packet(kH264StapA).Sps().Idr().SeqNum(2).Time(2).Marker().Build())
|
||||||
|
.packets,
|
||||||
|
IsEmpty());
|
||||||
|
|
||||||
|
EXPECT_THAT(packet_buffer
|
||||||
|
.InsertPacket(Packet(kH264StapA)
|
||||||
|
.Sps()
|
||||||
|
.Pps()
|
||||||
|
.Idr()
|
||||||
|
.SeqNum(3)
|
||||||
|
.Time(3)
|
||||||
|
.Marker()
|
||||||
|
.Build())
|
||||||
|
.packets,
|
||||||
|
SizeIs(1));
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST(H264PacketBufferTest, InsertingSpsPpsLastCompletesKeyframe) {
|
||||||
|
H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false);
|
||||||
|
|
||||||
|
RTC_UNUSED(packet_buffer.InsertPacket(
|
||||||
|
Packet(kH264SingleNalu).Idr().SeqNum(2).Time(1).Marker().Build()));
|
||||||
|
|
||||||
|
EXPECT_THAT(packet_buffer
|
||||||
|
.InsertPacket(
|
||||||
|
Packet(kH264StapA).Sps().Pps().SeqNum(1).Time(1).Build())
|
||||||
|
.packets,
|
||||||
|
SizeIs(2));
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST(H264PacketBufferTest, InsertingMidFuaCompletesFrame) {
|
||||||
|
H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false);
|
||||||
|
|
||||||
|
EXPECT_THAT(packet_buffer
|
||||||
|
.InsertPacket(Packet(kH264StapA)
|
||||||
|
.Sps()
|
||||||
|
.Pps()
|
||||||
|
.Idr()
|
||||||
|
.SeqNum(0)
|
||||||
|
.Time(0)
|
||||||
|
.Marker()
|
||||||
|
.Build())
|
||||||
|
.packets,
|
||||||
|
SizeIs(1));
|
||||||
|
|
||||||
|
RTC_UNUSED(packet_buffer.InsertPacket(
|
||||||
|
Packet(kH264FuA).Slice().SeqNum(1).Time(1).AsFirstFragment().Build()));
|
||||||
|
RTC_UNUSED(packet_buffer.InsertPacket(
|
||||||
|
Packet(kH264FuA).Slice().SeqNum(3).Time(1).Marker().Build()));
|
||||||
|
EXPECT_THAT(
|
||||||
|
packet_buffer
|
||||||
|
.InsertPacket(Packet(kH264FuA).Slice().SeqNum(2).Time(1).Build())
|
||||||
|
.packets,
|
||||||
|
SizeIs(3));
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST(H264PacketBufferTest, SeqNumJumpDoesNotCompleteFrame) {
|
||||||
|
H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false);
|
||||||
|
|
||||||
|
EXPECT_THAT(packet_buffer
|
||||||
|
.InsertPacket(Packet(kH264StapA)
|
||||||
|
.Sps()
|
||||||
|
.Pps()
|
||||||
|
.Idr()
|
||||||
|
.SeqNum(0)
|
||||||
|
.Time(0)
|
||||||
|
.Marker()
|
||||||
|
.Build())
|
||||||
|
.packets,
|
||||||
|
SizeIs(1));
|
||||||
|
|
||||||
|
EXPECT_THAT(
|
||||||
|
packet_buffer
|
||||||
|
.InsertPacket(Packet(kH264FuA).Slice().SeqNum(1).Time(1).Build())
|
||||||
|
.packets,
|
||||||
|
IsEmpty());
|
||||||
|
|
||||||
|
// Add `kBufferSize` to make the index of the sequence number wrap and end up
|
||||||
|
// where the packet with sequence number 2 would have ended up.
|
||||||
|
EXPECT_THAT(packet_buffer
|
||||||
|
.InsertPacket(Packet(kH264FuA)
|
||||||
|
.Slice()
|
||||||
|
.SeqNum(2 + kBufferSize)
|
||||||
|
.Time(3)
|
||||||
|
.Marker()
|
||||||
|
.Build())
|
||||||
|
.packets,
|
||||||
|
IsEmpty());
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST(H264PacketBufferTest, OldFramesAreNotCompletedAfterBufferWrap) {
|
||||||
|
H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false);
|
||||||
|
|
||||||
|
EXPECT_THAT(packet_buffer
|
||||||
|
.InsertPacket(Packet(kH264SingleNalu)
|
||||||
|
.Slice()
|
||||||
|
.SeqNum(1)
|
||||||
|
.Time(1)
|
||||||
|
.Marker()
|
||||||
|
.Build())
|
||||||
|
.packets,
|
||||||
|
IsEmpty());
|
||||||
|
|
||||||
|
// New keyframe, preceedes packet with sequence number 1 in the buffer.
|
||||||
|
EXPECT_THAT(packet_buffer
|
||||||
|
.InsertPacket(Packet(kH264StapA)
|
||||||
|
.Sps()
|
||||||
|
.Pps()
|
||||||
|
.Idr()
|
||||||
|
.SeqNum(kBufferSize)
|
||||||
|
.Time(kBufferSize)
|
||||||
|
.Marker()
|
||||||
|
.Build())
|
||||||
|
.packets,
|
||||||
|
SizeIs(1));
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST(H264PacketBufferTest, OldPacketsDontBlockNewPackets) {
|
||||||
|
H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false);
|
||||||
|
EXPECT_THAT(packet_buffer
|
||||||
|
.InsertPacket(Packet(kH264StapA)
|
||||||
|
.Sps()
|
||||||
|
.Pps()
|
||||||
|
.Idr()
|
||||||
|
.SeqNum(kBufferSize)
|
||||||
|
.Time(kBufferSize)
|
||||||
|
.Marker()
|
||||||
|
.Build())
|
||||||
|
.packets,
|
||||||
|
SizeIs(1));
|
||||||
|
|
||||||
|
RTC_UNUSED(packet_buffer.InsertPacket(Packet(kH264FuA)
|
||||||
|
.Slice()
|
||||||
|
.SeqNum(kBufferSize + 1)
|
||||||
|
.Time(kBufferSize + 1)
|
||||||
|
.AsFirstFragment()
|
||||||
|
.Build()));
|
||||||
|
|
||||||
|
RTC_UNUSED(packet_buffer.InsertPacket(Packet(kH264FuA)
|
||||||
|
.Slice()
|
||||||
|
.SeqNum(kBufferSize + 3)
|
||||||
|
.Time(kBufferSize + 1)
|
||||||
|
.Marker()
|
||||||
|
.Build()));
|
||||||
|
EXPECT_THAT(
|
||||||
|
packet_buffer
|
||||||
|
.InsertPacket(Packet(kH264FuA).Slice().SeqNum(2).Time(2).Build())
|
||||||
|
.packets,
|
||||||
|
IsEmpty());
|
||||||
|
|
||||||
|
EXPECT_THAT(packet_buffer
|
||||||
|
.InsertPacket(Packet(kH264FuA)
|
||||||
|
.Slice()
|
||||||
|
.SeqNum(kBufferSize + 2)
|
||||||
|
.Time(kBufferSize + 1)
|
||||||
|
.Build())
|
||||||
|
.packets,
|
||||||
|
SizeIs(3));
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST(H264PacketBufferTest, OldPacketDoesntCompleteFrame) {
|
||||||
|
H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false);
|
||||||
|
|
||||||
|
EXPECT_THAT(packet_buffer
|
||||||
|
.InsertPacket(Packet(kH264StapA)
|
||||||
|
.Sps()
|
||||||
|
.Pps()
|
||||||
|
.Idr()
|
||||||
|
.SeqNum(kBufferSize)
|
||||||
|
.Time(kBufferSize)
|
||||||
|
.Marker()
|
||||||
|
.Build())
|
||||||
|
.packets,
|
||||||
|
SizeIs(1));
|
||||||
|
|
||||||
|
EXPECT_THAT(packet_buffer
|
||||||
|
.InsertPacket(Packet(kH264FuA)
|
||||||
|
.Slice()
|
||||||
|
.SeqNum(kBufferSize + 3)
|
||||||
|
.Time(kBufferSize + 1)
|
||||||
|
.Marker()
|
||||||
|
.Build())
|
||||||
|
.packets,
|
||||||
|
IsEmpty());
|
||||||
|
|
||||||
|
EXPECT_THAT(
|
||||||
|
packet_buffer
|
||||||
|
.InsertPacket(
|
||||||
|
Packet(kH264FuA).Slice().SeqNum(2).Time(2).Marker().Build())
|
||||||
|
.packets,
|
||||||
|
IsEmpty());
|
||||||
|
|
||||||
|
EXPECT_THAT(packet_buffer
|
||||||
|
.InsertPacket(Packet(kH264FuA)
|
||||||
|
.Slice()
|
||||||
|
.SeqNum(kBufferSize + 1)
|
||||||
|
.Time(kBufferSize + 1)
|
||||||
|
.AsFirstFragment()
|
||||||
|
.Build())
|
||||||
|
.packets,
|
||||||
|
IsEmpty());
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST(H264PacketBufferTest, FrameBoundariesAreSet) {
|
||||||
|
H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false);
|
||||||
|
|
||||||
|
auto key = packet_buffer.InsertPacket(
|
||||||
|
Packet(kH264StapA).Sps().Pps().Idr().SeqNum(1).Time(1).Marker().Build());
|
||||||
|
|
||||||
|
ASSERT_THAT(key.packets, SizeIs(1));
|
||||||
|
EXPECT_TRUE(key.packets[0]->video_header.is_first_packet_in_frame);
|
||||||
|
EXPECT_TRUE(key.packets[0]->video_header.is_last_packet_in_frame);
|
||||||
|
|
||||||
|
RTC_UNUSED(packet_buffer.InsertPacket(
|
||||||
|
Packet(kH264FuA).Slice().SeqNum(2).Time(2).Build()));
|
||||||
|
RTC_UNUSED(packet_buffer.InsertPacket(
|
||||||
|
Packet(kH264FuA).Slice().SeqNum(3).Time(2).Build()));
|
||||||
|
auto delta = packet_buffer.InsertPacket(
|
||||||
|
Packet(kH264FuA).Slice().SeqNum(4).Time(2).Marker().Build());
|
||||||
|
|
||||||
|
ASSERT_THAT(delta.packets, SizeIs(3));
|
||||||
|
EXPECT_TRUE(delta.packets[0]->video_header.is_first_packet_in_frame);
|
||||||
|
EXPECT_FALSE(delta.packets[0]->video_header.is_last_packet_in_frame);
|
||||||
|
|
||||||
|
EXPECT_FALSE(delta.packets[1]->video_header.is_first_packet_in_frame);
|
||||||
|
EXPECT_FALSE(delta.packets[1]->video_header.is_last_packet_in_frame);
|
||||||
|
|
||||||
|
EXPECT_FALSE(delta.packets[2]->video_header.is_first_packet_in_frame);
|
||||||
|
EXPECT_TRUE(delta.packets[2]->video_header.is_last_packet_in_frame);
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST(H264PacketBufferTest, ResolutionSetOnFirstPacket) {
|
||||||
|
H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false);
|
||||||
|
|
||||||
|
RTC_UNUSED(packet_buffer.InsertPacket(
|
||||||
|
Packet(kH264SingleNalu).Aud().SeqNum(1).Time(1).Build()));
|
||||||
|
auto res = packet_buffer.InsertPacket(Packet(kH264StapA)
|
||||||
|
.SpsWithResolution({320, 240})
|
||||||
|
.Pps()
|
||||||
|
.Idr()
|
||||||
|
.SeqNum(2)
|
||||||
|
.Time(1)
|
||||||
|
.Marker()
|
||||||
|
.Build());
|
||||||
|
|
||||||
|
ASSERT_THAT(res.packets, SizeIs(2));
|
||||||
|
EXPECT_THAT(res.packets[0]->video_header.width, Eq(320));
|
||||||
|
EXPECT_THAT(res.packets[0]->video_header.height, Eq(240));
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST(H264PacketBufferTest, KeyframeAndDeltaFrameSetOnFirstPacket) {
|
||||||
|
H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false);
|
||||||
|
|
||||||
|
RTC_UNUSED(packet_buffer.InsertPacket(
|
||||||
|
Packet(kH264SingleNalu).Aud().SeqNum(1).Time(1).Build()));
|
||||||
|
auto key = packet_buffer.InsertPacket(
|
||||||
|
Packet(kH264StapA).Sps().Pps().Idr().SeqNum(2).Time(1).Marker().Build());
|
||||||
|
|
||||||
|
auto delta = packet_buffer.InsertPacket(
|
||||||
|
Packet(kH264SingleNalu).Slice().SeqNum(3).Time(2).Marker().Build());
|
||||||
|
|
||||||
|
ASSERT_THAT(key.packets, SizeIs(2));
|
||||||
|
EXPECT_THAT(key.packets[0]->video_header.frame_type,
|
||||||
|
Eq(VideoFrameType::kVideoFrameKey));
|
||||||
|
ASSERT_THAT(delta.packets, SizeIs(1));
|
||||||
|
EXPECT_THAT(delta.packets[0]->video_header.frame_type,
|
||||||
|
Eq(VideoFrameType::kVideoFrameDelta));
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST(H264PacketBufferTest, RtpSeqNumWrap) {
|
||||||
|
H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false);
|
||||||
|
|
||||||
|
RTC_UNUSED(packet_buffer.InsertPacket(
|
||||||
|
Packet(kH264StapA).Sps().Pps().SeqNum(0xffff).Time(0).Build()));
|
||||||
|
|
||||||
|
RTC_UNUSED(packet_buffer.InsertPacket(
|
||||||
|
Packet(kH264FuA).Idr().SeqNum(0).Time(0).Build()));
|
||||||
|
EXPECT_THAT(packet_buffer
|
||||||
|
.InsertPacket(
|
||||||
|
Packet(kH264FuA).Idr().SeqNum(1).Time(0).Marker().Build())
|
||||||
|
.packets,
|
||||||
|
SizeIs(3));
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST(H264PacketBufferTest, StapAFixedBitstream) {
|
||||||
|
H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false);
|
||||||
|
|
||||||
|
auto packets = packet_buffer
|
||||||
|
.InsertPacket(Packet(kH264StapA)
|
||||||
|
.Sps({1, 2, 3})
|
||||||
|
.Pps({4, 5, 6})
|
||||||
|
.Idr({7, 8, 9})
|
||||||
|
.SeqNum(0)
|
||||||
|
.Time(0)
|
||||||
|
.Marker()
|
||||||
|
.Build())
|
||||||
|
.packets;
|
||||||
|
|
||||||
|
ASSERT_THAT(packets, SizeIs(1));
|
||||||
|
EXPECT_THAT(PacketPayload(packets[0]),
|
||||||
|
ElementsAreArray(FlatVector({StartCode(),
|
||||||
|
{kSps, 1, 2, 3},
|
||||||
|
StartCode(),
|
||||||
|
{kPps, 4, 5, 6},
|
||||||
|
StartCode(),
|
||||||
|
{kIdr, 7, 8, 9}})));
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST(H264PacketBufferTest, SingleNaluFixedBitstream) {
|
||||||
|
H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false);
|
||||||
|
|
||||||
|
RTC_UNUSED(packet_buffer.InsertPacket(
|
||||||
|
Packet(kH264SingleNalu).Sps({1, 2, 3}).SeqNum(0).Time(0).Build()));
|
||||||
|
RTC_UNUSED(packet_buffer.InsertPacket(
|
||||||
|
Packet(kH264SingleNalu).Pps({4, 5, 6}).SeqNum(1).Time(0).Build()));
|
||||||
|
auto packets = packet_buffer
|
||||||
|
.InsertPacket(Packet(kH264SingleNalu)
|
||||||
|
.Idr({7, 8, 9})
|
||||||
|
.SeqNum(2)
|
||||||
|
.Time(0)
|
||||||
|
.Marker()
|
||||||
|
.Build())
|
||||||
|
.packets;
|
||||||
|
|
||||||
|
ASSERT_THAT(packets, SizeIs(3));
|
||||||
|
EXPECT_THAT(PacketPayload(packets[0]),
|
||||||
|
ElementsAreArray(FlatVector({StartCode(), {kSps, 1, 2, 3}})));
|
||||||
|
EXPECT_THAT(PacketPayload(packets[1]),
|
||||||
|
ElementsAreArray(FlatVector({StartCode(), {kPps, 4, 5, 6}})));
|
||||||
|
EXPECT_THAT(PacketPayload(packets[2]),
|
||||||
|
ElementsAreArray(FlatVector({StartCode(), {kIdr, 7, 8, 9}})));
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST(H264PacketBufferTest, StapaAndFuaFixedBitstream) {
|
||||||
|
H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false);
|
||||||
|
|
||||||
|
RTC_UNUSED(packet_buffer.InsertPacket(Packet(kH264StapA)
|
||||||
|
.Sps({1, 2, 3})
|
||||||
|
.Pps({4, 5, 6})
|
||||||
|
.SeqNum(0)
|
||||||
|
.Time(0)
|
||||||
|
.Build()));
|
||||||
|
RTC_UNUSED(packet_buffer.InsertPacket(Packet(kH264FuA)
|
||||||
|
.Idr({8, 8, 8})
|
||||||
|
.SeqNum(1)
|
||||||
|
.Time(0)
|
||||||
|
.AsFirstFragment()
|
||||||
|
.Build()));
|
||||||
|
auto packets = packet_buffer
|
||||||
|
.InsertPacket(Packet(kH264FuA)
|
||||||
|
.Idr({9, 9, 9})
|
||||||
|
.SeqNum(2)
|
||||||
|
.Time(0)
|
||||||
|
.Marker()
|
||||||
|
.Build())
|
||||||
|
.packets;
|
||||||
|
|
||||||
|
ASSERT_THAT(packets, SizeIs(3));
|
||||||
|
EXPECT_THAT(
|
||||||
|
PacketPayload(packets[0]),
|
||||||
|
ElementsAreArray(FlatVector(
|
||||||
|
{StartCode(), {kSps, 1, 2, 3}, StartCode(), {kPps, 4, 5, 6}})));
|
||||||
|
EXPECT_THAT(PacketPayload(packets[1]),
|
||||||
|
ElementsAreArray(FlatVector({StartCode(), {8, 8, 8}})));
|
||||||
|
// Third is a continuation of second, so only the payload is expected.
|
||||||
|
EXPECT_THAT(PacketPayload(packets[2]),
|
||||||
|
ElementsAreArray(FlatVector({{9, 9, 9}})));
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST(H264PacketBufferTest, FullPacketBufferDoesNotBlockKeyframe) {
|
||||||
|
H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false);
|
||||||
|
|
||||||
|
for (int i = 0; i < kBufferSize; ++i) {
|
||||||
|
EXPECT_THAT(
|
||||||
|
packet_buffer
|
||||||
|
.InsertPacket(
|
||||||
|
Packet(kH264SingleNalu).Slice().SeqNum(i).Time(0).Build())
|
||||||
|
.packets,
|
||||||
|
IsEmpty());
|
||||||
|
}
|
||||||
|
|
||||||
|
EXPECT_THAT(packet_buffer
|
||||||
|
.InsertPacket(Packet(kH264StapA)
|
||||||
|
.Sps()
|
||||||
|
.Pps()
|
||||||
|
.Idr()
|
||||||
|
.SeqNum(kBufferSize)
|
||||||
|
.Time(1)
|
||||||
|
.Marker()
|
||||||
|
.Build())
|
||||||
|
.packets,
|
||||||
|
SizeIs(1));
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST(H264PacketBufferTest, TooManyNalusInPacket) {
|
||||||
|
H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false);
|
||||||
|
|
||||||
|
std::unique_ptr<H264PacketBuffer::Packet> packet(
|
||||||
|
Packet(kH264StapA).Sps().Pps().Idr().SeqNum(1).Time(1).Marker().Build());
|
||||||
|
auto& h264_header =
|
||||||
|
absl::get<RTPVideoHeaderH264>(packet->video_header.video_type_header);
|
||||||
|
h264_header.nalus_length = kMaxNalusPerPacket + 1;
|
||||||
|
|
||||||
|
EXPECT_THAT(packet_buffer.InsertPacket(std::move(packet)).packets, IsEmpty());
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace
|
||||||
|
} // namespace webrtc
|
@ -197,6 +197,7 @@ webrtc_fuzzer_test("flexfec_receiver_fuzzer") {
|
|||||||
webrtc_fuzzer_test("packet_buffer_fuzzer") {
|
webrtc_fuzzer_test("packet_buffer_fuzzer") {
|
||||||
sources = [ "packet_buffer_fuzzer.cc" ]
|
sources = [ "packet_buffer_fuzzer.cc" ]
|
||||||
deps = [
|
deps = [
|
||||||
|
"../../modules/video_coding:packet_buffer",
|
||||||
"../../modules/video_coding/",
|
"../../modules/video_coding/",
|
||||||
"../../system_wrappers",
|
"../../system_wrappers",
|
||||||
]
|
]
|
||||||
|
@ -96,6 +96,7 @@ rtc_library("video") {
|
|||||||
"../modules/video_coding",
|
"../modules/video_coding",
|
||||||
"../modules/video_coding:codec_globals_headers",
|
"../modules/video_coding:codec_globals_headers",
|
||||||
"../modules/video_coding:nack_requester",
|
"../modules/video_coding:nack_requester",
|
||||||
|
"../modules/video_coding:packet_buffer",
|
||||||
"../modules/video_coding:video_codec_interface",
|
"../modules/video_coding:video_codec_interface",
|
||||||
"../modules/video_coding:video_coding_utility",
|
"../modules/video_coding:video_coding_utility",
|
||||||
"../modules/video_processing",
|
"../modules/video_processing",
|
||||||
@ -184,6 +185,7 @@ rtc_source_set("video_legacy") {
|
|||||||
"../modules/rtp_rtcp:rtp_video_header",
|
"../modules/rtp_rtcp:rtp_video_header",
|
||||||
"../modules/utility",
|
"../modules/utility",
|
||||||
"../modules/video_coding",
|
"../modules/video_coding",
|
||||||
|
"../modules/video_coding:packet_buffer",
|
||||||
"../modules/video_coding:video_codec_interface",
|
"../modules/video_coding:video_codec_interface",
|
||||||
"../modules/video_coding:video_coding_utility",
|
"../modules/video_coding:video_coding_utility",
|
||||||
"../modules/video_coding/deprecated:nack_module",
|
"../modules/video_coding/deprecated:nack_module",
|
||||||
@ -698,6 +700,7 @@ if (rtc_include_tests) {
|
|||||||
"../modules/video_coding",
|
"../modules/video_coding",
|
||||||
"../modules/video_coding:codec_globals_headers",
|
"../modules/video_coding:codec_globals_headers",
|
||||||
"../modules/video_coding:encoded_frame",
|
"../modules/video_coding:encoded_frame",
|
||||||
|
"../modules/video_coding:packet_buffer",
|
||||||
"../modules/video_coding:video_codec_interface",
|
"../modules/video_coding:video_codec_interface",
|
||||||
"../modules/video_coding:video_coding_utility",
|
"../modules/video_coding:video_coding_utility",
|
||||||
"../modules/video_coding:webrtc_h264",
|
"../modules/video_coding:webrtc_h264",
|
||||||
|
Reference in New Issue
Block a user