From 10dc1a6d8bf7252ddbda510e812635498dc506a1 Mon Sep 17 00:00:00 2001 From: philipel Date: Wed, 15 Sep 2021 10:45:43 +0200 Subject: [PATCH] New H264PacketBuffer consolidating a bunch of H264 specific hacks into one class. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bug: webrtc:12579 Change-Id: Idea35983e204e4a3f8628d5b4eb587bbdbff5877 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/227286 Reviewed-by: Niels Moller Reviewed-by: Erik Språng Reviewed-by: Danil Chapovalov Commit-Queue: Philip Eliasson Cr-Commit-Position: refs/heads/main@{#34999} --- api/video/BUILD.gn | 1 + modules/video_coding/BUILD.gn | 58 +- modules/video_coding/h264_packet_buffer.cc | 287 +++++++ modules/video_coding/h264_packet_buffer.h | 56 ++ .../h264_packet_buffer_unittest.cc | 779 ++++++++++++++++++ test/fuzzers/BUILD.gn | 1 + video/BUILD.gn | 3 + 7 files changed, 1183 insertions(+), 2 deletions(-) create mode 100644 modules/video_coding/h264_packet_buffer.cc create mode 100644 modules/video_coding/h264_packet_buffer.h create mode 100644 modules/video_coding/h264_packet_buffer_unittest.cc diff --git a/api/video/BUILD.gn b/api/video/BUILD.gn index e358b1f41e..ec74869962 100644 --- a/api/video/BUILD.gn +++ b/api/video/BUILD.gn @@ -159,6 +159,7 @@ rtc_library("rtp_video_frame_assembler") { ":encoded_frame", "../../modules/rtp_rtcp:rtp_rtcp", "../../modules/rtp_rtcp:rtp_rtcp_format", + "../../modules/video_coding:packet_buffer", "../../modules/video_coding:video_coding", "../../rtc_base:logging", ] diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn index 91ee4879e3..129d4eab5c 100644 --- a/modules/video_coding/BUILD.gn +++ b/modules/video_coding/BUILD.gn @@ -98,6 +98,56 @@ rtc_library("nack_requester") { ] } +rtc_library("packet_buffer") { + sources = [ + "packet_buffer.cc", + "packet_buffer.h", + ] + deps = [ + ":codec_globals_headers", + "../../api:array_view", + "../../api:rtp_packet_info", + "../../api/units:timestamp", + "../../api/video:encoded_image", + "../../api/video:video_frame_type", + "../../common_video", + "../../rtc_base:checks", + "../../rtc_base:logging", + "../../rtc_base:rtc_base_approved", + "../../rtc_base:rtc_numerics", + "../rtp_rtcp:rtp_rtcp_format", + "../rtp_rtcp:rtp_video_header", + ] + absl_deps = [ + "//third_party/abseil-cpp/absl/base:core_headers", + "//third_party/abseil-cpp/absl/types:variant", + ] +} + +rtc_library("h264_packet_buffer") { + sources = [ + "h264_packet_buffer.cc", + "h264_packet_buffer.h", + ] + deps = [ + ":codec_globals_headers", + ":packet_buffer", + "../../api:array_view", + "../../api:rtp_packet_info", + "../../api/units:timestamp", + "../../api/video:encoded_image", + "../../api/video:video_frame_type", + "../../common_video", + "../../rtc_base:checks", + "../../rtc_base:logging", + "../../rtc_base:rtc_base_approved", + "../../rtc_base:rtc_numerics", + "../rtp_rtcp:rtp_rtcp_format", + "../rtp_rtcp:rtp_video_header", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ] +} + rtc_library("video_coding") { visibility = [ "*" ] sources = [ @@ -128,8 +178,6 @@ rtc_library("video_coding") { "loss_notification_controller.h", "media_opt_util.cc", "media_opt_util.h", - "packet_buffer.cc", - "packet_buffer.h", "rtp_frame_id_only_ref_finder.cc", "rtp_frame_id_only_ref_finder.h", "rtp_frame_reference_finder.cc", @@ -158,6 +206,7 @@ rtc_library("video_coding") { deps = [ ":codec_globals_headers", ":encoded_frame", + ":packet_buffer", ":video_codec_interface", ":video_coding_utility", ":webrtc_vp9_helpers", @@ -962,6 +1011,7 @@ if (rtc_include_tests) { "frame_buffer2_unittest.cc", "frame_dependencies_calculator_unittest.cc", "generic_decoder_unittest.cc", + "h264_packet_buffer_unittest.cc", "h264_sprop_parameter_sets_unittest.cc", "h264_sps_pps_tracker_unittest.cc", "histogram_unittest.cc", @@ -1005,7 +1055,9 @@ if (rtc_include_tests) { ":codec_globals_headers", ":encoded_frame", ":frame_dependencies_calculator", + ":h264_packet_buffer", ":nack_requester", + ":packet_buffer", ":simulcast_test_fixture_impl", ":video_codec_interface", ":video_codecs_test_framework", @@ -1033,6 +1085,7 @@ if (rtc_include_tests) { "../../api/test/video:function_video_factory", "../../api/video:builtin_video_bitrate_allocator_factory", "../../api/video:encoded_frame", + "../../api/video:render_resolution", "../../api/video:video_adaptation", "../../api/video:video_bitrate_allocation", "../../api/video:video_bitrate_allocator", @@ -1055,6 +1108,7 @@ if (rtc_include_tests) { "../../rtc_base:task_queue_for_test", "../../rtc_base/experiments:jitter_upper_bound_experiment", "../../rtc_base/synchronization:mutex", + "../../rtc_base/system:unused", "../../system_wrappers", "../../system_wrappers:field_trial", "../../system_wrappers:metrics", diff --git a/modules/video_coding/h264_packet_buffer.cc b/modules/video_coding/h264_packet_buffer.cc new file mode 100644 index 0000000000..8968539b63 --- /dev/null +++ b/modules/video_coding/h264_packet_buffer.cc @@ -0,0 +1,287 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/video_coding/h264_packet_buffer.h" + +#include +#include +#include +#include + +#include "api/array_view.h" +#include "api/rtp_packet_info.h" +#include "api/video/video_frame_type.h" +#include "common_video/h264/h264_common.h" +#include "modules/rtp_rtcp/source/rtp_header_extensions.h" +#include "modules/rtp_rtcp/source/rtp_packet_received.h" +#include "modules/rtp_rtcp/source/rtp_video_header.h" +#include "modules/video_coding/codecs/h264/include/h264_globals.h" +#include "rtc_base/checks.h" +#include "rtc_base/copy_on_write_buffer.h" +#include "rtc_base/logging.h" +#include "rtc_base/numerics/sequence_number_util.h" + +namespace webrtc { +namespace { +int64_t EuclideanMod(int64_t n, int64_t div) { + RTC_DCHECK_GT(div, 0); + return (n %= div) < 0 ? n + div : n; +} + +rtc::ArrayView GetNaluInfos( + const RTPVideoHeaderH264& h264_header) { + if (h264_header.nalus_length > kMaxNalusPerPacket) { + return {}; + } + + return rtc::MakeArrayView(h264_header.nalus, h264_header.nalus_length); +} + +bool IsFirstPacketOfFragment(const RTPVideoHeaderH264& h264_header) { + return h264_header.nalus_length > 0; +} + +bool BeginningOfIdr(const H264PacketBuffer::Packet& packet) { + const auto& h264_header = + absl::get(packet.video_header.video_type_header); + const bool contains_idr_nalu = + absl::c_any_of(GetNaluInfos(h264_header), [](const auto& nalu_info) { + return nalu_info.type == H264::NaluType::kIdr; + }); + switch (h264_header.packetization_type) { + case kH264StapA: + case kH264SingleNalu: { + return contains_idr_nalu; + } + case kH264FuA: { + return contains_idr_nalu && IsFirstPacketOfFragment(h264_header); + } + } +} + +bool HasSps(const H264PacketBuffer::Packet& packet) { + auto& h264_header = + absl::get(packet.video_header.video_type_header); + return absl::c_any_of(GetNaluInfos(h264_header), [](const auto& nalu_info) { + return nalu_info.type == H264::NaluType::kSps; + }); +} + +// TODO(bugs.webrtc.org/13157): Update the H264 depacketizer so we don't have to +// fiddle with the payload at this point. +rtc::CopyOnWriteBuffer FixVideoPayload(rtc::ArrayView payload, + const RTPVideoHeader& video_header) { + constexpr uint8_t kStartCode[] = {0, 0, 0, 1}; + + const auto& h264_header = + absl::get(video_header.video_type_header); + + rtc::CopyOnWriteBuffer result; + switch (h264_header.packetization_type) { + case kH264StapA: { + const uint8_t* payload_end = payload.data() + payload.size(); + const uint8_t* nalu_ptr = payload.data() + 1; + while (nalu_ptr < payload_end - 1) { + // The first two bytes describe the length of the segment, where a + // segment is the nalu type plus nalu payload. + uint16_t segment_length = nalu_ptr[0] << 8 | nalu_ptr[1]; + nalu_ptr += 2; + + if (nalu_ptr + segment_length <= payload_end) { + result.AppendData(kStartCode); + result.AppendData(nalu_ptr, segment_length); + } + nalu_ptr += segment_length; + } + return result; + } + + case kH264FuA: { + if (IsFirstPacketOfFragment(h264_header)) { + result.AppendData(kStartCode); + } + result.AppendData(payload.data(), payload.size()); + return result; + } + + case kH264SingleNalu: { + result.AppendData(kStartCode); + result.AppendData(payload.data(), payload.size()); + return result; + } + } + + RTC_NOTREACHED(); + return result; +} + +} // namespace + +H264PacketBuffer::H264PacketBuffer(bool idr_only_keyframes_allowed) + : idr_only_keyframes_allowed_(idr_only_keyframes_allowed) {} + +H264PacketBuffer::InsertResult H264PacketBuffer::InsertPacket( + std::unique_ptr packet) { + RTC_DCHECK(packet->video_header.codec == kVideoCodecH264); + + InsertResult result; + if (!absl::holds_alternative( + packet->video_header.video_type_header)) { + return result; + } + + int64_t unwrapped_seq_num = seq_num_unwrapper_.Unwrap(packet->seq_num); + auto& packet_slot = GetPacket(unwrapped_seq_num); + if (packet_slot != nullptr && + AheadOrAt(packet_slot->timestamp, packet->timestamp)) { + // The incoming `packet` is old or a duplicate. + return result; + } else { + packet_slot = std::move(packet); + } + + result.packets = FindFrames(unwrapped_seq_num); + return result; +} + +std::unique_ptr& H264PacketBuffer::GetPacket( + int64_t unwrapped_seq_num) { + return buffer_[EuclideanMod(unwrapped_seq_num, kBufferSize)]; +} + +bool H264PacketBuffer::BeginningOfStream( + const H264PacketBuffer::Packet& packet) const { + return HasSps(packet) || + (idr_only_keyframes_allowed_ && BeginningOfIdr(packet)); +} + +std::vector> +H264PacketBuffer::FindFrames(int64_t unwrapped_seq_num) { + std::vector> found_frames; + + Packet* packet = GetPacket(unwrapped_seq_num).get(); + RTC_CHECK(packet != nullptr); + + // Check if the packet is continuous or the beginning of a new coded video + // sequence. + if (unwrapped_seq_num - 1 != last_continuous_unwrapped_seq_num_) { + if (unwrapped_seq_num <= last_continuous_unwrapped_seq_num_ || + !BeginningOfStream(*packet)) { + return found_frames; + } + + last_continuous_unwrapped_seq_num_ = unwrapped_seq_num; + } + + for (int64_t seq_num = unwrapped_seq_num; + seq_num < unwrapped_seq_num + kBufferSize;) { + RTC_DCHECK_GE(seq_num, *last_continuous_unwrapped_seq_num_); + + // Packets that were never assembled into a completed frame will stay in + // the 'buffer_'. Check that the `packet` sequence number match the expected + // unwrapped sequence number. + if (static_cast(seq_num) != packet->seq_num) { + return found_frames; + } + + last_continuous_unwrapped_seq_num_ = seq_num; + // Last packet of the frame, try to assemble the frame. + if (packet->marker_bit) { + uint32_t rtp_timestamp = packet->timestamp; + + // Iterate backwards to find where the frame starts. + for (int64_t seq_num_start = seq_num; + seq_num_start > seq_num - kBufferSize; --seq_num_start) { + auto& prev_packet = GetPacket(seq_num_start - 1); + + if (prev_packet == nullptr || prev_packet->timestamp != rtp_timestamp) { + if (MaybeAssembleFrame(seq_num_start, seq_num, found_frames)) { + // Frame was assembled, continue to look for more frames. + break; + } else { + // Frame was not assembled, no subsequent frame will be continuous. + return found_frames; + } + } + } + } + + seq_num++; + packet = GetPacket(seq_num).get(); + if (packet == nullptr) { + return found_frames; + } + } + + return found_frames; +} + +bool H264PacketBuffer::MaybeAssembleFrame( + int64_t start_seq_num_unwrapped, + int64_t end_sequence_number_unwrapped, + std::vector>& frames) { + bool has_sps = false; + bool has_pps = false; + bool has_idr = false; + + int width = -1; + int height = -1; + + for (int64_t seq_num = start_seq_num_unwrapped; + seq_num <= end_sequence_number_unwrapped; ++seq_num) { + const auto& packet = GetPacket(seq_num); + const auto& h264_header = + absl::get(packet->video_header.video_type_header); + for (const auto& nalu : GetNaluInfos(h264_header)) { + has_idr |= nalu.type == H264::NaluType::kIdr; + has_sps |= nalu.type == H264::NaluType::kSps; + has_pps |= nalu.type == H264::NaluType::kPps; + } + + width = std::max(packet->video_header.width, width); + height = std::max(packet->video_header.height, height); + } + + if (has_idr) { + if (!idr_only_keyframes_allowed_ && (!has_sps || !has_pps)) { + return false; + } + } + + for (int64_t seq_num = start_seq_num_unwrapped; + seq_num <= end_sequence_number_unwrapped; ++seq_num) { + auto& packet = GetPacket(seq_num); + + packet->video_header.is_first_packet_in_frame = + (seq_num == start_seq_num_unwrapped); + packet->video_header.is_last_packet_in_frame = + (seq_num == end_sequence_number_unwrapped); + + if (packet->video_header.is_first_packet_in_frame) { + if (width > 0 && height > 0) { + packet->video_header.width = width; + packet->video_header.height = height; + } + + packet->video_header.frame_type = has_idr + ? VideoFrameType::kVideoFrameKey + : VideoFrameType::kVideoFrameDelta; + } + + packet->video_payload = + FixVideoPayload(packet->video_payload, packet->video_header); + + frames.push_back(std::move(packet)); + } + + return true; +} + +} // namespace webrtc diff --git a/modules/video_coding/h264_packet_buffer.h b/modules/video_coding/h264_packet_buffer.h new file mode 100644 index 0000000000..1671fddb23 --- /dev/null +++ b/modules/video_coding/h264_packet_buffer.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_VIDEO_CODING_H264_PACKET_BUFFER_H_ +#define MODULES_VIDEO_CODING_H264_PACKET_BUFFER_H_ + +#include +#include +#include + +#include "absl/base/attributes.h" +#include "absl/types/optional.h" +#include "modules/video_coding/packet_buffer.h" +#include "rtc_base/numerics/sequence_number_util.h" + +namespace webrtc { + +class H264PacketBuffer { + public: + // The H264PacketBuffer does the same job as the PacketBuffer but for H264 + // only. To make it fit in with surronding code the PacketBuffer input/output + // classes are used. + using Packet = video_coding::PacketBuffer::Packet; + using InsertResult = video_coding::PacketBuffer::InsertResult; + + explicit H264PacketBuffer(bool idr_only_keyframes_allowed); + + ABSL_MUST_USE_RESULT InsertResult + InsertPacket(std::unique_ptr packet); + + private: + static constexpr int kBufferSize = 2048; + + std::unique_ptr& GetPacket(int64_t unwrapped_seq_num); + bool BeginningOfStream(const Packet& packet) const; + std::vector> FindFrames(int64_t unwrapped_seq_num); + bool MaybeAssembleFrame(int64_t start_seq_num_unwrapped, + int64_t end_sequence_number_unwrapped, + std::vector>& packets); + + const bool idr_only_keyframes_allowed_; + std::array, kBufferSize> buffer_; + absl::optional last_continuous_unwrapped_seq_num_; + SeqNumUnwrapper seq_num_unwrapper_; +}; + +} // namespace webrtc + +#endif // MODULES_VIDEO_CODING_H264_PACKET_BUFFER_H_ diff --git a/modules/video_coding/h264_packet_buffer_unittest.cc b/modules/video_coding/h264_packet_buffer_unittest.cc new file mode 100644 index 0000000000..3d2a432c54 --- /dev/null +++ b/modules/video_coding/h264_packet_buffer_unittest.cc @@ -0,0 +1,779 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "modules/video_coding/h264_packet_buffer.h" + +#include +#include +#include +#include +#include + +#include "api/array_view.h" +#include "api/video/render_resolution.h" +#include "common_video/h264/h264_common.h" +#include "rtc_base/system/unused.h" +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { + +using ::testing::ElementsAreArray; +using ::testing::Eq; +using ::testing::IsEmpty; +using ::testing::SizeIs; + +using H264::NaluType::kAud; +using H264::NaluType::kFuA; +using H264::NaluType::kIdr; +using H264::NaluType::kPps; +using H264::NaluType::kSlice; +using H264::NaluType::kSps; +using H264::NaluType::kStapA; + +constexpr int kBufferSize = 2048; + +std::vector StartCode() { + return {0, 0, 0, 1}; +} + +NaluInfo MakeNaluInfo(uint8_t type) { + NaluInfo res; + res.type = type; + res.sps_id = -1; + res.pps_id = -1; + return res; +} + +class Packet { + public: + explicit Packet(H264PacketizationTypes type); + + Packet& Idr(std::vector payload = {9, 9, 9}); + Packet& Slice(std::vector payload = {9, 9, 9}); + Packet& Sps(std::vector payload = {9, 9, 9}); + Packet& SpsWithResolution(RenderResolution resolution, + std::vector payload = {9, 9, 9}); + Packet& Pps(std::vector payload = {9, 9, 9}); + Packet& Aud(); + Packet& Marker(); + Packet& AsFirstFragment(); + Packet& Time(uint32_t rtp_timestamp); + Packet& SeqNum(uint16_t rtp_seq_num); + + std::unique_ptr Build(); + + private: + rtc::CopyOnWriteBuffer BuildFuaPayload() const; + rtc::CopyOnWriteBuffer BuildSingleNaluPayload() const; + rtc::CopyOnWriteBuffer BuildStapAPayload() const; + + RTPVideoHeaderH264& H264Header() { + return absl::get(video_header_.video_type_header); + } + const RTPVideoHeaderH264& H264Header() const { + return absl::get(video_header_.video_type_header); + } + + H264PacketizationTypes type_; + RTPVideoHeader video_header_; + bool first_fragment_ = false; + bool marker_bit_ = false; + uint32_t rtp_timestamp_ = 0; + uint16_t rtp_seq_num_ = 0; + std::vector> nalu_payloads_; +}; + +Packet::Packet(H264PacketizationTypes type) : type_(type) { + video_header_.video_type_header.emplace(); +} + +Packet& Packet::Idr(std::vector payload) { + auto& h264_header = H264Header(); + h264_header.nalus[h264_header.nalus_length++] = MakeNaluInfo(kIdr); + nalu_payloads_.push_back(std::move(payload)); + return *this; +} + +Packet& Packet::Slice(std::vector payload) { + auto& h264_header = H264Header(); + h264_header.nalus[h264_header.nalus_length++] = MakeNaluInfo(kSlice); + nalu_payloads_.push_back(std::move(payload)); + return *this; +} + +Packet& Packet::Sps(std::vector payload) { + auto& h264_header = H264Header(); + h264_header.nalus[h264_header.nalus_length++] = MakeNaluInfo(kSps); + nalu_payloads_.push_back(std::move(payload)); + return *this; +} + +Packet& Packet::SpsWithResolution(RenderResolution resolution, + std::vector payload) { + auto& h264_header = H264Header(); + h264_header.nalus[h264_header.nalus_length++] = MakeNaluInfo(kSps); + video_header_.width = resolution.Width(); + video_header_.height = resolution.Height(); + nalu_payloads_.push_back(std::move(payload)); + return *this; +} + +Packet& Packet::Pps(std::vector payload) { + auto& h264_header = H264Header(); + h264_header.nalus[h264_header.nalus_length++] = MakeNaluInfo(kPps); + nalu_payloads_.push_back(std::move(payload)); + return *this; +} + +Packet& Packet::Aud() { + auto& h264_header = H264Header(); + h264_header.nalus[h264_header.nalus_length++] = MakeNaluInfo(kAud); + nalu_payloads_.push_back({}); + return *this; +} + +Packet& Packet::Marker() { + marker_bit_ = true; + return *this; +} + +Packet& Packet::AsFirstFragment() { + first_fragment_ = true; + return *this; +} + +Packet& Packet::Time(uint32_t rtp_timestamp) { + rtp_timestamp_ = rtp_timestamp; + return *this; +} + +Packet& Packet::SeqNum(uint16_t rtp_seq_num) { + rtp_seq_num_ = rtp_seq_num; + return *this; +} + +std::unique_ptr Packet::Build() { + auto res = std::make_unique(); + + auto& h264_header = H264Header(); + switch (type_) { + case kH264FuA: { + RTC_CHECK_EQ(h264_header.nalus_length, 1); + res->video_payload = BuildFuaPayload(); + break; + } + case kH264SingleNalu: { + RTC_CHECK_EQ(h264_header.nalus_length, 1); + res->video_payload = BuildSingleNaluPayload(); + break; + } + case kH264StapA: { + RTC_CHECK_GT(h264_header.nalus_length, 1); + RTC_CHECK_LE(h264_header.nalus_length, kMaxNalusPerPacket); + res->video_payload = BuildStapAPayload(); + break; + } + } + + if (type_ == kH264FuA && !first_fragment_) { + h264_header.nalus_length = 0; + } + + h264_header.packetization_type = type_; + res->marker_bit = marker_bit_; + res->video_header = video_header_; + res->timestamp = rtp_timestamp_; + res->seq_num = rtp_seq_num_; + res->video_header.codec = kVideoCodecH264; + + return res; +} + +rtc::CopyOnWriteBuffer Packet::BuildFuaPayload() const { + return rtc::CopyOnWriteBuffer(nalu_payloads_[0].data(), + nalu_payloads_[0].size()); +} + +rtc::CopyOnWriteBuffer Packet::BuildSingleNaluPayload() const { + rtc::CopyOnWriteBuffer res; + auto& h264_header = H264Header(); + res.AppendData(&h264_header.nalus[0].type, 1); + res.AppendData(nalu_payloads_[0].data(), nalu_payloads_[0].size()); + return res; +} + +rtc::CopyOnWriteBuffer Packet::BuildStapAPayload() const { + rtc::CopyOnWriteBuffer res; + + const uint8_t indicator = H264::NaluType::kStapA; + res.AppendData(&indicator, 1); + + auto& h264_header = H264Header(); + for (size_t i = 0; i < h264_header.nalus_length; ++i) { + // The two first bytes indicates the nalu segment size. + uint8_t length_as_array[2] = { + 0, static_cast(nalu_payloads_[i].size() + 1)}; + res.AppendData(length_as_array); + + res.AppendData(&h264_header.nalus[i].type, 1); + res.AppendData(nalu_payloads_[i].data(), nalu_payloads_[i].size()); + } + return res; +} + +rtc::ArrayView PacketPayload( + const std::unique_ptr& packet) { + return packet->video_payload; +} + +std::vector FlatVector( + const std::vector>& elems) { + std::vector res; + for (const auto& elem : elems) { + res.insert(res.end(), elem.begin(), elem.end()); + } + return res; +} + +TEST(H264PacketBufferTest, IdrIsKeyframe) { + H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/true); + + EXPECT_THAT( + packet_buffer.InsertPacket(Packet(kH264SingleNalu).Idr().Marker().Build()) + .packets, + SizeIs(1)); +} + +TEST(H264PacketBufferTest, IdrIsNotKeyframe) { + H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false); + + EXPECT_THAT( + packet_buffer.InsertPacket(Packet(kH264SingleNalu).Idr().Marker().Build()) + .packets, + IsEmpty()); +} + +TEST(H264PacketBufferTest, IdrIsKeyframeFuaRequiresFirstFragmet) { + H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/true); + + // Not marked as the first fragment + EXPECT_THAT( + packet_buffer + .InsertPacket(Packet(kH264FuA).Idr().SeqNum(0).Time(0).Build()) + .packets, + IsEmpty()); + + EXPECT_THAT(packet_buffer + .InsertPacket( + Packet(kH264FuA).Idr().SeqNum(1).Time(0).Marker().Build()) + .packets, + IsEmpty()); + + // Marked as first fragment + EXPECT_THAT(packet_buffer + .InsertPacket(Packet(kH264FuA) + .Idr() + .SeqNum(2) + .Time(1) + .AsFirstFragment() + .Build()) + .packets, + IsEmpty()); + + EXPECT_THAT(packet_buffer + .InsertPacket( + Packet(kH264FuA).Idr().SeqNum(3).Time(1).Marker().Build()) + .packets, + SizeIs(2)); +} + +TEST(H264PacketBufferTest, SpsPpsIdrIsKeyframeSingleNalus) { + H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false); + + RTC_UNUSED(packet_buffer.InsertPacket( + Packet(kH264SingleNalu).Sps().SeqNum(0).Time(0).Build())); + RTC_UNUSED(packet_buffer.InsertPacket( + Packet(kH264SingleNalu).Pps().SeqNum(1).Time(0).Build())); + EXPECT_THAT( + packet_buffer + .InsertPacket( + Packet(kH264SingleNalu).Idr().SeqNum(2).Time(0).Marker().Build()) + .packets, + SizeIs(3)); +} + +TEST(H264PacketBufferTest, PpsIdrIsNotKeyframeSingleNalus) { + H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false); + + RTC_UNUSED(packet_buffer.InsertPacket( + Packet(kH264SingleNalu).Pps().SeqNum(0).Time(0).Build())); + EXPECT_THAT( + packet_buffer + .InsertPacket( + Packet(kH264SingleNalu).Idr().SeqNum(1).Time(0).Marker().Build()) + .packets, + IsEmpty()); +} + +TEST(H264PacketBufferTest, SpsIdrIsNotKeyframeSingleNalus) { + H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false); + + RTC_UNUSED(packet_buffer.InsertPacket( + Packet(kH264SingleNalu).Sps().SeqNum(0).Time(0).Build())); + EXPECT_THAT( + packet_buffer + .InsertPacket( + Packet(kH264SingleNalu).Idr().SeqNum(1).Time(0).Marker().Build()) + .packets, + IsEmpty()); +} + +TEST(H264PacketBufferTest, SpsPpsIdrIsKeyframeStapA) { + H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false); + + EXPECT_THAT(packet_buffer + .InsertPacket(Packet(kH264StapA) + .Sps() + .Pps() + .Idr() + .SeqNum(0) + .Time(0) + .Marker() + .Build()) + .packets, + SizeIs(1)); +} + +TEST(H264PacketBufferTest, PpsIdrIsNotKeyframeStapA) { + H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false); + + EXPECT_THAT( + packet_buffer + .InsertPacket( + Packet(kH264StapA).Pps().Idr().SeqNum(0).Time(0).Marker().Build()) + .packets, + IsEmpty()); +} + +TEST(H264PacketBufferTest, SpsIdrIsNotKeyframeStapA) { + H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false); + + EXPECT_THAT( + packet_buffer + .InsertPacket( + Packet(kH264StapA).Sps().Idr().SeqNum(2).Time(2).Marker().Build()) + .packets, + IsEmpty()); + + EXPECT_THAT(packet_buffer + .InsertPacket(Packet(kH264StapA) + .Sps() + .Pps() + .Idr() + .SeqNum(3) + .Time(3) + .Marker() + .Build()) + .packets, + SizeIs(1)); +} + +TEST(H264PacketBufferTest, InsertingSpsPpsLastCompletesKeyframe) { + H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false); + + RTC_UNUSED(packet_buffer.InsertPacket( + Packet(kH264SingleNalu).Idr().SeqNum(2).Time(1).Marker().Build())); + + EXPECT_THAT(packet_buffer + .InsertPacket( + Packet(kH264StapA).Sps().Pps().SeqNum(1).Time(1).Build()) + .packets, + SizeIs(2)); +} + +TEST(H264PacketBufferTest, InsertingMidFuaCompletesFrame) { + H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false); + + EXPECT_THAT(packet_buffer + .InsertPacket(Packet(kH264StapA) + .Sps() + .Pps() + .Idr() + .SeqNum(0) + .Time(0) + .Marker() + .Build()) + .packets, + SizeIs(1)); + + RTC_UNUSED(packet_buffer.InsertPacket( + Packet(kH264FuA).Slice().SeqNum(1).Time(1).AsFirstFragment().Build())); + RTC_UNUSED(packet_buffer.InsertPacket( + Packet(kH264FuA).Slice().SeqNum(3).Time(1).Marker().Build())); + EXPECT_THAT( + packet_buffer + .InsertPacket(Packet(kH264FuA).Slice().SeqNum(2).Time(1).Build()) + .packets, + SizeIs(3)); +} + +TEST(H264PacketBufferTest, SeqNumJumpDoesNotCompleteFrame) { + H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false); + + EXPECT_THAT(packet_buffer + .InsertPacket(Packet(kH264StapA) + .Sps() + .Pps() + .Idr() + .SeqNum(0) + .Time(0) + .Marker() + .Build()) + .packets, + SizeIs(1)); + + EXPECT_THAT( + packet_buffer + .InsertPacket(Packet(kH264FuA).Slice().SeqNum(1).Time(1).Build()) + .packets, + IsEmpty()); + + // Add `kBufferSize` to make the index of the sequence number wrap and end up + // where the packet with sequence number 2 would have ended up. + EXPECT_THAT(packet_buffer + .InsertPacket(Packet(kH264FuA) + .Slice() + .SeqNum(2 + kBufferSize) + .Time(3) + .Marker() + .Build()) + .packets, + IsEmpty()); +} + +TEST(H264PacketBufferTest, OldFramesAreNotCompletedAfterBufferWrap) { + H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false); + + EXPECT_THAT(packet_buffer + .InsertPacket(Packet(kH264SingleNalu) + .Slice() + .SeqNum(1) + .Time(1) + .Marker() + .Build()) + .packets, + IsEmpty()); + + // New keyframe, preceedes packet with sequence number 1 in the buffer. + EXPECT_THAT(packet_buffer + .InsertPacket(Packet(kH264StapA) + .Sps() + .Pps() + .Idr() + .SeqNum(kBufferSize) + .Time(kBufferSize) + .Marker() + .Build()) + .packets, + SizeIs(1)); +} + +TEST(H264PacketBufferTest, OldPacketsDontBlockNewPackets) { + H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false); + EXPECT_THAT(packet_buffer + .InsertPacket(Packet(kH264StapA) + .Sps() + .Pps() + .Idr() + .SeqNum(kBufferSize) + .Time(kBufferSize) + .Marker() + .Build()) + .packets, + SizeIs(1)); + + RTC_UNUSED(packet_buffer.InsertPacket(Packet(kH264FuA) + .Slice() + .SeqNum(kBufferSize + 1) + .Time(kBufferSize + 1) + .AsFirstFragment() + .Build())); + + RTC_UNUSED(packet_buffer.InsertPacket(Packet(kH264FuA) + .Slice() + .SeqNum(kBufferSize + 3) + .Time(kBufferSize + 1) + .Marker() + .Build())); + EXPECT_THAT( + packet_buffer + .InsertPacket(Packet(kH264FuA).Slice().SeqNum(2).Time(2).Build()) + .packets, + IsEmpty()); + + EXPECT_THAT(packet_buffer + .InsertPacket(Packet(kH264FuA) + .Slice() + .SeqNum(kBufferSize + 2) + .Time(kBufferSize + 1) + .Build()) + .packets, + SizeIs(3)); +} + +TEST(H264PacketBufferTest, OldPacketDoesntCompleteFrame) { + H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false); + + EXPECT_THAT(packet_buffer + .InsertPacket(Packet(kH264StapA) + .Sps() + .Pps() + .Idr() + .SeqNum(kBufferSize) + .Time(kBufferSize) + .Marker() + .Build()) + .packets, + SizeIs(1)); + + EXPECT_THAT(packet_buffer + .InsertPacket(Packet(kH264FuA) + .Slice() + .SeqNum(kBufferSize + 3) + .Time(kBufferSize + 1) + .Marker() + .Build()) + .packets, + IsEmpty()); + + EXPECT_THAT( + packet_buffer + .InsertPacket( + Packet(kH264FuA).Slice().SeqNum(2).Time(2).Marker().Build()) + .packets, + IsEmpty()); + + EXPECT_THAT(packet_buffer + .InsertPacket(Packet(kH264FuA) + .Slice() + .SeqNum(kBufferSize + 1) + .Time(kBufferSize + 1) + .AsFirstFragment() + .Build()) + .packets, + IsEmpty()); +} + +TEST(H264PacketBufferTest, FrameBoundariesAreSet) { + H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false); + + auto key = packet_buffer.InsertPacket( + Packet(kH264StapA).Sps().Pps().Idr().SeqNum(1).Time(1).Marker().Build()); + + ASSERT_THAT(key.packets, SizeIs(1)); + EXPECT_TRUE(key.packets[0]->video_header.is_first_packet_in_frame); + EXPECT_TRUE(key.packets[0]->video_header.is_last_packet_in_frame); + + RTC_UNUSED(packet_buffer.InsertPacket( + Packet(kH264FuA).Slice().SeqNum(2).Time(2).Build())); + RTC_UNUSED(packet_buffer.InsertPacket( + Packet(kH264FuA).Slice().SeqNum(3).Time(2).Build())); + auto delta = packet_buffer.InsertPacket( + Packet(kH264FuA).Slice().SeqNum(4).Time(2).Marker().Build()); + + ASSERT_THAT(delta.packets, SizeIs(3)); + EXPECT_TRUE(delta.packets[0]->video_header.is_first_packet_in_frame); + EXPECT_FALSE(delta.packets[0]->video_header.is_last_packet_in_frame); + + EXPECT_FALSE(delta.packets[1]->video_header.is_first_packet_in_frame); + EXPECT_FALSE(delta.packets[1]->video_header.is_last_packet_in_frame); + + EXPECT_FALSE(delta.packets[2]->video_header.is_first_packet_in_frame); + EXPECT_TRUE(delta.packets[2]->video_header.is_last_packet_in_frame); +} + +TEST(H264PacketBufferTest, ResolutionSetOnFirstPacket) { + H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false); + + RTC_UNUSED(packet_buffer.InsertPacket( + Packet(kH264SingleNalu).Aud().SeqNum(1).Time(1).Build())); + auto res = packet_buffer.InsertPacket(Packet(kH264StapA) + .SpsWithResolution({320, 240}) + .Pps() + .Idr() + .SeqNum(2) + .Time(1) + .Marker() + .Build()); + + ASSERT_THAT(res.packets, SizeIs(2)); + EXPECT_THAT(res.packets[0]->video_header.width, Eq(320)); + EXPECT_THAT(res.packets[0]->video_header.height, Eq(240)); +} + +TEST(H264PacketBufferTest, KeyframeAndDeltaFrameSetOnFirstPacket) { + H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false); + + RTC_UNUSED(packet_buffer.InsertPacket( + Packet(kH264SingleNalu).Aud().SeqNum(1).Time(1).Build())); + auto key = packet_buffer.InsertPacket( + Packet(kH264StapA).Sps().Pps().Idr().SeqNum(2).Time(1).Marker().Build()); + + auto delta = packet_buffer.InsertPacket( + Packet(kH264SingleNalu).Slice().SeqNum(3).Time(2).Marker().Build()); + + ASSERT_THAT(key.packets, SizeIs(2)); + EXPECT_THAT(key.packets[0]->video_header.frame_type, + Eq(VideoFrameType::kVideoFrameKey)); + ASSERT_THAT(delta.packets, SizeIs(1)); + EXPECT_THAT(delta.packets[0]->video_header.frame_type, + Eq(VideoFrameType::kVideoFrameDelta)); +} + +TEST(H264PacketBufferTest, RtpSeqNumWrap) { + H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false); + + RTC_UNUSED(packet_buffer.InsertPacket( + Packet(kH264StapA).Sps().Pps().SeqNum(0xffff).Time(0).Build())); + + RTC_UNUSED(packet_buffer.InsertPacket( + Packet(kH264FuA).Idr().SeqNum(0).Time(0).Build())); + EXPECT_THAT(packet_buffer + .InsertPacket( + Packet(kH264FuA).Idr().SeqNum(1).Time(0).Marker().Build()) + .packets, + SizeIs(3)); +} + +TEST(H264PacketBufferTest, StapAFixedBitstream) { + H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false); + + auto packets = packet_buffer + .InsertPacket(Packet(kH264StapA) + .Sps({1, 2, 3}) + .Pps({4, 5, 6}) + .Idr({7, 8, 9}) + .SeqNum(0) + .Time(0) + .Marker() + .Build()) + .packets; + + ASSERT_THAT(packets, SizeIs(1)); + EXPECT_THAT(PacketPayload(packets[0]), + ElementsAreArray(FlatVector({StartCode(), + {kSps, 1, 2, 3}, + StartCode(), + {kPps, 4, 5, 6}, + StartCode(), + {kIdr, 7, 8, 9}}))); +} + +TEST(H264PacketBufferTest, SingleNaluFixedBitstream) { + H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false); + + RTC_UNUSED(packet_buffer.InsertPacket( + Packet(kH264SingleNalu).Sps({1, 2, 3}).SeqNum(0).Time(0).Build())); + RTC_UNUSED(packet_buffer.InsertPacket( + Packet(kH264SingleNalu).Pps({4, 5, 6}).SeqNum(1).Time(0).Build())); + auto packets = packet_buffer + .InsertPacket(Packet(kH264SingleNalu) + .Idr({7, 8, 9}) + .SeqNum(2) + .Time(0) + .Marker() + .Build()) + .packets; + + ASSERT_THAT(packets, SizeIs(3)); + EXPECT_THAT(PacketPayload(packets[0]), + ElementsAreArray(FlatVector({StartCode(), {kSps, 1, 2, 3}}))); + EXPECT_THAT(PacketPayload(packets[1]), + ElementsAreArray(FlatVector({StartCode(), {kPps, 4, 5, 6}}))); + EXPECT_THAT(PacketPayload(packets[2]), + ElementsAreArray(FlatVector({StartCode(), {kIdr, 7, 8, 9}}))); +} + +TEST(H264PacketBufferTest, StapaAndFuaFixedBitstream) { + H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false); + + RTC_UNUSED(packet_buffer.InsertPacket(Packet(kH264StapA) + .Sps({1, 2, 3}) + .Pps({4, 5, 6}) + .SeqNum(0) + .Time(0) + .Build())); + RTC_UNUSED(packet_buffer.InsertPacket(Packet(kH264FuA) + .Idr({8, 8, 8}) + .SeqNum(1) + .Time(0) + .AsFirstFragment() + .Build())); + auto packets = packet_buffer + .InsertPacket(Packet(kH264FuA) + .Idr({9, 9, 9}) + .SeqNum(2) + .Time(0) + .Marker() + .Build()) + .packets; + + ASSERT_THAT(packets, SizeIs(3)); + EXPECT_THAT( + PacketPayload(packets[0]), + ElementsAreArray(FlatVector( + {StartCode(), {kSps, 1, 2, 3}, StartCode(), {kPps, 4, 5, 6}}))); + EXPECT_THAT(PacketPayload(packets[1]), + ElementsAreArray(FlatVector({StartCode(), {8, 8, 8}}))); + // Third is a continuation of second, so only the payload is expected. + EXPECT_THAT(PacketPayload(packets[2]), + ElementsAreArray(FlatVector({{9, 9, 9}}))); +} + +TEST(H264PacketBufferTest, FullPacketBufferDoesNotBlockKeyframe) { + H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false); + + for (int i = 0; i < kBufferSize; ++i) { + EXPECT_THAT( + packet_buffer + .InsertPacket( + Packet(kH264SingleNalu).Slice().SeqNum(i).Time(0).Build()) + .packets, + IsEmpty()); + } + + EXPECT_THAT(packet_buffer + .InsertPacket(Packet(kH264StapA) + .Sps() + .Pps() + .Idr() + .SeqNum(kBufferSize) + .Time(1) + .Marker() + .Build()) + .packets, + SizeIs(1)); +} + +TEST(H264PacketBufferTest, TooManyNalusInPacket) { + H264PacketBuffer packet_buffer(/*allow_idr_only_keyframes=*/false); + + std::unique_ptr packet( + Packet(kH264StapA).Sps().Pps().Idr().SeqNum(1).Time(1).Marker().Build()); + auto& h264_header = + absl::get(packet->video_header.video_type_header); + h264_header.nalus_length = kMaxNalusPerPacket + 1; + + EXPECT_THAT(packet_buffer.InsertPacket(std::move(packet)).packets, IsEmpty()); +} + +} // namespace +} // namespace webrtc diff --git a/test/fuzzers/BUILD.gn b/test/fuzzers/BUILD.gn index b024ce2dc6..171577aab7 100644 --- a/test/fuzzers/BUILD.gn +++ b/test/fuzzers/BUILD.gn @@ -197,6 +197,7 @@ webrtc_fuzzer_test("flexfec_receiver_fuzzer") { webrtc_fuzzer_test("packet_buffer_fuzzer") { sources = [ "packet_buffer_fuzzer.cc" ] deps = [ + "../../modules/video_coding:packet_buffer", "../../modules/video_coding/", "../../system_wrappers", ] diff --git a/video/BUILD.gn b/video/BUILD.gn index 7d15d27c37..99a6e76fad 100644 --- a/video/BUILD.gn +++ b/video/BUILD.gn @@ -96,6 +96,7 @@ rtc_library("video") { "../modules/video_coding", "../modules/video_coding:codec_globals_headers", "../modules/video_coding:nack_requester", + "../modules/video_coding:packet_buffer", "../modules/video_coding:video_codec_interface", "../modules/video_coding:video_coding_utility", "../modules/video_processing", @@ -184,6 +185,7 @@ rtc_source_set("video_legacy") { "../modules/rtp_rtcp:rtp_video_header", "../modules/utility", "../modules/video_coding", + "../modules/video_coding:packet_buffer", "../modules/video_coding:video_codec_interface", "../modules/video_coding:video_coding_utility", "../modules/video_coding/deprecated:nack_module", @@ -698,6 +700,7 @@ if (rtc_include_tests) { "../modules/video_coding", "../modules/video_coding:codec_globals_headers", "../modules/video_coding:encoded_frame", + "../modules/video_coding:packet_buffer", "../modules/video_coding:video_codec_interface", "../modules/video_coding:video_coding_utility", "../modules/video_coding:webrtc_h264",