diff --git a/api/rtp_parameters.cc b/api/rtp_parameters.cc index c3f14d8f32..2b580b1084 100644 --- a/api/rtp_parameters.cc +++ b/api/rtp_parameters.cc @@ -132,6 +132,9 @@ const char RtpExtension::kGenericFrameDescriptorUri00[] = "http://www.webrtc.org/experiments/rtp-hdrext/generic-frame-descriptor-00"; const char RtpExtension::kGenericFrameDescriptorUri01[] = "http://www.webrtc.org/experiments/rtp-hdrext/generic-frame-descriptor-01"; +const char RtpExtension::kDependencyDescriptorUri[] = + "https://aomediacodec.github.io/av1-rtp-spec/" + "#dependency-descriptor-rtp-header-extension"; const char RtpExtension::kGenericFrameDescriptorUri[] = "http://www.webrtc.org/experiments/rtp-hdrext/generic-frame-descriptor-00"; @@ -180,6 +183,7 @@ bool RtpExtension::IsSupportedForVideo(const std::string& uri) { uri == webrtc::RtpExtension::kFrameMarkingUri || uri == webrtc::RtpExtension::kGenericFrameDescriptorUri00 || uri == webrtc::RtpExtension::kGenericFrameDescriptorUri01 || + uri == webrtc::RtpExtension::kDependencyDescriptorUri || uri == webrtc::RtpExtension::kColorSpaceUri || uri == webrtc::RtpExtension::kRidUri || uri == webrtc::RtpExtension::kRepairedRidUri; diff --git a/api/rtp_parameters.h b/api/rtp_parameters.h index 01c6ed4fa3..4f45d65a3d 100644 --- a/api/rtp_parameters.h +++ b/api/rtp_parameters.h @@ -289,6 +289,7 @@ struct RTC_EXPORT RtpExtension { // Experimental codec agnostic frame descriptor. static const char kGenericFrameDescriptorUri00[]; static const char kGenericFrameDescriptorUri01[]; + static const char kDependencyDescriptorUri[]; // TODO(bugs.webrtc.org/10243): Remove once dependencies have been updated. static const char kGenericFrameDescriptorUri[]; diff --git a/call/rtp_video_sender.cc b/call/rtp_video_sender.cc index 42a03c05c6..8ec534e0c9 100644 --- a/call/rtp_video_sender.cc +++ b/call/rtp_video_sender.cc @@ -493,6 +493,16 @@ EncodedImageCallback::Result RtpVideoSender::OnEncodedImage( rtp_streams_[stream_index].rtp_rtcp->ExpectedRetransmissionTimeMs(); } + if (encoded_image._frameType == VideoFrameType::kVideoFrameKey) { + // If encoder adapter produce FrameDependencyStructure, pass it so that + // dependency descriptor rtp header extension can be used. + // If not supported, disable using dependency descriptor by passing nullptr. + rtp_streams_[stream_index].sender_video->SetVideoStructure( + (codec_specific_info && codec_specific_info->template_structure) + ? &*codec_specific_info->template_structure + : nullptr); + } + bool send_result = rtp_streams_[stream_index].sender_video->SendVideo( rtp_config_.payload_type, codec_type_, rtp_timestamp, encoded_image.capture_time_ms_, encoded_image, fragmentation, diff --git a/call/rtp_video_sender_unittest.cc b/call/rtp_video_sender_unittest.cc index 9f4aef4867..a7336dab6c 100644 --- a/call/rtp_video_sender_unittest.cc +++ b/call/rtp_video_sender_unittest.cc @@ -18,6 +18,7 @@ #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" #include "modules/rtp_rtcp/source/byte_io.h" #include "modules/rtp_rtcp/source/rtcp_packet/nack.h" +#include "modules/rtp_rtcp/source/rtp_dependency_descriptor_extension.h" #include "modules/rtp_rtcp/source/rtp_packet.h" #include "modules/video_coding/fec_controller_default.h" #include "modules/video_coding/include/video_codec_interface.h" @@ -33,10 +34,9 @@ #include "video/send_statistics_proxy.h" using ::testing::_; -using ::testing::Invoke; using ::testing::NiceMock; using ::testing::SaveArg; -using ::testing::Unused; +using ::testing::SizeIs; namespace webrtc { namespace { @@ -51,6 +51,7 @@ const int16_t kInitialTl0PicIdx1 = 99; const int16_t kInitialTl0PicIdx2 = 199; const int64_t kRetransmitWindowSizeMs = 500; const int kTransportsSequenceExtensionId = 7; +const int kDependencyDescriptorExtensionId = 8; class MockRtcpIntraFrameObserver : public RtcpIntraFrameObserver { public: @@ -104,6 +105,8 @@ VideoSendStream::Config CreateVideoSendStreamConfig( config.rtp.nack.rtp_history_ms = 1000; config.rtp.extensions.emplace_back(RtpExtension::kTransportSequenceNumberUri, kTransportsSequenceExtensionId); + config.rtp.extensions.emplace_back(RtpDependencyDescriptorExtension::kUri, + kDependencyDescriptorExtensionId); return config; } @@ -648,6 +651,135 @@ TEST(RtpVideoSenderTest, EarlyRetransmits) { test.AdvanceTime(TimeDelta::Millis(33)); } +TEST(RtpVideoSenderTest, SupportsDependencyDescriptor) { + test::ScopedFieldTrials trials("WebRTC-GenericDescriptor/Enabled/"); + + RtpVideoSenderTestFixture test({kSsrc1}, {}, kPayloadType, {}); + test.router()->SetActive(true); + + RtpHeaderExtensionMap extensions; + extensions.Register( + kDependencyDescriptorExtensionId); + std::vector sent_packets; + ON_CALL(test.transport(), SendRtp) + .WillByDefault([&](const uint8_t* packet, size_t length, + const PacketOptions& options) { + sent_packets.emplace_back(&extensions); + EXPECT_TRUE(sent_packets.back().Parse(packet, length)); + return true; + }); + + const uint8_t kPayload[1] = {'a'}; + EncodedImage encoded_image; + encoded_image.SetTimestamp(1); + encoded_image.capture_time_ms_ = 2; + encoded_image.SetEncodedData( + EncodedImageBuffer::Create(kPayload, sizeof(kPayload))); + + CodecSpecificInfo codec_specific; + codec_specific.codecType = VideoCodecType::kVideoCodecGeneric; + codec_specific.template_structure.emplace(); + codec_specific.template_structure->num_decode_targets = 1; + codec_specific.template_structure->templates = { + GenericFrameInfo::Builder().T(0).Dtis("S").Build(), + GenericFrameInfo::Builder().T(0).Dtis("S").Fdiffs({2}).Build(), + GenericFrameInfo::Builder().T(1).Dtis("D").Fdiffs({1}).Build(), + }; + + // Send two tiny images, mapping to single RTP packets. + // Send in key frame. + encoded_image._frameType = VideoFrameType::kVideoFrameKey; + codec_specific.generic_frame_info = + GenericFrameInfo::Builder().T(0).Dtis("S").Build(); + codec_specific.generic_frame_info->encoder_buffers = {{0, false, true}}; + EXPECT_EQ(test.router() + ->OnEncodedImage(encoded_image, &codec_specific, nullptr) + .error, + EncodedImageCallback::Result::OK); + test.AdvanceTime(TimeDelta::Millis(33)); + ASSERT_THAT(sent_packets, SizeIs(1)); + EXPECT_TRUE( + sent_packets.back().HasExtension()); + + // Send in delta frame. + encoded_image._frameType = VideoFrameType::kVideoFrameDelta; + codec_specific.template_structure = absl::nullopt; + codec_specific.generic_frame_info = + GenericFrameInfo::Builder().T(1).Dtis("D").Build(); + codec_specific.generic_frame_info->encoder_buffers = {{0, true, false}}; + EXPECT_EQ(test.router() + ->OnEncodedImage(encoded_image, &codec_specific, nullptr) + .error, + EncodedImageCallback::Result::OK); + test.AdvanceTime(TimeDelta::Millis(33)); + ASSERT_THAT(sent_packets, SizeIs(2)); + EXPECT_TRUE( + sent_packets.back().HasExtension()); +} + +TEST(RtpVideoSenderTest, SupportsStoppingUsingDependencyDescriptor) { + test::ScopedFieldTrials trials("WebRTC-GenericDescriptor/Enabled/"); + + RtpVideoSenderTestFixture test({kSsrc1}, {}, kPayloadType, {}); + test.router()->SetActive(true); + + RtpHeaderExtensionMap extensions; + extensions.Register( + kDependencyDescriptorExtensionId); + std::vector sent_packets; + ON_CALL(test.transport(), SendRtp) + .WillByDefault([&](const uint8_t* packet, size_t length, + const PacketOptions& options) { + sent_packets.emplace_back(&extensions); + EXPECT_TRUE(sent_packets.back().Parse(packet, length)); + return true; + }); + + const uint8_t kPayload[1] = {'a'}; + EncodedImage encoded_image; + encoded_image.SetTimestamp(1); + encoded_image.capture_time_ms_ = 2; + encoded_image.SetEncodedData( + EncodedImageBuffer::Create(kPayload, sizeof(kPayload))); + + CodecSpecificInfo codec_specific; + codec_specific.codecType = VideoCodecType::kVideoCodecGeneric; + codec_specific.template_structure.emplace(); + codec_specific.template_structure->num_decode_targets = 1; + codec_specific.template_structure->templates = { + GenericFrameInfo::Builder().T(0).Dtis("S").Build(), + GenericFrameInfo::Builder().T(0).Dtis("S").Fdiffs({2}).Build(), + GenericFrameInfo::Builder().T(1).Dtis("D").Fdiffs({1}).Build(), + }; + + // Send two tiny images, mapping to single RTP packets. + // Send in a key frame. + encoded_image._frameType = VideoFrameType::kVideoFrameKey; + codec_specific.generic_frame_info = + GenericFrameInfo::Builder().T(0).Dtis("S").Build(); + codec_specific.generic_frame_info->encoder_buffers = {{0, false, true}}; + EXPECT_EQ(test.router() + ->OnEncodedImage(encoded_image, &codec_specific, nullptr) + .error, + EncodedImageCallback::Result::OK); + test.AdvanceTime(TimeDelta::Millis(33)); + ASSERT_THAT(sent_packets, SizeIs(1)); + EXPECT_TRUE( + sent_packets.back().HasExtension()); + + // Send in a new key frame without the support for the dependency descriptor. + encoded_image._frameType = VideoFrameType::kVideoFrameKey; + codec_specific.template_structure = absl::nullopt; + EXPECT_EQ(test.router() + ->OnEncodedImage(encoded_image, &codec_specific, nullptr) + .error, + EncodedImageCallback::Result::OK); + test.AdvanceTime(TimeDelta::Millis(33)); + ASSERT_THAT(sent_packets, SizeIs(2)); + EXPECT_FALSE( + sent_packets.back().HasExtension()); +} + TEST(RtpVideoSenderTest, CanSetZeroBitrateWithOverhead) { test::ScopedFieldTrials trials("WebRTC-SendSideBwe-WithOverhead/Enabled/"); RtpVideoSenderTestFixture test({kSsrc1}, {kRtxSsrc1}, kPayloadType, {});