diff --git a/modules/rtp_rtcp/BUILD.gn b/modules/rtp_rtcp/BUILD.gn index 00431dc097..f7a0506804 100644 --- a/modules/rtp_rtcp/BUILD.gn +++ b/modules/rtp_rtcp/BUILD.gn @@ -398,6 +398,7 @@ rtc_library("rtp_video_header") { "source/rtp_video_header.h", ] deps = [ + "../../api:rtp_headers", "../../api/transport/rtp:dependency_descriptor", "../../api/video:video_frame", "../../api/video:video_frame_type", diff --git a/modules/rtp_rtcp/source/rtp_sender_video.cc b/modules/rtp_rtcp/source/rtp_sender_video.cc index 9e909c984e..c9fc782308 100644 --- a/modules/rtp_rtcp/source/rtp_sender_video.cc +++ b/modules/rtp_rtcp/source/rtp_sender_video.cc @@ -315,12 +315,10 @@ void RTPSenderVideo::SetVideoLayersAllocationInternal( allocation_ = std::move(allocation); } -void RTPSenderVideo::AddRtpHeaderExtensions( - const RTPVideoHeader& video_header, - const absl::optional& absolute_capture_time, - bool first_packet, - bool last_packet, - RtpPacketToSend* packet) const { +void RTPSenderVideo::AddRtpHeaderExtensions(const RTPVideoHeader& video_header, + bool first_packet, + bool last_packet, + RtpPacketToSend* packet) const { // Send color space when changed or if the frame is a key frame. Keep // sending color space information until the first base layer frame to // guarantee that the information is retrieved by the receiver. @@ -369,8 +367,9 @@ void RTPSenderVideo::AddRtpHeaderExtensions( packet->SetExtension(current_playout_delay_); } - if (first_packet && absolute_capture_time) { - packet->SetExtension(*absolute_capture_time); + if (first_packet && video_header.absolute_capture_time.has_value()) { + packet->SetExtension( + *video_header.absolute_capture_time); } if (video_header.generic) { @@ -477,8 +476,7 @@ bool RTPSenderVideo::SendVideo( int64_t capture_time_ms, rtc::ArrayView payload, RTPVideoHeader video_header, - absl::optional expected_retransmission_time_ms, - absl::optional estimated_capture_clock_offset_ms) { + absl::optional expected_retransmission_time_ms) { #if RTC_TRACE_EVENTS_ENABLED TRACE_EVENT_ASYNC_STEP1("webrtc", "Video", capture_time_ms, "Send", "type", FrameTypeToString(video_header.frame_type)); @@ -538,22 +536,31 @@ bool RTPSenderVideo::SendVideo( single_packet->SetTimestamp(rtp_timestamp); single_packet->set_capture_time_ms(capture_time_ms); - const absl::optional absolute_capture_time = + // Construct the absolute capture time extension if not provided. + if (!video_header.absolute_capture_time.has_value()) { + video_header.absolute_capture_time.emplace(); + video_header.absolute_capture_time->absolute_capture_timestamp = + Int64MsToUQ32x32( + clock_->ConvertTimestampToNtpTimeInMilliseconds(capture_time_ms)); + if (include_capture_clock_offset_) { + video_header.absolute_capture_time->estimated_capture_clock_offset = 0; + } + } + + // Let `absolute_capture_time_sender_` decide if the extension should be sent. + video_header.absolute_capture_time = absolute_capture_time_sender_.OnSendPacket( AbsoluteCaptureTimeSender::GetSource(single_packet->Ssrc(), single_packet->Csrcs()), single_packet->Timestamp(), kVideoPayloadTypeFrequency, - Int64MsToUQ32x32( - clock_->ConvertTimestampToNtpTimeInMilliseconds(capture_time_ms)), - /*estimated_capture_clock_offset=*/ - include_capture_clock_offset_ ? estimated_capture_clock_offset_ms - : absl::nullopt); + video_header.absolute_capture_time->absolute_capture_timestamp, + video_header.absolute_capture_time->estimated_capture_clock_offset); auto first_packet = std::make_unique(*single_packet); auto middle_packet = std::make_unique(*single_packet); auto last_packet = std::make_unique(*single_packet); // Simplest way to estimate how much extensions would occupy is to set them. - AddRtpHeaderExtensions(video_header, absolute_capture_time, + AddRtpHeaderExtensions(video_header, /*first_packet=*/true, /*last_packet=*/true, single_packet.get()); if (video_structure_ != nullptr && @@ -568,13 +575,13 @@ bool RTPSenderVideo::SendVideo( video_structure_ = nullptr; } - AddRtpHeaderExtensions(video_header, absolute_capture_time, + AddRtpHeaderExtensions(video_header, /*first_packet=*/true, /*last_packet=*/false, first_packet.get()); - AddRtpHeaderExtensions(video_header, absolute_capture_time, + AddRtpHeaderExtensions(video_header, /*first_packet=*/false, /*last_packet=*/false, middle_packet.get()); - AddRtpHeaderExtensions(video_header, absolute_capture_time, + AddRtpHeaderExtensions(video_header, /*first_packet=*/false, /*last_packet=*/true, last_packet.get()); diff --git a/modules/rtp_rtcp/source/rtp_sender_video.h b/modules/rtp_rtcp/source/rtp_sender_video.h index f4f15f3ba4..5164969489 100644 --- a/modules/rtp_rtcp/source/rtp_sender_video.h +++ b/modules/rtp_rtcp/source/rtp_sender_video.h @@ -90,20 +90,14 @@ class RTPSenderVideo { // expected_retransmission_time_ms.has_value() -> retransmission allowed. // `capture_time_ms` and `clock::CurrentTime` should be using the same epoch. - // Calls to this method is assumed to be externally serialized. - // `estimated_capture_clock_offset_ms` is an estimated clock offset between - // this sender and the original capturer, for this video packet. See - // http://www.webrtc.org/experiments/rtp-hdrext/abs-capture-time for more - // details. If the sender and the capture has the same clock, it is supposed - // to be zero valued, which is given as the default. + // Calls to this method are assumed to be externally serialized. bool SendVideo(int payload_type, absl::optional codec_type, uint32_t rtp_timestamp, int64_t capture_time_ms, rtc::ArrayView payload, RTPVideoHeader video_header, - absl::optional expected_retransmission_time_ms, - absl::optional estimated_capture_clock_offset_ms = 0); + absl::optional expected_retransmission_time_ms); bool SendEncodedImage( int payload_type, @@ -170,12 +164,10 @@ class RTPSenderVideo { const FrameDependencyStructure* video_structure); void SetVideoLayersAllocationInternal(VideoLayersAllocation allocation); - void AddRtpHeaderExtensions( - const RTPVideoHeader& video_header, - const absl::optional& absolute_capture_time, - bool first_packet, - bool last_packet, - RtpPacketToSend* packet) const + void AddRtpHeaderExtensions(const RTPVideoHeader& video_header, + bool first_packet, + bool last_packet, + RtpPacketToSend* packet) const RTC_EXCLUSIVE_LOCKS_REQUIRED(send_checker_); size_t FecPacketOverhead() const RTC_EXCLUSIVE_LOCKS_REQUIRED(send_checker_); diff --git a/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc b/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc index d3782be43e..d746ad1ff0 100644 --- a/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc +++ b/modules/rtp_rtcp/source/rtp_sender_video_unittest.cc @@ -16,6 +16,7 @@ #include #include "absl/memory/memory.h" +#include "api/rtp_headers.h" #include "api/test/mock_frame_encryptor.h" #include "api/transport/field_trial_based_config.h" #include "api/transport/rtp/dependency_descriptor.h" @@ -1181,24 +1182,27 @@ TEST_P(RtpSenderVideoTest, AbsoluteCaptureTime) { kAbsoluteCaptureTimestampMs, kFrame, hdr, kDefaultExpectedRetransmissionTimeMs); + absl::optional absolute_capture_time; + // It is expected that one and only one of the packets sent on this video - // frame has absolute capture time header extension. And no absolute capture - // time header extensions include capture clock offset. - int packets_with_abs_capture_time = 0; + // frame has absolute capture time header extension. for (const RtpPacketReceived& packet : transport_.sent_packets()) { - auto absolute_capture_time = - packet.GetExtension(); - if (absolute_capture_time) { - ++packets_with_abs_capture_time; - EXPECT_EQ( - absolute_capture_time->absolute_capture_timestamp, - Int64MsToUQ32x32(fake_clock_.ConvertTimestampToNtpTimeInMilliseconds( - kAbsoluteCaptureTimestampMs))); - EXPECT_FALSE( - absolute_capture_time->estimated_capture_clock_offset.has_value()); + if (absolute_capture_time.has_value()) { + EXPECT_FALSE(packet.HasExtension()); + } else { + absolute_capture_time = + packet.GetExtension(); } } - EXPECT_EQ(packets_with_abs_capture_time, 1); + + // Verify the capture timestamp and that the clock offset is not set. + ASSERT_TRUE(absolute_capture_time.has_value()); + EXPECT_EQ( + absolute_capture_time->absolute_capture_timestamp, + Int64MsToUQ32x32(fake_clock_.ConvertTimestampToNtpTimeInMilliseconds( + kAbsoluteCaptureTimestampMs))); + EXPECT_FALSE( + absolute_capture_time->estimated_capture_clock_offset.has_value()); } // Essentially the same test as AbsoluteCaptureTime but with a field trial. @@ -1214,31 +1218,64 @@ TEST_P(RtpSenderVideoTest, AbsoluteCaptureTimeWithCaptureClockOffset) { kAbsoluteCaptureTimeExtensionId); RTPVideoHeader hdr; - const absl::optional kExpectedCaptureClockOffset = - absl::make_optional(1234); hdr.frame_type = VideoFrameType::kVideoFrameKey; - rtp_sender_video_->SendVideo( - kPayload, kType, kTimestamp, kAbsoluteCaptureTimestampMs, kFrame, hdr, - kDefaultExpectedRetransmissionTimeMs, kExpectedCaptureClockOffset); + rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, + kAbsoluteCaptureTimestampMs, kFrame, hdr, + kDefaultExpectedRetransmissionTimeMs); + + absl::optional absolute_capture_time; // It is expected that one and only one of the packets sent on this video - // frame has absolute capture time header extension. And it includes capture - // clock offset. - int packets_with_abs_capture_time = 0; + // frame has absolute capture time header extension. for (const RtpPacketReceived& packet : transport_.sent_packets()) { - auto absolute_capture_time = - packet.GetExtension(); - if (absolute_capture_time) { - ++packets_with_abs_capture_time; - EXPECT_EQ( - absolute_capture_time->absolute_capture_timestamp, - Int64MsToUQ32x32(fake_clock_.ConvertTimestampToNtpTimeInMilliseconds( - kAbsoluteCaptureTimestampMs))); - EXPECT_EQ(kExpectedCaptureClockOffset, - absolute_capture_time->estimated_capture_clock_offset); + if (absolute_capture_time.has_value()) { + EXPECT_FALSE(packet.HasExtension()); + } else { + absolute_capture_time = + packet.GetExtension(); } } - EXPECT_EQ(packets_with_abs_capture_time, 1); + + // Verify the capture timestamp and that the clock offset is set to zero. + ASSERT_TRUE(absolute_capture_time.has_value()); + EXPECT_EQ( + absolute_capture_time->absolute_capture_timestamp, + Int64MsToUQ32x32(fake_clock_.ConvertTimestampToNtpTimeInMilliseconds( + kAbsoluteCaptureTimestampMs))); + EXPECT_EQ(absolute_capture_time->estimated_capture_clock_offset, 0); +} + +TEST_P(RtpSenderVideoTest, AbsoluteCaptureTimeWithExtensionProvided) { + constexpr AbsoluteCaptureTime kAbsoluteCaptureTime = { + 123, + absl::optional(456), + }; + uint8_t kFrame[kMaxPacketLength]; + rtp_module_->RegisterRtpHeaderExtension(AbsoluteCaptureTimeExtension::Uri(), + kAbsoluteCaptureTimeExtensionId); + + RTPVideoHeader hdr; + hdr.frame_type = VideoFrameType::kVideoFrameKey; + hdr.absolute_capture_time = kAbsoluteCaptureTime; + rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp, + /*capture_time_ms=*/789, kFrame, hdr, + kDefaultExpectedRetransmissionTimeMs); + + absl::optional absolute_capture_time; + + // It is expected that one and only one of the packets sent on this video + // frame has absolute capture time header extension. + for (const RtpPacketReceived& packet : transport_.sent_packets()) { + if (absolute_capture_time.has_value()) { + EXPECT_FALSE(packet.HasExtension()); + } else { + absolute_capture_time = + packet.GetExtension(); + } + } + + // Verify the extension. + EXPECT_EQ(absolute_capture_time, kAbsoluteCaptureTime); } TEST_P(RtpSenderVideoTest, PopulatesPlayoutDelay) { diff --git a/modules/rtp_rtcp/source/rtp_video_header.h b/modules/rtp_rtcp/source/rtp_video_header.h index c1be76fa4c..115b17d36d 100644 --- a/modules/rtp_rtcp/source/rtp_video_header.h +++ b/modules/rtp_rtcp/source/rtp_video_header.h @@ -16,6 +16,7 @@ #include "absl/container/inlined_vector.h" #include "absl/types/optional.h" #include "absl/types/variant.h" +#include "api/rtp_headers.h" #include "api/transport/rtp/dependency_descriptor.h" #include "api/video/color_space.h" #include "api/video/video_codec_type.h" @@ -81,6 +82,11 @@ struct RTPVideoHeader { // carries the webrtc::VideoFrame id field from the sender to the receiver. absl::optional video_frame_tracking_id; RTPVideoTypeHeader video_type_header; + + // When provided, is sent as is as an RTP header extension according to + // http://www.webrtc.org/experiments/rtp-hdrext/abs-capture-time. + // Otherwise, it is derived from other relevant information. + absl::optional absolute_capture_time; }; } // namespace webrtc