Allow providing the absolute capture time extension when packetizing a frame.

Bug: b/150859541
Change-Id: Iffb6ee84f49ffa64fdb0633248864d2dfd6e9ff3
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/234868
Commit-Queue: Paul Hallak <phallak@google.com>
Reviewed-by: Danil Chapovalov <danilchap@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#35194}

modules/rtp_rtcp/BUILD.gn

@@ -398,6 +398,7 @@ rtc_library("rtp_video_header") {
     "source/rtp_video_header.h",
   ]
   deps = [
+    "../../api:rtp_headers",
     "../../api/transport/rtp:dependency_descriptor",
     "../../api/video:video_frame",
     "../../api/video:video_frame_type",

modules/rtp_rtcp/source/rtp_sender_video.cc

@@ -315,9 +315,7 @@ void RTPSenderVideo::SetVideoLayersAllocationInternal(
   allocation_ = std::move(allocation);
 }
 
-void RTPSenderVideo::AddRtpHeaderExtensions(
+void RTPSenderVideo::AddRtpHeaderExtensions(const RTPVideoHeader& video_header,
-    const RTPVideoHeader& video_header,
-    const absl::optional<AbsoluteCaptureTime>& absolute_capture_time,
                                             bool first_packet,
                                             bool last_packet,
                                             RtpPacketToSend* packet) const {
@@ -369,8 +367,9 @@ void RTPSenderVideo::AddRtpHeaderExtensions(
     packet->SetExtension<PlayoutDelayLimits>(current_playout_delay_);
   }
 
-  if (first_packet && absolute_capture_time) {
+  if (first_packet && video_header.absolute_capture_time.has_value()) {
-    packet->SetExtension<AbsoluteCaptureTimeExtension>(*absolute_capture_time);
+    packet->SetExtension<AbsoluteCaptureTimeExtension>(
+        *video_header.absolute_capture_time);
   }
 
   if (video_header.generic) {
@@ -477,8 +476,7 @@ bool RTPSenderVideo::SendVideo(
     int64_t capture_time_ms,
     rtc::ArrayView<const uint8_t> payload,
     RTPVideoHeader video_header,
-    absl::optional<int64_t> expected_retransmission_time_ms,
+    absl::optional<int64_t> expected_retransmission_time_ms) {
-    absl::optional<int64_t> estimated_capture_clock_offset_ms) {
 #if RTC_TRACE_EVENTS_ENABLED
   TRACE_EVENT_ASYNC_STEP1("webrtc", "Video", capture_time_ms, "Send", "type",
                           FrameTypeToString(video_header.frame_type));
@@ -538,22 +536,31 @@ bool RTPSenderVideo::SendVideo(
   single_packet->SetTimestamp(rtp_timestamp);
   single_packet->set_capture_time_ms(capture_time_ms);
 
-  const absl::optional<AbsoluteCaptureTime> absolute_capture_time =
+  // Construct the absolute capture time extension if not provided.
+  if (!video_header.absolute_capture_time.has_value()) {
+    video_header.absolute_capture_time.emplace();
+    video_header.absolute_capture_time->absolute_capture_timestamp =
+        Int64MsToUQ32x32(
+            clock_->ConvertTimestampToNtpTimeInMilliseconds(capture_time_ms));
+    if (include_capture_clock_offset_) {
+      video_header.absolute_capture_time->estimated_capture_clock_offset = 0;
+    }
+  }
+
+  // Let `absolute_capture_time_sender_` decide if the extension should be sent.
+  video_header.absolute_capture_time =
       absolute_capture_time_sender_.OnSendPacket(
           AbsoluteCaptureTimeSender::GetSource(single_packet->Ssrc(),
                                                single_packet->Csrcs()),
           single_packet->Timestamp(), kVideoPayloadTypeFrequency,
-          Int64MsToUQ32x32(
+          video_header.absolute_capture_time->absolute_capture_timestamp,
-              clock_->ConvertTimestampToNtpTimeInMilliseconds(capture_time_ms)),
+          video_header.absolute_capture_time->estimated_capture_clock_offset);
-          /*estimated_capture_clock_offset=*/
-          include_capture_clock_offset_ ? estimated_capture_clock_offset_ms
-                                        : absl::nullopt);
 
   auto first_packet = std::make_unique<RtpPacketToSend>(*single_packet);
   auto middle_packet = std::make_unique<RtpPacketToSend>(*single_packet);
   auto last_packet = std::make_unique<RtpPacketToSend>(*single_packet);
   // Simplest way to estimate how much extensions would occupy is to set them.
-  AddRtpHeaderExtensions(video_header, absolute_capture_time,
+  AddRtpHeaderExtensions(video_header,
                          /*first_packet=*/true, /*last_packet=*/true,
                          single_packet.get());
   if (video_structure_ != nullptr &&
@@ -568,13 +575,13 @@ bool RTPSenderVideo::SendVideo(
     video_structure_ = nullptr;
   }
 
-  AddRtpHeaderExtensions(video_header, absolute_capture_time,
+  AddRtpHeaderExtensions(video_header,
                          /*first_packet=*/true, /*last_packet=*/false,
                          first_packet.get());
-  AddRtpHeaderExtensions(video_header, absolute_capture_time,
+  AddRtpHeaderExtensions(video_header,
                          /*first_packet=*/false, /*last_packet=*/false,
                          middle_packet.get());
-  AddRtpHeaderExtensions(video_header, absolute_capture_time,
+  AddRtpHeaderExtensions(video_header,
                          /*first_packet=*/false, /*last_packet=*/true,
                          last_packet.get());
 

modules/rtp_rtcp/source/rtp_sender_video.h

@@ -90,20 +90,14 @@ class RTPSenderVideo {
 
   // expected_retransmission_time_ms.has_value() -> retransmission allowed.
   // `capture_time_ms` and `clock::CurrentTime` should be using the same epoch.
-  // Calls to this method is assumed to be externally serialized.
+  // Calls to this method are assumed to be externally serialized.
-  // `estimated_capture_clock_offset_ms` is an estimated clock offset between
-  // this sender and the original capturer, for this video packet. See
-  // http://www.webrtc.org/experiments/rtp-hdrext/abs-capture-time for more
-  // details. If the sender and the capture has the same clock, it is supposed
-  // to be zero valued, which is given as the default.
   bool SendVideo(int payload_type,
                  absl::optional<VideoCodecType> codec_type,
                  uint32_t rtp_timestamp,
                  int64_t capture_time_ms,
                  rtc::ArrayView<const uint8_t> payload,
                  RTPVideoHeader video_header,
-                 absl::optional<int64_t> expected_retransmission_time_ms,
+                 absl::optional<int64_t> expected_retransmission_time_ms);
-                 absl::optional<int64_t> estimated_capture_clock_offset_ms = 0);
 
   bool SendEncodedImage(
       int payload_type,
@@ -170,9 +164,7 @@ class RTPSenderVideo {
       const FrameDependencyStructure* video_structure);
   void SetVideoLayersAllocationInternal(VideoLayersAllocation allocation);
 
-  void AddRtpHeaderExtensions(
+  void AddRtpHeaderExtensions(const RTPVideoHeader& video_header,
-      const RTPVideoHeader& video_header,
-      const absl::optional<AbsoluteCaptureTime>& absolute_capture_time,
                               bool first_packet,
                               bool last_packet,
                               RtpPacketToSend* packet) const

modules/rtp_rtcp/source/rtp_sender_video_unittest.cc

@@ -16,6 +16,7 @@
 #include <vector>
 
 #include "absl/memory/memory.h"
+#include "api/rtp_headers.h"
 #include "api/test/mock_frame_encryptor.h"
 #include "api/transport/field_trial_based_config.h"
 #include "api/transport/rtp/dependency_descriptor.h"
@@ -1181,24 +1182,27 @@ TEST_P(RtpSenderVideoTest, AbsoluteCaptureTime) {
                                kAbsoluteCaptureTimestampMs, kFrame, hdr,
                                kDefaultExpectedRetransmissionTimeMs);
 
+  absl::optional<AbsoluteCaptureTime> absolute_capture_time;
+
   // It is expected that one and only one of the packets sent on this video
-  // frame has absolute capture time header extension. And no absolute capture
+  // frame has absolute capture time header extension.
-  // time header extensions include capture clock offset.
-  int packets_with_abs_capture_time = 0;
   for (const RtpPacketReceived& packet : transport_.sent_packets()) {
-    auto absolute_capture_time =
+    if (absolute_capture_time.has_value()) {
+      EXPECT_FALSE(packet.HasExtension<AbsoluteCaptureTimeExtension>());
+    } else {
+      absolute_capture_time =
           packet.GetExtension<AbsoluteCaptureTimeExtension>();
-    if (absolute_capture_time) {
+    }
-      ++packets_with_abs_capture_time;
+  }
+
+  // Verify the capture timestamp and that the clock offset is not set.
+  ASSERT_TRUE(absolute_capture_time.has_value());
   EXPECT_EQ(
       absolute_capture_time->absolute_capture_timestamp,
       Int64MsToUQ32x32(fake_clock_.ConvertTimestampToNtpTimeInMilliseconds(
           kAbsoluteCaptureTimestampMs)));
   EXPECT_FALSE(
       absolute_capture_time->estimated_capture_clock_offset.has_value());
-    }
-  }
-  EXPECT_EQ(packets_with_abs_capture_time, 1);
 }
 
 // Essentially the same test as AbsoluteCaptureTime but with a field trial.
@@ -1214,31 +1218,64 @@ TEST_P(RtpSenderVideoTest, AbsoluteCaptureTimeWithCaptureClockOffset) {
                                           kAbsoluteCaptureTimeExtensionId);
 
   RTPVideoHeader hdr;
-  const absl::optional<int64_t> kExpectedCaptureClockOffset =
-      absl::make_optional(1234);
   hdr.frame_type = VideoFrameType::kVideoFrameKey;
-  rtp_sender_video_->SendVideo(
+  rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp,
-      kPayload, kType, kTimestamp, kAbsoluteCaptureTimestampMs, kFrame, hdr,
+                               kAbsoluteCaptureTimestampMs, kFrame, hdr,
-      kDefaultExpectedRetransmissionTimeMs, kExpectedCaptureClockOffset);
+                               kDefaultExpectedRetransmissionTimeMs);
+
+  absl::optional<AbsoluteCaptureTime> absolute_capture_time;
 
   // It is expected that one and only one of the packets sent on this video
-  // frame has absolute capture time header extension. And it includes capture
+  // frame has absolute capture time header extension.
-  // clock offset.
-  int packets_with_abs_capture_time = 0;
   for (const RtpPacketReceived& packet : transport_.sent_packets()) {
-    auto absolute_capture_time =
+    if (absolute_capture_time.has_value()) {
+      EXPECT_FALSE(packet.HasExtension<AbsoluteCaptureTimeExtension>());
+    } else {
+      absolute_capture_time =
           packet.GetExtension<AbsoluteCaptureTimeExtension>();
-    if (absolute_capture_time) {
+    }
-      ++packets_with_abs_capture_time;
+  }
+
+  // Verify the capture timestamp and that the clock offset is set to zero.
+  ASSERT_TRUE(absolute_capture_time.has_value());
   EXPECT_EQ(
       absolute_capture_time->absolute_capture_timestamp,
       Int64MsToUQ32x32(fake_clock_.ConvertTimestampToNtpTimeInMilliseconds(
           kAbsoluteCaptureTimestampMs)));
-      EXPECT_EQ(kExpectedCaptureClockOffset,
+  EXPECT_EQ(absolute_capture_time->estimated_capture_clock_offset, 0);
-                absolute_capture_time->estimated_capture_clock_offset);
+}
+
+TEST_P(RtpSenderVideoTest, AbsoluteCaptureTimeWithExtensionProvided) {
+  constexpr AbsoluteCaptureTime kAbsoluteCaptureTime = {
+      123,
+      absl::optional<int64_t>(456),
+  };
+  uint8_t kFrame[kMaxPacketLength];
+  rtp_module_->RegisterRtpHeaderExtension(AbsoluteCaptureTimeExtension::Uri(),
+                                          kAbsoluteCaptureTimeExtensionId);
+
+  RTPVideoHeader hdr;
+  hdr.frame_type = VideoFrameType::kVideoFrameKey;
+  hdr.absolute_capture_time = kAbsoluteCaptureTime;
+  rtp_sender_video_->SendVideo(kPayload, kType, kTimestamp,
+                               /*capture_time_ms=*/789, kFrame, hdr,
+                               kDefaultExpectedRetransmissionTimeMs);
+
+  absl::optional<AbsoluteCaptureTime> absolute_capture_time;
+
+  // It is expected that one and only one of the packets sent on this video
+  // frame has absolute capture time header extension.
+  for (const RtpPacketReceived& packet : transport_.sent_packets()) {
+    if (absolute_capture_time.has_value()) {
+      EXPECT_FALSE(packet.HasExtension<AbsoluteCaptureTimeExtension>());
+    } else {
+      absolute_capture_time =
+          packet.GetExtension<AbsoluteCaptureTimeExtension>();
     }
   }
-  EXPECT_EQ(packets_with_abs_capture_time, 1);
+
+  // Verify the extension.
+  EXPECT_EQ(absolute_capture_time, kAbsoluteCaptureTime);
 }
 
 TEST_P(RtpSenderVideoTest, PopulatesPlayoutDelay) {

modules/rtp_rtcp/source/rtp_video_header.h

@@ -16,6 +16,7 @@
 #include "absl/container/inlined_vector.h"
 #include "absl/types/optional.h"
 #include "absl/types/variant.h"
+#include "api/rtp_headers.h"
 #include "api/transport/rtp/dependency_descriptor.h"
 #include "api/video/color_space.h"
 #include "api/video/video_codec_type.h"
@@ -81,6 +82,11 @@ struct RTPVideoHeader {
   // carries the webrtc::VideoFrame id field from the sender to the receiver.
   absl::optional<uint16_t> video_frame_tracking_id;
   RTPVideoTypeHeader video_type_header;
+
+  // When provided, is sent as is as an RTP header extension according to
+  // http://www.webrtc.org/experiments/rtp-hdrext/abs-capture-time.
+  // Otherwise, it is derived from other relevant information.
+  absl::optional<AbsoluteCaptureTime> absolute_capture_time;
 };
 
 }  // namespace webrtc