Add absolute capture time property to rtp sources.

This part of the effort to implement A/V sync metric.

Bug: webrtc:10739
Change-Id: I4adba1b99b37b31868168e37d9aa8e03f8ea6d4e
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/159886
Commit-Queue: Ruslan Burakov <kuddai@webrtc.org>
Reviewed-by: Björn Terelius <terelius@webrtc.org>
Reviewed-by: Minyue Li <minyue@webrtc.org>
Reviewed-by: Danil Chapovalov <danilchap@webrtc.org>
Reviewed-by: Ruslan Burakov <kuddai@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#29849}

api/transport/rtp/BUILD.gn

@@ -14,6 +14,7 @@ rtc_source_set("rtp_source") {
     "rtp_source.h",
   ]
   deps = [
+    "../../../api:rtp_headers",
     "../../../rtc_base:checks",
     "//third_party/abseil-cpp/absl/types:optional",
   ]

api/transport/rtp/rtp_source.h

@@ -14,6 +14,7 @@
 #include <stdint.h>
 
 #include "absl/types/optional.h"
+#include "api/rtp_headers.h"
 #include "rtc_base/checks.h"
 
 namespace webrtc {
@@ -25,17 +26,35 @@ enum class RtpSourceType {
 
 class RtpSource {
  public:
+  struct Extensions {
+    absl::optional<uint8_t> audio_level;
+    absl::optional<AbsoluteCaptureTime> absolute_capture_time;
+  };
+
   RtpSource() = delete;
 
+  // TODO(bugs.webrtc.org/10739): Remove this constructor once all clients
+  // migrate to the version with absolute capture time.
   RtpSource(int64_t timestamp_ms,
             uint32_t source_id,
             RtpSourceType source_type,
             absl::optional<uint8_t> audio_level,
             uint32_t rtp_timestamp)
+      : RtpSource(timestamp_ms,
+                  source_id,
+                  source_type,
+                  rtp_timestamp,
+                  {audio_level, absl::nullopt}) {}
+
+  RtpSource(int64_t timestamp_ms,
+            uint32_t source_id,
+            RtpSourceType source_type,
+            uint32_t rtp_timestamp,
+            const RtpSource::Extensions& extensions)
       : timestamp_ms_(timestamp_ms),
         source_id_(source_id),
         source_type_(source_type),
-        audio_level_(audio_level),
+        extensions_(extensions),
         rtp_timestamp_(rtp_timestamp) {}
 
   RtpSource(const RtpSource&) = default;
@@ -54,16 +73,26 @@ class RtpSource {
   // The source can be either a contributing source or a synchronization source.
   RtpSourceType source_type() const { return source_type_; }
 
-  absl::optional<uint8_t> audio_level() const { return audio_level_; }
+  absl::optional<uint8_t> audio_level() const {
+    return extensions_.audio_level;
+  }
+
   void set_audio_level(const absl::optional<uint8_t>& level) {
-    audio_level_ = level;
+    extensions_.audio_level = level;
   }
 
   uint32_t rtp_timestamp() const { return rtp_timestamp_; }
 
+  absl::optional<AbsoluteCaptureTime> absolute_capture_time() const {
+    return extensions_.absolute_capture_time;
+  }
+
   bool operator==(const RtpSource& o) const {
     return timestamp_ms_ == o.timestamp_ms() && source_id_ == o.source_id() &&
-           source_type_ == o.source_type() && audio_level_ == o.audio_level_ &&
+           source_type_ == o.source_type() &&
+           extensions_.audio_level == o.extensions_.audio_level &&
+           extensions_.absolute_capture_time ==
+               o.extensions_.absolute_capture_time &&
            rtp_timestamp_ == o.rtp_timestamp();
   }
 
@@ -71,7 +100,7 @@ class RtpSource {
   int64_t timestamp_ms_;
   uint32_t source_id_;
   RtpSourceType source_type_;
-  absl::optional<uint8_t> audio_level_;
+  RtpSource::Extensions extensions_;
   uint32_t rtp_timestamp_;
 };
 

modules/rtp_rtcp/source/source_tracker.cc

@@ -34,6 +34,7 @@ void SourceTracker::OnFrameDelivered(const RtpPacketInfos& packet_infos) {
 
       entry.timestamp_ms = now_ms;
       entry.audio_level = packet_info.audio_level();
+      entry.absolute_capture_time = packet_info.absolute_capture_time();
       entry.rtp_timestamp = packet_info.rtp_timestamp();
     }
 
@@ -42,6 +43,7 @@ void SourceTracker::OnFrameDelivered(const RtpPacketInfos& packet_infos) {
 
     entry.timestamp_ms = now_ms;
     entry.audio_level = packet_info.audio_level();
+    entry.absolute_capture_time = packet_info.absolute_capture_time();
     entry.rtp_timestamp = packet_info.rtp_timestamp();
   }
 
@@ -60,8 +62,9 @@ std::vector<RtpSource> SourceTracker::GetSources() const {
     const SourceKey& key = pair.first;
     const SourceEntry& entry = pair.second;
 
-    sources.emplace_back(entry.timestamp_ms, key.source, key.source_type,
+    sources.emplace_back(
-                         entry.audio_level, entry.rtp_timestamp);
+        entry.timestamp_ms, key.source, key.source_type, entry.rtp_timestamp,
+        RtpSource::Extensions{entry.audio_level, entry.absolute_capture_time});
   }
 
   return sources;

modules/rtp_rtcp/source/source_tracker.h

@@ -90,6 +90,11 @@ class SourceTracker {
     // specs for `RTCRtpContributingSource` for more info.
     absl::optional<uint8_t> audio_level;
 
+    // Absolute capture time header extension received or interpolated from the
+    // most recent packet used to assemble the frame. For more info see
+    // https://webrtc.org/experiments/rtp-hdrext/abs-capture-time/
+    absl::optional<AbsoluteCaptureTime> absolute_capture_time;
+
     // RTP timestamp of the most recent packet used to assemble the frame
     // associated with |timestamp_ms|.
     uint32_t rtp_timestamp;

modules/rtp_rtcp/source/source_tracker_unittest.cc

@@ -18,6 +18,7 @@
 #include <utility>
 #include <vector>
 
+#include "absl/types/optional.h"
 #include "api/rtp_headers.h"
 #include "api/rtp_packet_info.h"
 #include "api/rtp_packet_infos.h"
@@ -46,15 +47,16 @@ class ExpectedSourceTracker {
     const int64_t now_ms = clock_->TimeInMilliseconds();
 
     for (const auto& packet_info : packet_infos) {
+      RtpSource::Extensions extensions = {packet_info.audio_level(),
+                                          packet_info.absolute_capture_time()};
+
       for (const auto& csrc : packet_info.csrcs()) {
         entries_.emplace_front(now_ms, csrc, RtpSourceType::CSRC,
-                               packet_info.audio_level(),
+                               packet_info.rtp_timestamp(), extensions);
-                               packet_info.rtp_timestamp());
       }
 
       entries_.emplace_front(now_ms, packet_info.ssrc(), RtpSourceType::SSRC,
-                             packet_info.audio_level(),
+                             packet_info.rtp_timestamp(), extensions);
-                             packet_info.rtp_timestamp());
     }
 
     PruneEntries(now_ms);
@@ -243,7 +245,9 @@ TEST(SourceTrackerTest, OnFrameDeliveredRecordsSources) {
   constexpr uint32_t kCsrcs1 = 21;
   constexpr uint32_t kRtpTimestamp = 40;
   constexpr absl::optional<uint8_t> kAudioLevel = 50;
-  constexpr absl::optional<AbsoluteCaptureTime> kAbsoluteCaptureTime = {};
+  constexpr absl::optional<AbsoluteCaptureTime> kAbsoluteCaptureTime =
+      AbsoluteCaptureTime{/*absolute_capture_timestamp=*/12,
+                          /*estimated_capture_clock_offset=*/absl::nullopt};
   constexpr int64_t kReceiveTimeMs = 60;
 
   SimulatedClock clock(1000000000000ULL);
@@ -254,14 +258,16 @@ TEST(SourceTrackerTest, OnFrameDeliveredRecordsSources) {
                      kAbsoluteCaptureTime, kReceiveTimeMs)}));
 
   int64_t timestamp_ms = clock.TimeInMilliseconds();
+  constexpr RtpSource::Extensions extensions = {kAudioLevel,
+                                                kAbsoluteCaptureTime};
 
   EXPECT_THAT(tracker.GetSources(),
               ElementsAre(RtpSource(timestamp_ms, kSsrc, RtpSourceType::SSRC,
-                                    kAudioLevel, kRtpTimestamp),
+                                    kRtpTimestamp, extensions),
                           RtpSource(timestamp_ms, kCsrcs1, RtpSourceType::CSRC,
-                                    kAudioLevel, kRtpTimestamp),
+                                    kRtpTimestamp, extensions),
                           RtpSource(timestamp_ms, kCsrcs0, RtpSourceType::CSRC,
-                                    kAudioLevel, kRtpTimestamp)));
+                                    kRtpTimestamp, extensions)));
 }
 
 TEST(SourceTrackerTest, OnFrameDeliveredUpdatesSources) {
@@ -273,7 +279,10 @@ TEST(SourceTrackerTest, OnFrameDeliveredUpdatesSources) {
   constexpr uint32_t kRtpTimestamp1 = 41;
   constexpr absl::optional<uint8_t> kAudioLevel0 = 50;
   constexpr absl::optional<uint8_t> kAudioLevel1 = absl::nullopt;
-  constexpr absl::optional<AbsoluteCaptureTime> kAbsoluteCaptureTime = {};
+  constexpr absl::optional<AbsoluteCaptureTime> kAbsoluteCaptureTime0 =
+      AbsoluteCaptureTime{12, 34};
+  constexpr absl::optional<AbsoluteCaptureTime> kAbsoluteCaptureTime1 =
+      AbsoluteCaptureTime{56, 78};
   constexpr int64_t kReceiveTimeMs0 = 60;
   constexpr int64_t kReceiveTimeMs1 = 61;
 
@@ -282,7 +291,7 @@ TEST(SourceTrackerTest, OnFrameDeliveredUpdatesSources) {
 
   tracker.OnFrameDelivered(RtpPacketInfos(
       {RtpPacketInfo(kSsrc, {kCsrcs0, kCsrcs1}, kRtpTimestamp0, kAudioLevel0,
-                     kAbsoluteCaptureTime, kReceiveTimeMs0)}));
+                     kAbsoluteCaptureTime0, kReceiveTimeMs0)}));
 
   int64_t timestamp_ms_0 = clock.TimeInMilliseconds();
 
@@ -290,20 +299,25 @@ TEST(SourceTrackerTest, OnFrameDeliveredUpdatesSources) {
 
   tracker.OnFrameDelivered(RtpPacketInfos(
       {RtpPacketInfo(kSsrc, {kCsrcs0, kCsrcs2}, kRtpTimestamp1, kAudioLevel1,
-                     kAbsoluteCaptureTime, kReceiveTimeMs1)}));
+                     kAbsoluteCaptureTime1, kReceiveTimeMs1)}));
 
   int64_t timestamp_ms_1 = clock.TimeInMilliseconds();
 
+  constexpr RtpSource::Extensions extensions0 = {kAudioLevel0,
+                                                 kAbsoluteCaptureTime0};
+  constexpr RtpSource::Extensions extensions1 = {kAudioLevel1,
+                                                 kAbsoluteCaptureTime1};
+
   EXPECT_THAT(
       tracker.GetSources(),
       ElementsAre(RtpSource(timestamp_ms_1, kSsrc, RtpSourceType::SSRC,
-                            kAudioLevel1, kRtpTimestamp1),
+                            kRtpTimestamp1, extensions1),
                   RtpSource(timestamp_ms_1, kCsrcs2, RtpSourceType::CSRC,
-                            kAudioLevel1, kRtpTimestamp1),
+                            kRtpTimestamp1, extensions1),
                   RtpSource(timestamp_ms_1, kCsrcs0, RtpSourceType::CSRC,
-                            kAudioLevel1, kRtpTimestamp1),
+                            kRtpTimestamp1, extensions1),
                   RtpSource(timestamp_ms_0, kCsrcs1, RtpSourceType::CSRC,
-                            kAudioLevel0, kRtpTimestamp0)));
+                            kRtpTimestamp0, extensions0)));
 }
 
 TEST(SourceTrackerTest, TimedOutSourcesAreRemoved) {
@@ -315,7 +329,10 @@ TEST(SourceTrackerTest, TimedOutSourcesAreRemoved) {
   constexpr uint32_t kRtpTimestamp1 = 41;
   constexpr absl::optional<uint8_t> kAudioLevel0 = 50;
   constexpr absl::optional<uint8_t> kAudioLevel1 = absl::nullopt;
-  constexpr absl::optional<AbsoluteCaptureTime> kAbsoluteCaptureTime = {};
+  constexpr absl::optional<AbsoluteCaptureTime> kAbsoluteCaptureTime0 =
+      AbsoluteCaptureTime{12, 34};
+  constexpr absl::optional<AbsoluteCaptureTime> kAbsoluteCaptureTime1 =
+      AbsoluteCaptureTime{56, 78};
   constexpr int64_t kReceiveTimeMs0 = 60;
   constexpr int64_t kReceiveTimeMs1 = 61;
 
@@ -324,26 +341,29 @@ TEST(SourceTrackerTest, TimedOutSourcesAreRemoved) {
 
   tracker.OnFrameDelivered(RtpPacketInfos(
       {RtpPacketInfo(kSsrc, {kCsrcs0, kCsrcs1}, kRtpTimestamp0, kAudioLevel0,
-                     kAbsoluteCaptureTime, kReceiveTimeMs0)}));
+                     kAbsoluteCaptureTime0, kReceiveTimeMs0)}));
 
   clock.AdvanceTimeMilliseconds(17);
 
   tracker.OnFrameDelivered(RtpPacketInfos(
       {RtpPacketInfo(kSsrc, {kCsrcs0, kCsrcs2}, kRtpTimestamp1, kAudioLevel1,
-                     kAbsoluteCaptureTime, kReceiveTimeMs1)}));
+                     kAbsoluteCaptureTime1, kReceiveTimeMs1)}));
 
   int64_t timestamp_ms_1 = clock.TimeInMilliseconds();
 
   clock.AdvanceTimeMilliseconds(SourceTracker::kTimeoutMs);
 
+  constexpr RtpSource::Extensions extensions1 = {kAudioLevel1,
+                                                 kAbsoluteCaptureTime1};
+
   EXPECT_THAT(
       tracker.GetSources(),
       ElementsAre(RtpSource(timestamp_ms_1, kSsrc, RtpSourceType::SSRC,
-                            kAudioLevel1, kRtpTimestamp1),
+                            kRtpTimestamp1, extensions1),
                   RtpSource(timestamp_ms_1, kCsrcs2, RtpSourceType::CSRC,
-                            kAudioLevel1, kRtpTimestamp1),
+                            kRtpTimestamp1, extensions1),
                   RtpSource(timestamp_ms_1, kCsrcs0, RtpSourceType::CSRC,
-                            kAudioLevel1, kRtpTimestamp1)));
+                            kRtpTimestamp1, extensions1)));
 }
 
 }  // namespace webrtc