diff --git a/api/transport/rtp/BUILD.gn b/api/transport/rtp/BUILD.gn index 04298e3902..777ff22639 100644 --- a/api/transport/rtp/BUILD.gn +++ b/api/transport/rtp/BUILD.gn @@ -14,6 +14,7 @@ rtc_source_set("rtp_source") { "rtp_source.h", ] deps = [ + "../../../api:rtp_headers", "../../../rtc_base:checks", "//third_party/abseil-cpp/absl/types:optional", ] diff --git a/api/transport/rtp/rtp_source.h b/api/transport/rtp/rtp_source.h index d26572c933..8c543cac0c 100644 --- a/api/transport/rtp/rtp_source.h +++ b/api/transport/rtp/rtp_source.h @@ -14,6 +14,7 @@ #include #include "absl/types/optional.h" +#include "api/rtp_headers.h" #include "rtc_base/checks.h" namespace webrtc { @@ -25,17 +26,35 @@ enum class RtpSourceType { class RtpSource { public: + struct Extensions { + absl::optional audio_level; + absl::optional absolute_capture_time; + }; + RtpSource() = delete; + // TODO(bugs.webrtc.org/10739): Remove this constructor once all clients + // migrate to the version with absolute capture time. RtpSource(int64_t timestamp_ms, uint32_t source_id, RtpSourceType source_type, absl::optional audio_level, uint32_t rtp_timestamp) + : RtpSource(timestamp_ms, + source_id, + source_type, + rtp_timestamp, + {audio_level, absl::nullopt}) {} + + RtpSource(int64_t timestamp_ms, + uint32_t source_id, + RtpSourceType source_type, + uint32_t rtp_timestamp, + const RtpSource::Extensions& extensions) : timestamp_ms_(timestamp_ms), source_id_(source_id), source_type_(source_type), - audio_level_(audio_level), + extensions_(extensions), rtp_timestamp_(rtp_timestamp) {} RtpSource(const RtpSource&) = default; @@ -54,16 +73,26 @@ class RtpSource { // The source can be either a contributing source or a synchronization source. RtpSourceType source_type() const { return source_type_; } - absl::optional audio_level() const { return audio_level_; } + absl::optional audio_level() const { + return extensions_.audio_level; + } + void set_audio_level(const absl::optional& level) { - audio_level_ = level; + extensions_.audio_level = level; } uint32_t rtp_timestamp() const { return rtp_timestamp_; } + absl::optional absolute_capture_time() const { + return extensions_.absolute_capture_time; + } + bool operator==(const RtpSource& o) const { return timestamp_ms_ == o.timestamp_ms() && source_id_ == o.source_id() && - source_type_ == o.source_type() && audio_level_ == o.audio_level_ && + source_type_ == o.source_type() && + extensions_.audio_level == o.extensions_.audio_level && + extensions_.absolute_capture_time == + o.extensions_.absolute_capture_time && rtp_timestamp_ == o.rtp_timestamp(); } @@ -71,7 +100,7 @@ class RtpSource { int64_t timestamp_ms_; uint32_t source_id_; RtpSourceType source_type_; - absl::optional audio_level_; + RtpSource::Extensions extensions_; uint32_t rtp_timestamp_; }; diff --git a/modules/rtp_rtcp/source/source_tracker.cc b/modules/rtp_rtcp/source/source_tracker.cc index 2878b11d41..22b887c5d2 100644 --- a/modules/rtp_rtcp/source/source_tracker.cc +++ b/modules/rtp_rtcp/source/source_tracker.cc @@ -34,6 +34,7 @@ void SourceTracker::OnFrameDelivered(const RtpPacketInfos& packet_infos) { entry.timestamp_ms = now_ms; entry.audio_level = packet_info.audio_level(); + entry.absolute_capture_time = packet_info.absolute_capture_time(); entry.rtp_timestamp = packet_info.rtp_timestamp(); } @@ -42,6 +43,7 @@ void SourceTracker::OnFrameDelivered(const RtpPacketInfos& packet_infos) { entry.timestamp_ms = now_ms; entry.audio_level = packet_info.audio_level(); + entry.absolute_capture_time = packet_info.absolute_capture_time(); entry.rtp_timestamp = packet_info.rtp_timestamp(); } @@ -60,8 +62,9 @@ std::vector SourceTracker::GetSources() const { const SourceKey& key = pair.first; const SourceEntry& entry = pair.second; - sources.emplace_back(entry.timestamp_ms, key.source, key.source_type, - entry.audio_level, entry.rtp_timestamp); + sources.emplace_back( + entry.timestamp_ms, key.source, key.source_type, entry.rtp_timestamp, + RtpSource::Extensions{entry.audio_level, entry.absolute_capture_time}); } return sources; diff --git a/modules/rtp_rtcp/source/source_tracker.h b/modules/rtp_rtcp/source/source_tracker.h index 3ea16e9db8..fcf99bf8b5 100644 --- a/modules/rtp_rtcp/source/source_tracker.h +++ b/modules/rtp_rtcp/source/source_tracker.h @@ -90,6 +90,11 @@ class SourceTracker { // specs for `RTCRtpContributingSource` for more info. absl::optional audio_level; + // Absolute capture time header extension received or interpolated from the + // most recent packet used to assemble the frame. For more info see + // https://webrtc.org/experiments/rtp-hdrext/abs-capture-time/ + absl::optional absolute_capture_time; + // RTP timestamp of the most recent packet used to assemble the frame // associated with |timestamp_ms|. uint32_t rtp_timestamp; diff --git a/modules/rtp_rtcp/source/source_tracker_unittest.cc b/modules/rtp_rtcp/source/source_tracker_unittest.cc index 8f6a5125a3..32f9f4b2a3 100644 --- a/modules/rtp_rtcp/source/source_tracker_unittest.cc +++ b/modules/rtp_rtcp/source/source_tracker_unittest.cc @@ -18,6 +18,7 @@ #include #include +#include "absl/types/optional.h" #include "api/rtp_headers.h" #include "api/rtp_packet_info.h" #include "api/rtp_packet_infos.h" @@ -46,15 +47,16 @@ class ExpectedSourceTracker { const int64_t now_ms = clock_->TimeInMilliseconds(); for (const auto& packet_info : packet_infos) { + RtpSource::Extensions extensions = {packet_info.audio_level(), + packet_info.absolute_capture_time()}; + for (const auto& csrc : packet_info.csrcs()) { entries_.emplace_front(now_ms, csrc, RtpSourceType::CSRC, - packet_info.audio_level(), - packet_info.rtp_timestamp()); + packet_info.rtp_timestamp(), extensions); } entries_.emplace_front(now_ms, packet_info.ssrc(), RtpSourceType::SSRC, - packet_info.audio_level(), - packet_info.rtp_timestamp()); + packet_info.rtp_timestamp(), extensions); } PruneEntries(now_ms); @@ -243,7 +245,9 @@ TEST(SourceTrackerTest, OnFrameDeliveredRecordsSources) { constexpr uint32_t kCsrcs1 = 21; constexpr uint32_t kRtpTimestamp = 40; constexpr absl::optional kAudioLevel = 50; - constexpr absl::optional kAbsoluteCaptureTime = {}; + constexpr absl::optional kAbsoluteCaptureTime = + AbsoluteCaptureTime{/*absolute_capture_timestamp=*/12, + /*estimated_capture_clock_offset=*/absl::nullopt}; constexpr int64_t kReceiveTimeMs = 60; SimulatedClock clock(1000000000000ULL); @@ -254,14 +258,16 @@ TEST(SourceTrackerTest, OnFrameDeliveredRecordsSources) { kAbsoluteCaptureTime, kReceiveTimeMs)})); int64_t timestamp_ms = clock.TimeInMilliseconds(); + constexpr RtpSource::Extensions extensions = {kAudioLevel, + kAbsoluteCaptureTime}; EXPECT_THAT(tracker.GetSources(), ElementsAre(RtpSource(timestamp_ms, kSsrc, RtpSourceType::SSRC, - kAudioLevel, kRtpTimestamp), + kRtpTimestamp, extensions), RtpSource(timestamp_ms, kCsrcs1, RtpSourceType::CSRC, - kAudioLevel, kRtpTimestamp), + kRtpTimestamp, extensions), RtpSource(timestamp_ms, kCsrcs0, RtpSourceType::CSRC, - kAudioLevel, kRtpTimestamp))); + kRtpTimestamp, extensions))); } TEST(SourceTrackerTest, OnFrameDeliveredUpdatesSources) { @@ -273,7 +279,10 @@ TEST(SourceTrackerTest, OnFrameDeliveredUpdatesSources) { constexpr uint32_t kRtpTimestamp1 = 41; constexpr absl::optional kAudioLevel0 = 50; constexpr absl::optional kAudioLevel1 = absl::nullopt; - constexpr absl::optional kAbsoluteCaptureTime = {}; + constexpr absl::optional kAbsoluteCaptureTime0 = + AbsoluteCaptureTime{12, 34}; + constexpr absl::optional kAbsoluteCaptureTime1 = + AbsoluteCaptureTime{56, 78}; constexpr int64_t kReceiveTimeMs0 = 60; constexpr int64_t kReceiveTimeMs1 = 61; @@ -282,7 +291,7 @@ TEST(SourceTrackerTest, OnFrameDeliveredUpdatesSources) { tracker.OnFrameDelivered(RtpPacketInfos( {RtpPacketInfo(kSsrc, {kCsrcs0, kCsrcs1}, kRtpTimestamp0, kAudioLevel0, - kAbsoluteCaptureTime, kReceiveTimeMs0)})); + kAbsoluteCaptureTime0, kReceiveTimeMs0)})); int64_t timestamp_ms_0 = clock.TimeInMilliseconds(); @@ -290,20 +299,25 @@ TEST(SourceTrackerTest, OnFrameDeliveredUpdatesSources) { tracker.OnFrameDelivered(RtpPacketInfos( {RtpPacketInfo(kSsrc, {kCsrcs0, kCsrcs2}, kRtpTimestamp1, kAudioLevel1, - kAbsoluteCaptureTime, kReceiveTimeMs1)})); + kAbsoluteCaptureTime1, kReceiveTimeMs1)})); int64_t timestamp_ms_1 = clock.TimeInMilliseconds(); + constexpr RtpSource::Extensions extensions0 = {kAudioLevel0, + kAbsoluteCaptureTime0}; + constexpr RtpSource::Extensions extensions1 = {kAudioLevel1, + kAbsoluteCaptureTime1}; + EXPECT_THAT( tracker.GetSources(), ElementsAre(RtpSource(timestamp_ms_1, kSsrc, RtpSourceType::SSRC, - kAudioLevel1, kRtpTimestamp1), + kRtpTimestamp1, extensions1), RtpSource(timestamp_ms_1, kCsrcs2, RtpSourceType::CSRC, - kAudioLevel1, kRtpTimestamp1), + kRtpTimestamp1, extensions1), RtpSource(timestamp_ms_1, kCsrcs0, RtpSourceType::CSRC, - kAudioLevel1, kRtpTimestamp1), + kRtpTimestamp1, extensions1), RtpSource(timestamp_ms_0, kCsrcs1, RtpSourceType::CSRC, - kAudioLevel0, kRtpTimestamp0))); + kRtpTimestamp0, extensions0))); } TEST(SourceTrackerTest, TimedOutSourcesAreRemoved) { @@ -315,7 +329,10 @@ TEST(SourceTrackerTest, TimedOutSourcesAreRemoved) { constexpr uint32_t kRtpTimestamp1 = 41; constexpr absl::optional kAudioLevel0 = 50; constexpr absl::optional kAudioLevel1 = absl::nullopt; - constexpr absl::optional kAbsoluteCaptureTime = {}; + constexpr absl::optional kAbsoluteCaptureTime0 = + AbsoluteCaptureTime{12, 34}; + constexpr absl::optional kAbsoluteCaptureTime1 = + AbsoluteCaptureTime{56, 78}; constexpr int64_t kReceiveTimeMs0 = 60; constexpr int64_t kReceiveTimeMs1 = 61; @@ -324,26 +341,29 @@ TEST(SourceTrackerTest, TimedOutSourcesAreRemoved) { tracker.OnFrameDelivered(RtpPacketInfos( {RtpPacketInfo(kSsrc, {kCsrcs0, kCsrcs1}, kRtpTimestamp0, kAudioLevel0, - kAbsoluteCaptureTime, kReceiveTimeMs0)})); + kAbsoluteCaptureTime0, kReceiveTimeMs0)})); clock.AdvanceTimeMilliseconds(17); tracker.OnFrameDelivered(RtpPacketInfos( {RtpPacketInfo(kSsrc, {kCsrcs0, kCsrcs2}, kRtpTimestamp1, kAudioLevel1, - kAbsoluteCaptureTime, kReceiveTimeMs1)})); + kAbsoluteCaptureTime1, kReceiveTimeMs1)})); int64_t timestamp_ms_1 = clock.TimeInMilliseconds(); clock.AdvanceTimeMilliseconds(SourceTracker::kTimeoutMs); + constexpr RtpSource::Extensions extensions1 = {kAudioLevel1, + kAbsoluteCaptureTime1}; + EXPECT_THAT( tracker.GetSources(), ElementsAre(RtpSource(timestamp_ms_1, kSsrc, RtpSourceType::SSRC, - kAudioLevel1, kRtpTimestamp1), + kRtpTimestamp1, extensions1), RtpSource(timestamp_ms_1, kCsrcs2, RtpSourceType::CSRC, - kAudioLevel1, kRtpTimestamp1), + kRtpTimestamp1, extensions1), RtpSource(timestamp_ms_1, kCsrcs0, RtpSourceType::CSRC, - kAudioLevel1, kRtpTimestamp1))); + kRtpTimestamp1, extensions1))); } } // namespace webrtc