diff --git a/api/rtp_packet_info.cc b/api/rtp_packet_info.cc index 68e78d7987..db818f7657 100644 --- a/api/rtp_packet_info.cc +++ b/api/rtp_packet_info.cc @@ -71,7 +71,9 @@ bool operator==(const RtpPacketInfo& lhs, const RtpPacketInfo& rhs) { (lhs.rtp_timestamp() == rhs.rtp_timestamp()) && (lhs.audio_level() == rhs.audio_level()) && (lhs.absolute_capture_time() == rhs.absolute_capture_time()) && - (lhs.receive_time() == rhs.receive_time()); + (lhs.receive_time() == rhs.receive_time() && + (lhs.local_capture_clock_offset() == + rhs.local_capture_clock_offset())); } } // namespace webrtc diff --git a/api/rtp_packet_info.h b/api/rtp_packet_info.h index ed484708d1..605620d638 100644 --- a/api/rtp_packet_info.h +++ b/api/rtp_packet_info.h @@ -75,6 +75,14 @@ class RTC_EXPORT RtpPacketInfo { absolute_capture_time_ = value; } + const absl::optional& local_capture_clock_offset() const { + return local_capture_clock_offset_; + } + + void set_local_capture_clock_offset(const absl::optional& value) { + local_capture_clock_offset_ = value; + } + Timestamp receive_time() const { return receive_time_; } void set_receive_time(Timestamp value) { receive_time_ = value; } // TODO(bugs.webrtc.org/12722): Deprecated, remove once downstream projects @@ -94,8 +102,17 @@ class RTC_EXPORT RtpPacketInfo { // Fields from the Absolute Capture Time header extension: // http://www.webrtc.org/experiments/rtp-hdrext/abs-capture-time + // To not be confused with |local_capture_clock_offset_|, the + // |estimated_capture_clock_offset| in |absolute_capture_time_| should + // represent the clock offset between a remote sender and the capturer, and + // thus equals to the corresponding values in the received RTP packets, + // subjected to possible interpolations. absl::optional absolute_capture_time_; + // Clock offset against capturer's clock. Should be derived from the estimated + // capture clock offset defined in the Absolute Capture Time header extension. + absl::optional local_capture_clock_offset_; + // Local |webrtc::Clock|-based timestamp of when the packet was received. Timestamp receive_time_; }; diff --git a/api/rtp_packet_info_unittest.cc b/api/rtp_packet_info_unittest.cc index 7af7323640..601d34f49e 100644 --- a/api/rtp_packet_info_unittest.cc +++ b/api/rtp_packet_info_unittest.cc @@ -149,6 +149,35 @@ TEST(RtpPacketInfoTest, AbsoluteCaptureTime) { EXPECT_EQ(rhs.absolute_capture_time(), value); } +TEST(RtpPacketInfoTest, LocalCaptureClockOffset) { + RtpPacketInfo lhs; + RtpPacketInfo rhs; + + EXPECT_TRUE(lhs == rhs); + EXPECT_FALSE(lhs != rhs); + + const absl::optional value = 10; + rhs.set_local_capture_clock_offset(value); + EXPECT_EQ(rhs.local_capture_clock_offset(), value); + + EXPECT_FALSE(lhs == rhs); + EXPECT_TRUE(lhs != rhs); + + lhs = rhs; + + EXPECT_TRUE(lhs == rhs); + EXPECT_FALSE(lhs != rhs); + + // Default local capture clock offset is null. + rhs = RtpPacketInfo(); + EXPECT_EQ(rhs.local_capture_clock_offset(), absl::nullopt); + + // Default local capture clock offset is null. + rhs = RtpPacketInfo({}, {}, {}, {}, AbsoluteCaptureTime{12, 34}, + Timestamp::Millis(0)); + EXPECT_EQ(rhs.local_capture_clock_offset(), absl::nullopt); +} + TEST(RtpPacketInfoTest, ReceiveTimeMs) { const Timestamp timestamp = Timestamp::Micros(8868963877546349045LL); diff --git a/audio/channel_receive.cc b/audio/channel_receive.cc index 6a57b983f1..fd33dbdf24 100644 --- a/audio/channel_receive.cc +++ b/audio/channel_receive.cc @@ -34,7 +34,8 @@ #include "modules/pacing/packet_router.h" #include "modules/rtp_rtcp/include/receive_statistics.h" #include "modules/rtp_rtcp/include/remote_ntp_time_estimator.h" -#include "modules/rtp_rtcp/source/absolute_capture_time_receiver.h" +#include "modules/rtp_rtcp/source/absolute_capture_time_interpolator.h" +#include "modules/rtp_rtcp/source/capture_clock_offset_updater.h" #include "modules/rtp_rtcp/source/rtp_header_extensions.h" #include "modules/rtp_rtcp/source/rtp_packet_received.h" #include "modules/rtp_rtcp/source/rtp_rtcp_config.h" @@ -276,7 +277,9 @@ class ChannelReceive : public ChannelReceiveInterface { rtc::scoped_refptr frame_decryptor_; webrtc::CryptoOptions crypto_options_; - webrtc::AbsoluteCaptureTimeReceiver absolute_capture_time_receiver_; + webrtc::AbsoluteCaptureTimeInterpolator absolute_capture_time_interpolator_; + + webrtc::CaptureClockOffsetUpdater capture_clock_offset_updater_; rtc::scoped_refptr frame_transformer_delegate_; @@ -434,6 +437,22 @@ AudioMixer::Source::AudioFrameInfo ChannelReceive::GetAudioFrameWithInfo( } } + // Fill in local capture clock offset in |audio_frame->packet_infos_|. + RtpPacketInfos::vector_type packet_infos; + for (auto& packet_info : audio_frame->packet_infos_) { + absl::optional local_capture_clock_offset; + if (packet_info.absolute_capture_time().has_value()) { + local_capture_clock_offset = + capture_clock_offset_updater_.AdjustEstimatedCaptureClockOffset( + packet_info.absolute_capture_time() + ->estimated_capture_clock_offset); + } + RtpPacketInfo new_packet_info(packet_info); + new_packet_info.set_local_capture_clock_offset(local_capture_clock_offset); + packet_infos.push_back(std::move(new_packet_info)); + } + audio_frame->packet_infos_ = RtpPacketInfos(packet_infos); + { RTC_HISTOGRAM_COUNTS_1000("WebRTC.Audio.TargetJitterBufferDelayMs", acm_receiver_.TargetDelayMs()); @@ -502,7 +521,7 @@ ChannelReceive::ChannelReceive( associated_send_channel_(nullptr), frame_decryptor_(frame_decryptor), crypto_options_(crypto_options), - absolute_capture_time_receiver_(clock) { + absolute_capture_time_interpolator_(clock) { RTC_DCHECK(module_process_thread_); RTC_DCHECK(audio_device_module); @@ -618,9 +637,9 @@ void ChannelReceive::OnRtpPacket(const RtpPacketReceived& packet) { // Interpolates absolute capture timestamp RTP header extension. header.extension.absolute_capture_time = - absolute_capture_time_receiver_.OnReceivePacket( - AbsoluteCaptureTimeReceiver::GetSource(header.ssrc, - header.arrOfCSRCs), + absolute_capture_time_interpolator_.OnReceivePacket( + AbsoluteCaptureTimeInterpolator::GetSource(header.ssrc, + header.arrOfCSRCs), header.timestamp, rtc::saturated_cast(packet_copy.payload_type_frequency()), header.extension.absolute_capture_time); @@ -713,7 +732,7 @@ void ChannelReceive::ReceivedRTCPPacket(const uint8_t* data, size_t length) { absl::optional remote_to_local_clock_offset_ms = ntp_estimator_.EstimateRemoteToLocalClockOffsetMs(); if (remote_to_local_clock_offset_ms.has_value()) { - absolute_capture_time_receiver_.SetRemoteToLocalClockOffset( + capture_clock_offset_updater_.SetRemoteToLocalClockOffset( Int64MsToQ32x32(*remote_to_local_clock_offset_ms)); } } diff --git a/modules/rtp_rtcp/BUILD.gn b/modules/rtp_rtcp/BUILD.gn index 8cf519456b..8204b6d724 100644 --- a/modules/rtp_rtcp/BUILD.gn +++ b/modules/rtp_rtcp/BUILD.gn @@ -138,12 +138,16 @@ rtc_library("rtp_rtcp") { "include/receive_statistics.h", "include/remote_ntp_time_estimator.h", "include/ulpfec_receiver.h", - "source/absolute_capture_time_receiver.cc", - "source/absolute_capture_time_receiver.h", + "source/absolute_capture_time_interpolator.cc", + "source/absolute_capture_time_interpolator.h", + "source/absolute_capture_time_receiver.cc", # DEPRECATED + "source/absolute_capture_time_receiver.h", # DEPRECATED "source/absolute_capture_time_sender.cc", "source/absolute_capture_time_sender.h", "source/active_decode_targets_helper.cc", "source/active_decode_targets_helper.h", + "source/capture_clock_offset_updater.cc", + "source/capture_clock_offset_updater.h", "source/create_video_rtp_depacketizer.cc", "source/create_video_rtp_depacketizer.h", "source/dtmf_queue.cc", @@ -473,10 +477,11 @@ if (rtc_include_tests) { testonly = true sources = [ - "source/absolute_capture_time_receiver_unittest.cc", + "source/absolute_capture_time_interpolator_unittest.cc", "source/absolute_capture_time_sender_unittest.cc", "source/active_decode_targets_helper_unittest.cc", "source/byte_io_unittest.cc", + "source/capture_clock_offset_updater_unittest.cc", "source/fec_private_tables_bursty_unittest.cc", "source/flexfec_header_reader_writer_unittest.cc", "source/flexfec_receiver_unittest.cc", diff --git a/modules/rtp_rtcp/source/absolute_capture_time_interpolator.cc b/modules/rtp_rtcp/source/absolute_capture_time_interpolator.cc new file mode 100644 index 0000000000..99fc030aca --- /dev/null +++ b/modules/rtp_rtcp/source/absolute_capture_time_interpolator.cc @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/rtp_rtcp/source/absolute_capture_time_interpolator.h" + +#include + +#include "rtc_base/checks.h" + +namespace webrtc { +namespace { + +constexpr Timestamp kInvalidLastReceiveTime = Timestamp::MinusInfinity(); +} // namespace + +constexpr TimeDelta AbsoluteCaptureTimeInterpolator::kInterpolationMaxInterval; + +AbsoluteCaptureTimeInterpolator::AbsoluteCaptureTimeInterpolator(Clock* clock) + : clock_(clock), last_receive_time_(kInvalidLastReceiveTime) {} + +uint32_t AbsoluteCaptureTimeInterpolator::GetSource( + uint32_t ssrc, + rtc::ArrayView csrcs) { + if (csrcs.empty()) { + return ssrc; + } + + return csrcs[0]; +} + +absl::optional +AbsoluteCaptureTimeInterpolator::OnReceivePacket( + uint32_t source, + uint32_t rtp_timestamp, + uint32_t rtp_clock_frequency, + const absl::optional& received_extension) { + const Timestamp receive_time = clock_->CurrentTime(); + + MutexLock lock(&mutex_); + + AbsoluteCaptureTime extension; + if (received_extension == absl::nullopt) { + if (!ShouldInterpolateExtension(receive_time, source, rtp_timestamp, + rtp_clock_frequency)) { + last_receive_time_ = kInvalidLastReceiveTime; + return absl::nullopt; + } + + extension.absolute_capture_timestamp = InterpolateAbsoluteCaptureTimestamp( + rtp_timestamp, rtp_clock_frequency, last_rtp_timestamp_, + last_absolute_capture_timestamp_); + extension.estimated_capture_clock_offset = + last_estimated_capture_clock_offset_; + } else { + last_source_ = source; + last_rtp_timestamp_ = rtp_timestamp; + last_rtp_clock_frequency_ = rtp_clock_frequency; + last_absolute_capture_timestamp_ = + received_extension->absolute_capture_timestamp; + last_estimated_capture_clock_offset_ = + received_extension->estimated_capture_clock_offset; + + last_receive_time_ = receive_time; + + extension = *received_extension; + } + + return extension; +} + +uint64_t AbsoluteCaptureTimeInterpolator::InterpolateAbsoluteCaptureTimestamp( + uint32_t rtp_timestamp, + uint32_t rtp_clock_frequency, + uint32_t last_rtp_timestamp, + uint64_t last_absolute_capture_timestamp) { + RTC_DCHECK_GT(rtp_clock_frequency, 0); + + return last_absolute_capture_timestamp + + static_cast( + rtc::dchecked_cast(rtp_timestamp - last_rtp_timestamp) + << 32) / + rtp_clock_frequency; +} + +bool AbsoluteCaptureTimeInterpolator::ShouldInterpolateExtension( + Timestamp receive_time, + uint32_t source, + uint32_t rtp_timestamp, + uint32_t rtp_clock_frequency) const { + // Shouldn't if we don't have a previously received extension stored. + if (last_receive_time_ == kInvalidLastReceiveTime) { + return false; + } + + // Shouldn't if the last received extension is too old. + if ((receive_time - last_receive_time_) > kInterpolationMaxInterval) { + return false; + } + + // Shouldn't if the source has changed. + if (last_source_ != source) { + return false; + } + + // Shouldn't if the RTP clock frequency has changed. + if (last_rtp_clock_frequency_ != rtp_clock_frequency) { + return false; + } + + // Shouldn't if the RTP clock frequency is invalid. + if (rtp_clock_frequency <= 0) { + return false; + } + + return true; +} + +} // namespace webrtc diff --git a/modules/rtp_rtcp/source/absolute_capture_time_interpolator.h b/modules/rtp_rtcp/source/absolute_capture_time_interpolator.h new file mode 100644 index 0000000000..89d7f0850c --- /dev/null +++ b/modules/rtp_rtcp/source/absolute_capture_time_interpolator.h @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_RTP_RTCP_SOURCE_ABSOLUTE_CAPTURE_TIME_INTERPOLATOR_H_ +#define MODULES_RTP_RTCP_SOURCE_ABSOLUTE_CAPTURE_TIME_INTERPOLATOR_H_ + +#include "api/array_view.h" +#include "api/rtp_headers.h" +#include "api/units/time_delta.h" +#include "api/units/timestamp.h" +#include "rtc_base/synchronization/mutex.h" +#include "rtc_base/thread_annotations.h" +#include "system_wrappers/include/clock.h" + +namespace webrtc { + +// +// Helper class for interpolating the |AbsoluteCaptureTime| header extension. +// +// Supports the "timestamp interpolation" optimization: +// A receiver SHOULD memorize the capture system (i.e. CSRC/SSRC), capture +// timestamp, and RTP timestamp of the most recently received abs-capture-time +// packet on each received stream. It can then use that information, in +// combination with RTP timestamps of packets without abs-capture-time, to +// extrapolate missing capture timestamps. +// +// See: https://webrtc.org/experiments/rtp-hdrext/abs-capture-time/ +// +class AbsoluteCaptureTimeInterpolator { + public: + static constexpr TimeDelta kInterpolationMaxInterval = + TimeDelta::Millis(5000); + + explicit AbsoluteCaptureTimeInterpolator(Clock* clock); + + // Returns the source (i.e. SSRC or CSRC) of the capture system. + static uint32_t GetSource(uint32_t ssrc, + rtc::ArrayView csrcs); + + // Returns a received header extension, an interpolated header extension, or + // |absl::nullopt| if it's not possible to interpolate a header extension. + absl::optional OnReceivePacket( + uint32_t source, + uint32_t rtp_timestamp, + uint32_t rtp_clock_frequency, + const absl::optional& received_extension); + + private: + friend class AbsoluteCaptureTimeSender; + + static uint64_t InterpolateAbsoluteCaptureTimestamp( + uint32_t rtp_timestamp, + uint32_t rtp_clock_frequency, + uint32_t last_rtp_timestamp, + uint64_t last_absolute_capture_timestamp); + + bool ShouldInterpolateExtension(Timestamp receive_time, + uint32_t source, + uint32_t rtp_timestamp, + uint32_t rtp_clock_frequency) const + RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + Clock* const clock_; + + Mutex mutex_; + + Timestamp last_receive_time_ RTC_GUARDED_BY(mutex_); + + uint32_t last_source_ RTC_GUARDED_BY(mutex_); + uint32_t last_rtp_timestamp_ RTC_GUARDED_BY(mutex_); + uint32_t last_rtp_clock_frequency_ RTC_GUARDED_BY(mutex_); + uint64_t last_absolute_capture_timestamp_ RTC_GUARDED_BY(mutex_); + absl::optional last_estimated_capture_clock_offset_ + RTC_GUARDED_BY(mutex_); +}; // AbsoluteCaptureTimeInterpolator + +} // namespace webrtc + +#endif // MODULES_RTP_RTCP_SOURCE_ABSOLUTE_CAPTURE_TIME_INTERPOLATOR_H_ diff --git a/modules/rtp_rtcp/source/absolute_capture_time_receiver_unittest.cc b/modules/rtp_rtcp/source/absolute_capture_time_interpolator_unittest.cc similarity index 61% rename from modules/rtp_rtcp/source/absolute_capture_time_receiver_unittest.cc rename to modules/rtp_rtcp/source/absolute_capture_time_interpolator_unittest.cc index ecf256734d..6a312f9b43 100644 --- a/modules/rtp_rtcp/source/absolute_capture_time_receiver_unittest.cc +++ b/modules/rtp_rtcp/source/absolute_capture_time_interpolator_unittest.cc @@ -8,7 +8,7 @@ * be found in the AUTHORS file in the root of the source tree. */ -#include "modules/rtp_rtcp/source/absolute_capture_time_receiver.h" +#include "modules/rtp_rtcp/source/absolute_capture_time_interpolator.h" #include "system_wrappers/include/ntp_time.h" #include "test/gmock.h" @@ -16,20 +16,21 @@ namespace webrtc { -TEST(AbsoluteCaptureTimeReceiverTest, GetSourceWithoutCsrcs) { +TEST(AbsoluteCaptureTimeInterpolatorTest, GetSourceWithoutCsrcs) { constexpr uint32_t kSsrc = 12; - EXPECT_EQ(AbsoluteCaptureTimeReceiver::GetSource(kSsrc, nullptr), kSsrc); + EXPECT_EQ(AbsoluteCaptureTimeInterpolator::GetSource(kSsrc, nullptr), kSsrc); } -TEST(AbsoluteCaptureTimeReceiverTest, GetSourceWithCsrcs) { +TEST(AbsoluteCaptureTimeInterpolatorTest, GetSourceWithCsrcs) { constexpr uint32_t kSsrc = 12; constexpr uint32_t kCsrcs[] = {34, 56, 78, 90}; - EXPECT_EQ(AbsoluteCaptureTimeReceiver::GetSource(kSsrc, kCsrcs), kCsrcs[0]); + EXPECT_EQ(AbsoluteCaptureTimeInterpolator::GetSource(kSsrc, kCsrcs), + kCsrcs[0]); } -TEST(AbsoluteCaptureTimeReceiverTest, ReceiveExtensionReturnsExtension) { +TEST(AbsoluteCaptureTimeInterpolatorTest, ReceiveExtensionReturnsExtension) { constexpr uint32_t kSource = 1337; constexpr uint32_t kRtpClockFrequency = 64000; constexpr uint32_t kRtpTimestamp0 = 1020300000; @@ -40,20 +41,19 @@ TEST(AbsoluteCaptureTimeReceiverTest, ReceiveExtensionReturnsExtension) { AbsoluteCaptureTime{Int64MsToUQ32x32(9020), absl::nullopt}; SimulatedClock clock(0); - AbsoluteCaptureTimeReceiver receiver(&clock); + AbsoluteCaptureTimeInterpolator interpolator(&clock); - receiver.SetRemoteToLocalClockOffset(0); - - EXPECT_EQ(receiver.OnReceivePacket(kSource, kRtpTimestamp0, - kRtpClockFrequency, kExtension0), + EXPECT_EQ(interpolator.OnReceivePacket(kSource, kRtpTimestamp0, + kRtpClockFrequency, kExtension0), kExtension0); - EXPECT_EQ(receiver.OnReceivePacket(kSource, kRtpTimestamp1, - kRtpClockFrequency, kExtension1), + EXPECT_EQ(interpolator.OnReceivePacket(kSource, kRtpTimestamp1, + kRtpClockFrequency, kExtension1), kExtension1); } -TEST(AbsoluteCaptureTimeReceiverTest, ReceiveNoExtensionReturnsNoExtension) { +TEST(AbsoluteCaptureTimeInterpolatorTest, + ReceiveNoExtensionReturnsNoExtension) { constexpr uint32_t kSource = 1337; constexpr uint32_t kRtpClockFrequency = 64000; constexpr uint32_t kRtpTimestamp0 = 1020300000; @@ -62,20 +62,18 @@ TEST(AbsoluteCaptureTimeReceiverTest, ReceiveNoExtensionReturnsNoExtension) { static const absl::optional kExtension1 = absl::nullopt; SimulatedClock clock(0); - AbsoluteCaptureTimeReceiver receiver(&clock); + AbsoluteCaptureTimeInterpolator interpolator(&clock); - receiver.SetRemoteToLocalClockOffset(0); - - EXPECT_EQ(receiver.OnReceivePacket(kSource, kRtpTimestamp0, - kRtpClockFrequency, kExtension0), + EXPECT_EQ(interpolator.OnReceivePacket(kSource, kRtpTimestamp0, + kRtpClockFrequency, kExtension0), absl::nullopt); - EXPECT_EQ(receiver.OnReceivePacket(kSource, kRtpTimestamp1, - kRtpClockFrequency, kExtension1), + EXPECT_EQ(interpolator.OnReceivePacket(kSource, kRtpTimestamp1, + kRtpClockFrequency, kExtension1), absl::nullopt); } -TEST(AbsoluteCaptureTimeReceiverTest, InterpolateLaterPacketArrivingLater) { +TEST(AbsoluteCaptureTimeInterpolatorTest, InterpolateLaterPacketArrivingLater) { constexpr uint32_t kSource = 1337; constexpr uint32_t kRtpClockFrequency = 64000; constexpr uint32_t kRtpTimestamp0 = 1020300000; @@ -87,15 +85,13 @@ TEST(AbsoluteCaptureTimeReceiverTest, InterpolateLaterPacketArrivingLater) { static const absl::optional kExtension2 = absl::nullopt; SimulatedClock clock(0); - AbsoluteCaptureTimeReceiver receiver(&clock); + AbsoluteCaptureTimeInterpolator interpolator(&clock); - receiver.SetRemoteToLocalClockOffset(0); - - EXPECT_EQ(receiver.OnReceivePacket(kSource, kRtpTimestamp0, - kRtpClockFrequency, kExtension0), + EXPECT_EQ(interpolator.OnReceivePacket(kSource, kRtpTimestamp0, + kRtpClockFrequency, kExtension0), kExtension0); - absl::optional extension = receiver.OnReceivePacket( + absl::optional extension = interpolator.OnReceivePacket( kSource, kRtpTimestamp1, kRtpClockFrequency, kExtension1); EXPECT_TRUE(extension.has_value()); EXPECT_EQ(UQ32x32ToInt64Ms(extension->absolute_capture_timestamp), @@ -103,8 +99,8 @@ TEST(AbsoluteCaptureTimeReceiverTest, InterpolateLaterPacketArrivingLater) { EXPECT_EQ(extension->estimated_capture_clock_offset, kExtension0->estimated_capture_clock_offset); - extension = receiver.OnReceivePacket(kSource, kRtpTimestamp2, - kRtpClockFrequency, kExtension2); + extension = interpolator.OnReceivePacket(kSource, kRtpTimestamp2, + kRtpClockFrequency, kExtension2); EXPECT_TRUE(extension.has_value()); EXPECT_EQ(UQ32x32ToInt64Ms(extension->absolute_capture_timestamp), UQ32x32ToInt64Ms(kExtension0->absolute_capture_timestamp) + 40); @@ -112,7 +108,8 @@ TEST(AbsoluteCaptureTimeReceiverTest, InterpolateLaterPacketArrivingLater) { kExtension0->estimated_capture_clock_offset); } -TEST(AbsoluteCaptureTimeReceiverTest, InterpolateEarlierPacketArrivingLater) { +TEST(AbsoluteCaptureTimeInterpolatorTest, + InterpolateEarlierPacketArrivingLater) { constexpr uint32_t kSource = 1337; constexpr uint32_t kRtpClockFrequency = 64000; constexpr uint32_t kRtpTimestamp0 = 1020300000; @@ -124,15 +121,13 @@ TEST(AbsoluteCaptureTimeReceiverTest, InterpolateEarlierPacketArrivingLater) { static const absl::optional kExtension2 = absl::nullopt; SimulatedClock clock(0); - AbsoluteCaptureTimeReceiver receiver(&clock); + AbsoluteCaptureTimeInterpolator interpolator(&clock); - receiver.SetRemoteToLocalClockOffset(0); - - EXPECT_EQ(receiver.OnReceivePacket(kSource, kRtpTimestamp0, - kRtpClockFrequency, kExtension0), + EXPECT_EQ(interpolator.OnReceivePacket(kSource, kRtpTimestamp0, + kRtpClockFrequency, kExtension0), kExtension0); - absl::optional extension = receiver.OnReceivePacket( + absl::optional extension = interpolator.OnReceivePacket( kSource, kRtpTimestamp1, kRtpClockFrequency, kExtension1); EXPECT_TRUE(extension.has_value()); EXPECT_EQ(UQ32x32ToInt64Ms(extension->absolute_capture_timestamp), @@ -140,8 +135,8 @@ TEST(AbsoluteCaptureTimeReceiverTest, InterpolateEarlierPacketArrivingLater) { EXPECT_EQ(extension->estimated_capture_clock_offset, kExtension0->estimated_capture_clock_offset); - extension = receiver.OnReceivePacket(kSource, kRtpTimestamp2, - kRtpClockFrequency, kExtension2); + extension = interpolator.OnReceivePacket(kSource, kRtpTimestamp2, + kRtpClockFrequency, kExtension2); EXPECT_TRUE(extension.has_value()); EXPECT_EQ(UQ32x32ToInt64Ms(extension->absolute_capture_timestamp), UQ32x32ToInt64Ms(kExtension0->absolute_capture_timestamp) - 40); @@ -149,7 +144,7 @@ TEST(AbsoluteCaptureTimeReceiverTest, InterpolateEarlierPacketArrivingLater) { kExtension0->estimated_capture_clock_offset); } -TEST(AbsoluteCaptureTimeReceiverTest, +TEST(AbsoluteCaptureTimeInterpolatorTest, InterpolateLaterPacketArrivingLaterWithRtpTimestampWrapAround) { constexpr uint32_t kSource = 1337; constexpr uint32_t kRtpClockFrequency = 64000; @@ -162,15 +157,13 @@ TEST(AbsoluteCaptureTimeReceiverTest, static const absl::optional kExtension2 = absl::nullopt; SimulatedClock clock(0); - AbsoluteCaptureTimeReceiver receiver(&clock); + AbsoluteCaptureTimeInterpolator interpolator(&clock); - receiver.SetRemoteToLocalClockOffset(0); - - EXPECT_EQ(receiver.OnReceivePacket(kSource, kRtpTimestamp0, - kRtpClockFrequency, kExtension0), + EXPECT_EQ(interpolator.OnReceivePacket(kSource, kRtpTimestamp0, + kRtpClockFrequency, kExtension0), kExtension0); - absl::optional extension = receiver.OnReceivePacket( + absl::optional extension = interpolator.OnReceivePacket( kSource, kRtpTimestamp1, kRtpClockFrequency, kExtension1); EXPECT_TRUE(extension.has_value()); EXPECT_EQ(UQ32x32ToInt64Ms(extension->absolute_capture_timestamp), @@ -178,8 +171,8 @@ TEST(AbsoluteCaptureTimeReceiverTest, EXPECT_EQ(extension->estimated_capture_clock_offset, kExtension0->estimated_capture_clock_offset); - extension = receiver.OnReceivePacket(kSource, kRtpTimestamp2, - kRtpClockFrequency, kExtension2); + extension = interpolator.OnReceivePacket(kSource, kRtpTimestamp2, + kRtpClockFrequency, kExtension2); EXPECT_TRUE(extension.has_value()); EXPECT_EQ(UQ32x32ToInt64Ms(extension->absolute_capture_timestamp), UQ32x32ToInt64Ms(kExtension0->absolute_capture_timestamp) + 40); @@ -187,7 +180,7 @@ TEST(AbsoluteCaptureTimeReceiverTest, kExtension0->estimated_capture_clock_offset); } -TEST(AbsoluteCaptureTimeReceiverTest, +TEST(AbsoluteCaptureTimeInterpolatorTest, InterpolateEarlierPacketArrivingLaterWithRtpTimestampWrapAround) { constexpr uint32_t kSource = 1337; constexpr uint32_t kRtpClockFrequency = 64000; @@ -200,15 +193,13 @@ TEST(AbsoluteCaptureTimeReceiverTest, static const absl::optional kExtension2 = absl::nullopt; SimulatedClock clock(0); - AbsoluteCaptureTimeReceiver receiver(&clock); + AbsoluteCaptureTimeInterpolator interpolator(&clock); - receiver.SetRemoteToLocalClockOffset(0); - - EXPECT_EQ(receiver.OnReceivePacket(kSource, kRtpTimestamp0, - kRtpClockFrequency, kExtension0), + EXPECT_EQ(interpolator.OnReceivePacket(kSource, kRtpTimestamp0, + kRtpClockFrequency, kExtension0), kExtension0); - absl::optional extension = receiver.OnReceivePacket( + absl::optional extension = interpolator.OnReceivePacket( kSource, kRtpTimestamp1, kRtpClockFrequency, kExtension1); EXPECT_TRUE(extension.has_value()); EXPECT_EQ(UQ32x32ToInt64Ms(extension->absolute_capture_timestamp), @@ -216,8 +207,8 @@ TEST(AbsoluteCaptureTimeReceiverTest, EXPECT_EQ(extension->estimated_capture_clock_offset, kExtension0->estimated_capture_clock_offset); - extension = receiver.OnReceivePacket(kSource, kRtpTimestamp2, - kRtpClockFrequency, kExtension2); + extension = interpolator.OnReceivePacket(kSource, kRtpTimestamp2, + kRtpClockFrequency, kExtension2); EXPECT_TRUE(extension.has_value()); EXPECT_EQ(UQ32x32ToInt64Ms(extension->absolute_capture_timestamp), UQ32x32ToInt64Ms(kExtension0->absolute_capture_timestamp) - 40); @@ -225,51 +216,7 @@ TEST(AbsoluteCaptureTimeReceiverTest, kExtension0->estimated_capture_clock_offset); } -TEST(AbsoluteCaptureTimeReceiverTest, - SkipEstimatedCaptureClockOffsetIfRemoteToLocalClockOffsetIsUnknown) { - constexpr uint32_t kSource = 1337; - constexpr uint32_t kRtpClockFrequency = 64000; - constexpr uint32_t kRtpTimestamp0 = 1020300000; - constexpr uint32_t kRtpTimestamp1 = kRtpTimestamp0 + 1280; - constexpr uint32_t kRtpTimestamp2 = kRtpTimestamp0 + 2560; - static const absl::optional kExtension0 = - AbsoluteCaptureTime{Int64MsToUQ32x32(9000), Int64MsToQ32x32(-350)}; - static const absl::optional kExtension1 = absl::nullopt; - static const absl::optional kExtension2 = absl::nullopt; - static const absl::optional kRemoteToLocalClockOffset2 = - Int64MsToQ32x32(-7000007); - - SimulatedClock clock(0); - AbsoluteCaptureTimeReceiver receiver(&clock); - - receiver.SetRemoteToLocalClockOffset(0); - - EXPECT_EQ(receiver.OnReceivePacket(kSource, kRtpTimestamp0, - kRtpClockFrequency, kExtension0), - kExtension0); - - receiver.SetRemoteToLocalClockOffset(absl::nullopt); - - absl::optional extension = receiver.OnReceivePacket( - kSource, kRtpTimestamp1, kRtpClockFrequency, kExtension1); - EXPECT_TRUE(extension.has_value()); - EXPECT_EQ(UQ32x32ToInt64Ms(extension->absolute_capture_timestamp), - UQ32x32ToInt64Ms(kExtension0->absolute_capture_timestamp) + 20); - EXPECT_EQ(extension->estimated_capture_clock_offset, absl::nullopt); - - receiver.SetRemoteToLocalClockOffset(kRemoteToLocalClockOffset2); - - extension = receiver.OnReceivePacket(kSource, kRtpTimestamp2, - kRtpClockFrequency, kExtension2); - EXPECT_TRUE(extension.has_value()); - EXPECT_EQ(UQ32x32ToInt64Ms(extension->absolute_capture_timestamp), - UQ32x32ToInt64Ms(kExtension0->absolute_capture_timestamp) + 40); - EXPECT_EQ(extension->estimated_capture_clock_offset, - *kExtension0->estimated_capture_clock_offset + - *kRemoteToLocalClockOffset2); -} - -TEST(AbsoluteCaptureTimeReceiverTest, SkipInterpolateIfTooLate) { +TEST(AbsoluteCaptureTimeInterpolatorTest, SkipInterpolateIfTooLate) { constexpr uint32_t kSource = 1337; constexpr uint32_t kRtpClockFrequency = 64000; constexpr uint32_t kRtpTimestamp0 = 1020300000; @@ -281,30 +228,28 @@ TEST(AbsoluteCaptureTimeReceiverTest, SkipInterpolateIfTooLate) { static const absl::optional kExtension2 = absl::nullopt; SimulatedClock clock(0); - AbsoluteCaptureTimeReceiver receiver(&clock); + AbsoluteCaptureTimeInterpolator interpolator(&clock); - receiver.SetRemoteToLocalClockOffset(0); - - EXPECT_EQ(receiver.OnReceivePacket(kSource, kRtpTimestamp0, - kRtpClockFrequency, kExtension0), + EXPECT_EQ(interpolator.OnReceivePacket(kSource, kRtpTimestamp0, + kRtpClockFrequency, kExtension0), kExtension0); - clock.AdvanceTime(AbsoluteCaptureTimeReceiver::kInterpolationMaxInterval); + clock.AdvanceTime(AbsoluteCaptureTimeInterpolator::kInterpolationMaxInterval); - EXPECT_TRUE(receiver + EXPECT_TRUE(interpolator .OnReceivePacket(kSource, kRtpTimestamp1, kRtpClockFrequency, kExtension1) .has_value()); clock.AdvanceTimeMilliseconds(1); - EXPECT_FALSE(receiver + EXPECT_FALSE(interpolator .OnReceivePacket(kSource, kRtpTimestamp2, kRtpClockFrequency, kExtension2) .has_value()); } -TEST(AbsoluteCaptureTimeReceiverTest, SkipInterpolateIfSourceChanged) { +TEST(AbsoluteCaptureTimeInterpolatorTest, SkipInterpolateIfSourceChanged) { constexpr uint32_t kSource0 = 1337; constexpr uint32_t kSource1 = 1338; constexpr uint32_t kRtpClockFrequency = 64000; @@ -315,21 +260,19 @@ TEST(AbsoluteCaptureTimeReceiverTest, SkipInterpolateIfSourceChanged) { static const absl::optional kExtension1 = absl::nullopt; SimulatedClock clock(0); - AbsoluteCaptureTimeReceiver receiver(&clock); + AbsoluteCaptureTimeInterpolator interpolator(&clock); - receiver.SetRemoteToLocalClockOffset(0); - - EXPECT_EQ(receiver.OnReceivePacket(kSource0, kRtpTimestamp0, - kRtpClockFrequency, kExtension0), + EXPECT_EQ(interpolator.OnReceivePacket(kSource0, kRtpTimestamp0, + kRtpClockFrequency, kExtension0), kExtension0); - EXPECT_FALSE(receiver + EXPECT_FALSE(interpolator .OnReceivePacket(kSource1, kRtpTimestamp1, kRtpClockFrequency, kExtension1) .has_value()); } -TEST(AbsoluteCaptureTimeReceiverTest, +TEST(AbsoluteCaptureTimeInterpolatorTest, SkipInterpolateIfRtpClockFrequencyChanged) { constexpr uint32_t kSource = 1337; constexpr uint32_t kRtpClockFrequency0 = 64000; @@ -341,21 +284,19 @@ TEST(AbsoluteCaptureTimeReceiverTest, static const absl::optional kExtension1 = absl::nullopt; SimulatedClock clock(0); - AbsoluteCaptureTimeReceiver receiver(&clock); + AbsoluteCaptureTimeInterpolator interpolator(&clock); - receiver.SetRemoteToLocalClockOffset(0); - - EXPECT_EQ(receiver.OnReceivePacket(kSource, kRtpTimestamp0, - kRtpClockFrequency0, kExtension0), + EXPECT_EQ(interpolator.OnReceivePacket(kSource, kRtpTimestamp0, + kRtpClockFrequency0, kExtension0), kExtension0); - EXPECT_FALSE(receiver + EXPECT_FALSE(interpolator .OnReceivePacket(kSource, kRtpTimestamp1, kRtpClockFrequency1, kExtension1) .has_value()); } -TEST(AbsoluteCaptureTimeReceiverTest, +TEST(AbsoluteCaptureTimeInterpolatorTest, SkipInterpolateIfRtpClockFrequencyIsInvalid) { constexpr uint32_t kSource = 1337; constexpr uint32_t kRtpClockFrequency = 0; @@ -366,21 +307,19 @@ TEST(AbsoluteCaptureTimeReceiverTest, static const absl::optional kExtension1 = absl::nullopt; SimulatedClock clock(0); - AbsoluteCaptureTimeReceiver receiver(&clock); + AbsoluteCaptureTimeInterpolator interpolator(&clock); - receiver.SetRemoteToLocalClockOffset(0); - - EXPECT_EQ(receiver.OnReceivePacket(kSource, kRtpTimestamp0, - kRtpClockFrequency, kExtension0), + EXPECT_EQ(interpolator.OnReceivePacket(kSource, kRtpTimestamp0, + kRtpClockFrequency, kExtension0), kExtension0); - EXPECT_FALSE(receiver + EXPECT_FALSE(interpolator .OnReceivePacket(kSource, kRtpTimestamp1, kRtpClockFrequency, kExtension1) .has_value()); } -TEST(AbsoluteCaptureTimeReceiverTest, SkipInterpolateIsSticky) { +TEST(AbsoluteCaptureTimeInterpolatorTest, SkipInterpolateIsSticky) { constexpr uint32_t kSource0 = 1337; constexpr uint32_t kSource1 = 1338; constexpr uint32_t kSource2 = 1337; @@ -394,20 +333,18 @@ TEST(AbsoluteCaptureTimeReceiverTest, SkipInterpolateIsSticky) { static const absl::optional kExtension2 = absl::nullopt; SimulatedClock clock(0); - AbsoluteCaptureTimeReceiver receiver(&clock); + AbsoluteCaptureTimeInterpolator interpolator(&clock); - receiver.SetRemoteToLocalClockOffset(0); - - EXPECT_EQ(receiver.OnReceivePacket(kSource0, kRtpTimestamp0, - kRtpClockFrequency, kExtension0), + EXPECT_EQ(interpolator.OnReceivePacket(kSource0, kRtpTimestamp0, + kRtpClockFrequency, kExtension0), kExtension0); - EXPECT_FALSE(receiver + EXPECT_FALSE(interpolator .OnReceivePacket(kSource1, kRtpTimestamp1, kRtpClockFrequency, kExtension1) .has_value()); - EXPECT_FALSE(receiver + EXPECT_FALSE(interpolator .OnReceivePacket(kSource2, kRtpTimestamp2, kRtpClockFrequency, kExtension2) .has_value()); diff --git a/modules/rtp_rtcp/source/absolute_capture_time_receiver.cc b/modules/rtp_rtcp/source/absolute_capture_time_receiver.cc index 529ed7eef6..efb75506d0 100644 --- a/modules/rtp_rtcp/source/absolute_capture_time_receiver.cc +++ b/modules/rtp_rtcp/source/absolute_capture_time_receiver.cc @@ -10,38 +10,14 @@ #include "modules/rtp_rtcp/source/absolute_capture_time_receiver.h" -#include - -#include "rtc_base/checks.h" - namespace webrtc { -namespace { - -constexpr Timestamp kInvalidLastReceiveTime = Timestamp::MinusInfinity(); -} // namespace - -constexpr TimeDelta AbsoluteCaptureTimeReceiver::kInterpolationMaxInterval; AbsoluteCaptureTimeReceiver::AbsoluteCaptureTimeReceiver(Clock* clock) - : clock_(clock), - remote_to_local_clock_offset_(absl::nullopt), - last_receive_time_(kInvalidLastReceiveTime) {} - -uint32_t AbsoluteCaptureTimeReceiver::GetSource( - uint32_t ssrc, - rtc::ArrayView csrcs) { - if (csrcs.empty()) { - return ssrc; - } - - return csrcs[0]; -} + : AbsoluteCaptureTimeInterpolator(clock) {} void AbsoluteCaptureTimeReceiver::SetRemoteToLocalClockOffset( absl::optional value_q32x32) { - MutexLock lock(&mutex_); - - remote_to_local_clock_offset_ = value_q32x32; + capture_clock_offset_updater_.SetRemoteToLocalClockOffset(value_q32x32); } absl::optional @@ -50,101 +26,16 @@ AbsoluteCaptureTimeReceiver::OnReceivePacket( uint32_t rtp_timestamp, uint32_t rtp_clock_frequency, const absl::optional& received_extension) { - const Timestamp receive_time = clock_->CurrentTime(); + auto extension = AbsoluteCaptureTimeInterpolator::OnReceivePacket( + source, rtp_timestamp, rtp_clock_frequency, received_extension); - MutexLock lock(&mutex_); - - AbsoluteCaptureTime extension; - if (received_extension == absl::nullopt) { - if (!ShouldInterpolateExtension(receive_time, source, rtp_timestamp, - rtp_clock_frequency)) { - last_receive_time_ = kInvalidLastReceiveTime; - return absl::nullopt; - } - - extension.absolute_capture_timestamp = InterpolateAbsoluteCaptureTimestamp( - rtp_timestamp, rtp_clock_frequency, last_rtp_timestamp_, - last_absolute_capture_timestamp_); - extension.estimated_capture_clock_offset = - last_estimated_capture_clock_offset_; - } else { - last_source_ = source; - last_rtp_timestamp_ = rtp_timestamp; - last_rtp_clock_frequency_ = rtp_clock_frequency; - last_absolute_capture_timestamp_ = - received_extension->absolute_capture_timestamp; - last_estimated_capture_clock_offset_ = - received_extension->estimated_capture_clock_offset; - - last_receive_time_ = receive_time; - - extension = *received_extension; + if (extension.has_value()) { + extension->estimated_capture_clock_offset = + capture_clock_offset_updater_.AdjustEstimatedCaptureClockOffset( + extension->estimated_capture_clock_offset); } - extension.estimated_capture_clock_offset = AdjustEstimatedCaptureClockOffset( - extension.estimated_capture_clock_offset); - return extension; } -uint64_t AbsoluteCaptureTimeReceiver::InterpolateAbsoluteCaptureTimestamp( - uint32_t rtp_timestamp, - uint32_t rtp_clock_frequency, - uint32_t last_rtp_timestamp, - uint64_t last_absolute_capture_timestamp) { - RTC_DCHECK_GT(rtp_clock_frequency, 0); - - return last_absolute_capture_timestamp + - static_cast( - rtc::dchecked_cast(rtp_timestamp - last_rtp_timestamp) - << 32) / - rtp_clock_frequency; -} - -bool AbsoluteCaptureTimeReceiver::ShouldInterpolateExtension( - Timestamp receive_time, - uint32_t source, - uint32_t rtp_timestamp, - uint32_t rtp_clock_frequency) const { - // Shouldn't if we don't have a previously received extension stored. - if (last_receive_time_ == kInvalidLastReceiveTime) { - return false; - } - - // Shouldn't if the last received extension is too old. - if ((receive_time - last_receive_time_) > kInterpolationMaxInterval) { - return false; - } - - // Shouldn't if the source has changed. - if (last_source_ != source) { - return false; - } - - // Shouldn't if the RTP clock frequency has changed. - if (last_rtp_clock_frequency_ != rtp_clock_frequency) { - return false; - } - - // Shouldn't if the RTP clock frequency is invalid. - if (rtp_clock_frequency <= 0) { - return false; - } - - return true; -} - -absl::optional -AbsoluteCaptureTimeReceiver::AdjustEstimatedCaptureClockOffset( - absl::optional received_value) const { - if (received_value == absl::nullopt || - remote_to_local_clock_offset_ == absl::nullopt) { - return absl::nullopt; - } - - // Do calculations as "unsigned" to make overflows deterministic. - return static_cast(*received_value) + - static_cast(*remote_to_local_clock_offset_); -} - } // namespace webrtc diff --git a/modules/rtp_rtcp/source/absolute_capture_time_receiver.h b/modules/rtp_rtcp/source/absolute_capture_time_receiver.h index ce3442b386..ad1bd7eb5d 100644 --- a/modules/rtp_rtcp/source/absolute_capture_time_receiver.h +++ b/modules/rtp_rtcp/source/absolute_capture_time_receiver.h @@ -11,89 +11,28 @@ #ifndef MODULES_RTP_RTCP_SOURCE_ABSOLUTE_CAPTURE_TIME_RECEIVER_H_ #define MODULES_RTP_RTCP_SOURCE_ABSOLUTE_CAPTURE_TIME_RECEIVER_H_ -#include "api/array_view.h" -#include "api/rtp_headers.h" -#include "api/units/time_delta.h" -#include "api/units/timestamp.h" -#include "rtc_base/synchronization/mutex.h" -#include "rtc_base/thread_annotations.h" +#include "modules/rtp_rtcp/source/absolute_capture_time_interpolator.h" +#include "modules/rtp_rtcp/source/capture_clock_offset_updater.h" #include "system_wrappers/include/clock.h" namespace webrtc { -// -// Helper class for receiving the |AbsoluteCaptureTime| header extension. -// -// Supports the "timestamp interpolation" optimization: -// A receiver SHOULD memorize the capture system (i.e. CSRC/SSRC), capture -// timestamp, and RTP timestamp of the most recently received abs-capture-time -// packet on each received stream. It can then use that information, in -// combination with RTP timestamps of packets without abs-capture-time, to -// extrapolate missing capture timestamps. -// -// See: https://webrtc.org/experiments/rtp-hdrext/abs-capture-time/ -// -class AbsoluteCaptureTimeReceiver { +// DEPRECATED. Use `AbsoluteCaptureTimeInterpolator` instead. +class AbsoluteCaptureTimeReceiver : public AbsoluteCaptureTimeInterpolator { public: - static constexpr TimeDelta kInterpolationMaxInterval = - TimeDelta::Millis(5000); - explicit AbsoluteCaptureTimeReceiver(Clock* clock); - // Returns the source (i.e. SSRC or CSRC) of the capture system. - static uint32_t GetSource(uint32_t ssrc, - rtc::ArrayView csrcs); - - // Sets the NTP clock offset between the sender system (which may be different - // from the capture system) and the local system. This information is normally - // provided by passing half the value of the Round-Trip Time estimation given - // by RTCP sender reports (see DLSR/DLRR). - // - // Note that the value must be in Q32.32-formatted fixed-point seconds. - void SetRemoteToLocalClockOffset(absl::optional value_q32x32); - - // Returns a received header extension, an interpolated header extension, or - // |absl::nullopt| if it's not possible to interpolate a header extension. absl::optional OnReceivePacket( uint32_t source, uint32_t rtp_timestamp, uint32_t rtp_clock_frequency, const absl::optional& received_extension); + void SetRemoteToLocalClockOffset(absl::optional value_q32x32); + private: - friend class AbsoluteCaptureTimeSender; - - static uint64_t InterpolateAbsoluteCaptureTimestamp( - uint32_t rtp_timestamp, - uint32_t rtp_clock_frequency, - uint32_t last_rtp_timestamp, - uint64_t last_absolute_capture_timestamp); - - bool ShouldInterpolateExtension(Timestamp receive_time, - uint32_t source, - uint32_t rtp_timestamp, - uint32_t rtp_clock_frequency) const - RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); - - absl::optional AdjustEstimatedCaptureClockOffset( - absl::optional received_value) const - RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); - - Clock* const clock_; - - Mutex mutex_; - - absl::optional remote_to_local_clock_offset_ RTC_GUARDED_BY(mutex_); - - Timestamp last_receive_time_ RTC_GUARDED_BY(mutex_); - - uint32_t last_source_ RTC_GUARDED_BY(mutex_); - uint32_t last_rtp_timestamp_ RTC_GUARDED_BY(mutex_); - uint32_t last_rtp_clock_frequency_ RTC_GUARDED_BY(mutex_); - uint64_t last_absolute_capture_timestamp_ RTC_GUARDED_BY(mutex_); - absl::optional last_estimated_capture_clock_offset_ - RTC_GUARDED_BY(mutex_); -}; // AbsoluteCaptureTimeReceiver + CaptureClockOffsetUpdater capture_clock_offset_updater_; +}; } // namespace webrtc diff --git a/modules/rtp_rtcp/source/capture_clock_offset_updater.cc b/modules/rtp_rtcp/source/capture_clock_offset_updater.cc new file mode 100644 index 0000000000..a5b12cb422 --- /dev/null +++ b/modules/rtp_rtcp/source/capture_clock_offset_updater.cc @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/rtp_rtcp/source/capture_clock_offset_updater.h" + +namespace webrtc { + +absl::optional +CaptureClockOffsetUpdater::AdjustEstimatedCaptureClockOffset( + absl::optional remote_capture_clock_offset) const { + if (remote_capture_clock_offset == absl::nullopt || + remote_to_local_clock_offset_ == absl::nullopt) { + return absl::nullopt; + } + + // Do calculations as "unsigned" to make overflows deterministic. + return static_cast(*remote_capture_clock_offset) + + static_cast(*remote_to_local_clock_offset_); +} + +void CaptureClockOffsetUpdater::SetRemoteToLocalClockOffset( + absl::optional offset_q32x32) { + remote_to_local_clock_offset_ = offset_q32x32; +} + +} // namespace webrtc diff --git a/modules/rtp_rtcp/source/capture_clock_offset_updater.h b/modules/rtp_rtcp/source/capture_clock_offset_updater.h new file mode 100644 index 0000000000..71d3eb4831 --- /dev/null +++ b/modules/rtp_rtcp/source/capture_clock_offset_updater.h @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_RTP_RTCP_SOURCE_CAPTURE_CLOCK_OFFSET_UPDATER_H_ +#define MODULES_RTP_RTCP_SOURCE_CAPTURE_CLOCK_OFFSET_UPDATER_H_ + +#include + +#include "absl/types/optional.h" + +namespace webrtc { + +// +// Helper class for calculating the clock offset against the capturer's clock. +// +// This is achieved by adjusting the estimated capture clock offset in received +// Absolute Capture Time RTP header extension (see +// https://webrtc.org/experiments/rtp-hdrext/abs-capture-time/), which +// represents the clock offset between a remote sender and the capturer, by +// adding local-to-remote clock offset. + +class CaptureClockOffsetUpdater { + public: + // Adjusts remote_capture_clock_offset, which originates from Absolute Capture + // Time RTP header extension, to get the local clock offset against the + // capturer's clock. + absl::optional AdjustEstimatedCaptureClockOffset( + absl::optional remote_capture_clock_offset) const; + + // Sets the NTP clock offset between the sender system (which may be different + // from the capture system) and the local system. This information is normally + // provided by passing half the value of the Round-Trip Time estimation given + // by RTCP sender reports (see DLSR/DLRR). + // + // Note that the value must be in Q32.32-formatted fixed-point seconds. + void SetRemoteToLocalClockOffset(absl::optional offset_q32x32); + + private: + absl::optional remote_to_local_clock_offset_; +}; + +} // namespace webrtc + +#endif // MODULES_RTP_RTCP_SOURCE_CAPTURE_CLOCK_OFFSET_UPDATER_H_ diff --git a/modules/rtp_rtcp/source/capture_clock_offset_updater_unittest.cc b/modules/rtp_rtcp/source/capture_clock_offset_updater_unittest.cc new file mode 100644 index 0000000000..43e1dd1379 --- /dev/null +++ b/modules/rtp_rtcp/source/capture_clock_offset_updater_unittest.cc @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/rtp_rtcp/source/capture_clock_offset_updater.h" + +#include "system_wrappers/include/ntp_time.h" +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { + +TEST(AbsoluteCaptureTimeReceiverTest, + SkipEstimatedCaptureClockOffsetIfRemoteToLocalClockOffsetIsUnknown) { + static const absl::optional kRemoteCaptureClockOffset = + Int64MsToQ32x32(-350); + CaptureClockOffsetUpdater updater; + updater.SetRemoteToLocalClockOffset(absl::nullopt); + EXPECT_EQ( + updater.AdjustEstimatedCaptureClockOffset(kRemoteCaptureClockOffset), + absl::nullopt); +} + +TEST(AbsoluteCaptureTimeReceiverTest, + SkipEstimatedCaptureClockOffsetIfRemoteCaptureClockOffsetIsUnknown) { + static const absl::optional kCaptureClockOffsetNull = absl::nullopt; + CaptureClockOffsetUpdater updater; + updater.SetRemoteToLocalClockOffset(0); + EXPECT_EQ(updater.AdjustEstimatedCaptureClockOffset(kCaptureClockOffsetNull), + kCaptureClockOffsetNull); + + static const absl::optional kRemoteCaptureClockOffset = + Int64MsToQ32x32(-350); + EXPECT_EQ( + updater.AdjustEstimatedCaptureClockOffset(kRemoteCaptureClockOffset), + kRemoteCaptureClockOffset); +} + +TEST(AbsoluteCaptureTimeReceiverTest, EstimatedCaptureClockOffsetArithmetic) { + static const absl::optional kRemoteCaptureClockOffset = + Int64MsToQ32x32(-350); + static const absl::optional kRemoteToLocalClockOffset = + Int64MsToQ32x32(-7000007); + CaptureClockOffsetUpdater updater; + updater.SetRemoteToLocalClockOffset(kRemoteToLocalClockOffset); + EXPECT_THAT( + updater.AdjustEstimatedCaptureClockOffset(kRemoteCaptureClockOffset), + ::testing::Optional(::testing::Eq(*kRemoteCaptureClockOffset + + *kRemoteToLocalClockOffset))); +} + +} // namespace webrtc diff --git a/video/rtp_video_stream_receiver.cc b/video/rtp_video_stream_receiver.cc index 5c877c5d87..7286a3ba4f 100644 --- a/video/rtp_video_stream_receiver.cc +++ b/video/rtp_video_stream_receiver.cc @@ -274,7 +274,7 @@ RtpVideoStreamReceiver::RtpVideoStreamReceiver( packet_buffer_(kPacketBufferStartSize, PacketBufferMaxSize()), has_received_frame_(false), frames_decryptable_(false), - absolute_capture_time_receiver_(clock) { + absolute_capture_time_interpolator_(clock) { constexpr bool remb_candidate = true; if (packet_router_) packet_router_->AddReceiveRtpModule(rtp_rtcp_.get(), remb_candidate); @@ -655,9 +655,9 @@ void RtpVideoStreamReceiver::OnReceivedPayloadData( // Try to extrapolate absolute capture time if it is missing. packet_info.set_absolute_capture_time( - absolute_capture_time_receiver_.OnReceivePacket( - AbsoluteCaptureTimeReceiver::GetSource(packet_info.ssrc(), - packet_info.csrcs()), + absolute_capture_time_interpolator_.OnReceivePacket( + AbsoluteCaptureTimeInterpolator::GetSource(packet_info.ssrc(), + packet_info.csrcs()), packet_info.rtp_timestamp(), // Assume frequency is the same one for all video frames. kVideoPayloadTypeFrequency, packet_info.absolute_capture_time())); @@ -1114,7 +1114,7 @@ bool RtpVideoStreamReceiver::DeliverRtcp(const uint8_t* rtcp_packet, absl::optional remote_to_local_clock_offset_ms = ntp_estimator_.EstimateRemoteToLocalClockOffsetMs(); if (remote_to_local_clock_offset_ms.has_value()) { - absolute_capture_time_receiver_.SetRemoteToLocalClockOffset( + capture_clock_offset_updater_.SetRemoteToLocalClockOffset( Int64MsToQ32x32(*remote_to_local_clock_offset_ms)); } } diff --git a/video/rtp_video_stream_receiver.h b/video/rtp_video_stream_receiver.h index aa86a0c29a..a302e1e77f 100644 --- a/video/rtp_video_stream_receiver.h +++ b/video/rtp_video_stream_receiver.h @@ -33,7 +33,8 @@ #include "modules/rtp_rtcp/include/rtp_header_extension_map.h" #include "modules/rtp_rtcp/include/rtp_rtcp.h" #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" -#include "modules/rtp_rtcp/source/absolute_capture_time_receiver.h" +#include "modules/rtp_rtcp/source/absolute_capture_time_interpolator.h" +#include "modules/rtp_rtcp/source/capture_clock_offset_updater.h" #include "modules/rtp_rtcp/source/rtp_dependency_descriptor_extension.h" #include "modules/rtp_rtcp/source/rtp_packet_received.h" #include "modules/rtp_rtcp/source/rtp_video_header.h" @@ -401,7 +402,10 @@ class RtpVideoStreamReceiver : public LossNotificationSender, std::atomic frames_decryptable_; absl::optional last_color_space_; - AbsoluteCaptureTimeReceiver absolute_capture_time_receiver_ + AbsoluteCaptureTimeInterpolator absolute_capture_time_interpolator_ + RTC_GUARDED_BY(worker_task_checker_); + + CaptureClockOffsetUpdater capture_clock_offset_updater_ RTC_GUARDED_BY(worker_task_checker_); int64_t last_completed_picture_id_ = 0; diff --git a/video/rtp_video_stream_receiver2.cc b/video/rtp_video_stream_receiver2.cc index 40fe8fed88..a45dbeb2ff 100644 --- a/video/rtp_video_stream_receiver2.cc +++ b/video/rtp_video_stream_receiver2.cc @@ -253,7 +253,7 @@ RtpVideoStreamReceiver2::RtpVideoStreamReceiver2( packet_buffer_(kPacketBufferStartSize, PacketBufferMaxSize()), has_received_frame_(false), frames_decryptable_(false), - absolute_capture_time_receiver_(clock) { + absolute_capture_time_interpolator_(clock) { constexpr bool remb_candidate = true; if (packet_router_) packet_router_->AddReceiveRtpModule(rtp_rtcp_.get(), remb_candidate); @@ -492,9 +492,9 @@ void RtpVideoStreamReceiver2::OnReceivedPayloadData( // Try to extrapolate absolute capture time if it is missing. packet_info.set_absolute_capture_time( - absolute_capture_time_receiver_.OnReceivePacket( - AbsoluteCaptureTimeReceiver::GetSource(packet_info.ssrc(), - packet_info.csrcs()), + absolute_capture_time_interpolator_.OnReceivePacket( + AbsoluteCaptureTimeInterpolator::GetSource(packet_info.ssrc(), + packet_info.csrcs()), packet_info.rtp_timestamp(), // Assume frequency is the same one for all video frames. kVideoPayloadTypeFrequency, packet_info.absolute_capture_time())); @@ -1036,7 +1036,7 @@ bool RtpVideoStreamReceiver2::DeliverRtcp(const uint8_t* rtcp_packet, absl::optional remote_to_local_clock_offset_ms = ntp_estimator_.EstimateRemoteToLocalClockOffsetMs(); if (remote_to_local_clock_offset_ms.has_value()) { - absolute_capture_time_receiver_.SetRemoteToLocalClockOffset( + capture_clock_offset_updater_.SetRemoteToLocalClockOffset( Int64MsToQ32x32(*remote_to_local_clock_offset_ms)); } } diff --git a/video/rtp_video_stream_receiver2.h b/video/rtp_video_stream_receiver2.h index a524f4b577..f8d5a26b65 100644 --- a/video/rtp_video_stream_receiver2.h +++ b/video/rtp_video_stream_receiver2.h @@ -29,7 +29,8 @@ #include "modules/rtp_rtcp/include/remote_ntp_time_estimator.h" #include "modules/rtp_rtcp/include/rtp_header_extension_map.h" #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" -#include "modules/rtp_rtcp/source/absolute_capture_time_receiver.h" +#include "modules/rtp_rtcp/source/absolute_capture_time_interpolator.h" +#include "modules/rtp_rtcp/source/capture_clock_offset_updater.h" #include "modules/rtp_rtcp/source/rtp_dependency_descriptor_extension.h" #include "modules/rtp_rtcp/source/rtp_packet_received.h" #include "modules/rtp_rtcp/source/rtp_rtcp_impl2.h" @@ -350,7 +351,10 @@ class RtpVideoStreamReceiver2 : public LossNotificationSender, bool frames_decryptable_ RTC_GUARDED_BY(worker_task_checker_); absl::optional last_color_space_; - AbsoluteCaptureTimeReceiver absolute_capture_time_receiver_ + AbsoluteCaptureTimeInterpolator absolute_capture_time_interpolator_ + RTC_GUARDED_BY(worker_task_checker_); + + CaptureClockOffsetUpdater capture_clock_offset_updater_ RTC_GUARDED_BY(worker_task_checker_); int64_t last_completed_picture_id_ = 0; diff --git a/video/video_receive_stream.cc b/video/video_receive_stream.cc index fc2364328b..dbdba388aa 100644 --- a/video/video_receive_stream.cc +++ b/video/video_receive_stream.cc @@ -506,6 +506,10 @@ void VideoReceiveStream::OnFrame(const VideoFrame& video_frame) { int64_t video_playout_ntp_ms; int64_t sync_offset_ms; double estimated_freq_khz; + + // TODO(bugs.webrtc.org/10739): we should set local capture clock offset for + // |video_frame.packet_infos|. But VideoFrame is const qualified here. + // TODO(tommi): GetStreamSyncOffsetInMs grabs three locks. One inside the // function itself, another in GetChannel() and a third in // GetPlayoutTimestamp. Seems excessive. Anyhow, I'm assuming the function diff --git a/video/video_receive_stream2.cc b/video/video_receive_stream2.cc index 2c7461ad0d..c9ec9e0123 100644 --- a/video/video_receive_stream2.cc +++ b/video/video_receive_stream2.cc @@ -526,6 +526,9 @@ int VideoReceiveStream2::GetBaseMinimumPlayoutDelayMs() const { void VideoReceiveStream2::OnFrame(const VideoFrame& video_frame) { VideoFrameMetaData frame_meta(video_frame, clock_->CurrentTime()); + // TODO(bugs.webrtc.org/10739): we should set local capture clock offset for + // |video_frame.packet_infos|. But VideoFrame is const qualified here. + worker_thread_->PostTask( ToQueuedTask(task_safety_, [frame_meta, this]() { RTC_DCHECK_RUN_ON(&worker_sequence_checker_);