diff --git a/api/media_stream_interface.cc b/api/media_stream_interface.cc index b55a8403c6..cf994cbeaa 100644 --- a/api/media_stream_interface.cc +++ b/api/media_stream_interface.cc @@ -32,7 +32,7 @@ const cricket::AudioOptions AudioSourceInterface::options() const { return {}; } -double AudioSourceInterface::GetLatency() const { +double MediaSourceInterface::GetLatency() const { return 0.0; } diff --git a/api/media_stream_interface.h b/api/media_stream_interface.h index e5203611ad..1249b85c96 100644 --- a/api/media_stream_interface.h +++ b/api/media_stream_interface.h @@ -61,6 +61,13 @@ class MediaSourceInterface : public rtc::RefCountInterface, virtual bool remote() const = 0; + // Sets the minimum latency of the remote source until audio playout. Actual + // observered latency may differ depending on the source. |latency| is in the + // range of [0.0, 10.0] seconds. + // TODO(kuddai) make pure virtual once not only remote tracks support latency. + virtual void SetLatency(double latency) {} + virtual double GetLatency() const; + protected: ~MediaSourceInterface() override = default; }; @@ -201,12 +208,6 @@ class AudioSourceInterface : public MediaSourceInterface { // be applied in the track in a way that does not affect clones of the track. virtual void SetVolume(double volume) {} - // Sets the minimum latency of the remote source until audio playout. Actual - // observered latency may differ depending on the source. |latency| is in the - // range of [0.0, 10.0] seconds. - virtual void SetLatency(double latency) {} - virtual double GetLatency() const; - // Registers/unregisters observers to the audio source. virtual void RegisterAudioObserver(AudioObserver* observer) {} virtual void UnregisterAudioObserver(AudioObserver* observer) {} diff --git a/api/video_track_source_proxy.h b/api/video_track_source_proxy.h index 820cdcb286..eb11befa32 100644 --- a/api/video_track_source_proxy.h +++ b/api/video_track_source_proxy.h @@ -32,6 +32,8 @@ PROXY_WORKER_METHOD2(void, rtc::VideoSinkInterface*, const rtc::VideoSinkWants&) PROXY_WORKER_METHOD1(void, RemoveSink, rtc::VideoSinkInterface*) +PROXY_WORKER_METHOD1(void, SetLatency, double) +PROXY_WORKER_CONSTMETHOD0(double, GetLatency) PROXY_METHOD1(void, RegisterObserver, ObserverInterface*) PROXY_METHOD1(void, UnregisterObserver, ObserverInterface*) END_PROXY_MAP() diff --git a/call/video_receive_stream.h b/call/video_receive_stream.h index ecff63d62b..0bcb6d3010 100644 --- a/call/video_receive_stream.h +++ b/call/video_receive_stream.h @@ -252,6 +252,15 @@ class VideoReceiveStream { virtual std::vector GetSources() const = 0; + // Sets a base minimum for the playout delay. Base minimum delay sets lower + // bound on minimum delay value determining lower bound on playout delay. + // + // Returns true if value was successfully set, false overwise. + virtual bool SetBaseMinimumPlayoutDelayMs(int delay_ms) = 0; + + // Returns current value of base minimum delay in milliseconds. + virtual int GetBaseMinimumPlayoutDelayMs() const = 0; + protected: virtual ~VideoReceiveStream() {} }; diff --git a/media/BUILD.gn b/media/BUILD.gn index b3195a1817..ccbfd38506 100644 --- a/media/BUILD.gn +++ b/media/BUILD.gn @@ -88,6 +88,7 @@ rtc_static_library("rtc_media_base") { "base/audio_source.h", "base/codec.cc", "base/codec.h", + "base/delayable.h", "base/media_channel.cc", "base/media_channel.h", "base/media_constants.cc", diff --git a/media/base/delayable.h b/media/base/delayable.h new file mode 100644 index 0000000000..90ce5d7089 --- /dev/null +++ b/media/base/delayable.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MEDIA_BASE_DELAYABLE_H_ +#define MEDIA_BASE_DELAYABLE_H_ + +#include + +#include "absl/types/optional.h" + +namespace cricket { + +// Delayable is used by user code through ApplyConstraints algorithm. Its +// methods must take precendence over similar functional in |syncable.h|. +class Delayable { + public: + virtual ~Delayable() {} + // Set base minimum delay of the receive stream with specified ssrc. + // Base minimum delay sets lower bound on minimum delay value which + // determines minimum delay until audio playout. + // Returns false if there is no stream with given ssrc. + virtual bool SetBaseMinimumPlayoutDelayMs(uint32_t ssrc, int delay_ms) = 0; + + // Returns current value of base minimum delay in milliseconds. + virtual absl::optional GetBaseMinimumPlayoutDelayMs( + uint32_t ssrc) const = 0; +}; + +} // namespace cricket + +#endif // MEDIA_BASE_DELAYABLE_H_ diff --git a/media/base/fake_media_engine.cc b/media/base/fake_media_engine.cc index 829790278a..e4e94caa7a 100644 --- a/media/base/fake_media_engine.cc +++ b/media/base/fake_media_engine.cc @@ -341,12 +341,14 @@ bool FakeVideoMediaChannel::AddRecvStream(const StreamParams& sp) { if (!RtpHelper::AddRecvStream(sp)) return false; sinks_[sp.first_ssrc()] = NULL; + output_delays_[sp.first_ssrc()] = 0; return true; } bool FakeVideoMediaChannel::RemoveRecvStream(uint32_t ssrc) { if (!RtpHelper::RemoveRecvStream(ssrc)) return false; sinks_.erase(ssrc); + output_delays_.erase(ssrc); return true; } void FakeVideoMediaChannel::FillBitrateInfo(BandwidthEstimationInfo* bwe_info) { @@ -358,6 +360,23 @@ std::vector FakeVideoMediaChannel::GetSources( uint32_t ssrc) const { return {}; } +bool FakeVideoMediaChannel::SetBaseMinimumPlayoutDelayMs(uint32_t ssrc, + int delay_ms) { + if (output_delays_.find(ssrc) == output_delays_.end()) { + return false; + } else { + output_delays_[ssrc] = delay_ms; + return true; + } +} +absl::optional FakeVideoMediaChannel::GetBaseMinimumPlayoutDelayMs( + uint32_t ssrc) const { + const auto it = output_delays_.find(ssrc); + if (it != output_delays_.end()) { + return it->second; + } + return absl::nullopt; +} bool FakeVideoMediaChannel::SetRecvCodecs( const std::vector& codecs) { if (fail_set_recv_codecs()) { diff --git a/media/base/fake_media_engine.h b/media/base/fake_media_engine.h index f5865806f0..3f1c28ba2c 100644 --- a/media/base/fake_media_engine.h +++ b/media/base/fake_media_engine.h @@ -441,6 +441,10 @@ class FakeVideoMediaChannel : public RtpHelper { std::vector GetSources(uint32_t ssrc) const override; + bool SetBaseMinimumPlayoutDelayMs(uint32_t ssrc, int delay_ms) override; + absl::optional GetBaseMinimumPlayoutDelayMs( + uint32_t ssrc) const override; + private: bool SetRecvCodecs(const std::vector& codecs); bool SetSendCodecs(const std::vector& codecs); @@ -452,6 +456,7 @@ class FakeVideoMediaChannel : public RtpHelper { std::vector send_codecs_; std::map*> sinks_; std::map*> sources_; + std::map output_delays_; VideoOptions options_; int max_bps_; }; diff --git a/media/base/media_channel.h b/media/base/media_channel.h index 2a08c68693..4106949605 100644 --- a/media/base/media_channel.h +++ b/media/base/media_channel.h @@ -32,6 +32,7 @@ #include "api/video/video_timing.h" #include "api/video_codecs/video_encoder_config.h" #include "media/base/codec.h" +#include "media/base/delayable.h" #include "media/base/media_config.h" #include "media/base/media_constants.h" #include "media/base/stream_params.h" @@ -707,7 +708,7 @@ struct AudioSendParameters : RtpSendParameters { struct AudioRecvParameters : RtpParameters {}; -class VoiceMediaChannel : public MediaChannel { +class VoiceMediaChannel : public MediaChannel, public Delayable { public: VoiceMediaChannel() {} explicit VoiceMediaChannel(const MediaConfig& config) @@ -738,13 +739,6 @@ class VoiceMediaChannel : public MediaChannel { AudioSource* source) = 0; // Set speaker output volume of the specified ssrc. virtual bool SetOutputVolume(uint32_t ssrc, double volume) = 0; - // Set base minimum delay of the receive stream with specified ssrc. - // Base minimum delay sets lower bound on minimum delay value which - // determines minimum delay until audio playout. - // Returns false if there is no stream with given ssrc. - virtual bool SetBaseMinimumPlayoutDelayMs(uint32_t ssrc, int delay_ms) = 0; - virtual absl::optional GetBaseMinimumPlayoutDelayMs( - uint32_t ssrc) const = 0; // Returns if the telephone-event has been negotiated. virtual bool CanInsertDtmf() = 0; // Send a DTMF |event|. The DTMF out-of-band signal will be used. @@ -783,7 +777,7 @@ struct VideoSendParameters : RtpSendParameters { // encapsulate all the parameters needed for a video RtpReceiver. struct VideoRecvParameters : RtpParameters {}; -class VideoMediaChannel : public MediaChannel { +class VideoMediaChannel : public MediaChannel, public Delayable { public: VideoMediaChannel() {} explicit VideoMediaChannel(const MediaConfig& config) diff --git a/media/engine/fake_webrtc_call.cc b/media/engine/fake_webrtc_call.cc index 662c6d2a9c..9ad99dea1d 100644 --- a/media/engine/fake_webrtc_call.cc +++ b/media/engine/fake_webrtc_call.cc @@ -410,6 +410,15 @@ const std::vector& FakeCall::GetVideoReceiveStreams() { return video_receive_streams_; } +const FakeVideoReceiveStream* FakeCall::GetVideoReceiveStream(uint32_t ssrc) { + for (const auto* p : GetVideoReceiveStreams()) { + if (p->GetConfig().rtp.remote_ssrc == ssrc) { + return p; + } + } + return nullptr; +} + const std::vector& FakeCall::GetAudioSendStreams() { return audio_send_streams_; } diff --git a/media/engine/fake_webrtc_call.h b/media/engine/fake_webrtc_call.h index 7df6b52919..71a54d66bf 100644 --- a/media/engine/fake_webrtc_call.h +++ b/media/engine/fake_webrtc_call.h @@ -222,6 +222,10 @@ class FakeVideoReceiveStream final : public webrtc::VideoReceiveStream { return std::vector(); } + int base_mininum_playout_delay_ms() const { + return base_mininum_playout_delay_ms_; + } + private: // webrtc::VideoReceiveStream implementation. void Start() override; @@ -229,10 +233,21 @@ class FakeVideoReceiveStream final : public webrtc::VideoReceiveStream { webrtc::VideoReceiveStream::Stats GetStats() const override; + bool SetBaseMinimumPlayoutDelayMs(int delay_ms) override { + base_mininum_playout_delay_ms_ = delay_ms; + return true; + } + + int GetBaseMinimumPlayoutDelayMs() const override { + return base_mininum_playout_delay_ms_; + } + webrtc::VideoReceiveStream::Config config_; bool receiving_; webrtc::VideoReceiveStream::Stats stats_; + int base_mininum_playout_delay_ms_ = 0; + int num_added_secondary_sinks_; int num_removed_secondary_sinks_; }; @@ -268,6 +283,7 @@ class FakeCall final : public webrtc::Call, public webrtc::PacketReceiver { const FakeAudioSendStream* GetAudioSendStream(uint32_t ssrc); const std::vector& GetAudioReceiveStreams(); const FakeAudioReceiveStream* GetAudioReceiveStream(uint32_t ssrc); + const FakeVideoReceiveStream* GetVideoReceiveStream(uint32_t ssrc); const std::vector& GetFlexfecReceiveStreams(); diff --git a/media/engine/webrtc_video_engine.cc b/media/engine/webrtc_video_engine.cc index 34489c07c4..144ebd7543 100644 --- a/media/engine/webrtc_video_engine.cc +++ b/media/engine/webrtc_video_engine.cc @@ -418,10 +418,17 @@ UnsignalledSsrcHandler::Action DefaultUnsignalledSsrcHandler::OnUnsignalledSsrc( RTC_LOG(LS_INFO) << "Creating default receive stream for SSRC=" << ssrc << "."; - if (!channel->AddRecvStream(sp, true)) { + if (!channel->AddRecvStream(sp, /*default_stream=*/true)) { RTC_LOG(LS_WARNING) << "Could not create default receive stream."; } + // SSRC 0 returns default_recv_base_minimum_delay_ms. + const int unsignaled_ssrc = 0; + int default_recv_base_minimum_delay_ms = + channel->GetBaseMinimumPlayoutDelayMs(unsignaled_ssrc).value_or(0); + // Set base minimum delay if it was set before for the default receive stream. + channel->SetBaseMinimumPlayoutDelayMs(ssrc, + default_recv_base_minimum_delay_ms); channel->SetSink(ssrc, default_sink_); return kDeliverPacket; } @@ -1497,6 +1504,51 @@ void WebRtcVideoChannel::SetFrameEncryptor( } } +bool WebRtcVideoChannel::SetBaseMinimumPlayoutDelayMs(uint32_t ssrc, + int delay_ms) { + absl::optional default_ssrc = GetDefaultReceiveStreamSsrc(); + rtc::CritScope stream_lock(&stream_crit_); + + // SSRC of 0 represents the default receive stream. + if (ssrc == 0) { + default_recv_base_minimum_delay_ms_ = delay_ms; + } + + if (ssrc == 0 && !default_ssrc) { + return true; + } + + if (ssrc == 0 && default_ssrc) { + ssrc = default_ssrc.value(); + } + + auto stream = receive_streams_.find(ssrc); + if (stream != receive_streams_.end()) { + stream->second->SetBaseMinimumPlayoutDelayMs(delay_ms); + return true; + } else { + RTC_LOG(LS_ERROR) << "No stream found to set base minimum playout delay"; + return false; + } +} + +absl::optional WebRtcVideoChannel::GetBaseMinimumPlayoutDelayMs( + uint32_t ssrc) const { + rtc::CritScope stream_lock(&stream_crit_); + // SSRC of 0 represents the default receive stream. + if (ssrc == 0) { + return default_recv_base_minimum_delay_ms_; + } + + auto stream = receive_streams_.find(ssrc); + if (stream != receive_streams_.end()) { + return stream->second->GetBaseMinimumPlayoutDelayMs(); + } else { + RTC_LOG(LS_ERROR) << "No stream found to get base minimum playout delay"; + return absl::nullopt; + } +} + absl::optional WebRtcVideoChannel::GetDefaultReceiveStreamSsrc() { rtc::CritScope stream_lock(&stream_crit_); absl::optional ssrc; @@ -2386,7 +2438,9 @@ void WebRtcVideoChannel::WebRtcVideoReceiveStream::SetRecvParameters( } void WebRtcVideoChannel::WebRtcVideoReceiveStream::RecreateWebRtcVideoStream() { + absl::optional base_minimum_playout_delay_ms; if (stream_) { + base_minimum_playout_delay_ms = stream_->GetBaseMinimumPlayoutDelayMs(); MaybeDissociateFlexfecFromVideo(); call_->DestroyVideoReceiveStream(stream_); stream_ = nullptr; @@ -2395,6 +2449,10 @@ void WebRtcVideoChannel::WebRtcVideoReceiveStream::RecreateWebRtcVideoStream() { config.rtp.protected_by_flexfec = (flexfec_stream_ != nullptr); config.stream_id = stream_params_.id; stream_ = call_->CreateVideoReceiveStream(std::move(config)); + if (base_minimum_playout_delay_ms) { + stream_->SetBaseMinimumPlayoutDelayMs( + base_minimum_playout_delay_ms.value()); + } MaybeAssociateFlexfecWithVideo(); stream_->Start(); } @@ -2457,6 +2515,16 @@ void WebRtcVideoChannel::WebRtcVideoReceiveStream::SetFrameDecryptor( } } +bool WebRtcVideoChannel::WebRtcVideoReceiveStream::SetBaseMinimumPlayoutDelayMs( + int delay_ms) { + return stream_ ? stream_->SetBaseMinimumPlayoutDelayMs(delay_ms) : false; +} + +int WebRtcVideoChannel::WebRtcVideoReceiveStream::GetBaseMinimumPlayoutDelayMs() + const { + return stream_ ? stream_->GetBaseMinimumPlayoutDelayMs() : 0; +} + void WebRtcVideoChannel::WebRtcVideoReceiveStream::SetSink( rtc::VideoSinkInterface* sink) { rtc::CritScope crit(&sink_lock_); diff --git a/media/engine/webrtc_video_engine.h b/media/engine/webrtc_video_engine.h index a3d5a2f42b..6604ab60d5 100644 --- a/media/engine/webrtc_video_engine.h +++ b/media/engine/webrtc_video_engine.h @@ -170,6 +170,11 @@ class WebRtcVideoChannel : public VideoMediaChannel, public webrtc::Transport { rtc::scoped_refptr frame_encryptor) override; + bool SetBaseMinimumPlayoutDelayMs(uint32_t ssrc, int delay_ms) override; + + absl::optional GetBaseMinimumPlayoutDelayMs( + uint32_t ssrc) const override; + // Implemented for VideoMediaChannelTest. bool sending() const { return sending_; } @@ -393,6 +398,10 @@ class WebRtcVideoChannel : public VideoMediaChannel, public webrtc::Transport { void SetFrameDecryptor( rtc::scoped_refptr frame_decryptor); + bool SetBaseMinimumPlayoutDelayMs(int delay_ms); + + int GetBaseMinimumPlayoutDelayMs() const; + void SetSink(rtc::VideoSinkInterface* sink); VideoReceiverInfo GetVideoReceiverInfo(bool log_stats); @@ -470,6 +479,9 @@ class WebRtcVideoChannel : public VideoMediaChannel, public webrtc::Transport { DefaultUnsignalledSsrcHandler default_unsignalled_ssrc_handler_; UnsignalledSsrcHandler* const unsignalled_ssrc_handler_; + // Delay for unsignaled streams, which may be set before the stream exists. + int default_recv_base_minimum_delay_ms_ = 0; + const MediaConfig::Video video_config_; rtc::CriticalSection stream_crit_; diff --git a/media/engine/webrtc_video_engine_unittest.cc b/media/engine/webrtc_video_engine_unittest.cc index 493c18c40e..3b14f27e38 100644 --- a/media/engine/webrtc_video_engine_unittest.cc +++ b/media/engine/webrtc_video_engine_unittest.cc @@ -5012,6 +5012,56 @@ TEST_F(WebRtcVideoChannelTest, RecvUnsignaledSsrcWithSignaledStreamId) { fake_call_->GetVideoReceiveStreams()[0]->GetConfig().sync_group.empty()); } +// Test BaseMinimumPlayoutDelayMs on receive streams. +TEST_F(WebRtcVideoChannelTest, BaseMinimumPlayoutDelayMs) { + // Test that set won't work for non-existing receive streams. + EXPECT_FALSE(channel_->SetBaseMinimumPlayoutDelayMs(kSsrc + 2, 200)); + // Test that get won't work for non-existing receive streams. + EXPECT_FALSE(channel_->GetBaseMinimumPlayoutDelayMs(kSsrc + 2)); + + EXPECT_TRUE(AddRecvStream()); + // Test that set works for the existing receive stream. + EXPECT_TRUE(channel_->SetBaseMinimumPlayoutDelayMs(last_ssrc_, 200)); + auto* recv_stream = fake_call_->GetVideoReceiveStream(last_ssrc_); + EXPECT_TRUE(recv_stream); + EXPECT_EQ(recv_stream->base_mininum_playout_delay_ms(), 200); + EXPECT_EQ(channel_->GetBaseMinimumPlayoutDelayMs(last_ssrc_).value_or(0), + 200); +} + +// Test BaseMinimumPlayoutDelayMs on unsignaled receive streams. +TEST_F(WebRtcVideoChannelTest, BaseMinimumPlayoutDelayMsUnsignaledRecvStream) { + absl::optional delay_ms; + const FakeVideoReceiveStream* recv_stream; + + // Set default stream with SSRC 0 + EXPECT_TRUE(channel_->SetBaseMinimumPlayoutDelayMs(0, 200)); + EXPECT_EQ(200, channel_->GetBaseMinimumPlayoutDelayMs(0).value_or(0)); + + // Spawn an unsignaled stream by sending a packet, it should inherit + // default delay 200. + const size_t kDataLength = 12; + uint8_t data[kDataLength]; + memset(data, 0, sizeof(data)); + rtc::SetBE32(&data[8], kIncomingUnsignalledSsrc); + rtc::CopyOnWriteBuffer packet(data, kDataLength); + channel_->OnPacketReceived(&packet, /* packet_time_us */ -1); + + recv_stream = fake_call_->GetVideoReceiveStream(kIncomingUnsignalledSsrc); + EXPECT_EQ(recv_stream->base_mininum_playout_delay_ms(), 200); + delay_ms = channel_->GetBaseMinimumPlayoutDelayMs(kIncomingUnsignalledSsrc); + EXPECT_EQ(200, delay_ms.value_or(0)); + + // Check that now if we change delay for SSRC 0 it will change delay for the + // default receiving stream as well. + EXPECT_TRUE(channel_->SetBaseMinimumPlayoutDelayMs(0, 300)); + EXPECT_EQ(300, channel_->GetBaseMinimumPlayoutDelayMs(0).value_or(0)); + delay_ms = channel_->GetBaseMinimumPlayoutDelayMs(kIncomingUnsignalledSsrc); + EXPECT_EQ(300, delay_ms.value_or(0)); + recv_stream = fake_call_->GetVideoReceiveStream(kIncomingUnsignalledSsrc); + EXPECT_EQ(recv_stream->base_mininum_playout_delay_ms(), 300); +} + void WebRtcVideoChannelTest::TestReceiveUnsignaledSsrcPacket( uint8_t payload_type, bool expect_created_receive_stream) { diff --git a/modules/video_coding/frame_buffer2.cc b/modules/video_coding/frame_buffer2.cc index 640111726c..e4def869d2 100644 --- a/modules/video_coding/frame_buffer2.cc +++ b/modules/video_coding/frame_buffer2.cc @@ -347,19 +347,6 @@ bool FrameBuffer::ValidReferences(const EncodedFrame& frame) const { return true; } -void FrameBuffer::UpdatePlayoutDelays(const EncodedFrame& frame) { - TRACE_EVENT0("webrtc", "FrameBuffer::UpdatePlayoutDelays"); - PlayoutDelay playout_delay = frame.EncodedImage().playout_delay_; - if (playout_delay.min_ms >= 0) - timing_->set_min_playout_delay(playout_delay.min_ms); - - if (playout_delay.max_ms >= 0) - timing_->set_max_playout_delay(playout_delay.max_ms); - - if (!frame.delayed_by_retransmission()) - timing_->IncomingTimestamp(frame.Timestamp(), frame.ReceivedTime()); -} - int64_t FrameBuffer::InsertFrame(std::unique_ptr frame) { TRACE_EVENT0("webrtc", "FrameBuffer::InsertFrame"); RTC_DCHECK(frame); @@ -449,7 +436,9 @@ int64_t FrameBuffer::InsertFrame(std::unique_ptr frame) { if (!UpdateFrameInfoWithIncomingFrame(*frame, info)) return last_continuous_picture_id; - UpdatePlayoutDelays(*frame); + + if (!frame->delayed_by_retransmission()) + timing_->IncomingTimestamp(frame->Timestamp(), frame->ReceivedTime()); info->second.frame = std::move(frame); diff --git a/modules/video_coding/frame_buffer2.h b/modules/video_coding/frame_buffer2.h index c71c19fcf1..2421c368f2 100644 --- a/modules/video_coding/frame_buffer2.h +++ b/modules/video_coding/frame_buffer2.h @@ -118,11 +118,6 @@ class FrameBuffer { // Check that the references of |frame| are valid. bool ValidReferences(const EncodedFrame& frame) const; - // Updates the minimal and maximal playout delays - // depending on the frame. - void UpdatePlayoutDelays(const EncodedFrame& frame) - RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_); - // Update all directly dependent and indirectly dependent frames and mark // them as continuous if all their references has been fulfilled. void PropagateContinuity(FrameMap::iterator start) diff --git a/modules/video_coding/frame_buffer2_unittest.cc b/modules/video_coding/frame_buffer2_unittest.cc index 94a5551da5..7dc8cb29ed 100644 --- a/modules/video_coding/frame_buffer2_unittest.cc +++ b/modules/video_coding/frame_buffer2_unittest.cc @@ -271,16 +271,6 @@ TEST_F(TestFrameBuffer2, OneSuperFrame) { CheckFrame(0, pid, 1); } -TEST_F(TestFrameBuffer2, SetPlayoutDelay) { - const PlayoutDelay kPlayoutDelayMs = {123, 321}; - std::unique_ptr test_frame(new FrameObjectFake()); - test_frame->id.picture_id = 0; - test_frame->SetPlayoutDelay(kPlayoutDelayMs); - buffer_->InsertFrame(std::move(test_frame)); - EXPECT_EQ(kPlayoutDelayMs.min_ms, timing_.min_playout_delay()); - EXPECT_EQ(kPlayoutDelayMs.max_ms, timing_.max_playout_delay()); -} - TEST_F(TestFrameBuffer2, ZeroPlayoutDelay) { VCMTiming timing(&clock_); buffer_.reset( diff --git a/pc/BUILD.gn b/pc/BUILD.gn index 59fe940bed..5c3fb0ef27 100644 --- a/pc/BUILD.gn +++ b/pc/BUILD.gn @@ -151,6 +151,10 @@ rtc_static_library("peerconnection") { "peer_connection_factory.cc", "peer_connection_factory.h", "peer_connection_internal.h", + "playout_latency.cc", + "playout_latency.h", + "playout_latency_interface.h", + "playout_latency_proxy.h", "remote_audio_source.cc", "remote_audio_source.h", "rtc_stats_collector.cc", @@ -396,6 +400,7 @@ if (rtc_include_tests) { "test/frame_generator_capturer_video_track_source.h", "test/mock_channel_interface.h", "test/mock_data_channel.h", + "test/mock_delayable.h", "test/mock_peer_connection_observers.h", "test/mock_rtp_receiver_internal.h", "test/mock_rtp_sender_internal.h", @@ -476,6 +481,7 @@ if (rtc_include_tests) { "peer_connection_simulcast_unittest.cc", "peer_connection_wrapper.cc", "peer_connection_wrapper.h", + "playout_latency_unittest.cc", "proxy_unittest.cc", "rtc_stats_collector_unittest.cc", "rtc_stats_integrationtest.cc", diff --git a/pc/playout_latency.cc b/pc/playout_latency.cc new file mode 100644 index 0000000000..be78439a90 --- /dev/null +++ b/pc/playout_latency.cc @@ -0,0 +1,87 @@ +/* + * Copyright 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "pc/playout_latency.h" + +#include "iostream" + +#include "rtc_base/checks.h" +#include "rtc_base/location.h" +#include "rtc_base/logging.h" +#include "rtc_base/numerics/safe_conversions.h" +#include "rtc_base/thread.h" +#include "rtc_base/thread_checker.h" + +namespace { +constexpr int kDefaultLatency = 0; +constexpr int kRoundToZeroThresholdMs = 10; +} // namespace + +namespace webrtc { + +PlayoutLatency::PlayoutLatency(rtc::Thread* worker_thread) + : signaling_thread_(rtc::Thread::Current()), worker_thread_(worker_thread) { + RTC_DCHECK(worker_thread_); +} + +void PlayoutLatency::OnStart(cricket::Delayable* media_channel, uint32_t ssrc) { + RTC_DCHECK_RUN_ON(signaling_thread_); + + media_channel_ = media_channel; + ssrc_ = ssrc; + + // Trying to apply cached latency for the audio stream. + if (cached_latency_) { + SetLatency(cached_latency_.value()); + } +} + +void PlayoutLatency::OnStop() { + RTC_DCHECK_RUN_ON(signaling_thread_); + // Assume that audio stream is no longer present for latency calls. + media_channel_ = nullptr; + ssrc_ = absl::nullopt; +} + +void PlayoutLatency::SetLatency(double latency) { + RTC_DCHECK_RUN_ON(worker_thread_); + RTC_DCHECK_GE(latency, 0); + RTC_DCHECK_LE(latency, 10); + + int delay_ms = rtc::dchecked_cast(latency * 1000); + // In JitterBuffer 0 delay has special meaning of being unconstrained value + // that is why we round delay to 0 if it is small enough during conversion + // from latency. + if (delay_ms <= kRoundToZeroThresholdMs) { + delay_ms = 0; + } + + cached_latency_ = latency; + if (media_channel_ && ssrc_) { + media_channel_->SetBaseMinimumPlayoutDelayMs(ssrc_.value(), delay_ms); + } +} + +double PlayoutLatency::GetLatency() const { + RTC_DCHECK_RUN_ON(worker_thread_); + + absl::optional delay_ms; + if (media_channel_ && ssrc_) { + delay_ms = media_channel_->GetBaseMinimumPlayoutDelayMs(ssrc_.value()); + } + + if (delay_ms) { + return delay_ms.value() / 1000.0; + } else { + return cached_latency_.value_or(kDefaultLatency); + } +} + +} // namespace webrtc diff --git a/pc/playout_latency.h b/pc/playout_latency.h new file mode 100644 index 0000000000..f51a927f6c --- /dev/null +++ b/pc/playout_latency.h @@ -0,0 +1,53 @@ +/* + * Copyright 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef PC_PLAYOUT_LATENCY_H_ +#define PC_PLAYOUT_LATENCY_H_ + +#include + +#include "absl/types/optional.h" +#include "media/base/delayable.h" +#include "pc/playout_latency_interface.h" +#include "rtc_base/thread.h" + +namespace webrtc { + +// PlayoutLatency converts latency measured in seconds to delay measured in +// milliseconds for the underlying media channel. It also handles cases when +// user sets Latency before the start of media_channel by caching its request. +// Note, this class is not thread safe. Its thread safe version is defined in +// pc/playout_latency_proxy.h +class PlayoutLatency : public PlayoutLatencyInterface { + public: + // Must be called on signaling thread. + explicit PlayoutLatency(rtc::Thread* worker_thread); + + void OnStart(cricket::Delayable* media_channel, uint32_t ssrc) override; + + void OnStop() override; + + void SetLatency(double latency) override; + + double GetLatency() const override; + + private: + // Throughout webrtc source, sometimes it is also called as |main_thread_|. + rtc::Thread* const signaling_thread_; + rtc::Thread* const worker_thread_; + // Media channel and ssrc together uniqely identify audio stream. + cricket::Delayable* media_channel_ = nullptr; + absl::optional ssrc_; + absl::optional cached_latency_; +}; + +} // namespace webrtc + +#endif // PC_PLAYOUT_LATENCY_H_ diff --git a/pc/playout_latency_interface.h b/pc/playout_latency_interface.h new file mode 100644 index 0000000000..e9957337fe --- /dev/null +++ b/pc/playout_latency_interface.h @@ -0,0 +1,43 @@ +/* + * Copyright 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef PC_PLAYOUT_LATENCY_INTERFACE_H_ +#define PC_PLAYOUT_LATENCY_INTERFACE_H_ + +#include + +#include "media/base/delayable.h" +#include "rtc_base/ref_count.h" + +namespace webrtc { + +// PlayoutLatency delivers user's latency queries to the underlying media +// channel. It can describe either video or audio latency for receiving stream. +// "Interface" suffix in the interface name is required to be compatible with +// api/proxy.cc +class PlayoutLatencyInterface : public rtc::RefCountInterface { + public: + // OnStart allows to uniqely identify to which receiving stream playout + // latency must correpond through |media_channel| and |ssrc| pair. + virtual void OnStart(cricket::Delayable* media_channel, uint32_t ssrc) = 0; + + // Indicates that underlying receiving stream is stopped. + virtual void OnStop() = 0; + + // Sets latency in seconds. + virtual void SetLatency(double latency) = 0; + + // Returns latency in seconds. + virtual double GetLatency() const = 0; +}; + +} // namespace webrtc + +#endif // PC_PLAYOUT_LATENCY_INTERFACE_H_ diff --git a/pc/playout_latency_proxy.h b/pc/playout_latency_proxy.h new file mode 100644 index 0000000000..22f02c5622 --- /dev/null +++ b/pc/playout_latency_proxy.h @@ -0,0 +1,32 @@ +/* + * Copyright 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef PC_PLAYOUT_LATENCY_PROXY_H_ +#define PC_PLAYOUT_LATENCY_PROXY_H_ + +#include + +#include "api/proxy.h" +#include "media/base/delayable.h" +#include "pc/playout_latency_interface.h" + +namespace webrtc { + +BEGIN_PROXY_MAP(PlayoutLatency) +PROXY_SIGNALING_THREAD_DESTRUCTOR() +PROXY_METHOD2(void, OnStart, cricket::Delayable*, uint32_t) +PROXY_METHOD0(void, OnStop) +PROXY_WORKER_METHOD1(void, SetLatency, double) +PROXY_WORKER_CONSTMETHOD0(double, GetLatency) +END_PROXY_MAP() + +} // namespace webrtc + +#endif // PC_PLAYOUT_LATENCY_PROXY_H_ diff --git a/pc/playout_latency_unittest.cc b/pc/playout_latency_unittest.cc new file mode 100644 index 0000000000..0a671618c2 --- /dev/null +++ b/pc/playout_latency_unittest.cc @@ -0,0 +1,103 @@ +/* + * Copyright 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "absl/types/optional.h" +#include "api/scoped_refptr.h" +#include "pc/playout_latency.h" +#include "pc/test/mock_delayable.h" +#include "rtc_base/ref_counted_object.h" +#include "rtc_base/thread.h" +#include "test/gmock.h" +#include "test/gtest.h" + +using ::testing::Return; + +namespace { +constexpr int kSsrc = 1234; +} // namespace + +namespace webrtc { + +class PlayoutLatencyTest : public testing::Test { + public: + PlayoutLatencyTest() + : latency_( + new rtc::RefCountedObject(rtc::Thread::Current())) { + } + + protected: + rtc::scoped_refptr latency_; + MockDelayable delayable_; +}; + +TEST_F(PlayoutLatencyTest, DefaultValue) { + EXPECT_DOUBLE_EQ(0.0, latency_->GetLatency()); +} + +TEST_F(PlayoutLatencyTest, GetLatency) { + latency_->OnStart(&delayable_, kSsrc); + + EXPECT_CALL(delayable_, GetBaseMinimumPlayoutDelayMs(kSsrc)) + .WillOnce(Return(2000)); + // Latency in seconds. + EXPECT_DOUBLE_EQ(2.0, latency_->GetLatency()); + + EXPECT_CALL(delayable_, GetBaseMinimumPlayoutDelayMs(kSsrc)) + .WillOnce(Return(absl::nullopt)); + // When no value is returned by GetBaseMinimumPlayoutDelayMs, and there are + // no caching, then return default value. + EXPECT_DOUBLE_EQ(0.0, latency_->GetLatency()); +} + +TEST_F(PlayoutLatencyTest, SetLatency) { + latency_->OnStart(&delayable_, kSsrc); + + EXPECT_CALL(delayable_, SetBaseMinimumPlayoutDelayMs(kSsrc, 3000)) + .WillOnce(Return(true)); + + // Latency in seconds. + latency_->SetLatency(3.0); +} + +TEST_F(PlayoutLatencyTest, Caching) { + // Check that value is cached before start. + latency_->SetLatency(4.0); + // Latency in seconds. + EXPECT_DOUBLE_EQ(4.0, latency_->GetLatency()); + + // Check that cached value applied on the start. + EXPECT_CALL(delayable_, SetBaseMinimumPlayoutDelayMs(kSsrc, 4000)) + .WillOnce(Return(true)); + latency_->OnStart(&delayable_, kSsrc); + + EXPECT_CALL(delayable_, GetBaseMinimumPlayoutDelayMs(kSsrc)) + .WillOnce(Return(absl::nullopt)); + // On false the latest cached value is returned. + EXPECT_DOUBLE_EQ(4.0, latency_->GetLatency()); + + latency_->OnStop(); + + // Check that after stop it returns last cached value. + EXPECT_DOUBLE_EQ(4.0, latency_->GetLatency()); +} + +TEST_F(PlayoutLatencyTest, Rounding) { + latency_->OnStart(&delayable_, kSsrc); + // In Jitter Buffer (Audio or Video) delay 0 has a special meaning of + // unconstrained variable, that is why here if latency is small enough we + // round it to 0 delay. + EXPECT_CALL(delayable_, SetBaseMinimumPlayoutDelayMs(kSsrc, 0)) + .WillOnce(Return(true)); + latency_->SetLatency(0.005); +} + +} // namespace webrtc diff --git a/pc/remote_audio_source.cc b/pc/remote_audio_source.cc index 63944c6d83..aba44006bf 100644 --- a/pc/remote_audio_source.cc +++ b/pc/remote_audio_source.cc @@ -16,6 +16,8 @@ #include "absl/algorithm/container.h" #include "absl/memory/memory.h" #include "api/scoped_refptr.h" +#include "pc/playout_latency.h" +#include "pc/playout_latency_proxy.h" #include "rtc_base/checks.h" #include "rtc_base/constructor_magic.h" #include "rtc_base/location.h" @@ -26,11 +28,6 @@ namespace webrtc { -namespace { -constexpr int kDefaultLatency = 0; -constexpr int kRoundToZeroThresholdMs = 10; -} // namespace - // This proxy is passed to the underlying media engine to receive audio data as // they come in. The data will then be passed back up to the RemoteAudioSource // which will fan it out to all the sinks that have been added to it. @@ -55,7 +52,11 @@ class RemoteAudioSource::AudioDataProxy : public AudioSinkInterface { RemoteAudioSource::RemoteAudioSource(rtc::Thread* worker_thread) : main_thread_(rtc::Thread::Current()), worker_thread_(worker_thread), - state_(MediaSourceInterface::kLive) { + state_(MediaSourceInterface::kLive), + latency_(PlayoutLatencyProxy::Create( + main_thread_, + worker_thread_, + new rtc::RefCountedObject(worker_thread))) { RTC_DCHECK(main_thread_); RTC_DCHECK(worker_thread_); } @@ -70,12 +71,6 @@ void RemoteAudioSource::Start(cricket::VoiceMediaChannel* media_channel, uint32_t ssrc) { RTC_DCHECK_RUN_ON(main_thread_); RTC_DCHECK(media_channel); - // Check that there are no consecutive start calls. - RTC_DCHECK(!media_channel_ && !ssrc_); - - // Remember media channel ssrc pair for latency calls. - media_channel_ = media_channel; - ssrc_ = ssrc; // Register for callbacks immediately before AddSink so that we always get // notified when a channel goes out of scope (signaled when "AudioDataProxy" @@ -85,10 +80,8 @@ void RemoteAudioSource::Start(cricket::VoiceMediaChannel* media_channel, absl::make_unique(this)); }); - // Trying to apply cached latency for the audio stream. - if (cached_latency_) { - SetLatency(cached_latency_.value()); - } + // Apply latency to the audio stream if |SetLatency| was called before. + latency_->OnStart(media_channel, ssrc); } void RemoteAudioSource::Stop(cricket::VoiceMediaChannel* media_channel, @@ -96,9 +89,7 @@ void RemoteAudioSource::Stop(cricket::VoiceMediaChannel* media_channel, RTC_DCHECK_RUN_ON(main_thread_); RTC_DCHECK(media_channel); - // Assume that audio stream is no longer present for latency calls. - media_channel_ = nullptr; - ssrc_ = absl::nullopt; + latency_->OnStop(); worker_thread_->Invoke( RTC_FROM_HERE, [&] { media_channel->SetRawAudioSink(ssrc, nullptr); }); @@ -123,50 +114,11 @@ void RemoteAudioSource::SetVolume(double volume) { } void RemoteAudioSource::SetLatency(double latency) { - RTC_DCHECK_GE(latency, 0); - RTC_DCHECK_LE(latency, 10); - - int delay_ms = rtc::dchecked_cast(latency * 1000); - // In NetEq 0 delay has special meaning of being unconstrained value that is - // why we round delay to 0 if it is small enough during conversion from - // latency. - if (delay_ms <= kRoundToZeroThresholdMs) { - delay_ms = 0; - } - - cached_latency_ = latency; - SetDelayMs(delay_ms); + latency_->SetLatency(latency); } double RemoteAudioSource::GetLatency() const { - absl::optional delay_ms = GetDelayMs(); - - if (delay_ms) { - return delay_ms.value() / 1000.0; - } else { - return cached_latency_.value_or(kDefaultLatency); - } -} - -bool RemoteAudioSource::SetDelayMs(int delay_ms) { - if (!media_channel_ || !ssrc_) { - return false; - } - - worker_thread_->Invoke(RTC_FROM_HERE, [&] { - media_channel_->SetBaseMinimumPlayoutDelayMs(ssrc_.value(), delay_ms); - }); - return true; -} - -absl::optional RemoteAudioSource::GetDelayMs() const { - if (!media_channel_ || !ssrc_) { - return absl::nullopt; - } - - return worker_thread_->Invoke>(RTC_FROM_HERE, [&] { - return media_channel_->GetBaseMinimumPlayoutDelayMs(ssrc_.value()); - }); + return latency_->GetLatency(); } void RemoteAudioSource::RegisterAudioObserver(AudioObserver* observer) { diff --git a/pc/remote_audio_source.h b/pc/remote_audio_source.h index 0773c38dcf..f4a5ec8257 100644 --- a/pc/remote_audio_source.h +++ b/pc/remote_audio_source.h @@ -17,6 +17,7 @@ #include "api/call/audio_sink.h" #include "api/notifier.h" #include "pc/channel.h" +#include "pc/playout_latency_interface.h" #include "rtc_base/critical_section.h" #include "rtc_base/message_handler.h" @@ -65,19 +66,15 @@ class RemoteAudioSource : public Notifier, void OnMessage(rtc::Message* msg) override; - bool SetDelayMs(int delay_ms); - absl::optional GetDelayMs() const; - rtc::Thread* const main_thread_; rtc::Thread* const worker_thread_; std::list audio_observers_; rtc::CriticalSection sink_lock_; std::list sinks_; SourceState state_; - // Media channel and ssrc together uniqely identify audio stream. - cricket::VoiceMediaChannel* media_channel_ = nullptr; - absl::optional ssrc_; - absl::optional cached_latency_; + // Allows to thread safely change playout latency. Handles caching cases if + // |SetLatency| is called before start. + rtc::scoped_refptr latency_; }; } // namespace webrtc diff --git a/pc/rtp_sender_receiver_unittest.cc b/pc/rtp_sender_receiver_unittest.cc index d55e4f46f6..c8d84f685e 100644 --- a/pc/rtp_sender_receiver_unittest.cc +++ b/pc/rtp_sender_receiver_unittest.cc @@ -456,6 +456,41 @@ class RtpSenderReceiverTest RunSetLastLayerAsInactiveTest(video_rtp_sender_.get()); } + void VerifyTrackLatencyBehaviour(cricket::Delayable* media_channel, + MediaStreamTrackInterface* track, + MediaSourceInterface* source, + uint32_t ssrc) { + absl::optional delay_ms; // In milliseconds. + double latency_s = 0.5; // In seconds. + + source->SetLatency(latency_s); + delay_ms = media_channel->GetBaseMinimumPlayoutDelayMs(ssrc); + EXPECT_DOUBLE_EQ(latency_s, delay_ms.value_or(0) / 1000.0); + + // Disabling the track should take no effect on previously set value. + track->set_enabled(false); + delay_ms = media_channel->GetBaseMinimumPlayoutDelayMs(ssrc); + EXPECT_DOUBLE_EQ(latency_s, delay_ms.value_or(0) / 1000.0); + + // When the track is disabled, we still should be able to set latency. + latency_s = 0.3; + source->SetLatency(latency_s); + delay_ms = media_channel->GetBaseMinimumPlayoutDelayMs(ssrc); + EXPECT_DOUBLE_EQ(latency_s, delay_ms.value_or(0) / 1000.0); + + // Enabling the track should take no effect on previously set value. + track->set_enabled(true); + delay_ms = media_channel->GetBaseMinimumPlayoutDelayMs(ssrc); + EXPECT_DOUBLE_EQ(latency_s, delay_ms.value_or(0) / 1000.0); + + // We still should be able to change latency. + latency_s = 0.0; + source->SetLatency(latency_s); + delay_ms = media_channel->GetBaseMinimumPlayoutDelayMs(ssrc); + EXPECT_EQ(0, delay_ms.value_or(-1)); + EXPECT_DOUBLE_EQ(latency_s, delay_ms.value_or(0) / 1000.0); + } + protected: rtc::Thread* const network_thread_; rtc::Thread* const worker_thread_; @@ -636,101 +671,36 @@ TEST_F(RtpSenderReceiverTest, RemoteAudioTrackSetVolume) { DestroyAudioRtpReceiver(); } -TEST_F(RtpSenderReceiverTest, RemoteAudioSourceLatencyCaching) { +TEST_F(RtpSenderReceiverTest, RemoteAudioSourceLatency) { absl::optional delay_ms; // In milliseconds. - double latency_s = 0.5; // In seconds. - rtc::scoped_refptr source = - new rtc::RefCountedObject(rtc::Thread::Current()); - - // Check default value. - EXPECT_DOUBLE_EQ(source->GetLatency(), 0.0); - - // Check caching behaviour. - source->SetLatency(latency_s); - EXPECT_DOUBLE_EQ(source->GetLatency(), latency_s); - - // Check that cached value applied on the start. - source->Start(voice_media_channel_, kAudioSsrc); - delay_ms = voice_media_channel_->GetBaseMinimumPlayoutDelayMs(kAudioSsrc); - EXPECT_DOUBLE_EQ(latency_s, delay_ms.value_or(0) / 1000.0); - - // Check that setting latency changes delay. - latency_s = 0.8; - source->SetLatency(latency_s); - delay_ms = voice_media_channel_->GetBaseMinimumPlayoutDelayMs(kAudioSsrc); - EXPECT_DOUBLE_EQ(latency_s, delay_ms.value_or(0) / 1000.0); - EXPECT_DOUBLE_EQ(latency_s, source->GetLatency()); - - // Check that if underlying delay is changed then remote source will reflect - // it. - delay_ms = 300; - voice_media_channel_->SetBaseMinimumPlayoutDelayMs(kAudioSsrc, - delay_ms.value()); - EXPECT_DOUBLE_EQ(source->GetLatency(), delay_ms.value() / 1000.0); - - // Check that after stop we get last cached value. - source->Stop(voice_media_channel_, kAudioSsrc); - EXPECT_DOUBLE_EQ(latency_s, source->GetLatency()); - - // Check that if we start source again with new ssrc then cached value is - // applied. - source->Start(voice_media_channel_, kAudioSsrc2); - delay_ms = voice_media_channel_->GetBaseMinimumPlayoutDelayMs(kAudioSsrc2); - EXPECT_DOUBLE_EQ(latency_s, delay_ms.value_or(0) / 1000.0); - - // Check rounding behavior. - source->SetLatency(2 / 1000.0); - delay_ms = voice_media_channel_->GetBaseMinimumPlayoutDelayMs(kAudioSsrc2); - EXPECT_EQ(0, delay_ms.value_or(-1)); - EXPECT_DOUBLE_EQ(0, source->GetLatency()); -} - -TEST_F(RtpSenderReceiverTest, RemoteAudioSourceLatencyNoCaching) { - int delay_ms = 300; // In milliseconds. rtc::scoped_refptr source = new rtc::RefCountedObject(rtc::Thread::Current()); // Set it to value different from default zero. - voice_media_channel_->SetBaseMinimumPlayoutDelayMs(kAudioSsrc, delay_ms); + voice_media_channel_->SetBaseMinimumPlayoutDelayMs(kAudioSsrc, 300); // Check that calling GetLatency on the source that hasn't been started yet - // won't trigger caching. + // won't trigger caching and return default value. EXPECT_DOUBLE_EQ(source->GetLatency(), 0); + + // Check that cached latency will be applied on start. + source->SetLatency(0.4); + EXPECT_DOUBLE_EQ(source->GetLatency(), 0.4); source->Start(voice_media_channel_, kAudioSsrc); - EXPECT_DOUBLE_EQ(source->GetLatency(), delay_ms / 1000.0); + delay_ms = voice_media_channel_->GetBaseMinimumPlayoutDelayMs(kAudioSsrc); + EXPECT_EQ(400, delay_ms); } -TEST_F(RtpSenderReceiverTest, RemoteAudioTrackSetLatency) { +TEST_F(RtpSenderReceiverTest, RemoteAudioTrackLatency) { CreateAudioRtpReceiver(); + VerifyTrackLatencyBehaviour(voice_media_channel_, audio_track_.get(), + audio_track_->GetSource(), kAudioSsrc); +} - absl::optional delay_ms; // In milliseconds. - double latency_s = 0.5; // In seconds. - audio_track_->GetSource()->SetLatency(latency_s); - delay_ms = voice_media_channel_->GetBaseMinimumPlayoutDelayMs(kAudioSsrc); - EXPECT_DOUBLE_EQ(latency_s, delay_ms.value_or(0) / 1000.0); - - // Disabling the track should take no effect on previously set value. - audio_track_->set_enabled(false); - delay_ms = voice_media_channel_->GetBaseMinimumPlayoutDelayMs(kAudioSsrc); - EXPECT_DOUBLE_EQ(latency_s, delay_ms.value_or(0) / 1000.0); - - // When the track is disabled, we still should be able to set latency. - latency_s = 0.3; - audio_track_->GetSource()->SetLatency(latency_s); - delay_ms = voice_media_channel_->GetBaseMinimumPlayoutDelayMs(kAudioSsrc); - EXPECT_DOUBLE_EQ(latency_s, delay_ms.value_or(0) / 1000.0); - - // Enabling the track should take no effect on previously set value. - audio_track_->set_enabled(true); - delay_ms = voice_media_channel_->GetBaseMinimumPlayoutDelayMs(kAudioSsrc); - EXPECT_DOUBLE_EQ(latency_s, delay_ms.value_or(0) / 1000.0); - - // We still should be able to change latency. - latency_s = 0.0; - audio_track_->GetSource()->SetLatency(latency_s); - delay_ms = voice_media_channel_->GetBaseMinimumPlayoutDelayMs(kAudioSsrc); - EXPECT_EQ(0, delay_ms.value_or(-1)); - EXPECT_DOUBLE_EQ(latency_s, delay_ms.value_or(0) / 1000.0); +TEST_F(RtpSenderReceiverTest, RemoteVideoTrackLatency) { + CreateVideoRtpReceiver(); + VerifyTrackLatencyBehaviour(video_media_channel_, video_track_.get(), + video_track_->GetSource(), kVideoSsrc); } // Test that the media channel isn't enabled for sending if the audio sender diff --git a/pc/test/mock_delayable.h b/pc/test/mock_delayable.h new file mode 100644 index 0000000000..548f9f8c0a --- /dev/null +++ b/pc/test/mock_delayable.h @@ -0,0 +1,31 @@ +/* + * Copyright 2019 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef PC_TEST_MOCK_DELAYABLE_H_ +#define PC_TEST_MOCK_DELAYABLE_H_ + +#include + +#include "absl/types/optional.h" +#include "media/base/delayable.h" +#include "test/gmock.h" + +namespace webrtc { + +class MockDelayable : public cricket::Delayable { + public: + MOCK_METHOD2(SetBaseMinimumPlayoutDelayMs, bool(uint32_t ssrc, int delay_ms)); + MOCK_CONST_METHOD1(GetBaseMinimumPlayoutDelayMs, + absl::optional(uint32_t ssrc)); +}; + +} // namespace webrtc + +#endif // PC_TEST_MOCK_DELAYABLE_H_ diff --git a/pc/video_rtp_receiver.cc b/pc/video_rtp_receiver.cc index 2d9a23fb0f..b112f048a9 100644 --- a/pc/video_rtp_receiver.cc +++ b/pc/video_rtp_receiver.cc @@ -39,7 +39,7 @@ VideoRtpReceiver::VideoRtpReceiver( const std::vector>& streams) : worker_thread_(worker_thread), id_(receiver_id), - source_(new RefCountedObject()), + source_(new RefCountedObject(worker_thread_)), track_(VideoTrackProxy::Create( rtc::Thread::Current(), worker_thread, @@ -123,6 +123,7 @@ void VideoRtpReceiver::Stop() { // media channel has already been deleted. SetSink(nullptr); } + source_->Stop(); stopped_ = true; } @@ -142,6 +143,8 @@ void VideoRtpReceiver::SetupMediaChannel(uint32_t ssrc) { // Attach any existing frame decryptor to the media channel. MaybeAttachFrameDecryptorToMediaChannel( ssrc_, worker_thread_, frame_decryptor_, media_channel_, stopped_); + + source_->Start(media_channel_, ssrc); } void VideoRtpReceiver::set_stream_ids(std::vector stream_ids) { diff --git a/pc/video_rtp_receiver.h b/pc/video_rtp_receiver.h index ab507aec64..2795e7a2bf 100644 --- a/pc/video_rtp_receiver.h +++ b/pc/video_rtp_receiver.h @@ -27,6 +27,8 @@ #include "api/video/video_source_interface.h" #include "media/base/media_channel.h" #include "media/base/video_broadcaster.h" +#include "pc/playout_latency.h" +#include "pc/playout_latency_proxy.h" #include "pc/rtp_receiver.h" #include "pc/video_track_source.h" #include "rtc_base/ref_counted_object.h" @@ -37,7 +39,7 @@ namespace webrtc { class VideoRtpReceiver : public rtc::RefCountedObject { public: // An SSRC of 0 will create a receiver that will match the first SSRC it - // sees. + // sees. Must be called on signaling thread. VideoRtpReceiver(rtc::Thread* worker_thread, std::string receiver_id, std::vector streams_ids); @@ -103,23 +105,42 @@ class VideoRtpReceiver : public rtc::RefCountedObject { std::vector GetSources() const override; - private: class VideoRtpTrackSource : public VideoTrackSource { public: - VideoRtpTrackSource() : VideoTrackSource(true /* remote */) {} + explicit VideoRtpTrackSource(rtc::Thread* worker_thread) + : VideoTrackSource(true /* remote */), + latency_(PlayoutLatencyProxy::Create( + rtc::Thread::Current(), + worker_thread, + new rtc::RefCountedObject(worker_thread))) {} rtc::VideoSourceInterface* source() override { return &broadcaster_; } rtc::VideoSinkInterface* sink() { return &broadcaster_; } + void SetLatency(double latency) override { latency_->SetLatency(latency); } + + void Start(cricket::VideoMediaChannel* media_channel, uint32_t ssrc) { + latency_->OnStart(media_channel, ssrc); + } + + void Stop() { latency_->OnStop(); } + + double GetLatency() const override { return latency_->GetLatency(); } + private: + // Allows to thread safely change playout latency. Handles caching cases if + // |SetLatency| is called before start. + rtc::scoped_refptr latency_; + // |broadcaster_| is needed since the decoder can only handle one sink. // It might be better if the decoder can handle multiple sinks and consider // the VideoSinkWants. rtc::VideoBroadcaster broadcaster_; }; + private: bool SetSink(rtc::VideoSinkInterface* sink); rtc::Thread* const worker_thread_; diff --git a/video/video_receive_stream.cc b/video/video_receive_stream.cc index c92a349a90..3cf786f266 100644 --- a/video/video_receive_stream.cc +++ b/video/video_receive_stream.cc @@ -54,6 +54,9 @@ namespace webrtc { namespace { +constexpr int kMinBaseMinimumDelayMs = 0; +constexpr int kMaxBaseMinimumDelayMs = 10000; + VideoCodec CreateDecoderVideoCodec(const VideoReceiveStream::Decoder& decoder) { VideoCodec codec; memset(&codec, 0, sizeof(codec)); @@ -166,12 +169,14 @@ VideoReceiveStream::VideoReceiveStream( PacketRouter* packet_router, VideoReceiveStream::Config config, ProcessThread* process_thread, - CallStats* call_stats) + CallStats* call_stats, + Clock* clock, + VCMTiming* timing) : transport_adapter_(config.rtcp_send_transport), config_(std::move(config)), num_cpu_cores_(num_cpu_cores), process_thread_(process_thread), - clock_(Clock::GetRealTimeClock()), + clock_(clock), decode_thread_(&DecodeThreadFunction, this, "DecodingThread", @@ -180,8 +185,11 @@ VideoReceiveStream::VideoReceiveStream( stats_proxy_(&config_, clock_), rtp_receive_statistics_( ReceiveStatistics::Create(clock_, &stats_proxy_, &stats_proxy_)), - timing_(new VCMTiming(clock_)), - video_receiver_(clock_, timing_.get(), this, this), + timing_(timing), + video_receiver_(clock_, + timing_.get(), + this, // NackSender + this), // KeyFrameRequestSender rtp_video_stream_receiver_(&transport_adapter_, call_stats, packet_router, @@ -201,6 +209,7 @@ VideoReceiveStream::VideoReceiveStream( RTC_DCHECK(call_stats_); module_process_sequence_checker_.Detach(); + network_sequence_checker_.Detach(); RTC_DCHECK(!config_.decoders.empty()); std::set decoder_payload_types; @@ -241,6 +250,22 @@ VideoReceiveStream::VideoReceiveStream( } } +VideoReceiveStream::VideoReceiveStream( + RtpStreamReceiverControllerInterface* receiver_controller, + int num_cpu_cores, + PacketRouter* packet_router, + VideoReceiveStream::Config config, + ProcessThread* process_thread, + CallStats* call_stats) + : VideoReceiveStream(receiver_controller, + num_cpu_cores, + packet_router, + std::move(config), + process_thread, + call_stats, + Clock::GetRealTimeClock(), + new VCMTiming(Clock::GetRealTimeClock())) {} + VideoReceiveStream::~VideoReceiveStream() { RTC_DCHECK_CALLED_SEQUENTIALLY(&worker_sequence_checker_); RTC_LOG(LS_INFO) << "~VideoReceiveStream: " << config_.ToString(); @@ -393,6 +418,25 @@ void VideoReceiveStream::RemoveSecondarySink( rtp_video_stream_receiver_.RemoveSecondarySink(sink); } +bool VideoReceiveStream::SetBaseMinimumPlayoutDelayMs(int delay_ms) { + RTC_DCHECK_CALLED_SEQUENTIALLY(&worker_sequence_checker_); + if (delay_ms < kMinBaseMinimumDelayMs || delay_ms > kMaxBaseMinimumDelayMs) { + return false; + } + + rtc::CritScope cs(&playout_delay_lock_); + base_minimum_playout_delay_ms_ = delay_ms; + UpdatePlayoutDelays(); + return true; +} + +int VideoReceiveStream::GetBaseMinimumPlayoutDelayMs() const { + RTC_DCHECK_CALLED_SEQUENTIALLY(&worker_sequence_checker_); + + rtc::CritScope cs(&playout_delay_lock_); + return base_minimum_playout_delay_ms_; +} + // TODO(tommi): This method grabs a lock 6 times. void VideoReceiveStream::OnFrame(const VideoFrame& video_frame) { int64_t sync_offset_ms; @@ -428,6 +472,7 @@ void VideoReceiveStream::RequestKeyFrame() { void VideoReceiveStream::OnCompleteFrame( std::unique_ptr frame) { + RTC_DCHECK_CALLED_SEQUENTIALLY(&network_sequence_checker_); // TODO(https://bugs.webrtc.org/9974): Consider removing this workaround. int64_t time_now_ms = rtc::TimeMillis(); if (last_complete_frame_time_ms_ > 0 && @@ -436,6 +481,19 @@ void VideoReceiveStream::OnCompleteFrame( } last_complete_frame_time_ms_ = time_now_ms; + const PlayoutDelay& playout_delay = frame->EncodedImage().playout_delay_; + if (playout_delay.min_ms >= 0) { + rtc::CritScope cs(&playout_delay_lock_); + frame_minimum_playout_delay_ms_ = playout_delay.min_ms; + UpdatePlayoutDelays(); + } + + if (playout_delay.max_ms >= 0) { + rtc::CritScope cs(&playout_delay_lock_); + frame_maximum_playout_delay_ms_ = playout_delay.max_ms; + UpdatePlayoutDelays(); + } + int64_t last_continuous_pid = frame_buffer_->InsertFrame(std::move(frame)); if (last_continuous_pid != -1) rtp_video_stream_receiver_.FrameContinuous(last_continuous_pid); @@ -482,7 +540,9 @@ uint32_t VideoReceiveStream::GetPlayoutTimestamp() const { void VideoReceiveStream::SetMinimumPlayoutDelay(int delay_ms) { RTC_DCHECK_CALLED_SEQUENTIALLY(&module_process_sequence_checker_); - video_receiver_.SetMinimumPlayoutDelay(delay_ms); + rtc::CritScope cs(&playout_delay_lock_); + syncable_minimum_playout_delay_ms_ = delay_ms; + UpdatePlayoutDelays(); } void VideoReceiveStream::DecodeThreadFunction(void* ptr) { @@ -565,6 +625,20 @@ bool VideoReceiveStream::Decode() { return true; } +void VideoReceiveStream::UpdatePlayoutDelays() const { + const int minimum_delay_ms = + std::max({frame_minimum_playout_delay_ms_, base_minimum_playout_delay_ms_, + syncable_minimum_playout_delay_ms_}); + if (minimum_delay_ms >= 0) { + timing_->set_min_playout_delay(minimum_delay_ms); + } + + const int maximum_delay_ms = frame_maximum_playout_delay_ms_; + if (maximum_delay_ms >= 0) { + timing_->set_max_playout_delay(maximum_delay_ms); + } +} + std::vector VideoReceiveStream::GetSources() const { return rtp_video_stream_receiver_.GetSources(); } diff --git a/video/video_receive_stream.h b/video/video_receive_stream.h index 30b227d695..978294db78 100644 --- a/video/video_receive_stream.h +++ b/video/video_receive_stream.h @@ -52,6 +52,14 @@ class VideoReceiveStream : public webrtc::VideoReceiveStream, public MediaTransportVideoSinkInterface, public MediaTransportRttObserver { public: + VideoReceiveStream(RtpStreamReceiverControllerInterface* receiver_controller, + int num_cpu_cores, + PacketRouter* packet_router, + VideoReceiveStream::Config config, + ProcessThread* process_thread, + CallStats* call_stats, + Clock* clock, + VCMTiming* timing); VideoReceiveStream(RtpStreamReceiverControllerInterface* receiver_controller, int num_cpu_cores, PacketRouter* packet_router, @@ -76,6 +84,12 @@ class VideoReceiveStream : public webrtc::VideoReceiveStream, void AddSecondarySink(RtpPacketSinkInterface* sink) override; void RemoveSecondarySink(const RtpPacketSinkInterface* sink) override; + // SetBaseMinimumPlayoutDelayMs and GetBaseMinimumPlayoutDelayMs are called + // from webrtc/api level and requested by user code. For e.g. blink/js layer + // in Chromium. + bool SetBaseMinimumPlayoutDelayMs(int delay_ms) override; + int GetBaseMinimumPlayoutDelayMs() const override; + // Implements rtc::VideoSinkInterface. void OnFrame(const VideoFrame& video_frame) override; @@ -104,6 +118,8 @@ class VideoReceiveStream : public webrtc::VideoReceiveStream, int id() const override; absl::optional GetInfo() const override; uint32_t GetPlayoutTimestamp() const override; + + // SetMinimumPlayoutDelay is only called by A/V sync. void SetMinimumPlayoutDelay(int delay_ms) override; std::vector GetSources() const override; @@ -111,9 +127,12 @@ class VideoReceiveStream : public webrtc::VideoReceiveStream, private: static void DecodeThreadFunction(void* ptr); bool Decode(); + void UpdatePlayoutDelays() const + RTC_EXCLUSIVE_LOCKS_REQUIRED(playout_delay_lock_); rtc::SequencedTaskChecker worker_sequence_checker_; rtc::SequencedTaskChecker module_process_sequence_checker_; + rtc::SequencedTaskChecker network_sequence_checker_; TransportAdapter transport_adapter_; const VideoReceiveStream::Config config_; @@ -158,6 +177,23 @@ class VideoReceiveStream : public webrtc::VideoReceiveStream, int64_t last_keyframe_request_ms_ = 0; int64_t last_complete_frame_time_ms_ = 0; + + rtc::CriticalSection playout_delay_lock_; + + // All of them tries to change current min_playout_delay on |timing_| but + // source of the change request is different in each case. Among them the + // biggest delay is used. -1 means use default value from the |timing_|. + // + // Minimum delay as decided by the RTP playout delay extension. + int frame_minimum_playout_delay_ms_ RTC_GUARDED_BY(playout_delay_lock_) = -1; + // Minimum delay as decided by the setLatency function in "webrtc/api". + int base_minimum_playout_delay_ms_ RTC_GUARDED_BY(playout_delay_lock_) = -1; + // Minimum delay as decided by the A/V synchronization feature. + int syncable_minimum_playout_delay_ms_ RTC_GUARDED_BY(playout_delay_lock_) = + -1; + + // Maximum delay as decided by the RTP playout delay extension. + int frame_maximum_playout_delay_ms_ RTC_GUARDED_BY(playout_delay_lock_) = -1; }; } // namespace internal } // namespace webrtc diff --git a/video/video_receive_stream_unittest.cc b/video/video_receive_stream_unittest.cc index 259026ad1e..0c76c86214 100644 --- a/video/video_receive_stream_unittest.cc +++ b/video/video_receive_stream_unittest.cc @@ -8,6 +8,7 @@ * be found in the AUTHORS file in the root of the source tree. */ +#include #include #include "test/gmock.h" @@ -59,6 +60,13 @@ class MockVideoDecoder : public VideoDecoder { const char* ImplementationName() const { return "MockVideoDecoder"; } }; +class FrameObjectFake : public video_coding::EncodedFrame { + public: + int64_t ReceivedTime() const override { return 0; } + + int64_t RenderTime() const override { return _renderTimeMs; } +}; + } // namespace class VideoReceiveStreamTest : public testing::Test { @@ -88,9 +96,12 @@ class VideoReceiveStreamTest : public testing::Test { null_decoder.decoder_factory = &null_decoder_factory_; config_.decoders.push_back(null_decoder); + Clock* clock = Clock::GetRealTimeClock(); + timing_ = new VCMTiming(clock); + video_receive_stream_.reset(new webrtc::internal::VideoReceiveStream( &rtp_stream_receiver_controller_, kDefaultNumCpuCores, &packet_router_, - config_.Copy(), process_thread_.get(), &call_stats_)); + config_.Copy(), process_thread_.get(), &call_stats_, clock, timing_)); } protected: @@ -106,6 +117,7 @@ class VideoReceiveStreamTest : public testing::Test { PacketRouter packet_router_; RtpStreamReceiverController rtp_stream_receiver_controller_; std::unique_ptr video_receive_stream_; + VCMTiming* timing_; }; TEST_F(VideoReceiveStreamTest, CreateFrameFromH264FmtpSpropAndIdr) { @@ -136,4 +148,65 @@ TEST_F(VideoReceiveStreamTest, CreateFrameFromH264FmtpSpropAndIdr) { init_decode_event_.Wait(kDefaultTimeOutMs); } +TEST_F(VideoReceiveStreamTest, PlayoutDelay) { + const PlayoutDelay kPlayoutDelayMs = {123, 321}; + std::unique_ptr test_frame(new FrameObjectFake()); + test_frame->id.picture_id = 0; + test_frame->SetPlayoutDelay(kPlayoutDelayMs); + + video_receive_stream_->OnCompleteFrame(std::move(test_frame)); + EXPECT_EQ(kPlayoutDelayMs.min_ms, timing_->min_playout_delay()); + EXPECT_EQ(kPlayoutDelayMs.max_ms, timing_->max_playout_delay()); + + // Check that the biggest minimum delay is chosen. + video_receive_stream_->SetMinimumPlayoutDelay(400); + EXPECT_EQ(400, timing_->min_playout_delay()); + + // Check base minimum delay validation. + EXPECT_FALSE(video_receive_stream_->SetBaseMinimumPlayoutDelayMs(12345)); + EXPECT_FALSE(video_receive_stream_->SetBaseMinimumPlayoutDelayMs(-1)); + EXPECT_TRUE(video_receive_stream_->SetBaseMinimumPlayoutDelayMs(500)); + EXPECT_EQ(500, timing_->min_playout_delay()); + + // Check that intermidiate values are remembered and the biggest remembered + // is chosen. + video_receive_stream_->SetBaseMinimumPlayoutDelayMs(0); + EXPECT_EQ(400, timing_->min_playout_delay()); + + video_receive_stream_->SetMinimumPlayoutDelay(0); + EXPECT_EQ(123, timing_->min_playout_delay()); +} + +TEST_F(VideoReceiveStreamTest, PlayoutDelayPreservesDefaultMaxValue) { + const int default_max_playout_latency = timing_->max_playout_delay(); + const PlayoutDelay kPlayoutDelayMs = {123, -1}; + + std::unique_ptr test_frame(new FrameObjectFake()); + test_frame->id.picture_id = 0; + test_frame->SetPlayoutDelay(kPlayoutDelayMs); + + video_receive_stream_->OnCompleteFrame(std::move(test_frame)); + + // Ensure that -1 preserves default maximum value from |timing_|. + EXPECT_EQ(kPlayoutDelayMs.min_ms, timing_->min_playout_delay()); + EXPECT_NE(kPlayoutDelayMs.max_ms, timing_->max_playout_delay()); + EXPECT_EQ(default_max_playout_latency, timing_->max_playout_delay()); +} + +TEST_F(VideoReceiveStreamTest, PlayoutDelayPreservesDefaultMinValue) { + const int default_min_playout_latency = timing_->min_playout_delay(); + const PlayoutDelay kPlayoutDelayMs = {-1, 321}; + + std::unique_ptr test_frame(new FrameObjectFake()); + test_frame->id.picture_id = 0; + test_frame->SetPlayoutDelay(kPlayoutDelayMs); + + video_receive_stream_->OnCompleteFrame(std::move(test_frame)); + + // Ensure that -1 preserves default minimum value from |timing_|. + EXPECT_NE(kPlayoutDelayMs.min_ms, timing_->min_playout_delay()); + EXPECT_EQ(kPlayoutDelayMs.max_ms, timing_->max_playout_delay()); + EXPECT_EQ(default_min_playout_latency, timing_->min_playout_delay()); +} + } // namespace webrtc