From 412a31bbf8913231740a46343858e8e1f452b6bc Mon Sep 17 00:00:00 2001 From: Marina Ciocea Date: Fri, 28 Feb 2020 16:02:06 +0100 Subject: [PATCH] Insert frame transformer between Depacketizer and Decoder. Add a new API in RTReceiverInterface, to be called from the browser side to insert a frame transformer between the Depacketizer and the Decoder. The frame transformer is passed from RTReceiverInterface through the library to be eventually set in RtpVideoStreamReceiver, where the frame transformation will occur in the follow-up CL https://webrtc-review.googlesource.com/c/src/+/169130. This change is part of the implementation of the Insertable Streams Web API: https://github.com/alvestrand/webrtc-media-streams/blob/master/explainer.md Design doc for WebRTC library changes: http://doc/1eiLkjNUkRy2FssCPLUp6eH08BZuXXoHfbbBP1ZN7EVk Bug: webrtc:11380 Change-Id: I6b73cd16e3907e8b7709b852d6a2540ee11b4fed Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/169129 Reviewed-by: Danil Chapovalov Reviewed-by: Magnus Flodman Reviewed-by: Harald Alvestrand Reviewed-by: Karl Wiberg Reviewed-by: Rasmus Brandt Commit-Queue: Marina Ciocea Cr-Commit-Position: refs/heads/master@{#30654} --- api/rtp_receiver_interface.cc | 3 ++ api/rtp_receiver_interface.h | 10 ++++++ call/video_receive_stream.h | 3 ++ media/base/media_channel.cc | 3 ++ media/base/media_channel.h | 3 ++ media/engine/webrtc_video_engine.cc | 18 +++++++++++ media/engine/webrtc_video_engine.h | 8 +++++ pc/video_rtp_receiver.cc | 17 +++++++++++ pc/video_rtp_receiver.h | 6 ++++ video/BUILD.gn | 1 + video/rtp_video_stream_receiver.cc | 34 +++++++++++++++++++-- video/rtp_video_stream_receiver.h | 25 +++++++++++++++ video/rtp_video_stream_receiver_unittest.cc | 2 +- video/video_receive_stream.cc | 3 +- 14 files changed, 132 insertions(+), 4 deletions(-) diff --git a/api/rtp_receiver_interface.cc b/api/rtp_receiver_interface.cc index d20516b67c..bc9aef5aef 100644 --- a/api/rtp_receiver_interface.cc +++ b/api/rtp_receiver_interface.cc @@ -38,4 +38,7 @@ RtpReceiverInterface::dtls_transport() const { return nullptr; } +void RtpReceiverInterface::SetDepacketizerToDecoderFrameTransformer( + rtc::scoped_refptr frame_transformer) {} + } // namespace webrtc diff --git a/api/rtp_receiver_interface.h b/api/rtp_receiver_interface.h index 6052763341..a15864e34a 100644 --- a/api/rtp_receiver_interface.h +++ b/api/rtp_receiver_interface.h @@ -19,6 +19,7 @@ #include "api/crypto/frame_decryptor_interface.h" #include "api/dtls_transport_interface.h" +#include "api/frame_transformer_interface.h" #include "api/media_stream_interface.h" #include "api/media_types.h" #include "api/proxy.h" @@ -107,6 +108,12 @@ class RTC_EXPORT RtpReceiverInterface : public rtc::RefCountInterface { // user. This can be used to update the state of the object. virtual rtc::scoped_refptr GetFrameDecryptor() const; + // Sets a frame transformer between the depacketizer and the decoder to enable + // client code to transform received frames according to their own processing + // logic. + virtual void SetDepacketizerToDecoderFrameTransformer( + rtc::scoped_refptr frame_transformer); + protected: ~RtpReceiverInterface() override = default; }; @@ -132,6 +139,9 @@ PROXY_METHOD1(void, rtc::scoped_refptr) PROXY_CONSTMETHOD0(rtc::scoped_refptr, GetFrameDecryptor) +PROXY_METHOD1(void, + SetDepacketizerToDecoderFrameTransformer, + rtc::scoped_refptr) END_PROXY_MAP() } // namespace webrtc diff --git a/call/video_receive_stream.h b/call/video_receive_stream.h index 0f5e8e043a..765d8027bf 100644 --- a/call/video_receive_stream.h +++ b/call/video_receive_stream.h @@ -21,6 +21,7 @@ #include "api/call/transport.h" #include "api/crypto/crypto_options.h" #include "api/crypto/frame_decryptor_interface.h" +#include "api/frame_transformer_interface.h" #include "api/rtp_headers.h" #include "api/rtp_parameters.h" #include "api/transport/rtp/rtp_source.h" @@ -262,6 +263,8 @@ class VideoReceiveStream { // Per PeerConnection cryptography options. CryptoOptions crypto_options; + + rtc::scoped_refptr frame_transformer; }; // Starts stream activity. diff --git a/media/base/media_channel.cc b/media/base/media_channel.cc index 29cf550655..3417924730 100644 --- a/media/base/media_channel.cc +++ b/media/base/media_channel.cc @@ -52,6 +52,9 @@ void MediaChannel::SetVideoCodecSwitchingEnabled(bool enabled) {} void MediaChannel::SetEncoderToPacketizerFrameTransformer( uint32_t ssrc, rtc::scoped_refptr frame_transformer) {} +void MediaChannel::SetDepacketizerToDecoderFrameTransformer( + uint32_t ssrc, + rtc::scoped_refptr frame_transformer) {} MediaSenderInfo::MediaSenderInfo() = default; MediaSenderInfo::~MediaSenderInfo() = default; diff --git a/media/base/media_channel.h b/media/base/media_channel.h index 9631722ff6..8ee4a238a1 100644 --- a/media/base/media_channel.h +++ b/media/base/media_channel.h @@ -291,6 +291,9 @@ class MediaChannel : public sigslot::has_slots<> { virtual void SetEncoderToPacketizerFrameTransformer( uint32_t ssrc, rtc::scoped_refptr frame_transformer); + virtual void SetDepacketizerToDecoderFrameTransformer( + uint32_t ssrc, + rtc::scoped_refptr frame_transformer); protected: bool DscpEnabled() const { return enable_dscp_; } diff --git a/media/engine/webrtc_video_engine.cc b/media/engine/webrtc_video_engine.cc index 3f43f7906b..0c23ff8b51 100644 --- a/media/engine/webrtc_video_engine.cc +++ b/media/engine/webrtc_video_engine.cc @@ -2941,6 +2941,13 @@ void WebRtcVideoChannel::WebRtcVideoReceiveStream::GenerateKeyFrame() { } } +void WebRtcVideoChannel::WebRtcVideoReceiveStream:: + SetDepacketizerToDecoderFrameTransformer( + rtc::scoped_refptr + frame_transformer) { + config_.frame_transformer = frame_transformer; +} + WebRtcVideoChannel::VideoCodecSettings::VideoCodecSettings() : flexfec_payload_type(-1), rtx_payload_type(-1) {} @@ -3152,6 +3159,17 @@ void WebRtcVideoChannel::SetEncoderToPacketizerFrameTransformer( } } +void WebRtcVideoChannel::SetDepacketizerToDecoderFrameTransformer( + uint32_t ssrc, + rtc::scoped_refptr frame_transformer) { + RTC_DCHECK_RUN_ON(&thread_checker_); + auto matching_stream = receive_streams_.find(ssrc); + if (matching_stream != receive_streams_.end()) { + matching_stream->second->SetDepacketizerToDecoderFrameTransformer( + std::move(frame_transformer)); + } +} + // TODO(bugs.webrtc.org/8785): Consider removing max_qp as member of // EncoderStreamFactory and instead set this value individually for each stream // in the VideoEncoderConfig.simulcast_layers. diff --git a/media/engine/webrtc_video_engine.h b/media/engine/webrtc_video_engine.h index b1cbd1bae8..418576f3f0 100644 --- a/media/engine/webrtc_video_engine.h +++ b/media/engine/webrtc_video_engine.h @@ -228,6 +228,10 @@ class WebRtcVideoChannel : public VideoMediaChannel, uint32_t ssrc, rtc::scoped_refptr frame_transformer) override; + void SetDepacketizerToDecoderFrameTransformer( + uint32_t ssrc, + rtc::scoped_refptr frame_transformer) + override; private: class WebRtcVideoReceiveStream; @@ -463,6 +467,10 @@ class WebRtcVideoChannel : public VideoMediaChannel, void ClearRecordableEncodedFrameCallback(); void GenerateKeyFrame(); + void SetDepacketizerToDecoderFrameTransformer( + rtc::scoped_refptr + frame_transformer); + private: void RecreateWebRtcVideoStream(); void MaybeRecreateWebRtcFlexfecStream(); diff --git a/pc/video_rtp_receiver.cc b/pc/video_rtp_receiver.cc index c6fb5430cd..a63a0f68f2 100644 --- a/pc/video_rtp_receiver.cc +++ b/pc/video_rtp_receiver.cc @@ -104,6 +104,18 @@ VideoRtpReceiver::GetFrameDecryptor() const { return frame_decryptor_; } +void VideoRtpReceiver::SetDepacketizerToDecoderFrameTransformer( + rtc::scoped_refptr frame_transformer) { + worker_thread_->Invoke(RTC_FROM_HERE, [&] { + RTC_DCHECK_RUN_ON(worker_thread_); + frame_transformer_ = std::move(frame_transformer); + if (media_channel_ && ssrc_.has_value() && !stopped_) { + media_channel_->SetDepacketizerToDecoderFrameTransformer( + *ssrc_, frame_transformer_); + } + }); +} + void VideoRtpReceiver::Stop() { // TODO(deadbeef): Need to do more here to fully stop receiving packets. if (stopped_) { @@ -144,6 +156,11 @@ void VideoRtpReceiver::RestartMediaChannel(absl::optional ssrc) { if (encoded_sink_enabled) { SetEncodedSinkEnabled(true); } + + if (frame_transformer_ && media_channel_ && ssrc_.has_value()) { + media_channel_->SetDepacketizerToDecoderFrameTransformer( + *ssrc_, frame_transformer_); + } }); // Attach any existing frame decryptor to the media channel. diff --git a/pc/video_rtp_receiver.h b/pc/video_rtp_receiver.h index 0b8a73da61..f66a8a7892 100644 --- a/pc/video_rtp_receiver.h +++ b/pc/video_rtp_receiver.h @@ -18,6 +18,7 @@ #include "absl/types/optional.h" #include "api/crypto/frame_decryptor_interface.h" +#include "api/frame_transformer_interface.h" #include "api/media_stream_interface.h" #include "api/media_types.h" #include "api/rtp_parameters.h" @@ -83,6 +84,9 @@ class VideoRtpReceiver : public rtc::RefCountedObject, rtc::scoped_refptr GetFrameDecryptor() const override; + void SetDepacketizerToDecoderFrameTransformer( + rtc::scoped_refptr frame_transformer) override; + // RtpReceiverInternal implementation. void Stop() override; void SetupMediaChannel(uint32_t ssrc) override; @@ -134,6 +138,8 @@ class VideoRtpReceiver : public rtc::RefCountedObject, int attachment_id_ = 0; rtc::scoped_refptr frame_decryptor_; rtc::scoped_refptr dtls_transport_; + rtc::scoped_refptr frame_transformer_ + RTC_GUARDED_BY(worker_thread_); // Allows to thread safely change jitter buffer delay. Handles caching cases // if |SetJitterBufferMinimumDelay| is called before start. rtc::scoped_refptr delay_; diff --git a/video/BUILD.gn b/video/BUILD.gn index 7bab757a24..2f7ec1788b 100644 --- a/video/BUILD.gn +++ b/video/BUILD.gn @@ -54,6 +54,7 @@ rtc_library("video") { ":frame_dumping_decoder", "../api:array_view", "../api:fec_controller_api", + "../api:frame_transformer_interface", "../api:libjingle_peerconnection_api", "../api:rtp_parameters", "../api:scoped_refptr", diff --git a/video/rtp_video_stream_receiver.cc b/video/rtp_video_stream_receiver.cc index 52f1014746..5a6c6270be 100644 --- a/video/rtp_video_stream_receiver.cc +++ b/video/rtp_video_stream_receiver.cc @@ -12,6 +12,7 @@ #include #include +#include #include #include @@ -194,7 +195,8 @@ RtpVideoStreamReceiver::RtpVideoStreamReceiver( NackSender* nack_sender, KeyFrameRequestSender* keyframe_request_sender, video_coding::OnCompleteFrameCallback* complete_frame_callback, - rtc::scoped_refptr frame_decryptor) + rtc::scoped_refptr frame_decryptor, + rtc::scoped_refptr frame_transformer) : clock_(clock), config_(*config), packet_router_(packet_router), @@ -221,7 +223,8 @@ RtpVideoStreamReceiver::RtpVideoStreamReceiver( packet_buffer_(clock_, kPacketBufferStartSize, PacketBufferMaxSize()), has_received_frame_(false), frames_decryptable_(false), - absolute_capture_time_receiver_(clock) { + absolute_capture_time_receiver_(clock), + frame_transformer_(frame_transformer) { constexpr bool remb_candidate = true; if (packet_router_) packet_router_->AddReceiveRtpModule(rtp_rtcp_.get(), remb_candidate); @@ -283,6 +286,33 @@ RtpVideoStreamReceiver::RtpVideoStreamReceiver( } } +RtpVideoStreamReceiver::RtpVideoStreamReceiver( + Clock* clock, + Transport* transport, + RtcpRttStats* rtt_stats, + PacketRouter* packet_router, + const VideoReceiveStream::Config* config, + ReceiveStatistics* rtp_receive_statistics, + ReceiveStatisticsProxy* receive_stats_proxy, + ProcessThread* process_thread, + NackSender* nack_sender, + KeyFrameRequestSender* keyframe_request_sender, + video_coding::OnCompleteFrameCallback* complete_frame_callback, + rtc::scoped_refptr frame_decryptor) + : RtpVideoStreamReceiver(clock, + transport, + rtt_stats, + packet_router, + config, + rtp_receive_statistics, + receive_stats_proxy, + process_thread, + nack_sender, + keyframe_request_sender, + complete_frame_callback, + frame_decryptor, + nullptr) {} + RtpVideoStreamReceiver::~RtpVideoStreamReceiver() { RTC_DCHECK(secondary_sinks_.empty()); diff --git a/video/rtp_video_stream_receiver.h b/video/rtp_video_stream_receiver.h index c0271f4cb1..98b324ca96 100644 --- a/video/rtp_video_stream_receiver.h +++ b/video/rtp_video_stream_receiver.h @@ -69,6 +69,29 @@ class RtpVideoStreamReceiver : public LossNotificationSender, public OnDecryptedFrameCallback, public OnDecryptionStatusChangeCallback { public: + RtpVideoStreamReceiver( + Clock* clock, + Transport* transport, + RtcpRttStats* rtt_stats, + // The packet router is optional; if provided, the RtpRtcp module for this + // stream is registered as a candidate for sending REMB and transport + // feedback. + PacketRouter* packet_router, + const VideoReceiveStream::Config* config, + ReceiveStatistics* rtp_receive_statistics, + ReceiveStatisticsProxy* receive_stats_proxy, + ProcessThread* process_thread, + NackSender* nack_sender, + // The KeyFrameRequestSender is optional; if not provided, key frame + // requests are sent via the internal RtpRtcp module. + KeyFrameRequestSender* keyframe_request_sender, + video_coding::OnCompleteFrameCallback* complete_frame_callback, + rtc::scoped_refptr frame_decryptor, + rtc::scoped_refptr frame_transformer); + + // TODO(bugs.webrtc.org/11380) remove after updating downstream dependencies + // to use the new constructor. + RTC_DEPRECATED RtpVideoStreamReceiver( Clock* clock, Transport* transport, @@ -346,6 +369,8 @@ class RtpVideoStreamReceiver : public LossNotificationSender, RTC_GUARDED_BY(worker_task_checker_); int64_t last_completed_picture_id_ = 0; + + rtc::scoped_refptr frame_transformer_; }; } // namespace webrtc diff --git a/video/rtp_video_stream_receiver_unittest.cc b/video/rtp_video_stream_receiver_unittest.cc index 44f544ffdb..d7d02b0731 100644 --- a/video/rtp_video_stream_receiver_unittest.cc +++ b/video/rtp_video_stream_receiver_unittest.cc @@ -155,7 +155,7 @@ class RtpVideoStreamReceiverTest : public ::testing::Test { Clock::GetRealTimeClock(), &mock_transport_, nullptr, nullptr, &config_, rtp_receive_statistics_.get(), nullptr, process_thread_.get(), &mock_nack_sender_, &mock_key_frame_request_sender_, - &mock_on_complete_frame_callback_, nullptr); + &mock_on_complete_frame_callback_, nullptr, nullptr); } RTPVideoHeader GetDefaultH264VideoHeader() { diff --git a/video/video_receive_stream.cc b/video/video_receive_stream.cc index 0a2d819ad1..246daadb81 100644 --- a/video/video_receive_stream.cc +++ b/video/video_receive_stream.cc @@ -215,7 +215,8 @@ VideoReceiveStream::VideoReceiveStream( this, // NackSender nullptr, // Use default KeyFrameRequestSender this, // OnCompleteFrameCallback - config_.frame_decryptor), + config_.frame_decryptor, + config_.frame_transformer), rtp_stream_sync_(this), max_wait_for_keyframe_ms_(KeyframeIntervalSettings::ParseFromFieldTrials() .MaxWaitForKeyframeMs()