diff --git a/api/rtp_receiver_interface.cc b/api/rtp_receiver_interface.cc index d20516b67c..bc9aef5aef 100644 --- a/api/rtp_receiver_interface.cc +++ b/api/rtp_receiver_interface.cc @@ -38,4 +38,7 @@ RtpReceiverInterface::dtls_transport() const { return nullptr; } +void RtpReceiverInterface::SetDepacketizerToDecoderFrameTransformer( + rtc::scoped_refptr frame_transformer) {} + } // namespace webrtc diff --git a/api/rtp_receiver_interface.h b/api/rtp_receiver_interface.h index 6052763341..a15864e34a 100644 --- a/api/rtp_receiver_interface.h +++ b/api/rtp_receiver_interface.h @@ -19,6 +19,7 @@ #include "api/crypto/frame_decryptor_interface.h" #include "api/dtls_transport_interface.h" +#include "api/frame_transformer_interface.h" #include "api/media_stream_interface.h" #include "api/media_types.h" #include "api/proxy.h" @@ -107,6 +108,12 @@ class RTC_EXPORT RtpReceiverInterface : public rtc::RefCountInterface { // user. This can be used to update the state of the object. virtual rtc::scoped_refptr GetFrameDecryptor() const; + // Sets a frame transformer between the depacketizer and the decoder to enable + // client code to transform received frames according to their own processing + // logic. + virtual void SetDepacketizerToDecoderFrameTransformer( + rtc::scoped_refptr frame_transformer); + protected: ~RtpReceiverInterface() override = default; }; @@ -132,6 +139,9 @@ PROXY_METHOD1(void, rtc::scoped_refptr) PROXY_CONSTMETHOD0(rtc::scoped_refptr, GetFrameDecryptor) +PROXY_METHOD1(void, + SetDepacketizerToDecoderFrameTransformer, + rtc::scoped_refptr) END_PROXY_MAP() } // namespace webrtc diff --git a/call/video_receive_stream.h b/call/video_receive_stream.h index 0f5e8e043a..765d8027bf 100644 --- a/call/video_receive_stream.h +++ b/call/video_receive_stream.h @@ -21,6 +21,7 @@ #include "api/call/transport.h" #include "api/crypto/crypto_options.h" #include "api/crypto/frame_decryptor_interface.h" +#include "api/frame_transformer_interface.h" #include "api/rtp_headers.h" #include "api/rtp_parameters.h" #include "api/transport/rtp/rtp_source.h" @@ -262,6 +263,8 @@ class VideoReceiveStream { // Per PeerConnection cryptography options. CryptoOptions crypto_options; + + rtc::scoped_refptr frame_transformer; }; // Starts stream activity. diff --git a/media/base/media_channel.cc b/media/base/media_channel.cc index 29cf550655..3417924730 100644 --- a/media/base/media_channel.cc +++ b/media/base/media_channel.cc @@ -52,6 +52,9 @@ void MediaChannel::SetVideoCodecSwitchingEnabled(bool enabled) {} void MediaChannel::SetEncoderToPacketizerFrameTransformer( uint32_t ssrc, rtc::scoped_refptr frame_transformer) {} +void MediaChannel::SetDepacketizerToDecoderFrameTransformer( + uint32_t ssrc, + rtc::scoped_refptr frame_transformer) {} MediaSenderInfo::MediaSenderInfo() = default; MediaSenderInfo::~MediaSenderInfo() = default; diff --git a/media/base/media_channel.h b/media/base/media_channel.h index 9631722ff6..8ee4a238a1 100644 --- a/media/base/media_channel.h +++ b/media/base/media_channel.h @@ -291,6 +291,9 @@ class MediaChannel : public sigslot::has_slots<> { virtual void SetEncoderToPacketizerFrameTransformer( uint32_t ssrc, rtc::scoped_refptr frame_transformer); + virtual void SetDepacketizerToDecoderFrameTransformer( + uint32_t ssrc, + rtc::scoped_refptr frame_transformer); protected: bool DscpEnabled() const { return enable_dscp_; } diff --git a/media/engine/webrtc_video_engine.cc b/media/engine/webrtc_video_engine.cc index 3f43f7906b..0c23ff8b51 100644 --- a/media/engine/webrtc_video_engine.cc +++ b/media/engine/webrtc_video_engine.cc @@ -2941,6 +2941,13 @@ void WebRtcVideoChannel::WebRtcVideoReceiveStream::GenerateKeyFrame() { } } +void WebRtcVideoChannel::WebRtcVideoReceiveStream:: + SetDepacketizerToDecoderFrameTransformer( + rtc::scoped_refptr + frame_transformer) { + config_.frame_transformer = frame_transformer; +} + WebRtcVideoChannel::VideoCodecSettings::VideoCodecSettings() : flexfec_payload_type(-1), rtx_payload_type(-1) {} @@ -3152,6 +3159,17 @@ void WebRtcVideoChannel::SetEncoderToPacketizerFrameTransformer( } } +void WebRtcVideoChannel::SetDepacketizerToDecoderFrameTransformer( + uint32_t ssrc, + rtc::scoped_refptr frame_transformer) { + RTC_DCHECK_RUN_ON(&thread_checker_); + auto matching_stream = receive_streams_.find(ssrc); + if (matching_stream != receive_streams_.end()) { + matching_stream->second->SetDepacketizerToDecoderFrameTransformer( + std::move(frame_transformer)); + } +} + // TODO(bugs.webrtc.org/8785): Consider removing max_qp as member of // EncoderStreamFactory and instead set this value individually for each stream // in the VideoEncoderConfig.simulcast_layers. diff --git a/media/engine/webrtc_video_engine.h b/media/engine/webrtc_video_engine.h index b1cbd1bae8..418576f3f0 100644 --- a/media/engine/webrtc_video_engine.h +++ b/media/engine/webrtc_video_engine.h @@ -228,6 +228,10 @@ class WebRtcVideoChannel : public VideoMediaChannel, uint32_t ssrc, rtc::scoped_refptr frame_transformer) override; + void SetDepacketizerToDecoderFrameTransformer( + uint32_t ssrc, + rtc::scoped_refptr frame_transformer) + override; private: class WebRtcVideoReceiveStream; @@ -463,6 +467,10 @@ class WebRtcVideoChannel : public VideoMediaChannel, void ClearRecordableEncodedFrameCallback(); void GenerateKeyFrame(); + void SetDepacketizerToDecoderFrameTransformer( + rtc::scoped_refptr + frame_transformer); + private: void RecreateWebRtcVideoStream(); void MaybeRecreateWebRtcFlexfecStream(); diff --git a/pc/video_rtp_receiver.cc b/pc/video_rtp_receiver.cc index c6fb5430cd..a63a0f68f2 100644 --- a/pc/video_rtp_receiver.cc +++ b/pc/video_rtp_receiver.cc @@ -104,6 +104,18 @@ VideoRtpReceiver::GetFrameDecryptor() const { return frame_decryptor_; } +void VideoRtpReceiver::SetDepacketizerToDecoderFrameTransformer( + rtc::scoped_refptr frame_transformer) { + worker_thread_->Invoke(RTC_FROM_HERE, [&] { + RTC_DCHECK_RUN_ON(worker_thread_); + frame_transformer_ = std::move(frame_transformer); + if (media_channel_ && ssrc_.has_value() && !stopped_) { + media_channel_->SetDepacketizerToDecoderFrameTransformer( + *ssrc_, frame_transformer_); + } + }); +} + void VideoRtpReceiver::Stop() { // TODO(deadbeef): Need to do more here to fully stop receiving packets. if (stopped_) { @@ -144,6 +156,11 @@ void VideoRtpReceiver::RestartMediaChannel(absl::optional ssrc) { if (encoded_sink_enabled) { SetEncodedSinkEnabled(true); } + + if (frame_transformer_ && media_channel_ && ssrc_.has_value()) { + media_channel_->SetDepacketizerToDecoderFrameTransformer( + *ssrc_, frame_transformer_); + } }); // Attach any existing frame decryptor to the media channel. diff --git a/pc/video_rtp_receiver.h b/pc/video_rtp_receiver.h index 0b8a73da61..f66a8a7892 100644 --- a/pc/video_rtp_receiver.h +++ b/pc/video_rtp_receiver.h @@ -18,6 +18,7 @@ #include "absl/types/optional.h" #include "api/crypto/frame_decryptor_interface.h" +#include "api/frame_transformer_interface.h" #include "api/media_stream_interface.h" #include "api/media_types.h" #include "api/rtp_parameters.h" @@ -83,6 +84,9 @@ class VideoRtpReceiver : public rtc::RefCountedObject, rtc::scoped_refptr GetFrameDecryptor() const override; + void SetDepacketizerToDecoderFrameTransformer( + rtc::scoped_refptr frame_transformer) override; + // RtpReceiverInternal implementation. void Stop() override; void SetupMediaChannel(uint32_t ssrc) override; @@ -134,6 +138,8 @@ class VideoRtpReceiver : public rtc::RefCountedObject, int attachment_id_ = 0; rtc::scoped_refptr frame_decryptor_; rtc::scoped_refptr dtls_transport_; + rtc::scoped_refptr frame_transformer_ + RTC_GUARDED_BY(worker_thread_); // Allows to thread safely change jitter buffer delay. Handles caching cases // if |SetJitterBufferMinimumDelay| is called before start. rtc::scoped_refptr delay_; diff --git a/video/BUILD.gn b/video/BUILD.gn index 7bab757a24..2f7ec1788b 100644 --- a/video/BUILD.gn +++ b/video/BUILD.gn @@ -54,6 +54,7 @@ rtc_library("video") { ":frame_dumping_decoder", "../api:array_view", "../api:fec_controller_api", + "../api:frame_transformer_interface", "../api:libjingle_peerconnection_api", "../api:rtp_parameters", "../api:scoped_refptr", diff --git a/video/rtp_video_stream_receiver.cc b/video/rtp_video_stream_receiver.cc index 52f1014746..5a6c6270be 100644 --- a/video/rtp_video_stream_receiver.cc +++ b/video/rtp_video_stream_receiver.cc @@ -12,6 +12,7 @@ #include #include +#include #include #include @@ -194,7 +195,8 @@ RtpVideoStreamReceiver::RtpVideoStreamReceiver( NackSender* nack_sender, KeyFrameRequestSender* keyframe_request_sender, video_coding::OnCompleteFrameCallback* complete_frame_callback, - rtc::scoped_refptr frame_decryptor) + rtc::scoped_refptr frame_decryptor, + rtc::scoped_refptr frame_transformer) : clock_(clock), config_(*config), packet_router_(packet_router), @@ -221,7 +223,8 @@ RtpVideoStreamReceiver::RtpVideoStreamReceiver( packet_buffer_(clock_, kPacketBufferStartSize, PacketBufferMaxSize()), has_received_frame_(false), frames_decryptable_(false), - absolute_capture_time_receiver_(clock) { + absolute_capture_time_receiver_(clock), + frame_transformer_(frame_transformer) { constexpr bool remb_candidate = true; if (packet_router_) packet_router_->AddReceiveRtpModule(rtp_rtcp_.get(), remb_candidate); @@ -283,6 +286,33 @@ RtpVideoStreamReceiver::RtpVideoStreamReceiver( } } +RtpVideoStreamReceiver::RtpVideoStreamReceiver( + Clock* clock, + Transport* transport, + RtcpRttStats* rtt_stats, + PacketRouter* packet_router, + const VideoReceiveStream::Config* config, + ReceiveStatistics* rtp_receive_statistics, + ReceiveStatisticsProxy* receive_stats_proxy, + ProcessThread* process_thread, + NackSender* nack_sender, + KeyFrameRequestSender* keyframe_request_sender, + video_coding::OnCompleteFrameCallback* complete_frame_callback, + rtc::scoped_refptr frame_decryptor) + : RtpVideoStreamReceiver(clock, + transport, + rtt_stats, + packet_router, + config, + rtp_receive_statistics, + receive_stats_proxy, + process_thread, + nack_sender, + keyframe_request_sender, + complete_frame_callback, + frame_decryptor, + nullptr) {} + RtpVideoStreamReceiver::~RtpVideoStreamReceiver() { RTC_DCHECK(secondary_sinks_.empty()); diff --git a/video/rtp_video_stream_receiver.h b/video/rtp_video_stream_receiver.h index c0271f4cb1..98b324ca96 100644 --- a/video/rtp_video_stream_receiver.h +++ b/video/rtp_video_stream_receiver.h @@ -69,6 +69,29 @@ class RtpVideoStreamReceiver : public LossNotificationSender, public OnDecryptedFrameCallback, public OnDecryptionStatusChangeCallback { public: + RtpVideoStreamReceiver( + Clock* clock, + Transport* transport, + RtcpRttStats* rtt_stats, + // The packet router is optional; if provided, the RtpRtcp module for this + // stream is registered as a candidate for sending REMB and transport + // feedback. + PacketRouter* packet_router, + const VideoReceiveStream::Config* config, + ReceiveStatistics* rtp_receive_statistics, + ReceiveStatisticsProxy* receive_stats_proxy, + ProcessThread* process_thread, + NackSender* nack_sender, + // The KeyFrameRequestSender is optional; if not provided, key frame + // requests are sent via the internal RtpRtcp module. + KeyFrameRequestSender* keyframe_request_sender, + video_coding::OnCompleteFrameCallback* complete_frame_callback, + rtc::scoped_refptr frame_decryptor, + rtc::scoped_refptr frame_transformer); + + // TODO(bugs.webrtc.org/11380) remove after updating downstream dependencies + // to use the new constructor. + RTC_DEPRECATED RtpVideoStreamReceiver( Clock* clock, Transport* transport, @@ -346,6 +369,8 @@ class RtpVideoStreamReceiver : public LossNotificationSender, RTC_GUARDED_BY(worker_task_checker_); int64_t last_completed_picture_id_ = 0; + + rtc::scoped_refptr frame_transformer_; }; } // namespace webrtc diff --git a/video/rtp_video_stream_receiver_unittest.cc b/video/rtp_video_stream_receiver_unittest.cc index 44f544ffdb..d7d02b0731 100644 --- a/video/rtp_video_stream_receiver_unittest.cc +++ b/video/rtp_video_stream_receiver_unittest.cc @@ -155,7 +155,7 @@ class RtpVideoStreamReceiverTest : public ::testing::Test { Clock::GetRealTimeClock(), &mock_transport_, nullptr, nullptr, &config_, rtp_receive_statistics_.get(), nullptr, process_thread_.get(), &mock_nack_sender_, &mock_key_frame_request_sender_, - &mock_on_complete_frame_callback_, nullptr); + &mock_on_complete_frame_callback_, nullptr, nullptr); } RTPVideoHeader GetDefaultH264VideoHeader() { diff --git a/video/video_receive_stream.cc b/video/video_receive_stream.cc index 0a2d819ad1..246daadb81 100644 --- a/video/video_receive_stream.cc +++ b/video/video_receive_stream.cc @@ -215,7 +215,8 @@ VideoReceiveStream::VideoReceiveStream( this, // NackSender nullptr, // Use default KeyFrameRequestSender this, // OnCompleteFrameCallback - config_.frame_decryptor), + config_.frame_decryptor, + config_.frame_transformer), rtp_stream_sync_(this), max_wait_for_keyframe_ms_(KeyframeIntervalSettings::ParseFromFieldTrials() .MaxWaitForKeyframeMs()