diff --git a/api/frame_transformer_interface.h b/api/frame_transformer_interface.h index 666a791f91..615c97466e 100644 --- a/api/frame_transformer_interface.h +++ b/api/frame_transformer_interface.h @@ -20,18 +20,54 @@ namespace webrtc { +// Owns the frame payload data. +class TransformableFrameInterface { + public: + virtual ~TransformableFrameInterface() = default; + + // Returns the frame payload data. The data is valid until the next non-const + // method call. + virtual rtc::ArrayView GetData() const = 0; + + // Copies |data| into the owned frame payload data. + virtual void SetData(rtc::ArrayView data) = 0; + + virtual uint32_t GetTimestamp() const = 0; + virtual uint32_t GetSsrc() const = 0; +}; + +class TransformableVideoFrameInterface : public TransformableFrameInterface { + public: + virtual ~TransformableVideoFrameInterface() = default; + virtual bool IsKeyFrame() const = 0; + + // Returns data needed in the frame transformation logic; for example, + // when the transformation applied to the frame is encryption/decryption, the + // additional data holds the serialized generic frame descriptor extension + // calculated in webrtc::RtpDescriptorAuthentication. + // TODO(bugs.webrtc.org/11380) remove from interface once + // webrtc::RtpDescriptorAuthentication is exposed in api/. + virtual std::vector GetAdditionalData() const = 0; +}; + // Objects implement this interface to be notified with the transformed frame. class TransformedFrameCallback : public rtc::RefCountInterface { public: + // TODO(bugs.webrtc.org/11380) remove after updating downstream dependencies + // to use new OnTransformedFrame signature. virtual void OnTransformedFrame( - std::unique_ptr transformed_frame) = 0; + std::unique_ptr transformed_frame) {} + // TODO(bugs.webrtc.org/11380) make pure virtual after updating usage + // downstream. + virtual void OnTransformedFrame( + std::unique_ptr transformed_frame) {} protected: ~TransformedFrameCallback() override = default; }; -// Transformes encoded frames. The transformed frame is sent in a callback using -// the TransformedFrameCallback interface (see below). +// Transforms encoded frames. The transformed frame is sent in a callback using +// the TransformedFrameCallback interface (see above). class FrameTransformerInterface : public rtc::RefCountInterface { public: // Transforms |frame| using the implementing class' processing logic. @@ -41,9 +77,17 @@ class FrameTransformerInterface : public rtc::RefCountInterface { // holds the serialized generic frame descriptor extension calculated in // webrtc::RtpDescriptorAuthentication, needed in the encryption/decryption // algorithms. + // TODO(bugs.webrtc.org/11380) remove after updating downstream dependencies + // to use new OnTransformedFrame() signature. virtual void TransformFrame(std::unique_ptr frame, std::vector additional_data, - uint32_t ssrc) = 0; + uint32_t ssrc) {} + + // Transforms |frame| using the implementing class' processing logic. + // TODO(bugs.webrtc.org/11380) make pure virtual after updating usage + // downstream. + virtual void Transform( + std::unique_ptr transformable_frame) {} virtual void RegisterTransformedFrameCallback( rtc::scoped_refptr) = 0; diff --git a/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc b/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc index 80753f3fb0..fa8309bac7 100644 --- a/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc +++ b/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc @@ -11,6 +11,7 @@ #include "modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h" #include +#include #include "absl/memory/memory.h" #include "modules/rtp_rtcp/source/rtp_descriptor_authentication.h" @@ -19,6 +20,88 @@ #include "rtc_base/task_utils/to_queued_task.h" namespace webrtc { +namespace { + +std::unique_ptr CreateFragmentationHeader( + const RTPFragmentationHeader* fragmentation_header) { + if (!fragmentation_header) + return nullptr; + auto ret = std::make_unique(); + ret->CopyFrom(*fragmentation_header); + return ret; +} + +class TransformableVideoSenderFrame : public TransformableVideoFrameInterface { + public: + TransformableVideoSenderFrame( + const EncodedImage& encoded_image, + const RTPVideoHeader& video_header, + int payload_type, + absl::optional codec_type, + uint32_t rtp_timestamp, + const RTPFragmentationHeader* fragmentation_header, + absl::optional expected_retransmission_time_ms, + uint32_t ssrc) + : encoded_data_(encoded_image.GetEncodedData()), + header_(video_header), + frame_type_(encoded_image._frameType), + payload_type_(payload_type), + codec_type_(codec_type), + timestamp_(rtp_timestamp), + capture_time_ms_(encoded_image.capture_time_ms_), + expected_retransmission_time_ms_(expected_retransmission_time_ms), + ssrc_(ssrc), + fragmentation_header_(CreateFragmentationHeader(fragmentation_header)) { + } + + ~TransformableVideoSenderFrame() override = default; + + // Implements TransformableVideoFrameInterface. + rtc::ArrayView GetData() const override { + return *encoded_data_; + } + + void SetData(rtc::ArrayView data) override { + encoded_data_ = EncodedImageBuffer::Create(data.data(), data.size()); + } + + uint32_t GetTimestamp() const override { return timestamp_; } + uint32_t GetSsrc() const override { return ssrc_; } + + bool IsKeyFrame() const override { + return frame_type_ == VideoFrameType::kVideoFrameKey; + } + + std::vector GetAdditionalData() const override { + return RtpDescriptorAuthentication(header_); + } + + const RTPVideoHeader& GetHeader() const { return header_; } + int GetPayloadType() const { return payload_type_; } + absl::optional GetCodecType() const { return codec_type_; } + int64_t GetCaptureTimeMs() const { return capture_time_ms_; } + + RTPFragmentationHeader* GetFragmentationHeader() const { + return fragmentation_header_.get(); + } + + const absl::optional& GetExpectedRetransmissionTimeMs() const { + return expected_retransmission_time_ms_; + } + + private: + rtc::scoped_refptr encoded_data_; + const RTPVideoHeader header_; + const VideoFrameType frame_type_; + const int payload_type_; + const absl::optional codec_type_ = absl::nullopt; + const uint32_t timestamp_; + const int64_t capture_time_ms_; + const absl::optional expected_retransmission_time_ms_; + const uint32_t ssrc_; + const std::unique_ptr fragmentation_header_; +}; +} // namespace RTPSenderVideoFrameTransformerDelegate::RTPSenderVideoFrameTransformerDelegate( RTPSenderVideo* sender, @@ -41,45 +124,84 @@ bool RTPSenderVideoFrameTransformerDelegate::TransformFrame( uint32_t ssrc) { if (!encoder_queue_) encoder_queue_ = TaskQueueBase::Current(); + // TODO(bugs.webrtc.org/11380) remove once this version of TransformFrame() is + // deprecated. frame_transformer_->TransformFrame( std::make_unique( encoded_image.GetEncodedData(), video_header, payload_type, codec_type, rtp_timestamp, encoded_image.capture_time_ms_, fragmentation, expected_retransmission_time_ms), RtpDescriptorAuthentication(video_header), ssrc); + frame_transformer_->Transform(std::make_unique( + encoded_image, video_header, payload_type, codec_type, rtp_timestamp, + fragmentation, expected_retransmission_time_ms, ssrc)); return true; } void RTPSenderVideoFrameTransformerDelegate::OnTransformedFrame( std::unique_ptr frame) { - { - rtc::CritScope lock(&sender_lock_); - if (!sender_) - return; - } - auto transformed_frame = absl::WrapUnique( - static_cast(frame.release())); + rtc::CritScope lock(&sender_lock_); + + // The encoder queue gets destroyed after the sender; as long as the sender is + // alive, it's safe to post. + if (!sender_) + return; rtc::scoped_refptr delegate = this; - encoder_queue_->PostTask( - ToQueuedTask([delegate = std::move(delegate), - transformed_frame = std::move(transformed_frame)]() { - delegate->SendVideo(*transformed_frame.get()); + encoder_queue_->PostTask(ToQueuedTask( + [delegate = std::move(delegate), frame = std::move(frame)]() mutable { + delegate->SendVideo(std::move(frame)); + })); +} + +void RTPSenderVideoFrameTransformerDelegate::OnTransformedFrame( + std::unique_ptr frame) { + rtc::CritScope lock(&sender_lock_); + + // The encoder queue gets destroyed after the sender; as long as the sender is + // alive, it's safe to post. + if (!sender_) + return; + rtc::scoped_refptr delegate = this; + encoder_queue_->PostTask(ToQueuedTask( + [delegate = std::move(delegate), frame = std::move(frame)]() mutable { + delegate->SendVideo(std::move(frame)); })); } void RTPSenderVideoFrameTransformerDelegate::SendVideo( - const TransformableEncodedFrame& transformed_frame) const { + std::unique_ptr frame) const { RTC_CHECK(encoder_queue_->IsCurrent()); rtc::CritScope lock(&sender_lock_); if (!sender_) return; + auto* transformed_frame = + static_cast(frame.get()); sender_->SendVideo( - transformed_frame.PayloadType(), transformed_frame.codec_type(), - transformed_frame.Timestamp(), transformed_frame.capture_time_ms(), - transformed_frame.EncodedImage(), - transformed_frame.fragmentation_header(), - transformed_frame.video_header(), - transformed_frame.expected_retransmission_time_ms()); + transformed_frame->PayloadType(), transformed_frame->codec_type(), + transformed_frame->Timestamp(), transformed_frame->capture_time_ms(), + transformed_frame->EncodedImage(), + transformed_frame->fragmentation_header(), + transformed_frame->video_header(), + transformed_frame->expected_retransmission_time_ms()); +} + +void RTPSenderVideoFrameTransformerDelegate::SendVideo( + std::unique_ptr transformed_frame) const { + RTC_CHECK(encoder_queue_->IsCurrent()); + rtc::CritScope lock(&sender_lock_); + if (!sender_) + return; + auto* transformed_video_frame = + static_cast(transformed_frame.get()); + sender_->SendVideo( + transformed_video_frame->GetPayloadType(), + transformed_video_frame->GetCodecType(), + transformed_video_frame->GetTimestamp(), + transformed_video_frame->GetCaptureTimeMs(), + transformed_video_frame->GetData(), + transformed_video_frame->GetFragmentationHeader(), + transformed_video_frame->GetHeader(), + transformed_video_frame->GetExpectedRetransmissionTimeMs()); } void RTPSenderVideoFrameTransformerDelegate::SetVideoStructureUnderLock( diff --git a/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h b/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h index 383c4d5990..4a6482b2de 100644 --- a/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h +++ b/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h @@ -48,9 +48,12 @@ class RTPSenderVideoFrameTransformerDelegate : public TransformedFrameCallback { // the transformed frame to be sent on the |encoder_queue_|. void OnTransformedFrame( std::unique_ptr frame) override; + void OnTransformedFrame( + std::unique_ptr frame) override; // Delegates the call to RTPSendVideo::SendVideo on the |encoder_queue_|. - void SendVideo(const TransformableEncodedFrame& transformed_frame) const; + void SendVideo(std::unique_ptr frame) const; + void SendVideo(std::unique_ptr frame) const; // Delegates the call to RTPSendVideo::SendVideo under |sender_lock_|. void SetVideoStructureUnderLock( diff --git a/video/rtp_video_stream_receiver_frame_transformer_delegate.cc b/video/rtp_video_stream_receiver_frame_transformer_delegate.cc index acef31cb96..8b63c7e4df 100644 --- a/video/rtp_video_stream_receiver_frame_transformer_delegate.cc +++ b/video/rtp_video_stream_receiver_frame_transformer_delegate.cc @@ -11,6 +11,7 @@ #include "video/rtp_video_stream_receiver_frame_transformer_delegate.h" #include +#include #include "absl/memory/memory.h" #include "modules/rtp_rtcp/source/rtp_descriptor_authentication.h" @@ -20,6 +21,45 @@ namespace webrtc { +namespace { +class TransformableVideoReceiverFrame + : public TransformableVideoFrameInterface { + public: + TransformableVideoReceiverFrame( + std::unique_ptr frame, + uint32_t ssrc) + : frame_(std::move(frame)), ssrc_(ssrc) {} + ~TransformableVideoReceiverFrame() override = default; + + // Implements TransformableVideoFrameInterface. + rtc::ArrayView GetData() const override { + return *frame_->GetEncodedData(); + } + + void SetData(rtc::ArrayView data) override { + frame_->SetEncodedData( + EncodedImageBuffer::Create(data.data(), data.size())); + } + + uint32_t GetTimestamp() const override { return frame_->Timestamp(); } + uint32_t GetSsrc() const override { return ssrc_; } + + bool IsKeyFrame() const override { return frame_->is_keyframe(); } + + std::vector GetAdditionalData() const override { + return RtpDescriptorAuthentication(frame_->GetRtpVideoHeader()); + } + + std::unique_ptr ExtractFrame() && { + return std::move(frame_); + } + + private: + std::unique_ptr frame_; + const uint32_t ssrc_; +}; +} // namespace + RtpVideoStreamReceiverFrameTransformerDelegate:: RtpVideoStreamReceiverFrameTransformerDelegate( RtpVideoStreamReceiver* receiver, @@ -46,10 +86,18 @@ void RtpVideoStreamReceiverFrameTransformerDelegate::TransformFrame( std::unique_ptr frame, uint32_t ssrc) { RTC_DCHECK_RUN_ON(&network_sequence_checker_); + // TODO(bugs.webrtc.org/11380) remove once this version of TransformFrame is + // deprecated. auto additional_data = RtpDescriptorAuthentication(frame->GetRtpVideoHeader()); - frame_transformer_->TransformFrame(std::move(frame), + auto frame_copy = + std::make_unique(*frame.get()); + frame_transformer_->TransformFrame(std::move(frame_copy), std::move(additional_data), ssrc); + + frame_transformer_->Transform( + std::make_unique(std::move(frame), + ssrc)); } void RtpVideoStreamReceiverFrameTransformerDelegate::OnTransformedFrame( @@ -62,6 +110,16 @@ void RtpVideoStreamReceiverFrameTransformerDelegate::OnTransformedFrame( })); } +void RtpVideoStreamReceiverFrameTransformerDelegate::OnTransformedFrame( + std::unique_ptr frame) { + rtc::scoped_refptr delegate = + this; + network_thread_->PostTask(ToQueuedTask( + [delegate = std::move(delegate), frame = std::move(frame)]() mutable { + delegate->ManageFrame(std::move(frame)); + })); +} + void RtpVideoStreamReceiverFrameTransformerDelegate::ManageFrame( std::unique_ptr frame) { RTC_DCHECK_RUN_ON(&network_sequence_checker_); @@ -72,4 +130,14 @@ void RtpVideoStreamReceiverFrameTransformerDelegate::ManageFrame( receiver_->ManageFrame(std::move(transformed_frame)); } +void RtpVideoStreamReceiverFrameTransformerDelegate::ManageFrame( + std::unique_ptr frame) { + RTC_DCHECK_RUN_ON(&network_sequence_checker_); + if (!receiver_) + return; + auto transformed_frame = absl::WrapUnique( + static_cast(frame.release())); + receiver_->ManageFrame(std::move(*transformed_frame).ExtractFrame()); +} + } // namespace webrtc diff --git a/video/rtp_video_stream_receiver_frame_transformer_delegate.h b/video/rtp_video_stream_receiver_frame_transformer_delegate.h index 2309796258..85fad1e002 100644 --- a/video/rtp_video_stream_receiver_frame_transformer_delegate.h +++ b/video/rtp_video_stream_receiver_frame_transformer_delegate.h @@ -43,10 +43,13 @@ class RtpVideoStreamReceiverFrameTransformerDelegate // the transformed frame to be managed on the |network_thread_|. void OnTransformedFrame( std::unique_ptr frame) override; + void OnTransformedFrame( + std::unique_ptr frame) override; // Delegates the call to RtpVideoReceiver::ManageFrame on the // |network_thread_|. void ManageFrame(std::unique_ptr frame); + void ManageFrame(std::unique_ptr frame); protected: ~RtpVideoStreamReceiverFrameTransformerDelegate() override = default;