Introduce TransformableFrameInterface.

Add a new frame interface to be used by frame transformers in Insertable
Streams. TransformableFrameInterface will replace
video_coding::EncodedFrame in a follow up CL, once downstream
dependecies are updated to use the new interface.

Until the functions using video_coding::EncodedFrame are removed from
the API, the video sender and receiver frame transformer delegates call
both function versions to avoid breaking tests downstream.

The TransformableFrameInterface will be used for both audio and video
frame transformers in follow-up CLs.

Bug: webrtc:11380
Change-Id: I9389a8549c156e13b1d8c938ff51eaa69c502f33
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/171863
Commit-Queue: Marina Ciocea <marinaciocea@webrtc.org>
Reviewed-by: Magnus Flodman <mflodman@webrtc.org>
Reviewed-by: Karl Wiberg <kwiberg@webrtc.org>
Reviewed-by: Danil Chapovalov <danilchap@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#30941}

api/frame_transformer_interface.h

@@ -20,18 +20,54 @@
 
 namespace webrtc {
 
+// Owns the frame payload data.
+class TransformableFrameInterface {
+ public:
+  virtual ~TransformableFrameInterface() = default;
+
+  // Returns the frame payload data. The data is valid until the next non-const
+  // method call.
+  virtual rtc::ArrayView<const uint8_t> GetData() const = 0;
+
+  // Copies |data| into the owned frame payload data.
+  virtual void SetData(rtc::ArrayView<const uint8_t> data) = 0;
+
+  virtual uint32_t GetTimestamp() const = 0;
+  virtual uint32_t GetSsrc() const = 0;
+};
+
+class TransformableVideoFrameInterface : public TransformableFrameInterface {
+ public:
+  virtual ~TransformableVideoFrameInterface() = default;
+  virtual bool IsKeyFrame() const = 0;
+
+  // Returns data needed in the frame transformation logic; for example,
+  // when the transformation applied to the frame is encryption/decryption, the
+  // additional data holds the serialized generic frame descriptor extension
+  // calculated in webrtc::RtpDescriptorAuthentication.
+  // TODO(bugs.webrtc.org/11380) remove from interface once
+  // webrtc::RtpDescriptorAuthentication is exposed in api/.
+  virtual std::vector<uint8_t> GetAdditionalData() const = 0;
+};
+
 // Objects implement this interface to be notified with the transformed frame.
 class TransformedFrameCallback : public rtc::RefCountInterface {
  public:
+  // TODO(bugs.webrtc.org/11380) remove after updating downstream dependencies
+  // to use new OnTransformedFrame signature.
   virtual void OnTransformedFrame(
-      std::unique_ptr<video_coding::EncodedFrame> transformed_frame) = 0;
+      std::unique_ptr<video_coding::EncodedFrame> transformed_frame) {}
+  // TODO(bugs.webrtc.org/11380) make pure virtual after updating usage
+  // downstream.
+  virtual void OnTransformedFrame(
+      std::unique_ptr<TransformableFrameInterface> transformed_frame) {}
 
  protected:
   ~TransformedFrameCallback() override = default;
 };
 
-// Transformes encoded frames. The transformed frame is sent in a callback using
+// Transforms encoded frames. The transformed frame is sent in a callback using
-// the TransformedFrameCallback interface (see below).
+// the TransformedFrameCallback interface (see above).
 class FrameTransformerInterface : public rtc::RefCountInterface {
  public:
   // Transforms |frame| using the implementing class' processing logic.
@@ -41,9 +77,17 @@ class FrameTransformerInterface : public rtc::RefCountInterface {
   // holds the serialized generic frame descriptor extension calculated in
   // webrtc::RtpDescriptorAuthentication, needed in the encryption/decryption
   // algorithms.
+  // TODO(bugs.webrtc.org/11380) remove after updating downstream dependencies
+  // to use new OnTransformedFrame() signature.
   virtual void TransformFrame(std::unique_ptr<video_coding::EncodedFrame> frame,
                               std::vector<uint8_t> additional_data,
-                              uint32_t ssrc) = 0;
+                              uint32_t ssrc) {}
+
+  // Transforms |frame| using the implementing class' processing logic.
+  // TODO(bugs.webrtc.org/11380) make pure virtual after updating usage
+  // downstream.
+  virtual void Transform(
+      std::unique_ptr<TransformableFrameInterface> transformable_frame) {}
 
   virtual void RegisterTransformedFrameCallback(
       rtc::scoped_refptr<TransformedFrameCallback>) = 0;

modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc

@@ -11,6 +11,7 @@
 #include "modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h"
 
 #include <utility>
+#include <vector>
 
 #include "absl/memory/memory.h"
 #include "modules/rtp_rtcp/source/rtp_descriptor_authentication.h"
@@ -19,6 +20,88 @@
 #include "rtc_base/task_utils/to_queued_task.h"
 
 namespace webrtc {
+namespace {
+
+std::unique_ptr<RTPFragmentationHeader> CreateFragmentationHeader(
+    const RTPFragmentationHeader* fragmentation_header) {
+  if (!fragmentation_header)
+    return nullptr;
+  auto ret = std::make_unique<RTPFragmentationHeader>();
+  ret->CopyFrom(*fragmentation_header);
+  return ret;
+}
+
+class TransformableVideoSenderFrame : public TransformableVideoFrameInterface {
+ public:
+  TransformableVideoSenderFrame(
+      const EncodedImage& encoded_image,
+      const RTPVideoHeader& video_header,
+      int payload_type,
+      absl::optional<VideoCodecType> codec_type,
+      uint32_t rtp_timestamp,
+      const RTPFragmentationHeader* fragmentation_header,
+      absl::optional<int64_t> expected_retransmission_time_ms,
+      uint32_t ssrc)
+      : encoded_data_(encoded_image.GetEncodedData()),
+        header_(video_header),
+        frame_type_(encoded_image._frameType),
+        payload_type_(payload_type),
+        codec_type_(codec_type),
+        timestamp_(rtp_timestamp),
+        capture_time_ms_(encoded_image.capture_time_ms_),
+        expected_retransmission_time_ms_(expected_retransmission_time_ms),
+        ssrc_(ssrc),
+        fragmentation_header_(CreateFragmentationHeader(fragmentation_header)) {
+  }
+
+  ~TransformableVideoSenderFrame() override = default;
+
+  // Implements TransformableVideoFrameInterface.
+  rtc::ArrayView<const uint8_t> GetData() const override {
+    return *encoded_data_;
+  }
+
+  void SetData(rtc::ArrayView<const uint8_t> data) override {
+    encoded_data_ = EncodedImageBuffer::Create(data.data(), data.size());
+  }
+
+  uint32_t GetTimestamp() const override { return timestamp_; }
+  uint32_t GetSsrc() const override { return ssrc_; }
+
+  bool IsKeyFrame() const override {
+    return frame_type_ == VideoFrameType::kVideoFrameKey;
+  }
+
+  std::vector<uint8_t> GetAdditionalData() const override {
+    return RtpDescriptorAuthentication(header_);
+  }
+
+  const RTPVideoHeader& GetHeader() const { return header_; }
+  int GetPayloadType() const { return payload_type_; }
+  absl::optional<VideoCodecType> GetCodecType() const { return codec_type_; }
+  int64_t GetCaptureTimeMs() const { return capture_time_ms_; }
+
+  RTPFragmentationHeader* GetFragmentationHeader() const {
+    return fragmentation_header_.get();
+  }
+
+  const absl::optional<int64_t>& GetExpectedRetransmissionTimeMs() const {
+    return expected_retransmission_time_ms_;
+  }
+
+ private:
+  rtc::scoped_refptr<EncodedImageBufferInterface> encoded_data_;
+  const RTPVideoHeader header_;
+  const VideoFrameType frame_type_;
+  const int payload_type_;
+  const absl::optional<VideoCodecType> codec_type_ = absl::nullopt;
+  const uint32_t timestamp_;
+  const int64_t capture_time_ms_;
+  const absl::optional<int64_t> expected_retransmission_time_ms_;
+  const uint32_t ssrc_;
+  const std::unique_ptr<RTPFragmentationHeader> fragmentation_header_;
+};
+}  // namespace
 
 RTPSenderVideoFrameTransformerDelegate::RTPSenderVideoFrameTransformerDelegate(
     RTPSenderVideo* sender,
@@ -41,45 +124,84 @@ bool RTPSenderVideoFrameTransformerDelegate::TransformFrame(
     uint32_t ssrc) {
   if (!encoder_queue_)
     encoder_queue_ = TaskQueueBase::Current();
+  // TODO(bugs.webrtc.org/11380) remove once this version of TransformFrame() is
+  // deprecated.
   frame_transformer_->TransformFrame(
       std::make_unique<TransformableEncodedFrame>(
           encoded_image.GetEncodedData(), video_header, payload_type,
           codec_type, rtp_timestamp, encoded_image.capture_time_ms_,
           fragmentation, expected_retransmission_time_ms),
       RtpDescriptorAuthentication(video_header), ssrc);
+  frame_transformer_->Transform(std::make_unique<TransformableVideoSenderFrame>(
+      encoded_image, video_header, payload_type, codec_type, rtp_timestamp,
+      fragmentation, expected_retransmission_time_ms, ssrc));
   return true;
 }
 
 void RTPSenderVideoFrameTransformerDelegate::OnTransformedFrame(
     std::unique_ptr<video_coding::EncodedFrame> frame) {
-  {
   rtc::CritScope lock(&sender_lock_);
+
+  // The encoder queue gets destroyed after the sender; as long as the sender is
+  // alive, it's safe to post.
   if (!sender_)
     return;
-  }
-  auto transformed_frame = absl::WrapUnique(
-      static_cast<TransformableEncodedFrame*>(frame.release()));
   rtc::scoped_refptr<RTPSenderVideoFrameTransformerDelegate> delegate = this;
-  encoder_queue_->PostTask(
+  encoder_queue_->PostTask(ToQueuedTask(
-      ToQueuedTask([delegate = std::move(delegate),
+      [delegate = std::move(delegate), frame = std::move(frame)]() mutable {
-                    transformed_frame = std::move(transformed_frame)]() {
+        delegate->SendVideo(std::move(frame));
-        delegate->SendVideo(*transformed_frame.get());
+      }));
+}
+
+void RTPSenderVideoFrameTransformerDelegate::OnTransformedFrame(
+    std::unique_ptr<TransformableFrameInterface> frame) {
+  rtc::CritScope lock(&sender_lock_);
+
+  // The encoder queue gets destroyed after the sender; as long as the sender is
+  // alive, it's safe to post.
+  if (!sender_)
+    return;
+  rtc::scoped_refptr<RTPSenderVideoFrameTransformerDelegate> delegate = this;
+  encoder_queue_->PostTask(ToQueuedTask(
+      [delegate = std::move(delegate), frame = std::move(frame)]() mutable {
+        delegate->SendVideo(std::move(frame));
       }));
 }
 
 void RTPSenderVideoFrameTransformerDelegate::SendVideo(
-    const TransformableEncodedFrame& transformed_frame) const {
+    std::unique_ptr<video_coding::EncodedFrame> frame) const {
   RTC_CHECK(encoder_queue_->IsCurrent());
   rtc::CritScope lock(&sender_lock_);
   if (!sender_)
     return;
+  auto* transformed_frame =
+      static_cast<TransformableEncodedFrame*>(frame.get());
   sender_->SendVideo(
-      transformed_frame.PayloadType(), transformed_frame.codec_type(),
+      transformed_frame->PayloadType(), transformed_frame->codec_type(),
-      transformed_frame.Timestamp(), transformed_frame.capture_time_ms(),
+      transformed_frame->Timestamp(), transformed_frame->capture_time_ms(),
-      transformed_frame.EncodedImage(),
+      transformed_frame->EncodedImage(),
-      transformed_frame.fragmentation_header(),
+      transformed_frame->fragmentation_header(),
-      transformed_frame.video_header(),
+      transformed_frame->video_header(),
-      transformed_frame.expected_retransmission_time_ms());
+      transformed_frame->expected_retransmission_time_ms());
+}
+
+void RTPSenderVideoFrameTransformerDelegate::SendVideo(
+    std::unique_ptr<TransformableFrameInterface> transformed_frame) const {
+  RTC_CHECK(encoder_queue_->IsCurrent());
+  rtc::CritScope lock(&sender_lock_);
+  if (!sender_)
+    return;
+  auto* transformed_video_frame =
+      static_cast<TransformableVideoSenderFrame*>(transformed_frame.get());
+  sender_->SendVideo(
+      transformed_video_frame->GetPayloadType(),
+      transformed_video_frame->GetCodecType(),
+      transformed_video_frame->GetTimestamp(),
+      transformed_video_frame->GetCaptureTimeMs(),
+      transformed_video_frame->GetData(),
+      transformed_video_frame->GetFragmentationHeader(),
+      transformed_video_frame->GetHeader(),
+      transformed_video_frame->GetExpectedRetransmissionTimeMs());
 }
 
 void RTPSenderVideoFrameTransformerDelegate::SetVideoStructureUnderLock(

modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h

@@ -48,9 +48,12 @@ class RTPSenderVideoFrameTransformerDelegate : public TransformedFrameCallback {
   // the transformed frame to be sent on the |encoder_queue_|.
   void OnTransformedFrame(
       std::unique_ptr<video_coding::EncodedFrame> frame) override;
+  void OnTransformedFrame(
+      std::unique_ptr<TransformableFrameInterface> frame) override;
 
   // Delegates the call to RTPSendVideo::SendVideo on the |encoder_queue_|.
-  void SendVideo(const TransformableEncodedFrame& transformed_frame) const;
+  void SendVideo(std::unique_ptr<video_coding::EncodedFrame> frame) const;
+  void SendVideo(std::unique_ptr<TransformableFrameInterface> frame) const;
 
   // Delegates the call to RTPSendVideo::SendVideo under |sender_lock_|.
   void SetVideoStructureUnderLock(

video/rtp_video_stream_receiver_frame_transformer_delegate.cc

@@ -11,6 +11,7 @@
 #include "video/rtp_video_stream_receiver_frame_transformer_delegate.h"
 
 #include <utility>
+#include <vector>
 
 #include "absl/memory/memory.h"
 #include "modules/rtp_rtcp/source/rtp_descriptor_authentication.h"
@@ -20,6 +21,45 @@
 
 namespace webrtc {
 
+namespace {
+class TransformableVideoReceiverFrame
+    : public TransformableVideoFrameInterface {
+ public:
+  TransformableVideoReceiverFrame(
+      std::unique_ptr<video_coding::RtpFrameObject> frame,
+      uint32_t ssrc)
+      : frame_(std::move(frame)), ssrc_(ssrc) {}
+  ~TransformableVideoReceiverFrame() override = default;
+
+  // Implements TransformableVideoFrameInterface.
+  rtc::ArrayView<const uint8_t> GetData() const override {
+    return *frame_->GetEncodedData();
+  }
+
+  void SetData(rtc::ArrayView<const uint8_t> data) override {
+    frame_->SetEncodedData(
+        EncodedImageBuffer::Create(data.data(), data.size()));
+  }
+
+  uint32_t GetTimestamp() const override { return frame_->Timestamp(); }
+  uint32_t GetSsrc() const override { return ssrc_; }
+
+  bool IsKeyFrame() const override { return frame_->is_keyframe(); }
+
+  std::vector<uint8_t> GetAdditionalData() const override {
+    return RtpDescriptorAuthentication(frame_->GetRtpVideoHeader());
+  }
+
+  std::unique_ptr<video_coding::RtpFrameObject> ExtractFrame() && {
+    return std::move(frame_);
+  }
+
+ private:
+  std::unique_ptr<video_coding::RtpFrameObject> frame_;
+  const uint32_t ssrc_;
+};
+}  // namespace
+
 RtpVideoStreamReceiverFrameTransformerDelegate::
     RtpVideoStreamReceiverFrameTransformerDelegate(
         RtpVideoStreamReceiver* receiver,
@@ -46,10 +86,18 @@ void RtpVideoStreamReceiverFrameTransformerDelegate::TransformFrame(
     std::unique_ptr<video_coding::RtpFrameObject> frame,
     uint32_t ssrc) {
   RTC_DCHECK_RUN_ON(&network_sequence_checker_);
+  // TODO(bugs.webrtc.org/11380) remove once this version of TransformFrame is
+  // deprecated.
   auto additional_data =
       RtpDescriptorAuthentication(frame->GetRtpVideoHeader());
-  frame_transformer_->TransformFrame(std::move(frame),
+  auto frame_copy =
+      std::make_unique<video_coding::RtpFrameObject>(*frame.get());
+  frame_transformer_->TransformFrame(std::move(frame_copy),
                                      std::move(additional_data), ssrc);
+
+  frame_transformer_->Transform(
+      std::make_unique<TransformableVideoReceiverFrame>(std::move(frame),
+                                                        ssrc));
 }
 
 void RtpVideoStreamReceiverFrameTransformerDelegate::OnTransformedFrame(
@@ -62,6 +110,16 @@ void RtpVideoStreamReceiverFrameTransformerDelegate::OnTransformedFrame(
       }));
 }
 
+void RtpVideoStreamReceiverFrameTransformerDelegate::OnTransformedFrame(
+    std::unique_ptr<TransformableFrameInterface> frame) {
+  rtc::scoped_refptr<RtpVideoStreamReceiverFrameTransformerDelegate> delegate =
+      this;
+  network_thread_->PostTask(ToQueuedTask(
+      [delegate = std::move(delegate), frame = std::move(frame)]() mutable {
+        delegate->ManageFrame(std::move(frame));
+      }));
+}
+
 void RtpVideoStreamReceiverFrameTransformerDelegate::ManageFrame(
     std::unique_ptr<video_coding::EncodedFrame> frame) {
   RTC_DCHECK_RUN_ON(&network_sequence_checker_);
@@ -72,4 +130,14 @@ void RtpVideoStreamReceiverFrameTransformerDelegate::ManageFrame(
   receiver_->ManageFrame(std::move(transformed_frame));
 }
 
+void RtpVideoStreamReceiverFrameTransformerDelegate::ManageFrame(
+    std::unique_ptr<TransformableFrameInterface> frame) {
+  RTC_DCHECK_RUN_ON(&network_sequence_checker_);
+  if (!receiver_)
+    return;
+  auto transformed_frame = absl::WrapUnique(
+      static_cast<TransformableVideoReceiverFrame*>(frame.release()));
+  receiver_->ManageFrame(std::move(*transformed_frame).ExtractFrame());
+}
+
 }  // namespace webrtc

video/rtp_video_stream_receiver_frame_transformer_delegate.h

@@ -43,10 +43,13 @@ class RtpVideoStreamReceiverFrameTransformerDelegate
   // the transformed frame to be managed on the |network_thread_|.
   void OnTransformedFrame(
       std::unique_ptr<video_coding::EncodedFrame> frame) override;
+  void OnTransformedFrame(
+      std::unique_ptr<TransformableFrameInterface> frame) override;
 
   // Delegates the call to RtpVideoReceiver::ManageFrame on the
   // |network_thread_|.
   void ManageFrame(std::unique_ptr<video_coding::EncodedFrame> frame);
+  void ManageFrame(std::unique_ptr<TransformableFrameInterface> frame);
 
  protected:
   ~RtpVideoStreamReceiverFrameTransformerDelegate() override = default;