Add a clone method to the video frame transformer API.

This will clone an encoded video frame into a sender frame,
preserving metadata as much as possible.

Bug: webrtc:14708
Change-Id: I6f68d2ee65ef85c32cc3c142a41346b81ba73533
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/284701
Commit-Queue: Harald Alvestrand <hta@webrtc.org>
Reviewed-by: Guido Urdaneta <guidou@webrtc.org>
Reviewed-by: Henrik Boström <hbos@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#38733}

api/BUILD.gn

@@ -1466,3 +1466,19 @@ rtc_library("field_trials") {
   ]
   absl_deps = [ "//third_party/abseil-cpp/absl/strings" ]
 }
+
+rtc_library("frame_transformer_factory") {
+  visibility = [ "*" ]
+  sources = [
+    "frame_transformer_factory.cc",
+    "frame_transformer_factory.h",
+  ]
+  deps = [
+    ":frame_transformer_interface",
+    ":scoped_refptr",
+    "../modules/rtp_rtcp",
+    "../rtc_base:refcount",
+    "video:encoded_frame",
+    "video:video_frame_metadata",
+  ]
+}

api/frame_transformer_factory.cc

@@ -0,0 +1,33 @@
+/*
+ *  Copyright 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "api/frame_transformer_factory.h"
+
+#include "modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h"
+
+namespace webrtc {
+
+std::unique_ptr<TransformableVideoFrameInterface> CreateVideoSenderFrame() {
+  RTC_CHECK_NOTREACHED();
+  return nullptr;
+}
+
+std::unique_ptr<TransformableVideoFrameInterface> CreateVideoReceiverFrame() {
+  RTC_CHECK_NOTREACHED();
+  return nullptr;
+}
+
+std::unique_ptr<TransformableVideoFrameInterface> CloneVideoFrame(
+    TransformableVideoFrameInterface* original) {
+  // At the moment, only making sender frames from receiver frames is supported.
+  return CloneSenderVideoFrame(original);
+}
+
+}  // namespace webrtc

api/frame_transformer_factory.h

@@ -0,0 +1,39 @@
+/*
+ *  Copyright 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef API_FRAME_TRANSFORMER_FACTORY_H_
+#define API_FRAME_TRANSFORMER_FACTORY_H_
+
+#include <memory>
+#include <vector>
+
+#include "api/frame_transformer_interface.h"
+#include "api/scoped_refptr.h"
+#include "api/video/encoded_frame.h"
+#include "api/video/video_frame_metadata.h"
+
+// This file contains EXPERIMENTAL functions to create video frames from
+// either an old video frame or directly from parameters.
+// These functions will be used in Chrome functionality to manipulate
+// encoded frames from Javascript.
+namespace webrtc {
+
+// TODO(bugs.webrtc.org/14708): Add the required parameters to these APIs.
+std::unique_ptr<TransformableVideoFrameInterface> CreateVideoSenderFrame();
+// TODO(bugs.webrtc.org/14708): Consider whether Receiver frames ever make sense
+// to create.
+std::unique_ptr<TransformableVideoFrameInterface> CreateVideoReceiverFrame();
+// Creates a new frame with the same metadata as the original.
+// The original can be a sender or receiver frame.
+RTC_EXPORT std::unique_ptr<TransformableVideoFrameInterface> CloneVideoFrame(
+    TransformableVideoFrameInterface* original);
+}  // namespace webrtc
+
+#endif  // API_FRAME_TRANSFORMER_FACTORY_H_

modules/rtp_rtcp/BUILD.gn

@@ -596,6 +596,7 @@ if (rtc_include_tests) {
     ]
     deps = [
       ":fec_test_helper",
+      ":frame_transformer_factory_unittest",
       ":mock_rtp_rtcp",
       ":rtcp_transceiver",
       ":rtp_packetizer_av1_test_helper",
@@ -605,6 +606,7 @@ if (rtc_include_tests) {
       "../../api:array_view",
       "../../api:create_time_controller",
       "../../api:field_trials_registry",
+      "../../api:frame_transformer_factory",
       "../../api:libjingle_peerconnection_api",
       "../../api:mock_frame_encryptor",
       "../../api:rtp_headers",
@@ -669,3 +671,19 @@ if (rtc_include_tests) {
     ]
   }
 }
+
+rtc_source_set("frame_transformer_factory_unittest") {
+  testonly = true
+  sources = [ "source/frame_transformer_factory_unittest.cc" ]
+  deps = [
+    "../../api:frame_transformer_factory",
+    "../../api:transport_api",
+    "../../call:video_stream_api",
+    "../../modules/rtp_rtcp",
+    "../../rtc_base:rtc_event",
+    "../../test:mock_frame_transformer",
+    "../../test:test_support",
+    "../../video",
+  ]
+  absl_deps = [ "//third_party/abseil-cpp/absl/memory" ]
+}

modules/rtp_rtcp/source/frame_transformer_factory_unittest.cc

@@ -0,0 +1,65 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "api/frame_transformer_factory.h"
+
+#include <cstdio>
+#include <memory>
+#include <utility>
+#include <vector>
+
+#include "absl/memory/memory.h"
+#include "api/call/transport.h"
+#include "call/video_receive_stream.h"
+#include "modules/rtp_rtcp/source/rtp_descriptor_authentication.h"
+#include "rtc_base/event.h"
+#include "test/gmock.h"
+#include "test/gtest.h"
+#include "test/mock_frame_transformer.h"
+
+namespace webrtc {
+namespace {
+
+using testing::NiceMock;
+using testing::Return;
+
+class MockTransformableVideoFrame
+    : public webrtc::TransformableVideoFrameInterface {
+ public:
+  MOCK_METHOD(rtc::ArrayView<const uint8_t>, GetData, (), (const override));
+  MOCK_METHOD(void, SetData, (rtc::ArrayView<const uint8_t> data), (override));
+  MOCK_METHOD(uint8_t, GetPayloadType, (), (const, override));
+  MOCK_METHOD(uint32_t, GetSsrc, (), (const, override));
+  MOCK_METHOD(uint32_t, GetTimestamp, (), (const, override));
+  MOCK_METHOD(TransformableFrameInterface::Direction,
+              GetDirection,
+              (),
+              (const, override));
+  MOCK_METHOD(bool, IsKeyFrame, (), (const, override));
+  MOCK_METHOD(std::vector<uint8_t>, GetAdditionalData, (), (const, override));
+  MOCK_METHOD(const webrtc::VideoFrameMetadata&,
+              GetMetadata,
+              (),
+              (const, override));
+};
+
+TEST(FrameTransformerFactory, CloneVideoFrame) {
+  NiceMock<MockTransformableVideoFrame> original_frame;
+  uint8_t data[10];
+  std::fill_n(data, 10, 5);
+  rtc::ArrayView<uint8_t> data_view(data);
+  EXPECT_CALL(original_frame, GetData()).WillRepeatedly(Return(data_view));
+  auto cloned_frame = CloneVideoFrame(&original_frame);
+  EXPECT_EQ(cloned_frame->GetData().size(), 10u);
+  EXPECT_THAT(cloned_frame->GetData(), testing::Each(5u));
+}
+
+}  // namespace
+}  // namespace webrtc

modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc

@@ -18,6 +18,7 @@
 #include "modules/rtp_rtcp/source/rtp_descriptor_authentication.h"
 #include "modules/rtp_rtcp/source/rtp_sender_video.h"
 #include "rtc_base/checks.h"
+#include "rtc_base/logging.h"
 
 namespace webrtc {
 namespace {
@@ -161,8 +162,9 @@ void RTPSenderVideoFrameTransformerDelegate::OnTransformedFrame(
 
   EnsureEncoderQueueCreated();
 
-  if (!sender_)
+  if (!sender_) {
     return;
+  }
   rtc::scoped_refptr<RTPSenderVideoFrameTransformerDelegate> delegate(this);
   encoder_queue_->PostTask(
       [delegate = std::move(delegate), frame = std::move(frame)]() mutable {
@@ -212,4 +214,37 @@ void RTPSenderVideoFrameTransformerDelegate::Reset() {
     sender_ = nullptr;
   }
 }
+
+std::unique_ptr<TransformableVideoFrameInterface> CloneSenderVideoFrame(
+    TransformableVideoFrameInterface* original) {
+  auto encoded_image_buffer = EncodedImageBuffer::Create(
+      original->GetData().data(), original->GetData().size());
+  EncodedImage encoded_image;
+  encoded_image.SetEncodedData(encoded_image_buffer);
+  RTPVideoHeader new_header;
+  absl::optional<VideoCodecType> new_codec_type;
+  // TODO(bugs.webrtc.org/14708): Figure out a way to get the header information
+  // without casting to TransformableVideoSenderFrame.
+  if (original->GetDirection() ==
+      TransformableFrameInterface::Direction::kSender) {
+    // TODO(bugs.webrtc.org/14708): Figure out a way to bulletproof this cast.
+    auto original_as_sender =
+        static_cast<TransformableVideoSenderFrame*>(original);
+    new_header = original_as_sender->GetHeader();
+    new_codec_type = original_as_sender->GetCodecType();
+  } else {
+    // TODO(bugs.webrtc.org/14708): Make this codec dependent
+    new_header.video_type_header.emplace<RTPVideoHeaderVP8>();
+    new_codec_type = kVideoCodecVP8;
+    // TODO(bugs.webrtc.org/14708): Fill in the new_header when it's not
+    // `Direction::kSender`
+  }
+  // TODO(bugs.webrtc.org/14708): Fill in other EncodedImage parameters
+  return std::make_unique<TransformableVideoSenderFrame>(
+      encoded_image, new_header, original->GetPayloadType(), new_codec_type,
+      original->GetTimestamp(),
+      absl::nullopt,  // expected_retransmission_time_ms
+      original->GetSsrc());
+}
+
 }  // namespace webrtc

modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h

@@ -87,6 +87,10 @@ class RTPSenderVideoFrameTransformerDelegate : public TransformedFrameCallback {
   std::unique_ptr<TaskQueueBase, TaskQueueDeleter> owned_encoder_queue_;
 };
 
+// Method to support cloning a Sender frame from another frame
+std::unique_ptr<TransformableVideoFrameInterface> CloneSenderVideoFrame(
+    TransformableVideoFrameInterface* original);
+
 }  // namespace webrtc
 
 #endif  // MODULES_RTP_RTCP_SOURCE_RTP_SENDER_VIDEO_FRAME_TRANSFORMER_DELEGATE_H_

modules/rtp_rtcp/source/rtp_sender_video_unittest.cc

@@ -17,6 +17,7 @@
 
 #include "absl/memory/memory.h"
 #include "api/field_trials_registry.h"
+#include "api/frame_transformer_factory.h"
 #include "api/rtp_headers.h"
 #include "api/task_queue/task_queue_base.h"
 #include "api/task_queue/task_queue_factory.h"
@@ -38,6 +39,7 @@
 #include "modules/rtp_rtcp/source/rtp_video_layers_allocation_extension.h"
 #include "rtc_base/arraysize.h"
 #include "rtc_base/checks.h"
+#include "rtc_base/logging.h"
 #include "rtc_base/rate_limiter.h"
 #include "rtc_base/thread.h"
 #include "test/gmock.h"
@@ -1645,5 +1647,45 @@ TEST_F(RtpSenderVideoWithFrameTransformerTest,
                                      kDefaultExpectedRetransmissionTimeMs);
 }
 
+TEST_F(RtpSenderVideoWithFrameTransformerTest,
+       OnTransformedFrameSendsVideoWhenCloned) {
+  auto mock_frame_transformer =
+      rtc::make_ref_counted<NiceMock<MockFrameTransformer>>();
+  rtc::scoped_refptr<TransformedFrameCallback> callback;
+  EXPECT_CALL(*mock_frame_transformer, RegisterTransformedFrameSinkCallback)
+      .WillOnce(SaveArg<0>(&callback));
+  std::unique_ptr<RTPSenderVideo> rtp_sender_video =
+      CreateSenderWithFrameTransformer(mock_frame_transformer);
+  ASSERT_TRUE(callback);
+
+  auto encoded_image = CreateDefaultEncodedImage();
+  RTPVideoHeader video_header;
+  video_header.frame_type = VideoFrameType::kVideoFrameKey;
+  ON_CALL(*mock_frame_transformer, Transform)
+      .WillByDefault(
+          [&callback](std::unique_ptr<TransformableFrameInterface> frame) {
+            auto clone = CloneVideoFrame(
+                static_cast<TransformableVideoFrameInterface*>(frame.get()));
+            EXPECT_TRUE(clone);
+            callback->OnTransformedFrame(std::move(clone));
+          });
+  auto encoder_queue = time_controller_.GetTaskQueueFactory()->CreateTaskQueue(
+      "encoder_queue", TaskQueueFactory::Priority::NORMAL);
+  encoder_queue->PostTask([&] {
+    rtp_sender_video->SendEncodedImage(kPayload, kType, kTimestamp,
+                                       *encoded_image, video_header,
+                                       kDefaultExpectedRetransmissionTimeMs);
+  });
+  time_controller_.AdvanceTime(TimeDelta::Zero());
+  EXPECT_EQ(transport_.packets_sent(), 1);
+  encoder_queue->PostTask([&] {
+    rtp_sender_video->SendEncodedImage(kPayload, kType, kTimestamp,
+                                       *encoded_image, video_header,
+                                       kDefaultExpectedRetransmissionTimeMs);
+  });
+  time_controller_.AdvanceTime(TimeDelta::Zero());
+  EXPECT_EQ(transport_.packets_sent(), 2);
+}
+
 }  // namespace
 }  // namespace webrtc