Accept NV12 frames into VP9

NV12 frames can be encoded by libvpx now, and this change allows for
encoding of them with VP9.

VP9 encode/decode tests now run with NV12 as well as I420.

Manually tested using video loopback with VP9 and NV12 generated frames.
  out/Default/video_loopback.app/Contents/MacOS/video_loopback --clip=GeneratorNV12 --codec="VP9"


Bug: webrtc:11635, webrtc:11974
Change-Id: Ifc5cbf77d2a27821cd5560c253d5d447c7a7cf53
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/185123
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Reviewed-by: Stefan Holmer <stefan@webrtc.org>
Commit-Queue: Evan Shrubsole <eshr@google.com>
Cr-Commit-Position: refs/heads/master@{#32220}

api/BUILD.gn

@@ -399,7 +399,10 @@ rtc_source_set("peer_connection_quality_test_fixture_api") {
 rtc_source_set("frame_generator_api") {
   visibility = [ "*" ]
   testonly = true
-  sources = [ "test/frame_generator_interface.h" ]
+  sources = [
+    "test/frame_generator_interface.cc",
+    "test/frame_generator_interface.h",
+  ]
 
   deps = [
     ":scoped_refptr",

api/test/frame_generator_interface.cc

@@ -0,0 +1,34 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "api/test/frame_generator_interface.h"
+
+namespace webrtc {
+namespace test {
+
+// static
+const char* FrameGeneratorInterface::OutputTypeToString(
+    FrameGeneratorInterface::OutputType type) {
+  switch (type) {
+    case OutputType::kI420:
+      return "I420";
+    case OutputType::kI420A:
+      return "I420A";
+    case OutputType::kI010:
+      return "I010";
+    case OutputType::kNV12:
+      return "NV12";
+    default:
+      RTC_NOTREACHED();
+  }
+}
+
+}  // namespace test
+}  // namespace webrtc

api/test/frame_generator_interface.h

@@ -33,6 +33,7 @@ class FrameGeneratorInterface {
   };
 
   enum class OutputType { kI420, kI420A, kI010, kNV12 };
+  static const char* OutputTypeToString(OutputType type);
 
   virtual ~FrameGeneratorInterface() = default;
 

modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc

@@ -89,11 +89,23 @@ class TestVp9Impl : public VideoCodecUnitTest {
   }
 };
 
+class TestVp9ImplForPixelFormat
+    : public TestVp9Impl,
+      public ::testing::WithParamInterface<
+          test::FrameGeneratorInterface::OutputType> {
+ protected:
+  void SetUp() override {
+    input_frame_generator_ = test::CreateSquareFrameGenerator(
+        kWidth, kHeight, GetParam(), absl::optional<int>());
+    TestVp9Impl::SetUp();
+  }
+};
+
 // Disabled on ios as flake, see https://crbug.com/webrtc/7057
 #if defined(WEBRTC_IOS)
-TEST_F(TestVp9Impl, DISABLED_EncodeDecode) {
+TEST_P(TestVp9ImplForPixelFormat, DISABLED_EncodeDecode) {
 #else
-TEST_F(TestVp9Impl, EncodeDecode) {
+TEST_P(TestVp9ImplForPixelFormat, EncodeDecode) {
 #endif
   VideoFrame input_frame = NextInputFrame();
   EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(input_frame, nullptr));
@@ -120,7 +132,7 @@ TEST_F(TestVp9Impl, EncodeDecode) {
             color_space.chroma_siting_vertical());
 }
 
-TEST_F(TestVp9Impl, DecodedColorSpaceFromBitstream) {
+TEST_P(TestVp9ImplForPixelFormat, DecodedColorSpaceFromBitstream) {
   EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(NextInputFrame(), nullptr));
   EncodedImage encoded_frame;
   CodecSpecificInfo codec_specific_info;
@@ -138,7 +150,7 @@ TEST_F(TestVp9Impl, DecodedColorSpaceFromBitstream) {
   EXPECT_FALSE(decoded_frame->color_space()->hdr_metadata());
 }
 
-TEST_F(TestVp9Impl, DecodedQpEqualsEncodedQp) {
+TEST_P(TestVp9ImplForPixelFormat, DecodedQpEqualsEncodedQp) {
   EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(NextInputFrame(), nullptr));
   EncodedImage encoded_frame;
   CodecSpecificInfo codec_specific_info;
@@ -154,6 +166,27 @@ TEST_F(TestVp9Impl, DecodedQpEqualsEncodedQp) {
   EXPECT_EQ(encoded_frame.qp_, *decoded_qp);
 }
 
+TEST_F(TestVp9Impl, SwitchInputPixelFormatsWithoutReconfigure) {
+  EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(NextInputFrame(), nullptr));
+  EncodedImage encoded_frame;
+  CodecSpecificInfo codec_specific_info;
+  ASSERT_TRUE(WaitForEncodedFrame(&encoded_frame, &codec_specific_info));
+
+  // Change the input frame type from I420 to NV12, encoding should still work.
+  input_frame_generator_ = test::CreateSquareFrameGenerator(
+      kWidth, kHeight, test::FrameGeneratorInterface::OutputType::kNV12,
+      absl::optional<int>());
+  EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(NextInputFrame(), nullptr));
+  ASSERT_TRUE(WaitForEncodedFrame(&encoded_frame, &codec_specific_info));
+
+  // Flipping back to I420, encoding should still work.
+  input_frame_generator_ = test::CreateSquareFrameGenerator(
+      kWidth, kHeight, test::FrameGeneratorInterface::OutputType::kI420,
+      absl::optional<int>());
+  EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, encoder_->Encode(NextInputFrame(), nullptr));
+  ASSERT_TRUE(WaitForEncodedFrame(&encoded_frame, &codec_specific_info));
+}
+
 TEST(Vp9ImplTest, ParserQpEqualsEncodedQp) {
   std::unique_ptr<VideoEncoder> encoder = VP9Encoder::Create();
   VideoCodec codec_settings = DefaultCodecSettings();
@@ -1741,4 +1774,12 @@ TEST_F(TestVp9Impl, HandlesEmptyInitDecode) {
   EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, decoder->Release());
 }
 
+INSTANTIATE_TEST_SUITE_P(
+    TestVp9ImplForPixelFormat,
+    TestVp9ImplForPixelFormat,
+    ::testing::Values(test::FrameGeneratorInterface::OutputType::kI420,
+                      test::FrameGeneratorInterface::OutputType::kNV12),
+    [](const auto& info) {
+      return test::FrameGeneratorInterface::OutputTypeToString(info.param);
+    });
 }  // namespace webrtc

modules/video_coding/codecs/vp9/vp9_impl.cc

@@ -978,12 +978,28 @@ int VP9EncoderImpl::Encode(const VideoFrame& input_image,
   input_image_ = &input_image;
 
   // Keep reference to buffer until encode completes.
-  rtc::scoped_refptr<I420BufferInterface> i420_buffer;
+  rtc::scoped_refptr<const VideoFrameBuffer> video_frame_buffer;
   const I010BufferInterface* i010_buffer;
   rtc::scoped_refptr<const I010BufferInterface> i010_copy;
   switch (profile_) {
     case VP9Profile::kProfile0: {
-      i420_buffer = input_image.video_frame_buffer()->ToI420();
+      if (input_image.video_frame_buffer()->type() ==
+          VideoFrameBuffer::Type::kNV12) {
+        const NV12BufferInterface* nv12_buffer =
+            input_image.video_frame_buffer()->GetNV12();
+        video_frame_buffer = nv12_buffer;
+        MaybeRewrapRawWithFormat(VPX_IMG_FMT_NV12);
+        raw_->planes[VPX_PLANE_Y] = const_cast<uint8_t*>(nv12_buffer->DataY());
+        raw_->planes[VPX_PLANE_U] = const_cast<uint8_t*>(nv12_buffer->DataUV());
+        raw_->planes[VPX_PLANE_V] = raw_->planes[VPX_PLANE_U] + 1;
+        raw_->stride[VPX_PLANE_Y] = nv12_buffer->StrideY();
+        raw_->stride[VPX_PLANE_U] = nv12_buffer->StrideUV();
+        raw_->stride[VPX_PLANE_V] = nv12_buffer->StrideUV();
+      } else {
+        rtc::scoped_refptr<I420BufferInterface> i420_buffer =
+            input_image.video_frame_buffer()->ToI420();
+        video_frame_buffer = i420_buffer;
+        MaybeRewrapRawWithFormat(VPX_IMG_FMT_I420);
         // Image in vpx_image_t format.
         // Input image is const. VPX's raw image is not defined as const.
         raw_->planes[VPX_PLANE_Y] = const_cast<uint8_t*>(i420_buffer->DataY());
@@ -992,6 +1008,7 @@ int VP9EncoderImpl::Encode(const VideoFrame& input_image,
         raw_->stride[VPX_PLANE_Y] = i420_buffer->StrideY();
         raw_->stride[VPX_PLANE_U] = i420_buffer->StrideU();
         raw_->stride[VPX_PLANE_V] = i420_buffer->StrideV();
+      }
       break;
     }
     case VP9Profile::kProfile1: {
@@ -1659,6 +1676,18 @@ VP9EncoderImpl::ParseQualityScalerConfig(std::string group_name) {
   return config;
 }
 
+void VP9EncoderImpl::MaybeRewrapRawWithFormat(const vpx_img_fmt fmt) {
+  if (!raw_) {
+    raw_ = vpx_img_wrap(nullptr, fmt, codec_.width, codec_.height, 1, nullptr);
+  } else if (raw_->fmt != fmt) {
+    RTC_LOG(INFO) << "Switching VP9 encoder pixel format to "
+                  << (fmt == VPX_IMG_FMT_NV12 ? "NV12" : "I420");
+    vpx_img_free(raw_);
+    raw_ = vpx_img_wrap(nullptr, fmt, codec_.width, codec_.height, 1, nullptr);
+  }
+  // else no-op since the image is already in the right format.
+}
+
 VP9DecoderImpl::VP9DecoderImpl()
     : decode_complete_callback_(nullptr),
       inited_(false),

modules/video_coding/codecs/vp9/vp9_impl.h

@@ -98,6 +98,8 @@ class VP9EncoderImpl : public VP9Encoder {
 
   size_t SteadyStateSize(int sid, int tid);
 
+  void MaybeRewrapRawWithFormat(const vpx_img_fmt fmt);
+
   EncodedImage encoded_image_;
   CodecSpecificInfo codec_specific_;
   EncodedImageCallback* encoded_complete_callback_;