Add support for I444 in VideoFrameBuffer

VideoFrameBuffer is currently hard coded to be either I420 or Native.
This CL makes VideoFrameBuffer more generic by moving the I420 specific
functions into their own class, and adds an enum tag that represents the
format and storage type of the buffer. Each buffer type is then
represented as a subclass. See webrtc/api/video/video_frame_buffer.h for
more info.

This CL also adds support for representing I444 in VideoFrameBuffer
using the new interface. Possible future buffer type candidates are
RGB and NV12.

BUG=webrtc:7632
TBR=stefan@webrtc.org

Review-Url: https://codereview.webrtc.org/2847383002
Cr-Commit-Position: refs/heads/master@{#18098}

webrtc/api/BUILD.gn

@@ -170,6 +170,7 @@ rtc_source_set("video_frame_api") {
     "video/i420_buffer.h",
     "video/video_frame.cc",
     "video/video_frame.h",
+    "video/video_frame_buffer.cc",
     "video/video_frame_buffer.h",
     "video/video_rotation.h",
   ]

webrtc/api/video/i420_buffer.cc

@@ -136,6 +136,10 @@ void I420Buffer::InitializeData() {
          I420DataSize(height_, stride_y_, stride_u_, stride_v_));
 }
 
+VideoFrameBuffer::Type I420Buffer::type() const {
+  return Type::kI420;
+}
+
 int I420Buffer::width() const {
   return width_;
 }
@@ -164,14 +168,6 @@ int I420Buffer::StrideV() const {
   return stride_v_;
 }
 
-void* I420Buffer::native_handle() const {
-  return nullptr;
-}
-
-rtc::scoped_refptr<VideoFrameBuffer> I420Buffer::NativeToI420Buffer() {
-  return this;
-}
-
 uint8_t* I420Buffer::MutableDataY() {
   return const_cast<uint8_t*>(DataY());
 }

webrtc/api/video/i420_buffer.h

@@ -20,7 +20,7 @@
 namespace webrtc {
 
 // Plain I420 buffer in standard memory.
-class I420Buffer : public VideoFrameBuffer {
+class I420Buffer : public PlanarYuvBuffer {
  public:
   static rtc::scoped_refptr<I420Buffer> Create(int width, int height);
   static rtc::scoped_refptr<I420Buffer> Create(int width,
@@ -53,6 +53,7 @@ class I420Buffer : public VideoFrameBuffer {
   // are resolved in a better way. Or in the mean time, use SetBlack.
   void InitializeData();
 
+  Type type() const override;
   int width() const override;
   int height() const override;
   const uint8_t* DataY() const override;
@@ -63,9 +64,6 @@ class I420Buffer : public VideoFrameBuffer {
   int StrideU() const override;
   int StrideV() const override;
 
-  void* native_handle() const override;
-  rtc::scoped_refptr<VideoFrameBuffer> NativeToI420Buffer() override;
-
   uint8_t* MutableDataY();
   uint8_t* MutableDataU();
   uint8_t* MutableDataV();

webrtc/api/video/video_frame_buffer.cc

@@ -0,0 +1,153 @@
+/*
+ *  Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "webrtc/api/video/video_frame_buffer.h"
+
+#include "libyuv/convert_from.h"
+#include "webrtc/api/video/i420_buffer.h"
+#include "webrtc/base/checks.h"
+
+namespace webrtc {
+
+namespace {
+
+// TODO(magjed): Remove this class. It is only used for providing a default
+// implementation of ToI420() until external clients are updated. ToI420() will
+// then be made pure virtual. This adapter adapts a VideoFrameBuffer (which is
+// expected to be in I420 format) to the PlanarYuvBuffer interface. The reason
+// this is needed is because of the return type mismatch in NativeToI420Buffer
+// (returns VideoFrameBuffer) vs ToI420 (returns PlanarYuvBuffer).
+class PlanarYuvBufferAdapter : public PlanarYuvBuffer {
+ public:
+  explicit PlanarYuvBufferAdapter(rtc::scoped_refptr<VideoFrameBuffer> buffer)
+      : buffer_(buffer) {}
+
+  Type type() const override { return Type::kI420; }
+
+  int width() const override { return buffer_->width(); }
+  int height() const override { return buffer_->height(); }
+
+  const uint8_t* DataY() const override { return buffer_->DataY(); }
+  const uint8_t* DataU() const override { return buffer_->DataU(); }
+  const uint8_t* DataV() const override { return buffer_->DataV(); }
+
+  int StrideY() const override { return buffer_->StrideY(); }
+  int StrideU() const override { return buffer_->StrideU(); }
+  int StrideV() const override { return buffer_->StrideV(); }
+
+ private:
+  rtc::scoped_refptr<VideoFrameBuffer> buffer_;
+};
+
+}  // namespace
+
+// TODO(magjed): The default implementations in VideoFrameBuffer are provided in
+// order to support the deprecated interface until external clients are updated.
+// Remove once done.
+VideoFrameBuffer::Type VideoFrameBuffer::type() const {
+  return native_handle() ? Type::kNative : Type::kI420;
+}
+
+const uint8_t* VideoFrameBuffer::DataY() const {
+  return const_cast<VideoFrameBuffer*>(this)->GetI420()->DataY();
+}
+
+const uint8_t* VideoFrameBuffer::DataU() const {
+  return const_cast<VideoFrameBuffer*>(this)->GetI420()->DataU();
+}
+
+const uint8_t* VideoFrameBuffer::DataV() const {
+  return const_cast<VideoFrameBuffer*>(this)->GetI420()->DataV();
+}
+
+// Returns the number of bytes between successive rows for a given plane.
+int VideoFrameBuffer::StrideY() const {
+  return const_cast<VideoFrameBuffer*>(this)->GetI420()->StrideY();
+}
+
+int VideoFrameBuffer::StrideU() const {
+  return const_cast<VideoFrameBuffer*>(this)->GetI420()->StrideU();
+}
+
+int VideoFrameBuffer::StrideV() const {
+  return const_cast<VideoFrameBuffer*>(this)->GetI420()->StrideV();
+}
+
+void* VideoFrameBuffer::native_handle() const {
+  RTC_DCHECK(type() != Type::kNative);
+  return nullptr;
+}
+
+rtc::scoped_refptr<VideoFrameBuffer> VideoFrameBuffer::NativeToI420Buffer() {
+  return ToI420();
+}
+
+rtc::scoped_refptr<PlanarYuvBuffer> VideoFrameBuffer::ToI420() {
+  return new rtc::RefCountedObject<PlanarYuvBufferAdapter>(
+      NativeToI420Buffer());
+}
+
+rtc::scoped_refptr<PlanarYuvBuffer> VideoFrameBuffer::GetI420() {
+  RTC_CHECK(type() == Type::kI420);
+  // TODO(magjed): static_cast to PlanarYuvBuffer instead once external clients
+  // are updated.
+  return new rtc::RefCountedObject<PlanarYuvBufferAdapter>(this);
+}
+
+rtc::scoped_refptr<PlanarYuvBuffer> VideoFrameBuffer::GetI444() {
+  RTC_CHECK(type() == Type::kI444);
+  return static_cast<PlanarYuvBuffer*>(this);
+}
+
+rtc::scoped_refptr<PlanarYuvBuffer> PlanarYuvBuffer::ToI420() {
+  switch (type()) {
+    case Type::kI420:
+      return this;
+    case Type::kI444: {
+      rtc::scoped_refptr<I420Buffer> i420_buffer =
+          I420Buffer::Create(width(), height());
+      libyuv::I420ToI444(DataY(), StrideY(), DataU(), StrideU(), DataV(),
+                         StrideV(), i420_buffer->MutableDataY(),
+                         i420_buffer->StrideY(), i420_buffer->MutableDataU(),
+                         i420_buffer->StrideU(), i420_buffer->MutableDataV(),
+                         i420_buffer->StrideV(), width(), height());
+      return i420_buffer;
+    }
+    default:
+      RTC_NOTREACHED();
+      return nullptr;
+  }
+}
+
+int PlanarYuvBuffer::ChromaWidth() const {
+  switch (type()) {
+    case Type::kI420:
+      return (width() + 1) / 2;
+    case Type::kI444:
+      return width();
+    default:
+      RTC_NOTREACHED();
+      return 0;
+  }
+}
+
+int PlanarYuvBuffer::ChromaHeight() const {
+  switch (type()) {
+    case Type::kI420:
+      return (height() + 1) / 2;
+    case Type::kI444:
+      return height();
+    default:
+      RTC_NOTREACHED();
+      return 0;
+  }
+}
+
+}  // namespace webrtc

webrtc/api/video/video_frame_buffer.h

@@ -18,38 +18,100 @@
 
 namespace webrtc {
 
-// Interface of a simple frame buffer containing pixel data. This interface does
-// not contain any frame metadata such as rotation, timestamp, pixel_width, etc.
+class PlanarYuvBuffer;
+
+// Base class for frame buffers of different types of pixel format and storage.
+// The tag in type() indicates how the data is represented, and each type is
+// implemented as a subclass. To access the pixel data, call the appropriate
+// GetXXX() function, where XXX represents the type. There is also a function
+// ToI420() that returns a frame buffer in I420 format, converting from the
+// underlying representation if necessary. I420 is the most widely accepted
+// format and serves as a fallback for video sinks that can only handle I420,
+// e.g. the internal WebRTC software encoders. A special enum value 'kNative' is
+// provided for external clients to implement their own frame buffer
+// representations, e.g. as textures. The external client can produce such
+// native frame buffers from custom video sources, and then cast it back to the
+// correct subclass in custom video sinks. The purpose of this is to improve
+// performance by providing an optimized path without intermediate conversions.
+// Frame metadata such as rotation and timestamp are stored in
+// webrtc::VideoFrame, and not here.
 class VideoFrameBuffer : public rtc::RefCountInterface {
  public:
+  // New frame buffer types will be added conservatively when there is an
+  // opportunity to optimize the path between some pair of video source and
+  // video sink.
+  enum class Type {
+    kNative,
+    kI420,
+    kI444,
+  };
+
+  // This function specifies in what pixel format the data is stored in.
+  virtual Type type() const;
+
   // The resolution of the frame in pixels. For formats where some planes are
   // subsampled, this is the highest-resolution plane.
   virtual int width() const = 0;
   virtual int height() const = 0;
 
+  // Returns a memory-backed frame buffer in I420 format. If the pixel data is
+  // in another format, a conversion will take place. All implementations must
+  // provide a fallback to I420 for compatibility with e.g. the internal WebRTC
+  // software encoders.
+  virtual rtc::scoped_refptr<PlanarYuvBuffer> ToI420();
+
+  // These functions should only be called if type() is of the correct type.
+  // Calling with a different type will result in a crash.
+  rtc::scoped_refptr<PlanarYuvBuffer> GetI420();
+  rtc::scoped_refptr<PlanarYuvBuffer> GetI444();
+
+  // Deprecated - use ToI420() first instead.
   // Returns pointer to the pixel data for a given plane. The memory is owned by
   // the VideoFrameBuffer object and must not be freed by the caller.
-  virtual const uint8_t* DataY() const = 0;
-  virtual const uint8_t* DataU() const = 0;
-  virtual const uint8_t* DataV() const = 0;
-
+  virtual const uint8_t* DataY() const;
+  virtual const uint8_t* DataU() const;
+  virtual const uint8_t* DataV() const;
   // Returns the number of bytes between successive rows for a given plane.
-  virtual int StrideY() const = 0;
-  virtual int StrideU() const = 0;
-  virtual int StrideV() const = 0;
+  virtual int StrideY() const;
+  virtual int StrideU() const;
+  virtual int StrideV() const;
 
+  // Deprecated - use type() to determine if the stored data is kNative, and
+  // then cast into the appropriate type.
   // Return the handle of the underlying video frame. This is used when the
   // frame is backed by a texture.
-  virtual void* native_handle() const = 0;
+  virtual void* native_handle() const;
 
-  // Returns a new memory-backed frame buffer converted from this buffer's
-  // native handle.
-  virtual rtc::scoped_refptr<VideoFrameBuffer> NativeToI420Buffer() = 0;
+  // Deprecated - use ToI420() instead.
+  virtual rtc::scoped_refptr<VideoFrameBuffer> NativeToI420Buffer();
 
  protected:
   ~VideoFrameBuffer() override {}
 };
 
+// This interface represents Type::kI420 and Type::kI444.
+class PlanarYuvBuffer : public VideoFrameBuffer {
+ public:
+  int ChromaWidth() const;
+  int ChromaHeight() const;
+
+  // Returns pointer to the pixel data for a given plane. The memory is owned by
+  // the VideoFrameBuffer object and must not be freed by the caller.
+  const uint8_t* DataY() const override = 0;
+  const uint8_t* DataU() const override = 0;
+  const uint8_t* DataV() const override = 0;
+
+  // Returns the number of bytes between successive rows for a given plane.
+  int StrideY() const override = 0;
+  int StrideU() const override = 0;
+  int StrideV() const override = 0;
+
+  rtc::scoped_refptr<PlanarYuvBuffer> ToI420() override;
+
+ protected:
+  ~PlanarYuvBuffer() override {}
+};
+
 }  // namespace webrtc
 
 #endif  // WEBRTC_API_VIDEO_VIDEO_FRAME_BUFFER_H_

webrtc/common_video/include/video_frame_buffer.h

@@ -27,6 +27,7 @@ class NativeHandleBuffer : public VideoFrameBuffer {
  public:
   NativeHandleBuffer(void* native_handle, int width, int height);
 
+  Type type() const override;
   int width() const override;
   int height() const override;
   const uint8_t* DataY() const override;
@@ -44,7 +45,7 @@ class NativeHandleBuffer : public VideoFrameBuffer {
   const int height_;
 };
 
-class WrappedI420Buffer : public webrtc::VideoFrameBuffer {
+class WrappedI420Buffer : public PlanarYuvBuffer {
  public:
   WrappedI420Buffer(int width,
                     int height,
@@ -55,6 +56,8 @@ class WrappedI420Buffer : public webrtc::VideoFrameBuffer {
                     const uint8_t* v_plane,
                     int v_stride,
                     const rtc::Callback0<void>& no_longer_used);
+  Type type() const override;
+
   int width() const override;
   int height() const override;
 
@@ -65,10 +68,6 @@ class WrappedI420Buffer : public webrtc::VideoFrameBuffer {
   int StrideU() const override;
   int StrideV() const override;
 
-  void* native_handle() const override;
-
-  rtc::scoped_refptr<VideoFrameBuffer> NativeToI420Buffer() override;
-
  private:
   friend class rtc::RefCountedObject<WrappedI420Buffer>;
   ~WrappedI420Buffer() override;

webrtc/common_video/video_frame_buffer.cc

@@ -30,6 +30,10 @@ NativeHandleBuffer::NativeHandleBuffer(void* native_handle,
   RTC_DCHECK_GT(height, 0);
 }
 
+VideoFrameBuffer::Type NativeHandleBuffer::type() const {
+  return Type::kNative;
+}
+
 int NativeHandleBuffer::width() const {
   return width_;
 }
@@ -92,6 +96,10 @@ WrappedI420Buffer::~WrappedI420Buffer() {
   no_longer_used_cb_();
 }
 
+VideoFrameBuffer::Type WrappedI420Buffer::type() const {
+  return Type::kI420;
+}
+
 int WrappedI420Buffer::width() const {
   return width_;
 }
@@ -120,13 +128,4 @@ int WrappedI420Buffer::StrideV() const {
   return v_stride_;
 }
 
-void* WrappedI420Buffer::native_handle() const {
-  return nullptr;
-}
-
-rtc::scoped_refptr<VideoFrameBuffer> WrappedI420Buffer::NativeToI420Buffer() {
-  RTC_NOTREACHED();
-  return nullptr;
-}
-
 }  // namespace webrtc