Use the underlying type of Java Video Buffer on Java -> C++ Frame Buffer

Just like the C++ API, add a method in Java VideoFrame.Buffer that
describes the underlying implementation.
Use this method to properly select AndroidVideoBuffer
or AndroidVideoI420Buffer in Java -> C++ Video Frame Conversion.

Also, add a test case for WrappedNativeI420Buffer
in VideoFrameBufferTest for consistency.

Bug: webrtc:12602
Change-Id: I4c0444e8af6f6a1109bc514e7ab6c2214f1f6d60
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/223080
Commit-Queue: Byoungchan Lee <daniel.l@hpcnt.com>
Reviewed-by: Niels Moller <nisse@webrtc.org>
Reviewed-by: Xavier Lepaul‎ <xalep@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#34545}

api/video/BUILD.gn

@@ -69,6 +69,12 @@ rtc_library("video_frame") {
   absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
 }
 
+if (is_android) {
+  java_cpp_enum("video_frame_enums") {
+    sources = [ "video_frame_buffer.h" ]
+  }
+}
+
 rtc_library("video_frame_i010") {
   visibility = [ "*" ]
   sources = [

api/video/video_frame_buffer.h

@@ -46,6 +46,8 @@ class RTC_EXPORT VideoFrameBuffer : public rtc::RefCountInterface {
   // New frame buffer types will be added conservatively when there is an
   // opportunity to optimize the path between some pair of video source and
   // video sink.
+  // GENERATED_JAVA_ENUM_PACKAGE: org.webrtc
+  // GENERATED_JAVA_CLASS_NAME_OVERRIDE: VideoFrameBufferType
   enum class Type {
     kNative,
     kI420,

sdk/android/BUILD.gn

@@ -202,7 +202,9 @@ if (is_android) {
       ":base_java",
       "//rtc_base:base_java",
       "//third_party/android_deps:com_android_support_support_annotations_java",
+      "//third_party/androidx:androidx_annotation_annotation_java",
     ]
+    srcjar_deps = [ "//api/video:video_frame_enums" ]
   }
 
   rtc_android_library("video_java") {
@@ -1486,11 +1488,16 @@ if (is_android) {
 
   rtc_library("instrumentationtests_jni") {
     testonly = true
-    sources = [ "instrumentationtests/loggable_test.cc" ]
+    sources = [
+      "instrumentationtests/loggable_test.cc",
+      "instrumentationtests/video_frame_buffer_test.cc",
+    ]
 
     deps = [
       ":base_jni",
       ":native_api_jni",
+      ":videoframe_jni",
+      "../../api/video:video_frame",
       "../../rtc_base:rtc_base_approved",
     ]
   }

sdk/android/api/org/webrtc/VideoFrame.java

@@ -34,6 +34,15 @@ public class VideoFrame implements RefCounted {
    * and the buffer needs to be returned to the VideoSource as soon as all references are gone.
    */
   public interface Buffer extends RefCounted {
+    /**
+     * Representation of the underlying buffer. Currently, only NATIVE and I420 are supported.
+     */
+    @CalledByNative("Buffer")
+    @VideoFrameBufferType
+    default int getBufferType() {
+      return VideoFrameBufferType.NATIVE;
+    }
+
     /**
      * Resolution of the buffer in pixels.
      */
@@ -63,6 +72,11 @@ public class VideoFrame implements RefCounted {
    * Interface for I420 buffers.
    */
   public interface I420Buffer extends Buffer {
+    @Override
+    default int getBufferType() {
+      return VideoFrameBufferType.I420;
+    }
+
     /**
      * Returns a direct ByteBuffer containing Y-plane data. The buffer capacity is at least
      * getStrideY() * getHeight() bytes. The position of the returned buffer is ignored and must

sdk/android/instrumentationtests/src/org/webrtc/VideoFrameBufferTest.java

@@ -32,6 +32,7 @@ import org.chromium.base.test.params.ParameterizedRunner;
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.junit.runner.RunWith;
+import org.webrtc.VideoFrame;
 
 /**
  * Test VideoFrame buffers of different kind of formats: I420, RGB, OES, NV12, NV21, and verify
@@ -45,7 +46,7 @@ public class VideoFrameBufferTest {
   /**
    * These tests are parameterized on this enum which represents the different VideoFrame.Buffers.
    */
-  private static enum BufferType { I420, RGB_TEXTURE, OES_TEXTURE, NV21, NV12 }
+  private static enum BufferType { I420_JAVA, I420_NATIVE, RGB_TEXTURE, OES_TEXTURE, NV21, NV12 }
 
   @ClassParameter private static List<ParameterSet> CLASS_PARAMS = new ArrayList<>();
 
@@ -75,17 +76,39 @@ public class VideoFrameBufferTest {
    */
   private static VideoFrame.Buffer createBufferWithType(
       BufferType bufferType, VideoFrame.I420Buffer i420Buffer) {
+    VideoFrame.Buffer buffer;
     switch (bufferType) {
-      case I420:
+      case I420_JAVA:
-        return i420Buffer.toI420();
+        buffer = i420Buffer;
+        buffer.retain();
+        assertEquals(VideoFrameBufferType.I420, buffer.getBufferType());
+        assertEquals(VideoFrameBufferType.I420, nativeGetBufferType(buffer));
+        return buffer;
+      case I420_NATIVE:
+        buffer = nativeGetNativeI420Buffer(i420Buffer);
+        assertEquals(VideoFrameBufferType.I420, buffer.getBufferType());
+        assertEquals(VideoFrameBufferType.I420, nativeGetBufferType(buffer));
+        return buffer;
       case RGB_TEXTURE:
-        return createRgbTextureBuffer(/* eglContext= */ null, i420Buffer);
+        buffer = createRgbTextureBuffer(/* eglContext= */ null, i420Buffer);
+        assertEquals(VideoFrameBufferType.NATIVE, buffer.getBufferType());
+        assertEquals(VideoFrameBufferType.NATIVE, nativeGetBufferType(buffer));
+        return buffer;
       case OES_TEXTURE:
-        return createOesTextureBuffer(/* eglContext= */ null, i420Buffer);
+        buffer = createOesTextureBuffer(/* eglContext= */ null, i420Buffer);
+        assertEquals(VideoFrameBufferType.NATIVE, buffer.getBufferType());
+        assertEquals(VideoFrameBufferType.NATIVE, nativeGetBufferType(buffer));
+        return buffer;
       case NV21:
-        return createNV21Buffer(i420Buffer);
+        buffer = createNV21Buffer(i420Buffer);
+        assertEquals(VideoFrameBufferType.NATIVE, buffer.getBufferType());
+        assertEquals(VideoFrameBufferType.NATIVE, nativeGetBufferType(buffer));
+        return buffer;
       case NV12:
-        return createNV12Buffer(i420Buffer);
+        buffer = createNV12Buffer(i420Buffer);
+        assertEquals(VideoFrameBufferType.NATIVE, buffer.getBufferType());
+        assertEquals(VideoFrameBufferType.NATIVE, nativeGetBufferType(buffer));
+        return buffer;
       default:
         throw new IllegalArgumentException("Unknown buffer type: " + bufferType);
     }
@@ -452,6 +475,7 @@ public class VideoFrameBufferTest {
     final VideoFrame.I420Buffer outputI420Buffer = bufferToTest.toI420();
     bufferToTest.release();
 
+    assertEquals(VideoFrameBufferType.I420, nativeGetBufferType(outputI420Buffer));
     assertAlmostEqualI420Buffers(referenceI420Buffer, outputI420Buffer);
     referenceI420Buffer.release();
     outputI420Buffer.release();
@@ -504,4 +528,10 @@ public class VideoFrameBufferTest {
     testCropAndScale(4 /* cropX= */, 4 /* cropY= */, /* cropWidth= */ 12, /* cropHeight= */ 12,
         /* scaleWidth= */ 8, /* scaleHeight= */ 8);
   }
+
+  @VideoFrameBufferType private static native int nativeGetBufferType(VideoFrame.Buffer buffer);
+
+  /** Returns the copy of I420Buffer using WrappedNativeI420Buffer. */
+  private static native VideoFrame.Buffer nativeGetNativeI420Buffer(
+      VideoFrame.I420Buffer i420Buffer);
 }

sdk/android/instrumentationtests/video_frame_buffer_test.cc

@@ -0,0 +1,45 @@
+/*
+ *  Copyright 2021 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "api/video/i420_buffer.h"
+#include "sdk/android/src/jni/jni_helpers.h"
+#include "sdk/android/src/jni/video_frame.h"
+#include "sdk/android/src/jni/wrapped_native_i420_buffer.h"
+
+namespace webrtc {
+namespace jni {
+
+JNI_FUNCTION_DECLARATION(jint,
+                         VideoFrameBufferTest_nativeGetBufferType,
+                         JNIEnv* jni,
+                         jclass,
+                         jobject video_frame_buffer) {
+  const JavaParamRef<jobject> j_video_frame_buffer(video_frame_buffer);
+  rtc::scoped_refptr<VideoFrameBuffer> buffer =
+      JavaToNativeFrameBuffer(jni, j_video_frame_buffer);
+  return static_cast<jint>(buffer->type());
+}
+
+JNI_FUNCTION_DECLARATION(jobject,
+                         VideoFrameBufferTest_nativeGetNativeI420Buffer,
+                         JNIEnv* jni,
+                         jclass,
+                         jobject i420_buffer) {
+  const JavaParamRef<jobject> j_i420_buffer(i420_buffer);
+  rtc::scoped_refptr<VideoFrameBuffer> buffer =
+      JavaToNativeFrameBuffer(jni, j_i420_buffer);
+  const I420BufferInterface* inputBuffer = buffer->GetI420();
+  RTC_DCHECK(inputBuffer != nullptr);
+  rtc::scoped_refptr<I420Buffer> outputBuffer = I420Buffer::Copy(*inputBuffer);
+  return WrapI420Buffer(jni, outputBuffer).Release();
+}
+
+}  // namespace jni
+}  // namespace webrtc

sdk/android/src/jni/video_frame.cc

@@ -70,6 +70,13 @@ class AndroidVideoBuffer : public VideoFrameBuffer {
 
 class AndroidVideoI420Buffer : public I420BufferInterface {
  public:
+  // Creates a native VideoFrameBuffer from a Java VideoFrame.I420Buffer.
+  static rtc::scoped_refptr<AndroidVideoI420Buffer> Create(
+      JNIEnv* jni,
+      int width,
+      int height,
+      const JavaRef<jobject>& j_video_frame_buffer);
+
   // Adopts and takes ownership of the Java VideoFrame.Buffer. I.e. retain()
   // will not be called, but release() will be called when the returned
   // AndroidVideoBuffer is destroyed.
@@ -113,11 +120,24 @@ class AndroidVideoI420Buffer : public I420BufferInterface {
   int stride_v_;
 };
 
+rtc::scoped_refptr<AndroidVideoI420Buffer> AndroidVideoI420Buffer::Create(
+    JNIEnv* jni,
+    int width,
+    int height,
+    const JavaRef<jobject>& j_video_frame_buffer) {
+  Java_Buffer_retain(jni, j_video_frame_buffer);
+  return AndroidVideoI420Buffer::Adopt(jni, width, height,
+                                       j_video_frame_buffer);
+}
+
 rtc::scoped_refptr<AndroidVideoI420Buffer> AndroidVideoI420Buffer::Adopt(
     JNIEnv* jni,
     int width,
     int height,
     const JavaRef<jobject>& j_video_frame_buffer) {
+  RTC_DCHECK_EQ(
+      static_cast<Type>(Java_Buffer_getBufferType(jni, j_video_frame_buffer)),
+      Type::kI420);
   return rtc::make_ref_counted<AndroidVideoI420Buffer>(jni, width, height,
                                                        j_video_frame_buffer);
 }
@@ -164,6 +184,9 @@ int64_t GetJavaVideoFrameTimestampNs(JNIEnv* jni,
 rtc::scoped_refptr<AndroidVideoBuffer> AndroidVideoBuffer::Adopt(
     JNIEnv* jni,
     const JavaRef<jobject>& j_video_frame_buffer) {
+  RTC_DCHECK_EQ(
+      static_cast<Type>(Java_Buffer_getBufferType(jni, j_video_frame_buffer)),
+      Type::kNative);
   return rtc::make_ref_counted<AndroidVideoBuffer>(jni, j_video_frame_buffer);
 }
 
@@ -233,7 +256,20 @@ rtc::scoped_refptr<I420BufferInterface> AndroidVideoBuffer::ToI420() {
 rtc::scoped_refptr<VideoFrameBuffer> JavaToNativeFrameBuffer(
     JNIEnv* jni,
     const JavaRef<jobject>& j_video_frame_buffer) {
-  return AndroidVideoBuffer::Create(jni, j_video_frame_buffer);
+  VideoFrameBuffer::Type type = static_cast<VideoFrameBuffer::Type>(
+      Java_Buffer_getBufferType(jni, j_video_frame_buffer));
+  switch (type) {
+    case VideoFrameBuffer::Type::kI420: {
+      const int width = Java_Buffer_getWidth(jni, j_video_frame_buffer);
+      const int height = Java_Buffer_getHeight(jni, j_video_frame_buffer);
+      return AndroidVideoI420Buffer::Create(jni, width, height,
+                                            j_video_frame_buffer);
+    }
+    case VideoFrameBuffer::Type::kNative:
+      return AndroidVideoBuffer::Create(jni, j_video_frame_buffer);
+    default:
+      RTC_CHECK_NOTREACHED();
+  }
 }
 
 VideoFrame JavaToNativeFrame(JNIEnv* jni,
@@ -243,8 +279,8 @@ VideoFrame JavaToNativeFrame(JNIEnv* jni,
       Java_VideoFrame_getBuffer(jni, j_video_frame);
   int rotation = Java_VideoFrame_getRotation(jni, j_video_frame);
   int64_t timestamp_ns = Java_VideoFrame_getTimestampNs(jni, j_video_frame);
-  rtc::scoped_refptr<AndroidVideoBuffer> buffer =
+  rtc::scoped_refptr<VideoFrameBuffer> buffer =
-      AndroidVideoBuffer::Create(jni, j_video_frame_buffer);
+      JavaToNativeFrameBuffer(jni, j_video_frame_buffer);
   return VideoFrame::Builder()
       .set_video_frame_buffer(buffer)
       .set_timestamp_rtp(timestamp_rtp)