Create AndroidVideoBuffer and allow renderers to consume it.

Bug: webrtc:7760
Change-Id: I3e3fddf48090ae27b226c65ddbb51f2c3d8dc544
Reviewed-on: https://chromium-review.googlesource.com/535638
Commit-Queue: Sami Kalliomäki <sakal@webrtc.org>
Reviewed-by: Magnus Jedvert <magjed@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#18624}

webrtc/sdk/android/api/org/webrtc/VideoRenderer.java

@@ -89,6 +89,38 @@ public class VideoRenderer {
       }
     }
 
+    /**
+     * Construct a frame of the given dimensions from VideoFrame.Buffer.
+     */
+    public I420Frame(int width, int height, int rotationDegree, float[] samplingMatrix,
+        VideoFrame.Buffer buffer, long nativeFramePointer) {
+      this.width = width;
+      this.height = height;
+      this.rotationDegree = rotationDegree;
+      if (rotationDegree % 90 != 0) {
+        throw new IllegalArgumentException("Rotation degree not multiple of 90: " + rotationDegree);
+      }
+      this.samplingMatrix = samplingMatrix;
+      if (buffer instanceof VideoFrame.TextureBuffer) {
+        VideoFrame.TextureBuffer textureBuffer = (VideoFrame.TextureBuffer) buffer;
+        this.yuvFrame = false;
+        this.textureId = textureBuffer.getTextureId();
+
+        this.yuvStrides = null;
+        this.yuvPlanes = null;
+      } else {
+        VideoFrame.I420Buffer i420Buffer = buffer.toI420();
+        this.yuvFrame = true;
+        this.yuvStrides =
+            new int[] {i420Buffer.getStrideY(), i420Buffer.getStrideU(), i420Buffer.getStrideV()};
+        this.yuvPlanes =
+            new ByteBuffer[] {i420Buffer.getDataY(), i420Buffer.getDataU(), i420Buffer.getDataV()};
+
+        this.textureId = 0;
+      }
+      this.nativeFramePointer = nativeFramePointer;
+    }
+
     public int rotatedWidth() {
       return (rotationDegree % 180 == 0) ? width : height;
     }

webrtc/sdk/android/src/jni/classreferenceholder.cc

@@ -66,33 +66,33 @@ ClassReferenceHolder::ClassReferenceHolder(JNIEnv* jni) {
   LoadClass(jni, "org/webrtc/EglBase$Context");
   LoadClass(jni, "org/webrtc/EglBase14$Context");
   LoadClass(jni, "org/webrtc/IceCandidate");
+  LoadClass(jni, "org/webrtc/MediaCodecVideoDecoder");
+  LoadClass(jni, "org/webrtc/MediaCodecVideoDecoder$DecodedOutputBuffer");
+  LoadClass(jni, "org/webrtc/MediaCodecVideoDecoder$DecodedTextureBuffer");
+  LoadClass(jni, "org/webrtc/MediaCodecVideoDecoder$VideoCodecType");
   LoadClass(jni, "org/webrtc/MediaCodecVideoEncoder");
   LoadClass(jni, "org/webrtc/MediaCodecVideoEncoder$OutputBufferInfo");
   LoadClass(jni, "org/webrtc/MediaCodecVideoEncoder$VideoCodecType");
-  LoadClass(jni, "org/webrtc/MediaCodecVideoDecoder");
-  LoadClass(jni, "org/webrtc/MediaCodecVideoDecoder$DecodedTextureBuffer");
-  LoadClass(jni, "org/webrtc/MediaCodecVideoDecoder$DecodedOutputBuffer");
-  LoadClass(jni, "org/webrtc/MediaCodecVideoDecoder$VideoCodecType");
   LoadClass(jni, "org/webrtc/MediaSource$State");
   LoadClass(jni, "org/webrtc/MediaStream");
-  LoadClass(jni, "org/webrtc/MediaStreamTrack$State");
   LoadClass(jni, "org/webrtc/MediaStreamTrack$MediaType");
+  LoadClass(jni, "org/webrtc/MediaStreamTrack$State");
   LoadClass(jni, "org/webrtc/NetworkMonitor");
   LoadClass(jni, "org/webrtc/NetworkMonitorAutoDetect$ConnectionType");
   LoadClass(jni, "org/webrtc/NetworkMonitorAutoDetect$IPAddress");
   LoadClass(jni, "org/webrtc/NetworkMonitorAutoDetect$NetworkInformation");
-  LoadClass(jni, "org/webrtc/PeerConnectionFactory");
   LoadClass(jni, "org/webrtc/PeerConnection$BundlePolicy");
+  LoadClass(jni, "org/webrtc/PeerConnection$CandidateNetworkPolicy");
   LoadClass(jni, "org/webrtc/PeerConnection$ContinualGatheringPolicy");
-  LoadClass(jni, "org/webrtc/PeerConnection$RtcpMuxPolicy");
   LoadClass(jni, "org/webrtc/PeerConnection$IceConnectionState");
   LoadClass(jni, "org/webrtc/PeerConnection$IceGatheringState");
   LoadClass(jni, "org/webrtc/PeerConnection$IceTransportsType");
+  LoadClass(jni, "org/webrtc/PeerConnection$KeyType");
+  LoadClass(jni, "org/webrtc/PeerConnection$RtcpMuxPolicy");
+  LoadClass(jni, "org/webrtc/PeerConnection$SignalingState");
   LoadClass(jni, "org/webrtc/PeerConnection$TcpCandidatePolicy");
   LoadClass(jni, "org/webrtc/PeerConnection$TlsCertPolicy");
-  LoadClass(jni, "org/webrtc/PeerConnection$CandidateNetworkPolicy");
+  LoadClass(jni, "org/webrtc/PeerConnectionFactory");
-  LoadClass(jni, "org/webrtc/PeerConnection$KeyType");
-  LoadClass(jni, "org/webrtc/PeerConnection$SignalingState");
   LoadClass(jni, "org/webrtc/RTCStats");
   LoadClass(jni, "org/webrtc/RTCStatsReport");
   LoadClass(jni, "org/webrtc/RtpReceiver");
@@ -103,6 +103,9 @@ ClassReferenceHolder::ClassReferenceHolder(JNIEnv* jni) {
   LoadClass(jni, "org/webrtc/StatsReport$Value");
   LoadClass(jni, "org/webrtc/SurfaceTextureHelper");
   LoadClass(jni, "org/webrtc/VideoCapturer");
+  LoadClass(jni, "org/webrtc/VideoFrame");
+  LoadClass(jni, "org/webrtc/VideoFrame$Buffer");
+  LoadClass(jni, "org/webrtc/VideoFrame$I420Buffer");
   LoadClass(jni, "org/webrtc/VideoRenderer$I420Frame");
   LoadClass(jni, "org/webrtc/VideoTrack");
   LoadClass(jni, "org/webrtc/WrappedNativeI420Buffer");

webrtc/sdk/android/src/jni/native_handle_impl.cc

@@ -17,7 +17,9 @@
 #include "webrtc/base/keep_ref_until_done.h"
 #include "webrtc/base/logging.h"
 #include "webrtc/base/scoped_ref_ptr.h"
+#include "webrtc/base/timeutils.h"
 #include "webrtc/common_video/include/video_frame_buffer.h"
+#include "webrtc/sdk/android/src/jni/classreferenceholder.h"
 #include "webrtc/sdk/android/src/jni/jni_helpers.h"
 #include "webrtc/system_wrappers/include/aligned_malloc.h"
 
@@ -32,7 +34,25 @@ Matrix::Matrix(JNIEnv* jni, jfloatArray a) {
   jni->ReleaseFloatArrayElements(a, ptr, 0);
 }
 
-jfloatArray Matrix::ToJava(JNIEnv* jni) {
+Matrix Matrix::fromAndroidGraphicsMatrix(JNIEnv* jni, jobject j_matrix) {
+  jfloatArray array_3x3 = jni->NewFloatArray(9);
+  jclass j_matrix_class = jni->FindClass("android/graphics/Matrix");
+  jni->CallVoidMethod(j_matrix,
+                      GetMethodID(jni, j_matrix_class, "getValues", "([F)V"),
+                      array_3x3);
+  jfloat* array_3x3_ptr = jni->GetFloatArrayElements(array_3x3, nullptr);
+  Matrix matrix;
+  memset(matrix.elem_, 0, sizeof(matrix.elem_));
+  for (int y = 0; y < 3; ++y) {
+    for (int x = 0; x < 3; ++x) {
+      matrix.elem_[y * 4 + x] = array_3x3_ptr[x + y * 3];
+    }
+  }
+  matrix.elem_[3 + 3 * 3] = 1;  // Bottom-right corner should be 1.
+  return matrix;
+}
+
+jfloatArray Matrix::ToJava(JNIEnv* jni) const {
   jfloatArray matrix = jni->NewFloatArray(16);
   jni->SetFloatArrayRegion(matrix, 0, 16, elem_);
   return matrix;
@@ -195,4 +215,123 @@ rtc::scoped_refptr<webrtc::I420BufferInterface> AndroidTextureBuffer::ToI420() {
   return copy;
 }
 
+AndroidVideoBuffer::AndroidVideoBuffer(JNIEnv* jni,
+                                       jmethodID j_retain_id,
+                                       jmethodID j_release_id,
+                                       int width,
+                                       int height,
+                                       const Matrix& matrix,
+                                       jobject j_video_frame_buffer)
+    : j_release_id_(j_release_id),
+      width_(width),
+      height_(height),
+      matrix_(matrix),
+      j_video_frame_buffer_(jni, j_video_frame_buffer) {
+  jni->CallVoidMethod(j_video_frame_buffer, j_retain_id);
+}
+
+AndroidVideoBuffer::~AndroidVideoBuffer() {
+  JNIEnv* jni = AttachCurrentThreadIfNeeded();
+  jni->CallVoidMethod(*j_video_frame_buffer_, j_release_id_);
+}
+
+jobject AndroidVideoBuffer::video_frame_buffer() const {
+  return *j_video_frame_buffer_;
+}
+
+webrtc::VideoFrameBuffer::Type AndroidVideoBuffer::type() const {
+  return Type::kNative;
+}
+
+int AndroidVideoBuffer::width() const {
+  return width_;
+}
+
+int AndroidVideoBuffer::height() const {
+  return height_;
+}
+
+rtc::scoped_refptr<webrtc::I420BufferInterface> AndroidVideoBuffer::ToI420() {
+  // TODO(magjed): Implement using Java ToI420.
+  return nullptr;
+}
+
+jobject AndroidVideoBuffer::ToJavaI420Frame(JNIEnv* jni,
+                                            int width,
+                                            int height,
+                                            int rotation) {
+  jclass j_byte_buffer_class = jni->FindClass("java/nio/ByteBuffer");
+  jclass j_i420_frame_class =
+      FindClass(jni, "org/webrtc/VideoRenderer$I420Frame");
+  jmethodID j_i420_frame_ctor_id =
+      GetMethodID(jni, j_i420_frame_class, "<init>",
+                  "(III[FLorg/webrtc/VideoFrame$Buffer;J)V");
+  // Java code just uses the native frame to hold a reference to the buffer so
+  // this is okay.
+  webrtc::VideoFrame* native_frame = new webrtc::VideoFrame(
+      this, 0 /* timestamp */, 0 /* render_time_ms */,
+      webrtc::VideoRotation::kVideoRotation_0 /* rotation */);
+  return jni->NewObject(j_i420_frame_class, j_i420_frame_ctor_id, width, height,
+                        rotation, matrix_.ToJava(jni), *j_video_frame_buffer_,
+                        jlongFromPointer(native_frame));
+}
+
+AndroidVideoBufferFactory::AndroidVideoBufferFactory(JNIEnv* jni)
+    : j_video_frame_class_(jni, FindClass(jni, "org/webrtc/VideoFrame")),
+      j_get_buffer_id_(GetMethodID(jni,
+                                   *j_video_frame_class_,
+                                   "getBuffer",
+                                   "()Lorg/webrtc/VideoFrame$Buffer;")),
+      j_get_width_id_(
+          GetMethodID(jni, *j_video_frame_class_, "getWidth", "()I")),
+      j_get_height_id_(
+          GetMethodID(jni, *j_video_frame_class_, "getHeight", "()I")),
+      j_get_rotation_id_(
+          GetMethodID(jni, *j_video_frame_class_, "getRotation", "()I")),
+      j_get_transform_matrix_id_(GetMethodID(jni,
+                                             *j_video_frame_class_,
+                                             "getTransformMatrix",
+                                             "()Landroid/graphics/Matrix;")),
+      j_get_timestamp_ns_id_(
+          GetMethodID(jni, *j_video_frame_class_, "getTimestampNs", "()J")),
+      j_video_frame_buffer_class_(
+          jni,
+          FindClass(jni, "org/webrtc/VideoFrame$Buffer")),
+      j_retain_id_(
+          GetMethodID(jni, *j_video_frame_buffer_class_, "retain", "()V")),
+      j_release_id_(
+          GetMethodID(jni, *j_video_frame_buffer_class_, "release", "()V")) {}
+
+webrtc::VideoFrame AndroidVideoBufferFactory::CreateFrame(
+    JNIEnv* jni,
+    jobject j_video_frame,
+    uint32_t timestamp_rtp) const {
+  jobject j_video_frame_buffer =
+      jni->CallObjectMethod(j_video_frame, j_get_buffer_id_);
+  int width = jni->CallIntMethod(j_video_frame, j_get_width_id_);
+  int height = jni->CallIntMethod(j_video_frame, j_get_height_id_);
+  int rotation = jni->CallIntMethod(j_video_frame, j_get_rotation_id_);
+  jobject j_matrix =
+      jni->CallObjectMethod(j_video_frame, j_get_transform_matrix_id_);
+  Matrix matrix = Matrix::fromAndroidGraphicsMatrix(jni, j_matrix);
+  uint32_t timestamp_ns =
+      jni->CallLongMethod(j_video_frame, j_get_timestamp_ns_id_);
+  rtc::scoped_refptr<AndroidVideoBuffer> buffer =
+      CreateBuffer(width, height, matrix, j_video_frame_buffer);
+  return webrtc::VideoFrame(buffer, timestamp_rtp,
+                            timestamp_ns / rtc::kNumNanosecsPerMillisec,
+                            static_cast<webrtc::VideoRotation>(rotation));
+}
+
+rtc::scoped_refptr<AndroidVideoBuffer> AndroidVideoBufferFactory::CreateBuffer(
+    int width,
+    int height,
+    const Matrix& matrix,
+    jobject j_video_frame_buffer) const {
+  JNIEnv* jni = AttachCurrentThreadIfNeeded();
+  return new rtc::RefCountedObject<AndroidVideoBuffer>(
+      jni, j_retain_id_, j_release_id_, width, height, matrix,
+      j_video_frame_buffer);
+}
+
 }  // namespace webrtc_jni

webrtc/sdk/android/src/jni/native_handle_impl.h

@@ -13,9 +13,11 @@
 
 #include <jni.h>
 
+#include "webrtc/api/video/video_frame.h"
 #include "webrtc/api/video/video_frame_buffer.h"
 #include "webrtc/api/video/video_rotation.h"
 #include "webrtc/base/callback.h"
+#include "webrtc/sdk/android/src/jni/jni_helpers.h"
 
 namespace webrtc_jni {
 
@@ -25,7 +27,9 @@ class Matrix {
  public:
   Matrix(JNIEnv* jni, jfloatArray a);
 
-  jfloatArray ToJava(JNIEnv* jni);
+  static Matrix fromAndroidGraphicsMatrix(JNIEnv* jni, jobject j_matrix);
+
+  jfloatArray ToJava(JNIEnv* jni) const;
 
   // Crop arguments are relative to original size.
   void Crop(float cropped_width,
@@ -36,6 +40,8 @@ class Matrix {
   void Rotate(webrtc::VideoRotation rotation);
 
  private:
+  Matrix() {}
+
   static void Multiply(const float a[16], const float b[16], float result[16]);
   float elem_[16];
 };
@@ -52,7 +58,18 @@ struct NativeHandleImpl {
   Matrix sampling_matrix;
 };
 
-class AndroidTextureBuffer : public webrtc::VideoFrameBuffer {
+// Base class to differentiate between the old texture frames and the new
+// Java-based frames.
+// TODO(sakal): Remove this and AndroidTextureBuffer once they are no longer
+// needed.
+class AndroidVideoFrameBuffer : public webrtc::VideoFrameBuffer {
+ public:
+  enum class AndroidType { kTextureBuffer, kJavaBuffer };
+
+  virtual AndroidType android_type() = 0;
+};
+
+class AndroidTextureBuffer : public AndroidVideoFrameBuffer {
  public:
   AndroidTextureBuffer(int width,
                        int height,
@@ -70,6 +87,8 @@ class AndroidTextureBuffer : public webrtc::VideoFrameBuffer {
 
   rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override;
 
+  AndroidType android_type() override { return AndroidType::kTextureBuffer; }
+
   const int width_;
   const int height_;
   NativeHandleImpl native_handle_;
@@ -82,6 +101,67 @@ class AndroidTextureBuffer : public webrtc::VideoFrameBuffer {
   rtc::Callback0<void> no_longer_used_cb_;
 };
 
+class AndroidVideoBuffer : public AndroidVideoFrameBuffer {
+ public:
+  AndroidVideoBuffer(JNIEnv* jni,
+                     jmethodID j_retain_id,
+                     jmethodID j_release_id,
+                     int width,
+                     int height,
+                     const Matrix& matrix,
+                     jobject j_video_frame_buffer);
+  ~AndroidVideoBuffer() override;
+
+  jobject video_frame_buffer() const;
+
+  // Returns an instance of VideoRenderer.I420Frame (deprecated)
+  jobject ToJavaI420Frame(JNIEnv* jni, int width, int height, int rotation);
+
+ private:
+  Type type() const override;
+  int width() const override;
+  int height() const override;
+
+  rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override;
+
+  AndroidType android_type() override { return AndroidType::kJavaBuffer; }
+
+  const jmethodID j_release_id_;
+  const int width_;
+  const int height_;
+  const Matrix matrix_;
+  // Holds a VideoFrame.Buffer.
+  ScopedGlobalRef<jobject> j_video_frame_buffer_;
+};
+
+class AndroidVideoBufferFactory {
+ public:
+  explicit AndroidVideoBufferFactory(JNIEnv* jni);
+
+  webrtc::VideoFrame CreateFrame(JNIEnv* jni,
+                                 jobject j_video_frame,
+                                 uint32_t timestamp_rtp) const;
+
+  rtc::scoped_refptr<AndroidVideoBuffer> CreateBuffer(
+      int width,
+      int height,
+      const Matrix& matrix,
+      jobject j_video_frame_buffer) const;
+
+ private:
+  ScopedGlobalRef<jclass> j_video_frame_class_;
+  jmethodID j_get_buffer_id_;
+  jmethodID j_get_width_id_;
+  jmethodID j_get_height_id_;
+  jmethodID j_get_rotation_id_;
+  jmethodID j_get_transform_matrix_id_;
+  jmethodID j_get_timestamp_ns_id_;
+
+  ScopedGlobalRef<jclass> j_video_frame_buffer_class_;
+  jmethodID j_retain_id_;
+  jmethodID j_release_id_;
+};
+
 }  // namespace webrtc_jni
 
 #endif  // WEBRTC_SDK_ANDROID_SRC_JNI_NATIVE_HANDLE_IMPL_H_

webrtc/sdk/android/src/jni/peerconnection_jni.cc

@@ -871,10 +871,29 @@ class JavaVideoRendererWrapper
 
   void OnFrame(const webrtc::VideoFrame& video_frame) override {
     ScopedLocalRefFrame local_ref_frame(jni());
-    jobject j_frame = (video_frame.video_frame_buffer()->type() ==
+
-                       webrtc::VideoFrameBuffer::Type::kNative)
+    jobject j_frame;
-                          ? ToJavaTextureFrame(&video_frame)
+    if (video_frame.video_frame_buffer()->type() ==
-                          : ToJavaI420Frame(&video_frame);
+        webrtc::VideoFrameBuffer::Type::kNative) {
+      AndroidVideoFrameBuffer* android_buffer =
+          static_cast<AndroidVideoFrameBuffer*>(
+              video_frame.video_frame_buffer().get());
+      switch (android_buffer->android_type()) {
+        case AndroidVideoFrameBuffer::AndroidType::kTextureBuffer:
+          j_frame = ToJavaTextureFrame(&video_frame);
+          break;
+        case AndroidVideoFrameBuffer::AndroidType::kJavaBuffer:
+          j_frame = static_cast<AndroidVideoBuffer*>(android_buffer)
+                        ->ToJavaI420Frame(jni(), video_frame.width(),
+                                          video_frame.height(),
+                                          video_frame.rotation());
+          break;
+        default:
+          RTC_NOTREACHED();
+      }
+    } else {
+      j_frame = ToJavaI420Frame(&video_frame);
+    }
     // |j_callbacks_| is responsible for releasing |j_frame| with
     // VideoRenderer.renderFrameDone().
     jni()->CallVoidMethod(*j_callbacks_, j_render_frame_id_, j_frame);