Add VideoSink interface to VideoFileRenderer.

Bug: webrtc:8776
Change-Id: I1782b0c197abf6f82a200a2808ddc87d1f250326
Reviewed-on: https://webrtc-review.googlesource.com/41320
Reviewed-by: Anders Carlsson <andersc@webrtc.org>
Commit-Queue: Sami Kalliomäki <sakal@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#21719}

sdk/android/BUILD.gn

@@ -139,7 +139,6 @@ generate_jni("generated_video_jni") {
     "api/org/webrtc/VideoEncoder.java",
     "api/org/webrtc/VideoEncoderFactory.java",
     "api/org/webrtc/VideoEncoderFallback.java",
-    "api/org/webrtc/VideoFileRenderer.java",
     "api/org/webrtc/VideoFrame.java",
     "api/org/webrtc/VideoRenderer.java",
     "api/org/webrtc/VideoSink.java",
@@ -197,7 +196,6 @@ rtc_static_library("video_jni") {
     "src/jni/videoencoderfallback.cc",
     "src/jni/videoencoderwrapper.cc",
     "src/jni/videoencoderwrapper.h",
-    "src/jni/videofilerenderer.cc",
     "src/jni/videoframe.cc",
     "src/jni/videoframe.h",
     "src/jni/videotrack.cc",

sdk/android/api/org/webrtc/VideoFileRenderer.java

@@ -17,17 +17,18 @@ import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.charset.Charset;
 import java.util.ArrayList;
+import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.LinkedBlockingQueue;
 
 /**
  * Can be used to save the video frames to file.
  */
 @JNINamespace("webrtc::jni")
-public class VideoFileRenderer implements VideoRenderer.Callbacks {
+public class VideoFileRenderer implements VideoRenderer.Callbacks, VideoSink {
   private static final String TAG = "VideoFileRenderer";
 
   private final HandlerThread renderThread;
-  private final Object handlerLock = new Object();
   private final Handler renderThreadHandler;
   private final FileOutputStream videoOutFile;
   private final String outputFileName;
@@ -73,61 +74,56 @@ public class VideoFileRenderer implements VideoRenderer.Callbacks {
   }
 
   @Override
-  public void renderFrame(final VideoRenderer.I420Frame frame) {
+  public void renderFrame(final VideoRenderer.I420Frame i420Frame) {
-    renderThreadHandler.post(new Runnable() {
+    final VideoFrame frame = i420Frame.toVideoFrame();
+    onFrame(frame);
+    frame.release();
+  }
+
   @Override
-      public void run() {
+  public void onFrame(VideoFrame frame) {
-        renderFrameOnRenderThread(frame);
+    frame.retain();
-      }
+    renderThreadHandler.post(() -> renderFrameOnRenderThread(frame));
-    });
   }
 
-  // TODO(sakal): yuvConverter.convert is deprecated. This will be removed once this file is updated
+  private void renderFrameOnRenderThread(VideoFrame frame) {
-  // to implement VideoSink instead of VideoRenderer.Callbacks.
+    final VideoFrame.Buffer buffer = frame.getBuffer();
-  @SuppressWarnings("deprecation")
-  private void renderFrameOnRenderThread(VideoRenderer.I420Frame frame) {
-    final float frameAspectRatio = (float) frame.rotatedWidth() / (float) frame.rotatedHeight();
 
-    final float[] rotatedSamplingMatrix =
+    // If the frame is rotated, it will be applied after cropAndScale. Therefore, if the frame is
-        RendererCommon.rotateTextureMatrix(frame.samplingMatrix, frame.rotationDegree);
+    // rotated by 90 degrees, swap width and height.
-    final float[] layoutMatrix = RendererCommon.getLayoutMatrix(
+    final int targetWidth = frame.getRotation() % 180 == 0 ? outputFileWidth : outputFileHeight;
-        false, frameAspectRatio, (float) outputFileWidth / outputFileHeight);
+    final int targetHeight = frame.getRotation() % 180 == 0 ? outputFileHeight : outputFileWidth;
-    final float[] texMatrix = RendererCommon.multiplyMatrices(rotatedSamplingMatrix, layoutMatrix);
 
-    try {
+    final float frameAspectRatio = (float) buffer.getWidth() / (float) buffer.getHeight();
-      ByteBuffer buffer = JniCommon.nativeAllocateByteBuffer(outputFrameSize);
+    final float fileAspectRatio = (float) targetWidth / (float) targetHeight;
-      if (!frame.yuvFrame) {
-        yuvConverter.convert(outputFrameBuffer, outputFileWidth, outputFileHeight, outputFileWidth,
-            frame.textureId, texMatrix);
 
-        int stride = outputFileWidth;
+    // Calculate cropping to equalize the aspect ratio.
-        byte[] data = outputFrameBuffer.array();
+    int cropWidth = buffer.getWidth();
-        int offset = outputFrameBuffer.arrayOffset();
+    int cropHeight = buffer.getHeight();
-
+    if (fileAspectRatio > frameAspectRatio) {
-        // Write Y
+      cropHeight *= frameAspectRatio / fileAspectRatio;
-        buffer.put(data, offset, outputFileWidth * outputFileHeight);
-
-        // Write U
-        for (int r = outputFileHeight; r < outputFileHeight * 3 / 2; ++r) {
-          buffer.put(data, offset + r * stride, stride / 2);
-        }
-
-        // Write V
-        for (int r = outputFileHeight; r < outputFileHeight * 3 / 2; ++r) {
-          buffer.put(data, offset + r * stride + stride / 2, stride / 2);
-        }
     } else {
-        nativeI420Scale(frame.yuvPlanes[0], frame.yuvStrides[0], frame.yuvPlanes[1],
+      cropWidth *= fileAspectRatio / frameAspectRatio;
-            frame.yuvStrides[1], frame.yuvPlanes[2], frame.yuvStrides[2], frame.width, frame.height,
+    }
-            outputFrameBuffer, outputFileWidth, outputFileHeight);
 
-        buffer.put(outputFrameBuffer.array(), outputFrameBuffer.arrayOffset(), outputFrameSize);
+    final int cropX = (buffer.getWidth() - cropWidth) / 2;
-      }
+    final int cropY = (buffer.getHeight() - cropHeight) / 2;
-      buffer.rewind();
+
-      rawFrames.add(buffer);
+    final VideoFrame.Buffer scaledBuffer =
-    } finally {
+        buffer.cropAndScale(cropX, cropY, cropWidth, cropHeight, targetWidth, targetHeight);
-      VideoRenderer.renderFrameDone(frame);
+    frame.release();
-    }
+
+    final VideoFrame.I420Buffer i420 = scaledBuffer.toI420();
+    scaledBuffer.release();
+
+    ByteBuffer byteBuffer = JniCommon.nativeAllocateByteBuffer(outputFrameSize);
+    YuvHelper.I420Rotate(i420.getDataY(), i420.getStrideY(), i420.getDataU(), i420.getStrideU(),
+        i420.getDataV(), i420.getStrideV(), byteBuffer, i420.getWidth(), i420.getHeight(),
+        frame.getRotation());
+    i420.release();
+
+    byteBuffer.rewind();
+    rawFrames.add(byteBuffer);
   }
 
   /**
@@ -135,14 +131,11 @@ public class VideoFileRenderer implements VideoRenderer.Callbacks {
    */
   public void release() {
     final CountDownLatch cleanupBarrier = new CountDownLatch(1);
-    renderThreadHandler.post(new Runnable() {
+    renderThreadHandler.post(() -> {
-      @Override
-      public void run() {
       yuvConverter.release();
       eglBase.release();
       renderThread.quit();
       cleanupBarrier.countDown();
-      }
     });
     ThreadUtils.awaitUninterruptibly(cleanupBarrier);
     try {
@@ -157,15 +150,12 @@ public class VideoFileRenderer implements VideoRenderer.Callbacks {
         JniCommon.nativeFreeByteBuffer(buffer);
       }
       videoOutFile.close();
-      Logging.d(TAG, "Video written to disk as " + outputFileName + ". Number frames are "
+      Logging.d(TAG,
-              + rawFrames.size() + " and the dimension of the frames are " + outputFileWidth + "x"
+          "Video written to disk as " + outputFileName + ". Number frames are " + rawFrames.size()
-              + outputFileHeight + ".");
+              + " and the dimension of the frames are " + outputFileWidth + "x" + outputFileHeight
+              + ".");
     } catch (IOException e) {
       Logging.e(TAG, "Error writing video to disk", e);
     }
   }
-
-  public static native void nativeI420Scale(ByteBuffer srcY, int strideY, ByteBuffer srcU,
-      int strideU, ByteBuffer srcV, int strideV, int width, int height, ByteBuffer dst,
-      int dstWidth, int dstHeight);
 }

sdk/android/api/org/webrtc/YuvHelper.java

@@ -66,6 +66,37 @@ public class YuvHelper {
         chromaWidth * 2, width, height);
   }
 
+  /** Helper method for rotating I420 to tightly packed destination buffer. */
+  public static void I420Rotate(ByteBuffer srcY, int srcStrideY, ByteBuffer srcU, int srcStrideU,
+      ByteBuffer srcV, int srcStrideV, ByteBuffer dst, int srcWidth, int srcHeight,
+      int rotationMode) {
+    final int dstWidth = rotationMode % 180 == 0 ? srcWidth : srcHeight;
+    final int dstHeight = rotationMode % 180 == 0 ? srcHeight : srcWidth;
+
+    final int dstChromaHeight = (dstHeight + 1) / 2;
+    final int dstChromaWidth = (dstWidth + 1) / 2;
+
+    final int minSize = dstWidth * dstHeight + dstChromaWidth * dstChromaHeight * 2;
+    if (dst.capacity() < minSize) {
+      throw new IllegalArgumentException("Expected destination buffer capacity to be at least "
+          + minSize + " was " + dst.capacity());
+    }
+
+    final int startY = 0;
+    final int startU = dstHeight * dstWidth;
+    final int startV = startU + dstChromaHeight * dstChromaWidth;
+
+    dst.position(startY);
+    final ByteBuffer dstY = dst.slice();
+    dst.position(startU);
+    final ByteBuffer dstU = dst.slice();
+    dst.position(startV);
+    final ByteBuffer dstV = dst.slice();
+
+    nativeI420Rotate(srcY, srcStrideY, srcU, srcStrideU, srcV, srcStrideV, dstY, dstWidth, dstU,
+        dstChromaWidth, dstV, dstChromaWidth, srcWidth, srcHeight, rotationMode);
+  }
+
   public static void I420Copy(ByteBuffer srcY, int srcStrideY, ByteBuffer srcU, int srcStrideU,
       ByteBuffer srcV, int srcStrideV, ByteBuffer dstY, int dstStrideY, ByteBuffer dstU,
       int dstStrideU, ByteBuffer dstV, int dstStrideV, int width, int height) {
@@ -80,10 +111,22 @@ public class YuvHelper {
         dstStrideUV, width, height);
   }
 
+  public static void I420Rotate(ByteBuffer srcY, int srcStrideY, ByteBuffer srcU, int srcStrideU,
+      ByteBuffer srcV, int srcStrideV, ByteBuffer dstY, int dstStrideY, ByteBuffer dstU,
+      int dstStrideU, ByteBuffer dstV, int dstStrideV, int srcWidth, int srcHeight,
+      int rotationMode) {
+    nativeI420Rotate(srcY, srcStrideY, srcU, srcStrideU, srcV, srcStrideV, dstY, dstStrideY, dstU,
+        dstStrideU, dstV, dstStrideV, srcWidth, srcHeight, rotationMode);
+  }
+
   private static native void nativeI420Copy(ByteBuffer srcY, int srcStrideY, ByteBuffer srcU,
       int srcStrideU, ByteBuffer srcV, int srcStrideV, ByteBuffer dstY, int dstStrideY,
       ByteBuffer dstU, int dstStrideU, ByteBuffer dstV, int dstStrideV, int width, int height);
   private static native void nativeI420ToNV12(ByteBuffer srcY, int srcStrideY, ByteBuffer srcU,
       int srcStrideU, ByteBuffer srcV, int srcStrideV, ByteBuffer dstY, int dstStrideY,
       ByteBuffer dstUV, int dstStrideUV, int width, int height);
+  private static native void nativeI420Rotate(ByteBuffer srcY, int srcStrideY, ByteBuffer srcU,
+      int srcStrideU, ByteBuffer srcV, int srcStrideV, ByteBuffer dstY, int dstStrideY,
+      ByteBuffer dstU, int dstStrideU, ByteBuffer dstV, int dstStrideV, int srcWidth, int srcHeight,
+      int rotationMode);
 }

sdk/android/instrumentationtests/src/org/webrtc/VideoFileRendererTest.java

@@ -51,22 +51,25 @@ public class VideoFileRendererTest {
     for (String frameStr : frames) {
       int[] planeSizes = {
           frameWidth * frameWidth, frameWidth * frameHeight / 4, frameWidth * frameHeight / 4};
+      int[] yuvStrides = {frameWidth, frameWidth / 2, frameWidth / 2};
 
-      byte[] frameBytes = frameStr.getBytes(Charset.forName("US-ASCII"));
       ByteBuffer[] yuvPlanes = new ByteBuffer[3];
+      byte[] frameBytes = frameStr.getBytes(Charset.forName("US-ASCII"));
       int pos = 0;
       for (int i = 0; i < 3; i++) {
         yuvPlanes[i] = ByteBuffer.allocateDirect(planeSizes[i]);
         yuvPlanes[i].put(frameBytes, pos, planeSizes[i]);
+        yuvPlanes[i].rewind();
         pos += planeSizes[i];
       }
 
-      int[] yuvStrides = {frameWidth, frameWidth / 2, frameWidth / 2};
+      VideoFrame.I420Buffer buffer =
+          JavaI420Buffer.wrap(frameWidth, frameHeight, yuvPlanes[0], yuvStrides[0], yuvPlanes[1],
+              yuvStrides[1], yuvPlanes[2], yuvStrides[2], null /* releaseCallback */);
 
-      VideoRenderer.I420Frame frame =
+      VideoFrame frame = new VideoFrame(buffer, 0 /* rotation */, 0 /* timestampNs */);
-          new VideoRenderer.I420Frame(frameWidth, frameHeight, 0, yuvStrides, yuvPlanes, 0);
+      videoFileRenderer.onFrame(frame);
-
+      frame.release();
-      videoFileRenderer.renderFrame(frame);
     }
     videoFileRenderer.release();
 

sdk/android/instrumentationtests/src/org/webrtc/YuvHelperTest.java

@@ -129,4 +129,36 @@ public class YuvHelperTest {
       assertEquals("Unexpected ByteBuffer contents at index: " + i, expected[i], test.get(i));
     }
   }
+
+  @SmallTest
+  @Test
+  public void testI420Rotate90() {
+    final int dstStrideY = TEST_HEIGHT;
+    final int dstStrideU = TEST_CHROMA_HEIGHT;
+    final int dstStrideV = TEST_CHROMA_HEIGHT;
+    final ByteBuffer dstY = ByteBuffer.allocateDirect(TEST_WIDTH * dstStrideY);
+    final ByteBuffer dstU = ByteBuffer.allocateDirect(TEST_CHROMA_WIDTH * dstStrideU);
+    final ByteBuffer dstV = ByteBuffer.allocateDirect(TEST_CHROMA_WIDTH * dstStrideV);
+
+    YuvHelper.I420Rotate(TEST_I420_Y, TEST_I420_STRIDE_Y, TEST_I420_U, TEST_I420_STRIDE_V,
+        TEST_I420_V, TEST_I420_STRIDE_U, dstY, dstStrideY, dstU, dstStrideU, dstV, dstStrideV,
+        TEST_WIDTH, TEST_HEIGHT, 90);
+
+    assertByteBufferContentEquals(new byte[] {7, 4, 1, 8, 5, 2, 9, 6, 3}, dstY);
+    assertByteBufferContentEquals(new byte[] {53, 51, 54, 52}, dstU);
+    assertByteBufferContentEquals(new byte[] {105, 101, 106, 102}, dstV);
+  }
+
+  @SmallTest
+  @Test
+  public void testI420Rotate90Tight() {
+    final ByteBuffer dst = ByteBuffer.allocateDirect(
+        TEST_WIDTH * TEST_HEIGHT + TEST_CHROMA_WIDTH * TEST_CHROMA_HEIGHT * 2);
+
+    YuvHelper.I420Rotate(TEST_I420_Y, TEST_I420_STRIDE_Y, TEST_I420_U, TEST_I420_STRIDE_V,
+        TEST_I420_V, TEST_I420_STRIDE_U, dst, TEST_WIDTH, TEST_HEIGHT, 90);
+
+    assertByteBufferContentEquals(
+        new byte[] {7, 4, 1, 8, 5, 2, 9, 6, 3, 53, 51, 54, 52, 105, 101, 106, 102}, dst);
+  }
 }

sdk/android/src/jni/videofilerenderer.cc

@@ -1,71 +0,0 @@
-/*
- *  Copyright 2017 The WebRTC project authors. All Rights Reserved.
- *
- *  Use of this source code is governed by a BSD-style license
- *  that can be found in the LICENSE file in the root of the source
- *  tree. An additional intellectual property rights grant can be found
- *  in the file PATENTS.  All contributing project authors may
- *  be found in the AUTHORS file in the root of the source tree.
- */
-
-#include <jni.h>
-
-#include "rtc_base/checks.h"
-#include "rtc_base/logging.h"
-#include "sdk/android/generated_video_jni/jni/VideoFileRenderer_jni.h"
-#include "sdk/android/src/jni/scoped_java_ref.h"
-#include "third_party/libyuv/include/libyuv/scale.h"
-
-namespace webrtc {
-namespace jni {
-
-static void JNI_VideoFileRenderer_I420Scale(
-    JNIEnv* jni,
-    const JavaParamRef<jclass>&,
-    const JavaParamRef<jobject>& j_src_buffer_y,
-    jint j_src_stride_y,
-    const JavaParamRef<jobject>& j_src_buffer_u,
-    jint j_src_stride_u,
-    const JavaParamRef<jobject>& j_src_buffer_v,
-    jint j_src_stride_v,
-    jint width,
-    jint height,
-    const JavaParamRef<jobject>& j_dst_buffer,
-    jint dstWidth,
-    jint dstHeight) {
-  size_t src_size_y = jni->GetDirectBufferCapacity(j_src_buffer_y.obj());
-  size_t src_size_u = jni->GetDirectBufferCapacity(j_src_buffer_u.obj());
-  size_t src_size_v = jni->GetDirectBufferCapacity(j_src_buffer_v.obj());
-  size_t dst_size = jni->GetDirectBufferCapacity(j_dst_buffer.obj());
-  int dst_stride = dstWidth;
-  RTC_CHECK_GE(src_size_y, j_src_stride_y * height);
-  RTC_CHECK_GE(src_size_u, j_src_stride_u * height / 4);
-  RTC_CHECK_GE(src_size_v, j_src_stride_v * height / 4);
-  RTC_CHECK_GE(dst_size, dst_stride * dstHeight * 3 / 2);
-  uint8_t* src_y = reinterpret_cast<uint8_t*>(
-      jni->GetDirectBufferAddress(j_src_buffer_y.obj()));
-  uint8_t* src_u = reinterpret_cast<uint8_t*>(
-      jni->GetDirectBufferAddress(j_src_buffer_u.obj()));
-  uint8_t* src_v = reinterpret_cast<uint8_t*>(
-      jni->GetDirectBufferAddress(j_src_buffer_v.obj()));
-  uint8_t* dst = reinterpret_cast<uint8_t*>(
-      jni->GetDirectBufferAddress(j_dst_buffer.obj()));
-
-  uint8_t* dst_y = dst;
-  size_t dst_stride_y = dst_stride;
-  uint8_t* dst_u = dst + dst_stride * dstHeight;
-  size_t dst_stride_u = dst_stride / 2;
-  uint8_t* dst_v = dst + dst_stride * dstHeight * 5 / 4;
-  size_t dst_stride_v = dst_stride / 2;
-
-  int ret = libyuv::I420Scale(
-      src_y, j_src_stride_y, src_u, j_src_stride_u, src_v, j_src_stride_v,
-      width, height, dst_y, dst_stride_y, dst_u, dst_stride_u, dst_v,
-      dst_stride_v, dstWidth, dstHeight, libyuv::kFilterBilinear);
-  if (ret) {
-    RTC_LOG(LS_ERROR) << "Error scaling I420 frame: " << ret;
-  }
-}
-
-}  // namespace jni
-}  // namespace webrtc

sdk/android/src/jni/yuvhelper.cc

@@ -81,5 +81,41 @@ static void JNI_YuvHelper_I420ToNV12(JNIEnv* jni,
                      width, height);
 }
 
+void JNI_YuvHelper_I420Rotate(JNIEnv* jni,
+                              const JavaParamRef<jclass>&,
+                              const JavaParamRef<jobject>& j_src_y,
+                              jint src_stride_y,
+                              const JavaParamRef<jobject>& j_src_u,
+                              jint src_stride_u,
+                              const JavaParamRef<jobject>& j_src_v,
+                              jint src_stride_v,
+                              const JavaParamRef<jobject>& j_dst_y,
+                              jint dst_stride_y,
+                              const JavaParamRef<jobject>& j_dst_u,
+                              jint dst_stride_u,
+                              const JavaParamRef<jobject>& j_dst_v,
+                              jint dst_stride_v,
+                              jint src_width,
+                              jint src_height,
+                              jint rotation_mode) {
+  const uint8_t* src_y =
+      static_cast<const uint8_t*>(jni->GetDirectBufferAddress(j_src_y.obj()));
+  const uint8_t* src_u =
+      static_cast<const uint8_t*>(jni->GetDirectBufferAddress(j_src_u.obj()));
+  const uint8_t* src_v =
+      static_cast<const uint8_t*>(jni->GetDirectBufferAddress(j_src_v.obj()));
+  uint8_t* dst_y =
+      static_cast<uint8_t*>(jni->GetDirectBufferAddress(j_dst_y.obj()));
+  uint8_t* dst_u =
+      static_cast<uint8_t*>(jni->GetDirectBufferAddress(j_dst_u.obj()));
+  uint8_t* dst_v =
+      static_cast<uint8_t*>(jni->GetDirectBufferAddress(j_dst_v.obj()));
+
+  libyuv::I420Rotate(src_y, src_stride_y, src_u, src_stride_u, src_v,
+                     src_stride_v, dst_y, dst_stride_y, dst_u, dst_stride_u,
+                     dst_v, dst_stride_v, src_width, src_height,
+                     static_cast<libyuv::RotationMode>(rotation_mode));
+}
+
 }  // namespace jni
 }  // namespace webrtc