zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add VideoSink interface to VideoFileRenderer.

Browse Source 
Bug: webrtc:8776
Change-Id: I1782b0c197abf6f82a200a2808ddc87d1f250326
Reviewed-on: https://webrtc-review.googlesource.com/41320
Reviewed-by: Anders Carlsson <andersc@webrtc.org>
Commit-Queue: Sami Kalliomäki <sakal@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#21719}
This commit is contained in:
Sami Kalliomäki
2018-01-22 14:19:16 +01:00
committed by Commit Bot
parent d8f6c167bb
commit 75db552b33
7 changed files with 175 additions and 144 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
sdk/android/BUILD.gn
View File

@ -139,7 +139,6 @@ generate_jni("generated_video_jni") {
"api/org/webrtc/VideoEncoder.java",
"api/org/webrtc/VideoEncoderFactory.java",
"api/org/webrtc/VideoEncoderFallback.java",
"api/org/webrtc/VideoFileRenderer.java",
"api/org/webrtc/VideoFrame.java",
"api/org/webrtc/VideoRenderer.java",
"api/org/webrtc/VideoSink.java",
@ -197,7 +196,6 @@ rtc_static_library("video_jni") {
"src/jni/videoencoderfallback.cc",
"src/jni/videoencoderwrapper.cc",
"src/jni/videoencoderwrapper.h",
"src/jni/videofilerenderer.cc",
"src/jni/videoframe.cc",
"src/jni/videoframe.h",
"src/jni/videotrack.cc",

110
sdk/android/api/org/webrtc/VideoFileRenderer.java
View File

@ -17,17 +17,18 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.LinkedBlockingQueue;
/**
* Can be used to save the video frames to file.
*/
@JNINamespace("webrtc::jni")
public class VideoFileRenderer implements VideoRenderer.Callbacks {
public class VideoFileRenderer implements VideoRenderer.Callbacks, VideoSink {
private static final String TAG = "VideoFileRenderer";
private final HandlerThread renderThread;
private final Object handlerLock = new Object();
private final Handler renderThreadHandler;
private final FileOutputStream videoOutFile;
private final String outputFileName;
@ -73,61 +74,56 @@ public class VideoFileRenderer implements VideoRenderer.Callbacks {
}
@Override
public void renderFrame(final VideoRenderer.I420Frame frame) {
renderThreadHandler.post(new Runnable() {
public void renderFrame(final VideoRenderer.I420Frame i420Frame) {
final VideoFrame frame = i420Frame.toVideoFrame();
onFrame(frame);
frame.release();
}
@Override
public void run() {
renderFrameOnRenderThread(frame);
}
});
public void onFrame(VideoFrame frame) {
frame.retain();
renderThreadHandler.post(() -> renderFrameOnRenderThread(frame));
}
// TODO(sakal): yuvConverter.convert is deprecated. This will be removed once this file is updated
// to implement VideoSink instead of VideoRenderer.Callbacks.
@SuppressWarnings("deprecation")
private void renderFrameOnRenderThread(VideoRenderer.I420Frame frame) {
final float frameAspectRatio = (float) frame.rotatedWidth() / (float) frame.rotatedHeight();
private void renderFrameOnRenderThread(VideoFrame frame) {
final VideoFrame.Buffer buffer = frame.getBuffer();
final float[] rotatedSamplingMatrix =
RendererCommon.rotateTextureMatrix(frame.samplingMatrix, frame.rotationDegree);
final float[] layoutMatrix = RendererCommon.getLayoutMatrix(
false, frameAspectRatio, (float) outputFileWidth / outputFileHeight);
final float[] texMatrix = RendererCommon.multiplyMatrices(rotatedSamplingMatrix, layoutMatrix);
// If the frame is rotated, it will be applied after cropAndScale. Therefore, if the frame is
// rotated by 90 degrees, swap width and height.
final int targetWidth = frame.getRotation() % 180 == 0 ? outputFileWidth : outputFileHeight;
final int targetHeight = frame.getRotation() % 180 == 0 ? outputFileHeight : outputFileWidth;
try {
ByteBuffer buffer = JniCommon.nativeAllocateByteBuffer(outputFrameSize);
if (!frame.yuvFrame) {
yuvConverter.convert(outputFrameBuffer, outputFileWidth, outputFileHeight, outputFileWidth,
frame.textureId, texMatrix);
final float frameAspectRatio = (float) buffer.getWidth() / (float) buffer.getHeight();
final float fileAspectRatio = (float) targetWidth / (float) targetHeight;
int stride = outputFileWidth;
byte[] data = outputFrameBuffer.array();
int offset = outputFrameBuffer.arrayOffset();
// Write Y
buffer.put(data, offset, outputFileWidth * outputFileHeight);
// Write U
for (int r = outputFileHeight; r < outputFileHeight * 3 / 2; ++r) {
buffer.put(data, offset + r * stride, stride / 2);
}
// Write V
for (int r = outputFileHeight; r < outputFileHeight * 3 / 2; ++r) {
buffer.put(data, offset + r * stride + stride / 2, stride / 2);
}
// Calculate cropping to equalize the aspect ratio.
int cropWidth = buffer.getWidth();
int cropHeight = buffer.getHeight();
if (fileAspectRatio > frameAspectRatio) {
cropHeight *= frameAspectRatio / fileAspectRatio;
} else {
nativeI420Scale(frame.yuvPlanes[0], frame.yuvStrides[0], frame.yuvPlanes[1],
frame.yuvStrides[1], frame.yuvPlanes[2], frame.yuvStrides[2], frame.width, frame.height,
outputFrameBuffer, outputFileWidth, outputFileHeight);
cropWidth *= fileAspectRatio / frameAspectRatio;
}
buffer.put(outputFrameBuffer.array(), outputFrameBuffer.arrayOffset(), outputFrameSize);
}
buffer.rewind();
rawFrames.add(buffer);
} finally {
VideoRenderer.renderFrameDone(frame);
}
final int cropX = (buffer.getWidth() - cropWidth) / 2;
final int cropY = (buffer.getHeight() - cropHeight) / 2;
final VideoFrame.Buffer scaledBuffer =
buffer.cropAndScale(cropX, cropY, cropWidth, cropHeight, targetWidth, targetHeight);
frame.release();
final VideoFrame.I420Buffer i420 = scaledBuffer.toI420();
scaledBuffer.release();
ByteBuffer byteBuffer = JniCommon.nativeAllocateByteBuffer(outputFrameSize);
YuvHelper.I420Rotate(i420.getDataY(), i420.getStrideY(), i420.getDataU(), i420.getStrideU(),
i420.getDataV(), i420.getStrideV(), byteBuffer, i420.getWidth(), i420.getHeight(),
frame.getRotation());
i420.release();
byteBuffer.rewind();
rawFrames.add(byteBuffer);
}
/**
@ -135,14 +131,11 @@ public class VideoFileRenderer implements VideoRenderer.Callbacks {
*/
public void release() {
final CountDownLatch cleanupBarrier = new CountDownLatch(1);
renderThreadHandler.post(new Runnable() {
@Override
public void run() {
renderThreadHandler.post(() -> {
yuvConverter.release();
eglBase.release();
renderThread.quit();
cleanupBarrier.countDown();
}
});
ThreadUtils.awaitUninterruptibly(cleanupBarrier);
try {
@ -157,15 +150,12 @@ public class VideoFileRenderer implements VideoRenderer.Callbacks {
JniCommon.nativeFreeByteBuffer(buffer);
}
videoOutFile.close();
Logging.d(TAG, "Video written to disk as " + outputFileName + ". Number frames are "
+ rawFrames.size() + " and the dimension of the frames are " + outputFileWidth + "x"
+ outputFileHeight + ".");
Logging.d(TAG,
"Video written to disk as " + outputFileName + ". Number frames are " + rawFrames.size()
+ " and the dimension of the frames are " + outputFileWidth + "x" + outputFileHeight
+ ".");
} catch (IOException e) {
Logging.e(TAG, "Error writing video to disk", e);
}
}
public static native void nativeI420Scale(ByteBuffer srcY, int strideY, ByteBuffer srcU,
int strideU, ByteBuffer srcV, int strideV, int width, int height, ByteBuffer dst,
int dstWidth, int dstHeight);
}

43
sdk/android/api/org/webrtc/YuvHelper.java
View File

@ -66,6 +66,37 @@ public class YuvHelper {
chromaWidth * 2, width, height);
}
/** Helper method for rotating I420 to tightly packed destination buffer. */
public static void I420Rotate(ByteBuffer srcY, int srcStrideY, ByteBuffer srcU, int srcStrideU,
ByteBuffer srcV, int srcStrideV, ByteBuffer dst, int srcWidth, int srcHeight,
int rotationMode) {
final int dstWidth = rotationMode % 180 == 0 ? srcWidth : srcHeight;
final int dstHeight = rotationMode % 180 == 0 ? srcHeight : srcWidth;
final int dstChromaHeight = (dstHeight + 1) / 2;
final int dstChromaWidth = (dstWidth + 1) / 2;
final int minSize = dstWidth * dstHeight + dstChromaWidth * dstChromaHeight * 2;
if (dst.capacity() < minSize) {
throw new IllegalArgumentException("Expected destination buffer capacity to be at least "
+ minSize + " was " + dst.capacity());
}
final int startY = 0;
final int startU = dstHeight * dstWidth;
final int startV = startU + dstChromaHeight * dstChromaWidth;
dst.position(startY);
final ByteBuffer dstY = dst.slice();
dst.position(startU);
final ByteBuffer dstU = dst.slice();
dst.position(startV);
final ByteBuffer dstV = dst.slice();
nativeI420Rotate(srcY, srcStrideY, srcU, srcStrideU, srcV, srcStrideV, dstY, dstWidth, dstU,
dstChromaWidth, dstV, dstChromaWidth, srcWidth, srcHeight, rotationMode);
}
public static void I420Copy(ByteBuffer srcY, int srcStrideY, ByteBuffer srcU, int srcStrideU,
ByteBuffer srcV, int srcStrideV, ByteBuffer dstY, int dstStrideY, ByteBuffer dstU,
int dstStrideU, ByteBuffer dstV, int dstStrideV, int width, int height) {
@ -80,10 +111,22 @@ public class YuvHelper {
dstStrideUV, width, height);
}
public static void I420Rotate(ByteBuffer srcY, int srcStrideY, ByteBuffer srcU, int srcStrideU,
ByteBuffer srcV, int srcStrideV, ByteBuffer dstY, int dstStrideY, ByteBuffer dstU,
int dstStrideU, ByteBuffer dstV, int dstStrideV, int srcWidth, int srcHeight,
int rotationMode) {
nativeI420Rotate(srcY, srcStrideY, srcU, srcStrideU, srcV, srcStrideV, dstY, dstStrideY, dstU,
dstStrideU, dstV, dstStrideV, srcWidth, srcHeight, rotationMode);
}
private static native void nativeI420Copy(ByteBuffer srcY, int srcStrideY, ByteBuffer srcU,
int srcStrideU, ByteBuffer srcV, int srcStrideV, ByteBuffer dstY, int dstStrideY,
ByteBuffer dstU, int dstStrideU, ByteBuffer dstV, int dstStrideV, int width, int height);
private static native void nativeI420ToNV12(ByteBuffer srcY, int srcStrideY, ByteBuffer srcU,
int srcStrideU, ByteBuffer srcV, int srcStrideV, ByteBuffer dstY, int dstStrideY,
ByteBuffer dstUV, int dstStrideUV, int width, int height);
private static native void nativeI420Rotate(ByteBuffer srcY, int srcStrideY, ByteBuffer srcU,
int srcStrideU, ByteBuffer srcV, int srcStrideV, ByteBuffer dstY, int dstStrideY,
ByteBuffer dstU, int dstStrideU, ByteBuffer dstV, int dstStrideV, int srcWidth, int srcHeight,
int rotationMode);
}

15
sdk/android/instrumentationtests/src/org/webrtc/VideoFileRendererTest.java
View File

@ -51,22 +51,25 @@ public class VideoFileRendererTest {
for (String frameStr : frames) {
int[] planeSizes = {
frameWidth * frameWidth, frameWidth * frameHeight / 4, frameWidth * frameHeight / 4};
int[] yuvStrides = {frameWidth, frameWidth / 2, frameWidth / 2};
byte[] frameBytes = frameStr.getBytes(Charset.forName("US-ASCII"));
ByteBuffer[] yuvPlanes = new ByteBuffer[3];
byte[] frameBytes = frameStr.getBytes(Charset.forName("US-ASCII"));
int pos = 0;
for (int i = 0; i < 3; i++) {
yuvPlanes[i] = ByteBuffer.allocateDirect(planeSizes[i]);
yuvPlanes[i].put(frameBytes, pos, planeSizes[i]);
yuvPlanes[i].rewind();
pos += planeSizes[i];
}
int[] yuvStrides = {frameWidth, frameWidth / 2, frameWidth / 2};
VideoFrame.I420Buffer buffer =
JavaI420Buffer.wrap(frameWidth, frameHeight, yuvPlanes[0], yuvStrides[0], yuvPlanes[1],
yuvStrides[1], yuvPlanes[2], yuvStrides[2], null /* releaseCallback */);
VideoRenderer.I420Frame frame =
new VideoRenderer.I420Frame(frameWidth, frameHeight, 0, yuvStrides, yuvPlanes, 0);
videoFileRenderer.renderFrame(frame);
VideoFrame frame = new VideoFrame(buffer, 0 /* rotation */, 0 /* timestampNs */);
videoFileRenderer.onFrame(frame);
frame.release();
}
videoFileRenderer.release();

32
sdk/android/instrumentationtests/src/org/webrtc/YuvHelperTest.java
View File

@ -129,4 +129,36 @@ public class YuvHelperTest {
assertEquals("Unexpected ByteBuffer contents at index: " + i, expected[i], test.get(i));
}
}
@SmallTest
@Test
public void testI420Rotate90() {
final int dstStrideY = TEST_HEIGHT;
final int dstStrideU = TEST_CHROMA_HEIGHT;
final int dstStrideV = TEST_CHROMA_HEIGHT;
final ByteBuffer dstY = ByteBuffer.allocateDirect(TEST_WIDTH * dstStrideY);
final ByteBuffer dstU = ByteBuffer.allocateDirect(TEST_CHROMA_WIDTH * dstStrideU);
final ByteBuffer dstV = ByteBuffer.allocateDirect(TEST_CHROMA_WIDTH * dstStrideV);
YuvHelper.I420Rotate(TEST_I420_Y, TEST_I420_STRIDE_Y, TEST_I420_U, TEST_I420_STRIDE_V,
TEST_I420_V, TEST_I420_STRIDE_U, dstY, dstStrideY, dstU, dstStrideU, dstV, dstStrideV,
TEST_WIDTH, TEST_HEIGHT, 90);
assertByteBufferContentEquals(new byte[] {7, 4, 1, 8, 5, 2, 9, 6, 3}, dstY);
assertByteBufferContentEquals(new byte[] {53, 51, 54, 52}, dstU);
assertByteBufferContentEquals(new byte[] {105, 101, 106, 102}, dstV);
}
@SmallTest
@Test
public void testI420Rotate90Tight() {
final ByteBuffer dst = ByteBuffer.allocateDirect(
TEST_WIDTH * TEST_HEIGHT + TEST_CHROMA_WIDTH * TEST_CHROMA_HEIGHT * 2);
YuvHelper.I420Rotate(TEST_I420_Y, TEST_I420_STRIDE_Y, TEST_I420_U, TEST_I420_STRIDE_V,
TEST_I420_V, TEST_I420_STRIDE_U, dst, TEST_WIDTH, TEST_HEIGHT, 90);
assertByteBufferContentEquals(
new byte[] {7, 4, 1, 8, 5, 2, 9, 6, 3, 53, 51, 54, 52, 105, 101, 106, 102}, dst);
}
}

71
sdk/android/src/jni/videofilerenderer.cc
View File

@ -1,71 +0,0 @@
/*
* Copyright 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include <jni.h>
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
#include "sdk/android/generated_video_jni/jni/VideoFileRenderer_jni.h"
#include "sdk/android/src/jni/scoped_java_ref.h"
#include "third_party/libyuv/include/libyuv/scale.h"
namespace webrtc {
namespace jni {
static void JNI_VideoFileRenderer_I420Scale(
JNIEnv* jni,
const JavaParamRef<jclass>&,
const JavaParamRef<jobject>& j_src_buffer_y,
jint j_src_stride_y,
const JavaParamRef<jobject>& j_src_buffer_u,
jint j_src_stride_u,
const JavaParamRef<jobject>& j_src_buffer_v,
jint j_src_stride_v,
jint width,
jint height,
const JavaParamRef<jobject>& j_dst_buffer,
jint dstWidth,
jint dstHeight) {
size_t src_size_y = jni->GetDirectBufferCapacity(j_src_buffer_y.obj());
size_t src_size_u = jni->GetDirectBufferCapacity(j_src_buffer_u.obj());
size_t src_size_v = jni->GetDirectBufferCapacity(j_src_buffer_v.obj());
size_t dst_size = jni->GetDirectBufferCapacity(j_dst_buffer.obj());
int dst_stride = dstWidth;
RTC_CHECK_GE(src_size_y, j_src_stride_y * height);
RTC_CHECK_GE(src_size_u, j_src_stride_u * height / 4);
RTC_CHECK_GE(src_size_v, j_src_stride_v * height / 4);
RTC_CHECK_GE(dst_size, dst_stride * dstHeight * 3 / 2);
uint8_t* src_y = reinterpret_cast<uint8_t*>(
jni->GetDirectBufferAddress(j_src_buffer_y.obj()));
uint8_t* src_u = reinterpret_cast<uint8_t*>(
jni->GetDirectBufferAddress(j_src_buffer_u.obj()));
uint8_t* src_v = reinterpret_cast<uint8_t*>(
jni->GetDirectBufferAddress(j_src_buffer_v.obj()));
uint8_t* dst = reinterpret_cast<uint8_t*>(
jni->GetDirectBufferAddress(j_dst_buffer.obj()));
uint8_t* dst_y = dst;
size_t dst_stride_y = dst_stride;
uint8_t* dst_u = dst + dst_stride * dstHeight;
size_t dst_stride_u = dst_stride / 2;
uint8_t* dst_v = dst + dst_stride * dstHeight * 5 / 4;
size_t dst_stride_v = dst_stride / 2;
int ret = libyuv::I420Scale(
src_y, j_src_stride_y, src_u, j_src_stride_u, src_v, j_src_stride_v,
width, height, dst_y, dst_stride_y, dst_u, dst_stride_u, dst_v,
dst_stride_v, dstWidth, dstHeight, libyuv::kFilterBilinear);
if (ret) {
RTC_LOG(LS_ERROR) << "Error scaling I420 frame: " << ret;
}
}
} // namespace jni
} // namespace webrtc

36
sdk/android/src/jni/yuvhelper.cc
View File

@ -81,5 +81,41 @@ static void JNI_YuvHelper_I420ToNV12(JNIEnv* jni,
width, height);
}
void JNI_YuvHelper_I420Rotate(JNIEnv* jni,
const JavaParamRef<jclass>&,
const JavaParamRef<jobject>& j_src_y,
jint src_stride_y,
const JavaParamRef<jobject>& j_src_u,
jint src_stride_u,
const JavaParamRef<jobject>& j_src_v,
jint src_stride_v,
const JavaParamRef<jobject>& j_dst_y,
jint dst_stride_y,
const JavaParamRef<jobject>& j_dst_u,
jint dst_stride_u,
const JavaParamRef<jobject>& j_dst_v,
jint dst_stride_v,
jint src_width,
jint src_height,
jint rotation_mode) {
const uint8_t* src_y =
static_cast<const uint8_t*>(jni->GetDirectBufferAddress(j_src_y.obj()));
const uint8_t* src_u =
static_cast<const uint8_t*>(jni->GetDirectBufferAddress(j_src_u.obj()));
const uint8_t* src_v =
static_cast<const uint8_t*>(jni->GetDirectBufferAddress(j_src_v.obj()));
uint8_t* dst_y =
static_cast<uint8_t*>(jni->GetDirectBufferAddress(j_dst_y.obj()));
uint8_t* dst_u =
static_cast<uint8_t*>(jni->GetDirectBufferAddress(j_dst_u.obj()));
uint8_t* dst_v =
static_cast<uint8_t*>(jni->GetDirectBufferAddress(j_dst_v.obj()));
libyuv::I420Rotate(src_y, src_stride_y, src_u, src_stride_u, src_v,
src_stride_v, dst_y, dst_stride_y, dst_u, dst_stride_u,
dst_v, dst_stride_v, src_width, src_height,
static_cast<libyuv::RotationMode>(rotation_mode));
}
} // namespace jni
} // namespace webrtc