zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move matrix from VideoFrame to TextureBuffer.

Browse Source 
Previously, the matrix in VideoFrame was used to crop and scale the
frame. This caused complications because webrtc::VideoFrame doesn't
include a matrix. cropAndScale method is added to VideoBuffer class for
cropping and scaling instead.

BUG=webrtc:7749, webrtc:7760

Review-Url: https://codereview.webrtc.org/2990583002
Cr-Commit-Position: refs/heads/master@{#19179}
This commit is contained in:
sakal
2017-07-28 07:12:23 -07:00
committed by Commit Bot
parent 5ba9730265
commit 836f60cda1
15 changed files with 369 additions and 257 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
webrtc/sdk/android/BUILD.gn
View File

@ -108,6 +108,7 @@ rtc_static_library("video_jni") {
"src/jni/videodecoderwrapper.cc",
"src/jni/videodecoderwrapper.h",
"src/jni/videofilerenderer_jni.cc",
"src/jni/videoframe_jni.cc",
"src/jni/videotrack_jni.cc",
"src/jni/wrapped_native_i420_buffer.cc",
"src/jni/wrapped_native_i420_buffer.h",
@ -393,8 +394,8 @@ android_library("libjingle_peerconnection_java") {
"api/org/webrtc/VideoSource.java",
"api/org/webrtc/VideoTrack.java",
"src/java/org/webrtc/AndroidVideoTrackSourceObserver.java",
"src/java/org/webrtc/BitrateAdjuster.java",
"src/java/org/webrtc/BaseBitrateAdjuster.java",
"src/java/org/webrtc/BitrateAdjuster.java",
"src/java/org/webrtc/Camera1Session.java",
"src/java/org/webrtc/Camera2Session.java",
"src/java/org/webrtc/CameraCapturer.java",
@ -407,9 +408,10 @@ android_library("libjingle_peerconnection_java") {
"src/java/org/webrtc/HardwareVideoEncoder.java",
"src/java/org/webrtc/Histogram.java",
"src/java/org/webrtc/I420BufferImpl.java",
"src/java/org/webrtc/VideoDecoderWrapperCallback.java",
"src/java/org/webrtc/MediaCodecUtils.java",
"src/java/org/webrtc/TextureBufferImpl.java",
"src/java/org/webrtc/VideoCodecType.java",
"src/java/org/webrtc/VideoDecoderWrapperCallback.java",
"src/java/org/webrtc/WrappedNativeI420Buffer.java",
"src/java/org/webrtc/YuvConverter.java",
]

90
webrtc/sdk/android/api/org/webrtc/SurfaceTextureHelper.java
View File

@ -10,6 +10,7 @@
package org.webrtc;
import android.graphics.Matrix;
import android.graphics.SurfaceTexture;
import android.opengl.GLES11Ext;
import android.opengl.GLES20;
@ -288,86 +289,13 @@ public class SurfaceTextureHelper {
* The returned TextureBuffer holds a reference to the SurfaceTextureHelper that created it. The
* buffer calls returnTextureFrame() when it is released.
*/
public TextureBuffer createTextureBuffer(int width, int height, float[] transformMatrix) {
return new OesTextureBuffer(oesTextureId, width, height, transformMatrix, this);
}
/**
* Android OES texture buffer backed by a SurfaceTextureHelper's texture. The buffer calls
* returnTextureFrame() when it is released.
*/
private static class OesTextureBuffer implements TextureBuffer {
private final int id;
private final int width;
private final int height;
private final float[] transformMatrix;
private final SurfaceTextureHelper helper;
private int refCount;
OesTextureBuffer(
int id, int width, int height, float[] transformMatrix, SurfaceTextureHelper helper) {
this.id = id;
this.width = width;
this.height = height;
this.transformMatrix = transformMatrix;
this.helper = helper;
this.refCount = 1; // Creator implicitly holds a reference.
}
@Override
public TextureBuffer.Type getType() {
return TextureBuffer.Type.OES;
}
@Override
public int getTextureId() {
return id;
}
@Override
public int getWidth() {
return width;
}
@Override
public int getHeight() {
return height;
}
@Override
public I420Buffer toI420() {
// SurfaceTextureHelper requires a stride that is divisible by 8. Round width up.
// See SurfaceTextureHelper for details on the size and format.
int stride = ((width + 7) / 8) * 8;
int uvHeight = (height + 1) / 2;
// Due to the layout used by SurfaceTextureHelper, vPos + stride * uvHeight would overrun the
// buffer. Add one row at the bottom to compensate for this. There will never be data in the
// extra row, but now other code does not have to deal with v stride * v height exceeding the
// buffer's capacity.
int size = stride * (height + uvHeight + 1);
ByteBuffer buffer = ByteBuffer.allocateDirect(size);
helper.textureToYUV(buffer, width, height, stride, id, transformMatrix);
int yPos = 0;
int uPos = yPos + stride * height;
// Rows of U and V alternate in the buffer, so V data starts after the first row of U.
int vPos = yPos + stride / 2;
// SurfaceTextureHelper uses the same stride for Y, U, and V data.
return new I420BufferImpl(
buffer, width, height, yPos, stride, uPos, stride, vPos, stride, null);
}
@Override
public void retain() {
++refCount;
}
@Override
public void release() {
if (--refCount == 0) {
helper.returnTextureFrame();
}
}
public TextureBuffer createTextureBuffer(int width, int height, Matrix transformMatrix) {
return new TextureBufferImpl(
width, height, TextureBuffer.Type.OES, oesTextureId, transformMatrix, this, new Runnable() {
@Override
public void run() {
returnTextureFrame();
}
});
}
}

78
webrtc/sdk/android/api/org/webrtc/VideoFrame.java
View File

@ -44,6 +44,13 @@ public class VideoFrame {
*/
void retain();
void release();
/**
* Crops a region defined by |cropx|, |cropY|, |cropWidth| and |cropHeight|. Scales it to size
* |scaleWidth| x |scaleHeight|.
*/
Buffer cropAndScale(
int cropX, int cropY, int cropWidth, int cropHeight, int scaleWidth, int scaleHeight);
}
/**
@ -67,24 +74,26 @@ public class VideoFrame {
Type getType();
int getTextureId();
/**
* Retrieve the transform matrix associated with the frame. This transform matrix maps 2D
* homogeneous coordinates of the form (s, t, 1) with s and t in the inclusive range [0, 1] to
* the coordinate that should be used to sample that location from the buffer.
*/
public Matrix getTransformMatrix();
}
private final Buffer buffer;
private final int rotation;
private final long timestampNs;
private final Matrix transformMatrix;
public VideoFrame(Buffer buffer, int rotation, long timestampNs, Matrix transformMatrix) {
public VideoFrame(Buffer buffer, int rotation, long timestampNs) {
if (buffer == null) {
throw new IllegalArgumentException("buffer not allowed to be null");
}
if (transformMatrix == null) {
throw new IllegalArgumentException("transformMatrix not allowed to be null");
}
this.buffer = buffer;
this.rotation = rotation;
this.timestampNs = timestampNs;
this.transformMatrix = transformMatrix;
}
public Buffer getBuffer() {
@ -105,26 +114,6 @@ public class VideoFrame {
return timestampNs;
}
/**
* Retrieve the transform matrix associated with the frame. This transform matrix maps 2D
* homogeneous coordinates of the form (s, t, 1) with s and t in the inclusive range [0, 1] to the
* coordinate that should be used to sample that location from the buffer.
*/
public Matrix getTransformMatrix() {
return transformMatrix;
}
/**
* Resolution of the frame in pixels.
*/
public int getWidth() {
return buffer.getWidth();
}
public int getHeight() {
return buffer.getHeight();
}
/**
* Reference counting of the underlying buffer.
*/
@ -135,4 +124,41 @@ public class VideoFrame {
public void release() {
buffer.release();
}
public static VideoFrame.Buffer cropAndScaleI420(final I420Buffer buffer, int cropX, int cropY,
int cropWidth, int cropHeight, int scaleWidth, int scaleHeight) {
if (cropWidth == scaleWidth && cropHeight == scaleHeight) {
// No scaling.
ByteBuffer dataY = buffer.getDataY();
ByteBuffer dataU = buffer.getDataU();
ByteBuffer dataV = buffer.getDataV();
dataY.position(cropX + cropY * buffer.getStrideY());
dataU.position(cropX / 2 + cropY / 2 * buffer.getStrideU());
dataV.position(cropX / 2 + cropY / 2 * buffer.getStrideV());
buffer.retain();
return new I420BufferImpl(buffer.getWidth(), buffer.getHeight(), dataY.slice(),
buffer.getStrideY(), dataU.slice(), buffer.getStrideU(), dataV.slice(),
buffer.getStrideV(), new Runnable() {
@Override
public void run() {
buffer.release();
}
});
}
I420BufferImpl newBuffer = I420BufferImpl.allocate(scaleWidth, scaleHeight);
nativeCropAndScaleI420(buffer.getDataY(), buffer.getStrideY(), buffer.getDataU(),
buffer.getStrideU(), buffer.getDataV(), buffer.getStrideV(), cropX, cropY, cropWidth,
cropHeight, newBuffer.getDataY(), newBuffer.getStrideY(), newBuffer.getDataU(),
newBuffer.getStrideU(), newBuffer.getDataV(), newBuffer.getStrideV(), scaleWidth,
scaleHeight);
return newBuffer;
}
private static native void nativeCropAndScaleI420(ByteBuffer srcY, int srcStrideY,
ByteBuffer srcU, int srcStrideU, ByteBuffer srcV, int srcStrideV, int cropX, int cropY,
int cropWidth, int cropHeight, ByteBuffer dstY, int dstStrideY, ByteBuffer dstU,
int dstStrideU, ByteBuffer dstV, int dstStrideV, int scaleWidth, int scaleHeight);
}

15
webrtc/sdk/android/api/org/webrtc/VideoRenderer.java
View File

@ -84,12 +84,11 @@ public class VideoRenderer {
}
/**
* Construct a frame of the given dimensions from VideoFrame.Buffer.
* Construct a frame from VideoFrame.Buffer.
*/
public I420Frame(int width, int height, int rotationDegree, float[] samplingMatrix,
VideoFrame.Buffer buffer, long nativeFramePointer) {
this.width = width;
this.height = height;
public I420Frame(int rotationDegree, VideoFrame.Buffer buffer, long nativeFramePointer) {
this.width = buffer.getWidth();
this.height = buffer.getHeight();
this.rotationDegree = rotationDegree;
if (rotationDegree % 90 != 0) {
throw new IllegalArgumentException("Rotation degree not multiple of 90: " + rotationDegree);
@ -98,7 +97,8 @@ public class VideoRenderer {
VideoFrame.TextureBuffer textureBuffer = (VideoFrame.TextureBuffer) buffer;
this.yuvFrame = false;
this.textureId = textureBuffer.getTextureId();
this.samplingMatrix = samplingMatrix;
this.samplingMatrix = RendererCommon.convertMatrixFromAndroidGraphicsMatrix(
textureBuffer.getTransformMatrix());
this.yuvStrides = null;
this.yuvPlanes = null;
@ -113,8 +113,7 @@ public class VideoRenderer {
// top-left corner of the image, but in glTexImage2D() the first element corresponds to the
// bottom-left corner. This discrepancy is corrected by multiplying the sampling matrix with
// a vertical flip matrix.
this.samplingMatrix =
RendererCommon.multiplyMatrices(samplingMatrix, RendererCommon.verticalFlipMatrix());
this.samplingMatrix = RendererCommon.verticalFlipMatrix();
this.textureId = 0;
}

21
webrtc/sdk/android/instrumentationtests/src/org/webrtc/HardwareVideoDecoderTest.java
View File

@ -18,8 +18,8 @@ import android.annotation.TargetApi;
import android.graphics.Matrix;
import android.support.test.filters.MediumTest;
import android.util.Log;
import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicReference;
import org.chromium.base.test.BaseJUnit4ClassRunner;
import org.junit.Test;
import org.junit.runner.RunWith;
@ -123,8 +123,7 @@ public final class HardwareVideoDecoderTest {
// First, encode a frame.
VideoFrame.I420Buffer buffer = I420BufferImpl.allocate(SETTINGS.width, SETTINGS.height);
VideoFrame frame =
new VideoFrame(buffer, rotation, presentationTimestampUs * 1000, new Matrix());
VideoFrame frame = new VideoFrame(buffer, rotation, presentationTimestampUs * 1000);
VideoEncoder.EncodeInfo info = new VideoEncoder.EncodeInfo(
new EncodedImage.FrameType[] {EncodedImage.FrameType.VideoFrameKey});
@ -141,9 +140,8 @@ public final class HardwareVideoDecoderTest {
frame = decoded.get();
assertEquals(frame.getRotation(), rotation);
assertEquals(frame.getTimestampNs(), presentationTimestampUs * 1000);
assertEquals(frame.getTransformMatrix(), new Matrix());
assertEquals(frame.getWidth(), SETTINGS.width);
assertEquals(frame.getHeight(), SETTINGS.height);
assertEquals(frame.getBuffer().getWidth(), SETTINGS.width);
assertEquals(frame.getBuffer().getHeight(), SETTINGS.height);
frame.release();
assertEquals(decoder.release(), VideoCodecStatus.OK);
@ -200,8 +198,7 @@ public final class HardwareVideoDecoderTest {
// First, encode a frame.
VideoFrame.I420Buffer buffer = I420BufferImpl.allocate(SETTINGS.width, SETTINGS.height);
VideoFrame frame =
new VideoFrame(buffer, rotation, presentationTimestampUs * 1000, new Matrix());
VideoFrame frame = new VideoFrame(buffer, rotation, presentationTimestampUs * 1000);
VideoEncoder.EncodeInfo info = new VideoEncoder.EncodeInfo(
new EncodedImage.FrameType[] {EncodedImage.FrameType.VideoFrameKey});
@ -218,13 +215,13 @@ public final class HardwareVideoDecoderTest {
frame = decoded.get();
assertEquals(frame.getRotation(), rotation);
assertEquals(frame.getTimestampNs(), presentationTimestampUs * 1000);
// TODO(mellem): Compare the matrix to whatever we expect to get back?
assertNotNull(frame.getTransformMatrix());
assertEquals(frame.getWidth(), SETTINGS.width);
assertEquals(frame.getHeight(), SETTINGS.height);
assertTrue(frame.getBuffer() instanceof VideoFrame.TextureBuffer);
VideoFrame.TextureBuffer textureBuffer = (VideoFrame.TextureBuffer) frame.getBuffer();
// TODO(mellem): Compare the matrix to whatever we expect to get back?
assertNotNull(textureBuffer.getTransformMatrix());
assertEquals(textureBuffer.getWidth(), SETTINGS.width);
assertEquals(textureBuffer.getHeight(), SETTINGS.height);
assertEquals(textureBuffer.getType(), VideoFrame.TextureBuffer.Type.OES);
assertEquals(decoder.release(), VideoCodecStatus.OK);

17
webrtc/sdk/android/instrumentationtests/src/org/webrtc/HardwareVideoEncoderTest.java
View File

@ -101,8 +101,7 @@ public class HardwareVideoEncoderTest {
assertEquals(encoder.initEncode(SETTINGS, callback), VideoCodecStatus.OK);
VideoFrame.I420Buffer buffer = I420BufferImpl.allocate(SETTINGS.width, SETTINGS.height);
VideoFrame frame =
new VideoFrame(buffer, 0 /* rotation */, presentationTimestampUs * 1000, new Matrix());
VideoFrame frame = new VideoFrame(buffer, 0 /* rotation */, presentationTimestampUs * 1000);
VideoEncoder.EncodeInfo info = new VideoEncoder.EncodeInfo(
new EncodedImage.FrameType[] {EncodedImage.FrameType.VideoFrameKey});
@ -162,6 +161,11 @@ public class HardwareVideoEncoderTest {
return oesTextureId;
}
@Override
public Matrix getTransformMatrix() {
return new Matrix();
}
@Override
public int getWidth() {
return SETTINGS.width;
@ -182,9 +186,14 @@ public class HardwareVideoEncoderTest {
@Override
public void release() {}
@Override
public VideoFrame.Buffer cropAndScale(
int cropX, int cropY, int cropWidth, int cropHeight, int scaleWidth, int scaleHeight) {
return null;
}
};
VideoFrame frame =
new VideoFrame(buffer, 0 /* rotation */, presentationTimestampUs * 1000, new Matrix());
VideoFrame frame = new VideoFrame(buffer, 0 /* rotation */, presentationTimestampUs * 1000);
VideoEncoder.EncodeInfo info = new VideoEncoder.EncodeInfo(
new EncodedImage.FrameType[] {EncodedImage.FrameType.VideoFrameKey});

27
webrtc/sdk/android/src/java/org/webrtc/HardwareVideoDecoder.java
View File

@ -417,12 +417,11 @@ class HardwareVideoDecoder
@Override
public void onTextureFrameAvailable(int oesTextureId, float[] transformMatrix, long timestampNs) {
VideoFrame.TextureBuffer oesBuffer = surfaceTextureHelper.createTextureBuffer(
renderedTextureMetadata.width, renderedTextureMetadata.height, transformMatrix);
Matrix matrix = RendererCommon.convertMatrixToAndroidGraphicsMatrix(transformMatrix);
renderedTextureMetadata.width, renderedTextureMetadata.height,
RendererCommon.convertMatrixToAndroidGraphicsMatrix(transformMatrix));
VideoFrame frame = new VideoFrame(oesBuffer, renderedTextureMetadata.rotation,
renderedTextureMetadata.presentationTimestampUs * 1000, matrix);
renderedTextureMetadata.presentationTimestampUs * 1000);
callback.onDecodedFrame(frame, renderedTextureMetadata.decodeTimeMs, null /* qp */);
frame.release();
}
@ -477,7 +476,7 @@ class HardwareVideoDecoder
}
long presentationTimeNs = info.presentationTimeUs * 1000;
VideoFrame frame = new VideoFrame(frameBuffer, rotation, presentationTimeNs, new Matrix());
VideoFrame frame = new VideoFrame(frameBuffer, rotation, presentationTimeNs);
// Note that qp is parsed on the C++ side.
callback.onDecodedFrame(frame, decodeTimeMs, null /* qp */);
@ -605,9 +604,9 @@ class HardwareVideoDecoder
activeOutputBuffers++;
}
I420BufferImpl.ReleaseCallback callback = new I420BufferImpl.ReleaseCallback() {
Runnable callback = new Runnable() {
@Override
public void onRelease() {
public void run() {
codec.releaseOutputBuffer(outputBufferIndex, false);
synchronized (activeOutputBuffersLock) {
activeOutputBuffers--;
@ -616,8 +615,20 @@ class HardwareVideoDecoder
}
};
buffer.position(yPos);
buffer.limit(uPos);
ByteBuffer dataY = buffer.slice();
buffer.position(uPos);
buffer.limit(vPos);
ByteBuffer dataU = buffer.slice();
buffer.position(vPos);
buffer.limit(vPos + uvStride * sliceHeight / 2);
ByteBuffer dataV = buffer.slice();
return new I420BufferImpl(
buffer, width, height, yPos, stride, uPos, uvStride, vPos, uvStride, callback);
width, height, dataY, stride, dataU, uvStride, dataV, uvStride, callback);
}
private static void copyI420(ByteBuffer src, int offset, VideoFrame.I420Buffer frameBuffer,

10
webrtc/sdk/android/src/java/org/webrtc/HardwareVideoEncoder.java
View File

@ -235,8 +235,8 @@ class HardwareVideoEncoder implements VideoEncoder {
}
// If input resolution changed, restart the codec with the new resolution.
int frameWidth = videoFrame.getWidth();
int frameHeight = videoFrame.getHeight();
int frameWidth = videoFrame.getBuffer().getWidth();
int frameHeight = videoFrame.getBuffer().getHeight();
if (frameWidth != width || frameHeight != height) {
VideoCodecStatus status = resetCodec(frameWidth, frameHeight);
if (status != VideoCodecStatus.OK) {
@ -271,8 +271,8 @@ class HardwareVideoEncoder implements VideoEncoder {
EncodedImage.Builder builder = EncodedImage.builder()
.setCaptureTimeMs(presentationTimestampMs)
.setCompleteFrame(true)
.setEncodedWidth(videoFrame.getWidth())
.setEncodedHeight(videoFrame.getHeight())
.setEncodedWidth(videoFrame.getBuffer().getWidth())
.setEncodedHeight(videoFrame.getBuffer().getHeight())
.setRotation(videoFrame.getRotation());
outputBuilders.offer(builder);
@ -293,7 +293,7 @@ class HardwareVideoEncoder implements VideoEncoder {
private VideoCodecStatus encodeTextureBuffer(
VideoFrame videoFrame, VideoFrame.TextureBuffer textureBuffer) {
Matrix matrix = videoFrame.getTransformMatrix();
Matrix matrix = textureBuffer.getTransformMatrix();
float[] transformationMatrix = RendererCommon.convertMatrixFromAndroidGraphicsMatrix(matrix);
try {

67
webrtc/sdk/android/src/java/org/webrtc/I420BufferImpl.java
View File

@ -15,32 +15,28 @@ import org.webrtc.VideoFrame.I420Buffer;
/** Implementation of an I420 VideoFrame buffer. */
class I420BufferImpl implements VideoFrame.I420Buffer {
private final ByteBuffer buffer;
private final int width;
private final int height;
private final int chromaHeight;
private final int yPos;
private final ByteBuffer dataY;
private final ByteBuffer dataU;
private final ByteBuffer dataV;
private final int strideY;
private final int uPos;
private final int strideU;
private final int vPos;
private final int strideV;
private final ReleaseCallback releaseCallback;
private final Runnable releaseCallback;
private int refCount;
/** Allocates an I420Buffer backed by existing data. */
I420BufferImpl(ByteBuffer buffer, int width, int height, int yPos, int strideY, int uPos,
int strideU, int vPos, int strideV, ReleaseCallback releaseCallback) {
this.buffer = buffer;
/** Constructs an I420Buffer backed by existing data. */
I420BufferImpl(int width, int height, ByteBuffer dataY, int strideY, ByteBuffer dataU,
int strideU, ByteBuffer dataV, int strideV, Runnable releaseCallback) {
this.width = width;
this.height = height;
this.chromaHeight = (height + 1) / 2;
this.yPos = yPos;
this.dataY = dataY;
this.dataU = dataU;
this.dataV = dataV;
this.strideY = strideY;
this.uPos = uPos;
this.strideU = strideU;
this.vPos = vPos;
this.strideV = strideV;
this.releaseCallback = releaseCallback;
@ -54,9 +50,22 @@ class I420BufferImpl implements VideoFrame.I420Buffer {
int yPos = 0;
int uPos = yPos + width * height;
int vPos = uPos + strideUV * chromaHeight;
ByteBuffer buffer = ByteBuffer.allocateDirect(width * height + 2 * strideUV * chromaHeight);
return new I420BufferImpl(
buffer, width, height, yPos, width, uPos, strideUV, vPos, strideUV, null);
buffer.position(yPos);
buffer.limit(uPos);
ByteBuffer dataY = buffer.slice();
buffer.position(uPos);
buffer.limit(vPos);
ByteBuffer dataU = buffer.slice();
buffer.position(vPos);
buffer.limit(vPos + strideUV * chromaHeight);
ByteBuffer dataV = buffer.slice();
return new I420BufferImpl(width, height, dataY, width, dataU, strideUV, dataV, strideUV, null);
}
@Override
@ -71,26 +80,17 @@ class I420BufferImpl implements VideoFrame.I420Buffer {
@Override
public ByteBuffer getDataY() {
ByteBuffer data = buffer.slice();
data.position(yPos);
data.limit(yPos + getStrideY() * height);
return data;
return dataY;
}
@Override
public ByteBuffer getDataU() {
ByteBuffer data = buffer.slice();
data.position(uPos);
data.limit(uPos + strideU * chromaHeight);
return data;
return dataU;
}
@Override
public ByteBuffer getDataV() {
ByteBuffer data = buffer.slice();
data.position(vPos);
data.limit(vPos + strideV * chromaHeight);
return data;
return dataV;
}
@Override
@ -121,13 +121,14 @@ class I420BufferImpl implements VideoFrame.I420Buffer {
@Override
public void release() {
if (--refCount == 0 && releaseCallback != null) {
releaseCallback.onRelease();
releaseCallback.run();
}
}
// Callback called when the frame is no longer referenced.
interface ReleaseCallback {
// Called when the frame is no longer referenced.
void onRelease();
@Override
public VideoFrame.Buffer cropAndScale(
int cropX, int cropY, int cropWidth, int cropHeight, int scaleWidth, int scaleHeight) {
return VideoFrame.cropAndScaleI420(
this, cropX, cropY, cropWidth, cropHeight, scaleWidth, scaleHeight);
}
}

131
webrtc/sdk/android/src/java/org/webrtc/TextureBufferImpl.java Normal file
View File

@ -0,0 +1,131 @@
/*
* Copyright 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
package org.webrtc;
import android.graphics.Matrix;
import java.nio.ByteBuffer;
/**
* Android texture buffer backed by a SurfaceTextureHelper's texture. The buffer calls
* |releaseCallback| when it is released.
*/
class TextureBufferImpl implements VideoFrame.TextureBuffer {
private final int width;
private final int height;
private final Type type;
private final int id;
private final Matrix transformMatrix;
private final SurfaceTextureHelper surfaceTextureHelper;
private final Runnable releaseCallback;
private int refCount;
public TextureBufferImpl(int width, int height, Type type, int id, Matrix transformMatrix,
SurfaceTextureHelper surfaceTextureHelper, Runnable releaseCallback) {
this.width = width;
this.height = height;
this.type = type;
this.id = id;
this.transformMatrix = transformMatrix;
this.surfaceTextureHelper = surfaceTextureHelper;
this.releaseCallback = releaseCallback;
this.refCount = 1; // Creator implicitly holds a reference.
}
@Override
public VideoFrame.TextureBuffer.Type getType() {
return type;
}
@Override
public int getTextureId() {
return id;
}
@Override
public Matrix getTransformMatrix() {
return transformMatrix;
}
@Override
public int getWidth() {
return width;
}
@Override
public int getHeight() {
return height;
}
@Override
public VideoFrame.I420Buffer toI420() {
// SurfaceTextureHelper requires a stride that is divisible by 8. Round width up.
// See SurfaceTextureHelper for details on the size and format.
int stride = ((width + 7) / 8) * 8;
int uvHeight = (height + 1) / 2;
// Due to the layout used by SurfaceTextureHelper, vPos + stride * uvHeight would overrun the
// buffer. Add one row at the bottom to compensate for this. There will never be data in the
// extra row, but now other code does not have to deal with v stride * v height exceeding the
// buffer's capacity.
int size = stride * (height + uvHeight + 1);
ByteBuffer buffer = ByteBuffer.allocateDirect(size);
surfaceTextureHelper.textureToYUV(buffer, width, height, stride, id,
RendererCommon.convertMatrixFromAndroidGraphicsMatrix(transformMatrix));
int yPos = 0;
int uPos = yPos + stride * height;
// Rows of U and V alternate in the buffer, so V data starts after the first row of U.
int vPos = uPos + stride / 2;
buffer.position(yPos);
buffer.limit(yPos + stride * height);
ByteBuffer dataY = buffer.slice();
buffer.position(uPos);
buffer.limit(uPos + stride * uvHeight);
ByteBuffer dataU = buffer.slice();
buffer.position(vPos);
buffer.limit(vPos + stride * uvHeight);
ByteBuffer dataV = buffer.slice();
// SurfaceTextureHelper uses the same stride for Y, U, and V data.
return new I420BufferImpl(width, height, dataY, stride, dataU, stride, dataV, stride, null);
}
@Override
public void retain() {
++refCount;
}
@Override
public void release() {
if (--refCount == 0) {
releaseCallback.run();
}
}
@Override
public VideoFrame.Buffer cropAndScale(
int cropX, int cropY, int cropWidth, int cropHeight, int scaleWidth, int scaleHeight) {
retain();
Matrix newMatrix = new Matrix(transformMatrix);
newMatrix.postScale(cropWidth / (float) width, cropHeight / (float) height);
newMatrix.postTranslate(cropX / (float) width, cropY / (float) height);
return new TextureBufferImpl(
scaleWidth, scaleHeight, type, id, newMatrix, surfaceTextureHelper, new Runnable() {
@Override
public void run() {
release();
}
});
}
}

7
webrtc/sdk/android/src/java/org/webrtc/WrappedNativeI420Buffer.java
View File

@ -94,6 +94,13 @@ class WrappedNativeI420Buffer implements VideoFrame.I420Buffer {
nativeRelease(nativeBuffer);
}
@Override
public VideoFrame.Buffer cropAndScale(
int cropX, int cropY, int cropWidth, int cropHeight, int scaleWidth, int scaleHeight) {
return VideoFrame.cropAndScaleI420(
this, cropX, cropY, cropWidth, cropHeight, scaleWidth, scaleHeight);
}
private static native long nativeAddRef(long nativeBuffer);
private static native long nativeRelease(long nativeBuffer);
}

79
webrtc/sdk/android/src/jni/native_handle_impl.cc
View File

@ -34,42 +34,6 @@ Matrix::Matrix(JNIEnv* jni, jfloatArray a) {
jni->ReleaseFloatArrayElements(a, ptr, 0);
}
Matrix Matrix::fromAndroidGraphicsMatrix(JNIEnv* jni, jobject j_matrix) {
jfloatArray array_3x3 = jni->NewFloatArray(9);
jclass j_matrix_class = jni->FindClass("android/graphics/Matrix");
jni->CallVoidMethod(j_matrix,
GetMethodID(jni, j_matrix_class, "getValues", "([F)V"),
array_3x3);
jfloat* array_3x3_ptr = jni->GetFloatArrayElements(array_3x3, nullptr);
Matrix matrix;
memset(matrix.elem_, 0, sizeof(matrix.elem_));
// The android.graphics.Matrix looks like this:
// [x1 y1 w1]
// [x2 y2 w2]
// [x3 y3 w3]
// We want to contruct a matrix that looks like this:
// [x1 y1 0 w1]
// [x2 y2 0 w2]
// [ 0 0 1 0]
// [x3 y3 0 w3]
// Since it is stored in column-major order, it looks like this:
// [x1 x2 0 x3
// y1 y2 0 y3
// 0 0 1 0
// w1 w2 0 w3]
matrix.elem_[0 * 4 + 0] = array_3x3_ptr[0 * 3 + 0];
matrix.elem_[0 * 4 + 1] = array_3x3_ptr[1 * 3 + 0];
matrix.elem_[0 * 4 + 3] = array_3x3_ptr[2 * 3 + 0];
matrix.elem_[1 * 4 + 0] = array_3x3_ptr[0 * 3 + 1];
matrix.elem_[1 * 4 + 1] = array_3x3_ptr[1 * 3 + 1];
matrix.elem_[1 * 4 + 3] = array_3x3_ptr[2 * 3 + 1];
matrix.elem_[2 * 4 + 2] = 1; // Z-scale should be 1.
matrix.elem_[3 * 4 + 0] = array_3x3_ptr[0 * 3 + 2];
matrix.elem_[3 * 4 + 1] = array_3x3_ptr[1 * 3 + 2];
matrix.elem_[3 * 4 + 3] = array_3x3_ptr[2 * 3 + 2];
return matrix;
}
jfloatArray Matrix::ToJava(JNIEnv* jni) const {
jfloatArray matrix = jni->NewFloatArray(16);
jni->SetFloatArrayRegion(matrix, 0, 16, elem_);
@ -237,12 +201,10 @@ AndroidVideoBuffer::AndroidVideoBuffer(JNIEnv* jni,
jmethodID j_release_id,
int width,
int height,
const Matrix& matrix,
jobject j_video_frame_buffer)
: j_release_id_(j_release_id),
width_(width),
height_(height),
matrix_(matrix),
j_video_frame_buffer_(jni, j_video_frame_buffer) {
jni->CallVoidMethod(j_video_frame_buffer, j_retain_id);
}
@ -274,23 +236,19 @@ rtc::scoped_refptr<webrtc::I420BufferInterface> AndroidVideoBuffer::ToI420() {
}
jobject AndroidVideoBuffer::ToJavaI420Frame(JNIEnv* jni,
int width,
int height,
int rotation) {
jclass j_byte_buffer_class = jni->FindClass("java/nio/ByteBuffer");
jclass j_i420_frame_class =
FindClass(jni, "org/webrtc/VideoRenderer$I420Frame");
jmethodID j_i420_frame_ctor_id =
GetMethodID(jni, j_i420_frame_class, "<init>",
"(III[FLorg/webrtc/VideoFrame$Buffer;J)V");
jmethodID j_i420_frame_ctor_id = GetMethodID(
jni, j_i420_frame_class, "<init>", "(ILorg/webrtc/VideoFrame$Buffer;J)V");
// Java code just uses the native frame to hold a reference to the buffer so
// this is okay.
webrtc::VideoFrame* native_frame = new webrtc::VideoFrame(
this, 0 /* timestamp */, 0 /* render_time_ms */,
webrtc::VideoRotation::kVideoRotation_0 /* rotation */);
return jni->NewObject(j_i420_frame_class, j_i420_frame_ctor_id, width, height,
rotation, matrix_.ToJava(jni), *j_video_frame_buffer_,
jlongFromPointer(native_frame));
return jni->NewObject(j_i420_frame_class, j_i420_frame_ctor_id, rotation,
*j_video_frame_buffer_, jlongFromPointer(native_frame));
}
AndroidVideoBufferFactory::AndroidVideoBufferFactory(JNIEnv* jni)
@ -299,16 +257,8 @@ AndroidVideoBufferFactory::AndroidVideoBufferFactory(JNIEnv* jni)
*j_video_frame_class_,
"getBuffer",
"()Lorg/webrtc/VideoFrame$Buffer;")),
j_get_width_id_(
GetMethodID(jni, *j_video_frame_class_, "getWidth", "()I")),
j_get_height_id_(
GetMethodID(jni, *j_video_frame_class_, "getHeight", "()I")),
j_get_rotation_id_(
GetMethodID(jni, *j_video_frame_class_, "getRotation", "()I")),
j_get_transform_matrix_id_(GetMethodID(jni,
*j_video_frame_class_,
"getTransformMatrix",
"()Landroid/graphics/Matrix;")),
j_get_timestamp_ns_id_(
GetMethodID(jni, *j_video_frame_class_, "getTimestampNs", "()J")),
j_video_frame_buffer_class_(
@ -317,7 +267,11 @@ AndroidVideoBufferFactory::AndroidVideoBufferFactory(JNIEnv* jni)
j_retain_id_(
GetMethodID(jni, *j_video_frame_buffer_class_, "retain", "()V")),
j_release_id_(
GetMethodID(jni, *j_video_frame_buffer_class_, "release", "()V")) {}
GetMethodID(jni, *j_video_frame_buffer_class_, "release", "()V")),
j_get_width_id_(
GetMethodID(jni, *j_video_frame_buffer_class_, "getWidth", "()I")),
j_get_height_id_(
GetMethodID(jni, *j_video_frame_buffer_class_, "getHeight", "()I")) {}
webrtc::VideoFrame AndroidVideoBufferFactory::CreateFrame(
JNIEnv* jni,
@ -325,30 +279,23 @@ webrtc::VideoFrame AndroidVideoBufferFactory::CreateFrame(
uint32_t timestamp_rtp) const {
jobject j_video_frame_buffer =
jni->CallObjectMethod(j_video_frame, j_get_buffer_id_);
int width = jni->CallIntMethod(j_video_frame, j_get_width_id_);
int height = jni->CallIntMethod(j_video_frame, j_get_height_id_);
int rotation = jni->CallIntMethod(j_video_frame, j_get_rotation_id_);
jobject j_matrix =
jni->CallObjectMethod(j_video_frame, j_get_transform_matrix_id_);
Matrix matrix = Matrix::fromAndroidGraphicsMatrix(jni, j_matrix);
uint32_t timestamp_ns =
jni->CallLongMethod(j_video_frame, j_get_timestamp_ns_id_);
rtc::scoped_refptr<AndroidVideoBuffer> buffer =
CreateBuffer(width, height, matrix, j_video_frame_buffer);
CreateBuffer(j_video_frame_buffer);
return webrtc::VideoFrame(buffer, timestamp_rtp,
timestamp_ns / rtc::kNumNanosecsPerMillisec,
static_cast<webrtc::VideoRotation>(rotation));
}
rtc::scoped_refptr<AndroidVideoBuffer> AndroidVideoBufferFactory::CreateBuffer(
int width,
int height,
const Matrix& matrix,
jobject j_video_frame_buffer) const {
JNIEnv* jni = AttachCurrentThreadIfNeeded();
int width = jni->CallIntMethod(j_video_frame_buffer, j_get_width_id_);
int height = jni->CallIntMethod(j_video_frame_buffer, j_get_height_id_);
return new rtc::RefCountedObject<AndroidVideoBuffer>(
jni, j_retain_id_, j_release_id_, width, height, matrix,
j_video_frame_buffer);
jni, j_retain_id_, j_release_id_, width, height, j_video_frame_buffer);
}
} // namespace webrtc_jni

12
webrtc/sdk/android/src/jni/native_handle_impl.h
View File

@ -108,14 +108,13 @@ class AndroidVideoBuffer : public AndroidVideoFrameBuffer {
jmethodID j_release_id,
int width,
int height,
const Matrix& matrix,
jobject j_video_frame_buffer);
~AndroidVideoBuffer() override;
jobject video_frame_buffer() const;
// Returns an instance of VideoRenderer.I420Frame (deprecated)
jobject ToJavaI420Frame(JNIEnv* jni, int width, int height, int rotation);
jobject ToJavaI420Frame(JNIEnv* jni, int rotation);
private:
Type type() const override;
@ -129,7 +128,6 @@ class AndroidVideoBuffer : public AndroidVideoFrameBuffer {
const jmethodID j_release_id_;
const int width_;
const int height_;
const Matrix matrix_;
// Holds a VideoFrame.Buffer.
ScopedGlobalRef<jobject> j_video_frame_buffer_;
};
@ -143,23 +141,19 @@ class AndroidVideoBufferFactory {
uint32_t timestamp_rtp) const;
rtc::scoped_refptr<AndroidVideoBuffer> CreateBuffer(
int width,
int height,
const Matrix& matrix,
jobject j_video_frame_buffer) const;
private:
ScopedGlobalRef<jclass> j_video_frame_class_;
jmethodID j_get_buffer_id_;
jmethodID j_get_width_id_;
jmethodID j_get_height_id_;
jmethodID j_get_rotation_id_;
jmethodID j_get_transform_matrix_id_;
jmethodID j_get_timestamp_ns_id_;
ScopedGlobalRef<jclass> j_video_frame_buffer_class_;
jmethodID j_retain_id_;
jmethodID j_release_id_;
jmethodID j_get_width_id_;
jmethodID j_get_height_id_;
};
} // namespace webrtc_jni

4
webrtc/sdk/android/src/jni/video_renderer_jni.cc
View File

@ -59,9 +59,7 @@ class JavaVideoRendererWrapper
break;
case AndroidVideoFrameBuffer::AndroidType::kJavaBuffer:
j_frame = static_cast<AndroidVideoBuffer*>(android_buffer)
->ToJavaI420Frame(jni(), video_frame.width(),
video_frame.height(),
video_frame.rotation());
->ToJavaI420Frame(jni(), video_frame.rotation());
break;
default:
RTC_NOTREACHED();

62
webrtc/sdk/android/src/jni/videoframe_jni.cc Normal file
View File

@ -0,0 +1,62 @@
/*
* Copyright 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include <jni.h>
#include "libyuv/scale.h"
#include "webrtc/rtc_base/checks.h"
namespace webrtc_jni {
extern "C" JNIEXPORT void JNICALL
Java_org_webrtc_VideoFrame_nativeCropAndScaleI420(JNIEnv* jni,
jclass,
jobject j_src_y,
jint src_stride_y,
jobject j_src_u,
jint src_stride_u,
jobject j_src_v,
jint src_stride_v,
jint crop_x,
jint crop_y,
jint crop_width,
jint crop_height,
jobject j_dst_y,
jint dst_stride_y,
jobject j_dst_u,
jint dst_stride_u,
jobject j_dst_v,
jint dst_stride_v,
jint scale_width,
jint scale_height) {
uint8_t const* src_y =
static_cast<uint8_t*>(jni->GetDirectBufferAddress(j_src_y));
uint8_t const* src_u =
static_cast<uint8_t*>(jni->GetDirectBufferAddress(j_src_u));
uint8_t const* src_v =
static_cast<uint8_t*>(jni->GetDirectBufferAddress(j_src_v));
uint8_t* dst_y = static_cast<uint8_t*>(jni->GetDirectBufferAddress(j_dst_y));
uint8_t* dst_u = static_cast<uint8_t*>(jni->GetDirectBufferAddress(j_dst_u));
uint8_t* dst_v = static_cast<uint8_t*>(jni->GetDirectBufferAddress(j_dst_v));
// Perform cropping using pointer arithmetic.
src_y += crop_x + crop_y * src_stride_y;
src_u += crop_x / 2 + crop_y / 2 * src_stride_u;
src_v += crop_x / 2 + crop_y / 2 * src_stride_v;
bool ret = libyuv::I420Scale(
src_y, src_stride_y, src_u, src_stride_u, src_v, src_stride_v, crop_width,
crop_height, dst_y, dst_stride_y, dst_u, dst_stride_u, dst_v,
dst_stride_v, scale_width, scale_height, libyuv::kFilterBox);
RTC_DCHECK_EQ(ret, 0) << "I420Scale failed";
}
} // namespace webrtc_jni