zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move matrix from VideoFrame to TextureBuffer.

Browse Source 
Previously, the matrix in VideoFrame was used to crop and scale the
frame. This caused complications because webrtc::VideoFrame doesn't
include a matrix. cropAndScale method is added to VideoBuffer class for
cropping and scaling instead.

BUG=webrtc:7749, webrtc:7760

Review-Url: https://codereview.webrtc.org/2990583002
Cr-Commit-Position: refs/heads/master@{#19179}
This commit is contained in:
sakal
2017-07-28 07:12:23 -07:00
committed by Commit Bot
parent 5ba9730265
commit 836f60cda1
15 changed files with 369 additions and 257 deletions
Download Patch File Download Diff File Expand all files Collapse all files

90
webrtc/sdk/android/api/org/webrtc/SurfaceTextureHelper.java
View File

@ -10,6 +10,7 @@
package org.webrtc;
import android.graphics.Matrix;
import android.graphics.SurfaceTexture;
import android.opengl.GLES11Ext;
import android.opengl.GLES20;
@ -288,86 +289,13 @@ public class SurfaceTextureHelper {
* The returned TextureBuffer holds a reference to the SurfaceTextureHelper that created it. The
* buffer calls returnTextureFrame() when it is released.
*/
public TextureBuffer createTextureBuffer(int width, int height, float[] transformMatrix) {
return new OesTextureBuffer(oesTextureId, width, height, transformMatrix, this);
}
/**
* Android OES texture buffer backed by a SurfaceTextureHelper's texture. The buffer calls
* returnTextureFrame() when it is released.
*/
private static class OesTextureBuffer implements TextureBuffer {
private final int id;
private final int width;
private final int height;
private final float[] transformMatrix;
private final SurfaceTextureHelper helper;
private int refCount;
OesTextureBuffer(
int id, int width, int height, float[] transformMatrix, SurfaceTextureHelper helper) {
this.id = id;
this.width = width;
this.height = height;
this.transformMatrix = transformMatrix;
this.helper = helper;
this.refCount = 1; // Creator implicitly holds a reference.
}
@Override
public TextureBuffer.Type getType() {
return TextureBuffer.Type.OES;
}
@Override
public int getTextureId() {
return id;
}
@Override
public int getWidth() {
return width;
}
@Override
public int getHeight() {
return height;
}
@Override
public I420Buffer toI420() {
// SurfaceTextureHelper requires a stride that is divisible by 8. Round width up.
// See SurfaceTextureHelper for details on the size and format.
int stride = ((width + 7) / 8) * 8;
int uvHeight = (height + 1) / 2;
// Due to the layout used by SurfaceTextureHelper, vPos + stride * uvHeight would overrun the
// buffer. Add one row at the bottom to compensate for this. There will never be data in the
// extra row, but now other code does not have to deal with v stride * v height exceeding the
// buffer's capacity.
int size = stride * (height + uvHeight + 1);
ByteBuffer buffer = ByteBuffer.allocateDirect(size);
helper.textureToYUV(buffer, width, height, stride, id, transformMatrix);
int yPos = 0;
int uPos = yPos + stride * height;
// Rows of U and V alternate in the buffer, so V data starts after the first row of U.
int vPos = yPos + stride / 2;
// SurfaceTextureHelper uses the same stride for Y, U, and V data.
return new I420BufferImpl(
buffer, width, height, yPos, stride, uPos, stride, vPos, stride, null);
}
@Override
public void retain() {
++refCount;
}
@Override
public void release() {
if (--refCount == 0) {
helper.returnTextureFrame();
}
}
public TextureBuffer createTextureBuffer(int width, int height, Matrix transformMatrix) {
return new TextureBufferImpl(
width, height, TextureBuffer.Type.OES, oesTextureId, transformMatrix, this, new Runnable() {
@Override
public void run() {
returnTextureFrame();
}
});
}
}

78
webrtc/sdk/android/api/org/webrtc/VideoFrame.java
View File

@ -44,6 +44,13 @@ public class VideoFrame {
*/
void retain();
void release();
/**
* Crops a region defined by |cropx|, |cropY|, |cropWidth| and |cropHeight|. Scales it to size
* |scaleWidth| x |scaleHeight|.
*/
Buffer cropAndScale(
int cropX, int cropY, int cropWidth, int cropHeight, int scaleWidth, int scaleHeight);
}
/**
@ -67,24 +74,26 @@ public class VideoFrame {
Type getType();
int getTextureId();
/**
* Retrieve the transform matrix associated with the frame. This transform matrix maps 2D
* homogeneous coordinates of the form (s, t, 1) with s and t in the inclusive range [0, 1] to
* the coordinate that should be used to sample that location from the buffer.
*/
public Matrix getTransformMatrix();
}
private final Buffer buffer;
private final int rotation;
private final long timestampNs;
private final Matrix transformMatrix;
public VideoFrame(Buffer buffer, int rotation, long timestampNs, Matrix transformMatrix) {
public VideoFrame(Buffer buffer, int rotation, long timestampNs) {
if (buffer == null) {
throw new IllegalArgumentException("buffer not allowed to be null");
}
if (transformMatrix == null) {
throw new IllegalArgumentException("transformMatrix not allowed to be null");
}
this.buffer = buffer;
this.rotation = rotation;
this.timestampNs = timestampNs;
this.transformMatrix = transformMatrix;
}
public Buffer getBuffer() {
@ -105,26 +114,6 @@ public class VideoFrame {
return timestampNs;
}
/**
* Retrieve the transform matrix associated with the frame. This transform matrix maps 2D
* homogeneous coordinates of the form (s, t, 1) with s and t in the inclusive range [0, 1] to the
* coordinate that should be used to sample that location from the buffer.
*/
public Matrix getTransformMatrix() {
return transformMatrix;
}
/**
* Resolution of the frame in pixels.
*/
public int getWidth() {
return buffer.getWidth();
}
public int getHeight() {
return buffer.getHeight();
}
/**
* Reference counting of the underlying buffer.
*/
@ -135,4 +124,41 @@ public class VideoFrame {
public void release() {
buffer.release();
}
public static VideoFrame.Buffer cropAndScaleI420(final I420Buffer buffer, int cropX, int cropY,
int cropWidth, int cropHeight, int scaleWidth, int scaleHeight) {
if (cropWidth == scaleWidth && cropHeight == scaleHeight) {
// No scaling.
ByteBuffer dataY = buffer.getDataY();
ByteBuffer dataU = buffer.getDataU();
ByteBuffer dataV = buffer.getDataV();
dataY.position(cropX + cropY * buffer.getStrideY());
dataU.position(cropX / 2 + cropY / 2 * buffer.getStrideU());
dataV.position(cropX / 2 + cropY / 2 * buffer.getStrideV());
buffer.retain();
return new I420BufferImpl(buffer.getWidth(), buffer.getHeight(), dataY.slice(),
buffer.getStrideY(), dataU.slice(), buffer.getStrideU(), dataV.slice(),
buffer.getStrideV(), new Runnable() {
@Override
public void run() {
buffer.release();
}
});
}
I420BufferImpl newBuffer = I420BufferImpl.allocate(scaleWidth, scaleHeight);
nativeCropAndScaleI420(buffer.getDataY(), buffer.getStrideY(), buffer.getDataU(),
buffer.getStrideU(), buffer.getDataV(), buffer.getStrideV(), cropX, cropY, cropWidth,
cropHeight, newBuffer.getDataY(), newBuffer.getStrideY(), newBuffer.getDataU(),
newBuffer.getStrideU(), newBuffer.getDataV(), newBuffer.getStrideV(), scaleWidth,
scaleHeight);
return newBuffer;
}
private static native void nativeCropAndScaleI420(ByteBuffer srcY, int srcStrideY,
ByteBuffer srcU, int srcStrideU, ByteBuffer srcV, int srcStrideV, int cropX, int cropY,
int cropWidth, int cropHeight, ByteBuffer dstY, int dstStrideY, ByteBuffer dstU,
int dstStrideU, ByteBuffer dstV, int dstStrideV, int scaleWidth, int scaleHeight);
}

15
webrtc/sdk/android/api/org/webrtc/VideoRenderer.java
View File

@ -84,12 +84,11 @@ public class VideoRenderer {
}
/**
* Construct a frame of the given dimensions from VideoFrame.Buffer.
* Construct a frame from VideoFrame.Buffer.
*/
public I420Frame(int width, int height, int rotationDegree, float[] samplingMatrix,
VideoFrame.Buffer buffer, long nativeFramePointer) {
this.width = width;
this.height = height;
public I420Frame(int rotationDegree, VideoFrame.Buffer buffer, long nativeFramePointer) {
this.width = buffer.getWidth();
this.height = buffer.getHeight();
this.rotationDegree = rotationDegree;
if (rotationDegree % 90 != 0) {
throw new IllegalArgumentException("Rotation degree not multiple of 90: " + rotationDegree);
@ -98,7 +97,8 @@ public class VideoRenderer {
VideoFrame.TextureBuffer textureBuffer = (VideoFrame.TextureBuffer) buffer;
this.yuvFrame = false;
this.textureId = textureBuffer.getTextureId();
this.samplingMatrix = samplingMatrix;
this.samplingMatrix = RendererCommon.convertMatrixFromAndroidGraphicsMatrix(
textureBuffer.getTransformMatrix());
this.yuvStrides = null;
this.yuvPlanes = null;
@ -113,8 +113,7 @@ public class VideoRenderer {
// top-left corner of the image, but in glTexImage2D() the first element corresponds to the
// bottom-left corner. This discrepancy is corrected by multiplying the sampling matrix with
// a vertical flip matrix.
this.samplingMatrix =
RendererCommon.multiplyMatrices(samplingMatrix, RendererCommon.verticalFlipMatrix());
this.samplingMatrix = RendererCommon.verticalFlipMatrix();
this.textureId = 0;
}