zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Android video rendering: Apply SurfaceTexture.getTransformationMatrix()

Browse Source 
This CL applies the transformation matrix instead of assuming it is always a vertical flip.

BUG=webrtc:4968,webrtc:4742
R=hbos@webrtc.org, pbos@webrtc.org

Review URL: https://codereview.webrtc.org/1318153007 .

Cr-Commit-Position: refs/heads/master@{#9905}
This commit is contained in:
Magnus Jedvert
2015-09-09 10:59:46 +02:00
parent 66f43392a3
commit 529528cc36
3 changed files with 153 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files

97
talk/app/webrtc/androidtests/src/org/webrtc/RendererCommonTest.java
View File

@ -35,7 +35,8 @@ import android.graphics.Point;
import static org.webrtc.RendererCommon.ScalingType.*;
import static org.webrtc.RendererCommon.getDisplaySize;
import static org.webrtc.RendererCommon.getTextureMatrix;
import static org.webrtc.RendererCommon.getLayoutMatrix;
import static org.webrtc.RendererCommon.getSamplingMatrix;
public class RendererCommonTest extends ActivityTestCase {
@SmallTest
@ -100,17 +101,60 @@ public class RendererCommonTest extends ActivityTestCase {
return doubleArray;
}
// Brief summary about matrix transformations:
// A coordinate p = [u, v, 0, 1] is transformed by matrix m like this p' = [u', v', 0, 1] = m * p.
// OpenGL uses column-major order, so:
// u' = u * m[0] + v * m[4] + m[12].
// v' = u * m[1] + v * m[5] + m[13].
@SmallTest
static public void testTexMatrixDefault() {
final float texMatrix[] = new float[16];
getTextureMatrix(texMatrix, 0, false, 1.0f, 1.0f);
// TODO(magjed): Every tex matrix contains a vertical flip, because we ignore the texture
// transform matrix from the SurfaceTexture (which contains a vertical flip). Update tests when
// this is fixed.
static public void testLayoutMatrixDefault() {
final float layoutMatrix[] = getLayoutMatrix(false, 1.0f, 1.0f);
// Assert:
// u' = u.
// v' = v.
MoreAsserts.assertEquals(round(layoutMatrix), new double[]
{1, 0, 0, 0,
0, 1, 0, 0,
0, 0, 1, 0,
0, 0, 0, 1});
}
@SmallTest
static public void testLayoutMatrixMirror() {
final float layoutMatrix[] = getLayoutMatrix(true, 1.0f, 1.0f);
// Assert:
// u' = 1 - u.
// v' = v.
MoreAsserts.assertEquals(round(layoutMatrix), new double[]
{-1, 0, 0, 0,
0, 1, 0, 0,
0, 0, 1, 0,
1, 0, 0, 1});
}
@SmallTest
static public void testLayoutMatrixScale() {
// Video has aspect ratio 2, but layout is square. This will cause only the center part of the
// video to be visible, i.e. the u coordinate will go from 0.25 to 0.75 instead of from 0 to 1.
final float layoutMatrix[] = getLayoutMatrix(false, 2.0f, 1.0f);
// Assert:
// u' = 0.25 + 0.5 u.
// v' = v.
MoreAsserts.assertEquals(round(layoutMatrix), new double[]
{ 0.5, 0, 0, 0,
0, 1, 0, 0,
0, 0, 1, 0,
0.25, 0, 0, 1});
}
@SmallTest
static public void testSamplingMatrixDefault() {
final float samplingMatrix[] = getSamplingMatrix(null, 0);
// Assert:
// u' = u.
// v' = 1 - v.
MoreAsserts.assertEquals(round(texMatrix), new double[]
MoreAsserts.assertEquals(round(samplingMatrix), new double[]
{1, 0, 0, 0,
0, -1, 0, 0,
0, 0, 1, 0,
@ -118,46 +162,15 @@ public class RendererCommonTest extends ActivityTestCase {
}
@SmallTest
static public void testTexMatrixMirror() {
final float texMatrix[] = new float[16];
getTextureMatrix(texMatrix, 0, true, 1.0f, 1.0f);
static public void testSamplingMatrixRotation90Deg() {
final float samplingMatrix[] = getSamplingMatrix(null, 90);
// Assert:
// u' = 1 - u.
// v' = 1 - v.
MoreAsserts.assertEquals(round(texMatrix), new double[]
{-1, 0, 0, 0,
0, -1, 0, 0,
0, 0, 1, 0,
1, 1, 0, 1});
}
@SmallTest
static public void testTexMatrixRotation90Deg() {
final float texMatrix[] = new float[16];
getTextureMatrix(texMatrix, 90, false, 1.0f, 1.0f);
// Assert:
// u' = 1 - v.
// v' = 1 - u.
MoreAsserts.assertEquals(round(texMatrix), new double[]
MoreAsserts.assertEquals(round(samplingMatrix), new double[]
{ 0, -1, 0, 0,
-1, 0, 0, 0,
0, 0, 1, 0,
1, 1, 0, 1});
}
@SmallTest
static public void testTexMatrixScale() {
final float texMatrix[] = new float[16];
// Video has aspect ratio 2, but layout is square. This will cause only the center part of the
// video to be visible, i.e. the u coordinate will go from 0.25 to 0.75 instead of from 0 to 1.
getTextureMatrix(texMatrix, 0, false, 2.0f, 1.0f);
// Assert:
// u' = 0.25 + 0.5 u.
// v' = 1 - v.
MoreAsserts.assertEquals(round(texMatrix), new double[]
{0.5, 0, 0, 0,
0, -1, 0, 0,
0, 0, 1, 0,
0.25, 1, 0, 1});
}
}

86
talk/app/webrtc/java/android/org/webrtc/RendererCommon.java
View File

@ -28,6 +28,7 @@
package org.webrtc;
import android.graphics.Point;
import android.graphics.SurfaceTexture;
import android.opengl.Matrix;
/**
@ -60,37 +61,62 @@ public class RendererCommon {
// The minimum fraction of the frame content that will be shown for |SCALE_ASPECT_BALANCED|.
// This limits excessive cropping when adjusting display size.
private static float BALANCED_VISIBLE_FRACTION = 0.5625f;
// Matrix with transform y' = 1 - y.
private static final float[] VERTICAL_FLIP = new float[] {
1, 0, 0, 0,
0, -1, 0, 0,
0, 0, 1, 0,
0, 1, 0, 1};
/**
* Calculates a texture transformation matrix based on rotation, mirror, and video vs display
* aspect ratio.
* Returns matrix that transforms standard coordinates to their proper sampling locations in
* the texture. This transform compensates for any properties of the video source that
* cause it to appear different from a normalized texture. If the video source is based on
* ByteBuffers, pass null in |surfaceTexture|.
*/
public static void getTextureMatrix(float[] outputTextureMatrix, float rotationDegree,
boolean mirror, float videoAspectRatio, float displayAspectRatio) {
// The matrix stack is using post-multiplication, which means that matrix operations:
// A; B; C; will end up as A * B * C. When you apply this to a vertex, it will result in:
// v' = A * B * C * v, i.e. the last matrix operation is the first thing that affects the
// vertex. This is the opposite of what you might expect.
Matrix.setIdentityM(outputTextureMatrix, 0);
// Move coordinates back to [0,1]x[0,1].
Matrix.translateM(outputTextureMatrix, 0, 0.5f, 0.5f, 0.0f);
// Rotate frame clockwise in the XY-plane (around the Z-axis).
Matrix.rotateM(outputTextureMatrix, 0, -rotationDegree, 0, 0, 1);
// Scale one dimension until video and display size have same aspect ratio.
if (displayAspectRatio > videoAspectRatio) {
Matrix.scaleM(outputTextureMatrix, 0, 1, videoAspectRatio / displayAspectRatio, 1);
public static float[] getSamplingMatrix(SurfaceTexture surfaceTexture, float rotationDegree) {
final float[] samplingMatrix;
if (surfaceTexture == null) {
// For ByteBuffers, row 0 specifies the top row, but for a texture, row 0 specifies the
// bottom row. Flip the image vertically to compensate for this.
samplingMatrix = VERTICAL_FLIP;
} else {
Matrix.scaleM(outputTextureMatrix, 0, displayAspectRatio / videoAspectRatio, 1, 1);
samplingMatrix = new float[16];
surfaceTexture.getTransformMatrix(samplingMatrix);
}
// Clockwise rotation matrix in the XY-plane (around the Z-axis).
final float[] rotationMatrix = new float[16];
Matrix.setRotateM(rotationMatrix, 0, -rotationDegree, 0, 0, 1);
adjustOrigin(rotationMatrix);
// Multiply matrices together.
final float[] tmpMatrix = new float[16];
Matrix.multiplyMM(tmpMatrix, 0, rotationMatrix, 0, samplingMatrix, 0);
return tmpMatrix;
}
/**
* Returns layout transformation matrix that applies an optional mirror effect and compensates
* for video vs display aspect ratio.
*/
public static float[] getLayoutMatrix(
boolean mirror, float videoAspectRatio, float displayAspectRatio) {
float scaleX = 1;
float scaleY = 1;
// Scale X or Y dimension so that video and display size have same aspect ratio.
if (displayAspectRatio > videoAspectRatio) {
scaleY = videoAspectRatio / displayAspectRatio;
} else {
scaleX = displayAspectRatio / videoAspectRatio;
}
// TODO(magjed): We currently ignore the texture transform matrix from the SurfaceTexture.
// It contains a vertical flip that is hardcoded here instead.
Matrix.scaleM(outputTextureMatrix, 0, 1, -1, 1);
// Apply optional horizontal flip.
if (mirror) {
Matrix.scaleM(outputTextureMatrix, 0, -1, 1, 1);
scaleX *= -1;
}
// Center coordinates around origin.
Matrix.translateM(outputTextureMatrix, 0, -0.5f, -0.5f, 0.0f);
final float matrix[] = new float[16];
Matrix.setIdentityM(matrix, 0);
Matrix.scaleM(matrix, 0, scaleX, scaleY, 1);
adjustOrigin(matrix);
return matrix;
}
/**
@ -102,6 +128,20 @@ public class RendererCommon {
maxDisplayWidth, maxDisplayHeight);
}
/**
* Move |matrix| transformation origin to (0.5, 0.5). This is the origin for texture coordinates
* that are in the range 0 to 1.
*/
private static void adjustOrigin(float[] matrix) {
// Note that OpenGL is using column-major order.
// Pre translate with -0.5 to move coordinates to range [-0.5, 0.5].
matrix[12] -= 0.5f * (matrix[0] + matrix[4]);
matrix[13] -= 0.5f * (matrix[1] + matrix[5]);
// Post translate with 0.5 to move coordinates to range [0, 1].
matrix[12] += 0.5f;
matrix[13] += 0.5f;
}
/**
* Each scaling type has a one-to-one correspondence to a numeric minimum fraction of the video
* that must remain visible.

51
talk/app/webrtc/java/android/org/webrtc/VideoRendererGui.java
View File

@ -41,6 +41,7 @@ import android.opengl.EGL14;
import android.opengl.EGLContext;
import android.opengl.GLES20;
import android.opengl.GLSurfaceView;
import android.opengl.Matrix;
import android.util.Log;
import org.webrtc.VideoRenderer.I420Frame;
@ -133,12 +134,15 @@ public class VideoRendererGui implements GLSurfaceView.Renderer {
// The actual view area in pixels. It is a centered subrectangle of the rectangle defined by
// |layoutInPercentage|.
private final Rect displayLayout = new Rect();
// Cached texture transformation matrix, calculated from current layout parameters.
private final float[] texMatrix = new float[16];
// Flag if texture vertices or coordinates update is needed.
private boolean updateTextureProperties;
// Texture properties update lock.
private final Object updateTextureLock = new Object();
// Cached layout transformation matrix, calculated from current layout parameters.
private float[] layoutMatrix;
// Flag if layout transformation matrix update is needed.
private boolean updateLayoutProperties;
// Layout properties update lock. Guards |updateLayoutProperties|, |screenWidth|,
// |screenHeight|, |videoWidth|, |videoHeight|, |rotationDegree|, |scalingType|, and |mirror|.
private final Object updateLayoutLock = new Object();
// Texture sampling matrix.
private float[] samplingMatrix;
// Viewport dimensions.
private int screenWidth;
private int screenHeight;
@ -160,7 +164,7 @@ public class VideoRendererGui implements GLSurfaceView.Renderer {
this.scalingType = scalingType;
this.mirror = mirror;
layoutInPercentage = new Rect(x, y, Math.min(100, x + width), Math.min(100, y + height));
updateTextureProperties = false;
updateLayoutProperties = false;
rotationDegree = 0;
}
@ -184,9 +188,9 @@ public class VideoRendererGui implements GLSurfaceView.Renderer {
}
}
private void checkAdjustTextureCoords() {
synchronized(updateTextureLock) {
if (!updateTextureProperties) {
private void updateLayoutMatrix() {
synchronized(updateLayoutLock) {
if (!updateLayoutProperties) {
return;
}
// Initialize to maximum allowed area. Round to integer coordinates inwards the layout
@ -209,9 +213,9 @@ public class VideoRendererGui implements GLSurfaceView.Renderer {
(displayLayout.height() - displaySize.y) / 2);
Log.d(TAG, " Adjusted display size: " + displayLayout.width() + " x "
+ displayLayout.height());
RendererCommon.getTextureMatrix(texMatrix, rotationDegree, mirror, videoAspectRatio,
(float) displayLayout.width() / displayLayout.height());
updateTextureProperties = false;
layoutMatrix = RendererCommon.getLayoutMatrix(
mirror, videoAspectRatio, (float) displayLayout.width() / displayLayout.height());
updateLayoutProperties = false;
Log.d(TAG, " AdjustTextureCoords done");
}
}
@ -229,10 +233,6 @@ public class VideoRendererGui implements GLSurfaceView.Renderer {
final boolean isNewFrame;
synchronized (pendingFrameLock) {
// Check if texture vertices/coordinates adjustment is required when
// screen orientation changes or video frame size changes.
checkAdjustTextureCoords();
isNewFrame = (pendingFrame != null);
if (isNewFrame && startTimeNs == -1) {
startTimeNs = now;
@ -255,12 +255,17 @@ public class VideoRendererGui implements GLSurfaceView.Renderer {
surfaceTexture.updateTexImage();
}
}
samplingMatrix = RendererCommon.getSamplingMatrix(
(SurfaceTexture) pendingFrame.textureObject, pendingFrame.rotationDegree);
copyTimeNs += (System.nanoTime() - now);
VideoRenderer.renderFrameDone(pendingFrame);
pendingFrame = null;
}
}
updateLayoutMatrix();
final float[] texMatrix = new float[16];
Matrix.multiplyMM(texMatrix, 0, samplingMatrix, 0, layoutMatrix, 0);
if (rendererType == RendererType.RENDERER_YUV) {
drawer.drawYuv(yuvTextures, texMatrix);
} else {
@ -291,7 +296,7 @@ public class VideoRendererGui implements GLSurfaceView.Renderer {
}
public void setScreenSize(final int screenWidth, final int screenHeight) {
synchronized(updateTextureLock) {
synchronized(updateLayoutLock) {
if (screenWidth == this.screenWidth && screenHeight == this.screenHeight) {
return;
}
@ -299,7 +304,7 @@ public class VideoRendererGui implements GLSurfaceView.Renderer {
screenWidth + " x " + screenHeight);
this.screenWidth = screenWidth;
this.screenHeight = screenHeight;
updateTextureProperties = true;
updateLayoutProperties = true;
}
}
@ -307,7 +312,7 @@ public class VideoRendererGui implements GLSurfaceView.Renderer {
RendererCommon.ScalingType scalingType, boolean mirror) {
final Rect layoutInPercentage =
new Rect(x, y, Math.min(100, x + width), Math.min(100, y + height));
synchronized(updateTextureLock) {
synchronized(updateLayoutLock) {
if (layoutInPercentage.equals(this.layoutInPercentage) && scalingType == this.scalingType
&& mirror == this.mirror) {
return;
@ -318,7 +323,7 @@ public class VideoRendererGui implements GLSurfaceView.Renderer {
this.layoutInPercentage.set(layoutInPercentage);
this.scalingType = scalingType;
this.mirror = mirror;
updateTextureProperties = true;
updateLayoutProperties = true;
}
}
@ -333,14 +338,14 @@ public class VideoRendererGui implements GLSurfaceView.Renderer {
rendererEvents.onFrameResolutionChanged(videoWidth, videoHeight, rotation);
}
synchronized (updateTextureLock) {
synchronized (updateLayoutLock) {
Log.d(TAG, "ID: " + id + ". YuvImageRenderer.setSize: " +
videoWidth + " x " + videoHeight + " rotation " + rotation);
this.videoWidth = videoWidth;
this.videoHeight = videoHeight;
rotationDegree = rotation;
updateTextureProperties = true;
updateLayoutProperties = true;
Log.d(TAG, " YuvImageRenderer.setSize done.");
}
}