Android video rendering: Apply SurfaceTexture.getTransformationMatrix()

This CL applies the transformation matrix instead of assuming it is always a vertical flip.

BUG=webrtc:4968,webrtc:4742
R=hbos@webrtc.org, pbos@webrtc.org

Review URL: https://codereview.webrtc.org/1318153007 .

Cr-Commit-Position: refs/heads/master@{#9905}

talk/app/webrtc/androidtests/src/org/webrtc/RendererCommonTest.java

@@ -35,7 +35,8 @@ import android.graphics.Point;
 
 import static org.webrtc.RendererCommon.ScalingType.*;
 import static org.webrtc.RendererCommon.getDisplaySize;
-import static org.webrtc.RendererCommon.getTextureMatrix;
+import static org.webrtc.RendererCommon.getLayoutMatrix;
+import static org.webrtc.RendererCommon.getSamplingMatrix;
 
 public class RendererCommonTest extends ActivityTestCase {
   @SmallTest
@@ -100,17 +101,60 @@ public class RendererCommonTest extends ActivityTestCase {
     return doubleArray;
   }
 
+  // Brief summary about matrix transformations:
+  // A coordinate p = [u, v, 0, 1] is transformed by matrix m like this p' = [u', v', 0, 1] = m * p.
+  // OpenGL uses column-major order, so:
+  // u' = u * m[0] + v * m[4] + m[12].
+  // v' = u * m[1] + v * m[5] + m[13].
+
   @SmallTest
-  static public void testTexMatrixDefault() {
+  static public void testLayoutMatrixDefault() {
-    final float texMatrix[] = new float[16];
+    final float layoutMatrix[] = getLayoutMatrix(false, 1.0f, 1.0f);
-    getTextureMatrix(texMatrix, 0, false, 1.0f, 1.0f);
+    // Assert:
-    // TODO(magjed): Every tex matrix contains a vertical flip, because we ignore the texture
+    // u' = u.
-    // transform matrix from the SurfaceTexture (which contains a vertical flip). Update tests when
+    // v' = v.
-    // this is fixed.
+    MoreAsserts.assertEquals(round(layoutMatrix), new double[]
+        {1, 0, 0, 0,
+         0, 1, 0, 0,
+         0, 0, 1, 0,
+         0, 0, 0, 1});
+  }
+
+  @SmallTest
+  static public void testLayoutMatrixMirror() {
+    final float layoutMatrix[] = getLayoutMatrix(true, 1.0f, 1.0f);
+    // Assert:
+    // u' = 1 - u.
+    // v' = v.
+    MoreAsserts.assertEquals(round(layoutMatrix), new double[]
+        {-1, 0, 0, 0,
+          0, 1, 0, 0,
+          0, 0, 1, 0,
+          1, 0, 0, 1});
+  }
+
+  @SmallTest
+  static public void testLayoutMatrixScale() {
+    // Video has aspect ratio 2, but layout is square. This will cause only the center part of the
+    // video to be visible, i.e. the u coordinate will go from 0.25 to 0.75 instead of from 0 to 1.
+    final float layoutMatrix[] = getLayoutMatrix(false, 2.0f, 1.0f);
+    // Assert:
+    // u' = 0.25 + 0.5 u.
+    // v' = v.
+    MoreAsserts.assertEquals(round(layoutMatrix), new double[]
+        { 0.5, 0, 0, 0,
+            0, 1, 0, 0,
+            0, 0, 1, 0,
+         0.25, 0, 0, 1});
+  }
+
+  @SmallTest
+  static public void testSamplingMatrixDefault() {
+    final float samplingMatrix[] = getSamplingMatrix(null, 0);
     // Assert:
     // u' = u.
     // v' = 1 - v.
-    MoreAsserts.assertEquals(round(texMatrix), new double[]
+    MoreAsserts.assertEquals(round(samplingMatrix), new double[]
         {1,  0, 0, 0,
          0, -1, 0, 0,
          0,  0, 1, 0,
@@ -118,46 +162,15 @@ public class RendererCommonTest extends ActivityTestCase {
   }
 
   @SmallTest
-  static public void testTexMatrixMirror() {
+  static public void testSamplingMatrixRotation90Deg() {
-    final float texMatrix[] = new float[16];
+    final float samplingMatrix[] = getSamplingMatrix(null, 90);
-    getTextureMatrix(texMatrix, 0, true, 1.0f, 1.0f);
     // Assert:
     // u' = 1 - u.
     // v' = 1 - v.
-    MoreAsserts.assertEquals(round(texMatrix), new double[]
+    MoreAsserts.assertEquals(round(samplingMatrix), new double[]
-        {-1, 0, 0, 0,
-         0, -1, 0, 0,
-         0, 0, 1, 0,
-         1, 1, 0, 1});
-  }
-
-  @SmallTest
-  static public void testTexMatrixRotation90Deg() {
-    final float texMatrix[] = new float[16];
-    getTextureMatrix(texMatrix, 90, false, 1.0f, 1.0f);
-    // Assert:
-    // u' = 1 - v.
-    // v' = 1 - u.
-    MoreAsserts.assertEquals(round(texMatrix), new double[]
         { 0, -1, 0, 0,
          -1,  0, 0, 0,
           0,  0, 1, 0,
           1,  1, 0, 1});
   }
-
-  @SmallTest
-  static public void testTexMatrixScale() {
-    final float texMatrix[] = new float[16];
-    // Video has aspect ratio 2, but layout is square. This will cause only the center part of the
-    // video to be visible, i.e. the u coordinate will go from 0.25 to 0.75 instead of from 0 to 1.
-    getTextureMatrix(texMatrix, 0, false, 2.0f, 1.0f);
-    // Assert:
-    // u' = 0.25 + 0.5 u.
-    // v' = 1 - v.
-    MoreAsserts.assertEquals(round(texMatrix), new double[]
-        {0.5, 0, 0, 0,
-         0, -1, 0, 0,
-         0, 0, 1, 0,
-         0.25, 1, 0, 1});
-  }
 }

talk/app/webrtc/java/android/org/webrtc/RendererCommon.java

@@ -28,6 +28,7 @@
 package org.webrtc;
 
 import android.graphics.Point;
+import android.graphics.SurfaceTexture;
 import android.opengl.Matrix;
 
 /**
@@ -60,37 +61,62 @@ public class RendererCommon {
   // The minimum fraction of the frame content that will be shown for |SCALE_ASPECT_BALANCED|.
   // This limits excessive cropping when adjusting display size.
   private static float BALANCED_VISIBLE_FRACTION = 0.5625f;
+  // Matrix with transform y' = 1 - y.
+  private static final float[] VERTICAL_FLIP = new float[] {
+      1,  0, 0, 0,
+      0, -1, 0, 0,
+      0,  0, 1, 0,
+      0,  1, 0, 1};
 
   /**
-   * Calculates a texture transformation matrix based on rotation, mirror, and video vs display
+   * Returns matrix that transforms standard coordinates to their proper sampling locations in
-   * aspect ratio.
+   * the texture. This transform compensates for any properties of the video source that
+   * cause it to appear different from a normalized texture. If the video source is based on
+   * ByteBuffers, pass null in |surfaceTexture|.
    */
-  public static void getTextureMatrix(float[] outputTextureMatrix, float rotationDegree,
+  public static float[] getSamplingMatrix(SurfaceTexture surfaceTexture, float rotationDegree) {
-      boolean mirror, float videoAspectRatio, float displayAspectRatio) {
+    final float[] samplingMatrix;
-    // The matrix stack is using post-multiplication, which means that matrix operations:
+    if (surfaceTexture == null) {
-    // A; B; C; will end up as A * B * C. When you apply this to a vertex, it will result in:
+      // For ByteBuffers, row 0 specifies the top row, but for a texture, row 0 specifies the
-    // v' = A * B * C * v, i.e. the last matrix operation is the first thing that affects the
+      // bottom row. Flip the image vertically to compensate for this.
-    // vertex. This is the opposite of what you might expect.
+      samplingMatrix = VERTICAL_FLIP;
-    Matrix.setIdentityM(outputTextureMatrix, 0);
-    // Move coordinates back to [0,1]x[0,1].
-    Matrix.translateM(outputTextureMatrix, 0, 0.5f, 0.5f, 0.0f);
-    // Rotate frame clockwise in the XY-plane (around the Z-axis).
-    Matrix.rotateM(outputTextureMatrix, 0, -rotationDegree, 0, 0, 1);
-    // Scale one dimension until video and display size have same aspect ratio.
-    if (displayAspectRatio > videoAspectRatio) {
-      Matrix.scaleM(outputTextureMatrix, 0, 1, videoAspectRatio / displayAspectRatio, 1);
     } else {
-      Matrix.scaleM(outputTextureMatrix, 0, displayAspectRatio / videoAspectRatio, 1, 1);
+      samplingMatrix = new float[16];
+      surfaceTexture.getTransformMatrix(samplingMatrix);
+    }
+    // Clockwise rotation matrix in the XY-plane (around the Z-axis).
+    final float[] rotationMatrix = new float[16];
+    Matrix.setRotateM(rotationMatrix, 0, -rotationDegree, 0, 0, 1);
+    adjustOrigin(rotationMatrix);
+    // Multiply matrices together.
+    final float[] tmpMatrix = new float[16];
+    Matrix.multiplyMM(tmpMatrix, 0, rotationMatrix, 0, samplingMatrix, 0);
+    return tmpMatrix;
+  }
+
+  /**
+   * Returns layout transformation matrix that applies an optional mirror effect and compensates
+   * for video vs display aspect ratio.
+   */
+  public static float[] getLayoutMatrix(
+      boolean mirror, float videoAspectRatio, float displayAspectRatio) {
+    float scaleX = 1;
+    float scaleY = 1;
+    // Scale X or Y dimension so that video and display size have same aspect ratio.
+    if (displayAspectRatio > videoAspectRatio) {
+      scaleY = videoAspectRatio / displayAspectRatio;
+    } else {
+      scaleX = displayAspectRatio / videoAspectRatio;
     }
-    // TODO(magjed): We currently ignore the texture transform matrix from the SurfaceTexture.
-    // It contains a vertical flip that is hardcoded here instead.
-    Matrix.scaleM(outputTextureMatrix, 0, 1, -1, 1);
     // Apply optional horizontal flip.
     if (mirror) {
-      Matrix.scaleM(outputTextureMatrix, 0, -1, 1, 1);
+      scaleX *= -1;
     }
-    // Center coordinates around origin.
+    final float matrix[] = new float[16];
-    Matrix.translateM(outputTextureMatrix, 0, -0.5f, -0.5f, 0.0f);
+    Matrix.setIdentityM(matrix, 0);
+    Matrix.scaleM(matrix, 0, scaleX, scaleY, 1);
+    adjustOrigin(matrix);
+    return matrix;
   }
 
   /**
@@ -102,6 +128,20 @@ public class RendererCommon {
         maxDisplayWidth, maxDisplayHeight);
   }
 
+  /**
+   * Move |matrix| transformation origin to (0.5, 0.5). This is the origin for texture coordinates
+   * that are in the range 0 to 1.
+   */
+  private static void adjustOrigin(float[] matrix) {
+    // Note that OpenGL is using column-major order.
+    // Pre translate with -0.5 to move coordinates to range [-0.5, 0.5].
+    matrix[12] -= 0.5f * (matrix[0] + matrix[4]);
+    matrix[13] -= 0.5f * (matrix[1] + matrix[5]);
+    // Post translate with 0.5 to move coordinates to range [0, 1].
+    matrix[12] += 0.5f;
+    matrix[13] += 0.5f;
+  }
+
   /**
    * Each scaling type has a one-to-one correspondence to a numeric minimum fraction of the video
    * that must remain visible.

talk/app/webrtc/java/android/org/webrtc/VideoRendererGui.java

@@ -41,6 +41,7 @@ import android.opengl.EGL14;
 import android.opengl.EGLContext;
 import android.opengl.GLES20;
 import android.opengl.GLSurfaceView;
+import android.opengl.Matrix;
 import android.util.Log;
 
 import org.webrtc.VideoRenderer.I420Frame;
@@ -133,12 +134,15 @@ public class VideoRendererGui implements GLSurfaceView.Renderer {
     // The actual view area in pixels. It is a centered subrectangle of the rectangle defined by
     // |layoutInPercentage|.
     private final Rect displayLayout = new Rect();
-    // Cached texture transformation matrix, calculated from current layout parameters.
+    // Cached layout transformation matrix, calculated from current layout parameters.
-    private final float[] texMatrix = new float[16];
+    private float[] layoutMatrix;
-    // Flag if texture vertices or coordinates update is needed.
+    // Flag if layout transformation matrix update is needed.
-    private boolean updateTextureProperties;
+    private boolean updateLayoutProperties;
-    // Texture properties update lock.
+    // Layout properties update lock. Guards |updateLayoutProperties|, |screenWidth|,
-    private final Object updateTextureLock = new Object();
+    // |screenHeight|, |videoWidth|, |videoHeight|, |rotationDegree|, |scalingType|, and |mirror|.
+    private final Object updateLayoutLock = new Object();
+    // Texture sampling matrix.
+    private float[] samplingMatrix;
     // Viewport dimensions.
     private int screenWidth;
     private int screenHeight;
@@ -160,7 +164,7 @@ public class VideoRendererGui implements GLSurfaceView.Renderer {
       this.scalingType = scalingType;
       this.mirror = mirror;
       layoutInPercentage = new Rect(x, y, Math.min(100, x + width), Math.min(100, y + height));
-      updateTextureProperties = false;
+      updateLayoutProperties = false;
       rotationDegree = 0;
     }
 
@@ -184,9 +188,9 @@ public class VideoRendererGui implements GLSurfaceView.Renderer {
       }
     }
 
-    private void checkAdjustTextureCoords() {
+    private void updateLayoutMatrix() {
-      synchronized(updateTextureLock) {
+      synchronized(updateLayoutLock) {
-        if (!updateTextureProperties) {
+        if (!updateLayoutProperties) {
           return;
         }
         // Initialize to maximum allowed area. Round to integer coordinates inwards the layout
@@ -209,9 +213,9 @@ public class VideoRendererGui implements GLSurfaceView.Renderer {
                             (displayLayout.height() - displaySize.y) / 2);
         Log.d(TAG, "  Adjusted display size: " + displayLayout.width() + " x "
             + displayLayout.height());
-        RendererCommon.getTextureMatrix(texMatrix, rotationDegree, mirror, videoAspectRatio,
+        layoutMatrix = RendererCommon.getLayoutMatrix(
-            (float) displayLayout.width() / displayLayout.height());
+            mirror, videoAspectRatio, (float) displayLayout.width() / displayLayout.height());
-        updateTextureProperties = false;
+        updateLayoutProperties = false;
         Log.d(TAG, "  AdjustTextureCoords done");
       }
     }
@@ -229,10 +233,6 @@ public class VideoRendererGui implements GLSurfaceView.Renderer {
 
       final boolean isNewFrame;
       synchronized (pendingFrameLock) {
-        // Check if texture vertices/coordinates adjustment is required when
-        // screen orientation changes or video frame size changes.
-        checkAdjustTextureCoords();
-
         isNewFrame = (pendingFrame != null);
         if (isNewFrame && startTimeNs == -1) {
           startTimeNs = now;
@@ -255,12 +255,17 @@ public class VideoRendererGui implements GLSurfaceView.Renderer {
               surfaceTexture.updateTexImage();
             }
           }
+          samplingMatrix = RendererCommon.getSamplingMatrix(
+              (SurfaceTexture) pendingFrame.textureObject, pendingFrame.rotationDegree);
           copyTimeNs += (System.nanoTime() - now);
           VideoRenderer.renderFrameDone(pendingFrame);
           pendingFrame = null;
         }
       }
 
+      updateLayoutMatrix();
+      final float[] texMatrix = new float[16];
+      Matrix.multiplyMM(texMatrix, 0, samplingMatrix, 0, layoutMatrix, 0);
       if (rendererType == RendererType.RENDERER_YUV) {
         drawer.drawYuv(yuvTextures, texMatrix);
       } else {
@@ -291,7 +296,7 @@ public class VideoRendererGui implements GLSurfaceView.Renderer {
     }
 
     public void setScreenSize(final int screenWidth, final int screenHeight) {
-      synchronized(updateTextureLock) {
+      synchronized(updateLayoutLock) {
         if (screenWidth == this.screenWidth && screenHeight == this.screenHeight) {
           return;
         }
@@ -299,7 +304,7 @@ public class VideoRendererGui implements GLSurfaceView.Renderer {
             screenWidth + " x " + screenHeight);
         this.screenWidth = screenWidth;
         this.screenHeight = screenHeight;
-        updateTextureProperties = true;
+        updateLayoutProperties = true;
       }
     }
 
@@ -307,7 +312,7 @@ public class VideoRendererGui implements GLSurfaceView.Renderer {
         RendererCommon.ScalingType scalingType, boolean mirror) {
       final Rect layoutInPercentage =
           new Rect(x, y, Math.min(100, x + width), Math.min(100, y + height));
-      synchronized(updateTextureLock) {
+      synchronized(updateLayoutLock) {
         if (layoutInPercentage.equals(this.layoutInPercentage) && scalingType == this.scalingType
             && mirror == this.mirror) {
           return;
@@ -318,7 +323,7 @@ public class VideoRendererGui implements GLSurfaceView.Renderer {
         this.layoutInPercentage.set(layoutInPercentage);
         this.scalingType = scalingType;
         this.mirror = mirror;
-        updateTextureProperties = true;
+        updateLayoutProperties = true;
       }
     }
 
@@ -333,14 +338,14 @@ public class VideoRendererGui implements GLSurfaceView.Renderer {
         rendererEvents.onFrameResolutionChanged(videoWidth, videoHeight, rotation);
       }
 
-      synchronized (updateTextureLock) {
+      synchronized (updateLayoutLock) {
         Log.d(TAG, "ID: " + id + ". YuvImageRenderer.setSize: " +
             videoWidth + " x " + videoHeight + " rotation " + rotation);
 
         this.videoWidth = videoWidth;
         this.videoHeight = videoHeight;
         rotationDegree = rotation;
-        updateTextureProperties = true;
+        updateLayoutProperties = true;
         Log.d(TAG, "  YuvImageRenderer.setSize done.");
       }
     }