Android: Respect input buffer layout of MediaFormat

On Android, MediaCodec can request a specific layout of the input buffer.
One can use the stride and slice height to calculate the layout from
the Encoder's MediaFormat. The current code assumes
a specific layout, which is a problematic in Android 12.
Fix this by honoring the stride and slice-height.

Bug: webrtc:13427
Change-Id: I2d3e429309e3add3ae668e0390460b51e6a49eb9
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/240680
Reviewed-by: Niels Moller <nisse@webrtc.org>
Reviewed-by: Harald Alvestrand <hta@webrtc.org>
Commit-Queue: Daniel.L (Byoungchan) Lee <daniel.l@hpcnt.com>
Cr-Commit-Position: refs/heads/main@{#36033}

sdk/android/src/java/org/webrtc/HardwareVideoEncoder.java

@@ -15,6 +15,7 @@ import android.media.MediaCodec;
 import android.media.MediaCodecInfo;
 import android.media.MediaFormat;
 import android.opengl.GLES20;
+import android.os.Build;
 import android.os.Bundle;
 import android.view.Surface;
 import androidx.annotation.Nullable;
@@ -149,6 +150,10 @@ class HardwareVideoEncoder implements VideoEncoder {
 
   private int width;
   private int height;
+  // Y-plane strides in the encoder's input
+  private int stride;
+  // Y-plane slice-height in the encoder's input
+  private int sliceHeight;
   private boolean useSurfaceMode;
 
   // --- Only accessed from the encoding thread.
@@ -280,6 +285,10 @@ class HardwareVideoEncoder implements VideoEncoder {
         textureEglBase.makeCurrent();
       }
 
+      MediaFormat inputFormat = codec.getInputFormat();
+      stride = getStride(inputFormat, width);
+      sliceHeight = getSliceHeight(inputFormat, height);
+
       codec.start();
       outputBuffers = codec.getOutputBuffers();
     } catch (IllegalStateException e) {
@@ -686,9 +695,25 @@ class HardwareVideoEncoder implements VideoEncoder {
     return sharedContext != null && surfaceColorFormat != null;
   }
 
+  private static int getStride(MediaFormat inputFormat, int width) {
+    if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M && inputFormat != null
+        && inputFormat.containsKey(MediaFormat.KEY_STRIDE)) {
+      return inputFormat.getInteger(MediaFormat.KEY_STRIDE);
+    }
+    return width;
+  }
+
+  private static int getSliceHeight(MediaFormat inputFormat, int height) {
+    if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M && inputFormat != null
+        && inputFormat.containsKey(MediaFormat.KEY_SLICE_HEIGHT)) {
+      return inputFormat.getInteger(MediaFormat.KEY_SLICE_HEIGHT);
+    }
+    return height;
+  }
+
   // Visible for testing.
   protected void fillInputBuffer(ByteBuffer buffer, VideoFrame.Buffer videoFrameBuffer) {
-    yuvFormat.fillBuffer(buffer, videoFrameBuffer);
+    yuvFormat.fillBuffer(buffer, videoFrameBuffer, stride, sliceHeight);
   }
 
   /**
@@ -697,24 +722,39 @@ class HardwareVideoEncoder implements VideoEncoder {
   private enum YuvFormat {
     I420 {
       @Override
-      void fillBuffer(ByteBuffer dstBuffer, VideoFrame.Buffer srcBuffer) {
+      void fillBuffer(
+          ByteBuffer dstBuffer, VideoFrame.Buffer srcBuffer, int dstStrideY, int dstSliceHeightY) {
+        /*
+         * According to the docs in Android MediaCodec, the stride of the U and V planes can be
+         * calculated based on the color format, though it is generally undefined and depends on the
+         * device and release.
+         * <p/> Assuming the width and height, dstStrideY and dstSliceHeightY are
+         * even, it works fine when we define the stride and slice-height of the dst U/V plane to be
+         * half of the dst Y plane.
+         */
+        int dstStrideU = dstStrideY / 2;
+        int dstSliceHeight = dstSliceHeightY / 2;
         VideoFrame.I420Buffer i420 = srcBuffer.toI420();
         YuvHelper.I420Copy(i420.getDataY(), i420.getStrideY(), i420.getDataU(), i420.getStrideU(),
-            i420.getDataV(), i420.getStrideV(), dstBuffer, i420.getWidth(), i420.getHeight());
+            i420.getDataV(), i420.getStrideV(), dstBuffer, i420.getWidth(), i420.getHeight(),
+            dstStrideY, dstSliceHeightY, dstStrideU, dstSliceHeight);
         i420.release();
       }
     },
     NV12 {
       @Override
-      void fillBuffer(ByteBuffer dstBuffer, VideoFrame.Buffer srcBuffer) {
+      void fillBuffer(
+          ByteBuffer dstBuffer, VideoFrame.Buffer srcBuffer, int dstStrideY, int dstSliceHeightY) {
         VideoFrame.I420Buffer i420 = srcBuffer.toI420();
         YuvHelper.I420ToNV12(i420.getDataY(), i420.getStrideY(), i420.getDataU(), i420.getStrideU(),
-            i420.getDataV(), i420.getStrideV(), dstBuffer, i420.getWidth(), i420.getHeight());
+            i420.getDataV(), i420.getStrideV(), dstBuffer, i420.getWidth(), i420.getHeight(),
+            dstStrideY, dstSliceHeightY);
         i420.release();
       }
     };
 
-    abstract void fillBuffer(ByteBuffer dstBuffer, VideoFrame.Buffer srcBuffer);
+    abstract void fillBuffer(
+        ByteBuffer dstBuffer, VideoFrame.Buffer srcBuffer, int dstStrideY, int dstSliceHeightY);
 
     static YuvFormat valueOf(int colorFormat) {
       switch (colorFormat) {

sdk/android/src/java/org/webrtc/MediaCodecWrapper.java

@@ -41,6 +41,8 @@ interface MediaCodecWrapper {
 
   void releaseOutputBuffer(int index, boolean render);
 
+  MediaFormat getInputFormat();
+
   MediaFormat getOutputFormat();
 
   ByteBuffer[] getInputBuffers();

sdk/android/src/java/org/webrtc/MediaCodecWrapperFactoryImpl.java

@@ -78,6 +78,11 @@ class MediaCodecWrapperFactoryImpl implements MediaCodecWrapperFactory {
       mediaCodec.releaseOutputBuffer(index, render);
     }
 
+    @Override
+    public MediaFormat getInputFormat() {
+      return mediaCodec.getInputFormat();
+    }
+
     @Override
     public MediaFormat getOutputFormat() {
       return mediaCodec.getOutputFormat();