Require 16x16 alignment when using HardwareVideoEncoder for encoding.

It seems the Android CTS tests only verify that 16x16 aligned resolutions
are supported.

This change checks the validity of input frame's size when initialing
or encoding processes are about to start using H/W MediaCodec.

This change has additional APIs to retrieve
|requested_resolution_alignment| and |apply_alignment_to_all_simulcast_layers|
from JAVA VideoEncoder class and its inherited classes. HardwareVideoEncoder
using MediaCodec has values of 16 and true for above variables.

Bug: webrtc:13089
Change-Id: I0c4ebf94eb36da29c2e384a3edf85b82e779b7f9
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/229460
Reviewed-by: Sergey Silkin <ssilkin@webrtc.org>
Reviewed-by: Åsa Persson <asapersson@webrtc.org>
Commit-Queue: Sergey Silkin <ssilkin@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#35169}

modules/video_coding/codecs/test/videocodec_test_mediacodec.cc

@@ -120,7 +120,7 @@ TEST(VideoCodecTestMediaCodec, ForemanMixedRes100kbpsVp8H264) {
   const std::vector<std::string> codecs = {cricket::kVp8CodecName,
                                            cricket::kH264CodecName};
   const std::vector<std::tuple<int, int>> resolutions = {
-      {128, 96}, {160, 120}, {176, 144}, {240, 136}, {320, 240}, {480, 272}};
+      {128, 96}, {176, 144}, {320, 240}, {480, 272}};
   const std::vector<RateProfile> rate_profiles = {
       {100, kForemanFramerateFps, 0}};
   const std::vector<QualityThresholds> quality_thresholds = {

sdk/android/api/org/webrtc/VideoEncoder.java

@@ -258,6 +258,39 @@ public interface VideoEncoder {
     }
   }
 
+  /**
+   * Metadata about the Encoder.
+   */
+  public class EncoderInfo {
+    /**
+     * The width and height of the incoming video frames should be divisible by
+     * |requested_resolution_alignment|
+     */
+    public final int requestedResolutionAlignment;
+
+    /**
+     * Same as above but if true, each simulcast layer should also be divisible by
+     * |requested_resolution_alignment|.
+     */
+    public final boolean applyAlignmentToAllSimulcastLayers;
+
+    public EncoderInfo(
+        int requestedResolutionAlignment, boolean applyAlignmentToAllSimulcastLayers) {
+      this.requestedResolutionAlignment = requestedResolutionAlignment;
+      this.applyAlignmentToAllSimulcastLayers = applyAlignmentToAllSimulcastLayers;
+    }
+
+    @CalledByNative("EncoderInfo")
+    public int getRequestedResolutionAlignment() {
+      return requestedResolutionAlignment;
+    }
+
+    @CalledByNative("EncoderInfo")
+    public boolean getApplyAlignmentToAllSimulcastLayers() {
+      return applyAlignmentToAllSimulcastLayers;
+    }
+  }
+
   public interface Callback {
     /**
      * Old encoders assume that the byte buffer held by `frame` is not accessed after the call to
@@ -343,4 +376,10 @@ public interface VideoEncoder {
    * called from arbitrary thread.
    */
   @CalledByNative String getImplementationName();
+
+  @CalledByNative
+  default EncoderInfo getEncoderInfo() {
+    return new EncoderInfo(
+        /* requestedResolutionAlignment= */ 1, /* applyAlignmentToAllSimulcastLayers= */ false);
+  }
 }

sdk/android/instrumentationtests/src/org/webrtc/AndroidVideoDecoderInstrumentationTest.java

@@ -73,14 +73,17 @@ public final class AndroidVideoDecoderInstrumentationTest {
 
   private static final boolean ENABLE_INTEL_VP8_ENCODER = true;
   private static final boolean ENABLE_H264_HIGH_PROFILE = true;
-  private static final VideoEncoder.Settings ENCODER_SETTINGS =
+  private static final VideoEncoder.Settings ENCODER_SETTINGS = new VideoEncoder.Settings(
-      new VideoEncoder.Settings(1 /* core */, TEST_FRAME_WIDTH, TEST_FRAME_HEIGHT, 300 /* kbps */,
+      1 /* core */,
-          30 /* fps */, 1 /* numberOfSimulcastStreams */, true /* automaticResizeOn */,
+      getAlignedNumber(TEST_FRAME_WIDTH, HardwareVideoEncoderTest.getPixelAlignmentRequired()),
+      getAlignedNumber(TEST_FRAME_HEIGHT, HardwareVideoEncoderTest.getPixelAlignmentRequired()),
+      300 /* kbps */, 30 /* fps */, 1 /* numberOfSimulcastStreams */, true /* automaticResizeOn */,
       /* capabilities= */ new VideoEncoder.Capabilities(false /* lossNotification */));
 
   private static final int DECODE_TIMEOUT_MS = 1000;
-  private static final VideoDecoder.Settings SETTINGS =
+  private static final VideoDecoder.Settings SETTINGS = new VideoDecoder.Settings(1 /* core */,
-      new VideoDecoder.Settings(1 /* core */, TEST_FRAME_WIDTH, TEST_FRAME_HEIGHT);
+      getAlignedNumber(TEST_FRAME_WIDTH, HardwareVideoEncoderTest.getPixelAlignmentRequired()),
+      getAlignedNumber(TEST_FRAME_HEIGHT, HardwareVideoEncoderTest.getPixelAlignmentRequired()));
 
   private static class MockDecodeCallback implements VideoDecoder.Callback {
     private BlockingQueue<VideoFrame> frameQueue = new LinkedBlockingQueue<>();
@@ -116,7 +119,10 @@ public final class AndroidVideoDecoderInstrumentationTest {
   private static VideoFrame.I420Buffer[] generateTestFrames() {
     VideoFrame.I420Buffer[] result = new VideoFrame.I420Buffer[TEST_FRAME_COUNT];
     for (int i = 0; i < TEST_FRAME_COUNT; i++) {
-      result[i] = JavaI420Buffer.allocate(TEST_FRAME_WIDTH, TEST_FRAME_HEIGHT);
+      result[i] = JavaI420Buffer.allocate(
+          getAlignedNumber(TEST_FRAME_WIDTH, HardwareVideoEncoderTest.getPixelAlignmentRequired()),
+          getAlignedNumber(
+              TEST_FRAME_HEIGHT, HardwareVideoEncoderTest.getPixelAlignmentRequired()));
       // TODO(sakal): Generate content for the test frames.
     }
     return result;
@@ -156,6 +162,10 @@ public final class AndroidVideoDecoderInstrumentationTest {
     assertEquals(VideoCodecStatus.OK, encoder.release());
   }
 
+  private static int getAlignedNumber(int number, int alignment) {
+    return (number / alignment) * alignment;
+  }
+
   @Before
   public void setUp() {
     NativeLibrary.initialize(new NativeLibrary.DefaultLoader(), TestConstants.NATIVE_LIBRARY);

sdk/android/instrumentationtests/src/org/webrtc/HardwareVideoEncoderTest.java

@@ -11,6 +11,7 @@
 package org.webrtc;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
@@ -75,6 +76,8 @@ public class HardwareVideoEncoderTest {
   private static final int NUM_TEST_FRAMES = 10;
   private static final int NUM_ENCODE_TRIES = 100;
   private static final int ENCODE_RETRY_SLEEP_MS = 1;
+  private static final int PIXEL_ALIGNMENT_REQUIRED = 16;
+  private static final boolean APPLY_ALIGNMENT_TO_ALL_SIMULCAST_LAYERS = false;
 
   // # Mock classes
   /**
@@ -322,7 +325,7 @@ public class HardwareVideoEncoderTest {
     return useTextures ? generateTextureFrame(width, height) : generateI420Frame(width, height);
   }
 
-  static void testEncodeFrame(
+  static VideoCodecStatus testEncodeFrame(
       VideoEncoder encoder, VideoFrame frame, VideoEncoder.EncodeInfo info) {
     int numTries = 0;
 
@@ -332,8 +335,10 @@ public class HardwareVideoEncoderTest {
 
       final VideoCodecStatus returnValue = encoder.encode(frame, info);
       switch (returnValue) {
-        case OK:
+        case OK: // Success
-          return; // Success
+                 // Fall through
+        case ERR_SIZE: // Wrong size
+          return returnValue;
         case NO_OUTPUT:
           if (numTries >= NUM_ENCODE_TRIES) {
             fail("encoder.encode keeps returning NO_OUTPUT");
@@ -350,6 +355,14 @@ public class HardwareVideoEncoderTest {
     }
   }
 
+  private static int getAlignedNumber(int number, int alignment) {
+    return (number / alignment) * alignment;
+  }
+
+  public static int getPixelAlignmentRequired() {
+    return PIXEL_ALIGNMENT_REQUIRED;
+  }
+
   // # Tests
   @Before
   public void setUp() {
@@ -446,11 +459,60 @@ public class HardwareVideoEncoderTest {
     callback.assertFrameEncoded(frame);
     frame.release();
 
-    frame = generateFrame(SETTINGS.width / 4, SETTINGS.height / 4);
+    // Android MediaCodec only guarantees of proper operation with 16-pixel-aligned input frame.
+    // Force the size of input frame with the greatest multiple of 16 below the original size.
+    frame = generateFrame(getAlignedNumber(SETTINGS.width / 4, PIXEL_ALIGNMENT_REQUIRED),
+        getAlignedNumber(SETTINGS.height / 4, PIXEL_ALIGNMENT_REQUIRED));
     testEncodeFrame(encoder, frame, info);
     callback.assertFrameEncoded(frame);
     frame.release();
 
     assertEquals(VideoCodecStatus.OK, encoder.release());
   }
+
+  @Test
+  @SmallTest
+  public void testEncodeAlignmentCheck() {
+    VideoEncoder encoder = createEncoder();
+    org.webrtc.HardwareVideoEncoderTest.MockEncoderCallback callback =
+        new org.webrtc.HardwareVideoEncoderTest.MockEncoderCallback();
+    assertEquals(VideoCodecStatus.OK, encoder.initEncode(SETTINGS, callback));
+
+    VideoFrame frame;
+    VideoEncoder.EncodeInfo info = new VideoEncoder.EncodeInfo(
+        new EncodedImage.FrameType[] {EncodedImage.FrameType.VideoFrameDelta});
+
+    frame = generateFrame(SETTINGS.width / 2, SETTINGS.height / 2);
+    assertEquals(VideoCodecStatus.OK, testEncodeFrame(encoder, frame, info));
+    frame.release();
+
+    // Android MediaCodec only guarantees of proper operation with 16-pixel-aligned input frame.
+    // Following input frame with non-aligned size would return ERR_SIZE.
+    frame = generateFrame(SETTINGS.width / 4, SETTINGS.height / 4);
+    assertNotEquals(VideoCodecStatus.OK, testEncodeFrame(encoder, frame, info));
+    frame.release();
+
+    // Since our encoder has returned with an error, we reinitialize the encoder.
+    assertEquals(VideoCodecStatus.OK, encoder.release());
+    assertEquals(VideoCodecStatus.OK, encoder.initEncode(SETTINGS, callback));
+
+    frame = generateFrame(getAlignedNumber(SETTINGS.width / 4, PIXEL_ALIGNMENT_REQUIRED),
+        getAlignedNumber(SETTINGS.height / 4, PIXEL_ALIGNMENT_REQUIRED));
+    assertEquals(VideoCodecStatus.OK, testEncodeFrame(encoder, frame, info));
+    frame.release();
+
+    assertEquals(VideoCodecStatus.OK, encoder.release());
+  }
+
+  @Test
+  @SmallTest
+  public void testGetEncoderInfo() {
+    VideoEncoder encoder = createEncoder();
+    assertEquals(VideoCodecStatus.OK, encoder.initEncode(SETTINGS, null));
+    VideoEncoder.EncoderInfo info = encoder.getEncoderInfo();
+    assertEquals(PIXEL_ALIGNMENT_REQUIRED, info.getRequestedResolutionAlignment());
+    assertEquals(
+        APPLY_ALIGNMENT_TO_ALL_SIMULCAST_LAYERS, info.getApplyAlignmentToAllSimulcastLayers());
+    assertEquals(VideoCodecStatus.OK, encoder.release());
+  }
 }

sdk/android/native_unittests/org/webrtc/FakeVideoEncoder.java

@@ -10,6 +10,8 @@
 
 package org.webrtc;
 
+import org.webrtc.VideoEncoder;
+
 /**
  * An implementation of VideoEncoder that is used for testing of functionalities of
  * VideoEncoderWrapper.

sdk/android/src/java/org/webrtc/HardwareVideoEncoder.java

@@ -54,6 +54,9 @@ class HardwareVideoEncoder implements VideoEncoder {
   private static final int MEDIA_CODEC_RELEASE_TIMEOUT_MS = 5000;
   private static final int DEQUEUE_OUTPUT_BUFFER_TIMEOUT_US = 100000;
 
+  // Size of the input frames should be multiple of 16 for the H/W encoder.
+  private static final int REQUIRED_RESOLUTION_ALIGNMENT = 16;
+
   /**
    * Keeps track of the number of output buffers that have been passed down the pipeline and not yet
    * released. We need to wait for this to go down to zero before operations invalidating the output
@@ -207,6 +210,12 @@ class HardwareVideoEncoder implements VideoEncoder {
 
     this.callback = callback;
     automaticResizeOn = settings.automaticResizeOn;
+
+    if (settings.width % REQUIRED_RESOLUTION_ALIGNMENT != 0
+        || settings.height % REQUIRED_RESOLUTION_ALIGNMENT != 0) {
+      Logging.e(TAG, "MediaCodec is only tested with resolutions that are 16x16 aligned.");
+      return VideoCodecStatus.ERR_SIZE;
+    }
     this.width = settings.width;
     this.height = settings.height;
     useSurfaceMode = canUseSurface();
@@ -498,12 +507,28 @@ class HardwareVideoEncoder implements VideoEncoder {
     return "HWEncoder";
   }
 
+  @Override
+  public EncoderInfo getEncoderInfo() {
+    // Since our MediaCodec is guaranteed to encode 16-pixel-aligned frames only, we set alignment
+    // value to be 16. Additionally, this encoder produces a single stream. So it should not require
+    // alignment for all layers.
+    return new EncoderInfo(
+        /* requestedResolutionAlignment= */ REQUIRED_RESOLUTION_ALIGNMENT,
+        /* applyAlignmentToAllSimulcastLayers= */ false);
+  }
+
   private VideoCodecStatus resetCodec(int newWidth, int newHeight, boolean newUseSurfaceMode) {
     encodeThreadChecker.checkIsOnValidThread();
     VideoCodecStatus status = release();
     if (status != VideoCodecStatus.OK) {
       return status;
     }
+
+    if (newWidth % REQUIRED_RESOLUTION_ALIGNMENT != 0
+        || newHeight % REQUIRED_RESOLUTION_ALIGNMENT != 0) {
+      Logging.e(TAG, "MediaCodec is only tested with resolutions that are 16x16 aligned.");
+      return VideoCodecStatus.ERR_SIZE;
+    }
     width = newWidth;
     height = newHeight;
     useSurfaceMode = newUseSurfaceMode;

sdk/android/src/jni/video_encoder_wrapper.cc

@@ -113,6 +113,12 @@ void VideoEncoderWrapper::UpdateEncoderInfo(JNIEnv* jni) {
 
   encoder_info_.resolution_bitrate_limits = JavaToNativeResolutionBitrateLimits(
       jni, Java_VideoEncoder_getResolutionBitrateLimits(jni, encoder_));
+
+  EncoderInfo info = GetEncoderInfoInternal(jni);
+  encoder_info_.requested_resolution_alignment =
+      info.requested_resolution_alignment;
+  encoder_info_.apply_alignment_to_all_simulcast_layers =
+      info.apply_alignment_to_all_simulcast_layers;
 }
 
 int32_t VideoEncoderWrapper::RegisterEncodeCompleteCallback(
@@ -230,6 +236,26 @@ VideoEncoderWrapper::GetScalingSettingsInternal(JNIEnv* jni) const {
   }
 }
 
+VideoEncoder::EncoderInfo VideoEncoderWrapper::GetEncoderInfoInternal(
+    JNIEnv* jni) const {
+  ScopedJavaLocalRef<jobject> j_encoder_info =
+      Java_VideoEncoder_getEncoderInfo(jni, encoder_);
+
+  jint requested_resolution_alignment =
+      Java_EncoderInfo_getRequestedResolutionAlignment(jni, j_encoder_info);
+
+  jboolean apply_alignment_to_all_simulcast_layers =
+      Java_EncoderInfo_getApplyAlignmentToAllSimulcastLayers(jni,
+                                                             j_encoder_info);
+
+  VideoEncoder::EncoderInfo info;
+  info.requested_resolution_alignment = requested_resolution_alignment;
+  info.apply_alignment_to_all_simulcast_layers =
+      apply_alignment_to_all_simulcast_layers;
+
+  return info;
+}
+
 void VideoEncoderWrapper::OnEncodedFrame(
     JNIEnv* jni,
     const JavaRef<jobject>& j_encoded_image) {

sdk/android/src/jni/video_encoder_wrapper.h

@@ -87,6 +87,8 @@ class VideoEncoderWrapper : public VideoEncoder {
   std::vector<ResolutionBitrateLimits> GetResolutionBitrateLimits(
       JNIEnv* jni) const;
 
+  VideoEncoder::EncoderInfo GetEncoderInfoInternal(JNIEnv* jni) const;
+
   const ScopedJavaGlobalRef<jobject> encoder_;
   const ScopedJavaGlobalRef<jclass> int_array_class_;