zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Require 16x16 alignment when using HardwareVideoEncoder for encoding.

Browse Source 
It seems the Android CTS tests only verify that 16x16 aligned resolutions
are supported.

This change checks the validity of input frame's size when initialing
or encoding processes are about to start using H/W MediaCodec.

This change has additional APIs to retrieve
|requested_resolution_alignment| and |apply_alignment_to_all_simulcast_layers|
from JAVA VideoEncoder class and its inherited classes. HardwareVideoEncoder
using MediaCodec has values of 16 and true for above variables.

Bug: webrtc:13089
Change-Id: I0c4ebf94eb36da29c2e384a3edf85b82e779b7f9
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/229460
Reviewed-by: Sergey Silkin <ssilkin@webrtc.org>
Reviewed-by: Åsa Persson <asapersson@webrtc.org>
Commit-Queue: Sergey Silkin <ssilkin@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#35169}
This commit is contained in:
Jiwon Jung
2021-10-08 16:41:01 +09:00
committed by WebRTC LUCI CQ
parent 3179fb2931
commit 5a79d28eba
8 changed files with 178 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
modules/video_coding/codecs/test/videocodec_test_mediacodec.cc
View File

@ -120,7 +120,7 @@ TEST(VideoCodecTestMediaCodec, ForemanMixedRes100kbpsVp8H264) {
const std::vector<std::string> codecs = {cricket::kVp8CodecName,
cricket::kH264CodecName};
const std::vector<std::tuple<int, int>> resolutions = {
{128, 96}, {160, 120}, {176, 144}, {240, 136}, {320, 240}, {480, 272}};
{128, 96}, {176, 144}, {320, 240}, {480, 272}};
const std::vector<RateProfile> rate_profiles = {
{100, kForemanFramerateFps, 0}};
const std::vector<QualityThresholds> quality_thresholds = {

39
sdk/android/api/org/webrtc/VideoEncoder.java
View File

@ -258,6 +258,39 @@ public interface VideoEncoder {
}
}
/**
* Metadata about the Encoder.
*/
public class EncoderInfo {
/**
* The width and height of the incoming video frames should be divisible by
* |requested_resolution_alignment|
*/
public final int requestedResolutionAlignment;
/**
* Same as above but if true, each simulcast layer should also be divisible by
* |requested_resolution_alignment|.
*/
public final boolean applyAlignmentToAllSimulcastLayers;
public EncoderInfo(
int requestedResolutionAlignment, boolean applyAlignmentToAllSimulcastLayers) {
this.requestedResolutionAlignment = requestedResolutionAlignment;
this.applyAlignmentToAllSimulcastLayers = applyAlignmentToAllSimulcastLayers;
}
@CalledByNative("EncoderInfo")
public int getRequestedResolutionAlignment() {
return requestedResolutionAlignment;
}
@CalledByNative("EncoderInfo")
public boolean getApplyAlignmentToAllSimulcastLayers() {
return applyAlignmentToAllSimulcastLayers;
}
}
public interface Callback {
/**
* Old encoders assume that the byte buffer held by `frame` is not accessed after the call to
@ -343,4 +376,10 @@ public interface VideoEncoder {
* called from arbitrary thread.
*/
@CalledByNative String getImplementationName();
@CalledByNative
default EncoderInfo getEncoderInfo() {
return new EncoderInfo(
/* requestedResolutionAlignment= */ 1, /* applyAlignmentToAllSimulcastLayers= */ false);
}
}

24
sdk/android/instrumentationtests/src/org/webrtc/AndroidVideoDecoderInstrumentationTest.java
View File

@ -73,14 +73,17 @@ public final class AndroidVideoDecoderInstrumentationTest {
private static final boolean ENABLE_INTEL_VP8_ENCODER = true;
private static final boolean ENABLE_H264_HIGH_PROFILE = true;
private static final VideoEncoder.Settings ENCODER_SETTINGS =
new VideoEncoder.Settings(1 /* core */, TEST_FRAME_WIDTH, TEST_FRAME_HEIGHT, 300 /* kbps */,
30 /* fps */, 1 /* numberOfSimulcastStreams */, true /* automaticResizeOn */,
/* capabilities= */ new VideoEncoder.Capabilities(false /* lossNotification */));
private static final VideoEncoder.Settings ENCODER_SETTINGS = new VideoEncoder.Settings(
1 /* core */,
getAlignedNumber(TEST_FRAME_WIDTH, HardwareVideoEncoderTest.getPixelAlignmentRequired()),
getAlignedNumber(TEST_FRAME_HEIGHT, HardwareVideoEncoderTest.getPixelAlignmentRequired()),
300 /* kbps */, 30 /* fps */, 1 /* numberOfSimulcastStreams */, true /* automaticResizeOn */,
/* capabilities= */ new VideoEncoder.Capabilities(false /* lossNotification */));
private static final int DECODE_TIMEOUT_MS = 1000;
private static final VideoDecoder.Settings SETTINGS =
new VideoDecoder.Settings(1 /* core */, TEST_FRAME_WIDTH, TEST_FRAME_HEIGHT);
private static final VideoDecoder.Settings SETTINGS = new VideoDecoder.Settings(1 /* core */,
getAlignedNumber(TEST_FRAME_WIDTH, HardwareVideoEncoderTest.getPixelAlignmentRequired()),
getAlignedNumber(TEST_FRAME_HEIGHT, HardwareVideoEncoderTest.getPixelAlignmentRequired()));
private static class MockDecodeCallback implements VideoDecoder.Callback {
private BlockingQueue<VideoFrame> frameQueue = new LinkedBlockingQueue<>();
@ -116,7 +119,10 @@ public final class AndroidVideoDecoderInstrumentationTest {
private static VideoFrame.I420Buffer[] generateTestFrames() {
VideoFrame.I420Buffer[] result = new VideoFrame.I420Buffer[TEST_FRAME_COUNT];
for (int i = 0; i < TEST_FRAME_COUNT; i++) {
result[i] = JavaI420Buffer.allocate(TEST_FRAME_WIDTH, TEST_FRAME_HEIGHT);
result[i] = JavaI420Buffer.allocate(
getAlignedNumber(TEST_FRAME_WIDTH, HardwareVideoEncoderTest.getPixelAlignmentRequired()),
getAlignedNumber(
TEST_FRAME_HEIGHT, HardwareVideoEncoderTest.getPixelAlignmentRequired()));
// TODO(sakal): Generate content for the test frames.
}
return result;
@ -156,6 +162,10 @@ public final class AndroidVideoDecoderInstrumentationTest {
assertEquals(VideoCodecStatus.OK, encoder.release());
}
private static int getAlignedNumber(int number, int alignment) {
return (number / alignment) * alignment;
}
@Before
public void setUp() {
NativeLibrary.initialize(new NativeLibrary.DefaultLoader(), TestConstants.NATIVE_LIBRARY);

70
sdk/android/instrumentationtests/src/org/webrtc/HardwareVideoEncoderTest.java
View File

@ -11,6 +11,7 @@
package org.webrtc;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@ -75,6 +76,8 @@ public class HardwareVideoEncoderTest {
private static final int NUM_TEST_FRAMES = 10;
private static final int NUM_ENCODE_TRIES = 100;
private static final int ENCODE_RETRY_SLEEP_MS = 1;
private static final int PIXEL_ALIGNMENT_REQUIRED = 16;
private static final boolean APPLY_ALIGNMENT_TO_ALL_SIMULCAST_LAYERS = false;
// # Mock classes
/**
@ -322,7 +325,7 @@ public class HardwareVideoEncoderTest {
return useTextures ? generateTextureFrame(width, height) : generateI420Frame(width, height);
}
static void testEncodeFrame(
static VideoCodecStatus testEncodeFrame(
VideoEncoder encoder, VideoFrame frame, VideoEncoder.EncodeInfo info) {
int numTries = 0;
@ -332,8 +335,10 @@ public class HardwareVideoEncoderTest {
final VideoCodecStatus returnValue = encoder.encode(frame, info);
switch (returnValue) {
case OK:
return; // Success
case OK: // Success
// Fall through
case ERR_SIZE: // Wrong size
return returnValue;
case NO_OUTPUT:
if (numTries >= NUM_ENCODE_TRIES) {
fail("encoder.encode keeps returning NO_OUTPUT");
@ -350,6 +355,14 @@ public class HardwareVideoEncoderTest {
}
}
private static int getAlignedNumber(int number, int alignment) {
return (number / alignment) * alignment;
}
public static int getPixelAlignmentRequired() {
return PIXEL_ALIGNMENT_REQUIRED;
}
// # Tests
@Before
public void setUp() {
@ -446,11 +459,60 @@ public class HardwareVideoEncoderTest {
callback.assertFrameEncoded(frame);
frame.release();
frame = generateFrame(SETTINGS.width / 4, SETTINGS.height / 4);
// Android MediaCodec only guarantees of proper operation with 16-pixel-aligned input frame.
// Force the size of input frame with the greatest multiple of 16 below the original size.
frame = generateFrame(getAlignedNumber(SETTINGS.width / 4, PIXEL_ALIGNMENT_REQUIRED),
getAlignedNumber(SETTINGS.height / 4, PIXEL_ALIGNMENT_REQUIRED));
testEncodeFrame(encoder, frame, info);
callback.assertFrameEncoded(frame);
frame.release();
assertEquals(VideoCodecStatus.OK, encoder.release());
}
@Test
@SmallTest
public void testEncodeAlignmentCheck() {
VideoEncoder encoder = createEncoder();
org.webrtc.HardwareVideoEncoderTest.MockEncoderCallback callback =
new org.webrtc.HardwareVideoEncoderTest.MockEncoderCallback();
assertEquals(VideoCodecStatus.OK, encoder.initEncode(SETTINGS, callback));
VideoFrame frame;
VideoEncoder.EncodeInfo info = new VideoEncoder.EncodeInfo(
new EncodedImage.FrameType[] {EncodedImage.FrameType.VideoFrameDelta});
frame = generateFrame(SETTINGS.width / 2, SETTINGS.height / 2);
assertEquals(VideoCodecStatus.OK, testEncodeFrame(encoder, frame, info));
frame.release();
// Android MediaCodec only guarantees of proper operation with 16-pixel-aligned input frame.
// Following input frame with non-aligned size would return ERR_SIZE.
frame = generateFrame(SETTINGS.width / 4, SETTINGS.height / 4);
assertNotEquals(VideoCodecStatus.OK, testEncodeFrame(encoder, frame, info));
frame.release();
// Since our encoder has returned with an error, we reinitialize the encoder.
assertEquals(VideoCodecStatus.OK, encoder.release());
assertEquals(VideoCodecStatus.OK, encoder.initEncode(SETTINGS, callback));
frame = generateFrame(getAlignedNumber(SETTINGS.width / 4, PIXEL_ALIGNMENT_REQUIRED),
getAlignedNumber(SETTINGS.height / 4, PIXEL_ALIGNMENT_REQUIRED));
assertEquals(VideoCodecStatus.OK, testEncodeFrame(encoder, frame, info));
frame.release();
assertEquals(VideoCodecStatus.OK, encoder.release());
}
@Test
@SmallTest
public void testGetEncoderInfo() {
VideoEncoder encoder = createEncoder();
assertEquals(VideoCodecStatus.OK, encoder.initEncode(SETTINGS, null));
VideoEncoder.EncoderInfo info = encoder.getEncoderInfo();
assertEquals(PIXEL_ALIGNMENT_REQUIRED, info.getRequestedResolutionAlignment());
assertEquals(
APPLY_ALIGNMENT_TO_ALL_SIMULCAST_LAYERS, info.getApplyAlignmentToAllSimulcastLayers());
assertEquals(VideoCodecStatus.OK, encoder.release());
}
}

2
sdk/android/native_unittests/org/webrtc/FakeVideoEncoder.java
View File

@ -10,6 +10,8 @@
package org.webrtc;
import org.webrtc.VideoEncoder;
/**
* An implementation of VideoEncoder that is used for testing of functionalities of
* VideoEncoderWrapper.

25
sdk/android/src/java/org/webrtc/HardwareVideoEncoder.java
View File

@ -54,6 +54,9 @@ class HardwareVideoEncoder implements VideoEncoder {
private static final int MEDIA_CODEC_RELEASE_TIMEOUT_MS = 5000;
private static final int DEQUEUE_OUTPUT_BUFFER_TIMEOUT_US = 100000;
// Size of the input frames should be multiple of 16 for the H/W encoder.
private static final int REQUIRED_RESOLUTION_ALIGNMENT = 16;
/**
* Keeps track of the number of output buffers that have been passed down the pipeline and not yet
* released. We need to wait for this to go down to zero before operations invalidating the output
@ -207,6 +210,12 @@ class HardwareVideoEncoder implements VideoEncoder {
this.callback = callback;
automaticResizeOn = settings.automaticResizeOn;
if (settings.width % REQUIRED_RESOLUTION_ALIGNMENT != 0
|| settings.height % REQUIRED_RESOLUTION_ALIGNMENT != 0) {
Logging.e(TAG, "MediaCodec is only tested with resolutions that are 16x16 aligned.");
return VideoCodecStatus.ERR_SIZE;
}
this.width = settings.width;
this.height = settings.height;
useSurfaceMode = canUseSurface();
@ -498,12 +507,28 @@ class HardwareVideoEncoder implements VideoEncoder {
return "HWEncoder";
}
@Override
public EncoderInfo getEncoderInfo() {
// Since our MediaCodec is guaranteed to encode 16-pixel-aligned frames only, we set alignment
// value to be 16. Additionally, this encoder produces a single stream. So it should not require
// alignment for all layers.
return new EncoderInfo(
/* requestedResolutionAlignment= */ REQUIRED_RESOLUTION_ALIGNMENT,
/* applyAlignmentToAllSimulcastLayers= */ false);
}
private VideoCodecStatus resetCodec(int newWidth, int newHeight, boolean newUseSurfaceMode) {
encodeThreadChecker.checkIsOnValidThread();
VideoCodecStatus status = release();
if (status != VideoCodecStatus.OK) {
return status;
}
if (newWidth % REQUIRED_RESOLUTION_ALIGNMENT != 0
|| newHeight % REQUIRED_RESOLUTION_ALIGNMENT != 0) {
Logging.e(TAG, "MediaCodec is only tested with resolutions that are 16x16 aligned.");
return VideoCodecStatus.ERR_SIZE;
}
width = newWidth;
height = newHeight;
useSurfaceMode = newUseSurfaceMode;

26
sdk/android/src/jni/video_encoder_wrapper.cc
View File

@ -113,6 +113,12 @@ void VideoEncoderWrapper::UpdateEncoderInfo(JNIEnv* jni) {
encoder_info_.resolution_bitrate_limits = JavaToNativeResolutionBitrateLimits(
jni, Java_VideoEncoder_getResolutionBitrateLimits(jni, encoder_));
EncoderInfo info = GetEncoderInfoInternal(jni);
encoder_info_.requested_resolution_alignment =
info.requested_resolution_alignment;
encoder_info_.apply_alignment_to_all_simulcast_layers =
info.apply_alignment_to_all_simulcast_layers;
}
int32_t VideoEncoderWrapper::RegisterEncodeCompleteCallback(
@ -230,6 +236,26 @@ VideoEncoderWrapper::GetScalingSettingsInternal(JNIEnv* jni) const {
}
}
VideoEncoder::EncoderInfo VideoEncoderWrapper::GetEncoderInfoInternal(
JNIEnv* jni) const {
ScopedJavaLocalRef<jobject> j_encoder_info =
Java_VideoEncoder_getEncoderInfo(jni, encoder_);
jint requested_resolution_alignment =
Java_EncoderInfo_getRequestedResolutionAlignment(jni, j_encoder_info);
jboolean apply_alignment_to_all_simulcast_layers =
Java_EncoderInfo_getApplyAlignmentToAllSimulcastLayers(jni,
j_encoder_info);
VideoEncoder::EncoderInfo info;
info.requested_resolution_alignment = requested_resolution_alignment;
info.apply_alignment_to_all_simulcast_layers =
apply_alignment_to_all_simulcast_layers;
return info;
}
void VideoEncoderWrapper::OnEncodedFrame(
JNIEnv* jni,
const JavaRef<jobject>& j_encoded_image) {

2
sdk/android/src/jni/video_encoder_wrapper.h
View File

@ -87,6 +87,8 @@ class VideoEncoderWrapper : public VideoEncoder {
std::vector<ResolutionBitrateLimits> GetResolutionBitrateLimits(
JNIEnv* jni) const;
VideoEncoder::EncoderInfo GetEncoderInfoInternal(JNIEnv* jni) const;
const ScopedJavaGlobalRef<jobject> encoder_;
const ScopedJavaGlobalRef<jclass> int_array_class_;