/* * Copyright 2018 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source * tree. An additional intellectual property rights grant can be found * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ package org.webrtc; import static com.google.common.truth.Truth.assertThat; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyInt; import static org.mockito.Matchers.anyLong; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static org.robolectric.Shadows.shadowOf; import android.graphics.Matrix; import android.graphics.SurfaceTexture; import android.media.MediaCodecInfo.CodecCapabilities; import android.media.MediaFormat; import android.os.Handler; import java.nio.ByteBuffer; import org.chromium.testing.local.LocalRobolectricTestRunner; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.ArgumentCaptor; import org.mockito.InOrder; import org.mockito.Mock; import org.mockito.MockitoAnnotations; import org.robolectric.annotation.Config; import org.webrtc.EncodedImage.FrameType; import org.webrtc.FakeMediaCodecWrapper.State; import org.webrtc.VideoDecoder.DecodeInfo; import org.webrtc.VideoFrame.I420Buffer; import org.webrtc.VideoFrame.TextureBuffer.Type; @RunWith(LocalRobolectricTestRunner.class) @Config(manifest = Config.NONE) public class AndroidVideoDecoderTest { private static final VideoDecoder.Settings TEST_DECODER_SETTINGS = new VideoDecoder.Settings(/* numberOfCores= */ 1, /* width= */ 640, /* height= */ 480); private static final int COLOR_FORMAT = CodecCapabilities.COLOR_FormatYUV420Planar; private static final long POLL_DELAY_MS = 10; private static final long DELIVER_DECODED_IMAGE_DELAY_MS = 10; private static final byte[] ENCODED_TEST_DATA = new byte[] {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; private class TestDecoder extends AndroidVideoDecoder { private final Object deliverDecodedFrameLock = new Object(); private boolean deliverDecodedFrameDone = true; public TestDecoder(MediaCodecWrapperFactory mediaCodecFactory, String codecName, VideoCodecType codecType, int colorFormat, EglBase.Context sharedContext) { super(mediaCodecFactory, codecName, codecType, colorFormat, sharedContext); } public void waitDeliverDecodedFrame() throws InterruptedException { synchronized (deliverDecodedFrameLock) { deliverDecodedFrameDone = false; deliverDecodedFrameLock.notifyAll(); while (!deliverDecodedFrameDone) { deliverDecodedFrameLock.wait(); } } } @SuppressWarnings("WaitNotInLoop") // This method is called inside a loop. @Override protected void deliverDecodedFrame() { synchronized (deliverDecodedFrameLock) { if (deliverDecodedFrameDone) { try { deliverDecodedFrameLock.wait(DELIVER_DECODED_IMAGE_DELAY_MS); } catch (InterruptedException e) { Thread.currentThread().interrupt(); return; } } if (deliverDecodedFrameDone) { return; } super.deliverDecodedFrame(); deliverDecodedFrameDone = true; deliverDecodedFrameLock.notifyAll(); } } @Override protected SurfaceTextureHelper createSurfaceTextureHelper() { return mockSurfaceTextureHelper; } @Override protected void releaseSurface() {} @Override protected VideoFrame.I420Buffer allocateI420Buffer(int width, int height) { int chromaHeight = (height + 1) / 2; int strideUV = (width + 1) / 2; int yPos = 0; int uPos = yPos + width * height; int vPos = uPos + strideUV * chromaHeight; ByteBuffer buffer = ByteBuffer.allocateDirect(width * height + 2 * strideUV * chromaHeight); buffer.position(yPos); buffer.limit(uPos); ByteBuffer dataY = buffer.slice(); buffer.position(uPos); buffer.limit(vPos); ByteBuffer dataU = buffer.slice(); buffer.position(vPos); buffer.limit(vPos + strideUV * chromaHeight); ByteBuffer dataV = buffer.slice(); return JavaI420Buffer.wrap(width, height, dataY, width, dataU, strideUV, dataV, strideUV, /* releaseCallback= */ null); } @Override protected void copyPlane( ByteBuffer src, int srcStride, ByteBuffer dst, int dstStride, int width, int height) { for (int y = 0; y < height; y++) { for (int x = 0; x < width; x++) { dst.put(y * dstStride + x, src.get(y * srcStride + x)); } } } } private class TestDecoderBuilder { private VideoCodecType codecType = VideoCodecType.VP8; private boolean useSurface = true; public TestDecoderBuilder setCodecType(VideoCodecType codecType) { this.codecType = codecType; return this; } public TestDecoderBuilder setUseSurface(boolean useSurface) { this.useSurface = useSurface; return this; } public TestDecoder build() { return new TestDecoder((String name) -> fakeMediaCodecWrapper, /* codecName= */ "org.webrtc.testdecoder", codecType, COLOR_FORMAT, useSurface ? mockEglBaseContext : null); } } private EncodedImage createTestEncodedImage() { return EncodedImage.builder() .setBuffer(ByteBuffer.wrap(ENCODED_TEST_DATA)) .setFrameType(FrameType.VideoFrameKey) .setCompleteFrame(true) .createEncodedImage(); } @Mock private EglBase.Context mockEglBaseContext; @Mock private SurfaceTextureHelper mockSurfaceTextureHelper; @Mock private VideoDecoder.Callback mockDecoderCallback; private FakeMediaCodecWrapper fakeMediaCodecWrapper; @Before public void setUp() { MockitoAnnotations.initMocks(this); when(mockSurfaceTextureHelper.getSurfaceTexture()) .thenReturn(new SurfaceTexture(/*texName=*/0)); MediaFormat outputFormat = new MediaFormat(); // TODO(sakal): Add more details to output format as needed. fakeMediaCodecWrapper = spy(new FakeMediaCodecWrapper(outputFormat)); } @Test public void testInit() { // Set-up. AndroidVideoDecoder decoder = new TestDecoderBuilder().setCodecType(VideoCodecType.VP8).build(); // Test. assertThat(decoder.initDecode(TEST_DECODER_SETTINGS, mockDecoderCallback)) .isEqualTo(VideoCodecStatus.OK); // Verify. assertThat(fakeMediaCodecWrapper.getState()).isEqualTo(State.EXECUTING_RUNNING); MediaFormat mediaFormat = fakeMediaCodecWrapper.getConfiguredFormat(); assertThat(mediaFormat).isNotNull(); assertThat(mediaFormat.getInteger(MediaFormat.KEY_WIDTH)) .isEqualTo(TEST_DECODER_SETTINGS.width); assertThat(mediaFormat.getInteger(MediaFormat.KEY_HEIGHT)) .isEqualTo(TEST_DECODER_SETTINGS.height); assertThat(mediaFormat.getString(MediaFormat.KEY_MIME)) .isEqualTo(VideoCodecType.VP8.mimeType()); } @Test public void testRelease() { // Set-up. AndroidVideoDecoder decoder = new TestDecoderBuilder().build(); decoder.initDecode(TEST_DECODER_SETTINGS, mockDecoderCallback); // Test. assertThat(decoder.release()).isEqualTo(VideoCodecStatus.OK); // Verify. assertThat(fakeMediaCodecWrapper.getState()).isEqualTo(State.RELEASED); } @Test public void testReleaseMultipleTimes() { // Set-up. AndroidVideoDecoder decoder = new TestDecoderBuilder().build(); decoder.initDecode(TEST_DECODER_SETTINGS, mockDecoderCallback); // Test. assertThat(decoder.release()).isEqualTo(VideoCodecStatus.OK); assertThat(decoder.release()).isEqualTo(VideoCodecStatus.OK); // Verify. assertThat(fakeMediaCodecWrapper.getState()).isEqualTo(State.RELEASED); } @Test public void testDecodeQueuesData() { // Set-up. AndroidVideoDecoder decoder = new TestDecoderBuilder().build(); decoder.initDecode(TEST_DECODER_SETTINGS, mockDecoderCallback); // Test. assertThat(decoder.decode(createTestEncodedImage(), new DecodeInfo(/* isMissingFrames= */ false, /* renderTimeMs= */ 0))) .isEqualTo(VideoCodecStatus.OK); // Verify. ArgumentCaptor indexCaptor = ArgumentCaptor.forClass(Integer.class); ArgumentCaptor offsetCaptor = ArgumentCaptor.forClass(Integer.class); ArgumentCaptor sizeCaptor = ArgumentCaptor.forClass(Integer.class); verify(fakeMediaCodecWrapper) .queueInputBuffer(indexCaptor.capture(), offsetCaptor.capture(), sizeCaptor.capture(), /* presentationTimeUs= */ anyLong(), /* flags= */ eq(0)); ByteBuffer inputBuffer = fakeMediaCodecWrapper.getInputBuffers()[indexCaptor.getValue()]; CodecTestHelper.assertEqualContents( ENCODED_TEST_DATA, inputBuffer, offsetCaptor.getValue(), sizeCaptor.getValue()); } @Test public void testDeliversOutputByteBuffers() throws InterruptedException { final byte[] testOutputData = CodecTestHelper.generateRandomData( TEST_DECODER_SETTINGS.width * TEST_DECODER_SETTINGS.height * 3 / 2); final I420Buffer expectedDeliveredBuffer = CodecTestHelper.wrapI420( TEST_DECODER_SETTINGS.width, TEST_DECODER_SETTINGS.height, testOutputData); // Set-up. TestDecoder decoder = new TestDecoderBuilder().setUseSurface(/* useSurface = */ false).build(); decoder.initDecode(TEST_DECODER_SETTINGS, mockDecoderCallback); decoder.decode(createTestEncodedImage(), new DecodeInfo(/* isMissingFrames= */ false, /* renderTimeMs= */ 0)); fakeMediaCodecWrapper.addOutputData( testOutputData, /* presentationTimestampUs= */ 0, /* flags= */ 0); // Test. decoder.waitDeliverDecodedFrame(); // Verify. ArgumentCaptor videoFrameCaptor = ArgumentCaptor.forClass(VideoFrame.class); verify(mockDecoderCallback) .onDecodedFrame(videoFrameCaptor.capture(), /* decodeTimeMs= */ any(Integer.class), /* qp= */ any()); VideoFrame videoFrame = videoFrameCaptor.getValue(); assertThat(videoFrame).isNotNull(); assertThat(videoFrame.getRotatedWidth()).isEqualTo(TEST_DECODER_SETTINGS.width); assertThat(videoFrame.getRotatedHeight()).isEqualTo(TEST_DECODER_SETTINGS.height); assertThat(videoFrame.getRotation()).isEqualTo(0); I420Buffer deliveredBuffer = videoFrame.getBuffer().toI420(); assertThat(deliveredBuffer.getDataY()).isEqualTo(expectedDeliveredBuffer.getDataY()); assertThat(deliveredBuffer.getDataU()).isEqualTo(expectedDeliveredBuffer.getDataU()); assertThat(deliveredBuffer.getDataV()).isEqualTo(expectedDeliveredBuffer.getDataV()); } @Test public void testRendersOutputTexture() throws InterruptedException { // Set-up. TestDecoder decoder = new TestDecoderBuilder().build(); decoder.initDecode(TEST_DECODER_SETTINGS, mockDecoderCallback); decoder.decode(createTestEncodedImage(), new DecodeInfo(/* isMissingFrames= */ false, /* renderTimeMs= */ 0)); int bufferIndex = fakeMediaCodecWrapper.addOutputTexture(/* presentationTimestampUs= */ 0, /* flags= */ 0); // Test. decoder.waitDeliverDecodedFrame(); // Verify. verify(fakeMediaCodecWrapper).releaseOutputBuffer(bufferIndex, /* render= */ true); } @Test public void testSurfaceTextureStall_FramesDropped() throws InterruptedException { final int numFrames = 10; // Maximum number of frame the decoder can keep queued on the output side. final int maxQueuedBuffers = 3; // Set-up. TestDecoder decoder = new TestDecoderBuilder().build(); decoder.initDecode(TEST_DECODER_SETTINGS, mockDecoderCallback); // Test. int[] bufferIndices = new int[numFrames]; for (int i = 0; i < 10; i++) { decoder.decode(createTestEncodedImage(), new DecodeInfo(/* isMissingFrames= */ false, /* renderTimeMs= */ 0)); bufferIndices[i] = fakeMediaCodecWrapper.addOutputTexture(/* presentationTimestampUs= */ 0, /* flags= */ 0); decoder.waitDeliverDecodedFrame(); } // Verify. InOrder releaseOrder = inOrder(fakeMediaCodecWrapper); releaseOrder.verify(fakeMediaCodecWrapper) .releaseOutputBuffer(bufferIndices[0], /* render= */ true); for (int i = 1; i < numFrames - maxQueuedBuffers; i++) { releaseOrder.verify(fakeMediaCodecWrapper) .releaseOutputBuffer(bufferIndices[i], /* render= */ false); } } @Test public void testDeliversRenderedBuffers() throws InterruptedException { // Set-up. TestDecoder decoder = new TestDecoderBuilder().build(); decoder.initDecode(TEST_DECODER_SETTINGS, mockDecoderCallback); decoder.decode(createTestEncodedImage(), new DecodeInfo(/* isMissingFrames= */ false, /* renderTimeMs= */ 0)); fakeMediaCodecWrapper.addOutputTexture(/* presentationTimestampUs= */ 0, /* flags= */ 0); // Render the output buffer. decoder.waitDeliverDecodedFrame(); ArgumentCaptor videoSinkCaptor = ArgumentCaptor.forClass(VideoSink.class); verify(mockSurfaceTextureHelper).startListening(videoSinkCaptor.capture()); // Test. Runnable releaseCallback = mock(Runnable.class); VideoFrame.TextureBuffer outputTextureBuffer = new TextureBufferImpl(TEST_DECODER_SETTINGS.width, TEST_DECODER_SETTINGS.height, Type.OES, /* id= */ 0, /* transformMatrix= */ new Matrix(), /* toI420Handler= */ new Handler(), new YuvConverter(), releaseCallback); VideoFrame outputVideoFrame = new VideoFrame(outputTextureBuffer, /* rotation= */ 0, /* timestampNs= */ 0); videoSinkCaptor.getValue().onFrame(outputVideoFrame); outputVideoFrame.release(); // Verify. ArgumentCaptor videoFrameCaptor = ArgumentCaptor.forClass(VideoFrame.class); verify(mockDecoderCallback) .onDecodedFrame(videoFrameCaptor.capture(), /* decodeTimeMs= */ any(Integer.class), /* qp= */ any()); VideoFrame videoFrame = videoFrameCaptor.getValue(); assertThat(videoFrame).isNotNull(); assertThat(videoFrame.getBuffer()).isEqualTo(outputTextureBuffer); verify(releaseCallback).run(); } @Test public void testConfigureExceptionTriggerSWFallback() { // Set-up. doThrow(new IllegalStateException("Fake error")) .when(fakeMediaCodecWrapper) .configure(any(), any(), any(), anyInt()); AndroidVideoDecoder decoder = new TestDecoderBuilder().build(); // Test. assertThat(decoder.initDecode(TEST_DECODER_SETTINGS, mockDecoderCallback)) .isEqualTo(VideoCodecStatus.FALLBACK_SOFTWARE); } @Test public void testStartExceptionTriggerSWFallback() { // Set-up. doThrow(new IllegalStateException("Fake error")).when(fakeMediaCodecWrapper).start(); AndroidVideoDecoder decoder = new TestDecoderBuilder().build(); // Test. assertThat(decoder.initDecode(TEST_DECODER_SETTINGS, mockDecoderCallback)) .isEqualTo(VideoCodecStatus.FALLBACK_SOFTWARE); } }