Android: Update MediaCodecVideoDecoder to output VideoFrames

Bug: webrtc:9181
Change-Id: I7eba15167536e453956c511a056143b039f52b92
Reviewed-on: https://webrtc-review.googlesource.com/71664
Reviewed-by: Sami Kalliomäki <sakal@webrtc.org>
Commit-Queue: Magnus Jedvert <magjed@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#22988}

sdk/android/api/org/webrtc/MediaCodecVideoDecoder.java

@@ -30,6 +30,8 @@ import java.util.Set;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import javax.annotation.Nullable;
+import org.webrtc.EglBase;
+import org.webrtc.VideoFrame;
 
 // Java-side of peerconnection.cc:MediaCodecVideoDecoder.
 // This class is an implementation detail of the Java PeerConnection API.
@@ -320,16 +322,16 @@ public class MediaCodecVideoDecoder {
     }
   }
 
-  // Pass null in |surfaceTextureHelper| to configure the codec for ByteBuffer output.
+  // Pass null in |eglContext| to configure the codec for ByteBuffer output.
   @CalledByNativeUnchecked
-  private boolean initDecode(VideoCodecType type, int width, int height,
+  private boolean initDecode(
-      @Nullable SurfaceTextureHelper surfaceTextureHelper) {
+      VideoCodecType type, int width, int height, @Nullable EglBase.Context eglContext) {
     if (mediaCodecThread != null) {
       throw new RuntimeException("initDecode: Forgot to release()?");
     }
 
     String mime = null;
-    useSurface = (surfaceTextureHelper != null);
+    useSurface = (eglContext != null);
     String[] supportedCodecPrefixes = null;
     if (type == VideoCodecType.VIDEO_CODEC_VP8) {
       mime = VP8_MIME_TYPE;
@@ -359,9 +361,14 @@ public class MediaCodecVideoDecoder {
       stride = width;
       sliceHeight = height;
 
-      if (useSurface && surfaceTextureHelper != null) {
+      if (useSurface) {
-        textureListener = new TextureListener(surfaceTextureHelper);
+        @Nullable
-        surface = new Surface(surfaceTextureHelper.getSurfaceTexture());
+        final SurfaceTextureHelper surfaceTextureHelper =
+            SurfaceTextureHelper.create("Decoder SurfaceTextureHelper", eglContext);
+        if (surfaceTextureHelper != null) {
+          textureListener = new TextureListener(surfaceTextureHelper);
+          surface = new Surface(surfaceTextureHelper.getSurfaceTexture());
+        }
       }
 
       MediaFormat format = MediaFormat.createVideoFormat(mime, width, height);
@@ -567,8 +574,7 @@ public class MediaCodecVideoDecoder {
 
   // Helper struct for dequeueTextureBuffer() below.
   private static class DecodedTextureBuffer {
-    private final int textureID;
+    private final VideoFrame.Buffer videoFrameBuffer;
-    private final float[] transformMatrix;
     // Presentation timestamp returned in dequeueOutputBuffer call.
     private final long presentationTimeStampMs;
     // C++ inputImage._timeStamp value for output frame.
@@ -585,11 +591,9 @@ public class MediaCodecVideoDecoder {
 
     // A DecodedTextureBuffer with zero |textureID| has special meaning and represents a frame
     // that was dropped.
-    public DecodedTextureBuffer(int textureID, float[] transformMatrix,
+    public DecodedTextureBuffer(VideoFrame.Buffer videoFrameBuffer, long presentationTimeStampMs,
-        long presentationTimeStampMs, long timeStampMs, long ntpTimeStampMs, long decodeTimeMs,
+        long timeStampMs, long ntpTimeStampMs, long decodeTimeMs, long frameDelay) {
-        long frameDelay) {
+      this.videoFrameBuffer = videoFrameBuffer;
-      this.textureID = textureID;
-      this.transformMatrix = transformMatrix;
       this.presentationTimeStampMs = presentationTimeStampMs;
       this.timeStampMs = timeStampMs;
       this.ntpTimeStampMs = ntpTimeStampMs;
@@ -598,13 +602,8 @@ public class MediaCodecVideoDecoder {
     }
 
     @CalledByNative("DecodedTextureBuffer")
-    int getTextureId() {
+    VideoFrame.Buffer getVideoFrameBuffer() {
-      return textureID;
+      return videoFrameBuffer;
-    }
-
-    @CalledByNative("DecodedTextureBuffer")
-    float[] getTransformMatrix() {
-      return transformMatrix;
     }
 
     @CalledByNative("DecodedTextureBuffer")
@@ -634,8 +633,7 @@ public class MediaCodecVideoDecoder {
   }
 
   // Poll based texture listener.
-  private static class TextureListener
+  private class TextureListener implements SurfaceTextureHelper.OnTextureFrameAvailableListener {
-      implements SurfaceTextureHelper.OnTextureFrameAvailableListener {
     private final SurfaceTextureHelper surfaceTextureHelper;
     // |newFrameLock| is used to synchronize arrival of new frames with wait()/notifyAll().
     private final Object newFrameLock = new Object();
@@ -674,9 +672,10 @@ public class MediaCodecVideoDecoder {
           throw new IllegalStateException("Already holding a texture.");
         }
         // |timestampNs| is always zero on some Android versions.
-        renderedBuffer = new DecodedTextureBuffer(oesTextureId, transformMatrix,
+        final VideoFrame.Buffer buffer = surfaceTextureHelper.createTextureBuffer(
-            bufferToRender.presentationTimeStampMs, bufferToRender.timeStampMs,
+            width, height, RendererCommon.convertMatrixToAndroidGraphicsMatrix(transformMatrix));
-            bufferToRender.ntpTimeStampMs, bufferToRender.decodeTimeMs,
+        renderedBuffer = new DecodedTextureBuffer(buffer, bufferToRender.presentationTimeStampMs,
+            bufferToRender.timeStampMs, bufferToRender.ntpTimeStampMs, bufferToRender.decodeTimeMs,
             SystemClock.elapsedRealtime() - bufferToRender.endDecodeTimeMs);
         bufferToRender = null;
         newFrameLock.notifyAll();
@@ -709,10 +708,11 @@ public class MediaCodecVideoDecoder {
       surfaceTextureHelper.stopListening();
       synchronized (newFrameLock) {
         if (renderedBuffer != null) {
-          surfaceTextureHelper.returnTextureFrame();
+          renderedBuffer.getVideoFrameBuffer().release();
           renderedBuffer = null;
         }
       }
+      surfaceTextureHelper.dispose();
     }
   }
 
@@ -844,8 +844,9 @@ public class MediaCodecVideoDecoder {
       }
 
       mediaCodec.releaseOutputBuffer(droppedFrame.index, false /* render */);
-      return new DecodedTextureBuffer(0, null, droppedFrame.presentationTimeStampMs,
+      return new DecodedTextureBuffer(null /* videoFrameBuffer */,
-          droppedFrame.timeStampMs, droppedFrame.ntpTimeStampMs, droppedFrame.decodeTimeMs,
+          droppedFrame.presentationTimeStampMs, droppedFrame.timeStampMs,
+          droppedFrame.ntpTimeStampMs, droppedFrame.decodeTimeMs,
           SystemClock.elapsedRealtime() - droppedFrame.endDecodeTimeMs);
     }
     return null;

sdk/android/src/jni/androidmediadecoder.cc

@@ -112,7 +112,6 @@ class MediaCodecVideoDecoder : public VideoDecoder, public rtc::MessageHandler {
   bool use_surface_;
   VideoCodec codec_;
   I420BufferPool decoded_frame_pool_;
-  rtc::scoped_refptr<SurfaceTextureHelper> surface_texture_helper_;
   DecodedImageCallback* callback_;
   int frames_received_;  // Number of frames received by decoder.
   int frames_decoded_;  // Number of frames decoded by decoder.
@@ -224,26 +223,12 @@ int32_t MediaCodecVideoDecoder::InitDecodeOnCodecThread() {
 
   ResetVariables();
 
-  if (use_surface_) {
-    surface_texture_helper_ = SurfaceTextureHelper::create(
-        jni, "Decoder SurfaceTextureHelper",
-        JavaParamRef<jobject>(render_egl_context_));
-    if (!surface_texture_helper_) {
-      ALOGE << "Couldn't create SurfaceTextureHelper - fallback to SW codec";
-      sw_fallback_required_ = true;
-      return WEBRTC_VIDEO_CODEC_ERROR;
-    }
-  }
-
   ScopedJavaLocalRef<jobject> j_video_codec_enum =
       Java_VideoCodecType_fromNativeIndex(jni, codecType_);
-  jobject j_surface_texture_helper =
+  jobject j_egl_context = use_surface_ ? render_egl_context_ : nullptr;
-      use_surface_
-          ? surface_texture_helper_->GetJavaSurfaceTextureHelper().obj()
-          : nullptr;
   bool success = Java_MediaCodecVideoDecoder_initDecode(
       jni, j_media_codec_video_decoder_, j_video_codec_enum, codec_.width,
-      codec_.height, JavaParamRef<jobject>(j_surface_texture_helper));
+      codec_.height, JavaParamRef<jobject>(j_egl_context));
 
   if (CheckException(jni) || !success) {
     ALOGE << "Codec initialization error - fallback to SW codec.";
@@ -325,7 +310,6 @@ int32_t MediaCodecVideoDecoder::ReleaseOnCodecThread() {
   ScopedLocalRefFrame local_ref_frame(jni);
   input_buffers_.clear();
   Java_MediaCodecVideoDecoder_release(jni, j_media_codec_video_decoder_);
-  surface_texture_helper_ = nullptr;
   inited_ = false;
   rtc::MessageQueueManager::Clear(this);
   if (CheckException(jni)) {
@@ -604,18 +588,14 @@ bool MediaCodecVideoDecoder::DeliverPendingOutputs(
     decode_time_ms =
         Java_DecodedTextureBuffer_getDecodeTimeMs(jni, j_decoder_output_buffer);
 
-    const int texture_id =
+    ScopedJavaLocalRef<jobject> j_video_frame_buffer =
-        Java_DecodedTextureBuffer_getTextureId(jni, j_decoder_output_buffer);
+        Java_DecodedTextureBuffer_getVideoFrameBuffer(jni,
-    if (texture_id != 0) {  // |texture_id| == 0 represents a dropped frame.
+                                                      j_decoder_output_buffer);
-      ScopedJavaLocalRef<jfloatArray> j_transform_matrix =
+    // |video_frame_buffer| == null represents a dropped frame.
-          Java_DecodedTextureBuffer_getTransformMatrix(jni,
+    if (!j_video_frame_buffer.is_null()) {
-                                                       j_decoder_output_buffer);
       frame_delayed_ms = Java_DecodedTextureBuffer_getFrameDelayMs(
           jni, j_decoder_output_buffer);
-
+      frame_buffer = AndroidVideoBuffer::Adopt(jni, j_video_frame_buffer);
-      // Create VideoFrameBuffer with native texture handle.
-      frame_buffer = surface_texture_helper_->CreateTextureFrame(
-          width, height, NativeHandleImpl(jni, texture_id, j_transform_matrix));
     } else {
       EnableFrameLogOnWarning();
     }