Add C++ SurfaceTextureHandler

This cl adds a C++ counterpart of the Java SurfaceTextureHandler. It can be used for creating a webrtc::VideoFrames from a native handle and also guarantee that the Java SurfaceTexture is notified when the VideoFrame is no longer in use.

BUG=webrtc:4993
R=magjed@webrtc.org

Review URL: https://codereview.webrtc.org/1366413003 .

Cr-Commit-Position: refs/heads/master@{#10084}

talk/app/webrtc/java/android/org/webrtc/SurfaceTextureHelper.java

@@ -37,6 +37,7 @@ import android.os.HandlerThread;
 import android.os.SystemClock;
 import android.util.Log;
 
+import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 
 /**
@@ -45,8 +46,11 @@ import java.util.concurrent.TimeUnit;
  * the frame. Only one texture frame can be in flight at once, so returnTextureFrame() must be
  * called in order to receive a new frame. Call disconnect() to stop receiveing new frames and
  * release all resources.
+ * Note that there is a C++ counter part of this class that optionally can be used. It is used for
+ * wrapping texture frames into webrtc::VideoFrames and also handles calling returnTextureFrame()
+ * when the webrtc::VideoFrame is no longer used.
  */
-public final class SurfaceTextureHelper {
+final class SurfaceTextureHelper {
   private static final String TAG = "SurfaceTextureHelper";
   /**
    * Callback interface for being notified that a new texture frame is available. The calls will be
@@ -65,18 +69,16 @@ public final class SurfaceTextureHelper {
   private final EglBase eglBase;
   private final SurfaceTexture surfaceTexture;
   private final int oesTextureId;
-  private final OnTextureFrameAvailableListener listener;
+  private OnTextureFrameAvailableListener listener;
   // The possible states of this class.
   private boolean hasPendingTexture = false;
   private boolean isTextureInUse = false;
   private boolean isQuitting = false;
 
   /**
-   * Construct a new SurfaceTextureHelper to stream textures to the given |listener|, sharing OpenGL
-   * resources with |sharedContext|.
+   * Construct a new SurfaceTextureHelper sharing OpenGL resources with |sharedContext|.
    */
-  public SurfaceTextureHelper(EGLContext sharedContext, OnTextureFrameAvailableListener listener) {
-    this.listener = listener;
+  public SurfaceTextureHelper(EGLContext sharedContext) {
     thread = new HandlerThread(TAG);
     thread.start();
     handler = new Handler(thread.getLooper());
@@ -87,13 +89,6 @@ public final class SurfaceTextureHelper {
 
     oesTextureId = GlUtil.generateTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES);
     surfaceTexture = new SurfaceTexture(oesTextureId);
-    surfaceTexture.setOnFrameAvailableListener(new SurfaceTexture.OnFrameAvailableListener() {
-      @Override
-      public void onFrameAvailable(SurfaceTexture surfaceTexture) {
-        hasPendingTexture = true;
-        tryDeliverTextureFrame();
-      }
-    }, handler);
 
     // Reattach EGL context to private thread.
     eglBase.detachCurrent();
@@ -104,6 +99,24 @@ public final class SurfaceTextureHelper {
     });
   }
 
+  /**
+   *  Start to stream textures to the given |listener|.
+   *  A Listener can only be set once.
+   */
+  public void setListener(OnTextureFrameAvailableListener listener) {
+    if (this.listener != null) {
+      throw new IllegalStateException("SurfaceTextureHelper listener has already been set.");
+    }
+    this.listener = listener;
+    surfaceTexture.setOnFrameAvailableListener(new SurfaceTexture.OnFrameAvailableListener() {
+      @Override
+      public void onFrameAvailable(SurfaceTexture surfaceTexture) {
+        hasPendingTexture = true;
+        tryDeliverTextureFrame();
+      }
+    }, handler);
+  }
+
   /**
    * Retrieve the underlying SurfaceTexture. The SurfaceTexture should be passed in to a video
    * producer such as a camera or decoder.
@@ -131,23 +144,34 @@ public final class SurfaceTextureHelper {
   }
 
   /**
-   * Call disconnect() to stop receiving frames and release all resources. This function will block
-   * until all frames are returned and all resoureces are released. You are guaranteed to not
-   * receive any more onTextureFrameAvailable() after this function returns.
+   * Call disconnect() to stop receiving frames. Resources are released when the texture frame has
+   * been returned by a call to returnTextureFrame(). You are guaranteed to not receive any more
+   * onTextureFrameAvailable() after this function returns.
    */
   public void disconnect() {
+    final CountDownLatch barrier = new CountDownLatch(1);
     handler.postAtFrontOfQueue(new Runnable() {
       @Override public void run() {
         isQuitting = true;
+        barrier.countDown();
         if (!isTextureInUse) {
           release();
         }
       }
     });
-    try {
-      thread.join();
-    } catch (InterruptedException e) {
-      Log.e(TAG, "SurfaceTexture thread was interrupted in join().");
+    boolean wasInterrupted = true;
+    while(true) {
+      try {
+        barrier.await();
+        break;
+      } catch (InterruptedException e) {
+        // Someone is asking us to return early at our convenience. We must wait until the
+        // |isQuitting| flag has been set but we should preserve the information and pass it along.
+        wasInterrupted = true;
+      }
+    }
+    if (wasInterrupted) {
+      Thread.currentThread().interrupt();
     }
   }
 

talk/app/webrtc/java/jni/surfacetexturehelper_jni.cc

@@ -0,0 +1,101 @@
+/*
+ * libjingle
+ * Copyright 2015 Google Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright notice,
+ *     this list of conditions and the following disclaimer in the documentation
+ *     and/or other materials provided with the distribution.
+ *  3. The name of the author may not be used to endorse or promote products
+ *     derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+ * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+
+#include "talk/app/webrtc/java/jni/surfacetexturehelper_jni.h"
+
+#include "talk/app/webrtc/java/jni/classreferenceholder.h"
+#include "webrtc/base/checks.h"
+
+namespace webrtc_jni {
+
+class SurfaceTextureHelper::TextureBuffer : public webrtc::NativeHandleBuffer {
+ public:
+  TextureBuffer(int width,
+                int height,
+                const rtc::scoped_refptr<SurfaceTextureHelper>& pool,
+                const NativeHandleImpl& native_handle)
+      : webrtc::NativeHandleBuffer(&native_handle_, width, height),
+        native_handle_(native_handle),
+        pool_(pool) {}
+
+  ~TextureBuffer() {
+    pool_->ReturnTextureFrame();
+  }
+
+  rtc::scoped_refptr<VideoFrameBuffer> NativeToI420Buffer() override {
+    RTC_NOTREACHED()
+        << "SurfaceTextureHelper::NativeToI420Buffer not implemented.";
+    return nullptr;
+  }
+
+ private:
+  NativeHandleImpl native_handle_;
+  const rtc::scoped_refptr<SurfaceTextureHelper> pool_;
+};
+
+SurfaceTextureHelper::SurfaceTextureHelper(JNIEnv* jni,
+                                           jobject egl_shared_context)
+    : j_surface_texture_helper_class_(
+          jni,
+          jni->FindClass("org/webrtc/SurfaceTextureHelper")),
+      j_surface_texture_helper_(
+          jni,
+          jni->NewObject(*j_surface_texture_helper_class_,
+                         GetMethodID(jni,
+                                     *j_surface_texture_helper_class_,
+                                     "<init>",
+                                     "(Landroid/opengl/EGLContext;)V"),
+                         egl_shared_context)),
+      j_return_texture_method_(
+          GetMethodID(jni,
+                      *j_surface_texture_helper_class_,
+                      "returnTextureFrame",
+                      "()V")) {
+  CHECK_EXCEPTION(jni) << "error during initialization of SurfaceTextureHelper";
+}
+
+SurfaceTextureHelper::~SurfaceTextureHelper() {
+}
+
+void SurfaceTextureHelper::ReturnTextureFrame() const {
+  JNIEnv* jni = AttachCurrentThreadIfNeeded();
+  jni->CallVoidMethod(*j_surface_texture_helper_, j_return_texture_method_);
+
+  CHECK_EXCEPTION(
+      jni) << "error during SurfaceTextureHelper.returnTextureFrame";
+}
+
+rtc::scoped_refptr<webrtc::VideoFrameBuffer>
+SurfaceTextureHelper::CreateTextureFrame(int width, int height,
+    const NativeHandleImpl& native_handle) {
+  return new rtc::RefCountedObject<TextureBuffer>(
+      width, height, this, native_handle);
+}
+
+}  // namespace webrtc_jni

talk/app/webrtc/java/jni/surfacetexturehelper_jni.h

@@ -0,0 +1,86 @@
+/*
+ * libjingle
+ * Copyright 2015 Google Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright notice,
+ *     this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright notice,
+ *     this list of conditions and the following disclaimer in the documentation
+ *     and/or other materials provided with the distribution.
+ *  3. The name of the author may not be used to endorse or promote products
+ *     derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+ * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef TALK_APP_WEBRTC_JAVA_JNI_SURFACETEXTUREHELPER_JNI_H_
+#define TALK_APP_WEBRTC_JAVA_JNI_SURFACETEXTUREHELPER_JNI_H_
+
+#include <jni.h>
+
+#include "talk/app/webrtc/java/jni/jni_helpers.h"
+#include "talk/app/webrtc/java/jni/native_handle_impl.h"
+#include "webrtc/base/refcount.h"
+#include "webrtc/base/scoped_ref_ptr.h"
+#include "webrtc/common_video/interface/video_frame_buffer.h"
+
+namespace webrtc_jni {
+
+// Helper class to create and synchronize access to an Android SurfaceTexture.
+// It is used for creating webrtc::VideoFrameBuffers from a SurfaceTexture when
+// the SurfaceTexture has been updated.
+// When the VideoFrameBuffer is released, this class returns the buffer to the
+// java SurfaceTextureHelper so it can be updated safely. The VideoFrameBuffer
+// can be released on an arbitrary thread.
+// SurfaceTextureHelper is reference counted to make sure that it is not
+// destroyed while a VideoFrameBuffer is in use.
+// This class is the C++ counterpart of the java class SurfaceTextureHelper.
+// Usage:
+// 1. Create an instance of this class.
+// 2. Call GetJavaSurfaceTextureHelper to get the Java SurfaceTextureHelper.
+// 3. Register a listener to the Java SurfaceListener and start producing
+// new buffers.
+// 3. Call CreateTextureFrame to wrap the Java texture in a VideoFrameBuffer.
+class SurfaceTextureHelper : public rtc::RefCountInterface {
+ public:
+  SurfaceTextureHelper(JNIEnv* jni, jobject shared_egl_context);
+
+  // Returns the Java SurfaceTextureHelper.
+  jobject GetJavaSurfaceTextureHelper() const {
+    return *j_surface_texture_helper_;
+  }
+
+  rtc::scoped_refptr<webrtc::VideoFrameBuffer> CreateTextureFrame(
+      int width,
+      int height,
+      const NativeHandleImpl& native_handle);
+
+ protected:
+  ~SurfaceTextureHelper();
+
+ private:
+  class TextureBuffer;
+  //  May be called on arbitrary thread.
+  void ReturnTextureFrame() const;
+
+  const ScopedGlobalRef<jclass> j_surface_texture_helper_class_;
+  const ScopedGlobalRef<jobject> j_surface_texture_helper_;
+  const jmethodID j_return_texture_method_;
+};
+
+}  // namespace webrtc_jni
+
+#endif  // TALK_APP_WEBRTC_JAVA_JNI_SURFACETEXTUREHELPER_JNI_H_

talk/libjingle.gyp

@@ -102,6 +102,8 @@
                 'app/webrtc/java/jni/androidmediadecoder_jni.h',
                 'app/webrtc/java/jni/androidmediaencoder_jni.cc',
                 'app/webrtc/java/jni/androidmediaencoder_jni.h',
+                'app/webrtc/java/jni/surfacetexturehelper_jni.cc',
+                'app/webrtc/java/jni/surfacetexturehelper_jni.h',
               ]
             }],
           ],