From 3e9eb4ba01acf8a49fa1949305fb30a7daf51964 Mon Sep 17 00:00:00 2001 From: Per Date: Mon, 28 Sep 2015 10:52:22 +0200 Subject: [PATCH] Add C++ SurfaceTextureHandler This cl adds a C++ counterpart of the Java SurfaceTextureHandler. It can be used for creating a webrtc::VideoFrames from a native handle and also guarantee that the Java SurfaceTexture is notified when the VideoFrame is no longer in use. BUG=webrtc:4993 R=magjed@webrtc.org Review URL: https://codereview.webrtc.org/1366413003 . Cr-Commit-Position: refs/heads/master@{#10084} --- .../org/webrtc/SurfaceTextureHelper.java | 64 +++++++---- .../java/jni/surfacetexturehelper_jni.cc | 101 ++++++++++++++++++ .../java/jni/surfacetexturehelper_jni.h | 86 +++++++++++++++ talk/libjingle.gyp | 2 + 4 files changed, 233 insertions(+), 20 deletions(-) create mode 100644 talk/app/webrtc/java/jni/surfacetexturehelper_jni.cc create mode 100644 talk/app/webrtc/java/jni/surfacetexturehelper_jni.h diff --git a/talk/app/webrtc/java/android/org/webrtc/SurfaceTextureHelper.java b/talk/app/webrtc/java/android/org/webrtc/SurfaceTextureHelper.java index dfd54544a5..9ab38210d4 100644 --- a/talk/app/webrtc/java/android/org/webrtc/SurfaceTextureHelper.java +++ b/talk/app/webrtc/java/android/org/webrtc/SurfaceTextureHelper.java @@ -37,6 +37,7 @@ import android.os.HandlerThread; import android.os.SystemClock; import android.util.Log; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; /** @@ -45,8 +46,11 @@ import java.util.concurrent.TimeUnit; * the frame. Only one texture frame can be in flight at once, so returnTextureFrame() must be * called in order to receive a new frame. Call disconnect() to stop receiveing new frames and * release all resources. + * Note that there is a C++ counter part of this class that optionally can be used. It is used for + * wrapping texture frames into webrtc::VideoFrames and also handles calling returnTextureFrame() + * when the webrtc::VideoFrame is no longer used. */ -public final class SurfaceTextureHelper { +final class SurfaceTextureHelper { private static final String TAG = "SurfaceTextureHelper"; /** * Callback interface for being notified that a new texture frame is available. The calls will be @@ -65,18 +69,16 @@ public final class SurfaceTextureHelper { private final EglBase eglBase; private final SurfaceTexture surfaceTexture; private final int oesTextureId; - private final OnTextureFrameAvailableListener listener; + private OnTextureFrameAvailableListener listener; // The possible states of this class. private boolean hasPendingTexture = false; private boolean isTextureInUse = false; private boolean isQuitting = false; /** - * Construct a new SurfaceTextureHelper to stream textures to the given |listener|, sharing OpenGL - * resources with |sharedContext|. + * Construct a new SurfaceTextureHelper sharing OpenGL resources with |sharedContext|. */ - public SurfaceTextureHelper(EGLContext sharedContext, OnTextureFrameAvailableListener listener) { - this.listener = listener; + public SurfaceTextureHelper(EGLContext sharedContext) { thread = new HandlerThread(TAG); thread.start(); handler = new Handler(thread.getLooper()); @@ -87,13 +89,6 @@ public final class SurfaceTextureHelper { oesTextureId = GlUtil.generateTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES); surfaceTexture = new SurfaceTexture(oesTextureId); - surfaceTexture.setOnFrameAvailableListener(new SurfaceTexture.OnFrameAvailableListener() { - @Override - public void onFrameAvailable(SurfaceTexture surfaceTexture) { - hasPendingTexture = true; - tryDeliverTextureFrame(); - } - }, handler); // Reattach EGL context to private thread. eglBase.detachCurrent(); @@ -104,6 +99,24 @@ public final class SurfaceTextureHelper { }); } + /** + * Start to stream textures to the given |listener|. + * A Listener can only be set once. + */ + public void setListener(OnTextureFrameAvailableListener listener) { + if (this.listener != null) { + throw new IllegalStateException("SurfaceTextureHelper listener has already been set."); + } + this.listener = listener; + surfaceTexture.setOnFrameAvailableListener(new SurfaceTexture.OnFrameAvailableListener() { + @Override + public void onFrameAvailable(SurfaceTexture surfaceTexture) { + hasPendingTexture = true; + tryDeliverTextureFrame(); + } + }, handler); + } + /** * Retrieve the underlying SurfaceTexture. The SurfaceTexture should be passed in to a video * producer such as a camera or decoder. @@ -131,23 +144,34 @@ public final class SurfaceTextureHelper { } /** - * Call disconnect() to stop receiving frames and release all resources. This function will block - * until all frames are returned and all resoureces are released. You are guaranteed to not - * receive any more onTextureFrameAvailable() after this function returns. + * Call disconnect() to stop receiving frames. Resources are released when the texture frame has + * been returned by a call to returnTextureFrame(). You are guaranteed to not receive any more + * onTextureFrameAvailable() after this function returns. */ public void disconnect() { + final CountDownLatch barrier = new CountDownLatch(1); handler.postAtFrontOfQueue(new Runnable() { @Override public void run() { isQuitting = true; + barrier.countDown(); if (!isTextureInUse) { release(); } } }); - try { - thread.join(); - } catch (InterruptedException e) { - Log.e(TAG, "SurfaceTexture thread was interrupted in join()."); + boolean wasInterrupted = true; + while(true) { + try { + barrier.await(); + break; + } catch (InterruptedException e) { + // Someone is asking us to return early at our convenience. We must wait until the + // |isQuitting| flag has been set but we should preserve the information and pass it along. + wasInterrupted = true; + } + } + if (wasInterrupted) { + Thread.currentThread().interrupt(); } } diff --git a/talk/app/webrtc/java/jni/surfacetexturehelper_jni.cc b/talk/app/webrtc/java/jni/surfacetexturehelper_jni.cc new file mode 100644 index 0000000000..b6259ed3b0 --- /dev/null +++ b/talk/app/webrtc/java/jni/surfacetexturehelper_jni.cc @@ -0,0 +1,101 @@ +/* + * libjingle + * Copyright 2015 Google Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO + * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + + +#include "talk/app/webrtc/java/jni/surfacetexturehelper_jni.h" + +#include "talk/app/webrtc/java/jni/classreferenceholder.h" +#include "webrtc/base/checks.h" + +namespace webrtc_jni { + +class SurfaceTextureHelper::TextureBuffer : public webrtc::NativeHandleBuffer { + public: + TextureBuffer(int width, + int height, + const rtc::scoped_refptr& pool, + const NativeHandleImpl& native_handle) + : webrtc::NativeHandleBuffer(&native_handle_, width, height), + native_handle_(native_handle), + pool_(pool) {} + + ~TextureBuffer() { + pool_->ReturnTextureFrame(); + } + + rtc::scoped_refptr NativeToI420Buffer() override { + RTC_NOTREACHED() + << "SurfaceTextureHelper::NativeToI420Buffer not implemented."; + return nullptr; + } + + private: + NativeHandleImpl native_handle_; + const rtc::scoped_refptr pool_; +}; + +SurfaceTextureHelper::SurfaceTextureHelper(JNIEnv* jni, + jobject egl_shared_context) + : j_surface_texture_helper_class_( + jni, + jni->FindClass("org/webrtc/SurfaceTextureHelper")), + j_surface_texture_helper_( + jni, + jni->NewObject(*j_surface_texture_helper_class_, + GetMethodID(jni, + *j_surface_texture_helper_class_, + "", + "(Landroid/opengl/EGLContext;)V"), + egl_shared_context)), + j_return_texture_method_( + GetMethodID(jni, + *j_surface_texture_helper_class_, + "returnTextureFrame", + "()V")) { + CHECK_EXCEPTION(jni) << "error during initialization of SurfaceTextureHelper"; +} + +SurfaceTextureHelper::~SurfaceTextureHelper() { +} + +void SurfaceTextureHelper::ReturnTextureFrame() const { + JNIEnv* jni = AttachCurrentThreadIfNeeded(); + jni->CallVoidMethod(*j_surface_texture_helper_, j_return_texture_method_); + + CHECK_EXCEPTION( + jni) << "error during SurfaceTextureHelper.returnTextureFrame"; +} + +rtc::scoped_refptr +SurfaceTextureHelper::CreateTextureFrame(int width, int height, + const NativeHandleImpl& native_handle) { + return new rtc::RefCountedObject( + width, height, this, native_handle); +} + +} // namespace webrtc_jni diff --git a/talk/app/webrtc/java/jni/surfacetexturehelper_jni.h b/talk/app/webrtc/java/jni/surfacetexturehelper_jni.h new file mode 100644 index 0000000000..49e2315236 --- /dev/null +++ b/talk/app/webrtc/java/jni/surfacetexturehelper_jni.h @@ -0,0 +1,86 @@ +/* + * libjingle + * Copyright 2015 Google Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO + * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef TALK_APP_WEBRTC_JAVA_JNI_SURFACETEXTUREHELPER_JNI_H_ +#define TALK_APP_WEBRTC_JAVA_JNI_SURFACETEXTUREHELPER_JNI_H_ + +#include + +#include "talk/app/webrtc/java/jni/jni_helpers.h" +#include "talk/app/webrtc/java/jni/native_handle_impl.h" +#include "webrtc/base/refcount.h" +#include "webrtc/base/scoped_ref_ptr.h" +#include "webrtc/common_video/interface/video_frame_buffer.h" + +namespace webrtc_jni { + +// Helper class to create and synchronize access to an Android SurfaceTexture. +// It is used for creating webrtc::VideoFrameBuffers from a SurfaceTexture when +// the SurfaceTexture has been updated. +// When the VideoFrameBuffer is released, this class returns the buffer to the +// java SurfaceTextureHelper so it can be updated safely. The VideoFrameBuffer +// can be released on an arbitrary thread. +// SurfaceTextureHelper is reference counted to make sure that it is not +// destroyed while a VideoFrameBuffer is in use. +// This class is the C++ counterpart of the java class SurfaceTextureHelper. +// Usage: +// 1. Create an instance of this class. +// 2. Call GetJavaSurfaceTextureHelper to get the Java SurfaceTextureHelper. +// 3. Register a listener to the Java SurfaceListener and start producing +// new buffers. +// 3. Call CreateTextureFrame to wrap the Java texture in a VideoFrameBuffer. +class SurfaceTextureHelper : public rtc::RefCountInterface { + public: + SurfaceTextureHelper(JNIEnv* jni, jobject shared_egl_context); + + // Returns the Java SurfaceTextureHelper. + jobject GetJavaSurfaceTextureHelper() const { + return *j_surface_texture_helper_; + } + + rtc::scoped_refptr CreateTextureFrame( + int width, + int height, + const NativeHandleImpl& native_handle); + + protected: + ~SurfaceTextureHelper(); + + private: + class TextureBuffer; + // May be called on arbitrary thread. + void ReturnTextureFrame() const; + + const ScopedGlobalRef j_surface_texture_helper_class_; + const ScopedGlobalRef j_surface_texture_helper_; + const jmethodID j_return_texture_method_; +}; + +} // namespace webrtc_jni + +#endif // TALK_APP_WEBRTC_JAVA_JNI_SURFACETEXTUREHELPER_JNI_H_ diff --git a/talk/libjingle.gyp b/talk/libjingle.gyp index a4ee36216d..804170a3fc 100755 --- a/talk/libjingle.gyp +++ b/talk/libjingle.gyp @@ -102,6 +102,8 @@ 'app/webrtc/java/jni/androidmediadecoder_jni.h', 'app/webrtc/java/jni/androidmediaencoder_jni.cc', 'app/webrtc/java/jni/androidmediaencoder_jni.h', + 'app/webrtc/java/jni/surfacetexturehelper_jni.cc', + 'app/webrtc/java/jni/surfacetexturehelper_jni.h', ] }], ],