Add Android Camera To Unity Plugin

The existing unity plugin (an example in webrtc codebase) does not support camera access on Android platform. This CL implements such functionality.

TBR=gyzhou@chromium.org

BUG=webrtc:8067

Review-Url: https://codereview.webrtc.org/2993273002
Cr-Commit-Position: refs/heads/master@{#19277}

webrtc/examples/unityplugin/ANDROID_INSTRUCTION

@@ -0,0 +1,33 @@
+Instruction of Running webrtc_unity_plugin on Android Unity
+
+1. On Linux machine, compile target webrtc_unity_plugin.
+   Checkout WebRTC codebase: fetch --no-hooks webrtc_android
+   If you already have a checkout for linux, add target_os=”android” into .gclient file.
+      Run gclient sync
+      Run gn args out/Android, and again set target_os=”android” in the args.gn
+   Modify file src/build/android/android_only_jni_exports.lst, to expose all functions. Namely, change "global" section to "*", and remove "local" section. Otherwise, Unity C# code will not be able to access the functions defined in the plugin.
+   Run ninja -C out/Android webrtc_unity_plugin
+
+2. On Linux machine, compile target libwebrtc_unity under webrtc checkout. This is the java code for webrtc to work on Android.
+
+3. Copy libwebrtc_unity.jar and libwebrtc_unity_plugin.so into Unity project folder, under Assets/Plugins/Android folder.
+
+4. Rename libwebrtc_unity_plugin.so to libjingle_peerconnection_so.so. Again, this is hacky, and the purpose is to let the java code in libwebrtc.jar to find their JNI implementation. And simultaneously, in your C# wrapper script for the native plugin libjingle_peerconnection_so.so, the dll_path should be set to “jingle_peerconnection_so”.
+
+5. In the Unity Main Scene’s Start method, write the following code to initialize the Java environment for webrtc (otherwise, webrtc will not be able to access audio device or camera from C++ code):
+
+#if UNITY_ANDROID
+        AndroidJavaClass playerClass = new AndroidJavaClass("com.unity3d.player.UnityPlayer");
+        AndroidJavaObject activity = playerClass.GetStatic<AndroidJavaObject>("currentActivity");
+        AndroidJavaClass webrtcClass = new AndroidJavaClass("org.webrtc.PeerConnectionFactory");
+        if (webrtcClass != null)
+        {
+            webrtcClass.CallStatic("initializeAndroidGlobals", new object[2] { activity, false });
+        }
+#endif
+
+6. Compile the unity project into an APK, and decompile the apk using apktool you can download from internet. And copy the AndroidManifest.xml to the Assets/Plugins/Android folder, and add two lines:
+  <uses-permission android:name=”android.permission.RECORD_AUDIO” />
+  <uses-permission android:name=”android.permission.CAMERA” />
+
+  The purpose of using apktool is to get a well-written android manifest xml file. If you know how to write manifest file from scratch, you can skip using apktool.

webrtc/examples/unityplugin/DEPS

@@ -0,0 +1,3 @@
+include_rules = [
+  "+webrtc/sdk",
+]

webrtc/examples/unityplugin/README

@@ -1,7 +1,10 @@
-This directory contains an example Unity native plugin for Windows OS.
+This directory contains an example Unity native plugin for Windows OS and Android.
+
 The APIs use Platform Invoke (P/Invoke) technology as required by Unity native plugin.
 This plugin dll can also be used by Windows C# applications other than Unity.
 
+For detailed build instruction on Android, see ANDROID_INSTRUCTION
+
 An example of wrapping native plugin into a C# managed class in Unity is given as following:
 
 using System;

webrtc/examples/unityplugin/classreferenceholder.cc

@@ -0,0 +1,88 @@
+/*
+ *  Copyright 2015 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+#include "webrtc/examples/unityplugin/classreferenceholder.h"
+
+#include <utility>
+
+#include "webrtc/sdk/android/src/jni/jni_helpers.h"
+
+namespace unity_plugin {
+
+// ClassReferenceHolder holds global reference to Java classes in app/webrtc.
+class ClassReferenceHolder {
+ public:
+  explicit ClassReferenceHolder(JNIEnv* jni);
+  ~ClassReferenceHolder();
+
+  void FreeReferences(JNIEnv* jni);
+  jclass GetClass(const std::string& name);
+
+  void LoadClass(JNIEnv* jni, const std::string& name);
+
+ private:
+  std::map<std::string, jclass> classes_;
+};
+
+// Allocated in LoadGlobalClassReferenceHolder(),
+// freed in FreeGlobalClassReferenceHolder().
+static ClassReferenceHolder* g_class_reference_holder = nullptr;
+
+void LoadGlobalClassReferenceHolder() {
+  RTC_CHECK(g_class_reference_holder == nullptr);
+  g_class_reference_holder = new ClassReferenceHolder(webrtc_jni::GetEnv());
+}
+
+void FreeGlobalClassReferenceHolder() {
+  g_class_reference_holder->FreeReferences(
+      webrtc_jni::AttachCurrentThreadIfNeeded());
+  delete g_class_reference_holder;
+  g_class_reference_holder = nullptr;
+}
+
+ClassReferenceHolder::ClassReferenceHolder(JNIEnv* jni) {
+  LoadClass(jni, "org/webrtc/UnityUtility");
+}
+
+ClassReferenceHolder::~ClassReferenceHolder() {
+  RTC_CHECK(classes_.empty()) << "Must call FreeReferences() before dtor!";
+}
+
+void ClassReferenceHolder::FreeReferences(JNIEnv* jni) {
+  for (std::map<std::string, jclass>::const_iterator it = classes_.begin();
+       it != classes_.end(); ++it) {
+    jni->DeleteGlobalRef(it->second);
+  }
+  classes_.clear();
+}
+
+jclass ClassReferenceHolder::GetClass(const std::string& name) {
+  std::map<std::string, jclass>::iterator it = classes_.find(name);
+  RTC_CHECK(it != classes_.end()) << "Unexpected GetClass() call for: " << name;
+  return it->second;
+}
+
+void ClassReferenceHolder::LoadClass(JNIEnv* jni, const std::string& name) {
+  jclass localRef = jni->FindClass(name.c_str());
+  CHECK_EXCEPTION(jni) << "error during FindClass: " << name;
+  RTC_CHECK(localRef) << name;
+  jclass globalRef = reinterpret_cast<jclass>(jni->NewGlobalRef(localRef));
+  CHECK_EXCEPTION(jni) << "error during NewGlobalRef: " << name;
+  RTC_CHECK(globalRef) << name;
+  bool inserted = classes_.insert(std::make_pair(name, globalRef)).second;
+  RTC_CHECK(inserted) << "Duplicate class name: " << name;
+}
+
+// Returns a global reference guaranteed to be valid for the lifetime of the
+// process.
+jclass FindClass(JNIEnv* jni, const char* name) {
+  return g_class_reference_holder->GetClass(name);
+}
+
+}  // namespace unity_plugin

webrtc/examples/unityplugin/classreferenceholder.h

@@ -0,0 +1,37 @@
+/*
+ *  Copyright 2015 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+// This is a supplement of webrtc_jni::ClassReferenceHolder.
+// The purpose of this ClassReferenceHolder is to load the example
+// specific java class into JNI c++ side, so that our c++ code can
+// call those java functions.
+
+#ifndef WEBRTC_EXAMPLES_UNITYPLUGIN_CLASSREFERENCEHOLDER_H_
+#define WEBRTC_EXAMPLES_UNITYPLUGIN_CLASSREFERENCEHOLDER_H_
+
+#include <jni.h>
+#include <map>
+#include <string>
+#include <vector>
+
+namespace unity_plugin {
+
+// LoadGlobalClassReferenceHolder must be called in JNI_OnLoad.
+void LoadGlobalClassReferenceHolder();
+// FreeGlobalClassReferenceHolder must be called in JNI_UnLoad.
+void FreeGlobalClassReferenceHolder();
+
+// Returns a global reference guaranteed to be valid for the lifetime of the
+// process.
+jclass FindClass(JNIEnv* jni, const char* name);
+
+}  // namespace unity_plugin
+
+#endif  // WEBRTC_EXAMPLES_UNITYPLUGIN_CLASSREFERENCEHOLDER_H_

webrtc/examples/unityplugin/java/src/org/webrtc/UnityUtility.java

@@ -0,0 +1,64 @@
+/*
+ *  Copyright 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+package org.webrtc;
+
+import android.content.Context;
+import java.util.List;
+
+public class UnityUtility {
+  private static final String VIDEO_CAPTURER_THREAD_NAME = "VideoCapturerThread";
+
+  public static SurfaceTextureHelper LoadSurfaceTextureHelper() {
+    final SurfaceTextureHelper surfaceTextureHelper =
+        SurfaceTextureHelper.create(VIDEO_CAPTURER_THREAD_NAME, null);
+    return surfaceTextureHelper;
+  }
+
+  private static boolean useCamera2() {
+    return Camera2Enumerator.isSupported(ContextUtils.getApplicationContext());
+  }
+
+  private static VideoCapturer createCameraCapturer(CameraEnumerator enumerator) {
+    final String[] deviceNames = enumerator.getDeviceNames();
+
+    for (String deviceName : deviceNames) {
+      if (enumerator.isFrontFacing(deviceName)) {
+        VideoCapturer videoCapturer = enumerator.createCapturer(deviceName, null);
+
+        if (videoCapturer != null) {
+          return videoCapturer;
+        }
+      }
+    }
+
+    return null;
+  }
+
+  public static VideoCapturer LinkCamera(
+      long nativeTrackSource, SurfaceTextureHelper surfaceTextureHelper) {
+    VideoCapturer capturer =
+        createCameraCapturer(new Camera2Enumerator(ContextUtils.getApplicationContext()));
+
+    VideoCapturer.CapturerObserver capturerObserver =
+        new AndroidVideoTrackSourceObserver(nativeTrackSource);
+
+    capturer.initialize(
+        surfaceTextureHelper, ContextUtils.getApplicationContext(), capturerObserver);
+
+    capturer.startCapture(720, 480, 30);
+    return capturer;
+  }
+
+  public static void StopCamera(VideoCapturer camera) throws InterruptedException {
+    camera.stopCapture();
+    camera.dispose();
+  }
+}

webrtc/examples/unityplugin/jni_onload.cc

@@ -0,0 +1,41 @@
+/*
+ *  Copyright 2015 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include <jni.h>
+#undef JNIEXPORT
+#define JNIEXPORT __attribute__((visibility("default")))
+
+#include "webrtc/examples/unityplugin/classreferenceholder.h"
+#include "webrtc/rtc_base/ssladapter.h"
+#include "webrtc/sdk/android/src/jni/classreferenceholder.h"
+#include "webrtc/sdk/android/src/jni/jni_helpers.h"
+
+namespace webrtc_jni {
+
+extern "C" jint JNIEXPORT JNICALL JNI_OnLoad(JavaVM* jvm, void* reserved) {
+  jint ret = InitGlobalJniVariables(jvm);
+  RTC_DCHECK_GE(ret, 0);
+  if (ret < 0)
+    return -1;
+
+  RTC_CHECK(rtc::InitializeSSL()) << "Failed to InitializeSSL()";
+  LoadGlobalClassReferenceHolder();
+  unity_plugin::LoadGlobalClassReferenceHolder();
+
+  return ret;
+}
+
+extern "C" void JNIEXPORT JNICALL JNI_OnUnLoad(JavaVM* jvm, void* reserved) {
+  FreeGlobalClassReferenceHolder();
+  unity_plugin::FreeGlobalClassReferenceHolder();
+  RTC_CHECK(rtc::CleanupSSL()) << "Failed to CleanupSSL()";
+}
+
+}  // namespace webrtc_jni

webrtc/examples/unityplugin/simple_peer_connection.cc

@@ -13,9 +13,16 @@
 #include <utility>
 
 #include "webrtc/api/test/fakeconstraints.h"
+#include "webrtc/api/videosourceproxy.h"
 #include "webrtc/media/engine/webrtcvideocapturerfactory.h"
 #include "webrtc/modules/video_capture/video_capture_factory.h"
 
+#if defined(WEBRTC_ANDROID)
+#include "webrtc/examples/unityplugin/classreferenceholder.h"
+#include "webrtc/sdk/android/src/jni/androidvideotracksource.h"
+#include "webrtc/sdk/android/src/jni/jni_helpers.h"
+#endif
+
 // Names used for media stream labels.
 const char kAudioLabel[] = "audio_label";
 const char kVideoLabel[] = "video_label";
@@ -27,6 +34,12 @@ static std::unique_ptr<rtc::Thread> g_worker_thread;
 static std::unique_ptr<rtc::Thread> g_signaling_thread;
 static rtc::scoped_refptr<webrtc::PeerConnectionFactoryInterface>
     g_peer_connection_factory;
+#if defined(WEBRTC_ANDROID)
+// Android case: the video track does not own the capturer, and it
+// relies on the app to dispose the capturer when the peerconnection
+// shuts down.
+static jobject g_camera = nullptr;
+#endif
 
 std::string GetEnvVarOrDefault(const char* env_var_name,
                                const char* default_value) {
@@ -149,6 +162,21 @@ bool SimplePeerConnection::CreatePeerConnection(const char** turn_urls,
 void SimplePeerConnection::DeletePeerConnection() {
   g_peer_count--;
 
+#if defined(WEBRTC_ANDROID)
+  if (g_camera) {
+    JNIEnv* env = webrtc_jni::GetEnv();
+    jclass pc_factory_class =
+        unity_plugin::FindClass(env, "org/webrtc/UnityUtility");
+    jmethodID stop_camera_method = webrtc_jni::GetStaticMethodID(
+        env, pc_factory_class, "StopCamera", "(Lorg/webrtc/VideoCapturer;)V");
+
+    env->CallStaticVoidMethod(pc_factory_class, stop_camera_method, g_camera);
+    CHECK_EXCEPTION(env);
+
+    g_camera = nullptr;
+  }
+#endif
+
   CloseDataChannel();
   peer_connection_ = nullptr;
   active_streams_.clear();
@@ -380,6 +408,41 @@ void SimplePeerConnection::AddStreams(bool audio_only) {
   stream->AddTrack(audio_track);
 
   if (!audio_only) {
+#if defined(WEBRTC_ANDROID)
+    JNIEnv* env = webrtc_jni::GetEnv();
+    jclass pc_factory_class =
+        unity_plugin::FindClass(env, "org/webrtc/UnityUtility");
+    jmethodID load_texture_helper_method = webrtc_jni::GetStaticMethodID(
+        env, pc_factory_class, "LoadSurfaceTextureHelper",
+        "()Lorg/webrtc/SurfaceTextureHelper;");
+    jobject texture_helper = env->CallStaticObjectMethod(
+        pc_factory_class, load_texture_helper_method);
+    CHECK_EXCEPTION(env);
+    RTC_DCHECK(texture_helper != nullptr)
+        << "Cannot get the Surface Texture Helper.";
+
+    rtc::scoped_refptr<webrtc::AndroidVideoTrackSource> source(
+        new rtc::RefCountedObject<webrtc::AndroidVideoTrackSource>(
+            g_signaling_thread.get(), env, texture_helper, false));
+    rtc::scoped_refptr<webrtc::VideoTrackSourceProxy> proxy_source =
+        webrtc::VideoTrackSourceProxy::Create(g_signaling_thread.get(),
+                                              g_worker_thread.get(), source);
+
+    // link with VideoCapturer (Camera);
+    jmethodID link_camera_method = webrtc_jni::GetStaticMethodID(
+        env, pc_factory_class, "LinkCamera",
+        "(JLorg/webrtc/SurfaceTextureHelper;)Lorg/webrtc/VideoCapturer;");
+    jobject camera_tmp =
+        env->CallStaticObjectMethod(pc_factory_class, link_camera_method,
+                                    (jlong)proxy_source.get(), texture_helper);
+    CHECK_EXCEPTION(env);
+    g_camera = (jobject)env->NewGlobalRef(camera_tmp);
+
+    rtc::scoped_refptr<webrtc::VideoTrackInterface> video_track(
+        g_peer_connection_factory->CreateVideoTrack(kVideoLabel,
+                                                    proxy_source.release()));
+    stream->AddTrack(video_track);
+#else
     std::unique_ptr<cricket::VideoCapturer> capture = OpenVideoCaptureDevice();
     if (capture) {
       rtc::scoped_refptr<webrtc::VideoTrackInterface> video_track(
@@ -388,10 +451,11 @@ void SimplePeerConnection::AddStreams(bool audio_only) {
                                std::move(capture), nullptr)));
 
       stream->AddTrack(video_track);
-      if (local_video_observer_ && !stream->GetVideoTracks().empty()) {
-        stream->GetVideoTracks()[0]->AddOrUpdateSink(
-            local_video_observer_.get(), rtc::VideoSinkWants());
-      }
+    }
+#endif
+    if (local_video_observer_ && !stream->GetVideoTracks().empty()) {
+      stream->GetVideoTracks()[0]->AddOrUpdateSink(local_video_observer_.get(),
+                                                   rtc::VideoSinkWants());
     }
   }
 

webrtc/examples/unityplugin/unity_plugin_apis.h

@@ -37,7 +37,11 @@ typedef void (*AUDIOBUSREADY_CALLBACK)(const void* audio_data,
                                        int number_of_channels,
                                        int number_of_frames);
 
+#if defined(WEBRTC_WIN)
 #define WEBRTC_PLUGIN_API __declspec(dllexport)
+#elif defined(WEBRTC_ANDROID)
+#define WEBRTC_PLUGIN_API __attribute__((visibility("default")))
+#endif
 extern "C" {
 // Create a peerconnection and return a unique peer connection id.
 WEBRTC_PLUGIN_API int CreatePeerConnection(const char** turn_urls,