Support injecting custom native codecs from Java.

This CL extends our support for injecting native codecs such that
downstream users can create Java codecs that are backed by custom
native codecs.

After this CL, the Java codec interfaces expose
createNativeVideo{En,Decoder}() methods that may return a value
representing a pointer to the backing native codec. Previously,
a similar mechanism was used for the special case of non-public
Java codecs extending from the internal
WrappedNativeVideo{En,De}coder classes.

Tested: AppRTCMobile on Pixel XL and Pixel 2.
Bug: webrtc:9495
Change-Id: I079ff744afc7bf9873ff983e775c136a6667266d
Reviewed-on: https://webrtc-review.googlesource.com/87264
Commit-Queue: Rasmus Brandt <brandtr@webrtc.org>
Reviewed-by: Sami Kalliomäki <sakal@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#23883}

sdk/android/BUILD.gn

@@ -282,8 +282,6 @@ generate_jni("generated_video_jni") {
     "src/java/org/webrtc/VideoDecoderWrapper.java",
     "src/java/org/webrtc/VideoEncoderWrapper.java",
     "src/java/org/webrtc/WrappedNativeI420Buffer.java",
-    "src/java/org/webrtc/WrappedNativeVideoDecoder.java",
-    "src/java/org/webrtc/WrappedNativeVideoEncoder.java",
   ]
   jni_package = ""
   namespace = "webrtc::jni"
@@ -327,8 +325,6 @@ rtc_static_library("video_jni") {
     "src/jni/videotrack.cc",
     "src/jni/wrapped_native_i420_buffer.cc",
     "src/jni/wrapped_native_i420_buffer.h",
-    "src/jni/wrappednativecodec.cc",
-    "src/jni/wrappednativecodec.h",
     "src/jni/yuvhelper.cc",
   ]
 

sdk/android/api/org/webrtc/VideoDecoder.java

@@ -51,6 +51,28 @@ public interface VideoDecoder {
     void onDecodedFrame(VideoFrame frame, Integer decodeTimeMs, Integer qp);
   }
 
+  /**
+   * The decoder implementation backing this interface is either 1) a Java
+   * decoder (e.g., an Android platform decoder), or alternatively 2) a native
+   * decoder (e.g., a software decoder or a C++ decoder adapter).
+   *
+   * For case 1), createNativeVideoDecoder() should return zero.
+   * In this case, we expect the native library to call the decoder through
+   * JNI using the Java interface declared below.
+   *
+   * For case 2), createNativeVideoDecoder() should return a non-zero value.
+   * In this case, we expect the native library to treat the returned value as
+   * a raw pointer of type webrtc::VideoDecoder* (ownership is transferred to
+   * the caller). The native library should then directly call the
+   * webrtc::VideoDecoder interface without going through JNI. All calls to
+   * the Java interface methods declared below should thus throw an
+   * UnsupportedOperationException.
+   */
+  @CalledByNative
+  default long createNativeVideoDecoder() {
+    return 0;
+  }
+
   /**
    * Initializes the decoding process with specified settings. Will be called on the decoding thread
    * before any decode calls.

sdk/android/api/org/webrtc/VideoDecoderFallback.java

@@ -23,7 +23,7 @@ public class VideoDecoderFallback extends WrappedNativeVideoDecoder {
   }
 
   @Override
-  long createNativeDecoder() {
+  public long createNativeVideoDecoder() {
     return nativeCreateDecoder(fallback, primary);
   }
 

sdk/android/api/org/webrtc/VideoEncoder.java

@@ -163,6 +163,36 @@ public interface VideoEncoder {
     void onEncodedFrame(EncodedImage frame, CodecSpecificInfo info);
   }
 
+  /**
+   * The encoder implementation backing this interface is either 1) a Java
+   * encoder (e.g., an Android platform encoder), or alternatively 2) a native
+   * encoder (e.g., a software encoder or a C++ encoder adapter).
+   *
+   * For case 1), createNativeVideoEncoder() should return zero.
+   * In this case, we expect the native library to call the encoder through
+   * JNI using the Java interface declared below.
+   *
+   * For case 2), createNativeVideoEncoder() should return a non-zero value.
+   * In this case, we expect the native library to treat the returned value as
+   * a raw pointer of type webrtc::VideoEncoder* (ownership is transferred to
+   * the caller). The native library should then directly call the
+   * webrtc::VideoEncoder interface without going through JNI. All calls to
+   * the Java interface methods declared below should thus throw an
+   * UnsupportedOperationException.
+   */
+  @CalledByNative
+  default long createNativeVideoEncoder() {
+    return 0;
+  }
+
+  /**
+   * Returns true if the encoder is backed by hardware.
+   */
+  @CalledByNative
+  default boolean isHardwareEncoder() {
+    return true;
+  }
+
   /**
    * Initializes the encoding process. Call before any calls to encode.
    */

sdk/android/api/org/webrtc/VideoEncoderFallback.java

@@ -23,13 +23,13 @@ public class VideoEncoderFallback extends WrappedNativeVideoEncoder {
   }
 
   @Override
-  long createNativeEncoder() {
+  public long createNativeVideoEncoder() {
     return nativeCreateEncoder(fallback, primary);
   }
 
   @Override
-  boolean isSoftwareEncoder() {
+  public boolean isHardwareEncoder() {
-    return isWrappedSoftwareEncoder(primary);
+    return primary.isHardwareEncoder();
   }
 
   private static native long nativeCreateEncoder(VideoEncoder fallback, VideoEncoder primary);

sdk/android/src/java/org/webrtc/VP8Decoder.java

@@ -12,7 +12,7 @@ package org.webrtc;
 
 class VP8Decoder extends WrappedNativeVideoDecoder {
   @Override
-  long createNativeDecoder() {
+  public long createNativeVideoDecoder() {
     return nativeCreateDecoder();
   }
 

sdk/android/src/java/org/webrtc/VP8Encoder.java

@@ -12,14 +12,14 @@ package org.webrtc;
 
 class VP8Encoder extends WrappedNativeVideoEncoder {
   @Override
-  long createNativeEncoder() {
+  public long createNativeVideoEncoder() {
     return nativeCreateEncoder();
   }
 
   static native long nativeCreateEncoder();
 
   @Override
-  boolean isSoftwareEncoder() {
+  public boolean isHardwareEncoder() {
-    return true;
+    return false;
   }
 }

sdk/android/src/java/org/webrtc/VP9Decoder.java

@@ -12,7 +12,7 @@ package org.webrtc;
 
 class VP9Decoder extends WrappedNativeVideoDecoder {
   @Override
-  long createNativeDecoder() {
+  public long createNativeVideoDecoder() {
     return nativeCreateDecoder();
   }
 

sdk/android/src/java/org/webrtc/VP9Encoder.java

@@ -12,15 +12,15 @@ package org.webrtc;
 
 class VP9Encoder extends WrappedNativeVideoEncoder {
   @Override
-  long createNativeEncoder() {
+  public long createNativeVideoEncoder() {
     return nativeCreateEncoder();
   }
 
   static native long nativeCreateEncoder();
 
   @Override
-  boolean isSoftwareEncoder() {
+  public boolean isHardwareEncoder() {
-    return true;
+    return false;
   }
 
   static native boolean nativeIsSupported();

sdk/android/src/java/org/webrtc/WrappedNativeVideoDecoder.java

@@ -14,7 +14,7 @@ package org.webrtc;
  * Wraps a native webrtc::VideoDecoder.
  */
 abstract class WrappedNativeVideoDecoder implements VideoDecoder {
-  @CalledByNative abstract long createNativeDecoder();
+  @Override public abstract long createNativeVideoDecoder();
 
   @Override
   public VideoCodecStatus initDecode(Settings settings, Callback decodeCallback) {
@@ -40,9 +40,4 @@ abstract class WrappedNativeVideoDecoder implements VideoDecoder {
   public String getImplementationName() {
     throw new UnsupportedOperationException("Not implemented.");
   }
-
-  @CalledByNative
-  static boolean isInstanceOf(VideoDecoder decoder) {
-    return decoder instanceof WrappedNativeVideoDecoder;
-  }
 }

sdk/android/src/java/org/webrtc/WrappedNativeVideoEncoder.java

@@ -14,9 +14,8 @@ package org.webrtc;
  * Wraps a native webrtc::VideoEncoder.
  */
 abstract class WrappedNativeVideoEncoder implements VideoEncoder {
-  @CalledByNative abstract long createNativeEncoder();
+  @Override public abstract long createNativeVideoEncoder();
-
+  @Override public abstract boolean isHardwareEncoder();
-  abstract boolean isSoftwareEncoder();
 
   @Override
   public VideoCodecStatus initEncode(Settings settings, Callback encodeCallback) {
@@ -52,15 +51,4 @@ abstract class WrappedNativeVideoEncoder implements VideoEncoder {
   public String getImplementationName() {
     throw new UnsupportedOperationException("Not implemented.");
   }
-
-  @CalledByNative
-  static boolean isWrappedSoftwareEncoder(VideoEncoder encoder) {
-    return (encoder instanceof WrappedNativeVideoEncoder)
-        && ((WrappedNativeVideoEncoder) encoder).isSoftwareEncoder();
-  }
-
-  @CalledByNative
-  static boolean isInstanceOf(VideoEncoder encoder) {
-    return encoder instanceof WrappedNativeVideoEncoder;
-  }
 }

sdk/android/src/jni/videodecoderfactorywrapper.cc

@@ -17,7 +17,7 @@
 #include "sdk/android/generated_video_jni/jni/VideoDecoderFactory_jni.h"
 #include "sdk/android/native_api/jni/java_types.h"
 #include "sdk/android/src/jni/videocodecinfo.h"
-#include "sdk/android/src/jni/wrappednativecodec.h"
+#include "sdk/android/src/jni/videodecoderwrapper.h"
 
 namespace webrtc {
 namespace jni {

sdk/android/src/jni/videodecoderfallback.cc

@@ -13,7 +13,7 @@
 #include "api/video_codecs/video_decoder_software_fallback_wrapper.h"
 #include "sdk/android/generated_video_jni/jni/VideoDecoderFallback_jni.h"
 #include "sdk/android/src/jni/jni_helpers.h"
-#include "sdk/android/src/jni/wrappednativecodec.h"
+#include "sdk/android/src/jni/videodecoderwrapper.h"
 
 namespace webrtc {
 namespace jni {

sdk/android/src/jni/videodecoderwrapper.cc

@@ -263,5 +263,19 @@ absl::optional<uint8_t> VideoDecoderWrapper::ParseQP(
   return qp;
 }
 
+std::unique_ptr<VideoDecoder> JavaToNativeVideoDecoder(
+    JNIEnv* jni,
+    const JavaRef<jobject>& j_decoder) {
+  const jlong native_decoder =
+      Java_VideoDecoder_createNativeVideoDecoder(jni, j_decoder);
+  VideoDecoder* decoder;
+  if (native_decoder == 0) {
+    decoder = new VideoDecoderWrapper(jni, j_decoder);
+  } else {
+    decoder = reinterpret_cast<VideoDecoder*>(native_decoder);
+  }
+  return std::unique_ptr<VideoDecoder>(decoder);
+}
+
 }  // namespace jni
 }  // namespace webrtc

sdk/android/src/jni/videodecoderwrapper.h

@@ -110,6 +110,13 @@ class VideoDecoderWrapper : public VideoDecoder {
       RTC_GUARDED_BY(frame_extra_infos_lock_);
 };
 
+/* If the j_decoder is a wrapped native decoder, unwrap it. If it is not,
+ * wrap it in a VideoDecoderWrapper.
+ */
+std::unique_ptr<VideoDecoder> JavaToNativeVideoDecoder(
+    JNIEnv* jni,
+    const JavaRef<jobject>& j_decoder);
+
 }  // namespace jni
 }  // namespace webrtc
 

sdk/android/src/jni/videoencoderfactorywrapper.cc

@@ -17,7 +17,7 @@
 #include "sdk/android/native_api/jni/class_loader.h"
 #include "sdk/android/native_api/jni/java_types.h"
 #include "sdk/android/src/jni/videocodecinfo.h"
-#include "sdk/android/src/jni/wrappednativecodec.h"
+#include "sdk/android/src/jni/videoencoderwrapper.h"
 
 namespace webrtc {
 namespace jni {
@@ -60,7 +60,7 @@ VideoEncoderFactory::CodecInfo VideoEncoderFactoryWrapper::QueryVideoEncoder(
 
   CodecInfo codec_info;
   // Check if this is a wrapped native software encoder implementation.
-  codec_info.is_hardware_accelerated = !IsWrappedSoftwareEncoder(jni, encoder);
+  codec_info.is_hardware_accelerated = IsHardwareVideoEncoder(jni, encoder);
   codec_info.has_internal_source = false;
   return codec_info;
 }

sdk/android/src/jni/videoencoderfallback.cc

@@ -13,7 +13,7 @@
 #include "api/video_codecs/video_encoder_software_fallback_wrapper.h"
 #include "sdk/android/generated_video_jni/jni/VideoEncoderFallback_jni.h"
 #include "sdk/android/src/jni/jni_helpers.h"
-#include "sdk/android/src/jni/wrappednativecodec.h"
+#include "sdk/android/src/jni/videoencoderwrapper.h"
 
 namespace webrtc {
 namespace jni {

sdk/android/src/jni/videoencoderwrapper.cc

@@ -453,5 +453,23 @@ std::string VideoEncoderWrapper::GetImplementationName(JNIEnv* jni) const {
       jni, Java_VideoEncoder_getImplementationName(jni, encoder_));
 }
 
+std::unique_ptr<VideoEncoder> JavaToNativeVideoEncoder(
+    JNIEnv* jni,
+    const JavaRef<jobject>& j_encoder) {
+  const jlong native_encoder =
+      Java_VideoEncoder_createNativeVideoEncoder(jni, j_encoder);
+  VideoEncoder* encoder;
+  if (native_encoder == 0) {
+    encoder = new VideoEncoderWrapper(jni, j_encoder);
+  } else {
+    encoder = reinterpret_cast<VideoEncoder*>(native_encoder);
+  }
+  return std::unique_ptr<VideoEncoder>(encoder);
+}
+
+bool IsHardwareVideoEncoder(JNIEnv* jni, const JavaRef<jobject>& j_encoder) {
+  return Java_VideoEncoder_isHardwareEncoder(jni, j_encoder);
+}
+
 }  // namespace jni
 }  // namespace webrtc

sdk/android/src/jni/videoencoderwrapper.h

@@ -112,6 +112,15 @@ class VideoEncoderWrapper : public VideoEncoder {
   size_t gof_idx_;
 };
 
+/* If the j_encoder is a wrapped native encoder, unwrap it. If it is not,
+ * wrap it in a VideoEncoderWrapper.
+ */
+std::unique_ptr<VideoEncoder> JavaToNativeVideoEncoder(
+    JNIEnv* jni,
+    const JavaRef<jobject>& j_encoder);
+
+bool IsHardwareVideoEncoder(JNIEnv* jni, const JavaRef<jobject>& j_encoder);
+
 }  // namespace jni
 }  // namespace webrtc
 

sdk/android/src/jni/wrappednativecodec.cc

@@ -1,59 +0,0 @@
-/*
- *  Copyright 2017 The WebRTC project authors. All Rights Reserved.
- *
- *  Use of this source code is governed by a BSD-style license
- *  that can be found in the LICENSE file in the root of the source
- *  tree. An additional intellectual property rights grant can be found
- *  in the file PATENTS.  All contributing project authors may
- *  be found in the AUTHORS file in the root of the source tree.
- */
-
-#include "sdk/android/src/jni/wrappednativecodec.h"
-
-#include "sdk/android/generated_video_jni/jni/WrappedNativeVideoDecoder_jni.h"
-#include "sdk/android/generated_video_jni/jni/WrappedNativeVideoEncoder_jni.h"
-#include "sdk/android/native_api/jni/class_loader.h"
-#include "sdk/android/src/jni/jni_helpers.h"
-#include "sdk/android/src/jni/videodecoderwrapper.h"
-#include "sdk/android/src/jni/videoencoderwrapper.h"
-
-namespace webrtc {
-namespace jni {
-
-std::unique_ptr<VideoDecoder> JavaToNativeVideoDecoder(
-    JNIEnv* jni,
-    const JavaRef<jobject>& j_decoder) {
-  VideoDecoder* decoder;
-  if (Java_WrappedNativeVideoDecoder_isInstanceOf(jni, j_decoder)) {
-    jlong native_decoder =
-        Java_WrappedNativeVideoDecoder_createNativeDecoder(jni, j_decoder);
-    decoder = reinterpret_cast<VideoDecoder*>(native_decoder);
-  } else {
-    decoder = new VideoDecoderWrapper(jni, j_decoder);
-  }
-
-  return std::unique_ptr<VideoDecoder>(decoder);
-}
-
-std::unique_ptr<VideoEncoder> JavaToNativeVideoEncoder(
-    JNIEnv* jni,
-    const JavaRef<jobject>& j_encoder) {
-  VideoEncoder* encoder;
-  if (Java_WrappedNativeVideoEncoder_isInstanceOf(jni, j_encoder)) {
-    jlong native_encoder =
-        Java_WrappedNativeVideoEncoder_createNativeEncoder(jni, j_encoder);
-    encoder = reinterpret_cast<VideoEncoder*>(native_encoder);
-  } else {
-    encoder = new VideoEncoderWrapper(jni, j_encoder);
-  }
-
-  return std::unique_ptr<VideoEncoder>(encoder);
-}
-
-bool IsWrappedSoftwareEncoder(JNIEnv* jni, const JavaRef<jobject>& j_encoder) {
-  return Java_WrappedNativeVideoEncoder_isWrappedSoftwareEncoder(jni,
-                                                                 j_encoder);
-}
-
-}  // namespace jni
-}  // namespace webrtc

sdk/android/src/jni/wrappednativecodec.h

@@ -1,43 +0,0 @@
-/*
- *  Copyright 2017 The WebRTC project authors. All Rights Reserved.
- *
- *  Use of this source code is governed by a BSD-style license
- *  that can be found in the LICENSE file in the root of the source
- *  tree. An additional intellectual property rights grant can be found
- *  in the file PATENTS.  All contributing project authors may
- *  be found in the AUTHORS file in the root of the source tree.
- */
-
-#ifndef SDK_ANDROID_SRC_JNI_WRAPPEDNATIVECODEC_H_
-#define SDK_ANDROID_SRC_JNI_WRAPPEDNATIVECODEC_H_
-
-#include <jni.h>
-#include <memory>
-
-#include "api/video_codecs/video_decoder.h"
-#include "api/video_codecs/video_encoder.h"
-#include "sdk/android/native_api/jni/scoped_java_ref.h"
-
-namespace webrtc {
-namespace jni {
-
-/* If the j_decoder is a wrapped native decoder, unwrap it. If it is not,
- * wrap it in a VideoDecoderWrapper.
- */
-std::unique_ptr<VideoDecoder> JavaToNativeVideoDecoder(
-    JNIEnv* jni,
-    const JavaRef<jobject>& j_decoder);
-
-/* If the j_encoder is a wrapped native encoder, unwrap it. If it is not,
- * wrap it in a VideoEncoderWrapper.
- */
-std::unique_ptr<VideoEncoder> JavaToNativeVideoEncoder(
-    JNIEnv* jni,
-    const JavaRef<jobject>& j_encoder);
-
-bool IsWrappedSoftwareEncoder(JNIEnv* jni, const JavaRef<jobject>& j_encoder);
-
-}  // namespace jni
-}  // namespace webrtc
-
-#endif  // SDK_ANDROID_SRC_JNI_WRAPPEDNATIVECODEC_H_