Injectable software codecs for Android.

This is similar to https://webrtc-review.googlesource.com/c/src/+/3620
for iOS.

Using the new WebRtcMediaEngineFactory::Create API, the built-in
software video codecs are no longer appended to the injected codecs.

To be able to use the software codecs, they are exposed as Java
classes through SoftwareVideoEncoderFactory etc.

There is also a new DefaultVideoEncoderFactory used by AppRTCMobile.
This factory tries to use hardware implementations where available,
but falls back to using the injected software codecs.

The HardwareVideoEncoderFactory is temporarily also falling back on
the software codecs in its default configuration in order to
maintain backwards compatibility.

Bug: webrtc:7925
Change-Id: I3e8c5ed492ccd160aca968986ad217d7978a951c
Reviewed-on: https://webrtc-review.googlesource.com/17480
Reviewed-by: Sami Kalliomäki <sakal@webrtc.org>
Reviewed-by: Magnus Jedvert <magjed@webrtc.org>
Commit-Queue: Anders Carlsson <andersc@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#20647}

sdk/android/api/org/webrtc/DefaultVideoDecoderFactory.java

@@ -0,0 +1,31 @@
+/*
+ *  Copyright 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+package org.webrtc;
+
+public class DefaultVideoDecoderFactory implements VideoDecoderFactory {
+  private final HardwareVideoDecoderFactory hardwareVideoDecoderFactory;
+  private final SoftwareVideoDecoderFactory softwareVideoDecoderFactory;
+
+  public DefaultVideoDecoderFactory(EglBase.Context eglContext) {
+    hardwareVideoDecoderFactory =
+        new HardwareVideoDecoderFactory(eglContext, false /* fallbackToSoftware */);
+    softwareVideoDecoderFactory = new SoftwareVideoDecoderFactory();
+  }
+
+  @Override
+  public VideoDecoder createDecoder(String codecType) {
+    VideoDecoder decoder = hardwareVideoDecoderFactory.createDecoder(codecType);
+    if (decoder != null) {
+      return decoder;
+    }
+    return softwareVideoDecoderFactory.createDecoder(codecType);
+  }
+}

sdk/android/api/org/webrtc/DefaultVideoEncoderFactory.java

@@ -0,0 +1,70 @@
+/*
+ *  Copyright 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+package org.webrtc;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+public class DefaultVideoEncoderFactory implements VideoEncoderFactory {
+  private final VideoEncoderFactory hardwareVideoEncoderFactory;
+  private final VideoEncoderFactory softwareVideoEncoderFactory;
+
+  public DefaultVideoEncoderFactory(
+      EglBase.Context eglContext, boolean enableIntelVp8Encoder, boolean enableH264HighProfile) {
+    hardwareVideoEncoderFactory = new HardwareVideoEncoderFactory(
+        eglContext, enableIntelVp8Encoder, enableH264HighProfile, false /* fallbackToSoftware */);
+    softwareVideoEncoderFactory = new SoftwareVideoEncoderFactory();
+  }
+
+  /* This is used for testing. */
+  DefaultVideoEncoderFactory(VideoEncoderFactory hardwareVideoEncoderFactory) {
+    this.hardwareVideoEncoderFactory = hardwareVideoEncoderFactory;
+    softwareVideoEncoderFactory = new SoftwareVideoEncoderFactory();
+  }
+
+  @Override
+  public VideoEncoder createEncoder(VideoCodecInfo info) {
+    List<VideoCodecInfo> hardwareSupportedCodecs =
+        Arrays.asList(hardwareVideoEncoderFactory.getSupportedCodecs());
+    if (containsSameCodec(hardwareSupportedCodecs, info)) {
+      return hardwareVideoEncoderFactory.createEncoder(info);
+    } else {
+      return softwareVideoEncoderFactory.createEncoder(info);
+    }
+  }
+
+  @Override
+  public VideoCodecInfo[] getSupportedCodecs() {
+    List<VideoCodecInfo> supportedCodecInfos = new ArrayList<>();
+
+    supportedCodecInfos.addAll(Arrays.asList(softwareVideoEncoderFactory.getSupportedCodecs()));
+
+    for (VideoCodecInfo info : hardwareVideoEncoderFactory.getSupportedCodecs()) {
+      if (!containsSameCodec(supportedCodecInfos, info)) {
+        supportedCodecInfos.add(info);
+      }
+    }
+
+    return supportedCodecInfos.toArray(new VideoCodecInfo[supportedCodecInfos.size()]);
+  }
+
+  private static boolean containsSameCodec(List<VideoCodecInfo> infos, VideoCodecInfo info) {
+    for (VideoCodecInfo otherInfo : infos) {
+      if (isSameCodec(info, otherInfo)) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  private static native boolean isSameCodec(VideoCodecInfo info1, VideoCodecInfo info2);
+}

sdk/android/api/org/webrtc/HardwareVideoDecoderFactory.java

@@ -15,7 +15,6 @@ import static org.webrtc.MediaCodecUtils.INTEL_PREFIX;
 import static org.webrtc.MediaCodecUtils.NVIDIA_PREFIX;
 import static org.webrtc.MediaCodecUtils.QCOM_PREFIX;
 
-import android.media.MediaCodec;
 import android.media.MediaCodecInfo;
 import android.media.MediaCodecInfo.CodecCapabilities;
 import android.media.MediaCodecList;
@@ -27,6 +26,7 @@ public class HardwareVideoDecoderFactory implements VideoDecoderFactory {
   private static final String TAG = "HardwareVideoDecoderFactory";
 
   private final EglBase.Context sharedContext;
+  private final boolean fallbackToSoftware;
 
   /** Creates a HardwareVideoDecoderFactory that does not use surface textures. */
   @Deprecated // Not removed yet to avoid breaking callers.
@@ -39,7 +39,12 @@ public class HardwareVideoDecoderFactory implements VideoDecoderFactory {
    * shared context.  The context may be null.  If it is null, then surface support is disabled.
    */
   public HardwareVideoDecoderFactory(EglBase.Context sharedContext) {
+    this(sharedContext, true /* fallbackToSoftware */);
+  }
+
+  HardwareVideoDecoderFactory(EglBase.Context sharedContext, boolean fallbackToSoftware) {
     this.sharedContext = sharedContext;
+    this.fallbackToSoftware = fallbackToSoftware;
   }
 
   @Override
@@ -48,7 +53,15 @@ public class HardwareVideoDecoderFactory implements VideoDecoderFactory {
     MediaCodecInfo info = findCodecForType(type);
 
     if (info == null) {
-      return null; // No support for this codec type.
+      // No hardware support for this type.
+      // TODO(andersc): This is for backwards compatibility. Remove when clients have migrated to
+      // new DefaultVideoEncoderFactory.
+      if (fallbackToSoftware) {
+        SoftwareVideoDecoderFactory softwareVideoDecoderFactory = new SoftwareVideoDecoderFactory();
+        return softwareVideoDecoderFactory.createDecoder(codecType);
+      } else {
+        return null;
+      }
     }
 
     CodecCapabilities capabilities = info.getCapabilitiesForType(type.mimeType());

sdk/android/api/org/webrtc/HardwareVideoEncoderFactory.java

@@ -42,9 +42,16 @@ public class HardwareVideoEncoderFactory implements VideoEncoderFactory {
   private final EglBase14.Context sharedContext;
   private final boolean enableIntelVp8Encoder;
   private final boolean enableH264HighProfile;
+  private final boolean fallbackToSoftware;
 
   public HardwareVideoEncoderFactory(
       EglBase.Context sharedContext, boolean enableIntelVp8Encoder, boolean enableH264HighProfile) {
+    this(
+        sharedContext, enableIntelVp8Encoder, enableH264HighProfile, true /* fallbackToSoftware */);
+  }
+
+  HardwareVideoEncoderFactory(EglBase.Context sharedContext, boolean enableIntelVp8Encoder,
+      boolean enableH264HighProfile, boolean fallbackToSoftware) {
     // Texture mode requires EglBase14.
     if (sharedContext instanceof EglBase14.Context) {
       this.sharedContext = (EglBase14.Context) sharedContext;
@@ -54,6 +61,7 @@ public class HardwareVideoEncoderFactory implements VideoEncoderFactory {
     }
     this.enableIntelVp8Encoder = enableIntelVp8Encoder;
     this.enableH264HighProfile = enableH264HighProfile;
+    this.fallbackToSoftware = fallbackToSoftware;
   }
 
   @Deprecated
@@ -67,7 +75,15 @@ public class HardwareVideoEncoderFactory implements VideoEncoderFactory {
     MediaCodecInfo info = findCodecForType(type);
 
     if (info == null) {
-      return null; // No support for this type.
+      // No hardware support for this type.
+      // TODO(andersc): This is for backwards compatibility. Remove when clients have migrated to
+      // new DefaultVideoEncoderFactory.
+      if (fallbackToSoftware) {
+        SoftwareVideoEncoderFactory softwareVideoEncoderFactory = new SoftwareVideoEncoderFactory();
+        return softwareVideoEncoderFactory.createEncoder(input);
+      } else {
+        return null;
+      }
     }
 
     String codecName = info.getName();
@@ -93,12 +109,23 @@ public class HardwareVideoEncoderFactory implements VideoEncoderFactory {
       if (codec != null) {
         String name = type.name();
         if (type == VideoCodecType.H264 && isH264HighProfileSupported(codec)) {
-          supportedCodecInfos.add(new VideoCodecInfo(0, name, getCodecProperties(type, true)));
+          supportedCodecInfos.add(new VideoCodecInfo(name, getCodecProperties(type, true)));
         }
 
-        supportedCodecInfos.add(new VideoCodecInfo(0, name, getCodecProperties(type, false)));
+        supportedCodecInfos.add(new VideoCodecInfo(name, getCodecProperties(type, false)));
       }
     }
+
+    // TODO(andersc): This is for backwards compatibility. Remove when clients have migrated to
+    // new DefaultVideoEncoderFactory.
+    if (fallbackToSoftware) {
+      for (VideoCodecInfo info : SoftwareVideoEncoderFactory.supportedCodecs()) {
+        if (!supportedCodecInfos.contains(info)) {
+          supportedCodecInfos.add(info);
+        }
+      }
+    }
+
     return supportedCodecInfos.toArray(new VideoCodecInfo[supportedCodecInfos.size()]);
   }
 

sdk/android/api/org/webrtc/SoftwareVideoDecoderFactory.java

@@ -0,0 +1,25 @@
+/*
+ *  Copyright 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+package org.webrtc;
+
+public class SoftwareVideoDecoderFactory implements VideoDecoderFactory {
+  @Override
+  public VideoDecoder createDecoder(String codecType) {
+    if (codecType.equalsIgnoreCase("VP8")) {
+      return new VP8Decoder();
+    }
+    if (codecType.equalsIgnoreCase("VP9")) {
+      return new VP9Decoder();
+    }
+
+    return null;
+  }
+}

sdk/android/api/org/webrtc/SoftwareVideoEncoderFactory.java

@@ -0,0 +1,39 @@
+/*
+ *  Copyright 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+package org.webrtc;
+
+import java.util.HashMap;
+
+public class SoftwareVideoEncoderFactory implements VideoEncoderFactory {
+  @Override
+  public VideoEncoder createEncoder(VideoCodecInfo info) {
+    if (info.name.equalsIgnoreCase("VP8")) {
+      return new VP8Encoder();
+    }
+    if (info.name.equalsIgnoreCase("VP9")) {
+      return new VP9Encoder();
+    }
+
+    return null;
+  }
+
+  @Override
+  public VideoCodecInfo[] getSupportedCodecs() {
+    return supportedCodecs();
+  }
+
+  public static VideoCodecInfo[] supportedCodecs() {
+    VideoCodecInfo vp8Info = new VideoCodecInfo("VP8", new HashMap<>());
+    VideoCodecInfo vp9Info = new VideoCodecInfo("VP9", new HashMap<>());
+
+    return new VideoCodecInfo[] {vp8Info, vp9Info};
+  }
+}

sdk/android/api/org/webrtc/VideoCodecInfo.java

@@ -29,10 +29,17 @@ public class VideoCodecInfo {
   public static final String H264_CONSTRAINED_BASELINE_3_1 =
       H264_PROFILE_CONSTRAINED_BASELINE + H264_LEVEL_3_1;
 
-  public final int payload;
   public final String name;
   public final Map<String, String> params;
+  @Deprecated public final int payload;
 
+  public VideoCodecInfo(String name, Map<String, String> params) {
+    this.payload = 0;
+    this.name = name;
+    this.params = params;
+  }
+
+  @Deprecated
   public VideoCodecInfo(int payload, String name, Map<String, String> params) {
     this.payload = payload;
     this.name = name;

sdk/android/api/org/webrtc/VideoDecoderFallback.java

@@ -0,0 +1,22 @@
+/*
+ *  Copyright 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+package org.webrtc;
+
+/**
+ * A combined video decoder that falls back on a secondary decoder if the primary decoder fails.
+ */
+public class VideoDecoderFallback extends WrappedNativeVideoDecoder {
+  public VideoDecoderFallback(VideoDecoder fallback, VideoDecoder primary) {
+    super(createNativeDecoder(fallback, primary));
+  }
+
+  private static native long createNativeDecoder(VideoDecoder fallback, VideoDecoder primary);
+}