Reland "Expose AV1 encoder&decoder from Android SDK."

This is a reland of fedd5029c584e9dc1352434b62a30cd8af2889d8

Original change's description:
> Expose AV1 encoder&decoder from Android SDK.
>
> Bug: None
> Change-Id: Ie32be36da498d4bed2a3cf51aa6abc8838e42da1
> Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/212024
> Reviewed-by: Xavier Lepaul‎ <xalep@webrtc.org>
> Commit-Queue: Yura Yaroshevich <yura.yaroshevich@gmail.com>
> Cr-Commit-Position: refs/heads/master@{#33743}

Bug: None
Change-Id: Ibfc7b860bd2314cf997444c7ab0d94d2b186e576
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/215586
Reviewed-by: Xavier Lepaul‎ <xalep@webrtc.org>
Commit-Queue: Yura Yaroshevich <yura.yaroshevich@gmail.com>
Cr-Commit-Position: refs/heads/master@{#33882}

examples/androidapp/res/values/arrays.xml

@@ -34,6 +34,7 @@
         <item>VP9</item>
         <item>H264 Baseline</item>
         <item>H264 High</item>
+        <item>AV1</item>
     </string-array>
 
     <string-array name="audioCodecs">

examples/androidapp/src/org/appspot/apprtc/PeerConnectionClient.java

@@ -95,6 +95,8 @@ public class PeerConnectionClient {
   private static final String VIDEO_CODEC_H264 = "H264";
   private static final String VIDEO_CODEC_H264_BASELINE = "H264 Baseline";
   private static final String VIDEO_CODEC_H264_HIGH = "H264 High";
+  private static final String VIDEO_CODEC_AV1 = "AV1";
+  private static final String VIDEO_CODEC_AV1_SDP_CODEC_NAME = "AV1X";
   private static final String AUDIO_CODEC_OPUS = "opus";
   private static final String AUDIO_CODEC_ISAC = "ISAC";
   private static final String VIDEO_CODEC_PARAM_START_BITRATE = "x-google-start-bitrate";
@@ -986,6 +988,8 @@ public class PeerConnectionClient {
         return VIDEO_CODEC_VP8;
       case VIDEO_CODEC_VP9:
         return VIDEO_CODEC_VP9;
+      case VIDEO_CODEC_AV1:
+        return VIDEO_CODEC_AV1_SDP_CODEC_NAME;
       case VIDEO_CODEC_H264_HIGH:
       case VIDEO_CODEC_H264_BASELINE:
         return VIDEO_CODEC_H264;

sdk/android/BUILD.gn

@@ -43,6 +43,7 @@ if (is_android) {
       ":filevideo_java",
       ":hwcodecs_java",
       ":java_audio_device_module_java",
+      ":libaom_av1_java",
       ":libjingle_peerconnection_java",
       ":libjingle_peerconnection_metrics_default_java",
       ":libvpx_vp8_java",
@@ -500,6 +501,20 @@ if (is_android) {
     ]
   }
 
+  rtc_android_library("libaom_av1_java") {
+    visibility = [ "*" ]
+    sources = [
+      "api/org/webrtc/LibaomAv1Decoder.java",
+      "api/org/webrtc/LibaomAv1Encoder.java",
+    ]
+    deps = [
+      ":base_java",
+      ":video_api_java",
+      ":video_java",
+      "//rtc_base:base_java",
+    ]
+  }
+
   rtc_android_library("swcodecs_java") {
     visibility = [ "*" ]
     sources = [
@@ -509,6 +524,7 @@ if (is_android) {
 
     deps = [
       ":base_java",
+      ":libaom_av1_java",
       ":libvpx_vp8_java",
       ":libvpx_vp9_java",
       ":video_api_java",
@@ -825,10 +841,24 @@ if (current_os == "linux" || is_android) {
     ]
   }
 
+  rtc_library("libaom_av1_jni") {
+    visibility = [ "*" ]
+    allow_poison = [ "software_video_codecs" ]
+    sources = [ "src/jni/av1_codec.cc" ]
+    deps = [
+      ":base_jni",
+      ":generated_libaom_av1_jni",
+      ":video_jni",
+      "../../modules/video_coding/codecs/av1:libaom_av1_decoder",
+      "../../modules/video_coding/codecs/av1:libaom_av1_encoder",
+    ]
+  }
+
   rtc_library("swcodecs_jni") {
     visibility = [ "*" ]
     allow_poison = [ "software_video_codecs" ]
     deps = [
+      ":libaom_av1_jni",
       ":libvpx_vp8_jni",
       ":libvpx_vp9_jni",
     ]
@@ -1264,6 +1294,16 @@ if (current_os == "linux" || is_android) {
     jni_generator_include = "//sdk/android/src/jni/jni_generator_helper.h"
   }
 
+  generate_jni("generated_libaom_av1_jni") {
+    sources = [
+      "api/org/webrtc/LibaomAv1Decoder.java",
+      "api/org/webrtc/LibaomAv1Encoder.java",
+    ]
+
+    namespace = "webrtc::jni"
+    jni_generator_include = "//sdk/android/src/jni/jni_generator_helper.h"
+  }
+
   generate_jni("generated_peerconnection_jni") {
     sources = [
       "api/org/webrtc/AddIceObserver.java",

sdk/android/api/org/webrtc/HardwareVideoEncoderFactory.java

@@ -137,9 +137,9 @@ public class HardwareVideoEncoderFactory implements VideoEncoderFactory {
 
     List<VideoCodecInfo> supportedCodecInfos = new ArrayList<VideoCodecInfo>();
     // Generate a list of supported codecs in order of preference:
-    // VP8, VP9, H264 (high profile), and H264 (baseline profile).
+    // VP8, VP9, H264 (high profile), H264 (baseline profile) and AV1.
-    for (VideoCodecMimeType type : new VideoCodecMimeType[] {
+    for (VideoCodecMimeType type : new VideoCodecMimeType[] {VideoCodecMimeType.VP8,
-             VideoCodecMimeType.VP8, VideoCodecMimeType.VP9, VideoCodecMimeType.H264}) {
+             VideoCodecMimeType.VP9, VideoCodecMimeType.H264, VideoCodecMimeType.AV1}) {
       MediaCodecInfo codec = findCodecForType(type);
       if (codec != null) {
         String name = type.name();
@@ -202,6 +202,8 @@ public class HardwareVideoEncoderFactory implements VideoEncoderFactory {
         return isHardwareSupportedInCurrentSdkVp9(info);
       case H264:
         return isHardwareSupportedInCurrentSdkH264(info);
+      case AV1:
+        return false;
     }
     return false;
   }
@@ -248,6 +250,7 @@ public class HardwareVideoEncoderFactory implements VideoEncoderFactory {
     switch (type) {
       case VP8: // Fallthrough intended.
       case VP9:
+      case AV1:
         return 100;
       case H264:
         return 20;

sdk/android/api/org/webrtc/LibaomAv1Decoder.java

@@ -0,0 +1,22 @@
+/*
+ *  Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+package org.webrtc;
+
+public class LibaomAv1Decoder extends WrappedNativeVideoDecoder {
+  @Override
+  public long createNativeVideoDecoder() {
+    return nativeCreateDecoder();
+  }
+
+  static native long nativeCreateDecoder();
+
+  static native boolean nativeIsSupported();
+}

sdk/android/api/org/webrtc/LibaomAv1Encoder.java

@@ -0,0 +1,27 @@
+/*
+ *  Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+package org.webrtc;
+
+public class LibaomAv1Encoder extends WrappedNativeVideoEncoder {
+  @Override
+  public long createNativeVideoEncoder() {
+    return nativeCreateEncoder();
+  }
+
+  static native long nativeCreateEncoder();
+
+  @Override
+  public boolean isHardwareEncoder() {
+    return false;
+  }
+
+  static native boolean nativeIsSupported();
+}

sdk/android/api/org/webrtc/SoftwareVideoDecoderFactory.java

@@ -32,6 +32,9 @@ public class SoftwareVideoDecoderFactory implements VideoDecoderFactory {
     if (codecType.getName().equalsIgnoreCase("VP9") && LibvpxVp9Decoder.nativeIsSupported()) {
       return new LibvpxVp9Decoder();
     }
+    if (codecType.getName().equalsIgnoreCase("AV1") && LibaomAv1Decoder.nativeIsSupported()) {
+      return new LibaomAv1Decoder();
+    }
 
     return null;
   }
@@ -48,6 +51,9 @@ public class SoftwareVideoDecoderFactory implements VideoDecoderFactory {
     if (LibvpxVp9Decoder.nativeIsSupported()) {
       codecs.add(new VideoCodecInfo("VP9", new HashMap<>()));
     }
+    if (LibaomAv1Decoder.nativeIsSupported()) {
+      codecs.add(new VideoCodecInfo("AV1", new HashMap<>()));
+    }
 
     return codecs.toArray(new VideoCodecInfo[codecs.size()]);
   }

sdk/android/api/org/webrtc/SoftwareVideoEncoderFactory.java

@@ -25,6 +25,9 @@ public class SoftwareVideoEncoderFactory implements VideoEncoderFactory {
     if (info.name.equalsIgnoreCase("VP9") && LibvpxVp9Encoder.nativeIsSupported()) {
       return new LibvpxVp9Encoder();
     }
+    if (info.name.equalsIgnoreCase("AV1") && LibaomAv1Encoder.nativeIsSupported()) {
+      return new LibaomAv1Encoder();
+    }
 
     return null;
   }
@@ -41,6 +44,9 @@ public class SoftwareVideoEncoderFactory implements VideoEncoderFactory {
     if (LibvpxVp9Encoder.nativeIsSupported()) {
       codecs.add(new VideoCodecInfo("VP9", new HashMap<>()));
     }
+    if (LibaomAv1Encoder.nativeIsSupported()) {
+      codecs.add(new VideoCodecInfo("AV1", new HashMap<>()));
+    }
 
     return codecs.toArray(new VideoCodecInfo[codecs.size()]);
   }

sdk/android/api/org/webrtc/VideoEncoder.java

@@ -86,6 +86,8 @@ public interface VideoEncoder {
 
   public class CodecSpecificInfoH264 extends CodecSpecificInfo {}
 
+  public class CodecSpecificInfoAV1 extends CodecSpecificInfo {}
+
   /**
    * Represents bitrate allocated for an encoder to produce frames. Bitrate can be divided between
    * spatial and temporal layers.

sdk/android/instrumentationtests/src/org/webrtc/DefaultVideoEncoderFactoryTest.java

@@ -70,13 +70,14 @@ public class DefaultVideoEncoderFactoryTest {
     VideoEncoderFactory hwFactory = new CustomHardwareVideoEncoderFactory(true, true);
     DefaultVideoEncoderFactory dvef = new DefaultVideoEncoderFactory(hwFactory);
     VideoCodecInfo[] videoCodecs = dvef.getSupportedCodecs();
-    assertEquals(4, videoCodecs.length);
+    assertEquals(5, videoCodecs.length);
     assertEquals("VP8", videoCodecs[0].name);
     assertEquals("VP9", videoCodecs[1].name);
-    assertEquals("H264", videoCodecs[2].name);
+    assertEquals("AV1", videoCodecs[2].name);
-    assertEquals("42e01f", videoCodecs[2].params.get("profile-level-id"));
     assertEquals("H264", videoCodecs[3].name);
-    assertEquals("640c1f", videoCodecs[3].params.get("profile-level-id"));
+    assertEquals("42e01f", videoCodecs[3].params.get("profile-level-id"));
+    assertEquals("H264", videoCodecs[4].name);
+    assertEquals("640c1f", videoCodecs[4].params.get("profile-level-id"));
   }
 
   @SmallTest
@@ -85,11 +86,12 @@ public class DefaultVideoEncoderFactoryTest {
     VideoEncoderFactory hwFactory = new CustomHardwareVideoEncoderFactory(true, false);
     DefaultVideoEncoderFactory dvef = new DefaultVideoEncoderFactory(hwFactory);
     VideoCodecInfo[] videoCodecs = dvef.getSupportedCodecs();
-    assertEquals(3, videoCodecs.length);
+    assertEquals(4, videoCodecs.length);
     assertEquals("VP8", videoCodecs[0].name);
     assertEquals("VP9", videoCodecs[1].name);
-    assertEquals("H264", videoCodecs[2].name);
+    assertEquals("AV1", videoCodecs[2].name);
-    assertEquals("42e01f", videoCodecs[2].params.get("profile-level-id"));
+    assertEquals("H264", videoCodecs[3].name);
+    assertEquals("42e01f", videoCodecs[3].params.get("profile-level-id"));
   }
 
   @SmallTest
@@ -98,12 +100,13 @@ public class DefaultVideoEncoderFactoryTest {
     VideoEncoderFactory hwFactory = new CustomHardwareVideoEncoderFactory(false, true);
     DefaultVideoEncoderFactory dvef = new DefaultVideoEncoderFactory(hwFactory);
     VideoCodecInfo[] videoCodecs = dvef.getSupportedCodecs();
-    assertEquals(4, videoCodecs.length);
+    assertEquals(5, videoCodecs.length);
     assertEquals("VP8", videoCodecs[0].name);
     assertEquals("VP9", videoCodecs[1].name);
-    assertEquals("H264", videoCodecs[2].name);
+    assertEquals("AV1", videoCodecs[2].name);
-    assertEquals("42e01f", videoCodecs[2].params.get("profile-level-id"));
     assertEquals("H264", videoCodecs[3].name);
-    assertEquals("640c1f", videoCodecs[3].params.get("profile-level-id"));
+    assertEquals("42e01f", videoCodecs[3].params.get("profile-level-id"));
+    assertEquals("H264", videoCodecs[4].name);
+    assertEquals("640c1f", videoCodecs[4].params.get("profile-level-id"));
   }
 }

sdk/android/src/java/org/webrtc/HardwareVideoEncoder.java

@@ -169,7 +169,7 @@ class HardwareVideoEncoder implements VideoEncoder {
    * intervals, and bitrateAdjuster.
    *
    * @param codecName the hardware codec implementation to use
-   * @param codecType the type of the given video codec (eg. VP8, VP9, or H264)
+   * @param codecType the type of the given video codec (eg. VP8, VP9, H264 or AV1)
    * @param surfaceColorFormat color format for surface mode or null if not available
    * @param yuvColorFormat color format for bytebuffer mode
    * @param keyFrameIntervalSec interval in seconds between key frames; used to initialize the codec

sdk/android/src/java/org/webrtc/MediaCodecUtils.java

@@ -91,6 +91,7 @@ class MediaCodecUtils {
     switch (type) {
       case VP8:
       case VP9:
+      case AV1:
         return new HashMap<String, String>();
       case H264:
         return H264Utils.getDefaultH264Params(highProfile);

sdk/android/src/java/org/webrtc/MediaCodecVideoDecoderFactory.java

@@ -64,8 +64,8 @@ class MediaCodecVideoDecoderFactory implements VideoDecoderFactory {
     List<VideoCodecInfo> supportedCodecInfos = new ArrayList<VideoCodecInfo>();
     // Generate a list of supported codecs in order of preference:
     // VP8, VP9, H264 (high profile), and H264 (baseline profile).
-    for (VideoCodecMimeType type : new VideoCodecMimeType[] {
+    for (VideoCodecMimeType type : new VideoCodecMimeType[] {VideoCodecMimeType.VP8,
-             VideoCodecMimeType.VP8, VideoCodecMimeType.VP9, VideoCodecMimeType.H264}) {
+             VideoCodecMimeType.VP9, VideoCodecMimeType.H264, VideoCodecMimeType.AV1}) {
       MediaCodecInfo codec = findCodecForType(type);
       if (codec != null) {
         String name = type.name();

sdk/android/src/java/org/webrtc/VideoCodecMimeType.java

@@ -14,7 +14,8 @@ package org.webrtc;
 enum VideoCodecMimeType {
   VP8("video/x-vnd.on2.vp8"),
   VP9("video/x-vnd.on2.vp9"),
-  H264("video/avc");
+  H264("video/avc"),
+  AV1("video/av01");
 
   private final String mimeType;
 

sdk/android/src/jni/av1_codec.cc

@@ -0,0 +1,39 @@
+/*
+ *  Copyright 2021 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include <jni.h>
+
+#include "modules/video_coding/codecs/av1/libaom_av1_decoder.h"
+#include "modules/video_coding/codecs/av1/libaom_av1_encoder.h"
+#include "sdk/android/generated_libaom_av1_jni/LibaomAv1Decoder_jni.h"
+#include "sdk/android/generated_libaom_av1_jni/LibaomAv1Encoder_jni.h"
+#include "sdk/android/src/jni/jni_helpers.h"
+
+namespace webrtc {
+namespace jni {
+
+static jlong JNI_LibaomAv1Encoder_CreateEncoder(JNIEnv* jni) {
+  return jlongFromPointer(webrtc::CreateLibaomAv1Encoder().release());
+}
+
+static jboolean JNI_LibaomAv1Encoder_IsSupported(JNIEnv* jni) {
+  return webrtc::kIsLibaomAv1EncoderSupported;
+}
+
+static jlong JNI_LibaomAv1Decoder_CreateDecoder(JNIEnv* jni) {
+  return jlongFromPointer(webrtc::CreateLibaomAv1Decoder().release());
+}
+
+static jboolean JNI_LibaomAv1Decoder_IsSupported(JNIEnv* jni) {
+  return webrtc::kIsLibaomAv1DecoderSupported;
+}
+
+}  // namespace jni
+}  // namespace webrtc

sdk/android/src/jni/video_codec_info.cc

@@ -19,18 +19,33 @@ namespace jni {
 
 SdpVideoFormat VideoCodecInfoToSdpVideoFormat(JNIEnv* jni,
                                               const JavaRef<jobject>& j_info) {
+  std::string codecName =
+      JavaToNativeString(jni, Java_VideoCodecInfo_getName(jni, j_info));
+  std::string sdpCodecName;
+  if (codecName == "AV1") {
+    // TODO(yyaroshevich): Undo mapping once AV1 sdp name is standardized
+    sdpCodecName = "AV1X";
+  } else {
+    sdpCodecName = codecName;
+  }
   return SdpVideoFormat(
-      JavaToNativeString(jni, Java_VideoCodecInfo_getName(jni, j_info)),
+      sdpCodecName,
       JavaToNativeStringMap(jni, Java_VideoCodecInfo_getParams(jni, j_info)));
 }
 
 ScopedJavaLocalRef<jobject> SdpVideoFormatToVideoCodecInfo(
     JNIEnv* jni,
     const SdpVideoFormat& format) {
+  std::string codecName;
+  if (format.name == "AV1X" || format.name == "AV1") {
+    codecName = "AV1";
+  } else {
+    codecName = format.name;
+  }
   ScopedJavaLocalRef<jobject> j_params =
       NativeToJavaStringMap(jni, format.parameters);
   return Java_VideoCodecInfo_Constructor(
-      jni, NativeToJavaString(jni, format.name), j_params);
+      jni, NativeToJavaString(jni, codecName), j_params);
 }
 
 }  // namespace jni