Add resolution bitrate thresholds to EncoderInfo.

When provided, these thresholds will be used instead of WebRTC default
limits specified in DropDueToSize() and GetMaxDefaultVideoBitrateKbps().

Bug: none
Change-Id: Ida45ea832041963b8b8475d69114b5c60a172fb7
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/142170
Commit-Queue: Sergey Silkin <ssilkin@webrtc.org>
Reviewed-by: Erik Språng <sprang@webrtc.org>
Reviewed-by: Niels Moller <nisse@webrtc.org>
Reviewed-by: Alex Glaznev <glaznev@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#28390}

api/video_codecs/video_encoder.h

@@ -122,6 +122,26 @@ class RTC_EXPORT VideoEncoder {
     ScalingSettings();
   };
 
+  // Bitrate thresholds for resolution.
+  struct ResolutionBitrateThresholds {
+    ResolutionBitrateThresholds(int frame_size_pixels,
+                                int min_start_bitrate_bps,
+                                int min_bitrate_bps,
+                                int max_bitrate_bps)
+        : frame_size_pixels(frame_size_pixels),
+          min_start_bitrate_bps(min_start_bitrate_bps),
+          min_bitrate_bps(min_bitrate_bps),
+          max_bitrate_bps(max_bitrate_bps) {}
+    // Size of video frame, in pixels, the bitrate thresholds are intended for.
+    int frame_size_pixels = 0;
+    // Recommended minimum bitrate to start encoding.
+    int min_start_bitrate_bps = 0;
+    // Recommended minimum bitrate.
+    int min_bitrate_bps = 0;
+    // Recommended maximum bitrate.
+    int max_bitrate_bps = 0;
+  };
+
   // Struct containing metadata about the encoder implementing this interface.
   struct EncoderInfo {
     static constexpr uint8_t kMaxFramerateFraction =
@@ -192,6 +212,9 @@ class RTC_EXPORT VideoEncoder {
     // with a 100% frame rate fraction.
     absl::InlinedVector<uint8_t, kMaxTemporalStreams>
         fps_allocation[kMaxSpatialLayers];
+
+    // Recommended bitrate thresholds for different resolutions.
+    std::vector<ResolutionBitrateThresholds> resolution_bitrate_thresholds;
   };
 
   struct RateControlParameters {

sdk/android/BUILD.gn

@@ -1507,6 +1507,7 @@ if (is_android) {
       "native_unittests/org/webrtc/ApplicationContextProvider.java",
       "native_unittests/org/webrtc/BuildInfo.java",
       "native_unittests/org/webrtc/CodecsWrapperTestHelper.java",
+      "native_unittests/org/webrtc/FakeVideoEncoder.java",
       "native_unittests/org/webrtc/JavaTypesTestHelper.java",
       "native_unittests/org/webrtc/JavaVideoSourceTestHelper.java",
       "native_unittests/org/webrtc/PeerConnectionFactoryInitializationHelper.java",

sdk/android/api/org/webrtc/VideoEncoder.java

@@ -11,6 +11,8 @@
 package org.webrtc;
 
 import android.support.annotation.Nullable;
+import java.util.Collections;
+import java.util.List;
 import org.webrtc.EncodedImage;
 
 /**
@@ -181,6 +183,59 @@ public interface VideoEncoder {
     }
   }
 
+  /**
+   * Bitrate thresholds for resolution.
+   */
+  public class ResolutionBitrateThresholds {
+    /**
+     * Maximum size of video frame, in pixels, the bitrate thresholds are intended for.
+     */
+    public final int frameSizePixels;
+
+    /**
+     * Recommended minimum bitrate to start encoding.
+     */
+    public final int minStartBitrateBps;
+
+    /**
+     * Recommended minimum bitrate.
+     */
+    public final int minBitrateBps;
+
+    /**
+     * Recommended maximum bitrate.
+     */
+    public final int maxBitrateBps;
+
+    public ResolutionBitrateThresholds(
+        int frameSizePixels, int minStartBitrateBps, int minBitrateBps, int maxBitrateBps) {
+      this.frameSizePixels = frameSizePixels;
+      this.minStartBitrateBps = minStartBitrateBps;
+      this.minBitrateBps = minBitrateBps;
+      this.maxBitrateBps = maxBitrateBps;
+    }
+
+    @CalledByNative("ResolutionBitrateThresholds")
+    public int getFrameSizePixels() {
+      return frameSizePixels;
+    }
+
+    @CalledByNative("ResolutionBitrateThresholds")
+    public int getMinStartBitrateBps() {
+      return minStartBitrateBps;
+    }
+
+    @CalledByNative("ResolutionBitrateThresholds")
+    public int getMinBitrateBps() {
+      return minBitrateBps;
+    }
+
+    @CalledByNative("ResolutionBitrateThresholds")
+    public int getMaxBitrateBps() {
+      return maxBitrateBps;
+    }
+  }
+
   public interface Callback {
     /**
      * Call to return an encoded frame. It is safe to assume the byte buffer held by |frame| is not
@@ -240,6 +295,14 @@ public interface VideoEncoder {
   /** Any encoder that wants to use WebRTC provided quality scaler must implement this method. */
   @CalledByNative ScalingSettings getScalingSettings();
 
+  /** Returns the list of resolution bitrate thresholds. */
+  @CalledByNative
+  default ResolutionBitrateThresholds[] getResolutionBitrateThresholds() {
+    // TODO(ssilkin): Update downstream projects and remove default implementation.
+    ResolutionBitrateThresholds thresholds[] = {};
+    return thresholds;
+  }
+
   /**
    * Should return a descriptive name for the implementation. Gets called once and cached. May be
    * called from arbitrary thread.

sdk/android/native_unittests/codecs/wrapper_unittest.cc

@@ -8,9 +8,13 @@
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
-#include "sdk/android/native_api/codecs/wrapper.h"
+#include <memory>
+
+#include "absl/memory/memory.h"
 #include "media/base/media_constants.h"
 #include "sdk/android/generated_native_unittests_jni/CodecsWrapperTestHelper_jni.h"
+#include "sdk/android/native_api/codecs/wrapper.h"
+#include "sdk/android/src/jni/video_encoder_wrapper.h"
 #include "test/gtest.h"
 
 namespace webrtc {
@@ -30,6 +34,25 @@ TEST(JavaCodecsWrapperTest, JavaToNativeVideoCodecInfo) {
   ASSERT_NE(it, video_format.parameters.end());
   EXPECT_EQ(cricket::kH264ProfileLevelConstrainedBaseline, it->second);
 }
+
+TEST(JavaCodecsWrapperTest, JavaToNativeResolutionBitrateThresholds) {
+  JNIEnv* env = AttachCurrentThreadIfNeeded();
+  ScopedJavaLocalRef<jobject> j_fake_encoder =
+      jni::Java_CodecsWrapperTestHelper_createFakeVideoEncoder(env);
+
+  auto encoder = jni::JavaToNativeVideoEncoder(env, j_fake_encoder);
+  ASSERT_TRUE(encoder);
+
+  // Check that the resolution bitrate thresholds are correctly passed from Java
+  // to native.
+  const std::vector<VideoEncoder::ResolutionBitrateThresholds> thresholds =
+      encoder->GetEncoderInfo().resolution_bitrate_thresholds;
+  ASSERT_EQ(thresholds.size(), 1u);
+  EXPECT_EQ(thresholds[0].frame_size_pixels, 640 * 360);
+  EXPECT_EQ(thresholds[0].min_start_bitrate_bps, 300000);
+  EXPECT_EQ(thresholds[0].min_bitrate_bps, 200000);
+  EXPECT_EQ(thresholds[0].max_bitrate_bps, 1000000);
+}
 }  // namespace
 }  // namespace test
 }  // namespace webrtc

sdk/android/native_unittests/org/webrtc/CodecsWrapperTestHelper.java

@@ -23,4 +23,9 @@ public class CodecsWrapperTestHelper {
     VideoCodecInfo codec_info = new VideoCodecInfo("H264", params);
     return codec_info;
   }
+
+  @CalledByNative
+  public static VideoEncoder createFakeVideoEncoder() {
+    return new FakeVideoEncoder();
+  }
 }

sdk/android/native_unittests/org/webrtc/FakeVideoEncoder.java

@@ -0,0 +1,58 @@
+/*
+ *  Copyright 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+package org.webrtc;
+
+/**
+ * An implementation of VideoEncoder that is used for testing of functionalities of
+ * VideoEncoderWrapper.
+ */
+class FakeVideoEncoder implements VideoEncoder {
+  @Override
+  public VideoCodecStatus initEncode(Settings settings, Callback encodeCallback) {
+    return VideoCodecStatus.OK;
+  }
+
+  @Override
+  public VideoCodecStatus release() {
+    return VideoCodecStatus.OK;
+  }
+
+  @Override
+  public VideoCodecStatus encode(VideoFrame frame, EncodeInfo info) {
+    return VideoCodecStatus.OK;
+  }
+
+  @Override
+  public VideoCodecStatus setRateAllocation(BitrateAllocation allocation, int framerate) {
+    return VideoCodecStatus.OK;
+  }
+
+  @Override
+  public ScalingSettings getScalingSettings() {
+    return ScalingSettings.OFF;
+  }
+
+  @Override
+  public ResolutionBitrateThresholds[] getResolutionBitrateThresholds() {
+    ResolutionBitrateThresholds resolution_bitrate_thresholds[] = {
+        new ResolutionBitrateThresholds(/* frameSizePixels = */ 640 * 360,
+            /* minStartBitrateBps = */ 300000,
+            /* minBitrateBps = */ 200000,
+            /* maxBitrateBps = */ 1000000)};
+
+    return resolution_bitrate_thresholds;
+  }
+
+  @Override
+  public String getImplementationName() {
+    return "FakeVideoEncoder";
+  }
+}

sdk/android/src/jni/video_encoder_wrapper.cc

@@ -35,6 +35,11 @@ VideoEncoderWrapper::VideoEncoderWrapper(JNIEnv* jni,
     : encoder_(jni, j_encoder), int_array_class_(GetClass(jni, "[I")) {
   initialized_ = false;
   num_resets_ = 0;
+
+  // Get bitrate thresholds in the constructor. This is a static property of the
+  // encoder and is expected to be available before it is initialized.
+  encoder_info_.resolution_bitrate_thresholds =
+      GetResolutionBitrateThresholds(jni);
 }
 VideoEncoderWrapper::~VideoEncoderWrapper() = default;
 
@@ -214,6 +219,38 @@ VideoEncoderWrapper::GetScalingSettingsInternal(JNIEnv* jni) const {
   }
 }
 
+std::vector<VideoEncoder::ResolutionBitrateThresholds>
+VideoEncoderWrapper::GetResolutionBitrateThresholds(JNIEnv* jni) const {
+  std::vector<VideoEncoder::ResolutionBitrateThresholds>
+      resolution_bitrate_thresholds;
+
+  ScopedJavaLocalRef<jobjectArray> j_thresholds_array =
+      Java_VideoEncoder_getResolutionBitrateThresholds(jni, encoder_);
+
+  const jsize num_thresholds = jni->GetArrayLength(j_thresholds_array.obj());
+  for (int i = 0; i < num_thresholds; ++i) {
+    ScopedJavaLocalRef<jobject> j_thresholds = ScopedJavaLocalRef<jobject>(
+        jni, jni->GetObjectArrayElement(j_thresholds_array.obj(), i));
+
+    jint frame_size_pixels =
+        Java_ResolutionBitrateThresholds_getFrameSizePixels(jni, j_thresholds);
+    jint min_start_bitrate_bps =
+        Java_ResolutionBitrateThresholds_getMinStartBitrateBps(jni,
+                                                               j_thresholds);
+    jint min_bitrate_bps =
+        Java_ResolutionBitrateThresholds_getMinBitrateBps(jni, j_thresholds);
+    jint max_bitrate_bps =
+        Java_ResolutionBitrateThresholds_getMaxBitrateBps(jni, j_thresholds);
+
+    resolution_bitrate_thresholds.push_back(
+        VideoEncoder::ResolutionBitrateThresholds(
+            frame_size_pixels, min_start_bitrate_bps, min_bitrate_bps,
+            max_bitrate_bps));
+  }
+
+  return resolution_bitrate_thresholds;
+}
+
 void VideoEncoderWrapper::OnEncodedFrame(
     JNIEnv* jni,
     const JavaRef<jobject>& j_caller,

sdk/android/src/jni/video_encoder_wrapper.h

@@ -79,6 +79,9 @@ class VideoEncoderWrapper : public VideoEncoder {
 
   ScalingSettings GetScalingSettingsInternal(JNIEnv* jni) const;
 
+  std::vector<ResolutionBitrateThresholds> GetResolutionBitrateThresholds(
+      JNIEnv* jni) const;
+
   const ScopedJavaGlobalRef<jobject> encoder_;
   const ScopedJavaGlobalRef<jclass> int_array_class_;