Non-integer frame rate in Android HW encoder

Bug: webrtc:10812
Change-Id: I4443dcfea851114bd5fbb10f11ca8a51cda12da8
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/229025
Commit-Queue: Sergey Silkin <ssilkin@webrtc.org>
Reviewed-by: Erik Språng <sprang@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#34813}

sdk/android/BUILD.gn

@@ -1615,6 +1615,7 @@ if (is_android) {
       "tests/src/org/webrtc/CodecTestHelper.java",
       "tests/src/org/webrtc/CryptoOptionsTest.java",
       "tests/src/org/webrtc/FakeMediaCodecWrapper.java",
+      "tests/src/org/webrtc/FramerateBitrateAdjusterTest.java",
       "tests/src/org/webrtc/GlGenericDrawerTest.java",
       "tests/src/org/webrtc/HardwareVideoEncoderTest.java",
       "tests/src/org/webrtc/IceCandidateTest.java",

sdk/android/api/org/webrtc/VideoEncoder.java

@@ -236,6 +236,28 @@ public interface VideoEncoder {
     }
   }
 
+  /** Rate control parameters. */
+  public class RateControlParameters {
+    /**
+     * Adjusted target bitrate, per spatial/temporal layer. May be lower or higher than the target
+     * depending on encoder behaviour.
+     */
+    public final BitrateAllocation bitrate;
+
+    /**
+     * Target framerate, in fps. A value <= 0.0 is invalid and should be interpreted as framerate
+     * target not available. In this case the encoder should fall back to the max framerate
+     * specified in `codec_settings` of the last InitEncode() call.
+     */
+    public final double framerateFps;
+
+    @CalledByNative("RateControlParameters")
+    public RateControlParameters(BitrateAllocation bitrate, double framerateFps) {
+      this.bitrate = bitrate;
+      this.framerateFps = framerateFps;
+    }
+  }
+
   public interface Callback {
     /**
      * Old encoders assume that the byte buffer held by `frame` is not accessed after the call to
@@ -296,7 +318,14 @@ public interface VideoEncoder {
   @CalledByNative VideoCodecStatus encode(VideoFrame frame, EncodeInfo info);
 
   /** Sets the bitrate allocation and the target framerate for the encoder. */
-  @CalledByNative VideoCodecStatus setRateAllocation(BitrateAllocation allocation, int framerate);
+  VideoCodecStatus setRateAllocation(BitrateAllocation allocation, int framerate);
+
+  /** Sets the bitrate allocation and the target framerate for the encoder. */
+  default @CalledByNative VideoCodecStatus setRates(RateControlParameters rcParameters) {
+    // Round frame rate up to avoid overshoots.
+    int framerateFps = (int) Math.ceil(rcParameters.framerateFps);
+    return setRateAllocation(rcParameters.bitrate, framerateFps);
+  }
 
   /** Any encoder that wants to use WebRTC provided quality scaler must implement this method. */
   @CalledByNative ScalingSettings getScalingSettings();

sdk/android/src/java/org/webrtc/BaseBitrateAdjuster.java

@@ -13,10 +13,10 @@ package org.webrtc;
 /** BitrateAdjuster that tracks bitrate and framerate but does not adjust them. */
 class BaseBitrateAdjuster implements BitrateAdjuster {
   protected int targetBitrateBps;
-  protected int targetFramerateFps;
+  protected double targetFramerateFps;
 
   @Override
-  public void setTargets(int targetBitrateBps, int targetFramerateFps) {
+  public void setTargets(int targetBitrateBps, double targetFramerateFps) {
     this.targetBitrateBps = targetBitrateBps;
     this.targetFramerateFps = targetFramerateFps;
   }
@@ -32,7 +32,7 @@ class BaseBitrateAdjuster implements BitrateAdjuster {
   }
 
   @Override
-  public int getAdjustedFramerateFps() {
+  public double getAdjustedFramerateFps() {
     return targetFramerateFps;
   }
 }

sdk/android/src/java/org/webrtc/BitrateAdjuster.java

@@ -15,7 +15,7 @@ interface BitrateAdjuster {
   /**
    * Sets the target bitrate in bits per second and framerate in frames per second.
    */
-  void setTargets(int targetBitrateBps, int targetFramerateFps);
+  void setTargets(int targetBitrateBps, double targetFramerateFps);
 
   /**
    * Should be used to report the size of an encoded frame to the bitrate adjuster. Use
@@ -27,5 +27,5 @@ interface BitrateAdjuster {
   int getAdjustedBitrateBps();
 
   /** Gets the current framerate. */
-  int getAdjustedFramerateFps();
+  double getAdjustedFramerateFps();
 }

sdk/android/src/java/org/webrtc/DynamicBitrateAdjuster.java

@@ -31,7 +31,7 @@ class DynamicBitrateAdjuster extends BaseBitrateAdjuster {
   private int bitrateAdjustmentScaleExp;
 
   @Override
-  public void setTargets(int targetBitrateBps, int targetFramerateFps) {
+  public void setTargets(int targetBitrateBps, double targetFramerateFps) {
     if (this.targetBitrateBps > 0 && targetBitrateBps < this.targetBitrateBps) {
       // Rescale the accumulator level if the accumulator max decreases
       deviationBytes = deviationBytes * targetBitrateBps / this.targetBitrateBps;

sdk/android/src/java/org/webrtc/FramerateBitrateAdjuster.java

@@ -18,9 +18,9 @@ class FramerateBitrateAdjuster extends BaseBitrateAdjuster {
   private static final int DEFAULT_FRAMERATE_FPS = 30;
 
   @Override
-  public void setTargets(int targetBitrateBps, int targetFramerateFps) {
+  public void setTargets(int targetBitrateBps, double targetFramerateFps) {
     // Keep frame rate unchanged and adjust bit rate.
     this.targetFramerateFps = DEFAULT_FRAMERATE_FPS;
-    this.targetBitrateBps = targetBitrateBps * DEFAULT_FRAMERATE_FPS / targetFramerateFps;
+    this.targetBitrateBps = (int) (targetBitrateBps * DEFAULT_FRAMERATE_FPS / targetFramerateFps);
   }
 }

sdk/android/src/java/org/webrtc/HardwareVideoEncoder.java

@@ -241,7 +241,8 @@ class HardwareVideoEncoder implements VideoEncoder {
       format.setInteger(MediaFormat.KEY_BIT_RATE, adjustedBitrate);
       format.setInteger(KEY_BITRATE_MODE, VIDEO_ControlRateConstant);
       format.setInteger(MediaFormat.KEY_COLOR_FORMAT, colorFormat);
-      format.setInteger(MediaFormat.KEY_FRAME_RATE, bitrateAdjuster.getAdjustedFramerateFps());
+      format.setFloat(
+          MediaFormat.KEY_FRAME_RATE, (float) bitrateAdjuster.getAdjustedFramerateFps());
       format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, keyFrameIntervalSec);
       if (codecType == VideoCodecMimeType.H264) {
         String profileLevelId = params.get(VideoCodecInfo.H264_FMTP_PROFILE_LEVEL_ID);
@@ -468,6 +469,13 @@ class HardwareVideoEncoder implements VideoEncoder {
     return VideoCodecStatus.OK;
   }
 
+  @Override
+  public VideoCodecStatus setRates(RateControlParameters rcParameters) {
+    encodeThreadChecker.checkIsOnValidThread();
+    bitrateAdjuster.setTargets(rcParameters.bitrate.getSum(), rcParameters.framerateFps);
+    return VideoCodecStatus.OK;
+  }
+
   @Override
   public ScalingSettings getScalingSettings() {
     encodeThreadChecker.checkIsOnValidThread();

sdk/android/src/jni/video_encoder_wrapper.cc

@@ -167,15 +167,14 @@ int32_t VideoEncoderWrapper::Encode(
   return HandleReturnCode(jni, ret, "encode");
 }
 
-void VideoEncoderWrapper::SetRates(const RateControlParameters& parameters) {
+void VideoEncoderWrapper::SetRates(const RateControlParameters& rc_parameters) {
   JNIEnv* jni = AttachCurrentThreadIfNeeded();
 
-  ScopedJavaLocalRef<jobject> j_bitrate_allocation =
+  ScopedJavaLocalRef<jobject> j_rc_parameters =
-      ToJavaBitrateAllocation(jni, parameters.bitrate);
+      ToJavaRateControlParameters(jni, rc_parameters);
-  ScopedJavaLocalRef<jobject> ret = Java_VideoEncoder_setRateAllocation(
+  ScopedJavaLocalRef<jobject> ret =
-      jni, encoder_, j_bitrate_allocation,
+      Java_VideoEncoder_setRates(jni, encoder_, j_rc_parameters);
-      (jint)(parameters.framerate_fps + 0.5));
+  HandleReturnCode(jni, ret, "setRates");
-  HandleReturnCode(jni, ret, "setRateAllocation");
 }
 
 VideoEncoder::EncoderInfo VideoEncoderWrapper::GetEncoderInfo() const {
@@ -410,6 +409,16 @@ ScopedJavaLocalRef<jobject> VideoEncoderWrapper::ToJavaBitrateAllocation(
   return Java_BitrateAllocation_Constructor(jni, j_allocation_array);
 }
 
+ScopedJavaLocalRef<jobject> VideoEncoderWrapper::ToJavaRateControlParameters(
+    JNIEnv* jni,
+    const VideoEncoder::RateControlParameters& rc_parameters) {
+  ScopedJavaLocalRef<jobject> j_bitrate_allocation =
+      ToJavaBitrateAllocation(jni, rc_parameters.bitrate);
+
+  return Java_RateControlParameters_Constructor(jni, j_bitrate_allocation,
+                                                rc_parameters.framerate_fps);
+}
+
 std::unique_ptr<VideoEncoder> JavaToNativeVideoEncoder(
     JNIEnv* jni,
     const JavaRef<jobject>& j_encoder) {

sdk/android/src/jni/video_encoder_wrapper.h

@@ -46,7 +46,7 @@ class VideoEncoderWrapper : public VideoEncoder {
   int32_t Encode(const VideoFrame& frame,
                  const std::vector<VideoFrameType>* frame_types) override;
 
-  void SetRates(const RateControlParameters& parameters) override;
+  void SetRates(const RateControlParameters& rc_parameters) override;
 
   EncoderInfo GetEncoderInfo() const override;
 
@@ -70,12 +70,19 @@ class VideoEncoderWrapper : public VideoEncoder {
                            const char* method_name);
 
   int ParseQp(rtc::ArrayView<const uint8_t> buffer);
+
   CodecSpecificInfo ParseCodecSpecificInfo(const EncodedImage& frame);
+
   ScopedJavaLocalRef<jobject> ToJavaBitrateAllocation(
       JNIEnv* jni,
       const VideoBitrateAllocation& allocation);
 
+  ScopedJavaLocalRef<jobject> ToJavaRateControlParameters(
+      JNIEnv* jni,
+      const VideoEncoder::RateControlParameters& rc_parameters);
+
   void UpdateEncoderInfo(JNIEnv* jni);
+
   ScalingSettings GetScalingSettingsInternal(JNIEnv* jni) const;
   std::vector<ResolutionBitrateLimits> GetResolutionBitrateLimits(
       JNIEnv* jni) const;

sdk/android/tests/src/org/webrtc/FramerateBitrateAdjusterTest.java

@@ -0,0 +1,46 @@
+/*
+ *  Copyright 2021 The WebRTC Project Authors. All rights reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+package org.webrtc;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import org.chromium.testing.local.LocalRobolectricTestRunner;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.robolectric.annotation.Config;
+import org.webrtc.VideoEncoder.ScalingSettings;
+
+@RunWith(LocalRobolectricTestRunner.class)
+@Config(manifest = Config.NONE)
+public class FramerateBitrateAdjusterTest {
+  @Test
+  public void getAdjustedFramerate_alwaysReturnsDefault() {
+    FramerateBitrateAdjuster bitrateAdjuster = new FramerateBitrateAdjuster();
+    bitrateAdjuster.setTargets(1000, 15);
+    assertThat(bitrateAdjuster.getAdjustedFramerateFps()).isEqualTo(30.0);
+  }
+
+  @Test
+  public void getAdjustedBitrate_defaultFramerate_returnsTargetBitrate() {
+    FramerateBitrateAdjuster bitrateAdjuster = new FramerateBitrateAdjuster();
+    bitrateAdjuster.setTargets(1000, 30);
+    assertThat(bitrateAdjuster.getAdjustedBitrateBps()).isEqualTo(1000);
+  }
+
+  @Test
+  public void getAdjustedBitrate_nonDefaultFramerate_returnsAdjustedBitrate() {
+    FramerateBitrateAdjuster bitrateAdjuster = new FramerateBitrateAdjuster();
+    bitrateAdjuster.setTargets(1000, 7.5);
+    // Target frame frame is x4 times smaller than the adjusted one (30fps). Adjusted bitrate should
+    // be x4 times larger then the target one.
+    assertThat(bitrateAdjuster.getAdjustedBitrateBps()).isEqualTo(4000);
+  }
+}

sdk/android/tests/src/org/webrtc/HardwareVideoEncoderTest.java

@@ -298,7 +298,7 @@ public class HardwareVideoEncoderTest {
         mockEncoderCallback);
 
     MediaFormat mediaFormat = fakeMediaCodecWrapper.getConfiguredFormat();
-    assertThat(mediaFormat.getInteger(MediaFormat.KEY_FRAME_RATE)).isEqualTo(30);
+    assertThat(mediaFormat.getFloat(MediaFormat.KEY_FRAME_RATE)).isEqualTo(30.0f);
   }
 
   @Test