Non-integer frame rate in Android HW encoder

Bug: webrtc:10812
Change-Id: I4443dcfea851114bd5fbb10f11ca8a51cda12da8
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/229025
Commit-Queue: Sergey Silkin <ssilkin@webrtc.org>
Reviewed-by: Erik Språng <sprang@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#34813}

sdk/android/src/java/org/webrtc/BaseBitrateAdjuster.java

@@ -13,10 +13,10 @@ package org.webrtc;
 /** BitrateAdjuster that tracks bitrate and framerate but does not adjust them. */
 class BaseBitrateAdjuster implements BitrateAdjuster {
   protected int targetBitrateBps;
-  protected int targetFramerateFps;
+  protected double targetFramerateFps;
 
   @Override
-  public void setTargets(int targetBitrateBps, int targetFramerateFps) {
+  public void setTargets(int targetBitrateBps, double targetFramerateFps) {
     this.targetBitrateBps = targetBitrateBps;
     this.targetFramerateFps = targetFramerateFps;
   }
@@ -32,7 +32,7 @@ class BaseBitrateAdjuster implements BitrateAdjuster {
   }
 
   @Override
-  public int getAdjustedFramerateFps() {
+  public double getAdjustedFramerateFps() {
     return targetFramerateFps;
   }
 }

sdk/android/src/java/org/webrtc/BitrateAdjuster.java

@@ -15,7 +15,7 @@ interface BitrateAdjuster {
   /**
    * Sets the target bitrate in bits per second and framerate in frames per second.
    */
-  void setTargets(int targetBitrateBps, int targetFramerateFps);
+  void setTargets(int targetBitrateBps, double targetFramerateFps);
 
   /**
    * Should be used to report the size of an encoded frame to the bitrate adjuster. Use
@@ -27,5 +27,5 @@ interface BitrateAdjuster {
   int getAdjustedBitrateBps();
 
   /** Gets the current framerate. */
-  int getAdjustedFramerateFps();
+  double getAdjustedFramerateFps();
 }

sdk/android/src/java/org/webrtc/DynamicBitrateAdjuster.java

@@ -31,7 +31,7 @@ class DynamicBitrateAdjuster extends BaseBitrateAdjuster {
   private int bitrateAdjustmentScaleExp;
 
   @Override
-  public void setTargets(int targetBitrateBps, int targetFramerateFps) {
+  public void setTargets(int targetBitrateBps, double targetFramerateFps) {
     if (this.targetBitrateBps > 0 && targetBitrateBps < this.targetBitrateBps) {
       // Rescale the accumulator level if the accumulator max decreases
       deviationBytes = deviationBytes * targetBitrateBps / this.targetBitrateBps;

sdk/android/src/java/org/webrtc/FramerateBitrateAdjuster.java

@@ -18,9 +18,9 @@ class FramerateBitrateAdjuster extends BaseBitrateAdjuster {
   private static final int DEFAULT_FRAMERATE_FPS = 30;
 
   @Override
-  public void setTargets(int targetBitrateBps, int targetFramerateFps) {
+  public void setTargets(int targetBitrateBps, double targetFramerateFps) {
     // Keep frame rate unchanged and adjust bit rate.
     this.targetFramerateFps = DEFAULT_FRAMERATE_FPS;
-    this.targetBitrateBps = targetBitrateBps * DEFAULT_FRAMERATE_FPS / targetFramerateFps;
+    this.targetBitrateBps = (int) (targetBitrateBps * DEFAULT_FRAMERATE_FPS / targetFramerateFps);
   }
 }

sdk/android/src/java/org/webrtc/HardwareVideoEncoder.java

@@ -241,7 +241,8 @@ class HardwareVideoEncoder implements VideoEncoder {
       format.setInteger(MediaFormat.KEY_BIT_RATE, adjustedBitrate);
       format.setInteger(KEY_BITRATE_MODE, VIDEO_ControlRateConstant);
       format.setInteger(MediaFormat.KEY_COLOR_FORMAT, colorFormat);
-      format.setInteger(MediaFormat.KEY_FRAME_RATE, bitrateAdjuster.getAdjustedFramerateFps());
+      format.setFloat(
+          MediaFormat.KEY_FRAME_RATE, (float) bitrateAdjuster.getAdjustedFramerateFps());
       format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, keyFrameIntervalSec);
       if (codecType == VideoCodecMimeType.H264) {
         String profileLevelId = params.get(VideoCodecInfo.H264_FMTP_PROFILE_LEVEL_ID);
@@ -468,6 +469,13 @@ class HardwareVideoEncoder implements VideoEncoder {
     return VideoCodecStatus.OK;
   }
 
+  @Override
+  public VideoCodecStatus setRates(RateControlParameters rcParameters) {
+    encodeThreadChecker.checkIsOnValidThread();
+    bitrateAdjuster.setTargets(rcParameters.bitrate.getSum(), rcParameters.framerateFps);
+    return VideoCodecStatus.OK;
+  }
+
   @Override
   public ScalingSettings getScalingSettings() {
     encodeThreadChecker.checkIsOnValidThread();

sdk/android/src/jni/video_encoder_wrapper.cc

@@ -167,15 +167,14 @@ int32_t VideoEncoderWrapper::Encode(
   return HandleReturnCode(jni, ret, "encode");
 }
 
-void VideoEncoderWrapper::SetRates(const RateControlParameters& parameters) {
+void VideoEncoderWrapper::SetRates(const RateControlParameters& rc_parameters) {
   JNIEnv* jni = AttachCurrentThreadIfNeeded();
 
-  ScopedJavaLocalRef<jobject> j_bitrate_allocation =
-      ToJavaBitrateAllocation(jni, parameters.bitrate);
-  ScopedJavaLocalRef<jobject> ret = Java_VideoEncoder_setRateAllocation(
-      jni, encoder_, j_bitrate_allocation,
-      (jint)(parameters.framerate_fps + 0.5));
-  HandleReturnCode(jni, ret, "setRateAllocation");
+  ScopedJavaLocalRef<jobject> j_rc_parameters =
+      ToJavaRateControlParameters(jni, rc_parameters);
+  ScopedJavaLocalRef<jobject> ret =
+      Java_VideoEncoder_setRates(jni, encoder_, j_rc_parameters);
+  HandleReturnCode(jni, ret, "setRates");
 }
 
 VideoEncoder::EncoderInfo VideoEncoderWrapper::GetEncoderInfo() const {
@@ -410,6 +409,16 @@ ScopedJavaLocalRef<jobject> VideoEncoderWrapper::ToJavaBitrateAllocation(
   return Java_BitrateAllocation_Constructor(jni, j_allocation_array);
 }
 
+ScopedJavaLocalRef<jobject> VideoEncoderWrapper::ToJavaRateControlParameters(
+    JNIEnv* jni,
+    const VideoEncoder::RateControlParameters& rc_parameters) {
+  ScopedJavaLocalRef<jobject> j_bitrate_allocation =
+      ToJavaBitrateAllocation(jni, rc_parameters.bitrate);
+
+  return Java_RateControlParameters_Constructor(jni, j_bitrate_allocation,
+                                                rc_parameters.framerate_fps);
+}
+
 std::unique_ptr<VideoEncoder> JavaToNativeVideoEncoder(
     JNIEnv* jni,
     const JavaRef<jobject>& j_encoder) {

sdk/android/src/jni/video_encoder_wrapper.h

@@ -46,7 +46,7 @@ class VideoEncoderWrapper : public VideoEncoder {
   int32_t Encode(const VideoFrame& frame,
                  const std::vector<VideoFrameType>* frame_types) override;
 
-  void SetRates(const RateControlParameters& parameters) override;
+  void SetRates(const RateControlParameters& rc_parameters) override;
 
   EncoderInfo GetEncoderInfo() const override;
 
@@ -70,12 +70,19 @@ class VideoEncoderWrapper : public VideoEncoder {
                            const char* method_name);
 
   int ParseQp(rtc::ArrayView<const uint8_t> buffer);
+
   CodecSpecificInfo ParseCodecSpecificInfo(const EncodedImage& frame);
+
   ScopedJavaLocalRef<jobject> ToJavaBitrateAllocation(
       JNIEnv* jni,
       const VideoBitrateAllocation& allocation);
 
+  ScopedJavaLocalRef<jobject> ToJavaRateControlParameters(
+      JNIEnv* jni,
+      const VideoEncoder::RateControlParameters& rc_parameters);
+
   void UpdateEncoderInfo(JNIEnv* jni);
+
   ScalingSettings GetScalingSettingsInternal(JNIEnv* jni) const;
   std::vector<ResolutionBitrateLimits> GetResolutionBitrateLimits(
       JNIEnv* jni) const;