zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Non-integer frame rate in Android HW encoder

Browse Source 
Bug: webrtc:10812
Change-Id: I4443dcfea851114bd5fbb10f11ca8a51cda12da8
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/229025
Commit-Queue: Sergey Silkin <ssilkin@webrtc.org>
Reviewed-by: Erik Språng <sprang@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#34813}
This commit is contained in:
Sergey Silkin
2021-08-18 09:46:16 +02:00
committed by WebRTC LUCI CQ
parent efece42aa5
commit 1db921e6f3
11 changed files with 119 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
sdk/android/BUILD.gn
View File

@ -1615,6 +1615,7 @@ if (is_android) {
"tests/src/org/webrtc/CodecTestHelper.java",
"tests/src/org/webrtc/CryptoOptionsTest.java",
"tests/src/org/webrtc/FakeMediaCodecWrapper.java",
"tests/src/org/webrtc/FramerateBitrateAdjusterTest.java",
"tests/src/org/webrtc/GlGenericDrawerTest.java",
"tests/src/org/webrtc/HardwareVideoEncoderTest.java",
"tests/src/org/webrtc/IceCandidateTest.java",

31
sdk/android/api/org/webrtc/VideoEncoder.java
View File

@ -236,6 +236,28 @@ public interface VideoEncoder {
}
}
/** Rate control parameters. */
public class RateControlParameters {
/**
* Adjusted target bitrate, per spatial/temporal layer. May be lower or higher than the target
* depending on encoder behaviour.
*/
public final BitrateAllocation bitrate;
/**
* Target framerate, in fps. A value <= 0.0 is invalid and should be interpreted as framerate
* target not available. In this case the encoder should fall back to the max framerate
* specified in `codec_settings` of the last InitEncode() call.
*/
public final double framerateFps;
@CalledByNative("RateControlParameters")
public RateControlParameters(BitrateAllocation bitrate, double framerateFps) {
this.bitrate = bitrate;
this.framerateFps = framerateFps;
}
}
public interface Callback {
/**
* Old encoders assume that the byte buffer held by `frame` is not accessed after the call to
@ -296,7 +318,14 @@ public interface VideoEncoder {
@CalledByNative VideoCodecStatus encode(VideoFrame frame, EncodeInfo info);
/** Sets the bitrate allocation and the target framerate for the encoder. */
@CalledByNative VideoCodecStatus setRateAllocation(BitrateAllocation allocation, int framerate);
VideoCodecStatus setRateAllocation(BitrateAllocation allocation, int framerate);
/** Sets the bitrate allocation and the target framerate for the encoder. */
default @CalledByNative VideoCodecStatus setRates(RateControlParameters rcParameters) {
// Round frame rate up to avoid overshoots.
int framerateFps = (int) Math.ceil(rcParameters.framerateFps);
return setRateAllocation(rcParameters.bitrate, framerateFps);
}
/** Any encoder that wants to use WebRTC provided quality scaler must implement this method. */
@CalledByNative ScalingSettings getScalingSettings();

6
sdk/android/src/java/org/webrtc/BaseBitrateAdjuster.java
View File

@ -13,10 +13,10 @@ package org.webrtc;
/** BitrateAdjuster that tracks bitrate and framerate but does not adjust them. */
class BaseBitrateAdjuster implements BitrateAdjuster {
protected int targetBitrateBps;
protected int targetFramerateFps;
protected double targetFramerateFps;
@Override
public void setTargets(int targetBitrateBps, int targetFramerateFps) {
public void setTargets(int targetBitrateBps, double targetFramerateFps) {
this.targetBitrateBps = targetBitrateBps;
this.targetFramerateFps = targetFramerateFps;
}
@ -32,7 +32,7 @@ class BaseBitrateAdjuster implements BitrateAdjuster {
}
@Override
public int getAdjustedFramerateFps() {
public double getAdjustedFramerateFps() {
return targetFramerateFps;
}
}

4
sdk/android/src/java/org/webrtc/BitrateAdjuster.java
View File

@ -15,7 +15,7 @@ interface BitrateAdjuster {
/**
* Sets the target bitrate in bits per second and framerate in frames per second.
*/
void setTargets(int targetBitrateBps, int targetFramerateFps);
void setTargets(int targetBitrateBps, double targetFramerateFps);
/**
* Should be used to report the size of an encoded frame to the bitrate adjuster. Use
@ -27,5 +27,5 @@ interface BitrateAdjuster {
int getAdjustedBitrateBps();
/** Gets the current framerate. */
int getAdjustedFramerateFps();
double getAdjustedFramerateFps();
}

2
sdk/android/src/java/org/webrtc/DynamicBitrateAdjuster.java
View File

@ -31,7 +31,7 @@ class DynamicBitrateAdjuster extends BaseBitrateAdjuster {
private int bitrateAdjustmentScaleExp;
@Override
public void setTargets(int targetBitrateBps, int targetFramerateFps) {
public void setTargets(int targetBitrateBps, double targetFramerateFps) {
if (this.targetBitrateBps > 0 && targetBitrateBps < this.targetBitrateBps) {
// Rescale the accumulator level if the accumulator max decreases
deviationBytes = deviationBytes * targetBitrateBps / this.targetBitrateBps;

4
sdk/android/src/java/org/webrtc/FramerateBitrateAdjuster.java
View File

@ -18,9 +18,9 @@ class FramerateBitrateAdjuster extends BaseBitrateAdjuster {
private static final int DEFAULT_FRAMERATE_FPS = 30;
@Override
public void setTargets(int targetBitrateBps, int targetFramerateFps) {
public void setTargets(int targetBitrateBps, double targetFramerateFps) {
// Keep frame rate unchanged and adjust bit rate.
this.targetFramerateFps = DEFAULT_FRAMERATE_FPS;
this.targetBitrateBps = targetBitrateBps * DEFAULT_FRAMERATE_FPS / targetFramerateFps;
this.targetBitrateBps = (int) (targetBitrateBps * DEFAULT_FRAMERATE_FPS / targetFramerateFps);
}
}

10
sdk/android/src/java/org/webrtc/HardwareVideoEncoder.java
View File

@ -241,7 +241,8 @@ class HardwareVideoEncoder implements VideoEncoder {
format.setInteger(MediaFormat.KEY_BIT_RATE, adjustedBitrate);
format.setInteger(KEY_BITRATE_MODE, VIDEO_ControlRateConstant);
format.setInteger(MediaFormat.KEY_COLOR_FORMAT, colorFormat);
format.setInteger(MediaFormat.KEY_FRAME_RATE, bitrateAdjuster.getAdjustedFramerateFps());
format.setFloat(
MediaFormat.KEY_FRAME_RATE, (float) bitrateAdjuster.getAdjustedFramerateFps());
format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, keyFrameIntervalSec);
if (codecType == VideoCodecMimeType.H264) {
String profileLevelId = params.get(VideoCodecInfo.H264_FMTP_PROFILE_LEVEL_ID);
@ -468,6 +469,13 @@ class HardwareVideoEncoder implements VideoEncoder {
return VideoCodecStatus.OK;
}
@Override
public VideoCodecStatus setRates(RateControlParameters rcParameters) {
encodeThreadChecker.checkIsOnValidThread();
bitrateAdjuster.setTargets(rcParameters.bitrate.getSum(), rcParameters.framerateFps);
return VideoCodecStatus.OK;
}
@Override
public ScalingSettings getScalingSettings() {
encodeThreadChecker.checkIsOnValidThread();

23
sdk/android/src/jni/video_encoder_wrapper.cc
View File

@ -167,15 +167,14 @@ int32_t VideoEncoderWrapper::Encode(
return HandleReturnCode(jni, ret, "encode");
}
void VideoEncoderWrapper::SetRates(const RateControlParameters& parameters) {
void VideoEncoderWrapper::SetRates(const RateControlParameters& rc_parameters) {
JNIEnv* jni = AttachCurrentThreadIfNeeded();
ScopedJavaLocalRef<jobject> j_bitrate_allocation =
ToJavaBitrateAllocation(jni, parameters.bitrate);
ScopedJavaLocalRef<jobject> ret = Java_VideoEncoder_setRateAllocation(
jni, encoder_, j_bitrate_allocation,
(jint)(parameters.framerate_fps + 0.5));
HandleReturnCode(jni, ret, "setRateAllocation");
ScopedJavaLocalRef<jobject> j_rc_parameters =
ToJavaRateControlParameters(jni, rc_parameters);
ScopedJavaLocalRef<jobject> ret =
Java_VideoEncoder_setRates(jni, encoder_, j_rc_parameters);
HandleReturnCode(jni, ret, "setRates");
}
VideoEncoder::EncoderInfo VideoEncoderWrapper::GetEncoderInfo() const {
@ -410,6 +409,16 @@ ScopedJavaLocalRef<jobject> VideoEncoderWrapper::ToJavaBitrateAllocation(
return Java_BitrateAllocation_Constructor(jni, j_allocation_array);
}
ScopedJavaLocalRef<jobject> VideoEncoderWrapper::ToJavaRateControlParameters(
JNIEnv* jni,
const VideoEncoder::RateControlParameters& rc_parameters) {
ScopedJavaLocalRef<jobject> j_bitrate_allocation =
ToJavaBitrateAllocation(jni, rc_parameters.bitrate);
return Java_RateControlParameters_Constructor(jni, j_bitrate_allocation,
rc_parameters.framerate_fps);
}
std::unique_ptr<VideoEncoder> JavaToNativeVideoEncoder(
JNIEnv* jni,
const JavaRef<jobject>& j_encoder) {

9
sdk/android/src/jni/video_encoder_wrapper.h
View File

@ -46,7 +46,7 @@ class VideoEncoderWrapper : public VideoEncoder {
int32_t Encode(const VideoFrame& frame,
const std::vector<VideoFrameType>* frame_types) override;
void SetRates(const RateControlParameters& parameters) override;
void SetRates(const RateControlParameters& rc_parameters) override;
EncoderInfo GetEncoderInfo() const override;
@ -70,12 +70,19 @@ class VideoEncoderWrapper : public VideoEncoder {
const char* method_name);
int ParseQp(rtc::ArrayView<const uint8_t> buffer);
CodecSpecificInfo ParseCodecSpecificInfo(const EncodedImage& frame);
ScopedJavaLocalRef<jobject> ToJavaBitrateAllocation(
JNIEnv* jni,
const VideoBitrateAllocation& allocation);
ScopedJavaLocalRef<jobject> ToJavaRateControlParameters(
JNIEnv* jni,
const VideoEncoder::RateControlParameters& rc_parameters);
void UpdateEncoderInfo(JNIEnv* jni);
ScalingSettings GetScalingSettingsInternal(JNIEnv* jni) const;
std::vector<ResolutionBitrateLimits> GetResolutionBitrateLimits(
JNIEnv* jni) const;

46
sdk/android/tests/src/org/webrtc/FramerateBitrateAdjusterTest.java Normal file
View File

@ -0,0 +1,46 @@
/*
* Copyright 2021 The WebRTC Project Authors. All rights reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
package org.webrtc;
import static com.google.common.truth.Truth.assertThat;
import org.chromium.testing.local.LocalRobolectricTestRunner;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.robolectric.annotation.Config;
import org.webrtc.VideoEncoder.ScalingSettings;
@RunWith(LocalRobolectricTestRunner.class)
@Config(manifest = Config.NONE)
public class FramerateBitrateAdjusterTest {
@Test
public void getAdjustedFramerate_alwaysReturnsDefault() {
FramerateBitrateAdjuster bitrateAdjuster = new FramerateBitrateAdjuster();
bitrateAdjuster.setTargets(1000, 15);
assertThat(bitrateAdjuster.getAdjustedFramerateFps()).isEqualTo(30.0);
}
@Test
public void getAdjustedBitrate_defaultFramerate_returnsTargetBitrate() {
FramerateBitrateAdjuster bitrateAdjuster = new FramerateBitrateAdjuster();
bitrateAdjuster.setTargets(1000, 30);
assertThat(bitrateAdjuster.getAdjustedBitrateBps()).isEqualTo(1000);
}
@Test
public void getAdjustedBitrate_nonDefaultFramerate_returnsAdjustedBitrate() {
FramerateBitrateAdjuster bitrateAdjuster = new FramerateBitrateAdjuster();
bitrateAdjuster.setTargets(1000, 7.5);
// Target frame frame is x4 times smaller than the adjusted one (30fps). Adjusted bitrate should
// be x4 times larger then the target one.
assertThat(bitrateAdjuster.getAdjustedBitrateBps()).isEqualTo(4000);
}
}

2
sdk/android/tests/src/org/webrtc/HardwareVideoEncoderTest.java
View File

@ -298,7 +298,7 @@ public class HardwareVideoEncoderTest {
mockEncoderCallback);
MediaFormat mediaFormat = fakeMediaCodecWrapper.getConfiguredFormat();
assertThat(mediaFormat.getInteger(MediaFormat.KEY_FRAME_RATE)).isEqualTo(30);
assertThat(mediaFormat.getFloat(MediaFormat.KEY_FRAME_RATE)).isEqualTo(30.0f);
}
@Test