zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Calculate frame timestamps based on target frame rate

Browse Source 
Before this change HardwareVideoEncoder used capture time as frame
timestamp passed to HW encoder. That led to buffer overshoots with
HW encoders which infer frame rate from timestamps when frames were
dropped before encoding (i.e., frame rate decreases according to frame
timestamps) or when FramerateBitrateAdjuster was used.

Fixed this by using synthetic monotonically increasing timestamps
calculated based on target frame rate provided by bitrate adjuster.

Bug: webrtc:12982
Change-Id: I2454cd4e574bbea1cb9855ced4d998104845415c
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/228902
Reviewed-by: Danil Chapovalov <danilchap@webrtc.org>
Commit-Queue: Sergey Silkin <ssilkin@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#34810}
This commit is contained in:
Sergey Silkin
2021-08-17 20:10:28 +02:00
committed by WebRTC LUCI CQ
parent 51238e6c28
commit c68796e260
6 changed files with 150 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
sdk/android/src/java/org/webrtc/BaseBitrateAdjuster.java
View File

@ -13,12 +13,12 @@ package org.webrtc;
/** BitrateAdjuster that tracks bitrate and framerate but does not adjust them. */
class BaseBitrateAdjuster implements BitrateAdjuster {
protected int targetBitrateBps;
protected int targetFps;
protected int targetFramerateFps;
@Override
public void setTargets(int targetBitrateBps, int targetFps) {
public void setTargets(int targetBitrateBps, int targetFramerateFps) {
this.targetBitrateBps = targetBitrateBps;
this.targetFps = targetFps;
this.targetFramerateFps = targetFramerateFps;
}
@Override
@ -32,7 +32,7 @@ class BaseBitrateAdjuster implements BitrateAdjuster {
}
@Override
public int getCodecConfigFramerate() {
return targetFps;
public int getAdjustedFramerateFps() {
return targetFramerateFps;
}
}

6
sdk/android/src/java/org/webrtc/BitrateAdjuster.java
View File

@ -15,7 +15,7 @@ interface BitrateAdjuster {
/**
* Sets the target bitrate in bits per second and framerate in frames per second.
*/
void setTargets(int targetBitrateBps, int targetFps);
void setTargets(int targetBitrateBps, int targetFramerateFps);
/**
* Should be used to report the size of an encoded frame to the bitrate adjuster. Use
@ -26,6 +26,6 @@ interface BitrateAdjuster {
/** Gets the current bitrate. */
int getAdjustedBitrateBps();
/** Gets the framerate for initial codec configuration. */
int getCodecConfigFramerate();
/** Gets the current framerate. */
int getAdjustedFramerateFps();
}

10
sdk/android/src/java/org/webrtc/DynamicBitrateAdjuster.java
View File

@ -31,24 +31,24 @@ class DynamicBitrateAdjuster extends BaseBitrateAdjuster {
private int bitrateAdjustmentScaleExp;
@Override
public void setTargets(int targetBitrateBps, int targetFps) {
public void setTargets(int targetBitrateBps, int targetFramerateFps) {
if (this.targetBitrateBps > 0 && targetBitrateBps < this.targetBitrateBps) {
// Rescale the accumulator level if the accumulator max decreases
deviationBytes = deviationBytes * targetBitrateBps / this.targetBitrateBps;
}
super.setTargets(targetBitrateBps, targetFps);
super.setTargets(targetBitrateBps, targetFramerateFps);
}
@Override
public void reportEncodedFrame(int size) {
if (targetFps == 0) {
if (targetFramerateFps == 0) {
return;
}
// Accumulate the difference between actual and expected frame sizes.
double expectedBytesPerFrame = (targetBitrateBps / BITS_PER_BYTE) / targetFps;
double expectedBytesPerFrame = (targetBitrateBps / BITS_PER_BYTE) / targetFramerateFps;
deviationBytes += (size - expectedBytesPerFrame);
timeSinceLastAdjustmentMs += 1000.0 / targetFps;
timeSinceLastAdjustmentMs += 1000.0 / targetFramerateFps;
// Adjust the bitrate when the encoder accumulates one second's worth of data in excess or
// shortfall of the target.

19
sdk/android/src/java/org/webrtc/FramerateBitrateAdjuster.java
View File

@ -15,21 +15,12 @@ package org.webrtc;
* hardware codecs that assume the framerate never changes.
*/
class FramerateBitrateAdjuster extends BaseBitrateAdjuster {
private static final int INITIAL_FPS = 30;
private static final int DEFAULT_FRAMERATE_FPS = 30;
@Override
public void setTargets(int targetBitrateBps, int targetFps) {
if (this.targetFps == 0) {
// Framerate-based bitrate adjustment always initializes to the same framerate.
targetFps = INITIAL_FPS;
}
super.setTargets(targetBitrateBps, targetFps);
this.targetBitrateBps = this.targetBitrateBps * INITIAL_FPS / this.targetFps;
}
@Override
public int getCodecConfigFramerate() {
return INITIAL_FPS;
public void setTargets(int targetBitrateBps, int targetFramerateFps) {
// Keep frame rate unchanged and adjust bit rate.
this.targetFramerateFps = DEFAULT_FRAMERATE_FPS;
this.targetBitrateBps = targetBitrateBps * DEFAULT_FRAMERATE_FPS / targetFramerateFps;
}
}

28
sdk/android/src/java/org/webrtc/HardwareVideoEncoder.java
View File

@ -149,6 +149,8 @@ class HardwareVideoEncoder implements VideoEncoder {
private boolean useSurfaceMode;
// --- Only accessed from the encoding thread.
// Presentation timestamp of next frame to encode.
private long nextPresentationTimestampUs;
// Presentation timestamp of the last requested (or forced) key frame.
private long lastKeyFrameNs;
@ -223,6 +225,7 @@ class HardwareVideoEncoder implements VideoEncoder {
private VideoCodecStatus initEncodeInternal() {
encodeThreadChecker.checkIsOnValidThread();
nextPresentationTimestampUs = 0;
lastKeyFrameNs = -1;
try {
@ -238,7 +241,7 @@ class HardwareVideoEncoder implements VideoEncoder {
format.setInteger(MediaFormat.KEY_BIT_RATE, adjustedBitrate);
format.setInteger(KEY_BITRATE_MODE, VIDEO_ControlRateConstant);
format.setInteger(MediaFormat.KEY_COLOR_FORMAT, colorFormat);
format.setInteger(MediaFormat.KEY_FRAME_RATE, bitrateAdjuster.getCodecConfigFramerate());
format.setInteger(MediaFormat.KEY_FRAME_RATE, bitrateAdjuster.getAdjustedFramerateFps());
format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, keyFrameIntervalSec);
if (codecType == VideoCodecMimeType.H264) {
String profileLevelId = params.get(VideoCodecInfo.H264_FMTP_PROFILE_LEVEL_ID);
@ -375,11 +378,18 @@ class HardwareVideoEncoder implements VideoEncoder {
.setRotation(videoFrame.getRotation());
outputBuilders.offer(builder);
long presentationTimestampUs = nextPresentationTimestampUs;
// Round frame duration down to avoid bitrate overshoot.
long frameDurationUs =
(long) (TimeUnit.SECONDS.toMicros(1) / bitrateAdjuster.getAdjustedFramerateFps());
nextPresentationTimestampUs += frameDurationUs;
final VideoCodecStatus returnValue;
if (useSurfaceMode) {
returnValue = encodeTextureBuffer(videoFrame);
returnValue = encodeTextureBuffer(videoFrame, presentationTimestampUs);
} else {
returnValue = encodeByteBuffer(videoFrame, videoFrameBuffer, bufferSize);
returnValue =
encodeByteBuffer(videoFrame, presentationTimestampUs, videoFrameBuffer, bufferSize);
}
// Check if the queue was successful.
@ -391,7 +401,8 @@ class HardwareVideoEncoder implements VideoEncoder {
return returnValue;
}
private VideoCodecStatus encodeTextureBuffer(VideoFrame videoFrame) {
private VideoCodecStatus encodeTextureBuffer(
VideoFrame videoFrame, long presentationTimestampUs) {
encodeThreadChecker.checkIsOnValidThread();
try {
// TODO(perkj): glClear() shouldn't be necessary since every pixel is covered anyway,
@ -401,7 +412,7 @@ class HardwareVideoEncoder implements VideoEncoder {
VideoFrame derotatedFrame =
new VideoFrame(videoFrame.getBuffer(), 0 /* rotation */, videoFrame.getTimestampNs());
videoFrameDrawer.drawFrame(derotatedFrame, textureDrawer, null /* additionalRenderMatrix */);
textureEglBase.swapBuffers(videoFrame.getTimestampNs());
textureEglBase.swapBuffers(TimeUnit.MICROSECONDS.toNanos(presentationTimestampUs));
} catch (RuntimeException e) {
Logging.e(TAG, "encodeTexture failed", e);
return VideoCodecStatus.ERROR;
@ -409,12 +420,9 @@ class HardwareVideoEncoder implements VideoEncoder {
return VideoCodecStatus.OK;
}
private VideoCodecStatus encodeByteBuffer(
VideoFrame videoFrame, VideoFrame.Buffer videoFrameBuffer, int bufferSize) {
private VideoCodecStatus encodeByteBuffer(VideoFrame videoFrame, long presentationTimestampUs,
VideoFrame.Buffer videoFrameBuffer, int bufferSize) {
encodeThreadChecker.checkIsOnValidThread();
// Frame timestamp rounded to the nearest microsecond.
long presentationTimestampUs = (videoFrame.getTimestampNs() + 500) / 1000;
// No timeout. Don't block for an input buffer, drop frames if the encoder falls behind.
int index;
try {