zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add dynamic bitrate tracker and adjustment for Exynos VP8 HW encoder.

Browse Source 
Exynos VP8 HW encoder may generate a bitrate which deviates too
much from target value causing frame drops and fps reduction or
BWE stuck at low values.

Add one more option to bitrate adjustment in HW encoder wrapper
which allows to track and dynamicaly scale bitrate used for
codec configuration.

BUG=b/30951236
R=wzh@webrtc.org

Review URL: https://codereview.webrtc.org/2308843002 .

Cr-Commit-Position: refs/heads/master@{#14095}
This commit is contained in:
Alex Glaznev
2016-09-06 14:08:19 -07:00
parent 37062ed16f
commit cfaca03b4b
1 changed files with 140 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

173
webrtc/api/android/java/src/org/webrtc/MediaCodecVideoEncoder.java
View File

@ -52,6 +52,12 @@ public class MediaCodecVideoEncoder {
private static final int DEQUEUE_TIMEOUT = 0; // Non-blocking, no wait.
private static final int BITRATE_ADJUSTMENT_FPS = 30;
private static final int MAXIMUM_INITIAL_FPS = 30;
private static final double BITRATE_CORRECTION_SEC = 3.0;
// Maximum bitrate correction scale - no more than 2 times.
private static final double BITRATE_CORRECTION_MAX_SCALE = 2;
// Amount of correction steps to reach correction maximum scale.
private static final int BITRATE_CORRECTION_STEPS = 10;
// Active running encoder instance. Set in initEncode() (called from native code)
// and reset to null in release() call.
private static MediaCodecVideoEncoder runningInstance = null;
@ -73,6 +79,19 @@ public class MediaCodecVideoEncoder {
private static final String VP9_MIME_TYPE = "video/x-vnd.on2.vp9";
private static final String H264_MIME_TYPE = "video/avc";
// Type of bitrate adjustment for video encoder.
public enum BitrateAdjustmentType {
// No adjustment - video encoder has no known bitrate problem.
NO_ADJUSTMENT,
// Framerate based bitrate adjustment is required - HW encoder does not use frame
// timestamps to calculate frame bitrate budget and instead is relying on initial
// fps configuration assuming that all frames are coming at fixed initial frame rate.
FRAMERATE_ADJUSTMENT,
// Dynamic bitrate adjustment is required - HW encoder used frame timestamps, but actual
// bitrate deviates too much from the target value.
DYNAMIC_ADJUSTMENT
}
// Class describing supported media codec properties.
private static class MediaCodecProperties {
public final String codecPrefix;
@ -81,39 +100,39 @@ public class MediaCodecVideoEncoder {
// Flag if encoder implementation does not use frame timestamps to calculate frame bitrate
// budget and instead is relying on initial fps configuration assuming that all frames are
// coming at fixed initial frame rate. Bitrate adjustment is required for this case.
public final boolean bitrateAdjustmentRequired;
public final BitrateAdjustmentType bitrateAdjustmentType;
MediaCodecProperties(
String codecPrefix, int minSdk, boolean bitrateAdjustmentRequired) {
String codecPrefix, int minSdk, BitrateAdjustmentType bitrateAdjustmentType) {
this.codecPrefix = codecPrefix;
this.minSdk = minSdk;
this.bitrateAdjustmentRequired = bitrateAdjustmentRequired;
this.bitrateAdjustmentType = bitrateAdjustmentType;
}
}
// List of supported HW VP8 encoders.
private static final MediaCodecProperties qcomVp8HwProperties = new MediaCodecProperties(
"OMX.qcom.", Build.VERSION_CODES.KITKAT, false /* bitrateAdjustmentRequired */);
"OMX.qcom.", Build.VERSION_CODES.KITKAT, BitrateAdjustmentType.NO_ADJUSTMENT);
private static final MediaCodecProperties exynosVp8HwProperties = new MediaCodecProperties(
"OMX.Exynos.", Build.VERSION_CODES.M, false /* bitrateAdjustmentRequired */);
"OMX.Exynos.", Build.VERSION_CODES.M, BitrateAdjustmentType.DYNAMIC_ADJUSTMENT);
private static final MediaCodecProperties[] vp8HwList = new MediaCodecProperties[] {
qcomVp8HwProperties, exynosVp8HwProperties
};
// List of supported HW VP9 encoders.
private static final MediaCodecProperties qcomVp9HwProperties = new MediaCodecProperties(
"OMX.qcom.", Build.VERSION_CODES.M, false /* bitrateAdjustmentRequired */);
"OMX.qcom.", Build.VERSION_CODES.M, BitrateAdjustmentType.NO_ADJUSTMENT);
private static final MediaCodecProperties exynosVp9HwProperties = new MediaCodecProperties(
"OMX.Exynos.", Build.VERSION_CODES.M, false /* bitrateAdjustmentRequired */);
"OMX.Exynos.", Build.VERSION_CODES.M, BitrateAdjustmentType.NO_ADJUSTMENT);
private static final MediaCodecProperties[] vp9HwList = new MediaCodecProperties[] {
qcomVp9HwProperties, exynosVp9HwProperties
};
// List of supported HW H.264 encoders.
private static final MediaCodecProperties qcomH264HwProperties = new MediaCodecProperties(
"OMX.qcom.", Build.VERSION_CODES.KITKAT, false /* bitrateAdjustmentRequired */);
"OMX.qcom.", Build.VERSION_CODES.KITKAT, BitrateAdjustmentType.NO_ADJUSTMENT);
private static final MediaCodecProperties exynosH264HwProperties = new MediaCodecProperties(
"OMX.Exynos.", Build.VERSION_CODES.LOLLIPOP, true /* bitrateAdjustmentRequired */);
"OMX.Exynos.", Build.VERSION_CODES.LOLLIPOP, BitrateAdjustmentType.FRAMERATE_ADJUSTMENT);
private static final MediaCodecProperties[] h264HwList = new MediaCodecProperties[] {
qcomH264HwProperties, exynosH264HwProperties
};
@ -146,7 +165,15 @@ public class MediaCodecVideoEncoder {
};
private VideoCodecType type;
private int colorFormat; // Used by native code.
private boolean bitrateAdjustmentRequired;
// Variables used for dynamic bitrate adjustment.
private BitrateAdjustmentType bitrateAdjustmentType = BitrateAdjustmentType.NO_ADJUSTMENT;
private double bitrateAccumulator;
private double bitrateAccumulatorMax;
private double bitrateObservationTimeMs;
private int bitrateAdjustmentScaleExp;
private int targetBitrateBps;
private int targetFps;
// SPS and PPS NALs (Config frame) for H.264.
private ByteBuffer configData = null;
@ -213,14 +240,15 @@ public class MediaCodecVideoEncoder {
// Helper struct for findHwEncoder() below.
private static class EncoderProperties {
public EncoderProperties(String codecName, int colorFormat, boolean bitrateAdjustment) {
public EncoderProperties(
String codecName, int colorFormat, BitrateAdjustmentType bitrateAdjustmentType) {
this.codecName = codecName;
this.colorFormat = colorFormat;
this.bitrateAdjustment = bitrateAdjustment;
this.bitrateAdjustmentType = bitrateAdjustmentType;
}
public final String codecName; // OpenMax component name for HW codec.
public final int colorFormat; // Color format supported by codec.
public final boolean bitrateAdjustment; // true if bitrate adjustment workaround is required.
public final BitrateAdjustmentType bitrateAdjustmentType; // Bitrate adjustment type
}
private static EncoderProperties findHwEncoder(
@ -264,7 +292,7 @@ public class MediaCodecVideoEncoder {
// Check if this is supported HW encoder.
boolean supportedCodec = false;
boolean bitrateAdjustmentRequired = false;
BitrateAdjustmentType bitrateAdjustmentType = BitrateAdjustmentType.NO_ADJUSTMENT;
for (MediaCodecProperties codecProperties : supportedHwCodecProperties) {
if (name.startsWith(codecProperties.codecPrefix)) {
if (Build.VERSION.SDK_INT < codecProperties.minSdk) {
@ -272,9 +300,10 @@ public class MediaCodecVideoEncoder {
Build.VERSION.SDK_INT);
continue;
}
if (codecProperties.bitrateAdjustmentRequired) {
Logging.w(TAG, "Codec " + name + " does not use frame timestamps.");
bitrateAdjustmentRequired = true;
if (codecProperties.bitrateAdjustmentType != BitrateAdjustmentType.NO_ADJUSTMENT) {
bitrateAdjustmentType = codecProperties.bitrateAdjustmentType;
Logging.w(TAG, "Codec " + name
+ " requires bitrate adjustment: " + bitrateAdjustmentType);
}
supportedCodec = true;
break;
@ -300,9 +329,10 @@ public class MediaCodecVideoEncoder {
for (int codecColorFormat : capabilities.colorFormats) {
if (codecColorFormat == supportedColorFormat) {
// Found supported HW encoder.
Logging.d(TAG, "Found target encoder for mime " + mime + " : " + name +
". Color: 0x" + Integer.toHexString(codecColorFormat));
return new EncoderProperties(name, codecColorFormat, bitrateAdjustmentRequired);
Logging.d(TAG, "Found target encoder for mime " + mime + " : " + name
+ ". Color: 0x" + Integer.toHexString(codecColorFormat)
+ ". Bitrate adjustment: " + bitrateAdjustmentType);
return new EncoderProperties(name, codecColorFormat, bitrateAdjustmentType);
}
}
}
@ -375,23 +405,29 @@ public class MediaCodecVideoEncoder {
}
runningInstance = this; // Encoder is now running and can be queried for stack traces.
colorFormat = properties.colorFormat;
bitrateAdjustmentRequired = properties.bitrateAdjustment;
if (bitrateAdjustmentRequired) {
bitrateAdjustmentType = properties.bitrateAdjustmentType;
if (bitrateAdjustmentType == BitrateAdjustmentType.FRAMERATE_ADJUSTMENT) {
fps = BITRATE_ADJUSTMENT_FPS;
} else {
fps = Math.min(fps, MAXIMUM_INITIAL_FPS);
}
Logging.d(TAG, "Color format: " + colorFormat +
". Bitrate adjustment: " + bitrateAdjustmentRequired +
". Bitrate adjustment: " + bitrateAdjustmentType +
". Initial fps: " + fps);
targetBitrateBps = 1000 * kbps;
targetFps = fps;
bitrateAccumulatorMax = targetBitrateBps / 8.0;
bitrateAccumulator = 0;
bitrateObservationTimeMs = 0;
bitrateAdjustmentScaleExp = 0;
mediaCodecThread = Thread.currentThread();
try {
MediaFormat format = MediaFormat.createVideoFormat(mime, width, height);
format.setInteger(MediaFormat.KEY_BIT_RATE, 1000 * kbps);
format.setInteger(MediaFormat.KEY_BIT_RATE, targetBitrateBps);
format.setInteger("bitrate-mode", VIDEO_ControlRateConstant);
format.setInteger(MediaFormat.KEY_COLOR_FORMAT, properties.colorFormat);
format.setInteger(MediaFormat.KEY_FRAME_RATE, fps);
format.setInteger(MediaFormat.KEY_FRAME_RATE, targetFps);
format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, keyFrameIntervalSec);
Logging.d(TAG, " Format: " + format);
mediaCodec = createByCodecName(properties.codecName);
@ -529,17 +565,36 @@ public class MediaCodecVideoEncoder {
private boolean setRates(int kbps, int frameRate) {
checkOnMediaCodecThread();
int codecBitrate = 1000 * kbps;
if (bitrateAdjustmentRequired && frameRate > 0) {
codecBitrate = BITRATE_ADJUSTMENT_FPS * codecBitrate / frameRate;
Logging.v(TAG, "setRates: " + kbps + " -> " + (codecBitrate / 1000)
+ " kbps. Fps: " + frameRate);
} else {
Logging.v(TAG, "setRates: " + kbps + " kbps. Fps: " + frameRate);
int codecBitrateBps = 1000 * kbps;
if (bitrateAdjustmentType == BitrateAdjustmentType.DYNAMIC_ADJUSTMENT) {
bitrateAccumulatorMax = codecBitrateBps / 8.0;
if (targetBitrateBps > 0 && codecBitrateBps < targetBitrateBps) {
// Rescale the accumulator level if the accumulator max decreases
bitrateAccumulator = bitrateAccumulator * codecBitrateBps / targetBitrateBps;
}
}
targetBitrateBps = codecBitrateBps;
targetFps = frameRate;
// Adjust actual encoder bitrate based on bitrate adjustment type.
if (bitrateAdjustmentType == BitrateAdjustmentType.FRAMERATE_ADJUSTMENT && targetFps > 0) {
codecBitrateBps = BITRATE_ADJUSTMENT_FPS * targetBitrateBps / targetFps;
Logging.v(TAG, "setRates: " + kbps + " -> " + (codecBitrateBps / 1000)
+ " kbps. Fps: " + targetFps);
} else if (bitrateAdjustmentType == BitrateAdjustmentType.DYNAMIC_ADJUSTMENT) {
Logging.v(TAG, "setRates: " + kbps + " kbps. Fps: " + targetFps
+ ". ExpScale: " + bitrateAdjustmentScaleExp);
if (bitrateAdjustmentScaleExp != 0) {
codecBitrateBps = (int)(codecBitrateBps * getBitrateScale(bitrateAdjustmentScaleExp));
}
} else {
Logging.v(TAG, "setRates: " + kbps + " kbps. Fps: " + targetFps);
}
try {
Bundle params = new Bundle();
params.putInt(MediaCodec.PARAMETER_KEY_VIDEO_BITRATE, codecBitrate);
params.putInt(MediaCodec.PARAMETER_KEY_VIDEO_BITRATE, codecBitrateBps);
mediaCodec.setParameters(params);
return true;
} catch (IllegalStateException e) {
@ -608,6 +663,8 @@ public class MediaCodecVideoEncoder {
ByteBuffer outputBuffer = outputBuffers[result].duplicate();
outputBuffer.position(info.offset);
outputBuffer.limit(info.offset + info.size);
reportEncodedFrame(info.size);
// Check key frame flag.
boolean isKeyFrame =
(info.flags & MediaCodec.BUFFER_FLAG_SYNC_FRAME) != 0;
@ -646,6 +703,56 @@ public class MediaCodecVideoEncoder {
}
}
private double getBitrateScale(int bitrateAdjustmentScaleExp) {
return Math.pow(BITRATE_CORRECTION_MAX_SCALE,
(double)bitrateAdjustmentScaleExp / BITRATE_CORRECTION_STEPS);
}
private void reportEncodedFrame(int size) {
if (targetFps == 0 || bitrateAdjustmentType != BitrateAdjustmentType.DYNAMIC_ADJUSTMENT) {
return;
}
// Accumulate the difference between actial and expected frame sizes.
double expectedBytesPerFrame = targetBitrateBps / (8.0 * targetFps);
bitrateAccumulator += (size - expectedBytesPerFrame);
bitrateObservationTimeMs += 1000.0 / targetFps;
// Put a cap on the accumulator, i.e., don't let it grow beyond some level to avoid
// using too old data for bitrate adjustment.
double bitrateAccumulatorCap = BITRATE_CORRECTION_SEC * bitrateAccumulatorMax;
bitrateAccumulator = Math.min(bitrateAccumulator, bitrateAccumulatorCap);
bitrateAccumulator = Math.max(bitrateAccumulator, -bitrateAccumulatorCap);
// Do bitrate adjustment every 3 seconds if actual encoder bitrate deviates too much
// form the target value.
if (bitrateObservationTimeMs > 1000 * BITRATE_CORRECTION_SEC) {
Logging.d(TAG, "Acc: " + (int)bitrateAccumulator
+ ". Max: " + (int)bitrateAccumulatorMax
+ ". ExpScale: " + bitrateAdjustmentScaleExp);
boolean bitrateAdjustmentScaleChanged = false;
if (bitrateAccumulator > bitrateAccumulatorMax) {
// Encoder generates too high bitrate - need to reduce the scale.
bitrateAccumulator = bitrateAccumulatorMax;
bitrateAdjustmentScaleExp--;
bitrateAdjustmentScaleChanged = true;
} else if (bitrateAccumulator < -bitrateAccumulatorMax) {
// Encoder generates too low bitrate - need to increase the scale.
bitrateAdjustmentScaleExp++;
bitrateAccumulator = -bitrateAccumulatorMax;
bitrateAdjustmentScaleChanged = true;
}
if (bitrateAdjustmentScaleChanged) {
bitrateAdjustmentScaleExp = Math.min(bitrateAdjustmentScaleExp, BITRATE_CORRECTION_STEPS);
bitrateAdjustmentScaleExp = Math.max(bitrateAdjustmentScaleExp, -BITRATE_CORRECTION_STEPS);
Logging.d(TAG, "Adjusting bitrate scale to " + bitrateAdjustmentScaleExp
+ ". Value: " + getBitrateScale(bitrateAdjustmentScaleExp));
setRates(targetBitrateBps / 1000, targetFps);
}
bitrateObservationTimeMs = 0;
}
}
// Release a dequeued output buffer back to the codec for re-use. Return
// false if the codec is no longer operable.
boolean releaseOutputBuffer(int index) {