zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Trigger framelisteners even on frames dropped by the FPS reduction by default.

Browse Source 
Modification affects EglRenderer on Android. Moves frame dropping to the
renderer thread. Frame listeners are triggered even when FPS reduction is
active unless applyFpsReduction is set to true.

BUG=webrtc:7149

Review-Url: https://codereview.webrtc.org/2688843002
Cr-Commit-Position: refs/heads/master@{#17206}
This commit is contained in:
sakal
2017-03-13 05:11:48 -07:00
committed by Commit bot
parent 7be1dcb98e
commit d15165222f
4 changed files with 117 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

137
webrtc/sdk/android/api/org/webrtc/EglRenderer.java
View File

@ -95,8 +95,6 @@ public class EglRenderer implements VideoRenderer.Callbacks {
private EglBase eglBase;
private final RendererCommon.YuvUploader yuvUploader = new RendererCommon.YuvUploader();
private RendererCommon.GlDrawer drawer;
// Texture ids for YUV frames. Allocated on first arrival of a YUV frame.
private int[] yuvTextures = null;
// Pending frame to render. Serves as a queue with size 1. Synchronized on |frameLock|.
private final Object frameLock = new Object();
@ -239,10 +237,7 @@ public class EglRenderer implements VideoRenderer.Callbacks {
drawer.release();
drawer = null;
}
if (yuvTextures != null) {
GLES20.glDeleteTextures(3, yuvTextures, 0);
yuvTextures = null;
}
yuvUploader.release();
if (bitmapTextureFramebuffer != null) {
bitmapTextureFramebuffer.release();
bitmapTextureFramebuffer = null;
@ -364,33 +359,31 @@ public class EglRenderer implements VideoRenderer.Callbacks {
* Register a callback to be invoked when a new video frame has been received. This version uses
* the drawer of the EglRenderer that was passed in init.
*
* @param listener The callback to be invoked.
* @param listener The callback to be invoked. The callback will be invoked on the render thread.
* It should be lightweight and must not call removeFrameListener.
* @param scale The scale of the Bitmap passed to the callback, or 0 if no Bitmap is
* required.
*/
public void addFrameListener(final FrameListener listener, final float scale) {
postToRenderThread(new Runnable() {
@Override
public void run() {
frameListeners.add(new FrameListenerAndParams(listener, scale, drawer));
}
});
addFrameListener(listener, scale, null);
}
/**
* Register a callback to be invoked when a new video frame has been received.
*
* @param listener The callback to be invoked.
* @param listener The callback to be invoked. The callback will be invoked on the render thread.
* It should be lightweight and must not call removeFrameListener.
* @param scale The scale of the Bitmap passed to the callback, or 0 if no Bitmap is
* required.
* @param drawer Custom drawer to use for this frame listener.
* @param drawer Custom drawer to use for this frame listener or null to use the default one.
*/
public void addFrameListener(
final FrameListener listener, final float scale, final RendererCommon.GlDrawer drawer) {
final FrameListener listener, final float scale, final RendererCommon.GlDrawer drawerParam) {
postToRenderThread(new Runnable() {
@Override
public void run() {
frameListeners.add(new FrameListenerAndParams(listener, scale, drawer));
final RendererCommon.GlDrawer listenerDrawer = drawerParam == null ? drawer : drawerParam;
frameListeners.add(new FrameListenerAndParams(listener, scale, listenerDrawer));
}
});
}
@ -403,6 +396,9 @@ public class EglRenderer implements VideoRenderer.Callbacks {
* @param runnable The callback to remove.
*/
public void removeFrameListener(final FrameListener listener) {
if (Thread.currentThread() == renderThreadHandler.getLooper().getThread()) {
throw new RuntimeException("removeFrameListener must not be called on the render thread.");
}
final CountDownLatch latch = new CountDownLatch(1);
postToRenderThread(new Runnable() {
@Override
@ -432,20 +428,6 @@ public class EglRenderer implements VideoRenderer.Callbacks {
VideoRenderer.renderFrameDone(frame);
return;
}
// Check if fps reduction is active.
synchronized (fpsReductionLock) {
if (minRenderPeriodNs > 0) {
final long currentTimeNs = System.nanoTime();
if (currentTimeNs < nextFrameTimeNs) {
logD("Dropping frame - fps reduction is active.");
VideoRenderer.renderFrameDone(frame);
return;
}
nextFrameTimeNs += minRenderPeriodNs;
// The time for the next frame should always be in the future.
nextFrameTimeNs = Math.max(nextFrameTimeNs, currentTimeNs);
}
}
synchronized (frameLock) {
dropOldFrame = (pendingFrame != null);
if (dropOldFrame) {
@ -543,6 +525,28 @@ public class EglRenderer implements VideoRenderer.Callbacks {
VideoRenderer.renderFrameDone(frame);
return;
}
// Check if fps reduction is active.
final boolean shouldRenderFrame;
synchronized (fpsReductionLock) {
if (minRenderPeriodNs == Long.MAX_VALUE) {
// Rendering is paused.
shouldRenderFrame = false;
} else if (minRenderPeriodNs <= 0) {
// FPS reduction is disabled.
shouldRenderFrame = true;
} else {
final long currentTimeNs = System.nanoTime();
if (currentTimeNs < nextFrameTimeNs) {
logD("Skipping frame rendering - fps reduction is active.");
shouldRenderFrame = false;
} else {
nextFrameTimeNs += minRenderPeriodNs;
// The time for the next frame should always be in the future.
nextFrameTimeNs = Math.max(nextFrameTimeNs, currentTimeNs);
shouldRenderFrame = true;
}
}
}
final long startTimeNs = System.nanoTime();
final float[] texMatrix =
@ -575,55 +579,61 @@ public class EglRenderer implements VideoRenderer.Callbacks {
drawMatrix = RendererCommon.multiplyMatrices(texMatrix, layoutMatrix);
}
GLES20.glClearColor(0 /* red */, 0 /* green */, 0 /* blue */, 0 /* alpha */);
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
boolean shouldUploadYuvTextures = false;
if (frame.yuvFrame) {
// Make sure YUV textures are allocated.
if (yuvTextures == null) {
yuvTextures = new int[3];
for (int i = 0; i < 3; i++) {
yuvTextures[i] = GlUtil.generateTexture(GLES20.GL_TEXTURE_2D);
shouldUploadYuvTextures = shouldRenderFrame;
// Check if there are frame listeners that we want to render a bitmap for regardless of if the
// frame was rendered. This is the case when there are frameListeners with scale != 0f.
if (!shouldUploadYuvTextures) {
for (FrameListenerAndParams listenerAndParams : frameListeners) {
if (listenerAndParams.scale != 0f) {
shouldUploadYuvTextures = true;
break;
}
}
}
}
final int[] yuvTextures = shouldUploadYuvTextures
? yuvUploader.uploadYuvData(frame.width, frame.height, frame.yuvStrides, frame.yuvPlanes)
: null;
yuvUploader.uploadYuvData(
yuvTextures, frame.width, frame.height, frame.yuvStrides, frame.yuvPlanes);
drawer.drawYuv(yuvTextures, drawMatrix, drawnFrameWidth, drawnFrameHeight, 0, 0,
eglBase.surfaceWidth(), eglBase.surfaceHeight());
} else {
drawer.drawOes(frame.textureId, drawMatrix, drawnFrameWidth, drawnFrameHeight, 0, 0,
eglBase.surfaceWidth(), eglBase.surfaceHeight());
if (shouldRenderFrame) {
GLES20.glClearColor(0 /* red */, 0 /* green */, 0 /* blue */, 0 /* alpha */);
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
if (frame.yuvFrame) {
drawer.drawYuv(yuvTextures, drawMatrix, drawnFrameWidth, drawnFrameHeight, 0, 0,
eglBase.surfaceWidth(), eglBase.surfaceHeight());
} else {
drawer.drawOes(frame.textureId, drawMatrix, drawnFrameWidth, drawnFrameHeight, 0, 0,
eglBase.surfaceWidth(), eglBase.surfaceHeight());
}
final long swapBuffersStartTimeNs = System.nanoTime();
eglBase.swapBuffers();
final long currentTimeNs = System.nanoTime();
synchronized (statisticsLock) {
++framesRendered;
renderTimeNs += (currentTimeNs - startTimeNs);
renderSwapBufferTimeNs += (currentTimeNs - swapBuffersStartTimeNs);
}
}
final long swapBuffersStartTimeNs = System.nanoTime();
eglBase.swapBuffers();
final long currentTimeNs = System.nanoTime();
synchronized (statisticsLock) {
++framesRendered;
renderTimeNs += (currentTimeNs - startTimeNs);
renderSwapBufferTimeNs += (currentTimeNs - swapBuffersStartTimeNs);
}
notifyCallbacks(frame, texMatrix);
notifyCallbacks(frame, yuvTextures, texMatrix);
VideoRenderer.renderFrameDone(frame);
}
private void notifyCallbacks(VideoRenderer.I420Frame frame, float[] texMatrix) {
// Make temporary copy of callback list to avoid ConcurrentModificationException, in case
// callbacks call addFramelistener or removeFrameListener.
final ArrayList<FrameListenerAndParams> tmpList;
private void notifyCallbacks(
VideoRenderer.I420Frame frame, int[] yuvTextures, float[] texMatrix) {
if (frameListeners.isEmpty())
return;
tmpList = new ArrayList<>(frameListeners);
frameListeners.clear();
final float[] bitmapMatrix = RendererCommon.multiplyMatrices(
RendererCommon.multiplyMatrices(texMatrix,
mirror ? RendererCommon.horizontalFlipMatrix() : RendererCommon.identityMatrix()),
RendererCommon.verticalFlipMatrix());
for (FrameListenerAndParams listenerAndParams : tmpList) {
for (FrameListenerAndParams listenerAndParams : frameListeners) {
final int scaledWidth = (int) (listenerAndParams.scale * frame.rotatedWidth());
final int scaledHeight = (int) (listenerAndParams.scale * frame.rotatedHeight());
@ -663,6 +673,7 @@ public class EglRenderer implements VideoRenderer.Callbacks {
bitmap.copyPixelsFromBuffer(bitmapBuffer);
listenerAndParams.listener.onFrame(bitmap);
}
frameListeners.clear();
}
private String averageTimeAsString(long sumTimeNs, int count) {

32
webrtc/sdk/android/api/org/webrtc/RendererCommon.java
View File

@ -14,7 +14,6 @@ import android.graphics.Point;
import android.opengl.GLES20;
import android.opengl.Matrix;
import android.view.View;
import java.nio.ByteBuffer;
/**
@ -63,13 +62,14 @@ public class RendererCommon {
// TODO(magjed): Investigate when GL_UNPACK_ROW_LENGTH is available, or make a custom shader
// that handles stride and compare performance with intermediate copy.
private ByteBuffer copyBuffer;
private int[] yuvTextures;
/**
* Upload |planes| into |outputYuvTextures|, taking stride into consideration.
* |outputYuvTextures| must have been generated in advance.
* Upload |planes| into OpenGL textures, taking stride into consideration.
*
* @return Array of three texture indices corresponding to Y-, U-, and V-plane respectively.
*/
public void uploadYuvData(
int[] outputYuvTextures, int width, int height, int[] strides, ByteBuffer[] planes) {
public int[] uploadYuvData(int width, int height, int[] strides, ByteBuffer[] planes) {
final int[] planeWidths = new int[] {width, width / 2, width / 2};
final int[] planeHeights = new int[] {height, height / 2, height / 2};
// Make a first pass to see if we need a temporary copy buffer.
@ -84,10 +84,17 @@ public class RendererCommon {
&& (copyBuffer == null || copyBuffer.capacity() < copyCapacityNeeded)) {
copyBuffer = ByteBuffer.allocateDirect(copyCapacityNeeded);
}
// Make sure YUV textures are allocated.
if (yuvTextures == null) {
yuvTextures = new int[3];
for (int i = 0; i < 3; i++) {
yuvTextures[i] = GlUtil.generateTexture(GLES20.GL_TEXTURE_2D);
}
}
// Upload each plane.
for (int i = 0; i < 3; ++i) {
GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + i);
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, outputYuvTextures[i]);
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, yuvTextures[i]);
// GLES only accepts packed data, i.e. stride == planeWidth.
final ByteBuffer packedByteBuffer;
if (strides[i] == planeWidths[i]) {
@ -101,6 +108,19 @@ public class RendererCommon {
GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_LUMINANCE, planeWidths[i],
planeHeights[i], 0, GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, packedByteBuffer);
}
return yuvTextures;
}
/**
* Releases cached resources. Uploader can still be used and the resources will be reallocated
* on first use.
*/
public void release() {
copyBuffer = null;
if (yuvTextures != null) {
GLES20.glDeleteTextures(3, yuvTextures, 0);
yuvTextures = null;
}
}
}

10
webrtc/sdk/android/api/org/webrtc/SurfaceViewRenderer.java
View File

@ -111,21 +111,23 @@ public class SurfaceViewRenderer
/**
* Register a callback to be invoked when a new video frame has been received.
*
* @param listener The callback to be invoked.
* @param listener The callback to be invoked. The callback will be invoked on the render thread.
* It should be lightweight and must not call removeFrameListener.
* @param scale The scale of the Bitmap passed to the callback, or 0 if no Bitmap is
* required.
* @param drawer Custom drawer to use for this frame listener.
*/
public void addFrameListener(
EglRenderer.FrameListener listener, float scale, final RendererCommon.GlDrawer drawer) {
eglRenderer.addFrameListener(listener, scale, drawer);
EglRenderer.FrameListener listener, float scale, RendererCommon.GlDrawer drawerParam) {
eglRenderer.addFrameListener(listener, scale, drawerParam);
}
/**
* Register a callback to be invoked when a new video frame has been received. This version uses
* the drawer of the EglRenderer that was passed in init.
*
* @param listener The callback to be invoked.
* @param listener The callback to be invoked. The callback will be invoked on the render thread.
* It should be lightweight and must not call removeFrameListener.
* @param scale The scale of the Bitmap passed to the callback, or 0 if no Bitmap is
* required.
*/