zqz/platform-external-webrtc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Android EglRenderer: Add Bitmap frame listener functionality.

Browse Source 
BUG=webrtc:6470

Review-Url: https://codereview.webrtc.org/2456873002
Cr-Commit-Position: refs/heads/master@{#14921}
This commit is contained in:
sakal
2016-11-03 09:15:34 -07:00
committed by Commit bot
parent 8848828708
commit fb0c573263
4 changed files with 390 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

128
webrtc/api/android/java/src/org/webrtc/EglRenderer.java
View File

@ -10,12 +10,16 @@
package org.webrtc;
import android.graphics.Bitmap;
import android.graphics.SurfaceTexture;
import android.opengl.GLES20;
import android.os.Handler;
import android.os.HandlerThread;
import android.os.Looper;
import android.view.Surface;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
@ -29,6 +33,18 @@ public class EglRenderer implements VideoRenderer.Callbacks {
private static final long LOG_INTERVAL_SEC = 4;
private static final int MAX_SURFACE_CLEAR_COUNT = 3;
public interface FrameListener { void onFrame(Bitmap frame); }
private static class ScaleAndFrameListener {
public final float scale;
public final FrameListener listener;
public ScaleAndFrameListener(float scale, FrameListener listener) {
this.scale = scale;
this.listener = listener;
}
}
private class EglSurfaceCreation implements Runnable {
private Object surface;
@ -60,6 +76,9 @@ public class EglRenderer implements VideoRenderer.Callbacks {
private final Object handlerLock = new Object();
private Handler renderThreadHandler;
private final Object frameListenerLock = new Object();
private final ArrayList<ScaleAndFrameListener> frameListeners = new ArrayList<>();
// Variables for fps reduction.
private final Object fpsReductionLock = new Object();
// Time for when next frame should be rendered.
@ -104,6 +123,9 @@ public class EglRenderer implements VideoRenderer.Callbacks {
// Time in ns spent by the render thread in the swapBuffers() function.
private long renderSwapBufferTimeNs;
// Used for bitmap capturing.
private GlTextureFrameBuffer bitmapTextureFramebuffer;
// Runnable for posting frames to render thread.
private final Runnable renderFrameRunnable = new Runnable() {
@Override
@ -220,6 +242,10 @@ public class EglRenderer implements VideoRenderer.Callbacks {
GLES20.glDeleteTextures(3, yuvTextures, 0);
yuvTextures = null;
}
if (bitmapTextureFramebuffer != null) {
bitmapTextureFramebuffer.release();
bitmapTextureFramebuffer = null;
}
if (eglBase != null) {
logD("eglBase detach and release.");
eglBase.detachCurrent();
@ -333,6 +359,36 @@ public class EglRenderer implements VideoRenderer.Callbacks {
setFpsReduction(0 /* fps */);
}
/**
* Register a callback to be invoked when a new video frame has been received.
*
* @param listener The callback to be invoked.
* @param scale The scale of the Bitmap passed to the callback, or 0 if no Bitmap is
* required.
*/
public void addFrameListener(FrameListener listener, float scale) {
synchronized (frameListenerLock) {
frameListeners.add(new ScaleAndFrameListener(scale, listener));
}
}
/**
* Remove any pending callback that was added with addFrameListener. If the callback is not in
* the queue, nothing happens.
*
* @param runnable The callback to remove.
*/
public void removeFrameListener(FrameListener listener) {
synchronized (frameListenerLock) {
final Iterator<ScaleAndFrameListener> iter = frameListeners.iterator();
while (iter.hasNext()) {
if (iter.next().listener == listener) {
iter.remove();
}
}
}
}
// VideoRenderer.Callbacks interface.
@Override
public void renderFrame(VideoRenderer.I420Frame frame) {
@ -472,8 +528,9 @@ public class EglRenderer implements VideoRenderer.Callbacks {
}
final long startTimeNs = System.nanoTime();
float[] texMatrix =
final float[] texMatrix =
RendererCommon.rotateTextureMatrix(frame.samplingMatrix, frame.rotationDegree);
final float[] drawMatrix;
// After a surface size change, the EGLSurface might still have a buffer of the old size in the
// pipeline. Querying the EGLSurface will show if the underlying buffer dimensions haven't yet
@ -487,7 +544,8 @@ public class EglRenderer implements VideoRenderer.Callbacks {
VideoRenderer.renderFrameDone(frame);
return;
}
logD("Surface size mismatch - clearing surface.");
logD("Surface size mismatch - clearing surface. Size: " + eglBase.surfaceWidth() + "x"
+ eglBase.surfaceHeight() + " Expected: " + surfaceWidth + "x" + surfaceHeight);
clearSurfaceOnRenderThread();
}
final float[] layoutMatrix;
@ -498,7 +556,7 @@ public class EglRenderer implements VideoRenderer.Callbacks {
layoutMatrix =
mirror ? RendererCommon.horizontalFlipMatrix() : RendererCommon.identityMatrix();
}
texMatrix = RendererCommon.multiplyMatrices(texMatrix, layoutMatrix);
drawMatrix = RendererCommon.multiplyMatrices(texMatrix, layoutMatrix);
}
GLES20.glClearColor(0 /* red */, 0 /* green */, 0 /* blue */, 0 /* alpha */);
@ -513,16 +571,15 @@ public class EglRenderer implements VideoRenderer.Callbacks {
}
yuvUploader.uploadYuvData(
yuvTextures, frame.width, frame.height, frame.yuvStrides, frame.yuvPlanes);
drawer.drawYuv(yuvTextures, texMatrix, frame.rotatedWidth(), frame.rotatedHeight(), 0, 0,
drawer.drawYuv(yuvTextures, drawMatrix, frame.rotatedWidth(), frame.rotatedHeight(), 0, 0,
surfaceWidth, surfaceHeight);
} else {
drawer.drawOes(frame.textureId, texMatrix, frame.rotatedWidth(), frame.rotatedHeight(), 0, 0,
drawer.drawOes(frame.textureId, drawMatrix, frame.rotatedWidth(), frame.rotatedHeight(), 0, 0,
surfaceWidth, surfaceHeight);
}
final long swapBuffersStartTimeNs = System.nanoTime();
eglBase.swapBuffers();
VideoRenderer.renderFrameDone(frame);
final long currentTimeNs = System.nanoTime();
synchronized (statisticsLock) {
@ -530,6 +587,65 @@ public class EglRenderer implements VideoRenderer.Callbacks {
renderTimeNs += (currentTimeNs - startTimeNs);
renderSwapBufferTimeNs += (currentTimeNs - swapBuffersStartTimeNs);
}
notifyCallbacks(frame, texMatrix);
VideoRenderer.renderFrameDone(frame);
}
private void notifyCallbacks(VideoRenderer.I420Frame frame, float[] texMatrix) {
// Make temporary copy of callback list to avoid ConcurrentModificationException, in case
// callbacks call addFramelistener or removeFrameListener.
final ArrayList<ScaleAndFrameListener> tmpList;
synchronized (frameListenerLock) {
if (frameListeners.isEmpty())
return;
tmpList = new ArrayList<>(frameListeners);
frameListeners.clear();
}
final float[] bitmapMatrix = RendererCommon.multiplyMatrices(
RendererCommon.multiplyMatrices(texMatrix,
mirror ? RendererCommon.horizontalFlipMatrix() : RendererCommon.identityMatrix()),
RendererCommon.verticalFlipMatrix());
for (ScaleAndFrameListener scaleAndListener : tmpList) {
final int scaledWidth = (int) (scaleAndListener.scale * frame.rotatedWidth());
final int scaledHeight = (int) (scaleAndListener.scale * frame.rotatedHeight());
if (scaledWidth == 0 || scaledHeight == 0) {
scaleAndListener.listener.onFrame(null);
continue;
}
if (bitmapTextureFramebuffer == null) {
bitmapTextureFramebuffer = new GlTextureFrameBuffer(GLES20.GL_RGBA);
}
bitmapTextureFramebuffer.setSize(scaledWidth, scaledHeight);
GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, bitmapTextureFramebuffer.getFrameBufferId());
GLES20.glFramebufferTexture2D(GLES20.GL_FRAMEBUFFER, GLES20.GL_COLOR_ATTACHMENT0,
GLES20.GL_TEXTURE_2D, bitmapTextureFramebuffer.getTextureId(), 0);
if (frame.yuvFrame) {
drawer.drawYuv(yuvTextures, bitmapMatrix, frame.rotatedWidth(), frame.rotatedHeight(),
0 /* viewportX */, 0 /* viewportY */, scaledWidth, scaledHeight);
} else {
drawer.drawOes(frame.textureId, bitmapMatrix, frame.rotatedWidth(), frame.rotatedHeight(),
0 /* viewportX */, 0 /* viewportY */, scaledWidth, scaledHeight);
}
final ByteBuffer bitmapBuffer = ByteBuffer.allocateDirect(scaledWidth * scaledHeight * 4);
GLES20.glViewport(0, 0, scaledWidth, scaledHeight);
GLES20.glReadPixels(
0, 0, scaledWidth, scaledHeight, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, bitmapBuffer);
GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0);
GlUtil.checkNoGLES2Error("EglRenderer.notifyCallbacks");
final Bitmap bitmap = Bitmap.createBitmap(scaledWidth, scaledHeight, Bitmap.Config.ARGB_8888);
bitmap.copyPixelsFromBuffer(bitmapBuffer);
scaleAndListener.listener.onFrame(bitmap);
}
}
private String averageTimeAsString(long sumTimeNs, int count) {

1
webrtc/api/android/java/src/org/webrtc/GlRectDrawer.java
View File

@ -12,7 +12,6 @@ package org.webrtc;
import android.opengl.GLES11Ext;
import android.opengl.GLES20;
import java.nio.FloatBuffer;
import java.util.IdentityHashMap;
import java.util.Map;