Android: Add error callback for GL_OUT_OF_MEMORY in EglRenderer

Encountering GL_OUT_OF_MEMORY is relatively common and we should give
clients a chance to deal with it in a non-fatal way.

Bug: webrtc:8154
Change-Id: Ifa9ca74392f21083692b02a5144dc5632a88d34d
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/144561
Commit-Queue: Magnus Jedvert <magjed@webrtc.org>
Reviewed-by: Sami Kalliomäki <sakal@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#28495}

sdk/android/api/org/webrtc/EglRenderer.java

@@ -37,6 +37,12 @@ public class EglRenderer implements VideoSink {
 
   public interface FrameListener { void onFrame(Bitmap frame); }
 
+  /** Callback for clients to be notified about errors encountered during rendering. */
+  public static interface ErrorCallback {
+    /** Called if GLES20.GL_OUT_OF_MEMORY is encountered during rendering. */
+    void onGlOutOfMemory();
+  }
+
   private static class FrameListenerAndParams {
     public final FrameListener listener;
     public final float scale;
@@ -112,6 +118,8 @@ public class EglRenderer implements VideoSink {
 
   private final ArrayList<FrameListenerAndParams> frameListeners = new ArrayList<>();
 
+  private volatile ErrorCallback errorCallback;
+
   // Variables for fps reduction.
   private final Object fpsReductionLock = new Object();
   // Time for when next frame should be rendered.
@@ -485,6 +493,11 @@ public class EglRenderer implements VideoSink {
     ThreadUtils.awaitUninterruptibly(latch);
   }
 
+  /** Can be set in order to be notified about errors encountered during rendering. */
+  public void setErrorCallback(ErrorCallback errorCallback) {
+    this.errorCallback = errorCallback;
+  }
+
   // VideoSink interface.
   @Override
   public void onFrame(VideoFrame frame) {
@@ -642,6 +655,7 @@ public class EglRenderer implements VideoSink {
     drawMatrix.preScale(scaleX, scaleY);
     drawMatrix.preTranslate(-0.5f, -0.5f);
 
+    try {
       if (shouldRenderFrame) {
         GLES20.glClearColor(0 /* red */, 0 /* green */, 0 /* blue */, 0 /* alpha */);
         GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
@@ -664,8 +678,22 @@ public class EglRenderer implements VideoSink {
       }
 
       notifyCallbacks(frame, shouldRenderFrame);
+    } catch (GlUtil.GlOutOfMemoryException e) {
+      logE("Error while drawing frame", e);
+      final ErrorCallback errorCallback = this.errorCallback;
+      if (errorCallback != null) {
+        errorCallback.onGlOutOfMemory();
+      }
+      // Attempt to free up some resources.
+      drawer.release();
+      frameDrawer.release();
+      bitmapTextureFramebuffer.release();
+      // Continue here on purpose and retry again for next frame. In worst case, this is a continous
+      // problem and no more frames will be drawn.
+    } finally {
       frame.release();
     }
+  }
 
   private void notifyCallbacks(VideoFrame frame, boolean wasRendered) {
     if (frameListeners.isEmpty())
@@ -743,6 +771,10 @@ public class EglRenderer implements VideoSink {
     }
   }
 
+  private void logE(String string, Throwable e) {
+    Logging.e(TAG, name + string, e);
+  }
+
   private void logD(String string) {
     Logging.d(TAG, name + string);
   }

sdk/android/api/org/webrtc/GlUtil.java

@@ -22,11 +22,19 @@ import java.nio.FloatBuffer;
 public class GlUtil {
   private GlUtil() {}
 
+  public static class GlOutOfMemoryException extends RuntimeException {
+    public GlOutOfMemoryException(String msg) {
+      super(msg);
+    }
+  }
+
   // Assert that no OpenGL ES 2.0 error has been raised.
   public static void checkNoGLES2Error(String msg) {
     int error = GLES20.glGetError();
     if (error != GLES20.GL_NO_ERROR) {
-      throw new RuntimeException(msg + ": GLES20 error: " + error);
+      throw error == GLES20.GL_OUT_OF_MEMORY
+          ? new GlOutOfMemoryException(msg)
+          : new RuntimeException(msg + ": GLES20 error: " + error);
     }
   }